class KeyDiscoveryAgent(Agent): """ Class to get server key, instance name and vip address of external/remote platforms """ def __init__(self, address, serverkey, identity, external_address_config, setup_mode, bind_web_address, *args, **kwargs): super(KeyDiscoveryAgent, self).__init__(identity, address, **kwargs) self._external_address_file = external_address_config self._ext_addresses = dict() self._grnlets = dict() self._vip_socket = None self._my_web_address = bind_web_address self.r = random.random() self._setup_mode = setup_mode if self._setup_mode: _log.debug("RUNNING IN MULTI-PLATFORM SETUP MODE") self._store_path = os.path.join(os.environ['VOLTTRON_HOME'], 'external_platform_discovery.json') self._ext_addresses_store = dict() self._ext_addresses_store_lock = Semaphore() @Core.receiver('onstart') def startup(self, sender, **kwargs): """ Try to get platform discovery info of all the remote platforms. If unsuccessful, setup events to try again later :param sender: caller :param kwargs: optional arguments :return: """ self._vip_socket = self.core.socket #If in setup mode, read the external_addresses.json to get web addresses to set up authorized connection with # external platforms. if self._setup_mode: if self._my_web_address is None: _log.error( "Web bind address is needed in multiplatform setup mode") return with self._ext_addresses_store_lock: try: self._ext_addresses_store = PersistentDict( filename=self._store_path, flag='c', format='json') except ValueError as exc: _log.error("Error in json format: {0}".format(exc)) #Delete the existing store. if self._ext_addresses_store: self._ext_addresses_store.clear() self._ext_addresses_store.async_sync() web_addresses = dict() #Read External web addresses file try: web_addresses = self._read_platform_address_file() try: web_addresses.remove(self._my_web_address) except ValueError: _log.debug( "My web address is not in the external bind web adress list" ) op = b'web-addresses' self._send_to_router(op, web_addresses) except IOError as exc: _log.error("Error in reading file: {}".format(exc)) return sec = random.random() * self.r + 10 delay = utils.get_aware_utc_now() + timedelta(seconds=sec) grnlt = self.core.schedule(delay, self._key_collection, web_addresses) else: #Use the existing store for platform discovery information with self._ext_addresses_store_lock: try: self._ext_addresses_store = PersistentDict( filename=self._store_path, flag='c', format='json') except ValueError as exc: _log.error("Error in json file format: {0}".format(exc)) for name, discovery_info in self._ext_addresses_store.items(): op = b'normalmode_platform_connection' self._send_to_router(op, discovery_info) def _key_collection(self, web_addresses): """ Collect platform discovery information (server key, instance name, vip-address) for all platforms. :param web_addresses: List of web addresses to get discovery info :return: """ for web_address in web_addresses: if web_address not in self._my_web_address: self._collect_key(web_address) def _collect_key(self, web_address): """ Try to get (server key, instance name, vip-address) of remote instance and send it to RoutingService to connect to the remote instance. If unsuccessful, try again later. :param name: instance name :param web_address: web address of remote instance :return: """ platform_info = dict() try: platform_info = self._get_platform_discovery(web_address) with self._ext_addresses_store_lock: _log.debug("Platform discovery info: {}".format(platform_info)) name = platform_info['instance-name'] self._ext_addresses_store[name] = platform_info self._ext_addresses_store.async_sync() except KeyError as exc: _log.error( "Discovery info does not contain instance name {}".format(exc)) except DiscoveryError: # If discovery error, try again later sec = random.random() * self.r + 30 delay = utils.get_aware_utc_now() + timedelta(seconds=sec) grnlet = self.core.schedule(delay, self._collect_key, web_address) except ConnectionError as e: _log.error("HTTP connection error {}".format(e)) #If platform discovery is successful, send the info to RoutingService #to establish connection with remote platform. if platform_info: op = b'setupmode_platform_connection' connection_settings = dict(platform_info) connection_settings['web-address'] = web_address self._send_to_router(op, connection_settings) def _send_to_router(self, op, platform_info): """ Send the platform discovery stats to the router to establish new connection :param platform_info: platform discovery stats :return: """ address = jsonapi.dumps(platform_info) frames = [op, address] try: self._vip_socket.send_vip(b'', 'routing_table', frames, copy=False) except ZMQError as ex: # Try sending later _log.error( "ZMQ error while sending external platform info to router: {}". format(ex)) def _read_platform_address_file(self): """ Read the external addresses file :return: """ try: with open(self._external_address_file) as fil: # Use gevent FileObject to avoid blocking the thread data = FileObject(fil, close=False).read() web_addresses = jsonapi.loads(data) if data else {} return web_addresses except IOError as e: _log.error("Error opening file {}".format( self._external_address_file)) raise except Exception: _log.exception('error loading %s', self._external_address_file) raise def _get_platform_discovery(self, web_address): """ Use http discovery call to get serverkey, instance name and vip-address of remote instance :param web_address: web address of remote instance :return: """ r = {} try: parsed = urlparse(web_address) assert parsed.scheme assert not parsed.path real_url = urljoin(web_address, "/discovery/") req = grequests.get(real_url) responses = grequests.map([req]) responses[0].raise_for_status() r = responses[0].json() return r except requests.exceptions.HTTPError: raise DiscoveryError( "Invalid discovery response from {}".format(real_url)) except requests.exceptions.Timeout: raise DiscoveryError("Timeout error from {}".format(real_url)) except AttributeError as e: raise DiscoveryError( "Invalid web_address passed {}".format(web_address)) except (ConnectionError, NewConnectionError) as e: raise DiscoveryError( "Connection to {} not available".format(real_url)) except Exception as e: raise DiscoveryError("Unknown Exception: {}".format(str(e)))
class AdminEndpoints(object): def __init__(self, rmq_mgmt, ssl_public_key): self._rmq_mgmt = rmq_mgmt self._ssl_public_key = ssl_public_key self._userdict = None self.reload_userdict() self._observer = Observer() self._observer.schedule( FileReloader("web-users.json", self.reload_userdict), get_home()) self._observer.start() self._certs = Certs() def reload_userdict(self): webuserpath = os.path.join(get_home(), 'web-users.json') self._userdict = PersistentDict(webuserpath) def get_routes(self): """ Returns a list of tuples with the routes for the adminstration endpoints available in it. :return: """ return [(re.compile('^/admin.*'), 'callable', self.admin)] def admin(self, env, data): if len(self._userdict) == 0: if env.get('REQUEST_METHOD') == 'POST': decoded = dict((k, v if len(v) > 1 else v[0]) for k, v in urlparse.parse_qs(data).iteritems()) username = decoded.get('username') pass1 = decoded.get('password1') pass2 = decoded.get('password2') if pass1 == pass2 and pass1 is not None: _log.debug("Setting master password") self.add_user(username, pass1, groups=['admin']) return Response('', status='302', headers={'Location': '/admin/login.html'}) template = template_env(env).get_template('first.html') return Response(template.render()) if 'login.html' in env.get('PATH_INFO') or '/admin/' == env.get( 'PATH_INFO'): template = template_env(env).get_template('login.html') return Response(template.render()) return self.verify_and_dispatch(env, data) def verify_and_dispatch(self, env, data): """ Verify that the user is an admin and dispatch :param env: web environment :param data: data associated with a web form or json/xml request data :return: Response object. """ from volttron.platform.web import get_user_claims, NotAuthorized try: claims = get_user_claims(env) except NotAuthorized: _log.error("Unauthorized user attempted to connect to {}".format( env.get('PATH_INFO'))) return Response('<h1>Unauthorized User</h1>', status="401 Unauthorized") # Make sure we have only admins for viewing this. if 'admin' not in claims.get('groups'): return Response('<h1>Unauthorized User</h1>', status="401 Unauthorized") # Make sure we have only admins for viewing this. if 'admin' not in claims.get('groups'): return Response('<h1>Unauthorized User</h1>', status="401 Unauthorized") path_info = env.get('PATH_INFO') if path_info.startswith('/admin/api/'): return self.__api_endpoint(path_info[len('/admin/api/'):], data) if path_info.endswith('html'): page = path_info.split('/')[-1] try: template = template_env(env).get_template(page) except TemplateNotFound: return Response("<h1>404 Not Found</h1>", status="404 Not Found") if page == 'list_certs.html': html = template.render( certs=self._certs.get_all_cert_subjects()) elif page == 'pending_csrs.html': html = template.render( csrs=self._certs.get_pending_csr_requests()) else: # A template with no params. html = template.render() return Response(html) template = template_env(env).get_template('index.html') resp = template.render() return Response(resp) def __api_endpoint(self, endpoint, data): _log.debug("Doing admin endpoint {}".format(endpoint)) if endpoint == 'certs': response = self.__cert_list_api() elif endpoint == 'pending_csrs': response = self.__pending_csrs_api() elif endpoint.startswith('approve_csr/'): response = self.__approve_csr_api(endpoint.split('/')[1]) elif endpoint.startswith('deny_csr/'): response = self.__deny_csr_api(endpoint.split('/')[1]) elif endpoint.startswith('delete_csr/'): response = self.__delete_csr_api(endpoint.split('/')[1]) else: response = Response( '{"status": "Unknown endpoint {}"}'.format(endpoint), content_type="application/json") return response def __approve_csr_api(self, common_name): try: _log.debug("Creating cert and permissions for user: {}".format( common_name)) self._certs.approve_csr(common_name) permissions = self._rmq_mgmt.get_default_permissions(common_name) self._rmq_mgmt.create_user_with_permissions( common_name, permissions, True) data = dict(status=self._certs.get_csr_status(common_name), cert=self._certs.get_cert_from_csr(common_name)) except ValueError as e: data = dict(status="ERROR", message=e.message) return Response(json.dumps(data), content_type="application/json") def __deny_csr_api(self, common_name): try: self._certs.deny_csr(common_name) data = dict(status="DENIED", message="The administrator has denied the request") except ValueError as e: data = dict(status="ERROR", message=e.message) return Response(json.dumps(data), content_type="application/json") def __delete_csr_api(self, common_name): try: self._certs.delete_csr(common_name) data = dict(status="DELETED", message="The administrator has denied the request") except ValueError as e: data = dict(status="ERROR", message=e.message) return Response(json.dumps(data), content_type="application/json") def __pending_csrs_api(self): csrs = [c for c in self._certs.get_pending_csr_requests()] return Response(json.dumps(csrs), content_type="application/json") def __cert_list_api(self): subjects = [ dict(common_name=x.common_name) for x in self._certs.get_all_cert_subjects() ] return Response(json.dumps(subjects), content_type="application/json") def add_user(self, username, unencrypted_pw, groups=[], overwrite=False): if self._userdict.get(username): raise ValueError("Already exists!") if groups is None: groups = [] hashed_pass = argon2.hash(unencrypted_pw) self._userdict[username] = dict(hashed_password=hashed_pass, groups=groups) self._userdict.async_sync()
class KeyDiscoveryAgent(Agent): """ Class to get server key, instance name and vip address of external/remote platforms """ def __init__(self, address, serverkey, identity, external_address_config, setup_mode, bind_web_address, *args, **kwargs): super(KeyDiscoveryAgent, self).__init__(identity, address, **kwargs) self._external_address_file = external_address_config self._ext_addresses = dict() self._grnlets = dict() self._vip_socket = None self._my_web_address = bind_web_address self.r = random.random() self._setup_mode = setup_mode if self._setup_mode: _log.debug("RUNNING IN MULTI-PLATFORM SETUP MODE") self._store_path = os.path.join(os.environ['VOLTTRON_HOME'], 'external_platform_discovery.json') self._ext_addresses_store = dict() self._ext_addresses_store_lock = Semaphore() @Core.receiver('onstart') def startup(self, sender, **kwargs): """ Try to get platform discovery info of all the remote platforms. If unsuccessful, setup events to try again later :param sender: caller :param kwargs: optional arguments :return: """ self._vip_socket = self.core.socket #If in setup mode, read the external_addresses.json to get web addresses to set up authorized connection with # external platforms. if self._setup_mode: if self._my_web_address is None: _log.error("Web bind address is needed in multiplatform setup mode") return with self._ext_addresses_store_lock: try: self._ext_addresses_store = PersistentDict(filename=self._store_path, flag='c', format='json') except ValueError as exc: _log.error("Error in json format: {0}".format(exc)) #Delete the existing store. if self._ext_addresses_store: self._ext_addresses_store.clear() self._ext_addresses_store.async_sync() web_addresses = dict() #Read External web addresses file try: web_addresses = self._read_platform_address_file() try: web_addresses.remove(self._my_web_address) except ValueError: _log.debug("My web address is not in the external bind web adress list") op = b'web-addresses' self._send_to_router(op, web_addresses) except IOError as exc: _log.error("Error in reading file: {}".format(exc)) return sec = random.random() * self.r + 10 delay = utils.get_aware_utc_now() + timedelta(seconds=sec) grnlt = self.core.schedule(delay, self._key_collection, web_addresses) else: #Use the existing store for platform discovery information with self._ext_addresses_store_lock: try: self._ext_addresses_store = PersistentDict(filename=self._store_path, flag='c', format='json') except ValueError as exc: _log.error("Error in json file format: {0}".format(exc)) for name, discovery_info in self._ext_addresses_store.items(): op = b'normalmode_platform_connection' self._send_to_router(op, discovery_info) def _key_collection(self, web_addresses): """ Collect platform discovery information (server key, instance name, vip-address) for all platforms. :param web_addresses: List of web addresses to get discovery info :return: """ for web_address in web_addresses: if web_address not in self._my_web_address: self._collect_key(web_address) def _collect_key(self, web_address): """ Try to get (server key, instance name, vip-address) of remote instance and send it to RoutingService to connect to the remote instance. If unsuccessful, try again later. :param name: instance name :param web_address: web address of remote instance :return: """ platform_info = dict() try: platform_info = self._get_platform_discovery(web_address) with self._ext_addresses_store_lock: _log.debug("Platform discovery info: {}".format(platform_info)) name = platform_info['instance-name'] self._ext_addresses_store[name] = platform_info self._ext_addresses_store.async_sync() except KeyError as exc: _log.error("Discovery info does not contain instance name {}".format(exc)) except DiscoveryError: # If discovery error, try again later sec = random.random() * self.r + 30 delay = utils.get_aware_utc_now() + timedelta(seconds=sec) grnlet = self.core.schedule(delay, self._collect_key, web_address) except ConnectionError as e: _log.error("HTTP connection error {}".format(e)) #If platform discovery is successful, send the info to RoutingService #to establish connection with remote platform. if platform_info: op = b'setupmode_platform_connection' connection_settings = dict(platform_info) connection_settings['web-address'] = web_address self._send_to_router(op, connection_settings) def _send_to_router(self, op, platform_info): """ Send the platform discovery stats to the router to establish new connection :param platform_info: platform discovery stats :return: """ address = jsonapi.dumps(platform_info) frames = [op, address] try: self._vip_socket.send_vip(b'', 'routing_table', frames, copy=False) except ZMQError as ex: # Try sending later _log.error("ZMQ error while sending external platform info to router: {}".format(ex)) def _read_platform_address_file(self): """ Read the external addresses file :return: """ try: with open(self._external_address_file) as fil: # Use gevent FileObject to avoid blocking the thread data = FileObject(fil, close=False).read() web_addresses = jsonapi.loads(data) if data else {} return web_addresses except IOError as e: _log.error("Error opening file {}".format(self._external_address_file)) raise except Exception: _log.exception('error loading %s', self._external_address_file) raise def _get_platform_discovery(self, web_address): """ Use http discovery call to get serverkey, instance name and vip-address of remote instance :param web_address: web address of remote instance :return: """ r = {} try: parsed = urlparse(web_address) assert parsed.scheme assert not parsed.path real_url = urljoin(web_address, "/discovery/") req = grequests.get(real_url) responses = grequests.map([req]) responses[0].raise_for_status() r = responses[0].json() return r except requests.exceptions.HTTPError: raise DiscoveryError( "Invalid discovery response from {}".format(real_url) ) except requests.exceptions.Timeout: raise DiscoveryError( "Timeout error from {}".format(real_url) ) except AttributeError as e: raise DiscoveryError( "Invalid web_address passed {}" .format(web_address) ) except (ConnectionError, NewConnectionError) as e: raise DiscoveryError( "Connection to {} not available".format(real_url) ) except Exception as e: raise DiscoveryError( "Unknown Exception".format(e.message) )