def __init__(self, rmq_mgmt, ssl_public_key): self._rmq_mgmt = rmq_mgmt self._ssl_public_key = ssl_public_key self._userdict = None self.reload_userdict() self._observer = Observer() self._observer.schedule( FileReloader("web-users.json", self.reload_userdict), get_home()) self._observer.start() self._certs = Certs()
def vip_main(agent_class, identity=None, version='0.1', **kwargs): """Default main entry point implementation for VIP agents.""" try: # If stdout is a pipe, re-open it line buffered if isapipe(sys.stdout): # Hold a reference to the previous file object so it doesn't # get garbage collected and close the underlying descriptor. stdout = sys.stdout sys.stdout = os.fdopen(stdout.fileno(), 'w', 1) # Quiet printing of KeyboardInterrupt by greenlets Hub = gevent.hub.Hub Hub.NOT_ERROR = Hub.NOT_ERROR + (KeyboardInterrupt,) config = os.environ.get('AGENT_CONFIG') identity = os.environ.get('AGENT_VIP_IDENTITY', identity) message_bus = os.environ.get('MESSAGEBUS', 'zmq') if identity is not None: if not is_valid_identity(identity): _log.warn('Deprecation warining') _log.warn( 'All characters in {identity} are not in the valid set.' .format(idenity=identity)) address = get_address() agent_uuid = os.environ.get('AGENT_UUID') volttron_home = get_home() from volttron.platform.certs import Certs certs = Certs() if os.path.isfile(certs.remote_cert_bundle_file()): os.environ['REQUESTS_CA_BUNDLE'] = certs.remote_cert_bundle_file() agent = agent_class(config_path=config, identity=identity, address=address, agent_uuid=agent_uuid, volttron_home=volttron_home, version=version, message_bus=message_bus, **kwargs) try: run = agent.run except AttributeError: run = agent.core.run task = gevent.spawn(run) try: task.join() finally: task.kill() except KeyboardInterrupt: pass
def __init__(self, serverkey, identity, address, bind_web_address, aip, volttron_central_address=None, volttron_central_rmq_address=None, web_ssl_key=None, web_ssl_cert=None, **kwargs): """Initialize the discovery service with the serverkey serverkey is the public key in order to access this volttron's bus. """ super(MasterWebService, self).__init__(identity, address, **kwargs) self.bind_web_address = bind_web_address self.serverkey = serverkey self.instance_name = None self.registeredroutes = [] self.peerroutes = defaultdict(list) self.pathroutes = defaultdict(list) # These will be used if set rather than the # any of the internal agent's certificates self.web_ssl_key = web_ssl_key self.web_ssl_cert = web_ssl_cert # Maps from endpoint to peer. self.endpoints = {} self.aip = aip self.volttron_central_address = volttron_central_address self.volttron_central_rmq_address = volttron_central_rmq_address # If vc is this instance then make the vc address the same as # the web address. if not self.volttron_central_address: self.volttron_central_address = bind_web_address if not mimetypes.inited: mimetypes.init() self._certs = Certs() self._csr_endpoints = None self.appContainer = None self._server_greenlet = None self._admin_endpoints = None
def test_vstart_expired_server_cert(request, instance): """ Test error when volttron is started with expired server cert when RMQ server is already running :param request: pytest request object :parma instance: volttron instance for testing """ # replace certs crts = instance.certsobj try: # overwrite certificates with quick expiry certs (root_ca, server_cert_name, admin_cert_name) = \ Certs.get_admin_cert_names(instance.instance_name) crts.create_ca_signed_cert(server_cert_name, type='server', fqdn=fqdn, valid_days=0.0001) gevent.sleep(9) try: instance.startup_platform(vip_address=get_rand_vip(), timeout=10) except Exception as e: assert e.message.startswith("Platform startup failed. Please check volttron.log") assert not (instance.is_running()) # Rabbitmq log would show # "TLS server: In state certify received CLIENT ALERT: Fatal - # Certificate Expired" except Exception as e: pytest.fail("Test failed with exception: {}".format(e))
def test_certificate_directories(temp_volttron_home): certs = Certs() paths = (certs.certs_pending_dir, certs.private_dir, certs.cert_dir, certs.remote_cert_dir, certs.csr_pending_dir, certs.ca_db_dir) for p in paths: assert os.path.exists(p)
def test_vstart_expired_admin_cert(request, instance): """ Test error when volttron is started with expired admin cert when RMQ server is already running :param request: pytest request object :param instance: volttron instance for testing """ # replace certs crts = instance.certsobj try: # overwrite certificates with quick expiry certs (root_ca, server_cert_name, admin_cert_name) = Certs.get_admin_cert_names(instance.instance_name) crts.create_ca_signed_cert(admin_cert_name, type='client', fqdn=fqdn, valid_days=0.0001) gevent.sleep(20) instance.startup_platform(vip_address=get_rand_vip()) gevent.sleep(5) assert instance.is_running() # MGMT PLUGIN DOES NOT COMPLAIN ABOUT EXPIRED ADMIN CERT?? May be because we send the password too ? cmd = ['volttron-ctl', 'rabbitmq', 'list-users'] process = subprocess.Popen(cmd, env=instance.env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) except Exception as e: pytest.fail("Test failed with exception: {}".format(e))
def setup(self): """Creates paths for used directories for the instance.""" for path in [self.run_dir, self.config_dir, self.install_dir]: if not os.path.exists(path): # others should have read and execute access to these directory # so explicitly set to 755. _log.debug("Setting up 755 permissions for path {}".format( path)) os.makedirs(path) os.chmod(path, 0o755) # Create certificates directory and its subdirectory at start of platform # so if volttron is run in secure mode, the first agent install would already have # the directories ready. In secure mode, agents will be run as separate user and will # not have access to create these directories Certs()
def test_expired_ca_cert_after_vstart(request, instance): """ Test error when CA cert expires after volttron has started. Once CA cert expires, can't install agent or can't get agent status. CA certificate needs to be recreated and client certs have to :param request: pytest request object :param instance: instance of volttron using rmq and ssl """ crts = instance.certsobj try: (root_ca, server_cert_name, admin_cert_name) = \ Certs.get_admin_cert_names(instance.instance_name) data = {'C': 'US', 'ST': 'Washington', 'L': 'Richland', 'O': 'pnnl', 'OU': 'volttron', 'CN': instance.instance_name + "_root_ca"} crts.create_root_ca(valid_days=0.0005, **data) print("current time after root ca:{}".format(datetime.datetime.utcnow())) copy(crts.cert_file(crts.root_ca_name), crts.cert_file(crts.trusted_ca_name)) crts.create_signed_cert_files(server_cert_name, cert_type='server', fqdn=fqdn) crts.create_signed_cert_files(admin_cert_name, cert_type='client') instance.startup_platform(vip_address=get_rand_vip()) print("current time after platform start:{}".format(datetime.datetime.utcnow())) gevent.sleep(30) # wait for CA to expire # Can't install new agent with pytest.raises(RuntimeError) as exec_info: agent = instance.install_agent( agent_dir=get_examples("ListenerAgent"), vip_identity="listener2", start=True) assert exec_info.type is RuntimeError except Exception as e: pytest.fail("Test failed with exception: {}".format(e)) finally: # can't do clean shutdown with expired ca. terminate process # and clean up manually instance.p_process.terminate() stop_rabbit(rmq_home=instance.rabbitmq_config_obj.rmq_home, env=instance.env, quite=True) if not instance.skip_cleanup: shutil.rmtree(instance.volttron_home)
def test_expired_server_cert_after_vstart(request, instance): """ Test error when server cert expires after volttron has started :param request: pytest request object :param instance: instance of volttron using rmq and ssl """ crts = instance.certsobj try: (root_ca, server_cert_name, admin_cert_name) = \ Certs.get_admin_cert_names(instance.instance_name) crts.create_signed_cert_files(server_cert_name, cert_type='server', fqdn=fqdn, valid_days=0.0004) # 34.5 seconds print("current time:{}".format(datetime.datetime.utcnow())) instance.startup_platform(vip_address=get_rand_vip()) print("current time:{}".format(datetime.datetime.utcnow())) agent = instance.install_agent( agent_dir=get_examples("ListenerAgent"), vip_identity="listener1", start=True) gevent.sleep(20) print("Attempting agent install after server certificate expiry") with pytest.raises(RuntimeError) as exec_info: agent = instance.install_agent( agent_dir=get_examples("ListenerAgent"), vip_identity="listener2", start=True) pytest.fail("Agent install should fail") assert exec_info.type is RuntimeError # Restore server cert and restart rmq ssl, wait for 30 seconds for volttron to reconnect crts.create_signed_cert_files(server_cert_name, cert_type='server', fqdn=fqdn) restart_ssl(rmq_home=instance.rabbitmq_config_obj.rmq_home, env=instance.env) gevent.sleep(15) # test setup sets the volttron reconnect wait to 5 seconds # status of first agent would still be fine and it would # continue to publish hearbeat. assert instance.is_agent_running(agent) instance.remove_agent(agent) except Exception as e: pytest.fail("Test failed with exception: {}".format(e))
def __init__(self, **kwargs): self.discovery_address = kwargs.pop('discovery_address') self.vip_address = kwargs.pop('vip-address', None) self.serverkey = kwargs.pop('serverkey', None) self.instance_name = kwargs.pop('instance-name') try: self.rmq_address = kwargs.pop('rmq-address') except KeyError: self.messagebus_type = 'zmq' else: self.messagebus_type = 'rmq' try: self.rmq_ca_cert = kwargs.pop('rmq-ca-cert') except KeyError: self.messagebus_type = 'zmq' else: self.messagebus_type = 'rmq' self.certs = Certs() assert len(kwargs) == 0
def test_vstart_expired_ca_cert(request, instance): """ Test error when volttron is started with expired CA cert when rabbitmq server is already running :param request: pytest request object :parma instance: volttron instance for testing """ crts = instance.certsobj try: # overwrite certificates with quick expiry certs (root_ca, server_cert_name, admin_cert_name) = \ Certs.get_admin_cert_names(instance.instance_name) data = {'C': 'US', 'ST': 'Washington', 'L': 'Richland', 'O': 'pnnl', 'OU': 'volttron', 'CN': instance.instance_name+"_root_ca"} crts.create_root_ca(valid_days=0.0001, **data) copy(crts.cert_file(crts.root_ca_name), crts.cert_file(crts.trusted_ca_name)) crts.create_ca_signed_cert(server_cert_name, type='server', fqdn=fqdn) crts.create_ca_signed_cert(admin_cert_name, type='client') gevent.sleep(9) print("Attempting to start volttron after cert expiry") try: # it fails fast. send a timeout instead of waiting for default timeout instance.startup_platform(vip_address=get_rand_vip(), timeout=10) pytest.fail("platform should not start") except Exception as e: assert e.message.startswith("Platform startup failed. Please check volttron.log") assert not (instance.is_running()) # Rabbitmq log would show Fatal certificate expired except Exception as e: pytest.fail("Test failed with exception: {}".format(e))
def certs_profile_1(certificate_dir, fqdn=None, num_server_certs=1, num_client_certs=3): """ Profile 1 generates the specified number of server and client certificates all signed by the same self-signed certificate. Usage: with certs_profile_1("/tmp/abc", 1, 2) as certs: ... :param certificate_dir: :return: """ certs = Certs(certificate_dir) data = { 'C': 'US', 'ST': 'Washington', 'L': 'Richland', 'O': 'pnnl', 'OU': 'volttron_test', 'CN': "myca" } ca_cert, ca_pk = certs.create_root_ca(**data) ns = SimpleNamespace(ca_cert=ca_cert, ca_key=ca_pk, ca_cert_file=certs.cert_file(certs.root_ca_name), ca_key_file=certs.private_key_file( certs.root_ca_name), server_certs=[], client_certs=[]) for x in range(num_server_certs): cert, key = certs.create_signed_cert_files(f"server{x}", cert_type="server", fqdn=fqdn) cert_ns = SimpleNamespace( key=key, cert=cert, cert_file=certs.cert_file(f"server{x}"), key_file=certs.private_key_file(f"server{x}")) ns.server_certs.append(cert_ns) for x in range(num_client_certs): cert, pk1 = certs.create_signed_cert_files(f"client{x}") cert_ns = SimpleNamespace( key=pk1, cert=cert, cert_file=certs.cert_file(f"client{x}"), key_file=certs.private_key_file(f"client{x}")) ns.client_certs.append(cert_ns) yield ns
def test_create_root_ca(temp_volttron_home): certs = Certs() assert not certs.ca_exists() certs.create_root_ca() assert certs.ca_exists()
def get_user_claim_from_bearer(bearer): certs = Certs() pubkey = certs.get_cert_public_key(get_fq_identity(MASTER_WEB)) return jwt.decode(bearer, pubkey, algorithms='RS256')
def certs_profile_2(certificate_dir, fqdn=None, num_server_certs=1, num_client_certs=3): """ Profile 2 generates the specified number of server and client certificates all signed by the same self-signed certificate. Usage: certs = certs_profile_1("/tmp/abc", 1, 2) ... :param certificate_dir: :return: ns """ certs = Certs(certificate_dir) data = {'C': 'US', 'ST': 'Washington', 'L': 'Richland', 'O': 'pnnl', 'OU': 'volttron_test', 'CN': "myca"} if not certs.ca_exists(): ca_cert, ca_pk = certs.create_root_ca(**data) # If the root ca already exists, get ca_cert and ca_pk from current root ca else: ca_cert = certs.cert(certs.root_ca_name) ca_pk = _load_key(certs.private_key_file(certs.root_ca_name)) # print(f"ca_cert: {ca_cert}") # print(f"ca_pk: {ca_pk}") # print(f"ca_pk_bytes: {certs.get_pk_bytes(certs.root_ca_name)}") ns = dict(ca_cert=ca_cert, ca_key=ca_pk, ca_cert_file=certs.cert_file(certs.root_ca_name), ca_key_file=certs.private_key_file(certs.root_ca_name), server_certs=[], client_certs=[]) for x in range(num_server_certs): cert, key = certs.create_signed_cert_files(f"server{x}", cert_type="server", fqdn=fqdn) cert_ns = dict(key=key, cert=cert, cert_file=certs.cert_file(f"server{x}"), key_file=certs.private_key_file(f"server{x}")) ns['server_certs'].append(cert_ns) for x in range(num_client_certs): cert, pk1 = certs.create_signed_cert_files(f"client{x}") cert_ns = dict(key=pk1, cert=cert, cert_file=certs.cert_file(f"client{x}"), key_file=certs.private_key_file(f"client{x}")) ns['client_certs'].append(cert_ns) return ns
class AdminEndpoints(object): def __init__(self, rmq_mgmt, ssl_public_key): self._rmq_mgmt = rmq_mgmt self._ssl_public_key = ssl_public_key self._userdict = None self.reload_userdict() self._observer = Observer() self._observer.schedule( FileReloader("web-users.json", self.reload_userdict), get_home()) self._observer.start() self._certs = Certs() def reload_userdict(self): webuserpath = os.path.join(get_home(), 'web-users.json') self._userdict = PersistentDict(webuserpath) def get_routes(self): """ Returns a list of tuples with the routes for the adminstration endpoints available in it. :return: """ return [(re.compile('^/admin.*'), 'callable', self.admin)] def admin(self, env, data): if len(self._userdict) == 0: if env.get('REQUEST_METHOD') == 'POST': decoded = dict((k, v if len(v) > 1 else v[0]) for k, v in urlparse.parse_qs(data).iteritems()) username = decoded.get('username') pass1 = decoded.get('password1') pass2 = decoded.get('password2') if pass1 == pass2 and pass1 is not None: _log.debug("Setting master password") self.add_user(username, pass1, groups=['admin']) return Response('', status='302', headers={'Location': '/admin/login.html'}) template = template_env(env).get_template('first.html') return Response(template.render()) if 'login.html' in env.get('PATH_INFO') or '/admin/' == env.get( 'PATH_INFO'): template = template_env(env).get_template('login.html') return Response(template.render()) return self.verify_and_dispatch(env, data) def verify_and_dispatch(self, env, data): """ Verify that the user is an admin and dispatch :param env: web environment :param data: data associated with a web form or json/xml request data :return: Response object. """ from volttron.platform.web import get_user_claims, NotAuthorized try: claims = get_user_claims(env) except NotAuthorized: _log.error("Unauthorized user attempted to connect to {}".format( env.get('PATH_INFO'))) return Response('<h1>Unauthorized User</h1>', status="401 Unauthorized") # Make sure we have only admins for viewing this. if 'admin' not in claims.get('groups'): return Response('<h1>Unauthorized User</h1>', status="401 Unauthorized") # Make sure we have only admins for viewing this. if 'admin' not in claims.get('groups'): return Response('<h1>Unauthorized User</h1>', status="401 Unauthorized") path_info = env.get('PATH_INFO') if path_info.startswith('/admin/api/'): return self.__api_endpoint(path_info[len('/admin/api/'):], data) if path_info.endswith('html'): page = path_info.split('/')[-1] try: template = template_env(env).get_template(page) except TemplateNotFound: return Response("<h1>404 Not Found</h1>", status="404 Not Found") if page == 'list_certs.html': html = template.render( certs=self._certs.get_all_cert_subjects()) elif page == 'pending_csrs.html': html = template.render( csrs=self._certs.get_pending_csr_requests()) else: # A template with no params. html = template.render() return Response(html) template = template_env(env).get_template('index.html') resp = template.render() return Response(resp) def __api_endpoint(self, endpoint, data): _log.debug("Doing admin endpoint {}".format(endpoint)) if endpoint == 'certs': response = self.__cert_list_api() elif endpoint == 'pending_csrs': response = self.__pending_csrs_api() elif endpoint.startswith('approve_csr/'): response = self.__approve_csr_api(endpoint.split('/')[1]) elif endpoint.startswith('deny_csr/'): response = self.__deny_csr_api(endpoint.split('/')[1]) elif endpoint.startswith('delete_csr/'): response = self.__delete_csr_api(endpoint.split('/')[1]) else: response = Response( '{"status": "Unknown endpoint {}"}'.format(endpoint), content_type="application/json") return response def __approve_csr_api(self, common_name): try: _log.debug("Creating cert and permissions for user: {}".format( common_name)) self._certs.approve_csr(common_name) permissions = self._rmq_mgmt.get_default_permissions(common_name) self._rmq_mgmt.create_user_with_permissions( common_name, permissions, True) data = dict(status=self._certs.get_csr_status(common_name), cert=self._certs.get_cert_from_csr(common_name)) except ValueError as e: data = dict(status="ERROR", message=e.message) return Response(json.dumps(data), content_type="application/json") def __deny_csr_api(self, common_name): try: self._certs.deny_csr(common_name) data = dict(status="DENIED", message="The administrator has denied the request") except ValueError as e: data = dict(status="ERROR", message=e.message) return Response(json.dumps(data), content_type="application/json") def __delete_csr_api(self, common_name): try: self._certs.delete_csr(common_name) data = dict(status="DELETED", message="The administrator has denied the request") except ValueError as e: data = dict(status="ERROR", message=e.message) return Response(json.dumps(data), content_type="application/json") def __pending_csrs_api(self): csrs = [c for c in self._certs.get_pending_csr_requests()] return Response(json.dumps(csrs), content_type="application/json") def __cert_list_api(self): subjects = [ dict(common_name=x.common_name) for x in self._certs.get_all_cert_subjects() ] return Response(json.dumps(subjects), content_type="application/json") def add_user(self, username, unencrypted_pw, groups=[], overwrite=False): if self._userdict.get(username): raise ValueError("Already exists!") if groups is None: groups = [] hashed_pass = argon2.hash(unencrypted_pw) self._userdict[username] = dict(hashed_password=hashed_pass, groups=groups) self._userdict.async_sync()
def __init__(self, core): self._core = weakref.ref(core) self._certs = Certs() self._auto_allow_csr = False
class CSREndpoints(object): def __init__(self, core): self._core = weakref.ref(core) self._certs = Certs() self._auto_allow_csr = False @property def auto_allow_csr(self): return self._auto_allow_csr @auto_allow_csr.setter def auto_allow_csr(self, value): self._auto_allow_csr = value def get_routes(self): """ Returns a list of tuples with the routes for authentication. Tuple should have the following: - regular expression for calling the endpoint - 'callable' keyword specifying that a method is being specified - the method that should be used to call when the regular expression matches code: return [ (re.compile('^/csr/request_new$'), 'callable', self._csr_request_new) ] :return: """ return [(re.compile('^/csr/request_new$'), 'callable', self._csr_request_new)] def _csr_request_new(self, env, data): _log.debug("New csr request") if not isinstance(data, dict): try: request_data = jsonapi.loads(data) except: _log.error( "Invalid data for csr request. Must be json serializable") return Response() else: request_data = data.copy() csr = request_data.get('csr').encode("utf-8") identity = self._certs.get_csr_common_name(csr) # The identity must start with the current instances name or it is a failure. if not identity.startswith(get_platform_instance_name() + "."): json_response = dict( status="ERROR", message="CSR must start with instance name: {}".format( get_platform_instance_name())) Response(jsonapi.dumps(json_response), content_type='application/json', headers={'Content-type': 'application/json'}) csr_file = self._certs.save_pending_csr_request( env.get('REMOTE_ADDR'), identity, csr) if self._auto_allow_csr: _log.debug( "Creating cert and permissions for user: {}".format(identity)) status = self._certs.get_csr_status(identity) json_response = dict(status=status) if status == 'APPROVED': try: json_response['cert'] = self._certs.get_cert_from_csr( identity) except Exception as e: _log.error(f"Exception getting cert from csr {e}") else: try: cert = self._certs.approve_csr(identity) #permissions = self._core().rmq_mgmt.get_default_permissions(identity) _log.debug(f"CREATING NEW RMQ USER: {identity}") permissions = dict(configure=".*", read=".*", write=".*") self._core().rmq_mgmt.create_user_with_permissions( identity, permissions, True) json_response = dict(status="SUCCESSFUL", cert=cert) except BaseException as e: _log.error( f"Exception in approving csr/creating user in auto_allow_csr mode: {e}" ) else: status = self._certs.get_csr_status(identity) cert = self._certs.get_cert_from_csr(identity) json_response = dict(status=status) if status == "APPROVED": json_response['cert'] = self._certs.get_cert_from_csr(identity) elif status == "PENDING": json_response[ 'message'] = "The request is pending admininstrator approval." elif status == "DENIED": json_response[ 'message'] = "The request has been denied by the administrator." elif status == "UNKNOWN": json_response[ 'message'] = "An unknown common name was specified to the server {}".format( identity) else: json_response[ 'message'] = "An unkonwn error has occured during the respons phase" response = None try: if json_response.get('cert', None): json_response['cert'] = json_response['cert'].decode('utf-8') response = Response(jsonapi.dumps(json_response), content_type='application/json', headers={'Content-type': 'application/json'}) except BaseException as e: _log.error(f"Exception creating Response {e}") return response
def request_cert(self, csr_server, fully_qualified_local_identity, discovery_info): """ Get a signed csr from the csr_server endpoint This method will create a csr request that is going to be sent to the signing server. :param csr_server: the http(s) location of the server to connect to. :return: """ if get_messagebus() != 'rmq': raise ValueError( "Only can create csr for rabbitmq based platform in ssl mode.") # from volttron.platform.web import DiscoveryInfo config = RMQConfig() if not config.is_ssl: raise ValueError( "Only can create csr for rabbitmq based platform in ssl mode.") # info = discovery_info # if info is None: # info = DiscoveryInfo.request_discovery_info(csr_server) certs = Certs() csr_request = certs.create_csr(fully_qualified_local_identity, discovery_info.instance_name) # The csr request requires the fully qualified identity that is # going to be connected to the external instance. # # The remote instance id is the instance name of the remote platform # concatenated with the identity of the local fully quallified identity. remote_cert_name = "{}.{}".format(discovery_info.instance_name, fully_qualified_local_identity) remote_ca_name = discovery_info.instance_name + "_ca" # if certs.cert_exists(remote_cert_name, True): # return certs.cert(remote_cert_name, True) json_request = dict( csr=csr_request, identity= remote_cert_name, # get_platform_instance_name()+"."+self._core().identity, hostname=config.hostname) response = requests.post(csr_server + "/csr/request_new", json=json.dumps(json_request), verify=False) _log.debug("The response: {}".format(response)) # from pprint import pprint # pprint(response.json()) j = response.json() status = j.get('status') cert = j.get('cert') message = j.get('message', '') if status == 'SUCCESSFUL' or status == 'APPROVED': certs.save_remote_info(fully_qualified_local_identity, remote_cert_name, cert, remote_ca_name, discovery_info.rmq_ca_cert) elif status == 'PENDING': _log.debug("Pending CSR request for {}".format(remote_cert_name)) elif status == 'DENIED': _log.error("Denied from remote machine. Shutting down agent.") status = Status.build( BAD_STATUS, context="Administrator denied remote connection. Shutting down" ) self._owner.vip.health.set_status(status.status, status.context) self._owner.vip.health.send_alert( self._core().identity + "_DENIED", status) self._core().stop() return None elif status == 'ERROR': err = "Error retrieving certificate from {}\n".format( config.hostname) err += "{}".format(message) raise ValueError(err) else: # No resposne return None certfile = certs.cert_file(remote_cert_name, remote=True) if certs.cert_exists(remote_cert_name, remote=True): return certfile else: return status, message
class MasterWebService(Agent): """The service that is responsible for managing and serving registered pages Agents can register either a directory of files to serve or an rpc method that will be called during the request process. """ def __init__(self, serverkey, identity, address, bind_web_address, aip, volttron_central_address=None, volttron_central_rmq_address=None, web_ssl_key=None, web_ssl_cert=None, **kwargs): """Initialize the discovery service with the serverkey serverkey is the public key in order to access this volttron's bus. """ super(MasterWebService, self).__init__(identity, address, **kwargs) self.bind_web_address = bind_web_address self.serverkey = serverkey self.instance_name = None self.registeredroutes = [] self.peerroutes = defaultdict(list) self.pathroutes = defaultdict(list) # These will be used if set rather than the # any of the internal agent's certificates self.web_ssl_key = web_ssl_key self.web_ssl_cert = web_ssl_cert # Maps from endpoint to peer. self.endpoints = {} self.aip = aip self.volttron_central_address = volttron_central_address self.volttron_central_rmq_address = volttron_central_rmq_address # If vc is this instance then make the vc address the same as # the web address. if not self.volttron_central_address: self.volttron_central_address = bind_web_address if not mimetypes.inited: mimetypes.init() self._certs = Certs() self._csr_endpoints = None self.appContainer = None self._server_greenlet = None self._admin_endpoints = None # pylint: disable=unused-argument @Core.receiver('onsetup') def onsetup(self, sender, **kwargs): self.vip.rpc.export(self._auto_allow_csr, 'auto_allow_csr') self.vip.rpc.export(self._is_auto_allow_csr, 'is_auto_allow_csr') def _is_auto_allow_csr(self): return self._csr_endpoints.auto_allow_csr def _auto_allow_csr(self, auto_allow_csr): self._csr_endpoints.auto_allow_csr = auto_allow_csr def remove_unconnnected_routes(self): peers = self.vip.peerlist().get() for p in self.peerroutes: if p not in peers: del self.peerroutes[p] @RPC.export def get_user_claims(self, bearer): from volttron.platform.web import get_user_claim_from_bearer return get_user_claim_from_bearer(bearer) @RPC.export def websocket_send(self, endpoint, message): _log.debug("Sending data to {} with message {}".format( endpoint, message)) self.appContainer.websocket_send(endpoint, message) @RPC.export def get_bind_web_address(self): return self.bind_web_address @RPC.export def get_serverkey(self): return self.serverkey @RPC.export def get_volttron_central_address(self): """Return address of external Volttron Central Note: this only applies to Volltron Central agents that are running on a different platform. """ return self.volttron_central_address @RPC.export def register_endpoint(self, endpoint, res_type): """ RPC method to register a dynamic route. :param endpoint: :return: """ _log.debug('Registering route with endpoint: {}'.format(endpoint)) # Get calling peer from the rpc context peer = bytes(self.vip.rpc.context.vip_message.peer) _log.debug('Route is associated with peer: {}'.format(peer)) if endpoint in self.endpoints: _log.error("Attempting to register an already existing endpoint.") _log.error("Ignoring registration.") raise DuplicateEndpointError( "Endpoint {} is already an endpoint".format(endpoint)) self.endpoints[endpoint] = (peer, res_type) @RPC.export def register_agent_route(self, regex, fn): """ Register an agent route to an exported function. When a http request is executed and matches the passed regular expression then the function on peer is executed. """ # Get calling peer from the rpc context peer = bytes(self.vip.rpc.context.vip_message.peer) _log.info( 'Registering agent route expression: {} peer: {} function: {}'. format(regex, peer, fn)) # TODO: inspect peer for function compiled = re.compile(regex) self.peerroutes[peer].append(compiled) self.registeredroutes.insert(0, (compiled, 'peer_route', (peer, fn))) @RPC.export def unregister_all_agent_routes(self): # Get calling peer from the rpc context peer = bytes(self.vip.rpc.context.vip_message.peer) _log.info('Unregistering agent routes for: {}'.format(peer)) for regex in self.peerroutes[peer]: out = [cp for cp in self.registeredroutes if cp[0] != regex] self.registeredroutes = out del self.peerroutes[peer] for regex in self.pathroutes[peer]: out = [cp for cp in self.registeredroutes if cp[0] != regex] self.registeredroutes = out del self.pathroutes[peer] _log.debug(self.endpoints) endpoints = self.endpoints.copy() endpoints = { i: endpoints[i] for i in endpoints if endpoints[i][0] != peer } _log.debug(endpoints) self.endpoints = endpoints @RPC.export def register_path_route(self, regex, root_dir): _log.info('Registering path route: {} {}'.format(regex, root_dir)) # Get calling peer from the rpc context peer = bytes(self.vip.rpc.context.vip_message.peer) compiled = re.compile(regex) self.pathroutes[peer].append(compiled) # in order for this agent to pass against the default route we want this # to be before the last route which will resolve to .* self.registeredroutes.insert( len(self.registeredroutes) - 1, (compiled, 'path', root_dir)) @RPC.export def register_websocket(self, endpoint): identity = bytes(self.vip.rpc.context.vip_message.peer) _log.debug('Caller identity: {}'.format(identity)) _log.debug('REGISTERING ENDPOINT: {}'.format(endpoint)) if self.appContainer: self.appContainer.create_ws_endpoint(endpoint, identity) else: _log.error('Attempting to register endpoint without web' 'subsystem initialized') raise AttributeError("self does not contain" " attribute appContainer") @RPC.export def unregister_websocket(self, endpoint): identity = bytes(self.vip.rpc.context.vip_message.peer) _log.debug('Caller identity: {}'.format(identity)) self.appContainer.destroy_ws_endpoint(endpoint) def _redirect_index(self, env, start_response, data=None): """ Redirect to the index page. @param env: @param start_response: @param data: @return: """ start_response('302 Found', [('Location', '/index.html')]) return ['1'] def _allow(self, environ, start_response, data=None): _log.info('Allowing new vc instance to connect to server.') jsondata = jsonapi.loads(data) json_validate_request(jsondata) assert jsondata.get('method') == 'allowvc' assert jsondata.get('params') params = jsondata.get('params') if isinstance(params, list): vcpublickey = params[0] else: vcpublickey = params.get('vcpublickey') assert vcpublickey assert len(vcpublickey) == 43 authfile = AuthFile() authentry = AuthEntry(credentials=vcpublickey) try: authfile.add(authentry) except AuthFileEntryAlreadyExists: pass start_response('200 OK', [('Content-Type', 'application/json')]) return jsonapi.dumps(json_result(jsondata['id'], "Added")) def _get_discovery(self, environ, start_response, data=None): q = query.Query(self.core) self.instance_name = q.query('instance-name').get(timeout=60) addreses = q.query('addresses').get(timeout=60) external_vip = None for x in addreses: try: if not is_ip_private(x): external_vip = x break except IndexError: pass return_dict = {} if external_vip and self.serverkey: return_dict['serverkey'] = encode_key(self.serverkey) return_dict['vip-address'] = external_vip if self.instance_name: return_dict['instance-name'] = self.instance_name if self.core.messagebus == 'rmq': config = RMQConfig() rmq_address = None if config.is_ssl: rmq_address = "amqps://{host}:{port}/{vhost}".format( host=config.hostname, port=config.amqp_port_ssl, vhost=config.virtual_host) else: rmq_address = "amqp://{host}:{port}/{vhost}".format( host=config.hostname, port=config.amqp_port, vhost=config.virtual_host) return_dict['rmq-address'] = rmq_address return_dict['rmq-ca-cert'] = self._certs.cert( self._certs.root_ca_name).public_bytes( serialization.Encoding.PEM) return Response(jsonapi.dumps(return_dict), content_type="application/json") def app_routing(self, env, start_response): """ The main routing function that maps the incoming request to a response. Depending on the registered routes map the request data onto an rpc function or a specific named file. """ path_info = env['PATH_INFO'] if path_info.startswith('/http://'): path_info = path_info[path_info.index('/', len('/http://')):] # only expose a partial list of the env variables to the registered # agents. envlist = [ 'HTTP_USER_AGENT', 'PATH_INFO', 'QUERY_STRING', 'REQUEST_METHOD', 'SERVER_PROTOCOL', 'REMOTE_ADDR', 'HTTP_ACCEPT_ENCODING', 'HTTP_COOKIE', 'CONTENT_TYPE', 'HTTP_AUTHORIZATION', 'SERVER_NAME', 'wsgi.url_scheme', 'HTTP_HOST' ] data = env['wsgi.input'].read() passenv = dict((envlist[i], env[envlist[i]]) for i in range(0, len(envlist)) if envlist[i] in env.keys()) _log.debug('path_info is: {}'.format(path_info)) # Get the peer responsible for dealing with the endpoint. If there # isn't a peer then fall back on the other methods of routing. (peer, res_type) = self.endpoints.get(path_info, (None, None)) _log.debug('Peer path_info is associated with: {}'.format(peer)) if self.is_json_content(env): data = json.loads(data) # Only if https available and rmq for the admin area. if env['wsgi.url_scheme'] == 'https' and self.core.messagebus == 'rmq': # Load the publickey that was used to sign the login message through the env # parameter so agents can use it to verify the Bearer has specific # jwt claims passenv['WEB_PUBLIC_KEY'] = env[ 'WEB_PUBLIC_KEY'] = self._certs.get_cert_public_key( get_fq_identity(self.core.identity)) # if we have a peer then we expect to call that peer's web subsystem # callback to perform whatever is required of the method. if peer: _log.debug('Calling peer {} back with env={} data={}'.format( peer, passenv, data)) res = self.vip.rpc.call(peer, 'route.callback', passenv, data).get(timeout=60) if res_type == "jsonrpc": return self.create_response(res, start_response) elif res_type == "raw": return self.create_raw_response(res, start_response) env['JINJA2_TEMPLATE_ENV'] = tplenv # if ws4pi.socket is set then this connection is a web socket # and so we return the websocket response. if 'ws4py.socket' in env: return env['ws4py.socket'](env, start_response) for k, t, v in self.registeredroutes: if k.match(path_info): _log.debug( "MATCHED:\npattern: {}, path_info: {}\n v: {}".format( k.pattern, path_info, v)) _log.debug('registered route t is: {}'.format(t)) if t == 'callable': # Generally for locally called items. # Changing signature of the "locally" called points to return # a Response object. Our response object then will in turn # be processed and the response will be written back to the # calling client. try: retvalue = v(env, start_response, data) except TypeError: retvalue = self.process_response( start_response, v(env, data)) if isinstance(retvalue, Response): return self.process_response(start_response, retvalue) else: return retvalue elif t == 'peer_route': # RPC calls from agents on the platform _log.debug('Matched peer_route with pattern {}'.format( k.pattern)) peer, fn = (v[0], v[1]) res = self.vip.rpc.call(peer, fn, passenv, data).get(timeout=120) _log.debug(res) return self.create_response(res, start_response) elif t == 'path': # File service from agents on the platform. if path_info == '/': return self._redirect_index(env, start_response) server_path = v + path_info # os.path.join(v, path_info) _log.debug('Serverpath: {}'.format(server_path)) return self._sendfile(env, start_response, server_path) start_response('404 Not Found', [('Content-Type', 'text/html')]) return [b'<h1>Not Found</h1>'] def is_json_content(self, env): ct = env.get('CONTENT_TYPE') if ct is not None and 'application/json' in ct: return True return False def process_response(self, start_responsee, response): # process the response start_responsee(response.status, response.headers) return bytes(response.content) def create_raw_response(self, res, start_response): # If this is a tuple then we know we are going to have a response # and a headers portion of the data. if isinstance(res, tuple) or isinstance(res, list): if len(res) == 1: status, = res headers = () if len(res) == 2: headers = () status, response = res if len(res) == 3: status, response, headers = res start_response(status, headers) return base64.b64decode(response) else: start_response("500 Programming Error", [('Content-Type', 'text/html')]) _log.error("Invalid length of response tuple (must be 1-3)") return [b'Invalid response tuple (must contain 1-3 elements)'] def create_response(self, res, start_response): # Dictionaries are going to be treated as if they are meant to be json # serialized with Content-Type of application/json if isinstance(res, dict): # Note this is specific to volttron central agent and should # probably not be at this level of abstraction. _log.debug('res is a dictionary.') if 'error' in res.keys(): if res['error']['code'] == UNAUTHORIZED: start_response('401 Unauthorized', [('Content-Type', 'text/html')]) message = res['error']['message'] code = res['error']['code'] return [ b'<h1>{}</h1>\n<h2>CODE:{}</h2>'.format(message, code) ] start_response('200 OK', [('Content-Type', 'application/json')]) return jsonapi.dumps(res) elif isinstance(res, list): _log.debug('list implies [content, headers]') if len(res) == 2: start_response('200 OK', res[1]) return res[0] elif len(res) == 3: start_response(res[0], res[2]) return res[1] # If this is a tuple then we know we are going to have a response # and a headers portion of the data. if isinstance(res, tuple) or isinstance(res, list): if len(res) != 2: start_response("500 Programming Error", [('Content-Type', 'text/html')]) _log.error("Invalid length of response tuple (must be 2)") return [b'Invalid response tuple (must contain 2 elements)'] response, headers = res header_dict = dict(headers) if header_dict.get('Content-Encoding', None) == 'gzip': gzip_compress = zlib.compressobj(9, zlib.DEFLATED, zlib.MAX_WBITS | 16) data = gzip_compress.compress(response) + gzip_compress.flush() start_response('200 OK', headers) return data else: return response else: start_response('200 OK', [('Content-Type', 'application/json')]) return jsonapi.dumps(res) def _sendfile(self, env, start_response, filename): from wsgiref.util import FileWrapper status = '200 OK' _log.debug('SENDING FILE: {}'.format(filename)) guess = mimetypes.guess_type(filename)[0] _log.debug('MIME GUESS: {}'.format(guess)) basename = os.path.dirname(filename) if not os.path.exists(basename): start_response('404 Not Found', [('Content-Type', 'text/html')]) return [b'<h1>Not Found</h1>'] elif not os.path.isfile(filename): start_response('404 Not Found', [('Content-Type', 'text/html')]) return [b'<h1>Not Found</h1>'] if not guess: guess = 'text/plain' response_headers = [ ('Content-type', guess), ] start_response(status, response_headers) return FileWrapper(open(filename, 'r')) @Core.receiver('onstart') def startupagent(self, sender, **kwargs): import urlparse parsed = urlparse.urlparse(self.bind_web_address) ssl_key = self.web_ssl_key ssl_cert = self.web_ssl_cert if parsed.scheme == 'https': # Admin interface is only availble to rmq at present. if self.core.messagebus == 'rmq': self._admin_endpoints = AdminEndpoints( self.core.rmq_mgmt, self._certs.get_cert_public_key( get_fq_identity(self.core.identity))) if ssl_key is None or ssl_cert is None: # Because the master.web service certificate is a client to rabbitmq we # can't use it directly therefore we use the -server on the file to specify # the server based file. base_filename = get_fq_identity(self.core.identity) + "-server" ssl_cert = self._certs.cert_file(base_filename) ssl_key = self._certs.private_key_file(base_filename) if not os.path.isfile(ssl_cert) or not os.path.isfile(ssl_key): self._certs.create_ca_signed_cert(base_filename, type='server') hostname = parsed.hostname port = parsed.port _log.info('Starting web server binding to {}://{}:{}.'.format( parsed.scheme, hostname, port)) # Handle the platform.web routes here. self.registeredroutes.append( (re.compile('^/discovery/$'), 'callable', self._get_discovery)) self.registeredroutes.append( (re.compile('^/discovery/allow$'), 'callable', self._allow)) # these routes are only available for rmq based message bus # at present. if self.core.messagebus == 'rmq': # We need reference to the object so we can change the behavior of # whether or not to have auto certs be created or not. self._csr_endpoints = CSREndpoints(self.core) for rt in self._csr_endpoints.get_routes(): self.registeredroutes.append(rt) for rt in self._admin_endpoints.get_routes(): self.registeredroutes.append(rt) ssl_private_key = self._certs.get_private_key( get_fq_identity(self.core.identity)) for rt in AuthenticateEndpoints(ssl_private_key).get_routes(): self.registeredroutes.append(rt) static_dir = os.path.join(os.path.dirname(__file__), "static") self.registeredroutes.append((re.compile('^/.*$'), 'path', static_dir)) port = int(port) vhome = os.environ.get('VOLTTRON_HOME') logdir = os.path.join(vhome, "log") if not os.path.exists(logdir): os.makedirs(logdir) self.appContainer = WebApplicationWrapper(self, hostname, port) if ssl_key and ssl_cert: svr = WSGIServer((hostname, port), self.appContainer, certfile=ssl_cert, keyfile=ssl_key) else: svr = WSGIServer((hostname, port), self.appContainer) self._server_greenlet = gevent.spawn(svr.serve_forever) def _authenticate_route(self, env, start_response, data): scheme = env.get('wsgi.url_scheme') if scheme != 'https': _log.warning("Authentication should be through https") start_response("401 Unauthorized", [('Content-Type', 'text/html')]) return "<html><body><h1>401 Unauthorized</h1></body></html>" from pprint import pprint pprint(env) import jwt jwt.encode()
def request_cert( self, csr_server, fully_qualified_local_identity, discovery_info ): """ Get a signed csr from the csr_server endpoint This method will create a csr request that is going to be sent to the signing server. :param csr_server: the http(s) location of the server to connect to. :return: """ if get_messagebus() != "rmq": raise ValueError( "Only can create csr for rabbitmq based platform in ssl mode." ) # from volttron.platform.web import DiscoveryInfo config = RMQConfig() if not config.is_ssl: raise ValueError( "Only can create csr for rabbitmq based platform in ssl mode." ) # info = discovery_info # if info is None: # info = DiscoveryInfo.request_discovery_info(csr_server) certs = Certs() csr_request = certs.create_csr( fully_qualified_local_identity, discovery_info.instance_name ) # The csr request requires the fully qualified identity that is # going to be connected to the external instance. # # The remote instance id is the instance name of the remote platform # concatenated with the identity of the local fully quallified # identity. remote_cert_name = "{}.{}".format( discovery_info.instance_name, fully_qualified_local_identity ) remote_ca_name = discovery_info.instance_name + "_ca" # if certs.cert_exists(remote_cert_name, True): # return certs.cert(remote_cert_name, True) json_request = dict( csr=csr_request.decode("utf-8"), identity=remote_cert_name, # get_platform_instance_name()+"."+self._core().identity, hostname=config.hostname, ) request = grequests.post( csr_server + "/csr/request_new", json=jsonapi.dumps(json_request), verify=False, ) response = grequests.map([request]) if response and isinstance(response, list): response[0].raise_for_status() response = response[0] # response = requests.post(csr_server + "/csr/request_new", # json=jsonapi.dumps(json_request), # verify=False) _log.debug("The response: %s", response) j = response.json() status = j.get("status") cert = j.get("cert") message = j.get("message", "") remote_certs_dir = self.get_remote_certs_dir() if status == "SUCCESSFUL" or status == "APPROVED": certs.save_agent_remote_info( remote_certs_dir, fully_qualified_local_identity, remote_cert_name, cert.encode("utf-8"), remote_ca_name, discovery_info.rmq_ca_cert.encode("utf-8"), ) os.environ["REQUESTS_CA_BUNDLE"] = os.path.join( remote_certs_dir, "requests_ca_bundle" ) _log.debug( "Set os.environ requests ca bundle to %s", os.environ["REQUESTS_CA_BUNDLE"], ) elif status == "PENDING": _log.debug("Pending CSR request for {}".format(remote_cert_name)) elif status == "DENIED": _log.error("Denied from remote machine. Shutting down agent.") status = Status.build( BAD_STATUS, context="Administrator denied remote " "connection. " "Shutting down", ) self._owner.vip.health.set_status(status.status, status.context) self._owner.vip.health.send_alert( self._core().identity + "_DENIED", status ) self._core().stop() return None elif status == "ERROR": err = "Error retrieving certificate from {}\n".format( config.hostname ) err += "{}".format(message) raise ValueError(err) else: # No resposne return None certfile = os.path.join(remote_certs_dir, remote_cert_name + ".crt") if os.path.exists(certfile): return certfile else: return status, message