def test_discovery(scheme): vhome = create_volttron_home() # creates a vhome level key store keystore = KeyStore() serverkey = decode_key(keystore.public) # Depending upon scheme we enable/disable password jwt and certificate based jwt. if scheme == 'https': with certs_profile_1('/'.join([vhome, 'certs'])) as certs: config_params = dict(web_ssl_key=certs.server_certs[0].key_file, web_ssl_cert=certs.server_certs[0].cert_file) else: config_params = dict(web_secret_key=get_random_key()) with get_test_volttron_home(messagebus='zmq', config_params=config_params): instance_name = "booballoon" host, port = get_hostname_and_random_port() # this is the vip address address = f"tcp://{host}:{port}" def _construct_query_mock(core): """ Internal function that creates a concrete response for the data. when query('instance-name').get() is called the passed instance name is returned """ nonlocal instance_name, address kv = {"instance-name": instance_name, "addresses": [address]} return MockQuery(**kv) with mock.patch('volttron.platform.vip.agent.subsystems.query.Query', _construct_query_mock): host, port = get_hostname_and_random_port() bind_web_address = f"{scheme}://{host}:{port}" serverkey = decode_key(keystore.public) mws = MasterWebService(serverkey=serverkey, identity=MASTER_WEB, address=address, bind_web_address=bind_web_address, **config_params) mws.startupagent(sender='testweb') env = get_test_web_env("/discovery/") mock_start_response = mock.Mock() # A closingiterator is returned from the response object so we use the next # on the returned response. Then we can do json responses. response = mws.app_routing(env, mock_start_response).__next__() # load json into a dict for testing responses. response = jsonapi.loads(response.decode('utf-8')) assert response.get('instance-name') is not None assert instance_name == response.get('instance-name') assert keystore.public == response.get('serverkey') assert address == response.get('vip-address')
def get_volttron_multi_msgbus_instances(instance_name1=None, instance_name2=None): print("volttron_multi_messagebus source: {} sink: {}".format( request.param['source'], request.param['sink'])) sink_address = get_rand_vip() if request.param['sink'] == 'rmq_web': hostname, port = get_hostname_and_random_port() web_address = 'https://{hostname}:{port}'.format(hostname=hostname, port=port) messagebus = 'rmq' ssl_auth = True else: web_address = "http://{}".format(get_rand_ip_and_port()) messagebus = 'zmq' ssl_auth = False sink = build_wrapper(sink_address, ssl_auth=ssl_auth, messagebus=messagebus, bind_web_address=web_address, volttron_central_address=web_address, instance_name="volttron1") source_address = get_rand_vip() messagebus = 'zmq' ssl_auth = False if request.param['source'] == 'rmq': messagebus = 'rmq' ssl_auth = True if sink.messagebus == 'rmq': # sink_ca_file = sink.certsobj.cert_file(sink.certsobj.root_ca_name) source = build_wrapper( source_address, ssl_auth=ssl_auth, messagebus=messagebus, volttron_central_address=sink.bind_web_address, remote_platform_ca=sink.certsobj.cert_file( sink.certsobj.root_ca_name), instance_name='volttron2') else: source = build_wrapper( source_address, ssl_auth=ssl_auth, messagebus=messagebus, volttron_central_address=sink.bind_web_address, instance_name='volttron2') get_volttron_multi_msgbus_instances.source = source get_volttron_multi_msgbus_instances.sink = sink return source, sink
def test_authenticate_endpoint(scheme): kwargs = {} # Note this is not a context wrapper, it just does the creation for us vhome = create_volttron_home() if scheme == 'https': with certs_profile_1(vhome) as certs: kwargs['web_ssl_key'] = certs.server_certs[0].key_file kwargs['web_ssl_cert'] = certs.server_certs[0].cert_file else: kwargs['web_secret_key'] = binascii.hexlify( os.urandom(65)).decode('utf-8') host, port = get_hostname_and_random_port() kwargs['bind_web_address'] = f"{scheme}://{host}:{port}" # We are specifying the volttron_home here so we don't create an additional one. with get_test_volttron_home(messagebus='zmq', config_params=kwargs, volttron_home=vhome): # add a user so that we can actually log in. user = '******' passwd = 'cat' adminep = AdminEndpoints() adminep.add_user(user, passwd, groups=['foo', 'read-only']) expected_claims = dict(groups=['foo', 'read-only']) with get_master_web(**kwargs) as mw: data = urlencode(dict(username=user, password=passwd)).encode('utf-8') assert len(data) > 0 # test not sending any parameters. env = get_test_web_env("/authenticate", input_data=data, method='POST') mocked_start_response, response = get_server_response(env, mw) assert 3 == len(response.split(".")) claims = mw.get_user_claims(response) assert claims assert not DeepDiff(expected_claims, claims)
def volttron_instance_web(request): print( "volttron_instance_web (messagebus {messagebus} ssl_auth {ssl_auth})". format(**request.param)) address = get_rand_vip() if request.param['ssl_auth']: hostname, port = get_hostname_and_random_port() web_address = 'https://{hostname}:{port}'.format(hostname=hostname, port=port) else: web_address = "http://{}".format(get_rand_ip_and_port()) wrapper = build_wrapper(address, ssl_auth=request.param['ssl_auth'], messagebus=request.param['messagebus'], bind_web_address=web_address, volttron_central_address=web_address) yield wrapper cleanup_wrapper(wrapper)
def volttron_multi_messagebus(request): """ This fixture allows multiple two message bus types to be configured to work together This case will create a source (where data comes from) and a sink (where data goes to) to allow connections from source to sink to be tested for the different cases. In particular, the case of VolttronCentralPlatform, Forwarder and DataMover agents should use this case. :param request: :return: """ print("volttron_multi_messagebus source: {} sink: {}".format( request.param['source'], request.param['sink'])) sink_address = get_rand_vip() if request.param['sink'] == 'rmq_web': hostname, port = get_hostname_and_random_port() web_address = 'https://{hostname}:{port}'.format(hostname=hostname, port=port) messagebus = 'rmq' ssl_auth = True else: web_address = "http://{}".format(get_rand_ip_and_port()) messagebus = 'zmq' ssl_auth = False sink = build_wrapper(sink_address, ssl_auth=ssl_auth, messagebus=messagebus, bind_web_address=web_address, volttron_central_address=web_address) source_address = get_rand_vip() messagebus = 'zmq' ssl_auth = False if request.param['source'] == 'rmq': messagebus = 'rmq' ssl_auth = True if sink.messagebus == 'rmq': # sink_ca_file = sink.certsobj.cert_file(sink.certsobj.root_ca_name) source = build_wrapper(source_address, ssl_auth=ssl_auth, messagebus=messagebus, volttron_central_address=sink.bind_web_address, remote_platform_ca=sink.certsobj.cert_file( sink.certsobj.root_ca_name)) if source.messagebus == 'rmq': # The _ca is how the auth subsystem saves the remote cert from discovery. We # are effectively doing that here instead of making the discovery call. source.certsobj.save_remote_cert( sink.certsobj.root_ca_name + "_ca", sink.certsobj.ca_cert(public_bytes=True)) else: source = build_wrapper(source_address, ssl_auth=ssl_auth, messagebus=messagebus, volttron_central_address=sink.bind_web_address) yield source, sink cleanup_wrapper(source) cleanup_wrapper(sink)
def create_rmq_volttron_setup(vhome=None, ssl_auth=False, env=None): """ Set-up rabbitmq broker for volttron testing: - Install config and rabbitmq_config.yml in VOLTTRON_HOME - Create virtual host, exchanges, certificates, and users - Start rabbitmq server :param instance_name: the canonical name for the instance being setup. :param vhome: volttron home directory, if None, use default from environment :param ssl_auth: ssl authentication, if true, all users of message queue must authenticate """ if vhome: os.environ['VOLTTRON_HOME'] = vhome else: vhome = get_home() # Build default config file object, which we will then update to fit the # current context the code is running in. rabbit_config_obj = RabbitTestConfig() # for docker this will be setup so we can always use this for the home if os.environ.get('RMQ_HOME'): rabbit_config_obj.rmq_home = os.environ.get('RMQ_HOME') else: rmq_home_env_file = os.path.expanduser("~/.volttron_rmq_home") if not os.path.isfile(rmq_home_env_file): raise ValueError("Rabbitmq home dir can't be found please\n run bootstrap.py --rabbitmq") with open(rmq_home_env_file, 'r') as f: rabbit_config_obj.rmq_home = f.read().strip() os.environ['RMQ_HOME'] = rabbit_config_obj.rmq_home # instance name is the basename of the volttron home now. rabbit_config_obj.instance_name = rabbit_config_obj.node_name = os.path.basename(vhome) os.mkdir(os.path.join(vhome, "rmq_node_data")) rabbit_config_obj.rmq_conf_file = os.path.join(vhome, "rmq_node_data", rabbit_config_obj.node_name + "-rmq.conf") rabbit_config_obj.rmq_env_file = os.path.join(vhome, "rmq_node_data", rabbit_config_obj.node_name + "-rmq-env.conf") env['RABBITMQ_CONF_ENV_FILE'] = rabbit_config_obj.rmq_env_file # Create rabbitmq config for test rabbit_config_obj.rabbitmq_config['ssl'] = str(ssl_auth) host, rabbit_config_obj.rabbitmq_config['amqp-port'] = get_hostname_and_random_port() host, rabbit_config_obj.rabbitmq_config['amqp-port-ssl'] = get_hostname_and_random_port() host, rabbit_config_obj.rabbitmq_config['mgmt-port'] = get_hostname_and_random_port(10000, 20000) host, rabbit_config_obj.rabbitmq_config['mgmt-port-ssl'] = get_hostname_and_random_port(10000, 20000) rabbit_config_obj.rabbitmq_config['host'] = host rabbit_config_obj.rabbitmq_config['certificate-data']['common-name'] = '{}_root_ca'.format(rabbit_config_obj.instance_name) from pprint import pprint print("RMQ Node Name: {} env: ".format(rabbit_config_obj.node_name)) pprint(env) # This is updating the volttron configuration file not the rabbitmq config file. instance_setup._update_config_file(instance_name=rabbit_config_obj.instance_name) vhome_config = os.path.join(vhome, 'rabbitmq_config.yml') if not os.path.isfile(vhome_config): with open(vhome_config, 'w') as yml_file: yaml.dump(rabbit_config_obj.rabbitmq_config, yml_file, default_flow_style=False) store_message_bus_config(message_bus='rmq', instance_name=rabbit_config_obj.instance_name) setup_rabbitmq_volttron('single', verbose=False, prompt=False, instance_name=rabbit_config_obj.instance_name, rmq_conf_file=rabbit_config_obj.rmq_conf_file, env=env) return rabbit_config_obj
def two_way_federated_rmq_instances(request, **kwargs): """ Create two rmq based volttron instances. Create bi-directional data flow channel by creating 2 federation links :return: 2 volttron instances - that are connected through federation """ instance_1_vip = get_rand_vip() instance_1_hostname, instance_1_https_port = get_hostname_and_random_port() instance_1_web_address = 'https://{hostname}:{port}'.format( hostname=instance_1_hostname, port=instance_1_https_port) instance_1 = build_wrapper(instance_1_vip, ssl_auth=True, messagebus='rmq', should_start=True, bind_web_address=instance_1_web_address, instance_name='volttron1', **kwargs) instance_1.enable_auto_csr() instance_2_vip = get_rand_vip() instance_2_hostname, instance_2_https_port = get_hostname_and_random_port() instance_2_webaddress = 'https://{hostname}:{port}'.format( hostname=instance_2_hostname, port=instance_2_https_port) instance_2 = build_wrapper(instance_2_vip, ssl_auth=True, messagebus='rmq', should_start=False, bind_web_address=instance_2_webaddress, instance_name='volttron2', **kwargs) instance_2_link_name = None instance_1_link_name = None try: # create federation config and setup federation link to instance_1 content = dict() fed = dict() fed[instance_1.rabbitmq_config_obj.rabbitmq_config["host"]] = { 'port': instance_1.rabbitmq_config_obj.rabbitmq_config["amqp-port-ssl"], 'virtual-host': instance_1.rabbitmq_config_obj.rabbitmq_config["virtual-host"], 'https-port': instance_1_https_port, 'federation-user': "******".format(instance_2.instance_name) } content['federation-upstream'] = fed import yaml config_path = os.path.join(instance_2.volttron_home, "rabbitmq_federation_config.yml") with open(config_path, 'w') as yaml_file: yaml.dump(content, yaml_file, default_flow_style=False) print(f"instance 2 Fed config path:{config_path}, content: {content}") instance_2.setup_federation(config_path) instance_2.startup_platform(vip_address=instance_2_vip, bind_web_address=instance_2_webaddress) instance_2.enable_auto_csr() # Check federation link status with with_os_environ(instance_2.env): rmq_mgmt = RabbitMQMgmt() links = rmq_mgmt.get_federation_links() print(f"instance 2 fed links state: {links[0]['status']}") assert links and links[0]['status'] == 'running' instance_2_link_name = links[0]['name'] instance_1.skip_cleanup = True instance_1.shutdown_platform() instance_1.skip_cleanup = False start_rabbit(rmq_home=instance_1.rabbitmq_config_obj.rmq_home, env=instance_1.env) # create federation config and setup federation to instance_2 content = dict() fed = dict() fed[instance_2.rabbitmq_config_obj.rabbitmq_config["host"]] = { 'port': instance_2.rabbitmq_config_obj.rabbitmq_config["amqp-port-ssl"], 'virtual-host': instance_2.rabbitmq_config_obj.rabbitmq_config["virtual-host"], 'https-port': instance_2_https_port, 'federation-user': "******".format(instance_1.instance_name) } content['federation-upstream'] = fed import yaml config_path = os.path.join(instance_1.volttron_home, "rabbitmq_federation_config.yml") with open(config_path, 'w') as yaml_file: yaml.dump(content, yaml_file, default_flow_style=False) print(f"instance 1 Fed config path:{config_path}, content: {content}") instance_1.setup_federation(config_path) instance_1.startup_platform(vip_address=instance_1_vip, bind_web_address=instance_1_web_address) import gevent gevent.sleep(10) # Check federation link status with with_os_environ(instance_1.env): rmq_mgmt = RabbitMQMgmt() links = rmq_mgmt.get_federation_links() print(f"instance 1 fed links state: {links[0]['status']}") assert links and links[0]['status'] == 'running' instance_1_link_name = links[0]['name'] except Exception as e: print(f"Exception setting up federation: {e}") instance_1.shutdown_platform() instance_2.shutdown_platform() raise e yield instance_1, instance_2 if instance_1_link_name: with with_os_environ(instance_1.env): rmq_mgmt = RabbitMQMgmt() rmq_mgmt.delete_multiplatform_parameter('federation-upstream', instance_1_link_name) if instance_2_link_name: with with_os_environ(instance_2.env): rmq_mgmt = RabbitMQMgmt() rmq_mgmt.delete_multiplatform_parameter('federation-upstream', instance_2_link_name) instance_1.shutdown_platform() instance_2.shutdown_platform()
def federated_rmq_instances(request, **kwargs): """ Create two rmq based volttron instances. One to act as producer of data and one to act as consumer of data producer is upstream instance and consumer is the downstream instance :return: 2 volttron instances - (producer, consumer) that are federated """ upstream_vip = get_rand_vip() upstream_hostname, upstream_https_port = get_hostname_and_random_port() web_address = 'https://{hostname}:{port}'.format( hostname=upstream_hostname, port=upstream_https_port) upstream = build_wrapper(upstream_vip, ssl_auth=True, messagebus='rmq', should_start=True, bind_web_address=web_address, instance_name='volttron1', **kwargs) upstream.enable_auto_csr() downstream_vip = get_rand_vip() hostname, https_port = get_hostname_and_random_port() downstream_web_address = 'https://{hostname}:{port}'.format( hostname=hostname, port=https_port) downstream = build_wrapper(downstream_vip, ssl_auth=True, messagebus='rmq', should_start=False, bind_web_address=downstream_web_address, instance_name='volttron2', **kwargs) link_name = None rmq_mgmt = None try: # create federation config and save in volttron home of 'downstream' instance content = dict() fed = dict() fed[upstream.rabbitmq_config_obj.rabbitmq_config["host"]] = { 'port': upstream.rabbitmq_config_obj.rabbitmq_config["amqp-port-ssl"], 'virtual-host': upstream.rabbitmq_config_obj.rabbitmq_config["virtual-host"], 'https-port': upstream_https_port, 'federation-user': "******".format(downstream.instance_name) } content['federation-upstream'] = fed import yaml config_path = os.path.join(downstream.volttron_home, "rabbitmq_federation_config.yml") with open(config_path, 'w') as yaml_file: yaml.dump(content, yaml_file, default_flow_style=False) # setup federation link from 'downstream' to 'upstream' instance downstream.setup_federation(config_path) downstream.startup_platform(vip_address=downstream_vip, bind_web_address=downstream_web_address) with with_os_environ(downstream.env): rmq_mgmt = RabbitMQMgmt() links = rmq_mgmt.get_federation_links() assert links and links[0]['status'] == 'running' link_name = links[0]['name'] except Exception as e: print("Exception setting up federation: {}".format(e)) upstream.shutdown_platform() if downstream.is_running(): downstream.shutdown_platform() raise e yield upstream, downstream if link_name and rmq_mgmt: rmq_mgmt.delete_multiplatform_parameter('federation-upstream', link_name) upstream.shutdown_platform() downstream.shutdown_platform()
def get_test_volttron_home(messagebus: str, web_https=False, web_http=False, has_vip=True, volttron_home: str = None, config_params: dict = None, env_options: dict = None): """ Create a full volttronn_home test environment with all of the options available in the environment (os.environ) and configuration file (volttron_home/config) in order to test from. @param messagebus: Currently supports rmq and zmq strings @param web_https: Determines if https should be used and enabled. If this is specified then the cert_fixtures.certs_profile_1 function will be used to generate certificates for the server and signed ca. Either web_https or web_http may be specified not both. @param has_vip: Allows the rmq message bus to not specify a vip address if backward compatibility is not needed. @param config_params: Configuration parameters that should go into the volttron configuration file, note if the basic ones are set via the previous arguments (i.e. web_https) then it is an error to specify bind-web-address (or other) duplicate. @param env_options: Other options that should be specified in the os.environ during the setup of this environment. """ # Make these not None so that we can use set operations on them to see if we have any overlap between # common configuration params and environment. if config_params is None: config_params = {} if env_options is None: env_options = {} # make a copy so we can restore in cleanup env_cpy = os.environ.copy() # start validating input assert messagebus in ( 'rmq', 'zmq'), 'Invalid messagebus specified, must be rmq or zmq.' if web_http and web_https: raise ValueError( "Incompatabile tyeps web_https and web_Http cannot both be specified as True" ) default_env_options = ('VOLTTRON_HOME', 'MESSAGEBUS') for v in default_env_options: if v in env_options: raise ValueError( f"Cannot specify {v} in env_options as it is set already.") # All is well.Create vhome if volttron_home: os.makedirs(volttron_home, exist_ok=True) else: volttron_home = create_volttron_home() # Create env envs = dict(VOLTTRON_HOME=volttron_home, MESSAGEBUS=messagebus) os.environ.update(envs) os.environ.update(env_options) # make the top level dirs os.mkdir(os.path.join(volttron_home, "agents")) os.mkdir(os.path.join(volttron_home, "configuration_store")) os.mkdir(os.path.join(volttron_home, "keystores")) os.mkdir(os.path.join(volttron_home, "run")) # create the certs. This will create the certs dirs web_certs_dir = os.path.join(volttron_home, "web_certs") web_certs = None if web_https: web_certs = certs_profile_1(web_certs_dir) vip_address = None bind_web_address = None web_ssl_cert = None web_ssl_key = None web_secret_key = None config_file = {} if messagebus == 'rmq': if has_vip: ip, port = get_rand_ip_and_port() vip_address = f"tcp://{ip}:{port}" web_https = True elif messagebus == 'zmq': if web_http or web_https: ip, port = get_rand_ip_and_port() vip_address = f"tcp://{ip}:{port}" if web_https: hostname, port = get_hostname_and_random_port() bind_web_address = f"https://{hostname}:{port}" web_ssl_cert = web_certs.server_certs[0].cert_file web_ssl_key = web_certs.server_certs[0].key_file elif web_http: hostname, port = get_hostname_and_random_port() bind_web_address = f"http://{hostname}:{port}" web_secret_key = get_random_key() if vip_address: config_file['vip-address'] = vip_address if bind_web_address: config_file['bind-web-address'] = bind_web_address if web_ssl_cert: config_file['web-ssl-cert'] = web_ssl_cert if web_ssl_key: config_file['web-ssl-key'] = web_ssl_key if web_secret_key: config_file['web-secret-key'] = web_secret_key config_intersect = set(config_file).intersection(set(config_params)) if len(config_intersect) > 0: raise ValueError( f"passed configuration params {list(config_intersect)} are built internally" ) config_file.update(config_params) update_platform_config(config_file) try: yield volttron_home finally: os.environ.clear() os.environ.update(env_cpy) if not os.environ.get("DEBUG", 0) != 1 and not os.environ.get( "DEBUG_MODE", 0): shutil.rmtree(volttron_home, ignore_errors=True)