def test_discovery(scheme):
vhome = create_volttron_home()
# creates a vhome level key store
keystore = KeyStore()
serverkey = decode_key(keystore.public)
# Depending upon scheme we enable/disable password jwt and certificate based jwt.
if scheme == 'https':
with certs_profile_1('/'.join([vhome, 'certs'])) as certs:
config_params = dict(web_ssl_key=certs.server_certs[0].key_file,
web_ssl_cert=certs.server_certs[0].cert_file)
else:
config_params = dict(web_secret_key=get_random_key())
with get_test_volttron_home(messagebus='zmq', config_params=config_params):
instance_name = "booballoon"
host, port = get_hostname_and_random_port()
# this is the vip address
address = f"tcp://{host}:{port}"
def _construct_query_mock(core):
"""
Internal function that creates a concrete response for the data.
when query('instance-name').get() is called the passed instance name
is returned
"""
nonlocal instance_name, address
kv = {"instance-name": instance_name, "addresses": [address]}
return MockQuery(**kv)
with mock.patch('volttron.platform.vip.agent.subsystems.query.Query',
_construct_query_mock):
host, port = get_hostname_and_random_port()
bind_web_address = f"{scheme}://{host}:{port}"
serverkey = decode_key(keystore.public)
mws = MasterWebService(serverkey=serverkey,
identity=MASTER_WEB,
address=address,
bind_web_address=bind_web_address,
**config_params)
mws.startupagent(sender='testweb')
env = get_test_web_env("/discovery/")
mock_start_response = mock.Mock()
# A closingiterator is returned from the response object so we use the next
# on the returned response. Then we can do json responses.
response = mws.app_routing(env, mock_start_response).__next__()
# load json into a dict for testing responses.
response = jsonapi.loads(response.decode('utf-8'))
assert response.get('instance-name') is not None
assert instance_name == response.get('instance-name')
assert keystore.public == response.get('serverkey')
assert address == response.get('vip-address')
def get_volttron_multi_msgbus_instances(instance_name1=None,
instance_name2=None):
print("volttron_multi_messagebus source: {} sink: {}".format(
request.param['source'], request.param['sink']))
sink_address = get_rand_vip()
if request.param['sink'] == 'rmq_web':
hostname, port = get_hostname_and_random_port()
web_address = 'https://{hostname}:{port}'.format(hostname=hostname,
port=port)
messagebus = 'rmq'
ssl_auth = True
else:
web_address = "http://{}".format(get_rand_ip_and_port())
messagebus = 'zmq'
ssl_auth = False
sink = build_wrapper(sink_address,
ssl_auth=ssl_auth,
messagebus=messagebus,
bind_web_address=web_address,
volttron_central_address=web_address,
instance_name="volttron1")
source_address = get_rand_vip()
messagebus = 'zmq'
ssl_auth = False
if request.param['source'] == 'rmq':
messagebus = 'rmq'
ssl_auth = True
if sink.messagebus == 'rmq':
# sink_ca_file = sink.certsobj.cert_file(sink.certsobj.root_ca_name)
source = build_wrapper(
source_address,
ssl_auth=ssl_auth,
messagebus=messagebus,
volttron_central_address=sink.bind_web_address,
remote_platform_ca=sink.certsobj.cert_file(
sink.certsobj.root_ca_name),
instance_name='volttron2')
else:
source = build_wrapper(
source_address,
ssl_auth=ssl_auth,
messagebus=messagebus,
volttron_central_address=sink.bind_web_address,
instance_name='volttron2')
get_volttron_multi_msgbus_instances.source = source
get_volttron_multi_msgbus_instances.sink = sink
return source, sink
def test_authenticate_endpoint(scheme):
kwargs = {}
# Note this is not a context wrapper, it just does the creation for us
vhome = create_volttron_home()
if scheme == 'https':
with certs_profile_1(vhome) as certs:
kwargs['web_ssl_key'] = certs.server_certs[0].key_file
kwargs['web_ssl_cert'] = certs.server_certs[0].cert_file
else:
kwargs['web_secret_key'] = binascii.hexlify(
os.urandom(65)).decode('utf-8')
host, port = get_hostname_and_random_port()
kwargs['bind_web_address'] = f"{scheme}://{host}:{port}"
# We are specifying the volttron_home here so we don't create an additional one.
with get_test_volttron_home(messagebus='zmq',
config_params=kwargs,
volttron_home=vhome):
# add a user so that we can actually log in.
user = '******'
passwd = 'cat'
adminep = AdminEndpoints()
adminep.add_user(user, passwd, groups=['foo', 'read-only'])
expected_claims = dict(groups=['foo', 'read-only'])
with get_master_web(**kwargs) as mw:
data = urlencode(dict(username=user,
password=passwd)).encode('utf-8')
assert len(data) > 0
# test not sending any parameters.
env = get_test_web_env("/authenticate",
input_data=data,
method='POST')
mocked_start_response, response = get_server_response(env, mw)
assert 3 == len(response.split("."))
claims = mw.get_user_claims(response)
assert claims
assert not DeepDiff(expected_claims, claims)
def volttron_instance_web(request):
print(
"volttron_instance_web (messagebus {messagebus} ssl_auth {ssl_auth})".
format(**request.param))
address = get_rand_vip()
if request.param['ssl_auth']:
hostname, port = get_hostname_and_random_port()
web_address = 'https://{hostname}:{port}'.format(hostname=hostname,
port=port)
else:
web_address = "http://{}".format(get_rand_ip_and_port())
wrapper = build_wrapper(address,
ssl_auth=request.param['ssl_auth'],
messagebus=request.param['messagebus'],
bind_web_address=web_address,
volttron_central_address=web_address)
yield wrapper
cleanup_wrapper(wrapper)
def volttron_multi_messagebus(request):
""" This fixture allows multiple two message bus types to be configured to work together
This case will create a source (where data comes from) and a sink (where data goes to) to
allow connections from source to sink to be tested for the different cases. In particular,
the case of VolttronCentralPlatform, Forwarder and DataMover agents should use this
case.
:param request:
:return:
"""
print("volttron_multi_messagebus source: {} sink: {}".format(
request.param['source'], request.param['sink']))
sink_address = get_rand_vip()
if request.param['sink'] == 'rmq_web':
hostname, port = get_hostname_and_random_port()
web_address = 'https://{hostname}:{port}'.format(hostname=hostname,
port=port)
messagebus = 'rmq'
ssl_auth = True
else:
web_address = "http://{}".format(get_rand_ip_and_port())
messagebus = 'zmq'
ssl_auth = False
sink = build_wrapper(sink_address,
ssl_auth=ssl_auth,
messagebus=messagebus,
bind_web_address=web_address,
volttron_central_address=web_address)
source_address = get_rand_vip()
messagebus = 'zmq'
ssl_auth = False
if request.param['source'] == 'rmq':
messagebus = 'rmq'
ssl_auth = True
if sink.messagebus == 'rmq':
# sink_ca_file = sink.certsobj.cert_file(sink.certsobj.root_ca_name)
source = build_wrapper(source_address,
ssl_auth=ssl_auth,
messagebus=messagebus,
volttron_central_address=sink.bind_web_address,
remote_platform_ca=sink.certsobj.cert_file(
sink.certsobj.root_ca_name))
if source.messagebus == 'rmq':
# The _ca is how the auth subsystem saves the remote cert from discovery. We
# are effectively doing that here instead of making the discovery call.
source.certsobj.save_remote_cert(
sink.certsobj.root_ca_name + "_ca",
sink.certsobj.ca_cert(public_bytes=True))
else:
source = build_wrapper(source_address,
ssl_auth=ssl_auth,
messagebus=messagebus,
volttron_central_address=sink.bind_web_address)
yield source, sink
cleanup_wrapper(source)
cleanup_wrapper(sink)
def create_rmq_volttron_setup(vhome=None, ssl_auth=False, env=None):
"""
Set-up rabbitmq broker for volttron testing:
- Install config and rabbitmq_config.yml in VOLTTRON_HOME
- Create virtual host, exchanges, certificates, and users
- Start rabbitmq server
:param instance_name: the canonical name for the instance being setup.
:param vhome: volttron home directory, if None, use default from environment
:param ssl_auth: ssl authentication, if true, all users of message queue must authenticate
"""
if vhome:
os.environ['VOLTTRON_HOME'] = vhome
else:
vhome = get_home()
# Build default config file object, which we will then update to fit the
# current context the code is running in.
rabbit_config_obj = RabbitTestConfig()
# for docker this will be setup so we can always use this for the home
if os.environ.get('RMQ_HOME'):
rabbit_config_obj.rmq_home = os.environ.get('RMQ_HOME')
else:
rmq_home_env_file = os.path.expanduser("~/.volttron_rmq_home")
if not os.path.isfile(rmq_home_env_file):
raise ValueError("Rabbitmq home dir can't be found please\n run bootstrap.py --rabbitmq")
with open(rmq_home_env_file, 'r') as f:
rabbit_config_obj.rmq_home = f.read().strip()
os.environ['RMQ_HOME'] = rabbit_config_obj.rmq_home
# instance name is the basename of the volttron home now.
rabbit_config_obj.instance_name = rabbit_config_obj.node_name = os.path.basename(vhome)
os.mkdir(os.path.join(vhome, "rmq_node_data"))
rabbit_config_obj.rmq_conf_file = os.path.join(vhome, "rmq_node_data", rabbit_config_obj.node_name + "-rmq.conf")
rabbit_config_obj.rmq_env_file = os.path.join(vhome, "rmq_node_data", rabbit_config_obj.node_name + "-rmq-env.conf")
env['RABBITMQ_CONF_ENV_FILE'] = rabbit_config_obj.rmq_env_file
# Create rabbitmq config for test
rabbit_config_obj.rabbitmq_config['ssl'] = str(ssl_auth)
host, rabbit_config_obj.rabbitmq_config['amqp-port'] = get_hostname_and_random_port()
host, rabbit_config_obj.rabbitmq_config['amqp-port-ssl'] = get_hostname_and_random_port()
host, rabbit_config_obj.rabbitmq_config['mgmt-port'] = get_hostname_and_random_port(10000, 20000)
host, rabbit_config_obj.rabbitmq_config['mgmt-port-ssl'] = get_hostname_and_random_port(10000, 20000)
rabbit_config_obj.rabbitmq_config['host'] = host
rabbit_config_obj.rabbitmq_config['certificate-data']['common-name'] = '{}_root_ca'.format(rabbit_config_obj.instance_name)
from pprint import pprint
print("RMQ Node Name: {} env: ".format(rabbit_config_obj.node_name))
pprint(env)
# This is updating the volttron configuration file not the rabbitmq config file.
instance_setup._update_config_file(instance_name=rabbit_config_obj.instance_name)
vhome_config = os.path.join(vhome, 'rabbitmq_config.yml')
if not os.path.isfile(vhome_config):
with open(vhome_config, 'w') as yml_file:
yaml.dump(rabbit_config_obj.rabbitmq_config, yml_file, default_flow_style=False)
store_message_bus_config(message_bus='rmq',
instance_name=rabbit_config_obj.instance_name)
setup_rabbitmq_volttron('single',
verbose=False,
prompt=False,
instance_name=rabbit_config_obj.instance_name,
rmq_conf_file=rabbit_config_obj.rmq_conf_file,
env=env)
return rabbit_config_obj
def two_way_federated_rmq_instances(request, **kwargs):
"""
Create two rmq based volttron instances. Create bi-directional data flow channel
by creating 2 federation links
:return: 2 volttron instances - that are connected through federation
"""
instance_1_vip = get_rand_vip()
instance_1_hostname, instance_1_https_port = get_hostname_and_random_port()
instance_1_web_address = 'https://{hostname}:{port}'.format(
hostname=instance_1_hostname, port=instance_1_https_port)
instance_1 = build_wrapper(instance_1_vip,
ssl_auth=True,
messagebus='rmq',
should_start=True,
bind_web_address=instance_1_web_address,
instance_name='volttron1',
**kwargs)
instance_1.enable_auto_csr()
instance_2_vip = get_rand_vip()
instance_2_hostname, instance_2_https_port = get_hostname_and_random_port()
instance_2_webaddress = 'https://{hostname}:{port}'.format(
hostname=instance_2_hostname, port=instance_2_https_port)
instance_2 = build_wrapper(instance_2_vip,
ssl_auth=True,
messagebus='rmq',
should_start=False,
bind_web_address=instance_2_webaddress,
instance_name='volttron2',
**kwargs)
instance_2_link_name = None
instance_1_link_name = None
try:
# create federation config and setup federation link to instance_1
content = dict()
fed = dict()
fed[instance_1.rabbitmq_config_obj.rabbitmq_config["host"]] = {
'port':
instance_1.rabbitmq_config_obj.rabbitmq_config["amqp-port-ssl"],
'virtual-host':
instance_1.rabbitmq_config_obj.rabbitmq_config["virtual-host"],
'https-port':
instance_1_https_port,
'federation-user':
"******".format(instance_2.instance_name)
}
content['federation-upstream'] = fed
import yaml
config_path = os.path.join(instance_2.volttron_home,
"rabbitmq_federation_config.yml")
with open(config_path, 'w') as yaml_file:
yaml.dump(content, yaml_file, default_flow_style=False)
print(f"instance 2 Fed config path:{config_path}, content: {content}")
instance_2.setup_federation(config_path)
instance_2.startup_platform(vip_address=instance_2_vip,
bind_web_address=instance_2_webaddress)
instance_2.enable_auto_csr()
# Check federation link status
with with_os_environ(instance_2.env):
rmq_mgmt = RabbitMQMgmt()
links = rmq_mgmt.get_federation_links()
print(f"instance 2 fed links state: {links[0]['status']}")
assert links and links[0]['status'] == 'running'
instance_2_link_name = links[0]['name']
instance_1.skip_cleanup = True
instance_1.shutdown_platform()
instance_1.skip_cleanup = False
start_rabbit(rmq_home=instance_1.rabbitmq_config_obj.rmq_home,
env=instance_1.env)
# create federation config and setup federation to instance_2
content = dict()
fed = dict()
fed[instance_2.rabbitmq_config_obj.rabbitmq_config["host"]] = {
'port':
instance_2.rabbitmq_config_obj.rabbitmq_config["amqp-port-ssl"],
'virtual-host':
instance_2.rabbitmq_config_obj.rabbitmq_config["virtual-host"],
'https-port':
instance_2_https_port,
'federation-user':
"******".format(instance_1.instance_name)
}
content['federation-upstream'] = fed
import yaml
config_path = os.path.join(instance_1.volttron_home,
"rabbitmq_federation_config.yml")
with open(config_path, 'w') as yaml_file:
yaml.dump(content, yaml_file, default_flow_style=False)
print(f"instance 1 Fed config path:{config_path}, content: {content}")
instance_1.setup_federation(config_path)
instance_1.startup_platform(vip_address=instance_1_vip,
bind_web_address=instance_1_web_address)
import gevent
gevent.sleep(10)
# Check federation link status
with with_os_environ(instance_1.env):
rmq_mgmt = RabbitMQMgmt()
links = rmq_mgmt.get_federation_links()
print(f"instance 1 fed links state: {links[0]['status']}")
assert links and links[0]['status'] == 'running'
instance_1_link_name = links[0]['name']
except Exception as e:
print(f"Exception setting up federation: {e}")
instance_1.shutdown_platform()
instance_2.shutdown_platform()
raise e
yield instance_1, instance_2
if instance_1_link_name:
with with_os_environ(instance_1.env):
rmq_mgmt = RabbitMQMgmt()
rmq_mgmt.delete_multiplatform_parameter('federation-upstream',
instance_1_link_name)
if instance_2_link_name:
with with_os_environ(instance_2.env):
rmq_mgmt = RabbitMQMgmt()
rmq_mgmt.delete_multiplatform_parameter('federation-upstream',
instance_2_link_name)
instance_1.shutdown_platform()
instance_2.shutdown_platform()
def federated_rmq_instances(request, **kwargs):
"""
Create two rmq based volttron instances. One to act as producer of data and one to act as consumer of data
producer is upstream instance and consumer is the downstream instance
:return: 2 volttron instances - (producer, consumer) that are federated
"""
upstream_vip = get_rand_vip()
upstream_hostname, upstream_https_port = get_hostname_and_random_port()
web_address = 'https://{hostname}:{port}'.format(
hostname=upstream_hostname, port=upstream_https_port)
upstream = build_wrapper(upstream_vip,
ssl_auth=True,
messagebus='rmq',
should_start=True,
bind_web_address=web_address,
instance_name='volttron1',
**kwargs)
upstream.enable_auto_csr()
downstream_vip = get_rand_vip()
hostname, https_port = get_hostname_and_random_port()
downstream_web_address = 'https://{hostname}:{port}'.format(
hostname=hostname, port=https_port)
downstream = build_wrapper(downstream_vip,
ssl_auth=True,
messagebus='rmq',
should_start=False,
bind_web_address=downstream_web_address,
instance_name='volttron2',
**kwargs)
link_name = None
rmq_mgmt = None
try:
# create federation config and save in volttron home of 'downstream' instance
content = dict()
fed = dict()
fed[upstream.rabbitmq_config_obj.rabbitmq_config["host"]] = {
'port':
upstream.rabbitmq_config_obj.rabbitmq_config["amqp-port-ssl"],
'virtual-host':
upstream.rabbitmq_config_obj.rabbitmq_config["virtual-host"],
'https-port':
upstream_https_port,
'federation-user':
"******".format(downstream.instance_name)
}
content['federation-upstream'] = fed
import yaml
config_path = os.path.join(downstream.volttron_home,
"rabbitmq_federation_config.yml")
with open(config_path, 'w') as yaml_file:
yaml.dump(content, yaml_file, default_flow_style=False)
# setup federation link from 'downstream' to 'upstream' instance
downstream.setup_federation(config_path)
downstream.startup_platform(vip_address=downstream_vip,
bind_web_address=downstream_web_address)
with with_os_environ(downstream.env):
rmq_mgmt = RabbitMQMgmt()
links = rmq_mgmt.get_federation_links()
assert links and links[0]['status'] == 'running'
link_name = links[0]['name']
except Exception as e:
print("Exception setting up federation: {}".format(e))
upstream.shutdown_platform()
if downstream.is_running():
downstream.shutdown_platform()
raise e
yield upstream, downstream
if link_name and rmq_mgmt:
rmq_mgmt.delete_multiplatform_parameter('federation-upstream',
link_name)
upstream.shutdown_platform()
downstream.shutdown_platform()
def get_test_volttron_home(messagebus: str,
web_https=False,
web_http=False,
has_vip=True,
volttron_home: str = None,
config_params: dict = None,
env_options: dict = None):
"""
Create a full volttronn_home test environment with all of the options available in the environment
(os.environ) and configuration file (volttron_home/config) in order to test from.
@param messagebus:
Currently supports rmq and zmq strings
@param web_https:
Determines if https should be used and enabled. If this is specified then the cert_fixtures.certs_profile_1
function will be used to generate certificates for the server and signed ca. Either web_https or web_http
may be specified not both.
@param has_vip:
Allows the rmq message bus to not specify a vip address if backward compatibility is not needed.
@param config_params:
Configuration parameters that should go into the volttron configuration file, note if the basic ones are
set via the previous arguments (i.e. web_https) then it is an error to specify bind-web-address (or other)
duplicate.
@param env_options:
Other options that should be specified in the os.environ during the setup of this environment.
"""
# Make these not None so that we can use set operations on them to see if we have any overlap between
# common configuration params and environment.
if config_params is None:
config_params = {}
if env_options is None:
env_options = {}
# make a copy so we can restore in cleanup
env_cpy = os.environ.copy()
# start validating input
assert messagebus in (
'rmq', 'zmq'), 'Invalid messagebus specified, must be rmq or zmq.'
if web_http and web_https:
raise ValueError(
"Incompatabile tyeps web_https and web_Http cannot both be specified as True"
)
default_env_options = ('VOLTTRON_HOME', 'MESSAGEBUS')
for v in default_env_options:
if v in env_options:
raise ValueError(
f"Cannot specify {v} in env_options as it is set already.")
# All is well.Create vhome
if volttron_home:
os.makedirs(volttron_home, exist_ok=True)
else:
volttron_home = create_volttron_home()
# Create env
envs = dict(VOLTTRON_HOME=volttron_home, MESSAGEBUS=messagebus)
os.environ.update(envs)
os.environ.update(env_options)
# make the top level dirs
os.mkdir(os.path.join(volttron_home, "agents"))
os.mkdir(os.path.join(volttron_home, "configuration_store"))
os.mkdir(os.path.join(volttron_home, "keystores"))
os.mkdir(os.path.join(volttron_home, "run"))
# create the certs. This will create the certs dirs
web_certs_dir = os.path.join(volttron_home, "web_certs")
web_certs = None
if web_https:
web_certs = certs_profile_1(web_certs_dir)
vip_address = None
bind_web_address = None
web_ssl_cert = None
web_ssl_key = None
web_secret_key = None
config_file = {}
if messagebus == 'rmq':
if has_vip:
ip, port = get_rand_ip_and_port()
vip_address = f"tcp://{ip}:{port}"
web_https = True
elif messagebus == 'zmq':
if web_http or web_https:
ip, port = get_rand_ip_and_port()
vip_address = f"tcp://{ip}:{port}"
if web_https:
hostname, port = get_hostname_and_random_port()
bind_web_address = f"https://{hostname}:{port}"
web_ssl_cert = web_certs.server_certs[0].cert_file
web_ssl_key = web_certs.server_certs[0].key_file
elif web_http:
hostname, port = get_hostname_and_random_port()
bind_web_address = f"http://{hostname}:{port}"
web_secret_key = get_random_key()
if vip_address:
config_file['vip-address'] = vip_address
if bind_web_address:
config_file['bind-web-address'] = bind_web_address
if web_ssl_cert:
config_file['web-ssl-cert'] = web_ssl_cert
if web_ssl_key:
config_file['web-ssl-key'] = web_ssl_key
if web_secret_key:
config_file['web-secret-key'] = web_secret_key
config_intersect = set(config_file).intersection(set(config_params))
if len(config_intersect) > 0:
raise ValueError(
f"passed configuration params {list(config_intersect)} are built internally"
)
config_file.update(config_params)
update_platform_config(config_file)
try:
yield volttron_home
finally:
os.environ.clear()
os.environ.update(env_cpy)
if not os.environ.get("DEBUG", 0) != 1 and not os.environ.get(
"DEBUG_MODE", 0):
shutil.rmtree(volttron_home, ignore_errors=True)