def get_volttron_multi_msgbus_instances(instance_name1=None,
instance_name2=None):
print("volttron_multi_messagebus source: {} sink: {}".format(
request.param['source'], request.param['sink']))
sink_address = get_rand_vip()
if request.param['sink'] == 'rmq_web':
hostname, port = get_hostname_and_random_port()
web_address = 'https://{hostname}:{port}'.format(hostname=hostname,
port=port)
messagebus = 'rmq'
ssl_auth = True
else:
web_address = "http://{}".format(get_rand_ip_and_port())
messagebus = 'zmq'
ssl_auth = False
sink = build_wrapper(sink_address,
ssl_auth=ssl_auth,
messagebus=messagebus,
bind_web_address=web_address,
volttron_central_address=web_address,
instance_name="volttron1")
source_address = get_rand_vip()
messagebus = 'zmq'
ssl_auth = False
if request.param['source'] == 'rmq':
messagebus = 'rmq'
ssl_auth = True
if sink.messagebus == 'rmq':
# sink_ca_file = sink.certsobj.cert_file(sink.certsobj.root_ca_name)
source = build_wrapper(
source_address,
ssl_auth=ssl_auth,
messagebus=messagebus,
volttron_central_address=sink.bind_web_address,
remote_platform_ca=sink.certsobj.cert_file(
sink.certsobj.root_ca_name),
instance_name='volttron2')
else:
source = build_wrapper(
source_address,
ssl_auth=ssl_auth,
messagebus=messagebus,
volttron_central_address=sink.bind_web_address,
instance_name='volttron2')
get_volttron_multi_msgbus_instances.source = source
get_volttron_multi_msgbus_instances.sink = sink
return source, sink
def volttron_instance_msgdebug(request):
print("building msgdebug instance")
wrapper = build_wrapper(get_rand_vip(),
msgdebug=True,
messagebus=request.param['messagebus'],
ssl_auth=request.param['ssl_auth'])
yield wrapper
cleanup_wrapper(wrapper)
def volttron_instance_encrypt(request):
print("building instance (using encryption)")
address = get_rand_vip()
wrapper = build_wrapper(address)
def cleanup():
cleanup_wrapper(wrapper)
request.addfinalizer(cleanup)
return wrapper
def volttron_instance_module_web(request):
print("building module instance (using web)")
address = get_rand_vip()
web_address = "http://{}".format(get_rand_ip_and_port())
wrapper = build_wrapper(address,
bind_web_address=web_address,
messagebus='zmq',
ssl_auth=False)
yield wrapper
cleanup_wrapper(wrapper)
def volttron_instance_zmq(request):
"""Fixture that returns a single instance of volttron platform for testing
@param request: pytest request object
@return: volttron platform instance
"""
address = get_rand_vip()
wrapper = build_wrapper(address)
yield wrapper
cleanup_wrapper(wrapper)
def volttron_instance_rmq(request):
"""Fixture that returns a single instance of volttron platform for testing
@param request: pytest request object
@return: volttron platform instance
"""
wrapper = None
address = get_rand_vip()
wrapper = build_wrapper(address, messagebus='rmq', ssl_auth=True)
yield wrapper
cleanup_wrapper(wrapper)
def get_n_volttron_instances(n, should_start=True, address_file=True):
get_n_volttron_instances.count = n
vip_addresses = []
web_addresses = []
instances = []
names = []
for i in range(0, n):
address = get_rand_vip()
web_address = "http://{}".format(get_rand_ip_and_port())
vip_addresses.append(address)
web_addresses.append(web_address)
nm = 'platform{}'.format(i + 1)
names.append(nm)
for i in range(0, n):
address = vip_addresses[i]
web_address = web_addresses[i]
wrapper = PlatformWrapper(messagebus='zmq', ssl_auth=False)
addr_file = os.path.join(wrapper.volttron_home,
'external_address.json')
if address_file:
with open(addr_file, 'w') as f:
jsonapi.dump(web_addresses, f)
gevent.sleep(0.5)
wrapper.startup_platform(address,
bind_web_address=web_address,
setupmode=True)
wrapper.skip_cleanup = True
instances.append(wrapper)
gevent.sleep(11)
for i in range(0, n):
instances[i].shutdown_platform()
gevent.sleep(1)
# del instances[:]
for i in range(0, n):
address = vip_addresses.pop(0)
web_address = web_addresses.pop(0)
print(address, web_address)
instances[i].startup_platform(address,
bind_web_address=web_address)
instances[i].allow_all_connections()
gevent.sleep(11)
instances = instances if n > 1 else instances[0]
get_n_volttron_instances.instances = instances
return instances
def volttron_instance(request, **kwargs):
"""Fixture that returns a single instance of volttron platform for testing
@param request: pytest request object
@return: volttron platform instance
"""
address = kwargs.pop("vip_address", get_rand_vip())
wrapper = build_wrapper(address,
messagebus=request.param['messagebus'],
ssl_auth=request.param['ssl_auth'],
**kwargs)
yield wrapper
cleanup_wrapper(wrapper)
def secure_volttron_instance(request):
"""
Fixture that returns a single instance of volttron platform for testing
"""
address = get_rand_vip()
wrapper = build_wrapper(address,
instance_name=request.param['instance_name'],
messagebus=request.param['messagebus'],
ssl_auth=request.param['ssl_auth'],
secure_agent_users=True)
gevent.sleep(3)
yield wrapper
cleanup_wrapper(wrapper)
def get_n_volttron_instances(n, should_start=True, **kwargs):
get_n_volttron_instances.count = n
instances = []
for i in range(0, n):
address = kwargs.pop("vip_address", get_rand_vip())
wrapper = build_wrapper(address,
should_start=should_start,
messagebus=request.param['messagebus'],
ssl_auth=request.param['ssl_auth'],
**kwargs)
instances.append(wrapper)
instances = instances if n > 1 else instances[0]
# setattr(get_n_volttron_instances, 'instances', instances)
get_n_volttron_instances.instances = instances
return instances
def aip(request):
volttron_home = tempfile.mkdtemp()
packaged_dir = os.path.join(volttron_home, "packaged")
os.makedirs(packaged_dir)
ipc = 'ipc://{}{}/run/'.format(
'@' if sys.platform.startswith('linux') else '',
volttron_home)
local_vip_address = ipc + 'vip.socket'
os.environ['VOLTTRON_HOME'] = volttron_home
opts = dict(
volttron_home=volttron_home,
vip_address=get_rand_vip(),
local_vip_address=ipc+'vip.socket',
publish_address=ipc+'publish',
subscribe_address=ipc+'subscribe',
log_level=logging.DEBUG
)
options = type('Options', (), opts)
# 'verify_agents': False,
# 'volttron_home': self.volttron_home,
# 'vip_address': vip_address,
# 'vip_local_address': ipc + 'vip.socket',
# 'publish_address': ipc + 'publish',
# 'subscribe_address': ipc + 'subscribe',
# 'bind_web_address': bind_web_address,
# 'volttron_central_address': volttron_central_address,
# 'volttron_central_serverkey': volttron_central_serverkey,
# 'platform_name': None,
# 'developer_mode': not encrypt,
# 'log': os.path.join(self.volttron_home, 'volttron.log'),
# 'log_config': None,
# 'monitor': True,
# 'autostart': True,
# 'log_level': logging.DEBUG,
# 'verboseness': logging.DEBUG
aip = AIPplatform(options)
aip.setup()
return aip
def aip(request):
volttron_home = tempfile.mkdtemp()
packaged_dir = os.path.join(volttron_home, "packaged")
os.makedirs(packaged_dir)
ipc = 'ipc://{}{}/run/'.format(
'@' if sys.platform.startswith('linux') else '',
volttron_home)
local_vip_address = ipc + 'vip.socket'
os.environ['VOLTTRON_HOME'] = volttron_home
opts = dict(
volttron_home=volttron_home,
vip_address=get_rand_vip(),
local_vip_address=ipc+'vip.socket',
publish_address=ipc+'publish',
subscribe_address=ipc+'subscribe',
log_level=logging.DEBUG
)
options = type('Options', (), opts)
# 'verify_agents': False,
# 'volttron_home': self.volttron_home,
# 'vip_address': vip_address,
# 'vip_local_address': ipc + 'vip.socket',
# 'publish_address': ipc + 'publish',
# 'subscribe_address': ipc + 'subscribe',
# 'bind_web_address': bind_web_address,
# 'volttron_central_address': volttron_central_address,
# 'volttron_central_serverkey': volttron_central_serverkey,
# 'platform_name': None,
# 'log': os.path.join(self.volttron_home, 'volttron.log'),
# 'log_config': None,
# 'monitor': True,
# 'autostart': True,
# 'log_level': logging.DEBUG,
# 'verboseness': logging.DEBUG
aip = AIPplatform(options)
aip.setup()
return aip
def volttron_instance_web(request):
print(
"volttron_instance_web (messagebus {messagebus} ssl_auth {ssl_auth})".
format(**request.param))
address = get_rand_vip()
if request.param['ssl_auth']:
hostname, port = get_hostname_and_random_port()
web_address = 'https://{hostname}:{port}'.format(hostname=hostname,
port=port)
else:
web_address = "http://{}".format(get_rand_ip_and_port())
wrapper = build_wrapper(address,
ssl_auth=request.param['ssl_auth'],
messagebus=request.param['messagebus'],
bind_web_address=web_address,
volttron_central_address=web_address)
yield wrapper
cleanup_wrapper(wrapper)
def aip(request):
volttron_home = tempfile.mkdtemp()
packaged_dir = os.path.join(volttron_home, "packaged")
os.makedirs(packaged_dir)
ipc = "ipc://{}{}/run/".format(
"@" if sys.platform.startswith("linux") else "", volttron_home)
local_vip_address = ipc + "vip.socket"
os.environ["VOLTTRON_HOME"] = volttron_home
opts = dict(
volttron_home=volttron_home,
vip_address=get_rand_vip(),
local_vip_address=ipc + "vip.socket",
publish_address=ipc + "publish",
subscribe_address=ipc + "subscribe",
log_level=logging.DEBUG,
)
options = type("Options", (), opts)
# 'verify_agents': False,
# 'volttron_home': self.volttron_home,
# 'vip_address': vip_address,
# 'vip_local_address': ipc + 'vip.socket',
# 'publish_address': ipc + 'publish',
# 'subscribe_address': ipc + 'subscribe',
# 'bind_web_address': bind_web_address,
# 'volttron_central_address': volttron_central_address,
# 'volttron_central_serverkey': volttron_central_serverkey,
# 'platform_name': None,
# 'log': os.path.join(self.volttron_home, 'volttron.log'),
# 'log_config': None,
# 'monitor': True,
# 'autostart': True,
# 'log_level': logging.DEBUG,
# 'verboseness': logging.DEBUG
aip = AIPplatform(options)
aip.setup()
return aip
def volttron_instance(request, **kwargs):
"""Fixture that returns a single instance of volttron platform for testing
@param request: pytest request object
@return: volttron platform instance
"""
address = kwargs.pop("vip_address", get_rand_vip())
wrapper = build_wrapper(address,
messagebus=request.param['messagebus'],
ssl_auth=request.param['ssl_auth'],
**kwargs)
wrapper_pid = wrapper.p_process.pid
try:
yield wrapper
except Exception as ex:
print(ex.args)
finally:
cleanup_wrapper(wrapper)
if not wrapper.debug_mode:
assert not Path(wrapper.volttron_home).exists()
# Final way to kill off the platform wrapper for the tests.
if psutil.pid_exists(wrapper_pid):
psutil.Process(wrapper_pid).kill()
def volttron_multi_messagebus(request):
""" This fixture allows multiple two message bus types to be configured to work together
This case will create a source (where data comes from) and a sink (where data goes to) to
allow connections from source to sink to be tested for the different cases. In particular,
the case of VolttronCentralPlatform, Forwarder and DataMover agents should use this
case.
:param request:
:return:
"""
print("volttron_multi_messagebus source: {} sink: {}".format(
request.param['source'], request.param['sink']))
sink_address = get_rand_vip()
if request.param['sink'] == 'rmq_web':
hostname, port = get_hostname_and_random_port()
web_address = 'https://{hostname}:{port}'.format(hostname=hostname,
port=port)
messagebus = 'rmq'
ssl_auth = True
else:
web_address = "http://{}".format(get_rand_ip_and_port())
messagebus = 'zmq'
ssl_auth = False
sink = build_wrapper(sink_address,
ssl_auth=ssl_auth,
messagebus=messagebus,
bind_web_address=web_address,
volttron_central_address=web_address)
source_address = get_rand_vip()
messagebus = 'zmq'
ssl_auth = False
if request.param['source'] == 'rmq':
messagebus = 'rmq'
ssl_auth = True
if sink.messagebus == 'rmq':
# sink_ca_file = sink.certsobj.cert_file(sink.certsobj.root_ca_name)
source = build_wrapper(source_address,
ssl_auth=ssl_auth,
messagebus=messagebus,
volttron_central_address=sink.bind_web_address,
remote_platform_ca=sink.certsobj.cert_file(
sink.certsobj.root_ca_name))
if source.messagebus == 'rmq':
# The _ca is how the auth subsystem saves the remote cert from discovery. We
# are effectively doing that here instead of making the discovery call.
source.certsobj.save_remote_cert(
sink.certsobj.root_ca_name + "_ca",
sink.certsobj.ca_cert(public_bytes=True))
else:
source = build_wrapper(source_address,
ssl_auth=ssl_auth,
messagebus=messagebus,
volttron_central_address=sink.bind_web_address)
yield source, sink
cleanup_wrapper(source)
cleanup_wrapper(sink)
def federated_rmq_instances(request, **kwargs):
"""
Create two rmq based volttron instances. One to act as producer of data and one to act as consumer of data
producer is upstream instance and consumer is the downstream instance
:return: 2 volttron instances - (producer, consumer) that are federated
"""
upstream_vip = get_rand_vip()
upstream_hostname, upstream_https_port = get_hostname_and_random_port()
web_address = 'https://{hostname}:{port}'.format(
hostname=upstream_hostname, port=upstream_https_port)
upstream = build_wrapper(upstream_vip,
ssl_auth=True,
messagebus='rmq',
should_start=True,
bind_web_address=web_address,
instance_name='volttron1',
**kwargs)
upstream.enable_auto_csr()
downstream_vip = get_rand_vip()
hostname, https_port = get_hostname_and_random_port()
downstream_web_address = 'https://{hostname}:{port}'.format(
hostname=hostname, port=https_port)
downstream = build_wrapper(downstream_vip,
ssl_auth=True,
messagebus='rmq',
should_start=False,
bind_web_address=downstream_web_address,
instance_name='volttron2',
**kwargs)
link_name = None
rmq_mgmt = None
try:
# create federation config and save in volttron home of 'downstream' instance
content = dict()
fed = dict()
fed[upstream.rabbitmq_config_obj.rabbitmq_config["host"]] = {
'port':
upstream.rabbitmq_config_obj.rabbitmq_config["amqp-port-ssl"],
'virtual-host':
upstream.rabbitmq_config_obj.rabbitmq_config["virtual-host"],
'https-port':
upstream_https_port,
'federation-user':
"******".format(downstream.instance_name)
}
content['federation-upstream'] = fed
import yaml
config_path = os.path.join(downstream.volttron_home,
"rabbitmq_federation_config.yml")
with open(config_path, 'w') as yaml_file:
yaml.dump(content, yaml_file, default_flow_style=False)
# setup federation link from 'downstream' to 'upstream' instance
downstream.setup_federation(config_path)
downstream.startup_platform(vip_address=downstream_vip,
bind_web_address=downstream_web_address)
with with_os_environ(downstream.env):
rmq_mgmt = RabbitMQMgmt()
links = rmq_mgmt.get_federation_links()
assert links and links[0]['status'] == 'running'
link_name = links[0]['name']
except Exception as e:
print("Exception setting up federation: {}".format(e))
upstream.shutdown_platform()
if downstream.is_running():
downstream.shutdown_platform()
raise e
yield upstream, downstream
if link_name and rmq_mgmt:
rmq_mgmt.delete_multiplatform_parameter('federation-upstream',
link_name)
upstream.shutdown_platform()
downstream.shutdown_platform()
def two_way_federated_rmq_instances(request, **kwargs):
"""
Create two rmq based volttron instances. Create bi-directional data flow channel
by creating 2 federation links
:return: 2 volttron instances - that are connected through federation
"""
instance_1_vip = get_rand_vip()
instance_1_hostname, instance_1_https_port = get_hostname_and_random_port()
instance_1_web_address = 'https://{hostname}:{port}'.format(
hostname=instance_1_hostname, port=instance_1_https_port)
instance_1 = build_wrapper(instance_1_vip,
ssl_auth=True,
messagebus='rmq',
should_start=True,
bind_web_address=instance_1_web_address,
instance_name='volttron1',
**kwargs)
instance_1.enable_auto_csr()
instance_2_vip = get_rand_vip()
instance_2_hostname, instance_2_https_port = get_hostname_and_random_port()
instance_2_webaddress = 'https://{hostname}:{port}'.format(
hostname=instance_2_hostname, port=instance_2_https_port)
instance_2 = build_wrapper(instance_2_vip,
ssl_auth=True,
messagebus='rmq',
should_start=False,
bind_web_address=instance_2_webaddress,
instance_name='volttron2',
**kwargs)
instance_2_link_name = None
instance_1_link_name = None
try:
# create federation config and setup federation link to instance_1
content = dict()
fed = dict()
fed[instance_1.rabbitmq_config_obj.rabbitmq_config["host"]] = {
'port':
instance_1.rabbitmq_config_obj.rabbitmq_config["amqp-port-ssl"],
'virtual-host':
instance_1.rabbitmq_config_obj.rabbitmq_config["virtual-host"],
'https-port':
instance_1_https_port,
'federation-user':
"******".format(instance_2.instance_name)
}
content['federation-upstream'] = fed
import yaml
config_path = os.path.join(instance_2.volttron_home,
"rabbitmq_federation_config.yml")
with open(config_path, 'w') as yaml_file:
yaml.dump(content, yaml_file, default_flow_style=False)
print(f"instance 2 Fed config path:{config_path}, content: {content}")
instance_2.setup_federation(config_path)
instance_2.startup_platform(vip_address=instance_2_vip,
bind_web_address=instance_2_webaddress)
instance_2.enable_auto_csr()
# Check federation link status
with with_os_environ(instance_2.env):
rmq_mgmt = RabbitMQMgmt()
links = rmq_mgmt.get_federation_links()
print(f"instance 2 fed links state: {links[0]['status']}")
assert links and links[0]['status'] == 'running'
instance_2_link_name = links[0]['name']
instance_1.skip_cleanup = True
instance_1.shutdown_platform()
instance_1.skip_cleanup = False
start_rabbit(rmq_home=instance_1.rabbitmq_config_obj.rmq_home,
env=instance_1.env)
# create federation config and setup federation to instance_2
content = dict()
fed = dict()
fed[instance_2.rabbitmq_config_obj.rabbitmq_config["host"]] = {
'port':
instance_2.rabbitmq_config_obj.rabbitmq_config["amqp-port-ssl"],
'virtual-host':
instance_2.rabbitmq_config_obj.rabbitmq_config["virtual-host"],
'https-port':
instance_2_https_port,
'federation-user':
"******".format(instance_1.instance_name)
}
content['federation-upstream'] = fed
import yaml
config_path = os.path.join(instance_1.volttron_home,
"rabbitmq_federation_config.yml")
with open(config_path, 'w') as yaml_file:
yaml.dump(content, yaml_file, default_flow_style=False)
print(f"instance 1 Fed config path:{config_path}, content: {content}")
instance_1.setup_federation(config_path)
instance_1.startup_platform(vip_address=instance_1_vip,
bind_web_address=instance_1_web_address)
import gevent
gevent.sleep(10)
# Check federation link status
with with_os_environ(instance_1.env):
rmq_mgmt = RabbitMQMgmt()
links = rmq_mgmt.get_federation_links()
print(f"instance 1 fed links state: {links[0]['status']}")
assert links and links[0]['status'] == 'running'
instance_1_link_name = links[0]['name']
except Exception as e:
print(f"Exception setting up federation: {e}")
instance_1.shutdown_platform()
instance_2.shutdown_platform()
raise e
yield instance_1, instance_2
if instance_1_link_name:
with with_os_environ(instance_1.env):
rmq_mgmt = RabbitMQMgmt()
rmq_mgmt.delete_multiplatform_parameter('federation-upstream',
instance_1_link_name)
if instance_2_link_name:
with with_os_environ(instance_2.env):
rmq_mgmt = RabbitMQMgmt()
rmq_mgmt.delete_multiplatform_parameter('federation-upstream',
instance_2_link_name)
instance_1.shutdown_platform()
instance_2.shutdown_platform()
def federated_rmq_instances(request, **kwargs):
"""
Create two rmq based volttron instances. One to act as producer of data and one to act as consumer of data
:return: 2 volttron instances - (producer, consumer) that are federated
"""
upstream_vip = get_rand_vip()
upstream = build_wrapper(upstream_vip,
ssl_auth=True,
messagebus='rmq',
should_start=False,
**kwargs)
downstream_vip = get_rand_vip()
downstream = build_wrapper(downstream_vip,
ssl_auth=True,
messagebus='rmq',
should_start=False,
**kwargs)
# exchange CA certs
stop_rabbit(rmq_home=upstream.rabbitmq_config_obj.rmq_home,
env=upstream.env,
quite=True)
stop_rabbit(rmq_home=downstream.rabbitmq_config_obj.rmq_home,
env=downstream.env,
quite=True)
with open(
os.path.join(upstream.certsobj.cert_dir,
upstream.instance_name + "-root-ca.crt"), "r") as uf:
with open(
os.path.join(downstream.certsobj.cert_dir,
downstream.instance_name + "-trusted-cas.crt"),
"a") as df:
df.write(uf.read())
with open(
os.path.join(downstream.certsobj.cert_dir,
downstream.instance_name + "-root-ca.crt"),
"r") as df:
with open(
os.path.join(upstream.certsobj.cert_dir,
upstream.instance_name + "-trusted-cas.crt"),
"a") as uf:
uf.write(df.read())
start_rabbit(rmq_home=downstream.rabbitmq_config_obj.rmq_home,
env=downstream.env)
gevent.sleep(1)
start_rabbit(rmq_home=upstream.rabbitmq_config_obj.rmq_home,
env=upstream.env)
gevent.sleep(1)
try:
# add downstream user ON UPSTREAM and give permissions
# ~/rabbitmq_server/rabbitmq_server-3.7.7/sbin/rabbitmqctl add_user <user> <password>
# ~/rabbitmq_server/rabbitmq_server-3.7.7/sbin/rabbitmqctl set_permissions -p <vhost> <user> ".*" ".*" ".*"
cmd = [
os.path.join(upstream.rabbitmq_config_obj.rmq_home,
"sbin/rabbitmqctl")
]
cmd.extend(['add_user', downstream.instance_name + "-admin", "test"])
execute_command(cmd,
env=upstream.env,
err_prefix="Error creating user in upstream server")
cmd = [
os.path.join(
upstream.rabbitmq_config_obj.rabbitmq_config['rmq-home'],
"sbin/rabbitmqctl")
]
cmd.extend([
'set_permissions', "-p",
upstream.rabbitmq_config_obj.rabbitmq_config["virtual-host"]
])
cmd.extend([downstream.instance_name + "-admin", ".*", ".*", ".*"])
execute_command(
cmd,
env=upstream.env,
err_prefix="Error setting user permission in upstream server")
gevent.sleep(1)
upstream.startup_platform(upstream_vip)
gevent.sleep(2)
print("After upstream start")
downstream.startup_platform(downstream_vip)
gevent.sleep(2)
# create federation config and setup federation
content = """federation-upstream:
{host}:
port: {port}
virtual-host: {vhost}
"""
config_path = os.path.join(downstream.volttron_home,
"federation.config")
with open(config_path, 'w') as conf:
conf.write(
content.format(
host=upstream.rabbitmq_config_obj.rabbitmq_config["host"],
port=upstream.rabbitmq_config_obj.
rabbitmq_config["amqp-port-ssl"],
vhost=upstream.rabbitmq_config_obj.
rabbitmq_config["virtual-host"]))
downstream.setup_federation(config_path)
except Exception as e:
print("Exception setting up federation: {}".format(e))
upstream.shutdown_platform()
downstream.shutdown_platform()
raise e
yield upstream, downstream
upstream.shutdown_platform()
downstream.shutdown_platform()
