def test_on_rmq_reconnect(volttron_instance_rmq, publisher_agent,
subscriber_agent):
"""
Test the fix for issue# 1702
:param request:
:param volttron_instance_rmq:
:return:
"""
publisher_agent.vip.pubsub.publish(peer='pubsub',
topic='test/test_message',
headers={},
message="This is test message")
gevent.sleep(0.5)
assert subscriber_agent.callback.call_count == 1
# Stop RabbitMQ server
rmq_cfg = RMQConfig()
stop_rabbit(rmq_cfg.rmq_home, env=volttron_instance_rmq.env)
gevent.sleep(1)
# Start RabbitMQ server again
start_rabbit(rmq_cfg.rmq_home, env=volttron_instance_rmq.env)
gevent.sleep(8)
publisher_agent.vip.pubsub.publish(
peer='pubsub',
topic='test/test_message',
headers={},
message="This is test message after rmq reconnect")
gevent.sleep(0.1)
assert subscriber_agent.callback.call_count == 2
def test_rmq_reconnect_with_publish(volttron_instance_rmq, publisher_agent,
subscriber_agent):
"""
Test the fix for issue# 1702
:param request:
:param volttron_instance_rmq:
:return:
"""
subscriber_agent.callback.reset_mock()
gevent.sleep(0.2)
publisher_agent.vip.pubsub.publish(peer='pubsub',
topic='test/test_message',
headers={},
message="This is test message")
gevent.sleep(0.2)
assert subscriber_agent.callback.call_count == 1
# Stop RabbitMQ server
rmq_cfg = RMQConfig()
stop_rabbit(rmq_cfg.rmq_home, env=volttron_instance_rmq.env)
gevent.sleep(2)
# Start RabbitMQ server
start_rabbit(rmq_cfg.rmq_home, env=volttron_instance_rmq.env)
gevent.sleep(2)
for i in range(5):
try:
publisher_agent.vip.pubsub.publish(peer='pubsub',
topic='test/test_message',
headers={},
message="This is test message")
except Unreachable:
# Apply back pressure and try again after sleep
gevent.sleep(1)
publisher_agent.vip.pubsub.publish(
peer='pubsub',
topic='test/test_message',
headers={},
message="This is test message after rmq reconnect")
gevent.sleep(0.1)
assert subscriber_agent.callback.call_count >= 2
def federated_rmq_instances(request, **kwargs):
"""
Create two rmq based volttron instances. One to act as producer of data and one to act as consumer of data
:return: 2 volttron instances - (producer, consumer) that are federated
"""
upstream_vip = get_rand_vip()
upstream = build_wrapper(upstream_vip,
ssl_auth=True,
messagebus='rmq',
should_start=False,
**kwargs)
downstream_vip = get_rand_vip()
downstream = build_wrapper(downstream_vip,
ssl_auth=True,
messagebus='rmq',
should_start=False,
**kwargs)
# exchange CA certs
stop_rabbit(rmq_home=upstream.rabbitmq_config_obj.rmq_home,
env=upstream.env,
quite=True)
stop_rabbit(rmq_home=downstream.rabbitmq_config_obj.rmq_home,
env=downstream.env,
quite=True)
with open(
os.path.join(upstream.certsobj.cert_dir,
upstream.instance_name + "-root-ca.crt"), "r") as uf:
with open(
os.path.join(downstream.certsobj.cert_dir,
downstream.instance_name + "-trusted-cas.crt"),
"a") as df:
df.write(uf.read())
with open(
os.path.join(downstream.certsobj.cert_dir,
downstream.instance_name + "-root-ca.crt"),
"r") as df:
with open(
os.path.join(upstream.certsobj.cert_dir,
upstream.instance_name + "-trusted-cas.crt"),
"a") as uf:
uf.write(df.read())
start_rabbit(rmq_home=downstream.rabbitmq_config_obj.rmq_home,
env=downstream.env)
gevent.sleep(1)
start_rabbit(rmq_home=upstream.rabbitmq_config_obj.rmq_home,
env=upstream.env)
gevent.sleep(1)
try:
# add downstream user ON UPSTREAM and give permissions
# ~/rabbitmq_server/rabbitmq_server-3.7.7/sbin/rabbitmqctl add_user <user> <password>
# ~/rabbitmq_server/rabbitmq_server-3.7.7/sbin/rabbitmqctl set_permissions -p <vhost> <user> ".*" ".*" ".*"
cmd = [
os.path.join(upstream.rabbitmq_config_obj.rmq_home,
"sbin/rabbitmqctl")
]
cmd.extend(['add_user', downstream.instance_name + "-admin", "test"])
execute_command(cmd,
env=upstream.env,
err_prefix="Error creating user in upstream server")
cmd = [
os.path.join(
upstream.rabbitmq_config_obj.rabbitmq_config['rmq-home'],
"sbin/rabbitmqctl")
]
cmd.extend([
'set_permissions', "-p",
upstream.rabbitmq_config_obj.rabbitmq_config["virtual-host"]
])
cmd.extend([downstream.instance_name + "-admin", ".*", ".*", ".*"])
execute_command(
cmd,
env=upstream.env,
err_prefix="Error setting user permission in upstream server")
gevent.sleep(1)
upstream.startup_platform(upstream_vip)
gevent.sleep(2)
print("After upstream start")
downstream.startup_platform(downstream_vip)
gevent.sleep(2)
# create federation config and setup federation
content = """federation-upstream:
{host}:
port: {port}
virtual-host: {vhost}
"""
config_path = os.path.join(downstream.volttron_home,
"federation.config")
with open(config_path, 'w') as conf:
conf.write(
content.format(
host=upstream.rabbitmq_config_obj.rabbitmq_config["host"],
port=upstream.rabbitmq_config_obj.
rabbitmq_config["amqp-port-ssl"],
vhost=upstream.rabbitmq_config_obj.
rabbitmq_config["virtual-host"]))
downstream.setup_federation(config_path)
except Exception as e:
print("Exception setting up federation: {}".format(e))
upstream.shutdown_platform()
downstream.shutdown_platform()
raise e
yield upstream, downstream
upstream.shutdown_platform()
downstream.shutdown_platform()
def two_way_federated_rmq_instances(request, **kwargs):
"""
Create two rmq based volttron instances. Create bi-directional data flow channel
by creating 2 federation links
:return: 2 volttron instances - that are connected through federation
"""
instance_1_vip = get_rand_vip()
instance_1_hostname, instance_1_https_port = get_hostname_and_random_port()
instance_1_web_address = 'https://{hostname}:{port}'.format(
hostname=instance_1_hostname, port=instance_1_https_port)
instance_1 = build_wrapper(instance_1_vip,
ssl_auth=True,
messagebus='rmq',
should_start=True,
bind_web_address=instance_1_web_address,
instance_name='volttron1',
**kwargs)
instance_1.enable_auto_csr()
instance_2_vip = get_rand_vip()
instance_2_hostname, instance_2_https_port = get_hostname_and_random_port()
instance_2_webaddress = 'https://{hostname}:{port}'.format(
hostname=instance_2_hostname, port=instance_2_https_port)
instance_2 = build_wrapper(instance_2_vip,
ssl_auth=True,
messagebus='rmq',
should_start=False,
bind_web_address=instance_2_webaddress,
instance_name='volttron2',
**kwargs)
instance_2_link_name = None
instance_1_link_name = None
try:
# create federation config and setup federation link to instance_1
content = dict()
fed = dict()
fed[instance_1.rabbitmq_config_obj.rabbitmq_config["host"]] = {
'port':
instance_1.rabbitmq_config_obj.rabbitmq_config["amqp-port-ssl"],
'virtual-host':
instance_1.rabbitmq_config_obj.rabbitmq_config["virtual-host"],
'https-port':
instance_1_https_port,
'federation-user':
"******".format(instance_2.instance_name)
}
content['federation-upstream'] = fed
import yaml
config_path = os.path.join(instance_2.volttron_home,
"rabbitmq_federation_config.yml")
with open(config_path, 'w') as yaml_file:
yaml.dump(content, yaml_file, default_flow_style=False)
print(f"instance 2 Fed config path:{config_path}, content: {content}")
instance_2.setup_federation(config_path)
instance_2.startup_platform(vip_address=instance_2_vip,
bind_web_address=instance_2_webaddress)
instance_2.enable_auto_csr()
# Check federation link status
with with_os_environ(instance_2.env):
rmq_mgmt = RabbitMQMgmt()
links = rmq_mgmt.get_federation_links()
print(f"instance 2 fed links state: {links[0]['status']}")
assert links and links[0]['status'] == 'running'
instance_2_link_name = links[0]['name']
instance_1.skip_cleanup = True
instance_1.shutdown_platform()
instance_1.skip_cleanup = False
start_rabbit(rmq_home=instance_1.rabbitmq_config_obj.rmq_home,
env=instance_1.env)
# create federation config and setup federation to instance_2
content = dict()
fed = dict()
fed[instance_2.rabbitmq_config_obj.rabbitmq_config["host"]] = {
'port':
instance_2.rabbitmq_config_obj.rabbitmq_config["amqp-port-ssl"],
'virtual-host':
instance_2.rabbitmq_config_obj.rabbitmq_config["virtual-host"],
'https-port':
instance_2_https_port,
'federation-user':
"******".format(instance_1.instance_name)
}
content['federation-upstream'] = fed
import yaml
config_path = os.path.join(instance_1.volttron_home,
"rabbitmq_federation_config.yml")
with open(config_path, 'w') as yaml_file:
yaml.dump(content, yaml_file, default_flow_style=False)
print(f"instance 1 Fed config path:{config_path}, content: {content}")
instance_1.setup_federation(config_path)
instance_1.startup_platform(vip_address=instance_1_vip,
bind_web_address=instance_1_web_address)
import gevent
gevent.sleep(10)
# Check federation link status
with with_os_environ(instance_1.env):
rmq_mgmt = RabbitMQMgmt()
links = rmq_mgmt.get_federation_links()
print(f"instance 1 fed links state: {links[0]['status']}")
assert links and links[0]['status'] == 'running'
instance_1_link_name = links[0]['name']
except Exception as e:
print(f"Exception setting up federation: {e}")
instance_1.shutdown_platform()
instance_2.shutdown_platform()
raise e
yield instance_1, instance_2
if instance_1_link_name:
with with_os_environ(instance_1.env):
rmq_mgmt = RabbitMQMgmt()
rmq_mgmt.delete_multiplatform_parameter('federation-upstream',
instance_1_link_name)
if instance_2_link_name:
with with_os_environ(instance_2.env):
rmq_mgmt = RabbitMQMgmt()
rmq_mgmt.delete_multiplatform_parameter('federation-upstream',
instance_2_link_name)
instance_1.shutdown_platform()
instance_2.shutdown_platform()