def test_basic_messaging_with_all_clients_concurrently(iqa: Instance,
length: int) -> None:
"""
Exchange messages through the router using three pairs of:
- Java Sender and Receiver
- Python Sender and Receiver, and
- NodeJS Sender and Receiver.
Expects that all messages are exchanged and all external clients complete successfully.
:param iqa:
:param length:
:return:
"""
receivers: List[ReceiverType] = iqa.get_clients(client_type=ReceiverType)
senders: List[SenderType] = iqa.get_clients(client_type=SenderType)
# Run all available clients in parallel
for receiver in receivers:
start_receiver(receiver)
for sender in senders:
start_sender(sender, length)
# Validate all results
for receiver, sender in zip(receivers, senders):
validate_client_results(receiver, sender)
def test_asynchronous_durable_subscription(self, topic_durable, broker,
iqa: Instance):
"""
This test must be defined as the second one (as tests defined in classes run sequentially in py.test).
With that said, the previous test left the durable subscriptions available in the related Broker instance.
So when this test runs, the Publishers will run first and will publish a pre-defined (self.MESSAGES) number
of messages into the related multicast address (topic_durable).
It waits till all publishers are done sending (or timed-out).
Next it will connect one Durable Subscriber instance with the "topic_durable" address across all routers
in the topology. Then it waits till all receivers are done receiving (or timed-out).
Then it validates:
- Number of messages sent
- Number of messages received by each receiver (expecting self.MESSAGES * len(routers))
- Integrity of received messages by comparing received body SHA1 with unique SHA1 sum from sent body.
:param topic_durable: Fixture that provides the topic to send/receive from
:param broker: A Broker component instance (not being used yet, but illustrates which broker is being used)
:param iqa: IQAInstance fixture that provides a list with all routers that will be used
:return:
"""
# Broker instance
broker_instance = iqa.get_brokers(broker)[0]
assert broker_instance
# List of routers to use
routers = iqa.get_routers()
# Create publisher list
publishers = self.create_publishers(routers, topic_durable)
# Wait till all senders have delivered their messages
[p.join() for p in publishers]
# Assert all senders sent correct amount of messages
self.validate_all_messages_sent(publishers)
# Create subscriber list
# At this point, as previous test (synchronous) completed, a durable subscription already
# exists, so subscribers should be able to retrieve their messages properly now
subscribers = self.create_subscribers(routers,
topic_durable,
durable=True)
# Wait till all subscribers are done
# the stopped flag will turn into true if any of them times out
[s.join() for s in subscribers]
# Assert that all receivers received expected amount of messages
self.validate_all_messages_received(publishers[0].message_body,
routers, subscribers)
def broker_s_internal(iqa: Instance) -> Broker:
"""
Returns the slave broker instance connected to internal 2 router
:param iqa:
:return:
"""
return iqa.get_brokers('Broker.S.I2')[0]
def broker_m_edge(iqa: Instance) -> Broker:
"""
Returns the master broker instance connected to edge 3 router
:param iqa:
:return:
"""
return iqa.get_brokers('Broker.M.E3')[0]
def broker_s_edge(iqa: Instance) -> Broker:
"""
Returns the slave broker instance connected to edge 3 router
:param iqa:
:return:
"""
return iqa.get_brokers('Broker.S.E3')[0]
class TestInstanceNodeDocker:
iqa_instance: Instance = Instance()
@pytest.fixture
def instance(self) -> Instance:
return TestInstanceNodeDocker.iqa_instance
@pytest.fixture
def node(self, instance: Instance) -> Node:
executor: ExecutorContainer = ExecutorContainer(
name="Docker executor", container_name='sshd-iqa')
example_node: NodeDocker = NodeDocker(hostname="sshd-iqa",
executor=executor)
instance.nodes.append(example_node)
node: Node = instance.nodes[0]
return node
def test_ping(self, node: NodeDocker) -> None:
node_ping: bool = node.ping()
assert node_ping
def test_get_ip(self, node: NodeDocker) -> None:
node_ip: str = node.get_ip()
assert node_ip is not None
def router_i3(iqa: Instance) -> Dispatch:
"""
Returns the router
:param iqa:
:return: Returns router instance
"""
return iqa.get_routers('Router.I3')[0]
def test_synchronous_nondurable_subscription(self, topic_nondurable,
broker, iqa: Instance):
"""
Connects one Non-Durable Subscriber instance to the "topic_durable" address across all routers.
Once all subscribers are connected, it starts one Publisher against each router
in the topology.
Next the test waits till all senders are done and till all receivers are done receiving (or timed-out).
Then it validates:
- Number of messages sent
- Number of messages received by each receiver (expecting self.MESSAGES * len(routers))
- Integrity of received messages by comparing received body SHA1 with unique SHA1 sum from sent body.
:param topic_nondurable: Fixture that provides the topic to send/receive from
:param broker: A Broker component instance (not being used yet, but illustrates which broker is being used)
:param iqa: IQAInstance fixture that provides a list with all routers that will be used
:return:
"""
# Broker instance
broker_instance = iqa.get_brokers(broker)[0]
assert broker_instance
# List of routers to use
routers = iqa.get_routers()
# Create subscriber list
subscribers = self.create_subscribers(routers,
topic_nondurable,
durable=False)
# Wait till all receivers have been created
while not all(s.receiver for s in subscribers):
time.sleep(TestDurableNonDurableSubscription.DELAY)
# Create publisher list
publishers = self.create_publishers(routers, topic_nondurable)
# Wait till all publishers and subscribers done
# the stopped flag will turn into true if any of them times out
[p.join() for p in publishers]
[s.join() for s in subscribers]
# Assert all senders sent correct amount of messages
self.validate_all_messages_sent(publishers)
# Assert that all receivers received expected amount of messages
self.validate_all_messages_received(publishers[0].message_body,
routers, subscribers)
def test_mesh_after_scale_down(router_cluster, iqa: Instance):
"""
Queries the router to validate that the number of Nodes in the topology is 1.
:param router_cluster:
:return:
"""
router: Dispatch = router_cluster[0]
validate_mesh_size(router, 1 * len(iqa.get_routers()))
def sender(request,
iqa: Instance) -> Union[SenderJava, SenderPython, SenderNodeJS]:
if "sender_" in request.param:
snd: List[str] = request.param.split('_')
sender_implementation: str = snd[1]
sender: SenderType = iqa.get_clients(SenderType,
sender_implementation)[0]
return sender
def nodejs_sender(iqa: Instance):
"""
Returns the first NodeJS Sender instance or None
:param iqa:
:return:
"""
assert iqa
return first_or_none(iqa.get_clients(SenderSubtype, 'nodejs'))
def python_sender(iqa: Instance):
"""
Returns the first Python Sender instance or None
:param iqa:
:return:
"""
assert iqa
return first_or_none(iqa.get_clients(SenderSubtype, 'python'))
def java_sender(iqa: Instance):
"""
Returns the first Java Sender instance or None
:param iqa:
:return:
"""
assert iqa
return first_or_none(iqa.get_clients(SenderSubtype, 'java'))
def router(iqa: Instance):
"""
Returns the first Router instance or None
:param iqa:
:return:
"""
assert iqa
return first_or_none(iqa.get_routers())
def receiver(
request,
iqa: Instance) -> Union[ReceiverJava, ReceiverPython, ReceiverNodeJS]:
if "receiver_" in request.param:
rcv: List[str] = request.param.split('_')
receiver_implementation: str = rcv[1]
receiver: ReceiverType = iqa.get_clients(ReceiverType,
receiver_implementation)[0]
return receiver
def test_router_mesh_after_scale_up(router_cluster: Tuple[Dispatch, str, str], iqa: Instance):
"""
Queries Router for all Node Entities available in the topology.
It expects the number of nodes matches number of PODs (mesh is correctly formed).
:param router_cluster:
:return:
"""
router: Dispatch = router_cluster[0]
validate_mesh_size(router, MESH_SIZE * len(iqa.get_routers()))
def router_cluster(request, iqa: Instance) -> Tuple[Dispatch, str, str]:
"""
Returns the router, cluster and token from parameter
:param request:
:param iqa:
:return: Returns router instance based on parametrized info
"""
if 'router_' in request.param:
router_number: int = int(request.param.split('_')[1])
return (iqa.get_routers()[router_number],
request.config.option.cluster[router_number],
request.config.option.token[router_number])
def sender(request,
iqa: Instance) -> Union[SenderJava, SenderPython, SenderNodeJS]:
"""
Fixture the first Sender instance
:param request:
:param iqa:
:return: Returns first Sender instance on 1 cluster instance
"""
if "sender_" in request.param:
s: list = request.param.split('_')
sender_implementation: str = s[1]
sender_number: int = int(s[2])
sender: SenderType = iqa.get_clients(Type[Sender],
sender_implementation)[0]
sender.set_url("amqp://%s:5672/address" %
request.config.option.cluster[sender_number])
return sender
def pytest_configure(config) -> None:
"""
Loads IQA instance based on provided environment and extra command line args.
All arguments will be available as variables that can be used inside the inventory.
The same can be done when using Ansible CLI (using -e cli_arg=value).
:param config:
:return:
"""
# Adding all arguments as environment variables, so child executions of Ansible
# will be able to use the same variables.
options: dict = dict(config.option.__dict__)
# Insert array elements with _0, _1, such as --router 1.1.1.1 and --router 2.2.2.2
# would become: router_0: 1.1.1.1 and router_1: 2.2.2.2
new_options: dict = dict()
for (key, value) in options.items():
if not isinstance(value, list):
continue
for n in range(len(value)):
new_options.update({'%s_%d' % (key, n): str(value[n])})
options.update(new_options)
options = {
key: str(value) for (key, value) in options.items() if key not in os.environ
}
os.environ.update(options)
# Loading the inventory
iqa: Instance = Instance(
inventory=config.getvalue('inventory'), cli_args=config.option.__dict__
)
# Adjusting clients timeout
for client in iqa.clients:
client.command.control.timeout = CLIENTS_TIMEOUT # type: ignore
config.iqa = iqa
# Clean up temporary files at exit
atexit.register(cleanup_files)
intro_message = """
This sample will iterate through all the 'broker' abstract defined at
the '%s' inventory file and it will then:
- Display the broker node hostname and its current status (if able to
communicate with it)
- Attempt to start the broker component if it is not running (using a
valid system service or a docker container)
Note: You can stop the broker component and validate if it gets started
- Iterate through all queues and displaying its FQQN and the message count
""" % inventory
print(intro_message)
# Loading the instance
print("Loading IQAInstance using inventory file: %s" % inventory)
iqa = Instance(inventory)
# Iterating through brokers
for broker in iqa.brokers: # type: (ServerComponent, Broker)
# List broker node hostname and its status
print("* Broker: %s - Status: %s" %
(broker.node.hostname, broker.service.status().name))
# If broker "service" is reported as not running, try starting it
if broker.service.status() != ServiceStatus.RUNNING:
print(" -> starting service: %s" % broker.service.name)
broker.service.start()
# Wait till broker web port is available
for attempt in range(MAX_ATTEMPTS):
# from messaging_components import clients, routers, brokers
from iqa.components.clients.external import nodejs, python
from iqa.components.clients.core.receiver import ReceiverCore
from iqa.components.clients.core.sender import SenderCore
from iqa.components.brokers.artemis import Artemis
from iqa.components.routers.dispatch import Dispatch
from iqa.instance.instance import Instance
from iqa.pytest.fixtures import iqa as pytest_iqa
############################
# Global python namespace #
############################
iqa_instance = Instance()
def pytest_namespace():
"""
Provide iqa_instance to pytest global namespace
"""
return {'iqa': iqa_instance}
@pytest.fixture
def iqa():
"""
IQA instance with accessible node, messaging_components
:return:
"""
def broker_master(request, iqa: Instance) -> BrokerType:
if "Broker.M." in request.param:
broker_hostname = request.param
return iqa.get_brokers(broker_hostname)[0]
def router_interior(request, iqa: Instance) -> RouterType:
if "Router.I" in request.param:
router_hostname = request.param
return iqa.get_routers(router_hostname)[0]
def router_with_broker(request, iqa: Instance) -> RouterType:
if "Router." in request.param:
router_hostname = request.param
return iqa.get_routers(router_hostname)[0]
intro_message: str = """
This sample will iterate through all the 'broker' abstract defined at
the '%s' inventory file and it will then:
- Display the broker node hostname and its current status (if able to
communicate with it)
- Attempt to start the broker component if it is not running (using a
valid system service or a docker container)
Note: You can stop the broker component and validate if it gets started
- Iterate through all queues and displaying its FQQN and the message count
""" % inventory
print(intro_message)
# Loading the instance
print("Loading IQAInstance using inventory file: %s" % inventory)
iqa: Instance = Instance(inventory)
# Iterating through brokers
for broker in iqa.brokers: # type: (ServerComponent, Broker)
# List broker node hostname and its status
print("* Broker: %s - Status: %s" %
(broker.node.hostname, broker.service.status().name))
# If broker "service" is reported as not running, try starting it
if broker.service.status() != ServiceStatus.RUNNING:
print(" -> starting service: %s" % broker.service.name)
broker.service.start()
# Wait till broker web port is available
for attempt in range(MAX_ATTEMPTS):
def test_address_translation_receiving(self, address, translates_to,
receiver, broker, router,
iqa: Instance):
"""
Receive messages from the provided "address" connecting with the "router" instance.
This test will execute an external client using the "receiver" instance and expect it to
consume RECV_COUNT messages from the given "address". The "address" used for receiving should resolve
as the "translates_to" value, which must be a queue name on the "broker" instance.
The number of messages in the respective queue must be equals or greater than RECV_COUNT.
This test will validate number of received messages as well as perform an SHA1 sum based
on the message's body, which must match the generated SHA1 sum from message sent earlier.
:param address:
:param translates_to:
:param receiver:
:param broker:
:param router:
:param iqa:
:return:
"""
# if not router.node.hostname.startswith('Router.I'):
# return
# Get broker instance for given broker name
broker_instance: Broker = iqa.get_brokers(broker)[0]
assert broker_instance
# Retrieving current number of messages in the destination address (queue)
queue = self._get_queue(broker_instance, translates_to)
initial_message_count = int(queue.message_count)
logging.info(
"Initial message count at queue %s - after receivers completed = %s"
% (translates_to, queue.message_count))
# Assert queue has been found and senders were able to send something
assert queue
assert initial_message_count >= self.RECV_COUNT
# Url to be used by receivers
url = "amqp://%s:%s/%s" % (router.node.get_ip(), router.port, address)
# Preparing receiver
logging.info("Receiving messages from %s - using %s" %
(url, receiver.implementation))
receiver.reset_command()
receiver.set_url(url)
receiver.command.control.count = self.RECV_COUNT
# cannot be used with cli-rhea as it is "waiting" for the given amount of time (causing a timeout to happen)
if receiver.implementation != 'nodejs':
receiver.command.control.timeout = self.TIMEOUT # Timeout flag for command to be executed
receiver.command.logging.log_msgs = 'dict'
receiver.command.timeout = self.TIMEOUT # Timeout for command
# Executes external receiver and waits until it finishes (or times out)
receiver.receive()
receiver.execution.wait()
# Validating results
logging.info(
"Receiver exit code: %s - timed out: %s" %
(receiver.execution.returncode, receiver.execution.timed_out))
assert receiver.execution.completed_successfully()
# Validating message integrity
stdout_lines = receiver.execution.read_stdout(lines=True)
assert len(stdout_lines) == self.RECV_COUNT
# Reading each message body and comparing SHA1 sum
for recv_msg in stdout_lines:
# Failing if a blank line was received
if not recv_msg:
pytest.fail("Not expecting an empty message")
continue
try:
recv_msg_dict = ast.literal_eval(recv_msg)
except ValueError:
pytest.fail("Invalid message body returned. Expecting a dict.")
# Failing if invalid content received from external client
if 'content' not in recv_msg_dict.keys():
pytest.fail(
'Expecting a content element as part of message dict.')
continue
# Failing if message returned with empty body
body = recv_msg_dict['content']
if not body:
pytest.fail("No message body available")
continue
# Validate integrity
assert hashlib.sha1(
body.encode('utf-8')).hexdigest() == self.MESSAGE_SHA1SUM
# Delaying 5 secs to clean up everything
time.sleep(5)
# Validates if all messages have been received
queue = self._get_queue(broker_instance, translates_to)
logging.info(
"Message count at queue %s - after receivers completed = %s" %
(translates_to, queue.message_count))
assert (initial_message_count - self.RECV_COUNT) == int(
queue.message_count)
def test_address_translation_sending(self, address, translates_to, sender,
broker, router, iqa: Instance):
"""
Send messages to the given "address", through the provided "router" instance. It uses the given
"sender" (ClientExternal) instance and expects the queue with name "translates_to" to exist in
the "broker" instance and that the number of messages in it will increase from initial message count
to the value defined in the SEND_COUNT constant.
:param address:
:param translates_to:
:param sender:
:param broker:
:param router:
:param iqa:
:return:
"""
# if not router.node.hostname.startswith('Router.I'):
# return
# Get broker instance for given broker name
broker_instance: Broker = iqa.get_brokers(broker)[0]
assert broker_instance
# Retrieving current number of messages in the destination address (queue)
queue = self._get_queue(broker_instance, translates_to)
initial_message_count = int(queue.message_count)
# Assert queue has been found
assert queue
assert initial_message_count is not None
# Url to be used by senders and receivers
url = "amqp://%s:%s/%s" % (router.node.get_ip(), router.port, address)
# Preparing the external sender
logging.info("Sending messages to %s - using %s" %
(url, sender.implementation))
sender.reset_command()
sender.set_url(url)
sender.command.control.count = self.SEND_COUNT
sender.command.control.timeout = self.TIMEOUT # Timeout flag for command to be executed
sender.command.timeout = self.TIMEOUT # Timeout for command (needed cause timeout flag is working properly)
# Defining the message to be sent
message = Message()
message.body = self.MESSAGE_BODY
# Sending and waiting for app to finish
sender.send(message)
sender.execution.wait()
# Validating sender completed successfully
logging.debug(
"Sender exit code: %s - timed out: %s" %
(sender.execution.returncode, sender.execution.timed_out))
assert sender.execution.completed_successfully()
# Delaying 5 secs to clean up everything
time.sleep(5)
# Validates if all messages have been delivered
queue = self._get_queue(broker_instance, translates_to)
logging.info(
"Message count at queue %s - after senders completed = %s" %
(translates_to, queue.message_count))
assert (self.SEND_COUNT + initial_message_count) == int(
queue.message_count)
def test_asynchronous_nondurable_subscription(self, topic_nondurable,
broker, iqa: Instance):
"""
Publishers run first and will publish a pre-defined (self.MESSAGES) number of messages into the related
multi-cast address (topic_nondurable).
It waits till all publishers are done sending (or timed-out).
Next it will connect one Non-Durable Subscriber instance with the "topic_nondurable" address across all routers
in the topology. Then it waits till all receivers time-out.
Then it validates:
- Number of messages sent
- Expect all receivers to time-out
- Number of messages received by each receiver (expecting 0)
:param topic_nondurable: Fixture that provides the topic to send/receive from
:param broker: A Broker component instance (not being used yet, but illustrates which broker is being used)
:param iqa: IQAInstance fixture that provides a list with all routers that will be used
:return:
"""
async_timeout = 30
broker_instance = iqa.get_brokers(broker)[0]
assert broker_instance
# List of routers to use
routers = iqa.get_routers()
# Create subscriber list
subscribers = self.create_subscribers(routers,
topic_nondurable,
durable=False,
timeout=async_timeout)
# Wait till all receivers have been created
while not all(s.receiver for s in subscribers):
time.sleep(TestDurableNonDurableSubscription.DELAY)
# Now stop all receivers to ensure non-durable subscription was discarded
[s.stop_receiver() for s in subscribers]
# Create publisher list
publishers = self.create_publishers(routers, topic_nondurable)
# Wait till all publishers are done sending
[p.join() for p in publishers]
# Create subscribers now with a small timeout and expect nothing to be received
subscribers = self.create_subscribers(routers,
topic_nondurable,
durable=False,
timeout=async_timeout)
# Wait till all subscribers timeout
[s.join() for s in subscribers]
# Assert all senders sent correct amount of messages
self.validate_all_messages_sent(publishers)
# Assert that all receivers did not receive any message and that all of them timed out
assert all([s.received == 0
for s in subscribers]), "Expecting no message received"
assert all([s.timeout_handler.timed_out() for s in subscribers
]), "Expecting all receivers to timeout"
from iqa.components.routers.dispatch.dispatch import Dispatch
from iqa.instance.instance import Instance
from iqa.system.executor import ExecutorBase
from iqa.system.node.node import Node
from iqa.system.service import Service
from pytest_iqa_instance.fixtures import iqa as pytest_iqa
############################
# Global python namespace #
############################
iqa_instance: Instance = Instance()
def pytest_namespace() -> Dict[str, Instance]:
"""
Provide iqa_instance to pytest global namespace
"""
return {'iqa': iqa_instance}
@pytest.fixture
def iqa():
"""
IQA instance with accessible node, messaging_components
:return:
"""
