def test_queue_api_params():
'''Check that timeout and loops are taking in consideration in the .send method of the Communication Api
for processes communicating via a queue as follows:
1) timeout is respected by the put method of the queue
2) num loops is respected by the put method of the queue. The loops are exhausted, an exception is thrown and
the put method is called loops times
'''
# (1)
queue_factory = factory.QueueCommunication()
timeout = 20
parent = queue_factory.parent(timeout=timeout)
with patch.object(parent.conn, 'put') as mock_put:
parent.send(mpq_protocol.REQ_TEST_PARENT)
message = [mpq_protocol.REQ_TEST_PARENT, parent.pid, None, None]
mock_put.assert_called_with(message, timeout=timeout)
# (2)
parent = queue_factory.parent(timeout=timeout, loops=20)
assert parent.loops == 20
mock_put.reset_mock()
with patch.object(parent.conn, 'put',
side_effect=queue.Full()) as mock_put:
with pytest.raises(errors.QueuesCommunicationError):
parent.send(mpq_protocol.REQ_TEST_PARENT)
assert mock_put.call_count == parent.loops
def test_send_thows_exception_queue_full():
'''Check .send throws an exception when queue is full and not other element can be placed
'''
max_size = 1 # max number of items that can be placed in the queue
queue_factory = factory.QueueCommunication(max_size=max_size)
test_comm = queue_factory.parent(timeout=0.1)
test_comm.send(mpq_protocol.REQ_FINISHED)
time.sleep(0.1)
assert not test_comm.conn.empty()
with pytest.raises(errors.QueuesCommunicationError):
test_comm.send(mpq_protocol.REQ_FINISHED)
def parent_full_duplex_communication_with_children(min_processes,
max_processes):
'''A parent send a message to a specific child and this, and only this, responds with its identifier. Once the the
parent confirms that such message has been received, it sends an DIE request to all children for them all to die.
'''
queue_factory = factory.QueueCommunication()
parent = queue_factory.parent()
parent_pid = multiprocessing.current_process().pid
# (1) Prepare list of processes to start and pass the value = 3 to each child process
child_processes = []
for offset in range(max_processes):
child_process = multiprocessing.Process(name=f'child_process_{offset}',
target=call_child,
args=(offset + 1, parent_pid,
queue_factory, None))
child_processes.append({'process': child_process})
# (2) Start processes
for child in child_processes:
child['process'].start()
child['pid'] = child['process'].pid
# (3) Send a message 3th process
identifier = min_processes
parent.send(mpq_protocol.REQ_DO,
recipient_pid=child_processes[identifier - 1]['pid'])
time.sleep(1)
# (4) Wait for answer from process 3th
stop = False
message = False
while not stop:
message = parent.receive(
func=lambda sms: sms[mpq_protocol.R_PID_OFFSET] == parent_pid)
if message:
stop = True
# (5) Only one message for us parent was in the queue and it should contain the identifier of process 3
assert message[mpq_protocol.S_PID_OFFSET + 2] == identifier
assert parent.queue_empty()
# (6) Indicate all process to die
for _ in range(len(child_processes)):
parent.send(mpq_protocol.REQ_DIE)
# (7) Wait for all processes to finish
for child in child_processes:
child['process'].join()
# Ensure the queue is actually empty - meaning all child processes did fetch their targeted DIE message
parent.queue_join()
def test_block_when_queue_is_empty(worker):
'''Ensure that workers block when the queue is empty but is awaiting for the next message to be popped in
'''
queue_factory = factory.QueueCommunication()
worker.task_queue = queue_factory.parent() # Now the worker will block in the queue
def add_message_to_queue_after_delay(q_factory):
'''Sends a message to the queue for the picker to die right away
'''
time.sleep(1)
child = q_factory.child()
child.send(mpq_protocol.REQ_DIE)
child_process = multiprocessing.Process(target=add_message_to_queue_after_delay, args=(queue_factory,))
child_process.start()
message = worker.run()
# If the worker would have not picked the message sent by the child, it would block indefinitely in the line below
worker.task_queue.queue_join()
assert message is not False
def __init__(self,
identity,
url,
remote_url,
max_processes,
ip_map,
linger=0):
super().__init__()
self.remote_url = remote_url
self.max_processes = max(MIN_PROCESSES, max_processes)
self.ip_map = ip_map # this should be a hash table for O(1) lookups
self.child_processes = []
self.queue_factory = factory.QueueCommunication()
self.conn = self.queue_factory.parent()
self.pid = multiprocessing.current_process().pid
self.sink = Sink(identity=identity,
url=url,
linger=linger,
s_type=zmq.PULL)
def test_children_to_parent_communication():
'''Simple test where all child processes send a message to the parent process
All children are initiated with a value that is sent to the parent for it to process it.
'''
queue_factory = factory.QueueCommunication()
parent = queue_factory.parent()
parent_pid = multiprocessing.current_process().pid
# Prepare list of processes to start and pass the value = 3 to each child process
child_processes = []
val = 3
for offset in range(5):
child_process = multiprocessing.Process(name=f'child_process_{offset}',
target=call_child,
args=(offset + 1, parent_pid,
queue_factory, val))
child_processes.append(child_process)
# Start processes
for child in child_processes:
child.start()
# Wait for the processes to finish
for child in child_processes:
child.join()
# Receive the data from all children
counter = 0
data_offset = mpq_protocol.S_PID_OFFSET + 2
while not parent.queue_empty():
message = parent.receive()
counter += message[data_offset]
# Ensure the queue is empty - no loose strings
parent.queue_join()
# Ensure we got the right data from children
assert counter == val * len(child_processes)
def test_receive_blocking_when_block_false():
'''Check that .receive blocks if nothing is in the queue and block flag is passed as false
'''
# (1) Sample with receive not blocking
queue_factory = factory.QueueCommunication()
parent = queue_factory.parent(timeout=0.1)
message = parent.receive()
# No blocking at receive() therefore receive will return False as no message was in the queue when we queried it
assert message is False
def add_message_to_queue_after_delay(q_factory):
time.sleep(1)
child = q_factory.child()
child.send(mpq_protocol.REQ_FINISHED)
# (2) Sample with receive blocking until something is ready in the queue
child_process = multiprocessing.Process(
target=add_message_to_queue_after_delay, args=(queue_factory, ))
child_process.start()
message = parent.receive(block=True) # we block for a while
parent.queue_join(
) # if parent did not block in the queue and pick the message we would wait here indefinitely
assert message is not False
def test_functionality(self):
'''Test the distributed task queue as a whole
'''
queue_factory = factory.QueueCommunication()
parent = queue_factory.parent()
# (1) Run Collector for 500 * 10(polling-in) = 3 seconds
max_workers = 10
child_collector = self.start_collector(
url=test_utils.TCP_COLLECTOR_URL_SOCKET,
publisher_url=test_utils.TCP_PUBLISHER_URL_SOCKET,
app=stubs.AppStub,
loops=500,
max_workers=max_workers)
assert child_collector.pid
# (2) Run Subscribers for 500 * 10 (polling-in) = 5 seconds
child_subscribers = []
for offset in range(max_workers):
child = self.start_subscriber(
url=test_utils.TCP_PUBLISHER_URL_SOCKET,
identity=f'subscriber_{offset}',
topic=offset,
loops=500,
comm=queue_factory.child())
child_subscribers.append(child)
time.sleep(
0.5) # ---> Give time to collector and subscribers to be ready
# (3) Run Producers
# --> 10 producers sends 10 tasks each to the collector => 100 tasks
child_producers = []
num_tasks = 10
tasks = [f'task_{x}' for x in range(num_tasks)]
for offset in range(max_workers):
child = self.start_producer(
url=test_utils.TCP_COLLECTOR_URL_SOCKET,
identity=f'producer_{offset}',
topic=offset,
tasks=tasks)
child_producers.append(child)
# Wait for producers
for child in child_producers:
child.join()
# Wait for subscribers
for child in child_subscribers:
child.join()
# Wait for collector child
child_collector.join()
# Assert the results
expected_results = max_workers * num_tasks
returned_results = 0
results = defaultdict(set)
while not parent.queue_empty():
returned_result = parent.receive()
topic, data = returned_result[-1]
results[topic].add(data)
returned_results += 1
assert returned_results == expected_results
# Ensure all topics received are the ones expected
assert all(_topic in results for _topic in range(num_tasks))
# Ensure all topics received does have num_tasks results
assert all(len(results[_topic]) == num_tasks for _topic in results)
def test_multiple_producer_one_sink():
'''Test that multiple simultaneous pushes can be handled by a sink. It also tests a practical case
of Multiprocessing queue communication.
It creates 5 child producers that will be sending data to the sink as a PUSH-ends. Then the same producers too
send the same data but now via Multiprocessing queues. At the end we compare that the two data received are
exactly the same to assert that the sink can handle multiple connections.
'''
queue_factory = factory.QueueCommunication()
parent_comm = queue_factory.parent(timeout=0.1)
parent_pid = multiprocessing.current_process().pid
# Create
cname = "ProducerMultiplePushesStub"
producers = []
for offset in range(5):
producers.append(
multiprocessing.Process(target=call_producer,
args=(stubs.ProducerMultiplePushesStub,
test_utils.TCP_CONNECT_URL_SOCKET,
cname + "_" + str(offset),
queue_factory, parent_pid)))
sink_pull = sink.Sink(url=test_utils.TCP_BIND_URL_SOCKET,
identity="SinkReceiveMultiplePushes",
s_type=zmq.PULL)
all_client_data = []
try:
for client in producers:
client.start()
# (1) Let's wait a bit for PUSH sockets to send their data so that they appear in our local PULL socket queues
time.sleep(0.5)
for _ in range(len(producers)):
client_data = sink_pull.run()
assert client_data is not False # The polling will always return real data after the waiting period
all_client_data.append(client_data)
assert len(all_client_data) == len(producers)
# (2) Wait for children processes to finish
for client in producers:
client.join()
# (3) Now we know all data sent by the client via PUSH it has too been done via queues
results = []
for _ in range(len(producers)):
results.append(
parent_comm.receive(func=lambda messages: messages[
mpq_protocol.R_PID_OFFSET] == parent_pid))
# (4) The queue should now be empty and all tasks done
assert parent_comm.conn.empty()
parent_comm.conn.join()
# (5) Check they are the same elements
a = set(message[mpq_protocol.S_PID_OFFSET + 2]['my_id']
for message in results)
b = set(message['my_id'] for message in all_client_data)
assert a == b
except KeyboardInterrupt:
pass
finally:
for client in producers:
client.join()
parent_comm.conn.join()
sink_pull.clean()
def test_multiple_subscriber_one_publisher(self):
'''Test that multiple simultaneous pushes can be handled by a sink. It also tests a practical case
of Multiprocessing queue communication.
It creates 5 child producers that will be sending data to the sink as a PUSH-ends. Then the same producers too
send the same data but now via Multiprocessing queues. At the end we compare that the two data received are
exactly the same to assert that the sink can handle multiple connections.
'''
queue_factory = factory.QueueCommunication()
parent_comm = queue_factory.parent(timeout=0.1)
# Create
subscribers = []
topics = ['A', 'B', 'C', 'D', 'E']
info = {}
for offset in range(5):
subscribers.append(
multiprocessing.Process(target=call_subscriber,
args=(stubs.SubscriberOKStub,
topics[offset],
test_utils.TCP_BIND_URL_SOCKET,
f"Subscriber_{offset}",
queue_factory.child(), 100)))
info[topics[offset]] = f"data for topic {topics[offset]}"
pub = publisher.Publisher(url=test_utils.TCP_BIND_URL_SOCKET,
identity='Publisher',
linger=0)
try:
for client in subscribers:
client.start()
# (1) Let's wait a bit for all subscriber to initialise before the publisher start sending info
time.sleep(0.5)
for offset in range(len(subscribers)):
pub.run(topics[offset], info[topics[offset]])
# (2) Wait for children processes to finish
for client in subscribers:
client.join()
# (3) Now we know all data sent by the publisher needs to be received from subscriber via a shared queue
results = []
for _ in range(len(subscribers)):
results.append(parent_comm.receive(func=lambda x: True)[-1])
# (4) The queue should now be empty and all tasks done
assert parent_comm.conn.empty()
parent_comm.conn.join()
# (5) Check they are the same elements
assert len(results) == len(topics)
for result in results:
_topic, data = result
assert _topic in topics
assert info[_topic] == data
except KeyboardInterrupt:
pass
finally:
for client in subscribers:
client.join()
parent_comm.conn.join()
pub.clean()
def queue_factory():
return factory.QueueCommunication()
def test_comm():
queue_factory = factory.QueueCommunication()
return queue_factory.parent(timeout=0.1)
