def test_incoming_message_threading(self):
max_wait = 30
thread_count = 10
thread_name_condition = threading.Condition()
thread_name = set()
event_topic = UuidGenerator.generate_id_as_string()
with self.create_client(
incoming_message_thread_pool_size=thread_count) as client:
client.connect()
event_callback = EventCallback()
def on_event(_):
with thread_name_condition:
thread_name.add(threading.current_thread())
if len(thread_name) == thread_count:
thread_name_condition.notify_all()
event_callback.on_event = on_event
client.add_event_callback(event_topic, event_callback)
for _ in range(0, 1000):
evt = Event(event_topic)
client.send_event(evt)
start = time.time()
with thread_name_condition:
while (time.time() - start < max_wait) and \
len(thread_name) < thread_count:
thread_name_condition.wait(max_wait)
self.assertEqual(thread_count, len(thread_name))
def test_execute_sync_during_callback(self):
event_topic = UuidGenerator.generate_id_as_string()
req_topic = UuidGenerator.generate_id_as_string()
with self.create_client() as client:
client.connect()
# callback
def event_callback(_):
with self.event_received_condition:
self.event_received = True
try:
req = Request(destination_topic=req_topic)
client.sync_request(req)
except Exception as ex: # pylint: disable=broad-except
self.request_exception_message = str(ex)
self.event_received_condition.notify_all()
callback = EventCallback()
callback.on_event = event_callback
client.add_event_callback(event_topic, callback)
event = Event(destination_topic=event_topic)
client.send_event(event)
start = time.time()
with self.event_received_condition:
while (time.time() - start < self.MAX_WAIT) and \
not self.event_received:
self.event_received_condition.wait(self.MAX_WAIT)
self.assertIsNotNone(self.request_exception_message)
self.assertIn("different thread", self.request_exception_message)
def test_client_receives_event_on_topic_only_after_subscribe(self):
"""
The idea of this test is to send an event to a topic which we are not
subscribed, so we shouldn't be notified. Then, we subscribe to that
topic and send a new event, we should get that last one.
"""
with self.create_client() as client:
test_topic = '/test/whatever/' + client.config._client_id
client.connect()
self.assertTrue(client.connected)
# Set request callback (use mock to easily check when it was called)
ecallback = EventCallback()
ecallback.on_event = Mock()
client.add_event_callback(test_topic, ecallback, False)
# Send event thru dxl fabric to a topic which we are *not* subscribed
msg = Event(destination_topic=test_topic)
client.send_event(msg)
time.sleep(1)
# We haven't been notified
self.assertEqual(ecallback.on_event.call_count, 0)
# Subscribe to topic
client.subscribe(test_topic)
# Send event thru dxl fabric again to that topic
msg = Event(destination_topic=test_topic)
client.send_event(msg)
time.sleep(1)
# Now we should have been notified of the event
self.assertEqual(ecallback.on_event.call_count, 1)
def test_execute_sync_during_callback(self):
event_topic = UuidGenerator.generate_id_as_string()
req_topic = UuidGenerator.generate_id_as_string()
with self.create_client() as client:
client.connect()
# callback
def event_callback(event):
try:
req = Request(destination_topic=req_topic)
client.sync_request(req)
except Exception, e:
self.exceptions.append(e)
raise e
ec = EventCallback()
ec.on_event = event_callback
client.add_event_callback(event_topic, ec)
time.sleep(self.SLEEP_TIME) # Check the time
evt = Event(destination_topic=event_topic)
client.send_event(evt)
time.sleep(self.SLEEP_TIME)
self.assertTrue(self.exceptions[0] is not None)
self.assertTrue("different thread" in self.exceptions[0].message)
def measure_performance(self, client, with_wildcard, topic_exists):
sub_count = 10000
query_multiplier = 10
topic_prefix = "/topic/" + UuidGenerator.generate_id_as_string() + "/"
event_count = [0]
message_ids = set()
payload = UuidGenerator.generate_id_as_string()
payload_as_bytes = payload.encode()
message_id_condition = Condition()
callback = EventCallback()
def on_event(event):
if event.payload == payload_as_bytes:
with message_id_condition:
event_count[0] += 1
message_ids.add(event.message_id)
message_id_condition.notify()
if len(message_ids) % sub_count == 0:
print("Messages size: " + str(len(message_ids)))
callback.on_event = on_event
client.add_event_callback("#", callback, False)
if with_wildcard:
client.subscribe(topic_prefix + "#")
for i in range(sub_count):
if i % 1000 == 0:
print("Subscribed: " + str(i))
client.subscribe(topic_prefix + str(i))
print("Subscribed.")
start_time = time.time()
for j in range(sub_count * query_multiplier):
evt = Event(topic_prefix +
str(j % sub_count +
(sub_count if not topic_exists else 0)))
evt.payload = payload
client.send_event(evt)
with message_id_condition:
while len(message_ids) != sub_count * query_multiplier \
or event_count[0] != sub_count * query_multiplier * (2 if with_wildcard and topic_exists else 1):
current_event = event_count[0]
message_id_condition.wait(5)
if current_event == event_count[0]:
self.fail("Event wait timeout")
self.assertEqual(sub_count * query_multiplier, len(message_ids))
self.assertEqual(
sub_count * query_multiplier *
(2 if with_wildcard and topic_exists else 1), event_count[0])
return time.time() - start_time
def test_client_handle_message_with_event_calls_event_callback(self):
event_callback = EventCallback()
event_callback.on_event = Mock()
self.client.add_event_callback(self.test_channel, event_callback)
# Create and process Event
evt = Event(destination_topic=self.test_channel)._to_bytes()
self.client._handle_message(self.test_channel, evt)
# Check that callback was called
self.assertEqual(event_callback.on_event.call_count, 1)
def measure_performance(self, client, with_wildcard, topic_exists):
SUB_COUNT = 10000
QUERY_MULTIPLIER = 10
TOPIC_PREFIX = "/topic/" + UuidGenerator.generate_id_as_string() + "/"
event_count = [0]
message_ids = set()
PAYLOAD = UuidGenerator.generate_id_as_string()
message_id_condition = Condition()
cb = EventCallback()
def on_event(event):
if event.payload == PAYLOAD:
with message_id_condition:
event_count[0] += 1
message_ids.add(event.message_id)
message_id_condition.notify()
if len(message_ids) % SUB_COUNT == 0:
print "Messages size: " + str(len(message_ids))
cb.on_event = on_event
client.add_event_callback("#", cb, False)
if with_wildcard:
client.subscribe(TOPIC_PREFIX + "#")
for i in range(SUB_COUNT):
if i % 1000 == 0:
print "Subscribed: " + str(i)
client.subscribe(TOPIC_PREFIX + str(i))
print "Subscribed."
start_time = time.time()
for j in range(SUB_COUNT * QUERY_MULTIPLIER):
evt = Event(TOPIC_PREFIX +
str(j % SUB_COUNT +
(SUB_COUNT if not topic_exists else 0)))
evt.payload = PAYLOAD
client.send_event(evt)
with message_id_condition:
while len(message_ids) != SUB_COUNT * QUERY_MULTIPLIER \
or event_count[0] != SUB_COUNT * QUERY_MULTIPLIER * (2 if with_wildcard and topic_exists else 1):
current_event = event_count[0]
message_id_condition.wait(5)
if current_event == event_count[0]:
self.fail("Event wait timeout")
self.assertEquals(SUB_COUNT * QUERY_MULTIPLIER, len(message_ids))
self.assertEquals(
SUB_COUNT * QUERY_MULTIPLIER *
(2 if with_wildcard and topic_exists else 1), event_count[0])
return time.time() - start_time
def test_execute_events(self):
with self.create_client(max_retries=0) as client:
try:
client.connect()
topic = UuidGenerator.generate_id_as_string()
# Create and register an event callback. Ensure that all sent events are received
# (via the outstanding events set). Also, track the number of total events received.
def event_callback(event):
with self.event_condition:
# Increment count of responses received
self.event_count += 1
# Remove from outstanding events
self.remove_outstanding_event(event.message_id)
# Notify that a response has been received (are we done yet?)
self.event_condition.notify_all()
callback = EventCallback()
callback.on_event = event_callback
client.add_event_callback(topic, callback)
for _ in range(0, self.EVENT_COUNT):
event = Event(topic)
self.append_outstanding_event(event.message_id)
client.send_event(event)
with self.event_condition:
while self.event_count != self.EVENT_COUNT:
current_count = self.event_count
self.event_condition.wait(self.MAX_EVENT_WAIT)
if current_count == self.event_count:
self.fail("Event wait timeout.")
self.assertEqual(0, len(self.outstanding_events))
print("Events test: PASSED")
except Exception as ex:
print(ex)
raise ex
def test_execute_message_other_fields(self):
with self.create_client(max_retries=0) as client:
client.connect()
topic = UuidGenerator.generate_id_as_string()
def on_event(event):
with self.event_received_condition:
try:
self.event_received = event
except Exception as ex: # pylint: disable=broad-except
print(ex)
self.event_received_condition.notify_all()
event_callback = EventCallback()
event_callback.on_event = on_event
client.add_event_callback(topic, event_callback)
event = Event(destination_topic=topic)
event.other_fields = {"key" + str(i): "value" + str(i)
for i in range(self.OTHER_FIELDS_COUNT)}
event.other_fields[b"key_as_bytes"] = b"val_as_bytes"
client.send_event(event)
# Bytes values for other field keys/values are expected to be
# converted to unicode strings as received from the DXL fabric.
del event.other_fields[b"key_as_bytes"]
event.other_fields[u"key_as_bytes"] = u"val_as_bytes"
start = time.time()
with self.event_received_condition:
while (time.time() - start < self.MAX_WAIT) and \
not self.event_received:
self.event_received_condition.wait(self.MAX_WAIT)
self.assertIsNotNone(self.event_received)
self.assertIsNotNone(self.event_received.other_fields)
for i in range(self.OTHER_FIELDS_COUNT):
self.assertEqual(
event.other_fields["key" + str(i)],
self.event_received.other_fields.get("key" + str(i), ""))
self.assertEqual(event.other_fields["key_as_bytes"],
self.event_received.other_fields.get(
"key_as_bytes", ""))
def test_incoming_message_threading(self):
thread_count = 10
thread_name = set()
event_topic = UuidGenerator.generate_id_as_string()
with self.create_client(incoming_message_thread_pool_size=thread_count) as client:
client.connect()
event_callback = EventCallback()
def on_event(event):
thread_name.add(threading.current_thread())
event_callback.on_event = on_event
client.add_event_callback(event_topic, event_callback)
for i in range(0, 1000):
evt = Event(event_topic)
client.send_event(evt)
time.sleep(30)
self.assertEquals(thread_count, len(thread_name))
def test_client_remove_call_for_unregistered_callback_does_not_error(self):
callback = EventCallback()
callback.on_event = Mock()
callback2 = EventCallback()
callback2.on_event = Mock()
self.client.add_event_callback(self.test_channel, callback)
self.client.add_event_callback(self.test_channel, callback2)
self.client.remove_event_callback(self.test_channel, callback)
self.client.remove_event_callback(self.test_channel, callback)
def test_wildcard_services(self):
max_wait = 10
with self.create_client() as client:
# The request message that the service receives
service_request_message = []
# The request message corresponding to the response received by the client
client_response_message_request = []
# The event that we received
client_event_message = []
client_event_message_condition = Condition()
# The payload that the service receives
service_request_message_receive_payload = []
client.connect()
info = ServiceRegistrationInfo(client, "myWildcardService")
meta = {}
# Transform events mapped to "test/#/" to "request/test/..."
meta["EventToRequestTopic"] = "/test/#"
meta["EventToRequestPrefix"] = "/request"
info.metadata = meta
rcb = RequestCallback()
def on_request(request):
print("## Request in service: " + request.destination_topic + ", " + str(request.message_id))
print("## Request in service - payload: " + request.payload)
service_request_message.append(request.message_id)
service_request_message_receive_payload.append(request.payload)
response = Response(request)
response.payload = "Request response - Event payload: " + request.payload
client.send_response(response)
rcb.on_request = on_request
info.add_topic("/request/test/#", rcb)
client.register_service_sync(info, 10)
evt = Event("/test/bar")
rcb = ResponseCallback()
def on_response(response):
# Only handle the response corresponding to the event we sent
if response.request_message_id == evt.message_id:
print("## received_response: " + response.request_message_id + ", " + response.__class__.__name__)
print("## received_response_payload: " + response.payload)
client_response_message_request[0] = response.request_message_id
rcb.on_response = on_response
client.add_response_callback("", rcb)
ecb = EventCallback()
def on_event(event):
print("## received event: " + event.destination_topic + ", " + event.message_id)
with client_event_message_condition:
client_event_message.append(event.message_id)
client_event_message_condition.notify_all()
ecb.on_event = on_event
client.add_event_callback("/test/#", ecb)
# Send our event
print("## Sending event: " + evt.destination_topic + ", " + evt.message_id)
evt.payload = "Unit test payload"
client.send_event(evt)
start = time.time()
with client_event_message_condition:
while (time.time() - start < max_wait) and \
not client_event_message:
client_event_message_condition.wait(max_wait)
# # Make sure the service received the request properly
# self.assertEqual(evt.message_id, service_request_message[0])
# # Make sure the service received the request payload from the event properly
# self.assertEqual(evt.payload, service_request_message_receive_payload[0])
# Make sure the response we received was for the request message
# self.assertEqual(evt.message_id, client_response_message_request[0])
# Make sure we received the correct event
self.assertGreater(len(client_event_message), 0)
self.assertEqual(evt.message_id, client_event_message[0])
def run():
try:
with client_factory(max_retries=0) as client:
retries = self.MAX_CONNECT_RETRIES
connected = False
while not connected and retries > 0:
try:
self.connect_time_start = time.time()
client.connect()
connected = True
except Exception:
if retries > 0:
retries -= 1
self.connect_retries += 1
self.assertTrue(connected,
"Unable to connect after retries")
def on_event(event):
with self.event_count_condition:
self.event_count += 1
current_count = self.event_count
self.event_count_condition.notify_all()
if current_count % 100 == 0:
print client.config._client_id + " : " + str(
current_count) + " : " + event.payload
# callback registration
callback = EventCallback()
callback.on_event = on_event
client.add_event_callback(event_topic, callback)
# Waiting all clients have connected
with self.connect_condition:
self.atomic_connect_count += 1
curr_count = self.atomic_connect_count
self.connect_condition.notify_all()
while self.atomic_connect_count != self.THREAD_COUNT:
self.connect_condition.wait(
timeout=self.MAX_CONNECT_WAIT)
if curr_count == self.atomic_connect_count:
self.fail(
"Timeout waiting for all threads to connect"
)
# Once all clients have connected, reset timing information
if self.requests_start_time == 0:
self.requests_start_time = time.time()
self.connect_time = self.requests_start_time - self.connect_time_start
for i in range(0, self.EVENT_COUNT):
event = Event(event_topic)
if i % 10 == 0:
print "###send: " + str(i)
event.payload = str(i)
send_client.send_event(event)
with self.event_count_condition:
while self.event_count != self.EVENT_COUNT * self.THREAD_COUNT:
curr_count = self.event_count
self.event_count_condition.wait(
timeout=self.MAX_CONNECT_WAIT)
if self.event_count == curr_count:
self.fail(
"Timed out while receiving events")
self.event_count_condition.notify_all()
if self.requests_end_time == 0:
self.requests_end_time = time.time()
except Exception, e:
logging.error(e.message)
raise e
def run():
try:
with client_factory(max_retries=0) as client:
retries = self.MAX_CONNECT_RETRIES
connected = False
while not connected and retries > 0:
try:
self.connect_time_start = time.time()
client.connect()
connected = True
except Exception: # pylint: disable=broad-except
if retries > 0:
retries -= 1
self.connect_retries += 1
self.assertTrue(connected,
"Unable to connect after retries")
def on_event(event):
with self.event_count_condition:
self.event_count += 1
current_count = self.event_count
if current_count == \
self.EVENT_COUNT * self.THREAD_COUNT:
self.event_count_condition.notify_all()
if current_count % 100 == 0:
print(client.config._client_id + " : " +
str(current_count) + " : " +
event.payload.decode("utf8"))
# callback registration
callback = EventCallback()
callback.on_event = on_event
client.add_event_callback(event_topic, callback)
# Waiting all clients have connected
with self.connect_condition:
self.atomic_connect_count += 1
if self.atomic_connect_count == self.THREAD_COUNT:
self.connect_condition.notify_all()
time_remaining = self.MAX_TIME
while self.atomic_connect_count != \
self.THREAD_COUNT and time_remaining > 0:
self.connect_condition.wait(
timeout=time_remaining)
time_remaining = start - time.time() + \
self.MAX_TIME
self.assertEqual(
self.THREAD_COUNT, self.atomic_connect_count,
"Timeout waiting for all threads to connect")
# Once all clients have connected, reset timing information
if self.requests_start_time == 0:
self.requests_start_time = time.time()
self.connect_time = \
self.requests_start_time - \
self.connect_time_start
for i in range(0, self.EVENT_COUNT):
event = Event(event_topic)
if i % 10 == 0:
print("###send: " + str(i))
event.payload = str(i)
send_client.send_event(event)
with self.event_count_condition:
time_remaining = self.MAX_TIME
while self.event_count != \
self.EVENT_COUNT * self.THREAD_COUNT and \
time_remaining > 0:
self.event_count_condition.wait(
timeout=time_remaining)
time_remaining = start - time.time() + \
self.MAX_TIME
self.assertEqual(
self.EVENT_COUNT * self.THREAD_COUNT,
self.event_count,
"Timed out while receiving events")
if self.requests_end_time == 0:
self.requests_end_time = time.time()
except Exception as ex:
logging.error(ex)
raise ex