def test_multiple_registrations(self): service_registration_count = 10 request_received_count = [0] topic = UuidGenerator.generate_id_as_string() with self.create_client() as service_client: service_client.connect() def my_request(request): request_received_count[0] += 1 response = Response(request) service_client.send_response(response) reg_info = ServiceRegistrationInfo(service_client, "multiple_registrations_test") callback = RequestCallback() callback.on_request = my_request reg_info.add_topic(topic, callback) with self.create_client() as request_client: request_client.connect() for _ in range(0, service_registration_count): service_client.register_service_sync(reg_info, self.DEFAULT_TIMEOUT) request = Request(topic) response = request_client.sync_request( request, timeout=self.RESPONSE_WAIT) self.assertNotIsInstance(response, ErrorResponse) self.assertEqual(request.message_id, response.request_message_id) service_client.unregister_service_sync(reg_info, self.DEFAULT_TIMEOUT) self.assertEqual(service_registration_count, request_received_count[0])
def test_error_message(self): with self.create_client() as client: test_service = TestService(client, 1) client.connect() error_code = 9090 error_message = "My error message" topic = UuidGenerator.generate_id_as_string() # # Create a test service that returns error messages # reg_info = ServiceRegistrationInfo(client, "testErrorMessageService") reg_info.add_topic(topic, test_service) client.register_service_sync(reg_info, self.DEFAULT_TIMEOUT) test_service.return_error = True test_service.error_code = error_code test_service.error_message = error_message client.add_request_callback(topic, test_service) # Send a request and ensure the response is an error message response = client.sync_request(Request(topic)) self.assertIsInstance(response, ErrorResponse, msg="Response is not an ErrorResponse") self.assertEqual(error_code, response.error_code) self.assertEqual(error_message, response.error_message)
def add_client_callbacks(self, client, on_client_request_callback=None): request_callback = RequestCallback() def on_request(request): logging.info(request.destination_topic) logging.info(request.payload) if on_client_request_callback: on_client_request_callback() response = Response(request) response.payload = "Ok" try: client.send_response(response) except DxlException as ex: print("Failed to send response" + str(ex)) request_callback.on_request = on_request self.info = ServiceRegistrationInfo(client, "/mcafee/service/JTI") self.info_registered = False self.info_registrations = 0 self.info_unregistrations = 0 service_id = self.info.service_id self.info.add_topic( "/mcafee/service/JTI/file/reputation/" + service_id, request_callback) self.info.add_topic( "/mcafee/service/JTI/cert/reputation/" + service_id, request_callback) def is_event_for_service(event): return json.loads(event.payload.decode("utf8").rstrip( "\0"))["serviceGuid"] == service_id class ServiceRegisteredCallback(EventCallback): def __init__(self, test): self.test = test super(ServiceRegisteredCallback, self).__init__() def on_event(self, event): if is_event_for_service(event): self.test.info_registrations += 1 self.test.info_registered = True class ServiceUnregisteredCallback(EventCallback): def __init__(self, test): self.test = test super(ServiceUnregisteredCallback, self).__init__() def on_event(self, event): if is_event_for_service(event): self.test.info_unregistrations += 1 self.test.info_registered = False client.add_event_callback(_ServiceManager.DXL_SERVICE_REGISTER_CHANNEL, ServiceRegisteredCallback(self)) client.add_event_callback( _ServiceManager.DXL_SERVICE_UNREGISTER_CHANNEL, ServiceUnregisteredCallback(self))
def create_service_reg_info(): reg_info = ServiceRegistrationInfo( service_client, "registry_specified_service_id_test") callback = RequestCallback() callback.on_request = \ lambda request: my_request(reg_info.service_id, request) reg_info.add_topic(topic, callback) service_client.register_service_sync(reg_info, self.DEFAULT_TIMEOUT) return reg_info
def test_response_service_not_found_no_service_id_at_client(self): request_received = [False] topic = UuidGenerator.generate_id_as_string() with self.create_client() as service_client: service_client.connect() def my_request(request): request_received[0] = True service_client.send_response(Response(request)) reg_info = ServiceRegistrationInfo( service_client, "response_service_not_found_no_service_id_at_client_test") callback = RequestCallback() callback.on_request = my_request reg_info.add_topic(topic, callback) reg_info.add_topic(topic, callback) service_client.register_service_sync(reg_info, self.DEFAULT_TIMEOUT) self.assertIsNotNone( self.query_service_registry_by_service( service_client, reg_info)) with self.create_client() as request_client: request_client.connect() # Remove the service's registration with the client-side # ServiceManager, avoiding unregistration of the service from # the broker. This should allow the broker to forward the # request on to the service client. registered_services = service_client._service_manager.services service = registered_services[reg_info.service_id] del registered_services[reg_info.service_id] request = Request(topic) response = request_client.sync_request( request, timeout=self.RESPONSE_WAIT) # Re-register the service with the internal ServiceManager so # that its resources (TTL timeout, etc.) can be cleaned up # properly at shutdown. registered_services[reg_info.service_id] = service # The request should receive an 'unavailable service' error # response because the service client should be unable to route # the request to an internally registered service. self.assertFalse(request_received[0]) self.assertIsInstance(response, ErrorResponse) self.assertEqual(reg_info.service_id, response.service_id) self.assertEqual( self.DXL_SERVICE_UNAVAILABLE_ERROR_CODE, BrokerServiceRegistryTest.normalized_error_code(response)) self.assertEqual(self.DXL_SERVICE_UNAVAILABLE_ERROR_MESSAGE, response.error_message) self.assertIsNone(self.query_service_registry_by_service( service_client, reg_info))
def test_execute_async_callback_timeout(self): # TODO: Set SYSPROP_ASYNC_CALLBACK_CHECK_INTERVAL = 10000 when it is available def resp_callback(): pass cb = ResponseCallback() cb.on_response = resp_callback with self.create_client() as client: client.connect() req_topic = UuidGenerator.generate_id_as_string() missing_topic = UuidGenerator.generate_id_as_string() test_service = TestService(client, 1) def empty_on_request(request): pass test_service.on_request = empty_on_request reg_info = ServiceRegistrationInfo(client, "async_callback_test_service") reg_info.add_topic(req_topic, test_service) # Register the service client.register_service_sync(reg_info, self.DEFAULT_TIMEOUT) async_req = Request(destination_topic=req_topic) client.async_request( async_req, cb) # TODO: Use the method with timeout when is will available for i in range(0, 10): req = Request(destination_topic=req_topic) client.async_request( req, cb ) # TODO: Use the updated method with timeout when it is available req_for_error = Request(destination_topic=missing_topic) client.async_request(req_for_error) async_callback_count = client._get_async_callback_count() self.assertEquals(11, async_callback_count) for i in range(0, 20): print "asyncCallbackCount = " + str( client._get_async_callback_count()) time.sleep(1) req = Request(destination_topic=req_topic) client.async_request(req, cb) self.assertEquals(1, async_callback_count)
def register_test_service(self, client, service_type=None): topic = "broker_service_registry_test_service_" + \ UuidGenerator.generate_id_as_string() reg_info = ServiceRegistrationInfo( client, service_type or "broker_service_registry_test_service_" + UuidGenerator.generate_id_as_string()) callback = RequestCallback() callback.on_request = \ lambda request: client.send_response(Response(request)) reg_info.add_topic(topic, callback) client.register_service_sync(reg_info, self.DEFAULT_TIMEOUT) return reg_info
def test_execute_message_payload(self): # Create a server that handles a request, unpacks the payload, and # asserts that the information in the payload was delivered successfully. with self.create_client(max_retries=0) as service_client: service_client.connect() topic = UuidGenerator.generate_id_as_string() reg_info = ServiceRegistrationInfo( service_client, "message_payload_runner_service") # callback definition def on_request(request): with self.request_complete_condition: try: self.request_received = request except Exception as ex: # pylint: disable=broad-except print(ex) self.request_complete_condition.notify_all() request_callback = RequestCallback() request_callback.on_request = on_request reg_info.add_topic(topic, request_callback) # Register the service service_client.register_service_sync(reg_info, self.DEFAULT_TIMEOUT) with self.create_client() as request_client: request_client.connect() packer = msgpack.Packer() # Send a request to the server with information contained # in the payload request = Request(destination_topic=topic) request.payload = packer.pack(self.TEST_STRING) request.payload += packer.pack(self.TEST_BYTE) request.payload += packer.pack(self.TEST_INT) request_client.async_request(request, request_callback) start = time.time() # Wait until the request has been processed with self.request_complete_condition: while (time.time() - start < self.MAX_WAIT) and \ not self.request_received: self.request_complete_condition.wait(self.MAX_WAIT) self.assertIsNotNone(self.request_received) unpacker = msgpack.Unpacker(file_like=BytesIO(request.payload)) self.assertEqual( next(unpacker).decode('utf8'), self.TEST_STRING) self.assertEqual(next(unpacker), self.TEST_BYTE) self.assertEqual(next(unpacker), self.TEST_INT)
def test_execute_message_payload(self): # Create a server that handles a request, unpacks the payload, and # asserts that the information in the payload was delivered successfully. with self.create_client(max_retries=0) as server: test_service = TestService(server, 1) server.connect() topic = UuidGenerator.generate_id_as_string() reg_info = ServiceRegistrationInfo( server, "message_payload_runner_service") # callback definition def on_request(request): unpacker = Unpacker( file_like=StringIO.StringIO(request.payload)) with self.request_complete_condition: try: self.assertEquals(unpacker.next(), self.TEST_STRING) self.assertEquals(unpacker.next(), self.TEST_BYTE) self.assertEquals(unpacker.next(), self.TEST_INT) self.received_request = True except Exception, e: print e.message self.request_complete_condition.notify_all() request_callback = RequestCallback() request_callback.on_request = on_request reg_info.add_topic(topic, request_callback) # Register the service server.register_service_sync(reg_info, self.DEFAULT_TIMEOUT) with self.create_client() as client: client.connect() packer = Packer() # Send a request to the server with information contained # in the payload request = Request(destination_topic=topic) request.payload = packer.pack(self.TEST_STRING) request.payload += packer.pack(self.TEST_BYTE) request.payload += packer.pack(self.TEST_INT) client.async_request(request, request_callback) with self.request_complete_condition: if not self.received_request: # Wait until the request has been processed self.request_complete_condition.wait(self.MAX_WAIT) if not self.received_request: self.fail("Request not received.")
class RegisterServiceClientTest(BaseClientTest): register_callback = EventCounterCallback() unregister_callback = EventCounterCallback() info = None request_callback = None def add_client_callbacks(self, client): self.request_callback = RequestCallback() def on_request(request): logging.info(request.destination_topic) logging.info(request.payload) response = Response(request) response.payload = bytes("Ok") try: client.send_response(response) except DxlException, ex: print "Failed to send response" + str(ex) self.request_callback.on_request = on_request client.add_event_callback(_ServiceManager.DXL_SERVICE_REGISTER_CHANNEL, self.register_callback) client.add_event_callback(_ServiceManager.DXL_SERVICE_UNREGISTER_CHANNEL, self.unregister_callback) self.info = ServiceRegistrationInfo(client, "/mcafee/service/JTI") self.info.add_topic("/mcafee/service/JTI/file/reputation/" + self.info.service_id, self.request_callback) self.info.add_topic("/mcafee/service/JTI/cert/reputation/" + self.info.service_id, self.request_callback)
def on_register_services(self): """ Invoked when services should be registered with the application """ api_methods = [] for api_name in self._api_names: api_method = self._get_api_method(api_name) if api_method: api_methods.append(api_method) else: logger.warning("MISP API name is invalid: %s", api_name) if api_methods: logger.info("Registering service: misp_service") service = ServiceRegistrationInfo(self._dxl_client, self._SERVICE_TYPE) for api_method in api_methods: api_method_name = api_method.__name__ topic = "{}{}/{}".format( self._SERVICE_TYPE, "/{}".format(self._service_unique_id) if self._service_unique_id else "", api_method_name) logger.info( "Registering request callback: %s%s_%s_%s. Topic: %s.", "misp", "_{}".format(self._service_unique_id) if self._service_unique_id else "", api_method_name, "requesthandler", topic) self.add_request_callback( service, topic, MispServiceRequestCallback(self, api_method), False) self.register_service(service)
def test_execute_sync_request(self): with self.create_client(max_retries=0) as client: # Create a test service that responds to requests on a particular topic. test_service = TestService(client, 1) client.connect() topic = "event_testing" # UuidGenerator.generate_id_as_string() reg_info = ServiceRegistrationInfo(client, "sync_request_runner_service") reg_info.add_topic(topic, test_service) # Register the service client.register_service_sync(reg_info, self.DEFAULT_TIMEOUT) executor = ThreadRunExecutor(self.REQUEST_COUNT) # Sends synchronous requests with a unique thread for each request. Ensure that the # response that is received corresponds with the request that was sent. Also, keep # track of the total number of responses received. def run(): try: request = Request(topic) response = client.sync_request(request, timeout=self.RESPONSE_WAIT) self.assertNotIsInstance(response, ErrorResponse) self.assertEqual(request.message_id, response.request_message_id) with self.response_count_condition: self.response_count += 1 if self.response_count % 100 == 0: print(self.response_count) self.response_count_condition.notify_all() except Exception as ex: print(ex) raise ex executor.execute(run) # Wait for all of the requests to complete with self.response_count_condition: while self.response_count != self.REQUEST_COUNT: current_count = self.response_count self.response_count_condition.wait(self.MAX_WAIT) if current_count == self.response_count: self.fail("Request wait timeout.") self.assertEqual(self.REQUEST_COUNT, self.response_count)
def test_response_service_not_found_no_channel(self): request_received = [False] topic = UuidGenerator.generate_id_as_string() with self.create_client() as service_client: service_client.connect() def my_request(request): request_received[0] = True service_client.send_response(Response(request)) reg_info = ServiceRegistrationInfo( service_client, "response_service_not_found_no_channel_test") callback = RequestCallback() callback.on_request = my_request reg_info.add_topic(topic, callback) service_client.register_service_sync(reg_info, self.DEFAULT_TIMEOUT) service_client.unsubscribe(topic) self.assertIsNotNone( self.query_service_registry_by_service( service_client, reg_info)) with self.create_client() as request_client: request_client.connect() request = Request(topic) response = request_client.sync_request( request, timeout=self.RESPONSE_WAIT) self.assertFalse(request_received[0]) self.assertIsInstance(response, ErrorResponse) self.assertEqual(reg_info.service_id, response.service_id) self.assertEqual( self.DXL_SERVICE_UNAVAILABLE_ERROR_CODE, BrokerServiceRegistryTest.normalized_error_code(response)) self.assertEqual(self.DXL_SERVICE_UNAVAILABLE_ERROR_MESSAGE, response.error_message) self.assertIsNone(self.query_service_registry_by_service( service_client, reg_info))
def test_multiple_services(self): with self.create_client() as service_client: service_client.connect() reg_info_topic_1 = "multiple_services_test_1_" + \ UuidGenerator.generate_id_as_string() reg_info_1 = ServiceRegistrationInfo( service_client, "multiple_services_test_1") def reg_info_request_1(request): response = Response(request) response.payload = "service1" service_client.send_response(response) reg_info_callback_1 = RequestCallback() reg_info_callback_1.on_request = reg_info_request_1 reg_info_1.add_topic(reg_info_topic_1, reg_info_callback_1) service_client.register_service_sync(reg_info_1, self.DEFAULT_TIMEOUT) reg_info_topic_2 = "multiple_services_test_2_" + \ UuidGenerator.generate_id_as_string() reg_info_2 = ServiceRegistrationInfo( service_client, "multiple_services_test_2") def reg_info_request_2(request): response = Response(request) response.payload = "service2" service_client.send_response(response) reg_info_callback_2 = RequestCallback() reg_info_callback_2.on_request = reg_info_request_2 reg_info_2.add_topic(reg_info_topic_2, reg_info_callback_2) service_client.register_service_sync(reg_info_2, self.DEFAULT_TIMEOUT) with self.create_client() as request_client: request_client.connect() response = request_client.sync_request( Request(reg_info_topic_1), self.DEFAULT_TIMEOUT) self.assertIsInstance(response, Response) self.assertEqual(response.payload.decode("utf8"), "service1") response = request_client.sync_request( Request(reg_info_topic_2), self.DEFAULT_TIMEOUT) self.assertIsInstance(response, Response) self.assertEqual(response.payload.decode("utf8"), "service2")
class RegisterServiceClientTest(BaseClientTest): SERVICE_REF_CLEANUP_DELAY = 30 info = None info_registered = False info_registrations = 0 info_unregistrations = 0 def add_client_callbacks(self, client, on_client_request_callback=None): request_callback = RequestCallback() def on_request(request): logging.info(request.destination_topic) logging.info(request.payload) if on_client_request_callback: on_client_request_callback() response = Response(request) response.payload = "Ok" try: client.send_response(response) except DxlException as ex: print("Failed to send response" + str(ex)) request_callback.on_request = on_request self.info = ServiceRegistrationInfo(client, "/mcafee/service/JTI") self.info_registered = False self.info_registrations = 0 self.info_unregistrations = 0 service_id = self.info.service_id self.info.add_topic( "/mcafee/service/JTI/file/reputation/" + service_id, request_callback) self.info.add_topic( "/mcafee/service/JTI/cert/reputation/" + service_id, request_callback) def is_event_for_service(event): return json.loads(event.payload.decode("utf8").rstrip( "\0"))["serviceGuid"] == service_id class ServiceRegisteredCallback(EventCallback): def __init__(self, test): self.test = test super(ServiceRegisteredCallback, self).__init__() def on_event(self, event): if is_event_for_service(event): self.test.info_registrations += 1 self.test.info_registered = True class ServiceUnregisteredCallback(EventCallback): def __init__(self, test): self.test = test super(ServiceUnregisteredCallback, self).__init__() def on_event(self, event): if is_event_for_service(event): self.test.info_unregistrations += 1 self.test.info_registered = False client.add_event_callback(_ServiceManager.DXL_SERVICE_REGISTER_CHANNEL, ServiceRegisteredCallback(self)) client.add_event_callback( _ServiceManager.DXL_SERVICE_UNREGISTER_CHANNEL, ServiceUnregisteredCallback(self)) def wait_info_registered_state(self, new_register_state): start = time.time() while (self.info_registered != new_register_state) and \ (time.time() - start < self.REG_DELAY): if self.info_registered != new_register_state: time.sleep(0.1) return self.info_registered def wait_info_registered(self): return self.wait_info_registered_state(True) def wait_info_not_registered(self): return not self.wait_info_registered_state(False) @staticmethod def service_ref_valid(client, service_id): services = client._service_manager.services return service_id in services and \ services[service_id].get_service() is not None def wait_for_service_reference_to_be_freed(self, client, service_id): start = time.time() while self.service_ref_valid(client, service_id) and \ (time.time() - start < self.SERVICE_REF_CLEANUP_DELAY): time.sleep(0.1) return not self.service_ref_valid(client, service_id) @attr('system') def test_register_service_before_connect(self): with self.create_client() as client: self.add_client_callbacks(client) client.register_service_async(self.info) client.connect() self.assertTrue(self.wait_info_registered()) client.unregister_service_sync(self.info, self.REG_DELAY) self.assertTrue(self.wait_info_not_registered()) self.assertEqual(1, self.info_registrations) self.assertEqual(1, self.info_unregistrations) @attr('system') def test_register_service_after_connect(self): with self.create_client() as client: self.add_client_callbacks(client) client.connect() client.register_service_sync(self.info, self.REG_DELAY) self.assertTrue(self.wait_info_registered()) client.unregister_service_sync(self.info, self.REG_DELAY) self.assertTrue(self.wait_info_not_registered()) self.assertEqual(1, self.info_registrations) self.assertEqual(1, self.info_unregistrations) @attr('system') def test_register_service_never_connect(self): with self.create_client() as client: self.add_client_callbacks(client) client.register_service_async(self.info) client.unregister_service_async(self.info) self.assertEqual(0, self.info_registrations) self.assertEqual(0, self.info_unregistrations) @attr('system') def test_register_unregister_service_before_connect(self): with self.create_client() as client: self.add_client_callbacks(client) client.register_service_async(self.info) client.unregister_service_async(self.info) client.connect() self.assertEqual(0, self.info_registrations) self.assertEqual(0, self.info_unregistrations) @attr('system') def test_register_service_and_send_request(self): with self.create_client() as client: self.add_client_callbacks(client) client.register_service_async(self.info) client.connect() self.assertTrue(self.wait_info_registered()) request = Request("/mcafee/service/JTI/file/reputation/" + self.info.service_id) request.payload = "Test" response = client.sync_request(request, self.POST_OP_DELAY) logging.info("Response payload: %s", response.payload.decode("utf8")) self.assertEqual("Ok", response.payload.decode("utf8")) @attr('system') def test_register_service_call_from_request_callback(self): # While in the request callback for a service invocation, attempt to # register a second service. Confirm that a call to the second service # is successful. This test ensures that concurrent add/remove service # calls and processing of incoming messages do not produce deadlocks. with self.create_client(self.DEFAULT_RETRIES, 2) as client: expected_second_service_response_payload = \ "Second service request okay too" second_service_callback = RequestCallback() def on_second_service_request(request): response = Response(request) response.payload = expected_second_service_response_payload try: client.send_response(response) except DxlException as ex: print("Failed to send response" + str(ex)) second_service_callback.on_request = on_second_service_request second_service_info = ServiceRegistrationInfo( client, "/mcafee/service/JTI2") second_service_info.add_topic( "/mcafee/service/JTI2/file/reputation/" + second_service_info.service_id, second_service_callback) def register_second_service(): client.register_service_sync(second_service_info, self.REG_DELAY) register_second_service_thread = threading.Thread( target=register_second_service) register_second_service_thread.daemon = True # Perform the second service registration from a separate thread # in order to ensure that locks taken by the callback and # service managers do not produce deadlocks between the # thread from which the service registration request is made and # any threads on which response messages are received from the # broker. def on_first_service_request(): register_second_service_thread.start() register_second_service_thread.join() self.add_client_callbacks(client, on_first_service_request) client.connect() client.register_service_sync(self.info, self.REG_DELAY) first_service_request = Request( "/mcafee/service/JTI/file/reputation/" + self.info.service_id) first_service_request.payload = "Test" first_service_response = client.sync_request( first_service_request, self.POST_OP_DELAY) first_service_response_payload = first_service_response. \ payload.decode("utf8") logging.info("First service response payload: %s", first_service_response_payload) self.assertEqual("Ok", first_service_response_payload) second_service_request = Request( "/mcafee/service/JTI2/file/reputation/" + second_service_info.service_id) second_service_request.payload = "Test" second_service_response = client.sync_request( second_service_request, self.POST_OP_DELAY) actual_second_service_response_payload = second_service_response. \ payload.decode("utf8") logging.info("Second service response payload: %s", actual_second_service_response_payload) self.assertEqual(expected_second_service_response_payload, actual_second_service_response_payload)
def test_wildcard_services(self): max_wait = 10 with self.create_client() as client: # The request message that the service receives service_request_message = [] # The request message corresponding to the response received by the client client_response_message_request = [] # The event that we received client_event_message = [] client_event_message_condition = Condition() # The payload that the service receives service_request_message_receive_payload = [] client.connect() info = ServiceRegistrationInfo(client, "myWildcardService") meta = {} # Transform events mapped to "test/#/" to "request/test/..." meta["EventToRequestTopic"] = "/test/#" meta["EventToRequestPrefix"] = "/request" info.metadata = meta rcb = RequestCallback() def on_request(request): print("## Request in service: " + request.destination_topic + ", " + str(request.message_id)) print("## Request in service - payload: " + request.payload) service_request_message.append(request.message_id) service_request_message_receive_payload.append(request.payload) response = Response(request) response.payload = "Request response - Event payload: " + request.payload client.send_response(response) rcb.on_request = on_request info.add_topic("/request/test/#", rcb) client.register_service_sync(info, 10) evt = Event("/test/bar") rcb = ResponseCallback() def on_response(response): # Only handle the response corresponding to the event we sent if response.request_message_id == evt.message_id: print("## received_response: " + response.request_message_id + ", " + response.__class__.__name__) print("## received_response_payload: " + response.payload) client_response_message_request[0] = response.request_message_id rcb.on_response = on_response client.add_response_callback("", rcb) ecb = EventCallback() def on_event(event): print("## received event: " + event.destination_topic + ", " + event.message_id) with client_event_message_condition: client_event_message.append(event.message_id) client_event_message_condition.notify_all() ecb.on_event = on_event client.add_event_callback("/test/#", ecb) # Send our event print("## Sending event: " + evt.destination_topic + ", " + evt.message_id) evt.payload = "Unit test payload" client.send_event(evt) start = time.time() with client_event_message_condition: while (time.time() - start < max_wait) and \ not client_event_message: client_event_message_condition.wait(max_wait) # # Make sure the service received the request properly # self.assertEqual(evt.message_id, service_request_message[0]) # # Make sure the service received the request payload from the event properly # self.assertEqual(evt.payload, service_request_message_receive_payload[0]) # Make sure the response we received was for the request message # self.assertEqual(evt.message_id, client_response_message_request[0]) # Make sure we received the correct event self.assertGreater(len(client_event_message), 0) self.assertEqual(evt.message_id, client_event_message[0])
def test_execute_async_request(self): # Create and register a response callback. Once the request has been processed # by the service, the requesting client will receive a notification via this # callback. At that point, we note that the request has been responded to (remove # it from the set of outstanding requests), and increment the count of responses # we have received via the callbacks. def my_response(response): with self.response_condition: # Increment count of responses received self.response_count += 1 # Remove from outstanding requests self.remove_outstanding_request(response.request_message_id) # Notify that a response has been received (are we done yet?) self.response_condition.notify_all() rc = ResponseCallback() rc.on_response = my_response with self.create_client(max_retries=0) as client: try: client.connect() # Create a test service that responds to requests on a particular topic. test_service = TestService(client, 1) topic = UuidGenerator.generate_id_as_string() reg_info = ServiceRegistrationInfo( client, "async_request_runner_service") reg_info.add_topic(topic, test_service) # Register the service client.register_service_sync(reg_info, self.DEFAULT_TIMEOUT) # Add a response callback (not channel-specific) client.add_response_callback("", rc) for i in range(0, self.REQ_COUNT): # Send a request without specifying a callback for the current request req = Request(topic) self.append_outstanding_request(req.message_id) client.async_request(req) # Send a request with a specific callback that is to receive the response. # The response will be received by two callbacks. req = Request(topic) self.append_outstanding_request(req.message_id) client.async_request(req, response_callback=rc) # Wait until all the responses are received via the response callbacks. # The "times 3" is due to the fact that 20000 requests were sent in total. # 20000 were handled via the global callback, an additional 10000 were handled # via the callback explicitly passed in the second set of requests. with self.response_condition: while self.response_count != self.REQ_COUNT * 3: current_count = self.response_count self.response_condition.wait(self.MAX_RESPONSE_WAIT) if current_count == self.response_count: self.fail("Response wait timeout") # Make sure there are no outstanding requests self.assertEqual(0, len(self.outstanding_requests)) print "Async request test: PASSED" except Exception, ex: print ex.message raise ex
def test_async_flood(self): channel = UuidGenerator.generate_id_as_string() with self.create_client() as client: self.m_info = ServiceRegistrationInfo(client, channel) client.connect() client.subscribe(channel) def my_request_callback(request): try: time.sleep(0.05) resp = Response(request) resp.payload = request.payload client.send_response(resp) except Exception as ex: # pylint: disable=broad-except print(ex) req_callback = RequestCallback() req_callback.on_request = my_request_callback self.m_info.add_topic(channel, req_callback) client.register_service_sync(self.m_info, 10) with self.create_client() as client2: client2.connect() def my_response_callback(response): if response.message_type == Message.MESSAGE_TYPE_ERROR: print("Received error response: " + response._error_response) with self.resp_condition: self.error_count += 1 self.resp_condition.notify_all() else: with self.resp_condition: if self.response_count % 10 == 0: print("Received request " + str(self.response_count)) self.response_count += 1 self.resp_condition.notify_all() callback = ResponseCallback() callback.on_response = my_response_callback client2.add_response_callback("", callback) for i in range(0, self.REQUEST_COUNT): if i % 100 == 0: print("Sent: " + str(i)) request = Request(channel) request.payload = str(i) client2.async_request(request) if self.error_count > 0: break # Wait for all responses, an error to occur, or we timeout start_time = time.time() with self.resp_condition: while (self.response_count != self.REQUEST_COUNT) and ( self.error_count == 0) and ( time.time() - start_time < self.WAIT_TIME): self.resp_condition.wait(5) if self.error_count != 0: raise Exception("Received an error response!") self.assertEqual(self.REQUEST_COUNT, self.response_count, "Did not receive all messages!")
def test_register_service_call_from_request_callback(self): # While in the request callback for a service invocation, attempt to # register a second service. Confirm that a call to the second service # is successful. This test ensures that concurrent add/remove service # calls and processing of incoming messages do not produce deadlocks. with self.create_client(self.DEFAULT_RETRIES, 2) as client: expected_second_service_response_payload = \ "Second service request okay too" second_service_callback = RequestCallback() def on_second_service_request(request): response = Response(request) response.payload = expected_second_service_response_payload try: client.send_response(response) except DxlException as ex: print("Failed to send response" + str(ex)) second_service_callback.on_request = on_second_service_request second_service_info = ServiceRegistrationInfo( client, "/mcafee/service/JTI2") second_service_info.add_topic( "/mcafee/service/JTI2/file/reputation/" + second_service_info.service_id, second_service_callback) def register_second_service(): client.register_service_sync(second_service_info, self.REG_DELAY) register_second_service_thread = threading.Thread( target=register_second_service) register_second_service_thread.daemon = True # Perform the second service registration from a separate thread # in order to ensure that locks taken by the callback and # service managers do not produce deadlocks between the # thread from which the service registration request is made and # any threads on which response messages are received from the # broker. def on_first_service_request(): register_second_service_thread.start() register_second_service_thread.join() self.add_client_callbacks(client, on_first_service_request) client.connect() client.register_service_sync(self.info, self.REG_DELAY) first_service_request = Request( "/mcafee/service/JTI/file/reputation/" + self.info.service_id) first_service_request.payload = "Test" first_service_response = client.sync_request( first_service_request, self.POST_OP_DELAY) first_service_response_payload = first_service_response. \ payload.decode("utf8") logging.info("First service response payload: %s", first_service_response_payload) self.assertEqual("Ok", first_service_response_payload) second_service_request = Request( "/mcafee/service/JTI2/file/reputation/" + second_service_info.service_id) second_service_request.payload = "Test" second_service_response = client.sync_request( second_service_request, self.POST_OP_DELAY) actual_second_service_response_payload = second_service_response. \ payload.decode("utf8") logging.info("Second service response payload: %s", actual_second_service_response_payload) self.assertEqual(expected_second_service_response_payload, actual_second_service_response_payload)
def execute_t(self, client_factory): with client_factory() as server_client: test_service = TestService(server_client, 1) connect_time_start = time.time() server_client.connect() topic = UuidGenerator.generate_id_as_string() reg_info = ServiceRegistrationInfo( server_client, "syncRequestThroughputRunnerService") reg_info.add_topic(topic, test_service) server_client.register_service_sync(reg_info, self.DEFAULT_TIMEOUT) # # Create a thread for each request. Wait for all of the clients to connect # to the broker before starting to calculate response related statistics. # executor = ThreadRunExecutor(self.THREAD_COUNT) def run(): try: with client_factory(max_retries=0) as client: retries = self.MAX_CONNECT_RETRIES connected = False while not connected and retries > 0: try: client.connect() connected = True except Exception: # pylint: disable=broad-except if retries > 0: retries -= 1 self.connect_retries += 1 self.assertTrue(connected, "Unable to connect after retries") # Waiting all clients have connected with self.connect_condition: self.atomic_connect_count += 1 self.connect_condition.notify_all() while self.atomic_connect_count != self.THREAD_COUNT: curr_count = self.atomic_connect_count self.connect_condition.wait( self.MAX_CONNECT_WAIT) if self.atomic_connect_count == curr_count: self.fail( "Timeout waiting for all threads to connect" ) # Once all clients have connected, reset timing information if self.requests_start_time == 0: self.requests_start_time = time.time() self.connect_time = self.requests_start_time - connect_time_start for _ in range(0, self.REQUEST_COUNT): req = Request(topic) call_start_time = time.time() response = client.sync_request( req, timeout=self.DEFAULT_TIMEOUT) response_time = time.time() - call_start_time self.assertNotIsInstance(response, ErrorResponse) with self.response_count_condition: self.response_count += 1 count = self.response_count if (self.requests_end_time == 0) and \ (count == (self.THREAD_COUNT * self.REQUEST_COUNT)): self.requests_end_time = time.time() if count % 100 is 0: print( str(count) + ", " + str(time.time() - self.requests_start_time)) # Calulate and track response times self.cummulative_response_time = self.cummulative_response_time + response_time self.response_times.append(response_time) except Exception as ex: print(ex) logging.info(ex) raise ex executor.execute(run) if self.THREAD_COUNT != self.response_count / self.REQUEST_COUNT: print("Failed! responseCount=" + str(self.response_count)) self.assertEqual(self.THREAD_COUNT, self.response_count / self.REQUEST_COUNT) server_client.unregister_service_sync(reg_info, self.DEFAULT_TIMEOUT)
class AsyncFloodTest(BaseClientTest): # The count of requests to send REQUEST_COUNT = 1000 # Amount of time to wait for the test to succeed WAIT_TIME = 90 # The service registration information m_info = None resp_condition = Condition() response_count = 0 error_count = 0 @attr('system') def test_async_flood(self): channel = UuidGenerator.generate_id_as_string() with self.create_client() as client: self.m_info = ServiceRegistrationInfo(client, channel) client.connect() client.subscribe(channel) def my_request_callback(rq): # print "request: " + str(rq.version) try: time.sleep(0.05) resp = Response(rq) resp.payload = rq.payload client.send_response(resp) except Exception, e: print e.message req_callback = RequestCallback() req_callback.on_request = my_request_callback self.m_info.add_topic(channel, req_callback) client.register_service_sync(self.m_info, 10) with self.create_client() as client2: client2.connect() def my_response_callback(response): # print "response" if response.message_type == Message.MESSAGE_TYPE_ERROR: print "Received error response: " + response._error_response with self.resp_condition: self.error_count += 1 self.resp_condition.notify_all() else: with self.resp_condition: if self.response_count % 10 == 0: print "Received request " + str(self.response_count) self.response_count += 1 self.resp_condition.notify_all() callback = ResponseCallback() callback.on_response = my_response_callback client2.add_response_callback("", callback) for i in range(0, self.REQUEST_COUNT): if i % 100 == 0: print "Sent: " + str(i) request = Request(channel) pl = str(i) request.payload = pl client2.async_request(request) if self.error_count > 0: break # Wait for all responses, an error to occur, or we timeout start_time = time.time() with self.resp_condition: while (self.response_count != self.REQUEST_COUNT) and (self.error_count == 0) and (time.time() - start_time < self.WAIT_TIME): self.resp_condition.wait(5) if self.error_count != 0: exc = Exception() exc.message = "Received an error response!" raise exc self.assertEquals(self.REQUEST_COUNT, self.response_count, "Did not receive all messages!")