class RemoteClient(object): """ A proxy client for any service or resource that forwards commands to the terrestrial endpoint for transmission. """ implements(ITerrestrialEndpoint) def __init__(self, iface=None, xs_name=None, resource_id=None, svc_name=None, process=None): """ Construct remote proxy client. Verify required parameters. Construct remote endpoint client. Set internal variables. Construct and add service method attributes. """ # Throw an exception if required interface arg not provided. if not isinstance(iface, InterfaceClass): raise ConfigNotFound('Invalid interface parameter.') # Throw an exception if the xs_name is not provided. if not isinstance(xs_name, str) or xs_name == '': raise ConfigNotFound('Invalid exchange space name parameter.') # Create the endpoint client. # Throw exception if unsuccessful. to_name = 'terrestrial_endpoint' + xs_name self._te_client = TerrestrialEndpointClient(process=process, to_name=to_name) # Must define a resource id or service name. if not resource_id and not svc_name: raise ConfigNotFound('No resource or service specified.') # Can't specify both a resource and a service. if resource_id and svc_name: raise ConfigNotFound( 'Can\'t specify both a resource and a service.') self._resource_id = resource_id self._xs_name = xs_name self._svc_name = svc_name # Grab the service method names. methods = iface.names() # Generate the service interface. # Each will forward to the terrestrial endpoint passing # the function name, args and kwargs. for m in methods: setattr(self, m, self.generate_service_method(m)) # Declare the dynamic interface. directlyProvides(self, iface) # Initialize the async results objects for blocking behavior. #self._async_result_evt = None def generate_service_method(self, name): """ A closure that returns a function for forwarding service calls. The service call name is stored as a kwarg. """ def func(*args, **kwargs): args = copy.deepcopy(args) kwargs = copy.deepcopy(kwargs) kwargs['func_name'] = name return self.forward(*args, **kwargs) return func def forward(self, *args, **kwargs): """ Forward a service method to the terrestrial endpoint through the service interface. """ func_name = kwargs.pop('func_name') try: link = kwargs.pop('link') except KeyError: link = True cid = '' try: remote_timeout = kwargs.pop('remote_timeout') if not isinstance(remote_timeout, int): remote_timeout = 0 elif remote_timeout < 0: remote_timeout = 0 elif remote_timeout == 0: pass else: cid = str(uuid.uuid4()) except KeyError: remote_timeout = 0 cmd = IonObject('RemoteCommand', resource_id=self._resource_id, svc_name=self._svc_name, command=func_name, command_id=cid, args=args, kwargs=kwargs) if remote_timeout == 0: return self._te_client.enqueue_command(cmd, link) else: if self._resource_id: origin = self._resource_id elif self._svc_name: origin = self._svc_name + self._xs_name pending_cmd = cmd async_result_evt = AsyncResult() def result_callback(evt, *args, **kwargs): """ Callback for subscriber retrive blocking results. """ #global async_result_evt if evt.type_ == 'RemoteCommandResult': cmd = evt.command if cmd.command_id == pending_cmd.command_id: async_result_evt.set(cmd) sub = EventSubscriber(event_type='RemoteCommandResult', origin=origin, callback=result_callback) sub.start() #self._pending_cmd = cmd cmd = self._te_client.enqueue_command(cmd, link) try: result = async_result_evt.get(timeout=remote_timeout) #self._pending_cmd = None sub.stop() except gevent.Timeout: #self._pending_cmd = None sub.stop() raise Timeout('Timed out waiting for remote result.') return result """ def _result_callback(self, evt, *args, **kwargs): #Callback for subscriber retrive blocking results. if evt.type_ == 'RemoteCommandResult': cmd = evt.command if self._pending_cmd: if cmd.command_id == self._pending_cmd.command_id: self._pending_cmd = None if self._async_result_evt: self._async_result_evt.set(cmd) """ def enqueue_command(self, command=None, link=False): """ Enqueue command with terrestrial endoint. """ return self._te_client(command, link) def get_queue(self): """ Get terrestrial endpoint queue for this resource/service. """ if self._resource_id: return self._te_client.get_queue(resource_id=self._resource_id) elif self._svc_name: return self._te_client.get_queue(svc_name=self._svc_name) def clear_queue(self): """ Clear terrestrial endpoint queue for this resource/service. """ if self._resource_id: return self._te_client.clear_queue(resource_id=self._resource_id) elif self._svc_name: return self._te_client.clear_queue(svc_name=self._svc_name) def pop_queue(self, command_id=''): """ Pop a command from the terrestrial endpoint queue. """ return self._te_client.pop_queue(command_id=command_id) def get_pending(self): """ Get pending commands for this resource/service. """ if self._resource_id: return self._te_client.get_pending(resource_id=self._resource_id) elif self._svc_name: return self._te_client.get_pending(svc_name=self._svc_name) def get_port(self): """ Not supported for remote proxy clients. """ raise BadRequest('get_port not available via remote client.') def set_client_port(self, port=0): """ Not supported for remote proxy clients. """ raise BadRequest('set_client_port not available via remote client.') def get_client_port(self): """ Not supported for remote proxy clients. """ raise BadRequest('get_client_port not available via remote client.')
class TestTerrestrialEndpoint(IonIntegrationTestCase): """ Test cases for 2CAA terrestrial endpoint. """ def setUp(self): """ Setup fake remote components. Start remote server. Set internal configuration and test variables. Start container. Start services. Spawn endpoint. Create and start subscribers. """ # Create fake remote client and server. # Add clean up to shut down properly. # Start remote server on a random port. self._remote_server = R3PCServer(self.consume_req, self.remote_server_close) self._remote_client = R3PCClient(self.consume_ack, self.remote_client_close) self.addCleanup(self._remote_server.stop) self.addCleanup(self._remote_client.stop) self._other_port = self._remote_server.start('*', 0) log.debug('Remote server binding to *:%i', self._other_port) # Set internal variables. self._other_host = 'localhost' self._xs_name = 'remote1' self._svc_name = 'terrestrial_endpoint' self._listen_name = self._svc_name + self._xs_name self._platform_resource_id = 'abc123' self._resource_id = 'fake_id' self._rmt_svc_name = 'fake_svc' self._no_requests = 10 self._requests_sent = {} self._results_recv = {} self._workers = [] self._done_evt = AsyncResult() self._queue_mod_evts = [] self._cmd_tx_evts = [] self._telem_evts = [] self._no_telem_evts = 0 self._no_queue_mod_evts = 0 self._no_cmd_tx_evts = 0 self._done_queue_mod_evts = AsyncResult() self._done_telem_evts = AsyncResult() self._done_cmd_tx_evts = AsyncResult() # Start container. log.debug('Staring capability container.') self._start_container() # Bring up services in a deploy file (no need to message) log.info('Staring deploy services.') self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Create a container client. log.debug('Creating container client.') self._container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) # The following spawn config creates the process with the remote # name tagged to the service name. """ listen_name = terrestrial_endpointremote1 2012-10-10 11:34:46,654 DEBUG ion.services.sa.tcaa.terrestrial_endpoint recv name: NP (ion_test_8257ab,terrestrial_endpointremote1,B: terrestrial_endpointremote1) 2012-10-10 11:34:46,654 DEBUG ion.services.sa.tcaa.terrestrial_endpoint startup listener recv name: NP (ion_test_8257ab,terrestrial_endpointremote1,B: terrestrial_endpointremote1) 2012-10-10 11:34:46,654 DEBUG ion.services.sa.tcaa.terrestrial_endpoint startup listener recv name: NP (ion_test_8257ab,Edwards-MacBook-Pro_local_2624.33,B: Edwards-MacBook-Pro_local_2624.33) """ # Create agent config. endpoint_config = { 'other_host' : self._other_host, 'other_port' : self._other_port, 'this_port' : 0, 'xs_name' : self._xs_name, 'platform_resource_id' : self._platform_resource_id, 'process' : { 'listen_name' : self._listen_name } } # Spawn the terrestrial enpoint process. log.debug('Spawning terrestrial endpoint process.') self._te_pid = self._container_client.spawn_process( name='remote_endpoint_1', module='ion.services.sa.tcaa.terrestrial_endpoint', cls='TerrestrialEndpoint', config=endpoint_config) log.debug('Endpoint pid=%s.', str(self._te_pid)) # Create an endpoint client. # The to_name may be either the process pid or # the listen_name, which for this remote bridge # is svc_name + remote_name as above. self.te_client = TerrestrialEndpointClient( process=FakeProcess(), to_name=self._listen_name) log.debug('Got te client %s.', str(self.te_client)) # Remember the terrestrial port. self._this_port = self.te_client.get_port() # Start the event publisher. self._event_publisher = EventPublisher() # Start the event subscriber. self._event_subscriber = EventSubscriber( event_type='PlatformEvent', callback=self.consume_event, origin=self._xs_name) self._event_subscriber.start() self._event_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._event_subscriber.stop) # Start the result subscriber. self._result_subscriber = EventSubscriber( event_type='RemoteCommandResult', origin=self._resource_id, callback=self.consume_event) self._result_subscriber.start() self._result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._result_subscriber.stop) def consume_event(self, evt, *args, **kwargs): """ Test callback for events. """ log.debug('Got event: %s, args: %s, kwargs: %s', str(evt), str(args), str(kwargs)) if evt.type_ == 'PublicPlatformTelemetryEvent': self._telem_evts.append(evt) if self._no_telem_evts > 0 and self._no_telem_evts == len(self._telem_evts): self._done_telem_evts.set() elif evt.type_ == 'RemoteQueueModifiedEvent': self._queue_mod_evts.append(evt) if self._no_queue_mod_evts > 0 and self._no_queue_mod_evts == len(self._queue_mod_evts): self._done_queue_mod_evts.set() elif evt.type_ == 'RemoteCommandTransmittedEvent': self._cmd_tx_evts.append(evt) if self._no_cmd_tx_evts > 0 and self._no_cmd_tx_evts == len(self._cmd_tx_evts): self._done_cmd_tx_evts.set() elif evt.type_ == 'RemoteCommandResult': cmd = evt.command self._results_recv[cmd.command_id] = cmd if len(self._results_recv) == self._no_requests: self._done_evt.set() def on_link_up(self): """ Called by a test to simulate turning the link on. """ log.debug('Remote client connecting to localhost:%i.', self._this_port) self._remote_client.start('localhost', self._this_port) # Publish a link up event to be caught by the endpoint. log.debug('Publishing telemetry event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin = self._platform_resource_id, status = TelemetryStatusType.AVAILABLE) def on_link_down(self): """ Called by a test to simulate turning the link off. """ self._remote_client.stop() # Publish a link down event to be caught by the endpoint. log.debug('Publishing telemetry event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin=self._platform_resource_id, status = TelemetryStatusType.UNAVAILABLE) def consume_req(self, request): """ Remote request callback. Fire a greenlet to do some fake work before returning via the remote client to terrestrial endpoint. """ # Spawn a greenlet to sleep briefly with each request and # then respond with a result through the remote client. log.debug('Remote endpoint got request: %s', str(request)) greenlet = gevent.spawn(self.process_remote_request, request) self._workers.append(greenlet) def consume_ack(self, request): """ Remote ack callback. """ log.debug('Remote endpoint got ack: %s', str(request)) def process_remote_request(self, request): """ Process remote request. Do random amount of fake work and enqueue result for return to terrestrial endpoint. """ worktime = random.uniform(.1,3) gevent.sleep(worktime) result = { 'command_id' : request.command_id, 'result' : 'fake_result' } log.debug('Finished processing request: %s', str(request)) self._remote_client.enqueue(result) def remote_server_close(self): """ Remote server closed callback. """ log.debug('The remote server closed.') def remote_client_close(self): """ Remoe client closed callback. """ log.debug('The remote client closed.') def make_fake_command(self, no): """ Build a fake command for use in tests. """ cmdstr = 'fake_cmd_%i' % no cmd = IonObject('RemoteCommand', resource_id=self._resource_id, command=cmdstr, args=['arg1', 23], kwargs={'kwargs1':'someval'}) return cmd def make_fake_svc_command(self, no): """ Build a fake command for use in tests. """ cmdstr = 'fake_cmd_%i' % no cmd = IonObject('RemoteCommand', svc_name=self._rmt_svc_name, command=cmdstr, args=['arg1', 23], kwargs={'kwargs1':'someval'}) return cmd def test_process_queued(self): """ test_process_queued Test forwarding of queued commands upon link up. """ self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._no_telem_evts = 2 for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) self.on_link_up() self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) pending = self.te_client.get_pending() self.assertEqual(len(pending), 0) self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) def test_process_online(self): """ test_process_online Test forwarding commands when the link is up. """ self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._no_telem_evts = 2 self.on_link_up() for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd gevent.sleep(.2) self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) pending = self.te_client.get_pending() self.assertEqual(len(pending), 0) self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) def test_remote_late(self): """ test_remote_late Test simulates behavior when the remote side is initially unavailable. """ self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._no_telem_evts = 2 self.on_link_up() gevent.sleep(2) self._remote_server.stop() self._remote_client.stop() for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) gevent.sleep(3) self._remote_client.start('localhost', self._this_port) self._remote_server.start('*', self._other_port) self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) pending = self.te_client.get_pending() self.assertEqual(len(pending), 0) self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) def test_get_clear_queue(self): """ test_get_clear_queue Test endpoint queue get and clear manipulators. """ # Set up for events expected. self._no_queue_mod_evts = self._no_requests * 2 # Queue commands. for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Queue commands. for i in range(self._no_requests): cmd = self.make_fake_svc_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Confirm queue mod events. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) # Confirm get queue with no id. queue = self.te_client.get_queue() self.assertEqual(len(queue), self._no_requests * 2) # Confirm get queue with id. queue = self.te_client.get_queue(resource_id=self._resource_id) self.assertEqual(len(queue), self._no_requests) # Confirm get queue with svc name. queue = self.te_client.get_queue(svc_name=self._rmt_svc_name) self.assertEqual(len(queue), self._no_requests) # Confirm get queue with bogus id. queue = self.te_client.get_queue(resource_id='bogus_id') self.assertEqual(len(queue), 0) # Confirm get queue with bogus id. queue = self.te_client.get_queue(svc_name='bogus_svc') self.assertEqual(len(queue), 0) # Reset queue mod expected events. self._queue_mod_evts = [] self._no_queue_mod_evts = 1 self._done_queue_mod_evts = AsyncResult() # Clear queue with no id. poped = self.te_client.clear_queue() # Confirm queue mod event and mods. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) queue = self.te_client.get_queue() self.assertEqual(len(poped), self._no_requests * 2) self.assertEqual(len(queue), 0) # Queue new commands and confirm event. self._queue_mod_evts = [] self._no_queue_mod_evts = self._no_requests * 2 self._done_queue_mod_evts = AsyncResult() self._requests_sent = {} for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd for i in range(self._no_requests): cmd = self.make_fake_svc_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) # Reset queue mod expected events. self._queue_mod_evts = [] self._no_queue_mod_evts = 1 self._done_queue_mod_evts = AsyncResult() # Clear queue with id. poped = self.te_client.clear_queue(resource_id=self._resource_id) # Confirm mods and mod events. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) queue = self.te_client.get_queue() self.assertEqual(len(poped), self._no_requests) self.assertEqual(len(queue), self._no_requests) # Reset queue mod expected events. self._queue_mod_evts = [] self._no_queue_mod_evts = 1 self._done_queue_mod_evts = AsyncResult() # Clear queue with id. poped = self.te_client.clear_queue(svc_name=self._rmt_svc_name) # Confirm mods and mod events. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) queue = self.te_client.get_queue() self.assertEqual(len(poped), self._no_requests) self.assertEqual(len(queue), 0) # Queue new commands and confirm events. self._queue_mod_evts = [] self._no_queue_mod_evts = self._no_requests self._done_queue_mod_evts = AsyncResult() self._requests_sent = {} for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) # Clear queue with bogus id. poped = self.te_client.clear_queue(resource_id='bogus id') queue = self.te_client.get_queue() self.assertEqual(len(poped), 0) self.assertEqual(len(queue), self._no_requests) # Clear queue with bogus svc name. poped = self.te_client.clear_queue(svc_name='bogus id') queue = self.te_client.get_queue() self.assertEqual(len(poped), 0) self.assertEqual(len(queue), self._no_requests) # Clear queue and confirm empty. self.te_client.clear_queue() queue = self.te_client.get_queue() self.assertEqual(len(queue), 0) # Turn on link and wait a few seconds. # Confirm no data or tx events arrive. self.on_link_up() gevent.sleep(2) self.assertEqual(len(self._cmd_tx_evts), 0) self.assertEqual(len(self._results_recv), 0) self._no_telem_evts = 2 self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) def test_pop_pending_queue(self): """ test_pop_pending_queue Test endpoint queue pop manipulators. """ # Set up for events expected. self._no_queue_mod_evts = self._no_requests # Queue commands. for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Confirm queue mod events. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) queue = self.te_client.get_queue() self.assertEqual(len(queue), self._no_requests) # Pop a few commands from the queue, confirm events. self._queue_mod_evts = [] self._no_queue_mod_evts = 3 self._done_queue_mod_evts = AsyncResult() cmd_ids = self._requests_sent.keys()[:3] poped = [] for x in cmd_ids: poped.append(self.te_client.pop_queue(x)) self._requests_sent.pop(x) # Try poping with illegal args. This should have no effect poped.append(self.te_client.pop_queue()) poped.append(self.te_client.pop_queue('bogus id')) poped = [x for x in poped if x != None] self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) queue = self.te_client.get_queue() self.assertEqual(len(poped), 3) self.assertEqual(len(queue), self._no_requests - 3) # Turn on the link and verify that only the remaining commands # get processed. self._no_telem_evts = 2 self._no_requests = self._no_requests - 3 self._no_cmd_tx_evts = self._no_requests self.on_link_up() self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) pending = self.te_client.get_pending() self.assertEqual(len(pending), 0) def test_repeated_clear_pop(self): """ test_repeated_clear_pop Test endpoint queue pop manipulators. """ # Set up for events expected. self._no_queue_mod_evts = self._no_requests for i in range(3): self._queue_mod_evts = [] self._no_queue_mod_evts = self._no_requests self._done_queue_mod_evts = AsyncResult() # Queue commands. self._requests_sent = {} for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Confirm queue mod events. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) # Confirm get queue with no id. queue = self.te_client.get_queue() self.assertEqual(len(queue), self._no_requests) # Reset queue mod expected events. self._queue_mod_evts = [] self._no_queue_mod_evts = 1 self._done_queue_mod_evts = AsyncResult() # Clear queue with no id. poped = self.te_client.clear_queue() # Confirm queue mod event and mods. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) queue = self.te_client.get_queue() self.assertEqual(len(poped), self._no_requests) self.assertEqual(len(queue), 0) self._queue_mod_evts = [] self._no_queue_mod_evts = self._no_requests self._done_queue_mod_evts = AsyncResult() # Queue commands. self._requests_sent = {} for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Confirm queue mod events. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) # Confirm get queue with no id. queue = self.te_client.get_queue() self.assertEqual(len(queue), self._no_requests) # Pop a few commands from the queue, confirm events. self._queue_mod_evts = [] self._no_queue_mod_evts = 3 self._done_queue_mod_evts = AsyncResult() cmd_ids = self._requests_sent.keys()[:3] poped = [] for x in cmd_ids: poped.append(self.te_client.pop_queue(x)) self._requests_sent.pop(x) self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) queue = self.te_client.get_queue() self.assertEqual(len(poped), 3) self.assertEqual(len(queue), self._no_requests - 3) self._no_telem_evts = 2 self._no_requests = self._no_requests - 3 self._no_cmd_tx_evts = self._no_requests self.on_link_up() self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) pending = self.te_client.get_pending() self.assertEqual(len(pending), 0) def test_get_pending(self): """ test_process_queued Test forwarding of queued commands upon link up. """ self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._no_telem_evts = 2 for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd self.on_link_up() self._no_requests = 3 self._done_evt.get(timeout=CFG.endpoint.receive.timeout) pending = self.te_client.get_pending(resource_id=self._resource_id) for x in pending: self.assertIn(x.command_id, self._requests_sent.keys()) self._no_requests = 10 self._done_evt = AsyncResult() self._done_evt.get(timeout=CFG.endpoint.receive.timeout) pending = self.te_client.get_pending() self.assertEqual(len(pending), 0) self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) #@unittest.skip('Wait for verification of resource registry use.') def test_persistent_queue(self): """ test_persistent_queue Test ability of endpoint to restore a persistent queue, survive reboot, etc. """ self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._no_telem_evts = 2 for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Confirm queue mod events. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) # Confirm get queue with no id. queue = self.te_client.get_queue() self.assertEqual(len(queue), self._no_requests) # Stop and restart the endpoint process. # Verify the queue is restored. self._container_client.terminate_process(self._te_pid) # Create agent config. endpoint_config = { 'other_host' : self._other_host, 'other_port' : self._other_port, 'this_port' : 0, 'xs_name' : self._xs_name, 'platform_resource_id' : self._platform_resource_id, 'process' : { 'listen_name' : self._listen_name } } # Spawn the terrestrial enpoint process. log.debug('Spawning terrestrial endpoint process.') self._te_pid = self._container_client.spawn_process( name='remote_endpoint_1', module='ion.services.sa.tcaa.terrestrial_endpoint', cls='TerrestrialEndpoint', config=endpoint_config) log.debug('Endpoint pid=%s.', str(self._te_pid)) # Create an endpoint client. # The to_name may be either the process pid or # the listen_name, which for this remote bridge # is svc_name + remote_name as above. self.te_client = TerrestrialEndpointClient( process=FakeProcess(), to_name=self._listen_name) log.debug('Got te client %s.', str(self.te_client)) # Remember the terrestrial port. self._this_port = self.te_client.get_port() # Confirm we restored the queue with the previous commands. queue = self.te_client.get_queue() self.assertEqual(len(queue), self._no_requests) self.on_link_up() self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) pending = self.te_client.get_pending() self.assertEqual(len(pending), 0) self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys())
class RemoteClient(object): """ A proxy client for any service or resource that forwards commands to the terrestrial endpoint for transmission. """ implements(ITerrestrialEndpoint) def __init__(self, iface=None, xs_name=None, resource_id=None, svc_name=None, process=None): """ Construct remote proxy client. Verify required parameters. Construct remote endpoint client. Set internal variables. Construct and add service method attributes. """ # Throw an exception if required interface arg not provided. if not isinstance(iface, InterfaceClass): raise ConfigNotFound('Invalid interface parameter.') # Throw an exception if the xs_name is not provided. if not isinstance(xs_name, str) or xs_name == '': raise ConfigNotFound('Invalid exchange space name parameter.') # Create the endpoint client. # Throw exception if unsuccessful. to_name = 'terrestrial_endpoint' + xs_name self._te_client = TerrestrialEndpointClient(process=process, to_name=to_name) # Must define a resource id or service name. if not resource_id and not svc_name: raise ConfigNotFound('No resource or service specified.') # Can't specify both a resource and a service. if resource_id and svc_name: raise ConfigNotFound('Can\'t specify both a resource and a service.') self._resource_id = resource_id self._xs_name = xs_name self._svc_name = svc_name # Grab the service method names. methods = iface.names() # Generate the service interface. # Each will forward to the terrestrial endpoint passing # the function name, args and kwargs. for m in methods: setattr(self, m, self.generate_service_method(m)) # Declare the dynamic interface. directlyProvides(self, iface) # Initialize the async results objects for blocking behavior. #self._async_result_evt = None def generate_service_method(self, name): """ A closure that returns a function for forwarding service calls. The service call name is stored as a kwarg. """ def func(*args, **kwargs): args = copy.deepcopy(args) kwargs = copy.deepcopy(kwargs) kwargs['func_name'] = name return self.forward(*args, **kwargs) return func def forward(self, *args, **kwargs): """ Forward a service method to the terrestrial endpoint through the service interface. """ func_name = kwargs.pop('func_name') try: link = kwargs.pop('link') except KeyError: link = True cid = '' try: remote_timeout = kwargs.pop('remote_timeout') if not isinstance(remote_timeout, int): remote_timeout = 0 elif remote_timeout < 0: remote_timeout = 0 elif remote_timeout == 0: pass else: cid = str(uuid.uuid4()) except KeyError: remote_timeout = 0 cmd = IonObject('RemoteCommand', resource_id=self._resource_id, svc_name=self._svc_name, command=func_name, command_id=cid, args= args, kwargs= kwargs) if remote_timeout == 0 : return self._te_client.enqueue_command(cmd, link) else: if self._resource_id: origin = self._resource_id elif self._svc_name: origin = self._svc_name + self._xs_name pending_cmd = cmd async_result_evt = AsyncResult() def result_callback(evt, *args, **kwargs): """ Callback for subscriber retrive blocking results. """ #global async_result_evt if evt.type_ == 'RemoteCommandResult': cmd = evt.command if cmd.command_id == pending_cmd.command_id: async_result_evt.set(cmd) sub = EventSubscriber( event_type='RemoteCommandResult', origin=origin, callback=result_callback) sub.start() #self._pending_cmd = cmd cmd = self._te_client.enqueue_command(cmd, link) try: result = async_result_evt.get(timeout=remote_timeout) #self._pending_cmd = None sub.stop() except gevent.Timeout: #self._pending_cmd = None sub.stop() raise Timeout('Timed out waiting for remote result.') return result """ def _result_callback(self, evt, *args, **kwargs): #Callback for subscriber retrive blocking results. if evt.type_ == 'RemoteCommandResult': cmd = evt.command if self._pending_cmd: if cmd.command_id == self._pending_cmd.command_id: self._pending_cmd = None if self._async_result_evt: self._async_result_evt.set(cmd) """ def enqueue_command(self, command=None, link=False): """ Enqueue command with terrestrial endoint. """ return self._te_client(command, link) def get_queue(self): """ Get terrestrial endpoint queue for this resource/service. """ if self._resource_id: return self._te_client.get_queue(resource_id=self._resource_id) elif self._svc_name: return self._te_client.get_queue(svc_name=self._svc_name) def clear_queue(self): """ Clear terrestrial endpoint queue for this resource/service. """ if self._resource_id: return self._te_client.clear_queue(resource_id=self._resource_id) elif self._svc_name: return self._te_client.clear_queue(svc_name=self._svc_name) def pop_queue(self, command_id=''): """ Pop a command from the terrestrial endpoint queue. """ return self._te_client.pop_queue(command_id=command_id) def get_pending(self): """ Get pending commands for this resource/service. """ if self._resource_id: return self._te_client.get_pending(resource_id=self._resource_id) elif self._svc_name: return self._te_client.get_pending(svc_name=self._svc_name) def get_port(self): """ Not supported for remote proxy clients. """ raise BadRequest('get_port not available via remote client.') def set_client_port(self, port=0): """ Not supported for remote proxy clients. """ raise BadRequest('set_client_port not available via remote client.') def get_client_port(self): """ Not supported for remote proxy clients. """ raise BadRequest('get_client_port not available via remote client.')
class TestTerrestrialEndpoint(IonIntegrationTestCase): """ Test cases for 2CAA terrestrial endpoint. """ def setUp(self): """ Setup fake remote components. Start remote server. Set internal configuration and test variables. Start container. Start services. Spawn endpoint. Create and start subscribers. """ # Create fake remote client and server. # Add clean up to shut down properly. # Start remote server on a random port. self._remote_server = R3PCServer(self.consume_req, self.remote_server_close) self._remote_client = R3PCClient(self.consume_ack, self.remote_client_close) self.addCleanup(self._remote_server.stop) self.addCleanup(self._remote_client.stop) self._other_port = self._remote_server.start('*', 0) log.debug('Remote server binding to *:%i', self._other_port) # Set internal variables. self._other_host = 'localhost' self._xs_name = 'remote1' self._svc_name = 'terrestrial_endpoint' self._listen_name = self._svc_name + self._xs_name self._platform_resource_id = 'abc123' self._resource_id = 'fake_id' self._rmt_svc_name = 'fake_svc' self._no_requests = 10 self._requests_sent = {} self._results_recv = {} self._workers = [] self._done_evt = AsyncResult() self._queue_mod_evts = [] self._cmd_tx_evts = [] self._telem_evts = [] self._no_telem_evts = 0 self._no_queue_mod_evts = 0 self._no_cmd_tx_evts = 0 self._done_queue_mod_evts = AsyncResult() self._done_telem_evts = AsyncResult() self._done_cmd_tx_evts = AsyncResult() # Start container. log.debug('Staring capability container.') self._start_container() # Bring up services in a deploy file (no need to message) log.info('Staring deploy services.') self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Create a container client. log.debug('Creating container client.') self._container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) # The following spawn config creates the process with the remote # name tagged to the service name. """ listen_name = terrestrial_endpointremote1 2012-10-10 11:34:46,654 DEBUG ion.services.sa.tcaa.terrestrial_endpoint recv name: NP (ion_test_8257ab,terrestrial_endpointremote1,B: terrestrial_endpointremote1) 2012-10-10 11:34:46,654 DEBUG ion.services.sa.tcaa.terrestrial_endpoint startup listener recv name: NP (ion_test_8257ab,terrestrial_endpointremote1,B: terrestrial_endpointremote1) 2012-10-10 11:34:46,654 DEBUG ion.services.sa.tcaa.terrestrial_endpoint startup listener recv name: NP (ion_test_8257ab,Edwards-MacBook-Pro_local_2624.33,B: Edwards-MacBook-Pro_local_2624.33) """ # Create agent config. endpoint_config = { 'other_host': self._other_host, 'other_port': self._other_port, 'this_port': 0, 'xs_name': self._xs_name, 'platform_resource_id': self._platform_resource_id, 'process': { 'listen_name': self._listen_name } } # Spawn the terrestrial enpoint process. log.debug('Spawning terrestrial endpoint process.') self._te_pid = self._container_client.spawn_process( name='remote_endpoint_1', module='ion.services.sa.tcaa.terrestrial_endpoint', cls='TerrestrialEndpoint', config=endpoint_config) log.debug('Endpoint pid=%s.', str(self._te_pid)) # Create an endpoint client. # The to_name may be either the process pid or # the listen_name, which for this remote bridge # is svc_name + remote_name as above. self.te_client = TerrestrialEndpointClient(process=FakeProcess(), to_name=self._listen_name) log.debug('Got te client %s.', str(self.te_client)) # Remember the terrestrial port. self._this_port = self.te_client.get_port() # Start the event publisher. self._event_publisher = EventPublisher() # Start the event subscriber. self._event_subscriber = EventSubscriber(event_type='PlatformEvent', callback=self.consume_event, origin=self._xs_name) self._event_subscriber.start() self._event_subscriber._ready_event.wait( timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._event_subscriber.stop) # Start the result subscriber. self._result_subscriber = EventSubscriber( event_type='RemoteCommandResult', origin=self._resource_id, callback=self.consume_event) self._result_subscriber.start() self._result_subscriber._ready_event.wait( timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._result_subscriber.stop) def consume_event(self, evt, *args, **kwargs): """ Test callback for events. """ log.debug('Got event: %s, args: %s, kwargs: %s', str(evt), str(args), str(kwargs)) if evt.type_ == 'PublicPlatformTelemetryEvent': self._telem_evts.append(evt) if self._no_telem_evts > 0 and self._no_telem_evts == len( self._telem_evts): self._done_telem_evts.set() elif evt.type_ == 'RemoteQueueModifiedEvent': self._queue_mod_evts.append(evt) if self._no_queue_mod_evts > 0 and self._no_queue_mod_evts == len( self._queue_mod_evts): self._done_queue_mod_evts.set() elif evt.type_ == 'RemoteCommandTransmittedEvent': self._cmd_tx_evts.append(evt) if self._no_cmd_tx_evts > 0 and self._no_cmd_tx_evts == len( self._cmd_tx_evts): self._done_cmd_tx_evts.set() elif evt.type_ == 'RemoteCommandResult': cmd = evt.command self._results_recv[cmd.command_id] = cmd if len(self._results_recv) == self._no_requests: self._done_evt.set() def on_link_up(self): """ Called by a test to simulate turning the link on. """ log.debug('Remote client connecting to localhost:%i.', self._this_port) self._remote_client.start('localhost', self._this_port) # Publish a link up event to be caught by the endpoint. log.debug('Publishing telemetry event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin=self._platform_resource_id, status=TelemetryStatusType.AVAILABLE) def on_link_down(self): """ Called by a test to simulate turning the link off. """ self._remote_client.stop() # Publish a link down event to be caught by the endpoint. log.debug('Publishing telemetry event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin=self._platform_resource_id, status=TelemetryStatusType.UNAVAILABLE) def consume_req(self, request): """ Remote request callback. Fire a greenlet to do some fake work before returning via the remote client to terrestrial endpoint. """ # Spawn a greenlet to sleep briefly with each request and # then respond with a result through the remote client. log.debug('Remote endpoint got request: %s', str(request)) greenlet = gevent.spawn(self.process_remote_request, request) self._workers.append(greenlet) def consume_ack(self, request): """ Remote ack callback. """ log.debug('Remote endpoint got ack: %s', str(request)) def process_remote_request(self, request): """ Process remote request. Do random amount of fake work and enqueue result for return to terrestrial endpoint. """ worktime = random.uniform(.1, 3) gevent.sleep(worktime) result = {'command_id': request.command_id, 'result': 'fake_result'} log.debug('Finished processing request: %s', str(request)) self._remote_client.enqueue(result) def remote_server_close(self): """ Remote server closed callback. """ log.debug('The remote server closed.') def remote_client_close(self): """ Remoe client closed callback. """ log.debug('The remote client closed.') def make_fake_command(self, no): """ Build a fake command for use in tests. """ cmdstr = 'fake_cmd_%i' % no cmd = IonObject('RemoteCommand', resource_id=self._resource_id, command=cmdstr, args=['arg1', 23], kwargs={'kwargs1': 'someval'}) return cmd def make_fake_svc_command(self, no): """ Build a fake command for use in tests. """ cmdstr = 'fake_cmd_%i' % no cmd = IonObject('RemoteCommand', svc_name=self._rmt_svc_name, command=cmdstr, args=['arg1', 23], kwargs={'kwargs1': 'someval'}) return cmd def test_process_queued(self): """ test_process_queued Test forwarding of queued commands upon link up. """ self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._no_telem_evts = 2 for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) self.on_link_up() self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) pending = self.te_client.get_pending() self.assertEqual(len(pending), 0) self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) def test_process_online(self): """ test_process_online Test forwarding commands when the link is up. """ self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._no_telem_evts = 2 self.on_link_up() for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd gevent.sleep(.2) self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) pending = self.te_client.get_pending() self.assertEqual(len(pending), 0) self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) def test_remote_late(self): """ test_remote_late Test simulates behavior when the remote side is initially unavailable. """ self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._no_telem_evts = 2 self.on_link_up() gevent.sleep(2) self._remote_server.stop() self._remote_client.stop() for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) gevent.sleep(3) self._remote_client.start('localhost', self._this_port) self._remote_server.start('*', self._other_port) self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) pending = self.te_client.get_pending() self.assertEqual(len(pending), 0) self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) def test_get_clear_queue(self): """ test_get_clear_queue Test endpoint queue get and clear manipulators. """ # Set up for events expected. self._no_queue_mod_evts = self._no_requests * 2 # Queue commands. for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Queue commands. for i in range(self._no_requests): cmd = self.make_fake_svc_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Confirm queue mod events. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) # Confirm get queue with no id. queue = self.te_client.get_queue() self.assertEqual(len(queue), self._no_requests * 2) # Confirm get queue with id. queue = self.te_client.get_queue(resource_id=self._resource_id) self.assertEqual(len(queue), self._no_requests) # Confirm get queue with svc name. queue = self.te_client.get_queue(svc_name=self._rmt_svc_name) self.assertEqual(len(queue), self._no_requests) # Confirm get queue with bogus id. queue = self.te_client.get_queue(resource_id='bogus_id') self.assertEqual(len(queue), 0) # Confirm get queue with bogus id. queue = self.te_client.get_queue(svc_name='bogus_svc') self.assertEqual(len(queue), 0) # Reset queue mod expected events. self._queue_mod_evts = [] self._no_queue_mod_evts = 1 self._done_queue_mod_evts = AsyncResult() # Clear queue with no id. poped = self.te_client.clear_queue() # Confirm queue mod event and mods. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) queue = self.te_client.get_queue() self.assertEqual(len(poped), self._no_requests * 2) self.assertEqual(len(queue), 0) # Queue new commands and confirm event. self._queue_mod_evts = [] self._no_queue_mod_evts = self._no_requests * 2 self._done_queue_mod_evts = AsyncResult() self._requests_sent = {} for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd for i in range(self._no_requests): cmd = self.make_fake_svc_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) # Reset queue mod expected events. self._queue_mod_evts = [] self._no_queue_mod_evts = 1 self._done_queue_mod_evts = AsyncResult() # Clear queue with id. poped = self.te_client.clear_queue(resource_id=self._resource_id) # Confirm mods and mod events. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) queue = self.te_client.get_queue() self.assertEqual(len(poped), self._no_requests) self.assertEqual(len(queue), self._no_requests) # Reset queue mod expected events. self._queue_mod_evts = [] self._no_queue_mod_evts = 1 self._done_queue_mod_evts = AsyncResult() # Clear queue with id. poped = self.te_client.clear_queue(svc_name=self._rmt_svc_name) # Confirm mods and mod events. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) queue = self.te_client.get_queue() self.assertEqual(len(poped), self._no_requests) self.assertEqual(len(queue), 0) # Queue new commands and confirm events. self._queue_mod_evts = [] self._no_queue_mod_evts = self._no_requests self._done_queue_mod_evts = AsyncResult() self._requests_sent = {} for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) # Clear queue with bogus id. poped = self.te_client.clear_queue(resource_id='bogus id') queue = self.te_client.get_queue() self.assertEqual(len(poped), 0) self.assertEqual(len(queue), self._no_requests) # Clear queue with bogus svc name. poped = self.te_client.clear_queue(svc_name='bogus id') queue = self.te_client.get_queue() self.assertEqual(len(poped), 0) self.assertEqual(len(queue), self._no_requests) # Clear queue and confirm empty. self.te_client.clear_queue() queue = self.te_client.get_queue() self.assertEqual(len(queue), 0) # Turn on link and wait a few seconds. # Confirm no data or tx events arrive. self.on_link_up() gevent.sleep(2) self.assertEqual(len(self._cmd_tx_evts), 0) self.assertEqual(len(self._results_recv), 0) self._no_telem_evts = 2 self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) def test_pop_pending_queue(self): """ test_pop_pending_queue Test endpoint queue pop manipulators. """ # Set up for events expected. self._no_queue_mod_evts = self._no_requests # Queue commands. for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Confirm queue mod events. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) queue = self.te_client.get_queue() self.assertEqual(len(queue), self._no_requests) # Pop a few commands from the queue, confirm events. self._queue_mod_evts = [] self._no_queue_mod_evts = 3 self._done_queue_mod_evts = AsyncResult() cmd_ids = self._requests_sent.keys()[:3] poped = [] for x in cmd_ids: poped.append(self.te_client.pop_queue(x)) self._requests_sent.pop(x) # Try poping with illegal args. This should have no effect poped.append(self.te_client.pop_queue()) poped.append(self.te_client.pop_queue('bogus id')) poped = [x for x in poped if x != None] self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) queue = self.te_client.get_queue() self.assertEqual(len(poped), 3) self.assertEqual(len(queue), self._no_requests - 3) # Turn on the link and verify that only the remaining commands # get processed. self._no_telem_evts = 2 self._no_requests = self._no_requests - 3 self._no_cmd_tx_evts = self._no_requests self.on_link_up() self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) pending = self.te_client.get_pending() self.assertEqual(len(pending), 0) def test_repeated_clear_pop(self): """ test_repeated_clear_pop Test endpoint queue pop manipulators. """ # Set up for events expected. self._no_queue_mod_evts = self._no_requests for i in range(3): self._queue_mod_evts = [] self._no_queue_mod_evts = self._no_requests self._done_queue_mod_evts = AsyncResult() # Queue commands. self._requests_sent = {} for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Confirm queue mod events. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) # Confirm get queue with no id. queue = self.te_client.get_queue() self.assertEqual(len(queue), self._no_requests) # Reset queue mod expected events. self._queue_mod_evts = [] self._no_queue_mod_evts = 1 self._done_queue_mod_evts = AsyncResult() # Clear queue with no id. poped = self.te_client.clear_queue() # Confirm queue mod event and mods. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) queue = self.te_client.get_queue() self.assertEqual(len(poped), self._no_requests) self.assertEqual(len(queue), 0) self._queue_mod_evts = [] self._no_queue_mod_evts = self._no_requests self._done_queue_mod_evts = AsyncResult() # Queue commands. self._requests_sent = {} for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Confirm queue mod events. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) # Confirm get queue with no id. queue = self.te_client.get_queue() self.assertEqual(len(queue), self._no_requests) # Pop a few commands from the queue, confirm events. self._queue_mod_evts = [] self._no_queue_mod_evts = 3 self._done_queue_mod_evts = AsyncResult() cmd_ids = self._requests_sent.keys()[:3] poped = [] for x in cmd_ids: poped.append(self.te_client.pop_queue(x)) self._requests_sent.pop(x) self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) queue = self.te_client.get_queue() self.assertEqual(len(poped), 3) self.assertEqual(len(queue), self._no_requests - 3) self._no_telem_evts = 2 self._no_requests = self._no_requests - 3 self._no_cmd_tx_evts = self._no_requests self.on_link_up() self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) pending = self.te_client.get_pending() self.assertEqual(len(pending), 0) def test_get_pending(self): """ test_process_queued Test forwarding of queued commands upon link up. """ self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._no_telem_evts = 2 for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd self.on_link_up() self._no_requests = 3 self._done_evt.get(timeout=CFG.endpoint.receive.timeout) pending = self.te_client.get_pending(resource_id=self._resource_id) for x in pending: self.assertIn(x.command_id, self._requests_sent.keys()) self._no_requests = 10 self._done_evt = AsyncResult() self._done_evt.get(timeout=CFG.endpoint.receive.timeout) pending = self.te_client.get_pending() self.assertEqual(len(pending), 0) self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) #@unittest.skip('Wait for verification of resource registry use.') def test_persistent_queue(self): """ test_persistent_queue Test ability of endpoint to restore a persistent queue, survive reboot, etc. """ self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._no_telem_evts = 2 for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Confirm queue mod events. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) # Confirm get queue with no id. queue = self.te_client.get_queue() self.assertEqual(len(queue), self._no_requests) # Stop and restart the endpoint process. # Verify the queue is restored. self._container_client.terminate_process(self._te_pid) # Create agent config. endpoint_config = { 'other_host': self._other_host, 'other_port': self._other_port, 'this_port': 0, 'xs_name': self._xs_name, 'platform_resource_id': self._platform_resource_id, 'process': { 'listen_name': self._listen_name } } # Spawn the terrestrial enpoint process. log.debug('Spawning terrestrial endpoint process.') self._te_pid = self._container_client.spawn_process( name='remote_endpoint_1', module='ion.services.sa.tcaa.terrestrial_endpoint', cls='TerrestrialEndpoint', config=endpoint_config) log.debug('Endpoint pid=%s.', str(self._te_pid)) # Create an endpoint client. # The to_name may be either the process pid or # the listen_name, which for this remote bridge # is svc_name + remote_name as above. self.te_client = TerrestrialEndpointClient(process=FakeProcess(), to_name=self._listen_name) log.debug('Got te client %s.', str(self.te_client)) # Remember the terrestrial port. self._this_port = self.te_client.get_port() # Confirm we restored the queue with the previous commands. queue = self.te_client.get_queue() self.assertEqual(len(queue), self._no_requests) self.on_link_up() self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) pending = self.te_client.get_pending() self.assertEqual(len(pending), 0) self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys())