def _start_terrestrial(self): """ Start up the terrestrial endpoint. """ # Create terrestrial config. terrestrial_endpoint_config = { 'other_host': 'localhost', 'other_port': self._remote_port, 'this_port': self._terrestrial_port, 'platform_resource_id': self._terrestrial_platform_id, 'xs_name': self._xs_name, 'process': { 'listen_name': self._terrestrial_listen_name } } # Spawn the terrestrial enpoint process. log.debug('Spawning terrestrial endpoint process.') self._terrestrial_pid = self._container_client.spawn_process( name=self._terrestrial_listen_name, module='ion.services.sa.tcaa.terrestrial_endpoint', cls='TerrestrialEndpoint', config=terrestrial_endpoint_config) log.debug('Terrestrial endpoint pid=%s.', str(self._terrestrial_pid)) # Create a terrestrial client. self.te_client = TerrestrialEndpointClient( process=FakeProcess(), to_name=self._terrestrial_listen_name) log.debug('Got te client %s.', str(self.te_client)) self._terrestrial_port = self.te_client.get_port() log.debug('Terrestrial port is: %i', self._terrestrial_port)
def __init__(self, iface=None, xs_name=None, resource_id=None, svc_name=None, process=None): """ Construct remote proxy client. Verify required parameters. Construct remote endpoint client. Set internal variables. Construct and add service method attributes. """ # Throw an exception if required interface arg not provided. if not isinstance(iface, InterfaceClass): raise ConfigNotFound('Invalid interface parameter.') # Throw an exception if the xs_name is not provided. if not isinstance(xs_name, str) or xs_name == '': raise ConfigNotFound('Invalid exchange space name parameter.') # Create the endpoint client. # Throw exception if unsuccessful. to_name = 'terrestrial_endpoint' + xs_name self._te_client = TerrestrialEndpointClient(process=process, to_name=to_name) # Must define a resource id or service name. if not resource_id and not svc_name: raise ConfigNotFound('No resource or service specified.') # Can't specify both a resource and a service. if resource_id and svc_name: raise ConfigNotFound( 'Can\'t specify both a resource and a service.') self._resource_id = resource_id self._xs_name = xs_name self._svc_name = svc_name # Grab the service method names. methods = iface.names() # Generate the service interface. # Each will forward to the terrestrial endpoint passing # the function name, args and kwargs. for m in methods: setattr(self, m, self.generate_service_method(m)) # Declare the dynamic interface. directlyProvides(self, iface)
def launch_terrestrial(): """ """ listen_name = 'terrestrial_endpoint' + tcaa_args['xs_name'] config = { 'other_host': tcaa_args['remote_host'], 'other_port': tcaa_args['remote_port'], 'this_port': tcaa_args['terrestrial_port'], 'platform_resource_id': tcaa_args['terrestrial_platform_id'], 'xs_name': tcaa_args['xs_name'], 'process': { 'listen_name': listen_name } } spargs = { 'name': listen_name, 'module': tcaa_args['terrestrial_module'], 'cls': tcaa_args['terrestrial_class'], 'config': config } pid = cc.spawn_process(**spargs) print 'terrestrial pid = %s' % pid tc = TerrestrialEndpointClient(process=FakeProcess(), to_name=listen_name) return tc
def _start_terrestrial(self): """ """ # Create terrestrial config. terrestrial_endpoint_config = { "other_host": "localhost", "other_port": self._remote_port, "this_port": self._terrestrial_port, "platform_resource_id": self._terrestrial_platform_id, "xs_name": self._xs_name, "process": {"listen_name": self._terrestrial_listen_name}, } # Spawn the terrestrial enpoint process. log.debug("Spawning terrestrial endpoint process.") self._terrestrial_pid = self._container_client.spawn_process( name=self._terrestrial_listen_name, module="ion.services.sa.tcaa.terrestrial_endpoint", cls="TerrestrialEndpoint", config=terrestrial_endpoint_config, ) log.debug("Terrestrial endpoint pid=%s.", str(self._terrestrial_pid)) # Create a terrestrial client. self.te_client = TerrestrialEndpointClient(process=FakeProcess(), to_name=self._terrestrial_listen_name) log.debug("Got te client %s.", str(self.te_client)) self._terrestrial_port = self.te_client.get_port() log.debug("Terrestrial port is: %i", self._terrestrial_port)
def _start_terrestrial(self): """ """ # Create terrestrial config. terrestrial_endpoint_config = { 'other_host' : 'localhost', 'other_port' : self._remote_port, 'this_port' : self._terrestrial_port, 'platform_resource_id' : self._terrestrial_platform_id, 'xs_name' : self._xs_name, 'process' : { 'listen_name' : self._terrestrial_listen_name } } # Spawn the terrestrial enpoint process. log.debug('Spawning terrestrial endpoint process.') self._terrestrial_pid = self._container_client.spawn_process( name=self._terrestrial_listen_name, module='ion.services.sa.tcaa.terrestrial_endpoint', cls='TerrestrialEndpoint', config=terrestrial_endpoint_config) log.debug('Terrestrial endpoint pid=%s.', str(self._terrestrial_pid)) # Create a terrestrial client. self.te_client = TerrestrialEndpointClient( process=FakeProcess(), to_name=self._terrestrial_listen_name) log.debug('Got te client %s.', str(self.te_client)) self._terrestrial_port = self.te_client.get_port() log.debug('Terrestrial port is: %i', self._terrestrial_port)
def __init__(self, iface=None, xs_name=None, resource_id=None, svc_name=None, process=None): """ Construct remote proxy client. Verify required parameters. Construct remote endpoint client. Set internal variables. Construct and add service method attributes. """ # Throw an exception if required interface arg not provided. if not isinstance(iface, InterfaceClass): raise ConfigNotFound('Invalid interface parameter.') # Throw an exception if the xs_name is not provided. if not isinstance(xs_name, str) or xs_name == '': raise ConfigNotFound('Invalid exchange space name parameter.') # Create the endpoint client. # Throw exception if unsuccessful. to_name = 'terrestrial_endpoint' + xs_name self._te_client = TerrestrialEndpointClient(process=process, to_name=to_name) # Must define a resource id or service name. if not resource_id and not svc_name: raise ConfigNotFound('No resource or service specified.') # Can't specify both a resource and a service. if resource_id and svc_name: raise ConfigNotFound('Can\'t specify both a resource and a service.') self._resource_id = resource_id self._xs_name = xs_name self._svc_name = svc_name # Grab the service method names. methods = iface.names() # Generate the service interface. # Each will forward to the terrestrial endpoint passing # the function name, args and kwargs. for m in methods: setattr(self, m, self.generate_service_method(m)) # Declare the dynamic interface. directlyProvides(self, iface) # Initialize the async results objects for blocking behavior. self._async_result_evt = None
def start_commander(delay, size): """ """ global commander global requests_sent global results_pending listen_name = 'terrestrial_endpoint' + tcaa_args['xs_name'] tc = TerrestrialEndpointClient(process=FakeProcess(), to_name=listen_name) def _loop(): """ """ while True: gevent.sleep(delay) cmd = make_fake_command(size) requests_sent[cmd.command_id] = cmd results_pending[cmd.command_id] = cmd tc.enqueue_command(cmd) #print '## sending command %s' % str(cmd) commander = gevent.spawn(_loop)
class RemoteClient(object): """ A proxy client for any service or resource that forwards commands to the terrestrial endpoint for transmission. """ implements(ITerrestrialEndpoint) def __init__(self, iface=None, xs_name=None, resource_id=None, svc_name=None, process=None): """ Construct remote proxy client. Verify required parameters. Construct remote endpoint client. Set internal variables. Construct and add service method attributes. """ # Throw an exception if required interface arg not provided. if not isinstance(iface, InterfaceClass): raise ConfigNotFound('Invalid interface parameter.') # Throw an exception if the xs_name is not provided. if not isinstance(xs_name, str) or xs_name == '': raise ConfigNotFound('Invalid exchange space name parameter.') # Create the endpoint client. # Throw exception if unsuccessful. to_name = 'terrestrial_endpoint' + xs_name self._te_client = TerrestrialEndpointClient(process=process, to_name=to_name) # Must define a resource id or service name. if not resource_id and not svc_name: raise ConfigNotFound('No resource or service specified.') # Can't specify both a resource and a service. if resource_id and svc_name: raise ConfigNotFound('Can\'t specify both a resource and a service.') self._resource_id = resource_id self._xs_name = xs_name self._svc_name = svc_name # Grab the service method names. methods = iface.names() # Generate the service interface. # Each will forward to the terrestrial endpoint passing # the function name, args and kwargs. for m in methods: setattr(self, m, self.generate_service_method(m)) # Declare the dynamic interface. directlyProvides(self, iface) # Initialize the async results objects for blocking behavior. #self._async_result_evt = None def generate_service_method(self, name): """ A closure that returns a function for forwarding service calls. The service call name is stored as a kwarg. """ def func(*args, **kwargs): args = copy.deepcopy(args) kwargs = copy.deepcopy(kwargs) kwargs['func_name'] = name return self.forward(*args, **kwargs) return func def forward(self, *args, **kwargs): """ Forward a service method to the terrestrial endpoint through the service interface. """ func_name = kwargs.pop('func_name') try: link = kwargs.pop('link') except KeyError: link = True cid = '' try: remote_timeout = kwargs.pop('remote_timeout') if not isinstance(remote_timeout, int): remote_timeout = 0 elif remote_timeout < 0: remote_timeout = 0 elif remote_timeout == 0: pass else: cid = str(uuid.uuid4()) except KeyError: remote_timeout = 0 cmd = IonObject('RemoteCommand', resource_id=self._resource_id, svc_name=self._svc_name, command=func_name, command_id=cid, args= args, kwargs= kwargs) if remote_timeout == 0 : return self._te_client.enqueue_command(cmd, link) else: if self._resource_id: origin = self._resource_id elif self._svc_name: origin = self._svc_name + self._xs_name pending_cmd = cmd async_result_evt = AsyncResult() def result_callback(evt, *args, **kwargs): """ Callback for subscriber retrive blocking results. """ #global async_result_evt if evt.type_ == 'RemoteCommandResult': cmd = evt.command if cmd.command_id == pending_cmd.command_id: async_result_evt.set(cmd) sub = EventSubscriber( event_type='RemoteCommandResult', origin=origin, callback=result_callback) sub.start() #self._pending_cmd = cmd cmd = self._te_client.enqueue_command(cmd, link) try: result = async_result_evt.get(timeout=remote_timeout) #self._pending_cmd = None sub.stop() except gevent.Timeout: #self._pending_cmd = None sub.stop() raise Timeout('Timed out waiting for remote result.') return result """ def _result_callback(self, evt, *args, **kwargs): #Callback for subscriber retrive blocking results. if evt.type_ == 'RemoteCommandResult': cmd = evt.command if self._pending_cmd: if cmd.command_id == self._pending_cmd.command_id: self._pending_cmd = None if self._async_result_evt: self._async_result_evt.set(cmd) """ def enqueue_command(self, command=None, link=False): """ Enqueue command with terrestrial endoint. """ return self._te_client(command, link) def get_queue(self): """ Get terrestrial endpoint queue for this resource/service. """ if self._resource_id: return self._te_client.get_queue(resource_id=self._resource_id) elif self._svc_name: return self._te_client.get_queue(svc_name=self._svc_name) def clear_queue(self): """ Clear terrestrial endpoint queue for this resource/service. """ if self._resource_id: return self._te_client.clear_queue(resource_id=self._resource_id) elif self._svc_name: return self._te_client.clear_queue(svc_name=self._svc_name) def pop_queue(self, command_id=''): """ Pop a command from the terrestrial endpoint queue. """ return self._te_client.pop_queue(command_id=command_id) def get_pending(self): """ Get pending commands for this resource/service. """ if self._resource_id: return self._te_client.get_pending(resource_id=self._resource_id) elif self._svc_name: return self._te_client.get_pending(svc_name=self._svc_name) def get_port(self): """ Not supported for remote proxy clients. """ raise BadRequest('get_port not available via remote client.') def set_client_port(self, port=0): """ Not supported for remote proxy clients. """ raise BadRequest('set_client_port not available via remote client.') def get_client_port(self): """ Not supported for remote proxy clients. """ raise BadRequest('get_client_port not available via remote client.')
class TestRemoteClient(IonIntegrationTestCase): """ Test cases for 2CAA remote clients. """ def setUp(self): """ Setup the test parameters. Start the container and retrieve client. Start the endpoints and resource agent. Start publisher and subscribers. """ ################################################################### # Internal parameters and container. ################################################################### # Internal parameters. self._terrestrial_platform_id = 'terrestrial_id' self._remote_platform_id = 'remote_id' self._resource_id = 'fake_id' self._xs_name = 'remote1' self._terrestrial_svc_name = 'terrestrial_endpoint' self._terrestrial_listen_name = self._terrestrial_svc_name + self._xs_name self._remote_svc_name = 'remote_endpoint' self._remote_listen_name = self._remote_svc_name + self._xs_name self._remote_port = 0 self._terrestrial_port = 0 self._te_client = None self._re_client = None self._remote_pid = None self._terrestrial_pid = None # Async test results. self._no_requests = 10 self._no_telem_evts = 2 self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._telem_evts = [] self._queue_mod_evts = [] self._cmd_tx_evts = [] self._results_recv = {} self._requests_sent = {} self._done_telem_evt = AsyncResult() self._done_queue_mod_evt = AsyncResult() self._done_cmd_tx_evt = AsyncResult() self._done_cmd_evt = AsyncResult() # Start container. log.debug('Staring capability container.') self._start_container() # Bring up services in a deploy file (no need to message). log.info('Staring deploy services.') self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Create a container client. log.debug('Creating container client.') self._container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) ################################################################### # Start endpoints, agent. ################################################################### self._start_terrestrial() self._start_remote() self._start_agent() ################################################################### # Assign client ports. # This is primarily for test purposes as the IP config in # deployment will be fixed in advance. ################################################################### self.te_client.set_client_port(self._remote_port) check_port = self.te_client.get_client_port() log.debug('Terrestrial client port is: %i', check_port) self.re_client.set_client_port(self._terrestrial_port) check_port = self.re_client.get_client_port() log.debug('Remote client port is: %i', check_port) ################################################################### # Start the event publisher and subscribers. # Used to send fake agent telemetry publications to the endpoints, # and to receive endpoint publications. ################################################################### self._event_publisher = EventPublisher() # Start the event subscriber for remote namespace platform events. # This event could be changed to RemoteNamespaceEvent. self._event_subscriber = EventSubscriber( event_type='PlatformEvent', callback=self.consume_event, origin=self._xs_name) self._event_subscriber.start() self._event_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._event_subscriber.stop) # Start the result subscriber for remote resource events. self._resource_result_subscriber = EventSubscriber( event_type='RemoteCommandResult', origin=IA_RESOURCE_ID, callback=self.consume_event) self._resource_result_subscriber.start() self._resource_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._resource_result_subscriber.stop) # Start the result subscriber for remote service events. self._service_result_subscriber = EventSubscriber( event_type='RemoteCommandResult', origin='resource_registry' + 'remote1', callback=self.consume_event) self._service_result_subscriber.start() self._service_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._service_result_subscriber.stop) # Start the result subscriber for fake resource results. self._fake_result_subscriber = EventSubscriber( event_type='RemoteCommandResult', origin=self._resource_id, callback=self.consume_event) self._fake_result_subscriber.start() self._fake_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._fake_result_subscriber.stop) ################################################################### # Start/stop helpers. ################################################################### def _start_agent(self): """ Start an instrument agent and client. """ log.info('Creating driver integration test support:') log.info('driver module: %s', DRV_MOD) log.info('driver class: %s', DRV_CLS) log.info('device address: %s', DEV_ADDR) log.info('device port: %s', DEV_PORT) log.info('log delimiter: %s', DELIM) log.info('work dir: %s', WORK_DIR) self._support = DriverIntegrationTestSupport(DRV_MOD, DRV_CLS, DEV_ADDR, DEV_PORT, DATA_PORT, CMD_PORT, PA_BINARY, DELIM, WORK_DIR) # Start port agent, add stop to cleanup. port = self._support.start_pagent() log.info('Port agent started at port %i',port) # Configure driver to use port agent port number. DVR_CONFIG['comms_config'] = { 'addr' : 'localhost', 'port' : port } self.addCleanup(self._support.stop_pagent) # Create agent config. agent_config = { 'driver_config' : DVR_CONFIG, 'stream_config' : {}, 'agent' : {'resource_id': IA_RESOURCE_ID}, 'test_mode' : True } # Start instrument agent. log.debug("Starting IA.") container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) ia_pid = container_client.spawn_process(name=IA_NAME, module=IA_MOD, cls=IA_CLS, config=agent_config) log.info('Agent pid=%s.', str(ia_pid)) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess()) log.info('Got ia client %s.', str(self._ia_client)) def _start_terrestrial(self): """ Start up the terrestrial endpoint. """ # Create terrestrial config. terrestrial_endpoint_config = { 'other_host' : 'localhost', 'other_port' : self._remote_port, 'this_port' : self._terrestrial_port, 'platform_resource_id' : self._terrestrial_platform_id, 'xs_name' : self._xs_name, 'process' : { 'listen_name' : self._terrestrial_listen_name } } # Spawn the terrestrial enpoint process. log.debug('Spawning terrestrial endpoint process.') self._terrestrial_pid = self._container_client.spawn_process( name=self._terrestrial_listen_name, module='ion.services.sa.tcaa.terrestrial_endpoint', cls='TerrestrialEndpoint', config=terrestrial_endpoint_config) log.debug('Terrestrial endpoint pid=%s.', str(self._terrestrial_pid)) # Create a terrestrial client. self.te_client = TerrestrialEndpointClient( process=FakeProcess(), to_name=self._terrestrial_listen_name) log.debug('Got te client %s.', str(self.te_client)) self._terrestrial_port = self.te_client.get_port() log.debug('Terrestrial port is: %i', self._terrestrial_port) def _start_remote(self): """ Start up the remote endpoint. """ # Create agent config. remote_endpoint_config = { 'other_host' : 'localhost', 'other_port' : self._terrestrial_port, 'this_port' : self._remote_port, 'platform_resource_id' : self._remote_platform_id, 'xs_name' : self._xs_name, 'process' : { 'listen_name' : self._remote_listen_name } } # Spawn the remote enpoint process. log.debug('Spawning remote endpoint process.') self._remote_pid = self._container_client.spawn_process( name=self._remote_listen_name, module='ion.services.sa.tcaa.remote_endpoint', cls='RemoteEndpoint', config=remote_endpoint_config) log.debug('Remote endpoint pid=%s.', str(self._remote_pid)) # Create an endpoint client. self.re_client = RemoteEndpointClient( process=FakeProcess(), to_name=self._remote_listen_name) log.debug('Got re client %s.', str(self.re_client)) # Remember the remote port. self._remote_port = self.re_client.get_port() log.debug('The remote port is: %i.', self._remote_port) ################################################################### # Telemetry publications to start/top endpoint. # (Normally be published by appropriate platform agents.) ################################################################### def terrestrial_link_up(self): """ Publish telemetry available to the terrestrial endpoint. """ # Publish a link up event to be caught by the terrestrial endpoint. log.debug('Publishing terrestrial telemetry available event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin=self._terrestrial_platform_id, status = TelemetryStatusType.AVAILABLE) def terrestrial_link_down(self): """ Publish telemetry unavailable to the terrestrial endpoint. """ # Publish a link up event to be caught by the terrestrial endpoint. log.debug('Publishing terrestrial telemetry unavailable event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin=self._terrestrial_platform_id, status = TelemetryStatusType.UNAVAILABLE) def remote_link_up(self): """ Publish telemetry available to the remote endpoint. """ # Publish a link up event to be caught by the remote endpoint. log.debug('Publishing remote telemetry available event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin=self._remote_platform_id, status = TelemetryStatusType.AVAILABLE) def remote_link_down(self): """ Publish telemetry unavailable to the remote endpoint. """ # Publish a link down event to be caught by the remote endpoint. log.debug('Publishing remote telemetry unavailable event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin=self._remote_platform_id, status = TelemetryStatusType.UNAVAILABLE) def consume_event(self, evt, *args, **kwargs): """ Test callback for events. """ log.debug('Test got event: %s, args: %s, kwargs: %s', str(evt), str(args), str(kwargs)) if evt.type_ == 'PublicPlatformTelemetryEvent': self._telem_evts.append(evt) if self._no_telem_evts > 0 and self._no_telem_evts == len(self._telem_evts): self._done_telem_evt.set() elif evt.type_ == 'RemoteQueueModifiedEvent': self._queue_mod_evts.append(evt) if self._no_queue_mod_evts > 0 and self._no_queue_mod_evts == len(self._queue_mod_evts): self._done_queue_mod_evt.set() elif evt.type_ == 'RemoteCommandTransmittedEvent': self._cmd_tx_evts.append(evt) if self._no_cmd_tx_evts > 0 and self._no_cmd_tx_evts == len(self._cmd_tx_evts): self._done_cmd_tx_evt.set() elif evt.type_ == 'RemoteCommandResult': cmd = evt.command self._results_recv[cmd.command_id] = cmd if len(self._results_recv) == self._no_requests: self._done_cmd_evt.set() ################################################################### # Misc helpers. ################################################################### def make_fake_command(self, no): """ Build a fake command for use in tests. """ cmdstr = 'fake_cmd_%i' % no cmd = IonObject('RemoteCommand', resource_id='fake_id', command=cmdstr, args=['arg1', 23], kwargs={'kwargs1':'someval'}) return cmd ################################################################### # Tests. ################################################################### def test_resource_client_online(self): """ test_recourse_client_online Test the client transparently forwards commands to the remote resource while link is up. """ # Publish link up events. self.terrestrial_link_up() self.remote_link_up() remote_client = RemoteClient(iface=IResourceAgent, xs_name=self._xs_name, resource_id='fake_id', process=FakeProcess()) # Queue up a series of fake commands to be handled by the remote side. for i in range(self._no_requests): cmd = remote_client.ping_agent() self._requests_sent[cmd.command_id] = cmd # Block on queue mod events. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) # Block on command transmissions and results. self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Publish link down events. self.terrestrial_link_down() self.remote_link_down() # Block on terrestrial public telemetry events. self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) #@unittest.skip('For some reason this bastard wont run on the builder.') def test_resource_client_blocking(self): """ test_resource_client_blocking Test the client can block on remote resource command results. """ # Publish link up events. self.terrestrial_link_up() self.remote_link_up() remote_client = RemoteClient(iface=IResourceAgent, xs_name=self._xs_name, resource_id=IA_RESOURCE_ID, process=FakeProcess()) # Queue up a series of fake commands to be handled by the remote side. """ {'time_completed': 1350421095.804607, 'resource_id': '123xyz', 'time_queued': 1350421095.623531, 'args': [], 'type_': 'RemoteCommand', 'command': 'ping_agent', 'result': 'ping from InstrumentAgent (name=Agent007,id=Edwards-MacBook-Pro_local_10126.35,type=agent), time: 1350421095757', 'kwargs': {}, 'svc_name': '', 'command_id': '76be11b4-a22c-49de-89cd-4e019463d7c9'} """ for i in range(self._no_requests): cmd = remote_client.ping_agent(remote_timeout=CFG.endpoint.receive.timeout) self.assertEqual(cmd.resource_id, IA_RESOURCE_ID) self.assertEqual(cmd.command, 'ping_agent') self.assertIn('ping from InstrumentAgent', cmd.result) # Publish link down events. self.terrestrial_link_down() self.remote_link_down() # Block on terrestrial public telemetry events. self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout) #@unittest.skip('For some reason this bastard wont run on the builder.') def test_service_client_blocking(self): """ test_service_client_blocking Test the client can command remote services and block on their results. """ # Publish link up events. self.terrestrial_link_up() self.remote_link_up() # Create remote client for the resource registry. remote_client = RemoteClient(iface=IResourceRegistryService, xs_name=self._xs_name, svc_name='resource_registry', process=FakeProcess()) # Create user object. obj = IonObject("UserInfo", name="some_name") result = remote_client.create(obj, remote_timeout=CFG.endpoint.receive.timeout) # Returns obj_id, obj_rev. obj_id, obj_rev = result.result # Confirm the results are valid. #Result is a tuple of strings. #{'result': ['ad183ff26bae4f329ddd85fd69d160a9', #'1-00a308c45fff459c7cda1db9a7314de6'], #'command_id': 'cc2ae00d-40b0-47d2-af61-8ffb87f1aca2'} self.assertIsInstance(obj_id, str) self.assertNotEqual(obj_id, '') self.assertIsInstance(obj_rev, str) self.assertNotEqual(obj_rev, '') # Read user object. result = remote_client.read(obj_id, remote_timeout=CFG.endpoint.receive.timeout) # Returns read_obj. read_obj = result.result # Confirm the results are valid. #Result is a user info object with the name set. #{'lcstate': 'DEPLOYED_AVAILABLE', #'_rev': '1-851f067bac3c34b2238c0188b3340d0f', #'description': '', #'ts_updated': '1349213207638', #'type_': 'UserInfo', #'contact': <interface.objects.ContactInformation object at 0x10d7df590>, #'_id': '27832d93f4cd4535a75ac75c06e00a7e', #'ts_created': '1349213207638', #'variables': [{'name': '', 'value': ''}], #'name': 'some_name'} self.assertIsInstance(read_obj, UserInfo) self.assertEquals(read_obj.name, 'some_name') # Update user object. read_obj.name = 'some_other_name' result = remote_client.update(read_obj, remote_timeout=CFG.endpoint.receive.timeout) # Returns nothing. # Read user object. result = remote_client.read(obj_id, remote_timeout=CFG.endpoint.receive.timeout) # Returns read_obj. read_obj = result.result # Confirm results are valid. self.assertIsInstance(read_obj, UserInfo) self.assertEquals(read_obj.name, 'some_other_name') # Delete user object. result = remote_client.delete(obj_id, remote_timeout=CFG.endpoint.receive.timeout) # Returns nothing. # Publish link down events. self.terrestrial_link_down() self.remote_link_down() # Block on terrestrial public telemetry events. self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout) def test_queue_manipulators(self): """ test_queue_manipulators Test ability to instpect and manipulate the command queue corresponding to this resource or service. """ remote_client = RemoteClient(iface=IResourceAgent, xs_name=self._xs_name, resource_id='fake_id', process=FakeProcess()) # Queue up a series of fake commands to be handled by the remote side. for i in range(self._no_requests): cmd = remote_client.ping_agent(link=False) self._requests_sent[cmd.command_id] = cmd # Block on queue mod events. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) queue = remote_client.get_queue() self.assertEqual(len(queue), self._no_requests) popped = remote_client.clear_queue() self.assertEqual(len(popped), self._no_requests) self._requests_sent = {} # Queue up a series of fake commands to be handled by the remote side. for i in range(self._no_requests): cmd = remote_client.ping_agent(link=False) self._requests_sent[cmd.command_id] = cmd # Pop the last three commands. cmd_ids = self._requests_sent.keys()[:3] poped = [] for x in cmd_ids: poped.append(remote_client.pop_queue(x)) self._requests_sent.pop(x) queue = remote_client.get_queue() self.assertEqual(len(queue), self._no_requests - 3) self.assertEqual(len(poped), 3) self._no_requests = self._no_requests - 3 self._no_cmd_tx_evts = self._no_requests # Publish link up events. self.terrestrial_link_up() self.remote_link_up() # Block on command transmissions and results. self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) pending = remote_client.get_pending() for x in pending: self.assertIn(x.command_id, self._requests_sent.keys()) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Publish link down events. self.terrestrial_link_down() self.remote_link_down() # Block on terrestrial public telemetry events. self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) def test_errors(self): """ test_errors Test various error conditions. """ # Constructed without a xs name. with self.assertRaises(ConfigNotFound): remote_client = RemoteClient(iface=IResourceAgent, resource_id=IA_RESOURCE_ID, process=FakeProcess()) # Constructed without an interface. with self.assertRaises(ConfigNotFound): remote_client = RemoteClient(xs_name=self._xs_name, resource_id=IA_RESOURCE_ID, process=FakeProcess()) # Construct with invalid interface. with self.assertRaises(ConfigNotFound): remote_client = RemoteClient('Bogus_interface', xs_name=self._xs_name, resource_id=IA_RESOURCE_ID, process=FakeProcess()) # Construct with no resource or service specified. with self.assertRaises(ConfigNotFound): remote_client = RemoteClient(iface=IResourceAgent, xs_name=self._xs_name, process=FakeProcess()) # Construct with both resource and service specified. with self.assertRaises(ConfigNotFound): remote_client = RemoteClient(iface=IResourceAgent, xs_name=self._xs_name, resource_id=IA_RESOURCE_ID, svc_name='resource_registry', process=FakeProcess()) # Create a valid resource client. remote_client = RemoteClient(iface=IResourceAgent, xs_name=self._xs_name, resource_id=IA_RESOURCE_ID, process=FakeProcess()) # Send a command while link is down. with self.assertRaises(Conflict): result = remote_client.ping_agent() # Test port manipulators refused by remote proxy client. with self.assertRaises(BadRequest): remote_client.get_port() with self.assertRaises(BadRequest): remote_client.set_client_port() with self.assertRaises(BadRequest): remote_client.get_client_port() def test_interfaces(self): """ test_interfaces Test that the client declare the correct interfaces. """ remote_client = RemoteClient(iface=IResourceAgent, xs_name=self._xs_name, resource_id='fake_id', process=FakeProcess()) interfaces = providedBy(remote_client) self.assertIn(IResourceAgent, interfaces) self.assertIn(ITerrestrialEndpoint, interfaces)
def test_persistent_queue(self): """ test_persistent_queue Test ability of endpoint to restore a persistent queue, survive reboot, etc. """ self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._no_telem_evts = 2 for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Confirm queue mod events. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) # Confirm get queue with no id. queue = self.te_client.get_queue() self.assertEqual(len(queue), self._no_requests) # Stop and restart the endpoint process. # Verify the queue is restored. self._container_client.terminate_process(self._te_pid) # Create agent config. endpoint_config = { 'other_host' : self._other_host, 'other_port' : self._other_port, 'this_port' : 0, 'xs_name' : self._xs_name, 'platform_resource_id' : self._platform_resource_id, 'process' : { 'listen_name' : self._listen_name } } # Spawn the terrestrial enpoint process. log.debug('Spawning terrestrial endpoint process.') self._te_pid = self._container_client.spawn_process( name='remote_endpoint_1', module='ion.services.sa.tcaa.terrestrial_endpoint', cls='TerrestrialEndpoint', config=endpoint_config) log.debug('Endpoint pid=%s.', str(self._te_pid)) # Create an endpoint client. # The to_name may be either the process pid or # the listen_name, which for this remote bridge # is svc_name + remote_name as above. self.te_client = TerrestrialEndpointClient( process=FakeProcess(), to_name=self._listen_name) log.debug('Got te client %s.', str(self.te_client)) # Remember the terrestrial port. self._this_port = self.te_client.get_port() # Confirm we restored the queue with the previous commands. queue = self.te_client.get_queue() self.assertEqual(len(queue), self._no_requests) self.on_link_up() self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) pending = self.te_client.get_pending() self.assertEqual(len(pending), 0) self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys())
class RemoteClient(object): """ A proxy client for any service or resource that forwards commands to the terrestrial endpoint for transmission. """ implements(ITerrestrialEndpoint) def __init__(self, iface=None, xs_name=None, resource_id=None, svc_name=None, process=None): """ Construct remote proxy client. Verify required parameters. Construct remote endpoint client. Set internal variables. Construct and add service method attributes. """ # Throw an exception if required interface arg not provided. if not isinstance(iface, InterfaceClass): raise ConfigNotFound('Invalid interface parameter.') # Throw an exception if the xs_name is not provided. if not isinstance(xs_name, str) or xs_name == '': raise ConfigNotFound('Invalid exchange space name parameter.') # Create the endpoint client. # Throw exception if unsuccessful. to_name = 'terrestrial_endpoint' + xs_name self._te_client = TerrestrialEndpointClient(process=process, to_name=to_name) # Must define a resource id or service name. if not resource_id and not svc_name: raise ConfigNotFound('No resource or service specified.') # Can't specify both a resource and a service. if resource_id and svc_name: raise ConfigNotFound( 'Can\'t specify both a resource and a service.') self._resource_id = resource_id self._xs_name = xs_name self._svc_name = svc_name # Grab the service method names. methods = iface.names() # Generate the service interface. # Each will forward to the terrestrial endpoint passing # the function name, args and kwargs. for m in methods: setattr(self, m, self.generate_service_method(m)) # Declare the dynamic interface. directlyProvides(self, iface) # Initialize the async results objects for blocking behavior. #self._async_result_evt = None def generate_service_method(self, name): """ A closure that returns a function for forwarding service calls. The service call name is stored as a kwarg. """ def func(*args, **kwargs): args = copy.deepcopy(args) kwargs = copy.deepcopy(kwargs) kwargs['func_name'] = name return self.forward(*args, **kwargs) return func def forward(self, *args, **kwargs): """ Forward a service method to the terrestrial endpoint through the service interface. """ func_name = kwargs.pop('func_name') try: link = kwargs.pop('link') except KeyError: link = True cid = '' try: remote_timeout = kwargs.pop('remote_timeout') if not isinstance(remote_timeout, int): remote_timeout = 0 elif remote_timeout < 0: remote_timeout = 0 elif remote_timeout == 0: pass else: cid = str(uuid.uuid4()) except KeyError: remote_timeout = 0 cmd = IonObject('RemoteCommand', resource_id=self._resource_id, svc_name=self._svc_name, command=func_name, command_id=cid, args=args, kwargs=kwargs) if remote_timeout == 0: return self._te_client.enqueue_command(cmd, link) else: if self._resource_id: origin = self._resource_id elif self._svc_name: origin = self._svc_name + self._xs_name pending_cmd = cmd async_result_evt = AsyncResult() def result_callback(evt, *args, **kwargs): """ Callback for subscriber retrive blocking results. """ #global async_result_evt if evt.type_ == 'RemoteCommandResult': cmd = evt.command if cmd.command_id == pending_cmd.command_id: async_result_evt.set(cmd) sub = EventSubscriber(event_type='RemoteCommandResult', origin=origin, callback=result_callback) sub.start() #self._pending_cmd = cmd cmd = self._te_client.enqueue_command(cmd, link) try: result = async_result_evt.get(timeout=remote_timeout) #self._pending_cmd = None sub.stop() except gevent.Timeout: #self._pending_cmd = None sub.stop() raise Timeout('Timed out waiting for remote result.') return result """ def _result_callback(self, evt, *args, **kwargs): #Callback for subscriber retrive blocking results. if evt.type_ == 'RemoteCommandResult': cmd = evt.command if self._pending_cmd: if cmd.command_id == self._pending_cmd.command_id: self._pending_cmd = None if self._async_result_evt: self._async_result_evt.set(cmd) """ def enqueue_command(self, command=None, link=False): """ Enqueue command with terrestrial endoint. """ return self._te_client(command, link) def get_queue(self): """ Get terrestrial endpoint queue for this resource/service. """ if self._resource_id: return self._te_client.get_queue(resource_id=self._resource_id) elif self._svc_name: return self._te_client.get_queue(svc_name=self._svc_name) def clear_queue(self): """ Clear terrestrial endpoint queue for this resource/service. """ if self._resource_id: return self._te_client.clear_queue(resource_id=self._resource_id) elif self._svc_name: return self._te_client.clear_queue(svc_name=self._svc_name) def pop_queue(self, command_id=''): """ Pop a command from the terrestrial endpoint queue. """ return self._te_client.pop_queue(command_id=command_id) def get_pending(self): """ Get pending commands for this resource/service. """ if self._resource_id: return self._te_client.get_pending(resource_id=self._resource_id) elif self._svc_name: return self._te_client.get_pending(svc_name=self._svc_name) def get_port(self): """ Not supported for remote proxy clients. """ raise BadRequest('get_port not available via remote client.') def set_client_port(self, port=0): """ Not supported for remote proxy clients. """ raise BadRequest('set_client_port not available via remote client.') def get_client_port(self): """ Not supported for remote proxy clients. """ raise BadRequest('get_client_port not available via remote client.')
def test_persistent_queue(self): """ test_persistent_queue Test ability of endpoint to restore a persistent queue, survive reboot, etc. """ self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._no_telem_evts = 2 for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Confirm queue mod events. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) # Confirm get queue with no id. queue = self.te_client.get_queue() self.assertEqual(len(queue), self._no_requests) # Stop and restart the endpoint process. # Verify the queue is restored. self._container_client.terminate_process(self._te_pid) # Create agent config. endpoint_config = { 'other_host': self._other_host, 'other_port': self._other_port, 'this_port': 0, 'xs_name': self._xs_name, 'platform_resource_id': self._platform_resource_id, 'process': { 'listen_name': self._listen_name } } # Spawn the terrestrial enpoint process. log.debug('Spawning terrestrial endpoint process.') self._te_pid = self._container_client.spawn_process( name='remote_endpoint_1', module='ion.services.sa.tcaa.terrestrial_endpoint', cls='TerrestrialEndpoint', config=endpoint_config) log.debug('Endpoint pid=%s.', str(self._te_pid)) # Create an endpoint client. # The to_name may be either the process pid or # the listen_name, which for this remote bridge # is svc_name + remote_name as above. self.te_client = TerrestrialEndpointClient(process=FakeProcess(), to_name=self._listen_name) log.debug('Got te client %s.', str(self.te_client)) # Remember the terrestrial port. self._this_port = self.te_client.get_port() # Confirm we restored the queue with the previous commands. queue = self.te_client.get_queue() self.assertEqual(len(queue), self._no_requests) self.on_link_up() self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) pending = self.te_client.get_pending() self.assertEqual(len(pending), 0) self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys())
def setUp(self): """ Setup fake remote components. Start remote server. Set internal configuration and test variables. Start container. Start services. Spawn endpoint. Create and start subscribers. """ # Create fake remote client and server. # Add clean up to shut down properly. # Start remote server on a random port. self._remote_server = R3PCServer(self.consume_req, self.remote_server_close) self._remote_client = R3PCClient(self.consume_ack, self.remote_client_close) self.addCleanup(self._remote_server.stop) self.addCleanup(self._remote_client.stop) self._other_port = self._remote_server.start('*', 0) log.debug('Remote server binding to *:%i', self._other_port) # Set internal variables. self._other_host = 'localhost' self._xs_name = 'remote1' self._svc_name = 'terrestrial_endpoint' self._listen_name = self._svc_name + self._xs_name self._platform_resource_id = 'abc123' self._resource_id = 'fake_id' self._rmt_svc_name = 'fake_svc' self._no_requests = 10 self._requests_sent = {} self._results_recv = {} self._workers = [] self._done_evt = AsyncResult() self._queue_mod_evts = [] self._cmd_tx_evts = [] self._telem_evts = [] self._no_telem_evts = 0 self._no_queue_mod_evts = 0 self._no_cmd_tx_evts = 0 self._done_queue_mod_evts = AsyncResult() self._done_telem_evts = AsyncResult() self._done_cmd_tx_evts = AsyncResult() # Start container. log.debug('Staring capability container.') self._start_container() # Bring up services in a deploy file (no need to message) log.info('Staring deploy services.') self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Create a container client. log.debug('Creating container client.') self._container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) # The following spawn config creates the process with the remote # name tagged to the service name. """ listen_name = terrestrial_endpointremote1 2012-10-10 11:34:46,654 DEBUG ion.services.sa.tcaa.terrestrial_endpoint recv name: NP (ion_test_8257ab,terrestrial_endpointremote1,B: terrestrial_endpointremote1) 2012-10-10 11:34:46,654 DEBUG ion.services.sa.tcaa.terrestrial_endpoint startup listener recv name: NP (ion_test_8257ab,terrestrial_endpointremote1,B: terrestrial_endpointremote1) 2012-10-10 11:34:46,654 DEBUG ion.services.sa.tcaa.terrestrial_endpoint startup listener recv name: NP (ion_test_8257ab,Edwards-MacBook-Pro_local_2624.33,B: Edwards-MacBook-Pro_local_2624.33) """ # Create agent config. endpoint_config = { 'other_host': self._other_host, 'other_port': self._other_port, 'this_port': 0, 'xs_name': self._xs_name, 'platform_resource_id': self._platform_resource_id, 'process': { 'listen_name': self._listen_name } } # Spawn the terrestrial enpoint process. log.debug('Spawning terrestrial endpoint process.') self._te_pid = self._container_client.spawn_process( name='remote_endpoint_1', module='ion.services.sa.tcaa.terrestrial_endpoint', cls='TerrestrialEndpoint', config=endpoint_config) log.debug('Endpoint pid=%s.', str(self._te_pid)) # Create an endpoint client. # The to_name may be either the process pid or # the listen_name, which for this remote bridge # is svc_name + remote_name as above. self.te_client = TerrestrialEndpointClient(process=FakeProcess(), to_name=self._listen_name) log.debug('Got te client %s.', str(self.te_client)) # Remember the terrestrial port. self._this_port = self.te_client.get_port() # Start the event publisher. self._event_publisher = EventPublisher() # Start the event subscriber. self._event_subscriber = EventSubscriber(event_type='PlatformEvent', callback=self.consume_event, origin=self._xs_name) self._event_subscriber.start() self._event_subscriber._ready_event.wait( timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._event_subscriber.stop) # Start the result subscriber. self._result_subscriber = EventSubscriber( event_type='RemoteCommandResult', origin=self._resource_id, callback=self.consume_event) self._result_subscriber.start() self._result_subscriber._ready_event.wait( timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._result_subscriber.stop)
class TestTerrestrialEndpoint(IonIntegrationTestCase): """ Test cases for 2CAA terrestrial endpoint. """ def setUp(self): """ Setup fake remote components. Start remote server. Set internal configuration and test variables. Start container. Start services. Spawn endpoint. Create and start subscribers. """ # Create fake remote client and server. # Add clean up to shut down properly. # Start remote server on a random port. self._remote_server = R3PCServer(self.consume_req, self.remote_server_close) self._remote_client = R3PCClient(self.consume_ack, self.remote_client_close) self.addCleanup(self._remote_server.stop) self.addCleanup(self._remote_client.stop) self._other_port = self._remote_server.start('*', 0) log.debug('Remote server binding to *:%i', self._other_port) # Set internal variables. self._other_host = 'localhost' self._xs_name = 'remote1' self._svc_name = 'terrestrial_endpoint' self._listen_name = self._svc_name + self._xs_name self._platform_resource_id = 'abc123' self._resource_id = 'fake_id' self._rmt_svc_name = 'fake_svc' self._no_requests = 10 self._requests_sent = {} self._results_recv = {} self._workers = [] self._done_evt = AsyncResult() self._queue_mod_evts = [] self._cmd_tx_evts = [] self._telem_evts = [] self._no_telem_evts = 0 self._no_queue_mod_evts = 0 self._no_cmd_tx_evts = 0 self._done_queue_mod_evts = AsyncResult() self._done_telem_evts = AsyncResult() self._done_cmd_tx_evts = AsyncResult() # Start container. log.debug('Staring capability container.') self._start_container() # Bring up services in a deploy file (no need to message) log.info('Staring deploy services.') self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Create a container client. log.debug('Creating container client.') self._container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) # The following spawn config creates the process with the remote # name tagged to the service name. """ listen_name = terrestrial_endpointremote1 2012-10-10 11:34:46,654 DEBUG ion.services.sa.tcaa.terrestrial_endpoint recv name: NP (ion_test_8257ab,terrestrial_endpointremote1,B: terrestrial_endpointremote1) 2012-10-10 11:34:46,654 DEBUG ion.services.sa.tcaa.terrestrial_endpoint startup listener recv name: NP (ion_test_8257ab,terrestrial_endpointremote1,B: terrestrial_endpointremote1) 2012-10-10 11:34:46,654 DEBUG ion.services.sa.tcaa.terrestrial_endpoint startup listener recv name: NP (ion_test_8257ab,Edwards-MacBook-Pro_local_2624.33,B: Edwards-MacBook-Pro_local_2624.33) """ # Create agent config. endpoint_config = { 'other_host': self._other_host, 'other_port': self._other_port, 'this_port': 0, 'xs_name': self._xs_name, 'platform_resource_id': self._platform_resource_id, 'process': { 'listen_name': self._listen_name } } # Spawn the terrestrial enpoint process. log.debug('Spawning terrestrial endpoint process.') self._te_pid = self._container_client.spawn_process( name='remote_endpoint_1', module='ion.services.sa.tcaa.terrestrial_endpoint', cls='TerrestrialEndpoint', config=endpoint_config) log.debug('Endpoint pid=%s.', str(self._te_pid)) # Create an endpoint client. # The to_name may be either the process pid or # the listen_name, which for this remote bridge # is svc_name + remote_name as above. self.te_client = TerrestrialEndpointClient(process=FakeProcess(), to_name=self._listen_name) log.debug('Got te client %s.', str(self.te_client)) # Remember the terrestrial port. self._this_port = self.te_client.get_port() # Start the event publisher. self._event_publisher = EventPublisher() # Start the event subscriber. self._event_subscriber = EventSubscriber(event_type='PlatformEvent', callback=self.consume_event, origin=self._xs_name) self._event_subscriber.start() self._event_subscriber._ready_event.wait( timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._event_subscriber.stop) # Start the result subscriber. self._result_subscriber = EventSubscriber( event_type='RemoteCommandResult', origin=self._resource_id, callback=self.consume_event) self._result_subscriber.start() self._result_subscriber._ready_event.wait( timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._result_subscriber.stop) def consume_event(self, evt, *args, **kwargs): """ Test callback for events. """ log.debug('Got event: %s, args: %s, kwargs: %s', str(evt), str(args), str(kwargs)) if evt.type_ == 'PublicPlatformTelemetryEvent': self._telem_evts.append(evt) if self._no_telem_evts > 0 and self._no_telem_evts == len( self._telem_evts): self._done_telem_evts.set() elif evt.type_ == 'RemoteQueueModifiedEvent': self._queue_mod_evts.append(evt) if self._no_queue_mod_evts > 0 and self._no_queue_mod_evts == len( self._queue_mod_evts): self._done_queue_mod_evts.set() elif evt.type_ == 'RemoteCommandTransmittedEvent': self._cmd_tx_evts.append(evt) if self._no_cmd_tx_evts > 0 and self._no_cmd_tx_evts == len( self._cmd_tx_evts): self._done_cmd_tx_evts.set() elif evt.type_ == 'RemoteCommandResult': cmd = evt.command self._results_recv[cmd.command_id] = cmd if len(self._results_recv) == self._no_requests: self._done_evt.set() def on_link_up(self): """ Called by a test to simulate turning the link on. """ log.debug('Remote client connecting to localhost:%i.', self._this_port) self._remote_client.start('localhost', self._this_port) # Publish a link up event to be caught by the endpoint. log.debug('Publishing telemetry event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin=self._platform_resource_id, status=TelemetryStatusType.AVAILABLE) def on_link_down(self): """ Called by a test to simulate turning the link off. """ self._remote_client.stop() # Publish a link down event to be caught by the endpoint. log.debug('Publishing telemetry event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin=self._platform_resource_id, status=TelemetryStatusType.UNAVAILABLE) def consume_req(self, request): """ Remote request callback. Fire a greenlet to do some fake work before returning via the remote client to terrestrial endpoint. """ # Spawn a greenlet to sleep briefly with each request and # then respond with a result through the remote client. log.debug('Remote endpoint got request: %s', str(request)) greenlet = gevent.spawn(self.process_remote_request, request) self._workers.append(greenlet) def consume_ack(self, request): """ Remote ack callback. """ log.debug('Remote endpoint got ack: %s', str(request)) def process_remote_request(self, request): """ Process remote request. Do random amount of fake work and enqueue result for return to terrestrial endpoint. """ worktime = random.uniform(.1, 3) gevent.sleep(worktime) result = {'command_id': request.command_id, 'result': 'fake_result'} log.debug('Finished processing request: %s', str(request)) self._remote_client.enqueue(result) def remote_server_close(self): """ Remote server closed callback. """ log.debug('The remote server closed.') def remote_client_close(self): """ Remoe client closed callback. """ log.debug('The remote client closed.') def make_fake_command(self, no): """ Build a fake command for use in tests. """ cmdstr = 'fake_cmd_%i' % no cmd = IonObject('RemoteCommand', resource_id=self._resource_id, command=cmdstr, args=['arg1', 23], kwargs={'kwargs1': 'someval'}) return cmd def make_fake_svc_command(self, no): """ Build a fake command for use in tests. """ cmdstr = 'fake_cmd_%i' % no cmd = IonObject('RemoteCommand', svc_name=self._rmt_svc_name, command=cmdstr, args=['arg1', 23], kwargs={'kwargs1': 'someval'}) return cmd def test_process_queued(self): """ test_process_queued Test forwarding of queued commands upon link up. """ self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._no_telem_evts = 2 for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) self.on_link_up() self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) pending = self.te_client.get_pending() self.assertEqual(len(pending), 0) self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) def test_process_online(self): """ test_process_online Test forwarding commands when the link is up. """ self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._no_telem_evts = 2 self.on_link_up() for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd gevent.sleep(.2) self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) pending = self.te_client.get_pending() self.assertEqual(len(pending), 0) self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) def test_remote_late(self): """ test_remote_late Test simulates behavior when the remote side is initially unavailable. """ self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._no_telem_evts = 2 self.on_link_up() gevent.sleep(2) self._remote_server.stop() self._remote_client.stop() for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) gevent.sleep(3) self._remote_client.start('localhost', self._this_port) self._remote_server.start('*', self._other_port) self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) pending = self.te_client.get_pending() self.assertEqual(len(pending), 0) self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) def test_get_clear_queue(self): """ test_get_clear_queue Test endpoint queue get and clear manipulators. """ # Set up for events expected. self._no_queue_mod_evts = self._no_requests * 2 # Queue commands. for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Queue commands. for i in range(self._no_requests): cmd = self.make_fake_svc_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Confirm queue mod events. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) # Confirm get queue with no id. queue = self.te_client.get_queue() self.assertEqual(len(queue), self._no_requests * 2) # Confirm get queue with id. queue = self.te_client.get_queue(resource_id=self._resource_id) self.assertEqual(len(queue), self._no_requests) # Confirm get queue with svc name. queue = self.te_client.get_queue(svc_name=self._rmt_svc_name) self.assertEqual(len(queue), self._no_requests) # Confirm get queue with bogus id. queue = self.te_client.get_queue(resource_id='bogus_id') self.assertEqual(len(queue), 0) # Confirm get queue with bogus id. queue = self.te_client.get_queue(svc_name='bogus_svc') self.assertEqual(len(queue), 0) # Reset queue mod expected events. self._queue_mod_evts = [] self._no_queue_mod_evts = 1 self._done_queue_mod_evts = AsyncResult() # Clear queue with no id. poped = self.te_client.clear_queue() # Confirm queue mod event and mods. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) queue = self.te_client.get_queue() self.assertEqual(len(poped), self._no_requests * 2) self.assertEqual(len(queue), 0) # Queue new commands and confirm event. self._queue_mod_evts = [] self._no_queue_mod_evts = self._no_requests * 2 self._done_queue_mod_evts = AsyncResult() self._requests_sent = {} for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd for i in range(self._no_requests): cmd = self.make_fake_svc_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) # Reset queue mod expected events. self._queue_mod_evts = [] self._no_queue_mod_evts = 1 self._done_queue_mod_evts = AsyncResult() # Clear queue with id. poped = self.te_client.clear_queue(resource_id=self._resource_id) # Confirm mods and mod events. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) queue = self.te_client.get_queue() self.assertEqual(len(poped), self._no_requests) self.assertEqual(len(queue), self._no_requests) # Reset queue mod expected events. self._queue_mod_evts = [] self._no_queue_mod_evts = 1 self._done_queue_mod_evts = AsyncResult() # Clear queue with id. poped = self.te_client.clear_queue(svc_name=self._rmt_svc_name) # Confirm mods and mod events. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) queue = self.te_client.get_queue() self.assertEqual(len(poped), self._no_requests) self.assertEqual(len(queue), 0) # Queue new commands and confirm events. self._queue_mod_evts = [] self._no_queue_mod_evts = self._no_requests self._done_queue_mod_evts = AsyncResult() self._requests_sent = {} for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) # Clear queue with bogus id. poped = self.te_client.clear_queue(resource_id='bogus id') queue = self.te_client.get_queue() self.assertEqual(len(poped), 0) self.assertEqual(len(queue), self._no_requests) # Clear queue with bogus svc name. poped = self.te_client.clear_queue(svc_name='bogus id') queue = self.te_client.get_queue() self.assertEqual(len(poped), 0) self.assertEqual(len(queue), self._no_requests) # Clear queue and confirm empty. self.te_client.clear_queue() queue = self.te_client.get_queue() self.assertEqual(len(queue), 0) # Turn on link and wait a few seconds. # Confirm no data or tx events arrive. self.on_link_up() gevent.sleep(2) self.assertEqual(len(self._cmd_tx_evts), 0) self.assertEqual(len(self._results_recv), 0) self._no_telem_evts = 2 self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) def test_pop_pending_queue(self): """ test_pop_pending_queue Test endpoint queue pop manipulators. """ # Set up for events expected. self._no_queue_mod_evts = self._no_requests # Queue commands. for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Confirm queue mod events. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) queue = self.te_client.get_queue() self.assertEqual(len(queue), self._no_requests) # Pop a few commands from the queue, confirm events. self._queue_mod_evts = [] self._no_queue_mod_evts = 3 self._done_queue_mod_evts = AsyncResult() cmd_ids = self._requests_sent.keys()[:3] poped = [] for x in cmd_ids: poped.append(self.te_client.pop_queue(x)) self._requests_sent.pop(x) # Try poping with illegal args. This should have no effect poped.append(self.te_client.pop_queue()) poped.append(self.te_client.pop_queue('bogus id')) poped = [x for x in poped if x != None] self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) queue = self.te_client.get_queue() self.assertEqual(len(poped), 3) self.assertEqual(len(queue), self._no_requests - 3) # Turn on the link and verify that only the remaining commands # get processed. self._no_telem_evts = 2 self._no_requests = self._no_requests - 3 self._no_cmd_tx_evts = self._no_requests self.on_link_up() self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) pending = self.te_client.get_pending() self.assertEqual(len(pending), 0) def test_repeated_clear_pop(self): """ test_repeated_clear_pop Test endpoint queue pop manipulators. """ # Set up for events expected. self._no_queue_mod_evts = self._no_requests for i in range(3): self._queue_mod_evts = [] self._no_queue_mod_evts = self._no_requests self._done_queue_mod_evts = AsyncResult() # Queue commands. self._requests_sent = {} for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Confirm queue mod events. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) # Confirm get queue with no id. queue = self.te_client.get_queue() self.assertEqual(len(queue), self._no_requests) # Reset queue mod expected events. self._queue_mod_evts = [] self._no_queue_mod_evts = 1 self._done_queue_mod_evts = AsyncResult() # Clear queue with no id. poped = self.te_client.clear_queue() # Confirm queue mod event and mods. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) queue = self.te_client.get_queue() self.assertEqual(len(poped), self._no_requests) self.assertEqual(len(queue), 0) self._queue_mod_evts = [] self._no_queue_mod_evts = self._no_requests self._done_queue_mod_evts = AsyncResult() # Queue commands. self._requests_sent = {} for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Confirm queue mod events. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) # Confirm get queue with no id. queue = self.te_client.get_queue() self.assertEqual(len(queue), self._no_requests) # Pop a few commands from the queue, confirm events. self._queue_mod_evts = [] self._no_queue_mod_evts = 3 self._done_queue_mod_evts = AsyncResult() cmd_ids = self._requests_sent.keys()[:3] poped = [] for x in cmd_ids: poped.append(self.te_client.pop_queue(x)) self._requests_sent.pop(x) self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) queue = self.te_client.get_queue() self.assertEqual(len(poped), 3) self.assertEqual(len(queue), self._no_requests - 3) self._no_telem_evts = 2 self._no_requests = self._no_requests - 3 self._no_cmd_tx_evts = self._no_requests self.on_link_up() self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) pending = self.te_client.get_pending() self.assertEqual(len(pending), 0) def test_get_pending(self): """ test_process_queued Test forwarding of queued commands upon link up. """ self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._no_telem_evts = 2 for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd self.on_link_up() self._no_requests = 3 self._done_evt.get(timeout=CFG.endpoint.receive.timeout) pending = self.te_client.get_pending(resource_id=self._resource_id) for x in pending: self.assertIn(x.command_id, self._requests_sent.keys()) self._no_requests = 10 self._done_evt = AsyncResult() self._done_evt.get(timeout=CFG.endpoint.receive.timeout) pending = self.te_client.get_pending() self.assertEqual(len(pending), 0) self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) #@unittest.skip('Wait for verification of resource registry use.') def test_persistent_queue(self): """ test_persistent_queue Test ability of endpoint to restore a persistent queue, survive reboot, etc. """ self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._no_telem_evts = 2 for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Confirm queue mod events. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) # Confirm get queue with no id. queue = self.te_client.get_queue() self.assertEqual(len(queue), self._no_requests) # Stop and restart the endpoint process. # Verify the queue is restored. self._container_client.terminate_process(self._te_pid) # Create agent config. endpoint_config = { 'other_host': self._other_host, 'other_port': self._other_port, 'this_port': 0, 'xs_name': self._xs_name, 'platform_resource_id': self._platform_resource_id, 'process': { 'listen_name': self._listen_name } } # Spawn the terrestrial enpoint process. log.debug('Spawning terrestrial endpoint process.') self._te_pid = self._container_client.spawn_process( name='remote_endpoint_1', module='ion.services.sa.tcaa.terrestrial_endpoint', cls='TerrestrialEndpoint', config=endpoint_config) log.debug('Endpoint pid=%s.', str(self._te_pid)) # Create an endpoint client. # The to_name may be either the process pid or # the listen_name, which for this remote bridge # is svc_name + remote_name as above. self.te_client = TerrestrialEndpointClient(process=FakeProcess(), to_name=self._listen_name) log.debug('Got te client %s.', str(self.te_client)) # Remember the terrestrial port. self._this_port = self.te_client.get_port() # Confirm we restored the queue with the previous commands. queue = self.te_client.get_queue() self.assertEqual(len(queue), self._no_requests) self.on_link_up() self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) pending = self.te_client.get_pending() self.assertEqual(len(pending), 0) self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys())
class TestRemoteClient(IonIntegrationTestCase): """ Test cases for 2CAA remote clients. """ def setUp(self): """ Setup the test parameters. Start the container and retrieve client. Start the endpoints and resource agent. Start publisher and subscribers. """ ################################################################### # Internal parameters and container. ################################################################### # Internal parameters. self._terrestrial_platform_id = 'terrestrial_id' self._remote_platform_id = 'remote_id' self._resource_id = 'fake_id' self._xs_name = 'remote1' self._terrestrial_svc_name = 'terrestrial_endpoint' self._terrestrial_listen_name = self._terrestrial_svc_name + self._xs_name self._remote_svc_name = 'remote_endpoint' self._remote_listen_name = self._remote_svc_name + self._xs_name self._remote_port = 0 self._terrestrial_port = 0 self._te_client = None self._re_client = None self._remote_pid = None self._terrestrial_pid = None # Async test results. self._no_requests = 10 self._no_telem_evts = 2 self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._telem_evts = [] self._queue_mod_evts = [] self._cmd_tx_evts = [] self._results_recv = {} self._requests_sent = {} self._done_telem_evt = AsyncResult() self._done_queue_mod_evt = AsyncResult() self._done_cmd_tx_evt = AsyncResult() self._done_cmd_evt = AsyncResult() # Start container. log.debug('Staring capability container.') self._start_container() # Bring up services in a deploy file (no need to message). log.info('Staring deploy services.') self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Create a container client. log.debug('Creating container client.') self._container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) ################################################################### # Start endpoints, agent. ################################################################### self._start_terrestrial() self._start_remote() self._start_agent() ################################################################### # Assign client ports. # This is primarily for test purposes as the IP config in # deployment will be fixed in advance. ################################################################### self.te_client.set_client_port(self._remote_port) check_port = self.te_client.get_client_port() log.debug('Terrestrial client port is: %i', check_port) self.re_client.set_client_port(self._terrestrial_port) check_port = self.re_client.get_client_port() log.debug('Remote client port is: %i', check_port) ################################################################### # Start the event publisher and subscribers. # Used to send fake agent telemetry publications to the endpoints, # and to receive endpoint publications. ################################################################### self._event_publisher = EventPublisher() # Start the event subscriber for remote namespace platform events. # This event could be changed to RemoteNamespaceEvent. self._event_subscriber = EventSubscriber(event_type='PlatformEvent', callback=self.consume_event, origin=self._xs_name) self._event_subscriber.start() self._event_subscriber._ready_event.wait( timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._event_subscriber.stop) # Start the result subscriber for remote resource events. self._resource_result_subscriber = EventSubscriber( event_type='RemoteCommandResult', origin=IA_RESOURCE_ID, callback=self.consume_event) self._resource_result_subscriber.start() self._resource_result_subscriber._ready_event.wait( timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._resource_result_subscriber.stop) # Start the result subscriber for remote service events. self._service_result_subscriber = EventSubscriber( event_type='RemoteCommandResult', origin='resource_registry' + 'remote1', callback=self.consume_event) self._service_result_subscriber.start() self._service_result_subscriber._ready_event.wait( timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._service_result_subscriber.stop) # Start the result subscriber for fake resource results. self._fake_result_subscriber = EventSubscriber( event_type='RemoteCommandResult', origin=self._resource_id, callback=self.consume_event) self._fake_result_subscriber.start() self._fake_result_subscriber._ready_event.wait( timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._fake_result_subscriber.stop) ################################################################### # Start/stop helpers. ################################################################### def _start_agent(self): """ Start an instrument agent and client. """ log.info('Creating driver integration test support:') log.info('driver module: %s', DRV_MOD) log.info('driver class: %s', DRV_CLS) log.info('device address: %s', DEV_ADDR) log.info('device port: %s', DEV_PORT) log.info('log delimiter: %s', DELIM) log.info('work dir: %s', WORK_DIR) self._support = DriverIntegrationTestSupport(DRV_MOD, DRV_CLS, DEV_ADDR, DEV_PORT, DATA_PORT, CMD_PORT, PA_BINARY, DELIM, WORK_DIR) # Start port agent, add stop to cleanup. port = self._support.start_pagent() log.info('Port agent started at port %i', port) # Configure driver to use port agent port number. DVR_CONFIG['comms_config'] = {'addr': 'localhost', 'port': port} self.addCleanup(self._support.stop_pagent) # Create agent config. agent_config = { 'driver_config': DVR_CONFIG, 'stream_config': {}, 'agent': { 'resource_id': IA_RESOURCE_ID }, 'test_mode': True } # Start instrument agent. log.debug("Starting IA.") container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) ia_pid = container_client.spawn_process(name=IA_NAME, module=IA_MOD, cls=IA_CLS, config=agent_config) log.info('Agent pid=%s.', str(ia_pid)) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess()) log.info('Got ia client %s.', str(self._ia_client)) def _start_terrestrial(self): """ Start up the terrestrial endpoint. """ # Create terrestrial config. terrestrial_endpoint_config = { 'other_host': 'localhost', 'other_port': self._remote_port, 'this_port': self._terrestrial_port, 'platform_resource_id': self._terrestrial_platform_id, 'xs_name': self._xs_name, 'process': { 'listen_name': self._terrestrial_listen_name } } # Spawn the terrestrial enpoint process. log.debug('Spawning terrestrial endpoint process.') self._terrestrial_pid = self._container_client.spawn_process( name=self._terrestrial_listen_name, module='ion.services.sa.tcaa.terrestrial_endpoint', cls='TerrestrialEndpoint', config=terrestrial_endpoint_config) log.debug('Terrestrial endpoint pid=%s.', str(self._terrestrial_pid)) # Create a terrestrial client. self.te_client = TerrestrialEndpointClient( process=FakeProcess(), to_name=self._terrestrial_listen_name) log.debug('Got te client %s.', str(self.te_client)) self._terrestrial_port = self.te_client.get_port() log.debug('Terrestrial port is: %i', self._terrestrial_port) def _start_remote(self): """ Start up the remote endpoint. """ # Create agent config. remote_endpoint_config = { 'other_host': 'localhost', 'other_port': self._terrestrial_port, 'this_port': self._remote_port, 'platform_resource_id': self._remote_platform_id, 'xs_name': self._xs_name, 'process': { 'listen_name': self._remote_listen_name } } # Spawn the remote enpoint process. log.debug('Spawning remote endpoint process.') self._remote_pid = self._container_client.spawn_process( name=self._remote_listen_name, module='ion.services.sa.tcaa.remote_endpoint', cls='RemoteEndpoint', config=remote_endpoint_config) log.debug('Remote endpoint pid=%s.', str(self._remote_pid)) # Create an endpoint client. self.re_client = RemoteEndpointClient(process=FakeProcess(), to_name=self._remote_listen_name) log.debug('Got re client %s.', str(self.re_client)) # Remember the remote port. self._remote_port = self.re_client.get_port() log.debug('The remote port is: %i.', self._remote_port) ################################################################### # Telemetry publications to start/top endpoint. # (Normally be published by appropriate platform agents.) ################################################################### def terrestrial_link_up(self): """ Publish telemetry available to the terrestrial endpoint. """ # Publish a link up event to be caught by the terrestrial endpoint. log.debug('Publishing terrestrial telemetry available event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin=self._terrestrial_platform_id, status=TelemetryStatusType.AVAILABLE) def terrestrial_link_down(self): """ Publish telemetry unavailable to the terrestrial endpoint. """ # Publish a link up event to be caught by the terrestrial endpoint. log.debug('Publishing terrestrial telemetry unavailable event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin=self._terrestrial_platform_id, status=TelemetryStatusType.UNAVAILABLE) def remote_link_up(self): """ Publish telemetry available to the remote endpoint. """ # Publish a link up event to be caught by the remote endpoint. log.debug('Publishing remote telemetry available event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin=self._remote_platform_id, status=TelemetryStatusType.AVAILABLE) def remote_link_down(self): """ Publish telemetry unavailable to the remote endpoint. """ # Publish a link down event to be caught by the remote endpoint. log.debug('Publishing remote telemetry unavailable event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin=self._remote_platform_id, status=TelemetryStatusType.UNAVAILABLE) def consume_event(self, evt, *args, **kwargs): """ Test callback for events. """ log.debug('Test got event: %s, args: %s, kwargs: %s', str(evt), str(args), str(kwargs)) if evt.type_ == 'PublicPlatformTelemetryEvent': self._telem_evts.append(evt) if self._no_telem_evts > 0 and self._no_telem_evts == len( self._telem_evts): self._done_telem_evt.set() elif evt.type_ == 'RemoteQueueModifiedEvent': self._queue_mod_evts.append(evt) if self._no_queue_mod_evts > 0 and self._no_queue_mod_evts == len( self._queue_mod_evts): self._done_queue_mod_evt.set() elif evt.type_ == 'RemoteCommandTransmittedEvent': self._cmd_tx_evts.append(evt) if self._no_cmd_tx_evts > 0 and self._no_cmd_tx_evts == len( self._cmd_tx_evts): self._done_cmd_tx_evt.set() elif evt.type_ == 'RemoteCommandResult': cmd = evt.command self._results_recv[cmd.command_id] = cmd if len(self._results_recv) == self._no_requests: self._done_cmd_evt.set() ################################################################### # Misc helpers. ################################################################### def make_fake_command(self, no): """ Build a fake command for use in tests. """ cmdstr = 'fake_cmd_%i' % no cmd = IonObject('RemoteCommand', resource_id='fake_id', command=cmdstr, args=['arg1', 23], kwargs={'kwargs1': 'someval'}) return cmd ################################################################### # Tests. ################################################################### def test_resource_client_online(self): """ test_recourse_client_online Test the client transparently forwards commands to the remote resource while link is up. """ # Publish link up events. self.terrestrial_link_up() self.remote_link_up() remote_client = RemoteClient(iface=IResourceAgent, xs_name=self._xs_name, resource_id='fake_id', process=FakeProcess()) # Queue up a series of fake commands to be handled by the remote side. for i in range(self._no_requests): cmd = remote_client.ping_agent() self._requests_sent[cmd.command_id] = cmd # Block on queue mod events. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) # Block on command transmissions and results. self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Publish link down events. self.terrestrial_link_down() self.remote_link_down() # Block on terrestrial public telemetry events. self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) #@unittest.skip('For some reason this bastard wont run on the builder.') def test_resource_client_blocking(self): """ test_resource_client_blocking Test the client can block on remote resource command results. """ # Publish link up events. self.terrestrial_link_up() self.remote_link_up() remote_client = RemoteClient(iface=IResourceAgent, xs_name=self._xs_name, resource_id=IA_RESOURCE_ID, process=FakeProcess()) # Queue up a series of fake commands to be handled by the remote side. """ {'time_completed': 1350421095.804607, 'resource_id': '123xyz', 'time_queued': 1350421095.623531, 'args': [], 'type_': 'RemoteCommand', 'command': 'ping_agent', 'result': 'ping from InstrumentAgent (name=Agent007,id=Edwards-MacBook-Pro_local_10126.35,type=agent), time: 1350421095757', 'kwargs': {}, 'svc_name': '', 'command_id': '76be11b4-a22c-49de-89cd-4e019463d7c9'} """ for i in range(self._no_requests): cmd = remote_client.ping_agent( remote_timeout=CFG.endpoint.receive.timeout) self.assertEqual(cmd.resource_id, IA_RESOURCE_ID) self.assertEqual(cmd.command, 'ping_agent') self.assertIn('ping from InstrumentAgent', cmd.result) # Publish link down events. self.terrestrial_link_down() self.remote_link_down() # Block on terrestrial public telemetry events. self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout) #@unittest.skip('For some reason this bastard wont run on the builder.') def test_service_client_blocking(self): """ test_service_client_blocking Test the client can command remote services and block on their results. """ # Publish link up events. self.terrestrial_link_up() self.remote_link_up() # Create remote client for the resource registry. remote_client = RemoteClient(iface=IResourceRegistryService, xs_name=self._xs_name, svc_name='resource_registry', process=FakeProcess()) # Create user object. obj = IonObject("UserInfo", name="some_name") result = remote_client.create( obj, remote_timeout=CFG.endpoint.receive.timeout) # Returns obj_id, obj_rev. obj_id, obj_rev = result.result # Confirm the results are valid. #Result is a tuple of strings. #{'result': ['ad183ff26bae4f329ddd85fd69d160a9', #'1-00a308c45fff459c7cda1db9a7314de6'], #'command_id': 'cc2ae00d-40b0-47d2-af61-8ffb87f1aca2'} self.assertIsInstance(obj_id, str) self.assertNotEqual(obj_id, '') self.assertIsInstance(obj_rev, str) self.assertNotEqual(obj_rev, '') # Read user object. result = remote_client.read( obj_id, remote_timeout=CFG.endpoint.receive.timeout) # Returns read_obj. read_obj = result.result # Confirm the results are valid. #Result is a user info object with the name set. #{'lcstate': 'DEPLOYED_AVAILABLE', #'_rev': '1-851f067bac3c34b2238c0188b3340d0f', #'description': '', #'ts_updated': '1349213207638', #'type_': 'UserInfo', #'contact': <interface.objects.ContactInformation object at 0x10d7df590>, #'_id': '27832d93f4cd4535a75ac75c06e00a7e', #'ts_created': '1349213207638', #'variables': [{'name': '', 'value': ''}], #'name': 'some_name'} self.assertIsInstance(read_obj, UserInfo) self.assertEquals(read_obj.name, 'some_name') # Update user object. read_obj.name = 'some_other_name' result = remote_client.update( read_obj, remote_timeout=CFG.endpoint.receive.timeout) # Returns nothing. # Read user object. result = remote_client.read( obj_id, remote_timeout=CFG.endpoint.receive.timeout) # Returns read_obj. read_obj = result.result # Confirm results are valid. self.assertIsInstance(read_obj, UserInfo) self.assertEquals(read_obj.name, 'some_other_name') # Delete user object. result = remote_client.delete( obj_id, remote_timeout=CFG.endpoint.receive.timeout) # Returns nothing. # Publish link down events. self.terrestrial_link_down() self.remote_link_down() # Block on terrestrial public telemetry events. self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout) def test_queue_manipulators(self): """ test_queue_manipulators Test ability to instpect and manipulate the command queue corresponding to this resource or service. """ remote_client = RemoteClient(iface=IResourceAgent, xs_name=self._xs_name, resource_id='fake_id', process=FakeProcess()) # Queue up a series of fake commands to be handled by the remote side. for i in range(self._no_requests): cmd = remote_client.ping_agent(link=False) self._requests_sent[cmd.command_id] = cmd # Block on queue mod events. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) queue = remote_client.get_queue() self.assertEqual(len(queue), self._no_requests) popped = remote_client.clear_queue() self.assertEqual(len(popped), self._no_requests) self._requests_sent = {} # Queue up a series of fake commands to be handled by the remote side. for i in range(self._no_requests): cmd = remote_client.ping_agent(link=False) self._requests_sent[cmd.command_id] = cmd # Pop the last three commands. cmd_ids = self._requests_sent.keys()[:3] poped = [] for x in cmd_ids: poped.append(remote_client.pop_queue(x)) self._requests_sent.pop(x) queue = remote_client.get_queue() self.assertEqual(len(queue), self._no_requests - 3) self.assertEqual(len(poped), 3) self._no_requests = self._no_requests - 3 self._no_cmd_tx_evts = self._no_requests # Publish link up events. self.terrestrial_link_up() self.remote_link_up() # Block on command transmissions and results. self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) pending = remote_client.get_pending() for x in pending: self.assertIn(x.command_id, self._requests_sent.keys()) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Publish link down events. self.terrestrial_link_down() self.remote_link_down() # Block on terrestrial public telemetry events. self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) def test_errors(self): """ test_errors Test various error conditions. """ # Constructed without a xs name. with self.assertRaises(ConfigNotFound): remote_client = RemoteClient(iface=IResourceAgent, resource_id=IA_RESOURCE_ID, process=FakeProcess()) # Constructed without an interface. with self.assertRaises(ConfigNotFound): remote_client = RemoteClient(xs_name=self._xs_name, resource_id=IA_RESOURCE_ID, process=FakeProcess()) # Construct with invalid interface. with self.assertRaises(ConfigNotFound): remote_client = RemoteClient('Bogus_interface', xs_name=self._xs_name, resource_id=IA_RESOURCE_ID, process=FakeProcess()) # Construct with no resource or service specified. with self.assertRaises(ConfigNotFound): remote_client = RemoteClient(iface=IResourceAgent, xs_name=self._xs_name, process=FakeProcess()) # Construct with both resource and service specified. with self.assertRaises(ConfigNotFound): remote_client = RemoteClient(iface=IResourceAgent, xs_name=self._xs_name, resource_id=IA_RESOURCE_ID, svc_name='resource_registry', process=FakeProcess()) # Create a valid resource client. remote_client = RemoteClient(iface=IResourceAgent, xs_name=self._xs_name, resource_id=IA_RESOURCE_ID, process=FakeProcess()) # Send a command while link is down. with self.assertRaises(Conflict): result = remote_client.ping_agent() # Test port manipulators refused by remote proxy client. with self.assertRaises(BadRequest): remote_client.get_port() with self.assertRaises(BadRequest): remote_client.set_client_port() with self.assertRaises(BadRequest): remote_client.get_client_port() def test_interfaces(self): """ test_interfaces Test that the client declare the correct interfaces. """ remote_client = RemoteClient(iface=IResourceAgent, xs_name=self._xs_name, resource_id='fake_id', process=FakeProcess()) interfaces = providedBy(remote_client) self.assertIn(IResourceAgent, interfaces) self.assertIn(ITerrestrialEndpoint, interfaces)
class Test2CAA(IonIntegrationTestCase): """ Test cases for 2CAA terrestrial endpoint. """ def setUp(self): """ """ ################################################################### # Internal parameters and container. ################################################################### # Internal parameters. self._terrestrial_platform_id = 'terrestrial_id' self._remote_platform_id = 'remote_id' self._resource_id = 'fake_id' self._xs_name = 'remote1' self._terrestrial_svc_name = 'terrestrial_endpoint' self._terrestrial_listen_name = self._terrestrial_svc_name + self._xs_name self._remote_svc_name = 'remote_endpoint' self._remote_listen_name = self._remote_svc_name + self._xs_name self._remote_port = 0 self._terrestrial_port = 0 self._te_client = None self._re_client = None self._remote_pid = None self._terrestrial_pid = None # Async test results. self._no_requests = 10 self._no_telem_evts = 2 self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._telem_evts = [] self._queue_mod_evts = [] self._cmd_tx_evts = [] self._results_recv = {} self._requests_sent = {} self._done_telem_evt = AsyncResult() self._done_queue_mod_evt = AsyncResult() self._done_cmd_tx_evt = AsyncResult() self._done_cmd_evt = AsyncResult() # Start container. log.debug('Staring capability container.') self._start_container() # Bring up services in a deploy file (no need to message). log.info('Staring deploy services.') self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Create a container client. log.debug('Creating container client.') self._container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) ################################################################### # Start endpoints, agent. ################################################################### self._start_terrestrial() self._start_remote() self._start_agent() ################################################################### # Assign client ports. # This is primarily for test purposes as the IP config in # deployment will be fixed in advance. ################################################################### self.te_client.set_client_port(self._remote_port) check_port = self.te_client.get_client_port() log.debug('Terrestrial client port is: %i', check_port) self.re_client.set_client_port(self._terrestrial_port) check_port = self.re_client.get_client_port() log.debug('Remote client port is: %i', check_port) ################################################################### # Start the event publisher and subscribers. # Used to send fake agent telemetry publications to the endpoints, # and to receive endpoint publications. ################################################################### self._event_publisher = EventPublisher() # Start the event subscriber for remote namespace platform events. # This event could be changed to RemoteNamespaceEvent. self._event_subscriber = EventSubscriber( event_type='PlatformEvent', callback=self.consume_event, origin=self._xs_name) self._event_subscriber.start() self._event_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._event_subscriber.stop) # Start the result subscriber for remote resource events. self._resource_result_subscriber = EventSubscriber( event_type='RemoteCommandResult', origin=IA_RESOURCE_ID, callback=self.consume_event) self._resource_result_subscriber.start() self._resource_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._resource_result_subscriber.stop) # Start the result subscriber for remote service events. self._service_result_subscriber = EventSubscriber( event_type='RemoteCommandResult', origin='resource_registry' + 'remote1', callback=self.consume_event) self._service_result_subscriber.start() self._service_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._service_result_subscriber.stop) # Start the result subscriber for fake resource results. self._fake_result_subscriber = EventSubscriber( event_type='RemoteCommandResult', origin=self._resource_id, callback=self.consume_event) self._fake_result_subscriber.start() self._fake_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._fake_result_subscriber.stop) ################################################################### # Start/stop helpers. ################################################################### def _start_agent(self): """ Start an instrument agent and client. """ log.info('Creating driver integration test support:') log.info('driver module: %s', DRV_MOD) log.info('driver class: %s', DRV_CLS) log.info('device address: %s', DEV_ADDR) log.info('device port: %s', DEV_PORT) log.info('log delimiter: %s', DELIM) log.info('work dir: %s', WORK_DIR) self._support = DriverIntegrationTestSupport(DRV_MOD, DRV_CLS, DEV_ADDR, DEV_PORT, DATA_PORT, CMD_PORT, PA_BINARY, DELIM, WORK_DIR) # Start port agent, add stop to cleanup. port = self._support.start_pagent() log.info('Port agent started at port %i',port) # Configure driver to use port agent port number. DVR_CONFIG['comms_config'] = { 'addr' : 'localhost', 'port' : port } self.addCleanup(self._support.stop_pagent) # Create agent config. agent_config = { 'driver_config' : DVR_CONFIG, 'stream_config' : {}, 'agent' : {'resource_id': IA_RESOURCE_ID}, 'test_mode' : True } # Start instrument agent. log.debug("Starting IA.") container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) ia_pid = container_client.spawn_process(name=IA_NAME, module=IA_MOD, cls=IA_CLS, config=agent_config) log.info('Agent pid=%s.', str(ia_pid)) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess()) log.info('Got ia client %s.', str(self._ia_client)) def _start_terrestrial(self): """ """ # Create terrestrial config. terrestrial_endpoint_config = { 'other_host' : 'localhost', 'other_port' : self._remote_port, 'this_port' : self._terrestrial_port, 'platform_resource_id' : self._terrestrial_platform_id, 'xs_name' : self._xs_name, 'process' : { 'listen_name' : self._terrestrial_listen_name } } # Spawn the terrestrial enpoint process. log.debug('Spawning terrestrial endpoint process.') self._terrestrial_pid = self._container_client.spawn_process( name=self._terrestrial_listen_name, module='ion.services.sa.tcaa.terrestrial_endpoint', cls='TerrestrialEndpoint', config=terrestrial_endpoint_config) log.debug('Terrestrial endpoint pid=%s.', str(self._terrestrial_pid)) # Create a terrestrial client. self.te_client = TerrestrialEndpointClient( process=FakeProcess(), to_name=self._terrestrial_listen_name) log.debug('Got te client %s.', str(self.te_client)) self._terrestrial_port = self.te_client.get_port() log.debug('Terrestrial port is: %i', self._terrestrial_port) def _start_remote(self): """ """ # Create agent config. remote_endpoint_config = { 'other_host' : 'localhost', 'other_port' : self._terrestrial_port, 'this_port' : self._remote_port, 'platform_resource_id' : self._remote_platform_id, 'xs_name' : self._xs_name, 'process' : { 'listen_name' : self._remote_listen_name } } # Spawn the remote enpoint process. log.debug('Spawning remote endpoint process.') self._remote_pid = self._container_client.spawn_process( name=self._remote_listen_name, module='ion.services.sa.tcaa.remote_endpoint', cls='RemoteEndpoint', config=remote_endpoint_config) log.debug('Remote endpoint pid=%s.', str(self._remote_pid)) # Create an endpoint client. self.re_client = RemoteEndpointClient( process=FakeProcess(), to_name=self._remote_listen_name) log.debug('Got re client %s.', str(self.re_client)) # Remember the remote port. self._remote_port = self.re_client.get_port() log.debug('The remote port is: %i.', self._remote_port) ################################################################### # Telemetry publications to start/top endpoint. # (Normally be published by appropriate platform agents.) ################################################################### def terrestrial_link_up(self): """ Publish telemetry available to the terrestrial endpoint. """ # Publish a link up event to be caught by the terrestrial endpoint. log.debug('Publishing terrestrial telemetry available event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin=self._terrestrial_platform_id, status = TelemetryStatusType.AVAILABLE) def terrestrial_link_down(self): """ Publish telemetry unavailable to the terrestrial endpoint. """ # Publish a link up event to be caught by the terrestrial endpoint. log.debug('Publishing terrestrial telemetry unavailable event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin=self._terrestrial_platform_id, status = TelemetryStatusType.UNAVAILABLE) def remote_link_up(self): """ Publish telemetry available to the remote endpoint. """ # Publish a link up event to be caught by the remote endpoint. log.debug('Publishing remote telemetry available event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin=self._remote_platform_id, status = TelemetryStatusType.AVAILABLE) def remote_link_down(self): """ Publish telemetry unavailable to the remote endpoint. """ # Publish a link down event to be caught by the remote endpoint. log.debug('Publishing remote telemetry unavailable event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin=self._remote_platform_id, status = TelemetryStatusType.UNAVAILABLE) def consume_event(self, evt, *args, **kwargs): """ Test callback for events. """ log.debug('Test got event: %s, args: %s, kwargs: %s', str(evt), str(args), str(kwargs)) if evt.type_ == 'PublicPlatformTelemetryEvent': self._telem_evts.append(evt) if self._no_telem_evts > 0 and self._no_telem_evts == len(self._telem_evts): self._done_telem_evt.set() elif evt.type_ == 'RemoteQueueModifiedEvent': self._queue_mod_evts.append(evt) if self._no_queue_mod_evts > 0 and self._no_queue_mod_evts == len(self._queue_mod_evts): self._done_queue_mod_evt.set() elif evt.type_ == 'RemoteCommandTransmittedEvent': self._cmd_tx_evts.append(evt) if self._no_cmd_tx_evts > 0 and self._no_cmd_tx_evts == len(self._cmd_tx_evts): self._done_cmd_tx_evt.set() elif evt.type_ == 'RemoteCommandResult': cmd = evt.command self._results_recv[cmd.command_id] = cmd if len(self._results_recv) == self._no_requests: self._done_cmd_evt.set() ################################################################### # Misc helpers. ################################################################### def make_fake_command(self, no): """ Build a fake command for use in tests. """ cmdstr = 'fake_cmd_%i' % no cmd = IonObject('RemoteCommand', resource_id='fake_id', command=cmdstr, args=['arg1', 23], kwargs={'kwargs1':'someval'}) return cmd ################################################################### # Tests. ################################################################### def test_queued_fake(self): """ test_queued_fake Test fake resource commands queued prior to linkup. """ # Queue up a series of fake commands to be handled by the remote side. for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Block on queue mod events. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) # Publish link up events. self.terrestrial_link_up() self.remote_link_up() # Block on command transmissions and results. self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Publish link down events. self.terrestrial_link_down() self.remote_link_down() # Block on terrestrial public telemetry events. self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) # The following error occurs when queue persistence is enabled. # Need to verify correct solution to enable persistent queues. """ 2012-11-06 14:27:13,517 INFO pyon.container.procs ProcManager.terminate_process: org_management -> pid=Edwards-MacBook-Pro_local_8975.8 Traceback (most recent call last): File "/Users/edward/Documents/Dev/code/coi-services/eggs/gevent-0.13.7-py2.7-macosx-10.5-intel.egg/gevent/greenlet.py", line 390, in run result = self._run(*self.args, **self.kwargs) File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/remote_endpoint.py", line 97, in command_loop self._callback(cmd_result) File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/remote_endpoint.py", line 284, in _result_complete self._client.enqueue(result) AttributeError: 'NoneType' object has no attribute 'enqueue' <Greenlet at 0x1059ca9b0: command_loop> failed with AttributeError 2012-11-06 14:27:13,586 INFO pyon.container.procs ProcManager.terminate_process: exchange_management -> pid=Edwards-MacBook-Pro_local_8975.7 2012-11-06 14:27:13,665 INFO pyon.container.procs ProcManager.terminate_process: policy_management -> pid=Edwards-MacBook-Pro_local_8975.6 2012-11-06 14:27:13,739 INFO pyon.container.procs ProcManager.terminate_process: identity_management -> pid=Edwards-MacBook-Pro_local_8975.5 2012-11-06 14:27:13,807 INFO pyon.container.procs ProcManager.terminate_process: directory -> pid=Edwards-MacBook-Pro_local_8975.4 2012-11-06 14:27:13,874 INFO pyon.container.procs ProcManager.terminate_process: resource_registry -> pid=Edwards-MacBook-Pro_local_8975.3 2012-11-06 14:27:13,941 INFO pyon.container.procs ProcManager.terminate_process: event_persister -> pid=Edwards-MacBook-Pro_local_8975.1 2012-11-06 14:27:13,945 INFO pyon.event.event EventSubscriber stopped. Event pattern=# 2012-11-06 14:27:14,124 INFO pyon.datastore.couchdb.couchdb_standalone Connecting to CouchDB server: http://localhost:5984 2012-11-06 14:27:14,399 INFO pyon.datastore.couchdb.couchdb_standalone Closing connection to CouchDB ====================================================================== ERROR: test_process_online ---------------------------------------------------------------------- Traceback (most recent call last): File "/Users/edward/Documents/Dev/code/coi-services/eggs/mock-0.8.0-py2.7.egg/mock.py", line 1605, in _inner return f(*args, **kw) File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/test/test_2caa.py", line 492, in test_process_online cmd = self.te_client.enqueue_command(cmd) File "/Users/edward/Documents/Dev/code/coi-services/interface/services/sa/iterrestrial_endpoint.py", line 188, in enqueue_command return self.request(IonObject('terrestrial_endpoint_enqueue_command_in', **{'command': command or None,'link': link}), op='enqueue_command', headers=headers, timeout=timeout) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 1012, in request return RequestResponseClient.request(self, msg, headers=headers, timeout=timeout) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 822, in request retval, headers = e.send(msg, headers=headers, timeout=timeout) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 310, in send result_data, result_headers = c.send(self.conv_type.server_role, msg, headers, **kwargs) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 126, in send return self._invite_and_send(to_role, msg, headers, **kwargs) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 145, in _invite_and_send return self._send(to_role, to_role_name, msg, header, **kwargs) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 169, in _send return self._end_point_unit._message_send(msg, headers, **kwargs) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 289, in _message_send return ProcessRPCRequestEndpointUnit.send(self, msg, headers, **kwargs) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 134, in send return self._send(_msg, _header, **kwargs) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 880, in _send raise ex Conflict: 409 - Object not based on most current version """ def test_process_online(self): """ test_process_online Test fake resource commands queued while link is up. """ # Publish link up events. self.terrestrial_link_up() self.remote_link_up() # Queue up a series of fake commands to be handled by the remote side. for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Block on queue mods, command transmissions and results. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Publish link down events. self.terrestrial_link_down() self.remote_link_down() # Block on terrestrial public telemetry events. self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) def test_remote_late(self): """ test_remote_late Test fake resource commands queued prior to linkup. Delay remote side linkup substantially to test terrestrial behavior when remote server not initially available. """ for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Block on queue mod events. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) # Actually stop the remote process, since the server is # available even if it hasn't linked up yet and the test is running # in the same container. The test will remember the appropriate # remote port numbers. self._container_client.terminate_process(self._remote_pid) self.terrestrial_link_up() gevent.sleep(10) # Restart remote side and publish remote link up. self._start_remote() self.remote_link_up() # Block for transmission and result events. self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Publish link down events. self.terrestrial_link_down() self.remote_link_down() self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) def test_resource_commands(self): """ test_resource_commands """ # Set up to verify the two commands queued. self._no_requests = 2 self._no_telem_evts = 2 self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests # Use IA client to verify IA. state = self._ia_client.get_agent_state() log.debug('Agent state is: %s', state) self.assertEqual(state, ResourceAgentState.UNINITIALIZED) retval = self._ia_client.ping_agent() log.debug('Agent ping is: %s', str(retval)) self.assertIn('ping from InstrumentAgent', retval) # Create and enqueue commands. state_cmd = IonObject('RemoteCommand', resource_id=IA_RESOURCE_ID, svc_name='', command='get_agent_state', args=[], kwargs={}) state_cmd = self.te_client.enqueue_command(state_cmd) self._requests_sent[state_cmd.command_id] = state_cmd ping_cmd = IonObject('RemoteCommand', resource_id=IA_RESOURCE_ID, svc_name='', command='ping_agent', args=[], kwargs={}) ping_cmd = self.te_client.enqueue_command(ping_cmd) self._requests_sent[ping_cmd.command_id] = ping_cmd # Block on queue mod events. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) # Publish link up events. self.terrestrial_link_up() self.remote_link_up() # Block on command transmissions and results. self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Publish link down events. self.terrestrial_link_down() self.remote_link_down() # Block on terrestrial public telemetry events. self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) self.assertEqual(self._results_recv[state_cmd.command_id].result, ResourceAgentState.UNINITIALIZED) self.assertIn('ping from InstrumentAgent', self._results_recv[ping_cmd.command_id].result) def test_service_command_sequence(self): """ test_service_commands """ # Publish link up events. self.terrestrial_link_up() self.remote_link_up() # Send commands one at a time. # Reset queues and events. self._no_requests = 1 self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._done_queue_mod_evt = AsyncResult() self._done_cmd_tx_evt = AsyncResult() self._done_cmd_evt = AsyncResult() self._queue_mod_evts = [] self._cmd_tx_evts = [] self._requests_sent = {} self._results_recv = {} # Create user object. obj = IonObject("UserInfo", name="some_name") cmd = IonObject('RemoteCommand', resource_id='', svc_name='resource_registry', command='create', args=[obj], kwargs='') cmd = self.te_client.enqueue_command(cmd) # Block for the queue to be modified, the command to be transmitted, # and the result to be received. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Returns obj_id, obj_rev. obj_id, obj_rev = self._results_recv[cmd.command_id].result # Confirm the results are valid. #Result is a tuple of strings. #{'result': ['ad183ff26bae4f329ddd85fd69d160a9', #'1-00a308c45fff459c7cda1db9a7314de6'], #'command_id': 'cc2ae00d-40b0-47d2-af61-8ffb87f1aca2'} self.assertIsInstance(obj_id, str) self.assertNotEqual(obj_id, '') self.assertIsInstance(obj_rev, str) self.assertNotEqual(obj_rev, '') # Send commands one at a time. # Reset queues and events. self._no_requests = 1 self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._done_queue_mod_evt = AsyncResult() self._done_cmd_tx_evt = AsyncResult() self._done_cmd_evt = AsyncResult() self._queue_mod_evts = [] self._cmd_tx_evts = [] self._requests_sent = {} self._results_recv = {} # Create user object. cmd = IonObject('RemoteCommand', resource_id='', svc_name='resource_registry', command='read', args=[obj_id], kwargs='') cmd = self.te_client.enqueue_command(cmd) # Block for the queue to be modified, the command to be transmitted, # and the result to be received. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Returns read_obj. read_obj = self._results_recv[cmd.command_id].result # Confirm the results are valid. #Result is a user info object with the name set. #{'lcstate': 'DEPLOYED_AVAILABLE', #'_rev': '1-851f067bac3c34b2238c0188b3340d0f', #'description': '', #'ts_updated': '1349213207638', #'type_': 'UserInfo', #'contact': <interface.objects.ContactInformation object at 0x10d7df590>, #'_id': '27832d93f4cd4535a75ac75c06e00a7e', #'ts_created': '1349213207638', #'variables': [{'name': '', 'value': ''}], #'name': 'some_name'} self.assertIsInstance(read_obj, UserInfo) self.assertEquals(read_obj.name, 'some_name') # Send commands one at a time. # Reset queues and events. self._no_requests = 1 self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._done_queue_mod_evt = AsyncResult() self._done_cmd_tx_evt = AsyncResult() self._done_cmd_evt = AsyncResult() self._queue_mod_evts = [] self._cmd_tx_evts = [] self._requests_sent = {} self._results_recv = {} # Create user object. read_obj.name = 'some_other_name' cmd = IonObject('RemoteCommand', resource_id='', svc_name='resource_registry', command='update', args=[read_obj], kwargs='') cmd = self.te_client.enqueue_command(cmd) # Block for the queue to be modified, the command to be transmitted, # and the result to be received. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Returns nothing. # Send commands one at a time. # Reset queues and events. self._no_requests = 1 self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._done_queue_mod_evt = AsyncResult() self._done_cmd_tx_evt = AsyncResult() self._done_cmd_evt = AsyncResult() self._queue_mod_evts = [] self._cmd_tx_evts = [] self._requests_sent = {} self._results_recv = {} # Create user object. cmd = IonObject('RemoteCommand', resource_id='', svc_name='resource_registry', command='read', args=[obj_id], kwargs='') cmd = self.te_client.enqueue_command(cmd) # Block for the queue to be modified, the command to be transmitted, # and the result to be received. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Returns read_obj. read_obj = self._results_recv[cmd.command_id].result self.assertIsInstance(read_obj, UserInfo) self.assertEquals(read_obj.name, 'some_other_name') # Send commands one at a time. # Reset queues and events. self._no_requests = 1 self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._done_queue_mod_evt = AsyncResult() self._done_cmd_tx_evt = AsyncResult() self._done_cmd_evt = AsyncResult() self._queue_mod_evts = [] self._cmd_tx_evts = [] self._requests_sent = {} self._results_recv = {} # Create user object. cmd = IonObject('RemoteCommand', resource_id='', svc_name='resource_registry', command='delete', args=[obj_id], kwargs='') cmd = self.te_client.enqueue_command(cmd) # Block for the queue to be modified, the command to be transmitted, # and the result to be received. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Returns nothing. # Publish link down events. self.terrestrial_link_down() self.remote_link_down()
class TestTerrestrialEndpoint(IonIntegrationTestCase): """ Test cases for 2CAA terrestrial endpoint. """ def setUp(self): """ Setup fake remote components. Start remote server. Set internal configuration and test variables. Start container. Start services. Spawn endpoint. Create and start subscribers. """ # Create fake remote client and server. # Add clean up to shut down properly. # Start remote server on a random port. self._remote_server = R3PCServer(self.consume_req, self.remote_server_close) self._remote_client = R3PCClient(self.consume_ack, self.remote_client_close) self.addCleanup(self._remote_server.stop) self.addCleanup(self._remote_client.stop) self._other_port = self._remote_server.start('*', 0) log.debug('Remote server binding to *:%i', self._other_port) # Set internal variables. self._other_host = 'localhost' self._xs_name = 'remote1' self._svc_name = 'terrestrial_endpoint' self._listen_name = self._svc_name + self._xs_name self._platform_resource_id = 'abc123' self._resource_id = 'fake_id' self._rmt_svc_name = 'fake_svc' self._no_requests = 10 self._requests_sent = {} self._results_recv = {} self._workers = [] self._done_evt = AsyncResult() self._queue_mod_evts = [] self._cmd_tx_evts = [] self._telem_evts = [] self._no_telem_evts = 0 self._no_queue_mod_evts = 0 self._no_cmd_tx_evts = 0 self._done_queue_mod_evts = AsyncResult() self._done_telem_evts = AsyncResult() self._done_cmd_tx_evts = AsyncResult() # Start container. log.debug('Staring capability container.') self._start_container() # Bring up services in a deploy file (no need to message) log.info('Staring deploy services.') self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Create a container client. log.debug('Creating container client.') self._container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) # The following spawn config creates the process with the remote # name tagged to the service name. """ listen_name = terrestrial_endpointremote1 2012-10-10 11:34:46,654 DEBUG ion.services.sa.tcaa.terrestrial_endpoint recv name: NP (ion_test_8257ab,terrestrial_endpointremote1,B: terrestrial_endpointremote1) 2012-10-10 11:34:46,654 DEBUG ion.services.sa.tcaa.terrestrial_endpoint startup listener recv name: NP (ion_test_8257ab,terrestrial_endpointremote1,B: terrestrial_endpointremote1) 2012-10-10 11:34:46,654 DEBUG ion.services.sa.tcaa.terrestrial_endpoint startup listener recv name: NP (ion_test_8257ab,Edwards-MacBook-Pro_local_2624.33,B: Edwards-MacBook-Pro_local_2624.33) """ # Create agent config. endpoint_config = { 'other_host' : self._other_host, 'other_port' : self._other_port, 'this_port' : 0, 'xs_name' : self._xs_name, 'platform_resource_id' : self._platform_resource_id, 'process' : { 'listen_name' : self._listen_name } } # Spawn the terrestrial enpoint process. log.debug('Spawning terrestrial endpoint process.') self._te_pid = self._container_client.spawn_process( name='remote_endpoint_1', module='ion.services.sa.tcaa.terrestrial_endpoint', cls='TerrestrialEndpoint', config=endpoint_config) log.debug('Endpoint pid=%s.', str(self._te_pid)) # Create an endpoint client. # The to_name may be either the process pid or # the listen_name, which for this remote bridge # is svc_name + remote_name as above. self.te_client = TerrestrialEndpointClient( process=FakeProcess(), to_name=self._listen_name) log.debug('Got te client %s.', str(self.te_client)) # Remember the terrestrial port. self._this_port = self.te_client.get_port() # Start the event publisher. self._event_publisher = EventPublisher() # Start the event subscriber. self._event_subscriber = EventSubscriber( event_type='PlatformEvent', callback=self.consume_event, origin=self._xs_name) self._event_subscriber.start() self._event_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._event_subscriber.stop) # Start the result subscriber. self._result_subscriber = EventSubscriber( event_type='RemoteCommandResult', origin=self._resource_id, callback=self.consume_event) self._result_subscriber.start() self._result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._result_subscriber.stop) def consume_event(self, evt, *args, **kwargs): """ Test callback for events. """ log.debug('Got event: %s, args: %s, kwargs: %s', str(evt), str(args), str(kwargs)) if evt.type_ == 'PublicPlatformTelemetryEvent': self._telem_evts.append(evt) if self._no_telem_evts > 0 and self._no_telem_evts == len(self._telem_evts): self._done_telem_evts.set() elif evt.type_ == 'RemoteQueueModifiedEvent': self._queue_mod_evts.append(evt) if self._no_queue_mod_evts > 0 and self._no_queue_mod_evts == len(self._queue_mod_evts): self._done_queue_mod_evts.set() elif evt.type_ == 'RemoteCommandTransmittedEvent': self._cmd_tx_evts.append(evt) if self._no_cmd_tx_evts > 0 and self._no_cmd_tx_evts == len(self._cmd_tx_evts): self._done_cmd_tx_evts.set() elif evt.type_ == 'RemoteCommandResult': cmd = evt.command self._results_recv[cmd.command_id] = cmd if len(self._results_recv) == self._no_requests: self._done_evt.set() def on_link_up(self): """ Called by a test to simulate turning the link on. """ log.debug('Remote client connecting to localhost:%i.', self._this_port) self._remote_client.start('localhost', self._this_port) # Publish a link up event to be caught by the endpoint. log.debug('Publishing telemetry event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin = self._platform_resource_id, status = TelemetryStatusType.AVAILABLE) def on_link_down(self): """ Called by a test to simulate turning the link off. """ self._remote_client.stop() # Publish a link down event to be caught by the endpoint. log.debug('Publishing telemetry event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin=self._platform_resource_id, status = TelemetryStatusType.UNAVAILABLE) def consume_req(self, request): """ Remote request callback. Fire a greenlet to do some fake work before returning via the remote client to terrestrial endpoint. """ # Spawn a greenlet to sleep briefly with each request and # then respond with a result through the remote client. log.debug('Remote endpoint got request: %s', str(request)) greenlet = gevent.spawn(self.process_remote_request, request) self._workers.append(greenlet) def consume_ack(self, request): """ Remote ack callback. """ log.debug('Remote endpoint got ack: %s', str(request)) def process_remote_request(self, request): """ Process remote request. Do random amount of fake work and enqueue result for return to terrestrial endpoint. """ worktime = random.uniform(.1,3) gevent.sleep(worktime) result = { 'command_id' : request.command_id, 'result' : 'fake_result' } log.debug('Finished processing request: %s', str(request)) self._remote_client.enqueue(result) def remote_server_close(self): """ Remote server closed callback. """ log.debug('The remote server closed.') def remote_client_close(self): """ Remoe client closed callback. """ log.debug('The remote client closed.') def make_fake_command(self, no): """ Build a fake command for use in tests. """ cmdstr = 'fake_cmd_%i' % no cmd = IonObject('RemoteCommand', resource_id=self._resource_id, command=cmdstr, args=['arg1', 23], kwargs={'kwargs1':'someval'}) return cmd def make_fake_svc_command(self, no): """ Build a fake command for use in tests. """ cmdstr = 'fake_cmd_%i' % no cmd = IonObject('RemoteCommand', svc_name=self._rmt_svc_name, command=cmdstr, args=['arg1', 23], kwargs={'kwargs1':'someval'}) return cmd def test_process_queued(self): """ test_process_queued Test forwarding of queued commands upon link up. """ self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._no_telem_evts = 2 for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) self.on_link_up() self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) pending = self.te_client.get_pending() self.assertEqual(len(pending), 0) self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) def test_process_online(self): """ test_process_online Test forwarding commands when the link is up. """ self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._no_telem_evts = 2 self.on_link_up() for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd gevent.sleep(.2) self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) pending = self.te_client.get_pending() self.assertEqual(len(pending), 0) self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) def test_remote_late(self): """ test_remote_late Test simulates behavior when the remote side is initially unavailable. """ self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._no_telem_evts = 2 self.on_link_up() gevent.sleep(2) self._remote_server.stop() self._remote_client.stop() for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) gevent.sleep(3) self._remote_client.start('localhost', self._this_port) self._remote_server.start('*', self._other_port) self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) pending = self.te_client.get_pending() self.assertEqual(len(pending), 0) self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) def test_get_clear_queue(self): """ test_get_clear_queue Test endpoint queue get and clear manipulators. """ # Set up for events expected. self._no_queue_mod_evts = self._no_requests * 2 # Queue commands. for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Queue commands. for i in range(self._no_requests): cmd = self.make_fake_svc_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Confirm queue mod events. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) # Confirm get queue with no id. queue = self.te_client.get_queue() self.assertEqual(len(queue), self._no_requests * 2) # Confirm get queue with id. queue = self.te_client.get_queue(resource_id=self._resource_id) self.assertEqual(len(queue), self._no_requests) # Confirm get queue with svc name. queue = self.te_client.get_queue(svc_name=self._rmt_svc_name) self.assertEqual(len(queue), self._no_requests) # Confirm get queue with bogus id. queue = self.te_client.get_queue(resource_id='bogus_id') self.assertEqual(len(queue), 0) # Confirm get queue with bogus id. queue = self.te_client.get_queue(svc_name='bogus_svc') self.assertEqual(len(queue), 0) # Reset queue mod expected events. self._queue_mod_evts = [] self._no_queue_mod_evts = 1 self._done_queue_mod_evts = AsyncResult() # Clear queue with no id. poped = self.te_client.clear_queue() # Confirm queue mod event and mods. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) queue = self.te_client.get_queue() self.assertEqual(len(poped), self._no_requests * 2) self.assertEqual(len(queue), 0) # Queue new commands and confirm event. self._queue_mod_evts = [] self._no_queue_mod_evts = self._no_requests * 2 self._done_queue_mod_evts = AsyncResult() self._requests_sent = {} for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd for i in range(self._no_requests): cmd = self.make_fake_svc_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) # Reset queue mod expected events. self._queue_mod_evts = [] self._no_queue_mod_evts = 1 self._done_queue_mod_evts = AsyncResult() # Clear queue with id. poped = self.te_client.clear_queue(resource_id=self._resource_id) # Confirm mods and mod events. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) queue = self.te_client.get_queue() self.assertEqual(len(poped), self._no_requests) self.assertEqual(len(queue), self._no_requests) # Reset queue mod expected events. self._queue_mod_evts = [] self._no_queue_mod_evts = 1 self._done_queue_mod_evts = AsyncResult() # Clear queue with id. poped = self.te_client.clear_queue(svc_name=self._rmt_svc_name) # Confirm mods and mod events. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) queue = self.te_client.get_queue() self.assertEqual(len(poped), self._no_requests) self.assertEqual(len(queue), 0) # Queue new commands and confirm events. self._queue_mod_evts = [] self._no_queue_mod_evts = self._no_requests self._done_queue_mod_evts = AsyncResult() self._requests_sent = {} for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) # Clear queue with bogus id. poped = self.te_client.clear_queue(resource_id='bogus id') queue = self.te_client.get_queue() self.assertEqual(len(poped), 0) self.assertEqual(len(queue), self._no_requests) # Clear queue with bogus svc name. poped = self.te_client.clear_queue(svc_name='bogus id') queue = self.te_client.get_queue() self.assertEqual(len(poped), 0) self.assertEqual(len(queue), self._no_requests) # Clear queue and confirm empty. self.te_client.clear_queue() queue = self.te_client.get_queue() self.assertEqual(len(queue), 0) # Turn on link and wait a few seconds. # Confirm no data or tx events arrive. self.on_link_up() gevent.sleep(2) self.assertEqual(len(self._cmd_tx_evts), 0) self.assertEqual(len(self._results_recv), 0) self._no_telem_evts = 2 self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) def test_pop_pending_queue(self): """ test_pop_pending_queue Test endpoint queue pop manipulators. """ # Set up for events expected. self._no_queue_mod_evts = self._no_requests # Queue commands. for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Confirm queue mod events. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) queue = self.te_client.get_queue() self.assertEqual(len(queue), self._no_requests) # Pop a few commands from the queue, confirm events. self._queue_mod_evts = [] self._no_queue_mod_evts = 3 self._done_queue_mod_evts = AsyncResult() cmd_ids = self._requests_sent.keys()[:3] poped = [] for x in cmd_ids: poped.append(self.te_client.pop_queue(x)) self._requests_sent.pop(x) # Try poping with illegal args. This should have no effect poped.append(self.te_client.pop_queue()) poped.append(self.te_client.pop_queue('bogus id')) poped = [x for x in poped if x != None] self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) queue = self.te_client.get_queue() self.assertEqual(len(poped), 3) self.assertEqual(len(queue), self._no_requests - 3) # Turn on the link and verify that only the remaining commands # get processed. self._no_telem_evts = 2 self._no_requests = self._no_requests - 3 self._no_cmd_tx_evts = self._no_requests self.on_link_up() self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) pending = self.te_client.get_pending() self.assertEqual(len(pending), 0) def test_repeated_clear_pop(self): """ test_repeated_clear_pop Test endpoint queue pop manipulators. """ # Set up for events expected. self._no_queue_mod_evts = self._no_requests for i in range(3): self._queue_mod_evts = [] self._no_queue_mod_evts = self._no_requests self._done_queue_mod_evts = AsyncResult() # Queue commands. self._requests_sent = {} for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Confirm queue mod events. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) # Confirm get queue with no id. queue = self.te_client.get_queue() self.assertEqual(len(queue), self._no_requests) # Reset queue mod expected events. self._queue_mod_evts = [] self._no_queue_mod_evts = 1 self._done_queue_mod_evts = AsyncResult() # Clear queue with no id. poped = self.te_client.clear_queue() # Confirm queue mod event and mods. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) queue = self.te_client.get_queue() self.assertEqual(len(poped), self._no_requests) self.assertEqual(len(queue), 0) self._queue_mod_evts = [] self._no_queue_mod_evts = self._no_requests self._done_queue_mod_evts = AsyncResult() # Queue commands. self._requests_sent = {} for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Confirm queue mod events. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) # Confirm get queue with no id. queue = self.te_client.get_queue() self.assertEqual(len(queue), self._no_requests) # Pop a few commands from the queue, confirm events. self._queue_mod_evts = [] self._no_queue_mod_evts = 3 self._done_queue_mod_evts = AsyncResult() cmd_ids = self._requests_sent.keys()[:3] poped = [] for x in cmd_ids: poped.append(self.te_client.pop_queue(x)) self._requests_sent.pop(x) self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) queue = self.te_client.get_queue() self.assertEqual(len(poped), 3) self.assertEqual(len(queue), self._no_requests - 3) self._no_telem_evts = 2 self._no_requests = self._no_requests - 3 self._no_cmd_tx_evts = self._no_requests self.on_link_up() self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) pending = self.te_client.get_pending() self.assertEqual(len(pending), 0) def test_get_pending(self): """ test_process_queued Test forwarding of queued commands upon link up. """ self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._no_telem_evts = 2 for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd self.on_link_up() self._no_requests = 3 self._done_evt.get(timeout=CFG.endpoint.receive.timeout) pending = self.te_client.get_pending(resource_id=self._resource_id) for x in pending: self.assertIn(x.command_id, self._requests_sent.keys()) self._no_requests = 10 self._done_evt = AsyncResult() self._done_evt.get(timeout=CFG.endpoint.receive.timeout) pending = self.te_client.get_pending() self.assertEqual(len(pending), 0) self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) #@unittest.skip('Wait for verification of resource registry use.') def test_persistent_queue(self): """ test_persistent_queue Test ability of endpoint to restore a persistent queue, survive reboot, etc. """ self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._no_telem_evts = 2 for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Confirm queue mod events. self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout) # Confirm get queue with no id. queue = self.te_client.get_queue() self.assertEqual(len(queue), self._no_requests) # Stop and restart the endpoint process. # Verify the queue is restored. self._container_client.terminate_process(self._te_pid) # Create agent config. endpoint_config = { 'other_host' : self._other_host, 'other_port' : self._other_port, 'this_port' : 0, 'xs_name' : self._xs_name, 'platform_resource_id' : self._platform_resource_id, 'process' : { 'listen_name' : self._listen_name } } # Spawn the terrestrial enpoint process. log.debug('Spawning terrestrial endpoint process.') self._te_pid = self._container_client.spawn_process( name='remote_endpoint_1', module='ion.services.sa.tcaa.terrestrial_endpoint', cls='TerrestrialEndpoint', config=endpoint_config) log.debug('Endpoint pid=%s.', str(self._te_pid)) # Create an endpoint client. # The to_name may be either the process pid or # the listen_name, which for this remote bridge # is svc_name + remote_name as above. self.te_client = TerrestrialEndpointClient( process=FakeProcess(), to_name=self._listen_name) log.debug('Got te client %s.', str(self.te_client)) # Remember the terrestrial port. self._this_port = self.te_client.get_port() # Confirm we restored the queue with the previous commands. queue = self.te_client.get_queue() self.assertEqual(len(queue), self._no_requests) self.on_link_up() self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout) self._done_evt.get(timeout=CFG.endpoint.receive.timeout) pending = self.te_client.get_pending() self.assertEqual(len(pending), 0) self.on_link_down() self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys())
def setUp(self): """ Setup fake remote components. Start remote server. Set internal configuration and test variables. Start container. Start services. Spawn endpoint. Create and start subscribers. """ # Create fake remote client and server. # Add clean up to shut down properly. # Start remote server on a random port. self._remote_server = R3PCServer(self.consume_req, self.remote_server_close) self._remote_client = R3PCClient(self.consume_ack, self.remote_client_close) self.addCleanup(self._remote_server.stop) self.addCleanup(self._remote_client.stop) self._other_port = self._remote_server.start('*', 0) log.debug('Remote server binding to *:%i', self._other_port) # Set internal variables. self._other_host = 'localhost' self._xs_name = 'remote1' self._svc_name = 'terrestrial_endpoint' self._listen_name = self._svc_name + self._xs_name self._platform_resource_id = 'abc123' self._resource_id = 'fake_id' self._rmt_svc_name = 'fake_svc' self._no_requests = 10 self._requests_sent = {} self._results_recv = {} self._workers = [] self._done_evt = AsyncResult() self._queue_mod_evts = [] self._cmd_tx_evts = [] self._telem_evts = [] self._no_telem_evts = 0 self._no_queue_mod_evts = 0 self._no_cmd_tx_evts = 0 self._done_queue_mod_evts = AsyncResult() self._done_telem_evts = AsyncResult() self._done_cmd_tx_evts = AsyncResult() # Start container. log.debug('Staring capability container.') self._start_container() # Bring up services in a deploy file (no need to message) log.info('Staring deploy services.') self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Create a container client. log.debug('Creating container client.') self._container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) # The following spawn config creates the process with the remote # name tagged to the service name. """ listen_name = terrestrial_endpointremote1 2012-10-10 11:34:46,654 DEBUG ion.services.sa.tcaa.terrestrial_endpoint recv name: NP (ion_test_8257ab,terrestrial_endpointremote1,B: terrestrial_endpointremote1) 2012-10-10 11:34:46,654 DEBUG ion.services.sa.tcaa.terrestrial_endpoint startup listener recv name: NP (ion_test_8257ab,terrestrial_endpointremote1,B: terrestrial_endpointremote1) 2012-10-10 11:34:46,654 DEBUG ion.services.sa.tcaa.terrestrial_endpoint startup listener recv name: NP (ion_test_8257ab,Edwards-MacBook-Pro_local_2624.33,B: Edwards-MacBook-Pro_local_2624.33) """ # Create agent config. endpoint_config = { 'other_host' : self._other_host, 'other_port' : self._other_port, 'this_port' : 0, 'xs_name' : self._xs_name, 'platform_resource_id' : self._platform_resource_id, 'process' : { 'listen_name' : self._listen_name } } # Spawn the terrestrial enpoint process. log.debug('Spawning terrestrial endpoint process.') self._te_pid = self._container_client.spawn_process( name='remote_endpoint_1', module='ion.services.sa.tcaa.terrestrial_endpoint', cls='TerrestrialEndpoint', config=endpoint_config) log.debug('Endpoint pid=%s.', str(self._te_pid)) # Create an endpoint client. # The to_name may be either the process pid or # the listen_name, which for this remote bridge # is svc_name + remote_name as above. self.te_client = TerrestrialEndpointClient( process=FakeProcess(), to_name=self._listen_name) log.debug('Got te client %s.', str(self.te_client)) # Remember the terrestrial port. self._this_port = self.te_client.get_port() # Start the event publisher. self._event_publisher = EventPublisher() # Start the event subscriber. self._event_subscriber = EventSubscriber( event_type='PlatformEvent', callback=self.consume_event, origin=self._xs_name) self._event_subscriber.start() self._event_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._event_subscriber.stop) # Start the result subscriber. self._result_subscriber = EventSubscriber( event_type='RemoteCommandResult', origin=self._resource_id, callback=self.consume_event) self._result_subscriber.start() self._result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._result_subscriber.stop)
class Test2CAA(IonIntegrationTestCase): """ Test cases for 2CAA terrestrial endpoint. """ def setUp(self): """ """ ################################################################### # Internal parameters and container. ################################################################### # Internal parameters. self._terrestrial_platform_id = 'terrestrial_id' self._remote_platform_id = 'remote_id' self._resource_id = 'fake_id' self._xs_name = 'remote1' self._terrestrial_svc_name = 'terrestrial_endpoint' self._terrestrial_listen_name = self._terrestrial_svc_name + self._xs_name self._remote_svc_name = 'remote_endpoint' self._remote_listen_name = self._remote_svc_name + self._xs_name self._remote_port = 0 self._terrestrial_port = 0 self._te_client = None self._re_client = None self._remote_pid = None self._terrestrial_pid = None # Async test results. self._no_requests = 10 self._no_telem_evts = 2 self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._telem_evts = [] self._queue_mod_evts = [] self._cmd_tx_evts = [] self._results_recv = {} self._requests_sent = {} self._done_telem_evt = AsyncResult() self._done_queue_mod_evt = AsyncResult() self._done_cmd_tx_evt = AsyncResult() self._done_cmd_evt = AsyncResult() # Start container. log.debug('Staring capability container.') self._start_container() # Bring up services in a deploy file (no need to message). log.info('Staring deploy services.') self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Create a container client. log.debug('Creating container client.') self._container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) ################################################################### # Start endpoints, agent. ################################################################### self._start_terrestrial() self._start_remote() self._start_agent() ################################################################### # Assign client ports. # This is primarily for test purposes as the IP config in # deployment will be fixed in advance. ################################################################### self.te_client.set_client_port(self._remote_port) check_port = self.te_client.get_client_port() log.debug('Terrestrial client port is: %i', check_port) self.re_client.set_client_port(self._terrestrial_port) check_port = self.re_client.get_client_port() log.debug('Remote client port is: %i', check_port) ################################################################### # Start the event publisher and subscribers. # Used to send fake agent telemetry publications to the endpoints, # and to receive endpoint publications. ################################################################### self._event_publisher = EventPublisher() # Start the event subscriber for remote namespace platform events. # This event could be changed to RemoteNamespaceEvent. self._event_subscriber = EventSubscriber( event_type='PlatformEvent', callback=self.consume_event, origin=self._xs_name) self._event_subscriber.start() self._event_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._event_subscriber.stop) # Start the result subscriber for remote resource events. self._resource_result_subscriber = EventSubscriber( event_type='RemoteCommandResult', origin=IA_RESOURCE_ID, callback=self.consume_event) self._resource_result_subscriber.start() self._resource_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._resource_result_subscriber.stop) # Start the result subscriber for remote service events. self._service_result_subscriber = EventSubscriber( event_type='RemoteCommandResult', origin='resource_registry' + 'remote1', callback=self.consume_event) self._service_result_subscriber.start() self._service_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._service_result_subscriber.stop) # Start the result subscriber for fake resource results. self._fake_result_subscriber = EventSubscriber( event_type='RemoteCommandResult', origin=self._resource_id, callback=self.consume_event) self._fake_result_subscriber.start() self._fake_result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout) self.addCleanup(self._fake_result_subscriber.stop) ################################################################### # Start/stop helpers. ################################################################### def _start_agent(self): """ Start an instrument agent and client. """ log.info('Creating driver integration test support:') log.info('driver module: %s', DRV_MOD) log.info('driver class: %s', DRV_CLS) log.info('device address: %s', DEV_ADDR) log.info('device port: %s', DEV_PORT) log.info('log delimiter: %s', DELIM) log.info('work dir: %s', WORK_DIR) self._support = DriverIntegrationTestSupport(DRV_MOD, DRV_CLS, DEV_ADDR, DEV_PORT, DATA_PORT, CMD_PORT, PA_BINARY, DELIM, WORK_DIR) # Start port agent, add stop to cleanup. port = self._support.start_pagent() log.info('Port agent started at port %i',port) # Configure driver to use port agent port number. DVR_CONFIG['comms_config'] = { 'addr' : 'localhost', 'port' : port } self.addCleanup(self._support.stop_pagent) # Create agent config. agent_config = { 'driver_config' : DVR_CONFIG, 'stream_config' : {}, 'agent' : {'resource_id': IA_RESOURCE_ID}, 'test_mode' : True } # Start instrument agent. log.debug("Starting IA.") container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) ia_pid = container_client.spawn_process(name=IA_NAME, module=IA_MOD, cls=IA_CLS, config=agent_config) log.info('Agent pid=%s.', str(ia_pid)) # Start a resource agent client to talk with the instrument agent. self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess()) log.info('Got ia client %s.', str(self._ia_client)) def _start_terrestrial(self): """ """ # Create terrestrial config. terrestrial_endpoint_config = { 'other_host' : 'localhost', 'other_port' : self._remote_port, 'this_port' : self._terrestrial_port, 'platform_resource_id' : self._terrestrial_platform_id, 'xs_name' : self._xs_name, 'process' : { 'listen_name' : self._terrestrial_listen_name } } # Spawn the terrestrial enpoint process. log.debug('Spawning terrestrial endpoint process.') self._terrestrial_pid = self._container_client.spawn_process( name=self._terrestrial_listen_name, module='ion.services.sa.tcaa.terrestrial_endpoint', cls='TerrestrialEndpoint', config=terrestrial_endpoint_config) log.debug('Terrestrial endpoint pid=%s.', str(self._terrestrial_pid)) # Create a terrestrial client. self.te_client = TerrestrialEndpointClient( process=FakeProcess(), to_name=self._terrestrial_listen_name) log.debug('Got te client %s.', str(self.te_client)) self._terrestrial_port = self.te_client.get_port() log.debug('Terrestrial port is: %i', self._terrestrial_port) def _start_remote(self): """ """ # Create agent config. remote_endpoint_config = { 'other_host' : 'localhost', 'other_port' : self._terrestrial_port, 'this_port' : self._remote_port, 'platform_resource_id' : self._remote_platform_id, 'xs_name' : self._xs_name, 'process' : { 'listen_name' : self._remote_listen_name } } # Spawn the remote enpoint process. log.debug('Spawning remote endpoint process.') self._remote_pid = self._container_client.spawn_process( name=self._remote_listen_name, module='ion.services.sa.tcaa.remote_endpoint', cls='RemoteEndpoint', config=remote_endpoint_config) log.debug('Remote endpoint pid=%s.', str(self._remote_pid)) # Create an endpoint client. self.re_client = RemoteEndpointClient( process=FakeProcess(), to_name=self._remote_listen_name) log.debug('Got re client %s.', str(self.re_client)) # Remember the remote port. self._remote_port = self.re_client.get_port() log.debug('The remote port is: %i.', self._remote_port) ################################################################### # Telemetry publications to start/top endpoint. # (Normally be published by appropriate platform agents.) ################################################################### def terrestrial_link_up(self): """ Publish telemetry available to the terrestrial endpoint. """ # Publish a link up event to be caught by the terrestrial endpoint. log.debug('Publishing terrestrial telemetry available event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin=self._terrestrial_platform_id, status = TelemetryStatusType.AVAILABLE) def terrestrial_link_down(self): """ Publish telemetry unavailable to the terrestrial endpoint. """ # Publish a link up event to be caught by the terrestrial endpoint. log.debug('Publishing terrestrial telemetry unavailable event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin=self._terrestrial_platform_id, status = TelemetryStatusType.UNAVAILABLE) def remote_link_up(self): """ Publish telemetry available to the remote endpoint. """ # Publish a link up event to be caught by the remote endpoint. log.debug('Publishing remote telemetry available event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin=self._remote_platform_id, status = TelemetryStatusType.AVAILABLE) def remote_link_down(self): """ Publish telemetry unavailable to the remote endpoint. """ # Publish a link down event to be caught by the remote endpoint. log.debug('Publishing remote telemetry unavailable event.') self._event_publisher.publish_event( event_type='PlatformTelemetryEvent', origin=self._remote_platform_id, status = TelemetryStatusType.UNAVAILABLE) def consume_event(self, evt, *args, **kwargs): """ Test callback for events. """ log.debug('Test got event: %s, args: %s, kwargs: %s', str(evt), str(args), str(kwargs)) if evt.type_ == 'PublicPlatformTelemetryEvent': self._telem_evts.append(evt) if self._no_telem_evts > 0 and self._no_telem_evts == len(self._telem_evts): self._done_telem_evt.set() elif evt.type_ == 'RemoteQueueModifiedEvent': self._queue_mod_evts.append(evt) if self._no_queue_mod_evts > 0 and self._no_queue_mod_evts == len(self._queue_mod_evts): self._done_queue_mod_evt.set() elif evt.type_ == 'RemoteCommandTransmittedEvent': self._cmd_tx_evts.append(evt) if self._no_cmd_tx_evts > 0 and self._no_cmd_tx_evts == len(self._cmd_tx_evts): self._done_cmd_tx_evt.set() elif evt.type_ == 'RemoteCommandResult': cmd = evt.command self._results_recv[cmd.command_id] = cmd if len(self._results_recv) == self._no_requests: self._done_cmd_evt.set() ################################################################### # Misc helpers. ################################################################### def make_fake_command(self, no): """ Build a fake command for use in tests. """ cmdstr = 'fake_cmd_%i' % no cmd = IonObject('RemoteCommand', resource_id='fake_id', command=cmdstr, args=['arg1', 23], kwargs={'worktime':3}) return cmd ################################################################### # Tests. ################################################################### def test_queued_fake(self): """ test_queued_fake Test fake resource commands queued prior to linkup. """ # Queue up a series of fake commands to be handled by the remote side. for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Block on queue mod events. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) # Publish link up events. self.terrestrial_link_up() self.remote_link_up() # Block on command transmissions and results. self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Publish link down events. self.terrestrial_link_down() self.remote_link_down() # Block on terrestrial public telemetry events. self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) # The following error occurs when queue persistence is enabled. # Need to verify correct solution to enable persistent queues. """ 2012-11-06 14:27:13,517 INFO pyon.container.procs ProcManager.terminate_process: org_management -> pid=Edwards-MacBook-Pro_local_8975.8 Traceback (most recent call last): File "/Users/edward/Documents/Dev/code/coi-services/eggs/gevent-0.13.7-py2.7-macosx-10.5-intel.egg/gevent/greenlet.py", line 390, in run result = self._run(*self.args, **self.kwargs) File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/remote_endpoint.py", line 97, in command_loop self._callback(cmd_result) File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/remote_endpoint.py", line 284, in _result_complete self._client.enqueue(result) AttributeError: 'NoneType' object has no attribute 'enqueue' <Greenlet at 0x1059ca9b0: command_loop> failed with AttributeError 2012-11-06 14:27:13,586 INFO pyon.container.procs ProcManager.terminate_process: exchange_management -> pid=Edwards-MacBook-Pro_local_8975.7 2012-11-06 14:27:13,665 INFO pyon.container.procs ProcManager.terminate_process: policy_management -> pid=Edwards-MacBook-Pro_local_8975.6 2012-11-06 14:27:13,739 INFO pyon.container.procs ProcManager.terminate_process: identity_management -> pid=Edwards-MacBook-Pro_local_8975.5 2012-11-06 14:27:13,807 INFO pyon.container.procs ProcManager.terminate_process: directory -> pid=Edwards-MacBook-Pro_local_8975.4 2012-11-06 14:27:13,874 INFO pyon.container.procs ProcManager.terminate_process: resource_registry -> pid=Edwards-MacBook-Pro_local_8975.3 2012-11-06 14:27:13,941 INFO pyon.container.procs ProcManager.terminate_process: event_persister -> pid=Edwards-MacBook-Pro_local_8975.1 2012-11-06 14:27:13,945 INFO pyon.event.event EventSubscriber stopped. Event pattern=# 2012-11-06 14:27:14,124 INFO pyon.datastore.couchdb.couchdb_standalone Connecting to CouchDB server: http://localhost:5984 2012-11-06 14:27:14,399 INFO pyon.datastore.couchdb.couchdb_standalone Closing connection to CouchDB ====================================================================== ERROR: test_process_online ---------------------------------------------------------------------- Traceback (most recent call last): File "/Users/edward/Documents/Dev/code/coi-services/eggs/mock-0.8.0-py2.7.egg/mock.py", line 1605, in _inner return f(*args, **kw) File "/Users/edward/Documents/Dev/code/coi-services/ion/services/sa/tcaa/test/test_2caa.py", line 492, in test_process_online cmd = self.te_client.enqueue_command(cmd) File "/Users/edward/Documents/Dev/code/coi-services/interface/services/sa/iterrestrial_endpoint.py", line 188, in enqueue_command return self.request(IonObject('terrestrial_endpoint_enqueue_command_in', **{'command': command or None,'link': link}), op='enqueue_command', headers=headers, timeout=timeout) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 1012, in request return RequestResponseClient.request(self, msg, headers=headers, timeout=timeout) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 822, in request retval, headers = e.send(msg, headers=headers, timeout=timeout) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 310, in send result_data, result_headers = c.send(self.conv_type.server_role, msg, headers, **kwargs) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 126, in send return self._invite_and_send(to_role, msg, headers, **kwargs) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 145, in _invite_and_send return self._send(to_role, to_role_name, msg, header, **kwargs) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 169, in _send return self._end_point_unit._message_send(msg, headers, **kwargs) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/ion/conversation.py", line 289, in _message_send return ProcessRPCRequestEndpointUnit.send(self, msg, headers, **kwargs) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 134, in send return self._send(_msg, _header, **kwargs) File "/Users/edward/Documents/Dev/code/coi-services/extern/pyon/pyon/net/endpoint.py", line 880, in _send raise ex Conflict: 409 - Object not based on most current version """ def test_process_online(self): """ test_process_online Test fake resource commands queued while link is up. """ # Publish link up events. self.terrestrial_link_up() self.remote_link_up() # Queue up a series of fake commands to be handled by the remote side. for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Block on queue mods, command transmissions and results. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Publish link down events. self.terrestrial_link_down() self.remote_link_down() # Block on terrestrial public telemetry events. self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) def test_remote_late(self): """ test_remote_late Test fake resource commands queued prior to linkup. Delay remote side linkup substantially to test terrestrial behavior when remote server not initially available. """ for i in range(self._no_requests): cmd = self.make_fake_command(i) cmd = self.te_client.enqueue_command(cmd) self._requests_sent[cmd.command_id] = cmd # Block on queue mod events. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) # Actually stop the remote process, since the server is # available even if it hasn't linked up yet and the test is running # in the same container. The test will remember the appropriate # remote port numbers. self._container_client.terminate_process(self._remote_pid) self.terrestrial_link_up() gevent.sleep(10) # Restart remote side and publish remote link up. self._start_remote() self.remote_link_up() # Block for transmission and result events. self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Publish link down events. self.terrestrial_link_down() self.remote_link_down() self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) def test_resource_commands(self): """ test_resource_commands """ # Set up to verify the two commands queued. self._no_requests = 2 self._no_telem_evts = 2 self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests # Use IA client to verify IA. state = self._ia_client.get_agent_state() log.debug('Agent state is: %s', state) self.assertEqual(state, ResourceAgentState.UNINITIALIZED) retval = self._ia_client.ping_agent() log.debug('Agent ping is: %s', str(retval)) self.assertIn('ping from InstrumentAgent', retval) # Create and enqueue commands. state_cmd = IonObject('RemoteCommand', resource_id=IA_RESOURCE_ID, svc_name='', command='get_agent_state', args=[], kwargs={}) state_cmd = self.te_client.enqueue_command(state_cmd) self._requests_sent[state_cmd.command_id] = state_cmd ping_cmd = IonObject('RemoteCommand', resource_id=IA_RESOURCE_ID, svc_name='', command='ping_agent', args=[], kwargs={}) ping_cmd = self.te_client.enqueue_command(ping_cmd) self._requests_sent[ping_cmd.command_id] = ping_cmd # Block on queue mod events. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) # Publish link up events. self.terrestrial_link_up() self.remote_link_up() # Block on command transmissions and results. self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Publish link down events. self.terrestrial_link_down() self.remote_link_down() # Block on terrestrial public telemetry events. self._done_telem_evt.get(timeout=CFG.endpoint.receive.timeout) self.assertItemsEqual(self._requests_sent.keys(), self._results_recv.keys()) self.assertEqual(self._results_recv[state_cmd.command_id].result, ResourceAgentState.UNINITIALIZED) self.assertIn('ping from InstrumentAgent', self._results_recv[ping_cmd.command_id].result) def test_service_command_sequence(self): """ test_service_commands """ # Publish link up events. self.terrestrial_link_up() self.remote_link_up() # Send commands one at a time. # Reset queues and events. self._no_requests = 1 self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._done_queue_mod_evt = AsyncResult() self._done_cmd_tx_evt = AsyncResult() self._done_cmd_evt = AsyncResult() self._queue_mod_evts = [] self._cmd_tx_evts = [] self._requests_sent = {} self._results_recv = {} # Create user object. obj = IonObject("UserInfo", name="some_name") cmd = IonObject('RemoteCommand', resource_id='', svc_name='resource_registry', command='create', args=[obj], kwargs='') cmd = self.te_client.enqueue_command(cmd) # Block for the queue to be modified, the command to be transmitted, # and the result to be received. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Returns obj_id, obj_rev. obj_id, obj_rev = self._results_recv[cmd.command_id].result # Confirm the results are valid. #Result is a tuple of strings. #{'result': ['ad183ff26bae4f329ddd85fd69d160a9', #'1-00a308c45fff459c7cda1db9a7314de6'], #'command_id': 'cc2ae00d-40b0-47d2-af61-8ffb87f1aca2'} self.assertIsInstance(obj_id, str) self.assertNotEqual(obj_id, '') self.assertIsInstance(obj_rev, str) self.assertNotEqual(obj_rev, '') # Send commands one at a time. # Reset queues and events. self._no_requests = 1 self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._done_queue_mod_evt = AsyncResult() self._done_cmd_tx_evt = AsyncResult() self._done_cmd_evt = AsyncResult() self._queue_mod_evts = [] self._cmd_tx_evts = [] self._requests_sent = {} self._results_recv = {} # Create user object. cmd = IonObject('RemoteCommand', resource_id='', svc_name='resource_registry', command='read', args=[obj_id], kwargs='') cmd = self.te_client.enqueue_command(cmd) # Block for the queue to be modified, the command to be transmitted, # and the result to be received. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Returns read_obj. read_obj = self._results_recv[cmd.command_id].result # Confirm the results are valid. #Result is a user info object with the name set. #{'lcstate': 'DEPLOYED_AVAILABLE', #'_rev': '1-851f067bac3c34b2238c0188b3340d0f', #'description': '', #'ts_updated': '1349213207638', #'type_': 'UserInfo', #'contact': <interface.objects.ContactInformation object at 0x10d7df590>, #'_id': '27832d93f4cd4535a75ac75c06e00a7e', #'ts_created': '1349213207638', #'variables': [{'name': '', 'value': ''}], #'name': 'some_name'} self.assertIsInstance(read_obj, UserInfo) self.assertEquals(read_obj.name, 'some_name') # Send commands one at a time. # Reset queues and events. self._no_requests = 1 self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._done_queue_mod_evt = AsyncResult() self._done_cmd_tx_evt = AsyncResult() self._done_cmd_evt = AsyncResult() self._queue_mod_evts = [] self._cmd_tx_evts = [] self._requests_sent = {} self._results_recv = {} # Create user object. read_obj.name = 'some_other_name' cmd = IonObject('RemoteCommand', resource_id='', svc_name='resource_registry', command='update', args=[read_obj], kwargs='') cmd = self.te_client.enqueue_command(cmd) # Block for the queue to be modified, the command to be transmitted, # and the result to be received. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Returns nothing. # Send commands one at a time. # Reset queues and events. self._no_requests = 1 self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._done_queue_mod_evt = AsyncResult() self._done_cmd_tx_evt = AsyncResult() self._done_cmd_evt = AsyncResult() self._queue_mod_evts = [] self._cmd_tx_evts = [] self._requests_sent = {} self._results_recv = {} # Create user object. cmd = IonObject('RemoteCommand', resource_id='', svc_name='resource_registry', command='read', args=[obj_id], kwargs='') cmd = self.te_client.enqueue_command(cmd) # Block for the queue to be modified, the command to be transmitted, # and the result to be received. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Returns read_obj. read_obj = self._results_recv[cmd.command_id].result self.assertIsInstance(read_obj, UserInfo) self.assertEquals(read_obj.name, 'some_other_name') # Send commands one at a time. # Reset queues and events. self._no_requests = 1 self._no_cmd_tx_evts = self._no_requests self._no_queue_mod_evts = self._no_requests self._done_queue_mod_evt = AsyncResult() self._done_cmd_tx_evt = AsyncResult() self._done_cmd_evt = AsyncResult() self._queue_mod_evts = [] self._cmd_tx_evts = [] self._requests_sent = {} self._results_recv = {} # Create user object. cmd = IonObject('RemoteCommand', resource_id='', svc_name='resource_registry', command='delete', args=[obj_id], kwargs='') cmd = self.te_client.enqueue_command(cmd) # Block for the queue to be modified, the command to be transmitted, # and the result to be received. self._done_queue_mod_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout) self._done_cmd_evt.get(timeout=CFG.endpoint.receive.timeout) # Returns nothing. # Publish link down events. self.terrestrial_link_down() self.remote_link_down()