class RemoteClient(object):
"""
A proxy client for any service or resource that forwards commands
to the terrestrial endpoint for transmission.
"""
implements(ITerrestrialEndpoint)
def __init__(self,
iface=None,
xs_name=None,
resource_id=None,
svc_name=None,
process=None):
"""
Construct remote proxy client.
Verify required parameters.
Construct remote endpoint client.
Set internal variables.
Construct and add service method attributes.
"""
# Throw an exception if required interface arg not provided.
if not isinstance(iface, InterfaceClass):
raise ConfigNotFound('Invalid interface parameter.')
# Throw an exception if the xs_name is not provided.
if not isinstance(xs_name, str) or xs_name == '':
raise ConfigNotFound('Invalid exchange space name parameter.')
# Create the endpoint client.
# Throw exception if unsuccessful.
to_name = 'terrestrial_endpoint' + xs_name
self._te_client = TerrestrialEndpointClient(process=process,
to_name=to_name)
# Must define a resource id or service name.
if not resource_id and not svc_name:
raise ConfigNotFound('No resource or service specified.')
# Can't specify both a resource and a service.
if resource_id and svc_name:
raise ConfigNotFound(
'Can\'t specify both a resource and a service.')
self._resource_id = resource_id
self._xs_name = xs_name
self._svc_name = svc_name
# Grab the service method names.
methods = iface.names()
# Generate the service interface.
# Each will forward to the terrestrial endpoint passing
# the function name, args and kwargs.
for m in methods:
setattr(self, m, self.generate_service_method(m))
# Declare the dynamic interface.
directlyProvides(self, iface)
# Initialize the async results objects for blocking behavior.
#self._async_result_evt = None
def generate_service_method(self, name):
"""
A closure that returns a function for forwarding service calls.
The service call name is stored as a kwarg.
"""
def func(*args, **kwargs):
args = copy.deepcopy(args)
kwargs = copy.deepcopy(kwargs)
kwargs['func_name'] = name
return self.forward(*args, **kwargs)
return func
def forward(self, *args, **kwargs):
"""
Forward a service method to the terrestrial endpoint
through the service interface.
"""
func_name = kwargs.pop('func_name')
try:
link = kwargs.pop('link')
except KeyError:
link = True
cid = ''
try:
remote_timeout = kwargs.pop('remote_timeout')
if not isinstance(remote_timeout, int):
remote_timeout = 0
elif remote_timeout < 0:
remote_timeout = 0
elif remote_timeout == 0:
pass
else:
cid = str(uuid.uuid4())
except KeyError:
remote_timeout = 0
cmd = IonObject('RemoteCommand',
resource_id=self._resource_id,
svc_name=self._svc_name,
command=func_name,
command_id=cid,
args=args,
kwargs=kwargs)
if remote_timeout == 0:
return self._te_client.enqueue_command(cmd, link)
else:
if self._resource_id:
origin = self._resource_id
elif self._svc_name:
origin = self._svc_name + self._xs_name
pending_cmd = cmd
async_result_evt = AsyncResult()
def result_callback(evt, *args, **kwargs):
"""
Callback for subscriber retrive blocking results.
"""
#global async_result_evt
if evt.type_ == 'RemoteCommandResult':
cmd = evt.command
if cmd.command_id == pending_cmd.command_id:
async_result_evt.set(cmd)
sub = EventSubscriber(event_type='RemoteCommandResult',
origin=origin,
callback=result_callback)
sub.start()
#self._pending_cmd = cmd
cmd = self._te_client.enqueue_command(cmd, link)
try:
result = async_result_evt.get(timeout=remote_timeout)
#self._pending_cmd = None
sub.stop()
except gevent.Timeout:
#self._pending_cmd = None
sub.stop()
raise Timeout('Timed out waiting for remote result.')
return result
"""
def _result_callback(self, evt, *args, **kwargs):
#Callback for subscriber retrive blocking results.
if evt.type_ == 'RemoteCommandResult':
cmd = evt.command
if self._pending_cmd:
if cmd.command_id == self._pending_cmd.command_id:
self._pending_cmd = None
if self._async_result_evt:
self._async_result_evt.set(cmd)
"""
def enqueue_command(self, command=None, link=False):
"""
Enqueue command with terrestrial endoint.
"""
return self._te_client(command, link)
def get_queue(self):
"""
Get terrestrial endpoint queue for this resource/service.
"""
if self._resource_id:
return self._te_client.get_queue(resource_id=self._resource_id)
elif self._svc_name:
return self._te_client.get_queue(svc_name=self._svc_name)
def clear_queue(self):
"""
Clear terrestrial endpoint queue for this resource/service.
"""
if self._resource_id:
return self._te_client.clear_queue(resource_id=self._resource_id)
elif self._svc_name:
return self._te_client.clear_queue(svc_name=self._svc_name)
def pop_queue(self, command_id=''):
"""
Pop a command from the terrestrial endpoint queue.
"""
return self._te_client.pop_queue(command_id=command_id)
def get_pending(self):
"""
Get pending commands for this resource/service.
"""
if self._resource_id:
return self._te_client.get_pending(resource_id=self._resource_id)
elif self._svc_name:
return self._te_client.get_pending(svc_name=self._svc_name)
def get_port(self):
"""
Not supported for remote proxy clients.
"""
raise BadRequest('get_port not available via remote client.')
def set_client_port(self, port=0):
"""
Not supported for remote proxy clients.
"""
raise BadRequest('set_client_port not available via remote client.')
def get_client_port(self):
"""
Not supported for remote proxy clients.
"""
raise BadRequest('get_client_port not available via remote client.')
class TestTerrestrialEndpoint(IonIntegrationTestCase):
"""
Test cases for 2CAA terrestrial endpoint.
"""
def setUp(self):
"""
Setup fake remote components.
Start remote server.
Set internal configuration and test variables.
Start container.
Start services.
Spawn endpoint.
Create and start subscribers.
"""
# Create fake remote client and server.
# Add clean up to shut down properly.
# Start remote server on a random port.
self._remote_server = R3PCServer(self.consume_req, self.remote_server_close)
self._remote_client = R3PCClient(self.consume_ack, self.remote_client_close)
self.addCleanup(self._remote_server.stop)
self.addCleanup(self._remote_client.stop)
self._other_port = self._remote_server.start('*', 0)
log.debug('Remote server binding to *:%i', self._other_port)
# Set internal variables.
self._other_host = 'localhost'
self._xs_name = 'remote1'
self._svc_name = 'terrestrial_endpoint'
self._listen_name = self._svc_name + self._xs_name
self._platform_resource_id = 'abc123'
self._resource_id = 'fake_id'
self._rmt_svc_name = 'fake_svc'
self._no_requests = 10
self._requests_sent = {}
self._results_recv = {}
self._workers = []
self._done_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._telem_evts = []
self._no_telem_evts = 0
self._no_queue_mod_evts = 0
self._no_cmd_tx_evts = 0
self._done_queue_mod_evts = AsyncResult()
self._done_telem_evts = AsyncResult()
self._done_cmd_tx_evts = AsyncResult()
# Start container.
log.debug('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message)
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Create a container client.
log.debug('Creating container client.')
self._container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
# The following spawn config creates the process with the remote
# name tagged to the service name.
"""
listen_name = terrestrial_endpointremote1
2012-10-10 11:34:46,654 DEBUG ion.services.sa.tcaa.terrestrial_endpoint recv name: NP (ion_test_8257ab,terrestrial_endpointremote1,B: terrestrial_endpointremote1)
2012-10-10 11:34:46,654 DEBUG ion.services.sa.tcaa.terrestrial_endpoint startup listener recv name: NP (ion_test_8257ab,terrestrial_endpointremote1,B: terrestrial_endpointremote1)
2012-10-10 11:34:46,654 DEBUG ion.services.sa.tcaa.terrestrial_endpoint startup listener recv name: NP (ion_test_8257ab,Edwards-MacBook-Pro_local_2624.33,B: Edwards-MacBook-Pro_local_2624.33)
"""
# Create agent config.
endpoint_config = {
'other_host' : self._other_host,
'other_port' : self._other_port,
'this_port' : 0,
'xs_name' : self._xs_name,
'platform_resource_id' : self._platform_resource_id,
'process' : {
'listen_name' : self._listen_name
}
}
# Spawn the terrestrial enpoint process.
log.debug('Spawning terrestrial endpoint process.')
self._te_pid = self._container_client.spawn_process(
name='remote_endpoint_1',
module='ion.services.sa.tcaa.terrestrial_endpoint',
cls='TerrestrialEndpoint',
config=endpoint_config)
log.debug('Endpoint pid=%s.', str(self._te_pid))
# Create an endpoint client.
# The to_name may be either the process pid or
# the listen_name, which for this remote bridge
# is svc_name + remote_name as above.
self.te_client = TerrestrialEndpointClient(
process=FakeProcess(),
to_name=self._listen_name)
log.debug('Got te client %s.', str(self.te_client))
# Remember the terrestrial port.
self._this_port = self.te_client.get_port()
# Start the event publisher.
self._event_publisher = EventPublisher()
# Start the event subscriber.
self._event_subscriber = EventSubscriber(
event_type='PlatformEvent',
callback=self.consume_event,
origin=self._xs_name)
self._event_subscriber.start()
self._event_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._event_subscriber.stop)
# Start the result subscriber.
self._result_subscriber = EventSubscriber(
event_type='RemoteCommandResult',
origin=self._resource_id,
callback=self.consume_event)
self._result_subscriber.start()
self._result_subscriber._ready_event.wait(timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._result_subscriber.stop)
def consume_event(self, evt, *args, **kwargs):
"""
Test callback for events.
"""
log.debug('Got event: %s, args: %s, kwargs: %s',
str(evt), str(args), str(kwargs))
if evt.type_ == 'PublicPlatformTelemetryEvent':
self._telem_evts.append(evt)
if self._no_telem_evts > 0 and self._no_telem_evts == len(self._telem_evts):
self._done_telem_evts.set()
elif evt.type_ == 'RemoteQueueModifiedEvent':
self._queue_mod_evts.append(evt)
if self._no_queue_mod_evts > 0 and self._no_queue_mod_evts == len(self._queue_mod_evts):
self._done_queue_mod_evts.set()
elif evt.type_ == 'RemoteCommandTransmittedEvent':
self._cmd_tx_evts.append(evt)
if self._no_cmd_tx_evts > 0 and self._no_cmd_tx_evts == len(self._cmd_tx_evts):
self._done_cmd_tx_evts.set()
elif evt.type_ == 'RemoteCommandResult':
cmd = evt.command
self._results_recv[cmd.command_id] = cmd
if len(self._results_recv) == self._no_requests:
self._done_evt.set()
def on_link_up(self):
"""
Called by a test to simulate turning the link on.
"""
log.debug('Remote client connecting to localhost:%i.',
self._this_port)
self._remote_client.start('localhost', self._this_port)
# Publish a link up event to be caught by the endpoint.
log.debug('Publishing telemetry event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin = self._platform_resource_id,
status = TelemetryStatusType.AVAILABLE)
def on_link_down(self):
"""
Called by a test to simulate turning the link off.
"""
self._remote_client.stop()
# Publish a link down event to be caught by the endpoint.
log.debug('Publishing telemetry event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._platform_resource_id,
status = TelemetryStatusType.UNAVAILABLE)
def consume_req(self, request):
"""
Remote request callback.
Fire a greenlet to do some fake work before returning via
the remote client to terrestrial endpoint.
"""
# Spawn a greenlet to sleep briefly with each request and
# then respond with a result through the remote client.
log.debug('Remote endpoint got request: %s', str(request))
greenlet = gevent.spawn(self.process_remote_request, request)
self._workers.append(greenlet)
def consume_ack(self, request):
"""
Remote ack callback.
"""
log.debug('Remote endpoint got ack: %s', str(request))
def process_remote_request(self, request):
"""
Process remote request.
Do random amount of fake work and enqueue result for return to
terrestrial endpoint.
"""
worktime = random.uniform(.1,3)
gevent.sleep(worktime)
result = {
'command_id' : request.command_id,
'result' : 'fake_result'
}
log.debug('Finished processing request: %s', str(request))
self._remote_client.enqueue(result)
def remote_server_close(self):
"""
Remote server closed callback.
"""
log.debug('The remote server closed.')
def remote_client_close(self):
"""
Remoe client closed callback.
"""
log.debug('The remote client closed.')
def make_fake_command(self, no):
"""
Build a fake command for use in tests.
"""
cmdstr = 'fake_cmd_%i' % no
cmd = IonObject('RemoteCommand',
resource_id=self._resource_id,
command=cmdstr,
args=['arg1', 23],
kwargs={'kwargs1':'someval'})
return cmd
def make_fake_svc_command(self, no):
"""
Build a fake command for use in tests.
"""
cmdstr = 'fake_cmd_%i' % no
cmd = IonObject('RemoteCommand',
svc_name=self._rmt_svc_name,
command=cmdstr,
args=['arg1', 23],
kwargs={'kwargs1':'someval'})
return cmd
def test_process_queued(self):
"""
test_process_queued
Test forwarding of queued commands upon link up.
"""
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._no_telem_evts = 2
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
self.on_link_up()
self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
pending = self.te_client.get_pending()
self.assertEqual(len(pending), 0)
self.on_link_down()
self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_process_online(self):
"""
test_process_online
Test forwarding commands when the link is up.
"""
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._no_telem_evts = 2
self.on_link_up()
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
gevent.sleep(.2)
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
pending = self.te_client.get_pending()
self.assertEqual(len(pending), 0)
self.on_link_down()
self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_remote_late(self):
"""
test_remote_late
Test simulates behavior when the remote side is initially unavailable.
"""
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._no_telem_evts = 2
self.on_link_up()
gevent.sleep(2)
self._remote_server.stop()
self._remote_client.stop()
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
gevent.sleep(3)
self._remote_client.start('localhost', self._this_port)
self._remote_server.start('*', self._other_port)
self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
pending = self.te_client.get_pending()
self.assertEqual(len(pending), 0)
self.on_link_down()
self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_get_clear_queue(self):
"""
test_get_clear_queue
Test endpoint queue get and clear manipulators.
"""
# Set up for events expected.
self._no_queue_mod_evts = self._no_requests * 2
# Queue commands.
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Queue commands.
for i in range(self._no_requests):
cmd = self.make_fake_svc_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Confirm queue mod events.
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
# Confirm get queue with no id.
queue = self.te_client.get_queue()
self.assertEqual(len(queue), self._no_requests * 2)
# Confirm get queue with id.
queue = self.te_client.get_queue(resource_id=self._resource_id)
self.assertEqual(len(queue), self._no_requests)
# Confirm get queue with svc name.
queue = self.te_client.get_queue(svc_name=self._rmt_svc_name)
self.assertEqual(len(queue), self._no_requests)
# Confirm get queue with bogus id.
queue = self.te_client.get_queue(resource_id='bogus_id')
self.assertEqual(len(queue), 0)
# Confirm get queue with bogus id.
queue = self.te_client.get_queue(svc_name='bogus_svc')
self.assertEqual(len(queue), 0)
# Reset queue mod expected events.
self._queue_mod_evts = []
self._no_queue_mod_evts = 1
self._done_queue_mod_evts = AsyncResult()
# Clear queue with no id.
poped = self.te_client.clear_queue()
# Confirm queue mod event and mods.
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
queue = self.te_client.get_queue()
self.assertEqual(len(poped), self._no_requests * 2)
self.assertEqual(len(queue), 0)
# Queue new commands and confirm event.
self._queue_mod_evts = []
self._no_queue_mod_evts = self._no_requests * 2
self._done_queue_mod_evts = AsyncResult()
self._requests_sent = {}
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
for i in range(self._no_requests):
cmd = self.make_fake_svc_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
# Reset queue mod expected events.
self._queue_mod_evts = []
self._no_queue_mod_evts = 1
self._done_queue_mod_evts = AsyncResult()
# Clear queue with id.
poped = self.te_client.clear_queue(resource_id=self._resource_id)
# Confirm mods and mod events.
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
queue = self.te_client.get_queue()
self.assertEqual(len(poped), self._no_requests)
self.assertEqual(len(queue), self._no_requests)
# Reset queue mod expected events.
self._queue_mod_evts = []
self._no_queue_mod_evts = 1
self._done_queue_mod_evts = AsyncResult()
# Clear queue with id.
poped = self.te_client.clear_queue(svc_name=self._rmt_svc_name)
# Confirm mods and mod events.
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
queue = self.te_client.get_queue()
self.assertEqual(len(poped), self._no_requests)
self.assertEqual(len(queue), 0)
# Queue new commands and confirm events.
self._queue_mod_evts = []
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evts = AsyncResult()
self._requests_sent = {}
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
# Clear queue with bogus id.
poped = self.te_client.clear_queue(resource_id='bogus id')
queue = self.te_client.get_queue()
self.assertEqual(len(poped), 0)
self.assertEqual(len(queue), self._no_requests)
# Clear queue with bogus svc name.
poped = self.te_client.clear_queue(svc_name='bogus id')
queue = self.te_client.get_queue()
self.assertEqual(len(poped), 0)
self.assertEqual(len(queue), self._no_requests)
# Clear queue and confirm empty.
self.te_client.clear_queue()
queue = self.te_client.get_queue()
self.assertEqual(len(queue), 0)
# Turn on link and wait a few seconds.
# Confirm no data or tx events arrive.
self.on_link_up()
gevent.sleep(2)
self.assertEqual(len(self._cmd_tx_evts), 0)
self.assertEqual(len(self._results_recv), 0)
self._no_telem_evts = 2
self.on_link_down()
self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout)
def test_pop_pending_queue(self):
"""
test_pop_pending_queue
Test endpoint queue pop manipulators.
"""
# Set up for events expected.
self._no_queue_mod_evts = self._no_requests
# Queue commands.
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Confirm queue mod events.
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
queue = self.te_client.get_queue()
self.assertEqual(len(queue), self._no_requests)
# Pop a few commands from the queue, confirm events.
self._queue_mod_evts = []
self._no_queue_mod_evts = 3
self._done_queue_mod_evts = AsyncResult()
cmd_ids = self._requests_sent.keys()[:3]
poped = []
for x in cmd_ids:
poped.append(self.te_client.pop_queue(x))
self._requests_sent.pop(x)
# Try poping with illegal args. This should have no effect
poped.append(self.te_client.pop_queue())
poped.append(self.te_client.pop_queue('bogus id'))
poped = [x for x in poped if x != None]
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
queue = self.te_client.get_queue()
self.assertEqual(len(poped), 3)
self.assertEqual(len(queue), self._no_requests - 3)
# Turn on the link and verify that only the remaining commands
# get processed.
self._no_telem_evts = 2
self._no_requests = self._no_requests - 3
self._no_cmd_tx_evts = self._no_requests
self.on_link_up()
self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
self.on_link_down()
self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
pending = self.te_client.get_pending()
self.assertEqual(len(pending), 0)
def test_repeated_clear_pop(self):
"""
test_repeated_clear_pop
Test endpoint queue pop manipulators.
"""
# Set up for events expected.
self._no_queue_mod_evts = self._no_requests
for i in range(3):
self._queue_mod_evts = []
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evts = AsyncResult()
# Queue commands.
self._requests_sent = {}
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Confirm queue mod events.
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
# Confirm get queue with no id.
queue = self.te_client.get_queue()
self.assertEqual(len(queue), self._no_requests)
# Reset queue mod expected events.
self._queue_mod_evts = []
self._no_queue_mod_evts = 1
self._done_queue_mod_evts = AsyncResult()
# Clear queue with no id.
poped = self.te_client.clear_queue()
# Confirm queue mod event and mods.
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
queue = self.te_client.get_queue()
self.assertEqual(len(poped), self._no_requests)
self.assertEqual(len(queue), 0)
self._queue_mod_evts = []
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evts = AsyncResult()
# Queue commands.
self._requests_sent = {}
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Confirm queue mod events.
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
# Confirm get queue with no id.
queue = self.te_client.get_queue()
self.assertEqual(len(queue), self._no_requests)
# Pop a few commands from the queue, confirm events.
self._queue_mod_evts = []
self._no_queue_mod_evts = 3
self._done_queue_mod_evts = AsyncResult()
cmd_ids = self._requests_sent.keys()[:3]
poped = []
for x in cmd_ids:
poped.append(self.te_client.pop_queue(x))
self._requests_sent.pop(x)
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
queue = self.te_client.get_queue()
self.assertEqual(len(poped), 3)
self.assertEqual(len(queue), self._no_requests - 3)
self._no_telem_evts = 2
self._no_requests = self._no_requests - 3
self._no_cmd_tx_evts = self._no_requests
self.on_link_up()
self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
self.on_link_down()
self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
pending = self.te_client.get_pending()
self.assertEqual(len(pending), 0)
def test_get_pending(self):
"""
test_process_queued
Test forwarding of queued commands upon link up.
"""
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._no_telem_evts = 2
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
self.on_link_up()
self._no_requests = 3
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
pending = self.te_client.get_pending(resource_id=self._resource_id)
for x in pending:
self.assertIn(x.command_id, self._requests_sent.keys())
self._no_requests = 10
self._done_evt = AsyncResult()
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
pending = self.te_client.get_pending()
self.assertEqual(len(pending), 0)
self.on_link_down()
self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
#@unittest.skip('Wait for verification of resource registry use.')
def test_persistent_queue(self):
"""
test_persistent_queue
Test ability of endpoint to restore a persistent queue, survive
reboot, etc.
"""
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._no_telem_evts = 2
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Confirm queue mod events.
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
# Confirm get queue with no id.
queue = self.te_client.get_queue()
self.assertEqual(len(queue), self._no_requests)
# Stop and restart the endpoint process.
# Verify the queue is restored.
self._container_client.terminate_process(self._te_pid)
# Create agent config.
endpoint_config = {
'other_host' : self._other_host,
'other_port' : self._other_port,
'this_port' : 0,
'xs_name' : self._xs_name,
'platform_resource_id' : self._platform_resource_id,
'process' : {
'listen_name' : self._listen_name
}
}
# Spawn the terrestrial enpoint process.
log.debug('Spawning terrestrial endpoint process.')
self._te_pid = self._container_client.spawn_process(
name='remote_endpoint_1',
module='ion.services.sa.tcaa.terrestrial_endpoint',
cls='TerrestrialEndpoint',
config=endpoint_config)
log.debug('Endpoint pid=%s.', str(self._te_pid))
# Create an endpoint client.
# The to_name may be either the process pid or
# the listen_name, which for this remote bridge
# is svc_name + remote_name as above.
self.te_client = TerrestrialEndpointClient(
process=FakeProcess(),
to_name=self._listen_name)
log.debug('Got te client %s.', str(self.te_client))
# Remember the terrestrial port.
self._this_port = self.te_client.get_port()
# Confirm we restored the queue with the previous commands.
queue = self.te_client.get_queue()
self.assertEqual(len(queue), self._no_requests)
self.on_link_up()
self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
pending = self.te_client.get_pending()
self.assertEqual(len(pending), 0)
self.on_link_down()
self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
class RemoteClient(object):
"""
A proxy client for any service or resource that forwards commands
to the terrestrial endpoint for transmission.
"""
implements(ITerrestrialEndpoint)
def __init__(self, iface=None, xs_name=None, resource_id=None,
svc_name=None, process=None):
"""
Construct remote proxy client.
Verify required parameters.
Construct remote endpoint client.
Set internal variables.
Construct and add service method attributes.
"""
# Throw an exception if required interface arg not provided.
if not isinstance(iface, InterfaceClass):
raise ConfigNotFound('Invalid interface parameter.')
# Throw an exception if the xs_name is not provided.
if not isinstance(xs_name, str) or xs_name == '':
raise ConfigNotFound('Invalid exchange space name parameter.')
# Create the endpoint client.
# Throw exception if unsuccessful.
to_name = 'terrestrial_endpoint' + xs_name
self._te_client = TerrestrialEndpointClient(process=process,
to_name=to_name)
# Must define a resource id or service name.
if not resource_id and not svc_name:
raise ConfigNotFound('No resource or service specified.')
# Can't specify both a resource and a service.
if resource_id and svc_name:
raise ConfigNotFound('Can\'t specify both a resource and a service.')
self._resource_id = resource_id
self._xs_name = xs_name
self._svc_name = svc_name
# Grab the service method names.
methods = iface.names()
# Generate the service interface.
# Each will forward to the terrestrial endpoint passing
# the function name, args and kwargs.
for m in methods:
setattr(self, m, self.generate_service_method(m))
# Declare the dynamic interface.
directlyProvides(self, iface)
# Initialize the async results objects for blocking behavior.
#self._async_result_evt = None
def generate_service_method(self, name):
"""
A closure that returns a function for forwarding service calls.
The service call name is stored as a kwarg.
"""
def func(*args, **kwargs):
args = copy.deepcopy(args)
kwargs = copy.deepcopy(kwargs)
kwargs['func_name'] = name
return self.forward(*args, **kwargs)
return func
def forward(self, *args, **kwargs):
"""
Forward a service method to the terrestrial endpoint
through the service interface.
"""
func_name = kwargs.pop('func_name')
try:
link = kwargs.pop('link')
except KeyError:
link = True
cid = ''
try:
remote_timeout = kwargs.pop('remote_timeout')
if not isinstance(remote_timeout, int):
remote_timeout = 0
elif remote_timeout < 0:
remote_timeout = 0
elif remote_timeout == 0:
pass
else:
cid = str(uuid.uuid4())
except KeyError:
remote_timeout = 0
cmd = IonObject('RemoteCommand',
resource_id=self._resource_id,
svc_name=self._svc_name,
command=func_name,
command_id=cid,
args= args,
kwargs= kwargs)
if remote_timeout == 0 :
return self._te_client.enqueue_command(cmd, link)
else:
if self._resource_id:
origin = self._resource_id
elif self._svc_name:
origin = self._svc_name + self._xs_name
pending_cmd = cmd
async_result_evt = AsyncResult()
def result_callback(evt, *args, **kwargs):
"""
Callback for subscriber retrive blocking results.
"""
#global async_result_evt
if evt.type_ == 'RemoteCommandResult':
cmd = evt.command
if cmd.command_id == pending_cmd.command_id:
async_result_evt.set(cmd)
sub = EventSubscriber(
event_type='RemoteCommandResult',
origin=origin,
callback=result_callback)
sub.start()
#self._pending_cmd = cmd
cmd = self._te_client.enqueue_command(cmd, link)
try:
result = async_result_evt.get(timeout=remote_timeout)
#self._pending_cmd = None
sub.stop()
except gevent.Timeout:
#self._pending_cmd = None
sub.stop()
raise Timeout('Timed out waiting for remote result.')
return result
"""
def _result_callback(self, evt, *args, **kwargs):
#Callback for subscriber retrive blocking results.
if evt.type_ == 'RemoteCommandResult':
cmd = evt.command
if self._pending_cmd:
if cmd.command_id == self._pending_cmd.command_id:
self._pending_cmd = None
if self._async_result_evt:
self._async_result_evt.set(cmd)
"""
def enqueue_command(self, command=None, link=False):
"""
Enqueue command with terrestrial endoint.
"""
return self._te_client(command, link)
def get_queue(self):
"""
Get terrestrial endpoint queue for this resource/service.
"""
if self._resource_id:
return self._te_client.get_queue(resource_id=self._resource_id)
elif self._svc_name:
return self._te_client.get_queue(svc_name=self._svc_name)
def clear_queue(self):
"""
Clear terrestrial endpoint queue for this resource/service.
"""
if self._resource_id:
return self._te_client.clear_queue(resource_id=self._resource_id)
elif self._svc_name:
return self._te_client.clear_queue(svc_name=self._svc_name)
def pop_queue(self, command_id=''):
"""
Pop a command from the terrestrial endpoint queue.
"""
return self._te_client.pop_queue(command_id=command_id)
def get_pending(self):
"""
Get pending commands for this resource/service.
"""
if self._resource_id:
return self._te_client.get_pending(resource_id=self._resource_id)
elif self._svc_name:
return self._te_client.get_pending(svc_name=self._svc_name)
def get_port(self):
"""
Not supported for remote proxy clients.
"""
raise BadRequest('get_port not available via remote client.')
def set_client_port(self, port=0):
"""
Not supported for remote proxy clients.
"""
raise BadRequest('set_client_port not available via remote client.')
def get_client_port(self):
"""
Not supported for remote proxy clients.
"""
raise BadRequest('get_client_port not available via remote client.')
class TestTerrestrialEndpoint(IonIntegrationTestCase):
"""
Test cases for 2CAA terrestrial endpoint.
"""
def setUp(self):
"""
Setup fake remote components.
Start remote server.
Set internal configuration and test variables.
Start container.
Start services.
Spawn endpoint.
Create and start subscribers.
"""
# Create fake remote client and server.
# Add clean up to shut down properly.
# Start remote server on a random port.
self._remote_server = R3PCServer(self.consume_req,
self.remote_server_close)
self._remote_client = R3PCClient(self.consume_ack,
self.remote_client_close)
self.addCleanup(self._remote_server.stop)
self.addCleanup(self._remote_client.stop)
self._other_port = self._remote_server.start('*', 0)
log.debug('Remote server binding to *:%i', self._other_port)
# Set internal variables.
self._other_host = 'localhost'
self._xs_name = 'remote1'
self._svc_name = 'terrestrial_endpoint'
self._listen_name = self._svc_name + self._xs_name
self._platform_resource_id = 'abc123'
self._resource_id = 'fake_id'
self._rmt_svc_name = 'fake_svc'
self._no_requests = 10
self._requests_sent = {}
self._results_recv = {}
self._workers = []
self._done_evt = AsyncResult()
self._queue_mod_evts = []
self._cmd_tx_evts = []
self._telem_evts = []
self._no_telem_evts = 0
self._no_queue_mod_evts = 0
self._no_cmd_tx_evts = 0
self._done_queue_mod_evts = AsyncResult()
self._done_telem_evts = AsyncResult()
self._done_cmd_tx_evts = AsyncResult()
# Start container.
log.debug('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message)
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Create a container client.
log.debug('Creating container client.')
self._container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
# The following spawn config creates the process with the remote
# name tagged to the service name.
"""
listen_name = terrestrial_endpointremote1
2012-10-10 11:34:46,654 DEBUG ion.services.sa.tcaa.terrestrial_endpoint recv name: NP (ion_test_8257ab,terrestrial_endpointremote1,B: terrestrial_endpointremote1)
2012-10-10 11:34:46,654 DEBUG ion.services.sa.tcaa.terrestrial_endpoint startup listener recv name: NP (ion_test_8257ab,terrestrial_endpointremote1,B: terrestrial_endpointremote1)
2012-10-10 11:34:46,654 DEBUG ion.services.sa.tcaa.terrestrial_endpoint startup listener recv name: NP (ion_test_8257ab,Edwards-MacBook-Pro_local_2624.33,B: Edwards-MacBook-Pro_local_2624.33)
"""
# Create agent config.
endpoint_config = {
'other_host': self._other_host,
'other_port': self._other_port,
'this_port': 0,
'xs_name': self._xs_name,
'platform_resource_id': self._platform_resource_id,
'process': {
'listen_name': self._listen_name
}
}
# Spawn the terrestrial enpoint process.
log.debug('Spawning terrestrial endpoint process.')
self._te_pid = self._container_client.spawn_process(
name='remote_endpoint_1',
module='ion.services.sa.tcaa.terrestrial_endpoint',
cls='TerrestrialEndpoint',
config=endpoint_config)
log.debug('Endpoint pid=%s.', str(self._te_pid))
# Create an endpoint client.
# The to_name may be either the process pid or
# the listen_name, which for this remote bridge
# is svc_name + remote_name as above.
self.te_client = TerrestrialEndpointClient(process=FakeProcess(),
to_name=self._listen_name)
log.debug('Got te client %s.', str(self.te_client))
# Remember the terrestrial port.
self._this_port = self.te_client.get_port()
# Start the event publisher.
self._event_publisher = EventPublisher()
# Start the event subscriber.
self._event_subscriber = EventSubscriber(event_type='PlatformEvent',
callback=self.consume_event,
origin=self._xs_name)
self._event_subscriber.start()
self._event_subscriber._ready_event.wait(
timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._event_subscriber.stop)
# Start the result subscriber.
self._result_subscriber = EventSubscriber(
event_type='RemoteCommandResult',
origin=self._resource_id,
callback=self.consume_event)
self._result_subscriber.start()
self._result_subscriber._ready_event.wait(
timeout=CFG.endpoint.receive.timeout)
self.addCleanup(self._result_subscriber.stop)
def consume_event(self, evt, *args, **kwargs):
"""
Test callback for events.
"""
log.debug('Got event: %s, args: %s, kwargs: %s', str(evt), str(args),
str(kwargs))
if evt.type_ == 'PublicPlatformTelemetryEvent':
self._telem_evts.append(evt)
if self._no_telem_evts > 0 and self._no_telem_evts == len(
self._telem_evts):
self._done_telem_evts.set()
elif evt.type_ == 'RemoteQueueModifiedEvent':
self._queue_mod_evts.append(evt)
if self._no_queue_mod_evts > 0 and self._no_queue_mod_evts == len(
self._queue_mod_evts):
self._done_queue_mod_evts.set()
elif evt.type_ == 'RemoteCommandTransmittedEvent':
self._cmd_tx_evts.append(evt)
if self._no_cmd_tx_evts > 0 and self._no_cmd_tx_evts == len(
self._cmd_tx_evts):
self._done_cmd_tx_evts.set()
elif evt.type_ == 'RemoteCommandResult':
cmd = evt.command
self._results_recv[cmd.command_id] = cmd
if len(self._results_recv) == self._no_requests:
self._done_evt.set()
def on_link_up(self):
"""
Called by a test to simulate turning the link on.
"""
log.debug('Remote client connecting to localhost:%i.', self._this_port)
self._remote_client.start('localhost', self._this_port)
# Publish a link up event to be caught by the endpoint.
log.debug('Publishing telemetry event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._platform_resource_id,
status=TelemetryStatusType.AVAILABLE)
def on_link_down(self):
"""
Called by a test to simulate turning the link off.
"""
self._remote_client.stop()
# Publish a link down event to be caught by the endpoint.
log.debug('Publishing telemetry event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._platform_resource_id,
status=TelemetryStatusType.UNAVAILABLE)
def consume_req(self, request):
"""
Remote request callback.
Fire a greenlet to do some fake work before returning via
the remote client to terrestrial endpoint.
"""
# Spawn a greenlet to sleep briefly with each request and
# then respond with a result through the remote client.
log.debug('Remote endpoint got request: %s', str(request))
greenlet = gevent.spawn(self.process_remote_request, request)
self._workers.append(greenlet)
def consume_ack(self, request):
"""
Remote ack callback.
"""
log.debug('Remote endpoint got ack: %s', str(request))
def process_remote_request(self, request):
"""
Process remote request.
Do random amount of fake work and enqueue result for return to
terrestrial endpoint.
"""
worktime = random.uniform(.1, 3)
gevent.sleep(worktime)
result = {'command_id': request.command_id, 'result': 'fake_result'}
log.debug('Finished processing request: %s', str(request))
self._remote_client.enqueue(result)
def remote_server_close(self):
"""
Remote server closed callback.
"""
log.debug('The remote server closed.')
def remote_client_close(self):
"""
Remoe client closed callback.
"""
log.debug('The remote client closed.')
def make_fake_command(self, no):
"""
Build a fake command for use in tests.
"""
cmdstr = 'fake_cmd_%i' % no
cmd = IonObject('RemoteCommand',
resource_id=self._resource_id,
command=cmdstr,
args=['arg1', 23],
kwargs={'kwargs1': 'someval'})
return cmd
def make_fake_svc_command(self, no):
"""
Build a fake command for use in tests.
"""
cmdstr = 'fake_cmd_%i' % no
cmd = IonObject('RemoteCommand',
svc_name=self._rmt_svc_name,
command=cmdstr,
args=['arg1', 23],
kwargs={'kwargs1': 'someval'})
return cmd
def test_process_queued(self):
"""
test_process_queued
Test forwarding of queued commands upon link up.
"""
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._no_telem_evts = 2
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
self.on_link_up()
self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
pending = self.te_client.get_pending()
self.assertEqual(len(pending), 0)
self.on_link_down()
self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_process_online(self):
"""
test_process_online
Test forwarding commands when the link is up.
"""
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._no_telem_evts = 2
self.on_link_up()
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
gevent.sleep(.2)
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
pending = self.te_client.get_pending()
self.assertEqual(len(pending), 0)
self.on_link_down()
self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_remote_late(self):
"""
test_remote_late
Test simulates behavior when the remote side is initially unavailable.
"""
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._no_telem_evts = 2
self.on_link_up()
gevent.sleep(2)
self._remote_server.stop()
self._remote_client.stop()
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
gevent.sleep(3)
self._remote_client.start('localhost', self._this_port)
self._remote_server.start('*', self._other_port)
self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
pending = self.te_client.get_pending()
self.assertEqual(len(pending), 0)
self.on_link_down()
self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_get_clear_queue(self):
"""
test_get_clear_queue
Test endpoint queue get and clear manipulators.
"""
# Set up for events expected.
self._no_queue_mod_evts = self._no_requests * 2
# Queue commands.
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Queue commands.
for i in range(self._no_requests):
cmd = self.make_fake_svc_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Confirm queue mod events.
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
# Confirm get queue with no id.
queue = self.te_client.get_queue()
self.assertEqual(len(queue), self._no_requests * 2)
# Confirm get queue with id.
queue = self.te_client.get_queue(resource_id=self._resource_id)
self.assertEqual(len(queue), self._no_requests)
# Confirm get queue with svc name.
queue = self.te_client.get_queue(svc_name=self._rmt_svc_name)
self.assertEqual(len(queue), self._no_requests)
# Confirm get queue with bogus id.
queue = self.te_client.get_queue(resource_id='bogus_id')
self.assertEqual(len(queue), 0)
# Confirm get queue with bogus id.
queue = self.te_client.get_queue(svc_name='bogus_svc')
self.assertEqual(len(queue), 0)
# Reset queue mod expected events.
self._queue_mod_evts = []
self._no_queue_mod_evts = 1
self._done_queue_mod_evts = AsyncResult()
# Clear queue with no id.
poped = self.te_client.clear_queue()
# Confirm queue mod event and mods.
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
queue = self.te_client.get_queue()
self.assertEqual(len(poped), self._no_requests * 2)
self.assertEqual(len(queue), 0)
# Queue new commands and confirm event.
self._queue_mod_evts = []
self._no_queue_mod_evts = self._no_requests * 2
self._done_queue_mod_evts = AsyncResult()
self._requests_sent = {}
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
for i in range(self._no_requests):
cmd = self.make_fake_svc_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
# Reset queue mod expected events.
self._queue_mod_evts = []
self._no_queue_mod_evts = 1
self._done_queue_mod_evts = AsyncResult()
# Clear queue with id.
poped = self.te_client.clear_queue(resource_id=self._resource_id)
# Confirm mods and mod events.
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
queue = self.te_client.get_queue()
self.assertEqual(len(poped), self._no_requests)
self.assertEqual(len(queue), self._no_requests)
# Reset queue mod expected events.
self._queue_mod_evts = []
self._no_queue_mod_evts = 1
self._done_queue_mod_evts = AsyncResult()
# Clear queue with id.
poped = self.te_client.clear_queue(svc_name=self._rmt_svc_name)
# Confirm mods and mod events.
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
queue = self.te_client.get_queue()
self.assertEqual(len(poped), self._no_requests)
self.assertEqual(len(queue), 0)
# Queue new commands and confirm events.
self._queue_mod_evts = []
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evts = AsyncResult()
self._requests_sent = {}
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
# Clear queue with bogus id.
poped = self.te_client.clear_queue(resource_id='bogus id')
queue = self.te_client.get_queue()
self.assertEqual(len(poped), 0)
self.assertEqual(len(queue), self._no_requests)
# Clear queue with bogus svc name.
poped = self.te_client.clear_queue(svc_name='bogus id')
queue = self.te_client.get_queue()
self.assertEqual(len(poped), 0)
self.assertEqual(len(queue), self._no_requests)
# Clear queue and confirm empty.
self.te_client.clear_queue()
queue = self.te_client.get_queue()
self.assertEqual(len(queue), 0)
# Turn on link and wait a few seconds.
# Confirm no data or tx events arrive.
self.on_link_up()
gevent.sleep(2)
self.assertEqual(len(self._cmd_tx_evts), 0)
self.assertEqual(len(self._results_recv), 0)
self._no_telem_evts = 2
self.on_link_down()
self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout)
def test_pop_pending_queue(self):
"""
test_pop_pending_queue
Test endpoint queue pop manipulators.
"""
# Set up for events expected.
self._no_queue_mod_evts = self._no_requests
# Queue commands.
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Confirm queue mod events.
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
queue = self.te_client.get_queue()
self.assertEqual(len(queue), self._no_requests)
# Pop a few commands from the queue, confirm events.
self._queue_mod_evts = []
self._no_queue_mod_evts = 3
self._done_queue_mod_evts = AsyncResult()
cmd_ids = self._requests_sent.keys()[:3]
poped = []
for x in cmd_ids:
poped.append(self.te_client.pop_queue(x))
self._requests_sent.pop(x)
# Try poping with illegal args. This should have no effect
poped.append(self.te_client.pop_queue())
poped.append(self.te_client.pop_queue('bogus id'))
poped = [x for x in poped if x != None]
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
queue = self.te_client.get_queue()
self.assertEqual(len(poped), 3)
self.assertEqual(len(queue), self._no_requests - 3)
# Turn on the link and verify that only the remaining commands
# get processed.
self._no_telem_evts = 2
self._no_requests = self._no_requests - 3
self._no_cmd_tx_evts = self._no_requests
self.on_link_up()
self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
self.on_link_down()
self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
pending = self.te_client.get_pending()
self.assertEqual(len(pending), 0)
def test_repeated_clear_pop(self):
"""
test_repeated_clear_pop
Test endpoint queue pop manipulators.
"""
# Set up for events expected.
self._no_queue_mod_evts = self._no_requests
for i in range(3):
self._queue_mod_evts = []
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evts = AsyncResult()
# Queue commands.
self._requests_sent = {}
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Confirm queue mod events.
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
# Confirm get queue with no id.
queue = self.te_client.get_queue()
self.assertEqual(len(queue), self._no_requests)
# Reset queue mod expected events.
self._queue_mod_evts = []
self._no_queue_mod_evts = 1
self._done_queue_mod_evts = AsyncResult()
# Clear queue with no id.
poped = self.te_client.clear_queue()
# Confirm queue mod event and mods.
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
queue = self.te_client.get_queue()
self.assertEqual(len(poped), self._no_requests)
self.assertEqual(len(queue), 0)
self._queue_mod_evts = []
self._no_queue_mod_evts = self._no_requests
self._done_queue_mod_evts = AsyncResult()
# Queue commands.
self._requests_sent = {}
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Confirm queue mod events.
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
# Confirm get queue with no id.
queue = self.te_client.get_queue()
self.assertEqual(len(queue), self._no_requests)
# Pop a few commands from the queue, confirm events.
self._queue_mod_evts = []
self._no_queue_mod_evts = 3
self._done_queue_mod_evts = AsyncResult()
cmd_ids = self._requests_sent.keys()[:3]
poped = []
for x in cmd_ids:
poped.append(self.te_client.pop_queue(x))
self._requests_sent.pop(x)
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
queue = self.te_client.get_queue()
self.assertEqual(len(poped), 3)
self.assertEqual(len(queue), self._no_requests - 3)
self._no_telem_evts = 2
self._no_requests = self._no_requests - 3
self._no_cmd_tx_evts = self._no_requests
self.on_link_up()
self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
self.on_link_down()
self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
pending = self.te_client.get_pending()
self.assertEqual(len(pending), 0)
def test_get_pending(self):
"""
test_process_queued
Test forwarding of queued commands upon link up.
"""
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._no_telem_evts = 2
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
self.on_link_up()
self._no_requests = 3
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
pending = self.te_client.get_pending(resource_id=self._resource_id)
for x in pending:
self.assertIn(x.command_id, self._requests_sent.keys())
self._no_requests = 10
self._done_evt = AsyncResult()
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
pending = self.te_client.get_pending()
self.assertEqual(len(pending), 0)
self.on_link_down()
self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
#@unittest.skip('Wait for verification of resource registry use.')
def test_persistent_queue(self):
"""
test_persistent_queue
Test ability of endpoint to restore a persistent queue, survive
reboot, etc.
"""
self._no_cmd_tx_evts = self._no_requests
self._no_queue_mod_evts = self._no_requests
self._no_telem_evts = 2
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
cmd = self.te_client.enqueue_command(cmd)
self._requests_sent[cmd.command_id] = cmd
# Confirm queue mod events.
self._done_queue_mod_evts.get(timeout=CFG.endpoint.receive.timeout)
# Confirm get queue with no id.
queue = self.te_client.get_queue()
self.assertEqual(len(queue), self._no_requests)
# Stop and restart the endpoint process.
# Verify the queue is restored.
self._container_client.terminate_process(self._te_pid)
# Create agent config.
endpoint_config = {
'other_host': self._other_host,
'other_port': self._other_port,
'this_port': 0,
'xs_name': self._xs_name,
'platform_resource_id': self._platform_resource_id,
'process': {
'listen_name': self._listen_name
}
}
# Spawn the terrestrial enpoint process.
log.debug('Spawning terrestrial endpoint process.')
self._te_pid = self._container_client.spawn_process(
name='remote_endpoint_1',
module='ion.services.sa.tcaa.terrestrial_endpoint',
cls='TerrestrialEndpoint',
config=endpoint_config)
log.debug('Endpoint pid=%s.', str(self._te_pid))
# Create an endpoint client.
# The to_name may be either the process pid or
# the listen_name, which for this remote bridge
# is svc_name + remote_name as above.
self.te_client = TerrestrialEndpointClient(process=FakeProcess(),
to_name=self._listen_name)
log.debug('Got te client %s.', str(self.te_client))
# Remember the terrestrial port.
self._this_port = self.te_client.get_port()
# Confirm we restored the queue with the previous commands.
queue = self.te_client.get_queue()
self.assertEqual(len(queue), self._no_requests)
self.on_link_up()
self._done_cmd_tx_evts.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
pending = self.te_client.get_pending()
self.assertEqual(len(pending), 0)
self.on_link_down()
self._done_telem_evts.get(timeout=CFG.endpoint.receive.timeout)
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
