def test_pub_on_different_subtypes(self): ar = event.AsyncResult() gq = queue.Queue() self.count = 0 def cb(event, *args, **kwargs): self.count += 1 gq.put(event) if event.description == "end": ar.set() sub = EventSubscriber(event_type="ResourceModifiedEvent", sub_type="st1", callback=cb) sub.activate() pub1 = EventPublisher(event_type="ResourceModifiedEvent") pub2 = EventPublisher(event_type="ContainerLifecycleEvent") pub1.publish_event(origin="two", sub_type="st2", description="2") pub2.publish_event(origin="three", sub_type="st1", description="3") pub1.publish_event(origin="one", sub_type="st1", description="1") pub1.publish_event(origin="four", sub_type="st1", description="end") ar.get(timeout=5) sub.deactivate() res = [] for x in xrange(self.count): res.append(gq.get(timeout=5)) self.assertEquals(len(res), 2) self.assertEquals(res[0].description, "1")
def test_pub_on_different_subtypes(self): ar = event.AsyncResult() gq = queue.Queue() self.count = 0 def cb(event, *args, **kwargs): self.count += 1 gq.put(event) if event.description == "end": ar.set() sub = EventSubscriber(event_type="ResourceModifiedEvent", sub_type="st1", callback=cb) sub.activate() pub1 = EventPublisher(event_type="ResourceModifiedEvent") pub2 = EventPublisher(event_type="ContainerLifecycleEvent") pub1.publish_event(origin="two", sub_type="st2", description="2") pub2.publish_event(origin="three", sub_type="st1", description="3") pub1.publish_event(origin="one", sub_type="st1", description="1") pub1.publish_event(origin="four", sub_type="st1", description="end") ar.get(timeout=5) sub.deactivate() res = [] for x in xrange(self.count): res.append(gq.get(timeout=5)) self.assertEquals(len(res), 2) self.assertEquals(res[0].description, "1")
def test_l4_ci_sa_rq_145_323(self): """ Instrument management shall update physical resource metadata when change occurs For example, when there is a change of state. note from maurice 2012-05-18: consider this to mean a change of stored RR data """ inst_obj = any_old(RT.InstrumentDevice) instrument_device_id, _ = self.RR.create(inst_obj) self.received_event = AsyncResult() #Create subscribers for agent and driver events. def consume_event(*args, **kwargs): self.received_event.set(True) log.info("L4-CI-SA-RQ-323") log.info("L4-CI-SA-RQ-145") event_sub = EventSubscriber(event_type="ResourceModifiedEvent", callback=consume_event) event_sub.activate() inst_obj = self.RR.read(instrument_device_id) inst_obj.description = "brand new description" self.RR.update(inst_obj) #wait for event result = self.received_event.get(timeout=10) event_sub.deactivate() self.assertTrue(result)
def _start_event_subscribers(self): """ Create subscribers for agent and driver events. """ def consume_event(*args, **kwargs): log.info('Test recieved ION event: args=%s kwargs=%s.', str(args), str(kwargs)) self._events_received.append(args[0]) if self._no_events and self._no_events == len(self._event_received): self._async_event_result.set() event_sub = EventSubscriber(event_type="DeviceEvent", callback=consume_event) event_sub.activate() self._event_subscribers.append(event_sub)
def _start_event_subscribers(self): """ Create subscribers for agent and driver events. """ def consume_event(*args, **kwargs): log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.', str(args), str(kwargs), str(args[0])) self._events_received.append(args[0]) if self._no_events and self._no_events == len(self._event_received): self._async_event_result.set() event_sub = EventSubscriber(event_type="DeviceEvent", callback=consume_event) event_sub.activate() self._event_subscribers.append(event_sub)
def __init__(self): # Start event subscribers, add stop to cleanup. self.no_events = None self.async_event_result = AsyncResult() self.events_received = [] self.event_subscribers = [] def consume_event(*args, **kwargs): log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.', str(args), str(kwargs), str(args[0])) self.events_received.append(args[0]) if self.no_events and self.no_events == len(self.event_received): self.async_event_result.set() event_sub = EventSubscriber(event_type="DeviceEvent", callback=consume_event) event_sub.activate() self.event_subscribers.append(event_sub)
class Notification(object): """ Encapsulates a notification's info and it's event subscriber @David - is this class needed? It does not seem to serve any purpose? """ def __init__(self, notification_request=None, subscriber_callback=None): self._res_obj = notification_request # The notification Request Resource Object # setup subscription using subscription_callback() # msg_recipientDO: make this walk the lists and set up a subscriber for every pair of # origin/event. This will require a list to hold all the subscribers so they can # be started and killed self.subscriber = EventSubscriber( origin=notification_request.origin, origin_type=notification_request.origin_type, event_type=notification_request.event_type, sub_type=notification_request.event_subtype, callback=subscriber_callback) self.notification_id = None def set_notification_id(self, id_=None): """ Set the notification id of the notification object @param notification id """ self.notification_id = id_ def activate(self): """ Start subscribing """ self.subscriber.activate() def deactivate(self): """ Stop subscribing """ self.subscriber.deactivate()
class Notification(object): """ Encapsulates a notification's info and it's event subscriber @David - is this class needed? It does not seem to serve any purpose? """ def __init__(self, notification_request=None, subscriber_callback=None): self._res_obj = notification_request # The notification Request Resource Object # setup subscription using subscription_callback() # msg_recipientDO: make this walk the lists and set up a subscriber for every pair of # origin/event. This will require a list to hold all the subscribers so they can # be started and killed self.subscriber = EventSubscriber(origin=notification_request.origin, origin_type = notification_request.origin_type, event_type=notification_request.event_type, sub_type=notification_request.event_subtype, callback=subscriber_callback) self.notification_id = None def set_notification_id(self, id_=None): """ Set the notification id of the notification object @param notification id """ self.notification_id = id_ def activate(self): """ Start subscribing """ self.subscriber.activate() def deactivate(self): """ Stop subscribing """ self.subscriber.deactivate()
def test_pub_on_different_subsubtypes(self): res_list = [DotDict(ar=event.AsyncResult(), gq=queue.Queue(), count=0) for i in xrange(4)] def cb_gen(num): def cb(event, *args, **kwargs): res_list[num].count += 1 res_list[num].gq.put(event) if event.description == "end": res_list[num].ar.set() return cb sub0 = EventSubscriber(event_type="ResourceModifiedEvent", sub_type="st1.*", callback=cb_gen(0)) sub0.activate() sub1 = EventSubscriber(event_type="ResourceModifiedEvent", sub_type="st1.a", callback=cb_gen(1)) sub1.activate() sub2 = EventSubscriber(event_type="ResourceModifiedEvent", sub_type="*.a", callback=cb_gen(2)) sub2.activate() sub3 = EventSubscriber(event_type="ResourceModifiedEvent", sub_type="st1", callback=cb_gen(3)) sub3.activate() pub1 = EventPublisher(event_type="ResourceModifiedEvent") pub1.publish_event(origin="one", sub_type="st1.a", description="1") pub1.publish_event(origin="two", sub_type="st1", description="2") pub1.publish_event(origin="three", sub_type="st1.b", description="3") pub1.publish_event(origin="four", sub_type="st2.a", description="4") pub1.publish_event(origin="five", sub_type="st2", description="5") pub1.publish_event(origin="six", sub_type="a", description="6") pub1.publish_event(origin="seven", sub_type="", description="7") pub1.publish_event(origin="end", sub_type="st1.a", description="end") pub1.publish_event(origin="end", sub_type="st1", description="end") [res_list[i].ar.get(timeout=5) for i in xrange(3)] sub0.deactivate() sub1.deactivate() sub2.deactivate() sub3.deactivate() for i in xrange(4): res_list[i].res = [] for x in xrange(res_list[i].count): res_list[i].res.append(res_list[i].gq.get(timeout=5)) self.assertEquals(len(res_list[0].res), 3) self.assertEquals(res_list[0].res[0].description, "1") self.assertEquals(len(res_list[1].res), 2) self.assertEquals(res_list[1].res[0].description, "1") self.assertEquals(len(res_list[2].res), 3) self.assertEquals(res_list[2].res[0].description, "1") self.assertEquals(len(res_list[3].res), 2) self.assertEquals(res_list[3].res[0].description, "2")
class ExternalDatasetAgentTestBase(object): # Agent parameters. EDA_RESOURCE_ID = '123xyz' EDA_NAME = 'ExampleEDA' EDA_MOD = 'ion.agents.data.external_dataset_agent' EDA_CLS = 'ExternalDatasetAgent' """ Test cases for instrument agent class. Functions in this class provide instrument agent integration tests and provide a tutorial on use of the agent setup and interface. """ def setUp(self): """ Initialize test members. """ # log.warn('Starting the container') # Start container. self._start_container() # Bring up services in a deploy file # log.warn('Starting the rel') self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Create a pubsub client to create streams. # log.warn('Init a pubsub client') self._pubsub_client = PubsubManagementServiceClient( node=self.container.node) # log.warn('Init a ContainerAgentClient') self._container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) # Data async and subscription TODO: Replace with new subscriber self._finished_count = None #TODO: Switch to gevent.queue.Queue self._async_finished_result = AsyncResult() self._finished_events_received = [] self._finished_event_subscriber = None self._start_finished_event_subscriber() self.addCleanup(self._stop_finished_event_subscriber) # TODO: Finish dealing with the resources and whatnot # TODO: DVR_CONFIG and (potentially) stream_config could both be reconfigured in self._setup_resources() self._setup_resources() #TG: Setup/configure the granule logger to log granules as they're published # Create agent config. agent_config = { 'driver_config': self.DVR_CONFIG, 'stream_config': {}, 'agent': { 'resource_id': self.EDA_RESOURCE_ID }, 'test_mode': True } # Start instrument agent. self._ia_pid = None log.debug('TestInstrumentAgent.setup(): starting EDA.') self._ia_pid = self._container_client.spawn_process( name=self.EDA_NAME, module=self.EDA_MOD, cls=self.EDA_CLS, config=agent_config) log.info('Agent pid=%s.', str(self._ia_pid)) # Start a resource agent client to talk with the instrument agent. self._ia_client = None self._ia_client = ResourceAgentClient(self.EDA_RESOURCE_ID, process=FakeProcess()) log.info('Got ia client %s.', str(self._ia_client)) ######################################## # Private "setup" functions ######################################## def _setup_resources(self): raise NotImplementedError( '_setup_resources must be implemented in the subclass') def create_stream_and_logger(self, name, stream_id=''): if not stream_id or stream_id is '': stream_id = self._pubsub_client.create_stream(name=name, encoding='ION R2') pid = self._container_client.spawn_process( name=name + '_logger', module='ion.processes.data.stream_granule_logger', cls='StreamGranuleLogger', config={'process': { 'stream_id': stream_id }}) log.info( 'Started StreamGranuleLogger \'{0}\' subscribed to stream_id={1}'. format(pid, stream_id)) return stream_id def _start_finished_event_subscriber(self): def consume_event(*args, **kwargs): if args[0].description == 'TestingFinished': log.debug('TestingFinished event received') self._finished_events_received.append(args[0]) if self._finished_count and self._finished_count == len( self._finished_events_received): log.debug('Finishing test...') self._async_finished_result.set( len(self._finished_events_received)) log.debug( 'Called self._async_finished_result.set({0})'.format( len(self._finished_events_received))) self._finished_event_subscriber = EventSubscriber( event_type='DeviceEvent', callback=consume_event) self._finished_event_subscriber.activate() def _stop_finished_event_subscriber(self): if self._finished_event_subscriber: self._finished_event_subscriber.deactivate() self._finished_event_subscriber = None ######################################## # Custom assertion functions ######################################## def assertListsEqual(self, lst1, lst2): lst1.sort() lst2.sort() return lst1 == lst2 def assertSampleDict(self, val): """ Verify the value is a sample dictionary for the sbe37. """ #{'p': [-6.945], 'c': [0.08707], 't': [20.002], 'time': [1333752198.450622]} self.assertTrue(isinstance(val, dict)) self.assertTrue(val.has_key('c')) self.assertTrue(val.has_key('t')) self.assertTrue(val.has_key('p')) self.assertTrue(val.has_key('time')) c = val['c'][0] t = val['t'][0] p = val['p'][0] time = val['time'][0] self.assertTrue(isinstance(c, float)) self.assertTrue(isinstance(t, float)) self.assertTrue(isinstance(p, float)) self.assertTrue(isinstance(time, float)) def assertParamDict(self, pd, all_params=False): """ Verify all device parameters exist and are correct type. """ if all_params: self.assertEqual(set(pd.keys()), set(PARAMS.keys())) for (key, type_val) in PARAMS.iteritems(): if type_val == list or type_val == tuple: self.assertTrue(isinstance(pd[key], (list, tuple))) else: self.assertTrue(isinstance(pd[key], type_val)) else: for (key, val) in pd.iteritems(): self.assertTrue(PARAMS.has_key(key)) self.assertTrue(isinstance(val, PARAMS[key])) def assertParamVals(self, params, correct_params): """ Verify parameters take the correct values. """ self.assertEqual(set(params.keys()), set(correct_params.keys())) for (key, val) in params.iteritems(): correct_val = correct_params[key] if isinstance(val, float): # Verify to 5% of the larger value. max_val = max(abs(val), abs(correct_val)) self.assertAlmostEqual(val, correct_val, delta=max_val * .01) elif isinstance(val, (list, tuple)): # list of tuple. self.assertEqual(list(val), list(correct_val)) else: # int, bool, str. self.assertEqual(val, correct_val) ######################################## # Test functions ######################################## def test_acquire_data(self): cmd = AgentCommand(command='initialize') _ = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='go_active') _ = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='run') _ = self._ia_client.execute_agent(cmd) self._finished_count = 3 log.info('Send an unconstrained request for data (\'new data\')') cmd = AgentCommand(command='acquire_data') self._ia_client.execute(cmd) log.info( 'Send a second unconstrained request for data (\'new data\'), should be rejected' ) cmd = AgentCommand(command='acquire_data') self._ia_client.execute(cmd) config_mods = {} log.info( 'Send a constrained request for data: constraints = HIST_CONSTRAINTS_1' ) config_mods['stream_id'] = self.create_stream_and_logger( name='stream_id_for_historical_1') config_mods['constraints'] = self.HIST_CONSTRAINTS_1 cmd = AgentCommand(command='acquire_data', args=[config_mods]) self._ia_client.execute(cmd) log.info( 'Send a second constrained request for data: constraints = HIST_CONSTRAINTS_2' ) config_mods['stream_id'] = self.create_stream_and_logger( name='stream_id_for_historical_2') config_mods['constraints'] = self.HIST_CONSTRAINTS_2 # config={'stream_id':'second_historical','TESTING':True, 'constraints':self.HIST_CONSTRAINTS_2} cmd = AgentCommand(command='acquire_data', args=[config_mods]) self._ia_client.execute(cmd) finished = self._async_finished_result.get(timeout=10) self.assertEqual(finished, self._finished_count) cmd = AgentCommand(command='reset') _ = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) def test_acquire_data_while_streaming(self): # Test instrument driver execute interface to start and stop streaming mode. cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) cmd = AgentCommand(command='initialize') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.INACTIVE) cmd = AgentCommand(command='go_active') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.IDLE) cmd = AgentCommand(command='run') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) # Make sure the polling interval is appropriate for a test params = {'POLLING_INTERVAL': 5} self._ia_client.set_param(params) self._finished_count = 2 # Begin streaming. cmd = AgentCommand(command='go_streaming') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.STREAMING) config = get_safe(self.DVR_CONFIG, 'dh_cfg', {}) log.info( 'Send a constrained request for data: constraints = HIST_CONSTRAINTS_1' ) config['stream_id'] = self.create_stream_and_logger( name='stream_id_for_historical_1') config['constraints'] = self.HIST_CONSTRAINTS_1 cmd = AgentCommand(command='acquire_data', args=[config]) reply = self._ia_client.execute(cmd) self.assertNotEqual(reply.status, 660) gevent.sleep(12) # Halt streaming. cmd = AgentCommand(command='go_observatory') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) # Assert that data was received self._async_finished_result.get(timeout=10) self.assertTrue(len(self._finished_events_received) >= 3) cmd = AgentCommand(command='reset') _ = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) def test_streaming(self): # Test instrument driver execute interface to start and stop streaming mode. cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) cmd = AgentCommand(command='initialize') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.INACTIVE) cmd = AgentCommand(command='go_active') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.IDLE) cmd = AgentCommand(command='run') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) # Make sure the polling interval is appropriate for a test params = {'POLLING_INTERVAL': 5} self._ia_client.set_param(params) self._finished_count = 3 # Begin streaming. cmd = AgentCommand(command='go_streaming') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.STREAMING) # Wait for some samples to roll in. gevent.sleep(12) # Halt streaming. cmd = AgentCommand(command='go_observatory') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) # Assert that data was received self._async_finished_result.get(timeout=10) self.assertTrue(len(self._finished_events_received) >= 3) cmd = AgentCommand(command='reset') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) def test_observatory(self): # Test instrument driver get and set interface. cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) cmd = AgentCommand(command='initialize') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.INACTIVE) cmd = AgentCommand(command='go_active') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.IDLE) cmd = AgentCommand(command='run') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) # Retrieve all resource parameters. reply = self._ia_client.get_param('DRIVER_PARAMETER_ALL') self.assertParamDict(reply, True) orig_config = reply ## Retrieve a subset of resource parameters. params = ['POLLING_INTERVAL'] reply = self._ia_client.get_param(params) self.assertParamDict(reply) orig_params = reply # Set a subset of resource parameters. new_params = { 'POLLING_INTERVAL': (orig_params['POLLING_INTERVAL'] * 2), } self._ia_client.set_param(new_params) check_new_params = self._ia_client.get_param(params) self.assertParamVals(check_new_params, new_params) cmd = AgentCommand(command='reset') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) def test_get_set_param(self): cmd = AgentCommand(command='initialize') _ = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='go_active') _ = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='run') _ = self._ia_client.execute_agent(cmd) # Get a couple parameters retval = self._ia_client.get_param( ['POLLING_INTERVAL', 'PATCHABLE_CONFIG_KEYS']) log.debug('Retrieved parameters from agent: {0}'.format(retval)) self.assertTrue(isinstance(retval, dict)) self.assertEqual(type(retval['POLLING_INTERVAL']), int) self.assertEqual(type(retval['PATCHABLE_CONFIG_KEYS']), list) # Attempt to get a parameter that doesn't exist log.debug('Try getting a non-existent parameter \'BAD_PARAM\'') self.assertRaises(InstParameterError, self._ia_client.get_param, ['BAD_PARAM']) # Set the polling_interval to a new value, then get it to make sure it set properly self._ia_client.set_param({'POLLING_INTERVAL': 10}) retval = self._ia_client.get_param(['POLLING_INTERVAL']) log.debug('Retrieved parameters from agent: {0}'.format(retval)) self.assertTrue(isinstance(retval, dict)) self.assertEqual(retval['POLLING_INTERVAL'], 10) # Attempt to set a parameter that doesn't exist log.debug('Try setting a non-existent parameter \'BAD_PARAM\'') self.assertRaises(InstParameterError, self._ia_client.set_param, {'BAD_PARAM': 'bad_val'}) # Attempt to set one parameter that does exist, and one that doesn't self.assertRaises(InstParameterError, self._ia_client.set_param, { 'POLLING_INTERVAL': 20, 'BAD_PARAM': 'bad_val' }) retval = self._ia_client.get_param(['POLLING_INTERVAL']) log.debug('Retrieved parameters from agent: {0}'.format(retval)) self.assertTrue(isinstance(retval, dict)) self.assertEqual(retval['POLLING_INTERVAL'], 20) cmd = AgentCommand(command='reset') _ = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) def test_initialize(self): # Test agent initialize command. This causes creation of driver process and transition to inactive. cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) cmd = AgentCommand(command='initialize') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.INACTIVE) cmd = AgentCommand(command='reset') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) def test_states(self): # Test agent state transitions. cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) cmd = AgentCommand(command='initialize') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.INACTIVE) cmd = AgentCommand(command='go_active') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.IDLE) cmd = AgentCommand(command='run') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) cmd = AgentCommand(command='pause') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.STOPPED) cmd = AgentCommand(command='resume') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) cmd = AgentCommand(command='clear') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.IDLE) cmd = AgentCommand(command='run') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) cmd = AgentCommand(command='pause') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.STOPPED) cmd = AgentCommand(command='clear') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.IDLE) cmd = AgentCommand(command='run') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) self._finished_count = 1 cmd = AgentCommand(command='go_streaming') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.STREAMING) gevent.sleep(5) cmd = AgentCommand(command='go_observatory') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) self._async_finished_result.get(timeout=5) cmd = AgentCommand(command='reset') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) def test_capabilities(self): # Test the ability to retrieve agent and resource parameter and command capabilities. acmds = self._ia_client.get_capabilities(['AGT_CMD']) log.debug('Agent Commands: {0}'.format(acmds)) acmds = [item[1] for item in acmds] self.assertListsEqual(acmds, AGT_CMDS.keys()) apars = self._ia_client.get_capabilities(['AGT_PAR']) log.debug('Agent Parameters: {0}'.format(apars)) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) cmd = AgentCommand(command='initialize') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.INACTIVE) rcmds = self._ia_client.get_capabilities(['RES_CMD']) log.debug('Resource Commands: {0}'.format(rcmds)) rcmds = [item[1] for item in rcmds] self.assertListsEqual(rcmds, CMDS.keys()) rpars = self._ia_client.get_capabilities(['RES_PAR']) log.debug('Resource Parameters: {0}'.format(rpars)) rpars = [item[1] for item in rpars] self.assertListsEqual(rpars, PARAMS.keys()) cmd = AgentCommand(command='reset') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) def test_errors(self): # Test illegal behavior and replies. cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) # Can't go active in unitialized state. # Status 660 is state error. cmd = AgentCommand(command='go_active') retval = self._ia_client.execute_agent(cmd) log.info('GO ACTIVE CMD %s', str(retval)) self.assertEquals(retval.status, 660) # Can't command driver in this state. cmd = AgentCommand(command='acquire_sample') reply = self._ia_client.execute(cmd) self.assertEqual(reply.status, 660) cmd = AgentCommand(command='initialize') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.INACTIVE) cmd = AgentCommand(command='go_active') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.IDLE) cmd = AgentCommand(command='run') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) # 404 unknown agent command. cmd = AgentCommand(command='kiss_edward') retval = self._ia_client.execute_agent(cmd) self.assertEquals(retval.status, 404) # 670 unknown driver command. cmd = AgentCommand(command='acquire_sample_please') retval = self._ia_client.execute(cmd) self.assertEqual(retval.status, 670) # 630 Parameter error. self.assertRaises(InstParameterError, self._ia_client.get_param, 'bogus bogus') cmd = AgentCommand(command='reset') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
class PolicyManagementService(BasePolicyManagementService): def on_init(self): self.event_pub = EventPublisher() self.policy_event_subscriber = EventSubscriber(event_type="ResourceModifiedEvent", origin_type="Policy", callback=self.policy_event_callback) self.policy_event_subscriber.activate() def on_quit(self): if self.policy_event_subscriber is not None: self.policy_event_subscriber.deactivate() """ Provides the interface to define and manage policy and a repository to store and retrieve policy and templates for policy definitions, aka attribute authority. """ def create_policy(self, policy=None): """Persists the provided Policy object for the specified Org id. The id string returned is the internal id by which Policy will be identified in the data store. @param policy Policy @retval policy_id str @throws BadRequest if object passed has _id or _rev attribute """ if not is_basic_identifier(policy.name): raise BadRequest("The policy name '%s' can only contain alphanumeric and underscore characters" % policy.name) policy.rule = policy.rule % (policy.name, policy.description) policy_id, version = self.clients.resource_registry.create(policy) return policy_id def update_policy(self, policy=None): """Updates the provided Policy object. Throws NotFound exception if an existing version of Policy is not found. Throws Conflict if the provided Policy object is not based on the latest persisted version of the object. @param policy Policy @throws NotFound object with specified id does not exist @throws BadRequest if object does not have _id or _rev attribute @throws Conflict object not based on latest persisted object version """ if not is_basic_identifier(policy.name): raise BadRequest("The policy name '%s' can only contain alphanumeric and underscore characters" % policy.name) self.clients.resource_registry.update(policy) def read_policy(self, policy_id=''): """Returns the Policy object for the specified policy id. Throws exception if id does not match any persisted Policy objects. @param policy_id str @retval policy Policy @throws NotFound object with specified id does not exist """ if not policy_id: raise BadRequest("The policy_id parameter is missing") policy = self.clients.resource_registry.read(policy_id) if not policy: raise NotFound("Policy %s does not exist" % policy_id) return policy def delete_policy(self, policy_id=''): """For now, permanently deletes Policy object with the specified id. Throws exception if id does not match any persisted Policy. @param policy_id str @throws NotFound object with specified id does not exist """ if not policy_id: raise BadRequest("The policy_id parameter is missing") policy = self.clients.resource_registry.read(policy_id) if not policy: raise NotFound("Policy %s does not exist" % policy_id) self.clients.resource_registry.delete(policy_id) def enable_policy(self, policy_id=''): """Sets a flag to enable the use of the policy rule @param policy_id str @throws NotFound object with specified id does not exist """ policy = self.read_policy(policy_id) policy.enabled = True self.update_policy(policy) def disable_policy(self, policy_id=''): """Resets a flag to disable the use of the policy rule @param policy_id str @throws NotFound object with specified id does not exist """ policy = self.read_policy(policy_id) policy.enabled = False self.update_policy(policy) def policy_event_callback(self, *args, **kwargs): """ This method is a callback function for receiving Policy Events. """ policy_event = args[0] policy_id = policy_event.origin log.debug("Policy modified: %s" % policy_id) try: policy = self.clients.resource_registry.read(policy_id) if policy: #Need to publish an event that a policy has changed for any associated resource res_list = self._find_resources_for_policy(policy_id) for res in res_list: self._publish_resource_policy_event(policy, res) except Exception, e: #If this is a delete operation, then don't bother with not finding the object. if policy_event.sub_type != 'DELETE': log.error(e)
class PolicyManagementService(BasePolicyManagementService): def on_init(self): self.event_pub = EventPublisher() self.policy_event_subscriber = EventSubscriber( event_type="ResourceModifiedEvent", origin_type="Policy", callback=self.policy_event_callback) self.policy_event_subscriber.activate() def on_quit(self): if self.policy_event_subscriber is not None: self.policy_event_subscriber.deactivate() """ Provides the interface to define and manage policy and a repository to store and retrieve policy and templates for policy definitions, aka attribute authority. """ def create_policy(self, policy=None): """Persists the provided Policy object for the specified Org id. The id string returned is the internal id by which Policy will be identified in the data store. @param policy Policy @retval policy_id str @throws BadRequest if object passed has _id or _rev attribute """ if not is_basic_identifier(policy.name): raise BadRequest( "The policy name '%s' can only contain alphanumeric and underscore characters" % policy.name) policy.rule = policy.rule % (policy.name, policy.description) policy_id, version = self.clients.resource_registry.create(policy) return policy_id def update_policy(self, policy=None): """Updates the provided Policy object. Throws NotFound exception if an existing version of Policy is not found. Throws Conflict if the provided Policy object is not based on the latest persisted version of the object. @param policy Policy @throws NotFound object with specified id does not exist @throws BadRequest if object does not have _id or _rev attribute @throws Conflict object not based on latest persisted object version """ if not is_basic_identifier(policy.name): raise BadRequest( "The policy name '%s' can only contain alphanumeric and underscore characters" % policy.name) self.clients.resource_registry.update(policy) def read_policy(self, policy_id=''): """Returns the Policy object for the specified policy id. Throws exception if id does not match any persisted Policy objects. @param policy_id str @retval policy Policy @throws NotFound object with specified id does not exist """ if not policy_id: raise BadRequest("The policy_id parameter is missing") policy = self.clients.resource_registry.read(policy_id) if not policy: raise NotFound("Policy %s does not exist" % policy_id) return policy def delete_policy(self, policy_id=''): """For now, permanently deletes Policy object with the specified id. Throws exception if id does not match any persisted Policy. @param policy_id str @throws NotFound object with specified id does not exist """ if not policy_id: raise BadRequest("The policy_id parameter is missing") policy = self.clients.resource_registry.read(policy_id) if not policy: raise NotFound("Policy %s does not exist" % policy_id) self.clients.resource_registry.delete(policy_id) def enable_policy(self, policy_id=''): """Sets a flag to enable the use of the policy rule @param policy_id str @throws NotFound object with specified id does not exist """ policy = self.read_policy(policy_id) policy.enabled = True self.update_policy(policy) def disable_policy(self, policy_id=''): """Resets a flag to disable the use of the policy rule @param policy_id str @throws NotFound object with specified id does not exist """ policy = self.read_policy(policy_id) policy.enabled = False self.update_policy(policy) def policy_event_callback(self, *args, **kwargs): """ This method is a callback function for receiving Policy Events. """ policy_event = args[0] policy_id = policy_event.origin log.debug("Policy modified: %s" % policy_id) try: policy = self.clients.resource_registry.read(policy_id) if policy: #Need to publish an event that a policy has changed for any associated resource res_list = self._find_resources_for_policy(policy_id) for res in res_list: self._publish_resource_policy_event(policy, res) except Exception, e: #If this is a delete operation, then don't bother with not finding the object. if policy_event.sub_type != 'DELETE': log.error(e)
class GovernanceController(object): def __init__(self, container): log.debug('GovernanceController.__init__()') self.container = container self.enabled = False self.interceptor_by_name_dict = dict() self.interceptor_order = [] self.policy_decision_point_manager = None self.governance_dispatcher = None def start(self): log.debug("GovernanceController starting ...") config = CFG.interceptor.interceptors.governance.config if config is None: config['enabled'] = False if "enabled" in config: self.enabled = config["enabled"] log.debug("GovernanceInterceptor enabled: %s" % str(self.enabled)) self.resource_policy_event_subscriber = None if self.enabled: self.initialize_from_config(config) self.resource_policy_event_subscriber = EventSubscriber( event_type="ResourcePolicyEvent", callback=self.policy_event_callback) self.resource_policy_event_subscriber.activate() self.rr_client = ResourceRegistryServiceProcessClient( node=self.container.node, process=self.container) self.policy_client = PolicyManagementServiceProcessClient( node=self.container.node, process=self.container) self.org_client = OrgManagementServiceProcessClient( node=self.container.node, process=self.container) def initialize_from_config(self, config): self.governance_dispatcher = GovernanceDispatcher() self.policy_decision_point_manager = PolicyDecisionPointManager() if 'interceptor_order' in config: self.interceptor_order = config['interceptor_order'] if 'governance_interceptors' in config: gov_ints = config['governance_interceptors'] for name in gov_ints: interceptor_def = gov_ints[name] # Instantiate and put in by_name array parts = interceptor_def["class"].split('.') modpath = ".".join(parts[:-1]) classname = parts[-1] module = __import__(modpath, fromlist=[classname]) classobj = getattr(module, classname) classinst = classobj() # Put in by_name_dict for possible re-use self.interceptor_by_name_dict[name] = classinst def stop(self): log.debug("GovernanceController stopping ...") if self.resource_policy_event_subscriber is not None: self.resource_policy_event_subscriber.deactivate() def process_incoming_message(self, invocation): self.process_message(invocation, self.interceptor_order, 'incoming') return self.governance_dispatcher.handle_incoming_message(invocation) def process_outgoing_message(self, invocation): self.process_message(invocation, reversed(self.interceptor_order), 'outgoing') return self.governance_dispatcher.handle_outgoing_message(invocation) def process_message(self, invocation, interceptor_list, method): for int_name in interceptor_list: class_inst = self.interceptor_by_name_dict[int_name] getattr(class_inst, method)(invocation) return invocation def policy_event_callback(self, *args, **kwargs): resource_policy_event = args[0] policy_id = resource_policy_event.origin resource_id = resource_policy_event.resource_id resource_type = resource_policy_event.resource_type resource_name = resource_policy_event.resource_name log.info("Resource policy modified: %s %s %s" % (policy_id, resource_id, resource_type)) if resource_type == 'ServiceDefinition': #TODO - REDO to have a configurable Org boundary by container ion_org = self.org_client.find_org() policy_rules = self.policy_client.get_active_service_policy_rules( ion_org._id, resource_name) self.update_resource_policy(resource_name, policy_rules) elif resource_type == 'Org': policy_rules = self.policy_client.get_active_resource_policy_rules( resource_id) if self.policy_decision_point_manager is not None: self.policy_decision_point_manager.load_org_policy_rules( policy_rules) self.update_all_resource_policies(resource_id) else: policy_rules = self.policy_client.get_active_resource_policy_rules( resource_id) self.update_resource_policy(resource_id, policy_rules) def update_resource_policy(self, resource_name, policy_rules): #Notify policy decision point of updated rules if self.policy_decision_point_manager is not None: log.debug("Loading policy for resource: %s" % resource_name) self.policy_decision_point_manager.load_policy_rules( resource_name, policy_rules) def update_all_resource_policies(self, org_id): #Notify policy decision point of updated rules for all existing service policies if self.policy_decision_point_manager is not None: for res_name in self.policy_decision_point_manager.policy_decision_point: try: policy_rules = self.policy_client.get_active_service_policy_rules( org_id, res_name) self.update_resource_policy(res_name, policy_rules) except Exception, e: log.error(e.message)
class VisualizationService(BaseVisualizationService): def on_start(self): # The data dictionary object holds a copy of all the viz products created by the service. The viz # products are indexed by the viz_product_type and data_product_id (which could be google_datatables or # mpl_graphs self.viz_data_dictionary = {} self.viz_data_dictionary['google_dt'] = {} self.viz_data_dictionary['google_realtime_dt'] = {} self.viz_data_dictionary['matplotlib_graphs'] = {} # Kind of redundant but we will maintain a separate list of data product_ids registered with the viz_service self.data_products = [] # Create clients to interface with PubSub, Transform Management Service and Resource Registry self.pubsub_cli = self.clients.pubsub_management self.tms_cli = self.clients.transform_management self.rr_cli = self.clients.resource_registry self.dr_cli = self.clients.data_retriever self.dsm_cli = self.clients.dataset_management """ # Create process definitions which will used to spawn off the transform processes self.matplotlib_proc_def = IonObject(RT.ProcessDefinition, name='viz_transform_process'+'.'+self.random_id_generator()) self.matplotlib_proc_def.executable = { 'module': 'ion.services.ans.visualization_service', 'class':'VizTransformProcForMatplotlibGraphs' } self.matplotlib_proc_def_id, _ = self.rr_cli.create(self.matplotlib_proc_def) self.google_dt_proc_def = IonObject(RT.ProcessDefinition, name='viz_transform_process'+'.'+self.random_id_generator()) self.google_dt_proc_def.executable = { 'module': 'ion.services.ans.visualization_service', 'class':'VizTransformProcForGoogleDT' } self.google_dt_proc_def_id, _ = self.rr_cli.create(self.google_dt_proc_def) """ # Query resource registry to get process definitions and streams ids made by the bootstrap proc_def_ids,_ = self.rr_cli.find_resources(restype=RT.ProcessDefinition, lcstate=None, name="viz_matplotlib_transform_process", id_only=True) self.matplotlib_proc_def_id = proc_def_ids[0] proc_def_ids,_ = self.rr_cli.find_resources(restype=RT.ProcessDefinition, lcstate=None, name="viz_google_dt_transform_process", id_only=True) self.google_dt_proc_def_id = proc_def_ids[0] # Create a stream that all the transform processes will use to submit data back to the viz service self.viz_service_submit_stream_id = self.pubsub_cli.create_stream(name="visualization_service_submit_stream." + self.random_id_generator()) # subscribe to this stream since all the results from transforms will be submitted here query = StreamQuery(stream_ids=[self.viz_service_submit_stream_id,]) self.viz_service_submit_stream_sub_id = self.pubsub_cli.create_subscription(query=query, exchange_name="visualization_service_submit_queue") submit_stream_subscriber_registrar = StreamSubscriberRegistrar(process = self.container, node = self.container.node ) submit_stream_subscriber = submit_stream_subscriber_registrar.create_subscriber(exchange_name='visualization_service_submit_queue', callback=self.process_submission) submit_stream_subscriber.start() self.pubsub_cli.activate_subscription(self.viz_service_submit_stream_sub_id) # Discover the existing data_product_ids active in the system sys_prod_ids, _ = self.rr_cli.find_resources(RT.DataProduct, None, None, True) # Register all the streams in the system, which will in turn start transform processes for dp_id in sys_prod_ids: self.register_new_data_product(dp_id) # listen for events when new data_products show up self.event_subscriber = EventSubscriber( event_type = "ResourceModifiedEvent", origin_type = "DataProduct", sub_type="UPDATE", callback=self.receive_new_dataproduct_event ) self.event_subscriber.activate() return def on_stop(self): self.event_subscriber.deactivate() super(VisualizationService, self).on_stop() return def process_submission(self, packet, headers): # The packet is a dictionary containing the data_product_id and viz_product_type. if(packet["viz_product_type"] == "google_realtime_dt"): self.submit_google_realtime_dt(data_product_id=packet["data_product_id"], data_table=packet["data_table"]) if(packet["viz_product_type"] == "google_dt"): self.submit_google_dt(data_product_id_token=packet["data_product_id_token"], data_table=packet["data_table"]) if(packet["viz_product_type"] == "matplotlib_graphs"): self.submit_mpl_image(data_product_id=packet["data_product_id"], image_obj=packet["image_obj"], image_name=packet["image_name"]) return def start_google_dt_transform(self, data_product_id='', query=''): """Request to fetch the datatable for a data product as specified in the query. Query will also specify whether its a realtime view or one-shot @param data_product_id str @param query str @retval datatable str @throws NotFound object with specified id, query does not exist """ # generate a token unique for this request data_product_id_token = data_product_id + "." + self.random_id_generator() # Get object asssociated with data_product_id dp_obj = self.rr_cli.read(data_product_id) if dp_obj.dataset_id == '': return None # define replay. If no filters are passed the entire ingested dataset is returned replay_id, replay_stream_id = self.dr_cli.define_replay(dataset_id=dp_obj.dataset_id) replay_stream_def_id = self.pubsub_cli.find_stream_definition(stream_id=replay_stream_id, id_only=True) # setup the transform to handle the data coming back from the replay # Init storage for the resulting data_table self.viz_data_dictionary['google_dt'][data_product_id_token] = {'data_table': [], 'ready_flag': False} # Create the subscription to the stream. This will be passed as parameter to the transform worker query = StreamQuery(stream_ids=[replay_stream_id,]) replay_subscription_id = self.pubsub_cli.create_subscription(query=query, exchange_name='viz_data_exchange.'+self.random_id_generator()) # maybe this is a good place to pass the couch DB table to use and other parameters configuration = {"stream_def_id": replay_stream_def_id, "data_product_id": data_product_id, "realtime_flag": "False", "data_product_id_token": data_product_id_token} # Launch the viz transform process viz_transform_id = self.tms_cli.create_transform( name='viz_transform_google_dt_' + self.random_id_generator()+'.'+data_product_id, in_subscription_id=replay_subscription_id, out_streams = {"visualization_service_submit_stream_id": self.viz_service_submit_stream_id }, process_definition_id=self.google_dt_proc_def_id, configuration=configuration) self.tms_cli.activate_transform(viz_transform_id) # keep a record of the the viz_transform_id self.viz_data_dictionary['google_dt'][data_product_id_token]['transform_proc'] = viz_transform_id # Start the replay and return the token self.dr_cli.start_replay(replay_id=replay_id) return "google_dt_transform_cb(\"" + data_product_id_token + "\")" def is_google_dt_ready(self, data_product_id_token=''): try: # check if token is valid if data_product_id_token in self.viz_data_dictionary['google_dt']: #check if the data table is ready if self.viz_data_dictionary['google_dt'][data_product_id_token]['ready_flag']: return "google_dt_status_cb(\"True\")" else: return "google_dt_status_cb(\"False\")" except AttributeError: return None def get_google_dt(self, data_product_id_token=''): try: # check if token is valid if data_product_id_token in self.viz_data_dictionary['google_dt']: #check if the data table is ready if not self.viz_data_dictionary['google_dt'][data_product_id_token]['ready_flag']: return None else: # Make a reference to the data_table and clean space in global dict data_table = self.viz_data_dictionary['google_dt'][data_product_id_token]['data_table'] # clean up the transform and space in global dict self.tms_cli.deactivate_transform(self.viz_data_dictionary['google_dt'][data_product_id_token]['transform_proc']) del self.viz_data_dictionary['google_dt'][data_product_id_token] # returning the reference to the data_table should mark the objects used by this tranform as ready # for deletion return data_table else: return None except AttributeError: return None def get_google_realtime_dt(self, data_product_id='', query=''): """Request to fetch the datatable for a data product as specified in the query. Query will also specify whether its a realtime view or one-shot @param data_product_id str @param query str @retval datatable str @throws NotFound object with specified id, query does not exist """ try: if data_product_id in self.viz_data_dictionary['google_realtime_dt']: # assign data_table to a temp var before returning it. This will ensure a complete object is returned # in case the data_table is being updated by a transform process data_table = self.viz_data_dictionary['google_realtime_dt'][data_product_id]['data_table'] return data_table else: return None except AttributeError: return None def get_list_of_mpl_images(self, data_product_id=''): """Request to fetch the list of Matplotlib generated images associated with a data product @param data_product_id str @retval image_list str @throws NotFound object with specified id does not exist """ # return a json version of the array stored in the data_dict try: if data_product_id in self.viz_data_dictionary['matplotlib_graphs']: # assign data_table to a temp var before returning it. This will ensure a complete object is returned img_list = self.viz_data_dictionary['matplotlib_graphs'][data_product_id]['list_of_images'] json_img_list = simplejson.dumps({'data': img_list}) return "image_list_callback("+json_img_list+")" else: return None except AttributeError: return None def get_image(self, data_product_id = '', image_name=''): """Request to fetch a file object from within the Visualization Service @param file_name str @retval file_obj str @throws NotFound object with specified id does not exist """ try: if data_product_id in self.viz_data_dictionary['matplotlib_graphs']: if image_name in self.viz_data_dictionary['matplotlib_graphs'][data_product_id]: img = self.viz_data_dictionary['matplotlib_graphs'][data_product_id][image_name] return img else: return None else: return None except AttributeError: return None def submit_google_dt(self, data_product_id_token='', data_table=''): """Send the rendered image to @param data_product_id str @param data_table str @param description str @throws BadRequest check data_product_id """ # Just copy the datatable in to the data dictionary self.viz_data_dictionary['google_dt'][data_product_id_token]['data_table'] = data_table self.viz_data_dictionary['google_dt'][data_product_id_token]['ready_flag'] = True #self.result.set(True) return def submit_google_realtime_dt(self, data_product_id='', data_table=''): """Send the rendered image to @param data_product_id str @param data_table str @param description str @throws BadRequest check data_product_id """ # Just copy the datatable in to the data dictionary self.viz_data_dictionary['google_realtime_dt'][data_product_id]['data_table'] = data_table return def submit_mpl_image(self, data_product_id='', image_obj='', image_name=''): """Send the rendered image to the visualization service @param data_product_id str @param image_obj str @param description str @throws BadRequest check data_product_id """ # Store the image object in the data dictionary and note the image in the list. # Check for existing entries before updating the list_of_images found = False for name in self.viz_data_dictionary['matplotlib_graphs'][data_product_id]['list_of_images']: if name == image_name: found = True break if not found: list_len = len(self.viz_data_dictionary['matplotlib_graphs'][data_product_id]['list_of_images']) self.viz_data_dictionary['matplotlib_graphs'][data_product_id]['list_of_images'].append(image_name) # Add binary data from the image to the dictionary self.viz_data_dictionary['matplotlib_graphs'][data_product_id][image_name] = image_obj return def register_new_data_product(self, data_product_id=''): """Apprise the Visualization service of a new data product in the system. This function inits transform processes for generating the matplotlib graphs of the new data product. It also creates transform processes which generate Google data-tables for the real-time streams (sliding window) coming in from the instruments. @param data_product_id str @throws BadRequest check data_product_id for duplicates """ # Check to see if the DP has already been registered. If yes, do nothing if (data_product_id in self.viz_data_dictionary['matplotlib_graphs']) or (data_product_id in self.viz_data_dictionary['google_realtime_dt']): log.warn("Data Product has already been registered with Visualization service. Ignoring.") return # extract the stream_id associated with the data_product_id viz_stream_id,_ = self.rr_cli.find_objects(data_product_id, PRED.hasStream, None, True) if viz_stream_id == []: log.warn ("Visualization_service: viz_stream_id is empty") return viz_stream_def_id = self.pubsub_cli.find_stream_definition(stream_id = viz_stream_id[0], id_only = True) # Go ahead only if the data product is unique if data_product_id in self.data_products: raise BadRequest self.data_products[len(self.data_products):] = data_product_id # init the space needed to store matplotlib_graphs and realtime Google data tables # For the matplotlib graphs, the list_of_images stores the names of the image files. The actual binary data for the # images is also stored in the same dictionary as {img_name1: binary_data1, img_name2: binary_data2 .. etc} self.viz_data_dictionary['matplotlib_graphs'][data_product_id] = {'transform_proc': "", 'list_of_images': []} # The 'data_table' key points to a JSON string self.viz_data_dictionary['google_realtime_dt'][data_product_id] = {'transform_proc': "", 'data_table': []} ############################################################################### # Create transform process for the matplotlib graphs. ############################################################################### # Create the subscription to the stream. This will be passed as parameter to the transform worker #query1 = StreamQuery(stream_ids=[viz_stream_id,]) query1 = StreamQuery(viz_stream_id) viz_subscription_id1 = self.pubsub_cli.create_subscription(query=query1, exchange_name='viz_data_exchange.'+self.random_id_generator()) # maybe this is a good place to pass the couch DB table to use and other parameters configuration1 = {"stream_def_id": viz_stream_def_id, "data_product_id": data_product_id} # Launch the viz transform process viz_transform_id1 = self.tms_cli.create_transform( name='viz_transform_matplotlib_'+ self.random_id_generator() + '.'+data_product_id, in_subscription_id=viz_subscription_id1, out_streams = {"visualization_service_submit_stream_id": self.viz_service_submit_stream_id }, process_definition_id=self.matplotlib_proc_def_id, configuration=configuration1) self.tms_cli.activate_transform(viz_transform_id1) # keep a record of the the viz_transform_id self.viz_data_dictionary['matplotlib_graphs'][data_product_id]['transform_proc'] = viz_transform_id1 ############################################################################### # Create transform process for the Google realtime datatables ############################################################################### # Create the subscription to the stream. This will be passed as parameter to the transform worker #query2 = StreamQuery(stream_ids=[viz_stream_id,]) query2 = StreamQuery(viz_stream_id) viz_subscription_id2 = self.pubsub_cli.create_subscription(query=query2, exchange_name='viz_data_exchange.'+self.random_id_generator()) # maybe this is a good place to pass the couch DB table to use and other parameters configuration2 = {"stream_def_id": viz_stream_def_id, "data_product_id": data_product_id, "realtime_flag": "True"} # Launch the viz transform process viz_transform_id2 = self.tms_cli.create_transform( name='viz_transform_realtime_google_dt_' + self.random_id_generator()+'.'+data_product_id, in_subscription_id=viz_subscription_id2, out_streams = {"visualization_service_submit_stream_id": self.viz_service_submit_stream_id }, process_definition_id=self.google_dt_proc_def_id, configuration=configuration2) self.tms_cli.activate_transform(viz_transform_id2) # keep a record of the the viz_transform_id self.viz_data_dictionary['google_realtime_dt'][data_product_id]['transform_proc'] = viz_transform_id2 def random_id_generator(self, size=8, chars=string.ascii_uppercase + string.digits): id = ''.join(random.choice(chars) for x in range(size)) return id def receive_new_dataproduct_event(self, event_msg, headers): """ For now we will start a thread that emulates an event handler """ # register the new data product self.register_new_data_product(event_msg.origin)
class GovernanceController(object): def __init__(self, container): log.debug('GovernanceController.__init__()') self.container = container self.enabled = False self.interceptor_by_name_dict = dict() self.interceptor_order = [] self.policy_decision_point_manager = None self.governance_dispatcher = None def start(self): log.debug("GovernanceController starting ...") config = CFG.interceptor.interceptors.governance.config if config is None: config['enabled'] = False if "enabled" in config: self.enabled = config["enabled"] log.debug("GovernanceInterceptor enabled: %s" % str(self.enabled)) self.event_subscriber = None if self.enabled: self.initialize_from_config(config) self.event_subscriber = EventSubscriber( event_type="ResourceModifiedEvent", origin_type="Policy", callback=self.policy_event_callback) self.event_subscriber.activate() self.rr_client = ResourceRegistryServiceProcessClient( node=self.container.node, process=self.container) self.policy_client = PolicyManagementServiceProcessClient( node=self.container.node, process=self.container) def initialize_from_config(self, config): self.governance_dispatcher = GovernanceDispatcher() self.policy_decision_point_manager = PolicyDecisionPointManager() if 'interceptor_order' in config: self.interceptor_order = config['interceptor_order'] if 'governance_interceptors' in config: gov_ints = config['governance_interceptors'] for name in gov_ints: interceptor_def = gov_ints[name] # Instantiate and put in by_name array parts = interceptor_def["class"].split('.') modpath = ".".join(parts[:-1]) classname = parts[-1] module = __import__(modpath, fromlist=[classname]) classobj = getattr(module, classname) classinst = classobj() # Put in by_name_dict for possible re-use self.interceptor_by_name_dict[name] = classinst def stop(self): log.debug("GovernanceController stopping ...") if self.event_subscriber is not None: self.event_subscriber.deactivate() def process_incoming_message(self, invocation): self.process_message(invocation, self.interceptor_order, 'incoming') return self.governance_dispatcher.handle_incoming_message(invocation) def process_outgoing_message(self, invocation): self.process_message(invocation, reversed(self.interceptor_order), 'outgoing') return self.governance_dispatcher.handle_outgoing_message(invocation) def process_message(self, invocation, interceptor_list, method): for int_name in interceptor_list: class_inst = self.interceptor_by_name_dict[int_name] getattr(class_inst, method)(invocation) return invocation def policy_event_callback(self, *args, **kwargs): policy_event = args[0] log.debug("Policy modified: %s" % policy_event.origin) self.trigger_policy_update(policy_event.origin) #This is a function which allows for a manual update of the policies as well. def trigger_policy_update(self, policy_id): try: #TODO - Find a better way to work with org_id - use ION Org for now. ion_org, _ = self.rr_client.find_resources( restype=RT.Org, name=CFG.system.root_org) resource_list, _ = self.rr_client.find_subjects( "", PRED.hasPolicy, policy_id) for res in resource_list: #TODO - may figure out a better way to get the name of the Resource Type - or maybe this is ok resource_type = res.__class__.__name__ #log.debug("Resource Type: %s" % resource_type) if resource_type == 'ServiceDefinition': policy_rules = self.policy_client.get_active_service_policy_rules( ion_org[0]._id, res.name) self.update_resource_policy(res.name, policy_rules) elif resource_type == 'Org': self.update_all_resource_policy(res._id) else: policy_rules = self.policy_client.get_active_resource_policy_rules( res._id) self.update_resource_policy(res._id, policy_rules) except Exception, e: log.error(e.message)
class ProcessDispatcherServiceIntTest(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2cei.yml') self.pd_cli = ProcessDispatcherServiceClient(node=self.container.node) self.process_definition = ProcessDefinition(name='test_process') self.process_definition.executable = { 'module': 'ion.services.cei.test.test_process_dispatcher', 'class': 'TestProcess' } self.process_definition_id = self.pd_cli.create_process_definition( self.process_definition) self.event_queue = queue.Queue() self.event_sub = None def tearDown(self): if self.event_sub: self.event_sub.deactivate() def _event_callback(self, event, *args, **kwargs): self.event_queue.put(event) def subscribe_events(self, origin): self.event_sub = EventSubscriber(event_type="ProcessLifecycleEvent", callback=self._event_callback, origin=origin, origin_type="DispatchedProcess") self.event_sub.activate() def await_state_event(self, pid, state): event = self.event_queue.get(timeout=5) log.debug("Got event: %s", event) self.assertEqual(event.origin, pid) self.assertEqual(event.state, state) return event def test_create_schedule_cancel(self): process_schedule = ProcessSchedule() pid = self.pd_cli.create_process(self.process_definition_id) self.subscribe_events(pid) pid2 = self.pd_cli.schedule_process(self.process_definition_id, process_schedule, configuration={}, process_id=pid) self.assertEqual(pid, pid2) self.await_state_event(pid, ProcessStateEnum.SPAWN) # now try communicating with the process to make sure it is really running test_client = TestClient() for i in range(5): # this timeout may be too low self.assertEqual(i + 1, test_client.count(timeout=1)) # kill the process and start it again self.pd_cli.cancel_process(pid) self.await_state_event(pid, ProcessStateEnum.TERMINATE) oldpid = pid pid = self.pd_cli.create_process(self.process_definition_id) self.subscribe_events(pid) pid2 = self.pd_cli.schedule_process(self.process_definition_id, process_schedule, configuration={}, process_id=pid) self.assertEqual(pid, pid2) self.assertNotEqual(oldpid, pid) self.await_state_event(pid, ProcessStateEnum.SPAWN) for i in range(5): # this timeout may be too low self.assertEqual(i + 1, test_client.count(timeout=1)) # kill the process for good self.pd_cli.cancel_process(pid) self.await_state_event(pid, ProcessStateEnum.TERMINATE) def test_schedule_bad_config(self): process_schedule = ProcessSchedule() # a non-JSON-serializable IonObject o = ProcessTarget() with self.assertRaises(BadRequest) as ar: self.pd_cli.schedule_process(self.process_definition_id, process_schedule, configuration={"bad": o}) self.assertTrue(ar.exception.message.startswith("bad configuration"))
class VisualizationService(BaseVisualizationService): def on_start(self): # The data dictionary object holds a copy of all the viz products created by the service. The viz # products are indexed by the viz_product_type and data_product_id (which could be google_datatables or # mpl_graphs self.viz_data_dictionary = {} self.viz_data_dictionary['google_dt'] = {} self.viz_data_dictionary['google_realtime_dt'] = {} self.viz_data_dictionary['matplotlib_graphs'] = {} # Kind of redundant but we will maintain a separate list of data product_ids registered with the viz_service self.data_products = [] # Create clients to interface with PubSub, Transform Management Service and Resource Registry self.pubsub_cli = self.clients.pubsub_management self.tms_cli = self.clients.transform_management self.rr_cli = self.clients.resource_registry self.dr_cli = self.clients.data_retriever self.dsm_cli = self.clients.dataset_management """ # Create process definitions which will used to spawn off the transform processes self.matplotlib_proc_def = IonObject(RT.ProcessDefinition, name='viz_transform_process'+'.'+self.random_id_generator()) self.matplotlib_proc_def.executable = { 'module': 'ion.services.ans.visualization_service', 'class':'VizTransformProcForMatplotlibGraphs' } self.matplotlib_proc_def_id, _ = self.rr_cli.create(self.matplotlib_proc_def) self.google_dt_proc_def = IonObject(RT.ProcessDefinition, name='viz_transform_process'+'.'+self.random_id_generator()) self.google_dt_proc_def.executable = { 'module': 'ion.services.ans.visualization_service', 'class':'VizTransformProcForGoogleDT' } self.google_dt_proc_def_id, _ = self.rr_cli.create(self.google_dt_proc_def) """ # Query resource registry to get process definitions and streams ids made by the bootstrap proc_def_ids, _ = self.rr_cli.find_resources( restype=RT.ProcessDefinition, lcstate=None, name="viz_matplotlib_transform_process", id_only=True) self.matplotlib_proc_def_id = proc_def_ids[0] proc_def_ids, _ = self.rr_cli.find_resources( restype=RT.ProcessDefinition, lcstate=None, name="viz_google_dt_transform_process", id_only=True) self.google_dt_proc_def_id = proc_def_ids[0] # Create a stream that all the transform processes will use to submit data back to the viz service self.viz_service_submit_stream_id = self.pubsub_cli.create_stream( name="visualization_service_submit_stream." + self.random_id_generator()) # subscribe to this stream since all the results from transforms will be submitted here query = StreamQuery(stream_ids=[ self.viz_service_submit_stream_id, ]) self.viz_service_submit_stream_sub_id = self.pubsub_cli.create_subscription( query=query, exchange_name="visualization_service_submit_queue") submit_stream_subscriber_registrar = StreamSubscriberRegistrar( process=self.container, node=self.container.node) submit_stream_subscriber = submit_stream_subscriber_registrar.create_subscriber( exchange_name='visualization_service_submit_queue', callback=self.process_submission) submit_stream_subscriber.start() self.pubsub_cli.activate_subscription( self.viz_service_submit_stream_sub_id) # Discover the existing data_product_ids active in the system sys_prod_ids, _ = self.rr_cli.find_resources(RT.DataProduct, None, None, True) # Register all the streams in the system, which will in turn start transform processes for dp_id in sys_prod_ids: self.register_new_data_product(dp_id) # listen for events when new data_products show up self.event_subscriber = EventSubscriber( event_type="ResourceModifiedEvent", origin_type="DataProduct", sub_type="UPDATE", callback=self.receive_new_dataproduct_event) self.event_subscriber.activate() return def on_stop(self): self.event_subscriber.deactivate() super(VisualizationService, self).on_stop() return def process_submission(self, packet, headers): # The packet is a dictionary containing the data_product_id and viz_product_type. if (packet["viz_product_type"] == "google_realtime_dt"): self.submit_google_realtime_dt( data_product_id=packet["data_product_id"], data_table=packet["data_table"]) if (packet["viz_product_type"] == "google_dt"): self.submit_google_dt( data_product_id_token=packet["data_product_id_token"], data_table=packet["data_table"]) if (packet["viz_product_type"] == "matplotlib_graphs"): self.submit_mpl_image(data_product_id=packet["data_product_id"], image_obj=packet["image_obj"], image_name=packet["image_name"]) return def start_google_dt_transform(self, data_product_id='', query=''): """Request to fetch the datatable for a data product as specified in the query. Query will also specify whether its a realtime view or one-shot @param data_product_id str @param query str @retval datatable str @throws NotFound object with specified id, query does not exist """ # generate a token unique for this request data_product_id_token = data_product_id + "." + self.random_id_generator( ) # Get object asssociated with data_product_id dp_obj = self.rr_cli.read(data_product_id) if dp_obj.dataset_id == '': return None # define replay. If no filters are passed the entire ingested dataset is returned replay_id, replay_stream_id = self.dr_cli.define_replay( dataset_id=dp_obj.dataset_id) replay_stream_def_id = self.pubsub_cli.find_stream_definition( stream_id=replay_stream_id, id_only=True) # setup the transform to handle the data coming back from the replay # Init storage for the resulting data_table self.viz_data_dictionary['google_dt'][data_product_id_token] = { 'data_table': [], 'ready_flag': False } # Create the subscription to the stream. This will be passed as parameter to the transform worker query = StreamQuery(stream_ids=[ replay_stream_id, ]) replay_subscription_id = self.pubsub_cli.create_subscription( query=query, exchange_name='viz_data_exchange.' + self.random_id_generator()) # maybe this is a good place to pass the couch DB table to use and other parameters configuration = { "stream_def_id": replay_stream_def_id, "data_product_id": data_product_id, "realtime_flag": "False", "data_product_id_token": data_product_id_token } # Launch the viz transform process viz_transform_id = self.tms_cli.create_transform( name='viz_transform_google_dt_' + self.random_id_generator() + '.' + data_product_id, in_subscription_id=replay_subscription_id, out_streams={ "visualization_service_submit_stream_id": self.viz_service_submit_stream_id }, process_definition_id=self.google_dt_proc_def_id, configuration=configuration) self.tms_cli.activate_transform(viz_transform_id) # keep a record of the the viz_transform_id self.viz_data_dictionary['google_dt'][data_product_id_token][ 'transform_proc'] = viz_transform_id # Start the replay and return the token self.dr_cli.start_replay(replay_id=replay_id) return "google_dt_transform_cb(\"" + data_product_id_token + "\")" def is_google_dt_ready(self, data_product_id_token=''): try: # check if token is valid if data_product_id_token in self.viz_data_dictionary['google_dt']: #check if the data table is ready if self.viz_data_dictionary['google_dt'][ data_product_id_token]['ready_flag']: return "google_dt_status_cb(\"True\")" else: return "google_dt_status_cb(\"False\")" except AttributeError: return None def get_google_dt(self, data_product_id_token=''): try: # check if token is valid if data_product_id_token in self.viz_data_dictionary['google_dt']: #check if the data table is ready if not self.viz_data_dictionary['google_dt'][ data_product_id_token]['ready_flag']: return None else: # Make a reference to the data_table and clean space in global dict data_table = self.viz_data_dictionary['google_dt'][ data_product_id_token]['data_table'] # clean up the transform and space in global dict self.tms_cli.deactivate_transform( self.viz_data_dictionary['google_dt'] [data_product_id_token]['transform_proc']) del self.viz_data_dictionary['google_dt'][ data_product_id_token] # returning the reference to the data_table should mark the objects used by this tranform as ready # for deletion return data_table else: return None except AttributeError: return None def get_google_realtime_dt(self, data_product_id='', query=''): """Request to fetch the datatable for a data product as specified in the query. Query will also specify whether its a realtime view or one-shot @param data_product_id str @param query str @retval datatable str @throws NotFound object with specified id, query does not exist """ try: if data_product_id in self.viz_data_dictionary[ 'google_realtime_dt']: # assign data_table to a temp var before returning it. This will ensure a complete object is returned # in case the data_table is being updated by a transform process data_table = self.viz_data_dictionary['google_realtime_dt'][ data_product_id]['data_table'] return data_table else: return None except AttributeError: return None def get_list_of_mpl_images(self, data_product_id=''): """Request to fetch the list of Matplotlib generated images associated with a data product @param data_product_id str @retval image_list str @throws NotFound object with specified id does not exist """ # return a json version of the array stored in the data_dict try: if data_product_id in self.viz_data_dictionary[ 'matplotlib_graphs']: # assign data_table to a temp var before returning it. This will ensure a complete object is returned img_list = self.viz_data_dictionary['matplotlib_graphs'][ data_product_id]['list_of_images'] json_img_list = simplejson.dumps({'data': img_list}) return "image_list_callback(" + json_img_list + ")" else: return None except AttributeError: return None def get_image(self, data_product_id='', image_name=''): """Request to fetch a file object from within the Visualization Service @param file_name str @retval file_obj str @throws NotFound object with specified id does not exist """ try: if data_product_id in self.viz_data_dictionary[ 'matplotlib_graphs']: if image_name in self.viz_data_dictionary['matplotlib_graphs'][ data_product_id]: img = self.viz_data_dictionary['matplotlib_graphs'][ data_product_id][image_name] return img else: return None else: return None except AttributeError: return None def submit_google_dt(self, data_product_id_token='', data_table=''): """Send the rendered image to @param data_product_id str @param data_table str @param description str @throws BadRequest check data_product_id """ # Just copy the datatable in to the data dictionary self.viz_data_dictionary['google_dt'][data_product_id_token][ 'data_table'] = data_table self.viz_data_dictionary['google_dt'][data_product_id_token][ 'ready_flag'] = True #self.result.set(True) return def submit_google_realtime_dt(self, data_product_id='', data_table=''): """Send the rendered image to @param data_product_id str @param data_table str @param description str @throws BadRequest check data_product_id """ # Just copy the datatable in to the data dictionary self.viz_data_dictionary['google_realtime_dt'][data_product_id][ 'data_table'] = data_table return def submit_mpl_image(self, data_product_id='', image_obj='', image_name=''): """Send the rendered image to the visualization service @param data_product_id str @param image_obj str @param description str @throws BadRequest check data_product_id """ # Store the image object in the data dictionary and note the image in the list. # Check for existing entries before updating the list_of_images found = False for name in self.viz_data_dictionary['matplotlib_graphs'][ data_product_id]['list_of_images']: if name == image_name: found = True break if not found: list_len = len(self.viz_data_dictionary['matplotlib_graphs'] [data_product_id]['list_of_images']) self.viz_data_dictionary['matplotlib_graphs'][data_product_id][ 'list_of_images'].append(image_name) # Add binary data from the image to the dictionary self.viz_data_dictionary['matplotlib_graphs'][data_product_id][ image_name] = image_obj return def register_new_data_product(self, data_product_id=''): """Apprise the Visualization service of a new data product in the system. This function inits transform processes for generating the matplotlib graphs of the new data product. It also creates transform processes which generate Google data-tables for the real-time streams (sliding window) coming in from the instruments. @param data_product_id str @throws BadRequest check data_product_id for duplicates """ # Check to see if the DP has already been registered. If yes, do nothing if (data_product_id in self.viz_data_dictionary['matplotlib_graphs'] ) or (data_product_id in self.viz_data_dictionary['google_realtime_dt']): log.warn( "Data Product has already been registered with Visualization service. Ignoring." ) return # extract the stream_id associated with the data_product_id viz_stream_id, _ = self.rr_cli.find_objects(data_product_id, PRED.hasStream, None, True) if viz_stream_id == []: log.warn("Visualization_service: viz_stream_id is empty") return viz_stream_def_id = self.pubsub_cli.find_stream_definition( stream_id=viz_stream_id[0], id_only=True) # Go ahead only if the data product is unique if data_product_id in self.data_products: raise BadRequest self.data_products[len(self.data_products):] = data_product_id # init the space needed to store matplotlib_graphs and realtime Google data tables # For the matplotlib graphs, the list_of_images stores the names of the image files. The actual binary data for the # images is also stored in the same dictionary as {img_name1: binary_data1, img_name2: binary_data2 .. etc} self.viz_data_dictionary['matplotlib_graphs'][data_product_id] = { 'transform_proc': "", 'list_of_images': [] } # The 'data_table' key points to a JSON string self.viz_data_dictionary['google_realtime_dt'][data_product_id] = { 'transform_proc': "", 'data_table': [] } ############################################################################### # Create transform process for the matplotlib graphs. ############################################################################### # Create the subscription to the stream. This will be passed as parameter to the transform worker #query1 = StreamQuery(stream_ids=[viz_stream_id,]) query1 = StreamQuery(viz_stream_id) viz_subscription_id1 = self.pubsub_cli.create_subscription( query=query1, exchange_name='viz_data_exchange.' + self.random_id_generator()) # maybe this is a good place to pass the couch DB table to use and other parameters configuration1 = { "stream_def_id": viz_stream_def_id, "data_product_id": data_product_id } # Launch the viz transform process viz_transform_id1 = self.tms_cli.create_transform( name='viz_transform_matplotlib_' + self.random_id_generator() + '.' + data_product_id, in_subscription_id=viz_subscription_id1, out_streams={ "visualization_service_submit_stream_id": self.viz_service_submit_stream_id }, process_definition_id=self.matplotlib_proc_def_id, configuration=configuration1) self.tms_cli.activate_transform(viz_transform_id1) # keep a record of the the viz_transform_id self.viz_data_dictionary['matplotlib_graphs'][data_product_id][ 'transform_proc'] = viz_transform_id1 ############################################################################### # Create transform process for the Google realtime datatables ############################################################################### # Create the subscription to the stream. This will be passed as parameter to the transform worker #query2 = StreamQuery(stream_ids=[viz_stream_id,]) query2 = StreamQuery(viz_stream_id) viz_subscription_id2 = self.pubsub_cli.create_subscription( query=query2, exchange_name='viz_data_exchange.' + self.random_id_generator()) # maybe this is a good place to pass the couch DB table to use and other parameters configuration2 = { "stream_def_id": viz_stream_def_id, "data_product_id": data_product_id, "realtime_flag": "True" } # Launch the viz transform process viz_transform_id2 = self.tms_cli.create_transform( name='viz_transform_realtime_google_dt_' + self.random_id_generator() + '.' + data_product_id, in_subscription_id=viz_subscription_id2, out_streams={ "visualization_service_submit_stream_id": self.viz_service_submit_stream_id }, process_definition_id=self.google_dt_proc_def_id, configuration=configuration2) self.tms_cli.activate_transform(viz_transform_id2) # keep a record of the the viz_transform_id self.viz_data_dictionary['google_realtime_dt'][data_product_id][ 'transform_proc'] = viz_transform_id2 def random_id_generator(self, size=8, chars=string.ascii_uppercase + string.digits): id = ''.join(random.choice(chars) for x in range(size)) return id def receive_new_dataproduct_event(self, event_msg, headers): """ For now we will start a thread that emulates an event handler """ # register the new data product self.register_new_data_product(event_msg.origin)
class ServiceGatewayService(BaseServiceGatewayService): """ The Service Gateway Service is the service that uses a gevent web server and Flask to bridge HTTP requests to AMQP RPC ION process service calls. """ def on_init(self): # defaults self.http_server = None # retain a pointer to this object for use in ProcessRPC calls global service_gateway_instance service_gateway_instance = self self.server_hostname = self.CFG.get_safe( "container.service_gateway.web_server.hostname", DEFAULT_WEB_SERVER_HOSTNAME ) self.server_port = self.CFG.get_safe("container.service_gateway.web_server.port", DEFAULT_WEB_SERVER_PORT) self.web_server_enabled = self.CFG.get_safe("container.service_gateway.web_server.enabled", True) self.logging = self.CFG.get_safe("container.service_gateway.web_server.log") # Optional list of trusted originators can be specified in config. self.trusted_originators = self.CFG.get_safe("container.service_gateway.trusted_originators") if not self.trusted_originators: self.trusted_originators = None log.info("Service Gateway will not check requests against trusted originators since none are configured.") # Get the user_cache_size self.user_cache_size = self.CFG.get_safe("container.service_gateway.user_cache_size", DEFAULT_USER_CACHE_SIZE) # Start the gevent web server unless disabled if self.web_server_enabled: log.info("Starting service gateway on %s:%s", self.server_hostname, self.server_port) self.start_service(self.server_hostname, self.server_port) self.user_role_event_subscriber = EventSubscriber( event_type="UserRoleModifiedEvent", origin_type="Org", callback=self.user_role_event_callback ) self.user_role_event_subscriber.activate() # Initialize an LRU Cache to keep user roles cached for performance reasons # maxSize = maximum number of elements to keep in cache # maxAgeMs = oldest entry to keep self.user_data_cache = LRUCache(self.user_cache_size, 0, 0) def on_quit(self): self.stop_service() if self.user_role_event_subscriber is not None: self.user_role_event_subscriber.deactivate() def start_service(self, hostname=DEFAULT_WEB_SERVER_HOSTNAME, port=DEFAULT_WEB_SERVER_PORT): """Responsible for starting the gevent based web server.""" if self.http_server is not None: self.stop_service() self.http_server = WSGIServer((hostname, port), app, log=self.logging) self.http_server.start() return True def stop_service(self): """Responsible for stopping the gevent based web server.""" if self.http_server is not None: self.http_server.stop() return True def is_trusted_address(self, requesting_address): if self.trusted_originators is None: return True for addr in self.trusted_originators: if requesting_address == addr: return True return False def user_role_event_callback(self, *args, **kwargs): """ This method is a callback function for receiving User Role Modified Events. """ user_role_event = args[0] org_id = user_role_event.origin user_id = user_role_event.user_id role_name = user_role_event.role_name log.debug("User Role modified: %s %s %s" % (org_id, user_id, role_name)) # Evict the user and their roles from the cache so that it gets updated with the next call. if service_gateway_instance.user_data_cache.has_key(user_id): log.debug("Evicting user from the user_data_cache: %s" % user_id) service_gateway_instance.user_data_cache.evict(user_id)
def test_pub_on_different_subsubtypes(self): res_list = [ DotDict(ar=event.AsyncResult(), gq=queue.Queue(), count=0) for i in xrange(4) ] def cb_gen(num): def cb(event, *args, **kwargs): res_list[num].count += 1 res_list[num].gq.put(event) if event.description == "end": res_list[num].ar.set() return cb sub0 = EventSubscriber(event_type="ResourceModifiedEvent", sub_type="st1.*", callback=cb_gen(0)) sub0.activate() sub1 = EventSubscriber(event_type="ResourceModifiedEvent", sub_type="st1.a", callback=cb_gen(1)) sub1.activate() sub2 = EventSubscriber(event_type="ResourceModifiedEvent", sub_type="*.a", callback=cb_gen(2)) sub2.activate() sub3 = EventSubscriber(event_type="ResourceModifiedEvent", sub_type="st1", callback=cb_gen(3)) sub3.activate() pub1 = EventPublisher(event_type="ResourceModifiedEvent") pub1.publish_event(origin="one", sub_type="st1.a", description="1") pub1.publish_event(origin="two", sub_type="st1", description="2") pub1.publish_event(origin="three", sub_type="st1.b", description="3") pub1.publish_event(origin="four", sub_type="st2.a", description="4") pub1.publish_event(origin="five", sub_type="st2", description="5") pub1.publish_event(origin="six", sub_type="a", description="6") pub1.publish_event(origin="seven", sub_type="", description="7") pub1.publish_event(origin="end", sub_type="st1.a", description="end") pub1.publish_event(origin="end", sub_type="st1", description="end") [res_list[i].ar.get(timeout=5) for i in xrange(3)] sub0.deactivate() sub1.deactivate() sub2.deactivate() sub3.deactivate() for i in xrange(4): res_list[i].res = [] for x in xrange(res_list[i].count): res_list[i].res.append(res_list[i].gq.get(timeout=5)) self.assertEquals(len(res_list[0].res), 3) self.assertEquals(res_list[0].res[0].description, "1") self.assertEquals(len(res_list[1].res), 2) self.assertEquals(res_list[1].res[0].description, "1") self.assertEquals(len(res_list[2].res), 3) self.assertEquals(res_list[2].res[0].description, "1") self.assertEquals(len(res_list[3].res), 2) self.assertEquals(res_list[3].res[0].description, "2")
class GovernanceController(object): def __init__(self,container): log.debug('GovernanceController.__init__()') self.container = container self.enabled = False self.interceptor_by_name_dict = dict() self.interceptor_order = [] self.policy_decision_point_manager = None self.governance_dispatcher = None def start(self): log.debug("GovernanceController starting ...") config = CFG.interceptor.interceptors.governance.config if config is None: config['enabled'] = False if "enabled" in config: self.enabled = config["enabled"] log.debug("GovernanceInterceptor enabled: %s" % str(self.enabled)) self.resource_policy_event_subscriber = None if self.enabled: self.initialize_from_config(config) self.resource_policy_event_subscriber = EventSubscriber(event_type="ResourcePolicyEvent", callback=self.policy_event_callback) self.resource_policy_event_subscriber.activate() self.rr_client = ResourceRegistryServiceProcessClient(node=self.container.node, process=self.container) self.policy_client = PolicyManagementServiceProcessClient(node=self.container.node, process=self.container) self.org_client = OrgManagementServiceProcessClient(node=self.container.node, process=self.container) def initialize_from_config(self, config): self.governance_dispatcher = GovernanceDispatcher() self.policy_decision_point_manager = PolicyDecisionPointManager() if 'interceptor_order' in config: self.interceptor_order = config['interceptor_order'] if 'governance_interceptors' in config: gov_ints = config['governance_interceptors'] for name in gov_ints: interceptor_def = gov_ints[name] # Instantiate and put in by_name array parts = interceptor_def["class"].split('.') modpath = ".".join(parts[:-1]) classname = parts[-1] module = __import__(modpath, fromlist=[classname]) classobj = getattr(module, classname) classinst = classobj() # Put in by_name_dict for possible re-use self.interceptor_by_name_dict[name] = classinst def stop(self): log.debug("GovernanceController stopping ...") if self.resource_policy_event_subscriber is not None: self.resource_policy_event_subscriber.deactivate() def process_incoming_message(self,invocation): self.process_message(invocation, self.interceptor_order,'incoming' ) return self.governance_dispatcher.handle_incoming_message(invocation) def process_outgoing_message(self,invocation): self.process_message(invocation, reversed(self.interceptor_order),'outgoing') return self.governance_dispatcher.handle_outgoing_message(invocation) def process_message(self,invocation,interceptor_list, method): for int_name in interceptor_list: class_inst = self.interceptor_by_name_dict[int_name] getattr(class_inst, method)(invocation) return invocation def policy_event_callback(self, *args, **kwargs): resource_policy_event = args[0] policy_id = resource_policy_event.origin resource_id = resource_policy_event.resource_id resource_type = resource_policy_event.resource_type resource_name = resource_policy_event.resource_name log.info("Resource policy modified: %s %s %s" % ( policy_id, resource_id, resource_type)) if resource_type == 'ServiceDefinition': #TODO - REDO to have a configurable Org boundary by container ion_org = self.org_client.find_org() policy_rules = self.policy_client.get_active_service_policy_rules(ion_org._id, resource_name) self.update_resource_policy(resource_name, policy_rules) elif resource_type == 'Org': policy_rules = self.policy_client.get_active_resource_policy_rules(resource_id) if self.policy_decision_point_manager is not None: self.policy_decision_point_manager.load_org_policy_rules(policy_rules) self.update_all_resource_policies(resource_id) else: policy_rules = self.policy_client.get_active_resource_policy_rules(resource_id) self.update_resource_policy(resource_id, policy_rules) def update_resource_policy(self, resource_name, policy_rules): #Notify policy decision point of updated rules if self.policy_decision_point_manager is not None: log.debug("Loading policy for resource: %s" % resource_name) self.policy_decision_point_manager.load_policy_rules(resource_name, policy_rules) def update_all_resource_policies(self, org_id): #Notify policy decision point of updated rules for all existing service policies if self.policy_decision_point_manager is not None: for res_name in self.policy_decision_point_manager.policy_decision_point: try: policy_rules = self.policy_client.get_active_service_policy_rules(org_id, res_name) self.update_resource_policy(res_name, policy_rules) except Exception, e: log.error(e.message)
class ProcessDispatcherServiceIntTest(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2cei.yml') self.pd_cli = ProcessDispatcherServiceClient(node=self.container.node) self.process_definition = ProcessDefinition(name='test_process') self.process_definition.executable = {'module': 'ion.services.cei.test.test_process_dispatcher', 'class':'TestProcess'} self.process_definition_id = self.pd_cli.create_process_definition(self.process_definition) self.event_queue = queue.Queue() self.event_sub = None def tearDown(self): if self.event_sub: self.event_sub.deactivate() def _event_callback(self, event, *args, **kwargs): self.event_queue.put(event) def subscribe_events(self, origin): self.event_sub = EventSubscriber(event_type="ProcessLifecycleEvent", callback=self._event_callback, origin=origin, origin_type="DispatchedProcess") self.event_sub.activate() def await_state_event(self, pid, state): event = self.event_queue.get(timeout=5) log.debug("Got event: %s", event) self.assertEqual(event.origin, pid) self.assertEqual(event.state, state) return event def test_create_schedule_cancel(self): process_schedule = ProcessSchedule() pid = self.pd_cli.create_process(self.process_definition_id) self.subscribe_events(pid) pid2 = self.pd_cli.schedule_process(self.process_definition_id, process_schedule, configuration={}, process_id=pid) self.assertEqual(pid, pid2) self.await_state_event(pid, ProcessStateEnum.SPAWN) # now try communicating with the process to make sure it is really running test_client = TestClient() for i in range(5): # this timeout may be too low self.assertEqual(i+1, test_client.count(timeout=1)) # kill the process and start it again self.pd_cli.cancel_process(pid) self.await_state_event(pid, ProcessStateEnum.TERMINATE) oldpid = pid pid = self.pd_cli.create_process(self.process_definition_id) self.subscribe_events(pid) pid2 = self.pd_cli.schedule_process(self.process_definition_id, process_schedule, configuration={}, process_id=pid) self.assertEqual(pid, pid2) self.assertNotEqual(oldpid, pid) self.await_state_event(pid, ProcessStateEnum.SPAWN) for i in range(5): # this timeout may be too low self.assertEqual(i+1, test_client.count(timeout=1)) # kill the process for good self.pd_cli.cancel_process(pid) self.await_state_event(pid, ProcessStateEnum.TERMINATE) def test_schedule_bad_config(self): process_schedule = ProcessSchedule() # a non-JSON-serializable IonObject o = ProcessTarget() with self.assertRaises(BadRequest) as ar: self.pd_cli.schedule_process(self.process_definition_id, process_schedule, configuration={"bad" : o}) self.assertTrue(ar.exception.message.startswith("bad configuration"))
class ExternalDatasetAgentTestBase(object): # Agent parameters. EDA_RESOURCE_ID = '123xyz' EDA_NAME = 'ExampleEDA' EDA_MOD = 'ion.agents.data.external_dataset_agent' EDA_CLS = 'ExternalDatasetAgent' """ Test cases for instrument agent class. Functions in this class provide instrument agent integration tests and provide a tutorial on use of the agent setup and interface. """ def setUp(self): """ Initialize test members. """ # log.warn('Starting the container') # Start container. self._start_container() # Bring up services in a deploy file # log.warn('Starting the rel') self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Create a pubsub client to create streams. # log.warn('Init a pubsub client') self._pubsub_client = PubsubManagementServiceClient(node=self.container.node) # log.warn('Init a ContainerAgentClient') self._container_client = ContainerAgentClient(node=self.container.node, name=self.container.name) # Data async and subscription TODO: Replace with new subscriber self._finished_count = None #TODO: Switch to gevent.queue.Queue self._async_finished_result = AsyncResult() self._finished_events_received = [] self._finished_event_subscriber = None self._start_finished_event_subscriber() self.addCleanup(self._stop_finished_event_subscriber) # TODO: Finish dealing with the resources and whatnot # TODO: DVR_CONFIG and (potentially) stream_config could both be reconfigured in self._setup_resources() self._setup_resources() #TG: Setup/configure the granule logger to log granules as they're published # Create agent config. agent_config = { 'driver_config' : self.DVR_CONFIG, 'stream_config' : {}, 'agent' : {'resource_id': self.EDA_RESOURCE_ID}, 'test_mode' : True } # Start instrument agent. self._ia_pid = None log.debug('TestInstrumentAgent.setup(): starting EDA.') self._ia_pid = self._container_client.spawn_process( name=self.EDA_NAME, module=self.EDA_MOD, cls=self.EDA_CLS, config=agent_config ) log.info('Agent pid=%s.', str(self._ia_pid)) # Start a resource agent client to talk with the instrument agent. self._ia_client = None self._ia_client = ResourceAgentClient(self.EDA_RESOURCE_ID, process=FakeProcess()) log.info('Got ia client %s.', str(self._ia_client)) ######################################## # Private "setup" functions ######################################## def _setup_resources(self): raise NotImplementedError('_setup_resources must be implemented in the subclass') def create_stream_and_logger(self, name, stream_id=''): if not stream_id or stream_id is '': stream_id = self._pubsub_client.create_stream(name=name, encoding='ION R2') pid = self._container_client.spawn_process( name=name+'_logger', module='ion.processes.data.stream_granule_logger', cls='StreamGranuleLogger', config={'process':{'stream_id':stream_id}} ) log.info('Started StreamGranuleLogger \'{0}\' subscribed to stream_id={1}'.format(pid, stream_id)) return stream_id def _start_finished_event_subscriber(self): def consume_event(*args,**kwargs): if args[0].description == 'TestingFinished': log.debug('TestingFinished event received') self._finished_events_received.append(args[0]) if self._finished_count and self._finished_count == len(self._finished_events_received): log.debug('Finishing test...') self._async_finished_result.set(len(self._finished_events_received)) log.debug('Called self._async_finished_result.set({0})'.format(len(self._finished_events_received))) self._finished_event_subscriber = EventSubscriber(event_type='DeviceEvent', callback=consume_event) self._finished_event_subscriber.activate() def _stop_finished_event_subscriber(self): if self._finished_event_subscriber: self._finished_event_subscriber.deactivate() self._finished_event_subscriber = None ######################################## # Custom assertion functions ######################################## def assertListsEqual(self, lst1, lst2): lst1.sort() lst2.sort() return lst1 == lst2 def assertSampleDict(self, val): """ Verify the value is a sample dictionary for the sbe37. """ #{'p': [-6.945], 'c': [0.08707], 't': [20.002], 'time': [1333752198.450622]} self.assertTrue(isinstance(val, dict)) self.assertTrue(val.has_key('c')) self.assertTrue(val.has_key('t')) self.assertTrue(val.has_key('p')) self.assertTrue(val.has_key('time')) c = val['c'][0] t = val['t'][0] p = val['p'][0] time = val['time'][0] self.assertTrue(isinstance(c, float)) self.assertTrue(isinstance(t, float)) self.assertTrue(isinstance(p, float)) self.assertTrue(isinstance(time, float)) def assertParamDict(self, pd, all_params=False): """ Verify all device parameters exist and are correct type. """ if all_params: self.assertEqual(set(pd.keys()), set(PARAMS.keys())) for (key, type_val) in PARAMS.iteritems(): if type_val == list or type_val == tuple: self.assertTrue(isinstance(pd[key], (list, tuple))) else: self.assertTrue(isinstance(pd[key], type_val)) else: for (key, val) in pd.iteritems(): self.assertTrue(PARAMS.has_key(key)) self.assertTrue(isinstance(val, PARAMS[key])) def assertParamVals(self, params, correct_params): """ Verify parameters take the correct values. """ self.assertEqual(set(params.keys()), set(correct_params.keys())) for (key, val) in params.iteritems(): correct_val = correct_params[key] if isinstance(val, float): # Verify to 5% of the larger value. max_val = max(abs(val), abs(correct_val)) self.assertAlmostEqual(val, correct_val, delta=max_val*.01) elif isinstance(val, (list, tuple)): # list of tuple. self.assertEqual(list(val), list(correct_val)) else: # int, bool, str. self.assertEqual(val, correct_val) ######################################## # Test functions ######################################## def test_acquire_data(self): cmd=AgentCommand(command='initialize') _ = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='go_active') _ = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='run') _ = self._ia_client.execute_agent(cmd) self._finished_count = 3 log.info('Send an unconstrained request for data (\'new data\')') cmd = AgentCommand(command='acquire_data') self._ia_client.execute(cmd) log.info('Send a second unconstrained request for data (\'new data\'), should be rejected') cmd = AgentCommand(command='acquire_data') self._ia_client.execute(cmd) config_mods={} log.info('Send a constrained request for data: constraints = HIST_CONSTRAINTS_1') config_mods['stream_id'] = self.create_stream_and_logger(name='stream_id_for_historical_1') config_mods['constraints']=self.HIST_CONSTRAINTS_1 cmd = AgentCommand(command='acquire_data', args=[config_mods]) self._ia_client.execute(cmd) log.info('Send a second constrained request for data: constraints = HIST_CONSTRAINTS_2') config_mods['stream_id'] = self.create_stream_and_logger(name='stream_id_for_historical_2') config_mods['constraints']=self.HIST_CONSTRAINTS_2 # config={'stream_id':'second_historical','TESTING':True, 'constraints':self.HIST_CONSTRAINTS_2} cmd = AgentCommand(command='acquire_data', args=[config_mods]) self._ia_client.execute(cmd) finished = self._async_finished_result.get(timeout=10) self.assertEqual(finished,self._finished_count) cmd = AgentCommand(command='reset') _ = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) def test_acquire_data_while_streaming(self): # Test instrument driver execute interface to start and stop streaming mode. cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) cmd = AgentCommand(command='initialize') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.INACTIVE) cmd = AgentCommand(command='go_active') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.IDLE) cmd = AgentCommand(command='run') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) # Make sure the polling interval is appropriate for a test params = { 'POLLING_INTERVAL':5 } self._ia_client.set_param(params) self._finished_count = 2 # Begin streaming. cmd = AgentCommand(command='go_streaming') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.STREAMING) config = get_safe(self.DVR_CONFIG, 'dh_cfg', {}) log.info('Send a constrained request for data: constraints = HIST_CONSTRAINTS_1') config['stream_id'] = self.create_stream_and_logger(name='stream_id_for_historical_1') config['constraints']=self.HIST_CONSTRAINTS_1 cmd = AgentCommand(command='acquire_data', args=[config]) reply = self._ia_client.execute(cmd) self.assertNotEqual(reply.status, 660) gevent.sleep(12) # Halt streaming. cmd = AgentCommand(command='go_observatory') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) # Assert that data was received self._async_finished_result.get(timeout=10) self.assertTrue(len(self._finished_events_received) >= 3) cmd = AgentCommand(command='reset') _ = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) def test_streaming(self): # Test instrument driver execute interface to start and stop streaming mode. cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) cmd = AgentCommand(command='initialize') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.INACTIVE) cmd = AgentCommand(command='go_active') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.IDLE) cmd = AgentCommand(command='run') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) # Make sure the polling interval is appropriate for a test params = { 'POLLING_INTERVAL':5 } self._ia_client.set_param(params) self._finished_count = 3 # Begin streaming. cmd = AgentCommand(command='go_streaming') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.STREAMING) # Wait for some samples to roll in. gevent.sleep(12) # Halt streaming. cmd = AgentCommand(command='go_observatory') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) # Assert that data was received self._async_finished_result.get(timeout=10) self.assertTrue(len(self._finished_events_received) >= 3) cmd = AgentCommand(command='reset') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) def test_observatory(self): # Test instrument driver get and set interface. cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) cmd = AgentCommand(command='initialize') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.INACTIVE) cmd = AgentCommand(command='go_active') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.IDLE) cmd = AgentCommand(command='run') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) # Retrieve all resource parameters. reply = self._ia_client.get_param('DRIVER_PARAMETER_ALL') self.assertParamDict(reply, True) orig_config = reply ## Retrieve a subset of resource parameters. params = [ 'POLLING_INTERVAL' ] reply = self._ia_client.get_param(params) self.assertParamDict(reply) orig_params = reply # Set a subset of resource parameters. new_params = { 'POLLING_INTERVAL' : (orig_params['POLLING_INTERVAL'] * 2), } self._ia_client.set_param(new_params) check_new_params = self._ia_client.get_param(params) self.assertParamVals(check_new_params, new_params) cmd = AgentCommand(command='reset') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) def test_get_set_param(self): cmd=AgentCommand(command='initialize') _ = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='go_active') _ = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='run') _ = self._ia_client.execute_agent(cmd) # Get a couple parameters retval = self._ia_client.get_param(['POLLING_INTERVAL','PATCHABLE_CONFIG_KEYS']) log.debug('Retrieved parameters from agent: {0}'.format(retval)) self.assertTrue(isinstance(retval,dict)) self.assertEqual(type(retval['POLLING_INTERVAL']),int) self.assertEqual(type(retval['PATCHABLE_CONFIG_KEYS']),list) # Attempt to get a parameter that doesn't exist log.debug('Try getting a non-existent parameter \'BAD_PARAM\'') self.assertRaises(InstParameterError, self._ia_client.get_param,['BAD_PARAM']) # Set the polling_interval to a new value, then get it to make sure it set properly self._ia_client.set_param({'POLLING_INTERVAL':10}) retval = self._ia_client.get_param(['POLLING_INTERVAL']) log.debug('Retrieved parameters from agent: {0}'.format(retval)) self.assertTrue(isinstance(retval,dict)) self.assertEqual(retval['POLLING_INTERVAL'],10) # Attempt to set a parameter that doesn't exist log.debug('Try setting a non-existent parameter \'BAD_PARAM\'') self.assertRaises(InstParameterError, self._ia_client.set_param, {'BAD_PARAM':'bad_val'}) # Attempt to set one parameter that does exist, and one that doesn't self.assertRaises(InstParameterError, self._ia_client.set_param, {'POLLING_INTERVAL':20,'BAD_PARAM':'bad_val'}) retval = self._ia_client.get_param(['POLLING_INTERVAL']) log.debug('Retrieved parameters from agent: {0}'.format(retval)) self.assertTrue(isinstance(retval,dict)) self.assertEqual(retval['POLLING_INTERVAL'],20) cmd = AgentCommand(command='reset') _ = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) def test_initialize(self): # Test agent initialize command. This causes creation of driver process and transition to inactive. cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) cmd = AgentCommand(command='initialize') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.INACTIVE) cmd = AgentCommand(command='reset') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) def test_states(self): # Test agent state transitions. cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) cmd = AgentCommand(command='initialize') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.INACTIVE) cmd = AgentCommand(command='go_active') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.IDLE) cmd = AgentCommand(command='run') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) cmd = AgentCommand(command='pause') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.STOPPED) cmd = AgentCommand(command='resume') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) cmd = AgentCommand(command='clear') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.IDLE) cmd = AgentCommand(command='run') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) cmd = AgentCommand(command='pause') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.STOPPED) cmd = AgentCommand(command='clear') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.IDLE) cmd = AgentCommand(command='run') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) self._finished_count = 1 cmd = AgentCommand(command='go_streaming') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.STREAMING) gevent.sleep(5) cmd = AgentCommand(command='go_observatory') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) self._async_finished_result.get(timeout=5) cmd = AgentCommand(command='reset') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) def test_capabilities(self): # Test the ability to retrieve agent and resource parameter and command capabilities. acmds = self._ia_client.get_capabilities(['AGT_CMD']) log.debug('Agent Commands: {0}'.format(acmds)) acmds = [item[1] for item in acmds] self.assertListsEqual(acmds, AGT_CMDS.keys()) apars = self._ia_client.get_capabilities(['AGT_PAR']) log.debug('Agent Parameters: {0}'.format(apars)) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) cmd = AgentCommand(command='initialize') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.INACTIVE) rcmds = self._ia_client.get_capabilities(['RES_CMD']) log.debug('Resource Commands: {0}'.format(rcmds)) rcmds = [item[1] for item in rcmds] self.assertListsEqual(rcmds, CMDS.keys()) rpars = self._ia_client.get_capabilities(['RES_PAR']) log.debug('Resource Parameters: {0}'.format(rpars)) rpars = [item[1] for item in rpars] self.assertListsEqual(rpars, PARAMS.keys()) cmd = AgentCommand(command='reset') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) def test_errors(self): # Test illegal behavior and replies. cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED) # Can't go active in unitialized state. # Status 660 is state error. cmd = AgentCommand(command='go_active') retval = self._ia_client.execute_agent(cmd) log.info('GO ACTIVE CMD %s',str(retval)) self.assertEquals(retval.status, 660) # Can't command driver in this state. cmd = AgentCommand(command='acquire_sample') reply = self._ia_client.execute(cmd) self.assertEqual(reply.status, 660) cmd = AgentCommand(command='initialize') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.INACTIVE) cmd = AgentCommand(command='go_active') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.IDLE) cmd = AgentCommand(command='run') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.OBSERVATORY) # 404 unknown agent command. cmd = AgentCommand(command='kiss_edward') retval = self._ia_client.execute_agent(cmd) self.assertEquals(retval.status, 404) # 670 unknown driver command. cmd = AgentCommand(command='acquire_sample_please') retval = self._ia_client.execute(cmd) self.assertEqual(retval.status, 670) # 630 Parameter error. self.assertRaises(InstParameterError, self._ia_client.get_param, 'bogus bogus') cmd = AgentCommand(command='reset') retval = self._ia_client.execute_agent(cmd) cmd = AgentCommand(command='get_current_state') retval = self._ia_client.execute_agent(cmd) state = retval.result self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)