def test_pub_on_different_subtypes(self):
ar = event.AsyncResult()
gq = queue.Queue()
self.count = 0
def cb(event, *args, **kwargs):
self.count += 1
gq.put(event)
if event.description == "end":
ar.set()
sub = EventSubscriber(event_type="ResourceModifiedEvent",
sub_type="st1",
callback=cb)
sub.activate()
pub1 = EventPublisher(event_type="ResourceModifiedEvent")
pub2 = EventPublisher(event_type="ContainerLifecycleEvent")
pub1.publish_event(origin="two", sub_type="st2", description="2")
pub2.publish_event(origin="three", sub_type="st1", description="3")
pub1.publish_event(origin="one", sub_type="st1", description="1")
pub1.publish_event(origin="four", sub_type="st1", description="end")
ar.get(timeout=5)
sub.deactivate()
res = []
for x in xrange(self.count):
res.append(gq.get(timeout=5))
self.assertEquals(len(res), 2)
self.assertEquals(res[0].description, "1")
def test_pub_on_different_subtypes(self):
ar = event.AsyncResult()
gq = queue.Queue()
self.count = 0
def cb(event, *args, **kwargs):
self.count += 1
gq.put(event)
if event.description == "end":
ar.set()
sub = EventSubscriber(event_type="ResourceModifiedEvent", sub_type="st1", callback=cb)
sub.activate()
pub1 = EventPublisher(event_type="ResourceModifiedEvent")
pub2 = EventPublisher(event_type="ContainerLifecycleEvent")
pub1.publish_event(origin="two", sub_type="st2", description="2")
pub2.publish_event(origin="three", sub_type="st1", description="3")
pub1.publish_event(origin="one", sub_type="st1", description="1")
pub1.publish_event(origin="four", sub_type="st1", description="end")
ar.get(timeout=5)
sub.deactivate()
res = []
for x in xrange(self.count):
res.append(gq.get(timeout=5))
self.assertEquals(len(res), 2)
self.assertEquals(res[0].description, "1")
def test_l4_ci_sa_rq_145_323(self):
"""
Instrument management shall update physical resource metadata when change occurs
For example, when there is a change of state.
note from maurice 2012-05-18: consider this to mean a change of stored RR data
"""
inst_obj = any_old(RT.InstrumentDevice)
instrument_device_id, _ = self.RR.create(inst_obj)
self.received_event = AsyncResult()
#Create subscribers for agent and driver events.
def consume_event(*args, **kwargs):
self.received_event.set(True)
log.info("L4-CI-SA-RQ-323")
log.info("L4-CI-SA-RQ-145")
event_sub = EventSubscriber(event_type="ResourceModifiedEvent", callback=consume_event)
event_sub.activate()
inst_obj = self.RR.read(instrument_device_id)
inst_obj.description = "brand new description"
self.RR.update(inst_obj)
#wait for event
result = self.received_event.get(timeout=10)
event_sub.deactivate()
self.assertTrue(result)
def _start_event_subscribers(self):
"""
Create subscribers for agent and driver events.
"""
def consume_event(*args, **kwargs):
log.info('Test recieved ION event: args=%s kwargs=%s.', str(args), str(kwargs))
self._events_received.append(args[0])
if self._no_events and self._no_events == len(self._event_received):
self._async_event_result.set()
event_sub = EventSubscriber(event_type="DeviceEvent", callback=consume_event)
event_sub.activate()
self._event_subscribers.append(event_sub)
def _start_event_subscribers(self):
"""
Create subscribers for agent and driver events.
"""
def consume_event(*args, **kwargs):
log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.',
str(args), str(kwargs), str(args[0]))
self._events_received.append(args[0])
if self._no_events and self._no_events == len(self._event_received):
self._async_event_result.set()
event_sub = EventSubscriber(event_type="DeviceEvent", callback=consume_event)
event_sub.activate()
self._event_subscribers.append(event_sub)
def __init__(self):
# Start event subscribers, add stop to cleanup.
self.no_events = None
self.async_event_result = AsyncResult()
self.events_received = []
self.event_subscribers = []
def consume_event(*args, **kwargs):
log.info('Test recieved ION event: args=%s, kwargs=%s, event=%s.',
str(args), str(kwargs), str(args[0]))
self.events_received.append(args[0])
if self.no_events and self.no_events == len(self.event_received):
self.async_event_result.set()
event_sub = EventSubscriber(event_type="DeviceEvent", callback=consume_event)
event_sub.activate()
self.event_subscribers.append(event_sub)
class Notification(object):
"""
Encapsulates a notification's info and it's event subscriber
@David - is this class needed? It does not seem to serve any purpose?
"""
def __init__(self, notification_request=None, subscriber_callback=None):
self._res_obj = notification_request # The notification Request Resource Object
# setup subscription using subscription_callback()
# msg_recipientDO: make this walk the lists and set up a subscriber for every pair of
# origin/event. This will require a list to hold all the subscribers so they can
# be started and killed
self.subscriber = EventSubscriber(
origin=notification_request.origin,
origin_type=notification_request.origin_type,
event_type=notification_request.event_type,
sub_type=notification_request.event_subtype,
callback=subscriber_callback)
self.notification_id = None
def set_notification_id(self, id_=None):
"""
Set the notification id of the notification object
@param notification id
"""
self.notification_id = id_
def activate(self):
"""
Start subscribing
"""
self.subscriber.activate()
def deactivate(self):
"""
Stop subscribing
"""
self.subscriber.deactivate()
class Notification(object):
"""
Encapsulates a notification's info and it's event subscriber
@David - is this class needed? It does not seem to serve any purpose?
"""
def __init__(self, notification_request=None, subscriber_callback=None):
self._res_obj = notification_request # The notification Request Resource Object
# setup subscription using subscription_callback()
# msg_recipientDO: make this walk the lists and set up a subscriber for every pair of
# origin/event. This will require a list to hold all the subscribers so they can
# be started and killed
self.subscriber = EventSubscriber(origin=notification_request.origin,
origin_type = notification_request.origin_type,
event_type=notification_request.event_type,
sub_type=notification_request.event_subtype,
callback=subscriber_callback)
self.notification_id = None
def set_notification_id(self, id_=None):
"""
Set the notification id of the notification object
@param notification id
"""
self.notification_id = id_
def activate(self):
"""
Start subscribing
"""
self.subscriber.activate()
def deactivate(self):
"""
Stop subscribing
"""
self.subscriber.deactivate()
def test_pub_on_different_subsubtypes(self):
res_list = [DotDict(ar=event.AsyncResult(), gq=queue.Queue(), count=0) for i in xrange(4)]
def cb_gen(num):
def cb(event, *args, **kwargs):
res_list[num].count += 1
res_list[num].gq.put(event)
if event.description == "end":
res_list[num].ar.set()
return cb
sub0 = EventSubscriber(event_type="ResourceModifiedEvent", sub_type="st1.*", callback=cb_gen(0))
sub0.activate()
sub1 = EventSubscriber(event_type="ResourceModifiedEvent", sub_type="st1.a", callback=cb_gen(1))
sub1.activate()
sub2 = EventSubscriber(event_type="ResourceModifiedEvent", sub_type="*.a", callback=cb_gen(2))
sub2.activate()
sub3 = EventSubscriber(event_type="ResourceModifiedEvent", sub_type="st1", callback=cb_gen(3))
sub3.activate()
pub1 = EventPublisher(event_type="ResourceModifiedEvent")
pub1.publish_event(origin="one", sub_type="st1.a", description="1")
pub1.publish_event(origin="two", sub_type="st1", description="2")
pub1.publish_event(origin="three", sub_type="st1.b", description="3")
pub1.publish_event(origin="four", sub_type="st2.a", description="4")
pub1.publish_event(origin="five", sub_type="st2", description="5")
pub1.publish_event(origin="six", sub_type="a", description="6")
pub1.publish_event(origin="seven", sub_type="", description="7")
pub1.publish_event(origin="end", sub_type="st1.a", description="end")
pub1.publish_event(origin="end", sub_type="st1", description="end")
[res_list[i].ar.get(timeout=5) for i in xrange(3)]
sub0.deactivate()
sub1.deactivate()
sub2.deactivate()
sub3.deactivate()
for i in xrange(4):
res_list[i].res = []
for x in xrange(res_list[i].count):
res_list[i].res.append(res_list[i].gq.get(timeout=5))
self.assertEquals(len(res_list[0].res), 3)
self.assertEquals(res_list[0].res[0].description, "1")
self.assertEquals(len(res_list[1].res), 2)
self.assertEquals(res_list[1].res[0].description, "1")
self.assertEquals(len(res_list[2].res), 3)
self.assertEquals(res_list[2].res[0].description, "1")
self.assertEquals(len(res_list[3].res), 2)
self.assertEquals(res_list[3].res[0].description, "2")
class ExternalDatasetAgentTestBase(object):
# Agent parameters.
EDA_RESOURCE_ID = '123xyz'
EDA_NAME = 'ExampleEDA'
EDA_MOD = 'ion.agents.data.external_dataset_agent'
EDA_CLS = 'ExternalDatasetAgent'
"""
Test cases for instrument agent class. Functions in this class provide
instrument agent integration tests and provide a tutorial on use of
the agent setup and interface.
"""
def setUp(self):
"""
Initialize test members.
"""
# log.warn('Starting the container')
# Start container.
self._start_container()
# Bring up services in a deploy file
# log.warn('Starting the rel')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Create a pubsub client to create streams.
# log.warn('Init a pubsub client')
self._pubsub_client = PubsubManagementServiceClient(
node=self.container.node)
# log.warn('Init a ContainerAgentClient')
self._container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
# Data async and subscription TODO: Replace with new subscriber
self._finished_count = None
#TODO: Switch to gevent.queue.Queue
self._async_finished_result = AsyncResult()
self._finished_events_received = []
self._finished_event_subscriber = None
self._start_finished_event_subscriber()
self.addCleanup(self._stop_finished_event_subscriber)
# TODO: Finish dealing with the resources and whatnot
# TODO: DVR_CONFIG and (potentially) stream_config could both be reconfigured in self._setup_resources()
self._setup_resources()
#TG: Setup/configure the granule logger to log granules as they're published
# Create agent config.
agent_config = {
'driver_config': self.DVR_CONFIG,
'stream_config': {},
'agent': {
'resource_id': self.EDA_RESOURCE_ID
},
'test_mode': True
}
# Start instrument agent.
self._ia_pid = None
log.debug('TestInstrumentAgent.setup(): starting EDA.')
self._ia_pid = self._container_client.spawn_process(
name=self.EDA_NAME,
module=self.EDA_MOD,
cls=self.EDA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(self._ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = None
self._ia_client = ResourceAgentClient(self.EDA_RESOURCE_ID,
process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
########################################
# Private "setup" functions
########################################
def _setup_resources(self):
raise NotImplementedError(
'_setup_resources must be implemented in the subclass')
def create_stream_and_logger(self, name, stream_id=''):
if not stream_id or stream_id is '':
stream_id = self._pubsub_client.create_stream(name=name,
encoding='ION R2')
pid = self._container_client.spawn_process(
name=name + '_logger',
module='ion.processes.data.stream_granule_logger',
cls='StreamGranuleLogger',
config={'process': {
'stream_id': stream_id
}})
log.info(
'Started StreamGranuleLogger \'{0}\' subscribed to stream_id={1}'.
format(pid, stream_id))
return stream_id
def _start_finished_event_subscriber(self):
def consume_event(*args, **kwargs):
if args[0].description == 'TestingFinished':
log.debug('TestingFinished event received')
self._finished_events_received.append(args[0])
if self._finished_count and self._finished_count == len(
self._finished_events_received):
log.debug('Finishing test...')
self._async_finished_result.set(
len(self._finished_events_received))
log.debug(
'Called self._async_finished_result.set({0})'.format(
len(self._finished_events_received)))
self._finished_event_subscriber = EventSubscriber(
event_type='DeviceEvent', callback=consume_event)
self._finished_event_subscriber.activate()
def _stop_finished_event_subscriber(self):
if self._finished_event_subscriber:
self._finished_event_subscriber.deactivate()
self._finished_event_subscriber = None
########################################
# Custom assertion functions
########################################
def assertListsEqual(self, lst1, lst2):
lst1.sort()
lst2.sort()
return lst1 == lst2
def assertSampleDict(self, val):
"""
Verify the value is a sample dictionary for the sbe37.
"""
#{'p': [-6.945], 'c': [0.08707], 't': [20.002], 'time': [1333752198.450622]}
self.assertTrue(isinstance(val, dict))
self.assertTrue(val.has_key('c'))
self.assertTrue(val.has_key('t'))
self.assertTrue(val.has_key('p'))
self.assertTrue(val.has_key('time'))
c = val['c'][0]
t = val['t'][0]
p = val['p'][0]
time = val['time'][0]
self.assertTrue(isinstance(c, float))
self.assertTrue(isinstance(t, float))
self.assertTrue(isinstance(p, float))
self.assertTrue(isinstance(time, float))
def assertParamDict(self, pd, all_params=False):
"""
Verify all device parameters exist and are correct type.
"""
if all_params:
self.assertEqual(set(pd.keys()), set(PARAMS.keys()))
for (key, type_val) in PARAMS.iteritems():
if type_val == list or type_val == tuple:
self.assertTrue(isinstance(pd[key], (list, tuple)))
else:
self.assertTrue(isinstance(pd[key], type_val))
else:
for (key, val) in pd.iteritems():
self.assertTrue(PARAMS.has_key(key))
self.assertTrue(isinstance(val, PARAMS[key]))
def assertParamVals(self, params, correct_params):
"""
Verify parameters take the correct values.
"""
self.assertEqual(set(params.keys()), set(correct_params.keys()))
for (key, val) in params.iteritems():
correct_val = correct_params[key]
if isinstance(val, float):
# Verify to 5% of the larger value.
max_val = max(abs(val), abs(correct_val))
self.assertAlmostEqual(val, correct_val, delta=max_val * .01)
elif isinstance(val, (list, tuple)):
# list of tuple.
self.assertEqual(list(val), list(correct_val))
else:
# int, bool, str.
self.assertEqual(val, correct_val)
########################################
# Test functions
########################################
def test_acquire_data(self):
cmd = AgentCommand(command='initialize')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='go_active')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='run')
_ = self._ia_client.execute_agent(cmd)
self._finished_count = 3
log.info('Send an unconstrained request for data (\'new data\')')
cmd = AgentCommand(command='acquire_data')
self._ia_client.execute(cmd)
log.info(
'Send a second unconstrained request for data (\'new data\'), should be rejected'
)
cmd = AgentCommand(command='acquire_data')
self._ia_client.execute(cmd)
config_mods = {}
log.info(
'Send a constrained request for data: constraints = HIST_CONSTRAINTS_1'
)
config_mods['stream_id'] = self.create_stream_and_logger(
name='stream_id_for_historical_1')
config_mods['constraints'] = self.HIST_CONSTRAINTS_1
cmd = AgentCommand(command='acquire_data', args=[config_mods])
self._ia_client.execute(cmd)
log.info(
'Send a second constrained request for data: constraints = HIST_CONSTRAINTS_2'
)
config_mods['stream_id'] = self.create_stream_and_logger(
name='stream_id_for_historical_2')
config_mods['constraints'] = self.HIST_CONSTRAINTS_2
# config={'stream_id':'second_historical','TESTING':True, 'constraints':self.HIST_CONSTRAINTS_2}
cmd = AgentCommand(command='acquire_data', args=[config_mods])
self._ia_client.execute(cmd)
finished = self._async_finished_result.get(timeout=10)
self.assertEqual(finished, self._finished_count)
cmd = AgentCommand(command='reset')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_acquire_data_while_streaming(self):
# Test instrument driver execute interface to start and stop streaming mode.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Make sure the polling interval is appropriate for a test
params = {'POLLING_INTERVAL': 5}
self._ia_client.set_param(params)
self._finished_count = 2
# Begin streaming.
cmd = AgentCommand(command='go_streaming')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STREAMING)
config = get_safe(self.DVR_CONFIG, 'dh_cfg', {})
log.info(
'Send a constrained request for data: constraints = HIST_CONSTRAINTS_1'
)
config['stream_id'] = self.create_stream_and_logger(
name='stream_id_for_historical_1')
config['constraints'] = self.HIST_CONSTRAINTS_1
cmd = AgentCommand(command='acquire_data', args=[config])
reply = self._ia_client.execute(cmd)
self.assertNotEqual(reply.status, 660)
gevent.sleep(12)
# Halt streaming.
cmd = AgentCommand(command='go_observatory')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Assert that data was received
self._async_finished_result.get(timeout=10)
self.assertTrue(len(self._finished_events_received) >= 3)
cmd = AgentCommand(command='reset')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_streaming(self):
# Test instrument driver execute interface to start and stop streaming mode.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Make sure the polling interval is appropriate for a test
params = {'POLLING_INTERVAL': 5}
self._ia_client.set_param(params)
self._finished_count = 3
# Begin streaming.
cmd = AgentCommand(command='go_streaming')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STREAMING)
# Wait for some samples to roll in.
gevent.sleep(12)
# Halt streaming.
cmd = AgentCommand(command='go_observatory')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Assert that data was received
self._async_finished_result.get(timeout=10)
self.assertTrue(len(self._finished_events_received) >= 3)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_observatory(self):
# Test instrument driver get and set interface.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Retrieve all resource parameters.
reply = self._ia_client.get_param('DRIVER_PARAMETER_ALL')
self.assertParamDict(reply, True)
orig_config = reply
## Retrieve a subset of resource parameters.
params = ['POLLING_INTERVAL']
reply = self._ia_client.get_param(params)
self.assertParamDict(reply)
orig_params = reply
# Set a subset of resource parameters.
new_params = {
'POLLING_INTERVAL': (orig_params['POLLING_INTERVAL'] * 2),
}
self._ia_client.set_param(new_params)
check_new_params = self._ia_client.get_param(params)
self.assertParamVals(check_new_params, new_params)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_get_set_param(self):
cmd = AgentCommand(command='initialize')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='go_active')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='run')
_ = self._ia_client.execute_agent(cmd)
# Get a couple parameters
retval = self._ia_client.get_param(
['POLLING_INTERVAL', 'PATCHABLE_CONFIG_KEYS'])
log.debug('Retrieved parameters from agent: {0}'.format(retval))
self.assertTrue(isinstance(retval, dict))
self.assertEqual(type(retval['POLLING_INTERVAL']), int)
self.assertEqual(type(retval['PATCHABLE_CONFIG_KEYS']), list)
# Attempt to get a parameter that doesn't exist
log.debug('Try getting a non-existent parameter \'BAD_PARAM\'')
self.assertRaises(InstParameterError, self._ia_client.get_param,
['BAD_PARAM'])
# Set the polling_interval to a new value, then get it to make sure it set properly
self._ia_client.set_param({'POLLING_INTERVAL': 10})
retval = self._ia_client.get_param(['POLLING_INTERVAL'])
log.debug('Retrieved parameters from agent: {0}'.format(retval))
self.assertTrue(isinstance(retval, dict))
self.assertEqual(retval['POLLING_INTERVAL'], 10)
# Attempt to set a parameter that doesn't exist
log.debug('Try setting a non-existent parameter \'BAD_PARAM\'')
self.assertRaises(InstParameterError, self._ia_client.set_param,
{'BAD_PARAM': 'bad_val'})
# Attempt to set one parameter that does exist, and one that doesn't
self.assertRaises(InstParameterError, self._ia_client.set_param, {
'POLLING_INTERVAL': 20,
'BAD_PARAM': 'bad_val'
})
retval = self._ia_client.get_param(['POLLING_INTERVAL'])
log.debug('Retrieved parameters from agent: {0}'.format(retval))
self.assertTrue(isinstance(retval, dict))
self.assertEqual(retval['POLLING_INTERVAL'], 20)
cmd = AgentCommand(command='reset')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_initialize(self):
# Test agent initialize command. This causes creation of driver process and transition to inactive.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_states(self):
# Test agent state transitions.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='pause')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command='resume')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='clear')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='pause')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command='clear')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
self._finished_count = 1
cmd = AgentCommand(command='go_streaming')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STREAMING)
gevent.sleep(5)
cmd = AgentCommand(command='go_observatory')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
self._async_finished_result.get(timeout=5)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_capabilities(self):
# Test the ability to retrieve agent and resource parameter and command capabilities.
acmds = self._ia_client.get_capabilities(['AGT_CMD'])
log.debug('Agent Commands: {0}'.format(acmds))
acmds = [item[1] for item in acmds]
self.assertListsEqual(acmds, AGT_CMDS.keys())
apars = self._ia_client.get_capabilities(['AGT_PAR'])
log.debug('Agent Parameters: {0}'.format(apars))
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
rcmds = self._ia_client.get_capabilities(['RES_CMD'])
log.debug('Resource Commands: {0}'.format(rcmds))
rcmds = [item[1] for item in rcmds]
self.assertListsEqual(rcmds, CMDS.keys())
rpars = self._ia_client.get_capabilities(['RES_PAR'])
log.debug('Resource Parameters: {0}'.format(rpars))
rpars = [item[1] for item in rpars]
self.assertListsEqual(rpars, PARAMS.keys())
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_errors(self):
# Test illegal behavior and replies.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
# Can't go active in unitialized state.
# Status 660 is state error.
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
log.info('GO ACTIVE CMD %s', str(retval))
self.assertEquals(retval.status, 660)
# Can't command driver in this state.
cmd = AgentCommand(command='acquire_sample')
reply = self._ia_client.execute(cmd)
self.assertEqual(reply.status, 660)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# 404 unknown agent command.
cmd = AgentCommand(command='kiss_edward')
retval = self._ia_client.execute_agent(cmd)
self.assertEquals(retval.status, 404)
# 670 unknown driver command.
cmd = AgentCommand(command='acquire_sample_please')
retval = self._ia_client.execute(cmd)
self.assertEqual(retval.status, 670)
# 630 Parameter error.
self.assertRaises(InstParameterError, self._ia_client.get_param,
'bogus bogus')
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
class PolicyManagementService(BasePolicyManagementService):
def on_init(self):
self.event_pub = EventPublisher()
self.policy_event_subscriber = EventSubscriber(event_type="ResourceModifiedEvent", origin_type="Policy", callback=self.policy_event_callback)
self.policy_event_subscriber.activate()
def on_quit(self):
if self.policy_event_subscriber is not None:
self.policy_event_subscriber.deactivate()
"""
Provides the interface to define and manage policy and a repository to store and retrieve policy and templates for
policy definitions, aka attribute authority.
"""
def create_policy(self, policy=None):
"""Persists the provided Policy object for the specified Org id. The id string returned
is the internal id by which Policy will be identified in the data store.
@param policy Policy
@retval policy_id str
@throws BadRequest if object passed has _id or _rev attribute
"""
if not is_basic_identifier(policy.name):
raise BadRequest("The policy name '%s' can only contain alphanumeric and underscore characters" % policy.name)
policy.rule = policy.rule % (policy.name, policy.description)
policy_id, version = self.clients.resource_registry.create(policy)
return policy_id
def update_policy(self, policy=None):
"""Updates the provided Policy object. Throws NotFound exception if
an existing version of Policy is not found. Throws Conflict if
the provided Policy object is not based on the latest persisted
version of the object.
@param policy Policy
@throws NotFound object with specified id does not exist
@throws BadRequest if object does not have _id or _rev attribute
@throws Conflict object not based on latest persisted object version
"""
if not is_basic_identifier(policy.name):
raise BadRequest("The policy name '%s' can only contain alphanumeric and underscore characters" % policy.name)
self.clients.resource_registry.update(policy)
def read_policy(self, policy_id=''):
"""Returns the Policy object for the specified policy id.
Throws exception if id does not match any persisted Policy
objects.
@param policy_id str
@retval policy Policy
@throws NotFound object with specified id does not exist
"""
if not policy_id:
raise BadRequest("The policy_id parameter is missing")
policy = self.clients.resource_registry.read(policy_id)
if not policy:
raise NotFound("Policy %s does not exist" % policy_id)
return policy
def delete_policy(self, policy_id=''):
"""For now, permanently deletes Policy object with the specified
id. Throws exception if id does not match any persisted Policy.
@param policy_id str
@throws NotFound object with specified id does not exist
"""
if not policy_id:
raise BadRequest("The policy_id parameter is missing")
policy = self.clients.resource_registry.read(policy_id)
if not policy:
raise NotFound("Policy %s does not exist" % policy_id)
self.clients.resource_registry.delete(policy_id)
def enable_policy(self, policy_id=''):
"""Sets a flag to enable the use of the policy rule
@param policy_id str
@throws NotFound object with specified id does not exist
"""
policy = self.read_policy(policy_id)
policy.enabled = True
self.update_policy(policy)
def disable_policy(self, policy_id=''):
"""Resets a flag to disable the use of the policy rule
@param policy_id str
@throws NotFound object with specified id does not exist
"""
policy = self.read_policy(policy_id)
policy.enabled = False
self.update_policy(policy)
def policy_event_callback(self, *args, **kwargs):
"""
This method is a callback function for receiving Policy Events.
"""
policy_event = args[0]
policy_id = policy_event.origin
log.debug("Policy modified: %s" % policy_id)
try:
policy = self.clients.resource_registry.read(policy_id)
if policy:
#Need to publish an event that a policy has changed for any associated resource
res_list = self._find_resources_for_policy(policy_id)
for res in res_list:
self._publish_resource_policy_event(policy, res)
except Exception, e:
#If this is a delete operation, then don't bother with not finding the object.
if policy_event.sub_type != 'DELETE':
log.error(e)
class PolicyManagementService(BasePolicyManagementService):
def on_init(self):
self.event_pub = EventPublisher()
self.policy_event_subscriber = EventSubscriber(
event_type="ResourceModifiedEvent",
origin_type="Policy",
callback=self.policy_event_callback)
self.policy_event_subscriber.activate()
def on_quit(self):
if self.policy_event_subscriber is not None:
self.policy_event_subscriber.deactivate()
"""
Provides the interface to define and manage policy and a repository to store and retrieve policy and templates for
policy definitions, aka attribute authority.
"""
def create_policy(self, policy=None):
"""Persists the provided Policy object for the specified Org id. The id string returned
is the internal id by which Policy will be identified in the data store.
@param policy Policy
@retval policy_id str
@throws BadRequest if object passed has _id or _rev attribute
"""
if not is_basic_identifier(policy.name):
raise BadRequest(
"The policy name '%s' can only contain alphanumeric and underscore characters"
% policy.name)
policy.rule = policy.rule % (policy.name, policy.description)
policy_id, version = self.clients.resource_registry.create(policy)
return policy_id
def update_policy(self, policy=None):
"""Updates the provided Policy object. Throws NotFound exception if
an existing version of Policy is not found. Throws Conflict if
the provided Policy object is not based on the latest persisted
version of the object.
@param policy Policy
@throws NotFound object with specified id does not exist
@throws BadRequest if object does not have _id or _rev attribute
@throws Conflict object not based on latest persisted object version
"""
if not is_basic_identifier(policy.name):
raise BadRequest(
"The policy name '%s' can only contain alphanumeric and underscore characters"
% policy.name)
self.clients.resource_registry.update(policy)
def read_policy(self, policy_id=''):
"""Returns the Policy object for the specified policy id.
Throws exception if id does not match any persisted Policy
objects.
@param policy_id str
@retval policy Policy
@throws NotFound object with specified id does not exist
"""
if not policy_id:
raise BadRequest("The policy_id parameter is missing")
policy = self.clients.resource_registry.read(policy_id)
if not policy:
raise NotFound("Policy %s does not exist" % policy_id)
return policy
def delete_policy(self, policy_id=''):
"""For now, permanently deletes Policy object with the specified
id. Throws exception if id does not match any persisted Policy.
@param policy_id str
@throws NotFound object with specified id does not exist
"""
if not policy_id:
raise BadRequest("The policy_id parameter is missing")
policy = self.clients.resource_registry.read(policy_id)
if not policy:
raise NotFound("Policy %s does not exist" % policy_id)
self.clients.resource_registry.delete(policy_id)
def enable_policy(self, policy_id=''):
"""Sets a flag to enable the use of the policy rule
@param policy_id str
@throws NotFound object with specified id does not exist
"""
policy = self.read_policy(policy_id)
policy.enabled = True
self.update_policy(policy)
def disable_policy(self, policy_id=''):
"""Resets a flag to disable the use of the policy rule
@param policy_id str
@throws NotFound object with specified id does not exist
"""
policy = self.read_policy(policy_id)
policy.enabled = False
self.update_policy(policy)
def policy_event_callback(self, *args, **kwargs):
"""
This method is a callback function for receiving Policy Events.
"""
policy_event = args[0]
policy_id = policy_event.origin
log.debug("Policy modified: %s" % policy_id)
try:
policy = self.clients.resource_registry.read(policy_id)
if policy:
#Need to publish an event that a policy has changed for any associated resource
res_list = self._find_resources_for_policy(policy_id)
for res in res_list:
self._publish_resource_policy_event(policy, res)
except Exception, e:
#If this is a delete operation, then don't bother with not finding the object.
if policy_event.sub_type != 'DELETE':
log.error(e)
class GovernanceController(object):
def __init__(self, container):
log.debug('GovernanceController.__init__()')
self.container = container
self.enabled = False
self.interceptor_by_name_dict = dict()
self.interceptor_order = []
self.policy_decision_point_manager = None
self.governance_dispatcher = None
def start(self):
log.debug("GovernanceController starting ...")
config = CFG.interceptor.interceptors.governance.config
if config is None:
config['enabled'] = False
if "enabled" in config:
self.enabled = config["enabled"]
log.debug("GovernanceInterceptor enabled: %s" % str(self.enabled))
self.resource_policy_event_subscriber = None
if self.enabled:
self.initialize_from_config(config)
self.resource_policy_event_subscriber = EventSubscriber(
event_type="ResourcePolicyEvent",
callback=self.policy_event_callback)
self.resource_policy_event_subscriber.activate()
self.rr_client = ResourceRegistryServiceProcessClient(
node=self.container.node, process=self.container)
self.policy_client = PolicyManagementServiceProcessClient(
node=self.container.node, process=self.container)
self.org_client = OrgManagementServiceProcessClient(
node=self.container.node, process=self.container)
def initialize_from_config(self, config):
self.governance_dispatcher = GovernanceDispatcher()
self.policy_decision_point_manager = PolicyDecisionPointManager()
if 'interceptor_order' in config:
self.interceptor_order = config['interceptor_order']
if 'governance_interceptors' in config:
gov_ints = config['governance_interceptors']
for name in gov_ints:
interceptor_def = gov_ints[name]
# Instantiate and put in by_name array
parts = interceptor_def["class"].split('.')
modpath = ".".join(parts[:-1])
classname = parts[-1]
module = __import__(modpath, fromlist=[classname])
classobj = getattr(module, classname)
classinst = classobj()
# Put in by_name_dict for possible re-use
self.interceptor_by_name_dict[name] = classinst
def stop(self):
log.debug("GovernanceController stopping ...")
if self.resource_policy_event_subscriber is not None:
self.resource_policy_event_subscriber.deactivate()
def process_incoming_message(self, invocation):
self.process_message(invocation, self.interceptor_order, 'incoming')
return self.governance_dispatcher.handle_incoming_message(invocation)
def process_outgoing_message(self, invocation):
self.process_message(invocation, reversed(self.interceptor_order),
'outgoing')
return self.governance_dispatcher.handle_outgoing_message(invocation)
def process_message(self, invocation, interceptor_list, method):
for int_name in interceptor_list:
class_inst = self.interceptor_by_name_dict[int_name]
getattr(class_inst, method)(invocation)
return invocation
def policy_event_callback(self, *args, **kwargs):
resource_policy_event = args[0]
policy_id = resource_policy_event.origin
resource_id = resource_policy_event.resource_id
resource_type = resource_policy_event.resource_type
resource_name = resource_policy_event.resource_name
log.info("Resource policy modified: %s %s %s" %
(policy_id, resource_id, resource_type))
if resource_type == 'ServiceDefinition': #TODO - REDO to have a configurable Org boundary by container
ion_org = self.org_client.find_org()
policy_rules = self.policy_client.get_active_service_policy_rules(
ion_org._id, resource_name)
self.update_resource_policy(resource_name, policy_rules)
elif resource_type == 'Org':
policy_rules = self.policy_client.get_active_resource_policy_rules(
resource_id)
if self.policy_decision_point_manager is not None:
self.policy_decision_point_manager.load_org_policy_rules(
policy_rules)
self.update_all_resource_policies(resource_id)
else:
policy_rules = self.policy_client.get_active_resource_policy_rules(
resource_id)
self.update_resource_policy(resource_id, policy_rules)
def update_resource_policy(self, resource_name, policy_rules):
#Notify policy decision point of updated rules
if self.policy_decision_point_manager is not None:
log.debug("Loading policy for resource: %s" % resource_name)
self.policy_decision_point_manager.load_policy_rules(
resource_name, policy_rules)
def update_all_resource_policies(self, org_id):
#Notify policy decision point of updated rules for all existing service policies
if self.policy_decision_point_manager is not None:
for res_name in self.policy_decision_point_manager.policy_decision_point:
try:
policy_rules = self.policy_client.get_active_service_policy_rules(
org_id, res_name)
self.update_resource_policy(res_name, policy_rules)
except Exception, e:
log.error(e.message)
class VisualizationService(BaseVisualizationService):
def on_start(self):
# The data dictionary object holds a copy of all the viz products created by the service. The viz
# products are indexed by the viz_product_type and data_product_id (which could be google_datatables or
# mpl_graphs
self.viz_data_dictionary = {}
self.viz_data_dictionary['google_dt'] = {}
self.viz_data_dictionary['google_realtime_dt'] = {}
self.viz_data_dictionary['matplotlib_graphs'] = {}
# Kind of redundant but we will maintain a separate list of data product_ids registered with the viz_service
self.data_products = []
# Create clients to interface with PubSub, Transform Management Service and Resource Registry
self.pubsub_cli = self.clients.pubsub_management
self.tms_cli = self.clients.transform_management
self.rr_cli = self.clients.resource_registry
self.dr_cli = self.clients.data_retriever
self.dsm_cli = self.clients.dataset_management
"""
# Create process definitions which will used to spawn off the transform processes
self.matplotlib_proc_def = IonObject(RT.ProcessDefinition, name='viz_transform_process'+'.'+self.random_id_generator())
self.matplotlib_proc_def.executable = {
'module': 'ion.services.ans.visualization_service',
'class':'VizTransformProcForMatplotlibGraphs'
}
self.matplotlib_proc_def_id, _ = self.rr_cli.create(self.matplotlib_proc_def)
self.google_dt_proc_def = IonObject(RT.ProcessDefinition, name='viz_transform_process'+'.'+self.random_id_generator())
self.google_dt_proc_def.executable = {
'module': 'ion.services.ans.visualization_service',
'class':'VizTransformProcForGoogleDT'
}
self.google_dt_proc_def_id, _ = self.rr_cli.create(self.google_dt_proc_def)
"""
# Query resource registry to get process definitions and streams ids made by the bootstrap
proc_def_ids,_ = self.rr_cli.find_resources(restype=RT.ProcessDefinition, lcstate=None, name="viz_matplotlib_transform_process", id_only=True)
self.matplotlib_proc_def_id = proc_def_ids[0]
proc_def_ids,_ = self.rr_cli.find_resources(restype=RT.ProcessDefinition, lcstate=None, name="viz_google_dt_transform_process", id_only=True)
self.google_dt_proc_def_id = proc_def_ids[0]
# Create a stream that all the transform processes will use to submit data back to the viz service
self.viz_service_submit_stream_id = self.pubsub_cli.create_stream(name="visualization_service_submit_stream." + self.random_id_generator())
# subscribe to this stream since all the results from transforms will be submitted here
query = StreamQuery(stream_ids=[self.viz_service_submit_stream_id,])
self.viz_service_submit_stream_sub_id = self.pubsub_cli.create_subscription(query=query, exchange_name="visualization_service_submit_queue")
submit_stream_subscriber_registrar = StreamSubscriberRegistrar(process = self.container, node = self.container.node )
submit_stream_subscriber = submit_stream_subscriber_registrar.create_subscriber(exchange_name='visualization_service_submit_queue', callback=self.process_submission)
submit_stream_subscriber.start()
self.pubsub_cli.activate_subscription(self.viz_service_submit_stream_sub_id)
# Discover the existing data_product_ids active in the system
sys_prod_ids, _ = self.rr_cli.find_resources(RT.DataProduct, None, None, True)
# Register all the streams in the system, which will in turn start transform processes
for dp_id in sys_prod_ids:
self.register_new_data_product(dp_id)
# listen for events when new data_products show up
self.event_subscriber = EventSubscriber(
event_type = "ResourceModifiedEvent",
origin_type = "DataProduct",
sub_type="UPDATE",
callback=self.receive_new_dataproduct_event
)
self.event_subscriber.activate()
return
def on_stop(self):
self.event_subscriber.deactivate()
super(VisualizationService, self).on_stop()
return
def process_submission(self, packet, headers):
# The packet is a dictionary containing the data_product_id and viz_product_type.
if(packet["viz_product_type"] == "google_realtime_dt"):
self.submit_google_realtime_dt(data_product_id=packet["data_product_id"], data_table=packet["data_table"])
if(packet["viz_product_type"] == "google_dt"):
self.submit_google_dt(data_product_id_token=packet["data_product_id_token"], data_table=packet["data_table"])
if(packet["viz_product_type"] == "matplotlib_graphs"):
self.submit_mpl_image(data_product_id=packet["data_product_id"], image_obj=packet["image_obj"],
image_name=packet["image_name"])
return
def start_google_dt_transform(self, data_product_id='', query=''):
"""Request to fetch the datatable for a data product as specified in the query. Query will also specify whether its a realtime view or one-shot
@param data_product_id str
@param query str
@retval datatable str
@throws NotFound object with specified id, query does not exist
"""
# generate a token unique for this request
data_product_id_token = data_product_id + "." + self.random_id_generator()
# Get object asssociated with data_product_id
dp_obj = self.rr_cli.read(data_product_id)
if dp_obj.dataset_id == '':
return None
# define replay. If no filters are passed the entire ingested dataset is returned
replay_id, replay_stream_id = self.dr_cli.define_replay(dataset_id=dp_obj.dataset_id)
replay_stream_def_id = self.pubsub_cli.find_stream_definition(stream_id=replay_stream_id, id_only=True)
# setup the transform to handle the data coming back from the replay
# Init storage for the resulting data_table
self.viz_data_dictionary['google_dt'][data_product_id_token] = {'data_table': [], 'ready_flag': False}
# Create the subscription to the stream. This will be passed as parameter to the transform worker
query = StreamQuery(stream_ids=[replay_stream_id,])
replay_subscription_id = self.pubsub_cli.create_subscription(query=query, exchange_name='viz_data_exchange.'+self.random_id_generator())
# maybe this is a good place to pass the couch DB table to use and other parameters
configuration = {"stream_def_id": replay_stream_def_id, "data_product_id": data_product_id, "realtime_flag": "False", "data_product_id_token": data_product_id_token}
# Launch the viz transform process
viz_transform_id = self.tms_cli.create_transform( name='viz_transform_google_dt_' + self.random_id_generator()+'.'+data_product_id,
in_subscription_id=replay_subscription_id,
out_streams = {"visualization_service_submit_stream_id": self.viz_service_submit_stream_id },
process_definition_id=self.google_dt_proc_def_id,
configuration=configuration)
self.tms_cli.activate_transform(viz_transform_id)
# keep a record of the the viz_transform_id
self.viz_data_dictionary['google_dt'][data_product_id_token]['transform_proc'] = viz_transform_id
# Start the replay and return the token
self.dr_cli.start_replay(replay_id=replay_id)
return "google_dt_transform_cb(\"" + data_product_id_token + "\")"
def is_google_dt_ready(self, data_product_id_token=''):
try:
# check if token is valid
if data_product_id_token in self.viz_data_dictionary['google_dt']:
#check if the data table is ready
if self.viz_data_dictionary['google_dt'][data_product_id_token]['ready_flag']:
return "google_dt_status_cb(\"True\")"
else:
return "google_dt_status_cb(\"False\")"
except AttributeError:
return None
def get_google_dt(self, data_product_id_token=''):
try:
# check if token is valid
if data_product_id_token in self.viz_data_dictionary['google_dt']:
#check if the data table is ready
if not self.viz_data_dictionary['google_dt'][data_product_id_token]['ready_flag']:
return None
else:
# Make a reference to the data_table and clean space in global dict
data_table = self.viz_data_dictionary['google_dt'][data_product_id_token]['data_table']
# clean up the transform and space in global dict
self.tms_cli.deactivate_transform(self.viz_data_dictionary['google_dt'][data_product_id_token]['transform_proc'])
del self.viz_data_dictionary['google_dt'][data_product_id_token]
# returning the reference to the data_table should mark the objects used by this tranform as ready
# for deletion
return data_table
else:
return None
except AttributeError:
return None
def get_google_realtime_dt(self, data_product_id='', query=''):
"""Request to fetch the datatable for a data product as specified in the query. Query will also specify whether its a realtime view or one-shot
@param data_product_id str
@param query str
@retval datatable str
@throws NotFound object with specified id, query does not exist
"""
try:
if data_product_id in self.viz_data_dictionary['google_realtime_dt']:
# assign data_table to a temp var before returning it. This will ensure a complete object is returned
# in case the data_table is being updated by a transform process
data_table = self.viz_data_dictionary['google_realtime_dt'][data_product_id]['data_table']
return data_table
else:
return None
except AttributeError:
return None
def get_list_of_mpl_images(self, data_product_id=''):
"""Request to fetch the list of Matplotlib generated images associated with a data product
@param data_product_id str
@retval image_list str
@throws NotFound object with specified id does not exist
"""
# return a json version of the array stored in the data_dict
try:
if data_product_id in self.viz_data_dictionary['matplotlib_graphs']:
# assign data_table to a temp var before returning it. This will ensure a complete object is returned
img_list = self.viz_data_dictionary['matplotlib_graphs'][data_product_id]['list_of_images']
json_img_list = simplejson.dumps({'data': img_list})
return "image_list_callback("+json_img_list+")"
else:
return None
except AttributeError:
return None
def get_image(self, data_product_id = '', image_name=''):
"""Request to fetch a file object from within the Visualization Service
@param file_name str
@retval file_obj str
@throws NotFound object with specified id does not exist
"""
try:
if data_product_id in self.viz_data_dictionary['matplotlib_graphs']:
if image_name in self.viz_data_dictionary['matplotlib_graphs'][data_product_id]:
img = self.viz_data_dictionary['matplotlib_graphs'][data_product_id][image_name]
return img
else:
return None
else:
return None
except AttributeError:
return None
def submit_google_dt(self, data_product_id_token='', data_table=''):
"""Send the rendered image to
@param data_product_id str
@param data_table str
@param description str
@throws BadRequest check data_product_id
"""
# Just copy the datatable in to the data dictionary
self.viz_data_dictionary['google_dt'][data_product_id_token]['data_table'] = data_table
self.viz_data_dictionary['google_dt'][data_product_id_token]['ready_flag'] = True
#self.result.set(True)
return
def submit_google_realtime_dt(self, data_product_id='', data_table=''):
"""Send the rendered image to
@param data_product_id str
@param data_table str
@param description str
@throws BadRequest check data_product_id
"""
# Just copy the datatable in to the data dictionary
self.viz_data_dictionary['google_realtime_dt'][data_product_id]['data_table'] = data_table
return
def submit_mpl_image(self, data_product_id='', image_obj='', image_name=''):
"""Send the rendered image to the visualization service
@param data_product_id str
@param image_obj str
@param description str
@throws BadRequest check data_product_id
"""
# Store the image object in the data dictionary and note the image in the list.
# Check for existing entries before updating the list_of_images
found = False
for name in self.viz_data_dictionary['matplotlib_graphs'][data_product_id]['list_of_images']:
if name == image_name:
found = True
break
if not found:
list_len = len(self.viz_data_dictionary['matplotlib_graphs'][data_product_id]['list_of_images'])
self.viz_data_dictionary['matplotlib_graphs'][data_product_id]['list_of_images'].append(image_name)
# Add binary data from the image to the dictionary
self.viz_data_dictionary['matplotlib_graphs'][data_product_id][image_name] = image_obj
return
def register_new_data_product(self, data_product_id=''):
"""Apprise the Visualization service of a new data product in the system. This function inits transform
processes for generating the matplotlib graphs of the new data product. It also creates transform processes which
generate Google data-tables for the real-time streams (sliding window) coming in from the instruments.
@param data_product_id str
@throws BadRequest check data_product_id for duplicates
"""
# Check to see if the DP has already been registered. If yes, do nothing
if (data_product_id in self.viz_data_dictionary['matplotlib_graphs']) or (data_product_id in self.viz_data_dictionary['google_realtime_dt']):
log.warn("Data Product has already been registered with Visualization service. Ignoring.")
return
# extract the stream_id associated with the data_product_id
viz_stream_id,_ = self.rr_cli.find_objects(data_product_id, PRED.hasStream, None, True)
if viz_stream_id == []:
log.warn ("Visualization_service: viz_stream_id is empty")
return
viz_stream_def_id = self.pubsub_cli.find_stream_definition(stream_id = viz_stream_id[0], id_only = True)
# Go ahead only if the data product is unique
if data_product_id in self.data_products:
raise BadRequest
self.data_products[len(self.data_products):] = data_product_id
# init the space needed to store matplotlib_graphs and realtime Google data tables
# For the matplotlib graphs, the list_of_images stores the names of the image files. The actual binary data for the
# images is also stored in the same dictionary as {img_name1: binary_data1, img_name2: binary_data2 .. etc}
self.viz_data_dictionary['matplotlib_graphs'][data_product_id] = {'transform_proc': "", 'list_of_images': []}
# The 'data_table' key points to a JSON string
self.viz_data_dictionary['google_realtime_dt'][data_product_id] = {'transform_proc': "", 'data_table': []}
###############################################################################
# Create transform process for the matplotlib graphs.
###############################################################################
# Create the subscription to the stream. This will be passed as parameter to the transform worker
#query1 = StreamQuery(stream_ids=[viz_stream_id,])
query1 = StreamQuery(viz_stream_id)
viz_subscription_id1 = self.pubsub_cli.create_subscription(query=query1, exchange_name='viz_data_exchange.'+self.random_id_generator())
# maybe this is a good place to pass the couch DB table to use and other parameters
configuration1 = {"stream_def_id": viz_stream_def_id, "data_product_id": data_product_id}
# Launch the viz transform process
viz_transform_id1 = self.tms_cli.create_transform( name='viz_transform_matplotlib_'+ self.random_id_generator() + '.'+data_product_id,
in_subscription_id=viz_subscription_id1,
out_streams = {"visualization_service_submit_stream_id": self.viz_service_submit_stream_id },
process_definition_id=self.matplotlib_proc_def_id,
configuration=configuration1)
self.tms_cli.activate_transform(viz_transform_id1)
# keep a record of the the viz_transform_id
self.viz_data_dictionary['matplotlib_graphs'][data_product_id]['transform_proc'] = viz_transform_id1
###############################################################################
# Create transform process for the Google realtime datatables
###############################################################################
# Create the subscription to the stream. This will be passed as parameter to the transform worker
#query2 = StreamQuery(stream_ids=[viz_stream_id,])
query2 = StreamQuery(viz_stream_id)
viz_subscription_id2 = self.pubsub_cli.create_subscription(query=query2, exchange_name='viz_data_exchange.'+self.random_id_generator())
# maybe this is a good place to pass the couch DB table to use and other parameters
configuration2 = {"stream_def_id": viz_stream_def_id, "data_product_id": data_product_id, "realtime_flag": "True"}
# Launch the viz transform process
viz_transform_id2 = self.tms_cli.create_transform( name='viz_transform_realtime_google_dt_' + self.random_id_generator()+'.'+data_product_id,
in_subscription_id=viz_subscription_id2,
out_streams = {"visualization_service_submit_stream_id": self.viz_service_submit_stream_id },
process_definition_id=self.google_dt_proc_def_id,
configuration=configuration2)
self.tms_cli.activate_transform(viz_transform_id2)
# keep a record of the the viz_transform_id
self.viz_data_dictionary['google_realtime_dt'][data_product_id]['transform_proc'] = viz_transform_id2
def random_id_generator(self, size=8, chars=string.ascii_uppercase + string.digits):
id = ''.join(random.choice(chars) for x in range(size))
return id
def receive_new_dataproduct_event(self, event_msg, headers):
"""
For now we will start a thread that emulates an event handler
"""
# register the new data product
self.register_new_data_product(event_msg.origin)
class GovernanceController(object):
def __init__(self, container):
log.debug('GovernanceController.__init__()')
self.container = container
self.enabled = False
self.interceptor_by_name_dict = dict()
self.interceptor_order = []
self.policy_decision_point_manager = None
self.governance_dispatcher = None
def start(self):
log.debug("GovernanceController starting ...")
config = CFG.interceptor.interceptors.governance.config
if config is None:
config['enabled'] = False
if "enabled" in config:
self.enabled = config["enabled"]
log.debug("GovernanceInterceptor enabled: %s" % str(self.enabled))
self.event_subscriber = None
if self.enabled:
self.initialize_from_config(config)
self.event_subscriber = EventSubscriber(
event_type="ResourceModifiedEvent",
origin_type="Policy",
callback=self.policy_event_callback)
self.event_subscriber.activate()
self.rr_client = ResourceRegistryServiceProcessClient(
node=self.container.node, process=self.container)
self.policy_client = PolicyManagementServiceProcessClient(
node=self.container.node, process=self.container)
def initialize_from_config(self, config):
self.governance_dispatcher = GovernanceDispatcher()
self.policy_decision_point_manager = PolicyDecisionPointManager()
if 'interceptor_order' in config:
self.interceptor_order = config['interceptor_order']
if 'governance_interceptors' in config:
gov_ints = config['governance_interceptors']
for name in gov_ints:
interceptor_def = gov_ints[name]
# Instantiate and put in by_name array
parts = interceptor_def["class"].split('.')
modpath = ".".join(parts[:-1])
classname = parts[-1]
module = __import__(modpath, fromlist=[classname])
classobj = getattr(module, classname)
classinst = classobj()
# Put in by_name_dict for possible re-use
self.interceptor_by_name_dict[name] = classinst
def stop(self):
log.debug("GovernanceController stopping ...")
if self.event_subscriber is not None:
self.event_subscriber.deactivate()
def process_incoming_message(self, invocation):
self.process_message(invocation, self.interceptor_order, 'incoming')
return self.governance_dispatcher.handle_incoming_message(invocation)
def process_outgoing_message(self, invocation):
self.process_message(invocation, reversed(self.interceptor_order),
'outgoing')
return self.governance_dispatcher.handle_outgoing_message(invocation)
def process_message(self, invocation, interceptor_list, method):
for int_name in interceptor_list:
class_inst = self.interceptor_by_name_dict[int_name]
getattr(class_inst, method)(invocation)
return invocation
def policy_event_callback(self, *args, **kwargs):
policy_event = args[0]
log.debug("Policy modified: %s" % policy_event.origin)
self.trigger_policy_update(policy_event.origin)
#This is a function which allows for a manual update of the policies as well.
def trigger_policy_update(self, policy_id):
try:
#TODO - Find a better way to work with org_id - use ION Org for now.
ion_org, _ = self.rr_client.find_resources(
restype=RT.Org, name=CFG.system.root_org)
resource_list, _ = self.rr_client.find_subjects(
"", PRED.hasPolicy, policy_id)
for res in resource_list:
#TODO - may figure out a better way to get the name of the Resource Type - or maybe this is ok
resource_type = res.__class__.__name__
#log.debug("Resource Type: %s" % resource_type)
if resource_type == 'ServiceDefinition':
policy_rules = self.policy_client.get_active_service_policy_rules(
ion_org[0]._id, res.name)
self.update_resource_policy(res.name, policy_rules)
elif resource_type == 'Org':
self.update_all_resource_policy(res._id)
else:
policy_rules = self.policy_client.get_active_resource_policy_rules(
res._id)
self.update_resource_policy(res._id, policy_rules)
except Exception, e:
log.error(e.message)
class ProcessDispatcherServiceIntTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2cei.yml')
self.pd_cli = ProcessDispatcherServiceClient(node=self.container.node)
self.process_definition = ProcessDefinition(name='test_process')
self.process_definition.executable = {
'module': 'ion.services.cei.test.test_process_dispatcher',
'class': 'TestProcess'
}
self.process_definition_id = self.pd_cli.create_process_definition(
self.process_definition)
self.event_queue = queue.Queue()
self.event_sub = None
def tearDown(self):
if self.event_sub:
self.event_sub.deactivate()
def _event_callback(self, event, *args, **kwargs):
self.event_queue.put(event)
def subscribe_events(self, origin):
self.event_sub = EventSubscriber(event_type="ProcessLifecycleEvent",
callback=self._event_callback,
origin=origin,
origin_type="DispatchedProcess")
self.event_sub.activate()
def await_state_event(self, pid, state):
event = self.event_queue.get(timeout=5)
log.debug("Got event: %s", event)
self.assertEqual(event.origin, pid)
self.assertEqual(event.state, state)
return event
def test_create_schedule_cancel(self):
process_schedule = ProcessSchedule()
pid = self.pd_cli.create_process(self.process_definition_id)
self.subscribe_events(pid)
pid2 = self.pd_cli.schedule_process(self.process_definition_id,
process_schedule,
configuration={},
process_id=pid)
self.assertEqual(pid, pid2)
self.await_state_event(pid, ProcessStateEnum.SPAWN)
# now try communicating with the process to make sure it is really running
test_client = TestClient()
for i in range(5):
# this timeout may be too low
self.assertEqual(i + 1, test_client.count(timeout=1))
# kill the process and start it again
self.pd_cli.cancel_process(pid)
self.await_state_event(pid, ProcessStateEnum.TERMINATE)
oldpid = pid
pid = self.pd_cli.create_process(self.process_definition_id)
self.subscribe_events(pid)
pid2 = self.pd_cli.schedule_process(self.process_definition_id,
process_schedule,
configuration={},
process_id=pid)
self.assertEqual(pid, pid2)
self.assertNotEqual(oldpid, pid)
self.await_state_event(pid, ProcessStateEnum.SPAWN)
for i in range(5):
# this timeout may be too low
self.assertEqual(i + 1, test_client.count(timeout=1))
# kill the process for good
self.pd_cli.cancel_process(pid)
self.await_state_event(pid, ProcessStateEnum.TERMINATE)
def test_schedule_bad_config(self):
process_schedule = ProcessSchedule()
# a non-JSON-serializable IonObject
o = ProcessTarget()
with self.assertRaises(BadRequest) as ar:
self.pd_cli.schedule_process(self.process_definition_id,
process_schedule,
configuration={"bad": o})
self.assertTrue(ar.exception.message.startswith("bad configuration"))
class VisualizationService(BaseVisualizationService):
def on_start(self):
# The data dictionary object holds a copy of all the viz products created by the service. The viz
# products are indexed by the viz_product_type and data_product_id (which could be google_datatables or
# mpl_graphs
self.viz_data_dictionary = {}
self.viz_data_dictionary['google_dt'] = {}
self.viz_data_dictionary['google_realtime_dt'] = {}
self.viz_data_dictionary['matplotlib_graphs'] = {}
# Kind of redundant but we will maintain a separate list of data product_ids registered with the viz_service
self.data_products = []
# Create clients to interface with PubSub, Transform Management Service and Resource Registry
self.pubsub_cli = self.clients.pubsub_management
self.tms_cli = self.clients.transform_management
self.rr_cli = self.clients.resource_registry
self.dr_cli = self.clients.data_retriever
self.dsm_cli = self.clients.dataset_management
"""
# Create process definitions which will used to spawn off the transform processes
self.matplotlib_proc_def = IonObject(RT.ProcessDefinition, name='viz_transform_process'+'.'+self.random_id_generator())
self.matplotlib_proc_def.executable = {
'module': 'ion.services.ans.visualization_service',
'class':'VizTransformProcForMatplotlibGraphs'
}
self.matplotlib_proc_def_id, _ = self.rr_cli.create(self.matplotlib_proc_def)
self.google_dt_proc_def = IonObject(RT.ProcessDefinition, name='viz_transform_process'+'.'+self.random_id_generator())
self.google_dt_proc_def.executable = {
'module': 'ion.services.ans.visualization_service',
'class':'VizTransformProcForGoogleDT'
}
self.google_dt_proc_def_id, _ = self.rr_cli.create(self.google_dt_proc_def)
"""
# Query resource registry to get process definitions and streams ids made by the bootstrap
proc_def_ids, _ = self.rr_cli.find_resources(
restype=RT.ProcessDefinition,
lcstate=None,
name="viz_matplotlib_transform_process",
id_only=True)
self.matplotlib_proc_def_id = proc_def_ids[0]
proc_def_ids, _ = self.rr_cli.find_resources(
restype=RT.ProcessDefinition,
lcstate=None,
name="viz_google_dt_transform_process",
id_only=True)
self.google_dt_proc_def_id = proc_def_ids[0]
# Create a stream that all the transform processes will use to submit data back to the viz service
self.viz_service_submit_stream_id = self.pubsub_cli.create_stream(
name="visualization_service_submit_stream." +
self.random_id_generator())
# subscribe to this stream since all the results from transforms will be submitted here
query = StreamQuery(stream_ids=[
self.viz_service_submit_stream_id,
])
self.viz_service_submit_stream_sub_id = self.pubsub_cli.create_subscription(
query=query, exchange_name="visualization_service_submit_queue")
submit_stream_subscriber_registrar = StreamSubscriberRegistrar(
process=self.container, node=self.container.node)
submit_stream_subscriber = submit_stream_subscriber_registrar.create_subscriber(
exchange_name='visualization_service_submit_queue',
callback=self.process_submission)
submit_stream_subscriber.start()
self.pubsub_cli.activate_subscription(
self.viz_service_submit_stream_sub_id)
# Discover the existing data_product_ids active in the system
sys_prod_ids, _ = self.rr_cli.find_resources(RT.DataProduct, None,
None, True)
# Register all the streams in the system, which will in turn start transform processes
for dp_id in sys_prod_ids:
self.register_new_data_product(dp_id)
# listen for events when new data_products show up
self.event_subscriber = EventSubscriber(
event_type="ResourceModifiedEvent",
origin_type="DataProduct",
sub_type="UPDATE",
callback=self.receive_new_dataproduct_event)
self.event_subscriber.activate()
return
def on_stop(self):
self.event_subscriber.deactivate()
super(VisualizationService, self).on_stop()
return
def process_submission(self, packet, headers):
# The packet is a dictionary containing the data_product_id and viz_product_type.
if (packet["viz_product_type"] == "google_realtime_dt"):
self.submit_google_realtime_dt(
data_product_id=packet["data_product_id"],
data_table=packet["data_table"])
if (packet["viz_product_type"] == "google_dt"):
self.submit_google_dt(
data_product_id_token=packet["data_product_id_token"],
data_table=packet["data_table"])
if (packet["viz_product_type"] == "matplotlib_graphs"):
self.submit_mpl_image(data_product_id=packet["data_product_id"],
image_obj=packet["image_obj"],
image_name=packet["image_name"])
return
def start_google_dt_transform(self, data_product_id='', query=''):
"""Request to fetch the datatable for a data product as specified in the query. Query will also specify whether its a realtime view or one-shot
@param data_product_id str
@param query str
@retval datatable str
@throws NotFound object with specified id, query does not exist
"""
# generate a token unique for this request
data_product_id_token = data_product_id + "." + self.random_id_generator(
)
# Get object asssociated with data_product_id
dp_obj = self.rr_cli.read(data_product_id)
if dp_obj.dataset_id == '':
return None
# define replay. If no filters are passed the entire ingested dataset is returned
replay_id, replay_stream_id = self.dr_cli.define_replay(
dataset_id=dp_obj.dataset_id)
replay_stream_def_id = self.pubsub_cli.find_stream_definition(
stream_id=replay_stream_id, id_only=True)
# setup the transform to handle the data coming back from the replay
# Init storage for the resulting data_table
self.viz_data_dictionary['google_dt'][data_product_id_token] = {
'data_table': [],
'ready_flag': False
}
# Create the subscription to the stream. This will be passed as parameter to the transform worker
query = StreamQuery(stream_ids=[
replay_stream_id,
])
replay_subscription_id = self.pubsub_cli.create_subscription(
query=query,
exchange_name='viz_data_exchange.' + self.random_id_generator())
# maybe this is a good place to pass the couch DB table to use and other parameters
configuration = {
"stream_def_id": replay_stream_def_id,
"data_product_id": data_product_id,
"realtime_flag": "False",
"data_product_id_token": data_product_id_token
}
# Launch the viz transform process
viz_transform_id = self.tms_cli.create_transform(
name='viz_transform_google_dt_' + self.random_id_generator() +
'.' + data_product_id,
in_subscription_id=replay_subscription_id,
out_streams={
"visualization_service_submit_stream_id":
self.viz_service_submit_stream_id
},
process_definition_id=self.google_dt_proc_def_id,
configuration=configuration)
self.tms_cli.activate_transform(viz_transform_id)
# keep a record of the the viz_transform_id
self.viz_data_dictionary['google_dt'][data_product_id_token][
'transform_proc'] = viz_transform_id
# Start the replay and return the token
self.dr_cli.start_replay(replay_id=replay_id)
return "google_dt_transform_cb(\"" + data_product_id_token + "\")"
def is_google_dt_ready(self, data_product_id_token=''):
try:
# check if token is valid
if data_product_id_token in self.viz_data_dictionary['google_dt']:
#check if the data table is ready
if self.viz_data_dictionary['google_dt'][
data_product_id_token]['ready_flag']:
return "google_dt_status_cb(\"True\")"
else:
return "google_dt_status_cb(\"False\")"
except AttributeError:
return None
def get_google_dt(self, data_product_id_token=''):
try:
# check if token is valid
if data_product_id_token in self.viz_data_dictionary['google_dt']:
#check if the data table is ready
if not self.viz_data_dictionary['google_dt'][
data_product_id_token]['ready_flag']:
return None
else:
# Make a reference to the data_table and clean space in global dict
data_table = self.viz_data_dictionary['google_dt'][
data_product_id_token]['data_table']
# clean up the transform and space in global dict
self.tms_cli.deactivate_transform(
self.viz_data_dictionary['google_dt']
[data_product_id_token]['transform_proc'])
del self.viz_data_dictionary['google_dt'][
data_product_id_token]
# returning the reference to the data_table should mark the objects used by this tranform as ready
# for deletion
return data_table
else:
return None
except AttributeError:
return None
def get_google_realtime_dt(self, data_product_id='', query=''):
"""Request to fetch the datatable for a data product as specified in the query. Query will also specify whether its a realtime view or one-shot
@param data_product_id str
@param query str
@retval datatable str
@throws NotFound object with specified id, query does not exist
"""
try:
if data_product_id in self.viz_data_dictionary[
'google_realtime_dt']:
# assign data_table to a temp var before returning it. This will ensure a complete object is returned
# in case the data_table is being updated by a transform process
data_table = self.viz_data_dictionary['google_realtime_dt'][
data_product_id]['data_table']
return data_table
else:
return None
except AttributeError:
return None
def get_list_of_mpl_images(self, data_product_id=''):
"""Request to fetch the list of Matplotlib generated images associated with a data product
@param data_product_id str
@retval image_list str
@throws NotFound object with specified id does not exist
"""
# return a json version of the array stored in the data_dict
try:
if data_product_id in self.viz_data_dictionary[
'matplotlib_graphs']:
# assign data_table to a temp var before returning it. This will ensure a complete object is returned
img_list = self.viz_data_dictionary['matplotlib_graphs'][
data_product_id]['list_of_images']
json_img_list = simplejson.dumps({'data': img_list})
return "image_list_callback(" + json_img_list + ")"
else:
return None
except AttributeError:
return None
def get_image(self, data_product_id='', image_name=''):
"""Request to fetch a file object from within the Visualization Service
@param file_name str
@retval file_obj str
@throws NotFound object with specified id does not exist
"""
try:
if data_product_id in self.viz_data_dictionary[
'matplotlib_graphs']:
if image_name in self.viz_data_dictionary['matplotlib_graphs'][
data_product_id]:
img = self.viz_data_dictionary['matplotlib_graphs'][
data_product_id][image_name]
return img
else:
return None
else:
return None
except AttributeError:
return None
def submit_google_dt(self, data_product_id_token='', data_table=''):
"""Send the rendered image to
@param data_product_id str
@param data_table str
@param description str
@throws BadRequest check data_product_id
"""
# Just copy the datatable in to the data dictionary
self.viz_data_dictionary['google_dt'][data_product_id_token][
'data_table'] = data_table
self.viz_data_dictionary['google_dt'][data_product_id_token][
'ready_flag'] = True
#self.result.set(True)
return
def submit_google_realtime_dt(self, data_product_id='', data_table=''):
"""Send the rendered image to
@param data_product_id str
@param data_table str
@param description str
@throws BadRequest check data_product_id
"""
# Just copy the datatable in to the data dictionary
self.viz_data_dictionary['google_realtime_dt'][data_product_id][
'data_table'] = data_table
return
def submit_mpl_image(self,
data_product_id='',
image_obj='',
image_name=''):
"""Send the rendered image to the visualization service
@param data_product_id str
@param image_obj str
@param description str
@throws BadRequest check data_product_id
"""
# Store the image object in the data dictionary and note the image in the list.
# Check for existing entries before updating the list_of_images
found = False
for name in self.viz_data_dictionary['matplotlib_graphs'][
data_product_id]['list_of_images']:
if name == image_name:
found = True
break
if not found:
list_len = len(self.viz_data_dictionary['matplotlib_graphs']
[data_product_id]['list_of_images'])
self.viz_data_dictionary['matplotlib_graphs'][data_product_id][
'list_of_images'].append(image_name)
# Add binary data from the image to the dictionary
self.viz_data_dictionary['matplotlib_graphs'][data_product_id][
image_name] = image_obj
return
def register_new_data_product(self, data_product_id=''):
"""Apprise the Visualization service of a new data product in the system. This function inits transform
processes for generating the matplotlib graphs of the new data product. It also creates transform processes which
generate Google data-tables for the real-time streams (sliding window) coming in from the instruments.
@param data_product_id str
@throws BadRequest check data_product_id for duplicates
"""
# Check to see if the DP has already been registered. If yes, do nothing
if (data_product_id in self.viz_data_dictionary['matplotlib_graphs']
) or (data_product_id
in self.viz_data_dictionary['google_realtime_dt']):
log.warn(
"Data Product has already been registered with Visualization service. Ignoring."
)
return
# extract the stream_id associated with the data_product_id
viz_stream_id, _ = self.rr_cli.find_objects(data_product_id,
PRED.hasStream, None, True)
if viz_stream_id == []:
log.warn("Visualization_service: viz_stream_id is empty")
return
viz_stream_def_id = self.pubsub_cli.find_stream_definition(
stream_id=viz_stream_id[0], id_only=True)
# Go ahead only if the data product is unique
if data_product_id in self.data_products:
raise BadRequest
self.data_products[len(self.data_products):] = data_product_id
# init the space needed to store matplotlib_graphs and realtime Google data tables
# For the matplotlib graphs, the list_of_images stores the names of the image files. The actual binary data for the
# images is also stored in the same dictionary as {img_name1: binary_data1, img_name2: binary_data2 .. etc}
self.viz_data_dictionary['matplotlib_graphs'][data_product_id] = {
'transform_proc': "",
'list_of_images': []
}
# The 'data_table' key points to a JSON string
self.viz_data_dictionary['google_realtime_dt'][data_product_id] = {
'transform_proc': "",
'data_table': []
}
###############################################################################
# Create transform process for the matplotlib graphs.
###############################################################################
# Create the subscription to the stream. This will be passed as parameter to the transform worker
#query1 = StreamQuery(stream_ids=[viz_stream_id,])
query1 = StreamQuery(viz_stream_id)
viz_subscription_id1 = self.pubsub_cli.create_subscription(
query=query1,
exchange_name='viz_data_exchange.' + self.random_id_generator())
# maybe this is a good place to pass the couch DB table to use and other parameters
configuration1 = {
"stream_def_id": viz_stream_def_id,
"data_product_id": data_product_id
}
# Launch the viz transform process
viz_transform_id1 = self.tms_cli.create_transform(
name='viz_transform_matplotlib_' + self.random_id_generator() +
'.' + data_product_id,
in_subscription_id=viz_subscription_id1,
out_streams={
"visualization_service_submit_stream_id":
self.viz_service_submit_stream_id
},
process_definition_id=self.matplotlib_proc_def_id,
configuration=configuration1)
self.tms_cli.activate_transform(viz_transform_id1)
# keep a record of the the viz_transform_id
self.viz_data_dictionary['matplotlib_graphs'][data_product_id][
'transform_proc'] = viz_transform_id1
###############################################################################
# Create transform process for the Google realtime datatables
###############################################################################
# Create the subscription to the stream. This will be passed as parameter to the transform worker
#query2 = StreamQuery(stream_ids=[viz_stream_id,])
query2 = StreamQuery(viz_stream_id)
viz_subscription_id2 = self.pubsub_cli.create_subscription(
query=query2,
exchange_name='viz_data_exchange.' + self.random_id_generator())
# maybe this is a good place to pass the couch DB table to use and other parameters
configuration2 = {
"stream_def_id": viz_stream_def_id,
"data_product_id": data_product_id,
"realtime_flag": "True"
}
# Launch the viz transform process
viz_transform_id2 = self.tms_cli.create_transform(
name='viz_transform_realtime_google_dt_' +
self.random_id_generator() + '.' + data_product_id,
in_subscription_id=viz_subscription_id2,
out_streams={
"visualization_service_submit_stream_id":
self.viz_service_submit_stream_id
},
process_definition_id=self.google_dt_proc_def_id,
configuration=configuration2)
self.tms_cli.activate_transform(viz_transform_id2)
# keep a record of the the viz_transform_id
self.viz_data_dictionary['google_realtime_dt'][data_product_id][
'transform_proc'] = viz_transform_id2
def random_id_generator(self,
size=8,
chars=string.ascii_uppercase + string.digits):
id = ''.join(random.choice(chars) for x in range(size))
return id
def receive_new_dataproduct_event(self, event_msg, headers):
"""
For now we will start a thread that emulates an event handler
"""
# register the new data product
self.register_new_data_product(event_msg.origin)
class ServiceGatewayService(BaseServiceGatewayService):
"""
The Service Gateway Service is the service that uses a gevent web server and Flask
to bridge HTTP requests to AMQP RPC ION process service calls.
"""
def on_init(self):
# defaults
self.http_server = None
# retain a pointer to this object for use in ProcessRPC calls
global service_gateway_instance
service_gateway_instance = self
self.server_hostname = self.CFG.get_safe(
"container.service_gateway.web_server.hostname", DEFAULT_WEB_SERVER_HOSTNAME
)
self.server_port = self.CFG.get_safe("container.service_gateway.web_server.port", DEFAULT_WEB_SERVER_PORT)
self.web_server_enabled = self.CFG.get_safe("container.service_gateway.web_server.enabled", True)
self.logging = self.CFG.get_safe("container.service_gateway.web_server.log")
# Optional list of trusted originators can be specified in config.
self.trusted_originators = self.CFG.get_safe("container.service_gateway.trusted_originators")
if not self.trusted_originators:
self.trusted_originators = None
log.info("Service Gateway will not check requests against trusted originators since none are configured.")
# Get the user_cache_size
self.user_cache_size = self.CFG.get_safe("container.service_gateway.user_cache_size", DEFAULT_USER_CACHE_SIZE)
# Start the gevent web server unless disabled
if self.web_server_enabled:
log.info("Starting service gateway on %s:%s", self.server_hostname, self.server_port)
self.start_service(self.server_hostname, self.server_port)
self.user_role_event_subscriber = EventSubscriber(
event_type="UserRoleModifiedEvent", origin_type="Org", callback=self.user_role_event_callback
)
self.user_role_event_subscriber.activate()
# Initialize an LRU Cache to keep user roles cached for performance reasons
# maxSize = maximum number of elements to keep in cache
# maxAgeMs = oldest entry to keep
self.user_data_cache = LRUCache(self.user_cache_size, 0, 0)
def on_quit(self):
self.stop_service()
if self.user_role_event_subscriber is not None:
self.user_role_event_subscriber.deactivate()
def start_service(self, hostname=DEFAULT_WEB_SERVER_HOSTNAME, port=DEFAULT_WEB_SERVER_PORT):
"""Responsible for starting the gevent based web server."""
if self.http_server is not None:
self.stop_service()
self.http_server = WSGIServer((hostname, port), app, log=self.logging)
self.http_server.start()
return True
def stop_service(self):
"""Responsible for stopping the gevent based web server."""
if self.http_server is not None:
self.http_server.stop()
return True
def is_trusted_address(self, requesting_address):
if self.trusted_originators is None:
return True
for addr in self.trusted_originators:
if requesting_address == addr:
return True
return False
def user_role_event_callback(self, *args, **kwargs):
"""
This method is a callback function for receiving User Role Modified Events.
"""
user_role_event = args[0]
org_id = user_role_event.origin
user_id = user_role_event.user_id
role_name = user_role_event.role_name
log.debug("User Role modified: %s %s %s" % (org_id, user_id, role_name))
# Evict the user and their roles from the cache so that it gets updated with the next call.
if service_gateway_instance.user_data_cache.has_key(user_id):
log.debug("Evicting user from the user_data_cache: %s" % user_id)
service_gateway_instance.user_data_cache.evict(user_id)
def test_pub_on_different_subsubtypes(self):
res_list = [
DotDict(ar=event.AsyncResult(), gq=queue.Queue(), count=0)
for i in xrange(4)
]
def cb_gen(num):
def cb(event, *args, **kwargs):
res_list[num].count += 1
res_list[num].gq.put(event)
if event.description == "end":
res_list[num].ar.set()
return cb
sub0 = EventSubscriber(event_type="ResourceModifiedEvent",
sub_type="st1.*",
callback=cb_gen(0))
sub0.activate()
sub1 = EventSubscriber(event_type="ResourceModifiedEvent",
sub_type="st1.a",
callback=cb_gen(1))
sub1.activate()
sub2 = EventSubscriber(event_type="ResourceModifiedEvent",
sub_type="*.a",
callback=cb_gen(2))
sub2.activate()
sub3 = EventSubscriber(event_type="ResourceModifiedEvent",
sub_type="st1",
callback=cb_gen(3))
sub3.activate()
pub1 = EventPublisher(event_type="ResourceModifiedEvent")
pub1.publish_event(origin="one", sub_type="st1.a", description="1")
pub1.publish_event(origin="two", sub_type="st1", description="2")
pub1.publish_event(origin="three", sub_type="st1.b", description="3")
pub1.publish_event(origin="four", sub_type="st2.a", description="4")
pub1.publish_event(origin="five", sub_type="st2", description="5")
pub1.publish_event(origin="six", sub_type="a", description="6")
pub1.publish_event(origin="seven", sub_type="", description="7")
pub1.publish_event(origin="end", sub_type="st1.a", description="end")
pub1.publish_event(origin="end", sub_type="st1", description="end")
[res_list[i].ar.get(timeout=5) for i in xrange(3)]
sub0.deactivate()
sub1.deactivate()
sub2.deactivate()
sub3.deactivate()
for i in xrange(4):
res_list[i].res = []
for x in xrange(res_list[i].count):
res_list[i].res.append(res_list[i].gq.get(timeout=5))
self.assertEquals(len(res_list[0].res), 3)
self.assertEquals(res_list[0].res[0].description, "1")
self.assertEquals(len(res_list[1].res), 2)
self.assertEquals(res_list[1].res[0].description, "1")
self.assertEquals(len(res_list[2].res), 3)
self.assertEquals(res_list[2].res[0].description, "1")
self.assertEquals(len(res_list[3].res), 2)
self.assertEquals(res_list[3].res[0].description, "2")
class GovernanceController(object):
def __init__(self,container):
log.debug('GovernanceController.__init__()')
self.container = container
self.enabled = False
self.interceptor_by_name_dict = dict()
self.interceptor_order = []
self.policy_decision_point_manager = None
self.governance_dispatcher = None
def start(self):
log.debug("GovernanceController starting ...")
config = CFG.interceptor.interceptors.governance.config
if config is None:
config['enabled'] = False
if "enabled" in config:
self.enabled = config["enabled"]
log.debug("GovernanceInterceptor enabled: %s" % str(self.enabled))
self.resource_policy_event_subscriber = None
if self.enabled:
self.initialize_from_config(config)
self.resource_policy_event_subscriber = EventSubscriber(event_type="ResourcePolicyEvent", callback=self.policy_event_callback)
self.resource_policy_event_subscriber.activate()
self.rr_client = ResourceRegistryServiceProcessClient(node=self.container.node, process=self.container)
self.policy_client = PolicyManagementServiceProcessClient(node=self.container.node, process=self.container)
self.org_client = OrgManagementServiceProcessClient(node=self.container.node, process=self.container)
def initialize_from_config(self, config):
self.governance_dispatcher = GovernanceDispatcher()
self.policy_decision_point_manager = PolicyDecisionPointManager()
if 'interceptor_order' in config:
self.interceptor_order = config['interceptor_order']
if 'governance_interceptors' in config:
gov_ints = config['governance_interceptors']
for name in gov_ints:
interceptor_def = gov_ints[name]
# Instantiate and put in by_name array
parts = interceptor_def["class"].split('.')
modpath = ".".join(parts[:-1])
classname = parts[-1]
module = __import__(modpath, fromlist=[classname])
classobj = getattr(module, classname)
classinst = classobj()
# Put in by_name_dict for possible re-use
self.interceptor_by_name_dict[name] = classinst
def stop(self):
log.debug("GovernanceController stopping ...")
if self.resource_policy_event_subscriber is not None:
self.resource_policy_event_subscriber.deactivate()
def process_incoming_message(self,invocation):
self.process_message(invocation, self.interceptor_order,'incoming' )
return self.governance_dispatcher.handle_incoming_message(invocation)
def process_outgoing_message(self,invocation):
self.process_message(invocation, reversed(self.interceptor_order),'outgoing')
return self.governance_dispatcher.handle_outgoing_message(invocation)
def process_message(self,invocation,interceptor_list, method):
for int_name in interceptor_list:
class_inst = self.interceptor_by_name_dict[int_name]
getattr(class_inst, method)(invocation)
return invocation
def policy_event_callback(self, *args, **kwargs):
resource_policy_event = args[0]
policy_id = resource_policy_event.origin
resource_id = resource_policy_event.resource_id
resource_type = resource_policy_event.resource_type
resource_name = resource_policy_event.resource_name
log.info("Resource policy modified: %s %s %s" % ( policy_id, resource_id, resource_type))
if resource_type == 'ServiceDefinition': #TODO - REDO to have a configurable Org boundary by container
ion_org = self.org_client.find_org()
policy_rules = self.policy_client.get_active_service_policy_rules(ion_org._id, resource_name)
self.update_resource_policy(resource_name, policy_rules)
elif resource_type == 'Org':
policy_rules = self.policy_client.get_active_resource_policy_rules(resource_id)
if self.policy_decision_point_manager is not None:
self.policy_decision_point_manager.load_org_policy_rules(policy_rules)
self.update_all_resource_policies(resource_id)
else:
policy_rules = self.policy_client.get_active_resource_policy_rules(resource_id)
self.update_resource_policy(resource_id, policy_rules)
def update_resource_policy(self, resource_name, policy_rules):
#Notify policy decision point of updated rules
if self.policy_decision_point_manager is not None:
log.debug("Loading policy for resource: %s" % resource_name)
self.policy_decision_point_manager.load_policy_rules(resource_name, policy_rules)
def update_all_resource_policies(self, org_id):
#Notify policy decision point of updated rules for all existing service policies
if self.policy_decision_point_manager is not None:
for res_name in self.policy_decision_point_manager.policy_decision_point:
try:
policy_rules = self.policy_client.get_active_service_policy_rules(org_id, res_name)
self.update_resource_policy(res_name, policy_rules)
except Exception, e:
log.error(e.message)
class ProcessDispatcherServiceIntTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2cei.yml')
self.pd_cli = ProcessDispatcherServiceClient(node=self.container.node)
self.process_definition = ProcessDefinition(name='test_process')
self.process_definition.executable = {'module': 'ion.services.cei.test.test_process_dispatcher',
'class':'TestProcess'}
self.process_definition_id = self.pd_cli.create_process_definition(self.process_definition)
self.event_queue = queue.Queue()
self.event_sub = None
def tearDown(self):
if self.event_sub:
self.event_sub.deactivate()
def _event_callback(self, event, *args, **kwargs):
self.event_queue.put(event)
def subscribe_events(self, origin):
self.event_sub = EventSubscriber(event_type="ProcessLifecycleEvent",
callback=self._event_callback, origin=origin, origin_type="DispatchedProcess")
self.event_sub.activate()
def await_state_event(self, pid, state):
event = self.event_queue.get(timeout=5)
log.debug("Got event: %s", event)
self.assertEqual(event.origin, pid)
self.assertEqual(event.state, state)
return event
def test_create_schedule_cancel(self):
process_schedule = ProcessSchedule()
pid = self.pd_cli.create_process(self.process_definition_id)
self.subscribe_events(pid)
pid2 = self.pd_cli.schedule_process(self.process_definition_id,
process_schedule, configuration={}, process_id=pid)
self.assertEqual(pid, pid2)
self.await_state_event(pid, ProcessStateEnum.SPAWN)
# now try communicating with the process to make sure it is really running
test_client = TestClient()
for i in range(5):
# this timeout may be too low
self.assertEqual(i+1, test_client.count(timeout=1))
# kill the process and start it again
self.pd_cli.cancel_process(pid)
self.await_state_event(pid, ProcessStateEnum.TERMINATE)
oldpid = pid
pid = self.pd_cli.create_process(self.process_definition_id)
self.subscribe_events(pid)
pid2 = self.pd_cli.schedule_process(self.process_definition_id,
process_schedule, configuration={}, process_id=pid)
self.assertEqual(pid, pid2)
self.assertNotEqual(oldpid, pid)
self.await_state_event(pid, ProcessStateEnum.SPAWN)
for i in range(5):
# this timeout may be too low
self.assertEqual(i+1, test_client.count(timeout=1))
# kill the process for good
self.pd_cli.cancel_process(pid)
self.await_state_event(pid, ProcessStateEnum.TERMINATE)
def test_schedule_bad_config(self):
process_schedule = ProcessSchedule()
# a non-JSON-serializable IonObject
o = ProcessTarget()
with self.assertRaises(BadRequest) as ar:
self.pd_cli.schedule_process(self.process_definition_id,
process_schedule, configuration={"bad" : o})
self.assertTrue(ar.exception.message.startswith("bad configuration"))
class ExternalDatasetAgentTestBase(object):
# Agent parameters.
EDA_RESOURCE_ID = '123xyz'
EDA_NAME = 'ExampleEDA'
EDA_MOD = 'ion.agents.data.external_dataset_agent'
EDA_CLS = 'ExternalDatasetAgent'
"""
Test cases for instrument agent class. Functions in this class provide
instrument agent integration tests and provide a tutorial on use of
the agent setup and interface.
"""
def setUp(self):
"""
Initialize test members.
"""
# log.warn('Starting the container')
# Start container.
self._start_container()
# Bring up services in a deploy file
# log.warn('Starting the rel')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Create a pubsub client to create streams.
# log.warn('Init a pubsub client')
self._pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# log.warn('Init a ContainerAgentClient')
self._container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
# Data async and subscription TODO: Replace with new subscriber
self._finished_count = None
#TODO: Switch to gevent.queue.Queue
self._async_finished_result = AsyncResult()
self._finished_events_received = []
self._finished_event_subscriber = None
self._start_finished_event_subscriber()
self.addCleanup(self._stop_finished_event_subscriber)
# TODO: Finish dealing with the resources and whatnot
# TODO: DVR_CONFIG and (potentially) stream_config could both be reconfigured in self._setup_resources()
self._setup_resources()
#TG: Setup/configure the granule logger to log granules as they're published
# Create agent config.
agent_config = {
'driver_config' : self.DVR_CONFIG,
'stream_config' : {},
'agent' : {'resource_id': self.EDA_RESOURCE_ID},
'test_mode' : True
}
# Start instrument agent.
self._ia_pid = None
log.debug('TestInstrumentAgent.setup(): starting EDA.')
self._ia_pid = self._container_client.spawn_process(
name=self.EDA_NAME,
module=self.EDA_MOD,
cls=self.EDA_CLS,
config=agent_config
)
log.info('Agent pid=%s.', str(self._ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = None
self._ia_client = ResourceAgentClient(self.EDA_RESOURCE_ID, process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
########################################
# Private "setup" functions
########################################
def _setup_resources(self):
raise NotImplementedError('_setup_resources must be implemented in the subclass')
def create_stream_and_logger(self, name, stream_id=''):
if not stream_id or stream_id is '':
stream_id = self._pubsub_client.create_stream(name=name, encoding='ION R2')
pid = self._container_client.spawn_process(
name=name+'_logger',
module='ion.processes.data.stream_granule_logger',
cls='StreamGranuleLogger',
config={'process':{'stream_id':stream_id}}
)
log.info('Started StreamGranuleLogger \'{0}\' subscribed to stream_id={1}'.format(pid, stream_id))
return stream_id
def _start_finished_event_subscriber(self):
def consume_event(*args,**kwargs):
if args[0].description == 'TestingFinished':
log.debug('TestingFinished event received')
self._finished_events_received.append(args[0])
if self._finished_count and self._finished_count == len(self._finished_events_received):
log.debug('Finishing test...')
self._async_finished_result.set(len(self._finished_events_received))
log.debug('Called self._async_finished_result.set({0})'.format(len(self._finished_events_received)))
self._finished_event_subscriber = EventSubscriber(event_type='DeviceEvent', callback=consume_event)
self._finished_event_subscriber.activate()
def _stop_finished_event_subscriber(self):
if self._finished_event_subscriber:
self._finished_event_subscriber.deactivate()
self._finished_event_subscriber = None
########################################
# Custom assertion functions
########################################
def assertListsEqual(self, lst1, lst2):
lst1.sort()
lst2.sort()
return lst1 == lst2
def assertSampleDict(self, val):
"""
Verify the value is a sample dictionary for the sbe37.
"""
#{'p': [-6.945], 'c': [0.08707], 't': [20.002], 'time': [1333752198.450622]}
self.assertTrue(isinstance(val, dict))
self.assertTrue(val.has_key('c'))
self.assertTrue(val.has_key('t'))
self.assertTrue(val.has_key('p'))
self.assertTrue(val.has_key('time'))
c = val['c'][0]
t = val['t'][0]
p = val['p'][0]
time = val['time'][0]
self.assertTrue(isinstance(c, float))
self.assertTrue(isinstance(t, float))
self.assertTrue(isinstance(p, float))
self.assertTrue(isinstance(time, float))
def assertParamDict(self, pd, all_params=False):
"""
Verify all device parameters exist and are correct type.
"""
if all_params:
self.assertEqual(set(pd.keys()), set(PARAMS.keys()))
for (key, type_val) in PARAMS.iteritems():
if type_val == list or type_val == tuple:
self.assertTrue(isinstance(pd[key], (list, tuple)))
else:
self.assertTrue(isinstance(pd[key], type_val))
else:
for (key, val) in pd.iteritems():
self.assertTrue(PARAMS.has_key(key))
self.assertTrue(isinstance(val, PARAMS[key]))
def assertParamVals(self, params, correct_params):
"""
Verify parameters take the correct values.
"""
self.assertEqual(set(params.keys()), set(correct_params.keys()))
for (key, val) in params.iteritems():
correct_val = correct_params[key]
if isinstance(val, float):
# Verify to 5% of the larger value.
max_val = max(abs(val), abs(correct_val))
self.assertAlmostEqual(val, correct_val, delta=max_val*.01)
elif isinstance(val, (list, tuple)):
# list of tuple.
self.assertEqual(list(val), list(correct_val))
else:
# int, bool, str.
self.assertEqual(val, correct_val)
########################################
# Test functions
########################################
def test_acquire_data(self):
cmd=AgentCommand(command='initialize')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='go_active')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='run')
_ = self._ia_client.execute_agent(cmd)
self._finished_count = 3
log.info('Send an unconstrained request for data (\'new data\')')
cmd = AgentCommand(command='acquire_data')
self._ia_client.execute(cmd)
log.info('Send a second unconstrained request for data (\'new data\'), should be rejected')
cmd = AgentCommand(command='acquire_data')
self._ia_client.execute(cmd)
config_mods={}
log.info('Send a constrained request for data: constraints = HIST_CONSTRAINTS_1')
config_mods['stream_id'] = self.create_stream_and_logger(name='stream_id_for_historical_1')
config_mods['constraints']=self.HIST_CONSTRAINTS_1
cmd = AgentCommand(command='acquire_data', args=[config_mods])
self._ia_client.execute(cmd)
log.info('Send a second constrained request for data: constraints = HIST_CONSTRAINTS_2')
config_mods['stream_id'] = self.create_stream_and_logger(name='stream_id_for_historical_2')
config_mods['constraints']=self.HIST_CONSTRAINTS_2
# config={'stream_id':'second_historical','TESTING':True, 'constraints':self.HIST_CONSTRAINTS_2}
cmd = AgentCommand(command='acquire_data', args=[config_mods])
self._ia_client.execute(cmd)
finished = self._async_finished_result.get(timeout=10)
self.assertEqual(finished,self._finished_count)
cmd = AgentCommand(command='reset')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_acquire_data_while_streaming(self):
# Test instrument driver execute interface to start and stop streaming mode.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Make sure the polling interval is appropriate for a test
params = {
'POLLING_INTERVAL':5
}
self._ia_client.set_param(params)
self._finished_count = 2
# Begin streaming.
cmd = AgentCommand(command='go_streaming')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STREAMING)
config = get_safe(self.DVR_CONFIG, 'dh_cfg', {})
log.info('Send a constrained request for data: constraints = HIST_CONSTRAINTS_1')
config['stream_id'] = self.create_stream_and_logger(name='stream_id_for_historical_1')
config['constraints']=self.HIST_CONSTRAINTS_1
cmd = AgentCommand(command='acquire_data', args=[config])
reply = self._ia_client.execute(cmd)
self.assertNotEqual(reply.status, 660)
gevent.sleep(12)
# Halt streaming.
cmd = AgentCommand(command='go_observatory')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Assert that data was received
self._async_finished_result.get(timeout=10)
self.assertTrue(len(self._finished_events_received) >= 3)
cmd = AgentCommand(command='reset')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_streaming(self):
# Test instrument driver execute interface to start and stop streaming mode.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Make sure the polling interval is appropriate for a test
params = {
'POLLING_INTERVAL':5
}
self._ia_client.set_param(params)
self._finished_count = 3
# Begin streaming.
cmd = AgentCommand(command='go_streaming')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STREAMING)
# Wait for some samples to roll in.
gevent.sleep(12)
# Halt streaming.
cmd = AgentCommand(command='go_observatory')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Assert that data was received
self._async_finished_result.get(timeout=10)
self.assertTrue(len(self._finished_events_received) >= 3)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_observatory(self):
# Test instrument driver get and set interface.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Retrieve all resource parameters.
reply = self._ia_client.get_param('DRIVER_PARAMETER_ALL')
self.assertParamDict(reply, True)
orig_config = reply
## Retrieve a subset of resource parameters.
params = [
'POLLING_INTERVAL'
]
reply = self._ia_client.get_param(params)
self.assertParamDict(reply)
orig_params = reply
# Set a subset of resource parameters.
new_params = {
'POLLING_INTERVAL' : (orig_params['POLLING_INTERVAL'] * 2),
}
self._ia_client.set_param(new_params)
check_new_params = self._ia_client.get_param(params)
self.assertParamVals(check_new_params, new_params)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_get_set_param(self):
cmd=AgentCommand(command='initialize')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='go_active')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='run')
_ = self._ia_client.execute_agent(cmd)
# Get a couple parameters
retval = self._ia_client.get_param(['POLLING_INTERVAL','PATCHABLE_CONFIG_KEYS'])
log.debug('Retrieved parameters from agent: {0}'.format(retval))
self.assertTrue(isinstance(retval,dict))
self.assertEqual(type(retval['POLLING_INTERVAL']),int)
self.assertEqual(type(retval['PATCHABLE_CONFIG_KEYS']),list)
# Attempt to get a parameter that doesn't exist
log.debug('Try getting a non-existent parameter \'BAD_PARAM\'')
self.assertRaises(InstParameterError, self._ia_client.get_param,['BAD_PARAM'])
# Set the polling_interval to a new value, then get it to make sure it set properly
self._ia_client.set_param({'POLLING_INTERVAL':10})
retval = self._ia_client.get_param(['POLLING_INTERVAL'])
log.debug('Retrieved parameters from agent: {0}'.format(retval))
self.assertTrue(isinstance(retval,dict))
self.assertEqual(retval['POLLING_INTERVAL'],10)
# Attempt to set a parameter that doesn't exist
log.debug('Try setting a non-existent parameter \'BAD_PARAM\'')
self.assertRaises(InstParameterError, self._ia_client.set_param, {'BAD_PARAM':'bad_val'})
# Attempt to set one parameter that does exist, and one that doesn't
self.assertRaises(InstParameterError, self._ia_client.set_param, {'POLLING_INTERVAL':20,'BAD_PARAM':'bad_val'})
retval = self._ia_client.get_param(['POLLING_INTERVAL'])
log.debug('Retrieved parameters from agent: {0}'.format(retval))
self.assertTrue(isinstance(retval,dict))
self.assertEqual(retval['POLLING_INTERVAL'],20)
cmd = AgentCommand(command='reset')
_ = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_initialize(self):
# Test agent initialize command. This causes creation of driver process and transition to inactive.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_states(self):
# Test agent state transitions.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='pause')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command='resume')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='clear')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='pause')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command='clear')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
self._finished_count = 1
cmd = AgentCommand(command='go_streaming')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STREAMING)
gevent.sleep(5)
cmd = AgentCommand(command='go_observatory')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
self._async_finished_result.get(timeout=5)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_capabilities(self):
# Test the ability to retrieve agent and resource parameter and command capabilities.
acmds = self._ia_client.get_capabilities(['AGT_CMD'])
log.debug('Agent Commands: {0}'.format(acmds))
acmds = [item[1] for item in acmds]
self.assertListsEqual(acmds, AGT_CMDS.keys())
apars = self._ia_client.get_capabilities(['AGT_PAR'])
log.debug('Agent Parameters: {0}'.format(apars))
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
rcmds = self._ia_client.get_capabilities(['RES_CMD'])
log.debug('Resource Commands: {0}'.format(rcmds))
rcmds = [item[1] for item in rcmds]
self.assertListsEqual(rcmds, CMDS.keys())
rpars = self._ia_client.get_capabilities(['RES_PAR'])
log.debug('Resource Parameters: {0}'.format(rpars))
rpars = [item[1] for item in rpars]
self.assertListsEqual(rpars, PARAMS.keys())
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
def test_errors(self):
# Test illegal behavior and replies.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
# Can't go active in unitialized state.
# Status 660 is state error.
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
log.info('GO ACTIVE CMD %s',str(retval))
self.assertEquals(retval.status, 660)
# Can't command driver in this state.
cmd = AgentCommand(command='acquire_sample')
reply = self._ia_client.execute(cmd)
self.assertEqual(reply.status, 660)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# 404 unknown agent command.
cmd = AgentCommand(command='kiss_edward')
retval = self._ia_client.execute_agent(cmd)
self.assertEquals(retval.status, 404)
# 670 unknown driver command.
cmd = AgentCommand(command='acquire_sample_please')
retval = self._ia_client.execute(cmd)
self.assertEqual(retval.status, 670)
# 630 Parameter error.
self.assertRaises(InstParameterError, self._ia_client.get_param, 'bogus bogus')
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)