def test_pub_on_different_origins(self):
ar = event.AsyncResult()
gq = queue.Queue()
self.count = 0
def cb(*args, **kwargs):
self.count += 1
gq.put(args[0])
if self.count == 3:
ar.set()
sub = EventSubscriber(event_type="ResourceEvent", callback=cb)
pub = EventPublisher(event_type="ResourceEvent")
self._listen(sub)
pub.publish_event(origin="one", description="1")
pub.publish_event(origin="two", description="2")
pub.publish_event(origin="three", description="3")
ar.get(timeout=5)
res = []
for x in xrange(self.count):
res.append(gq.get(timeout=5))
self.assertEquals(len(res), 3)
self.assertEquals(res[0].description, "1")
self.assertEquals(res[1].description, "2")
self.assertEquals(res[2].description, "3")
def publish_event_for_diagnostics(): # pragma: no cover
"""
Convenient method to do the publication of the event to generate diagnostic
information about the statuses kept in each running platform agent.
><> from ion.agents.platform.status_manager import publish_event_for_diagnostics
><> publish_event_for_diagnostics()
and something like the following will be logged out:
2013-05-17 17:25:16,076 INFO Dummy-247 ion.agents.platform.status_manager:760 'MJ01C': (99cb3e71302a4e5ca0c137292103e357) statuses:
AGGREGATE_COMMS AGGREGATE_DATA AGGREGATE_LOCATION AGGREGATE_POWER
d231ccba8d674b4691b039ceecec8d95 : STATUS_UNKNOWN STATUS_UNKNOWN STATUS_UNKNOWN STATUS_UNKNOWN
40c787fc727a4734b219fde7c8df7543 : STATUS_UNKNOWN STATUS_UNKNOWN STATUS_UNKNOWN STATUS_UNKNOWN
55ee7225435444e3a862d7ceaa9d1875 : STATUS_OK STATUS_OK STATUS_OK STATUS_OK
1d27e0c2723149cc9692488dced7dd95 : STATUS_UNKNOWN STATUS_UNKNOWN STATUS_UNKNOWN STATUS_UNKNOWN
aggstatus : STATUS_OK STATUS_OK STATUS_OK STATUS_OK
rollup_status : STATUS_OK STATUS_OK STATUS_OK STATUS_OK
"""
from pyon.event.event import EventPublisher
ep = EventPublisher()
evt = dict(event_type='DeviceStatusEvent',
sub_type='diagnoser',
origin='command_line')
print("publishing: %s" % str(evt))
ep.publish_event(**evt)
def _acquire_data(cls, config, publisher, unlock_new_data_callback):
"""
Ensures required keys (such as stream_id) are available from config, configures the publisher and then calls:
BaseDataHandler._new_data_constraints (only if config does not contain 'constraints')
BaseDataHandler._publish_data passing BaseDataHandler._get_data as a parameter
@param config Dict containing configuration parameters, may include constraints, formatters, etc
@param unlock_new_data_callback BaseDataHandler callback function to allow conditional unlocking of the BaseDataHandler._semaphore
"""
log.debug('start _acquire_data: config={0}'.format(config))
cls._init_acquisition_cycle(config)
constraints = get_safe(config, 'constraints')
if not constraints:
gevent.getcurrent().link(unlock_new_data_callback)
constraints = cls._new_data_constraints(config)
if constraints is None:
raise InstrumentParameterException(
"Data constraints returned from _new_data_constraints cannot be None"
)
config['constraints'] = constraints
cls._publish_data(publisher, cls._get_data(config))
# Publish a 'TestFinished' event
if get_safe(config, 'TESTING'):
log.debug('Publish TestingFinished event')
pub = EventPublisher('DeviceCommonLifecycleEvent')
pub.publish_event(origin='BaseDataHandler._acquire_data',
description='TestingFinished')
class SystemManagementService(BaseSystemManagementService):
""" container management requests are handled by the event listener
ion.processes.event.container_manager.ContainerManager
which must be running on each container.
"""
def on_start(self,*a,**b):
super(SystemManagementService,self).on_start(*a,**b)
self.sender = EventPublisher()
def on_quit(self,*a,**b):
self.sender.close()
def perform_action(self, predicate, action):
userid = None # get from context
self.sender.publish_event(event_type=OT.ContainerManagementRequest, origin=userid, predicate=predicate, action=action)
def set_log_level(self, logger='', level='', recursive=False):
self.perform_action(ALL_CONTAINERS_INSTANCE, IonObject(OT.ChangeLogLevel, logger=logger, level=level, recursive=recursive))
def reset_policy_cache(self, headers=None, timeout=None):
"""Clears and reloads the policy caches in all of the containers.
@throws BadRequest None
"""
self.perform_action(ALL_CONTAINERS_INSTANCE, IonObject(OT.ResetPolicyCache))
def trigger_garbage_collection(self):
"""Triggers a garbage collection in all containers
@throws BadRequest None
"""
self.perform_action(ALL_CONTAINERS_INSTANCE, IonObject(OT.TriggerGarbageCollection))
def trigger_container_snapshot(self, snapshot_id='', include_snapshots=None, exclude_snapshots=None,
take_at_time='', clear_all=False, persist_snapshot=True, snapshot_kwargs=None):
if not snapshot_id:
snapshot_id = get_ion_ts()
if not snapshot_kwargs:
snapshot_kwargs = {}
self.perform_action(ALL_CONTAINERS_INSTANCE, IonObject(OT.TriggerContainerSnapshot,
snapshot_id=snapshot_id,
include_snapshots=include_snapshots,
exclude_snapshots=exclude_snapshots,
take_at_time=take_at_time,
clear_all=clear_all,
persist_snapshot=persist_snapshot,
snapshot_kwargs=snapshot_kwargs))
log.info("Event to trigger container snapshots sent. snapshot_id=%s" % snapshot_id)
def start_gevent_block(self, alarm_mode=False):
self.perform_action(ALL_CONTAINERS_INSTANCE, IonObject(OT.StartGeventBlock, alarm_mode=alarm_mode))
def stop_gevent_block(self):
self.perform_action(ALL_CONTAINERS_INSTANCE, IonObject(OT.StopGeventBlock))
def prepare_system_shutdown(self, mode=''):
self.perform_action(ALL_CONTAINERS_INSTANCE, IonObject(OT.PrepareSystemShutdown, mode=mode))
def publish_event_for_diagnostics(): # pragma: no cover
"""
Convenient method to do the publication of the event to generate diagnostic
information about the statuses kept in each running platform agent.
><> from ion.agents.platform.status_manager import publish_event_for_diagnostics
><> publish_event_for_diagnostics()
and something like the following will be logged out:
2013-05-17 17:25:16,076 INFO Dummy-247 ion.agents.platform.status_manager:760 'MJ01C': (99cb3e71302a4e5ca0c137292103e357) statuses:
AGGREGATE_COMMS AGGREGATE_DATA AGGREGATE_LOCATION AGGREGATE_POWER
d231ccba8d674b4691b039ceecec8d95 : STATUS_UNKNOWN STATUS_UNKNOWN STATUS_UNKNOWN STATUS_UNKNOWN
40c787fc727a4734b219fde7c8df7543 : STATUS_UNKNOWN STATUS_UNKNOWN STATUS_UNKNOWN STATUS_UNKNOWN
55ee7225435444e3a862d7ceaa9d1875 : STATUS_OK STATUS_OK STATUS_OK STATUS_OK
1d27e0c2723149cc9692488dced7dd95 : STATUS_UNKNOWN STATUS_UNKNOWN STATUS_UNKNOWN STATUS_UNKNOWN
aggstatus : STATUS_OK STATUS_OK STATUS_OK STATUS_OK
rollup_status : STATUS_OK STATUS_OK STATUS_OK STATUS_OK
"""
from pyon.event.event import EventPublisher
ep = EventPublisher()
evt = dict(event_type='DeviceStatusEvent', sub_type='diagnoser', origin='command_line')
print("publishing: %s" % str(evt))
ep.publish_event(**evt)
def test_base_subscriber_as_catchall(self):
ar = event.AsyncResult()
gq = queue.Queue()
self.count = 0
def cb(*args, **kwargs):
self.count += 1
gq.put(args[0])
if self.count == 2:
ar.set()
sub = EventSubscriber(callback=cb)
pub1 = EventPublisher(event_type="ResourceEvent")
pub2 = EventPublisher(event_type="ContainerLifecycleEvent")
self._listen(sub)
pub1.publish_event(origin="some", description="1")
pub2.publish_event(origin="other", description="2")
ar.get(timeout=5)
res = []
for x in xrange(self.count):
res.append(gq.get(timeout=5))
self.assertEquals(len(res), 2)
self.assertEquals(res[0].description, "1")
self.assertEquals(res[1].description, "2")
def test_pub_with_event_repo(self):
pub = EventPublisher(event_type="ResourceEvent",
node=self.container.node)
pub.publish_event(origin="specifics", description="hallo")
evs = self.container.event_repository.find_events(origin='specifics')
self.assertEquals(len(evs), 1)
def test_sub(self):
#start interaction observer
io = InteractionObserver()
io.start()
#publish an event
ev_pub = EventPublisher(event_type="ResourceEvent")
ev_pub.publish_event(origin="specific", description="event")
# publish a message
msg_pub = Publisher()
msg_pub.publish(to_name='anyone', msg="msg")
# give 2 seconds for the messages to arrive
time.sleep(2)
#verify that two messages (an event and a message) are seen
self.assertEquals(len(io.msg_log), 2)
#iterate through the messages observed
for item in io.msg_log:
# if event
if item[2]:
#verify that the origin is what we sent
self.assertEquals(item[1]['origin'], 'specific')
dump = io._get_data(io.msg_log,{})
sump = dump
def publish_alert(self):
"""
"""
event_data = self.make_event_data()
print '########## publishing: ' + event_data['sub_type']
pub = EventPublisher()
pub.publish_event(**event_data)
def subscription_callback(self, message, headers):
filter_field = self.notification._res_obj.delivery_config.processing[
'filter_field']
condition = self.notification._res_obj.delivery_config.processing[
'condition']
try:
comparator = self.notification._res_obj.delivery_config.processing[
'comparator']
comparator_func = DetectionEventProcessor.comparators[comparator]
except KeyError:
raise BadRequest(
"Bad comparator specified in Detection filter: '%s'" %
comparator)
field_val = getattr(message, filter_field)
if field_val is not None and comparator_func(field_val, condition):
log.info('Detected an event')
event_publisher = EventPublisher("DetectionEvent")
message = str(self.notification._res_obj.delivery_config)
#@David What should the origin and origin type be for Detection Events
event_publisher.publish_event(
origin='DetectionEventProcessor',
message="Event Detected by DetectionEventProcessor",
description="Event was detected by DetectionEventProcessor",
condition=
message, # Concatenate the filter and make it a message
original_origin=self.notification._res_obj.origin,
original_type=self.notification._res_obj.origin_type)
def _acquire_data(cls, config, publisher, unlock_new_data_callback):
"""
Ensures required keys (such as stream_id) are available from config, configures the publisher and then calls:
BaseDataHandler._new_data_constraints (only if config does not contain 'constraints')
BaseDataHandler._publish_data passing BaseDataHandler._get_data as a parameter
@param config Dict containing configuration parameters, may include constraints, formatters, etc
@param unlock_new_data_callback BaseDataHandler callback function to allow conditional unlocking of the BaseDataHandler._semaphore
"""
log.debug('start _acquire_data: config={0}'.format(config))
cls._init_acquisition_cycle(config)
constraints = get_safe(config,'constraints')
if not constraints:
gevent.getcurrent().link(unlock_new_data_callback)
constraints = cls._new_data_constraints(config)
if constraints is None:
raise InstrumentParameterException("Data constraints returned from _new_data_constraints cannot be None")
config['constraints'] = constraints
cls._publish_data(publisher, cls._get_data(config))
# Publish a 'TestFinished' event
if get_safe(config,'TESTING'):
log.debug('Publish TestingFinished event')
pub = EventPublisher('DeviceCommonLifecycleEvent')
pub.publish_event(origin='BaseDataHandler._acquire_data', description='TestingFinished')
def test_sub(self):
#start interaction observer
io = InteractionObserver()
io.start()
#publish an event
ev_pub = EventPublisher(event_type="ResourceEvent")
ev_pub.publish_event(origin="specific", description="event")
# publish a message
msg_pub = Publisher()
msg_pub.publish(to_name='anyone', msg="msg")
# give 2 seconds for the messages to arrive
time.sleep(2)
#verify that two messages (an event and a message) are seen
self.assertEquals(len(io.msg_log), 2)
#iterate through the messages observed
for item in io.msg_log:
# if event
if item[2]:
#verify that the origin is what we sent
self.assertEquals(item[1]['origin'], 'specific')
dump = io._get_data(io.msg_log,{})
sump = dump
def test_base_subscriber_as_catchall(self):
ar = event.AsyncResult()
gq = queue.Queue()
self.count = 0
def cb(*args, **kwargs):
self.count += 1
gq.put(args[0])
if self.count == 2:
ar.set()
sub = EventSubscriber(callback=cb)
pub1 = EventPublisher(event_type="ResourceEvent")
pub2 = EventPublisher(event_type="ContainerLifecycleEvent")
self._listen(sub)
pub1.publish_event(origin="some", description="1")
pub2.publish_event(origin="other", description="2")
ar.get(timeout=5)
res = []
for x in xrange(self.count):
res.append(gq.get(timeout=5))
self.assertEquals(len(res), 2)
self.assertEquals(res[0].description, "1")
self.assertEquals(res[1].description, "2")
def process_oms_event():
json_params = {}
# oms direct request
if request.data:
json_params = json_loads(str(request.data))
log.debug('ServiceGatewayService:process_oms_event request.data: %s', json_params)
#validate payload
if 'platform_id' not in json_params or 'message' not in json_params:
log.warning('Invalid OMS event format. payload_data: %s', json_params)
#return gateway_json_response(OMS_BAD_REQUEST_RESPONSE)
#prepare the event information
try:
#create a publisher to relay OMS events into the system as DeviceEvents
event_publisher = EventPublisher()
event_publisher.publish_event(
event_type='OMSDeviceStatusEvent',
origin_type='OMS Platform',
origin=json_params.get('platform_id', 'NOT PROVIDED'),
sub_type='',
description = json_params.get('message', ''),
status_details = json_params)
except Exception, e:
log.error('Could not publish OMS event: %s. Event data: %s', e.message, json_params)
def _acquire_data(cls, config, unlock_new_data_callback):
"""
Ensures required keys (such as stream_id) are available from config, configures the publisher and then calls:
BaseDataHandler._new_data_constraints (only if config does not contain 'constraints')
BaseDataHandler._publish_data passing BaseDataHandler._get_data as a parameter
@param config Dict containing configuration parameters, may include constraints, formatters, etc
@param unlock_new_data_callback BaseDataHandler callback function to allow conditional unlocking of the BaseDataHandler._semaphore
"""
stream_id = get_safe(config, 'stream_id')
if not stream_id:
raise ConfigurationError('Configuration does not contain required \'stream_id\' key')
#TODO: Configure the publisher
publisher=None
constraints = get_safe(config,'constraints')
if not constraints:
gevent.getcurrent().link(unlock_new_data_callback)
constraints = cls._new_data_constraints(config)
config['constraints']=constraints
cls._publish_data(publisher, config, cls._get_data(config))
# Publish a 'TestFinished' event
if get_safe(config,'TESTING'):
log.debug('Publish TestingFinished event')
pub = EventPublisher('DeviceCommonLifecycleEvent')
pub.publish_event(origin='BaseDataHandler._acquire_data', description='TestingFinished')
def test_pub_on_different_origins(self):
ar = event.AsyncResult()
gq = queue.Queue()
self.count = 0
def cb(*args, **kwargs):
self.count += 1
gq.put(args[0])
if self.count == 3:
ar.set()
sub = EventSubscriber(event_type="ResourceEvent", callback=cb)
pub = EventPublisher(event_type="ResourceEvent")
self._listen(sub)
pub.publish_event(origin="one", description="1")
pub.publish_event(origin="two", description="2")
pub.publish_event(origin="three", description="3")
ar.get(timeout=5)
res = []
for x in xrange(self.count):
res.append(gq.get(timeout=5))
self.assertEquals(len(res), 3)
self.assertEquals(res[0].description, "1")
self.assertEquals(res[1].description, "2")
self.assertEquals(res[2].description, "3")
def process_oms_event():
json_params = {}
# oms direct request
if request.data:
json_params = simplejson.loads(str(request.data))
log.debug('ServiceGatewayService:process_oms_event request.data: %s',
json_params)
#validate payload
if 'platform_id' not in json_params or 'message' not in json_params:
log.warning('Invalid OMS event format. payload_data: %s', json_params)
#return gateway_json_response(OMS_BAD_REQUEST_RESPONSE)
#prepare the event information
try:
#create a publisher to relay OMS events into the system as DeviceEvents
event_publisher = EventPublisher()
event_publisher.publish_event(
event_type='OMSDeviceStatusEvent',
origin_type='OMS Platform',
origin=json_params.get('platform_id', 'NOT PROVIDED'),
sub_type='',
description=json_params.get('message', ''),
status_details=json_params)
except Exception, e:
log.error('Could not publish OMS event: %s. Event data: %s',
e.message, json_params)
class SystemManagementService(BaseSystemManagementService):
""" container management requests are handled by the event listener
ion.processes.event.container_manager.ContainerManager
which must be running on each container.
"""
def on_start(self,*a,**b):
super(SystemManagementService,self).on_start(*a,**b)
self.sender = EventPublisher()
def on_quit(self,*a,**b):
self.sender.close()
def perform_action(self, predicate, action):
userid = None # get from context
self.sender.publish_event(event_type=OT.ContainerManagementRequest, origin=userid, predicate=predicate, action=action)
def set_log_level(self, logger='', level='', recursive=False):
self.perform_action(ALL_CONTAINERS_INSTANCE, IonObject(OT.ChangeLogLevel, logger=logger, level=level, recursive=recursive))
def reset_policy_cache(self, headers=None, timeout=None):
"""Clears and reloads the policy caches in all of the containers.
@throws BadRequest None
"""
self.perform_action(ALL_CONTAINERS_INSTANCE, IonObject(OT.ResetPolicyCache))
def trigger_garbage_collection(self):
"""Triggers a garbage collection in all containers
@throws BadRequest None
"""
self.perform_action(ALL_CONTAINERS_INSTANCE, IonObject(OT.TriggerGarbageCollection))
def test_send_notification_emails(self):
# create user with email address in RR
user_identty_object = IonObject(RT.ActorIdentity, name="user1")
user_id = self.imc.create_actor_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {"name":"user1_info", "contact":{"email":'*****@*****.**'}})
self.imc.create_user_info(user_id, user_info_object)
# create first notification
notification_object = IonObject(RT.NotificationRequest, {"name":"notification1",
"origin_list":['Some_Resource_Agent_ID1'],
"events_list":['ResourceLifecycleEvent']})
self.unsc.create_notification(notification_object, user_id)
# create second notification
notification_object = IonObject(RT.NotificationRequest, {"name":"notification2",
"origin_list":['Some_Resource_Agent_ID2'],
"events_list":['DataEvent']})
self.unsc.create_notification(notification_object, user_id)
# publish an event for each notification to generate the emails
# this can't be easily check in SW so need to check for these at the [email protected] account
rle_publisher = EventPublisher("ResourceLifecycleEvent")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1', description="RLE test event")
de_publisher = EventPublisher("DataEvent")
de_publisher.publish_event(origin='Some_Resource_Agent_ID2', description="DE test event")
gevent.sleep(1)
def test_event_in_stream_out_transform(self):
"""
Test the event-in/stream-out transform
"""
stream_id, _ = self.pubsub.create_stream('test_stream', exchange_point='science_data')
self.exchange_cleanup.append('science_data')
#---------------------------------------------------------------------------------------------
# Launch a ctd transform
#---------------------------------------------------------------------------------------------
# Create the process definition
process_definition = ProcessDefinition(
name='EventToStreamTransform',
description='For testing an event-in/stream-out transform')
process_definition.executable['module']= 'ion.processes.data.transforms.event_in_stream_out_transform'
process_definition.executable['class'] = 'EventToStreamTransform'
proc_def_id = self.process_dispatcher.create_process_definition(process_definition=process_definition)
# Build the config
config = DotDict()
config.process.queue_name = 'test_queue'
config.process.exchange_point = 'science_data'
config.process.publish_streams.output = stream_id
config.process.event_type = 'ExampleDetectableEvent'
config.process.variables = ['voltage', 'temperature' ]
# Schedule the process
pid = self.process_dispatcher.schedule_process(process_definition_id=proc_def_id, configuration=config)
self.addCleanup(self.process_dispatcher.cancel_process,pid)
#---------------------------------------------------------------------------------------------
# Create a subscriber for testing
#---------------------------------------------------------------------------------------------
ar_cond = gevent.event.AsyncResult()
def subscriber_callback(m, r, s):
ar_cond.set(m)
sub = StandaloneStreamSubscriber('sub', subscriber_callback)
self.addCleanup(sub.stop)
sub_id = self.pubsub.create_subscription('subscription_cond',
stream_ids=[stream_id],
exchange_name='sub')
self.pubsub.activate_subscription(sub_id)
self.queue_cleanup.append(sub.xn.queue)
sub.start()
gevent.sleep(4)
#---------------------------------------------------------------------------------------------
# Publish an event. The transform has been configured to receive this event
#---------------------------------------------------------------------------------------------
event_publisher = EventPublisher("ExampleDetectableEvent")
event_publisher.publish_event(origin = 'fake_origin', voltage = '5', temperature = '273')
# Assert that the transform processed the event and published data on the output stream
result_cond = ar_cond.get(timeout=10)
self.assertTrue(result_cond)
def publish_alert(self):
"""
Publishes the alert to ION.
"""
event_data = self.make_event_data()
log.trace("publishing alert: %s", event_data)
pub = EventPublisher()
pub.publish_event(**event_data)
def call_process(self, packet, headers=None):
try:
self.process(packet)
except Exception as e:
log.exception('Unhandled caught in transform process')
event_publisher = EventPublisher()
event_publisher.publish_event(origin=self._transform_id, event_type='ExceptionEvent',
exception_type=str(type(e)), exception_message=e.message)
def publish_alert(self):
"""
Publishes the alert to ION.
"""
event_data = self.make_event_data()
log.trace("publishing alert: %s", event_data)
pub = EventPublisher()
pub.publish_event(**event_data)
def call_process(self, packet):
try:
self.process(packet)
except Exception as e:
log.exception('Unhandled caught in transform process')
event_publisher = EventPublisher()
event_publisher.publish_event(origin=self._transform_id,
event_type='ExceptionEvent',
exception_type=str(type(e)),
exception_message=e.message)
def validate_salinity_array(a, context={}):
from pyon.agent.agent import ResourceAgentState
from pyon.event.event import EventPublisher
from pyon.public import OT
stream_id = context['stream_id']
dataprocess_id = context['dataprocess_id']
event_publisher = EventPublisher(OT.DeviceStatusAlertEvent)
event_publisher.publish_event( origin = stream_id, values=[dataprocess_id], description="Invalid value for salinity")
def _acquire_sample(cls, config, publisher, unlock_new_data_callback,
update_new_data_check_attachment):
"""
Ensures required keys (such as stream_id) are available from config, configures the publisher and then calls:
BaseDataHandler._constraints_for_new_request (only if config does not contain 'constraints')
BaseDataHandler._publish_data passing BaseDataHandler._get_data as a parameter
@param config Dict containing configuration parameters, may include constraints, formatters, etc
@param publisher the publisher used to publish data
@param unlock_new_data_callback BaseDataHandler callback function to allow conditional unlocking of the BaseDataHandler._semaphore
@param update_new_data_check_attachment classmethod to update the external dataset resources file list attachment
@throws InstrumentParameterException if the data constraints are not a dictionary
@retval None
"""
log.debug('start _acquire_sample: config={0}'.format(config))
cls._init_acquisition_cycle(config)
constraints = get_safe(config, 'constraints')
if not constraints:
gevent.getcurrent().link(unlock_new_data_callback)
try:
constraints = cls._constraints_for_new_request(config)
except NoNewDataWarning:
#log.info(nndw.message)
if get_safe(config, 'TESTING'):
#log.debug('Publish TestingFinished event')
pub = EventPublisher('DeviceCommonLifecycleEvent')
pub.publish_event(origin='BaseDataHandler._acquire_sample',
description='TestingFinished')
return
if constraints is None:
raise InstrumentParameterException(
"Data constraints returned from _constraints_for_new_request cannot be None"
)
config['constraints'] = constraints
elif isinstance(constraints, dict):
addnl_constr = cls._constraints_for_historical_request(config)
if not addnl_constr is None and isinstance(addnl_constr, dict):
constraints.update(addnl_constr)
else:
raise InstrumentParameterException(
'Data constraints must be of type \'dict\': {0}'.format(
constraints))
cls._publish_data(publisher, cls._get_data(config), config,
update_new_data_check_attachment)
# Publish a 'TestFinished' event
if get_safe(config, 'TESTING'):
#log.debug('Publish TestingFinished event')
pub = EventPublisher(OT.DeviceCommonLifecycleEvent)
pub.publish_event(origin='BaseDataHandler._acquire_sample',
description='TestingFinished')
def test_two_sided_interval(self):
"""
test_two_sided_interval
Test interval alarm and alarm event publishing for a closed
inteval.
"""
kwargs = {
'name': 'temp_high_warning',
'stream_name': 'fakestreamname',
'value_id': 'temp',
'message': 'Temperature is above normal range.',
'type': StreamAlarmType.WARNING,
'lower_bound': 10.0,
'lower_rel_op': '<',
'upper_bound': 20.0,
'upper_rel_op': '<'
}
# Create alarm object.
alarm = IonObject('IntervalAlarmDef', **kwargs)
alarm = construct_alarm_expression(alarm)
# This sequence will produce 5 alarms.
# 5.5 warning
# 10.2 all clear
# 23.3 warning
# 17.5 all clear
# 8.8 warning
self._event_count = 5
test_vals = [
5.5, 5.5, 5.4, 4.6, 4.5, 10.2, 10.3, 10.5, 15.5, 23.3, 23.3, 24.8,
17.5, 16.5, 12.5, 8.8, 7.7
]
pub = EventPublisher(event_type="StreamAlarmEvent",
node=self.container.node)
for x in test_vals:
event_data = None
eval_alarm(alarm, x)
if alarm.first_time == 1:
event_data = make_event_data(alarm)
elif alarm.first_time > 1:
if alarm.status != alarm.old_status:
event_data = make_event_data(alarm)
if event_data:
pub.publish_event(origin=self._resource_id, **event_data)
self._async_event_result.get(timeout=30)
"""
class ProcessDispatcherSimpleAPIClient(object):
# State to use when state returned from PD is None
unknown_state = "400-PENDING"
state_map = {
ProcessStateEnum.SPAWN: '500-RUNNING',
ProcessStateEnum.TERMINATE: '700-TERMINATED',
ProcessStateEnum.ERROR: '850-FAILED'
}
def __init__(self, name, **kwargs):
self.real_client = ProcessDispatcherServiceClient(to_name=name, **kwargs)
self.event_pub = EventPublisher()
def dispatch_process(self, upid, spec, subscribers, constraints=None,
immediate=False):
name = spec.get('name')
self.event_pub.publish_event(event_type="ProcessLifecycleEvent",
origin=name, origin_type="DispatchedHAProcess",
state=ProcessStateEnum.SPAWN)
process_def = ProcessDefinition(name=name)
process_def.executable = {'module': spec.get('module'),
'class': spec.get('class')}
process_def_id = self.real_client.create_process_definition(process_def)
pid = self.real_client.create_process(process_def_id)
process_schedule = ProcessSchedule()
sched_pid = self.real_client.schedule_process(process_def_id,
process_schedule, configuration={}, process_id=pid)
proc = self.real_client.read_process(sched_pid)
dict_proc = {'upid': proc.process_id,
'state': self.state_map.get(proc.process_state, self.unknown_state),
}
return dict_proc
def terminate_process(self, pid):
return self.real_client.cancel_process(pid)
def describe_processes(self):
procs = self.real_client.list_processes()
dict_procs = []
for proc in procs:
dict_proc = {'upid': proc.process_id,
'state': self.state_map.get(proc.process_state, self.unknown_state),
}
dict_procs.append(dict_proc)
return dict_procs
def validate_salinity_array(a, context={}):
from pyon.agent.agent import ResourceAgentState
from pyon.event.event import EventPublisher
from pyon.public import OT
stream_id = context['stream_id']
dataprocess_id = context['dataprocess_id']
event_publisher = EventPublisher(OT.DeviceStatusAlertEvent)
event_publisher.publish_event(origin=stream_id,
values=[dataprocess_id],
description="Invalid value for salinity")
def test_two_sided_interval(self):
"""
test_two_sided_interval
Test interval alarm and alarm event publishing for a closed
inteval.
"""
kwargs = {
"name": "temp_high_warning",
"stream_name": "fakestreamname",
"value_id": "temp",
"message": "Temperature is above normal range.",
"type": StreamAlarmType.WARNING,
"lower_bound": 10.0,
"lower_rel_op": "<",
"upper_bound": 20.0,
"upper_rel_op": "<",
}
# Create alarm object.
alarm = IonObject("IntervalAlarmDef", **kwargs)
alarm = construct_alarm_expression(alarm)
# This sequence will produce 5 alarms.
# 5.5 warning
# 10.2 all clear
# 23.3 warning
# 17.5 all clear
# 8.8 warning
self._event_count = 5
test_vals = [5.5, 5.5, 5.4, 4.6, 4.5, 10.2, 10.3, 10.5, 15.5, 23.3, 23.3, 24.8, 17.5, 16.5, 12.5, 8.8, 7.7]
pub = EventPublisher(event_type="StreamAlarmEvent", node=self.container.node)
for x in test_vals:
event_data = None
eval_alarm(alarm, x)
if alarm.first_time == 1:
event_data = make_event_data(alarm)
elif alarm.first_time > 1:
if alarm.status != alarm.old_status:
event_data = make_event_data(alarm)
if event_data:
pub.publish_event(origin=self._resource_id, **event_data)
self._async_event_result.get(timeout=30)
"""
def test_less_than_interval(self):
"""
test_less_than_interval
Test interval alarm and alarm event publishing for a less than
inteval.
"""
kwargs = {
'name': 'reserve_power_warning',
'stream_name': 'fakestreamname',
'value_id': 'battery_level',
'message': 'Battery is below normal range.',
'type': StreamAlarmType.WARNING,
'upper_bound': 4.0,
'upper_rel_op': '<'
}
# Create alarm object.
alarm = IonObject('IntervalAlarmDef', **kwargs)
alarm = construct_alarm_expression(alarm)
# This sequence will produce 5 alarms:
# 5.5 warning
# 3.3 all clear
# 4.5 warning
# 3.3 all clear
# 4.8 warning
self._event_count = 5
test_vals = [
5.5, 5.5, 5.4, 4.6, 4.5, 3.3, 3.3, 4.5, 4.5, 3.3, 3.3, 4.8
]
pub = EventPublisher(event_type="StreamAlarmEvent",
node=self.container.node)
for x in test_vals:
event_data = None
eval_alarm(alarm, x)
if alarm.first_time == 1:
event_data = make_event_data(alarm)
elif alarm.first_time > 1:
if alarm.status != alarm.old_status:
event_data = make_event_data(alarm)
if event_data:
pub.publish_event(origin=self._resource_id, **event_data)
self._async_event_result.get(timeout=30)
"""
def test_pub_on_different_subtypes(self):
ar = event.AsyncResult()
gq = queue.Queue()
self.count = 0
def cb(event, *args, **kwargs):
self.count += 1
gq.put(event)
if event.description == "end":
ar.set()
sub = EventSubscriber(event_type="ResourceModifiedEvent",
sub_type="st1",
callback=cb)
sub.activate()
pub1 = EventPublisher(event_type="ResourceModifiedEvent")
pub2 = EventPublisher(event_type="ContainerLifecycleEvent")
pub1.publish_event(origin="two", sub_type="st2", description="2")
pub2.publish_event(origin="three", sub_type="st1", description="3")
pub1.publish_event(origin="one", sub_type="st1", description="1")
pub1.publish_event(origin="four", sub_type="st1", description="end")
ar.get(timeout=5)
sub.deactivate()
res = []
for x in xrange(self.count):
res.append(gq.get(timeout=5))
self.assertEquals(len(res), 2)
self.assertEquals(res[0].description, "1")
def publish_link_event(up_down, terrestrial_remote=2):
"""
"""
status = TelemetryStatusType.AVAILABLE if up_down \
else TelemetryStatusType.UNAVAILABLE
platform_id = tcaa_args['terrestrial_platform_id'] if terrestrial_remote \
else tcaa_args['remote_platform_id']
pub = EventPublisher()
if terrestrial_remote == 0:
pub.publish_event(
event_type='PlatformTelemetryEvent',
origin=tcaa_args['terrestrial_platform_id'],
status = status)
elif terrestrial_remote == 1:
pub.publish_event(
event_type='PlatformTelemetryEvent',
origin=tcaa_args['remote_platform_id'],
status = status)
elif terrestrial_remote == 2:
pub.publish_event(
event_type='PlatformTelemetryEvent',
origin=tcaa_args['terrestrial_platform_id'],
status = status)
pub.publish_event(
event_type='PlatformTelemetryEvent',
origin=tcaa_args['remote_platform_id'],
status = status)
else:
raise ValueError('terrestrial_remote must be in range [0,2].')
def test_pub_and_sub(self):
ar = event.AsyncResult()
def cb(*args, **kwargs):
ar.set(args)
sub = EventSubscriber(event_type="ResourceEvent", callback=cb, origin="specific")
pub = EventPublisher(event_type="ResourceEvent")
self._listen(sub)
pub.publish_event(origin="specific", description="hello")
evmsg, evheaders = ar.get(timeout=5)
self.assertEquals(evmsg.description, "hello")
self.assertAlmostEquals(int(evmsg.ts_created), int(get_ion_ts()), delta=5000)
def test_greater_than_interval(self):
"""
test_greater_than_interval
Test interval alarm and alarm event publishing for a greater than
inteval.
"""
kwargs = {
'name': 'current_warning_interval',
'stream_name': 'fakestreamname',
'value_id': 'port_current',
'message': 'Current is above normal range.',
'type': StreamAlarmType.WARNING,
'lower_bound': 10.5,
'lower_rel_op': '<'
}
# Create alarm object.
alarm = IonObject('IntervalAlarmDef', **kwargs)
alarm = construct_alarm_expression(alarm)
# This sequence will produce 5 alarms:
# Warning on the first value,
# All clear on 30,
# Warning on 5.5
# All clear on 15.1
# Warning on 3.3
self._event_count = 5
test_vals = [
5.5, 5.4, 5.5, 5.6, 30, 30.4, 5.5, 5.6, 15.1, 15.2, 15.3, 3.3, 3.4
]
pub = EventPublisher(event_type="StreamAlarmEvent",
node=self.container.node)
for x in test_vals:
event_data = None
eval_alarm(alarm, x)
if alarm.first_time == 1:
event_data = make_event_data(alarm)
elif alarm.first_time > 1:
if alarm.status != alarm.old_status:
event_data = make_event_data(alarm)
if event_data:
pub.publish_event(origin=self._resource_id, **event_data)
self._async_event_result.get(timeout=30)
"""
def test_greater_than_interval(self):
"""
test_greater_than_interval
Test interval alarm and alarm event publishing for a greater than
inteval.
"""
kwargs = {
'name' : 'current_warning_interval',
'stream_name' : 'fakestreamname',
'value_id' : 'port_current',
'message' : 'Current is above normal range.',
'type' : StreamAlarmType.WARNING,
'lower_bound' : 10.5,
'lower_rel_op' : '<'
}
if TEST_ION_OBJECTS:
# Create alarm object.
alarm = IonObject('IntervalAlarmDef', **kwargs)
alarm = construct_alarm_expression(alarm)
else:
alarm = IntervalAlarm(**kwargs)
# This sequence will produce 5 alarms:
# All clear on the first value,
# Warning on the first 30,
# All clear on the following 5.5,
# Warning on the 15.1,
# All clear on the following 3.3.
self._event_count = 5
test_vals = [5.5, 5.4, 5.5, 5.6, 30, 30.4, 5.5, 5.6, 15.1, 15.2,
15.3, 3.3, 3.4]
pub = EventPublisher(event_type="StreamAlarmEvent",
node=self.container.node)
for x in test_vals:
if TEST_ION_OBJECTS:
(alarm, event_data) = eval_alarm(alarm, x)
else:
event_data = alarm.eval_alarm(x)
if event_data:
pub.publish_event(origin=self._resource_id, **event_data)
self._async_event_result.get(timeout=30)
"""
def test_pub_on_different_subtypes(self):
ar = event.AsyncResult()
gq = queue.Queue()
self.count = 0
def cb(event, *args, **kwargs):
self.count += 1
gq.put(event)
if event.description == "end":
ar.set()
sub = EventSubscriber(event_type="ResourceModifiedEvent", sub_type="st1", callback=cb)
sub.activate()
pub1 = EventPublisher(event_type="ResourceModifiedEvent")
pub2 = EventPublisher(event_type="ContainerLifecycleEvent")
pub1.publish_event(origin="two", sub_type="st2", description="2")
pub2.publish_event(origin="three", sub_type="st1", description="3")
pub1.publish_event(origin="one", sub_type="st1", description="1")
pub1.publish_event(origin="four", sub_type="st1", description="end")
ar.get(timeout=5)
sub.deactivate()
res = []
for x in xrange(self.count):
res.append(gq.get(timeout=5))
self.assertEquals(len(res), 2)
self.assertEquals(res[0].description, "1")
def test_pub_and_sub(self):
ar = event.AsyncResult()
def cb(*args, **kwargs):
ar.set(args)
sub = EventSubscriber(event_type="ResourceEvent", callback=cb, origin="specific")
pub = EventPublisher(event_type="ResourceEvent")
self._listen(sub)
pub.publish_event(origin="specific", description="hello")
evmsg, evheaders = ar.get(timeout=5)
self.assertEquals(evmsg.description, "hello")
self.assertAlmostEquals(int(evmsg.ts_created), int(get_ion_ts()), delta=5000)
def publish_link_event(up_down, terrestrial_remote=2):
"""
"""
status = TelemetryStatusType.AVAILABLE if up_down \
else TelemetryStatusType.UNAVAILABLE
platform_id = tcaa_args['terrestrial_platform_id'] if terrestrial_remote \
else tcaa_args['remote_platform_id']
pub = EventPublisher()
if terrestrial_remote == 0:
pub.publish_event(event_type='PlatformTelemetryEvent',
origin=tcaa_args['terrestrial_platform_id'],
status=status)
elif terrestrial_remote == 1:
pub.publish_event(event_type='PlatformTelemetryEvent',
origin=tcaa_args['remote_platform_id'],
status=status)
elif terrestrial_remote == 2:
pub.publish_event(event_type='PlatformTelemetryEvent',
origin=tcaa_args['terrestrial_platform_id'],
status=status)
pub.publish_event(event_type='PlatformTelemetryEvent',
origin=tcaa_args['remote_platform_id'],
status=status)
else:
raise ValueError('terrestrial_remote must be in range [0,2].')
def test_two_sided_interval(self):
"""
test_two_sided_interval
Test interval alarm and alarm event publishing for a closed
inteval.
"""
kwargs = {
'name' : 'temp_high_warning',
'stream_name' : 'fakestreamname',
'value_id' : 'temp',
'message' : 'Temperature is above normal range.',
'type' : StreamAlarmType.WARNING,
'lower_bound' : 10.0,
'lower_rel_op' : '<',
'upper_bound' : 20.0,
'upper_rel_op' : '<'
}
if TEST_ION_OBJECTS:
# Create alarm object.
alarm = IonObject('IntervalAlarmDef', **kwargs)
alarm = construct_alarm_expression(alarm)
else:
alarm = IntervalAlarm(**kwargs)
# This sequence will produce 5 alarms.
self._event_count = 5
test_vals = [5.5, 5.5, 5.4, 4.6, 4.5, 10.2, 10.3, 10.5, 15.5,
23.3, 23.3, 24.8, 17.5, 16.5, 12.5, 8.8, 7.7]
pub = EventPublisher(event_type="StreamAlarmEvent",
node=self.container.node)
for x in test_vals:
if TEST_ION_OBJECTS:
(alarm, event_data) = eval_alarm(alarm, x)
else:
event_data = alarm.eval_alarm(x)
if event_data:
pub.publish_event(origin=self._resource_id, **event_data)
self._async_event_result.get(timeout=30)
class EventAlertTransform(TransformEventListener):
def on_start(self):
log.warn("EventAlertTransform.on_start()")
super(EventAlertTransform, self).on_start()
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
# get the algorithm to use
# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
self.max_count = self.CFG.get_safe("process.max_count", 1)
self.time_window = self.CFG.get_safe("process.time_window", 0)
self.counter = 0
self.event_times = []
# -------------------------------------------------------------------------------------
# Create the publisher that will publish the Alert message
# -------------------------------------------------------------------------------------
self.event_publisher = EventPublisher()
def process_event(self, msg, headers):
"""
The callback method.
If the events satisfy the criteria, publish an alert event.
"""
self.counter += 1
self.event_times.append(msg.ts_created)
if self.counter == self.max_count:
time_diff = self.event_times[self.max_count - 1] - self.event_times[0]
if time_diff <= self.time_window:
self.publish()
self.counter = 0
self.event_times = []
def publish(self):
# -------------------------------------------------------------------------------------
# publish an alert event
# -------------------------------------------------------------------------------------
self.event_publisher.publish_event(
event_type="DeviceEvent", origin="EventAlertTransform", description="An alert event being published."
)
def _acquire_sample(cls, config, publisher, unlock_new_data_callback, update_new_data_check_attachment):
"""
Ensures required keys (such as stream_id) are available from config, configures the publisher and then calls:
BaseDataHandler._constraints_for_new_request (only if config does not contain 'constraints')
BaseDataHandler._publish_data passing BaseDataHandler._get_data as a parameter
@param config Dict containing configuration parameters, may include constraints, formatters, etc
@param publisher the publisher used to publish data
@param unlock_new_data_callback BaseDataHandler callback function to allow conditional unlocking of the BaseDataHandler._semaphore
@param update_new_data_check_attachment classmethod to update the external dataset resources file list attachment
@throws InstrumentParameterException if the data constraints are not a dictionary
@retval None
"""
log.debug('start _acquire_sample: config={0}'.format(config))
cls._init_acquisition_cycle(config)
constraints = get_safe(config, 'constraints')
if not constraints:
gevent.getcurrent().link(unlock_new_data_callback)
try:
constraints = cls._constraints_for_new_request(config)
except NoNewDataWarning:
#log.info(nndw.message)
if get_safe(config, 'TESTING'):
#log.debug('Publish TestingFinished event')
pub = EventPublisher('DeviceCommonLifecycleEvent')
pub.publish_event(origin='BaseDataHandler._acquire_sample', description='TestingFinished')
return
if constraints is None:
raise InstrumentParameterException("Data constraints returned from _constraints_for_new_request cannot be None")
config['constraints'] = constraints
elif isinstance(constraints, dict):
addnl_constr = cls._constraints_for_historical_request(config)
if not addnl_constr is None and isinstance(addnl_constr, dict):
constraints.update(addnl_constr)
else:
raise InstrumentParameterException('Data constraints must be of type \'dict\': {0}'.format(constraints))
cls._publish_data(publisher, cls._get_data(config))
if 'set_new_data_check' in config:
update_new_data_check_attachment(config['external_dataset_res_id'], config['set_new_data_check'])
# Publish a 'TestFinished' event
if get_safe(config, 'TESTING'):
#log.debug('Publish TestingFinished event')
pub = EventPublisher('DeviceCommonLifecycleEvent')
pub.publish_event(origin='BaseDataHandler._acquire_sample', description='TestingFinished')
class SystemManagementService(BaseSystemManagementService):
""" container management requests are handled by the event listener
ion.processes.event.container_manager.ContainerManager
which must be running on each container.
"""
def on_start(self,*a,**b):
super(SystemManagementService,self).on_start(*a,**b)
self.sender = EventPublisher()
def on_quit(self,*a,**b):
self.sender.close()
def perform_action(self, predicate, action):
userid = None # get from context
self.sender.publish_event(event_type="ContainerManagementRequest", origin=userid, predicate=predicate, action=action)
def set_log_level(self, logger='', level='', recursive=False):
self.perform_action(ALL_CONTAINERS_INSTANCE, IonObject('ChangeLogLevel', logger=logger, level=level, recursive=recursive))
def test_greater_than_interval(self):
"""
test_greater_than_interval
Test interval alarm and alarm event publishing for a greater than
inteval.
"""
kwargs = {
"name": "current_warning_interval",
"stream_name": "fakestreamname",
"value_id": "port_current",
"message": "Current is above normal range.",
"type": StreamAlarmType.WARNING,
"lower_bound": 10.5,
"lower_rel_op": "<",
}
# Create alarm object.
alarm = IonObject("IntervalAlarmDef", **kwargs)
alarm = construct_alarm_expression(alarm)
# This sequence will produce 5 alarms:
# Warning on the first value,
# All clear on 30,
# Warning on 5.5
# All clear on 15.1
# Warning on 3.3
self._event_count = 5
test_vals = [5.5, 5.4, 5.5, 5.6, 30, 30.4, 5.5, 5.6, 15.1, 15.2, 15.3, 3.3, 3.4]
pub = EventPublisher(event_type="StreamAlarmEvent", node=self.container.node)
for x in test_vals:
event_data = None
eval_alarm(alarm, x)
if alarm.first_time == 1:
event_data = make_event_data(alarm)
elif alarm.first_time > 1:
if alarm.status != alarm.old_status:
event_data = make_event_data(alarm)
if event_data:
pub.publish_event(origin=self._resource_id, **event_data)
self._async_event_result.get(timeout=30)
"""
def test_less_than_interval(self):
"""
test_less_than_interval
Test interval alarm and alarm event publishing for a less than
inteval.
"""
kwargs = {
"name": "reserve_power_warning",
"stream_name": "fakestreamname",
"value_id": "battery_level",
"message": "Battery is below normal range.",
"type": StreamAlarmType.WARNING,
"upper_bound": 4.0,
"upper_rel_op": "<",
}
# Create alarm object.
alarm = IonObject("IntervalAlarmDef", **kwargs)
alarm = construct_alarm_expression(alarm)
# This sequence will produce 5 alarms:
# 5.5 warning
# 3.3 all clear
# 4.5 warning
# 3.3 all clear
# 4.8 warning
self._event_count = 5
test_vals = [5.5, 5.5, 5.4, 4.6, 4.5, 3.3, 3.3, 4.5, 4.5, 3.3, 3.3, 4.8]
pub = EventPublisher(event_type="StreamAlarmEvent", node=self.container.node)
for x in test_vals:
event_data = None
eval_alarm(alarm, x)
if alarm.first_time == 1:
event_data = make_event_data(alarm)
elif alarm.first_time > 1:
if alarm.status != alarm.old_status:
event_data = make_event_data(alarm)
if event_data:
pub.publish_event(origin=self._resource_id, **event_data)
self._async_event_result.get(timeout=30)
"""
def test_pub_and_sub(self):
ar = event.AsyncResult()
gq = queue.Queue()
self.count = 0
def cb(*args, **kwargs):
self.count += 1
gq.put(args[0])
if self.count == 2:
ar.set()
sub = EventSubscriber(event_type="ResourceEvent", callback=cb, origin="specific")
pub = EventPublisher(event_type="ResourceEvent")
self._listen(sub)
pub.publish_event(origin="specific", description="hello")
event_obj = bootstrap.IonObject('ResourceEvent', origin='specific', description='more testing')
self.assertEqual(event_obj, pub.publish_event_object(event_obj))
with self.assertRaises(BadRequest) as cm:
event_obj = bootstrap.IonObject('ResourceEvent', origin='specific', description='more testing', ts_created='2423')
pub.publish_event_object(event_obj)
self.assertIn( 'The ts_created value is not a valid timestamp',cm.exception.message)
with self.assertRaises(BadRequest) as cm:
event_obj = bootstrap.IonObject('ResourceEvent', origin='specific', description='more testing', ts_created='1000494978462')
pub.publish_event_object(event_obj)
self.assertIn( 'This ts_created value is too old',cm.exception.message)
with self.assertRaises(BadRequest) as cm:
event_obj = bootstrap.IonObject('ResourceEvent', origin='specific', description='more testing')
event_obj._id = '343434'
pub.publish_event_object(event_obj)
self.assertIn( 'The event object cannot contain a _id field',cm.exception.message)
ar.get(timeout=5)
res = []
for x in xrange(self.count):
res.append(gq.get(timeout=5))
self.assertEquals(len(res), self.count)
self.assertEquals(res[0].description, "hello")
self.assertAlmostEquals(int(res[0].ts_created), int(get_ion_ts()), delta=5000)
self.assertEquals(res[1].description, "more testing")
self.assertAlmostEquals(int(res[1].ts_created), int(get_ion_ts()), delta=5000)
def test_event_triggered_transform_B(self):
'''
Test that packets are processed by the event triggered transform
'''
#---------------------------------------------------------------------------------------------
# Launch a ctd transform
#---------------------------------------------------------------------------------------------
# Create the process definition
process_definition = ProcessDefinition(
name='EventTriggeredTransform_B',
description='For testing EventTriggeredTransform_B')
process_definition.executable[
'module'] = 'ion.processes.data.transforms.event_triggered_transform'
process_definition.executable['class'] = 'EventTriggeredTransform_B'
event_transform_proc_def_id = self.process_dispatcher.create_process_definition(
process_definition=process_definition)
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub.create_stream_definition(
'stream_def', parameter_dictionary_id=pdict_id)
stream_id, _ = self.pubsub.create_stream(
'test_stream',
exchange_point='science_data',
stream_definition_id=stream_def_id)
# Build the config
config = DotDict()
config.process.queue_name = self.exchange_name
config.process.exchange_point = self.exchange_point
config.process.publish_streams.output = stream_id
config.process.event_type = 'ResourceLifecycleEvent'
config.process.stream_id = stream_id
# Schedule the process
self.process_dispatcher.schedule_process(
process_definition_id=event_transform_proc_def_id,
configuration=config)
#---------------------------------------------------------------------------------------------
# Publish an event to wake up the event triggered transform
#---------------------------------------------------------------------------------------------
event_publisher = EventPublisher("ResourceLifecycleEvent")
event_publisher.publish_event(origin='fake_origin')
def generate_event(self, msg):
'''
Publish an event
'''
log.info('Detected an event')
event_publisher = EventPublisher("DetectionEvent")
message = str(self.notification._res_obj.delivery_config.processing['search_string'])
#@David What should the origin and origin type be for Detection Events
event_publisher.publish_event(origin='DetectionEventProcessor',
message=msg,
description="Event was detected by DetectionEventProcessor",
condition = message, # Concatenate the filter and make it a message
original_origin = self.notification._res_obj.origin,
original_type = self.notification._res_obj.origin_type)
class StreamAlertTransform(TransformStreamListener):
def on_start(self):
self.queue_name = self.CFG.get_safe("process.queue_name", self.id)
self.value = self.CFG.get_safe("process.value", 0)
self.subscriber = SimpleStreamSubscriber.new_subscriber(self.container, self.queue_name, self.recv_packet)
self.subscriber.start()
# Create the publisher that will publish the Alert message
self.event_publisher = EventPublisher()
def recv_packet(self, msg, headers):
"""
The callback method.
If the events satisfy the criteria, publish an alert event.
"""
value = self._extract_parameters_from_stream(msg, "VALUE")
if msg.find("PUBLISH") > -1 and (value < self.value):
self.publish()
def publish(self):
"""
Publish an alert event
"""
self.event_publisher.publish_event(
event_type="DeviceEvent", origin="StreamAlertTransform", description="An alert event being published."
)
def _extract_parameters_from_stream(self, msg, field):
tokens = msg.split(" ")
try:
for token in tokens:
token = token.strip()
if token == "=":
i = tokens.index(token)
if tokens[i - 1] == field:
return int(tokens[i + 1].strip())
except IndexError:
log.warning("Could not extract value from the message. Please check its format.")
return self.value
class SystemManagementService(BaseSystemManagementService):
""" container management requests are handled by the event listener
ion.processes.event.container_manager.ContainerManager
which must be running on each container.
"""
def on_start(self, *a, **b):
super(SystemManagementService, self).on_start(*a, **b)
self.sender = EventPublisher()
def on_quit(self, *a, **b):
self.sender.close()
def perform_action(self, predicate, action):
userid = None # get from context
self.sender.publish_event(event_type=OT.ContainerManagementRequest,
origin=userid,
predicate=predicate,
action=action)
def set_log_level(self, logger='', level='', recursive=False):
self.perform_action(
ALL_CONTAINERS_INSTANCE,
IonObject(OT.ChangeLogLevel,
logger=logger,
level=level,
recursive=recursive))
def reset_policy_cache(self, headers=None, timeout=None):
"""Clears and reloads the policy caches in all of the containers.
@throws BadRequest None
"""
self.perform_action(ALL_CONTAINERS_INSTANCE,
IonObject(OT.ResetPolicyCache))
def trigger_garbage_collection(self):
"""Triggers a garbage collection in all containers
@throws BadRequest None
"""
self.perform_action(ALL_CONTAINERS_INSTANCE,
IonObject(OT.TriggerGarbageCollection))
def process_oms_event():
if not request.data:
log.warning('process_oms_event: invalid OMS event payload: %r',
request.data)
return gateway_json_response(OMS_BAD_REQUEST_RESPONSE)
payload = json_loads(str(request.data))
if not isinstance(payload, list):
log.warning(
'process_oms_event: invalid OMS event payload: '
'expecting array but got: %r', payload)
return gateway_json_response(OMS_BAD_REQUEST_RESPONSE)
log.debug('process_oms_event: payload=%s', payload)
event_publisher = EventPublisher()
for obj in payload:
for k in ['event_id', 'platform_id', 'message']:
if k not in obj:
log.warning(
'process_oms_event: invalid OMS event: %r missing. '
'Received object: %s', k, obj)
#return gateway_json_response(OMS_BAD_REQUEST_RESPONSE)
# note the the external event_id is captured in the sub_type field:
evt = dict(event_type='OMSDeviceStatusEvent',
origin_type='OMS Platform',
origin=obj.get('platform_id', 'platform_id NOT PROVIDED'),
sub_type=obj.get('event_id', 'event_id NOT PROVIDED'),
description=obj.get('message', ''),
status_details=obj)
try:
event_publisher.publish_event(**evt)
log.debug('process_oms_event: published: %s', evt)
except Exception as e:
log.exception('process_oms_event: could not publish OMS event: %s',
evt)
return gateway_json_response(OMS_ACCEPTED_RESPONSE)
def test_send_notification_emails(self):
# create user with email address in RR
user_identty_object = IonObject(RT.ActorIdentity, name="user1")
user_id = self.imc.create_actor_identity(user_identty_object)
user_info_object = IonObject(RT.UserInfo, {
"name": "user1_info",
"contact": {
"email": '*****@*****.**'
}
})
self.imc.create_user_info(user_id, user_info_object)
# create first notification
notification_object = IonObject(
RT.NotificationRequest, {
"name": "notification1",
"origin_list": ['Some_Resource_Agent_ID1'],
"events_list": ['ResourceLifecycleEvent']
})
self.unsc.create_notification(notification_object, user_id)
# create second notification
notification_object = IonObject(
RT.NotificationRequest, {
"name": "notification2",
"origin_list": ['Some_Resource_Agent_ID2'],
"events_list": ['DataEvent']
})
self.unsc.create_notification(notification_object, user_id)
# publish an event for each notification to generate the emails
# this can't be easily check in SW so need to check for these at the [email protected] account
rle_publisher = EventPublisher("ResourceLifecycleEvent")
rle_publisher.publish_event(origin='Some_Resource_Agent_ID1',
description="RLE test event")
de_publisher = EventPublisher("DataEvent")
de_publisher.publish_event(origin='Some_Resource_Agent_ID2',
description="DE test event")
gevent.sleep(1)
def test_subscriber_listening_for_specific_origin(self):
ar = event.AsyncResult()
self.count = 0
def cb(*args, **kwargs):
self.count += 1
ar.set(args[0])
sub = EventSubscriber(event_type="ResourceEvent", origin="specific", callback=cb)
pub = EventPublisher(event_type="ResourceEvent", node=self.container.node)
self._listen(sub)
pub.publish_event(origin="notspecific", description="1")
pub.publish_event(origin="notspecific", description="2")
pub.publish_event(origin="specific", description="3")
pub.publish_event(origin="notspecific", description="4")
evmsg = ar.get(timeout=5)
self.assertEquals(self.count, 1)
self.assertEquals(evmsg.description, "3")
class TestRemoteEndpoint(IonIntegrationTestCase):
"""
Test cases for 2CAA terrestrial endpoint.
"""
def setUp(self):
"""
Start fake terrestrial components and add cleanup.
Start terrestrial server and retrieve port.
Set internal variables.
Start container.
Start deployment.
Start container agent.
Spawn remote endpoint process.
Create remote endpoint client and retrieve remote server port.
Create event publisher.
"""
self._terrestrial_server = R3PCServer(self.consume_req, self.terrestrial_server_close)
self._terrestrial_client = R3PCClient(self.consume_ack, self.terrestrial_client_close)
self.addCleanup(self._terrestrial_server.stop)
self.addCleanup(self._terrestrial_client.stop)
self._other_port = self._terrestrial_server.start('*', 0)
log.debug('Terrestrial server binding to *:%i', self._other_port)
self._other_host = 'localhost'
self._platform_resource_id = 'abc123'
self._resource_id = 'fake_id'
self._no_requests = 10
self._requests_sent = {}
self._results_recv = {}
self._no_telem_events = 0
self._done_evt = AsyncResult()
self._done_telem_evts = AsyncResult()
self._cmd_tx_evt = AsyncResult()
# Start container.
log.debug('Staring capability container.')
self._start_container()
# Bring up services in a deploy file (no need to message).
log.info('Staring deploy services.')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Create a container client.
log.debug('Creating container client.')
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
# Create agent config.
endpoint_config = {
'other_host' : self._other_host,
'other_port' : self._other_port,
'this_port' : 0,
'platform_resource_id' : self._platform_resource_id
}
# Spawn the remote enpoint process.
log.debug('Spawning remote endpoint process.')
re_pid = container_client.spawn_process(
name='remote_endpoint_1',
module='ion.services.sa.tcaa.remote_endpoint',
cls='RemoteEndpoint',
config=endpoint_config)
log.debug('Endpoint pid=%s.', str(re_pid))
# Create an endpoint client.
self.re_client = RemoteEndpointClient(
process=FakeProcess(),
to_name=re_pid)
log.debug('Got re client %s.', str(self.re_client))
# Remember the remote port.
self._this_port = self.re_client.get_port()
log.debug('The remote port is: %i.', self._this_port)
# Start the event publisher.
self._event_publisher = EventPublisher()
######################################################################
# Helpers.
######################################################################
def on_link_up(self):
"""
Called by a test to simulate turning the link on.
"""
log.debug('Terrestrial client connecting to localhost:%i.',
self._this_port)
self._terrestrial_client.start('localhost', self._this_port)
# Publish a link up event to be caught by the endpoint.
log.debug('Publishing telemetry event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._platform_resource_id,
status = TelemetryStatusType.AVAILABLE)
def on_link_down(self):
"""
Called by a test to simulate turning the link off.
"""
self._terrestrial_client.stop()
# Publish a link down event to be caught by the endpoint.
log.debug('Publishing telemetry event.')
self._event_publisher.publish_event(
event_type='PlatformTelemetryEvent',
origin=self._platform_resource_id,
status = TelemetryStatusType.UNAVAILABLE)
def consume_req(self, res):
"""
Consume a terrestrial request setting async event when necessary.
"""
command_id = res['command_id']
self._results_recv[command_id] = res
if len(self._results_recv) == self._no_requests:
self._done_evt.set()
def consume_ack(self, cmd):
"""
Consume terrestrial ack setting async event when necessary.
"""
self._requests_sent[cmd.command_id] = cmd
if len(self._requests_sent) == self._no_requests:
self._cmd_tx_evt.set()
def terrestrial_server_close(self):
"""
Callback when terrestrial server closes.
"""
pass
def terrestrial_client_close(self):
"""
Callback when terrestrial client closes.
"""
pass
def make_fake_command(self, no):
"""
Build a fake command for use in tests.
"""
cmdstr = 'fake_cmd_%i' % no
cmd = IonObject('RemoteCommand',
resource_id=self._resource_id,
command=cmdstr,
args=['arg1', 23],
kwargs={'worktime':3},
command_id = str(uuid.uuid4()))
return cmd
def start_agent(self):
"""
Start an instrument agent and client.
"""
log.info('Creating driver integration test support:')
log.info('driver module: %s', DRV_MOD)
log.info('driver class: %s', DRV_CLS)
log.info('device address: %s', DEV_ADDR)
log.info('device port: %s', DEV_PORT)
log.info('log delimiter: %s', DELIM)
log.info('work dir: %s', WORK_DIR)
self._support = DriverIntegrationTestSupport(DRV_MOD,
DRV_CLS,
DEV_ADDR,
DEV_PORT,
DATA_PORT,
CMD_PORT,
PA_BINARY,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
port = self._support.start_pagent()
log.info('Port agent started at port %i',port)
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {
'addr' : 'localhost',
'port' : port,
'cmd_port' : CMD_PORT
}
self.addCleanup(self._support.stop_pagent)
# Create agent config.
agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : {},
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True
}
# Start instrument agent.
log.debug("Starting IA.")
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
######################################################################
# Tests.
######################################################################
def test_process_queued(self):
"""
test_process_queued
Test that queued commands are forwarded to and handled by
remote endpoint when link comes up.
"""
# Create and enqueue some requests.
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
self._terrestrial_client.enqueue(cmd)
# Publish a telemetry available event.
# This will cause the endpoint clients to wake up and connect.
self.on_link_up()
# Wait for all the enqueued commands to be acked.
# Wait for all the responses to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish a telemetry unavailable event.
# This will cause the endpoint clients to disconnect and go to sleep.
self.on_link_down()
# Confirm the results match the commands sent.
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_process_online(self):
"""
test_process_online
Test commands are forwarded and handled while link is up.
"""
# Publish a telemetry available event.
# This will cause the endpoint clients to wake up and connect.
self.on_link_up()
# Wait for the link to be up.
# The remote side does not publish public telemetry events
# so we can't wait for that.
gevent.sleep(1)
# Create and enqueue some requests.
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
self._terrestrial_client.enqueue(cmd)
# Wait for all the enqueued commands to be acked.
# Wait for all the responses to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish a telemetry unavailable event.
# This will cause the endpoint clients to disconnect and go to sleep.
self.on_link_down()
# Confirm the results match the commands sent.
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_terrestrial_late(self):
"""
test_terrestrial_late
Test queued commands are forwarded and handled by remote endpoint
when terrestrial side is late to come up.
"""
# Publish a telemetry available event.
# This will cause the endpoint clients to wake up and connect.
self.on_link_up()
# Wait for the link to be up.
# The remote side does not publish public telemetry events
# so we can't wait for that.
gevent.sleep(1)
# Manually stop the terrestrial endpoint.
# This will cause it to be unavailable when commands are queued
# to simulate stability during asynchronous wake ups.
self._terrestrial_server.stop()
self._terrestrial_client.stop()
# Create and enqueue some requests.
for i in range(self._no_requests):
cmd = self.make_fake_command(i)
self._terrestrial_client.enqueue(cmd)
# Remote side awaits the terrestrial waking up.
gevent.sleep(3)
# Terrestrail endpoint eventually wakes up and starts transmitting.
self._terrestrial_client.start('localhost', self._this_port)
self._terrestrial_server.start('*', self._other_port)
# Wait for all the enqueued commands to be acked.
# Wait for all the responses to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Publish a telemetry unavailable event.
# This will cause the endpoint clients to disconnect and go to sleep.
self.on_link_down()
# Confirm the results match the commands sent.
self.assertItemsEqual(self._requests_sent.keys(),
self._results_recv.keys())
def test_service_commands(self):
"""
test_service_commands
Test that real service commands are handled by the remote endpoint.
"""
# Publish a telemetry available event.
# This will cause the endpoint clients to wake up and connect.
self.on_link_up()
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Create user object.
obj = IonObject("UserInfo", name="some_name")
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='create',
args=[obj],
kwargs='',
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns obj_id, obj_rev.
obj_id, obj_rev = self._results_recv[cmd.command_id]['result']
# Confirm the results are valid.
"""
Result is a tuple of strings.
{'result': ['ad183ff26bae4f329ddd85fd69d160a9',
'1-00a308c45fff459c7cda1db9a7314de6'],
'command_id': 'cc2ae00d-40b0-47d2-af61-8ffb87f1aca2'}
"""
self.assertIsInstance(obj_id, str)
self.assertNotEqual(obj_id, '')
self.assertIsInstance(obj_rev, str)
self.assertNotEqual(obj_rev, '')
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Read user object.
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='read',
args=[obj_id],
kwargs='',
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns read_obj.
read_obj = self._results_recv[cmd.command_id]['result']
# Confirm the results are valid.
"""
Result is a user info object with the name set.
{'lcstate': 'DEPLOYED_AVAILABLE',
'_rev': '1-851f067bac3c34b2238c0188b3340d0f',
'description': '',
'ts_updated': '1349213207638',
'type_': 'UserInfo',
'contact': <interface.objects.ContactInformation object at 0x10d7df590>,
'_id': '27832d93f4cd4535a75ac75c06e00a7e',
'ts_created': '1349213207638',
'variables': [{'name': '', 'value': ''}],
'name': 'some_name'}
"""
self.assertIsInstance(read_obj, UserInfo)
self.assertEquals(read_obj.name, 'some_name')
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Update user object.
read_obj.name = 'some_other_name'
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='update',
args=[read_obj],
kwargs='',
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns nothing.
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Read user object.
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='read',
args=[obj_id],
kwargs='',
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns read_obj.
read_obj = self._results_recv[cmd.command_id]['result']
self.assertIsInstance(read_obj, UserInfo)
self.assertEquals(read_obj.name, 'some_other_name')
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Delete user object.
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='delete',
args=[obj_id],
kwargs='',
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns nothing.
# Publish a telemetry unavailable event.
# This will cause the endpoint clients to disconnect and go to sleep.
self.on_link_down()
gevent.sleep(1)
def test_resource_commands(self):
"""
test_resource_commands
Test that real resource commands are handled by the remote endpoint.
"""
# Start the IA and check it's out there and behaving.
self.start_agent()
state = self._ia_client.get_agent_state()
log.debug('Agent state is: %s', state)
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
retval = self._ia_client.ping_agent()
log.debug('Agent ping is: %s', str(retval))
self.assertIn('ping from InstrumentAgent', retval)
# Publish a telemetry available event.
# This will cause the endpoint clients to wake up and connect.
self.on_link_up()
# Wait for the link to be up.
# The remote side does not publish public telemetry events
# so we can't wait for that.
gevent.sleep(1)
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Get agent state via remote endpoint.
cmd = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='get_agent_state',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns agent state.
state = self._results_recv[cmd.command_id]['result']
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Ping agent via remote endpoint.
cmd = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='ping_agent',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns agent state.
ping = self._results_recv[cmd.command_id]['result']
self.assertIn('ping from InstrumentAgent', ping)
# Publish a telemetry unavailable event.
# This will cause the endpoint clients to disconnect and go to sleep.
self.on_link_down()
gevent.sleep(1)
def test_bad_service_name_resource_id(self):
"""
test_bad_service_name_resource_id
Test for proper exception behavior when a bad service name or
resource id is used in a command forwarded to the remote endpoint.
"""
# Publish a telemetry available event.
# This will cause the endpoint clients to wake up and connect.
self.on_link_up()
# Wait for the link to be up.
# The remote side does not publish public telemetry events
# so we can't wait for that.
gevent.sleep(1)
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Create user object.
obj = IonObject("UserInfo", name="some_name")
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='bogus_service',
command='create',
args=[obj],
kwargs='',
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns NotFound.
result = self._results_recv[cmd.command_id]['result']
self.assertIsInstance(result, NotFound)
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Get agent state via remote endpoint.
cmd = IonObject('RemoteCommand',
resource_id='bogus_resource_id',
svc_name='',
command='get_agent_state',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns NotFound.
result = self._results_recv[cmd.command_id]['result']
self.assertIsInstance(result, NotFound)
# Publish a telemetry unavailable event.
# This will cause the endpoint clients to disconnect and go to sleep.
self.on_link_down()
gevent.sleep(1)
def test_bad_commands(self):
"""
test_bad_commands
Test for correct exception behavior if a bad command name is forwarded
to a remote service or resource.
"""
# Start the IA and check it's out there and behaving.
self.start_agent()
state = self._ia_client.get_agent_state()
log.debug('Agent state is: %s', state)
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
retval = self._ia_client.ping_agent()
log.debug('Agent ping is: %s', str(retval))
self.assertIn('ping from InstrumentAgent', retval)
# Publish a telemetry available event.
# This will cause the endpoint clients to wake up and connect.
self.on_link_up()
# Wait for the link to be up.
# The remote side does not publish public telemetry events
# so we can't wait for that.
gevent.sleep(1)
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Create user object.
obj = IonObject("UserInfo", name="some_name")
cmd = IonObject('RemoteCommand',
resource_id='',
svc_name='resource_registry',
command='what_the_flunk',
args=[obj],
kwargs='',
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns BadRequest.
result = self._results_recv[cmd.command_id]['result']
self.assertIsInstance(result, BadRequest)
# Send commands one at a time.
# Reset queues and events.
self._no_requests = 1
self._done_evt = AsyncResult()
self._cmd_tx_evt = AsyncResult()
self._requests_sent = {}
self._results_recv = {}
# Get agent state via remote endpoint.
cmd = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='what_the_flunk',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd)
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Returns NotFound.
result = self._results_recv[cmd.command_id]['result']
self.assertIsInstance(result, BadRequest)
# Publish a telemetry unavailable event.
# This will cause the endpoint clients to disconnect and go to sleep.
self.on_link_down()
gevent.sleep(1)
def test_resource_command_sequence(self):
"""
test_resource_command_sequence
Test for successful completion of a properly ordered sequence of
resource commands queued for forwarding to the remote endpoint.
"""
# Start the IA and check it's out there and behaving.
self.start_agent()
state = self._ia_client.get_agent_state()
log.debug('Agent state is: %s', state)
self.assertEqual(state, ResourceAgentState.UNINITIALIZED)
retval = self._ia_client.ping_agent()
log.debug('Agent ping is: %s', str(retval))
self.assertIn('ping from InstrumentAgent', retval)
# We execute a sequence of twelve consecutive events.
self._no_requests = 12
# Get agent state.
cmd1 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='get_agent_state',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd1)
# Initialize agent.
cmd2 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='execute_agent',
args=[AgentCommand(command=ResourceAgentEvent.INITIALIZE)],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd2)
# Get agent state.
cmd3 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='get_agent_state',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd3)
# Go active.
cmd4 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='execute_agent',
args=[AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd4)
# Get agent state.
cmd5 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='get_agent_state',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd5)
# Run.
cmd6 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='execute_agent',
args=[AgentCommand(command=ResourceAgentEvent.RUN)],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd6)
# Get agent state.
cmd7 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='get_agent_state',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd7)
# Acquire sample.
cmd8 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='execute_resource',
args=[AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd8)
# Acquire sample
cmd9 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='execute_resource',
args=[AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd9)
# Acquire sample.
cmd10 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='execute_resource',
args=[AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd10)
# Reset.
cmd11 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='execute_agent',
args=[AgentCommand(command=ResourceAgentEvent.RESET)],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd11)
# Get agent state.
cmd12 = IonObject('RemoteCommand',
resource_id=IA_RESOURCE_ID,
svc_name='',
command='get_agent_state',
args=[],
kwargs={},
command_id = str(uuid.uuid4()))
self._terrestrial_client.enqueue(cmd12)
# Publish a telemetry available event.
# This will cause the endpoint clients to wake up and connect.
self.on_link_up()
# Wait for command request to be acked.
# Wait for response to arrive.
self._cmd_tx_evt.get(timeout=CFG.endpoint.receive.timeout)
self._done_evt.get(timeout=CFG.endpoint.receive.timeout)
# Check results of command sequence.
"""
0ccf1e10-eeca-400d-aefe-f9d6888ec963 {'result': 'RESOURCE_AGENT_STATE_INACTIVE', 'command_id': '0ccf1e10-eeca-400d-aefe-f9d6888ec963'}
92531bdf-c2c8-4aa8-817d-5107c7311b37 {'result': <interface.objects.AgentCommandResult object at 0x10d7f11d0>, 'command_id': '92531bdf-c2c8-4aa8-817d-5107c7311b37'}
509934a1-5038-40d8-8014-591e2d8042b6 {'result': 'RESOURCE_AGENT_STATE_COMMAND', 'command_id': '509934a1-5038-40d8-8014-591e2d8042b6'}
88bacbb7-5366-4d27-9ecf-fff2bec34b2c {'result': <interface.objects.AgentCommandResult object at 0x10d389190>, 'command_id': '88bacbb7-5366-4d27-9ecf-fff2bec34b2c'}
f8b4d3fa-a249-439b-8bd4-ac212b6100aa {'result': <interface.objects.AgentCommandResult object at 0x10d3893d0>, 'command_id': 'f8b4d3fa-a249-439b-8bd4-ac212b6100aa'}
8ae98e39-fdb3-4218-ad8f-584620397d9f {'result': <interface.objects.AgentCommandResult object at 0x10d739990>, 'command_id': '8ae98e39-fdb3-4218-ad8f-584620397d9f'}
746364a1-c4c7-400f-96d4-ee36df5dc1a4 {'result': BadRequest('Execute argument "command" not set.',), 'command_id': '746364a1-c4c7-400f-96d4-ee36df5dc1a4'}
d516d3d9-e4f9-4ea5-80e0-34639a6377b5 {'result': <interface.objects.AgentCommandResult object at 0x10d3b2350>, 'command_id': 'd516d3d9-e4f9-4ea5-80e0-34639a6377b5'}
c7da03f5-59bc-420a-9e10-0a7794266599 {'result': 'RESOURCE_AGENT_STATE_IDLE', 'command_id': 'c7da03f5-59bc-420a-9e10-0a7794266599'}
678d870a-bf18-424a-afb0-f80ecf3277e2 {'result': <interface.objects.AgentCommandResult object at 0x10d739590>, 'command_id': '678d870a-bf18-424a-afb0-f80ecf3277e2'}
750c6a30-56eb-4535-99c2-a81fefab1b1f {'result': 'RESOURCE_AGENT_STATE_COMMAND', 'command_id': '750c6a30-56eb-4535-99c2-a81fefab1b1f'}
c17bd658-3775-4aa3-8844-02df70a0e3c0 {'result': 'RESOURCE_AGENT_STATE_UNINITIALIZED', 'command_id': 'c17bd658-3775-4aa3-8844-02df70a0e3c0'}
"""
# First result is a state string.
result1 = self._results_recv[cmd1.command_id]['result']
self.assertEqual(result1, ResourceAgentState.UNINITIALIZED)
# Second result is an empty AgentCommandResult.
result2 = self._results_recv[cmd2.command_id]['result']
# Third result is a state string.
result3 = self._results_recv[cmd3.command_id]['result']
self.assertEqual(result3, ResourceAgentState.INACTIVE)
# Fourth result is an empty AgentCommandResult.
result4 = self._results_recv[cmd4.command_id]['result']
# Fifth result is a state string.
result5 = self._results_recv[cmd5.command_id]['result']
self.assertEqual(result5, ResourceAgentState.IDLE)
# Sixth result is an empty AgentCommandResult.
result6 = self._results_recv[cmd6.command_id]['result']
# Seventh result is a state string.
result7 = self._results_recv[cmd7.command_id]['result']
self.assertEqual(result7, ResourceAgentState.COMMAND)
"""
{'raw': {'quality_flag': 'ok', 'preferred_timestamp': 'driver_timestamp',
'stream_name': 'raw', 'pkt_format_id': 'JSON_Data',
'pkt_version': 1, '
values': [{'binary': True, 'value_id': 'raw',
'value': 'NzkuNDM3MywxNy4yMDU2NCwgNzYxLjg4NSwgICA2LjIxOTgsIDE1MDYuMzk3LCAwMSBGZWIgMjAwMSwgMDE6MDE6MDA='}],
'driver_timestamp': 3558286748.8039923},
'parsed': {'quality_flag': 'ok', 'preferred_timestamp': 'driver_timestamp',
'stream_name': 'parsed', 'pkt_format_id': 'JSON_Data', 'pkt_version': 1,
'values': [{'value_id': 'temp', 'value': 79.4373},
{'value_id': 'conductivity', 'value': 17.20564},
{'value_id': 'pressure', 'value': 761.885}],
'driver_timestamp': 3558286748.8039923}}
"""
# Eigth result is an AgentCommandResult containing a sample.
result8 = self._results_recv[cmd8.command_id]['result']
self.assertTrue('parsed',result8.result )
# Ninth result is an AgentCommandResult containing a sample.
result9 = self._results_recv[cmd9.command_id]['result']
self.assertTrue('parsed',result9.result )
# Tenth result is an AgentCommandResult containing a sample.
result10 = self._results_recv[cmd10.command_id]['result']
self.assertTrue('parsed',result10.result )
# Eleventh result is an empty AgentCommandResult.
result11 = self._results_recv[cmd11.command_id]['result']
# Twelth result is a state string.
result12 = self._results_recv[cmd12.command_id]['result']
self.assertEqual(result1, ResourceAgentState.UNINITIALIZED)
# Publish a telemetry unavailable event.
# This will cause the endpoint clients to disconnect and go to sleep.
self.on_link_down()
gevent.sleep(1)
class NotificationSentScanner(object):
def __init__(self, container=None):
self.container = container or bootstrap.container_instance
self.object_store = self.container.object_store
self.resource_registry = self.container.resource_registry
self.event_publisher = EventPublisher()
# next_midnight is used to flush the counts (see NOTE in method)
self.next_midnight = self._midnight(days=1)
self.persist_interval = 300 # interval in seconds to persist/reload counts TODO: use CFG
self.time_last_persist = 0
# initalize volatile counts (memory only, should be routinely persisted)
self._initialize_counts()
def process_events(self, event_list):
notifications = set(
) # set() of notifications to disable, _disable_notifications can happen >1 depending on len(event_list)
for e in event_list:
# skip if not a NotificationEvent
if e.type_ not in NOTIFICATION_EVENTS:
continue
user_id = e.user_id
notification_id = e.notification_id
notification_max = e.notification_max # default value is zero indicating no max
# initialize user_id if necessary
if user_id not in self.counts:
self.counts[user_id] = Counter()
# increment counts (user_id key to allow ALL to be counted)
self.counts[user_id][
'all'] += 1 # tracks total notifications by user
self.counts[user_id][notification_id] += 1
self.counts_updated_since_persist = True
# disable notification if notification_max reached
if notification_max:
if self.counts[user_id][notification_id] >= notification_max:
# TODO this could be dict so key could be checked for insertion
notifications.add(
self._disable_notification(notification_id))
# update notifications that have been disabled
if notifications:
self._update_notifications(notifications)
# only attempt to persist counts if there was an update
if self.counts_updated_since_persist:
if time.time() > (self.time_last_persist + self.persist_interval):
self._persist_counts()
# reset counts if reset_interval has elapsed
if time.time() > self.next_midnight:
self._reset_counts()
# NOTE: ObjectStore 'objects' contain '_id' and '_rev'
def _initialize_counts(self):
""" initialize the volatile (memory only) counts from ObjectStore if available """
try:
self.counts_obj = self.object_store.read_doc('notification_counts')
# persisted as standard dicts, convert to Counter objects ignoring the ObjectStore '_id' and '_rev'
self.counts = {
k: Counter(v)
for k, v in self.counts_obj.items()
if not (k == '_id' or k == '_rev')
}
except NotFound:
self.counts = {}
self._persist_counts()
def _persist_counts(self):
""" persist the counts to ObjectStore """
try:
self.counts_obj = self.object_store.read_doc('notification_counts')
except NotFound:
self.object_store.create_doc({}, 'notification_counts')
self.counts_obj = self.object_store.read_doc('notification_counts')
# Counter objects cannot be persisted, convert to standard dicts (leaves '_id', '_rev' untouched)
self.counts_obj.update({k: dict(v) for k, v in self.counts.items()})
self.object_store.update_doc(self.counts_obj)
self.time_last_persist = time.time()
self.counts_updated_since_persist = False # boolean to check if counts should be persisted
def _reset_counts(self):
""" clears the persisted counts """
self.object_store.delete_doc('notification_counts')
self._initialize_counts(
) # NOTE: NotificationRequest boolean disabled_by_system reset by UNS
self.next_midnight = self._midnight(days=1)
def _disable_notification(self, notification_id):
""" set the disabled_by_system boolean to True """
notification = self.resource_registry.read(notification_id)
notification.disabled_by_system = True
return notification
def _update_notifications(self, notifications):
""" updates notifications then publishes ReloadUserInfoEvent """
for n in notifications:
self.resource_registry.update(n)
self.event_publisher.publish_event(event_type=OT.ReloadUserInfoEvent)
def _midnight(self, days=0):
""" NOTE: this is midnight PDT (+0700) """
dt = datetime.combine(date.today(), datetime.min.time()) + timedelta(
days=days, hours=7)
return (dt - datetime.utcfromtimestamp(0)).total_seconds()
class Container(BaseContainerAgent):
"""
The Capability Container. Its purpose is to spawn/monitor processes and services
that do the bulk of the work in the ION system. It also manages connections to the Exchange
and the various forms of datastores in the systems.
"""
# Singleton static variables
#node = None
id = None
name = None
pidfile = None
instance = None
def __init__(self, *args, **kwargs):
BaseContainerAgent.__init__(self, *args, **kwargs)
self._is_started = False
self._capabilities = []
# set container id and cc_agent name (as they are set in base class call)
self.id = get_default_container_id()
self.name = "cc_agent_%s" % self.id
Container.instance = self
from pyon.core import bootstrap
bootstrap.container_instance = self
log.debug("Container (sysname=%s) initializing ..." % bootstrap.get_sys_name())
# DatastoreManager - controls access to Datastores (both mock and couch backed)
self.datastore_manager = DatastoreManager()
self.datastore_manager.start()
self._capabilities.append("DATASTORE_MANAGER")
# Keep track of the overrides from the command-line, so they can trump app/rel file data
self.spawn_args = kwargs
# Instantiate Directory and self-register
# Has the additional side effect of either
# bootstrapping the configuration into the
# directory or read the configuration based
# in the value of the auto_bootstrap setting
self.directory = Directory()
# Create this Container's specific ExchangeManager instance
self.ex_manager = ExchangeManager(self)
# Create this Container's specific ProcManager instance
self.proc_manager = ProcManager(self)
# Create this Container's specific AppManager instance
self.app_manager = AppManager(self)
# File System - Interface to the OS File System, using correct path names and setups
self.file_system = FileSystem(CFG)
# Governance Controller - manages the governance related interceptors
self.governance_controller = GovernanceController(self)
# sFlow manager - controls sFlow stat emission
self.sflow_manager = SFlowManager(self)
# Coordinates the container start
self._status = "INIT"
# protection for when the container itself is used as a Process for clients
self.container = self
log.debug("Container initialized, OK.")
def start(self):
log.debug("Container starting...")
if self._is_started:
raise ContainerError("Container already started")
# Check if this UNIX process already runs a Container.
self.pidfile = "cc-pid-%d" % os.getpid()
if os.path.exists(self.pidfile):
raise ContainerError("Container.on_start(): Container is a singleton per UNIX process. Existing pid file found: %s" % self.pidfile)
# write out a PID file containing our agent messaging name
with open(self.pidfile, 'w') as f:
pid_contents = {'messaging': dict(CFG.server.amqp),
'container-agent': self.name,
'container-xp': bootstrap.get_sys_name() }
f.write(msgpack.dumps(pid_contents))
atexit.register(self._cleanup_pid)
self._capabilities.append("PID_FILE")
# set up abnormal termination handler for this container
def handl(signum, frame):
try:
self._cleanup_pid() # cleanup the pidfile first
self.quit() # now try to quit - will not error on second cleanup pidfile call
finally:
signal.signal(signal.SIGTERM, self._normal_signal)
os.kill(os.getpid(), signal.SIGTERM)
self._normal_signal = signal.signal(signal.SIGTERM, handl)
self.datastore_manager.start()
self._capabilities.append("DATASTORE_MANAGER")
# Self-register with Directory
self.directory.register("/Containers", self.id, cc_agent=self.name)
self.directory.register("/Containers/%s" % self.id, "Processes")
self._capabilities.append("DIRECTORY")
# Event repository
self.event_repository = EventRepository()
self.event_pub = EventPublisher()
self._capabilities.append("EVENT_REPOSITORY")
# Local resource registry
self.resource_registry = ResourceRegistry()
self._capabilities.append("RESOURCE_REGISTRY")
# Persistent objects
self.datastore_manager.get_datastore("objects", DataStore.DS_PROFILE.OBJECTS)
# State repository
self.state_repository = StateRepository()
self._capabilities.append("STATE_REPOSITORY")
# Start ExchangeManager, which starts the node (broker connection)
self.ex_manager.start()
self._capabilities.append("EXCHANGE_MANAGER")
self.proc_manager.start()
self._capabilities.append("PROC_MANAGER")
self.app_manager.start()
self._capabilities.append("APP_MANAGER")
self.governance_controller.start()
self._capabilities.append("GOVERNANCE_CONTROLLER")
if CFG.container.get('sflow', {}).get('enabled', False):
self.sflow_manager.start()
self._capabilities.append("SFLOW_MANAGER")
# Start the CC-Agent API
rsvc = ProcessRPCServer(node=self.node, from_name=self.name, service=self, process=self)
# Start an ION process with the right kind of endpoint factory
proc = self.proc_manager.proc_sup.spawn(name=self.name, listeners=[rsvc], service=self)
self.proc_manager.proc_sup.ensure_ready(proc)
self._capabilities.append("CONTAINER_AGENT")
self.event_pub.publish_event(event_type="ContainerLifecycleEvent",
origin=self.id, origin_type="CapabilityContainer",
sub_type="START",
state=ContainerStateEnum.START)
self._is_started = True
self._status = "RUNNING"
log.info("Container started, OK.")
@property
def node(self):
"""
Returns the active/default Node that should be used for most communication in the system.
Defers to exchange manager, but only if it has been started, otherwise returns None.
"""
if "EXCHANGE_MANAGER" in self._capabilities:
return self.ex_manager.default_node
return None
@contextmanager
def _push_status(self, new_status):
"""
Temporarily sets the internal status flag.
Use this as a decorator or in a with-statement before calling a temporary status changing
method, like start_rel_from_url.
"""
curstatus = self._status
self._status = new_status
try:
yield
finally:
self._status = curstatus
def serve_forever(self):
""" Run the container until killed. """
log.debug("In Container.serve_forever")
if not self.proc_manager.proc_sup.running:
self.start()
# serve forever short-circuits if immediate is on and children len is ok
num_procs = len(self.proc_manager.proc_sup.children)
immediate = CFG.system.get('immediate', False)
if not (immediate and num_procs == 1): # only spawned greenlet is the CC-Agent
# print a warning just in case
if immediate and num_procs != 1:
log.warn("CFG.system.immediate=True but number of spawned processes is not 1 (%d)", num_procs)
try:
# This just waits in this Greenlet for all child processes to complete,
# which is triggered somewhere else.
self.proc_manager.proc_sup.join_children()
except (KeyboardInterrupt, SystemExit) as ex:
log.info('Received a kill signal, shutting down the container.')
watch_parent = CFG.system.get('watch_parent', None)
if watch_parent:
watch_parent.kill()
except:
log.exception('Unhandled error! Forcing container shutdown')
else:
log.debug("Container.serve_forever short-circuiting due to CFG.system.immediate")
self.proc_manager.proc_sup.shutdown(CFG.cc.timeout.shutdown)
def status(self):
"""
Returns the internal status.
"""
return self._status
def _cleanup_pid(self):
if self.pidfile:
log.debug("Cleanup pidfile: %s", self.pidfile)
try:
os.remove(self.pidfile)
except Exception, e:
log.warn("Pidfile could not be deleted: %s" % str(e))
self.pidfile = None
class IngestionWorker(TransformDataProcess):
"""
Instances of this class acts as Ingestion Workers. They receive packets and send them to couchdb datastore or
hdf storage according to the policy in the data stream or the default policy of the ingestion configuration
"""
def dataset_configs_event_test_hook(self, msg, headers):
pass
def ingest_process_test_hook(self, msg, headers):
pass
def on_init(self):
self.event_pub = EventPublisher()
def on_start(self):
super(IngestionWorker, self).on_start()
#----------------------------------------------
# Start up couch
#----------------------------------------------
self.couch_config = self.CFG.get('couch_storage')
self.hdf_storage = self.CFG.get('hdf_storage')
self.number_of_workers = self.CFG.get('number_of_workers')
self.description = self.CFG.get('description')
self.ingest_config_id = self.CFG.get('configuration_id')
self.datastore_name = self.couch_config.get('datastore_name',
None) or 'dm_datastore'
try:
self.datastore_profile = getattr(
DataStore.DS_PROFILE,
self.couch_config.get('datastore_profile', 'SCIDATA'))
except AttributeError:
log.exception(
'Invalid datastore profile passed to ingestion worker. Defaulting to SCIDATA'
)
self.datastore_profile = DataStore.DS_PROFILE.SCIDATA
log.debug('datastore_profile %s' % self.datastore_profile)
self.db = self.container.datastore_manager.get_datastore(
ds_name=self.datastore_name,
profile=self.datastore_profile,
config=self.CFG)
self.resource_reg_client = ResourceRegistryServiceClient(
node=self.container.node)
self.dataset_configs = {}
# update the policy
def receive_dataset_config_event(event_msg, headers):
log.info('Updating dataset config in ingestion worker: %s',
event_msg)
if event_msg.type != DatasetIngestionTypeEnum.DATASETINGESTIONBYSTREAM:
raise IngestionWorkerException(
'Received invalid type in dataset config event.')
stream_id = event_msg.configuration.stream_id
if event_msg.deleted:
try:
del self.dataset_configs[stream_id]
except KeyError:
log.info(
'Tried to remove dataset config that does not exist!')
else:
self.dataset_configs[stream_id] = event_msg
# Hook to override just before processing is complete
self.dataset_configs_event_test_hook(event_msg, headers)
#Start the event subscriber - really - what a mess!
self.event_subscriber = EventSubscriber(
event_type="DatasetIngestionConfigurationEvent",
origin=self.ingest_config_id,
callback=receive_dataset_config_event)
self.gl = spawn(self.event_subscriber.listen)
self.event_subscriber._ready_event.wait(timeout=5)
log.info(str(self.db))
def process(self, packet):
"""Process incoming data!!!!
"""
if isinstance(packet, (Granule, CompoundGranule)):
log.info('Received new granule, but ingestion doesnt work yet!')
return
# Ignoring any packet that is not a stream granule!
if not isinstance(packet, StreamGranuleContainer):
raise IngestionWorkerException(
'Received invalid message type: "%s"', type(packet))
# Get the dataset config for this stream
dset_config = self.get_dataset_config(packet)
# Process the packet
ingest_attributes = self.process_stream(packet, dset_config)
#@todo - get this data from the dataset config...
if dset_config:
dataset_id = dset_config.dataset_id
stream_id = dset_config.stream_id
self.event_pub.publish_event(event_type="GranuleIngestedEvent",
sub_type="DatasetIngest",
origin=dataset_id,
status=200,
ingest_attributes=ingest_attributes,
stream_id=stream_id)
headers = ''
# Hook to override just before processing is complete
self.ingest_process_test_hook(packet, headers)
def persist_immutable(self, obj):
"""
This method is not functional yet - the doc object is python specific. The sha1 must be of a language independent form.
"""
doc = self.db._ion_object_to_persistence_dict(obj)
sha1 = sha1hex(doc)
try:
self.db.create_doc(doc, object_id=sha1)
log.debug('Persisted document %s', type(obj))
except BadRequest:
# Deduplication in action!
#@TODO why are we getting so many duplicate comments?
log.exception('Failed to write packet!\n%s' % obj)
# Do the id or revision have a purpose? do we need a return value?
def process_stream(self, packet, dset_config):
"""
Accepts a stream. Also accepts instruction (a dset_config). According to the received dset_config it processes the
stream such as store in hfd_storage, couch_storage.
@param: packet The incoming data stream of type stream.
@param: dset_config The dset_config telling this method what to do with the incoming data stream.
"""
ingestion_attributes = {
'variables': [],
'number_of_records': -1,
'updated_metadata': False,
'updated_data': False
}
if dset_config is None:
log.info('No dataset config for this stream!')
return
values_string = ''
sha1 = ''
encoding_type = ''
for key, value in packet.identifiables.iteritems():
if isinstance(value, DataStream):
values_string = value.values
value.values = ''
elif isinstance(value, Encoding):
sha1 = value.sha1
encoding_type = value.encoding_type
elif isinstance(value, Coverage):
ingestion_attributes['variables'].append(key)
elif isinstance(value, CountElement):
ingestion_attributes['number_of_records'] = value.value
if dset_config.archive_metadata is True:
log.debug("Persisting data....")
ingestion_attributes['updated_metadata'] = True
self.persist_immutable(packet)
if dset_config.archive_data is True:
#@todo - grab the filepath to save the hdf string somewhere..
ingestion_attributes['updated_data'] = True
if values_string:
calculated_sha1 = hashlib.sha1(
values_string).hexdigest().upper()
filename = FileSystem.get_hierarchical_url(
FS.CACHE, calculated_sha1, ".%s" % encoding_type)
if sha1 != calculated_sha1:
raise IngestionWorkerException(
'The sha1 stored is different than the calculated from the received hdf_string'
)
#log.warn('writing to filename: %s' % filename)
with open(filename, mode='wb') as f:
f.write(values_string)
f.close()
else:
log.warn("Nothing to write!")
return ingestion_attributes
def on_stop(self):
TransformDataProcess.on_stop(self)
# close event subscriber safely
self.event_subscriber.close()
self.gl.join(timeout=5)
self.gl.kill()
self.db.close()
def on_quit(self):
TransformDataProcess.on_quit(self)
# close event subscriber safely
self.event_subscriber.close()
self.gl.join(timeout=5)
self.gl.kill()
self.db.close()
def get_dataset_config(self, incoming_packet):
"""
Gets the dset_config for the data stream
"""
try:
stream_id = incoming_packet.stream_resource_id
except AttributeError:
log.info(
'Packet does not have a data_stream_id: using default policy')
return None
dset_config = self.dataset_configs.get(stream_id, None)
configuration = None
if dset_config is None:
log.info('No config found for stream id: %s ' % stream_id)
else:
log.info('Got config: %s for stream id: %s' %
(dset_config, stream_id))
configuration = dset_config.configuration
# return the extracted instruction
return configuration
class TestActivateInstrumentIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
super(TestActivateInstrumentIntegration, self).setUp()
config = DotDict()
config.bootstrap.use_es = True
self._start_container()
self.addCleanup(TestActivateInstrumentIntegration.es_cleanup)
self.container.start_rel_from_url('res/deploy/r2deploy.yml', config)
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dpclient = DataProductManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataretrieverclient = DataRetrieverServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.usernotificationclient = UserNotificationServiceClient()
#setup listerner vars
self._data_greenlets = []
self._no_samples = None
self._samples_received = []
self.event_publisher = EventPublisher()
@staticmethod
def es_cleanup():
es_host = CFG.get_safe('server.elasticsearch.host', 'localhost')
es_port = CFG.get_safe('server.elasticsearch.port', '9200')
es = ep.ElasticSearch(host=es_host, port=es_port, timeout=10)
indexes = STD_INDEXES.keys()
indexes.append('%s_resources_index' % get_sys_name().lower())
indexes.append('%s_events_index' % get_sys_name().lower())
for index in indexes:
IndexManagementService._es_call(es.river_couchdb_delete, index)
IndexManagementService._es_call(es.index_delete, index)
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name + '_logger')
producer_definition.executable = {
'module': 'ion.processes.data.stream_granule_logger',
'class': 'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(
process_definition=producer_definition)
configuration = {
'process': {
'stream_id': stream_id,
}
}
pid = self.processdispatchclient.schedule_process(
process_definition_id=logger_procdef_id,
configuration=configuration)
return pid
def _create_notification(self,
user_name='',
instrument_id='',
product_id=''):
#--------------------------------------------------------------------------------------
# Make notification request objects
#--------------------------------------------------------------------------------------
notification_request_1 = NotificationRequest(
name='notification_1',
origin=instrument_id,
origin_type="instrument",
event_type='ResourceLifecycleEvent')
notification_request_2 = NotificationRequest(
name='notification_2',
origin=product_id,
origin_type="data product",
event_type='DetectionEvent')
#--------------------------------------------------------------------------------------
# Create a user and get the user_id
#--------------------------------------------------------------------------------------
user = UserInfo()
user.name = user_name
user.contact.email = '*****@*****.**' % user_name
user_id, _ = self.rrclient.create(user)
#--------------------------------------------------------------------------------------
# Create notification
#--------------------------------------------------------------------------------------
self.usernotificationclient.create_notification(
notification=notification_request_1, user_id=user_id)
self.usernotificationclient.create_notification(
notification=notification_request_2, user_id=user_id)
log.debug(
"test_activateInstrumentSample: create_user_notifications user_id %s",
str(user_id))
return user_id
def get_datastore(self, dataset_id):
dataset = self.datasetclient.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(
datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def _check_computed_attributes_of_extended_instrument(
self, expected_instrument_device_id='', extended_instrument=None):
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(extended_instrument.computed.firmware_version,
ComputedFloatValue)
self.assertIsInstance(
extended_instrument.computed.last_data_received_datetime,
ComputedFloatValue)
self.assertIsInstance(
extended_instrument.computed.last_calibration_datetime,
ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.uptime,
ComputedStringValue)
self.assertIsInstance(
extended_instrument.computed.power_status_roll_up,
ComputedIntValue)
self.assertIsInstance(
extended_instrument.computed.communications_status_roll_up,
ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.data_status_roll_up,
ComputedIntValue)
self.assertIsInstance(
extended_instrument.computed.location_status_roll_up,
ComputedIntValue)
# the following assert will not work without elasticsearch.
#self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) )
self.assertEqual(
extended_instrument.computed.communications_status_roll_up.value,
StatusType.STATUS_WARNING)
self.assertEqual(
extended_instrument.computed.data_status_roll_up.value,
StatusType.STATUS_OK)
self.assertEqual(
extended_instrument.computed.power_status_roll_up.value,
StatusType.STATUS_WARNING)
# Verify the computed attribute for user notification requests
self.assertEqual(
1,
len(extended_instrument.computed.user_notification_requests.value))
notifications = extended_instrument.computed.user_notification_requests.value
notification = notifications[0]
self.assertEqual(notification.origin, expected_instrument_device_id)
self.assertEqual(notification.origin_type, "instrument")
self.assertEqual(notification.event_type, 'ResourceLifecycleEvent')
def _check_computed_attributes_of_extended_product(
self, expected_data_product_id='', extended_data_product=None):
self.assertEqual(expected_data_product_id, extended_data_product._id)
log.debug("extended_data_product.computed: %s",
extended_data_product.computed)
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(
extended_data_product.computed.product_download_size_estimated,
ComputedIntValue)
self.assertIsInstance(
extended_data_product.computed.number_active_subscriptions,
ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.data_url,
ComputedStringValue)
self.assertIsInstance(extended_data_product.computed.stored_data_size,
ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.recent_granules,
ComputedDictValue)
self.assertIsInstance(extended_data_product.computed.parameters,
ComputedListValue)
self.assertIsInstance(extended_data_product.computed.recent_events,
ComputedEventListValue)
self.assertIsInstance(extended_data_product.computed.provenance,
ComputedDictValue)
self.assertIsInstance(
extended_data_product.computed.user_notification_requests,
ComputedListValue)
self.assertIsInstance(
extended_data_product.computed.active_user_subscriptions,
ComputedListValue)
self.assertIsInstance(
extended_data_product.computed.past_user_subscriptions,
ComputedListValue)
self.assertIsInstance(extended_data_product.computed.last_granule,
ComputedDictValue)
self.assertIsInstance(extended_data_product.computed.is_persisted,
ComputedIntValue)
self.assertIsInstance(
extended_data_product.computed.data_contents_updated,
ComputedStringValue)
self.assertIsInstance(extended_data_product.computed.data_datetime,
ComputedListValue)
# exact text here keeps changing to fit UI capabilities. keep assertion general...
self.assertTrue('ok' in extended_data_product.computed.last_granule.
value['quality_flag'])
self.assertEqual(
2, len(extended_data_product.computed.data_datetime.value))
notifications = extended_data_product.computed.user_notification_requests.value
notification = notifications[0]
self.assertEqual(notification.origin, expected_data_product_id)
self.assertEqual(notification.origin_type, "data product")
self.assertEqual(notification.event_type, 'DetectionEvent')
@attr('LOCOINT')
@unittest.skipIf(not use_es, 'No ElasticSearch')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
@patch.dict(CFG, {'endpoint': {'receive': {'timeout': 60}}})
def test_activateInstrumentSample(self):
self.loggerpids = []
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
log.debug('new InstrumentModel id = %s ', instModel_id)
#Create stream alarms
"""
test_two_sided_interval
Test interval alarm and alarm event publishing for a closed
inteval.
"""
# kwargs = {
# 'name' : 'test_sim_warning',
# 'stream_name' : 'parsed',
# 'value_id' : 'temp',
# 'message' : 'Temperature is above test range of 5.0.',
# 'type' : StreamAlarmType.WARNING,
# 'upper_bound' : 5.0,
# 'upper_rel_op' : '<'
# }
kwargs = {
'name': 'temperature_warning_interval',
'stream_name': 'parsed',
'value_id': 'temp',
'message':
'Temperature is below the normal range of 50.0 and above.',
'type': StreamAlarmType.WARNING,
'lower_bound': 50.0,
'lower_rel_op': '<'
}
# Create alarm object.
alarm = {}
alarm['type'] = 'IntervalAlarmDef'
alarm['kwargs'] = kwargs
raw_config = StreamConfiguration(
stream_name='raw',
parameter_dictionary_name='ctd_raw_param_dict',
records_per_granule=2,
granule_publish_rate=5)
parsed_config = StreamConfiguration(
stream_name='parsed',
parameter_dictionary_name='ctd_parsed_param_dict',
records_per_granule=2,
granule_publish_rate=5,
alarms=[alarm])
# Create InstrumentAgent
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri=
"http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1a-py2.7.egg",
stream_configurations=[raw_config, parsed_config])
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
log.debug('new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(
instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug(
'test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) '
)
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345")
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, instDevice_id)
log.debug(
"test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ",
instDevice_id)
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance,
name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config=port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubcli.create_stream_definition(
name='parsed', parameter_dictionary_id=parsed_pdict_id)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.pubsubcli.create_stream_definition(
name='raw', parameter_dictionary_id=raw_pdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id1 = self.dpclient.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug('new dp_id = %s', data_product_id1)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id1)
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1,
PRED.hasStream, None, True)
log.debug('Data product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id1,
PRED.hasDataset,
RT.Dataset, True)
log.debug('Data set for data_product_id1 = %s', dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
pid = self.create_logger('ctd_parsed', stream_ids[0])
self.loggerpids.append(pid)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id2 = self.dpclient.create_data_product(
data_product=dp_obj, stream_definition_id=raw_stream_def_id)
log.debug('new dp_id = %s', data_product_id2)
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id2)
# setup notifications for the device and parsed data product
user_id_1 = self._create_notification(user_name='user_1',
instrument_id=instDevice_id,
product_id=data_product_id1)
#---------- Create notifications for another user and verify that we see different computed subscriptions for the two users ---------
user_id_2 = self._create_notification(user_name='user_2',
instrument_id=instDevice_id,
product_id=data_product_id2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2,
PRED.hasStream, None, True)
log.debug('Data product streams2 = %s', str(stream_ids))
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id2,
PRED.hasDataset,
RT.Dataset, True)
log.debug('Data set for data_product_id2 = %s', dataset_ids[0])
self.raw_dataset = dataset_ids[0]
#elastic search debug
es_indexes, _ = self.container.resource_registry.find_resources(
restype='ElasticSearchIndex')
log.debug('ElasticSearch indexes: %s', [i.name for i in es_indexes])
log.debug('Bootstrap %s', CFG.bootstrap.use_es)
def start_instrument_agent():
self.imsclient.start_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id)
gevent.joinall([gevent.spawn(start_instrument_agent)])
#setup a subscriber to alarm events from the device
self._events_received = []
self._event_count = 0
self._samples_out_of_range = 0
self._samples_complete = False
self._async_sample_result = AsyncResult()
def consume_event(*args, **kwargs):
log.debug(
'TestActivateInstrument recieved ION event: args=%s, kwargs=%s, event=%s.',
str(args), str(kwargs), str(args[0]))
self._events_received.append(args[0])
self._event_count = len(self._events_received)
self._async_sample_result.set()
self._event_subscriber = EventSubscriber(
event_type=
'StreamWarningAlarmEvent', #'StreamWarningAlarmEvent', # StreamAlarmEvent
callback=consume_event,
origin=instDevice_id)
self._event_subscriber.start()
#cleanup
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
def stop_subscriber():
self._event_subscriber.stop()
self._event_subscriber = None
self.addCleanup(stop_subscriber)
#wait for start
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(
instAgentInstance_id)
gate = ProcessStateGate(self.processdispatchclient.read_process,
inst_agent_instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(
gate. await (30),
"The instrument agent instance (%s) did not spawn in 30 seconds" %
inst_agent_instance_obj.agent_process_id)
log.debug('Instrument agent instance obj: = %s',
str(inst_agent_instance_obj))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(
instDevice_id,
to_name=inst_agent_instance_obj.agent_process_id,
process=FakeProcess())
log.debug("test_activateInstrumentSample: got ia client %s",
str(self._ia_client))
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: initialize %s", str(retval))
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
log.debug("(L4-CI-SA-RQ-334): Sending go_active command ")
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrument: return value from go_active %s",
str(reply))
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug(
"(L4-CI-SA-RQ-334): current state after sending go_active command %s",
str(state))
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: run %s", str(reply))
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.PAUSE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.STOPPED)
cmd = AgentCommand(command=ResourceAgentEvent.RESUME)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.CLEAR)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
for i in xrange(10):
retval = self._ia_client.execute_resource(cmd)
log.debug("test_activateInstrumentSample: return from sample %s",
str(retval))
log.debug("test_activateInstrumentSample: calling reset ")
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: return from reset %s",
str(reply))
self._samples_complete = True
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data = self.dataretrieverclient.retrieve(self.parsed_dataset)
self.assertIsInstance(replay_data, Granule)
rdt = RecordDictionaryTool.load_from_granule(replay_data)
log.debug("test_activateInstrumentSample: RDT parsed: %s",
str(rdt.pretty_print()))
temp_vals = rdt['temp']
self.assertEquals(len(temp_vals), 10)
log.debug("test_activateInstrumentSample: all temp_vals: %s",
temp_vals)
#out_of_range_temp_vals = [i for i in temp_vals if i > 5]
out_of_range_temp_vals = [i for i in temp_vals if i < 50.0]
log.debug("test_activateInstrumentSample: Out_of_range_temp_vals: %s",
out_of_range_temp_vals)
self._samples_out_of_range = len(out_of_range_temp_vals)
# if no bad values were produced, then do not wait for an event
if self._samples_out_of_range == 0:
self._async_sample_result.set()
log.debug("test_activateInstrumentSample: _events_received: %s",
self._events_received)
log.debug("test_activateInstrumentSample: _event_count: %s",
self._event_count)
self._async_sample_result.get(timeout=CFG.endpoint.receive.timeout)
replay_data = self.dataretrieverclient.retrieve(self.raw_dataset)
self.assertIsInstance(replay_data, Granule)
rdt = RecordDictionaryTool.load_from_granule(replay_data)
log.debug("RDT raw: %s", str(rdt.pretty_print()))
raw_vals = rdt['raw']
self.assertEquals(len(raw_vals), 10)
log.debug("l4-ci-sa-rq-138")
"""
Physical resource control shall be subject to policy
Instrument management control capabilities shall be subject to policy
The actor accessing the control capabilities must be authorized to send commands.
note from maurice 2012-05-18: Talk to tim M to verify that this is policy. If it is then talk with Stephen to
get an example of a policy test and use that to create a test stub that will be
completed when we have instrument policies.
Tim M: The "actor", aka observatory operator, will access the instrument through ION.
"""
#--------------------------------------------------------------------------------
# Get the extended data product to see if it contains the granules
#--------------------------------------------------------------------------------
extended_product = self.dpclient.get_data_product_extension(
data_product_id=data_product_id1, user_id=user_id_1)
def poller(extended_product):
return len(extended_product.computed.user_notification_requests.
value) == 1
poll(poller, extended_product, timeout=30)
self._check_computed_attributes_of_extended_product(
expected_data_product_id=data_product_id1,
extended_data_product=extended_product)
#--------------------------------------------------------------------------------
#put some events into the eventsdb to test - this should set the comms and data status to WARNING
#--------------------------------------------------------------------------------
t = get_ion_ts()
self.event_publisher.publish_event(ts_created=t,
event_type='DeviceStatusEvent',
origin=instDevice_id,
state=DeviceStatusType.OUT_OF_RANGE,
values=[200])
self.event_publisher.publish_event(
ts_created=t,
event_type='DeviceCommsEvent',
origin=instDevice_id,
state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION,
lapse_interval_seconds=20)
#--------------------------------------------------------------------------------
# Get the extended instrument
#--------------------------------------------------------------------------------
extended_instrument = self.imsclient.get_instrument_device_extension(
instrument_device_id=instDevice_id, user_id=user_id_1)
self._check_computed_attributes_of_extended_instrument(
expected_instrument_device_id=instDevice_id,
extended_instrument=extended_instrument)
#--------------------------------------------------------------------------------
# For the second user, check the extended data product and the extended intrument
#--------------------------------------------------------------------------------
extended_product = self.dpclient.get_data_product_extension(
data_product_id=data_product_id2, user_id=user_id_2)
self._check_computed_attributes_of_extended_product(
expected_data_product_id=data_product_id2,
extended_data_product=extended_product)
#---------- Put some events into the eventsdb to test - this should set the comms and data status to WARNING ---------
t = get_ion_ts()
self.event_publisher.publish_event(ts_created=t,
event_type='DeviceStatusEvent',
origin=instDevice_id,
state=DeviceStatusType.OUT_OF_RANGE,
values=[200])
self.event_publisher.publish_event(
ts_created=t,
event_type='DeviceCommsEvent',
origin=instDevice_id,
state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION,
lapse_interval_seconds=20)
#--------------------------------------------------------------------------------
# Get the extended instrument
#--------------------------------------------------------------------------------
extended_instrument = self.imsclient.get_instrument_device_extension(
instrument_device_id=instDevice_id, user_id=user_id_2)
self._check_computed_attributes_of_extended_instrument(
expected_instrument_device_id=instDevice_id,
extended_instrument=extended_instrument)
#--------------------------------------------------------------------------------
# Deactivate loggers
#--------------------------------------------------------------------------------
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
self.dpclient.delete_data_product(data_product_id1)
self.dpclient.delete_data_product(data_product_id2)
class SchedulerService(BaseSchedulerService):
def __init__(self, *args, **kwargs):
BaseSchedulerService.__init__(self, *args, **kwargs)
self.schedule_entries = {}
self._no_reschedule = False
def on_start(self):
if CFG.get_safe("process.start_mode") == "RESTART":
self.on_system_restart()
self.pub = EventPublisher(event_type="ResourceEvent")
def on_quit(self):
self.pub.close()
# throw killswitch on future reschedules
self._no_reschedule = True
# terminate any pending spawns
self._stop_pending_timers()
def __notify(self, task, id, index):
log.debug("SchedulerService:__notify: - " + task.event_origin +
" - Time: " + str(self.__now()) + " - ID: " + id +
" -Index:" + str(index))
self.pub.publish_event(origin=task.event_origin)
def __now(self):
return datetime.utcnow()
def __now_posix(self, now):
return time.mktime(now.timetuple())
def _expire_callback(self, id, index):
task = self.__get_entry(id)
self.__notify(task, id, index)
if not self.__reschedule(id, index):
self.__delete(id, index)
def __calculate_next_interval(self, task, current_time):
if task.start_time < current_time:
next_interval = task.start_time
while (next_interval < current_time):
next_interval = next_interval + task.interval
return (next_interval - current_time)
else:
return (task.start_time - current_time) + task.interval
def __get_expire_time(self, task):
now = self.__now()
now_posix = self.__now_posix(now)
expires_in = []
if type(task) == TimeOfDayTimer:
for time_of_day in task.times_of_day:
expire_time = datetime(now.year, now.month, now.day,
time_of_day['hour'],
time_of_day['minute'],
time_of_day['second'])
expires_in.append(ceil((expire_time - now).total_seconds()))
elif type(task) == IntervalTimer and (task.end_time == -1 or (
(now_posix + task.interval) <= task.end_time)):
expires_in = [(self.__calculate_next_interval(task, now_posix))]
return expires_in
def __get_reschedule_expire_time(self, task, index):
expires_in = False
now = self.__now()
now_posix = self.__now_posix(now)
if type(task) == TimeOfDayTimer:
if task.expires > now_posix:
time_of_day = task.times_of_day[index]
tomorrow = now + timedelta(days=1)
expire_time = datetime(tomorrow.year, tomorrow.month,
tomorrow.day, time_of_day['hour'],
time_of_day['minute'],
time_of_day['second'])
expires_in = (ceil((expire_time - now).total_seconds()))
else:
expires_in = False
elif type(task) == IntervalTimer and (task.end_time == -1 or (
(now_posix + task.interval) <= task.end_time)):
if task.start_time <= now_posix:
expires_in = (task.interval)
else:
expires_in = ((task.start_time - now_posix) + task.interval)
return expires_in
def __validate_expire_times(self, expire_times):
for index, expire_time in enumerate(expire_times):
if expire_time < 0:
return False
return True
def __schedule(self, scheduler_entry, id=False):
# if "id" is set, it means scheduler_entry is already in Resource Regsitry. This can occur during a sytsem restart
spawns = []
task = scheduler_entry.entry
expire_times = self.__get_expire_time(task)
if not self.__validate_expire_times(expire_times):
log.error(
"SchedulerService:__schedule: scheduling: expire time is less than zero: "
)
return False
if not id:
id, _ = self.clients.resource_registry.create(scheduler_entry)
self.__create_entry(task, spawns, id)
for index, expire_time in enumerate(expire_times):
log.debug("SchedulerService:__schedule: scheduling: - " +
task.event_origin + " - Now: " + str(self.__now()) +
" - Expire: " + str(expire_time) + " - ID: " + id +
" - Index:" + str(index))
spawn = gevent.spawn_later(expire_time, self._expire_callback, id,
index)
spawns.append(spawn)
return id
def __reschedule(self, id, index):
if self._no_reschedule:
log.debug(
"SchedulerService:__reschedule: process quitting, refusing to reschedule %s",
id)
return False
task = self.__get_entry(id)
expire_time = self.__get_reschedule_expire_time(task, index)
if expire_time:
log.debug("SchedulerService:__reschedule: rescheduling: - " +
task.event_origin + " - Now: " + str(self.__now()) +
" - Expire: " + str(expire_time) + " - ID: " + id +
" -Index:" + str(index))
spawn = gevent.spawn_later(expire_time, self._expire_callback, id,
index)
self.__update_entry(id=id, index=index, spawn=spawn)
return True
else:
log.debug(
"SchedulerService:__reschedule: timer expired. Removed from RR : - "
+ task.event_origin + " - Now: " + str(self.__now()) +
" - Expire: " + str(expire_time) + " - ID: " + id +
" -Index:" + str(index))
return False
def __create_entry(self, task, spawns, id):
self.schedule_entries[id] = {"task": task, "spawns": spawns}
def __update_entry(self, id, index, spawn=None, interval=None):
if spawn is not None:
self.schedule_entries[id]["spawns"][index] = spawn
if interval is not None:
self.schedule_entries[id]["task"].interval = interval
def __get_entry_all(self, id):
return self.schedule_entries[id]
def __get_spawns(self, id):
return self.schedule_entries[id]["spawns"]
def __get_entry(self, id):
return self.schedule_entries[id]["task"]
def __delete(self, id, index, force=False):
if id in self.schedule_entries:
task = self.__get_entry(id)
if force and type(task) == TimeOfDayTimer:
log.debug("SchedulerService:__delete: entry deleted " + id +
" -Index:" + str(index))
del self.schedule_entries[id]
self.clients.resource_registry.delete(id)
elif type(task) == TimeOfDayTimer:
task = self.__get_entry(id)
task.times_of_day[index] = None
# Delete if all the timers are set to none
are_all_timers_expired = True
for time_of_day in task.times_of_day:
if time_of_day is not None:
are_all_timers_expired = False
break
if are_all_timers_expired:
log.debug("SchedulerService:__delete: entry deleted " +
id + " -Index:" + str(index))
del self.schedule_entries[id]
self.clients.resource_registry.delete(id)
else:
log.debug("SchedulerService:__delete: entry deleted " + id +
" -Index:" + str(index))
del self.schedule_entries[id]
self.clients.resource_registry.delete(id)
return True
return False
def __is_timer_valid(self, task):
# Validate event_origin is set
if not task.event_origin:
log.error(
"SchedulerService.__is_timer_valid: event_origin is not set")
return False
# Validate the timer is set correctly
if type(task) == IntervalTimer:
if (task.end_time != -1
and (self.__now_posix(self.__now()) >= task.end_time)):
log.error(
"SchedulerService.__is_timer_valid: IntervalTimer is set to incorrect value"
)
return False
elif type(task) == TimeOfDayTimer:
for time_of_day in task.times_of_day:
time_of_day['hour'] = int(time_of_day['hour'])
time_of_day['minute'] = int(time_of_day['minute'])
time_of_day['second'] = int(time_of_day['second'])
if ((time_of_day['hour'] < 0 or time_of_day['hour'] > 23) or
(time_of_day['minute'] < 0 or time_of_day['minute'] > 59)
or
(time_of_day['second'] < 0 or time_of_day['second'] > 61)):
log.error(
"SchedulerService.__is_timer_valid: TimeOfDayTimer is set to incorrect value"
)
return False
else:
return False
return True
def _stop_pending_timers(self):
"""
Safely stops all pending and active timers.
For all timers still waiting to run, calls kill on them. For active timers, let
them exit naturally and prevent the reschedule by setting the _no_reschedule flag.
"""
# prevent reschedules
self._no_reschedule = True
gls = []
for timer_id in self.schedule_entries:
spawns = self.__get_spawns(timer_id)
for spawn in spawns:
gls.append(spawn)
# only kill spawns that haven't started yet
if spawn._start_event is not None:
spawn.kill()
log.debug("_stop_pending_timers: timer %s deleted", timer_id)
self.schedule_entries.clear()
# wait for running gls to finish up
gevent.joinall(gls, timeout=10)
# allow reschedules from here on out
self._no_reschedule = False
def on_system_restart(self):
'''
On system restart, get timer data from Resource Registry and restore the Scheduler state
'''
# Remove all active timers
# When this method is called, there should not be any active timers but if it is called from test, this helps
# to remove current active timer and restore them from Resource Regstiry
self._stop_pending_timers()
# Restore the timer from Resource Registry
scheduler_entries, _ = self.clients.resource_registry.find_resources(
RT.SchedulerEntry, id_only=False)
for scheduler_entry in scheduler_entries:
self.__schedule(scheduler_entry, scheduler_entry._id)
log.debug("SchedulerService:on_system_restart: timer restored: " +
scheduler_entry._id)
def create_timer(self, scheduler_entry=None):
"""
Create a timer which will send TimerEvents as requested for a given schedule.
The schedule request is expressed through a specific subtype of TimerSchedulerEntry.
The task is delivered as a TimeEvent to which processes can subscribe. The creator
defines the fields of the task. A GUID-based id prefixed by readable process name
is recommended for the origin. Because the delivery of the task is via the ION Exchange
there is potential for a small deviation in precision.
Returns a timer_id which can be used to cancel the timer.
@param timer__schedule TimerSchedulerEntry
@retval timer_id str
@throws BadRequest if timer is misformed and can not be scheduled
"""
##scheduler_entry = scheduler_entry.entry
status = self.__is_timer_valid(scheduler_entry.entry)
if not status:
raise BadRequest
id = self.__schedule(scheduler_entry)
if not id:
raise BadRequest
return id
def cancel_timer(self, timer_id=''):
"""
Cancels an existing timer which has not reached its expire time.
@param timer_id str
@throws NotFound if timer_id doesn't exist
"""
#try:
try:
spawns = self.__get_spawns(timer_id)
for spawn in spawns:
spawn.kill()
log.debug("SchedulerService: cancel_timer: id: " + str(timer_id))
self.__delete(id=timer_id, index=None, force=True)
except:
log.error(
"SchedulerService: cancel_timer: timer id doesn't exist: " +
str(timer_id))
raise BadRequest
def create_interval_timer(self,
start_time="",
interval=0,
end_time="",
event_origin="",
event_subtype=""):
if (end_time != -1 and
(self.__now_posix(self.__now()) >= end_time)) or not event_origin:
log.error(
"SchedulerService.create_interval_timer: event_origin is not set"
)
raise BadRequest
if start_time == "now":
start_time = self.__now_posix(self.__now())
log.debug(
"SchedulerService:create_interval_timer start_time: %s interval: %s end_time: %s event_origin: %s"
% (start_time, interval, end_time, event_origin))
interval_timer = IonObject(
"IntervalTimer", {
"start_time": start_time,
"interval": interval,
"end_time": end_time,
"event_origin": event_origin,
"event_subtype": event_subtype
})
se = IonObject(RT.SchedulerEntry, {"entry": interval_timer})
return self.create_timer(se)
def create_time_of_day_timer(self,
times_of_day=None,
expires='',
event_origin='',
event_subtype=''):
# Validate the timer
if not event_origin:
log.error(
"SchedulerService.create_time_of_day_timer: event_origin is set to invalid value"
)
raise BadRequest
for time_of_day in times_of_day:
time_of_day['hour'] = int(time_of_day['hour'])
time_of_day['minute'] = int(time_of_day['minute'])
time_of_day['second'] = int(time_of_day['second'])
log.debug(
"SchedulerService:create_time_of_day_timer - hour: %d minute: %d second: %d expires: %d event_origin: %s"
% (time_of_day['hour'], time_of_day['minute'],
time_of_day['second'], time_of_day['second'], event_origin))
if ((time_of_day['hour'] < 0 or time_of_day['hour'] > 23) or
(time_of_day['minute'] < 0 or time_of_day['minute'] > 59) or
(time_of_day['second'] < 0 or time_of_day['second'] > 61)):
log.error(
"SchedulerService:create_time_of_day_timer: TimeOfDayTimer is set to invalid value"
)
raise BadRequest
time_of_day_timer = IonObject(
"TimeOfDayTimer", {
"times_of_day": times_of_day,
"expires": expires,
"event_origin": event_origin,
"event_subtype": event_subtype
})
se = IonObject(RT.SchedulerEntry, {"entry": time_of_day_timer})
return self.create_timer(se)