class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
# @unittest.skip('this test just for debugging setup')
# def test_just_the_setup(self):
# return
@attr('EXT')
def test_resources_associations_extensions(self):
"""
create one of each resource and association used by IMS
to guard against problems in ion-definitions
"""
#stuff we control
instrument_agent_instance_id, _ = self.RR.create(any_old(RT.InstrumentAgentInstance))
instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent))
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel))
instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice))
platform_agent_instance_id, _ = self.RR.create(any_old(RT.PlatformAgentInstance))
platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent))
platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice))
platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel))
sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice))
sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel))
#stuff we associate to
data_producer_id, _ = self.RR.create(any_old(RT.DataProducer))
org_id, _ = self.RR.create(any_old(RT.Org))
#instrument_agent_instance_id #is only a target
#instrument_agent
self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id)
#instrument_device
self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id)
self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id)
self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id)
self.RR.create_association(org_id, PRED.hasResource, instrument_device_id)
instrument_model_id #is only a target
platform_agent_instance_id #is only a target
#platform_agent
self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id)
#platform_device
self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id)
self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id)
platform_model_id #is only a target
#sensor_device
self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id)
self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id)
sensor_model_id #is only a target
#create a parsed product for this instrument output
tdom, sdom = time_series_domain()
tdom = tdom.dump()
sdom = sdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
processing_level_code='Parsed_Canonical',
temporal_domain = tdom,
spatial_domain = sdom)
pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id)
data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1)
def addInstOwner(inst_id, subject):
actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject})
user_id = self.IDS.create_actor_identity(actor_identity_obj)
user_info_obj = any_old(RT.UserInfo)
user_info_id = self.IDS.create_user_info(user_id, user_info_obj)
self.RR.create_association(inst_id, PRED.hasOwner, user_id)
#Testing multiple instrument owners
addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254")
addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256")
extended_instrument = self.IMS.get_instrument_device_extension(instrument_device_id)
self.assertEqual(instrument_device_id, extended_instrument._id)
self.assertEqual(len(extended_instrument.owners), 2)
self.assertEqual(extended_instrument.instrument_model._id, instrument_model_id)
# Lifecycle
self.assertEquals(len(extended_instrument.lcstate_transitions), 7)
self.assertEquals(set(extended_instrument.lcstate_transitions.keys()), set(['enable', 'develop', 'deploy', 'retire', 'plan', 'integrate', 'announce']))
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(extended_instrument.computed.firmware_version, ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.last_data_received_datetime, ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.last_calibration_datetime, ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue)
self.assertIsInstance(extended_instrument.computed.power_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.communications_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.location_status_roll_up, ComputedIntValue)
log.debug("extended_instrument.computed: %s", extended_instrument.computed)
#check model
inst_model_obj = self.RR.read(instrument_model_id)
self.assertEqual(inst_model_obj.name, extended_instrument.instrument_model.name)
#check agent instance
inst_agent_instance_obj = self.RR.read(instrument_agent_instance_id)
self.assertEqual(inst_agent_instance_obj.name, extended_instrument.agent_instance.name)
#check agent
inst_agent_obj = self.RR.read(instrument_agent_id)
#compound assoc return list of lists so check the first element
self.assertEqual(inst_agent_obj.name, extended_instrument.instrument_agent.name)
#check platform device
plat_device_obj = self.RR.read(platform_device_id)
self.assertEqual(plat_device_obj.name, extended_instrument.platform_device.name)
extended_platform = self.IMS.get_platform_device_extension(platform_device_id)
self.assertEqual(1, len(extended_platform.instrument_devices))
self.assertEqual(instrument_device_id, extended_platform.instrument_devices[0]._id)
self.assertEqual(1, len(extended_platform.instrument_models))
self.assertEqual(instrument_model_id, extended_platform.instrument_models[0]._id)
self.assertEquals(extended_platform.platform_agent._id, platform_agent_id)
self.assertEquals(len(extended_platform.lcstate_transitions), 7)
self.assertEquals(set(extended_platform.lcstate_transitions.keys()), set(['enable', 'develop', 'deploy', 'retire', 'plan', 'integrate', 'announce']))
#check sensor devices
self.assertEqual(1, len(extended_instrument.sensor_devices))
#check data_product_parameters_set
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_instrument.computed.data_product_parameters_set.status)
self.assertTrue( 'Parsed_Canonical' in extended_instrument.computed.data_product_parameters_set.value)
# the ctd parameters should include 'temp'
self.assertTrue( 'temp' in extended_instrument.computed.data_product_parameters_set.value['Parsed_Canonical'])
#none of these will work because there is no agent
self.assertEqual(ComputedValueAvailability.NOTAVAILABLE,
extended_instrument.computed.firmware_version.status)
# self.assertEqual(ComputedValueAvailability.NOTAVAILABLE,
# extended_instrument.computed.operational_state.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.power_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.communications_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.data_status_roll_up.status)
# self.assertEqual(StatusType.STATUS_OK,
# extended_instrument.computed.data_status_roll_up.value)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.location_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.recent_events.status)
# self.assertEqual([], extended_instrument.computed.recent_events.value)
# cleanup
c = DotDict()
c.resource_registry = self.RR
self.RR2.pluck(instrument_agent_id)
self.RR2.pluck(instrument_model_id)
self.RR2.pluck(instrument_device_id)
self.RR2.pluck(platform_agent_id)
self.RR2.pluck(sensor_device_id)
self.IMS.force_delete_instrument_agent(instrument_agent_id)
self.IMS.force_delete_instrument_model(instrument_model_id)
self.IMS.force_delete_instrument_device(instrument_device_id)
self.IMS.force_delete_platform_agent_instance(platform_agent_instance_id)
self.IMS.force_delete_platform_agent(platform_agent_id)
self.IMS.force_delete_platform_device(platform_device_id)
self.IMS.force_delete_platform_model(platform_model_id)
self.IMS.force_delete_sensor_device(sensor_device_id)
self.IMS.force_delete_sensor_model(sensor_model_id)
#stuff we associate to
self.RR.delete(data_producer_id)
self.RR.delete(org_id)
def test_custom_attributes(self):
"""
Test assignment of custom attributes
"""
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel,
{"custom_attributes":
{"favorite_color": "attr desc goes here"}
}))
instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice,
{"custom_attributes":
{"favorite_color": "red",
"bogus_attr": "should raise warning"
}
}))
self.IMS.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id)
# cleanup
self.IMS.force_delete_instrument_device(instrument_device_id)
self.IMS.force_delete_instrument_model(instrument_model_id)
def _get_datastore(self, dataset_id):
dataset = self.DSC.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def test_resource_state_save_restore(self):
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.IMS.create_instrument_model(instModel_obj)
log.debug( 'new InstrumentModel id = %s ', instModel_id)
# Create InstrumentAgent
raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 )
parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5 )
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg",
stream_configurations = [raw_config, parsed_config] )
instAgent_id = self.IMS.create_instrument_agent(instAgent_obj)
log.debug( 'new InstrumentAgent id = %s', instAgent_id)
self.IMS.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 '
+ '(SA Req: L4-CI-SA-RQ-241) ')
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345" )
instDevice_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj)
self.IMS.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ",
instDevice_id)
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config = port_agent_config)
instAgentInstance_id = self.IMS.create_instrument_agent_instance(instAgentInstance_obj,
instAgent_id,
instDevice_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
spdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=spdict_id)
rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id1 = self.DP.create_data_product(data_product=dp_obj,
stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1)
self.DP.activate_data_product_persistence(data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True)
log.debug( 'Data product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True)
log.debug( 'Data set for data_product_id1 = %s', dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
#create the datastore at the beginning of each int test that persists data
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id2 = self.DP.create_data_product(data_product=dp_obj,
stream_definition_id=raw_stream_def_id)
log.debug( 'new dp_id = %s', str(data_product_id2))
self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2)
self.DP.activate_data_product_persistence(data_product_id=data_product_id2)
# spin up agent
self.IMS.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
self.addCleanup(self.IMS.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
#wait for start
instance_obj = self.IMS.read_instrument_agent_instance(instAgentInstance_id)
gate = ProcessStateGate(self.PDC.read_process,
instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" %
instance_obj.agent_process_id)
# take snapshot of config
snap_id = self.IMS.save_resource_state(instDevice_id, "xyzzy snapshot")
snap_obj = self.RR.read_attachment(snap_id, include_content=True)
print "Saved config:"
print snap_obj.content
#modify config
instance_obj.driver_config["comms_config"] = "BAD_DATA"
self.RR.update(instance_obj)
#restore config
self.IMS.restore_resource_state(instDevice_id, snap_id)
instance_obj = self.RR.read(instAgentInstance_id)
self.assertNotEqual("BAD_DATA", instance_obj.driver_config["comms_config"])
self.DP.delete_data_product(data_product_id1)
self.DP.delete_data_product(data_product_id2)
def test_agent_instance_config(self):
"""
Verify that agent configurations are being built properly
"""
clients = DotDict()
clients.resource_registry = self.RR
clients.pubsub_management = self.PSC
clients.dataset_management = self.DSC
pconfig_builder = PlatformAgentConfigurationBuilder(clients)
iconfig_builder = InstrumentAgentConfigurationBuilder(clients)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
org_obj = any_old(RT.Org)
org_id = self.RR2.create(org_obj)
inst_startup_config = {'startup': 'config'}
generic_alerts_config = {'lvl1': {'lvl2': 'lvl3val'}}
required_config_keys = [
'org_name',
'device_type',
'agent',
'driver_config',
'stream_config',
'startup_config',
'aparam_alert_config',
'children']
def verify_instrument_config(config, device_id):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual(org_obj.name, config['org_name'])
self.assertEqual(RT.InstrumentDevice, config['device_type'])
self.assertIn('driver_config', config)
driver_config = config['driver_config']
expected_driver_fields = {'process_type': ('ZMQPyClassDriverLauncher',),
}
for k, v in expected_driver_fields.iteritems():
self.assertIn(k, driver_config)
self.assertEqual(v, driver_config[k])
self.assertEqual
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertEqual(inst_startup_config, config['startup_config'])
self.assertIn('aparam_alert_config', config)
self.assertEqual(generic_alerts_config, config['aparam_alert_config'])
self.assertIn('stream_config', config)
for key in ['children']:
self.assertEqual({}, config[key])
def verify_child_config(config, device_id, inst_device_id=None):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual(org_obj.name, config['org_name'])
self.assertEqual(RT.PlatformDevice, config['device_type'])
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertIn('aparam_alert_config', config)
self.assertEqual(generic_alerts_config, config['aparam_alert_config'])
self.assertIn('stream_config', config)
self.assertIn('driver_config', config)
self.assertIn('foo', config['driver_config'])
self.assertEqual('bar', config['driver_config']['foo'])
self.assertIn('process_type', config['driver_config'])
self.assertEqual(('ZMQPyClassDriverLauncher',), config['driver_config']['process_type'])
if None is inst_device_id:
for key in ['children', 'startup_config']:
self.assertEqual({}, config[key])
else:
for key in ['startup_config']:
self.assertEqual({}, config[key])
self.assertIn(inst_device_id, config['children'])
verify_instrument_config(config['children'][inst_device_id], inst_device_id)
def verify_parent_config(config, parent_device_id, child_device_id, inst_device_id=None):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual(org_obj.name, config['org_name'])
self.assertEqual(RT.PlatformDevice, config['device_type'])
self.assertIn('process_type', config['driver_config'])
self.assertEqual(('ZMQPyClassDriverLauncher',), config['driver_config']['process_type'])
self.assertEqual({'resource_id': parent_device_id}, config['agent'])
self.assertIn('aparam_alert_config', config)
self.assertEqual(generic_alerts_config, config['aparam_alert_config'])
self.assertIn('stream_config', config)
for key in ['startup_config']:
self.assertEqual({}, config[key])
self.assertIn(child_device_id, config['children'])
verify_child_config(config['children'][child_device_id], child_device_id, inst_device_id)
rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id)
#todo: create org and figure out which agent resource needs to get assigned to it
def _make_platform_agent_structure(agent_config=None):
if None is agent_config: agent_config = {}
# instance creation
platform_agent_instance_obj = any_old(RT.PlatformAgentInstance, {'driver_config': {'foo': 'bar'},
'alerts': generic_alerts_config})
platform_agent_instance_obj.agent_config = agent_config
platform_agent_instance_id = self.IMS.create_platform_agent_instance(platform_agent_instance_obj)
# agent creation
raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 )
platform_agent_obj = any_old(RT.PlatformAgent, {"stream_configurations":[raw_config]})
platform_agent_id = self.IMS.create_platform_agent(platform_agent_obj)
# device creation
platform_device_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice))
# data product creation
dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom})
dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id)
self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
# assignments
self.RR2.assign_platform_agent_instance_to_platform_device(platform_agent_instance_id, platform_device_id)
self.RR2.assign_platform_agent_to_platform_agent_instance(platform_agent_id, platform_agent_instance_id)
self.RR2.assign_platform_device_to_org_with_has_resource(platform_agent_instance_id, org_id)
return platform_agent_instance_id, platform_agent_id, platform_device_id
def _make_instrument_agent_structure(agent_config=None):
if None is agent_config: agent_config = {}
# instance creation
instrument_agent_instance_obj = any_old(RT.InstrumentAgentInstance, {"startup_config": inst_startup_config,
'alerts': generic_alerts_config})
instrument_agent_instance_obj.agent_config = agent_config
instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(instrument_agent_instance_obj)
# agent creation
raw_config = StreamConfiguration(stream_name='raw',
parameter_dictionary_name='ctd_raw_param_dict',
records_per_granule=2,
granule_publish_rate=5 )
instrument_agent_obj = any_old(RT.InstrumentAgent, {"stream_configurations":[raw_config]})
instrument_agent_id = self.IMS.create_instrument_agent(instrument_agent_obj)
# device creation
instrument_device_id = self.IMS.create_instrument_device(any_old(RT.InstrumentDevice))
# data product creation
dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom})
dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
# assignments
self.RR2.assign_instrument_agent_instance_to_instrument_device(instrument_agent_instance_id, instrument_device_id)
self.RR2.assign_instrument_agent_to_instrument_agent_instance(instrument_agent_id, instrument_agent_instance_id)
self.RR2.assign_instrument_device_to_org_with_has_resource(instrument_agent_instance_id, org_id)
return instrument_agent_instance_id, instrument_agent_id, instrument_device_id
# can't do anything without an agent instance obj
log.debug("Testing that preparing a launcher without agent instance raises an error")
self.assertRaises(AssertionError, pconfig_builder.prepare, will_launch=False)
log.debug("Making the structure for a platform agent, which will be the child")
platform_agent_instance_child_id, _, platform_device_child_id = _make_platform_agent_structure()
platform_agent_instance_child_obj = self.RR2.read(platform_agent_instance_child_id)
log.debug("Preparing a valid agent instance launch, for config only")
pconfig_builder.set_agent_instance_object(platform_agent_instance_child_obj)
child_config = pconfig_builder.prepare(will_launch=False)
verify_child_config(child_config, platform_device_child_id)
log.debug("Making the structure for a platform agent, which will be the parent")
platform_agent_instance_parent_id, _, platform_device_parent_id = _make_platform_agent_structure()
platform_agent_instance_parent_obj = self.RR2.read(platform_agent_instance_parent_id)
log.debug("Testing child-less parent as a child config")
pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj)
parent_config = pconfig_builder.prepare(will_launch=False)
verify_child_config(parent_config, platform_device_parent_id)
log.debug("assigning child platform to parent")
self.RR2.assign_platform_device_to_platform_device(platform_device_child_id, platform_device_parent_id)
child_device_ids = self.RR2.find_platform_device_ids_of_device(platform_device_parent_id)
self.assertNotEqual(0, len(child_device_ids))
log.debug("Testing parent + child as parent config")
pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj)
parent_config = pconfig_builder.prepare(will_launch=False)
verify_parent_config(parent_config, platform_device_parent_id, platform_device_child_id)
log.debug("making the structure for an instrument agent")
instrument_agent_instance_id, _, instrument_device_id = _make_instrument_agent_structure()
instrument_agent_instance_obj = self.RR2.read(instrument_agent_instance_id)
log.debug("Testing instrument config")
iconfig_builder.set_agent_instance_object(instrument_agent_instance_obj)
instrument_config = iconfig_builder.prepare(will_launch=False)
verify_instrument_config(instrument_config, instrument_device_id)
log.debug("assigning instrument to platform")
self.RR2.assign_instrument_device_to_platform_device(instrument_device_id, platform_device_child_id)
child_device_ids = self.RR2.find_instrument_device_ids_of_device(platform_device_child_id)
self.assertNotEqual(0, len(child_device_ids))
log.debug("Testing entire config")
pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj)
full_config = pconfig_builder.prepare(will_launch=False)
verify_parent_config(full_config, platform_device_parent_id, platform_device_child_id, instrument_device_id)
class TestPlatformLaunch(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient()
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
# Use the network definition provided by RSN OMS directly.
rsn_oms = CIOMSClientFactory.create_instance(DVR_CONFIG['oms_uri'])
self._network_definition = RsnOmsUtil.build_network_definition(rsn_oms)
# get serialized version for the configuration:
self._network_definition_ser = NetworkUtil.serialize_network_definition(self._network_definition)
if log.isEnabledFor(logging.TRACE):
log.trace("NetworkDefinition serialization:\n%s", self._network_definition_ser)
self._async_data_result = AsyncResult()
self._data_subscribers = []
self._samples_received = []
self.addCleanup(self._stop_data_subscribers)
self._async_event_result = AsyncResult()
self._event_subscribers = []
self._events_received = []
self.addCleanup(self._stop_event_subscribers)
self._start_event_subscriber()
def _start_data_subscriber(self, stream_name, stream_id):
"""
Starts data subscriber for the given stream_name and stream_config
"""
def consume_data(message, stream_route, stream_id):
# A callback for processing subscribed-to data.
log.info('Subscriber received data message: %s.', str(message))
self._samples_received.append(message)
self._async_data_result.set()
log.info('_start_data_subscriber stream_name=%r stream_id=%r',
stream_name, stream_id)
# Create subscription for the stream
exchange_name = '%s_queue' % stream_name
self.container.ex_manager.create_xn_queue(exchange_name).purge()
sub = StandaloneStreamSubscriber(exchange_name, consume_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = self.PSC.create_subscription(name=exchange_name, stream_ids=[stream_id])
self.PSC.activate_subscription(sub_id)
sub.subscription_id = sub_id
def _stop_data_subscribers(self):
"""
Stop the data subscribers on cleanup.
"""
try:
for sub in self._data_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self.PSC.deactivate_subscription(sub.subscription_id)
except:
pass
self.PSC.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._data_subscribers = []
def _start_event_subscriber(self, event_type="DeviceEvent", sub_type="platform_event"):
"""
Starts event subscriber for events of given event_type ("DeviceEvent"
by default) and given sub_type ("platform_event" by default).
"""
def consume_event(evt, *args, **kwargs):
# A callback for consuming events.
log.info('Event subscriber received evt: %s.', str(evt))
self._events_received.append(evt)
self._async_event_result.set(evt)
sub = EventSubscriber(event_type=event_type,
sub_type=sub_type,
callback=consume_event)
sub.start()
log.info("registered event subscriber for event_type=%r, sub_type=%r",
event_type, sub_type)
self._event_subscribers.append(sub)
sub._ready_event.wait(timeout=EVENT_TIMEOUT)
def _stop_event_subscribers(self):
"""
Stops the event subscribers on cleanup.
"""
try:
for sub in self._event_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self.PSC.deactivate_subscription(sub.subscription_id)
except:
pass
self.PSC.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._event_subscribers = []
def _create_platform_configuration(self):
"""
Verify that agent configurations are being built properly
"""
#
# This method is an adaptation of test_agent_instance_config in
# test_instrument_management_service_integration.py
#
clients = DotDict()
clients.resource_registry = self.RR
clients.pubsub_management = self.PSC
clients.dataset_management = self.DSC
pconfig_builder = PlatformAgentConfigurationBuilder(clients)
iconfig_builder = InstrumentAgentConfigurationBuilder(clients)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
org_id = self.RR2.create(any_old(RT.Org))
inst_startup_config = {'startup': 'config'}
required_config_keys = [
'org_name',
'device_type',
'agent',
'driver_config',
'stream_config',
'startup_config',
'alarm_defs',
'children']
def verify_instrument_config(config, device_id):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual('Org_1', config['org_name'])
self.assertEqual(RT.InstrumentDevice, config['device_type'])
self.assertIn('driver_config', config)
driver_config = config['driver_config']
expected_driver_fields = {'process_type': ('ZMQPyClassDriverLauncher',),
}
for k, v in expected_driver_fields.iteritems():
self.assertIn(k, driver_config)
self.assertEqual(v, driver_config[k])
self.assertEqual
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertEqual(inst_startup_config, config['startup_config'])
self.assertIn('stream_config', config)
for key in ['alarm_defs', 'children']:
self.assertEqual({}, config[key])
def verify_child_config(config, device_id, inst_device_id=None):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual('Org_1', config['org_name'])
self.assertEqual(RT.PlatformDevice, config['device_type'])
self.assertEqual({'process_type': ('ZMQPyClassDriverLauncher',)}, config['driver_config'])
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertIn('stream_config', config)
if None is inst_device_id:
for key in ['alarm_defs', 'children', 'startup_config']:
self.assertEqual({}, config[key])
else:
for key in ['alarm_defs', 'startup_config']:
self.assertEqual({}, config[key])
self.assertIn(inst_device_id, config['children'])
verify_instrument_config(config['children'][inst_device_id], inst_device_id)
def verify_parent_config(config, parent_device_id, child_device_id, inst_device_id=None):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual('Org_1', config['org_name'])
self.assertEqual(RT.PlatformDevice, config['device_type'])
self.assertEqual({'process_type': ('ZMQPyClassDriverLauncher',)}, config['driver_config'])
self.assertEqual({'resource_id': parent_device_id}, config['agent'])
self.assertIn('stream_config', config)
for key in ['alarm_defs', 'startup_config']:
self.assertEqual({}, config[key])
self.assertIn(child_device_id, config['children'])
verify_child_config(config['children'][child_device_id], child_device_id, inst_device_id)
parsed_rpdict_id = self.DSC.read_parameter_dictionary_by_name(
'platform_eng_parsed',
id_only=True)
self.parsed_stream_def_id = self.PSC.create_stream_definition(
name='parsed',
parameter_dictionary_id=parsed_rpdict_id)
rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id)
#todo: create org and figure out which agent resource needs to get assigned to it
def _make_platform_agent_structure(agent_config=None):
if None is agent_config: agent_config = {}
# instance creation
platform_agent_instance_obj = any_old(RT.PlatformAgentInstance)
platform_agent_instance_obj.agent_config = agent_config
platform_agent_instance_id = self.IMS.create_platform_agent_instance(platform_agent_instance_obj)
# agent creation
raw_config = StreamConfiguration(stream_name='parsed',
parameter_dictionary_name='platform_eng_parsed',
records_per_granule=2,
granule_publish_rate=5)
platform_agent_obj = any_old(RT.PlatformAgent, {"stream_configurations":[raw_config]})
platform_agent_id = self.IMS.create_platform_agent(platform_agent_obj)
# device creation
platform_device_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice))
# data product creation
dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom})
# dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id)
dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=self.parsed_stream_def_id)
self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
# assignments
self.RR2.assign_platform_agent_instance_to_platform_device(platform_agent_instance_id, platform_device_id)
self.RR2.assign_platform_agent_to_platform_agent_instance(platform_agent_id, platform_agent_instance_id)
self.RR2.assign_platform_device_to_org_with_has_resource(platform_agent_instance_id, org_id)
return platform_agent_instance_id, platform_agent_id, platform_device_id
def _make_instrument_agent_structure(agent_config=None):
if None is agent_config: agent_config = {}
# instance creation
instrument_agent_instance_obj = any_old(RT.InstrumentAgentInstance, {"startup_config": inst_startup_config})
instrument_agent_instance_obj.agent_config = agent_config
instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(instrument_agent_instance_obj)
# agent creation
raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 )
instrument_agent_obj = any_old(RT.InstrumentAgent, {"stream_configurations":[raw_config]})
instrument_agent_id = self.IMS.create_instrument_agent(instrument_agent_obj)
# device creation
instrument_device_id = self.IMS.create_instrument_device(any_old(RT.InstrumentDevice))
# data product creation
dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom})
dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
# assignments
self.RR2.assign_instrument_agent_instance_to_instrument_device(instrument_agent_instance_id, instrument_device_id)
self.RR2.assign_instrument_agent_to_instrument_agent_instance(instrument_agent_id, instrument_agent_instance_id)
self.RR2.assign_instrument_device_to_org_with_has_resource(instrument_agent_instance_id, org_id)
return instrument_agent_instance_id, instrument_agent_id, instrument_device_id
# can't do anything without an agent instance obj
log.debug("Testing that preparing a launcher without agent instance raises an error")
self.assertRaises(AssertionError, pconfig_builder.prepare, will_launch=False)
log.debug("Making the structure for a platform agent, which will be the child")
platform_agent_instance_child_id, _, platform_device_child_id = _make_platform_agent_structure()
platform_agent_instance_child_obj = self.RR2.read(platform_agent_instance_child_id)
log.debug("Preparing a valid agent instance launch, for config only")
pconfig_builder.set_agent_instance_object(platform_agent_instance_child_obj)
child_config = pconfig_builder.prepare(will_launch=False)
verify_child_config(child_config, platform_device_child_id)
log.debug("Making the structure for a platform agent, which will be the parent")
platform_agent_instance_parent_id, _, platform_device_parent_id = _make_platform_agent_structure()
platform_agent_instance_parent_obj = self.RR2.read(platform_agent_instance_parent_id)
log.debug("Testing child-less parent as a child config")
pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj)
parent_config = pconfig_builder.prepare(will_launch=False)
verify_child_config(parent_config, platform_device_parent_id)
log.debug("assigning child platform to parent")
self.RR2.assign_platform_device_to_platform_device(platform_device_child_id, platform_device_parent_id)
child_device_ids = self.RR2.find_platform_device_ids_of_device(platform_device_parent_id)
self.assertNotEqual(0, len(child_device_ids))
log.debug("Testing parent + child as parent config")
pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj)
parent_config = pconfig_builder.prepare(will_launch=False)
verify_parent_config(parent_config, platform_device_parent_id, platform_device_child_id)
log.debug("making the structure for an instrument agent")
instrument_agent_instance_id, _, instrument_device_id = _make_instrument_agent_structure()
instrument_agent_instance_obj = self.RR2.read(instrument_agent_instance_id)
log.debug("Testing instrument config")
iconfig_builder.set_agent_instance_object(instrument_agent_instance_obj)
instrument_config = iconfig_builder.prepare(will_launch=False)
verify_instrument_config(instrument_config, instrument_device_id)
log.debug("assigning instrument to platform")
self.RR2.assign_instrument_device_to_platform_device(instrument_device_id, platform_device_child_id)
child_device_ids = self.RR2.find_instrument_device_ids_of_device(platform_device_child_id)
self.assertNotEqual(0, len(child_device_ids))
log.debug("Testing entire config")
pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj)
full_config = pconfig_builder.prepare(will_launch=False)
verify_parent_config(full_config, platform_device_parent_id, platform_device_child_id, instrument_device_id)
if log.isEnabledFor(logging.TRACE):
import pprint
pp = pprint.PrettyPrinter()
pp.pprint(full_config)
log.trace("full_config = %s", pp.pformat(full_config))
return full_config
def get_streamConfigs(self):
#
# This method is an adaptation of get_streamConfigs in
# test_driver_egg.py
#
return [
StreamConfiguration(stream_name='parsed',
parameter_dictionary_name='platform_eng_parsed',
records_per_granule=2,
granule_publish_rate=5)
# TODO enable something like the following when also
# incorporating "raw" data:
#,
#StreamConfiguration(stream_name='raw',
# parameter_dictionary_name='ctd_raw_param_dict',
# records_per_granule=2,
# granule_publish_rate=5)
]
@skip("Still needs alignment with new configuration structure")
def test_hierarchy(self):
# TODO re-implement.
pass
def test_single_platform(self):
full_config = self._create_platform_configuration()
platform_id = 'LJ01D'
stream_configurations = self.get_streamConfigs()
agent__obj = IonObject(RT.PlatformAgent,
name='%s_PlatformAgent' % platform_id,
description='%s_PlatformAgent platform agent' % platform_id,
stream_configurations=stream_configurations)
agent_id = self.IMS.create_platform_agent(agent__obj)
device__obj = IonObject(RT.PlatformDevice,
name='%s_PlatformDevice' % platform_id,
description='%s_PlatformDevice platform device' % platform_id,
# ports=port_objs,
# platform_monitor_attributes = monitor_attribute_objs
)
self.device_id = self.IMS.create_platform_device(device__obj)
#######################################
# data product (adapted from test_instrument_management_service_integration)
tdom, sdom = time_series_domain()
tdom = tdom.dump()
sdom = sdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='DataProduct test',
processing_level_code='Parsed_Canonical',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=self.parsed_stream_def_id)
log.debug('data_product_id1 = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=self.device_id, data_product_id=data_product_id1)
self.DP.activate_data_product_persistence(data_product_id=data_product_id1)
#######################################
#######################################
# dataset
stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True)
log.debug( 'Data product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True)
log.debug('Data set for data_product_id1 = %s', dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
#######################################
full_config['platform_config'] = {
'platform_id': platform_id,
'driver_config': DVR_CONFIG,
'network_definition' : self._network_definition_ser
}
agent_instance_obj = IonObject(RT.PlatformAgentInstance,
name='%s_PlatformAgentInstance' % platform_id,
description="%s_PlatformAgentInstance" % platform_id,
agent_config=full_config)
agent_instance_id = self.IMS.create_platform_agent_instance(
platform_agent_instance = agent_instance_obj,
platform_agent_id = agent_id,
platform_device_id = self.device_id)
stream_id = stream_ids[0]
self._start_data_subscriber(agent_instance_id, stream_id)
log.debug("about to call imsclient.start_platform_agent_instance with id=%s", agent_instance_id)
pid = self.IMS.start_platform_agent_instance(platform_agent_instance_id=agent_instance_id)
log.debug("start_platform_agent_instance returned pid=%s", pid)
#wait for start
instance_obj = self.IMS.read_platform_agent_instance(agent_instance_id)
gate = ProcessStateGate(self.PDC.read_process,
instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(90), "The platform agent instance did not spawn in 90 seconds")
agent_instance_obj= self.IMS.read_instrument_agent_instance(agent_instance_id)
log.debug('Platform agent instance obj')
# Start a resource agent client to talk with the instrument agent.
self._pa_client = ResourceAgentClient('paclient',
name=agent_instance_obj.agent_process_id,
process=FakeProcess())
log.debug("got platform agent client %s", str(self._pa_client))
# ping_agent can be issued before INITIALIZE
retval = self._pa_client.ping_agent(timeout=TIMEOUT)
log.debug('Base Platform ping_agent = %s', str(retval) )
cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug( 'Base Platform INITIALIZE = %s', str(retval) )
# GO_ACTIVE
cmd = AgentCommand(command=PlatformAgentEvent.GO_ACTIVE)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug( 'Base Platform GO_ACTIVE = %s', str(retval) )
# RUN:
cmd = AgentCommand(command=PlatformAgentEvent.RUN)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug( 'Base Platform RUN = %s', str(retval) )
# START_MONITORING:
cmd = AgentCommand(command=PlatformAgentEvent.START_MONITORING)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug( 'Base Platform START_MONITORING = %s', str(retval) )
# wait for data sample
# just wait for at least one -- see consume_data above
log.info("waiting for reception of a data sample...")
self._async_data_result.get(timeout=DATA_TIMEOUT)
self.assertTrue(len(self._samples_received) >= 1)
log.info("waiting a bit more for reception of more data samples...")
sleep(15)
log.info("Got data samples: %d", len(self._samples_received))
# wait for event
# just wait for at least one event -- see consume_event above
log.info("waiting for reception of an event...")
self._async_event_result.get(timeout=EVENT_TIMEOUT)
log.info("Received events: %s", len(self._events_received))
#get the extended platfrom which wil include platform aggreate status fields
# extended_platform = self.IMS.get_platform_device_extension(self.device_id)
# log.debug( 'test_single_platform extended_platform: %s', str(extended_platform) )
# log.debug( 'test_single_platform power_status_roll_up: %s', str(extended_platform.computed.power_status_roll_up.value) )
# log.debug( 'test_single_platform comms_status_roll_up: %s', str(extended_platform.computed.communications_status_roll_up.value) )
# STOP_MONITORING:
cmd = AgentCommand(command=PlatformAgentEvent.STOP_MONITORING)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug( 'Base Platform STOP_MONITORING = %s', str(retval) )
# GO_INACTIVE
cmd = AgentCommand(command=PlatformAgentEvent.GO_INACTIVE)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug( 'Base Platform GO_INACTIVE = %s', str(retval) )
# RESET: Resets the base platform agent, which includes termination of
# its sub-platforms processes:
cmd = AgentCommand(command=PlatformAgentEvent.RESET)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug( 'Base Platform RESET = %s', str(retval) )
#-------------------------------
# Stop Base Platform AgentInstance
#-------------------------------
self.IMS.stop_platform_agent_instance(platform_agent_instance_id=agent_instance_id)
class BaseIntTestPlatform(IonIntegrationTestCase, HelperTestMixin):
"""
A base class with several conveniences supporting specific platform agent
integration tests, see:
- ion/agents/platform/test/test_platform_agent_with_rsn.py
- ion/services/sa/observatory/test/test_platform_launch.py
The platform IDs used here are organized as follows:
Node1D -> MJ01C -> LJ01D
where -> goes from parent platform to child platform.
This is a subset of the whole topology defined in the simulated platform
network (network.yml), which in turn is used by the RSN OMS simulator.
- 'LJ01D' is the root platform used in test_single_platform
- 'Node1D' is the root platform used in test_hierarchy
Methods are provided to construct specific platform topologies, but
subclasses decide which to use.
"""
@classmethod
def setUpClass(cls):
HelperTestMixin.setUpClass()
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient()
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
self.org_id = self.RR2.create(any_old(RT.Org))
log.debug("Org created: %s", self.org_id)
# Use the network definition provided by RSN OMS directly.
rsn_oms = CIOMSClientFactory.create_instance(DVR_CONFIG['oms_uri'])
self._network_definition = RsnOmsUtil.build_network_definition(rsn_oms)
CIOMSClientFactory.destroy_instance(rsn_oms)
# get serialized version for the configuration:
self._network_definition_ser = NetworkUtil.serialize_network_definition(self._network_definition)
log.trace("NetworkDefinition serialization:\n%s", self._network_definition_ser)
# set attributes for the platforms:
self._platform_attributes = {}
for platform_id in self._network_definition.pnodes:
pnode = self._network_definition.pnodes[platform_id]
dic = dict((attr.attr_id, attr.defn) for attr in pnode.attrs.itervalues())
self._platform_attributes[platform_id] = dic
log.trace("_platform_attributes: %s", self._platform_attributes)
# set ports for the platforms:
self._platform_ports = {}
for platform_id in self._network_definition.pnodes:
pnode = self._network_definition.pnodes[platform_id]
dic = {}
for port_id, port in pnode.ports.iteritems():
dic[port_id] = dict(port_id=port_id,
network=port.network)
self._platform_ports[platform_id] = dic
log.trace("_platform_ports: %s", self._platform_attributes)
self._async_data_result = AsyncResult()
self._data_subscribers = []
self._samples_received = []
self.addCleanup(self._stop_data_subscribers)
self._async_event_result = AsyncResult()
self._event_subscribers = []
self._events_received = []
self.addCleanup(self._stop_event_subscribers)
self._start_event_subscriber()
#################################################################
# data subscribers handling
#################################################################
def _start_data_subscriber(self, stream_name, stream_id):
"""
Starts data subscriber for the given stream_name and stream_config
"""
def consume_data(message, stream_route, stream_id):
# A callback for processing subscribed-to data.
log.info('Subscriber received data message: %s. stream_name=%r stream_id=%r',
str(message), stream_name, stream_id)
self._samples_received.append(message)
self._async_data_result.set()
log.info('_start_data_subscriber stream_name=%r stream_id=%r',
stream_name, stream_id)
# Create subscription for the stream
exchange_name = '%s_queue' % stream_name
self.container.ex_manager.create_xn_queue(exchange_name).purge()
sub = StandaloneStreamSubscriber(exchange_name, consume_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = self.PSC.create_subscription(name=exchange_name, stream_ids=[stream_id])
self.PSC.activate_subscription(sub_id)
sub.subscription_id = sub_id
def _stop_data_subscribers(self):
"""
Stop the data subscribers on cleanup.
"""
try:
for sub in self._data_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self.PSC.deactivate_subscription(sub.subscription_id)
except:
pass
self.PSC.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._data_subscribers = []
#################################################################
# event subscribers handling
#################################################################
def _start_event_subscriber(self, event_type="DeviceEvent", sub_type="platform_event"):
"""
Starts event subscriber for events of given event_type ("DeviceEvent"
by default) and given sub_type ("platform_event" by default).
"""
def consume_event(evt, *args, **kwargs):
# A callback for consuming events.
log.info('Event subscriber received evt: %s.', str(evt))
self._events_received.append(evt)
self._async_event_result.set(evt)
sub = EventSubscriber(event_type=event_type,
sub_type=sub_type,
callback=consume_event)
sub.start()
log.info("registered event subscriber for event_type=%r, sub_type=%r",
event_type, sub_type)
self._event_subscribers.append(sub)
sub._ready_event.wait(timeout=EVENT_TIMEOUT)
def _stop_event_subscribers(self):
"""
Stops the event subscribers on cleanup.
"""
try:
for sub in self._event_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self.PSC.deactivate_subscription(sub.subscription_id)
except:
pass
self.PSC.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._event_subscribers = []
#################################################################
# config supporting methods
#################################################################
def _create_platform_config_builder(self):
clients = DotDict()
clients.resource_registry = self.RR
clients.pubsub_management = self.PSC
clients.dataset_management = self.DSC
pconfig_builder = PlatformAgentConfigurationBuilder(clients)
# can't do anything without an agent instance obj
log.debug("Testing that preparing a launcher without agent instance raises an error")
self.assertRaises(AssertionError, pconfig_builder.prepare, will_launch=False)
return pconfig_builder
def _generate_parent_with_child_config(self, p_parent, p_child):
log.debug("Testing parent platform + child platform as parent config")
pconfig_builder = self._create_platform_config_builder()
pconfig_builder.set_agent_instance_object(p_parent.platform_agent_instance_obj)
parent_config = pconfig_builder.prepare(will_launch=False)
self._verify_parent_config(parent_config,
p_parent.platform_device_id,
p_child.platform_device_id,
is_platform=True)
self._debug_config(parent_config,
"platform_agent_config_%s_->_%s.txt" % (
p_parent.platform_id, p_child.platform_id))
def _generate_platform_with_instrument_config(self, p_obj, i_obj):
log.debug("Testing parent platform + child instrument as parent config")
pconfig_builder = self._create_platform_config_builder()
pconfig_builder.set_agent_instance_object(p_obj.platform_agent_instance_obj)
parent_config = pconfig_builder.prepare(will_launch=False)
self._verify_parent_config(parent_config,
p_obj.platform_device_id,
i_obj.instrument_device_id,
is_platform=False)
self._debug_config(parent_config,
"platform_agent_config_%s_->_%s.txt" % (
p_obj.platform_id, i_obj.instrument_device_id))
def _generate_config(self, platform_agent_instance_obj, platform_id, suffix=''):
pconfig_builder = self._create_platform_config_builder()
pconfig_builder.set_agent_instance_object(platform_agent_instance_obj)
config = pconfig_builder.prepare(will_launch=False)
self._debug_config(config, "platform_agent_config_%s%s.txt" % (platform_id, suffix))
return config
def _get_platform_stream_configs(self):
"""
This method is an adaptation of get_streamConfigs in
test_driver_egg.py
"""
return [
StreamConfiguration(stream_name='parsed',
parameter_dictionary_name='platform_eng_parsed',
records_per_granule=2,
granule_publish_rate=5)
# TODO include a "raw" stream?
]
def _get_instrument_stream_configs(self):
"""
configs copied from test_activate_instrument.py
"""
return [
StreamConfiguration(stream_name='ctd_raw',
parameter_dictionary_name='ctd_raw_param_dict',
records_per_granule=2,
granule_publish_rate=5),
StreamConfiguration(stream_name='ctd_parsed',
parameter_dictionary_name='ctd_parsed_param_dict',
records_per_granule=2, granule_publish_rate=5)
]
def _create_instrument_config_builder(self):
clients = DotDict()
clients.resource_registry = self.RR
clients.pubsub_management = self.PSC
clients.dataset_management = self.DSC
iconfig_builder = InstrumentAgentConfigurationBuilder(clients)
return iconfig_builder
def _generate_instrument_config(self, instrument_agent_instance_obj, instrument_id, suffix=''):
pconfig_builder = self._create_instrument_config_builder()
pconfig_builder.set_agent_instance_object(instrument_agent_instance_obj)
config = pconfig_builder.prepare(will_launch=False)
self._debug_config(config, "instrument_agent_config_%s%s.txt" % (instrument_id, suffix))
return config
def _debug_config(self, config, outname):
if log.isEnabledFor(logging.DEBUG):
import pprint
outname = "logs/%s" % outname
try:
pprint.PrettyPrinter(stream=file(outname, "w")).pprint(config)
log.debug("config pretty-printed to %s", outname)
except Exception as e:
log.warn("error printing config to %s: %s", outname, e)
def _verify_child_config(self, config, device_id, is_platform):
for key in required_config_keys:
self.assertIn(key, config)
if is_platform:
self.assertEqual(RT.PlatformDevice, config['device_type'])
for key in DVR_CONFIG.iterkeys():
self.assertIn(key, config['driver_config'])
for key in ['children', 'startup_config']:
self.assertEqual({}, config[key])
else:
self.assertEqual(RT.InstrumentDevice, config['device_type'])
for key in ['children']:
self.assertEqual({}, config[key])
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertIn('stream_config', config)
def _verify_parent_config(self, config, parent_device_id,
child_device_id, is_platform):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual(RT.PlatformDevice, config['device_type'])
for key in DVR_CONFIG.iterkeys():
self.assertIn(key, config['driver_config'])
self.assertEqual({'resource_id': parent_device_id}, config['agent'])
self.assertIn('stream_config', config)
for key in ['startup_config']:
self.assertEqual({}, config[key])
self.assertIn(child_device_id, config['children'])
self._verify_child_config(config['children'][child_device_id],
child_device_id, is_platform)
def _create_platform_configuration(self, platform_id, parent_platform_id=None):
"""
This method is an adaptation of test_agent_instance_config in
test_instrument_management_service_integration.py
@param platform_id
@param parent_platform_id
@return a DotDict with various of the constructed elements associated
to the platform.
"""
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
#
# TODO will each platform have its own param dictionary?
#
param_dict_name = 'platform_eng_parsed'
parsed_rpdict_id = self.DSC.read_parameter_dictionary_by_name(
param_dict_name,
id_only=True)
self.parsed_stream_def_id = self.PSC.create_stream_definition(
name='parsed',
parameter_dictionary_id=parsed_rpdict_id)
def _make_platform_agent_structure(agent_config=None):
if None is agent_config: agent_config = {}
driver_config = copy.deepcopy(DVR_CONFIG)
driver_config['attributes'] = self._platform_attributes[platform_id]
driver_config['ports'] = self._platform_ports[platform_id]
log.debug("driver_config: %s", driver_config)
# instance creation
platform_agent_instance_obj = any_old(RT.PlatformAgentInstance, {
'driver_config': driver_config})
platform_agent_instance_obj.agent_config = agent_config
platform_agent_instance_id = self.IMS.create_platform_agent_instance(platform_agent_instance_obj)
# agent creation
platform_agent_obj = any_old(RT.PlatformAgent, {
"stream_configurations": self._get_platform_stream_configs(),
'driver_module': DVR_MOD,
'driver_class': DVR_CLS})
platform_agent_id = self.IMS.create_platform_agent(platform_agent_obj)
# device creation
platform_device_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice))
# data product creation
dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom})
dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=self.parsed_stream_def_id)
self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
# assignments
self.RR2.assign_platform_agent_instance_to_platform_device(platform_agent_instance_id, platform_device_id)
self.RR2.assign_platform_agent_to_platform_agent_instance(platform_agent_id, platform_agent_instance_id)
self.RR2.assign_platform_device_to_org_with_has_resource(platform_agent_instance_id, self.org_id)
#######################################
# dataset
log.debug('data product = %s', dp_id)
stream_ids, _ = self.RR.find_objects(dp_id, PRED.hasStream, None, True)
log.debug('Data product stream_ids = %s', stream_ids)
stream_id = stream_ids[0]
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.RR.find_objects(dp_id, PRED.hasDataset, RT.Dataset, True)
log.debug('Data set for data_product_id1 = %s', dataset_ids[0])
#######################################
return platform_agent_instance_id, platform_agent_id, platform_device_id, stream_id
log.debug("Making the structure for a platform agent")
# TODO Note: the 'platform_config' entry is a mechanism that the
# platform agent expects to know the platform_id and parent_platform_id.
# Determine how to finally indicate this info.
platform_config = {
'platform_id': platform_id,
'parent_platform_id': parent_platform_id,
}
child_agent_config = {
'platform_config': platform_config
}
platform_agent_instance_child_id, _, platform_device_child_id, stream_id = \
_make_platform_agent_structure(child_agent_config)
platform_agent_instance_child_obj = self.RR2.read(platform_agent_instance_child_id)
child_config = self._generate_config(platform_agent_instance_child_obj, platform_id)
self._verify_child_config(child_config, platform_device_child_id,
is_platform=True)
self.platform_device_parent_id = platform_device_child_id
p_obj = DotDict()
p_obj.platform_id = platform_id
p_obj.parent_platform_id = parent_platform_id
p_obj.agent_config = child_config
p_obj.platform_agent_instance_obj = platform_agent_instance_child_obj
p_obj.platform_device_id = platform_device_child_id
p_obj.platform_agent_instance_id = platform_agent_instance_child_id
p_obj.stream_id = stream_id
return p_obj
def _create_platform(self, platform_id, parent_platform_id=None):
"""
The main method to create a platform configuration and do other
preparations for a given platform.
"""
p_obj = self._create_platform_configuration(platform_id, parent_platform_id)
# start corresponding data subscriber:
self._start_data_subscriber(p_obj.platform_agent_instance_id,
p_obj.stream_id)
return p_obj
#################################################################
# platform child-parent linking
#################################################################
def _assign_child_to_parent(self, p_child, p_parent):
log.debug("assigning child platform %r to parent %r",
p_child.platform_id, p_parent.platform_id)
self.RR2.assign_platform_device_to_platform_device(p_child.platform_device_id,
p_parent.platform_device_id)
child_device_ids = self.RR2.find_platform_device_ids_of_device(p_parent.platform_device_id)
self.assertNotEqual(0, len(child_device_ids))
self._generate_parent_with_child_config(p_parent, p_child)
#################################################################
# instrument
#################################################################
def _set_up_pre_environment_for_instrument(self):
"""
From test_instrument_agent.py
Basically, this method launches the port agent and the completes the
instrument driver configuration used to properly set up the
instrument agent.
@return instrument_driver_config
"""
import sys
from ion.agents.instrument.driver_process import DriverProcessType
from ion.agents.instrument.driver_process import ZMQEggDriverProcess
# A seabird driver.
DRV_URI = 'http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.7-py2.7.egg'
DRV_MOD = 'mi.instrument.seabird.sbe37smb.ooicore.driver'
DRV_CLS = 'SBE37Driver'
WORK_DIR = '/tmp/'
DELIM = ['<<', '>>']
instrument_driver_config = {
'dvr_egg' : DRV_URI,
'dvr_mod' : DRV_MOD,
'dvr_cls' : DRV_CLS,
'workdir' : WORK_DIR,
'process_type' : None
}
# Launch from egg or a local MI repo.
LAUNCH_FROM_EGG=True
if LAUNCH_FROM_EGG:
# Dynamically load the egg into the test path
launcher = ZMQEggDriverProcess(instrument_driver_config)
egg = launcher._get_egg(DRV_URI)
if not egg in sys.path: sys.path.insert(0, egg)
instrument_driver_config['process_type'] = (DriverProcessType.EGG,)
else:
mi_repo = os.getcwd() + os.sep + 'extern' + os.sep + 'mi_repo'
if not mi_repo in sys.path: sys.path.insert(0, mi_repo)
instrument_driver_config['process_type'] = (DriverProcessType.PYTHON_MODULE,)
instrument_driver_config['mi_repo'] = mi_repo
DEV_ADDR = CFG.device.sbe37.host
DEV_PORT = CFG.device.sbe37.port
DATA_PORT = CFG.device.sbe37.port_agent_data_port
CMD_PORT = CFG.device.sbe37.port_agent_cmd_port
PA_BINARY = CFG.device.sbe37.port_agent_binary
self._support = DriverIntegrationTestSupport(None,
None,
DEV_ADDR,
DEV_PORT,
DATA_PORT,
CMD_PORT,
PA_BINARY,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
port = self._support.start_pagent()
log.info('Port agent started at port %i', port)
self.addCleanup(self._support.stop_pagent)
# Configure instrument driver to use port agent port number.
instrument_driver_config['comms_config'] = {
'addr': 'localhost',
'port': port,
'cmd_port': CMD_PORT
}
return instrument_driver_config
def _make_instrument_agent_structure(self, org_obj, agent_config=None):
if None is agent_config: agent_config = {}
# from test_activate_instrument:test_activateInstrumentSample
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.IMS.create_instrument_model(instModel_obj)
log.debug('new InstrumentModel id = %s ', instModel_id)
# agent creation
instrument_agent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1a-py2.7.egg",
stream_configurations=self._get_instrument_stream_configs())
instrument_agent_id = self.IMS.create_instrument_agent(instrument_agent_obj)
log.debug('new InstrumentAgent id = %s', instrument_agent_id)
self.IMS.assign_instrument_model_to_instrument_agent(instModel_id, instrument_agent_id)
# device creation
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345")
instrument_device_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj)
self.IMS.assign_instrument_model_to_instrument_device(instModel_id, instrument_device_id)
log.debug("new InstrumentDevice id = %s ", instrument_device_id)
#Create stream alarms
alert_def = {
'name' : 'temperature_warning_interval',
'stream_name' : 'ctd_parsed',
'message' : 'Temperature is below the normal range of 50.0 and above.',
'alert_type' : StreamAlertType.WARNING,
'value_id' : 'temp',
'resource_id' : instrument_device_id,
'origin_type' : 'device',
'lower_bound' : 50.0,
'lower_rel_op' : '<',
'alert_class' : 'IntervalAlert'
}
instrument_driver_config = self._set_up_pre_environment_for_instrument()
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
# instance creation
instrument_agent_instance_obj = IonObject(RT.InstrumentAgentInstance,
name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
driver_config=instrument_driver_config,
port_agent_config=port_agent_config,
alerts=[alert_def])
instrument_agent_instance_obj.agent_config = agent_config
instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(instrument_agent_instance_obj)
# data products
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
org_id = self.RR2.create(org_obj)
# parsed:
parsed_pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.PSC.create_stream_definition(
name='ctd_parsed',
parameter_dictionary_id=parsed_pdict_id)
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id1 = self.DP.create_data_product(data_product=dp_obj,
stream_definition_id=parsed_stream_def_id)
self.DP.activate_data_product_persistence(data_product_id=data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id,
data_product_id=data_product_id1)
# raw:
raw_pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.PSC.create_stream_definition(
name='ctd_raw',
parameter_dictionary_id=raw_pdict_id)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id2 = self.DP.create_data_product(data_product=dp_obj,
stream_definition_id=raw_stream_def_id)
self.DP.activate_data_product_persistence(data_product_id=data_product_id2)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id,
data_product_id=data_product_id2)
# assignments
self.RR2.assign_instrument_agent_instance_to_instrument_device(instrument_agent_instance_id, instrument_device_id)
self.RR2.assign_instrument_agent_to_instrument_agent_instance(instrument_agent_id, instrument_agent_instance_id)
self.RR2.assign_instrument_device_to_org_with_has_resource(instrument_agent_instance_id, org_id)
i_obj = DotDict()
i_obj.instrument_agent_id = instrument_agent_id
i_obj.instrument_device_id = instrument_device_id
i_obj.instrument_agent_instance_id = instrument_agent_instance_id
return i_obj
def verify_instrument_config(self, config, org_obj, device_id):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual(org_obj.name, config['org_name'])
self.assertEqual(RT.InstrumentDevice, config['device_type'])
self.assertIn('driver_config', config)
driver_config = config['driver_config']
expected_driver_fields = {'process_type': ('ZMQEggDriverLauncher',),
}
for k, v in expected_driver_fields.iteritems():
self.assertIn(k, driver_config)
self.assertEqual(v, driver_config[k])
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertIn('stream_config', config)
for key in ['children']:
self.assertEqual({}, config[key])
def _create_instrument(self):
"""
The main method to create an instrument configuration.
"""
iconfig_builder = self._create_instrument_config_builder()
org_obj = any_old(RT.Org)
log.debug("making the structure for an instrument agent")
i_obj = self._make_instrument_agent_structure(org_obj)
instrument_agent_instance_obj = self.RR2.read(i_obj.instrument_agent_instance_id)
log.debug("Testing instrument config")
iconfig_builder.set_agent_instance_object(instrument_agent_instance_obj)
instrument_config = iconfig_builder.prepare(will_launch=False)
self.verify_instrument_config(instrument_config, org_obj,
i_obj.instrument_device_id)
self._generate_instrument_config(instrument_agent_instance_obj,
i_obj.instrument_agent_instance_id)
return i_obj
#################################################################
# instrument-platform linking
#################################################################
def _assign_instrument_to_platform(self, i_obj, p_obj):
log.debug("assigning instrument %r to platform %r",
i_obj.instrument_agent_instance_id, p_obj.platform_id)
self.RR2.assign_instrument_device_to_platform_device(
i_obj.instrument_device_id,
p_obj.platform_device_id)
child_device_ids = self.RR2.find_instrument_device_ids_of_device(p_obj.platform_device_id)
self.assertNotEqual(0, len(child_device_ids))
self._generate_platform_with_instrument_config(p_obj, i_obj)
#################################################################
# some platform topologies
#################################################################
def _create_single_platform(self):
"""
Creates and prepares a platform corresponding to the
platform ID 'LJ01D', which is a leaf in the simulated network.
"""
p_root = self._create_platform('LJ01D')
return p_root
def _create_small_hierarchy(self):
"""
Creates a small platform network consisting of 3 platforms as follows:
Node1D -> MJ01C -> LJ01D
where -> goes from parent to child.
"""
p_root = self._create_platform('Node1D')
p_child = self._create_platform('MJ01C', parent_platform_id='Node1D')
p_grandchild = self._create_platform('LJ01D', parent_platform_id='MJ01C')
self._assign_child_to_parent(p_child, p_root)
self._assign_child_to_parent(p_grandchild, p_child)
self._generate_config(p_root.platform_agent_instance_obj, p_root.platform_id, "_final")
return p_root
#################################################################
# start / stop platform
#################################################################
def _start_platform(self, agent_instance_id):
log.debug("about to call start_platform_agent_instance with id=%s", agent_instance_id)
pid = self.IMS.start_platform_agent_instance(platform_agent_instance_id=agent_instance_id)
log.debug("start_platform_agent_instance returned pid=%s", pid)
#wait for start
agent_instance_obj = self.IMS.read_platform_agent_instance(agent_instance_id)
gate = ProcessStateGate(self.PDC.read_process,
agent_instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(90), "The platform agent instance did not spawn in 90 seconds")
# Start a resource agent client to talk with the agent.
self._pa_client = ResourceAgentClient('paclient',
name=agent_instance_obj.agent_process_id,
process=FakeProcess())
log.debug("got platform agent client %s", str(self._pa_client))
def _stop_platform(self, agent_instance_id):
self.IMS.stop_platform_agent_instance(platform_agent_instance_id=agent_instance_id)
#################################################################
# start / stop instrument
#################################################################
def _start_instrument(self, agent_instance_id):
log.debug("about to call start_instrument_agent_instance with id=%s", agent_instance_id)
pid = self.IMS.start_instrument_agent_instance(instrument_agent_instance_id=agent_instance_id)
log.debug("start_instrument_agent_instance returned pid=%s", pid)
#wait for start
agent_instance_obj = self.IMS.read_instrument_agent_instance(agent_instance_id)
gate = ProcessStateGate(self.PDC.read_process,
agent_instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(90), "The instrument agent instance did not spawn in 90 seconds")
# Start a resource agent client to talk with the agent.
self._ia_client = ResourceAgentClient('paclient',
name=agent_instance_obj.agent_process_id,
process=FakeProcess())
log.debug("got instrument agent client %s", str(self._ia_client))
def _stop_instrument(self, agent_instance_id):
self.IMS.stop_instrument_agent_instance(instrument_agent_instance_id=agent_instance_id)
#################################################################
# misc convenience methods
#################################################################
def _get_state(self):
state = self._pa_client.get_agent_state()
return state
def _assert_state(self, state):
self.assertEquals(self._get_state(), state)
def _execute_agent(self, cmd):
log.info("_execute_agent: cmd=%r kwargs=%r ...", cmd.command, cmd.kwargs)
time_start = time.time()
#retval = self._pa_client.execute_agent(cmd, timeout=timeout)
retval = self._pa_client.execute_agent(cmd)
elapsed_time = time.time() - time_start
log.info("_execute_agent: cmd=%r elapsed_time=%s, retval = %s",
cmd.command, elapsed_time, str(retval))
return retval
#################################################################
# commands that concrete tests can call
#################################################################
def _ping_agent(self):
retval = self._pa_client.ping_agent()
self.assertIsInstance(retval, str)
def _ping_resource(self):
cmd = AgentCommand(command=PlatformAgentEvent.PING_RESOURCE)
if self._get_state() == PlatformAgentState.UNINITIALIZED:
# should get ServerError: "Command not handled in current state"
with self.assertRaises(ServerError):
#self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
self._pa_client.execute_agent(cmd)
else:
# In all other states the command should be accepted:
retval = self._execute_agent(cmd)
self.assertEquals("PONG", retval.result)
def _get_metadata(self):
cmd = AgentCommand(command=PlatformAgentEvent.GET_METADATA)
retval = self._execute_agent(cmd)
md = retval.result
self.assertIsInstance(md, dict)
# TODO verify possible subset of required entries in the dict.
log.info("GET_METADATA = %s", md)
def _get_ports(self):
cmd = AgentCommand(command=PlatformAgentEvent.GET_PORTS)
retval = self._execute_agent(cmd)
md = retval.result
self.assertIsInstance(md, dict)
# TODO verify possible subset of required entries in the dict.
log.info("GET_PORTS = %s", md)
def _initialize(self):
self._assert_state(PlatformAgentState.UNINITIALIZED)
cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.INACTIVE)
def _go_active(self):
cmd = AgentCommand(command=PlatformAgentEvent.GO_ACTIVE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.IDLE)
def _run(self):
cmd = AgentCommand(command=PlatformAgentEvent.RUN)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.COMMAND)
def _start_resource_monitoring(self):
cmd = AgentCommand(command=PlatformAgentEvent.START_MONITORING)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.MONITORING)
def _wait_for_a_data_sample(self):
log.info("waiting for reception of a data sample...")
# just wait for at least one -- see consume_data
self._async_data_result.get(timeout=DATA_TIMEOUT)
self.assertTrue(len(self._samples_received) >= 1)
log.info("Received samples: %s", len(self._samples_received))
def _wait_for_external_event(self):
log.info("waiting for reception of an external event...")
# just wait for at least one -- see consume_event
self._async_event_result.get(timeout=EVENT_TIMEOUT)
self.assertTrue(len(self._events_received) >= 1)
log.info("Received events: %s", len(self._events_received))
def _stop_resource_monitoring(self):
cmd = AgentCommand(command=PlatformAgentEvent.STOP_MONITORING)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.COMMAND)
def _pause(self):
cmd = AgentCommand(command=PlatformAgentEvent.PAUSE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.STOPPED)
def _resume(self):
cmd = AgentCommand(command=PlatformAgentEvent.RESUME)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.COMMAND)
def _clear(self):
cmd = AgentCommand(command=PlatformAgentEvent.CLEAR)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.IDLE)
def _go_inactive(self):
cmd = AgentCommand(command=PlatformAgentEvent.GO_INACTIVE)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.INACTIVE)
def _reset(self):
cmd = AgentCommand(command=PlatformAgentEvent.RESET)
retval = self._execute_agent(cmd)
self._assert_state(PlatformAgentState.UNINITIALIZED)
def _check_sync(self):
cmd = AgentCommand(command=PlatformAgentEvent.CHECK_SYNC)
retval = self._execute_agent(cmd)
log.info("CHECK_SYNC result: %s", retval.result)
self.assertTrue(retval.result is not None)
self.assertEquals(retval.result[0:3], "OK:")
return retval.result
class TestPlatformLaunch(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient()
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
# Use the network definition provided by RSN OMS directly.
rsn_oms = CIOMSClientFactory.create_instance(DVR_CONFIG['oms_uri'])
self._network_definition = RsnOmsUtil.build_network_definition(rsn_oms)
# get serialized version for the configuration:
self._network_definition_ser = NetworkUtil.serialize_network_definition(
self._network_definition)
if log.isEnabledFor(logging.TRACE):
log.trace("NetworkDefinition serialization:\n%s",
self._network_definition_ser)
self._async_data_result = AsyncResult()
self._data_subscribers = []
self._samples_received = []
self.addCleanup(self._stop_data_subscribers)
self._async_event_result = AsyncResult()
self._event_subscribers = []
self._events_received = []
self.addCleanup(self._stop_event_subscribers)
self._start_event_subscriber()
def _start_data_subscriber(self, stream_name, stream_id):
"""
Starts data subscriber for the given stream_name and stream_config
"""
def consume_data(message, stream_route, stream_id):
# A callback for processing subscribed-to data.
log.info('Subscriber received data message: %s.', str(message))
self._samples_received.append(message)
self._async_data_result.set()
log.info('_start_data_subscriber stream_name=%r stream_id=%r',
stream_name, stream_id)
# Create subscription for the stream
exchange_name = '%s_queue' % stream_name
self.container.ex_manager.create_xn_queue(exchange_name).purge()
sub = StandaloneStreamSubscriber(exchange_name, consume_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = self.PSC.create_subscription(name=exchange_name,
stream_ids=[stream_id])
self.PSC.activate_subscription(sub_id)
sub.subscription_id = sub_id
def _stop_data_subscribers(self):
"""
Stop the data subscribers on cleanup.
"""
try:
for sub in self._data_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self.PSC.deactivate_subscription(sub.subscription_id)
except:
pass
self.PSC.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._data_subscribers = []
def _start_event_subscriber(self,
event_type="DeviceEvent",
sub_type="platform_event"):
"""
Starts event subscriber for events of given event_type ("DeviceEvent"
by default) and given sub_type ("platform_event" by default).
"""
def consume_event(evt, *args, **kwargs):
# A callback for consuming events.
log.info('Event subscriber received evt: %s.', str(evt))
self._events_received.append(evt)
self._async_event_result.set(evt)
sub = EventSubscriber(event_type=event_type,
sub_type=sub_type,
callback=consume_event)
sub.start()
log.info("registered event subscriber for event_type=%r, sub_type=%r",
event_type, sub_type)
self._event_subscribers.append(sub)
sub._ready_event.wait(timeout=EVENT_TIMEOUT)
def _stop_event_subscribers(self):
"""
Stops the event subscribers on cleanup.
"""
try:
for sub in self._event_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self.PSC.deactivate_subscription(sub.subscription_id)
except:
pass
self.PSC.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._event_subscribers = []
def _create_platform_configuration(self):
"""
Verify that agent configurations are being built properly
"""
#
# This method is an adaptation of test_agent_instance_config in
# test_instrument_management_service_integration.py
#
clients = DotDict()
clients.resource_registry = self.RR
clients.pubsub_management = self.PSC
clients.dataset_management = self.DSC
pconfig_builder = PlatformAgentConfigurationBuilder(clients)
iconfig_builder = InstrumentAgentConfigurationBuilder(clients)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
org_id = self.RR2.create(any_old(RT.Org))
inst_startup_config = {'startup': 'config'}
required_config_keys = [
'org_name', 'device_type', 'agent', 'driver_config',
'stream_config', 'startup_config', 'alarm_defs', 'children'
]
def verify_instrument_config(config, device_id):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual('Org_1', config['org_name'])
self.assertEqual(RT.InstrumentDevice, config['device_type'])
self.assertIn('driver_config', config)
driver_config = config['driver_config']
expected_driver_fields = {
'process_type': ('ZMQPyClassDriverLauncher', ),
}
for k, v in expected_driver_fields.iteritems():
self.assertIn(k, driver_config)
self.assertEqual(v, driver_config[k])
self.assertEqual
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertEqual(inst_startup_config, config['startup_config'])
self.assertIn('stream_config', config)
for key in ['alarm_defs', 'children']:
self.assertEqual({}, config[key])
def verify_child_config(config, device_id, inst_device_id=None):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual('Org_1', config['org_name'])
self.assertEqual(RT.PlatformDevice, config['device_type'])
self.assertEqual({'process_type': ('ZMQPyClassDriverLauncher', )},
config['driver_config'])
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertIn('stream_config', config)
if None is inst_device_id:
for key in ['alarm_defs', 'children', 'startup_config']:
self.assertEqual({}, config[key])
else:
for key in ['alarm_defs', 'startup_config']:
self.assertEqual({}, config[key])
self.assertIn(inst_device_id, config['children'])
verify_instrument_config(config['children'][inst_device_id],
inst_device_id)
def verify_parent_config(config,
parent_device_id,
child_device_id,
inst_device_id=None):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual('Org_1', config['org_name'])
self.assertEqual(RT.PlatformDevice, config['device_type'])
self.assertEqual({'process_type': ('ZMQPyClassDriverLauncher', )},
config['driver_config'])
self.assertEqual({'resource_id': parent_device_id},
config['agent'])
self.assertIn('stream_config', config)
for key in ['alarm_defs', 'startup_config']:
self.assertEqual({}, config[key])
self.assertIn(child_device_id, config['children'])
verify_child_config(config['children'][child_device_id],
child_device_id, inst_device_id)
parsed_rpdict_id = self.DSC.read_parameter_dictionary_by_name(
'platform_eng_parsed', id_only=True)
self.parsed_stream_def_id = self.PSC.create_stream_definition(
name='parsed', parameter_dictionary_id=parsed_rpdict_id)
rpdict_id = self.DSC.read_parameter_dictionary_by_name(
'ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.PSC.create_stream_definition(
name='raw', parameter_dictionary_id=rpdict_id)
#todo: create org and figure out which agent resource needs to get assigned to it
def _make_platform_agent_structure(agent_config=None):
if None is agent_config: agent_config = {}
# instance creation
platform_agent_instance_obj = any_old(RT.PlatformAgentInstance)
platform_agent_instance_obj.agent_config = agent_config
platform_agent_instance_id = self.IMS.create_platform_agent_instance(
platform_agent_instance_obj)
# agent creation
raw_config = StreamConfiguration(
stream_name='parsed',
parameter_dictionary_name='platform_eng_parsed',
records_per_granule=2,
granule_publish_rate=5)
platform_agent_obj = any_old(
RT.PlatformAgent, {"stream_configurations": [raw_config]})
platform_agent_id = self.IMS.create_platform_agent(
platform_agent_obj)
# device creation
platform_device_id = self.IMS.create_platform_device(
any_old(RT.PlatformDevice))
# data product creation
dp_obj = any_old(RT.DataProduct, {
"temporal_domain": tdom,
"spatial_domain": sdom
})
# dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id)
dp_id = self.DP.create_data_product(
data_product=dp_obj,
stream_definition_id=self.parsed_stream_def_id)
self.DAMS.assign_data_product(input_resource_id=platform_device_id,
data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
# assignments
self.RR2.assign_platform_agent_instance_to_platform_device(
platform_agent_instance_id, platform_device_id)
self.RR2.assign_platform_agent_to_platform_agent_instance(
platform_agent_id, platform_agent_instance_id)
self.RR2.assign_platform_device_to_org_with_has_resource(
platform_agent_instance_id, org_id)
return platform_agent_instance_id, platform_agent_id, platform_device_id
def _make_instrument_agent_structure(agent_config=None):
if None is agent_config: agent_config = {}
# instance creation
instrument_agent_instance_obj = any_old(
RT.InstrumentAgentInstance,
{"startup_config": inst_startup_config})
instrument_agent_instance_obj.agent_config = agent_config
instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(
instrument_agent_instance_obj)
# agent creation
raw_config = StreamConfiguration(
stream_name='raw',
parameter_dictionary_name='ctd_raw_param_dict',
records_per_granule=2,
granule_publish_rate=5)
instrument_agent_obj = any_old(
RT.InstrumentAgent, {"stream_configurations": [raw_config]})
instrument_agent_id = self.IMS.create_instrument_agent(
instrument_agent_obj)
# device creation
instrument_device_id = self.IMS.create_instrument_device(
any_old(RT.InstrumentDevice))
# data product creation
dp_obj = any_old(RT.DataProduct, {
"temporal_domain": tdom,
"spatial_domain": sdom
})
dp_id = self.DP.create_data_product(
data_product=dp_obj, stream_definition_id=raw_stream_def_id)
self.DAMS.assign_data_product(
input_resource_id=instrument_device_id, data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
# assignments
self.RR2.assign_instrument_agent_instance_to_instrument_device(
instrument_agent_instance_id, instrument_device_id)
self.RR2.assign_instrument_agent_to_instrument_agent_instance(
instrument_agent_id, instrument_agent_instance_id)
self.RR2.assign_instrument_device_to_org_with_has_resource(
instrument_agent_instance_id, org_id)
return instrument_agent_instance_id, instrument_agent_id, instrument_device_id
# can't do anything without an agent instance obj
log.debug(
"Testing that preparing a launcher without agent instance raises an error"
)
self.assertRaises(AssertionError,
pconfig_builder.prepare,
will_launch=False)
log.debug(
"Making the structure for a platform agent, which will be the child"
)
platform_agent_instance_child_id, _, platform_device_child_id = _make_platform_agent_structure(
)
platform_agent_instance_child_obj = self.RR2.read(
platform_agent_instance_child_id)
log.debug("Preparing a valid agent instance launch, for config only")
pconfig_builder.set_agent_instance_object(
platform_agent_instance_child_obj)
child_config = pconfig_builder.prepare(will_launch=False)
verify_child_config(child_config, platform_device_child_id)
log.debug(
"Making the structure for a platform agent, which will be the parent"
)
platform_agent_instance_parent_id, _, platform_device_parent_id = _make_platform_agent_structure(
)
platform_agent_instance_parent_obj = self.RR2.read(
platform_agent_instance_parent_id)
log.debug("Testing child-less parent as a child config")
pconfig_builder.set_agent_instance_object(
platform_agent_instance_parent_obj)
parent_config = pconfig_builder.prepare(will_launch=False)
verify_child_config(parent_config, platform_device_parent_id)
log.debug("assigning child platform to parent")
self.RR2.assign_platform_device_to_platform_device(
platform_device_child_id, platform_device_parent_id)
child_device_ids = self.RR2.find_platform_device_ids_of_device(
platform_device_parent_id)
self.assertNotEqual(0, len(child_device_ids))
log.debug("Testing parent + child as parent config")
pconfig_builder.set_agent_instance_object(
platform_agent_instance_parent_obj)
parent_config = pconfig_builder.prepare(will_launch=False)
verify_parent_config(parent_config, platform_device_parent_id,
platform_device_child_id)
log.debug("making the structure for an instrument agent")
instrument_agent_instance_id, _, instrument_device_id = _make_instrument_agent_structure(
)
instrument_agent_instance_obj = self.RR2.read(
instrument_agent_instance_id)
log.debug("Testing instrument config")
iconfig_builder.set_agent_instance_object(
instrument_agent_instance_obj)
instrument_config = iconfig_builder.prepare(will_launch=False)
verify_instrument_config(instrument_config, instrument_device_id)
log.debug("assigning instrument to platform")
self.RR2.assign_instrument_device_to_platform_device(
instrument_device_id, platform_device_child_id)
child_device_ids = self.RR2.find_instrument_device_ids_of_device(
platform_device_child_id)
self.assertNotEqual(0, len(child_device_ids))
log.debug("Testing entire config")
pconfig_builder.set_agent_instance_object(
platform_agent_instance_parent_obj)
full_config = pconfig_builder.prepare(will_launch=False)
verify_parent_config(full_config, platform_device_parent_id,
platform_device_child_id, instrument_device_id)
if log.isEnabledFor(logging.TRACE):
import pprint
pp = pprint.PrettyPrinter()
pp.pprint(full_config)
log.trace("full_config = %s", pp.pformat(full_config))
return full_config
def get_streamConfigs(self):
#
# This method is an adaptation of get_streamConfigs in
# test_driver_egg.py
#
return [
StreamConfiguration(
stream_name='parsed',
parameter_dictionary_name='platform_eng_parsed',
records_per_granule=2,
granule_publish_rate=5)
# TODO enable something like the following when also
# incorporating "raw" data:
#,
#StreamConfiguration(stream_name='raw',
# parameter_dictionary_name='ctd_raw_param_dict',
# records_per_granule=2,
# granule_publish_rate=5)
]
@skip("Still needs alignment with new configuration structure")
def test_hierarchy(self):
# TODO re-implement.
pass
def test_single_platform(self):
full_config = self._create_platform_configuration()
platform_id = 'LJ01D'
stream_configurations = self.get_streamConfigs()
agent__obj = IonObject(RT.PlatformAgent,
name='%s_PlatformAgent' % platform_id,
description='%s_PlatformAgent platform agent' %
platform_id,
stream_configurations=stream_configurations)
agent_id = self.IMS.create_platform_agent(agent__obj)
device__obj = IonObject(
RT.PlatformDevice,
name='%s_PlatformDevice' % platform_id,
description='%s_PlatformDevice platform device' % platform_id,
# ports=port_objs,
# platform_monitor_attributes = monitor_attribute_objs
)
self.device_id = self.IMS.create_platform_device(device__obj)
#######################################
# data product (adapted from test_instrument_management_service_integration)
tdom, sdom = time_series_domain()
tdom = tdom.dump()
sdom = sdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='DataProduct test',
processing_level_code='Parsed_Canonical',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id1 = self.DP.create_data_product(
data_product=dp_obj,
stream_definition_id=self.parsed_stream_def_id)
log.debug('data_product_id1 = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=self.device_id,
data_product_id=data_product_id1)
self.DP.activate_data_product_persistence(
data_product_id=data_product_id1)
#######################################
#######################################
# dataset
stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream,
None, True)
log.debug('Data product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.RR.find_objects(data_product_id1,
PRED.hasDataset, RT.Dataset,
True)
log.debug('Data set for data_product_id1 = %s', dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
#######################################
full_config['platform_config'] = {
'platform_id': platform_id,
'driver_config': DVR_CONFIG,
'network_definition': self._network_definition_ser
}
agent_instance_obj = IonObject(
RT.PlatformAgentInstance,
name='%s_PlatformAgentInstance' % platform_id,
description="%s_PlatformAgentInstance" % platform_id,
agent_config=full_config)
agent_instance_id = self.IMS.create_platform_agent_instance(
platform_agent_instance=agent_instance_obj,
platform_agent_id=agent_id,
platform_device_id=self.device_id)
stream_id = stream_ids[0]
self._start_data_subscriber(agent_instance_id, stream_id)
log.debug(
"about to call imsclient.start_platform_agent_instance with id=%s",
agent_instance_id)
pid = self.IMS.start_platform_agent_instance(
platform_agent_instance_id=agent_instance_id)
log.debug("start_platform_agent_instance returned pid=%s", pid)
#wait for start
instance_obj = self.IMS.read_platform_agent_instance(agent_instance_id)
gate = ProcessStateGate(self.PDC.read_process,
instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(
gate. await (90),
"The platform agent instance did not spawn in 90 seconds")
agent_instance_obj = self.IMS.read_instrument_agent_instance(
agent_instance_id)
log.debug('Platform agent instance obj')
# Start a resource agent client to talk with the instrument agent.
self._pa_client = ResourceAgentClient(
'paclient',
name=agent_instance_obj.agent_process_id,
process=FakeProcess())
log.debug("got platform agent client %s", str(self._pa_client))
# ping_agent can be issued before INITIALIZE
retval = self._pa_client.ping_agent(timeout=TIMEOUT)
log.debug('Base Platform ping_agent = %s', str(retval))
cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform INITIALIZE = %s', str(retval))
# GO_ACTIVE
cmd = AgentCommand(command=PlatformAgentEvent.GO_ACTIVE)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform GO_ACTIVE = %s', str(retval))
# RUN:
cmd = AgentCommand(command=PlatformAgentEvent.RUN)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform RUN = %s', str(retval))
# START_MONITORING:
cmd = AgentCommand(command=PlatformAgentEvent.START_MONITORING)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform START_MONITORING = %s', str(retval))
# wait for data sample
# just wait for at least one -- see consume_data above
log.info("waiting for reception of a data sample...")
self._async_data_result.get(timeout=DATA_TIMEOUT)
self.assertTrue(len(self._samples_received) >= 1)
log.info("waiting a bit more for reception of more data samples...")
sleep(15)
log.info("Got data samples: %d", len(self._samples_received))
# wait for event
# just wait for at least one event -- see consume_event above
log.info("waiting for reception of an event...")
self._async_event_result.get(timeout=EVENT_TIMEOUT)
log.info("Received events: %s", len(self._events_received))
#get the extended platfrom which wil include platform aggreate status fields
# extended_platform = self.IMS.get_platform_device_extension(self.device_id)
# log.debug( 'test_single_platform extended_platform: %s', str(extended_platform) )
# log.debug( 'test_single_platform power_status_roll_up: %s', str(extended_platform.computed.power_status_roll_up.value) )
# log.debug( 'test_single_platform comms_status_roll_up: %s', str(extended_platform.computed.communications_status_roll_up.value) )
# STOP_MONITORING:
cmd = AgentCommand(command=PlatformAgentEvent.STOP_MONITORING)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform STOP_MONITORING = %s', str(retval))
# GO_INACTIVE
cmd = AgentCommand(command=PlatformAgentEvent.GO_INACTIVE)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform GO_INACTIVE = %s', str(retval))
# RESET: Resets the base platform agent, which includes termination of
# its sub-platforms processes:
cmd = AgentCommand(command=PlatformAgentEvent.RESET)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform RESET = %s', str(retval))
#-------------------------------
# Stop Base Platform AgentInstance
#-------------------------------
self.IMS.stop_platform_agent_instance(
platform_agent_instance_id=agent_instance_id)
class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
print 'started services'
# @unittest.skip('this test just for debugging setup')
# def test_just_the_setup(self):
# return
@attr('EXT')
def test_resources_associations_extensions(self):
"""
create one of each resource and association used by IMS
to guard against problems in ion-definitions
"""
#stuff we control
instrument_agent_instance_id, _ = self.RR.create(any_old(RT.InstrumentAgentInstance))
instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent))
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel))
instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice))
platform_agent_instance_id, _ = self.RR.create(any_old(RT.PlatformAgentInstance))
platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent))
platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice))
platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel))
sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice))
sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel))
#stuff we associate to
data_producer_id, _ = self.RR.create(any_old(RT.DataProducer))
org_id, _ = self.RR.create(any_old(RT.Org))
#instrument_agent_instance_id #is only a target
#instrument_agent
self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id)
#instrument_device
self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id)
self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id)
self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id)
self.RR.create_association(org_id, PRED.hasResource, instrument_device_id)
instrument_model_id #is only a target
platform_agent_instance_id #is only a target
#platform_agent
self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id)
#platform_device
self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id)
self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id)
platform_model_id #is only a target
#sensor_device
self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id)
self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id)
sensor_model_id #is only a target
#create a parsed product for this instrument output
tdom, sdom = time_series_domain()
tdom = tdom.dump()
sdom = sdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
processing_level_code='Parsed_Canonical',
temporal_domain = tdom,
spatial_domain = sdom)
pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id)
data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1)
def addInstOwner(inst_id, subject):
actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject})
user_id = self.IDS.create_actor_identity(actor_identity_obj)
user_info_obj = any_old(RT.UserInfo)
user_info_id = self.IDS.create_user_info(user_id, user_info_obj)
self.RR.create_association(inst_id, PRED.hasOwner, user_id)
#Testing multiple instrument owners
addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254")
addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256")
extended_instrument = self.IMS.get_instrument_device_extension(instrument_device_id)
self.assertEqual(instrument_device_id, extended_instrument._id)
self.assertEqual(len(extended_instrument.owners), 2)
self.assertEqual(extended_instrument.instrument_model._id, instrument_model_id)
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(extended_instrument.computed.firmware_version, ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.last_data_received_datetime, ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.last_calibration_datetime, ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue)
self.assertIsInstance(extended_instrument.computed.power_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.communications_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.location_status_roll_up, ComputedIntValue)
log.debug("extended_instrument.computed: %s", extended_instrument.computed)
#check model
inst_model_obj = self.RR.read(instrument_model_id)
self.assertEqual(inst_model_obj.name, extended_instrument.instrument_model.name)
#check agent instance
inst_agent_instance_obj = self.RR.read(instrument_agent_instance_id)
self.assertEqual(inst_agent_instance_obj.name, extended_instrument.agent_instance.name)
#check agent
inst_agent_obj = self.RR.read(instrument_agent_id)
#compound assoc return list of lists so check the first element
self.assertEqual(inst_agent_obj.name, extended_instrument.instrument_agent[0].name)
#check platform device
plat_device_obj = self.RR.read(platform_device_id)
self.assertEqual(plat_device_obj.name, extended_instrument.platform_device.name)
extended_platform = self.IMS.get_platform_device_extension(platform_device_id)
self.assertEqual(1, len(extended_platform.instrument_devices))
self.assertEqual(instrument_device_id, extended_platform.instrument_devices[0]._id)
self.assertEqual(1, len(extended_platform.instrument_models))
self.assertEqual(instrument_model_id, extended_platform.instrument_models[0]._id)
#check sensor devices
self.assertEqual(1, len(extended_instrument.sensor_devices))
#check data_product_parameters_set
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_instrument.computed.data_product_parameters_set.status)
self.assertTrue( 'Parsed_Canonical' in extended_instrument.computed.data_product_parameters_set.value)
# the ctd parameters should include 'temp'
self.assertTrue( 'temp' in extended_instrument.computed.data_product_parameters_set.value['Parsed_Canonical'])
#none of these will work because there is no agent
self.assertEqual(ComputedValueAvailability.NOTAVAILABLE,
extended_instrument.computed.firmware_version.status)
# self.assertEqual(ComputedValueAvailability.NOTAVAILABLE,
# extended_instrument.computed.operational_state.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.power_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.communications_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.data_status_roll_up.status)
# self.assertEqual(StatusType.STATUS_OK,
# extended_instrument.computed.data_status_roll_up.value)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.location_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.recent_events.status)
# self.assertEqual([], extended_instrument.computed.recent_events.value)
# cleanup
c = DotDict()
c.resource_registry = self.RR
self.RR2.pluck(instrument_agent_id)
self.RR2.pluck(instrument_model_id)
self.RR2.pluck(instrument_device_id)
self.RR2.pluck(platform_agent_id)
self.IMS.force_delete_instrument_agent(instrument_agent_id)
self.IMS.force_delete_instrument_model(instrument_model_id)
self.IMS.force_delete_instrument_device(instrument_device_id)
self.IMS.force_delete_platform_agent_instance(platform_agent_instance_id)
self.IMS.force_delete_platform_agent(platform_agent_id)
self.IMS.force_delete_platform_device(platform_device_id)
self.IMS.force_delete_platform_model(platform_model_id)
self.IMS.force_delete_sensor_device(sensor_device_id)
self.IMS.force_delete_sensor_model(sensor_model_id)
#stuff we associate to
self.RR.delete(data_producer_id)
self.RR.delete(org_id)
def test_custom_attributes(self):
"""
Test assignment of custom attributes
"""
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel,
{"custom_attributes":
{"favorite_color": "attr desc goes here"}
}))
instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice,
{"custom_attributes":
{"favorite_color": "red",
"bogus_attr": "should raise warning"
}
}))
self.IMS.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id)
# cleanup
self.IMS.force_delete_instrument_device(instrument_device_id)
self.IMS.force_delete_instrument_model(instrument_model_id)
def _get_datastore(self, dataset_id):
dataset = self.DSC.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def test_resource_state_save_restore(self):
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.IMS.create_instrument_model(instModel_obj)
log.debug( 'new InstrumentModel id = %s ', instModel_id)
# Create InstrumentAgent
raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 )
parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5 )
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg",
stream_configurations = [raw_config, parsed_config] )
instAgent_id = self.IMS.create_instrument_agent(instAgent_obj)
log.debug( 'new InstrumentAgent id = %s', instAgent_id)
self.IMS.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 '
+ '(SA Req: L4-CI-SA-RQ-241) ')
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345" )
instDevice_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj)
self.IMS.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ",
instDevice_id)
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config = port_agent_config)
instAgentInstance_id = self.IMS.create_instrument_agent_instance(instAgentInstance_obj,
instAgent_id,
instDevice_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
spdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=spdict_id)
rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id1 = self.DP.create_data_product(data_product=dp_obj,
stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1)
self.DP.activate_data_product_persistence(data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True)
log.debug( 'Data product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True)
log.debug( 'Data set for data_product_id1 = %s', dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
#create the datastore at the beginning of each int test that persists data
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id2 = self.DP.create_data_product(data_product=dp_obj,
stream_definition_id=raw_stream_def_id)
log.debug( 'new dp_id = %s', str(data_product_id2))
self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2)
self.DP.activate_data_product_persistence(data_product_id=data_product_id2)
# spin up agent
self.IMS.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
self.addCleanup(self.IMS.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
#wait for start
instance_obj = self.IMS.read_instrument_agent_instance(instAgentInstance_id)
gate = ProcessStateGate(self.PDC.read_process,
instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" %
instance_obj.agent_process_id)
# take snapshot of config
snap_id = self.IMS.save_resource_state(instDevice_id, "xyzzy snapshot")
snap_obj = self.RR.read_attachment(snap_id, include_content=True)
print "Saved config:"
print snap_obj.content
#modify config
instance_obj.driver_config["comms_config"] = "BAD_DATA"
self.RR.update(instance_obj)
#restore config
self.IMS.restore_resource_state(instDevice_id, snap_id)
instance_obj = self.RR.read(instAgentInstance_id)
self.assertNotEqual("BAD_DATA", instance_obj.driver_config["comms_config"])
self.DP.delete_data_product(data_product_id1)
self.DP.delete_data_product(data_product_id2)
def test_agent_instance_config(self):
"""
Verify that agent configurations are being built properly
"""
clients = DotDict()
clients.resource_registry = self.RR
clients.pubsub_management = self.PSC
clients.dataset_management = self.DSC
pconfig_builder = PlatformAgentConfigurationBuilder(clients)
iconfig_builder = InstrumentAgentConfigurationBuilder(clients)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
org_id = self.RR2.create(any_old(RT.Org))
inst_startup_config = {'startup': 'config'}
required_config_keys = [
'org_name',
'device_type',
'agent',
'driver_config',
'stream_config',
'startup_config',
'alarm_defs',
'children']
def verify_instrument_config(config, device_id):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual('Org_1', config['org_name'])
self.assertEqual(RT.InstrumentDevice, config['device_type'])
self.assertIn('driver_config', config)
driver_config = config['driver_config']
expected_driver_fields = {'process_type': ('ZMQPyClassDriverLauncher',),
}
for k, v in expected_driver_fields.iteritems():
self.assertIn(k, driver_config)
self.assertEqual(v, driver_config[k])
self.assertEqual
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertEqual(inst_startup_config, config['startup_config'])
self.assertIn('stream_config', config)
for key in ['alarm_defs', 'children']:
self.assertEqual({}, config[key])
def verify_child_config(config, device_id, inst_device_id=None):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual('Org_1', config['org_name'])
self.assertEqual(RT.PlatformDevice, config['device_type'])
self.assertEqual({'process_type': ('ZMQPyClassDriverLauncher',)}, config['driver_config'])
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertIn('stream_config', config)
if None is inst_device_id:
for key in ['alarm_defs', 'children', 'startup_config']:
self.assertEqual({}, config[key])
else:
for key in ['alarm_defs', 'startup_config']:
self.assertEqual({}, config[key])
self.assertIn(inst_device_id, config['children'])
verify_instrument_config(config['children'][inst_device_id], inst_device_id)
def verify_parent_config(config, parent_device_id, child_device_id, inst_device_id=None):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual('Org_1', config['org_name'])
self.assertEqual(RT.PlatformDevice, config['device_type'])
self.assertEqual({'process_type': ('ZMQPyClassDriverLauncher',)}, config['driver_config'])
self.assertEqual({'resource_id': parent_device_id}, config['agent'])
self.assertIn('stream_config', config)
for key in ['alarm_defs', 'startup_config']:
self.assertEqual({}, config[key])
self.assertIn(child_device_id, config['children'])
verify_child_config(config['children'][child_device_id], child_device_id, inst_device_id)
rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id)
#todo: create org and figure out which agent resource needs to get assigned to it
def _make_platform_agent_structure(agent_config=None):
if None is agent_config: agent_config = {}
# instance creation
platform_agent_instance_obj = any_old(RT.PlatformAgentInstance)
platform_agent_instance_obj.agent_config = agent_config
platform_agent_instance_id = self.IMS.create_platform_agent_instance(platform_agent_instance_obj)
# agent creation
raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 )
platform_agent_obj = any_old(RT.PlatformAgent, {"stream_configurations":[raw_config]})
platform_agent_id = self.IMS.create_platform_agent(platform_agent_obj)
# device creation
platform_device_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice))
# data product creation
dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom})
dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id)
self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
# assignments
self.RR2.assign_platform_agent_instance_to_platform_device(platform_agent_instance_id, platform_device_id)
self.RR2.assign_platform_agent_to_platform_agent_instance(platform_agent_id, platform_agent_instance_id)
self.RR2.assign_platform_device_to_org_with_has_resource(platform_agent_instance_id, org_id)
return platform_agent_instance_id, platform_agent_id, platform_device_id
def _make_instrument_agent_structure(agent_config=None):
if None is agent_config: agent_config = {}
# instance creation
instrument_agent_instance_obj = any_old(RT.InstrumentAgentInstance, {"startup_config": inst_startup_config})
instrument_agent_instance_obj.agent_config = agent_config
instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(instrument_agent_instance_obj)
# agent creation
raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 )
instrument_agent_obj = any_old(RT.InstrumentAgent, {"stream_configurations":[raw_config]})
instrument_agent_id = self.IMS.create_instrument_agent(instrument_agent_obj)
# device creation
instrument_device_id = self.IMS.create_instrument_device(any_old(RT.InstrumentDevice))
# data product creation
dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom})
dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
# assignments
self.RR2.assign_instrument_agent_instance_to_instrument_device(instrument_agent_instance_id, instrument_device_id)
self.RR2.assign_instrument_agent_to_instrument_agent_instance(instrument_agent_id, instrument_agent_instance_id)
self.RR2.assign_instrument_device_to_org_with_has_resource(instrument_agent_instance_id, org_id)
return instrument_agent_instance_id, instrument_agent_id, instrument_device_id
# can't do anything without an agent instance obj
log.debug("Testing that preparing a launcher without agent instance raises an error")
self.assertRaises(AssertionError, pconfig_builder.prepare, will_launch=False)
log.debug("Making the structure for a platform agent, which will be the child")
platform_agent_instance_child_id, _, platform_device_child_id = _make_platform_agent_structure()
platform_agent_instance_child_obj = self.RR2.read(platform_agent_instance_child_id)
log.debug("Preparing a valid agent instance launch, for config only")
pconfig_builder.set_agent_instance_object(platform_agent_instance_child_obj)
child_config = pconfig_builder.prepare(will_launch=False)
verify_child_config(child_config, platform_device_child_id)
log.debug("Making the structure for a platform agent, which will be the parent")
platform_agent_instance_parent_id, _, platform_device_parent_id = _make_platform_agent_structure()
platform_agent_instance_parent_obj = self.RR2.read(platform_agent_instance_parent_id)
log.debug("Testing child-less parent as a child config")
pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj)
parent_config = pconfig_builder.prepare(will_launch=False)
verify_child_config(parent_config, platform_device_parent_id)
log.debug("assigning child platform to parent")
self.RR2.assign_platform_device_to_platform_device(platform_device_child_id, platform_device_parent_id)
child_device_ids = self.RR2.find_platform_device_ids_of_device(platform_device_parent_id)
self.assertNotEqual(0, len(child_device_ids))
log.debug("Testing parent + child as parent config")
pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj)
parent_config = pconfig_builder.prepare(will_launch=False)
verify_parent_config(parent_config, platform_device_parent_id, platform_device_child_id)
log.debug("making the structure for an instrument agent")
instrument_agent_instance_id, _, instrument_device_id = _make_instrument_agent_structure()
instrument_agent_instance_obj = self.RR2.read(instrument_agent_instance_id)
log.debug("Testing instrument config")
iconfig_builder.set_agent_instance_object(instrument_agent_instance_obj)
instrument_config = iconfig_builder.prepare(will_launch=False)
verify_instrument_config(instrument_config, instrument_device_id)
log.debug("assigning instrument to platform")
self.RR2.assign_instrument_device_to_platform_device(instrument_device_id, platform_device_child_id)
child_device_ids = self.RR2.find_instrument_device_ids_of_device(platform_device_child_id)
self.assertNotEqual(0, len(child_device_ids))
log.debug("Testing entire config")
pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj)
full_config = pconfig_builder.prepare(will_launch=False)
verify_parent_config(full_config, platform_device_parent_id, platform_device_child_id, instrument_device_id)
#self.fail(parent_config)
#plauncher.prepare(will_launch=False)
def sample_nested_platform_agent_instance_config(self):
"""
for informational purposes
"""
ret = {'org_name': 'Org_1',
'alarm_defs': {},
'driver_config': {'process_type': ('ZMQPyClassDriverLauncher',)},
'stream_config': {'parameter_dictionary': 'lots of stuff'},
'agent': {'resource_id': '33e54106c4444444862da082098bc123'},
'startup_config': {},
'device_type': 'PlatformDevice',
'children': {'76a39596eeff4fd5b409c4cb93f0e581':
{'org_name': 'Org_1',
'alarm_defs': {},
'driver_config': {'process_type': ('ZMQPyClassDriverLauncher',)},
'stream_config': {'parameter_dictionary': 'lots of stuff'},
'agent': {'resource_id': '76a39596eeff4fd5b409c4cb93f0e581'},
'startup_config': {},
'device_type': 'PlatformDevice',
'children': {}}}}
return ret
