def load_data_product(self):
dset_i = 0
dataset_management = DatasetManagementServiceClient()
pubsub_management = PubsubManagementServiceClient()
data_product_management = DataProductManagementServiceClient()
resource_registry = self.container.instance.resource_registry
dp_obj = DataProduct(
name='instrument_data_product_%i' % dset_i,
description='ctd stream test',
processing_level_code='Parsed_Canonical')
pdict_id = dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
stream_def_id = pubsub_management.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id)
self.addCleanup(pubsub_management.delete_stream_definition, stream_def_id)
data_product_id = data_product_management.create_data_product(data_product=dp_obj, stream_definition_id=stream_def_id)
self.addCleanup(data_product_management.delete_data_product, data_product_id)
data_product_management.activate_data_product_persistence(data_product_id)
self.addCleanup(data_product_management.suspend_data_product_persistence, data_product_id)
stream_ids, assocs = resource_registry.find_objects(subject=data_product_id, predicate='hasStream', id_only=True)
stream_id = stream_ids[0]
route = pubsub_management.read_stream_route(stream_id)
dataset_ids, assocs = resource_registry.find_objects(subject=data_product_id, predicate='hasDataset', id_only=True)
dataset_id = dataset_ids[0]
return data_product_id, stream_id, route, stream_def_id, dataset_id
def load_data_product(self):
dset_i = 0
dataset_management = DatasetManagementServiceClient()
pubsub_management = PubsubManagementServiceClient()
data_product_management = DataProductManagementServiceClient()
resource_registry = self.container.instance.resource_registry
tdom, sdom = time_series_domain()
tdom = tdom.dump()
sdom = sdom.dump()
dp_obj = DataProduct(
name='instrument_data_product_%i' % dset_i,
description='ctd stream test',
processing_level_code='Parsed_Canonical',
temporal_domain = tdom,
spatial_domain = sdom)
pdict_id = dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
stream_def_id = pubsub_management.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id)
self.addCleanup(pubsub_management.delete_stream_definition, stream_def_id)
data_product_id = data_product_management.create_data_product(data_product=dp_obj, stream_definition_id=stream_def_id)
self.addCleanup(data_product_management.delete_data_product, data_product_id)
data_product_management.activate_data_product_persistence(data_product_id)
self.addCleanup(data_product_management.suspend_data_product_persistence, data_product_id)
stream_ids, assocs = resource_registry.find_objects(subject=data_product_id, predicate='hasStream', id_only=True)
stream_id = stream_ids[0]
route = pubsub_management.read_stream_route(stream_id)
dataset_ids, assocs = resource_registry.find_objects(subject=data_product_id, predicate='hasDataset', id_only=True)
dataset_id = dataset_ids[0]
return data_product_id, stream_id, route, stream_def_id, dataset_id
class DatasetLoadTest(IonIntegrationTestCase):
"""
The following integration tests (INTMAN) are to ONLY be run manually
"""
def setUp(self):
self.username = CFG.get_safe('eoi.geoserver.user_name', 'admin')
self.PASSWORD = CFG.get_safe('eoi.geoserver.password', 'geoserver')
self.gs_host = CFG.get_safe('eoi.geoserver.server', 'http://*****:*****@unittest.skipIf(not (CFG.get_safe('eoi.meta.use_eoi_services', False)), 'Skip test in TABLE LOADER as services are not loaded')
def test_create_dataset(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition('example', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
dp = DataProduct(name='example')
data_product_id = self.data_product_management.create_data_product(dp, stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product, data_product_id)
self.data_product_management.activate_data_product_persistence(data_product_id)
self.addCleanup(self.data_product_management.suspend_data_product_persistence, data_product_id)
dataset_id = self.resource_registry.find_objects(data_product_id, PRED.hasDataset, id_only=True)[0][0]
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
rdt = ph.get_rdt(stream_def_id)
ph.fill_rdt(rdt, 100)
ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(monitor.event.wait(10))
# Yield to other greenlets, had an issue with connectivity
gevent.sleep(1)
log.debug("--------------------------------")
log.debug(dataset_id)
coverage_path = DatasetManagementService()._get_coverage_path(dataset_id)
log.debug(coverage_path)
log.debug("--------------------------------")
breakpoint(locals(), globals())
class DMTestCase(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.resource_registry = self.container.resource_registry
self.RR2 = EnhancedResourceRegistryClient(self.resource_registry)
self.data_acquisition_management = DataAcquisitionManagementServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.instrument_management = InstrumentManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.data_process_management = DataProcessManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
self.data_retriever = DataRetrieverServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.user_notification = UserNotificationServiceClient()
self.workflow_management = WorkflowManagementServiceClient()
self.visualization = VisualizationServiceClient()
def create_stream_definition(self, *args, **kwargs):
stream_def_id = self.pubsub_management.create_stream_definition(*args, **kwargs)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
return stream_def_id
def create_data_product(self,name, stream_def_id='', param_dict_name='', pdict_id=''):
if not (stream_def_id or param_dict_name or pdict_id):
raise AssertionError('Attempted to create a Data Product without a parameter dictionary')
tdom, sdom = time_series_domain()
dp = DataProduct(name=name,
spatial_domain = sdom.dump(),
temporal_domain = tdom.dump(),
)
stream_def_id = stream_def_id or self.create_stream_definition('%s stream def' % name,
parameter_dictionary_id=pdict_id or self.RR2.find_resource_by_name(RT.ParameterDictionary,
param_dict_name, id_only=True))
data_product_id = self.data_product_management.create_data_product(dp, stream_definition_id=stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product, data_product_id)
return data_product_id
def activate_data_product(self, data_product_id):
self.data_product_management.activate_data_product_persistence(data_product_id)
self.addCleanup(self.data_product_management.suspend_data_product_persistence, data_product_id)
def data_product_by_id(self, alt_id):
data_products, _ = self.container.resource_registry.find_resources_ext(alt_id=alt_id, alt_id_ns='PRE', id_only=True)
if data_products:
return data_products[0]
return None
def dataset_of_data_product(self, data_product_id):
return self.resource_registry.find_objects(data_product_id, PRED.hasDataset, id_only=True)[0][0]
def _start_raw_ingestion(self):
dpsc_cli = DataProductManagementServiceClient()
rrclient = ResourceRegistryServiceClient()
RR2 = EnhancedResourceRegistryClient(rrclient)
# Generic time-series data domain creation
tdom, sdom = time_series_domain()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom.dump(),
spatial_domain = sdom.dump())
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 10.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -10.0
dp_obj.ooi_product_name = "PRODNAME"
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
log.info("Create data product... raw stream id: %s", self._raw_stream_id)
dp_id = dpsc_cli.create_data_product_(data_product= dp_obj)
dataset_id = self.create_dataset(self._raw_stream_pdict_id)
RR2.assign_stream_definition_to_data_product_with_has_stream_definition(self._raw_stream_def_id, dp_id)
RR2.assign_stream_to_data_product_with_has_stream(self._raw_stream_id, dp_id)
RR2.assign_dataset_to_data_product_with_has_dataset(dataset_id, dp_id)
self._raw_dataset_id = dataset_id
log.info("Create data product...Complete")
# Assert that the data product has an associated stream at this stage
stream_ids, _ = rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True)
self.assertNotEquals(len(stream_ids), 0)
# Assert that the data product has an associated stream def at this stage
stream_ids, _ = rrclient.find_objects(dp_id, PRED.hasStreamDefinition, RT.StreamDefinition, True)
self.assertNotEquals(len(stream_ids), 0)
log.info("Activate data product persistence")
dpsc_cli.activate_data_product_persistence(dp_id)
log.info("Read data product")
dp_obj = dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0)
log.debug('Created data product %s', dp_obj)
def _start_raw_ingestion(self):
dpsc_cli = DataProductManagementServiceClient()
rrclient = ResourceRegistryServiceClient()
RR2 = EnhancedResourceRegistryClient(rrclient)
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp')
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 10.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -10.0
dp_obj.ooi_product_name = "PRODNAME"
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
log.info("Create data product... raw stream id: %s",
self._raw_stream_id)
dp_id = dpsc_cli.create_data_product_(data_product=dp_obj)
dataset_id = self.create_dataset(self._raw_stream_pdict_id)
RR2.assign_stream_definition_to_data_product_with_has_stream_definition(
self._raw_stream_def_id, dp_id)
RR2.assign_stream_to_data_product_with_has_stream(
self._raw_stream_id, dp_id)
RR2.assign_dataset_to_data_product_with_has_dataset(dataset_id, dp_id)
self._raw_dataset_id = dataset_id
log.info("Create data product...Complete")
# Assert that the data product has an associated stream at this stage
stream_ids, _ = rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream,
True)
self.assertNotEquals(len(stream_ids), 0)
# Assert that the data product has an associated stream def at this stage
stream_ids, _ = rrclient.find_objects(dp_id, PRED.hasStreamDefinition,
RT.StreamDefinition, True)
self.assertNotEquals(len(stream_ids), 0)
log.info("Activate data product persistence")
dpsc_cli.activate_data_product_persistence(dp_id)
log.info("Read data product")
dp_obj = dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0)
log.debug('Created data product %s', dp_obj)
class DMTestCase(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.resource_registry = self.container.resource_registry
self.RR2 = EnhancedResourceRegistryClient(self.resource_registry)
self.data_acquisition_management = DataAcquisitionManagementServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.instrument_management = InstrumentManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.data_process_management = DataProcessManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
self.data_retriever = DataRetrieverServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.user_notification = UserNotificationServiceClient()
self.workflow_management = WorkflowManagementServiceClient()
self.visualization = VisualizationServiceClient()
def create_stream_definition(self, *args, **kwargs):
stream_def_id = self.pubsub_management.create_stream_definition(*args, **kwargs)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
return stream_def_id
def create_data_product(self,name, stream_def_id='', param_dict_name='', pdict_id=''):
if not (stream_def_id or param_dict_name or pdict_id):
raise AssertionError('Attempted to create a Data Product without a parameter dictionary')
tdom, sdom = time_series_domain()
dp = DataProduct(name=name,
spatial_domain = sdom.dump(),
temporal_domain = tdom.dump(),
)
stream_def_id = stream_def_id or self.create_stream_definition('%s stream def' % name,
parameter_dictionary_id=pdict_id or self.RR2.find_resource_by_name(RT.ParameterDictionary,
param_dict_name, id_only=True))
data_product_id = self.data_product_management.create_data_product(dp, stream_definition_id=stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product, data_product_id)
return data_product_id
def activate_data_product(self, data_product_id):
self.data_product_management.activate_data_product_persistence(data_product_id)
self.addCleanup(self.data_product_management.suspend_data_product_persistence, data_product_id)
class InstrumentSimulator(ApeComponent):
def __init__(self, component_id, agent, configuration):
ApeComponent.__init__(self, component_id, agent, configuration)
model_id = None
keep_running = True
emit_granules = False
thread = None
def _start(self):
# interact with container
self.resource_registry = ResourceRegistryServiceClient(node=self.agent.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.agent.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.agent.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.agent.container.node)
self.data_product_client = DataProductManagementServiceClient(node=self.agent.container.node)
# add appropriate entries to the pycc registry
if self.configuration.easy_registration:
self.register()
# begin loop to generate/send data
self.thread = self._DataEmitter(self)
self.thread.start()
def _stop(self):
log.debug("stopping instrument")
self.stop_sending()
self.keep_running = False
def perform_action(self, request):
log.debug('instrument simulator performing action ' + str(request))
if isinstance(request, StartRequest):
self.start_sending()
elif isinstance(request, StopRequest):
self.stop_sending()
elif isinstance(request, RegisterWithContainer):
self.register()
else:
raise ApeException('instrument simulator does not know how to: ' + str(request))
def register(self):
''' create entries in resource registry for this instrument '''
# TODO: handle more than one ID returned from registry
# TODO: fail if request register but already in registry (and not easy_registration)
# TODO: should register device agent
log.debug('registering instrument')
product_ids,_ = self.resource_registry.find_resources(RT.DataProduct, None, self.configuration.data_product, id_only=True)
if len(product_ids) > 0:
log.debug('data product was already in the registry')
self.data_product_id = product_ids[0]
# if self.configuration.easy_registration:
# self.data_product_id = product_ids[0]
# else:
# #raise ApeException('should not find data product in registry until i add it: ' + self.configuration.data_product)
# ## TODO need ID from name: self.data_product_id = self.data_product_client.read_data_product(data_product=self.configuration.data_product)
# pass
else:
log.debug('adding data product to the registry')
# Create InstrumentDevice
device = IonObject(RT.InstrumentDevice, name=self.configuration.data_product, description='instrument simulator', serial_number=self.configuration.data_product )
device_id = self.imsclient.create_instrument_device(instrument_device=device)
self.imsclient.assign_instrument_model_to_instrument_device(instrument_model_id=self._get_model_id(), instrument_device_id=device_id)
#@TODO: Do not use CoverageCraft ever.
craft = CoverageCraft
sdom, tdom = craft.create_domains()
sdom = sdom.dump()
tdom = tdom.dump()
parameter_dictionary = craft.create_parameters()
parameter_dictionary = parameter_dictionary.dump()
data_product = IonObject(RT.DataProduct,name=self.configuration.data_product,description='ape producer', spatial_domain=sdom, temporal_domain=tdom)
stream_def_id = self._get_streamdef_id(parameter_dictionary)
self.data_product_id = self.data_product_client.create_data_product(data_product=data_product, stream_definition_id=stream_def_id)
self.damsclient.assign_data_product(input_resource_id=device_id, data_product_id=self.data_product_id)
if self.configuration.persist_product:
self.data_product_client.activate_data_product_persistence(data_product_id=self.data_product_id, persist_data=True, persist_metadata=True)
# get/create producer id
# producer_ids, _ = self.clients.resource_registry.find_objects(self.data_product_id, PRED.hasDataProducer, RT.DataProducer, id_only=True)
# if len(producer_ids)>0:
# log.debug('data producer already in the registry')
# self.data_producer_id = producer_ids[0]
# else:
self.data_producer_id = 'UNUSED'
self._create_publisher()
def _get_model_id(self):
if self.model_id is None:
try:
self.model_id = self._read_model_id()
self.model_id = self._read_model_id()
except:
model = IonObject(RT.InstrumentModel, name=self.configuration.instrument.model_name) #, description=self.configuration.model_name, model_label=self.data_source_name)
self.model_id = self.imsclient.create_instrument_model(model)
return self.model_id
def _read_model_id(self):
# debug reading instrument model
try_this = self.resource_registry.find_resources(restype=RT.InstrumentModel, id_only=True)
model_ids = self.resource_registry.find_resources(restype=RT.InstrumentModel, id_only=True, name=self.configuration.instrument.model_name)
return model_ids[0][0]
def _get_streamdef_id(self, pdict=None):
#TODO: figure out how to read existing RAW definition instead of just creating
#try:
#stream_type = self.pubsubclient.read_stream_definition()
stream_def_id = self.pubsubclient.create_stream_definition(name='RAW stream',parameter_dictionary=pdict)
return stream_def_id
def _create_publisher(self):
stream_ids, _ = self.resource_registry.find_objects(self.data_product_id, PRED.hasStream, None, True)
stream_id = stream_ids[0]
self.publisher = StreamPublisher(process=self.agent, stream_id=stream_id)
class _DataEmitter(Thread):
''' do not make outer class a Thread b/c start() is already meaningful to a pyon process '''
def __init__(self, instrument):
Thread.__init__(self)
self.instrument = instrument
self.daemon = True
def run(self):
self.instrument.run()
def start_sending(self):
self.emit_granules = True
def stop_sending(self):
self.emit_granules = False
def _get_parameter_dictionary(self):
pdict = ParameterDictionary()
cond_ctxt = ParameterContext('salinity', param_type=QuantityType(value_encoding=np.float64))
cond_ctxt.uom = 'unknown'
cond_ctxt.fill_value = 0e0
pdict.add_context(cond_ctxt)
pres_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=np.float64))
pres_ctxt.uom = 'unknown'
pres_ctxt.fill_value = 0x0
pdict.add_context(pres_ctxt)
temp_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=np.float64))
temp_ctxt.uom = 'unknown'
temp_ctxt.fill_value = 0x0
pdict.add_context(temp_ctxt)
oxy_ctxt = ParameterContext('oxygen', param_type=QuantityType(value_encoding=np.float64))
oxy_ctxt.uom = 'unknown'
oxy_ctxt.fill_value = 0x0
pdict.add_context(oxy_ctxt)
internal_ts_ctxt = ParameterContext(name='internal_timestamp', param_type=QuantityType(value_encoding=np.float64))
internal_ts_ctxt._derived_from_name = 'time'
internal_ts_ctxt.uom = 'seconds'
internal_ts_ctxt.fill_value = -1
pdict.add_context(internal_ts_ctxt, is_temporal=True)
driver_ts_ctxt = ParameterContext(name='driver_timestamp', param_type=QuantityType(value_encoding=np.float64))
driver_ts_ctxt._derived_from_name = 'time'
driver_ts_ctxt.uom = 'seconds'
driver_ts_ctxt.fill_value = -1
pdict.add_context(driver_ts_ctxt)
return pdict
def run(self):
try:
self.do_run()
except:
log.exception('instrument simulator thread shutting down')
def do_run(self):
''' main loop: essentially create granule, publish granule, then pause to remain at target rate.
three lines of the useful code marked with ###, everything else is timing/reporting.
'''
target_iteration_time = sleep_time = self.configuration.interval
first_batch = True
granule_count = iteration_count = granule_sum_size = 0
granule_elapsed_seconds = publish_elapsed_seconds = sleep_elapsed_seconds = iteration_elapsed_seconds = 0.
do_timing = self.configuration.report_timing or self.configuration.log_timing
parameter_dictionary = self._get_parameter_dictionary() #DatasetManagementServiceClient(node=self.agent.container.node).read_parameter_dictionary_by_name('ctd_parsed_param_dict')
if not parameter_dictionary:
log.error('failed to find parameter dictionary: ctd_parsed_param_dict (producer thread aborting)')
return
if type(parameter_dictionary) == dict:
log.debug('pdict is dict')
elif isinstance(parameter_dictionary,ParameterDictionary):
log.debug('pdict is ParameterDictionary')
else:
log.warn('invalid pdict: %r', parameter_dictionary)
parameter_dictionary = parameter_dictionary.__dict__
while self.keep_running:
if self.emit_granules:
adjust_timing = True
iteration_start = granule_start = time()
if do_timing:
granule_count+=1
### step 1: create granule
granule = self.configuration.instrument.get_granule(time=time(), pd=parameter_dictionary)
if do_timing:
granule_end = publish_start = time()
try:
### step 2: publish granule
log.debug(self.component_id + ' publishing granule')
self.publisher.publish(granule)
except Exception as e:
if self.keep_running:
raise e
else:
# while blocking on publish(), async call may have closed connection
# so eat exception and return cleanly
return
if do_timing:
publish_end = time()
granule_elapsed_seconds += (granule_end - granule_start)
publish_elapsed_seconds += (publish_end - publish_start)
pickled_granule = pickle.dumps(granule)
granule_size = len(pickled_granule)
granule_sum_size += granule_size
sleep_start = time()
### step 3: sleep to maintain configured rate
if sleep_time>0 or self.configuration.sleep_even_zero:
sleep(sleep_time)
if do_timing:
sleep_end = time()
sleep_elapsed_seconds += (sleep_end-sleep_start)
if granule_count%self.configuration.timing_rate==0:
# skip initial batch of timing -- can be skewed b/c one instrument start before the other,
# so first instrument works against a more idle system
if first_batch:
granule_count = granule_elapsed_seconds = publish_elapsed_seconds = sleep_elapsed_seconds = 0.
first_batch = False
run_start_time = time()
else:
adjust_timing = False
if self.configuration.log_timing:
log.info('%s: granule: %f, publish: %f, sleep: %f, size %f' %
(self.configuration.data_product, granule_elapsed_seconds/granule_count,
publish_elapsed_seconds/granule_count, sleep_elapsed_seconds/granule_count,
granule_sum_size/granule_count))
if self.configuration.report_timing:
report = { 'count': granule_count,
'time': time() - run_start_time,
'publish':publish_elapsed_seconds/granule_count,
'granule':granule_elapsed_seconds/granule_count,
'sleep': sleep_elapsed_seconds/granule_count,
'iteration': iteration_elapsed_seconds/iteration_count }
if self.configuration.include_size:
report['size'] = granule_sum_size
message = PerformanceResult(report)
self.report(message)
if adjust_timing:
iteration_count+=1
iteration_end = time()
actual_iteration_time = iteration_end - iteration_start
iteration_elapsed_seconds += actual_iteration_time
adjustment = target_iteration_time - actual_iteration_time
sleep_time = max(sleep_time + adjustment, 0)
else:
# if not emitting granules, don't do timing either
# and don't allow to sleep for too short a time and hog CPU
sleep(max(10,self.configuration.interval))
log.debug('not emitting')
class VisualizationIntegrationTestHelper(IonIntegrationTestCase):
def create_ctd_input_stream_and_data_product(self, data_product_name='ctd_parsed'):
cc = self.container
assertions = self.assertTrue
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.workflowclient = WorkflowManagementServiceClient(node=self.container.node)
self.process_dispatcher = ProcessDispatcherServiceClient(node=self.container.node)
self.vis_client = VisualizationServiceClient(node=self.container.node)
#-------------------------------
# Create CTD Parsed as the initial data product
#-------------------------------
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def, name='Simulated CTD data')
log.debug('Creating new CDM data product with a stream definition')
craft = CoverageCraft
sdom, tdom = craft.create_domains()
sdom = sdom.dump()
tdom = tdom.dump()
parameter_dictionary = craft.create_parameters()
parameter_dictionary = parameter_dictionary.dump()
dp_obj = IonObject(RT.DataProduct,
name=data_product_name,
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_parsed_data_product_id = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary)
log.debug('new ctd_parsed_data_product_id = %s' % ctd_parsed_data_product_id)
#Only ever need one device for testing purposes.
instDevice_obj,_ = self.rrclient.find_resources(restype=RT.InstrumentDevice, name='SBE37IMDevice')
if instDevice_obj:
instDevice_id = instDevice_obj[0]._id
else:
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product_id)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_id, PRED.hasStream, None, True)
assertions(len(stream_ids) > 0 )
ctd_stream_id = stream_ids[0]
return ctd_stream_id, ctd_parsed_data_product_id
def create_data_product(self, dp_name = "", dp_description = ""):
craft = CoverageCraft
sdom, tdom = craft.create_domains()
sdom = sdom.dump()
tdom = tdom.dump()
parameter_dictionary = craft.create_parameters() # this creates a ParameterDictionary object
parameter_dictionary = parameter_dictionary.dump() # this returns a python dictionary
data_prod_obj = IonObject(RT.DataProduct,
name=dp_name,
description=dp_description,
temporal_domain = tdom,
spatial_domain = sdom)
data_prod_id = self.create_data_product(data_prod_obj, stream_definition_id, parameter_dictionary)
return data_prod_id, data_prod_obj
def start_simple_input_stream_process(self, ctd_stream_id):
return self.start_input_stream_process(ctd_stream_id)
def start_sinusoidal_input_stream_process(self, ctd_stream_id):
return self.start_input_stream_process(ctd_stream_id, 'ion.processes.data.sinusoidal_stream_publisher', 'SinusoidalCtdPublisher')
def start_input_stream_process(self, ctd_stream_id, module = 'ion.processes.data.ctd_stream_publisher', class_name= 'SimpleCtdPublisher'):
###
### Start the process for producing the CTD data
###
# process definition for the ctd simulator...
producer_definition = ProcessDefinition()
producer_definition.executable = {
'module':module,
'class':class_name
}
ctd_sim_procdef_id = self.process_dispatcher.create_process_definition(process_definition=producer_definition)
# Start the ctd simulator to produce some data
configuration = {
'process':{
'stream_id':ctd_stream_id,
}
}
ctd_sim_pid = self.process_dispatcher.schedule_process(process_definition_id=ctd_sim_procdef_id, configuration=configuration)
return ctd_sim_pid
def start_output_stream_and_listen(self, ctd_stream_id, data_product_stream_ids, message_count_per_stream=10):
cc = self.container
assertions = self.assertTrue
###
### Make a subscriber in the test to listen for transformed data
###
salinity_subscription_id = self.pubsubclient.create_subscription(
query=StreamQuery(data_product_stream_ids),
exchange_name = 'workflow_test',
exchange_point = 'science_data',
name = "test workflow transformations",
)
pid = cc.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = cc.proc_manager.procs[pid]
subscriber_registrar = StreamSubscriberRegistrar(process=dummy_process, container=cc)
result = gevent.event.AsyncResult()
results = []
message_count = len(data_product_stream_ids) * message_count_per_stream
def message_received(message, headers):
# Heads
results.append(message)
if len(results) >= message_count: #Only wait for so many messages - per stream
result.set(True)
subscriber = subscriber_registrar.create_subscriber(exchange_name='workflow_test', callback=message_received)
subscriber.start()
# after the queue has been created it is safe to activate the subscription
self.pubsubclient.activate_subscription(subscription_id=salinity_subscription_id)
#Start the input stream process
if ctd_stream_id is not None:
ctd_sim_pid = self.start_simple_input_stream_process(ctd_stream_id)
# Assert that we have received data
assertions(result.get(timeout=30))
# stop the flow parse the messages...
if ctd_stream_id is not None:
self.process_dispatcher.cancel_process(ctd_sim_pid) # kill the ctd simulator process - that is enough data
self.pubsubclient.deactivate_subscription(subscription_id=salinity_subscription_id)
subscriber.stop()
return results
def validate_messages(self, results):
cc = self.container
assertions = self.assertTrue
first_salinity_values = None
for message in results:
rdt = RecordDictionaryTool.load_from_granule(message)
try:
temp = get_safe(rdt, 'temp')
# psd = PointSupplementStreamParser(stream_definition=self.ctd_stream_def, stream_granule=message)
# temp = psd.get_values('temperature')
# log.info(psd.list_field_names())
except KeyError as ke:
temp = None
if temp is not None:
assertions(isinstance(temp, numpy.ndarray))
log.info( 'temperature=' + str(numpy.nanmin(temp)))
first_salinity_values = None
else:
#psd = PointSupplementStreamParser(stream_definition=SalinityTransform.outgoing_stream_def, stream_granule=message)
#log.info( psd.list_field_names())
# Test the handy info method for the names of fields in the stream def
#assertions('salinity' in psd.list_field_names())
# you have to know the name of the coverage in stream def
salinity = get_safe(rdt, 'salinity')
#salinity = psd.get_values('salinity')
log.info( 'salinity=' + str(numpy.nanmin(salinity)))
# Check to see if salinity has values
assertions(salinity != None)
assertions(isinstance(salinity, numpy.ndarray))
assertions(numpy.nanmin(salinity) > 0.0) # salinity should always be greater than 0
if first_salinity_values is None:
first_salinity_values = salinity.tolist()
else:
second_salinity_values = salinity.tolist()
assertions(len(first_salinity_values) == len(second_salinity_values))
for idx in range(0,len(first_salinity_values)):
assertions(first_salinity_values[idx]*2.0 == second_salinity_values[idx])
def validate_data_ingest_retrieve(self, dataset_id):
assertions = self.assertTrue
self.data_retriever = DataRetrieverServiceClient(node=self.container.node)
#validate that data was ingested
replay_granule = self.data_retriever.retrieve_last_granule(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(replay_granule)
salinity = get_safe(rdt, 'salinity')
assertions(salinity != None)
#retrieve all the granules from the database and check the values
replay_granule_all = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(replay_granule_all)
for k, v in rdt.iteritems():
if k == 'salinity':
for val in numpy.nditer(v):
assertions(val > 0)
def create_salinity_data_process_definition(self):
# Salinity: Data Process Definition
#First look to see if it exists and if not, then create it
dpd,_ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='ctd_salinity')
if len(dpd) > 0:
return dpd[0]
log.debug("Create data process definition SalinityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_salinity',
description='create a salinity data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform',
process_source='SalinityTransform source code here...')
try:
ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except Excpetion as ex:
self.fail("failed to create new SalinityTransform data process definition: %s" %ex)
# create a stream definition for the data from the salinity Transform
sal_stream_def_id = self.pubsubclient.create_stream_definition(container=SalinityTransform.outgoing_stream_def, name='Salinity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(sal_stream_def_id, ctd_L2_salinity_dprocdef_id )
return ctd_L2_salinity_dprocdef_id
def create_salinity_doubler_data_process_definition(self):
#First look to see if it exists and if not, then create it
dpd,_ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='salinity_doubler')
if len(dpd) > 0:
return dpd[0]
# Salinity Doubler: Data Process Definition
log.debug("Create data process definition SalinityDoublerTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='salinity_doubler',
description='create a salinity doubler data product',
module='ion.processes.data.transforms.example_double_salinity',
class_name='SalinityDoubler',
process_source='SalinityDoubler source code here...')
try:
salinity_doubler_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except Exception as ex:
self.fail("failed to create new SalinityDoubler data process definition: %s" %ex)
# create a stream definition for the data from the salinity Transform
salinity_double_stream_def_id = self.pubsubclient.create_stream_definition(container=SalinityDoubler.outgoing_stream_def, name='SalinityDoubler')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(salinity_double_stream_def_id, salinity_doubler_dprocdef_id )
return salinity_doubler_dprocdef_id
def create_transform_process(self, data_process_definition_id, data_process_input_dp_id):
data_process_definition = self.rrclient.read(data_process_definition_id)
# Find the link between the output Stream Definition resource and the Data Process Definition resource
stream_ids,_ = self.rrclient.find_objects(data_process_definition._id, PRED.hasStreamDefinition, RT.StreamDefinition, id_only=True)
if not stream_ids:
raise Inconsistent("The data process definition %s is missing an association to an output stream definition" % data_process_definition._id )
process_output_stream_def_id = stream_ids[0]
#Concatenate the name of the workflow and data process definition for the name of the data product output
data_process_name = data_process_definition.name
# Create the output data product of the transform
transform_dp_obj = IonObject(RT.DataProduct, name=data_process_name,description=data_process_definition.description)
transform_dp_id = self.dataproductclient.create_data_product(transform_dp_obj, process_output_stream_def_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=transform_dp_id)
#last one out of the for loop is the output product id
output_data_product_id = transform_dp_id
# Create the transform data process
log.debug("create data_process and start it")
data_process_id = self.dataprocessclient.create_data_process(data_process_definition._id, [data_process_input_dp_id], {'output':transform_dp_id})
self.dataprocessclient.activate_data_process(data_process_id)
#Find the id of the output data stream
stream_ids, _ = self.rrclient.find_objects(transform_dp_id, PRED.hasStream, None, True)
if not stream_ids:
raise Inconsistent("The data process %s is missing an association to an output stream" % data_process_id )
return data_process_id, output_data_product_id
def create_google_dt_data_process_definition(self):
#First look to see if it exists and if not, then create it
dpd,_ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='google_dt_transform')
if len(dpd) > 0:
return dpd[0]
# Data Process Definition
log.debug("Create data process definition GoogleDtTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='google_dt_transform',
description='Convert data streams to Google DataTables',
module='ion.processes.data.transforms.viz.google_dt',
class_name='VizTransformGoogleDT',
process_source='VizTransformGoogleDT source code here...')
try:
procdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except Exception as ex:
self.fail("failed to create new VizTransformGoogleDT data process definition: %s" %ex)
# create a stream definition for the data from the
stream_def_id = self.pubsubclient.create_stream_definition(container=VizTransformGoogleDT.outgoing_stream_def, name='VizTransformGoogleDT')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(stream_def_id, procdef_id )
return procdef_id
def validate_google_dt_transform_results(self, results):
cc = self.container
assertions = self.assertTrue
# if its just one granule, wrap it up in a list so we can use the following for loop for a couple of cases
if isinstance(results,Granule):
results =[results]
for g in results:
if isinstance(g,Granule):
tx = TaxyTool.load_from_granule(g)
rdt = RecordDictionaryTool.load_from_granule(g)
gdt_data = get_safe(rdt, 'google_dt_components')
# IF this granule does not contains google dt, skip
if gdt_data == None:
continue
gdt = gdt_data[0]
assertions(gdt['viz_product_type'] == 'google_dt' )
assertions(len(gdt['data_description']) >= 0) # Need to come up with a better check
assertions(len(gdt['data_content']) >= 0)
def create_mpl_graphs_data_process_definition(self):
#First look to see if it exists and if not, then create it
dpd,_ = self.rrclient.find_resources(restype=RT.DataProcessDefinition, name='mpl_graphs_transform')
if len(dpd) > 0:
return dpd[0]
#Data Process Definition
log.debug("Create data process definition MatplotlibGraphsTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='mpl_graphs_transform',
description='Convert data streams to Matplotlib graphs',
module='ion.processes.data.transforms.viz.matplotlib_graphs',
class_name='VizTransformMatplotlibGraphs',
process_source='VizTransformMatplotlibGraphs source code here...')
try:
procdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except Exception as ex:
self.fail("failed to create new VizTransformMatplotlibGraphs data process definition: %s" %ex)
# create a stream definition for the data
stream_def_id = self.pubsubclient.create_stream_definition(container=VizTransformMatplotlibGraphs.outgoing_stream_def, name='VizTransformMatplotlibGraphs')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(stream_def_id, procdef_id )
return procdef_id
def validate_mpl_graphs_transform_results(self, results):
cc = self.container
assertions = self.assertTrue
# if its just one granule, wrap it up in a list so we can use the following for loop for a couple of cases
if isinstance(results,Granule):
results =[results]
for g in results:
if isinstance(g,Granule):
tx = TaxyTool.load_from_granule(g)
rdt = RecordDictionaryTool.load_from_granule(g)
graphs = get_safe(rdt, 'matplotlib_graphs')
if graphs == None:
continue
for graph in graphs[0]:
# At this point only dictionaries containing image data should be passed
# For some reason non dictionary values are filtering through.
if not isinstance(graph, dict):
continue
assertions(graph['viz_product_type'] == 'matplotlib_graphs' )
# check to see if the list (numpy array) contains actual images
assertions(imghdr.what(graph['image_name'], h = graph['image_obj']) == 'png')
def validate_vis_service_google_dt_results(self, results):
assertions = self.assertTrue
assertions(results)
gdt_str = (results.lstrip("google.visualization.Query.setResponse(")).rstrip(")")
assertions(len(gdt_str) > 0)
return
def validate_vis_service_mpl_graphs_results(self, results):
assertions = self.assertTrue
assertions(results)
# check to see if the object passed is a dictionary with a valid image object in it
image_format = results["content_type"].lstrip("image/")
assertions(imghdr.what(results['image_name'], h = base64.decodestring(results['image_obj'])) == image_format)
return
class TestDataProductManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dpsc_cli = DataProductManagementServiceClient()
self.rrclient = ResourceRegistryServiceClient()
self.damsclient = DataAcquisitionManagementServiceClient()
self.pubsubcli = PubsubManagementServiceClient()
self.ingestclient = IngestionManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.unsc = UserNotificationServiceClient()
self.data_retriever = DataRetrieverServiceClient()
#------------------------------------------
# Create the environment
#------------------------------------------
self.stream_def_id = self.pubsubcli.create_stream_definition(
name='SBE37_CDM')
self.process_definitions = {}
ingestion_worker_definition = ProcessDefinition(
name='ingestion worker')
ingestion_worker_definition.executable = {
'module':
'ion.processes.data.ingestion.science_granule_ingestion_worker',
'class': 'ScienceGranuleIngestionWorker'
}
process_definition_id = self.process_dispatcher.create_process_definition(
process_definition=ingestion_worker_definition)
self.process_definitions['ingestion_worker'] = process_definition_id
self.pids = []
self.exchange_points = []
self.exchange_names = []
#------------------------------------------------------------------------------------------------
# First launch the ingestors
#------------------------------------------------------------------------------------------------
self.exchange_space = 'science_granule_ingestion'
self.exchange_point = 'science_data'
config = DotDict()
config.process.datastore_name = 'datasets'
config.process.queue_name = self.exchange_space
self.exchange_names.append(self.exchange_space)
self.exchange_points.append(self.exchange_point)
pid = self.process_dispatcher.schedule_process(
self.process_definitions['ingestion_worker'], configuration=config)
log.debug("the ingestion worker process id: %s", pid)
self.pids.append(pid)
self.addCleanup(self.cleaning_up)
def cleaning_up(self):
for pid in self.pids:
log.debug("number of pids to be terminated: %s", len(self.pids))
try:
self.process_dispatcher.cancel_process(pid)
log.debug("Terminated the process: %s", pid)
except:
log.debug("could not terminate the process id: %s" % pid)
IngestionManagementIntTest.clean_subscriptions()
for xn in self.exchange_names:
xni = self.container.ex_manager.create_xn_queue(xn)
xni.delete()
for xp in self.exchange_points:
xpi = self.container.ex_manager.create_xp(xp)
xpi.delete()
def get_datastore(self, dataset_id):
dataset = self.dataset_management.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(
datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
@attr('EXT')
@attr('PREP')
def test_create_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
parameter_dictionary = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict')
ctd_stream_def_id = self.pubsubcli.create_stream_definition(
name='Simulated CTD data',
parameter_dictionary_id=parameter_dictionary._id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp')
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 10.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -10.0
dp_obj.ooi_product_name = "PRODNAME"
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product(
data_product=dp_obj, stream_definition_id=ctd_stream_def_id)
# Assert that the data product has an associated stream at this stage
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream,
RT.Stream, True)
self.assertNotEquals(len(stream_ids), 0)
# Assert that the data product has an associated stream def at this stage
stream_ids, _ = self.rrclient.find_objects(dp_id,
PRED.hasStreamDefinition,
RT.StreamDefinition, True)
self.assertNotEquals(len(stream_ids), 0)
self.dpsc_cli.activate_data_product_persistence(dp_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0)
log.debug('Created data product %s', dp_obj)
#------------------------------------------------------------------------------------------------
# test creating a new data product with a stream definition
#------------------------------------------------------------------------------------------------
log.debug('Creating new data product with a stream definition')
dp_obj = IonObject(RT.DataProduct,
name='DP2',
description='some new dp')
dp_id2 = self.dpsc_cli.create_data_product(dp_obj, ctd_stream_def_id)
self.dpsc_cli.activate_data_product_persistence(dp_id2)
log.debug('new dp_id = %s' % dp_id2)
#------------------------------------------------------------------------------------------------
#make sure data product is associated with stream def
#------------------------------------------------------------------------------------------------
streamdefs = []
streams, _ = self.rrclient.find_objects(dp_id2, PRED.hasStream,
RT.Stream, True)
for s in streams:
log.debug("Checking stream %s" % s)
sdefs, _ = self.rrclient.find_objects(s, PRED.hasStreamDefinition,
RT.StreamDefinition, True)
for sd in sdefs:
log.debug("Checking streamdef %s" % sd)
streamdefs.append(sd)
self.assertIn(ctd_stream_def_id, streamdefs)
group_names = self.dpsc_cli.get_data_product_group_list()
self.assertIn("PRODNAME", group_names)
# test reading a non-existent data product
log.debug('reading non-existent data product')
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product('some_fake_id')
# update a data product (tests read also)
log.debug('Updating data product')
# first get the existing dp object
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# now tweak the object
dp_obj.description = 'the very first dp'
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 20.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -20.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 20.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -20.0
# now write the dp back to the registry
update_result = self.dpsc_cli.update_data_product(dp_obj)
# now get the dp back to see if it was updated
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertEquals(dp_obj.description, 'the very first dp')
self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0)
log.debug('Updated data product %s', dp_obj)
#test extension
extended_product = self.dpsc_cli.get_data_product_extension(dp_id)
self.assertEqual(dp_id, extended_product._id)
self.assertEqual(
ComputedValueAvailability.PROVIDED,
extended_product.computed.product_download_size_estimated.status)
self.assertEqual(
0, extended_product.computed.product_download_size_estimated.value)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_product.computed.parameters.status)
#log.debug("test_create_data_product: parameters %s" % extended_product.computed.parameters.value)
def ion_object_encoder(obj):
return obj.__dict__
#test prepare for create
data_product_data = self.dpsc_cli.prepare_data_product_support()
#print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2)
self.assertEqual(data_product_data._id, "")
self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport)
self.assertEqual(
len(data_product_data.associations['StreamDefinition'].resources),
2)
self.assertEqual(
len(data_product_data.associations['Dataset'].resources), 0)
self.assertEqual(
len(data_product_data.associations['StreamDefinition'].
associated_resources), 0)
self.assertEqual(
len(data_product_data.associations['Dataset'].associated_resources
), 0)
#test prepare for update
data_product_data = self.dpsc_cli.prepare_data_product_support(dp_id)
#print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2)
self.assertEqual(data_product_data._id, dp_id)
self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport)
self.assertEqual(
len(data_product_data.associations['StreamDefinition'].resources),
2)
self.assertEqual(
len(data_product_data.associations['Dataset'].resources), 1)
self.assertEqual(
len(data_product_data.associations['StreamDefinition'].
associated_resources), 1)
self.assertEqual(
data_product_data.associations['StreamDefinition'].
associated_resources[0].s, dp_id)
self.assertEqual(
len(data_product_data.associations['Dataset'].associated_resources
), 1)
self.assertEqual(
data_product_data.associations['Dataset'].associated_resources[0].
s, dp_id)
# now 'delete' the data product
log.debug("deleting data product: %s" % dp_id)
self.dpsc_cli.delete_data_product(dp_id)
# Assert that there are no associated streams leftover after deleting the data product
stream_ids, assoc_ids = self.rrclient.find_objects(
dp_id, PRED.hasStream, RT.Stream, True)
self.assertEquals(len(stream_ids), 0)
self.assertEquals(len(assoc_ids), 0)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# Get the events corresponding to the data product
ret = self.unsc.get_recent_events(resource_id=dp_id)
events = ret.value
for event in events:
log.debug("event time: %s" % event.ts_created)
self.assertTrue(len(events) > 0)
def test_data_product_stream_def(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(
name='Simulated CTD data', parameter_dictionary_id=pdict_id)
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp')
dp_id = self.dpsc_cli.create_data_product(
data_product=dp_obj, stream_definition_id=ctd_stream_def_id)
stream_def_id = self.dpsc_cli.get_data_product_stream_definition(dp_id)
self.assertEquals(ctd_stream_def_id, stream_def_id)
def test_derived_data_product(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(
name='ctd parsed', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsubcli.delete_stream_definition,
ctd_stream_def_id)
dp = DataProduct(name='Instrument DP')
dp_id = self.dpsc_cli.create_data_product(
dp, stream_definition_id=ctd_stream_def_id)
self.addCleanup(self.dpsc_cli.force_delete_data_product, dp_id)
self.dpsc_cli.activate_data_product_persistence(dp_id)
self.addCleanup(self.dpsc_cli.suspend_data_product_persistence, dp_id)
dataset_ids, _ = self.rrclient.find_objects(subject=dp_id,
predicate=PRED.hasDataset,
id_only=True)
if not dataset_ids:
raise NotFound("Data Product %s dataset does not exist" %
str(dp_id))
dataset_id = dataset_ids[0]
# Make the derived data product
simple_stream_def_id = self.pubsubcli.create_stream_definition(
name='TEMPWAT stream def',
parameter_dictionary_id=pdict_id,
available_fields=['time', 'temp'])
tempwat_dp = DataProduct(name='TEMPWAT',
category=DataProductTypeEnum.DERIVED)
tempwat_dp_id = self.dpsc_cli.create_data_product(
tempwat_dp,
stream_definition_id=simple_stream_def_id,
parent_data_product_id=dp_id)
self.addCleanup(self.dpsc_cli.delete_data_product, tempwat_dp_id)
# Check that the streams associated with the data product are persisted with
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream,
RT.Stream, True)
for stream_id in stream_ids:
self.assertTrue(self.ingestclient.is_persisted(stream_id))
stream_id = stream_ids[0]
route = self.pubsubcli.read_stream_route(stream_id=stream_id)
rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
rdt['pressure'] = np.arange(20)
publisher = StandaloneStreamPublisher(stream_id, route)
dataset_modified = Event()
def cb(*args, **kwargs):
dataset_modified.set()
es = EventSubscriber(event_type=OT.DatasetModified,
callback=cb,
origin=dataset_id,
auto_delete=True)
es.start()
self.addCleanup(es.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
tempwat_dataset_ids, _ = self.rrclient.find_objects(tempwat_dp_id,
PRED.hasDataset,
id_only=True)
tempwat_dataset_id = tempwat_dataset_ids[0]
granule = self.data_retriever.retrieve(
tempwat_dataset_id, delivery_format=simple_stream_def_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt['time'], np.arange(20))
self.assertEquals(set(rdt.fields), set(['time', 'temp']))
def test_activate_suspend_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(
name='Simulated CTD data', parameter_dictionary_id=pdict_id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
# Construct temporal and spatial Coordinate Reference System objects
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp')
log.debug("Created an IonObject for a data product: %s" % dp_obj)
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product(
data_product=dp_obj, stream_definition_id=ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# Subscribe to persist events
#------------------------------------------------------------------------------------------------
queue = gevent.queue.Queue()
def info_event_received(message, headers):
queue.put(message)
es = EventSubscriber(event_type=OT.InformationContentStatusEvent,
callback=info_event_received,
origin=dp_id,
auto_delete=True)
es.start()
self.addCleanup(es.stop)
#------------------------------------------------------------------------------------------------
# test activate and suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.activate_data_product_persistence(dp_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
dataset_ids, _ = self.rrclient.find_objects(subject=dp_id,
predicate=PRED.hasDataset,
id_only=True)
if not dataset_ids:
raise NotFound("Data Product %s dataset does not exist" %
str(dp_id))
dataset_id = dataset_ids[0]
# Check that the streams associated with the data product are persisted with
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream,
RT.Stream, True)
for stream_id in stream_ids:
self.assertTrue(self.ingestclient.is_persisted(stream_id))
stream_id = stream_ids[0]
route = self.pubsubcli.read_stream_route(stream_id=stream_id)
rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
publisher = StandaloneStreamPublisher(stream_id, route)
dataset_modified = Event()
def cb(*args, **kwargs):
dataset_modified.set()
es = EventSubscriber(event_type=OT.DatasetModified,
callback=cb,
origin=dataset_id,
auto_delete=True)
es.start()
self.addCleanup(es.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data = self.data_retriever.retrieve(dataset_ids[0])
self.assertIsInstance(replay_data, Granule)
log.debug(
"The data retriever was able to replay the dataset that was attached to the data product "
"we wanted to be persisted. Therefore the data product was indeed persisted with "
"otherwise we could not have retrieved its dataset using the data retriever. Therefore "
"this demonstration shows that L4-CI-SA-RQ-267 is satisfied: 'Data product management shall persist data products'"
)
data_product_object = self.rrclient.read(dp_id)
self.assertEquals(data_product_object.name, 'DP1')
self.assertEquals(data_product_object.description, 'some new dp')
log.debug(
"Towards L4-CI-SA-RQ-308: 'Data product management shall persist data product metadata'. "
" Attributes in create for the data product obj, name= '%s', description='%s', match those of object from the "
"resource registry, name='%s', desc='%s'" %
(dp_obj.name, dp_obj.description, data_product_object.name,
data_product_object.description))
#------------------------------------------------------------------------------------------------
# test suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.suspend_data_product_persistence(dp_id)
dataset_modified.clear()
rdt['time'] = np.arange(20, 40)
publisher.publish(rdt.to_granule())
self.assertFalse(dataset_modified.wait(2))
self.dpsc_cli.activate_data_product_persistence(dp_id)
dataset_modified.clear()
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_almost_equal(rdt['time'], np.arange(40))
dataset_ids, _ = self.rrclient.find_objects(dp_id,
PRED.hasDataset,
id_only=True)
self.assertEquals(len(dataset_ids), 1)
self.dpsc_cli.suspend_data_product_persistence(dp_id)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
with self.assertRaises(NotFound):
dp_obj = self.rrclient.read(dp_id)
info_event_counter = 0
runtime = 0
starttime = time.time()
caught_events = []
#check that the four InfoStatusEvents were received
while info_event_counter < 4 and runtime < 60:
a = queue.get(timeout=60)
caught_events.append(a)
info_event_counter += 1
runtime = time.time() - starttime
self.assertEquals(info_event_counter, 4)
class TestAgentLaunchOps(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
unittest # suppress an pycharm inspector error if all unittest.skip references are commented out
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.DSC = DatasetManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.OMS = ObservatoryManagementServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
# @unittest.skip('this test just for debugging setup')
# def test_just_the_setup(self):
# return
def test_get_agent_client_noprocess(self):
inst_device_id = self.RR2.create(any_old(RT.InstrumentDevice))
iap = ResourceAgentClient._get_agent_process_id(inst_device_id)
# should be no running agent
self.assertIsNone(iap)
# should raise NotFound
self.assertRaises(NotFound, ResourceAgentClient, inst_device_id)
def test_resource_state_save_restore(self):
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.IMS.create_instrument_model(instModel_obj)
log.debug('new InstrumentModel id = %s ', instModel_id)
# Create InstrumentAgent
raw_config = StreamConfiguration(
stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict')
parsed_config = StreamConfiguration(
stream_name='parsed',
parameter_dictionary_name='ctd_parsed_param_dict')
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri=DRV_URI_GOOD,
stream_configurations=[raw_config, parsed_config])
instAgent_id = self.IMS.create_instrument_agent(instAgent_obj)
log.debug('new InstrumentAgent id = %s', instAgent_id)
self.IMS.assign_instrument_model_to_instrument_agent(
instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug(
'test_activateInstrumentSample: Create instrument resource to represent the SBE37 '
+ '(SA Req: L4-CI-SA-RQ-241) ')
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345")
instDevice_id = self.IMS.create_instrument_device(
instrument_device=instDevice_obj)
self.IMS.assign_instrument_model_to_instrument_device(
instModel_id, instDevice_id)
log.debug(
"test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ",
instDevice_id)
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance,
name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config=port_agent_config)
instAgentInstance_id = self.IMS.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id)
spdict_id = self.DSC.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.PSC.create_stream_definition(
name='parsed', parameter_dictionary_id=spdict_id)
rpdict_id = self.DSC.read_parameter_dictionary_by_name(
'ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.PSC.create_stream_definition(
name='raw', parameter_dictionary_id=rpdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test')
data_product_id1 = self.DP.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug('new dp_id = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id1)
self.DP.activate_data_product_persistence(
data_product_id=data_product_id1)
self.addCleanup(self.DP.suspend_data_product_persistence,
data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream,
None, True)
log.debug('Data product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.RR.find_objects(data_product_id1,
PRED.hasDataset, RT.Dataset,
True)
log.debug('Data set for data_product_id1 = %s', dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
#create the datastore at the beginning of each int test that persists data
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test')
data_product_id2 = self.DP.create_data_product(
data_product=dp_obj, stream_definition_id=raw_stream_def_id)
log.debug('new dp_id = %s', str(data_product_id2))
self.DAMS.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id2)
self.DP.activate_data_product_persistence(
data_product_id=data_product_id2)
self.addCleanup(self.DP.suspend_data_product_persistence,
data_product_id2)
# spin up agent
self.IMS.start_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id)
self.addCleanup(self.IMS.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
#wait for start
instance_obj = self.IMS.read_instrument_agent_instance(
instAgentInstance_id)
gate = AgentProcessStateGate(self.PDC.read_process, instDevice_id,
ProcessStateEnum.RUNNING)
self.assertTrue(
gate. await (30),
"The instrument agent instance (%s) did not spawn in 30 seconds" %
gate.process_id)
# take snapshot of config
snap_id = self.IMS.save_resource_state(instDevice_id, "xyzzy snapshot")
snap_obj = self.RR.read_attachment(snap_id, include_content=True)
#modify config
instance_obj.driver_config["comms_config"] = "BAD_DATA"
self.RR.update(instance_obj)
#restore config
self.IMS.restore_resource_state(instDevice_id, snap_id)
instance_obj = self.RR.read(instAgentInstance_id)
if "BAD_DATA" == instance_obj.driver_config["comms_config"]:
print "Saved config:"
print snap_obj.content
self.fail("Saved config was not properly restored")
self.assertNotEqual("BAD_DATA",
instance_obj.driver_config["comms_config"])
self.DP.delete_data_product(data_product_id1)
self.DP.delete_data_product(data_product_id2)
def test_agent_instance_config_hasDevice(self):
def assign_fn(child_device_id, parent_device_id):
self.RR2.create_association(parent_device_id, PRED.hasDevice,
child_device_id)
def find_fn(parent_device_id):
ret, _ = self.RR.find_objects(subject=parent_device_id,
predicate=PRED.hasDevice,
id_only=True)
return ret
self.base_agent_instance_config(assign_fn, find_fn)
log.info("END test_agent_instance_config_hasDevice")
def test_agent_instance_config_hasNetworkParent(self):
def assign_fn(child_device_id, parent_device_id):
self.RR2.create_association(child_device_id, PRED.hasNetworkParent,
parent_device_id)
def find_fn(parent_device_id):
ret, _ = self.RR.find_subjects(object=parent_device_id,
predicate=PRED.hasNetworkParent,
id_only=True)
return ret
self.base_agent_instance_config(assign_fn, find_fn)
log.info("END test_agent_instance_config_hasNetworkParent")
def base_agent_instance_config(
self, assign_child_platform_to_parent_platform_fn,
find_child_platform_ids_of_parent_platform_fn):
"""
Verify that agent configurations are being built properly
"""
clients = DotDict()
clients.resource_registry = self.RR
clients.pubsub_management = self.PSC
clients.dataset_management = self.DSC
config_builder = DotDict
config_builder.i = None
config_builder.p = None
def refresh_pconfig_builder_hack(config_builder):
"""
ugly hack to get around "idempotent" RR2 caching
remove after https://github.com/ooici/coi-services/pull/1190
"""
config_builder.p = PlatformAgentConfigurationBuilder(clients)
def refresh_iconfig_builder_hack(config_builder):
"""
ugly hack to get around "idempotent" RR2 caching
remove after https://github.com/ooici/coi-services/pull/1190
"""
config_builder.i = InstrumentAgentConfigurationBuilder(clients)
org_obj = any_old(RT.Org)
org_id = self.RR2.create(org_obj)
inst_startup_config = {'startup': 'config'}
generic_alerts_config = [{'lvl2': 'lvl3val'}]
required_config_keys = [
'org_governance_name', 'device_type', 'agent', 'driver_config',
'stream_config', 'startup_config', 'aparam_alerts_config',
'children'
]
def verify_instrument_config(config, device_id):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual(org_obj.org_governance_name,
config['org_governance_name'])
self.assertEqual(RT.InstrumentDevice, config['device_type'])
self.assertIn('driver_config', config)
driver_config = config['driver_config']
expected_driver_fields = {
'process_type': ('ZMQPyClassDriverLauncher', ),
}
for k, v in expected_driver_fields.iteritems():
self.assertIn(k, driver_config)
self.assertEqual(v, driver_config[k])
self.assertEqual
self.assertIn('resource_id', config['agent'])
self.assertEqual(device_id, config['agent']['resource_id'])
self.assertEqual(inst_startup_config, config['startup_config'])
self.assertIn('aparam_alerts_config', config)
self.assertEqual(generic_alerts_config,
config['aparam_alerts_config'])
self.assertIn('stream_config', config)
for key in ['children']:
self.assertEqual({}, config[key])
# TODO(OOIION-1495) review the asserts below related with
# requiring 'ports' to be present in the driver_config.
# See recent adjustment in agent_configuration_builder.py,
# which I did to avoid other tests to fail.
# The asserts below would make the following tests fail:
# test_agent_instance_config_hasDevice
# test_agent_instance_config_hasNetworkParent
def verify_child_config(config, device_id, inst_device_id=None):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual(org_obj.org_governance_name,
config['org_governance_name'])
self.assertEqual(RT.PlatformDevice, config['device_type'])
self.assertIn('resource_id', config['agent'])
self.assertEqual(device_id, config['agent']['resource_id'])
self.assertIn('aparam_alerts_config', config)
self.assertEqual(generic_alerts_config,
config['aparam_alerts_config'])
self.assertIn('stream_config', config)
self.assertIn('driver_config', config)
self.assertIn('foo', config['driver_config'])
"""
self.assertIn('ports', config['driver_config'])
"""
self.assertEqual('bar', config['driver_config']['foo'])
self.assertIn('process_type', config['driver_config'])
self.assertEqual(('ZMQPyClassDriverLauncher', ),
config['driver_config']['process_type'])
if None is inst_device_id:
for key in ['children', 'startup_config']:
self.assertEqual({}, config[key])
else:
for key in ['startup_config']:
self.assertEqual({}, config[key])
self.assertIn(inst_device_id, config['children'])
verify_instrument_config(config['children'][inst_device_id],
inst_device_id)
"""
if config['driver_config']['ports']:
self.assertTrue( isinstance(config['driver_config']['ports'], dict) )
"""
def verify_parent_config(config,
parent_device_id,
child_device_id,
inst_device_id=None):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual(org_obj.org_governance_name,
config['org_governance_name'])
self.assertEqual(RT.PlatformDevice, config['device_type'])
self.assertIn('process_type', config['driver_config'])
"""
self.assertIn('ports', config['driver_config'])
"""
self.assertEqual(('ZMQPyClassDriverLauncher', ),
config['driver_config']['process_type'])
self.assertIn('resource_id', config['agent'])
self.assertEqual(parent_device_id, config['agent']['resource_id'])
self.assertIn('aparam_alerts_config', config)
self.assertEqual(generic_alerts_config,
config['aparam_alerts_config'])
self.assertIn('stream_config', config)
for key in ['startup_config']:
self.assertEqual({}, config[key])
"""
if config['driver_config']['ports']:
self.assertTrue( isinstance(config['driver_config']['ports'], dict) )
"""
self.assertIn(child_device_id, config['children'])
verify_child_config(config['children'][child_device_id],
child_device_id, inst_device_id)
rpdict_id = self.DSC.read_parameter_dictionary_by_name(
'ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.PSC.create_stream_definition(
name='raw', parameter_dictionary_id=rpdict_id)
#todo: create org and figure out which agent resource needs to get assigned to it
def _make_platform_agent_structure(name='', agent_config=None):
if None is agent_config: agent_config = {}
# instance creation
platform_agent_instance_obj = any_old(
RT.PlatformAgentInstance, {
'driver_config': {
'foo': 'bar'
},
'alerts': generic_alerts_config
})
platform_agent_instance_obj.agent_config = agent_config
platform_agent_instance_id = self.IMS.create_platform_agent_instance(
platform_agent_instance_obj)
# agent creation
raw_config = StreamConfiguration(
stream_name='raw',
parameter_dictionary_name='ctd_raw_param_dict')
platform_agent_obj = any_old(
RT.PlatformAgent, {"stream_configurations": [raw_config]})
platform_agent_id = self.IMS.create_platform_agent(
platform_agent_obj)
# device creation
platform_device_id = self.IMS.create_platform_device(
any_old(RT.PlatformDevice))
# data product creation
dp_obj = any_old(RT.DataProduct)
dp_id = self.DP.create_data_product(
data_product=dp_obj, stream_definition_id=raw_stream_def_id)
self.DAMS.assign_data_product(input_resource_id=platform_device_id,
data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
self.addCleanup(self.DP.suspend_data_product_persistence, dp_id)
#deployment creation
site_obj = IonObject(RT.PlatformSite, name='sitePlatform')
site_id = self.OMS.create_platform_site(platform_site=site_obj)
# find current deployment using time constraints
current_time = int(calendar.timegm(time.gmtime()))
# two years on either side of current time
start = current_time - 63115200
end = current_time + 63115200
temporal_bounds = IonObject(OT.TemporalBounds,
name='planned',
start_datetime=str(start),
end_datetime=str(end))
platform_port_obj = IonObject(
OT.PlatformPort,
reference_designator='GA01SUMO-FI003-09-CTDMO0999',
port_type=PortTypeEnum.UPLINK,
ip_address=0)
deployment_obj = IonObject(
RT.Deployment,
name='TestPlatformDeployment_' + name,
description='some new deployment',
context=IonObject(OT.CabledNodeDeploymentContext),
constraint_list=[temporal_bounds],
port_assignments={platform_device_id: platform_port_obj})
deploy_id = self.OMS.create_deployment(
deployment=deployment_obj,
site_id=site_id,
device_id=platform_device_id)
# assignments
self.RR2.assign_platform_agent_instance_to_platform_device_with_has_agent_instance(
platform_agent_instance_id, platform_device_id)
self.RR2.assign_platform_agent_to_platform_agent_instance_with_has_agent_definition(
platform_agent_id, platform_agent_instance_id)
self.RR2.assign_platform_device_to_org_with_has_resource(
platform_agent_instance_id, org_id)
return platform_agent_instance_id, platform_agent_id, platform_device_id
def _make_instrument_agent_structure(agent_config=None):
if None is agent_config: agent_config = {}
# instance creation
instrument_agent_instance_obj = any_old(
RT.InstrumentAgentInstance, {
"startup_config": inst_startup_config,
'alerts': generic_alerts_config
})
instrument_agent_instance_obj.agent_config = agent_config
instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(
instrument_agent_instance_obj)
# agent creation
raw_config = StreamConfiguration(
stream_name='raw',
parameter_dictionary_name='ctd_raw_param_dict')
instrument_agent_obj = any_old(
RT.InstrumentAgent, {"stream_configurations": [raw_config]})
instrument_agent_id = self.IMS.create_instrument_agent(
instrument_agent_obj)
# device creation
instrument_device_id = self.IMS.create_instrument_device(
any_old(RT.InstrumentDevice))
# data product creation
dp_obj = any_old(RT.DataProduct)
dp_id = self.DP.create_data_product(
data_product=dp_obj, stream_definition_id=raw_stream_def_id)
self.DAMS.assign_data_product(
input_resource_id=instrument_device_id, data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
self.addCleanup(self.DP.suspend_data_product_persistence, dp_id)
#deployment creation
site_obj = IonObject(RT.InstrumentSite, name='siteInstrument')
site_id = self.OMS.create_instrument_site(instrument_site=site_obj)
# find current deployment using time constraints
current_time = int(calendar.timegm(time.gmtime()))
# two years on either side of current time
start = current_time - 63115200
end = current_time + 63115200
temporal_bounds = IonObject(OT.TemporalBounds,
name='planned',
start_datetime=str(start),
end_datetime=str(end))
platform_port_obj = IonObject(
OT.PlatformPort,
reference_designator='GA01SUMO-FI003-08-CTDMO0888',
port_type=PortTypeEnum.PAYLOAD,
ip_address=0)
deployment_obj = IonObject(
RT.Deployment,
name='TestDeployment for Cabled Instrument',
description='some new deployment',
context=IonObject(OT.CabledInstrumentDeploymentContext),
constraint_list=[temporal_bounds],
port_assignments={instrument_device_id: platform_port_obj})
deploy_id = self.OMS.create_deployment(
deployment=deployment_obj,
site_id=site_id,
device_id=instrument_device_id)
# assignments
self.RR2.assign_instrument_agent_instance_to_instrument_device_with_has_agent_instance(
instrument_agent_instance_id, instrument_device_id)
self.RR2.assign_instrument_agent_to_instrument_agent_instance_with_has_agent_definition(
instrument_agent_id, instrument_agent_instance_id)
self.RR2.assign_instrument_device_to_org_with_has_resource(
instrument_agent_instance_id, org_id)
return instrument_agent_instance_id, instrument_agent_id, instrument_device_id
# can't do anything without an agent instance obj
log.debug(
"Testing that preparing a launcher without agent instance raises an error"
)
refresh_pconfig_builder_hack(
config_builder
) # associations have changed since builder was instantiated
self.assertRaises(AssertionError,
config_builder.p.prepare,
will_launch=False)
log.debug(
"Making the structure for a platform agent, which will be the child"
)
platform_agent_instance_child_id, _, platform_device_child_id = _make_platform_agent_structure(
name='child')
platform_agent_instance_child_obj = self.RR2.read(
platform_agent_instance_child_id)
log.debug("Preparing a valid agent instance launch, for config only")
refresh_pconfig_builder_hack(
config_builder
) # associations have changed since builder was instantiated
config_builder.p.set_agent_instance_object(
platform_agent_instance_child_obj)
child_config = config_builder.p.prepare(will_launch=False)
verify_child_config(child_config, platform_device_child_id)
log.debug(
"Making the structure for a platform agent, which will be the parent"
)
platform_agent_instance_parent_id, _, platform_device_parent_id = _make_platform_agent_structure(
name='parent')
platform_agent_instance_parent_obj = self.RR2.read(
platform_agent_instance_parent_id)
log.debug("Testing child-less parent as a child config")
refresh_pconfig_builder_hack(
config_builder
) # associations have changed since builder was instantiated
config_builder.p.set_agent_instance_object(
platform_agent_instance_parent_obj)
parent_config = config_builder.p.prepare(will_launch=False)
verify_child_config(parent_config, platform_device_parent_id)
log.debug("assigning child platform to parent")
assign_child_platform_to_parent_platform_fn(platform_device_child_id,
platform_device_parent_id)
child_device_ids = find_child_platform_ids_of_parent_platform_fn(
platform_device_parent_id)
self.assertNotEqual(0, len(child_device_ids))
log.debug("Testing parent + child as parent config")
refresh_pconfig_builder_hack(
config_builder
) # associations have changed since builder was instantiated
config_builder.p.set_agent_instance_object(
platform_agent_instance_parent_obj)
parent_config = config_builder.p.prepare(will_launch=False)
verify_parent_config(parent_config, platform_device_parent_id,
platform_device_child_id)
log.debug("making the structure for an instrument agent")
instrument_agent_instance_id, _, instrument_device_id = _make_instrument_agent_structure(
)
instrument_agent_instance_obj = self.RR2.read(
instrument_agent_instance_id)
log.debug("Testing instrument config")
refresh_iconfig_builder_hack(
config_builder
) # associations have changed since builder was instantiated
config_builder.i.set_agent_instance_object(
instrument_agent_instance_obj)
instrument_config = config_builder.i.prepare(will_launch=False)
verify_instrument_config(instrument_config, instrument_device_id)
log.debug("assigning instrument to platform")
self.RR2.assign_instrument_device_to_platform_device_with_has_device(
instrument_device_id, platform_device_child_id)
child_device_ids = self.RR2.find_instrument_device_ids_of_platform_device_using_has_device(
platform_device_child_id)
self.assertNotEqual(0, len(child_device_ids))
log.debug("Testing entire config")
refresh_pconfig_builder_hack(
config_builder
) # associations have changed since builder was instantiated
config_builder.p.set_agent_instance_object(
platform_agent_instance_parent_obj)
full_config = config_builder.p.prepare(will_launch=False)
verify_parent_config(full_config, platform_device_parent_id,
platform_device_child_id, instrument_device_id)
#self.fail(parent_config)
#plauncher.prepare(will_launch=False)
log.info("END base_agent_instance_config")
class TestDataProductManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dpsc_cli = DataProductManagementServiceClient(node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.unsc = UserNotificationServiceClient()
self.data_retriever = DataRetrieverServiceClient()
#------------------------------------------
# Create the environment
#------------------------------------------
datastore_name = CACHE_DATASTORE_NAME
self.db = self.container.datastore_manager.get_datastore(datastore_name)
self.stream_def_id = self.pubsubcli.create_stream_definition(name='SBE37_CDM')
self.process_definitions = {}
ingestion_worker_definition = ProcessDefinition(name='ingestion worker')
ingestion_worker_definition.executable = {
'module':'ion.processes.data.ingestion.science_granule_ingestion_worker',
'class' :'ScienceGranuleIngestionWorker'
}
process_definition_id = self.process_dispatcher.create_process_definition(process_definition=ingestion_worker_definition)
self.process_definitions['ingestion_worker'] = process_definition_id
self.pids = []
self.exchange_points = []
self.exchange_names = []
#------------------------------------------------------------------------------------------------
# First launch the ingestors
#------------------------------------------------------------------------------------------------
self.exchange_space = 'science_granule_ingestion'
self.exchange_point = 'science_data'
config = DotDict()
config.process.datastore_name = 'datasets'
config.process.queue_name = self.exchange_space
self.exchange_names.append(self.exchange_space)
self.exchange_points.append(self.exchange_point)
pid = self.process_dispatcher.schedule_process(self.process_definitions['ingestion_worker'],configuration=config)
log.debug("the ingestion worker process id: %s", pid)
self.pids.append(pid)
self.addCleanup(self.cleaning_up)
def cleaning_up(self):
for pid in self.pids:
log.debug("number of pids to be terminated: %s", len(self.pids))
try:
self.process_dispatcher.cancel_process(pid)
log.debug("Terminated the process: %s", pid)
except:
log.debug("could not terminate the process id: %s" % pid)
IngestionManagementIntTest.clean_subscriptions()
for xn in self.exchange_names:
xni = self.container.ex_manager.create_xn_queue(xn)
xni.delete()
for xp in self.exchange_points:
xpi = self.container.ex_manager.create_xp(xp)
xpi.delete()
def get_datastore(self, dataset_id):
dataset = self.dataset_management.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def test_create_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
parameter_dictionary_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict')
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=parameter_dictionary_id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
# Generic time-series data domain creation
tdom, sdom = time_series_domain()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom.dump(),
spatial_domain = sdom.dump())
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 200.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = 100.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 50.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = 100.0
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product( data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
self.dpsc_cli.activate_data_product_persistence(dp_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
self.assertEquals(dp_obj.geospatial_point_center.lat, 150.0)
log.debug('Created data product %s', dp_obj)
#------------------------------------------------------------------------------------------------
# test creating a new data product with a stream definition
#------------------------------------------------------------------------------------------------
log.debug('Creating new data product with a stream definition')
dp_obj = IonObject(RT.DataProduct,
name='DP2',
description='some new dp',
temporal_domain = tdom.dump(),
spatial_domain = sdom.dump())
dp_id2 = self.dpsc_cli.create_data_product(dp_obj, ctd_stream_def_id)
self.dpsc_cli.activate_data_product_persistence(dp_id2)
log.debug('new dp_id = %s' % dp_id2)
#------------------------------------------------------------------------------------------------
#make sure data product is associated with stream def
#------------------------------------------------------------------------------------------------
streamdefs = []
streams, _ = self.rrclient.find_objects(dp_id2, PRED.hasStream, RT.Stream, True)
for s in streams:
log.debug("Checking stream %s" % s)
sdefs, _ = self.rrclient.find_objects(s, PRED.hasStreamDefinition, RT.StreamDefinition, True)
for sd in sdefs:
log.debug("Checking streamdef %s" % sd)
streamdefs.append(sd)
self.assertIn(ctd_stream_def_id, streamdefs)
# test reading a non-existent data product
log.debug('reading non-existent data product')
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product('some_fake_id')
# update a data product (tests read also)
log.debug('Updating data product')
# first get the existing dp object
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# now tweak the object
dp_obj.description = 'the very first dp'
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 300.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = 200.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 150.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = 200.0
# now write the dp back to the registry
update_result = self.dpsc_cli.update_data_product(dp_obj)
# now get the dp back to see if it was updated
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertEquals(dp_obj.description,'the very first dp')
self.assertEquals(dp_obj.geospatial_point_center.lat, 250.0)
log.debug('Updated data product %s', dp_obj)
#test extension
extended_product = self.dpsc_cli.get_data_product_extension(dp_id)
self.assertEqual(dp_id, extended_product._id)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_product.computed.product_download_size_estimated.status)
self.assertEqual(0, extended_product.computed.product_download_size_estimated.value)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_product.computed.parameters.status)
#log.debug("test_create_data_product: parameters %s" % extended_product.computed.parameters.value)
# now 'delete' the data product
log.debug("deleting data product: %s" % dp_id)
self.dpsc_cli.delete_data_product(dp_id)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# Get the events corresponding to the data product
ret = self.unsc.get_recent_events(resource_id=dp_id)
events = ret.value
for event in events:
log.debug("event time: %s" % event.ts_created)
self.assertTrue(len(events) > 0)
def test_data_product_stream_def(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
stream_def_id = self.dpsc_cli.get_data_product_stream_definition(dp_id)
self.assertEquals(ctd_stream_def_id, stream_def_id)
def test_activate_suspend_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
log.debug("Created an IonObject for a data product: %s" % dp_obj)
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test activate and suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.activate_data_product_persistence(dp_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True)
if not dataset_ids:
raise NotFound("Data Product %s dataset does not exist" % str(dp_id))
self.get_datastore(dataset_ids[0])
# Check that the streams associated with the data product are persisted with
stream_ids, _ = self.rrclient.find_objects(dp_id,PRED.hasStream,RT.Stream,True)
for stream_id in stream_ids:
self.assertTrue(self.ingestclient.is_persisted(stream_id))
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data = self.data_retriever.retrieve(dataset_ids[0])
self.assertIsInstance(replay_data, Granule)
log.debug("The data retriever was able to replay the dataset that was attached to the data product "
"we wanted to be persisted. Therefore the data product was indeed persisted with "
"otherwise we could not have retrieved its dataset using the data retriever. Therefore "
"this demonstration shows that L4-CI-SA-RQ-267 is satisfied: 'Data product management shall persist data products'")
data_product_object = self.rrclient.read(dp_id)
self.assertEquals(data_product_object.name,'DP1')
self.assertEquals(data_product_object.description,'some new dp')
log.debug("Towards L4-CI-SA-RQ-308: 'Data product management shall persist data product metadata'. "
" Attributes in create for the data product obj, name= '%s', description='%s', match those of object from the "
"resource registry, name='%s', desc='%s'" % (dp_obj.name, dp_obj.description,data_product_object.name,
data_product_object.description))
#------------------------------------------------------------------------------------------------
# test suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.suspend_data_product_persistence(dp_id)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
with self.assertRaises(NotFound):
dp_obj = self.rrclient.read(dp_id)
class BulkIngestBase(object):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.pubsub_management = PubsubManagementServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
self.data_acquisition_management = DataAcquisitionManagementServiceClient()
self.data_retriever = DataRetrieverServiceClient()
self.process_dispatch_client = ProcessDispatcherServiceClient(node=self.container.node)
self.resource_registry = self.container.resource_registry
self.context_ids = self.build_param_contexts()
self.setup_resources()
def build_param_contexts(self):
raise NotImplementedError('build_param_contexts must be implemented in child classes')
def create_external_dataset(self):
raise NotImplementedError('create_external_dataset must be implemented in child classes')
def get_dvr_config(self):
raise NotImplementedError('get_dvr_config must be implemented in child classes')
def get_retrieve_client(self, dataset_id=''):
raise NotImplementedError('get_retrieve_client must be implemented in child classes')
def test_data_ingest(self):
self.pdict_id = self.create_parameter_dict(self.name)
self.stream_def_id = self.create_stream_def(self.name, self.pdict_id)
self.data_product_id = self.create_data_product(self.name, self.description, self.stream_def_id)
self.dataset_id = self.get_dataset_id(self.data_product_id)
self.stream_id, self.route = self.get_stream_id_and_route(self.data_product_id)
self.external_dataset_id = self.create_external_dataset()
self.data_producer_id = self.register_external_dataset(self.external_dataset_id)
self.start_agent()
def create_parameter_dict(self, name=''):
return self.dataset_management.create_parameter_dictionary(name=name, parameter_context_ids=self.context_ids, temporal_context='time')
def create_stream_def(self, name='', pdict_id=''):
return self.pubsub_management.create_stream_definition(name=name, parameter_dictionary_id=pdict_id)
def create_data_product(self, name='', description='', stream_def_id=''):
tdom, sdom = time_series_domain()
tdom = tdom.dump()
sdom = sdom.dump()
dp_obj = DataProduct(
name=name,
description=description,
processing_level_code='Parsed_Canonical',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id = self.data_product_management.create_data_product(data_product=dp_obj, stream_definition_id=stream_def_id)
self.data_product_management.activate_data_product_persistence(data_product_id)
return data_product_id
def register_external_dataset(self, external_dataset_id=''):
return self.data_acquisition_management.register_external_data_set(external_dataset_id=external_dataset_id)
def get_dataset_id(self, data_product_id=''):
dataset_ids, assocs = self.resource_registry.find_objects(subject=data_product_id, predicate='hasDataset', id_only=True)
return dataset_ids[0]
def get_stream_id_and_route(self, data_product_id):
stream_ids, _ = self.resource_registry.find_objects(data_product_id, PRED.hasStream, RT.Stream, id_only=True)
stream_id = stream_ids[0]
route = self.pubsub_management.read_stream_route(stream_id)
#self.create_logger(self.name, stream_id)
return stream_id, route
def start_agent(self):
agent_config = {
'driver_config': self.get_dvr_config(),
'stream_config': {},
'agent': {'resource_id': self.external_dataset_id},
'test_mode': True
}
_ia_pid = self.container.spawn_process(
name=self.EDA_NAME,
module=self.EDA_MOD,
cls=self.EDA_CLS,
config=agent_config)
self._ia_client = ResourceAgentClient(self.external_dataset_id, process=FakeProcess())
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command=DriverEvent.START_AUTOSAMPLE)
self._ia_client.execute_resource(command=cmd)
self.start_listener(self.dataset_id)
def stop_agent(self):
cmd = AgentCommand(command=DriverEvent.STOP_AUTOSAMPLE)
self._ia_client.execute_resource(cmd)
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
self._ia_client.execute_agent(cmd)
def start_listener(self, dataset_id=''):
dataset_modified = Event()
#callback to use retrieve to get data from the coverage
def cb(*args, **kwargs):
self.get_retrieve_client(dataset_id=dataset_id)
#callback to keep execution going once dataset has been fully ingested
def cb2(*args, **kwargs):
dataset_modified.set()
es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id)
es.start()
es2 = EventSubscriber(event_type=OT.DeviceCommonLifecycleEvent, callback=cb2, origin='BaseDataHandler._acquire_sample')
es2.start()
self.addCleanup(es.stop)
self.addCleanup(es2.stop)
#let it go for up to 120 seconds, then stop the agent and reset it
dataset_modified.wait(120)
self.stop_agent()
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name+'_logger')
producer_definition.executable = {
'module':'ion.processes.data.stream_granule_logger',
'class':'StreamGranuleLogger'
}
logger_procdef_id = self.process_dispatch_client.create_process_definition(process_definition=producer_definition)
configuration = {
'process':{
'stream_id':stream_id,
}
}
pid = self.process_dispatch_client.schedule_process(process_definition_id=logger_procdef_id, configuration=configuration)
return pid
class TestPlatformInstrument(BaseIntTestPlatform):
def setUp(self):
self._start_container()
self._pp = pprint.PrettyPrinter()
log.debug("oms_uri = %s", OMS_URI)
self.oms = CIOMSClientFactory.create_instance(OMS_URI)
#url = OmsTestMixin.start_http_server()
#log.debug("TestPlatformInstrument:setup http url %s", url)
#
#result = self.oms.event.register_event_listener(url)
#log.debug("TestPlatformInstrument:setup register_event_listener result %s", result)
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.dpclient = DataProductManagementServiceClient(
node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.RR2 = EnhancedResourceRegistryClient(self.rrclient)
self.org_id = self.RR2.create(any_old(RT.Org))
log.debug("Org created: %s", self.org_id)
# see _set_receive_timeout
self._receive_timeout = 300
self.instrument_device_id = ''
self.platform_device_id = ''
self.platform_site_id = ''
self.platform_agent_instance_id = ''
self._pa_client = ''
def done():
CIOMSClientFactory.destroy_instance(self.oms)
event_notifications = OmsTestMixin.stop_http_server()
log.info("event_notifications = %s" % str(event_notifications))
self.addCleanup(done)
@unittest.skip('Must be run locally...')
def test_platform_with_instrument_streaming(self):
#
# The following is with just a single platform and the single
# instrument "SBE37_SIM_08", which corresponds to the one on port 4008.
#
#load the paramaters and the param dicts necesssary for the VEL3D
log.debug(
"load params------------------------------------------------------------------------------"
)
self._load_params()
log.debug(
" _register_oms_listener------------------------------------------------------------------------------"
)
self._register_oms_listener()
#create the instrument device/agent/mode
log.debug("---------- create_instrument_resources ----------")
self._create_instrument_resources()
#create the platform device, agent and instance
log.debug("---------- create_platform_configuration ----------")
self._create_platform_configuration('LPJBox_CI_Ben_Hall')
self.rrclient.create_association(subject=self.platform_device_id,
predicate=PRED.hasDevice,
object=self.instrument_device_id)
log.debug("---------- start_platform ----------")
self._start_platform()
self.addCleanup(self._stop_platform)
# get everything in command mode:
self._ping_agent()
log.debug(" ---------- initialize ----------")
self._initialize()
_ia_client = ResourceAgentClient(self.instrument_device_id,
process=FakeProcess())
state = _ia_client.get_agent_state()
log.info("TestPlatformInstrument get_agent_state %s", state)
log.debug(" ---------- go_active ----------")
self._go_active()
state = _ia_client.get_agent_state()
log.info("TestPlatformInstrument get_agent_state %s", state)
log.debug("---------- run ----------")
self._run()
gevent.sleep(2)
log.debug(" ---------- _start_resource_monitoring ----------")
self._start_resource_monitoring()
gevent.sleep(2)
#
# # verify the instrument is command state:
# state = ia_client.get_agent_state()
# log.debug(" TestPlatformInstrument get_agent_state: %s", state)
# self.assertEqual(state, ResourceAgentState.COMMAND) _stop_resource_monitoring
log.debug(" ---------- _stop_resource_monitoring ----------")
self._stop_resource_monitoring()
gevent.sleep(2)
log.debug(" ---------- go_inactive ----------")
self._go_inactive()
state = _ia_client.get_agent_state()
log.info("TestPlatformInstrument get_agent_state %s", state)
self._reset()
self._shutdown()
def _get_platform_attributes(self):
log.debug(" ----------get_platform_attributes ----------")
attr_infos = self.oms.attr.get_platform_attributes(
'LPJBox_CI_Ben_Hall')
log.debug('_get_platform_attributes: %s', self._pp.pformat(attr_infos))
attrs = attr_infos['LPJBox_CI_Ben_Hall']
for attrid, arrinfo in attrs.iteritems():
arrinfo['attr_id'] = attrid
log.debug('_get_platform_attributes: %s', self._pp.pformat(attrs))
return attrs
def _load_params(self):
log.info(" ---------- load_params ----------")
# load_parameter_scenarios
self.container.spawn_process(
"Loader",
"ion.processes.bootstrap.ion_loader",
"IONLoader",
config=dict(
op="load",
scenario="BETA",
path="master",
categories=
"ParameterFunctions,ParameterDefs,ParameterDictionary,StreamDefinition",
clearcols="owner_id,org_ids",
assets="res/preload/r2_ioc/ooi_assets",
parseooi="True",
))
def _create_platform_configuration(self,
platform_id,
parent_platform_id=None):
"""
This method is an adaptation of test_agent_instance_config in
test_instrument_management_service_integration.py
@param platform_id
@param parent_platform_id
@return a DotDict with various of the constructed elements associated
to the platform.
"""
param_dict_name = 'platform_eng_parsed'
parsed_rpdict_id = self.dataset_management.read_parameter_dictionary_by_name(
param_dict_name, id_only=True)
self.parsed_stream_def_id = self.pubsubclient.create_stream_definition(
name='parsed', parameter_dictionary_id=parsed_rpdict_id)
driver_config = PLTFRM_DVR_CONFIG
driver_config['attributes'] = self._get_platform_attributes(
) #self._platform_attributes[platform_id]
#OMS returning an error for port.get_platform_ports
#driver_config['ports'] = self._platform_ports[platform_id]
log.debug("driver_config: %s", driver_config)
# instance creation
platform_agent_instance_obj = any_old(RT.PlatformAgentInstance,
{'driver_config': driver_config})
platform_agent_instance_obj.agent_config = {
'platform_config': {
'platform_id': 'LPJBox_CI_Ben_Hall',
'parent_platform_id': None
}
}
self.platform_agent_instance_id = self.imsclient.create_platform_agent_instance(
platform_agent_instance_obj)
# agent creation
platform_agent_obj = any_old(
RT.PlatformAgent, {
"stream_configurations": self._get_platform_stream_configs(),
'driver_module': PLTFRM_DVR_MOD,
'driver_class': PLTFRM_DVR_CLS
})
platform_agent_id = self.imsclient.create_platform_agent(
platform_agent_obj)
# device creation
self.platform_device_id = self.imsclient.create_platform_device(
any_old(RT.PlatformDevice))
# data product creation
dp_obj = any_old(RT.DataProduct)
dp_id = self.dpclient.create_data_product(
data_product=dp_obj,
stream_definition_id=self.parsed_stream_def_id)
self.damsclient.assign_data_product(
input_resource_id=self.platform_device_id, data_product_id=dp_id)
self.dpclient.activate_data_product_persistence(data_product_id=dp_id)
self.addCleanup(self.dpclient.delete_data_product, dp_id)
# assignments
self.RR2.assign_platform_agent_instance_to_platform_device_with_has_agent_instance(
self.platform_agent_instance_id, self.platform_device_id)
self.RR2.assign_platform_agent_to_platform_agent_instance_with_has_agent_definition(
platform_agent_id, self.platform_agent_instance_id)
self.RR2.assign_platform_device_to_org_with_has_resource(
self.platform_agent_instance_id, self.org_id)
#######################################
# dataset
log.debug('data product = %s', dp_id)
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None,
True)
log.debug('Data product stream_ids = %s', stream_ids)
stream_id = stream_ids[0]
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasDataset,
RT.Dataset, True)
log.debug('Data set for data_product_id1 = %s', dataset_ids[0])
#######################################
log.debug(
'_create_platform_site_and_deployment platform_device_id: %s',
self.platform_device_id)
site_object = IonObject(RT.PlatformSite, name='PlatformSite1')
self.platform_site_id = self.omsclient.create_platform_site(
platform_site=site_object, parent_id='')
log.debug('_create_platform_site_and_deployment site id: %s',
self.platform_site_id)
#create supporting objects for the Deployment resource
# 1. temporal constraint
# find current deployment using time constraints
current_time = int(calendar.timegm(time.gmtime()))
# two years on either side of current time
start = current_time - 63115200
end = current_time + 63115200
temporal_bounds = IonObject(OT.TemporalBounds,
name='planned',
start_datetime=str(start),
end_datetime=str(end))
# 2. PlatformPort object which defines device to port map
platform_port_obj = IonObject(
OT.PlatformPort,
reference_designator='GA01SUMO-FI003-01-CTDMO0999',
port_type=PortTypeEnum.UPLINK,
ip_address='0')
# now create the Deployment
deployment_obj = IonObject(
RT.Deployment,
name='TestPlatformDeployment',
description='some new deployment',
context=IonObject(OT.CabledNodeDeploymentContext),
constraint_list=[temporal_bounds],
port_assignments={self.platform_device_id: platform_port_obj})
platform_deployment_id = self.omsclient.create_deployment(
deployment=deployment_obj,
site_id=self.platform_site_id,
device_id=self.platform_device_id)
log.debug('_create_platform_site_and_deployment deployment_id: %s',
platform_deployment_id)
deploy_obj2 = self.omsclient.read_deployment(platform_deployment_id)
log.debug('_create_platform_site_and_deployment deploy_obj2 : %s',
deploy_obj2)
return self.platform_site_id, platform_deployment_id
def _create_instrument_resources(self):
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='VEL3D',
description="VEL3D")
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
log.debug('new InstrumentModel id = %s ', instModel_id)
raw_config = StreamConfiguration(stream_name='raw',
parameter_dictionary_name='raw')
vel3d_b_sample = StreamConfiguration(
stream_name='vel3d_b_sample',
parameter_dictionary_name='vel3d_b_sample')
vel3d_b_engineering = StreamConfiguration(
stream_name='vel3d_b_engineering',
parameter_dictionary_name='vel3d_b_engineering')
# Create InstrumentAgent
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri=
"http://sddevrepo.oceanobservatories.org/releases/nobska_mavs4_ooicore-0.0.7-py2.7.egg",
stream_configurations=[
raw_config, vel3d_b_sample, vel3d_b_engineering
])
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
log.debug('new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(
instModel_id, instAgent_id)
# Create InstrumentDevice
instDevice_obj = IonObject(RT.InstrumentDevice,
name='VEL3DDevice',
description="VEL3DDevice",
serial_number="12345")
self.instrument_device_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, self.instrument_device_id)
port_agent_config = {
'device_addr': '10.180.80.6',
'device_port': 2101,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': 1025,
'data_port': 1026,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance,
name='VEL3DAgentInstance',
description="VEL3DAgentInstance",
port_agent_config=port_agent_config,
alerts=[])
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, self.instrument_device_id)
self._start_port_agent(
self.imsclient.read_instrument_agent_instance(
instAgentInstance_id))
vel3d_b_sample_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'vel3d_b_sample', id_only=True)
vel3d_b_sample_stream_def_id = self.pubsubclient.create_stream_definition(
name='vel3d_b_sample',
parameter_dictionary_id=vel3d_b_sample_pdict_id)
vel3d_b_engineering_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'vel3d_b_engineering', id_only=True)
vel3d_b_engineering_stream_def_id = self.pubsubclient.create_stream_definition(
name='vel3d_b_engineering',
parameter_dictionary_id=vel3d_b_engineering_pdict_id)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'raw', id_only=True)
raw_stream_def_id = self.pubsubclient.create_stream_definition(
name='raw', parameter_dictionary_id=raw_pdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='vel3d_b_sample',
description='vel3d_b_sample')
data_product_id1 = self.dpclient.create_data_product(
data_product=dp_obj,
stream_definition_id=vel3d_b_sample_stream_def_id)
self.damsclient.assign_data_product(
input_resource_id=self.instrument_device_id,
data_product_id=data_product_id1)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id1)
dp_obj = IonObject(RT.DataProduct,
name='vel3d_b_engineering',
description='vel3d_b_engineering')
data_product_id2 = self.dpclient.create_data_product(
data_product=dp_obj,
stream_definition_id=vel3d_b_engineering_stream_def_id)
self.damsclient.assign_data_product(
input_resource_id=self.instrument_device_id,
data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id2)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test')
data_product_id3 = self.dpclient.create_data_product(
data_product=dp_obj, stream_definition_id=raw_stream_def_id)
self.damsclient.assign_data_product(
input_resource_id=self.instrument_device_id,
data_product_id=data_product_id3)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id3)
#create instrument site and associated deployment
site_object = IonObject(RT.InstrumentSite, name='InstrumentSite1')
instrument_site_id = self.omsclient.create_instrument_site(
instrument_site=site_object, parent_id=self.platform_site_id)
log.debug('_create_instrument_site_and_deployment site id: %s',
instrument_site_id)
#create supporting objects for the Deployment resource
# 1. temporal constraint
# find current deployment using time constraints
current_time = int(calendar.timegm(time.gmtime()))
# two years on either side of current time
start = current_time - 63115200
end = current_time + 63115200
temporal_bounds = IonObject(OT.TemporalBounds,
name='planned',
start_datetime=str(start),
end_datetime=str(end))
# 2. PlatformPort object which defines device to port map
platform_port_obj = IonObject(
OT.PlatformPort,
reference_designator='GA01SUMO-FI003-03-CTDMO0999',
port_type=PortTypeEnum.PAYLOAD,
ip_address='0')
# now create the Deployment
deployment_obj = IonObject(
RT.Deployment,
name='TestInstrumentDeployment',
description='some new deployment',
context=IonObject(OT.CabledInstrumentDeploymentContext),
constraint_list=[temporal_bounds],
port_assignments={self.instrument_device_id: platform_port_obj})
instrument_deployment_id = self.omsclient.create_deployment(
deployment=deployment_obj,
site_id=instrument_site_id,
device_id=self.instrument_device_id)
log.debug('_create_instrument_site_and_deployment deployment_id: %s',
instrument_deployment_id)
def _start_port_agent(self, instrument_agent_instance_obj=None):
"""
Construct and start the port agent, ONLY NEEDED FOR INSTRUMENT AGENTS.
"""
_port_agent_config = instrument_agent_instance_obj.port_agent_config
# It blocks until the port agent starts up or a timeout
_pagent = PortAgentProcess.launch_process(_port_agent_config,
test_mode=True)
pid = _pagent.get_pid()
port = _pagent.get_data_port()
cmd_port = _pagent.get_command_port()
log.info(
"IMS:_start_pagent returned from PortAgentProcess.launch_process pid: %s ",
pid)
# Hack to get ready for DEMO. Further though needs to be put int
# how we pass this config info around.
host = 'localhost'
driver_config = instrument_agent_instance_obj.driver_config
comms_config = driver_config.get('comms_config')
if comms_config:
host = comms_config.get('addr')
else:
log.warn("No comms_config specified, using '%s'" % host)
# Configure driver to use port agent port number.
instrument_agent_instance_obj.driver_config['comms_config'] = {
'addr': host,
'cmd_port': cmd_port,
'port': port
}
instrument_agent_instance_obj.driver_config['pagent_pid'] = pid
self.imsclient.update_instrument_agent_instance(
instrument_agent_instance_obj)
return self.imsclient.read_instrument_agent_instance(
instrument_agent_instance_obj._id)
def _start_platform(self):
"""
Starts the given platform waiting for it to transition to the
UNINITIALIZED state (note that the agent starts in the LAUNCHING state).
More in concrete the sequence of steps here are:
- prepares subscriber to receive the UNINITIALIZED state transition
- launches the platform process
- waits for the start of the process
- waits for the transition to the UNINITIALIZED state
"""
##############################################################
# prepare to receive the UNINITIALIZED state transition:
async_res = AsyncResult()
def consume_event(evt, *args, **kwargs):
log.debug("Got ResourceAgentStateEvent %s from origin %r",
evt.state, evt.origin)
if evt.state == PlatformAgentState.UNINITIALIZED:
async_res.set(evt)
# start subscriber:
sub = EventSubscriber(event_type="ResourceAgentStateEvent",
origin=self.platform_device_id,
callback=consume_event)
sub.start()
log.info(
"registered event subscriber to wait for state=%r from origin %r",
PlatformAgentState.UNINITIALIZED, self.platform_device_id)
#self._event_subscribers.append(sub)
sub._ready_event.wait(timeout=EVENT_TIMEOUT)
##############################################################
# now start the platform:
agent_instance_id = self.platform_agent_instance_id
log.debug("about to call start_platform_agent_instance with id=%s",
agent_instance_id)
pid = self.imsclient.start_platform_agent_instance(
platform_agent_instance_id=agent_instance_id)
log.debug("start_platform_agent_instance returned pid=%s", pid)
#wait for start
agent_instance_obj = self.imsclient.read_platform_agent_instance(
agent_instance_id)
gate = AgentProcessStateGate(self.processdispatchclient.read_process,
self.platform_device_id,
ProcessStateEnum.RUNNING)
self.assertTrue(
gate. await (90),
"The platform agent instance did not spawn in 90 seconds")
# Start a resource agent client to talk with the agent.
self._pa_client = ResourceAgentClient(self.platform_device_id,
name=gate.process_id,
process=FakeProcess())
log.debug("got platform agent client %s", str(self._pa_client))
##############################################################
# wait for the UNINITIALIZED event:
async_res.get(timeout=self._receive_timeout)
def _register_oms_listener(self):
#load the paramaters and the param dicts necesssary for the VEL3D
log.debug("---------- connect_to_oms ---------- ")
log.debug("oms_uri = %s", OMS_URI)
self.oms = CIOMSClientFactory.create_instance(OMS_URI)
#buddha url
url = "http://10.22.88.168:5000/ion-service/oms_event"
log.info("test_oms_events_receive:setup http url %s", url)
result = self.oms.event.register_event_listener(url)
log.debug("_register_oms_listener register_event_listener result %s",
result)
#-------------------------------------------------------------------------------------
# Set up the subscriber to catch the alert event
#-------------------------------------------------------------------------------------
def callback_for_alert(event, *args, **kwargs):
log.debug("caught an OMSDeviceStatusEvent: %s", event)
self.catch_alert.put(event)
self.event_subscriber = EventSubscriber(
event_type='OMSDeviceStatusEvent', callback=callback_for_alert)
self.event_subscriber.start()
self.addCleanup(self.event_subscriber.stop)
result = self.oms.event.generate_test_event({
'platform_id': 'fake_platform_id',
'message':
"fake event triggered from CI using OMS' generate_test_event",
'severity': '3',
'group ': 'power'
})
log.debug("_register_oms_listener generate_test_event result %s",
result)
def _stop_platform(self):
try:
self.IMS.stop_platform_agent_instance(
self.platform_agent_instance_id)
except Exception:
log.warn(
"platform_id=%r: Exception in IMS.stop_platform_agent_instance with "
"platform_agent_instance_id = %r. Perhaps already dead.",
self.platform_device_id, self.platform_agent_instance_id)
class TestActivateInstrumentIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url("res/deploy/r2deploy.yml")
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
# setup listerner vars
self._data_greenlets = []
self._no_samples = None
self._samples_received = []
def create_logger(self, name, stream_id=""):
# logger process
producer_definition = ProcessDefinition(name=name + "_logger")
producer_definition.executable = {
"module": "ion.processes.data.stream_granule_logger",
"class": "StreamGranuleLogger",
}
logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition)
configuration = {"process": {"stream_id": stream_id}}
pid = self.processdispatchclient.schedule_process(
process_definition_id=logger_procdef_id, configuration=configuration
)
return pid
@unittest.skip("TBD")
def test_activateInstrumentSample(self):
self.loggerpids = []
# Create InstrumentModel
instModel_obj = IonObject(
RT.InstrumentModel, name="SBE37IMModel", description="SBE37IMModel", model="SBE37IMModel"
)
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" % ex)
log.debug("new InstrumentModel id = %s ", instModel_id)
# Create InstrumentAgent
instAgent_obj = IonObject(
RT.InstrumentAgent,
name="agent007",
description="SBE37IMAgent",
driver_module="ion.agents.instrument.instrument_agent",
driver_class="InstrumentAgent",
)
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" % ex)
log.debug("new InstrumentAgent id = %s", instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug(
"test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) "
)
instDevice_obj = IonObject(
RT.InstrumentDevice, name="SBE37IMDevice", description="SBE37IMDevice", serial_number="12345"
)
try:
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" % ex)
log.debug(
"test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id
)
instAgentInstance_obj = IonObject(
RT.InstrumentAgentInstance,
name="SBE37IMAgentInstance",
description="SBE37IMAgentInstance",
driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
driver_class="SBE37Driver",
comms_device_address="sbe37-simulator.oceanobservatories.org",
comms_device_port=4001,
port_agent_work_dir="/tmp/",
port_agent_delimeter=["<<", ">>"],
)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id
)
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubcli.create_stream_definition(container=ctd_stream_def)
log.debug("new Stream Definition id = %s", instDevice_id)
log.debug("Creating new CDM data product with a stream definition")
craft = CoverageCraft
sdom, tdom = craft.create_domains()
sdom = sdom.dump()
tdom = tdom.dump()
parameter_dictionary = craft.create_parameters()
parameter_dictionary = parameter_dictionary.dump()
dp_obj = IonObject(
RT.DataProduct,
name="the parsed data",
description="ctd stream test",
temporal_domain=tdom,
spatial_domain=sdom,
)
data_product_id1 = self.dpclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary)
log.debug("new dp_id = %s", data_product_id1)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True)
log.debug("Data product streams1 = %s", stream_ids)
pid = self.create_logger("ctd_parsed", stream_ids[0])
self.loggerpids.append(pid)
# -------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
# -------------------------------
log.debug("test_activateInstrumentSample: create data process definition ctd_L0_all")
dpd_obj = IonObject(
RT.DataProcessDefinition,
name="ctd_L0_all",
description="transform ctd package into three separate L0 streams",
module="ion.processes.data.transforms.ctd.ctd_L0_all",
class_name="ctd_L0_all",
process_source="some_source_reference",
)
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" % ex)
# -------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
# -------------------------------
outgoing_stream_l0_conductivity = L0_conductivity_stream_definition()
outgoing_stream_l0_conductivity_id = self.pubsubcli.create_stream_definition(
container=outgoing_stream_l0_conductivity, name="L0_Conductivity"
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id
)
outgoing_stream_l0_pressure = L0_pressure_stream_definition()
outgoing_stream_l0_pressure_id = self.pubsubcli.create_stream_definition(
container=outgoing_stream_l0_pressure, name="L0_Pressure"
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id
)
outgoing_stream_l0_temperature = L0_temperature_stream_definition()
outgoing_stream_l0_temperature_id = self.pubsubcli.create_stream_definition(
container=outgoing_stream_l0_temperature, name="L0_Temperature"
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id
)
self.output_products = {}
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject(
RT.DataProduct,
name="L0_Conductivity",
description="transform output conductivity",
temporal_domain=tdom,
spatial_domain=sdom,
)
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id, parameter_dictionary
)
self.output_products["conductivity"] = ctd_l0_conductivity_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id)
stream_ids, _ = self.rrclient.find_objects(ctd_l0_conductivity_output_dp_id, PRED.hasStream, None, True)
log.debug(" ctd_l0_conductivity stream id = %s", str(stream_ids))
pid = self.create_logger(" ctd_l1_conductivity", stream_ids[0])
self.loggerpids.append(pid)
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject(
RT.DataProduct,
name="L0_Pressure",
description="transform output pressure",
temporal_domain=tdom,
spatial_domain=sdom,
)
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id, parameter_dictionary
)
self.output_products["pressure"] = ctd_l0_pressure_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id)
stream_ids, _ = self.rrclient.find_objects(ctd_l0_pressure_output_dp_id, PRED.hasStream, None, True)
log.debug(" ctd_l0_pressure stream id = %s", str(stream_ids))
pid = self.create_logger(" ctd_l0_pressure", stream_ids[0])
self.loggerpids.append(pid)
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject(
RT.DataProduct,
name="L0_Temperature",
description="transform output temperature",
temporal_domain=tdom,
spatial_domain=sdom,
)
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id, parameter_dictionary
)
self.output_products["temperature"] = ctd_l0_temperature_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id)
stream_ids, _ = self.rrclient.find_objects(ctd_l0_temperature_output_dp_id, PRED.hasStream, None, True)
log.debug(" ctd_l0_temperature stream id = %s", str(stream_ids))
pid = self.create_logger(" ctd_l0_temperature", stream_ids[0])
self.loggerpids.append(pid)
# -------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
# -------------------------------
log.debug("test_activateInstrumentSample: create L0 all data_process start")
try:
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(
ctd_L0_all_dprocdef_id, [data_product_id1], self.output_products
)
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" % ex)
log.debug("test_createTransformsThenActivateInstrument: create L0 all data_process return")
log.debug("Creating new RAW data product with a stream definition")
raw_stream_def = SBE37_RAW_stream_definition()
raw_stream_def_id = self.pubsubcli.create_stream_definition(container=raw_stream_def)
dp_obj = IonObject(
RT.DataProduct,
name="the raw data",
description="raw stream test",
temporal_domain=tdom,
spatial_domain=sdom,
)
data_product_id2 = self.dpclient.create_data_product(dp_obj, raw_stream_def_id, parameter_dictionary)
log.debug("new dp_id = %s", str(data_product_id2))
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True)
log.debug("Data product streams2 = %s", str(stream_ids))
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
log.debug("Instrument agent instance obj: = %s", str(inst_agent_instance_obj))
# Start a resource agent client to talk with the instrument agent.
# self._ia_client = ResourceAgentClient('123xyz', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess())
self._ia_client = ResourceAgentClient(instDevice_id, process=FakeProcess())
log.debug("test_activateInstrumentSample: got ia client %s", str(self._ia_client))
cmd = AgentCommand(command="initialize")
retval = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: initialize %s", str(retval))
time.sleep(1)
log.debug("test_activateInstrumentSample: Sending go_active command (L4-CI-SA-RQ-334)")
cmd = AgentCommand(command="go_active")
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrument: return value from go_active %s", str(reply))
time.sleep(1)
cmd = AgentCommand(command="get_current_state")
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug(
"test_activateInstrumentSample: current state after sending go_active command %s (L4-CI-SA-RQ-334)",
str(state),
)
cmd = AgentCommand(command="run")
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: run %s", str(reply))
time.sleep(1)
log.debug("test_activateInstrumentSample: calling acquire_sample ")
cmd = AgentCommand(command="acquire_sample")
reply = self._ia_client.execute(cmd)
log.debug("test_activateInstrumentSample: return from acquire_sample %s", str(reply))
time.sleep(1)
log.debug("test_activateInstrumentSample: calling acquire_sample 2")
cmd = AgentCommand(command="acquire_sample")
reply = self._ia_client.execute(cmd)
log.debug("test_activateInstrumentSample: return from acquire_sample 2 %s", str(reply))
time.sleep(1)
log.debug("test_activateInstrumentSample: calling acquire_sample 3")
cmd = AgentCommand(command="acquire_sample")
reply = self._ia_client.execute(cmd)
log.debug("test_activateInstrumentSample: return from acquire_sample 3 %s", str(reply))
time.sleep(2)
log.debug("test_activateInstrumentSample: calling reset ")
cmd = AgentCommand(command="reset")
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: return from reset %s", str(reply))
time.sleep(1)
# -------------------------------
# Deactivate InstrumentAgentInstance
# -------------------------------
self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
@unittest.skip("TBD")
def test_activateInstrumentStream(self):
self.loggerpids = []
# Create InstrumentModel
instModel_obj = IonObject(
RT.InstrumentModel, name="SBE37IMModel", description="SBE37IMModel", model="SBE37IMModel"
)
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" % ex)
log.debug("new InstrumentModel id = %s ", instModel_id)
# Create InstrumentAgent
instAgent_obj = IonObject(
RT.InstrumentAgent,
name="agent007",
description="SBE37IMAgent",
driver_module="ion.agents.instrument.instrument_agent",
driver_class="InstrumentAgent",
)
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" % ex)
log.debug("new InstrumentAgent id = %s", instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug(
"test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) "
)
instDevice_obj = IonObject(
RT.InstrumentDevice, name="SBE37IMDevice", description="SBE37IMDevice", serial_number="12345"
)
try:
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" % ex)
log.debug(
"test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id
)
instAgentInstance_obj = IonObject(
RT.InstrumentAgentInstance,
name="SBE37IMAgentInstance",
description="SBE37IMAgentInstance",
driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
driver_class="SBE37Driver",
comms_device_address="sbe37-simulator.oceanobservatories.org",
comms_device_port=4001,
port_agent_work_dir="/tmp/",
port_agent_delimeter=["<<", ">>"],
)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id
)
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubcli.create_stream_definition(container=ctd_stream_def)
log.debug("new Stream Definition id = %s", instDevice_id)
log.debug("Creating new CDM data product with a stream definition")
craft = CoverageCraft
sdom, tdom = craft.create_domains()
sdom = sdom.dump()
tdom = tdom.dump()
parameter_dictionary = craft.create_parameters()
parameter_dictionary = parameter_dictionary.dump()
dp_obj = IonObject(
RT.DataProduct,
name="the parsed data",
description="ctd stream test",
temporal_domain=tdom,
spatial_domain=sdom,
)
data_product_id1 = self.dpclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary)
log.debug("new dp_id = %s", data_product_id1)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True)
log.debug("Data product streams1 = %s", stream_ids)
pid = self.create_logger("ctd_parsed", stream_ids[0])
self.loggerpids.append(pid)
# simdata_subscription_id = self.pubsubcli.create_subscription(
# query=StreamQuery([stream_ids[0]]),
# exchange_name='Sim_data_queue',
# name='SimDataSubscription',
# description='SimData SubscriptionDescription'
# )
#
#
# def simdata_message_received(message, headers):
# input = str(message)
# log.debug("test_activateInstrumentStream: granule received: %s", input)
#
#
# subscriber_registrar = StreamSubscriberRegistrar(process=self.container, container=self.container)
# simdata_subscriber = subscriber_registrar.create_subscriber(exchange_name='Sim_data_queue', callback=simdata_message_received)
#
# # Start subscribers
# simdata_subscriber.start()
#
# # Activate subscriptions
# self.pubsubcli.activate_subscription(simdata_subscription_id)
log.debug("Creating new RAW data product with a stream definition")
raw_stream_def = SBE37_RAW_stream_definition()
raw_stream_def_id = self.pubsubcli.create_stream_definition(container=raw_stream_def)
dp_obj = IonObject(
RT.DataProduct,
name="the raw data",
description="raw stream test",
temporal_domain=tdom,
spatial_domain=sdom,
)
data_product_id2 = self.dpclient.create_data_product(dp_obj, raw_stream_def_id, parameter_dictionary)
log.debug("new dp_id = %s", str(data_product_id2))
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True)
log.debug("Data product streams2 = %s", str(stream_ids))
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
log.debug("test_activateInstrumentStream Instrument agent instance obj: = %s", str(inst_agent_instance_obj))
# Start a resource agent client to talk with the instrument agent.
# self._ia_client = ResourceAgentClient('123xyz', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess())
self._ia_client = ResourceAgentClient(instDevice_id, process=FakeProcess())
log.debug("test_activateInstrumentStream: got ia client %s", str(self._ia_client))
cmd = AgentCommand(command="initialize")
retval = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentStream: initialize %s", str(retval))
time.sleep(2)
log.debug("test_activateInstrumentStream: Sending go_active command (L4-CI-SA-RQ-334)")
cmd = AgentCommand(command="go_active")
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentStream: return value from go_active %s", str(reply))
time.sleep(2)
cmd = AgentCommand(command="get_current_state")
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug(
"test_activateInstrumentStream: current state after sending go_active command %s (L4-CI-SA-RQ-334)",
str(state),
)
cmd = AgentCommand(command="run")
reply = self._ia_client.execute_agent(cmd)
time.sleep(2)
cmd = AgentCommand(command="get_current_state")
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("test_activateInstrumentStream: return from run state: %s", str(state))
# Make sure the sampling rate and transmission are sane.
params = {SBE37Parameter.NAVG: 1, SBE37Parameter.INTERVAL: 5, SBE37Parameter.TXREALTIME: True}
self._ia_client.set_param(params)
time.sleep(2)
log.debug("test_activateInstrumentStream: calling go_streaming ")
cmd = AgentCommand(command="go_streaming")
reply = self._ia_client.execute_agent(cmd)
time.sleep(2)
cmd = AgentCommand(command="get_current_state")
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("test_activateInstrumentStream: return from go_streaming state: %s", str(state))
time.sleep(5)
log.debug("test_activateInstrumentStream: calling go_observatory")
cmd = AgentCommand(command="go_observatory")
reply = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command="get_current_state")
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("test_activateInstrumentStream: return from go_observatory state %s", str(state))
log.debug("test_activateInstrumentStream: calling reset ")
cmd = AgentCommand(command="reset")
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentStream: return from reset state:%s", str(reply.result))
time.sleep(2)
# -------------------------------
# Deactivate InstrumentAgentInstance
# -------------------------------
self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
class TestWorkflowManagementIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(
node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
self.workflowclient = WorkflowManagementServiceClient(
node=self.container.node)
self.process_dispatcher = ProcessDispatcherServiceClient(
node=self.container.node)
self.ctd_stream_def = SBE37_CDM_stream_definition()
def _create_ctd_input_stream_and_data_product(
self, data_product_name='ctd_parsed'):
cc = self.container
assertions = self.assertTrue
#-------------------------------
# Create CTD Parsed as the initial data product
#-------------------------------
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(
container=self.ctd_stream_def, name='Simulated CTD data')
log.debug('Creating new CDM data product with a stream definition')
dp_obj = IonObject(RT.DataProduct,
name=data_product_name,
description='ctd stream test')
try:
ctd_parsed_data_product_id = self.dataproductclient.create_data_product(
dp_obj, ctd_stream_def_id)
except Exception as ex:
self.fail("failed to create new data product: %s" % ex)
log.debug('new ctd_parsed_data_product_id = ',
ctd_parsed_data_product_id)
#Only ever need one device for testing purposes.
instDevice_obj, _ = self.rrclient.find_resources(
restype=RT.InstrumentDevice, name='SBE37IMDevice')
if instDevice_obj:
instDevice_id = instDevice_obj[0]._id
else:
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345")
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.damsclient.assign_data_product(
input_resource_id=instDevice_id,
data_product_id=ctd_parsed_data_product_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_parsed_data_product_id,
persist_data=False,
persist_metadata=False)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product_id,
PRED.hasStream, None, True)
assertions(len(stream_ids) > 0)
ctd_stream_id = stream_ids[0]
return ctd_stream_id, ctd_parsed_data_product_id
def _start_simple_input_stream_process(self, ctd_stream_id):
return self._start_input_stream_process(ctd_stream_id)
def _start_sinusoidal_input_stream_process(self, ctd_stream_id):
return self._start_input_stream_process(
ctd_stream_id, 'ion.processes.data.sinusoidal_stream_publisher',
'SinusoidalCtdPublisher')
def _start_input_stream_process(
self,
ctd_stream_id,
module='ion.processes.data.ctd_stream_publisher',
class_name='SimpleCtdPublisher'):
###
### Start the process for producing the CTD data
###
# process definition for the ctd simulator...
producer_definition = ProcessDefinition()
producer_definition.executable = {
'module': module,
'class': class_name
}
ctd_sim_procdef_id = self.process_dispatcher.create_process_definition(
process_definition=producer_definition)
# Start the ctd simulator to produce some data
configuration = {
'process': {
'stream_id': ctd_stream_id,
}
}
ctd_sim_pid = self.process_dispatcher.schedule_process(
process_definition_id=ctd_sim_procdef_id,
configuration=configuration)
return ctd_sim_pid
def _start_output_stream_listener(self,
data_product_stream_ids,
message_count_per_stream=10):
cc = self.container
assertions = self.assertTrue
###
### Make a subscriber in the test to listen for transformed data
###
salinity_subscription_id = self.pubsubclient.create_subscription(
query=StreamQuery(data_product_stream_ids),
exchange_name='workflow_test',
name="test workflow transformations",
)
pid = cc.spawn_process(name='dummy_process_for_test',
module='pyon.ion.process',
cls='SimpleProcess',
config={})
dummy_process = cc.proc_manager.procs[pid]
subscriber_registrar = StreamSubscriberRegistrar(process=dummy_process,
node=cc.node)
result = gevent.event.AsyncResult()
results = []
def message_received(message, headers):
# Heads
log.warn(' data received!')
results.append(message)
if len(results) >= len(
data_product_stream_ids
) * message_count_per_stream: #Only wait for so many messages - per stream
result.set(True)
subscriber = subscriber_registrar.create_subscriber(
exchange_name='workflow_test', callback=message_received)
subscriber.start()
# after the queue has been created it is safe to activate the subscription
self.pubsubclient.activate_subscription(
subscription_id=salinity_subscription_id)
# Assert that we have received data
assertions(result.get(timeout=30))
self.pubsubclient.deactivate_subscription(
subscription_id=salinity_subscription_id)
subscriber.stop()
return results
def _validate_messages(self, results):
cc = self.container
assertions = self.assertTrue
first_salinity_values = None
for message in results:
try:
psd = PointSupplementStreamParser(
stream_definition=self.ctd_stream_def,
stream_granule=message)
temp = psd.get_values('temperature')
log.info(psd.list_field_names())
except KeyError as ke:
temp = None
if temp is not None:
assertions(isinstance(temp, numpy.ndarray))
log.info('temperature=' + str(numpy.nanmin(temp)))
first_salinity_values = None
else:
psd = PointSupplementStreamParser(
stream_definition=SalinityTransform.outgoing_stream_def,
stream_granule=message)
log.info(psd.list_field_names())
# Test the handy info method for the names of fields in the stream def
assertions('salinity' in psd.list_field_names())
# you have to know the name of the coverage in stream def
salinity = psd.get_values('salinity')
log.info('salinity=' + str(numpy.nanmin(salinity)))
assertions(isinstance(salinity, numpy.ndarray))
assertions(numpy.nanmin(salinity) >
0.0) # salinity should always be greater than 0
if first_salinity_values is None:
first_salinity_values = salinity.tolist()
else:
second_salinity_values = salinity.tolist()
assertions(
len(first_salinity_values) == len(
second_salinity_values))
for idx in range(0, len(first_salinity_values)):
assertions(first_salinity_values[idx] *
2.0 == second_salinity_values[idx])
def _create_salinity_data_process_definition(self):
# Salinity: Data Process Definition
#First look to see if it exists and if not, then create it
dpd, _ = self.rrclient.find_resources(restype=RT.DataProcessDefinition,
name='ctd_salinity')
if len(dpd) > 0:
return dpd[0]
log.debug("Create data process definition SalinityTransform")
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_salinity',
description='create a salinity data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform',
process_source='SalinityTransform source code here...')
try:
ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except Excpetion as ex:
self.fail(
"failed to create new SalinityTransform data process definition: %s"
% ex)
# create a stream definition for the data from the salinity Transform
sal_stream_def_id = self.pubsubclient.create_stream_definition(
container=SalinityTransform.outgoing_stream_def, name='Salinity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
sal_stream_def_id, ctd_L2_salinity_dprocdef_id)
return ctd_L2_salinity_dprocdef_id
def _create_salinity_doubler_data_process_definition(self):
#First look to see if it exists and if not, then create it
dpd, _ = self.rrclient.find_resources(restype=RT.DataProcessDefinition,
name='salinity_doubler')
if len(dpd) > 0:
return dpd[0]
# Salinity Doubler: Data Process Definition
log.debug("Create data process definition SalinityDoublerTransform")
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='salinity_doubler',
description='create a salinity doubler data product',
module='ion.processes.data.transforms.example_double_salinity',
class_name='SalinityDoubler',
process_source='SalinityDoubler source code here...')
try:
salinity_doubler_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except Exception as ex:
self.fail(
"failed to create new SalinityDoubler data process definition: %s"
% ex)
# create a stream definition for the data from the salinity Transform
salinity_double_stream_def_id = self.pubsubclient.create_stream_definition(
container=SalinityDoubler.outgoing_stream_def,
name='SalinityDoubler')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
salinity_double_stream_def_id, salinity_doubler_dprocdef_id)
return salinity_doubler_dprocdef_id
def create_transform_process(self, data_process_definition_id,
data_process_input_dp_id):
data_process_definition = self.rrclient.read(
data_process_definition_id)
# Find the link between the output Stream Definition resource and the Data Process Definition resource
stream_ids, _ = self.rrclient.find_objects(data_process_definition._id,
PRED.hasStreamDefinition,
RT.StreamDefinition,
id_only=True)
if not stream_ids:
raise Inconsistent(
"The data process definition %s is missing an association to an output stream definition"
% data_process_definition._id)
process_output_stream_def_id = stream_ids[0]
#Concatenate the name of the workflow and data process definition for the name of the data product output
data_process_name = data_process_definition.name
# Create the output data product of the transform
transform_dp_obj = IonObject(
RT.DataProduct,
name=data_process_name,
description=data_process_definition.description)
transform_dp_id = self.dataproductclient.create_data_product(
transform_dp_obj, process_output_stream_def_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=transform_dp_id,
persist_data=True,
persist_metadata=True)
#last one out of the for loop is the output product id
output_data_product_id = transform_dp_id
# Create the transform data process
log.debug("create data_process and start it")
data_process_id = self.dataprocessclient.create_data_process(
data_process_definition._id, [data_process_input_dp_id],
{'output': transform_dp_id})
self.dataprocessclient.activate_data_process(data_process_id)
#Find the id of the output data stream
stream_ids, _ = self.rrclient.find_objects(transform_dp_id,
PRED.hasStream, None, True)
if not stream_ids:
raise Inconsistent(
"The data process %s is missing an association to an output stream"
% data_process_id)
return data_process_id, output_data_product_id
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Not integrated for CEI')
#@unittest.skip("Skipping for debugging ")
def test_SA_transform_components(self):
assertions = self.assertTrue
#The list of data product streams to monitor
data_product_stream_ids = list()
#Create the input data product
ctd_stream_id, ctd_parsed_data_product_id = self._create_ctd_input_stream_and_data_product(
)
data_product_stream_ids.append(ctd_stream_id)
###
### Setup the first transformation
###
# Salinity: Data Process Definition
ctd_L2_salinity_dprocdef_id = self._create_salinity_data_process_definition(
)
l2_salinity_all_data_process_id, ctd_l2_salinity_output_dp_id = self.create_transform_process(
ctd_L2_salinity_dprocdef_id, ctd_parsed_data_product_id)
## get the stream id for the transform outputs
stream_ids, _ = self.rrclient.find_objects(
ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True)
assertions(len(stream_ids) > 0)
sal_stream_id = stream_ids[0]
data_product_stream_ids.append(sal_stream_id)
###
### Setup the second transformation
###
# Salinity Doubler: Data Process Definition
salinity_doubler_dprocdef_id = self._create_salinity_doubler_data_process_definition(
)
salinity_double_data_process_id, salinity_doubler_output_dp_id = self.create_transform_process(
salinity_doubler_dprocdef_id, ctd_l2_salinity_output_dp_id)
stream_ids, _ = self.rrclient.find_objects(
salinity_doubler_output_dp_id, PRED.hasStream, None, True)
assertions(len(stream_ids) > 0)
sal_dbl_stream_id = stream_ids[0]
data_product_stream_ids.append(sal_dbl_stream_id)
#Start the input stream process
ctd_sim_pid = self._start_simple_input_stream_process(ctd_stream_id)
#Start te output stream listener to monitor and verify messages
results = self._start_output_stream_listener(data_product_stream_ids)
#Stop the transform processes
self.dataprocessclient.deactivate_data_process(
salinity_double_data_process_id)
self.dataprocessclient.deactivate_data_process(
l2_salinity_all_data_process_id)
# stop the flow parse the messages...
self.process_dispatcher.cancel_process(
ctd_sim_pid
) # kill the ctd simulator process - that is enough data
#Validate the data from each of the messages along the way
self._validate_messages(results)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Not integrated for CEI')
#@unittest.skip("Skipping for debugging ")
def test_transform_workflow(self):
assertions = self.assertTrue
# Build the workflow definition
workflow_def_obj = IonObject(
RT.WorkflowDefinition,
name='Salinity_Test_Workflow',
description='tests a workflow of multiple transform data processes'
)
workflow_data_product_name = 'TEST-Workflow_Output_Product' #Set a specific output product name
#Add a transformation process definition
ctd_L2_salinity_dprocdef_id = self._create_salinity_data_process_definition(
)
workflow_step_obj = IonObject(
'DataProcessWorkflowStep',
data_process_definition_id=ctd_L2_salinity_dprocdef_id,
persist_process_output_data=False
) #Don't persist the intermediate data product
workflow_def_obj.workflow_steps.append(workflow_step_obj)
#Add a transformation process definition
salinity_doubler_dprocdef_id = self._create_salinity_doubler_data_process_definition(
)
workflow_step_obj = IonObject(
'DataProcessWorkflowStep',
data_process_definition_id=salinity_doubler_dprocdef_id,
output_data_product_name=workflow_data_product_name)
workflow_def_obj.workflow_steps.append(workflow_step_obj)
#Create it in the resource registry
workflow_def_id = self.workflowclient.create_workflow_definition(
workflow_def_obj)
aids = self.rrclient.find_associations(workflow_def_id,
PRED.hasDataProcessDefinition)
assertions(len(aids) == 2)
#The list of data product streams to monitor
data_product_stream_ids = list()
#Create the input data product
ctd_stream_id, ctd_parsed_data_product_id = self._create_ctd_input_stream_and_data_product(
)
data_product_stream_ids.append(ctd_stream_id)
#Create and start the workflow
workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow(
workflow_def_id, ctd_parsed_data_product_id, timeout=30)
workflow_output_ids, _ = self.rrclient.find_subjects(
RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True)
assertions(len(workflow_output_ids) == 1)
#Verify the output data product name matches what was specified in the workflow definition
workflow_product = self.rrclient.read(workflow_product_id)
assertions(workflow_product.name == workflow_data_product_name)
#Walk the associations to find the appropriate output data streams to validate the messages
workflow_dp_ids, _ = self.rrclient.find_objects(
workflow_id, PRED.hasDataProduct, RT.DataProduct, True)
assertions(len(workflow_dp_ids) == 2)
for dp_id in workflow_dp_ids:
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream,
None, True)
assertions(len(stream_ids) == 1)
data_product_stream_ids.append(stream_ids[0])
#Start the input stream process
ctd_sim_pid = self._start_simple_input_stream_process(ctd_stream_id)
#Start the output stream listener to monitor and verify messages
results = self._start_output_stream_listener(data_product_stream_ids)
#Stop the workflow processes
self.workflowclient.terminate_data_process_workflow(
workflow_id, False, timeout=15) # Should test true at some point
#Make sure the Workflow object was removed
objs, _ = self.rrclient.find_resources(restype=RT.Workflow)
assertions(len(objs) == 0)
# stop the flow parse the messages...
self.process_dispatcher.cancel_process(
ctd_sim_pid
) # kill the ctd simulator process - that is enough data
#Validate the data from each of the messages along the way
self._validate_messages(results)
#Cleanup to make sure delete is correct.
self.workflowclient.delete_workflow_definition(workflow_def_id)
workflow_def_ids, _ = self.rrclient.find_resources(
restype=RT.WorkflowDefinition)
assertions(len(workflow_def_ids) == 0)
def _create_google_dt_data_process_definition(self):
#First look to see if it exists and if not, then create it
dpd, _ = self.rrclient.find_resources(restype=RT.DataProcessDefinition,
name='google_dt_transform')
if len(dpd) > 0:
return dpd[0]
# Data Process Definition
log.debug("Create data process definition GoogleDtTransform")
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='google_dt_transform',
description='Convert data streams to Google DataTables',
module='ion.processes.data.transforms.viz.google_dt',
class_name='VizTransformGoogleDT',
process_source='VizTransformGoogleDT source code here...')
try:
procdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except Exception as ex:
self.fail(
"failed to create new VizTransformGoogleDT data process definition: %s"
% ex)
# create a stream definition for the data from the
stream_def_id = self.pubsubclient.create_stream_definition(
container=VizTransformGoogleDT.outgoing_stream_def,
name='VizTransformGoogleDT')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
stream_def_id, procdef_id)
return procdef_id
def _validate_google_dt_results(self, results_stream_def, results):
cc = self.container
assertions = self.assertTrue
for g in results:
if isinstance(g, Granule):
tx = TaxyTool.load_from_granule(g)
rdt = RecordDictionaryTool.load_from_granule(g)
#log.warn(tx.pretty_print())
#log.warn(rdt.pretty_print())
gdt_component = rdt['google_dt_components'][0]
assertions(
gdt_component['viz_product_type'] == 'google_realtime_dt')
gdt_description = gdt_component['data_table_description']
gdt_content = gdt_component['data_table_content']
assertions(gdt_description[0][0] == 'time')
assertions(len(gdt_description) > 1)
assertions(len(gdt_content) >= 0)
return
def _create_mpl_graphs_data_process_definition(self):
#First look to see if it exists and if not, then create it
dpd, _ = self.rrclient.find_resources(restype=RT.DataProcessDefinition,
name='mpl_graphs_transform')
if len(dpd) > 0:
return dpd[0]
#Data Process Definition
log.debug("Create data process definition MatplotlibGraphsTransform")
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='mpl_graphs_transform',
description='Convert data streams to Matplotlib graphs',
module='ion.processes.data.transforms.viz.matplotlib_graphs',
class_name='VizTransformMatplotlibGraphs',
process_source='VizTransformMatplotlibGraphs source code here...')
try:
procdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
except Exception as ex:
self.fail(
"failed to create new VizTransformMatplotlibGraphs data process definition: %s"
% ex)
# create a stream definition for the data
stream_def_id = self.pubsubclient.create_stream_definition(
container=VizTransformMatplotlibGraphs.outgoing_stream_def,
name='VizTransformMatplotlibGraphs')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
stream_def_id, procdef_id)
return procdef_id
def _validate_mpl_graphs_results(self, results_stream_def, results):
cc = self.container
assertions = self.assertTrue
for g in results:
if isinstance(g, Granule):
tx = TaxyTool.load_from_granule(g)
rdt = RecordDictionaryTool.load_from_granule(g)
#log.warn(tx.pretty_print())
#log.warn(rdt.pretty_print())
graphs = rdt['matplotlib_graphs']
for graph in graphs:
assertions(
graph['viz_product_type'] == 'matplotlib_graphs')
# check to see if the list (numpy array) contians actual images
assertions(
imghdr.what(graph['image_name'], graph['image_obj']) ==
'png')
return
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Not integrated for CEI')
#unittest.skip("Skipping for debugging ")
def test_google_dt_transform_workflow(self):
assertions = self.assertTrue
# Build the workflow definition
workflow_def_obj = IonObject(
RT.WorkflowDefinition,
name='GoogleDT_Test_Workflow',
description=
'Tests the workflow of converting stream data to Google DT')
#Add a transformation process definition
google_dt_procdef_id = self._create_google_dt_data_process_definition()
workflow_step_obj = IonObject(
'DataProcessWorkflowStep',
data_process_definition_id=google_dt_procdef_id)
workflow_def_obj.workflow_steps.append(workflow_step_obj)
#Create it in the resource registry
workflow_def_id = self.workflowclient.create_workflow_definition(
workflow_def_obj)
#The list of data product streams to monitor
data_product_stream_ids = list()
#Create the input data product
ctd_stream_id, ctd_parsed_data_product_id = self._create_ctd_input_stream_and_data_product(
)
data_product_stream_ids.append(ctd_stream_id)
#Create and start the workflow
workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow(
workflow_def_id, ctd_parsed_data_product_id)
workflow_output_ids, _ = self.rrclient.find_subjects(
RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True)
assertions(len(workflow_output_ids) == 1)
#Walk the associations to find the appropriate output data streams to validate the messages
workflow_dp_ids, _ = self.rrclient.find_objects(
workflow_id, PRED.hasDataProduct, RT.DataProduct, True)
assertions(len(workflow_dp_ids) == 1)
for dp_id in workflow_dp_ids:
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream,
None, True)
assertions(len(stream_ids) == 1)
data_product_stream_ids.append(stream_ids[0])
#Start the input stream process
ctd_sim_pid = self._start_simple_input_stream_process(ctd_stream_id)
#Start the output stream listener to monitor and verify messages
results = self._start_output_stream_listener(data_product_stream_ids)
#Stop the workflow processes
self.workflowclient.terminate_data_process_workflow(
workflow_id, False) # Should test true at some point
# stop the flow parse the messages...
self.process_dispatcher.cancel_process(
ctd_sim_pid
) # kill the ctd simulator process - that is enough data
#Validate the data from each of the messages along the way
self._validate_google_dt_results(
VizTransformGoogleDT.outgoing_stream_def, results)
#Cleanup to make sure delete is correct.
self.workflowclient.delete_workflow_definition(workflow_def_id)
workflow_def_ids, _ = self.rrclient.find_resources(
restype=RT.WorkflowDefinition)
assertions(len(workflow_def_ids) == 0)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Not integrated for CEI')
#@unittest.skip("Skipping for debugging ")
def test_mpl_graphs_transform_workflow(self):
assertions = self.assertTrue
# Build the workflow definition
workflow_def_obj = IonObject(
RT.WorkflowDefinition,
name='Mpl_Graphs_Test_Workflow',
description=
'Tests the workflow of converting stream data to Matplotlib graphs'
)
#Add a transformation process definition
mpl_graphs_procdef_id = self._create_mpl_graphs_data_process_definition(
)
workflow_step_obj = IonObject(
'DataProcessWorkflowStep',
data_process_definition_id=mpl_graphs_procdef_id)
workflow_def_obj.workflow_steps.append(workflow_step_obj)
#Create it in the resource registry
workflow_def_id = self.workflowclient.create_workflow_definition(
workflow_def_obj)
#The list of data product streams to monitor
data_product_stream_ids = list()
#Create the input data product
ctd_stream_id, ctd_parsed_data_product_id = self._create_ctd_input_stream_and_data_product(
)
data_product_stream_ids.append(ctd_stream_id)
#Create and start the workflow
workflow_id, workflow_product_id = self.workflowclient.create_data_process_workflow(
workflow_def_id, ctd_parsed_data_product_id)
workflow_output_ids, _ = self.rrclient.find_subjects(
RT.Workflow, PRED.hasOutputProduct, workflow_product_id, True)
assertions(len(workflow_output_ids) == 1)
#Walk the associations to find the appropriate output data streams to validate the messages
workflow_dp_ids, _ = self.rrclient.find_objects(
workflow_id, PRED.hasDataProduct, RT.DataProduct, True)
assertions(len(workflow_dp_ids) == 1)
for dp_id in workflow_dp_ids:
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream,
None, True)
assertions(len(stream_ids) == 1)
data_product_stream_ids.append(stream_ids[0])
#Start the input stream process
ctd_sim_pid = self._start_sinusoidal_input_stream_process(
ctd_stream_id)
#Start the output stream listener to monitor and verify messages
results = self._start_output_stream_listener(data_product_stream_ids)
#Stop the workflow processes
self.workflowclient.terminate_data_process_workflow(
workflow_id, False) # Should test true at some point
# stop the flow parse the messages...
self.process_dispatcher.cancel_process(
ctd_sim_pid
) # kill the ctd simulator process - that is enough data
#Validate the data from each of the messages along the way
self._validate_mpl_graphs_results(
VizTransformGoogleDT.outgoing_stream_def, results)
#Cleanup to make sure delete is correct.
self.workflowclient.delete_workflow_definition(workflow_def_id)
workflow_def_ids, _ = self.rrclient.find_resources(
restype=RT.WorkflowDefinition)
assertions(len(workflow_def_ids) == 0)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Not integrated for CEI')
#@unittest.skip("Skipping for debugging ")
def test_multiple_workflow_instances(self):
assertions = self.assertTrue
# Build the workflow definition
workflow_def_obj = IonObject(
RT.WorkflowDefinition,
name='Multiple_Test_Workflow',
description='Tests the workflow of converting stream data')
#Add a transformation process definition
google_dt_procdef_id = self._create_google_dt_data_process_definition()
workflow_step_obj = IonObject(
'DataProcessWorkflowStep',
data_process_definition_id=google_dt_procdef_id)
workflow_def_obj.workflow_steps.append(workflow_step_obj)
#Create it in the resource registry
workflow_def_id = self.workflowclient.create_workflow_definition(
workflow_def_obj)
#The list of data product streams to monitor
data_product_stream_ids = list()
#Create the first input data product
ctd_stream_id1, ctd_parsed_data_product_id1 = self._create_ctd_input_stream_and_data_product(
'ctd_parsed1')
data_product_stream_ids.append(ctd_stream_id1)
#Create and start the first workflow
workflow_id1, workflow_product_id1 = self.workflowclient.create_data_process_workflow(
workflow_def_id, ctd_parsed_data_product_id1)
#Create the second input data product
ctd_stream_id2, ctd_parsed_data_product_id2 = self._create_ctd_input_stream_and_data_product(
'ctd_parsed2')
data_product_stream_ids.append(ctd_stream_id2)
#Create and start the first workflow
workflow_id2, workflow_product_id2 = self.workflowclient.create_data_process_workflow(
workflow_def_id, ctd_parsed_data_product_id2)
#Walk the associations to find the appropriate output data streams to validate the messages
workflow_ids, _ = self.rrclient.find_resources(restype=RT.Workflow)
assertions(len(workflow_ids) == 2)
#Start the first input stream process
ctd_sim_pid1 = self._start_sinusoidal_input_stream_process(
ctd_stream_id1)
#Start the second input stream process
ctd_sim_pid2 = self._start_simple_input_stream_process(ctd_stream_id2)
#Start the output stream listener to monitor a set number of messages being sent through the workflows
results = self._start_output_stream_listener(
data_product_stream_ids, message_count_per_stream=5)
# stop the flow of messages...
self.process_dispatcher.cancel_process(
ctd_sim_pid1
) # kill the ctd simulator process - that is enough data
self.process_dispatcher.cancel_process(ctd_sim_pid2)
#Stop the first workflow processes
self.workflowclient.terminate_data_process_workflow(
workflow_id1, False) # Should test true at some point
#Stop the second workflow processes
self.workflowclient.terminate_data_process_workflow(
workflow_id2, False) # Should test true at some point
workflow_ids, _ = self.rrclient.find_resources(restype=RT.Workflow)
assertions(len(workflow_ids) == 0)
#Cleanup to make sure delete is correct.
self.workflowclient.delete_workflow_definition(workflow_def_id)
workflow_def_ids, _ = self.rrclient.find_resources(
restype=RT.WorkflowDefinition)
assertions(len(workflow_def_ids) == 0)
aid_list = self.rrclient.find_associations(
workflow_def_id, PRED.hasDataProcessDefinition)
assertions(len(aid_list) == 0)
class ServiceTests(IonIntegrationTestCase):
"""
Tests the GeoServer and Foreign Data Wrapper (FDW) services.
"""
def setUp(self):
self.username = CFG.get_safe('eoi.geoserver.user_name', 'admin')
self.PASSWORD = CFG.get_safe('eoi.geoserver.password', 'geoserver')
self.gs_host = CFG.get_safe('eoi.geoserver.server',
'http://*****:*****@unittest.skipIf(not (CFG.get_safe('eoi.meta.use_eoi_services', False)),
'Skip test in TABLE LOADER as services are not loaded')
def test_reset_store(self):
# Makes sure store is empty
self.assertTrue(self.reset_store())
url = ''.join([self.gs_rest_url, '/layers.json'])
# Asserts layers were able to be retrieved
r = requests.get(url, auth=(self.username, self.PASSWORD))
self.assertTrue(r.status_code == 200)
# Asserts there are no layers in the ooi store
layers = json.loads(r.content)
self.assertTrue(len(layers['layers']) == 0)
@unittest.skipIf(not (CFG.get_safe('eoi.meta.use_eoi_services', False)),
'Skip test in TABLE LOADER as services are not loaded')
def test_create_dataset_verify_geoserver_layer(self):
#generate layer and check that the service created it in geoserver
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition(
'example', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition,
stream_def_id)
dp = DataProduct(name='example')
data_product_id = self.data_product_management.create_data_product(
dp, stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product,
data_product_id)
self.data_product_management.activate_data_product_persistence(
data_product_id)
self.addCleanup(
self.data_product_management.suspend_data_product_persistence,
data_product_id)
dataset_id = self.resource_registry.find_objects(data_product_id,
PRED.hasDataset,
id_only=True)[0][0]
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
rdt = ph.get_rdt(stream_def_id)
ph.fill_rdt(rdt, 100)
ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(monitor.event.wait(10))
gevent.sleep(
1) # Yield to other greenlets, had an issue with connectivity
log.debug("--------------------------------")
log.debug(dataset_id)
coverage_path = DatasetManagementService()._get_coverage_path(
dataset_id)
log.debug(coverage_path)
log.debug("--------------------------------")
# verify that the layer exists in geoserver
try:
r = requests.get(self.gs_rest_url + '/layers/ooi_' + dataset_id +
'_ooi.xml',
auth=(self.username, self.PASSWORD))
self.assertTrue(r.status_code == 200)
except Exception as e:
log.error("check service and layer exist...%s", e)
self.assertTrue(False)
@unittest.skipIf(not (CFG.get_safe('eoi.meta.use_eoi_services', False)),
'Skip test in TABLE LOADER as services are not loaded')
def test_verify_importer_service_online(self):
try:
r = requests.get(self.importer_service_url)
self.assertTrue(r.status_code == 200)
except Exception as e:
#make it fail
log.error("check service is started on port...%s", e)
self.assertTrue(False)
@unittest.skip('Not yet implemented')
def test_add_geoserver_layer(self):
# pass the create command to the service and check that the layer exists in geoserver similar to the one above
# send add layer directly to localhost 8844 with some params
# store gets reset every time container is started
# Makes sure store is empty
self.assertTrue(self.reset_store())
params = {
'temp_L1': 'real',
'conductivity_L1': 'real',
'temp': 'real',
'density': 'real',
'pressure_L1': 'real',
'lon': 'real',
'lat_lookup': 'real',
'density_lookup': 'real',
'pressure': 'real',
'lon_lookup': 'real',
'geom': 'geom',
'time': 'time',
'lat': 'real',
'salinity': 'real',
'conductivity': 'real'
}
r = requests.get(
self.importer_service_url +
'/service=addlayer&name=45a6a3cea12e470b90f3e5a769f22161&id=45a6a3cea12e470b90f3e5a769f22161¶ms='
+ str(params))
self.assertTrue(r.status_code == 200)
@unittest.skip('Not yet implemented')
def test_fdt_created_during(self):
# generate a data product and check that the FDT exists
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition(
'example', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition,
stream_def_id)
dp = DataProduct(name='example')
data_product_id = self.data_product_management.create_data_product(
dp, stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product,
data_product_id)
self.data_product_management.activate_data_product_persistence(
data_product_id)
self.addCleanup(
self.data_product_management.suspend_data_product_persistence,
data_product_id)
dataset_id = self.resource_registry.find_objects(data_product_id,
PRED.hasDataset,
id_only=True)[0][0]
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
rdt = ph.get_rdt(stream_def_id)
ph.fill_rdt(rdt, 100)
ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(monitor.event.wait(10))
gevent.sleep(
1) # Yield to other greenlets, had an issue with connectivity
print "--------------------------------"
print dataset_id
coverage_path = DatasetManagementService()._get_coverage_path(
dataset_id)
print coverage_path
print "--------------------------------"
#assert that the time exists in the DB
#compare the params in geoserver to those in the table
#Verify DB contains table
#eg sql request String sqlcmd = "select column_name from information_schema.columns where table_name = \'" + dataset_id + "\';";
#Verify Geoserver
#r = requests.get('http://*****:*****@unittest.skip('Not yet implemented')
def test_remove_geolayer_directory(self):
# pass the remove command to the service and check that the layer exists in geoserver similar to the one above
# send remove layer directly to localhost 8844 with some params
# check store
print ""
pass
@unittest.skip('Not yet implemented')
def test_update_geolayer_directory(self):
# pass the update command to the service and check that the layer exists in geoserver similar to the one above
# send update layer directly to localhost 8844 with some params
# check store
#does add then remove
print ""
pass
@unittest.skip('Not yet implemented')
def test_get_data_from_FDW(self):
# generate a data product and check that the FDW can get data
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition(
'example', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition,
stream_def_id)
dp = DataProduct(name='example')
data_product_id = self.data_product_management.create_data_product(
dp, stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product,
data_product_id)
self.data_product_management.activate_data_product_persistence(
data_product_id)
self.addCleanup(
self.data_product_management.suspend_data_product_persistence,
data_product_id)
dataset_id = self.resource_registry.find_objects(data_product_id,
PRED.hasDataset,
id_only=True)[0][0]
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
rdt = ph.get_rdt(stream_def_id)
ph.fill_rdt(rdt, 100)
ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(monitor.event.wait(10))
gevent.sleep(
1) # Yield to other greenlets, had an issue with connectivity
print "--------------------------------"
print dataset_id
coverage_path = DatasetManagementService()._get_coverage_path(
dataset_id)
print coverage_path
print "--------------------------------"
#verify table exists in the DB (similar to above)
# ....code...
# check that the geoserver layer exists as above
# ... code ....
# make a WMS/WFS request...somet like this (or both)
url = self.gs_host + '/geoserver/geonode/ows?service=WFS&version=1.0.0&request=GetFeature&typeName=geonode:ooi_' + dataset_id + '_ooi&maxFeatures=1&outputFormat=csv'
r = requests.get(url)
assertTrue(r.status_code == 200)
#check r.text does not contain <ServiceException code="InvalidParameterValue" locator="typeName">
@unittest.skipIf(not (CFG.get_safe('eoi.meta.use_eoi_services', False)),
'Skip test in TABLE LOADER as services are not loaded')
def test_sos_response(self):
expected_content = 'SOS SERVICE IS UP.....Hello World!'
url = self.gs_ows_url + '?request=echo&service=sos'
r = requests.get(url)
self.assertEqual(r.content, expected_content)
@unittest.skipIf(not (CFG.get_safe('eoi.meta.use_eoi_services', False)),
'Skip test in TABLE LOADER as services are not loaded')
def test_sos_get_capabilities(self):
# Validates reponse is not an exception, assues valid otherwise
self.setup_resource()
expected_content = ''
url = self.gs_ows_url + '?request=getCapabilities&service=sos&version=1.0.0&offering=_' + self.offering_id + '_view'
r = requests.get(url)
self.assertEquals(r.status_code, 200)
self.assertTrue(r.content.find('<sos:Capabilities') >= 0)
@unittest.skipIf(not (CFG.get_safe('eoi.meta.use_eoi_services', False)),
'Skip test in TABLE LOADER as services are not loaded')
def test_sos_get_offering(self):
# Validates reponse is not an exception, assues valid otherwise
# TODO: Use deterministic <swe:values> for comparison
self.setup_resource()
expected_content = ''
url = self.gs_ows_url + '?request=getObservation&service=sos&version=1.0.0&offering=_' + self.offering_id + '_view&observedproperty=time,temp,density&responseformat=text/xml'
r = requests.get(url)
self.assertEquals(r.status_code, 200)
self.assertTrue(r.content.find('<om:ObservationCollection') >= 0)
self.assertTrue(r.content.find('ExceptionReport') == -1)
def reset_store(self, store_name='ooi', store_id='ooi'):
"""
Posts a resetstore request to the ImporterService
"""
try:
url = ''.join([
self.importer_service_url, '/service=resetstore&name=',
store_name, '&id=', store_id
])
r = requests.post(url)
if r.status_code == 200:
return True
else:
return False
except Exception as e:
log.error('Unable to reset GeoServer Store. %s', e)
return False
class TestDataProductVersions(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
print 'started services'
# Now create client to DataProductManagementService
self.client = DataProductManagementServiceClient(
node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.rrclient)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(
node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
#@unittest.skip('not working')
@patch.dict(CFG, {'endpoint': {'receive': {'timeout': 60}}})
def test_createDataProductVersionSimple(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(
name='test', parameter_dictionary_id=pdict_id)
# test creating a new data product which will also create the initial/default version
log.debug('Creating new data product with a stream definition')
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='DP',
description='some new dp',
temporal_domain=tdom,
spatial_domain=sdom)
dp_id = self.client.create_data_product(dp_obj, ctd_stream_def_id)
log.debug('new dp_id = %s', str(dp_id))
dpc_id = self.client.create_data_product_collection(
data_product_id=dp_id,
collection_name='firstCollection',
collection_description='collection desc')
#test that the links exist
version_ids, _ = self.rrclient.find_objects(subject=dpc_id,
predicate=PRED.hasVersion,
id_only=True)
log.debug('version_ids = %s', str(version_ids))
self.assertTrue(
version_ids,
'Failed to connect the data product to the version collection.')
self.assertTrue(
version_ids[0] == dp_id,
'Failed to connect the data product to the version collection.')
# test creating a subsequent data product version which will update the data product pointers
dp2_obj = IonObject(RT.DataProduct,
name='DP2',
description='a second dp',
temporal_domain=tdom,
spatial_domain=sdom)
dp2_id = self.client.create_data_product(dp2_obj, ctd_stream_def_id)
log.debug('second dp_id = %s', dp2_id)
log.debug("adding product %s to collection %s", dp2_id, dpc_id)
self.client.add_data_product_version_to_collection(
data_product_id=dp2_id,
data_product_collection_id=dpc_id,
version_name="second version",
version_description="a second version created")
#test that the links exist
version_ids = self.RR2.find_data_product_ids_of_data_product_collection_using_has_version(
dpc_id)
self.assertEqual(2, len(version_ids))
recent_version_id = self.client.get_current_version(dpc_id)
self.assertEquals(recent_version_id, dp2_id)
base_version_id = self.client.get_base_version(dpc_id)
self.assertEquals(base_version_id, dp_id)
#---------------------------------------------------------------------------------------------
# Now check that we can subscribe to the stream for the data product version
#---------------------------------------------------------------------------------------------
# Activating the data products contained in the versions held by the data product collection
data_product_collection_obj = self.rrclient.read(dpc_id)
version_list = data_product_collection_obj.version_list
self.assertEquals(len(version_list), 2)
for version in version_list:
data_product_id = version.data_product_id
self.client.activate_data_product_persistence(data_product_id)
streams = self.RR2.find_streams_of_data_product_using_has_stream(
data_product_id)
self.assertNotEqual(0, len(streams))
for stream in streams:
self.assertTrue(stream.persisted)
log.debug(
"This satisfies L4-CI-DM-RQ-053: 'The dynamic data distribution services shall support multiple "
"versions of a given data topic.' This is true because we have shown above that we can persist the "
"streams associated to the data product versions, and therefore we can subscribe to them. In "
"test_oms_launch2.py, we have tested that activated data products like the ones we have here can be "
"used by data processes to gather streaming data and it all works together correctly. This therefore "
"completes the demonstration of req L4-CI-DM-RQ-053.")
#---------------------------------------------------------------------------------------------
# Now delete all the created stuff and look for possible problems in doing so
#---------------------------------------------------------------------------------------------
log.debug("deleting all data productgs")
self.client.delete_data_product(dp_id)
self.client.delete_data_product(dp2_id)
self.client.delete_data_product_collection(dpc_id)
self.client.force_delete_data_product_collection(dpc_id)
# now try to get the deleted dp object
self.assertRaises(NotFound, self.client.read_data_product_collection,
dpc_id)
class TestDataProcessWithLookupTable(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
#@unittest.skip('not working')
def test_lookupTableProcessing(self):
#-------------------------------
# Create InstrumentModel
#-------------------------------
instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", model_label="SBE37IMModel" )
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" %ex)
print 'test_createTransformsThenActivateInstrument: new InstrumentModel id = ', instModel_id
#-------------------------------
# Create InstrumentAgent
#-------------------------------
instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="ion.agents.instrument.instrument_agent", driver_class="InstrumentAgent" )
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" %ex)
print 'test_createTransformsThenActivateInstrument: new InstrumentAgent id = ', instAgent_id
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
#-------------------------------
# Create InstrumentDevice and attachment for lookup table
#-------------------------------
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
try:
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" %ex)
print 'test_createTransformsThenActivateInstrument: new InstrumentDevice id = ', instDevice_id
contents = "this is the lookup table contents, replace with a file..."
att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII)
deviceAttachment = self.rrclient.create_attachment(instDevice_id, att)
print 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = ', deviceAttachment
#-------------------------------
# Create InstrumentAgentInstance to hold configuration information
#-------------------------------
driver_config = {
'dvr_mod' : 'ion.agents.instrument.drivers.sbe37.sbe37_driver',
'dvr_cls' : 'SBE37Driver',
'workdir' : '/tmp/',
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance", driver_config = driver_config,
comms_device_address='sbe37-simulator.oceanobservatories.org', comms_device_port=4001, port_agent_work_dir='/tmp/', port_agent_delimeter=['<<','>>'] )
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id)
#-------------------------------
# Create CTD Parsed as the first data product
#-------------------------------
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def)
print 'TestDataProcessWithLookupTable: new Stream Definition id = ', instDevice_id
print 'Creating new CDM data product with a stream definition'
dp_obj = IonObject(RT.DataProduct,name='ctd_parsed',description='ctd stream test')
try:
ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" %ex)
print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product, persist_data=True, persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True)
print 'TestDataProcessWithLookupTable: Data product streams1 = ', stream_ids
#-------------------------------
# Create CTD Raw as the second data product
#-------------------------------
print 'TestDataProcessWithLookupTable: Creating new RAW data product with a stream definition'
raw_stream_def = SBE37_RAW_stream_definition()
raw_stream_def_id = self.pubsubclient.create_stream_definition(container=raw_stream_def)
dp_obj = IonObject(RT.DataProduct,name='ctd_raw',description='raw stream test')
try:
ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" %ex)
print 'new ctd_raw_data_product_id = ', ctd_raw_data_product
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_raw_data_product, persist_data=True, persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True)
print 'Data product streams2 = ', stream_ids
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug("TestDataProcessWithLookupTable: create data process definition ctd_L0_all")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all',
process_source='some_source_reference')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" %ex)
contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process Definition, replace with a file..."
att = IonObject(RT.Attachment, name='processDefinitionLookupTable',content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII)
processDefinitionAttachment = self.rrclient.create_attachment(ctd_L0_all_dprocdef_id, att)
log.debug("TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment id %s", str(processDefinitionAttachment) )
processDefinitionAttachment_obj = self.rrclient.read(processDefinitionAttachment)
log.debug("TestDataProcessWithLookupTable:test_createTransformsThenActivateInstrument: InstrumentDevice attachment obj %s", str(processDefinitionAttachment_obj) )
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity = L0_conductivity_stream_definition()
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_conductivity, name='L0_Conductivity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id )
outgoing_stream_l0_pressure = L0_pressure_stream_definition()
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_pressure, name='L0_Pressure')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id )
outgoing_stream_l0_temperature = L0_temperature_stream_definition()
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_temperature, name='L0_Temperature')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id )
self.output_products={}
log.debug("TestDataProcessWithLookupTable: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject(RT.DataProduct, name='L0_Conductivity',description='transform output conductivity')
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id)
self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id, persist_data=True, persist_metadata=True)
log.debug("TestDataProcessWithLookupTable: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject(RT.DataProduct, name='L0_Pressure',description='transform output pressure')
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id)
self.output_products['pressure'] = ctd_l0_pressure_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id, persist_data=True, persist_metadata=True)
log.debug("TestDataProcessWithLookupTable: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject(RT.DataProduct, name='L0_Temperature',description='transform output temperature')
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id)
self.output_products['temperature'] = ctd_l0_temperature_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id, persist_data=True, persist_metadata=True)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
log.debug("TestDataProcessWithLookupTable: create L0 all data_process start")
try:
in_prods = []
in_prods.append(ctd_parsed_data_product)
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, in_prods, self.output_products)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("TestDataProcessWithLookupTable: create L0 all data_process return")
contents = "this is the lookup table contents for L0 Conductivity - Temperature - Pressure: Data Process , replace with a file..."
att = IonObject(RT.Attachment, name='processLookupTable',content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII)
processAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att)
print 'TestDataProcessWithLookupTable: InstrumentDevice attachment id = ', processAttachment
class DatasetLoadTest(IonIntegrationTestCase):
"""
The following integration tests (INTMAN) are to ONLY be run manually
"""
def setUp(self):
self.username = CFG.get_safe('eoi.geoserver.user_name', 'admin')
self.PASSWORD = CFG.get_safe('eoi.geoserver.password', 'geoserver')
self.gs_host = CFG.get_safe('eoi.geoserver.server',
'http://*****:*****@unittest.skipIf(not (CFG.get_safe('eoi.meta.use_eoi_services', False)),
'Skip test in TABLE LOADER as services are not loaded')
def test_create_dataset(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition(
'example', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition,
stream_def_id)
dp = DataProduct(name='example')
data_product_id = self.data_product_management.create_data_product(
dp, stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product,
data_product_id)
self.data_product_management.activate_data_product_persistence(
data_product_id)
self.addCleanup(
self.data_product_management.suspend_data_product_persistence,
data_product_id)
dataset_id = self.resource_registry.find_objects(data_product_id,
PRED.hasDataset,
id_only=True)[0][0]
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
rdt = ph.get_rdt(stream_def_id)
ph.fill_rdt(rdt, 100)
ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(monitor.event.wait(10))
# Yield to other greenlets, had an issue with connectivity
gevent.sleep(1)
log.debug("--------------------------------")
log.debug(dataset_id)
coverage_path = DatasetManagementService()._get_coverage_path(
dataset_id)
log.debug(coverage_path)
log.debug("--------------------------------")
breakpoint(locals(), globals())
class TestDataProductManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dpsc_cli = DataProductManagementServiceClient(node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.unsc = UserNotificationServiceClient()
#------------------------------------------
# Create the environment
#------------------------------------------
datastore_name = CACHE_DATASTORE_NAME
self.db = self.container.datastore_manager.get_datastore(datastore_name)
self.stream_def_id = self.pubsubcli.create_stream_definition(name='SBE37_CDM')
self.process_definitions = {}
ingestion_worker_definition = ProcessDefinition(name='ingestion worker')
ingestion_worker_definition.executable = {
'module':'ion.processes.data.ingestion.science_granule_ingestion_worker',
'class' :'ScienceGranuleIngestionWorker'
}
process_definition_id = self.process_dispatcher.create_process_definition(process_definition=ingestion_worker_definition)
self.process_definitions['ingestion_worker'] = process_definition_id
#------------------------------------------------------------------------------------------------
# First launch the ingestors
#------------------------------------------------------------------------------------------------
self.exchange_space = 'science_granule_ingestion'
self.exchange_point = 'science_data'
config = DotDict()
config.process.datastore_name = 'datasets'
config.process.queue_name = self.exchange_space
self.process_dispatcher.schedule_process(self.process_definitions['ingestion_worker'],configuration=config)
def get_datastore(self, dataset_id):
dataset = self.dataset_management.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
@unittest.skip('OBE')
def test_get_last_update(self):
# Construct temporal and spatial Coordinate Reference System objects
tcrs = CRS([AxisTypeEnum.TIME])
scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT])
# Construct temporal and spatial Domain objects
tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline)
sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE) # 1d spatial topology (station/trajectory)
sdom = sdom.dump()
tdom = tdom.dump()
#@TODO: DO NOT DO THIS, WHEN THIS TEST IS REWRITTEN GET RID OF THIS, IT WILL FAIL, thanks -Luke
parameter_dictionary = get_param_dict('ctd_parsed_param_dict')
parameter_dictionary = parameter_dictionary.dump()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id = self.dpsc_cli.create_data_product(data_product=dp_obj, stream_definition_id=self.stream_def_id, parameter_dictionary=parameter_dictionary)
stream_ids, garbage = self.rrclient.find_objects(data_product_id, PRED.hasStream, id_only=True)
stream_id = stream_ids[0]
fake_lu = LastUpdate()
fake_lu_doc = self.db._ion_object_to_persistence_dict(fake_lu)
self.db.create_doc(fake_lu_doc, object_id=stream_id)
#------------------------------------------
# Now execute
#------------------------------------------
res = self.dpsc_cli.get_last_update(data_product_id=data_product_id)
self.assertTrue(isinstance(res[stream_id], LastUpdate), 'retrieving documents failed')
def test_create_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
parameter_dictionary_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict')
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=parameter_dictionary_id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
log.debug('test_createDataProduct: Creating new data product w/o a stream definition (L4-CI-SA-RQ-308)')
# Generic time-series data domain creation
tdom, sdom = time_series_domain()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom.dump(),
spatial_domain = sdom.dump())
log.debug("Created an IonObject for a data product: %s" % dp_obj)
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product( data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
log.debug('new dp_id = %s' % dp_id)
log.debug("test_createDataProduct: Data product info from registry %s (L4-CI-SA-RQ-308)", str(dp_obj))
#------------------------------------------------------------------------------------------------
# test creating a new data product with a stream definition
#------------------------------------------------------------------------------------------------
log.debug('Creating new data product with a stream definition')
dp_obj = IonObject(RT.DataProduct,
name='DP2',
description='some new dp',
temporal_domain = tdom.dump(),
spatial_domain = sdom.dump())
dp_id2 = self.dpsc_cli.create_data_product(dp_obj, ctd_stream_def_id)
log.debug('new dp_id = %s' % dp_id2)
#------------------------------------------------------------------------------------------------
#make sure data product is associated with stream def
#------------------------------------------------------------------------------------------------
streamdefs = []
streams, _ = self.rrclient.find_objects(dp_id2, PRED.hasStream, RT.Stream, True)
for s in streams:
log.debug("Checking stream %s" % s)
sdefs, _ = self.rrclient.find_objects(s, PRED.hasStreamDefinition, RT.StreamDefinition, True)
for sd in sdefs:
log.debug("Checking streamdef %s" % sd)
streamdefs.append(sd)
self.assertIn(ctd_stream_def_id, streamdefs)
# test reading a non-existent data product
log.debug('reading non-existent data product')
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product('some_fake_id')
# update a data product (tests read also)
log.debug('Updating data product')
# first get the existing dp object
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# now tweak the object
dp_obj.description = 'the very first dp'
# now write the dp back to the registry
update_result = self.dpsc_cli.update_data_product(dp_obj)
# now get the dp back to see if it was updated
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertEquals(dp_obj.description,'the very first dp')
#test extension
extended_product = self.dpsc_cli.get_data_product_extension(dp_id)
self.assertEqual(dp_id, extended_product._id)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_product.computed.product_download_size_estimated.status)
self.assertEqual(0, extended_product.computed.product_download_size_estimated.value)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_product.computed.parameters.status)
#log.debug("test_create_data_product: parameters %s" % extended_product.computed.parameters.value)
# now 'delete' the data product
log.debug("deleting data product: %s" % dp_id)
self.dpsc_cli.delete_data_product(dp_id)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
try:
dp_obj = self.dpsc_cli.read_data_product(dp_id)
except NotFound as ex:
pass
else:
self.fail("force deleted data product was found during read")
# Get the events corresponding to the data product
ret = self.unsc.get_recent_events(resource_id=dp_id)
events = ret.value
for event in events:
log.debug("event time: %s" % event.ts_created)
# now try to get the deleted dp object
#todo: the RR should perhaps not return retired data products
# dp_obj = self.dpsc_cli.read_data_product(dp_id)
# now try to delete the already deleted dp object
# log.debug( "deleting non-existing data product")
# self.dpsc_cli.delete_data_product(dp_id)
# Shut down container
#container.stop()
def test_activate_suspend_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
log.debug('test_createDataProduct: Creating new data product w/o a stream definition (L4-CI-SA-RQ-308)')
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
log.debug("Created an IonObject for a data product: %s" % dp_obj)
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
log.debug('new dp_id = %s' % dp_id)
log.debug("test_createDataProduct: Data product info from registry %s (L4-CI-SA-RQ-308)", str(dp_obj))
dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True)
if not dataset_ids:
raise NotFound("Data Product %s dataset does not exist" % str(dp_id))
self.get_datastore(dataset_ids[0])
#------------------------------------------------------------------------------------------------
# test activate and suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.activate_data_product_persistence(dp_id)
#------------------------------------------------------------------------------------------------
# test suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.suspend_data_product_persistence(dp_id)
# pid = self.container.spawn_process(name='dummy_process_for_test',
# module='pyon.ion.process',
# cls='SimpleProcess',
# config={})
# dummy_process = self.container.proc_manager.procs[pid]
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
try:
dp_obj = self.rrclient.read(dp_id)
except NotFound as ex:
pass
else:
self.fail("force_deleted data product was found during read")
class VisStreamLauncher(ImmediateProcess):
"""
Class emulates a stream source from a NetCDF file. It emits a record of data every few seconds on a
stream identified by a routing key.
"""
def on_init(self):
log.debug("VizStreamProducer init. Self.id=%s" % self.id)
def on_start(self):
log.debug("VizStreamProducer start")
self.data_source_name = self.CFG.get_safe('name',
'sine_wave_generator')
self.dataset = self.CFG.get_safe('dataset', 'sinusoidal')
# create a pubsub client and a resource registry client
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(
node=self.container.node)
# Dummy instrument related clients
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.dpclient = DataProductManagementServiceClient(
node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
# create the pubsub client
self.pubsubclient = PubsubManagementServiceClient(
node=self.container.node)
# Additional code for creating a dummy instrument
"""
# Set up the preconditions. Look for an existing ingestion config
while True:
log.info("VisStreamLauncher:on_start: Waiting for an ingestion configuration to be available.")
ingestion_cfgs, _ = self.rrclient.find_resources(RT.IngestionConfiguration, None, None, True)
if len(ingestion_cfgs) > 0:
break
else:
gevent.sleep(1)
"""
# Check to see if the data_product already exists in the system (for e.g re launching the code after a crash)
dp_ids, _ = self.rrclient.find_resources(RT.DataProduct, None,
self.data_source_name, True)
if len(dp_ids) > 0:
data_product_id = dp_ids[0]
else:
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name=self.data_source_name,
description=self.data_source_name)
instModel_id = self.imsclient.create_instrument_model(
instModel_obj)
# Create InstrumentDevice
instDevice_obj = IonObject(RT.InstrumentDevice,
name=self.data_source_name,
description=self.data_source_name,
serial_number="12345")
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, instDevice_id)
# create a stream definition for the data from the ctd simulator
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubclient.create_stream_definition(
name="SBE37_CDM",
description="SBE37_CDM",
parameter_dictionary_id=pdict_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name=self.data_source_name,
description='Example ctd stream',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id = self.dpclient.create_data_product(
dp_obj, stream_definition_id=ctd_stream_def_id)
self.damsclient.assign_data_product(
input_resource_id=instDevice_id,
data_product_id=data_product_id)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id)
print ">>>>>>>>>>>>> Dataproduct for sine wave generator : ", data_product_id
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id,
PRED.hasStream, None, True)
if self.dataset == 'sinusoidal':
self.container.spawn_process(
name='ctd_test.' + self.data_source_name,
module='ion.processes.data.sinusoidal_stream_publisher',
cls='SinusoidalCtdPublisher',
config={'process': {
'stream_id': stream_ids[0]
}})
else:
self.container.spawn_process(
name='ctd_test.' + self.data_source_name,
module='ion.processes.data.ctd_stream_publisher',
cls='SimpleCtdPublisher',
config={'process': {
'stream_id': stream_ids[0]
}})
"""
workflow_def_ids,_ = self.rrclient.find_resources(restype=RT.WorkflowDefinition, name='Realtime_HighCharts', id_only=True)
if not len(workflow_def_ids):
helper_create_highcharts_workflow_def(self.container)
"""
def on_quit(self):
super(VisStreamLauncher, self).on_quit()
log.debug("VizStreamProducer quit")
class TestActivateInstrumentIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
super(TestActivateInstrumentIntegration, self).setUp()
config = DotDict()
config.bootstrap.use_es = True
self._start_container()
self.addCleanup(TestActivateInstrumentIntegration.es_cleanup)
self.container.start_rel_from_url('res/deploy/r2deploy.yml', config)
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dpclient = DataProductManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataretrieverclient = DataRetrieverServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.usernotificationclient = UserNotificationServiceClient()
#setup listerner vars
self._data_greenlets = []
self._no_samples = None
self._samples_received = []
self.event_publisher = EventPublisher()
@staticmethod
def es_cleanup():
es_host = CFG.get_safe('server.elasticsearch.host', 'localhost')
es_port = CFG.get_safe('server.elasticsearch.port', '9200')
es = ep.ElasticSearch(host=es_host, port=es_port, timeout=10)
indexes = STD_INDEXES.keys()
indexes.append('%s_resources_index' % get_sys_name().lower())
indexes.append('%s_events_index' % get_sys_name().lower())
for index in indexes:
IndexManagementService._es_call(es.river_couchdb_delete, index)
IndexManagementService._es_call(es.index_delete, index)
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name + '_logger')
producer_definition.executable = {
'module': 'ion.processes.data.stream_granule_logger',
'class': 'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(
process_definition=producer_definition)
configuration = {
'process': {
'stream_id': stream_id,
}
}
pid = self.processdispatchclient.schedule_process(
process_definition_id=logger_procdef_id,
configuration=configuration)
return pid
def _create_notification(self,
user_name='',
instrument_id='',
product_id=''):
#--------------------------------------------------------------------------------------
# Make notification request objects
#--------------------------------------------------------------------------------------
notification_request_1 = NotificationRequest(
name='notification_1',
origin=instrument_id,
origin_type="instrument",
event_type='ResourceLifecycleEvent')
notification_request_2 = NotificationRequest(
name='notification_2',
origin=product_id,
origin_type="data product",
event_type='DetectionEvent')
#--------------------------------------------------------------------------------------
# Create a user and get the user_id
#--------------------------------------------------------------------------------------
user = UserInfo()
user.name = user_name
user.contact.email = '*****@*****.**' % user_name
user_id, _ = self.rrclient.create(user)
#--------------------------------------------------------------------------------------
# Create notification
#--------------------------------------------------------------------------------------
self.usernotificationclient.create_notification(
notification=notification_request_1, user_id=user_id)
self.usernotificationclient.create_notification(
notification=notification_request_2, user_id=user_id)
log.debug(
"test_activateInstrumentSample: create_user_notifications user_id %s",
str(user_id))
return user_id
def get_datastore(self, dataset_id):
dataset = self.datasetclient.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(
datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def _check_computed_attributes_of_extended_instrument(
self, expected_instrument_device_id='', extended_instrument=None):
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(extended_instrument.computed.firmware_version,
ComputedFloatValue)
self.assertIsInstance(
extended_instrument.computed.last_data_received_datetime,
ComputedFloatValue)
self.assertIsInstance(
extended_instrument.computed.last_calibration_datetime,
ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.uptime,
ComputedStringValue)
self.assertIsInstance(
extended_instrument.computed.power_status_roll_up,
ComputedIntValue)
self.assertIsInstance(
extended_instrument.computed.communications_status_roll_up,
ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.data_status_roll_up,
ComputedIntValue)
self.assertIsInstance(
extended_instrument.computed.location_status_roll_up,
ComputedIntValue)
# the following assert will not work without elasticsearch.
#self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) )
self.assertEqual(
extended_instrument.computed.communications_status_roll_up.value,
StatusType.STATUS_WARNING)
self.assertEqual(
extended_instrument.computed.data_status_roll_up.value,
StatusType.STATUS_OK)
self.assertEqual(
extended_instrument.computed.power_status_roll_up.value,
StatusType.STATUS_WARNING)
# Verify the computed attribute for user notification requests
self.assertEqual(
1,
len(extended_instrument.computed.user_notification_requests.value))
notifications = extended_instrument.computed.user_notification_requests.value
notification = notifications[0]
self.assertEqual(notification.origin, expected_instrument_device_id)
self.assertEqual(notification.origin_type, "instrument")
self.assertEqual(notification.event_type, 'ResourceLifecycleEvent')
def _check_computed_attributes_of_extended_product(
self, expected_data_product_id='', extended_data_product=None):
self.assertEqual(expected_data_product_id, extended_data_product._id)
log.debug("extended_data_product.computed: %s",
extended_data_product.computed)
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(
extended_data_product.computed.product_download_size_estimated,
ComputedIntValue)
self.assertIsInstance(
extended_data_product.computed.number_active_subscriptions,
ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.data_url,
ComputedStringValue)
self.assertIsInstance(extended_data_product.computed.stored_data_size,
ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.recent_granules,
ComputedDictValue)
self.assertIsInstance(extended_data_product.computed.parameters,
ComputedListValue)
self.assertIsInstance(extended_data_product.computed.recent_events,
ComputedEventListValue)
self.assertIsInstance(extended_data_product.computed.provenance,
ComputedDictValue)
self.assertIsInstance(
extended_data_product.computed.user_notification_requests,
ComputedListValue)
self.assertIsInstance(
extended_data_product.computed.active_user_subscriptions,
ComputedListValue)
self.assertIsInstance(
extended_data_product.computed.past_user_subscriptions,
ComputedListValue)
self.assertIsInstance(extended_data_product.computed.last_granule,
ComputedDictValue)
self.assertIsInstance(extended_data_product.computed.is_persisted,
ComputedIntValue)
self.assertIsInstance(
extended_data_product.computed.data_contents_updated,
ComputedStringValue)
self.assertIsInstance(extended_data_product.computed.data_datetime,
ComputedListValue)
# exact text here keeps changing to fit UI capabilities. keep assertion general...
self.assertTrue('ok' in extended_data_product.computed.last_granule.
value['quality_flag'])
self.assertEqual(
2, len(extended_data_product.computed.data_datetime.value))
notifications = extended_data_product.computed.user_notification_requests.value
notification = notifications[0]
self.assertEqual(notification.origin, expected_data_product_id)
self.assertEqual(notification.origin_type, "data product")
self.assertEqual(notification.event_type, 'DetectionEvent')
@attr('LOCOINT')
@unittest.skipIf(not use_es, 'No ElasticSearch')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
@patch.dict(CFG, {'endpoint': {'receive': {'timeout': 60}}})
def test_activateInstrumentSample(self):
self.loggerpids = []
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
log.debug('new InstrumentModel id = %s ', instModel_id)
#Create stream alarms
"""
test_two_sided_interval
Test interval alarm and alarm event publishing for a closed
inteval.
"""
# kwargs = {
# 'name' : 'test_sim_warning',
# 'stream_name' : 'parsed',
# 'value_id' : 'temp',
# 'message' : 'Temperature is above test range of 5.0.',
# 'type' : StreamAlarmType.WARNING,
# 'upper_bound' : 5.0,
# 'upper_rel_op' : '<'
# }
kwargs = {
'name': 'temperature_warning_interval',
'stream_name': 'parsed',
'value_id': 'temp',
'message':
'Temperature is below the normal range of 50.0 and above.',
'type': StreamAlarmType.WARNING,
'lower_bound': 50.0,
'lower_rel_op': '<'
}
# Create alarm object.
alarm = {}
alarm['type'] = 'IntervalAlarmDef'
alarm['kwargs'] = kwargs
raw_config = StreamConfiguration(
stream_name='raw',
parameter_dictionary_name='ctd_raw_param_dict',
records_per_granule=2,
granule_publish_rate=5)
parsed_config = StreamConfiguration(
stream_name='parsed',
parameter_dictionary_name='ctd_parsed_param_dict',
records_per_granule=2,
granule_publish_rate=5,
alarms=[alarm])
# Create InstrumentAgent
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri=
"http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1a-py2.7.egg",
stream_configurations=[raw_config, parsed_config])
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
log.debug('new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(
instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug(
'test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) '
)
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345")
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, instDevice_id)
log.debug(
"test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ",
instDevice_id)
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance,
name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config=port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubcli.create_stream_definition(
name='parsed', parameter_dictionary_id=parsed_pdict_id)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.pubsubcli.create_stream_definition(
name='raw', parameter_dictionary_id=raw_pdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id1 = self.dpclient.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug('new dp_id = %s', data_product_id1)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id1)
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1,
PRED.hasStream, None, True)
log.debug('Data product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id1,
PRED.hasDataset,
RT.Dataset, True)
log.debug('Data set for data_product_id1 = %s', dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
pid = self.create_logger('ctd_parsed', stream_ids[0])
self.loggerpids.append(pid)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id2 = self.dpclient.create_data_product(
data_product=dp_obj, stream_definition_id=raw_stream_def_id)
log.debug('new dp_id = %s', data_product_id2)
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id2)
# setup notifications for the device and parsed data product
user_id_1 = self._create_notification(user_name='user_1',
instrument_id=instDevice_id,
product_id=data_product_id1)
#---------- Create notifications for another user and verify that we see different computed subscriptions for the two users ---------
user_id_2 = self._create_notification(user_name='user_2',
instrument_id=instDevice_id,
product_id=data_product_id2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2,
PRED.hasStream, None, True)
log.debug('Data product streams2 = %s', str(stream_ids))
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id2,
PRED.hasDataset,
RT.Dataset, True)
log.debug('Data set for data_product_id2 = %s', dataset_ids[0])
self.raw_dataset = dataset_ids[0]
#elastic search debug
es_indexes, _ = self.container.resource_registry.find_resources(
restype='ElasticSearchIndex')
log.debug('ElasticSearch indexes: %s', [i.name for i in es_indexes])
log.debug('Bootstrap %s', CFG.bootstrap.use_es)
def start_instrument_agent():
self.imsclient.start_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id)
gevent.joinall([gevent.spawn(start_instrument_agent)])
#setup a subscriber to alarm events from the device
self._events_received = []
self._event_count = 0
self._samples_out_of_range = 0
self._samples_complete = False
self._async_sample_result = AsyncResult()
def consume_event(*args, **kwargs):
log.debug(
'TestActivateInstrument recieved ION event: args=%s, kwargs=%s, event=%s.',
str(args), str(kwargs), str(args[0]))
self._events_received.append(args[0])
self._event_count = len(self._events_received)
self._async_sample_result.set()
self._event_subscriber = EventSubscriber(
event_type=
'StreamWarningAlarmEvent', #'StreamWarningAlarmEvent', # StreamAlarmEvent
callback=consume_event,
origin=instDevice_id)
self._event_subscriber.start()
#cleanup
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
def stop_subscriber():
self._event_subscriber.stop()
self._event_subscriber = None
self.addCleanup(stop_subscriber)
#wait for start
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(
instAgentInstance_id)
gate = ProcessStateGate(self.processdispatchclient.read_process,
inst_agent_instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(
gate. await (30),
"The instrument agent instance (%s) did not spawn in 30 seconds" %
inst_agent_instance_obj.agent_process_id)
log.debug('Instrument agent instance obj: = %s',
str(inst_agent_instance_obj))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(
instDevice_id,
to_name=inst_agent_instance_obj.agent_process_id,
process=FakeProcess())
log.debug("test_activateInstrumentSample: got ia client %s",
str(self._ia_client))
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: initialize %s", str(retval))
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
log.debug("(L4-CI-SA-RQ-334): Sending go_active command ")
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrument: return value from go_active %s",
str(reply))
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug(
"(L4-CI-SA-RQ-334): current state after sending go_active command %s",
str(state))
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: run %s", str(reply))
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.PAUSE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.STOPPED)
cmd = AgentCommand(command=ResourceAgentEvent.RESUME)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.CLEAR)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
for i in xrange(10):
retval = self._ia_client.execute_resource(cmd)
log.debug("test_activateInstrumentSample: return from sample %s",
str(retval))
log.debug("test_activateInstrumentSample: calling reset ")
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: return from reset %s",
str(reply))
self._samples_complete = True
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data = self.dataretrieverclient.retrieve(self.parsed_dataset)
self.assertIsInstance(replay_data, Granule)
rdt = RecordDictionaryTool.load_from_granule(replay_data)
log.debug("test_activateInstrumentSample: RDT parsed: %s",
str(rdt.pretty_print()))
temp_vals = rdt['temp']
self.assertEquals(len(temp_vals), 10)
log.debug("test_activateInstrumentSample: all temp_vals: %s",
temp_vals)
#out_of_range_temp_vals = [i for i in temp_vals if i > 5]
out_of_range_temp_vals = [i for i in temp_vals if i < 50.0]
log.debug("test_activateInstrumentSample: Out_of_range_temp_vals: %s",
out_of_range_temp_vals)
self._samples_out_of_range = len(out_of_range_temp_vals)
# if no bad values were produced, then do not wait for an event
if self._samples_out_of_range == 0:
self._async_sample_result.set()
log.debug("test_activateInstrumentSample: _events_received: %s",
self._events_received)
log.debug("test_activateInstrumentSample: _event_count: %s",
self._event_count)
self._async_sample_result.get(timeout=CFG.endpoint.receive.timeout)
replay_data = self.dataretrieverclient.retrieve(self.raw_dataset)
self.assertIsInstance(replay_data, Granule)
rdt = RecordDictionaryTool.load_from_granule(replay_data)
log.debug("RDT raw: %s", str(rdt.pretty_print()))
raw_vals = rdt['raw']
self.assertEquals(len(raw_vals), 10)
log.debug("l4-ci-sa-rq-138")
"""
Physical resource control shall be subject to policy
Instrument management control capabilities shall be subject to policy
The actor accessing the control capabilities must be authorized to send commands.
note from maurice 2012-05-18: Talk to tim M to verify that this is policy. If it is then talk with Stephen to
get an example of a policy test and use that to create a test stub that will be
completed when we have instrument policies.
Tim M: The "actor", aka observatory operator, will access the instrument through ION.
"""
#--------------------------------------------------------------------------------
# Get the extended data product to see if it contains the granules
#--------------------------------------------------------------------------------
extended_product = self.dpclient.get_data_product_extension(
data_product_id=data_product_id1, user_id=user_id_1)
def poller(extended_product):
return len(extended_product.computed.user_notification_requests.
value) == 1
poll(poller, extended_product, timeout=30)
self._check_computed_attributes_of_extended_product(
expected_data_product_id=data_product_id1,
extended_data_product=extended_product)
#--------------------------------------------------------------------------------
#put some events into the eventsdb to test - this should set the comms and data status to WARNING
#--------------------------------------------------------------------------------
t = get_ion_ts()
self.event_publisher.publish_event(ts_created=t,
event_type='DeviceStatusEvent',
origin=instDevice_id,
state=DeviceStatusType.OUT_OF_RANGE,
values=[200])
self.event_publisher.publish_event(
ts_created=t,
event_type='DeviceCommsEvent',
origin=instDevice_id,
state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION,
lapse_interval_seconds=20)
#--------------------------------------------------------------------------------
# Get the extended instrument
#--------------------------------------------------------------------------------
extended_instrument = self.imsclient.get_instrument_device_extension(
instrument_device_id=instDevice_id, user_id=user_id_1)
self._check_computed_attributes_of_extended_instrument(
expected_instrument_device_id=instDevice_id,
extended_instrument=extended_instrument)
#--------------------------------------------------------------------------------
# For the second user, check the extended data product and the extended intrument
#--------------------------------------------------------------------------------
extended_product = self.dpclient.get_data_product_extension(
data_product_id=data_product_id2, user_id=user_id_2)
self._check_computed_attributes_of_extended_product(
expected_data_product_id=data_product_id2,
extended_data_product=extended_product)
#---------- Put some events into the eventsdb to test - this should set the comms and data status to WARNING ---------
t = get_ion_ts()
self.event_publisher.publish_event(ts_created=t,
event_type='DeviceStatusEvent',
origin=instDevice_id,
state=DeviceStatusType.OUT_OF_RANGE,
values=[200])
self.event_publisher.publish_event(
ts_created=t,
event_type='DeviceCommsEvent',
origin=instDevice_id,
state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION,
lapse_interval_seconds=20)
#--------------------------------------------------------------------------------
# Get the extended instrument
#--------------------------------------------------------------------------------
extended_instrument = self.imsclient.get_instrument_device_extension(
instrument_device_id=instDevice_id, user_id=user_id_2)
self._check_computed_attributes_of_extended_instrument(
expected_instrument_device_id=instDevice_id,
extended_instrument=extended_instrument)
#--------------------------------------------------------------------------------
# Deactivate loggers
#--------------------------------------------------------------------------------
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
self.dpclient.delete_data_product(data_product_id1)
self.dpclient.delete_data_product(data_product_id2)
class ServiceTests(IonIntegrationTestCase):
"""
Tests the GeoServer and Foreign Data Wrapper (FDW) services.
"""
def setUp(self):
self.username = CFG.get_safe('eoi.geoserver.user_name', 'admin')
self.PASSWORD = CFG.get_safe('eoi.geoserver.password', 'geoserver')
self.gs_host = CFG.get_safe('eoi.geoserver.server', 'http://*****:*****@unittest.skipIf(not (CFG.get_safe('eoi.meta.use_eoi_services', False)), 'Skip test in TABLE LOADER as services are not loaded')
def test_reset_store(self):
# Makes sure store is empty
self.assertTrue(self.reset_store())
url = ''.join([self.gs_rest_url, '/layers.json'])
# Asserts layers were able to be retrieved
r = requests.get(url, auth=(self.username, self.PASSWORD))
self.assertTrue(r.status_code == 200)
# Asserts there are no layers in the ooi store
layers = json.loads(r.content)
self.assertTrue(len(layers['layers']) == 0)
@unittest.skipIf( not (CFG.get_safe('eoi.meta.use_eoi_services', False)), 'Skip test in TABLE LOADER as services are not loaded')
def test_create_dataset_verify_geoserver_layer(self):
#generate layer and check that the service created it in geoserver
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition('example', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
tdom, sdom = time_series_domain()
dp = DataProduct(name='example')
dp.spatial_domain = sdom.dump()
dp.temporal_domain = tdom.dump()
data_product_id = self.data_product_management.create_data_product(dp, stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product, data_product_id)
self.data_product_management.activate_data_product_persistence(data_product_id)
self.addCleanup(self.data_product_management.suspend_data_product_persistence, data_product_id)
dataset_id = self.resource_registry.find_objects(data_product_id, PRED.hasDataset, id_only=True)[0][0]
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
rdt = ph.get_rdt(stream_def_id)
ph.fill_rdt(rdt, 100)
ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(monitor.event.wait(10))
gevent.sleep(1) # Yield to other greenlets, had an issue with connectivity
log.debug("--------------------------------")
log.debug(dataset_id)
coverage_path = DatasetManagementService()._get_coverage_path(dataset_id)
log.debug(coverage_path)
log.debug("--------------------------------")
# verify that the layer exists in geoserver
try:
r = requests.get(self.gs_rest_url + '/layers/ooi_' + dataset_id + '_ooi.xml', auth=(self.username, self.PASSWORD))
self.assertTrue(r.status_code == 200)
except Exception as e:
log.error("check service and layer exist...%s", e)
self.assertTrue(False)
@unittest.skipIf( not (CFG.get_safe('eoi.meta.use_eoi_services', False)), 'Skip test in TABLE LOADER as services are not loaded')
def test_verify_importer_service_online(self):
try:
r = requests.get(self.importer_service_url)
self.assertTrue(r.status_code == 200)
except Exception as e:
#make it fail
log.error("check service is started on port...%s", e)
self.assertTrue(False)
@unittest.skip('Not yet implemented')
def test_add_geoserver_layer(self):
# pass the create command to the service and check that the layer exists in geoserver similar to the one above
# send add layer directly to localhost 8844 with some params
# store gets reset every time container is started
# Makes sure store is empty
self.assertTrue(self.reset_store())
params = {'temp_L1': 'real', 'conductivity_L1': 'real', 'temp': 'real', 'density': 'real',
'pressure_L1': 'real', 'lon': 'real', 'lat_lookup': 'real', 'density_lookup': 'real',
'pressure': 'real', 'lon_lookup': 'real', 'geom': 'geom', 'time': 'time', 'lat': 'real',
'salinity': 'real', 'conductivity': 'real'}
r = requests.get(self.importer_service_url + '/service=addlayer&name=45a6a3cea12e470b90f3e5a769f22161&id=45a6a3cea12e470b90f3e5a769f22161¶ms=' + str(params))
self.assertTrue(r.status_code == 200)
@unittest.skip('Not yet implemented')
def test_fdt_created_during(self):
# generate a data product and check that the FDT exists
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition('example', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
tdom, sdom = time_series_domain()
dp = DataProduct(name='example')
dp.spatial_domain = sdom.dump()
dp.temporal_domain = tdom.dump()
data_product_id = self.data_product_management.create_data_product(dp, stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product, data_product_id)
self.data_product_management.activate_data_product_persistence(data_product_id)
self.addCleanup(self.data_product_management.suspend_data_product_persistence, data_product_id)
dataset_id = self.resource_registry.find_objects(data_product_id, PRED.hasDataset, id_only=True)[0][0]
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
rdt = ph.get_rdt(stream_def_id)
ph.fill_rdt(rdt, 100)
ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(monitor.event.wait(10))
gevent.sleep(1) # Yield to other greenlets, had an issue with connectivity
print "--------------------------------"
print dataset_id
coverage_path = DatasetManagementService()._get_coverage_path(dataset_id)
print coverage_path
print "--------------------------------"
#assert that the time exists in the DB
#compare the params in geoserver to those in the table
#Verify DB contains table
#eg sql request String sqlcmd = "select column_name from information_schema.columns where table_name = \'" + dataset_id + "\';";
#Verify Geoserver
#r = requests.get('http://*****:*****@unittest.skip('Not yet implemented')
def test_remove_geolayer_directory(self):
# pass the remove command to the service and check that the layer exists in geoserver similar to the one above
# send remove layer directly to localhost 8844 with some params
# check store
print ""
pass
@unittest.skip('Not yet implemented')
def test_update_geolayer_directory(self):
# pass the update command to the service and check that the layer exists in geoserver similar to the one above
# send update layer directly to localhost 8844 with some params
# check store
#does add then remove
print ""
pass
@unittest.skip('Not yet implemented')
def test_get_data_from_FDW(self):
# generate a data product and check that the FDW can get data
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition('example', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
tdom, sdom = time_series_domain()
dp = DataProduct(name='example')
dp.spatial_domain = sdom.dump()
dp.temporal_domain = tdom.dump()
data_product_id = self.data_product_management.create_data_product(dp, stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product, data_product_id)
self.data_product_management.activate_data_product_persistence(data_product_id)
self.addCleanup(self.data_product_management.suspend_data_product_persistence, data_product_id)
dataset_id = self.resource_registry.find_objects(data_product_id, PRED.hasDataset, id_only=True)[0][0]
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
rdt = ph.get_rdt(stream_def_id)
ph.fill_rdt(rdt, 100)
ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(monitor.event.wait(10))
gevent.sleep(1) # Yield to other greenlets, had an issue with connectivity
print "--------------------------------"
print dataset_id
coverage_path = DatasetManagementService()._get_coverage_path(dataset_id)
print coverage_path
print "--------------------------------"
#verify table exists in the DB (similar to above)
# ....code...
# check that the geoserver layer exists as above
# ... code ....
# make a WMS/WFS request...somet like this (or both)
url = self.gs_host+'/geoserver/geonode/ows?service=WFS&version=1.0.0&request=GetFeature&typeName=geonode:ooi_' + dataset_id + '_ooi&maxFeatures=1&outputFormat=csv'
r = requests.get(url)
assertTrue(r.status_code == 200)
#check r.text does not contain <ServiceException code="InvalidParameterValue" locator="typeName">
@unittest.skipIf(not (CFG.get_safe('eoi.meta.use_eoi_services', False)), 'Skip test in TABLE LOADER as services are not loaded')
def test_sos_response(self):
expected_content = 'SOS SERVICE IS UP.....Hello World!'
url = self.gs_ows_url + '?request=echo&service=sos'
r = requests.get(url)
self.assertEqual(r.content, expected_content)
@unittest.skipIf(not (CFG.get_safe('eoi.meta.use_eoi_services', False)), 'Skip test in TABLE LOADER as services are not loaded')
def test_sos_get_capabilities(self):
# Validates reponse is not an exception, assues valid otherwise
self.setup_resource()
expected_content = ''
url = self.gs_ows_url + '?request=getCapabilities&service=sos&version=1.0.0&offering=_' + self.offering_id + '_view'
r = requests.get(url)
self.assertEquals(r.status_code, 200)
self.assertTrue(r.content.find('<sos:Capabilities') >= 0)
@unittest.skipIf(not (CFG.get_safe('eoi.meta.use_eoi_services', False)), 'Skip test in TABLE LOADER as services are not loaded')
def test_sos_get_offering(self):
# Validates reponse is not an exception, assues valid otherwise
# TODO: Use deterministic <swe:values> for comparison
self.setup_resource()
expected_content = ''
url = self.gs_ows_url + '?request=getObservation&service=sos&version=1.0.0&offering=_' + self.offering_id + '_view&observedproperty=time,temp,density&responseformat=text/xml'
r = requests.get(url)
self.assertEquals(r.status_code, 200)
self.assertTrue(r.content.find('<om:ObservationCollection') >= 0)
self.assertTrue(r.content.find('ExceptionReport') == -1)
def reset_store(self, store_name='ooi', store_id='ooi'):
"""
Posts a resetstore request to the ImporterService
"""
try:
url = ''.join([self.importer_service_url, '/service=resetstore&name=', store_name, '&id=', store_id])
r = requests.post(url)
if r.status_code == 200:
return True
else:
return False
except Exception as e:
log.error('Unable to reset GeoServer Store. %s', e)
return False
class TestDataProductManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dpsc_cli = DataProductManagementServiceClient()
self.rrclient = ResourceRegistryServiceClient()
self.damsclient = DataAcquisitionManagementServiceClient()
self.pubsubcli = PubsubManagementServiceClient()
self.ingestclient = IngestionManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.unsc = UserNotificationServiceClient()
self.data_retriever = DataRetrieverServiceClient()
self.identcli = IdentityManagementServiceClient()
#------------------------------------------
# Create the environment
#------------------------------------------
self.stream_def_id = self.pubsubcli.create_stream_definition(name='SBE37_CDM')
self.process_definitions = {}
ingestion_worker_definition = ProcessDefinition(name='ingestion worker')
ingestion_worker_definition.executable = {
'module':'ion.processes.data.ingestion.science_granule_ingestion_worker',
'class' :'ScienceGranuleIngestionWorker'
}
process_definition_id = self.process_dispatcher.create_process_definition(process_definition=ingestion_worker_definition)
self.process_definitions['ingestion_worker'] = process_definition_id
self.pids = []
self.exchange_points = []
self.exchange_names = []
#------------------------------------------------------------------------------------------------
# First launch the ingestors
#------------------------------------------------------------------------------------------------
self.exchange_space = 'science_granule_ingestion'
self.exchange_point = 'science_data'
config = DotDict()
config.process.datastore_name = 'datasets'
config.process.queue_name = self.exchange_space
self.exchange_names.append(self.exchange_space)
self.exchange_points.append(self.exchange_point)
pid = self.process_dispatcher.schedule_process(self.process_definitions['ingestion_worker'],configuration=config)
log.debug("the ingestion worker process id: %s", pid)
self.pids.append(pid)
self.addCleanup(self.cleaning_up)
def cleaning_up(self):
for pid in self.pids:
log.debug("number of pids to be terminated: %s", len(self.pids))
try:
self.process_dispatcher.cancel_process(pid)
log.debug("Terminated the process: %s", pid)
except:
log.debug("could not terminate the process id: %s" % pid)
IngestionManagementIntTest.clean_subscriptions()
for xn in self.exchange_names:
xni = self.container.ex_manager.create_xn_queue(xn)
xni.delete()
for xp in self.exchange_points:
xpi = self.container.ex_manager.create_xp(xp)
xpi.delete()
def get_datastore(self, dataset_id):
dataset = self.dataset_management.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
@attr('EXT')
@attr('PREP')
def test_create_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
parameter_dictionary = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict')
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=parameter_dictionary._id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp')
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 10.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -10.0
dp_obj.ooi_product_name = "PRODNAME"
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product( data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
# Assert that the data product has an associated stream at this stage
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True)
self.assertNotEquals(len(stream_ids), 0)
# Assert that the data product has an associated stream def at this stage
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStreamDefinition, RT.StreamDefinition, True)
self.assertNotEquals(len(stream_ids), 0)
self.dpsc_cli.activate_data_product_persistence(dp_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0)
log.debug('Created data product %s', dp_obj)
#------------------------------------------------------------------------------------------------
# test creating a new data product with a stream definition
#------------------------------------------------------------------------------------------------
log.debug('Creating new data product with a stream definition')
dp_obj = IonObject(RT.DataProduct,
name='DP2',
description='some new dp')
dp_id2 = self.dpsc_cli.create_data_product(dp_obj, ctd_stream_def_id)
self.dpsc_cli.activate_data_product_persistence(dp_id2)
log.debug('new dp_id = %s' % dp_id2)
#------------------------------------------------------------------------------------------------
#make sure data product is associated with stream def
#------------------------------------------------------------------------------------------------
streamdefs = []
streams, _ = self.rrclient.find_objects(dp_id2, PRED.hasStream, RT.Stream, True)
for s in streams:
log.debug("Checking stream %s" % s)
sdefs, _ = self.rrclient.find_objects(s, PRED.hasStreamDefinition, RT.StreamDefinition, True)
for sd in sdefs:
log.debug("Checking streamdef %s" % sd)
streamdefs.append(sd)
self.assertIn(ctd_stream_def_id, streamdefs)
group_names = self.dpsc_cli.get_data_product_group_list()
self.assertIn("PRODNAME", group_names)
#----------------------------------------------------------------------------------------
# Create users then notifications to this data product for each user
#----------------------------------------------------------------------------------------
# user_1
user_1 = UserInfo()
user_1.name = 'user_1'
user_1.contact.email = '*****@*****.**'
# user_2
user_2 = UserInfo()
user_2.name = 'user_2'
user_2.contact.email = '*****@*****.**'
#user1 is a complete user
self.subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254"
actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject})
actor_id = self.identcli.create_actor_identity(actor_identity_obj)
user_credentials_obj = IonObject("UserCredentials", {"name": self.subject})
self.identcli.register_user_credentials(actor_id, user_credentials_obj)
user_id_1 = self.identcli.create_user_info(actor_id, user_1)
user_id_2, _ = self.rrclient.create(user_2)
delivery_config1a = IonObject(OT.DeliveryConfiguration, email='*****@*****.**', mode=DeliveryModeEnum.EMAIL, frequency=NotificationFrequencyEnum.BATCH)
delivery_config1b = IonObject(OT.DeliveryConfiguration, email='*****@*****.**', mode=DeliveryModeEnum.EMAIL, frequency=NotificationFrequencyEnum.BATCH)
notification_request_1 = NotificationRequest( name = "notification_1",
origin=dp_id,
origin_type="type_1",
event_type=OT.ResourceLifecycleEvent,
disabled_by_system = False,
delivery_configurations=[delivery_config1a, delivery_config1b])
delivery_config2a = IonObject(OT.DeliveryConfiguration, email='*****@*****.**', mode=DeliveryModeEnum.EMAIL, frequency=NotificationFrequencyEnum.BATCH)
delivery_config2b = IonObject(OT.DeliveryConfiguration, email='*****@*****.**', mode=DeliveryModeEnum.EMAIL, frequency=NotificationFrequencyEnum.BATCH)
notification_request_2 = NotificationRequest( name = "notification_2",
origin=dp_id,
origin_type="type_2",
disabled_by_system = False,
event_type=OT.DetectionEvent,
delivery_configurations=[delivery_config2a, delivery_config2b])
notification_request_1_id = self.unsc.create_notification(notification=notification_request_1, user_id=user_id_1)
notification_request_2_id = self.unsc.create_notification(notification=notification_request_2, user_id=user_id_2)
self.unsc.delete_notification(notification_request_1_id)
# test reading a non-existent data product
log.debug('reading non-existent data product')
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product('some_fake_id')
# update a data product (tests read also)
log.debug('Updating data product')
# first get the existing dp object
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# now tweak the object
dp_obj.description = 'the very first dp'
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 20.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -20.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 20.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -20.0
# now write the dp back to the registry
update_result = self.dpsc_cli.update_data_product(dp_obj)
# now get the dp back to see if it was updated
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertEquals(dp_obj.description,'the very first dp')
self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0)
log.debug('Updated data product %s', dp_obj)
#test extension
extended_product = self.dpsc_cli.get_data_product_extension(dp_id)
#validate that there is one active and one retired user notification for this data product
self.assertEqual(1, len(extended_product.computed.active_user_subscriptions.value))
self.assertEqual(1, len(extended_product.computed.past_user_subscriptions.value))
self.assertEqual(dp_id, extended_product._id)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_product.computed.product_download_size_estimated.status)
self.assertEqual(0, extended_product.computed.product_download_size_estimated.value)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_product.computed.parameters.status)
#log.debug("test_create_data_product: parameters %s" % extended_product.computed.parameters.value)
def ion_object_encoder(obj):
return obj.__dict__
#test prepare for create
data_product_data = self.dpsc_cli.prepare_data_product_support()
#print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2)
self.assertEqual(data_product_data._id, "")
self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport)
self.assertEqual(len(data_product_data.associations['StreamDefinition'].resources), 2)
self.assertEqual(len(data_product_data.associations['Dataset'].resources), 0)
self.assertEqual(len(data_product_data.associations['StreamDefinition'].associated_resources), 0)
self.assertEqual(len(data_product_data.associations['Dataset'].associated_resources), 0)
#test prepare for update
data_product_data = self.dpsc_cli.prepare_data_product_support(dp_id)
#print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2)
self.assertEqual(data_product_data._id, dp_id)
self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport)
self.assertEqual(len(data_product_data.associations['StreamDefinition'].resources), 2)
self.assertEqual(len(data_product_data.associations['Dataset'].resources), 1)
self.assertEqual(len(data_product_data.associations['StreamDefinition'].associated_resources), 1)
self.assertEqual(data_product_data.associations['StreamDefinition'].associated_resources[0].s, dp_id)
self.assertEqual(len(data_product_data.associations['Dataset'].associated_resources), 1)
self.assertEqual(data_product_data.associations['Dataset'].associated_resources[0].s, dp_id)
# now 'delete' the data product
log.debug("deleting data product: %s" % dp_id)
self.dpsc_cli.delete_data_product(dp_id)
# Assert that there are no associated streams leftover after deleting the data product
stream_ids, assoc_ids = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True)
self.assertEquals(len(stream_ids), 0)
self.assertEquals(len(assoc_ids), 0)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# Get the events corresponding to the data product
ret = self.unsc.get_recent_events(resource_id=dp_id)
events = ret.value
for event in events:
log.debug("event time: %s" % event.ts_created)
self.assertTrue(len(events) > 0)
def test_data_product_stream_def(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id)
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp')
dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
stream_def_id = self.dpsc_cli.get_data_product_stream_definition(dp_id)
self.assertEquals(ctd_stream_def_id, stream_def_id)
def test_derived_data_product(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='ctd parsed', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsubcli.delete_stream_definition, ctd_stream_def_id)
dp = DataProduct(name='Instrument DP')
dp_id = self.dpsc_cli.create_data_product(dp, stream_definition_id=ctd_stream_def_id)
self.addCleanup(self.dpsc_cli.force_delete_data_product, dp_id)
self.dpsc_cli.activate_data_product_persistence(dp_id)
self.addCleanup(self.dpsc_cli.suspend_data_product_persistence, dp_id)
dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True)
if not dataset_ids:
raise NotFound("Data Product %s dataset does not exist" % str(dp_id))
dataset_id = dataset_ids[0]
# Make the derived data product
simple_stream_def_id = self.pubsubcli.create_stream_definition(name='TEMPWAT stream def', parameter_dictionary_id=pdict_id, available_fields=['time','temp'])
tempwat_dp = DataProduct(name='TEMPWAT', category=DataProductTypeEnum.DERIVED)
tempwat_dp_id = self.dpsc_cli.create_data_product(tempwat_dp, stream_definition_id=simple_stream_def_id, parent_data_product_id=dp_id)
self.addCleanup(self.dpsc_cli.delete_data_product, tempwat_dp_id)
# Check that the streams associated with the data product are persisted with
stream_ids, _ = self.rrclient.find_objects(dp_id,PRED.hasStream,RT.Stream,True)
for stream_id in stream_ids:
self.assertTrue(self.ingestclient.is_persisted(stream_id))
stream_id = stream_ids[0]
route = self.pubsubcli.read_stream_route(stream_id=stream_id)
rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
rdt['pressure'] = np.arange(20)
publisher = StandaloneStreamPublisher(stream_id,route)
dataset_modified = Event()
def cb(*args, **kwargs):
dataset_modified.set()
es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id, auto_delete=True)
es.start()
self.addCleanup(es.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
tempwat_dataset_ids, _ = self.rrclient.find_objects(tempwat_dp_id, PRED.hasDataset, id_only=True)
tempwat_dataset_id = tempwat_dataset_ids[0]
granule = self.data_retriever.retrieve(tempwat_dataset_id, delivery_format=simple_stream_def_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt['time'], np.arange(20))
self.assertEquals(set(rdt.fields), set(['time','temp']))
def test_activate_suspend_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
# Construct temporal and spatial Coordinate Reference System objects
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp')
log.debug("Created an IonObject for a data product: %s" % dp_obj)
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# Subscribe to persist events
#------------------------------------------------------------------------------------------------
queue = gevent.queue.Queue()
def info_event_received(message, headers):
queue.put(message)
es = EventSubscriber(event_type=OT.InformationContentStatusEvent, callback=info_event_received, origin=dp_id, auto_delete=True)
es.start()
self.addCleanup(es.stop)
#------------------------------------------------------------------------------------------------
# test activate and suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.activate_data_product_persistence(dp_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True)
if not dataset_ids:
raise NotFound("Data Product %s dataset does not exist" % str(dp_id))
dataset_id = dataset_ids[0]
# Check that the streams associated with the data product are persisted with
stream_ids, _ = self.rrclient.find_objects(dp_id,PRED.hasStream,RT.Stream,True)
for stream_id in stream_ids:
self.assertTrue(self.ingestclient.is_persisted(stream_id))
stream_id = stream_ids[0]
route = self.pubsubcli.read_stream_route(stream_id=stream_id)
rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
publisher = StandaloneStreamPublisher(stream_id,route)
dataset_modified = Event()
def cb(*args, **kwargs):
dataset_modified.set()
es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id, auto_delete=True)
es.start()
self.addCleanup(es.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data = self.data_retriever.retrieve(dataset_ids[0])
self.assertIsInstance(replay_data, Granule)
log.debug("The data retriever was able to replay the dataset that was attached to the data product "
"we wanted to be persisted. Therefore the data product was indeed persisted with "
"otherwise we could not have retrieved its dataset using the data retriever. Therefore "
"this demonstration shows that L4-CI-SA-RQ-267 is satisfied: 'Data product management shall persist data products'")
data_product_object = self.rrclient.read(dp_id)
self.assertEquals(data_product_object.name,'DP1')
self.assertEquals(data_product_object.description,'some new dp')
log.debug("Towards L4-CI-SA-RQ-308: 'Data product management shall persist data product metadata'. "
" Attributes in create for the data product obj, name= '%s', description='%s', match those of object from the "
"resource registry, name='%s', desc='%s'" % (dp_obj.name, dp_obj.description,data_product_object.name,
data_product_object.description))
#------------------------------------------------------------------------------------------------
# test suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.suspend_data_product_persistence(dp_id)
dataset_modified.clear()
rdt['time'] = np.arange(20,40)
publisher.publish(rdt.to_granule())
self.assertFalse(dataset_modified.wait(2))
self.dpsc_cli.activate_data_product_persistence(dp_id)
dataset_modified.clear()
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_almost_equal(rdt['time'], np.arange(40))
dataset_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasDataset, id_only=True)
self.assertEquals(len(dataset_ids), 1)
self.dpsc_cli.suspend_data_product_persistence(dp_id)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
with self.assertRaises(NotFound):
dp_obj = self.rrclient.read(dp_id)
info_event_counter = 0
runtime = 0
starttime = time.time()
caught_events = []
#check that the four InfoStatusEvents were received
while info_event_counter < 4 and runtime < 60 :
a = queue.get(timeout=60)
caught_events.append(a)
info_event_counter += 1
runtime = time.time() - starttime
self.assertEquals(info_event_counter, 4)
class TestObservatoryManagementFullIntegration(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient()
self.RR2 = EnhancedResourceRegistryClient(self.RR)
self.OMS = ObservatoryManagementServiceClient()
self.org_management_service = OrgManagementServiceClient()
self.IMS = InstrumentManagementServiceClient()
self.dpclient = DataProductManagementServiceClient()
self.pubsubcli = PubsubManagementServiceClient()
self.damsclient = DataAcquisitionManagementServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.data_retriever = DataRetrieverServiceClient()
self.data_product_management = DataProductManagementServiceClient()
self._load_stage = 0
self._resources = {}
def preload_ooi(self, stage=STAGE_LOAD_ASSETS):
# Preloads OOI up to a given stage
if self._load_stage >= stage:
return
if self._load_stage < STAGE_LOAD_ORGS:
log.info(
"--------------------------------------------------------------------------------------------------------"
)
log.info("Preloading stage: %s (OOIR2 Orgs, users, roles)",
STAGE_LOAD_ORGS)
# load_OOIR2_scenario
self.container.spawn_process("Loader",
"ion.processes.bootstrap.ion_loader",
"IONLoader",
config=dict(
op="load",
scenario="OOIR2",
path="master",
))
self._load_stage = STAGE_LOAD_ORGS
if self._load_stage < STAGE_LOAD_PARAMS:
log.info(
"--------------------------------------------------------------------------------------------------------"
)
log.info("Preloading stage: %s (BASE params, streamdefs)",
STAGE_LOAD_PARAMS)
# load_parameter_scenarios
self.container.spawn_process(
"Loader",
"ion.processes.bootstrap.ion_loader",
"IONLoader",
config=dict(
op="load",
scenario="BETA",
path="master",
categories=
"ParameterFunctions,ParameterDefs,ParameterDictionary,StreamDefinition",
clearcols="owner_id,org_ids",
assets="res/preload/r2_ioc/ooi_assets",
parseooi="True",
))
self._load_stage = STAGE_LOAD_PARAMS
if self._load_stage < STAGE_LOAD_AGENTS:
log.info(
"--------------------------------------------------------------------------------------------------------"
)
log.info("Preloading stage: %s (OOIR2_I agents, model links)",
STAGE_LOAD_AGENTS)
# load_OOIR2_agents
self.container.spawn_process("Loader",
"ion.processes.bootstrap.ion_loader",
"IONLoader",
config=dict(
op="load",
scenario="OOIR2_I",
path="master",
))
self._load_stage = STAGE_LOAD_AGENTS
if self._load_stage < STAGE_LOAD_ASSETS:
log.info(
"--------------------------------------------------------------------------------------------------------"
)
log.info(
"Preloading stage: %s (OOI assets linked to params, agents)",
STAGE_LOAD_ASSETS)
# load_ooi_assets
self.container.spawn_process(
"Loader",
"ion.processes.bootstrap.ion_loader",
"IONLoader",
config=dict(
op="load",
loadooi="True",
path="master",
assets="res/preload/r2_ioc/ooi_assets",
bulk="True",
debug="True",
ooiuntil="9/1/2013",
ooiparams="True",
#excludecategories: DataProduct,DataProductLink,Deployment,Workflow,WorkflowDefinition
))
self._load_stage = STAGE_LOAD_ASSETS
# 'DataProduct,DataProductLink,WorkflowDefinition,ExternalDataProvider,ExternalDatasetModel,ExternalDataset,ExternalDatasetAgent,ExternalDatasetAgentInstance',
@unittest.skip('Work in progress')
def test_observatory(self):
self._load_stage = 0
self._resources = {}
passing = True
self.assertTrue(True)
# LOAD STEP 1
self.preload_ooi(stage=STAGE_LOAD_ORGS)
passing &= self.orguserrole_assertions()
# LOAD STEP 2
self.preload_ooi(stage=STAGE_LOAD_PARAMS)
passing &= self.parameter_assertions()
# LOAD STEP 3
self.preload_ooi(stage=STAGE_LOAD_AGENTS)
passing &= self.agent_assertions()
# LOAD STEP 4
self.preload_ooi(stage=STAGE_LOAD_ASSETS)
# Check OOI preloaded resources to see if they match needs for this test and for correctness
passing &= self.sites_assertions()
passing &= self.device_assertions()
passing &= self.deployment_assertions()
# Extensive tests on select RSN nodes
passing &= self.rsn_node_checks()
# Extensive tests on select RSN instruments
passing &= self.check_rsn_instrument()
passing &= self.check_rsn_instrument_data_product()
# Extensive tests on a glider
#passing &= self.check_glider()
# Extensive tests on a CG assembly
#passing &= self.check_cg_assembly()
# Add a new instrument agent
# Add a new instrument agent instance
# Check DataProducts
# Check Provenance
IonIntegrationTestCase.assertTrue(self, passing)
# -------------------------------------------------------------------------
def orguserrole_assertions(self):
passing = True
passing &= self._check_marine_facility("MF_CGSN")
passing &= self._check_marine_facility("MF_RSN")
passing &= self._check_marine_facility("MF_EA")
return passing
def _check_marine_facility(self, preload_id):
passing = True
log.debug("Checking marine facility %s and associations", preload_id)
mf_obj = self.retrieve_ooi_asset(preload_id)
mf_id = mf_obj._id
self._resources[preload_id] = mf_id
passing &= self.assertEquals(mf_obj.lcstate, LCS.DEPLOYED)
res_list, _ = self.RR.find_objects(subject=mf_id,
predicate=PRED.hasMembership,
id_only=True)
passing &= self.assertTrue(len(res_list) >= 3)
res_list, _ = self.RR.find_objects(subject=mf_id,
predicate=PRED.hasRole,
id_only=False)
passing &= self.assertTrue(len(res_list) >= 5)
passing &= self._check_role_assignments(res_list, "ORG_MANAGER")
passing &= self._check_role_assignments(res_list,
"OBSERVATORY_OPERATOR")
passing &= self._check_role_assignments(res_list,
"INSTRUMENT_OPERATOR")
return passing
def _check_role_assignments(self, role_list, role_name):
passing = True
role_obj = self._find_resource_in_list(role_list, "governance_name",
role_name)
if role_obj:
res_list = self.RR.find_subjects(predicate=PRED.hasRole,
object=role_obj._id,
id_only=True)
passing &= self.assertTrue(len(res_list) >= 1)
return passing
def parameter_assertions(self):
passing = True
pctx_list, _ = self.RR.find_resources_ext(restype=RT.ParameterContext)
passing &= self.assertTrue(len(pctx_list) >= 10)
pdict_list, _ = self.RR.find_resources_ext(
restype=RT.ParameterDictionary)
passing &= self.assertTrue(len(pdict_list) >= 10)
sdef_list, _ = self.RR.find_resources_ext(restype=RT.StreamDefinition)
passing &= self.assertTrue(len(sdef_list) >= 10)
# Verify that a PDict has the appropriate QC parameters defined
pdicts, _ = self.RR.find_resources_ext(restype=RT.ParameterDictionary,
alt_id_ns='PRE',
alt_id='DICT110')
passing &= self.assertTrue(len(pdicts) == 1)
if not pdicts:
return passing
pdict = pdicts[0]
# According to the latest SAF, density should NOT have trend
parameters, _ = self.RR.find_objects(pdict, PRED.hasParameterContext)
names = [i.name for i in parameters if i.name.startswith('density')]
passing &= self.assertTrue('density_trndtst_qc' not in names)
return passing
def agent_assertions(self):
passing = True
# TODO: More tests?
return passing
def sites_assertions(self):
passing = True
observatory_list, _ = self.RR.find_resources_ext(
restype=RT.Observatory)
passing &= self.assertTrue(len(observatory_list) >= 40)
for obs in observatory_list:
passing &= self.assertEquals(obs.lcstate, LCS.DEPLOYED)
platform_site_list, _ = self.RR.find_resources(RT.PlatformSite,
id_only=False)
log.debug('platform sites: %s', [ps.name for ps in platform_site_list])
passing &= self.assertTrue(len(platform_site_list) >= 30)
return passing
def device_assertions(self):
passing = True
platform_device_list, _ = self.RR.find_resources(RT.PlatformDevice,
id_only=False)
passing &= self.assertTrue(len(platform_device_list) >= 30)
for pdev in platform_device_list:
log.debug('platform device: %s', pdev.name)
passing &= self.assertEquals(pdev.lcstate, LCS.PLANNED)
platform_agent_list, _ = self.RR.find_resources(RT.PlatformAgent,
id_only=False)
passing &= self.assertTrue(len(platform_agent_list) >= 2)
for pagent in platform_agent_list:
log.debug('platform agent: %s', pagent.name)
passing &= self.assertEquals(pagent.lcstate, LCS.DEPLOYED)
instrument_agent_list, _ = self.RR.find_resources(RT.InstrumentAgent,
id_only=False)
passing &= self.assertTrue(len(instrument_agent_list) >= 3)
for iagent in instrument_agent_list:
log.debug('instrument agent: %s', iagent.name)
passing &= self.assertEquals(iagent.lcstate, LCS.DEPLOYED)
model_list, _ = self.RR.find_objects(subject=iagent._id,
predicate=PRED.hasModel,
id_only=True)
passing &= self.assertTrue(
len(model_list) >= 1, "IA %s" % iagent.name)
return passing
def deployment_assertions(self):
passing = True
deployment_list, _ = self.RR.find_resources(RT.Deployment,
id_only=False)
passing &= self.assertTrue(len(deployment_list) >= 30)
for deploy in deployment_list:
log.debug('deployment: %s', deploy.name)
passing &= self.assertEquals(deploy.lcstate, LCS.DEPLOYED)
return passing
def rsn_node_checks(self):
"""
Current preload creates:
- PlatformDevice in PLANNED
- PlatformSite in DEPLOYED
- Deployment in DEPLOYED
- Deployment is NOT activated
"""
passing = True
dp_obj = self.retrieve_ooi_asset("CE04OSHY-PN01C_DEP")
passing &= self.assertEquals(dp_obj.lcstate, LCS.DEPLOYED)
passing &= self.assertEquals(dp_obj.availability, AS.AVAILABLE)
log.debug(
'test_observatory retrieve CE04OSHY-PN01C_DEP deployment: %s',
dp_obj)
# Check existing RSN node CE04OSHY-LV01C Deployment (PLANNED lcstate)
CE04OSHY_LV01C_deployment = self.retrieve_ooi_asset(
'CE04OSHY-LV01C_DEP')
passing &= self.assertEquals(CE04OSHY_LV01C_deployment.lcstate,
LCS.DEPLOYED)
passing &= self.assertEquals(CE04OSHY_LV01C_deployment.availability,
AS.AVAILABLE)
#self.dump_deployment(CE04OSHY_LV01C_deployment._id)
log.debug(
'test_observatory retrieve RSN node CE04OSHY-LV01C Deployment: %s',
CE04OSHY_LV01C_deployment)
CE04OSHY_LV01C_device = self.retrieve_ooi_asset('CE04OSHY-LV01C_PD')
# Set CE04OSHY-LV01C device to DEVELOPED state
passing &= self.transition_lcs_then_verify(
resource_id=CE04OSHY_LV01C_device._id,
new_lcs_state=LCE.DEVELOP,
verify=LCS.DEVELOPED)
# Set CE04OSHY-LV01C device to INTEGRATED state
passing &= self.transition_lcs_then_verify(
resource_id=CE04OSHY_LV01C_device._id,
new_lcs_state=LCE.INTEGRATE,
verify=LCS.INTEGRATED)
# Set CE04OSHY-LV01C device to DEPLOYED state
passing &= self.transition_lcs_then_verify(
resource_id=CE04OSHY_LV01C_device._id,
new_lcs_state=LCE.DEPLOY,
verify=LCS.DEPLOYED)
# Set CE04OSHY-LV01C Deployment to DEPLOYED state
# NOTE: Deployments are created in DEPLOYED state, currently
#self.transition_lcs_then_verify(resource_id=CE04OSHY_LV01C_deployment._id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED')
# Activate Deployment for CE04OSHY-LV01C
self.OMS.activate_deployment(CE04OSHY_LV01C_deployment._id)
log.debug(
'--------- activate_deployment CE04OSHY_LV01C_deployment -------------- '
)
self.dump_deployment(CE04OSHY_LV01C_deployment._id)
passing &= self.validate_deployment_activated(
CE04OSHY_LV01C_deployment._id)
# (optional) Start CE04OSHY-LV01C platform agent with simulator
# NOTE: DataProduct is generated in DEPLOYED state
# # Set DataProduct for CE04OSHY-LV01C platform to DEPLOYED state
# output_data_product_ids, assns =self.RR.find_objects(subject=CE04OSHY_LV01C_device._id, predicate=PRED.hasOutputProduct, id_only=True)
# if output_data_product_ids:
# #self.assertEquals(len(child_devs), 3)
# for output_data_product_id in output_data_product_ids:
# log.debug('DataProduct for CE04OSHY-LV01C platform: %s', output_data_product_id)
# self.transition_lcs_then_verify(resource_id=output_data_product_id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED')
# Check events for CE04OSHY-LV01C platform
# Check existing CE04OSBP-LJ01C Deployment (PLANNED lcstate)
# dp_list, _ = self.RR.find_resources_ext(alt_id_ns="PRE", alt_id="CE04OSBP-LJ01C_DEP")
# self.assertEquals(len(dp_list), 1)
# CE04OSHY_LV01C_deployment = dp_list[0]
# self.assertEquals(CE04OSHY_LV01C_deployment.lcstate, 'PLANNED')
# log.debug('test_observatory retrieve RSN node CE04OSBP-LJ01C Deployment: %s', CE04OSHY_LV01C_deployment)
# Set CE04OSBP-LJ01C Deployment to DEPLOYED state
# Update description and other attributes for CE04OSBP-LJ01C device resource
# Create attachment (JPG image) for CE04OSBP-LJ01C device resource
# Activate Deployment for CE04OSBP-LJ01C
# (optional) Add/register CE04OSBP-LJ01C platform agent to parent agent
# (optional) Start CE04OSBP-LJ01C platform agent
return passing
def check_rsn_instrument(self):
"""
Check existing RSN instrument CE04OSBP-LJ01C-06-CTDBPO108 Deployment (PLANNED lcstate)
Current preload creates:
- InstrumentDevice in PLANNED
- InstrumentSite in DEPLOYED
- Deployment in DEPLOYED
- Deployment is activated
"""
passing = True
CE04OSBP_LJ01C_06_CTDBPO108_deploy = self.retrieve_ooi_asset(
'CE04OSBP-LJ01C-06-CTDBPO108_DEP')
self.dump_deployment(CE04OSBP_LJ01C_06_CTDBPO108_deploy._id)
#passing &= self.assertEquals(CE04OSBP_LJ01C_06_CTDBPO108_deploy.lcstate, 'PLANNED')
# Set CE04OSBP-LJ01C-06-CTDBPO108 device to DEVELOPED state
CE04OSBP_LJ01C_06_CTDBPO108_device = self.retrieve_ooi_asset(
'CE04OSBP-LJ01C-06-CTDBPO108_ID')
passing &= self.transition_lcs_then_verify(
resource_id=CE04OSBP_LJ01C_06_CTDBPO108_device._id,
new_lcs_state=LCE.DEVELOP,
verify='DEVELOPED')
# Set CE04OSBP-LJ01C-06-CTDBPO108 device to INTEGRATED state
passing &= self.transition_lcs_then_verify(
resource_id=CE04OSBP_LJ01C_06_CTDBPO108_device._id,
new_lcs_state=LCE.INTEGRATE,
verify='INTEGRATED')
# Set CE04OSBP-LJ01C-06-CTDBPO108 device to DEPLOYED state
passing &= self.transition_lcs_then_verify(
resource_id=CE04OSBP_LJ01C_06_CTDBPO108_device._id,
new_lcs_state=LCE.DEPLOY,
verify='DEPLOYED')
# Set CE04OSBP-LJ01C-06-CTDBPO108 Deployment to DEPLOYED state
#self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_deploy._id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED')
# Activate Deployment for CE04OSBP-LJ01C-06-CTDBPO108 instrument
log.debug(
'--------- activate_deployment CE04OSBP-LJ01C-06-CTDBPO108 deployment -------------- '
)
self.OMS.activate_deployment(CE04OSBP_LJ01C_06_CTDBPO108_deploy._id)
passing &= self.validate_deployment_activated(
CE04OSBP_LJ01C_06_CTDBPO108_deploy._id)
# (optional) Add/register CE04OSBP-LJ01C-06-CTDBPO108 instrument agent to parent agent
# (optional) Start CE04OSBP-LJ01C-06-CTDBPO108 instrument agent with simulator
# Set all DataProducts for CE04OSBP-LJ01C-06-CTDBPO108 to DEPLOYED state
# (optional) Create a substitute Deployment for site CE04OSBP-LJ01C-06-CTDBPO108 with a comparable device
CE04OSBP_LJ01C_06_CTDBPO108_isite = self.retrieve_ooi_asset(
'CE04OSBP-LJ01C-06-CTDBPO108')
## create device here: retrieve CTD Mooring on Mooring Riser 001 - similiar?
GP03FLMB_RI001_10_CTDMOG999_ID_idevice = self.retrieve_ooi_asset(
'GP03FLMB-RI001-10-CTDMOG999_ID')
deploy_id_2 = self.create_basic_deployment(
name='CE04OSBP-LJ01C-06-CTDBPO108_DEP2',
description=
'substitute Deployment for site CE04OSBP-LJ01C-06-CTDBPO108 with a comparable device'
)
self.OMS.assign_device_to_deployment(
instrument_device_id=GP03FLMB_RI001_10_CTDMOG999_ID_idevice._id,
deployment_id=deploy_id_2)
self.OMS.assign_site_to_deployment(
instrument_site_id=CE04OSBP_LJ01C_06_CTDBPO108_isite._id,
deployment_id=deploy_id_2)
self.dump_deployment(deploy_id_2)
# (optional) Activate this second deployment - check first deployment is deactivated
self.OMS.deactivate_deployment(CE04OSBP_LJ01C_06_CTDBPO108_deploy._id)
passing &= self.validate_deployment_deactivated(
CE04OSBP_LJ01C_06_CTDBPO108_deploy._id)
# log.debug('Activate deployment deploy_id_2')
# self.get_deployment_ids(deploy_id_2)
# self.dump_deployment(deploy_id_2, "deploy_id_2")
# self.OMS.activate_deployment(deploy_id_2)
# passing &= self.validate_deployment_deactivated(CE04OSBP_LJ01C_06_CTDBPO108_deploy._id)
#
# # (optional) Set first CE04OSBP-LJ01C-06-CTDBPO108 Deployment to INTEGRATED state
# passing &= self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_deploy._id, new_lcs_state=LCE.INTEGRATE, verify='INTEGRATED')
#
# # Set first CE04OSBP-LJ01C-06-CTDBPO108 device to INTEGRATED state
# passing &= self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_device._id, new_lcs_state=LCE.INTEGRATE, verify='INTEGRATED')
#
#
# # (optional) Create a third Deployment for site CE04OSBP-LJ01C-06-CTDBPO108 with a same device from first deployment
# deploy_id_3 = self.create_basic_deployment(name='CE04OSBP-LJ01C-06-CTDBPO108_DEP3', description='substitute Deployment for site CE04OSBP-LJ01C-06-CTDBPO108 with same device as first')
# self.IMS.deploy_instrument_device(instrument_device_id=GP03FLMB_RI001_10_CTDMOG999_ID_idevice._id, deployment_id=deploy_id_3)
# self.OMS.deploy_instrument_site(instrument_site_id=CE04OSBP_LJ01C_06_CTDBPO108_isite._id, deployment_id=deploy_id_3)
# self.dump_deployment(deploy_id_3)
#
#
# # Set first CE04OSBP-LJ01C-06-CTDBPO108 device to DEPLOYED state
# passing &= self.transition_lcs_then_verify(resource_id=CE04OSBP_LJ01C_06_CTDBPO108_device._id, new_lcs_state=LCE.DEPLOY, verify='DEPLOYED')
#
# # (optional) Activate this third deployment - check second deployment is deactivated
# log.debug('Activate deployment deploy_id_3')
# self.dump_deployment(deploy_id_3)
# self.OMS.activate_deployment(deploy_id_3)
# #todo: check second deployment is deactivated
return passing
def check_data_product_reference(self, reference_designator, output=[]):
passing = True
data_product_ids, _ = self.RR.find_resources_ext(
alt_id_ns='PRE',
alt_id='%s_DPI1' % reference_designator,
id_only=True) # Assuming DPI1 is parsed
passing &= self.assertEquals(len(data_product_ids), 1)
if not data_product_ids:
return passing
# Let's go ahead and activate it
data_product_id = data_product_ids[0]
self.dpclient.activate_data_product_persistence(data_product_id)
self.addCleanup(self.dpclient.suspend_data_product_persistence,
data_product_id)
dataset_ids, _ = self.RR.find_objects(data_product_id,
PRED.hasDataset,
id_only=True)
passing &= self.assertEquals(len(dataset_ids), 1)
if not dataset_ids:
return passing
dataset_id = dataset_ids[0]
stream_def_ids, _ = self.RR.find_objects(data_product_id,
PRED.hasStreamDefinition,
id_only=True)
passing &= self.assertEquals(len(dataset_ids), 1)
if not stream_def_ids:
return passing
stream_def_id = stream_def_ids[0]
output.append((data_product_id, stream_def_id, dataset_id))
return passing
def check_tempsf_instrument_data_product(self, reference_designator):
passing = True
info_list = []
passing &= self.check_data_product_reference(reference_designator,
info_list)
if not passing: return passing
data_product_id, stream_def_id, dataset_id = info_list.pop()
now = time.time()
ntp_now = now + 2208988800
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [ntp_now]
rdt['temperature'] = [[
25.3884, 26.9384, 24.3394, 23.3401, 22.9832, 29.4434, 26.9873,
15.2883, 16.3374, 14.5883, 15.7253, 18.4383, 15.3488, 17.2993,
10.2111, 11.5993, 10.9345, 9.4444, 9.9876, 10.9834, 11.0098,
5.3456, 4.2994, 4.3009
]]
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt)
passing &= self.assertTrue(dataset_monitor.wait())
if not passing: return passing
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
passing &= self.assert_array_almost_equal(rdt['time'], [ntp_now])
passing &= self.assert_array_almost_equal(rdt['temperature'], [[
25.3884, 26.9384, 24.3394, 23.3401, 22.9832, 29.4434, 26.9873,
15.2883, 16.3374, 14.5883, 15.7253, 18.4383, 15.3488, 17.2993,
10.2111, 11.5993, 10.9345, 9.4444, 9.9876, 10.9834, 11.0098,
5.3456, 4.2994, 4.3009
]])
return passing
def check_trhph_instrument_data_products(self, reference_designator):
passing = True
info_list = []
passing &= self.check_data_product_reference(reference_designator,
info_list)
if not passing:
return passing
data_product_id, stream_def_id, dataset_id = info_list.pop()
pdict = self.RR2.find_parameter_dictionary_of_stream_definition_using_has_parameter_dictionary(
stream_def_id)
passing &= self.assertEquals(pdict.name, 'trhph_sample')
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
# calibration constants
a = 1.98e-9
b = -2.45e-6
c = 9.28e-4
d = -0.0888
e = 0.731
V_s = 1.506
V_c = 0.
T = 11.8
r1 = 0.906
r2 = 4.095
r3 = 4.095
ORP_V = 1.806
Cl = np.nan
offset = 2008
gain = 4.0
# Normally this would be 50 per the DPS but the precision is %4.0f which truncates the values to the nearest 1...
ORP = ((ORP_V * 1000.) - offset) / gain
ntp_now = time.time() + 2208988800
rdt['cc_a'] = [a]
rdt['cc_b'] = [b]
rdt['cc_c'] = [c]
rdt['cc_d'] = [d]
rdt['cc_e'] = [e]
rdt['ref_temp_volts'] = [V_s]
rdt['resistivity_temp_volts'] = [V_c]
rdt['eh_sensor'] = [ORP_V]
rdt['resistivity_5'] = [r1]
rdt['resistivity_x1'] = [r2]
rdt['resistivity_x5'] = [r3]
rdt['cc_offset'] = [offset]
rdt['cc_gain'] = [gain]
rdt['time'] = [ntp_now]
passing &= self.assert_array_almost_equal(rdt['vent_fluid_temperaure'],
[T], 2)
passing &= self.assert_array_almost_equal(
rdt['vent_fluid_chloride_conc'], [Cl], 4)
passing &= self.assert_array_almost_equal(rdt['vent_fluid_orp'], [ORP],
4)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt)
passing &= self.assertTrue(dataset_monitor.wait())
if not passing: return passing
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
passing &= self.assert_array_almost_equal(rdt['vent_fluid_temperaure'],
[T], 2)
passing &= self.assert_array_almost_equal(
rdt['vent_fluid_chloride_conc'], [Cl], 4)
passing &= self.assert_array_almost_equal(rdt['vent_fluid_orp'], [ORP],
4)
return passing
def check_vel3d_instrument_data_products(self, reference_designator):
passing = True
info_list = []
passing &= self.check_data_product_reference(reference_designator,
info_list)
if not passing:
return passing
data_product_id, stream_def_id, dataset_id = info_list.pop()
pdict = self.RR2.find_parameter_dictionary_of_stream_definition_using_has_parameter_dictionary(
stream_def_id)
self.assertEquals(pdict.name, 'vel3d_b_sample')
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
lat = 14.6846
lon = -51.044
ts = np.array([
3319563600, 3319567200, 3319570800, 3319574400, 3319578000,
3319581600, 3319585200, 3319588800, 3319592400, 3319596000
],
dtype=np.float)
ve = np.array([-3.2, 0.1, 0., 2.3, -0.1, 5.6, 5.1, 5.8, 8.8, 10.3])
vn = np.array([18.2, 9.9, 12., 6.6, 7.4, 3.4, -2.6, 0.2, -1.5, 4.1])
vu = np.array([-1.1, -0.6, -1.4, -2, -1.7, -2, 1.3, -1.6, -1.1, -4.5])
ve_expected = np.array([
-0.085136, -0.028752, -0.036007, 0.002136, -0.023158, 0.043218,
0.056451, 0.054727, 0.088446, 0.085952
])
vn_expected = np.array([
0.164012, 0.094738, 0.114471, 0.06986, 0.07029, 0.049237,
-0.009499, 0.019311, 0.012096, 0.070017
])
vu_expected = np.array([
-0.011, -0.006, -0.014, -0.02, -0.017, -0.02, 0.013, -0.016,
-0.011, -0.045
])
rdt['time'] = ts
rdt['lat'] = [lat] * 10
rdt['lon'] = [lon] * 10
rdt['turbulent_velocity_east'] = ve
rdt['turbulent_velocity_north'] = vn
rdt['turbulent_velocity_up'] = vu
passing &= self.assert_array_almost_equal(
rdt['eastward_turbulent_velocity'], ve_expected)
passing &= self.assert_array_almost_equal(
rdt['northward_turbulent_velocity'], vn_expected)
passing &= self.assert_array_almost_equal(
rdt['upward_turbulent_velocity'], vu_expected)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt)
passing &= self.assertTrue(dataset_monitor.wait())
if not passing: return passing
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
passing &= self.assert_array_almost_equal(
rdt['eastward_turbulent_velocity'], ve_expected)
passing &= self.assert_array_almost_equal(
rdt['northward_turbulent_velocity'], vn_expected)
passing &= self.assert_array_almost_equal(
rdt['upward_turbulent_velocity'], vu_expected)
return passing
def check_presta_instrument_data_products(self, reference_designator):
# Check the parsed data product make sure it's got everything it needs and can be published persisted etc.
# Absolute Pressure (SFLPRES_L0) is what comes off the instrumnet, SFLPRES_L1 is a pfunc
# Let's go ahead and publish some fake data!!!
# According to https://alfresco.oceanobservatories.org/alfresco/d/d/workspace/SpacesStore/63e16865-9d9e-4b11-b0b3-d5658faa5080/1341-00230_Data_Product_Spec_SFLPRES_OOI.pdf
# Appendix A. Example 1.
# p_psia_tide = 14.8670
# the tide should be 10.2504
passing = True
info_list = []
passing &= self.check_data_product_reference(reference_designator,
info_list)
if not passing:
return passing
data_product_id, stream_def_id, dataset_id = info_list.pop()
now = time.time()
ntp_now = now + 2208988800.
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [ntp_now]
rdt['absolute_pressure'] = [14.8670]
passing &= self.assert_array_almost_equal(rdt['seafloor_pressure'],
[10.2504], 4)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ParameterHelper.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(
dataset_monitor.wait()) # Bumped to 20 to keep buildbot happy
if not passing: return passing
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
passing &= self.assert_array_almost_equal(rdt['time'], [ntp_now])
passing &= self.assert_array_almost_equal(rdt['seafloor_pressure'],
[10.2504], 4)
passing &= self.assert_array_almost_equal(rdt['absolute_pressure'],
[14.8670], 4)
return passing
def check_rsn_instrument_data_product(self):
passing = True
# for RS03AXBS-MJ03A-06-PRESTA301 (PREST-A) there are a few listed data products
# Parsed, Engineering
# SFLPRES-0 SFLPRES-1
# Check for the two data products and make sure they have the proper parameters
# SFLPRES-0 should
data_products, _ = self.RR.find_resources_ext(
alt_id_ns='PRE',
alt_id='RS03AXBS-MJ03A-06-PRESTA301_SFLPRES_L0_DPID',
id_only=True)
passing &= self.assertTrue(len(data_products) == 1)
if not data_products:
return passing
data_product_id = data_products[0]
stream_defs, _ = self.RR.find_objects(data_product_id,
PRED.hasStreamDefinition,
id_only=False)
passing &= self.assertTrue(len(stream_defs) == 1)
if not stream_defs:
return passing
# Assert that the stream definition has the correct reference designator
stream_def = stream_defs[0]
passing &= self.assertEquals(
stream_def.stream_configuration['reference_designator'],
'RS03AXBS-MJ03A-06-PRESTA301')
# Get the pdict and make sure that the parameters corresponding to the available fields
# begin with the appropriate data product identifier
pdict_ids, _ = self.RR.find_objects(stream_def,
PRED.hasParameterDictionary,
id_only=True)
passing &= self.assertEquals(len(pdict_ids), 1)
if not pdict_ids:
return passing
pdict_id = pdict_ids[0]
pdict = DatasetManagementService.get_parameter_dictionary(pdict_id)
available_params = [
pdict.get_context(i) for i in pdict.keys()
if i in stream_def.available_fields
]
for p in available_params:
if p.name == 'time': # Ignore the domain parameter
continue
passing &= self.assertTrue(p.ooi_short_name.startswith('SFLPRES'))
passing &= self.check_presta_instrument_data_products(
'RS01SLBS-MJ01A-06-PRESTA101')
passing &= self.check_vel3d_instrument_data_products(
'RS01SLBS-MJ01A-12-VEL3DB101')
passing &= self.check_presta_instrument_data_products(
'RS03AXBS-MJ03A-06-PRESTA301')
passing &= self.check_vel3d_instrument_data_products(
'RS03AXBS-MJ03A-12-VEL3DB301')
passing &= self.check_tempsf_instrument_data_product(
'RS03ASHS-MJ03B-07-TMPSFA301')
passing &= self.check_vel3d_instrument_data_products(
'RS03INT2-MJ03D-12-VEL3DB304')
passing &= self.check_trhph_instrument_data_products(
'RS03INT1-MJ03C-10-TRHPHA301')
self.data_product_management.activate_data_product_persistence(
data_product_id)
dataset_id = self.RR2.find_dataset_id_of_data_product_using_has_dataset(
data_product_id)
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
self.assert_array_almost_equal(rdt['seafloor_pressure'], [10.2504], 4)
self.assert_array_almost_equal(rdt['absolute_pressure'], [14.8670], 4)
self.data_product_management.suspend_data_product_persistence(
data_product_id) # Should do nothing and not raise anything
return passing
def check_glider(self):
'''
# Check that glider GP05MOAS-GL001 assembly is defined by OOI preload (3 instruments)
'''
passing = True
GP05MOAS_GL001_device = self.retrieve_ooi_asset('GP05MOAS-GL001_PD')
child_devs, assns = self.RR.find_objects(
subject=GP05MOAS_GL001_device._id,
predicate=PRED.hasDevice,
id_only=True)
passing &= self.assertEquals(len(child_devs), 3)
# Set GP05MOAS-GL001 Deployment to DEPLOYED
GP05MOAS_GL001_deploy = self.retrieve_ooi_asset('GP05MOAS-GL001_DEP')
passing &= self.transition_lcs_then_verify(
resource_id=GP05MOAS_GL001_deploy._id,
new_lcs_state=LCE.DEPLOY,
verify='DEPLOYED')
# Activate Deployment for GP05MOAS-GL001
#self.OMS.activate_deployment(GP05MOAS_GL001_deploy._id)
# Deactivate Deployment for GP05MOAS-GL001
#self.OMS.deactivate_deployment(GP05MOAS_GL001_deploy._id)
# Create a new Deployment resource X without any assignment
x_deploy_id = self.create_basic_deployment(
name='X_Deployment',
description='new Deployment resource X without any assignment')
# Assign Deployment X to site GP05MOAS-GL001
GP05MOAS_GL001_psite = self.retrieve_ooi_asset('GP05MOAS-GL001')
self.OMS.assign_site_to_deployment(GP05MOAS_GL001_psite._id,
x_deploy_id)
# Assign Deployment X to first device for GP05MOAS-GL001
GP05MOAS_GL001_device = self.retrieve_ooi_asset('GP05MOAS-GL001_PD')
self.OMS.assign_device_to_deployment(GP05MOAS_GL001_device._id,
x_deploy_id)
# Set GP05MOAS-GL001 Deployment to PLANNED state
#self.transition_lcs_then_verify(resource_id=x_deploy_id, new_lcs_state=LCE.PLAN, verify='PLANNED')
# ??? already in planned
# Set second GP05MOAS-GL001 Deployment to DEPLOYED
passing &= self.transition_lcs_then_verify(resource_id=x_deploy_id,
new_lcs_state=LCE.DEPLOY,
verify='DEPLOYED')
self.dump_deployment(x_deploy_id)
# Activate second Deployment for GP05MOAS-GL001
#self.OMS.activate_deployment(x_deploy_id)
# Deactivate second Deployment for GP05MOAS-GL001
#self.OMS.deactivate_deployment(x_deploy_id)
return passing
def check_cg_assembly(self):
passing = True
# Set several CE01ISSM-RI002-* instrument devices to DEVELOPED state
# Assemble several CE01ISSM-RI002-* instruments to a CG CE01ISSM-RI002 component platform
# Set several CE01ISSM-RI002-* instrument devices to INTEGRATED state
# Assemble CE01ISSM-RI002 platform to CG CE01ISSM-LM001 station platform
# Set CE01ISSM-RI002 component device to INTEGRATED state
# Set CE01ISSM-LM001 station device to INTEGRATED state
# Set CE01ISSM-LM001 station device to DEPLOYED state (children maybe too?)
# Set CE01ISSM-LM001 Deployment to DEPLOYED
# Activate CE01ISSM-LM001 platform assembly deployment
# Dectivate CE01ISSM-LM001 platform assembly deployment
# Set CE01ISSM-LM001 Deployment to INTEGRATED state
# Set CE01ISSM-LM001 station device to INTEGRATED state
# Set CE01ISSM-RI002 component device to INTEGRATED state
# Set CE01ISSM-RI002 component device to INTEGRATED state
# Disassemble CE01ISSM-RI002 platform from CG CE01ISSM-LM001 station platform
# Disassemble all CE01ISSM-RI002-* instruments from a CG CE01ISSM-RI002 component platform
# Retire instrument one for CE01ISSM-RI002-*
# Retire device one for CE01ISSM-RI002
# Retire device one for CE01ISSM-LM001
return passing
# -------------------------------------------------------------------------
def retrieve_ooi_asset(self, alt_id='', namespace='PRE'):
dp_list, _ = self.RR.find_resources_ext(alt_id_ns=namespace,
alt_id=alt_id)
self.assertEquals(len(dp_list), 1)
return dp_list[0]
def transition_lcs_then_verify(self, resource_id, new_lcs_state, verify):
ret = self.RR2.advance_lcs(resource_id, new_lcs_state)
resource_obj = self.RR.read(resource_id)
return self.assertEquals(resource_obj.lcstate, verify)
def create_basic_deployment(self, name='', description=''):
start = IonTime(datetime.datetime(2013, 1, 1))
end = IonTime(datetime.datetime(2014, 1, 1))
temporal_bounds = IonObject(OT.TemporalBounds,
name='planned',
start_datetime=start.to_string(),
end_datetime=end.to_string())
deployment_obj = IonObject(RT.Deployment,
name=name,
description=description,
context=IonObject(
OT.CabledNodeDeploymentContext),
constraint_list=[temporal_bounds])
return self.OMS.create_deployment(deployment_obj)
def validate_deployment_activated(self, deployment_id=''):
site_id, device_id = self.get_deployment_ids(deployment_id)
assocs = self.RR.find_associations(subject=site_id,
predicate=PRED.hasDevice,
object=device_id)
return self.assertEquals(len(assocs), 1)
def validate_deployment_deactivated(self, deployment_id=''):
site_id, device_id = self.get_deployment_ids(deployment_id)
assocs = self.RR.find_associations(subject=site_id,
predicate=PRED.hasDevice,
object=device_id)
return self.assertEquals(len(assocs), 0)
def dump_deployment(self, deployment_id='', name=""):
#site_id, device_id = self.get_deployment_ids(deployment_id)
resource_list, _ = self.RR.find_subjects(predicate=PRED.hasDeployment,
object=deployment_id,
id_only=True)
resource_list.append(deployment_id)
resources = self.RR.read_mult(resource_list)
log.debug('--------- dump_deployment %s summary---------------',
name)
for resource in resources:
log.debug('%s: %s (%s)', resource._get_type(), resource.name,
resource._id)
log.debug('--------- dump_deployment %s full dump ---------------',
name)
for resource in resources:
log.debug('resource: %s ', resource)
log.debug('--------- dump_deployment %s end ---------------', name)
#assocs = self.container.resource_registry.find_assoctiations(anyside=deployment_id)
# assocs = Container.instance.resource_registry.find_assoctiations(anyside=deployment_id)
# log.debug('--------- dump_deployment ---------------')
# for assoc in assocs:
# log.debug('SUBJECT: %s PREDICATE: %s OBJET: %s', assoc.s, assoc.p, assoc.o)
# log.debug('--------- dump_deployment end ---------------')
def get_deployment_ids(self, deployment_id=''):
devices = []
sites = []
idevice_list, _ = self.RR.find_subjects(RT.InstrumentDevice,
PRED.hasDeployment,
deployment_id,
id_only=True)
pdevice_list, _ = self.RR.find_subjects(RT.PlatformDevice,
PRED.hasDeployment,
deployment_id,
id_only=True)
devices = idevice_list + pdevice_list
self.assertEquals(1, len(devices))
isite_list, _ = self.RR.find_subjects(RT.InstrumentSite,
PRED.hasDeployment,
deployment_id,
id_only=True)
psite_list, _ = self.RR.find_subjects(RT.PlatformSite,
PRED.hasDeployment,
deployment_id,
id_only=True)
sites = isite_list + psite_list
self.assertEquals(1, len(sites))
return sites[0], devices[0]
def _find_resource_in_list(self,
res_list,
attr,
attr_val,
assert_found=True):
for res in res_list:
v = getattr(res, attr, None)
if v == attr_val:
return res
if assert_found:
self.assertTrue(
False,
"Attribute %s value %s not found in list" % (attr, attr_val))
return None
# -------------------------------------------------------------------------
def _get_caller(self):
s = inspect.stack()
return "%s:%s" % (s[2][1], s[2][2])
@assertion_wrapper
def assert_array_almost_equal(self, *args, **kwargs):
np.testing.assert_array_almost_equal(*args, **kwargs)
@assertion_wrapper
def assertEquals(self, *args, **kwargs):
IonIntegrationTestCase.assertEquals(self, *args, **kwargs)
@assertion_wrapper
def assertTrue(self, *args, **kwargs):
IonIntegrationTestCase.assertTrue(self, *args, **kwargs)
class TestActivateRSNVel3DInstrument(IonIntegrationTestCase):
def setUp(self):
# Start container
super(TestActivateRSNVel3DInstrument, self).setUp()
config = DotDict()
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml', config)
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dpclient = DataProductManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataretrieverclient = DataRetrieverServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name + '_logger')
producer_definition.executable = {
'module': 'ion.processes.data.stream_granule_logger',
'class': 'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(
process_definition=producer_definition)
configuration = {
'process': {
'stream_id': stream_id,
}
}
pid = self.processdispatchclient.schedule_process(
process_definition_id=logger_procdef_id,
configuration=configuration)
return pid
@attr('LOCOINT')
@unittest.skip('under construction')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
@patch.dict(CFG, {'endpoint': {'receive': {'timeout': 180}}})
def test_activate_rsn_vel3d(self):
log.info(
"--------------------------------------------------------------------------------------------------------"
)
# load_parameter_scenarios
self.container.spawn_process(
"Loader",
"ion.processes.bootstrap.ion_loader",
"IONLoader",
config=dict(
op="load",
scenario="BETA",
path="master",
categories=
"ParameterFunctions,ParameterDefs,ParameterDictionary,StreamDefinition",
clearcols="owner_id,org_ids",
assets="res/preload/r2_ioc/ooi_assets",
parseooi="True",
))
self.loggerpids = []
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='Vel3DMModel',
description="Vel3DMModel")
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
log.debug('test_activate_rsn_vel3d new InstrumentModel id = %s ',
instModel_id)
raw_config = StreamConfiguration(stream_name='raw',
parameter_dictionary_name='raw')
vel3d_b_sample = StreamConfiguration(
stream_name='vel3d_b_sample',
parameter_dictionary_name='vel3d_b_sample')
vel3d_b_engineering = StreamConfiguration(
stream_name='vel3d_b_engineering',
parameter_dictionary_name='vel3d_b_engineering')
RSN_VEL3D_01 = {
'DEV_ADDR': "10.180.80.6",
'DEV_PORT': 2101,
'DATA_PORT': 1026,
'CMD_PORT': 1025,
'PA_BINARY': "port_agent"
}
# Create InstrumentAgent
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='Vel3DAgent',
description="Vel3DAgent",
driver_uri=
"http://sddevrepo.oceanobservatories.org/releases/nobska_mavs4_ooicore-0.0.7-py2.7.egg",
stream_configurations=[
raw_config, vel3d_b_sample, vel3d_b_engineering
])
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
log.debug('test_activate_rsn_vel3d new InstrumentAgent id = %s',
instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(
instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug(
'test_activate_rsn_vel3d: Create instrument resource to represent the Vel3D '
)
instDevice_obj = IonObject(RT.InstrumentDevice,
name='Vel3DDevice',
description="Vel3DDevice",
serial_number="12345")
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, instDevice_id)
log.debug("test_activate_rsn_vel3d: new InstrumentDevice id = %s ",
instDevice_id)
port_agent_config = {
'device_addr': '10.180.80.6',
'device_port': 2101,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': 1025,
'data_port': 1026,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance,
name='Vel3DAgentInstance',
description="Vel3DAgentInstance",
port_agent_config=port_agent_config,
alerts=[])
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id)
parsed_sample_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'vel3d_b_sample', id_only=True)
parsed_sample_stream_def_id = self.pubsubcli.create_stream_definition(
name='vel3d_b_sample',
parameter_dictionary_id=parsed_sample_pdict_id)
parsed_eng_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'vel3d_b_engineering', id_only=True)
parsed_eng_stream_def_id = self.pubsubcli.create_stream_definition(
name='vel3d_b_engineering',
parameter_dictionary_id=parsed_eng_pdict_id)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'raw', id_only=True)
raw_stream_def_id = self.pubsubcli.create_stream_definition(
name='raw', parameter_dictionary_id=raw_pdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='vel3d_b_sample',
description='vel3d_b_sample')
sample_data_product_id = self.dpclient.create_data_product(
data_product=dp_obj,
stream_definition_id=parsed_sample_stream_def_id)
log.debug('new dp_id = %s', sample_data_product_id)
self.dpclient.activate_data_product_persistence(
data_product_id=sample_data_product_id)
self.damsclient.assign_data_product(
input_resource_id=instDevice_id,
data_product_id=sample_data_product_id)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(sample_data_product_id,
PRED.hasStream, None, True)
log.debug('sample_data_product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(sample_data_product_id,
PRED.hasDataset,
RT.Dataset, True)
log.debug('Data set for sample_data_product = %s', dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
pid = self.create_logger('vel3d_b_sample', stream_ids[0])
self.loggerpids.append(pid)
dp_obj = IonObject(RT.DataProduct,
name='vel3d_b_engineering',
description='vel3d_b_engineering')
eng_data_product_id = self.dpclient.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_eng_stream_def_id)
log.debug('new dp_id = %s', eng_data_product_id)
self.dpclient.activate_data_product_persistence(
data_product_id=eng_data_product_id)
self.damsclient.assign_data_product(
input_resource_id=instDevice_id,
data_product_id=eng_data_product_id)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test')
data_product_id2 = self.dpclient.create_data_product(
data_product=dp_obj, stream_definition_id=raw_stream_def_id)
log.debug('new dp_id = %s', data_product_id2)
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2,
PRED.hasStream, None, True)
log.debug('test_activate_rsn_vel3d Data product streams2 = %s',
str(stream_ids))
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id2,
PRED.hasDataset,
RT.Dataset, True)
log.debug('test_activate_rsn_vel3d Data set for data_product_id2 = %s',
dataset_ids[0])
self.raw_dataset = dataset_ids[0]
def start_instrument_agent():
self.imsclient.start_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id)
gevent.joinall([gevent.spawn(start_instrument_agent)])
#cleanup
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
#wait for start
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(
instAgentInstance_id)
gate = AgentProcessStateGate(self.processdispatchclient.read_process,
instDevice_id, ProcessStateEnum.RUNNING)
self.assertTrue(
gate. await (30),
"The instrument agent instance (%s) did not spawn in 30 seconds" %
gate.process_id)
#log.trace('Instrument agent instance obj: = %s' , str(inst_agent_instance_obj))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(instDevice_id,
to_name=gate.process_id,
process=FakeProcess())
def check_state(label, desired_state):
actual_state = self._ia_client.get_agent_state()
log.debug("%s instrument agent is in state '%s'", label,
actual_state)
self.assertEqual(desired_state, actual_state)
log.debug("test_activate_rsn_vel3d: got ia client %s",
str(self._ia_client))
check_state("just-spawned", ResourceAgentState.UNINITIALIZED)
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
log.debug("test_activate_rsn_vel3d: initialize %s", str(retval))
check_state("initialized", ResourceAgentState.INACTIVE)
log.debug("test_activate_rsn_vel3d Sending go_active command ")
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activate_rsn_vel3d: return value from go_active %s",
str(reply))
check_state("activated", ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("current state after sending go_active command %s",
str(state))
#
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activate_rsn_vel3d: run %s", str(reply))
check_state("commanded", ResourceAgentState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("current state after sending run command %s", str(state))
# cmd = AgentCommand(command=ProtocolEvent.START_AUTOSAMPLE)
# reply = self._ia_client.execute_agent(cmd)
# log.debug("test_activate_rsn_vel3d: run %s" , str(reply))
# state = self._ia_client.get_agent_state()
# self.assertEqual(ResourceAgentState.COMMAND, state)
#
# gevent.sleep(5)
#
# cmd = AgentCommand(command=ProtocolEvent.STOP_AUTOSAMPLE)
# reply = self._ia_client.execute_agent(cmd)
# log.debug("test_activate_rsn_vel3d: run %s" , str(reply))
# state = self._ia_client.get_agent_state()
# self.assertEqual(ResourceAgentState.COMMAND, state)
#
# cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
# retval = self._ia_client.execute_agent(cmd)
# state = retval.result
# log.debug("current state after sending STOP_AUTOSAMPLE command %s" , str(state))
#
# cmd = AgentCommand(command=ResourceAgentEvent.PAUSE)
# retval = self._ia_client.execute_agent(cmd)
# state = self._ia_client.get_agent_state()
# self.assertEqual(ResourceAgentState.STOPPED, state)
#
# cmd = AgentCommand(command=ResourceAgentEvent.RESUME)
# retval = self._ia_client.execute_agent(cmd)
# state = self._ia_client.get_agent_state()
# self.assertEqual(ResourceAgentState.COMMAND, state)
#
# cmd = AgentCommand(command=ResourceAgentEvent.CLEAR)
# retval = self._ia_client.execute_agent(cmd)
# state = self._ia_client.get_agent_state()
# self.assertEqual(ResourceAgentState.IDLE, state)
#
# cmd = AgentCommand(command=ResourceAgentEvent.RUN)
# retval = self._ia_client.execute_agent(cmd)
# state = self._ia_client.get_agent_state()
# self.assertEqual(ResourceAgentState.COMMAND, state)
log.debug("test_activate_rsn_vel3d: calling reset ")
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activate_rsn_vel3d: return from reset %s", str(reply))
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data_raw = self.dataretrieverclient.retrieve(self.raw_dataset)
self.assertIsInstance(replay_data_raw, Granule)
rdt_raw = RecordDictionaryTool.load_from_granule(replay_data_raw)
log.debug("RDT raw: %s", str(rdt_raw.pretty_print()))
self.assertIn('raw', rdt_raw)
raw_vals = rdt_raw['raw']
#--------------------------------------------------------------------------------
# Deactivate loggers
#--------------------------------------------------------------------------------
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
self.dpclient.delete_data_product(sample_data_product_id)
self.dpclient.delete_data_product(eng_data_product_id)
self.dpclient.delete_data_product(data_product_id2)
class RegistrationProcessTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dataset_management = DatasetManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.resource_registry = self.container.resource_registry
@attr('LOCOINT')
@unittest.skipIf(os.getenv(
'CEI_LAUNCH_TEST', False
), 'Host requires file-system access to coverage files, CEI mode does not support.'
)
def test_get_dataset_to_xml(self):
def init(self):
super(RegistrationProcess, self).__init__()
self.CFG = CFG
RegistrationProcess.__init__ = init
self.rp = RegistrationProcess()
self.rp.on_start()
dataset_id = self._make_dataset()
coverage_path = DatasetManagementService()._get_coverage_path(
dataset_id)
cov = SimplexCoverage.load(coverage_path)
xml_str = self.rp.get_dataset_xml(coverage_path)
dom = parseString(xml_str)
node = dom.getElementsByTagName('addAttributes')
metadata = node[0]
for n in metadata.childNodes:
if n.nodeType != 3:
if n.attributes["name"].value == "title":
self.assertEquals(cov.name, n.childNodes[0].nodeValue)
if n.attributes["name"].value == "institution":
self.assertEquals('OOI', n.childNodes[0].nodeValue)
if n.attributes["name"].value == "infoUrl":
self.assertEquals(self.rp.pydap_url + cov.name,
n.childNodes[0].nodeValue)
parameters = []
node = dom.getElementsByTagName('sourceName')
for n in node:
if n.nodeType != 3:
parameters.append(str(n.childNodes[0].nodeValue))
cov_params = [key for key in cov.list_parameters()]
for p in parameters:
self.assertIn(p, cov_params)
cov.close()
def _make_dataset(self):
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
parameter_dict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
dataset_id = self.dataset_management.create_dataset(
'test_dataset',
parameter_dictionary_id=parameter_dict_id,
spatial_domain=sdom,
temporal_domain=tdom)
return dataset_id
def test_pydap(self):
if not CFG.get_safe('bootstrap.use_pydap', False):
raise unittest.SkipTest('PyDAP is off (bootstrap.use_pydap)')
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition(
'example', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition,
stream_def_id)
tdom, sdom = time_series_domain()
dp = DataProduct(name='example')
dp.spatial_domain = sdom.dump()
dp.temporal_domain = tdom.dump()
data_product_id = self.data_product_management.create_data_product(
dp, stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product,
data_product_id)
self.data_product_management.activate_data_product_persistence(
data_product_id)
self.addCleanup(
self.data_product_management.suspend_data_product_persistence,
data_product_id)
dataset_id = self.resource_registry.find_objects(data_product_id,
PRED.hasDataset,
id_only=True)[0][0]
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
rdt = ph.get_rdt(stream_def_id)
ph.fill_rdt(rdt, 10)
ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(monitor.event.wait(10))
gevent.sleep(
1) # Yield to other greenlets, had an issue with connectivity
pydap_host = CFG.get_safe('server.pydap.host', 'localhost')
pydap_port = CFG.get_safe('server.pydap.port', 8001)
url = 'http://%s:%s/%s' % (pydap_host, pydap_port, dataset_id)
for i in xrange(
3
): # Do it three times to test that the cache doesn't corrupt the requests/responses
ds = open_url(url)
np.testing.assert_array_equal(ds['time'][:], np.arange(10))
untested = []
for k, v in rdt.iteritems():
if k == rdt.temporal_parameter:
continue
context = rdt.context(k)
if isinstance(context.param_type, QuantityType):
np.testing.assert_array_equal(ds[k][k][:][0], rdt[k])
elif isinstance(context.param_type, ArrayType):
if context.param_type.inner_encoding is None:
values = np.empty(rdt[k].shape, dtype='O')
for i, obj in enumerate(rdt[k]):
values[i] = str(obj)
np.testing.assert_array_equal(ds[k][k][:][0], values)
elif len(rdt[k].shape) > 1:
values = np.empty(rdt[k].shape[0], dtype='O')
for i in xrange(rdt[k].shape[0]):
values[i] = ','.join(
map(lambda x: str(x), rdt[k][i].tolist()))
elif isinstance(context.param_type, ConstantType):
np.testing.assert_array_equal(ds[k][k][:][0], rdt[k])
elif isinstance(context.param_type, CategoryType):
np.testing.assert_array_equal(ds[k][k][:][0], rdt[k])
else:
untested.append('%s (%s)' % (k, context.param_type))
if untested:
raise AssertionError('Untested parameters: %s' % untested)
class TestInstrumentAlerts(IonIntegrationTestCase):
pdict_id = None
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.catch_alert= gevent.queue.Queue()
def _create_instrument_model(self):
instModel_obj = IonObject( RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel" )
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
self.addCleanup(self.imsclient.delete_instrument_model, instModel_id)
return instModel_id
def _create_instrument_agent(self, instModel_id):
raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 )
parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5 )
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.4-py2.7.egg",
stream_configurations = [raw_config, parsed_config] )
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
self.addCleanup(self.imsclient.delete_instrument_agent, instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
return instAgent_id
def _create_instrument_device(self, instModel_id):
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
self.addCleanup(self.imsclient.delete_instrument_device, instDevice_id)
log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id)
return instDevice_id
def _create_instrument_stream_alarms(self, instDevice_id):
#Create stream alarms
"""
test_two_sided_interval
Test interval alarm and alarm event publishing for a closed
inteval.
"""
temp_alert_def = {
'name' : 'temperature_warning_interval',
'stream_name' : 'parsed',
'description' : 'Temperature is below the normal range of 50.0 and above.',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_DATA,
'value_id' : 'temp',
'resource_id' : instDevice_id,
'origin_type' : 'device',
'lower_bound' : 50.0,
'lower_rel_op' : '<',
'alert_class' : 'IntervalAlert'
}
late_data_alert_def = {
'name' : 'late_data_warning',
'stream_name' : 'parsed',
'description' : 'Expected data has not arrived.',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS,
'value_id' : None,
'resource_id' : instDevice_id,
'origin_type' : 'device',
'time_delta' : 2,
'alert_class' : 'LateDataAlert'
}
return temp_alert_def, late_data_alert_def
def _create_instrument_agent_instance(self, instAgent_id, instDevice_id):
# port_agent_config = {
# 'device_addr': CFG.device.sbe37.host,
# 'device_port': CFG.device.sbe37.port,
# 'process_type': PortAgentProcessType.UNIX,
# 'binary_path': "port_agent",
# 'port_agent_addr': 'localhost',
# 'command_port': CFG.device.sbe37.port_agent_cmd_port,
# 'data_port': CFG.device.sbe37.port_agent_data_port,
# 'log_level': 5,
# 'type': PortAgentType.ETHERNET
# }
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': 4008,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
temp_alert, late_data_alert = self._create_instrument_stream_alarms(instDevice_id)
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config = port_agent_config,
alerts= [temp_alert, late_data_alert]
)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj,
instAgent_id,
instDevice_id)
self.addCleanup(self.imsclient.delete_instrument_agent_instance, instAgentInstance_id)
return instAgentInstance_id
def test_alerts(self):
#-------------------------------------------------------------------------------------
# Create InstrumentModel
#-------------------------------------------------------------------------------------
instModel_id = self._create_instrument_model()
#-------------------------------------------------------------------------------------
# Create InstrumentAgent
#-------------------------------------------------------------------------------------
instAgent_id = self._create_instrument_agent(instModel_id)
#-------------------------------------------------------------------------------------
# Create InstrumentDevice
#-------------------------------------------------------------------------------------
instDevice_id = self._create_instrument_device(instModel_id)
# It is necessary for the instrument device to be associated with atleast one output data product
tdom, sdom = time_series_domain()
parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubclient.create_stream_definition(name='parsed', parameter_dictionary_id=parsed_pdict_id)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('raw', id_only=True)
raw_stream_def_id = self.pubsubclient.create_stream_definition(name='raw', parameter_dictionary_id=raw_pdict_id)
# We are creating two data products here, one for parsed and another raw
dp_obj_parsed = IonObject(RT.DataProduct,
name='parsed_data_product',
description='Parsed output data product for instrument',
temporal_domain = tdom.dump(),
spatial_domain = sdom.dump())
dp_obj_raw = IonObject(RT.DataProduct,
name='raw_data_prod',
description='Raw output data product for instrument',
temporal_domain = tdom.dump(),
spatial_domain = sdom.dump())
parsed_out_data_prod_id = self.dataproductclient.create_data_product(data_product=dp_obj_parsed, stream_definition_id=parsed_stream_def_id)
raw_out_data_prod_id = self.dataproductclient.create_data_product(data_product=dp_obj_raw, stream_definition_id=raw_stream_def_id)
self.addCleanup(self.dataproductclient.delete_data_product, parsed_out_data_prod_id)
self.addCleanup(self.dataproductclient.delete_data_product, raw_out_data_prod_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=parsed_out_data_prod_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=raw_out_data_prod_id)
# todo: note that the generated config on the instruments will be done for both raw and parsed stream defs since these two data products constructed with each are associated as output data products with the instrument
# todo: if the config is not generated for a stream def, then the instrument agent will complain if the simulator generates data corresponding to a stream def that is not there in the stream config as a mentioned stream def
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=parsed_out_data_prod_id)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=raw_out_data_prod_id)
log.debug("assigned instdevice id: %s to data product: %s", instDevice_id, raw_out_data_prod_id)
#-------------------------------------------------------------------------------------
# Create Instrument Agent Instance
#-------------------------------------------------------------------------------------
instAgentInstance_id = self._create_instrument_agent_instance(instAgent_id,instDevice_id )
#-------------------------------------------------------------------------------------
# Launch InstrumentAgentInstance, connect to the resource agent client
#-------------------------------------------------------------------------------------
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
# Wait for instrument agent to spawn
gate = ProcessStateGate(self.processdispatchclient.read_process,
inst_agent_instance_obj.agent_process_id, ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(15), "The instrument agent instance did not spawn in 15 seconds")
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(instDevice_id,
to_name=inst_agent_instance_obj.agent_process_id,
process=FakeProcess())
#-------------------------------------------------------------------------------------
# Set up the subscriber to catch the alert event
#-------------------------------------------------------------------------------------
def callback_for_alert(event, *args, **kwargs):
log.debug("caught an alert: %s", event)
self.catch_alert.put(event)
self.event_subscriber = EventSubscriber(event_type='DeviceStatusAlertEvent',
origin=instDevice_id,
callback=callback_for_alert)
self.event_subscriber.start()
self.addCleanup(self.event_subscriber.stop)
#-------------------------------------------------------------------------------------
# Running the instrument....
#-------------------------------------------------------------------------------------
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("(L4-CI-SA-RQ-334): current state after sending go_active command %s", str(state))
self.assertTrue(state, 'DRIVER_STATE_COMMAND')
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE)
retval = self._ia_client.execute_resource(cmd)
got_bad_temp = False
got_late_data = False
runtime = 0
starttime = time.time()
caught_events = []
while (got_bad_temp == False or got_late_data == False) and \
runtime < 120:
a = self.catch_alert.get(timeout=90)
caught_events.append(a)
if a.name == 'temperature_warning_interval' and \
a.description == 'Temperature is below the normal range of 50.0 and above.':
got_bad_temp = True
if a.name == 'late_data_warning' and \
a.description == 'Expected data has not arrived.':
got_late_data = True
runtime = time.time() - starttime
log.debug("caught_events: %s", [c.name for c in caught_events])
for c in caught_events:
self.assertIn(c.name, ['temperature_warning_interval', 'late_data_warning'])
self.assertEqual(c.origin, instDevice_id)
self.assertEqual(c.type_, 'DeviceStatusAlertEvent')
self.assertEqual(c.origin_type, 'InstrumentDevice')
self.assertTrue(got_bad_temp)
self.assertTrue(got_late_data)
class TestActivateInstrumentIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
super(TestActivateInstrumentIntegration, self).setUp()
config = DotDict()
config.bootstrap.use_es = True
self._start_container()
self.addCleanup(TestActivateInstrumentIntegration.es_cleanup)
self.container.start_rel_from_url('res/deploy/r2deploy.yml', config)
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dpclient = DataProductManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataretrieverclient = DataRetrieverServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.usernotificationclient = UserNotificationServiceClient()
#setup listerner vars
self._data_greenlets = []
self._no_samples = None
self._samples_received = []
self.event_publisher = EventPublisher()
@staticmethod
def es_cleanup():
es_host = CFG.get_safe('server.elasticsearch.host', 'localhost')
es_port = CFG.get_safe('server.elasticsearch.port', '9200')
es = ep.ElasticSearch(host=es_host, port=es_port, timeout=10)
indexes = STD_INDEXES.keys()
indexes.append('%s_resources_index' % get_sys_name().lower())
indexes.append('%s_events_index' % get_sys_name().lower())
for index in indexes:
IndexManagementService._es_call(es.river_couchdb_delete, index)
IndexManagementService._es_call(es.index_delete, index)
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name + '_logger')
producer_definition.executable = {
'module': 'ion.processes.data.stream_granule_logger',
'class': 'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(
process_definition=producer_definition)
configuration = {
'process': {
'stream_id': stream_id,
}
}
pid = self.processdispatchclient.schedule_process(
process_definition_id=logger_procdef_id,
configuration=configuration)
return pid
def _create_notification(self,
user_name='',
instrument_id='',
product_id=''):
#--------------------------------------------------------------------------------------
# Make notification request objects
#--------------------------------------------------------------------------------------
notification_request_1 = NotificationRequest(
name='notification_1',
origin=instrument_id,
origin_type="instrument",
event_type='ResourceLifecycleEvent')
notification_request_2 = NotificationRequest(
name='notification_2',
origin=product_id,
origin_type="data product",
event_type='DetectionEvent')
#--------------------------------------------------------------------------------------
# Create a user and get the user_id
#--------------------------------------------------------------------------------------
user = UserInfo()
user.name = user_name
user.contact.email = '*****@*****.**' % user_name
user_id, _ = self.rrclient.create(user)
#--------------------------------------------------------------------------------------
# Create notification
#--------------------------------------------------------------------------------------
self.usernotificationclient.create_notification(
notification=notification_request_1, user_id=user_id)
self.usernotificationclient.create_notification(
notification=notification_request_2, user_id=user_id)
log.debug(
"test_activateInstrumentSample: create_user_notifications user_id %s",
str(user_id))
return user_id
def get_datastore(self, dataset_id):
dataset = self.datasetclient.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(
datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def _check_computed_attributes_of_extended_instrument(
self, expected_instrument_device_id='', extended_instrument=None):
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(
extended_instrument.computed.last_data_received_datetime,
ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.uptime,
ComputedStringValue)
self.assertIsInstance(
extended_instrument.computed.power_status_roll_up,
ComputedIntValue)
self.assertIsInstance(
extended_instrument.computed.communications_status_roll_up,
ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.data_status_roll_up,
ComputedIntValue)
self.assertIsInstance(
extended_instrument.computed.location_status_roll_up,
ComputedIntValue)
# the following assert will not work without elasticsearch.
#self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) )
# Verify the computed attribute for user notification requests
self.assertEqual(
1,
len(extended_instrument.computed.user_notification_requests.value))
notifications = extended_instrument.computed.user_notification_requests.value
notification = notifications[0]
self.assertEqual(expected_instrument_device_id, notification.origin)
self.assertEqual("instrument", notification.origin_type)
self.assertEqual('ResourceLifecycleEvent', notification.event_type)
def _check_computed_attributes_of_extended_product(
self, expected_data_product_id='', extended_data_product=None):
self.assertEqual(expected_data_product_id, extended_data_product._id)
log.debug("extended_data_product.computed: %s",
extended_data_product.computed)
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(
extended_data_product.computed.product_download_size_estimated,
ComputedFloatValue)
self.assertIsInstance(
extended_data_product.computed.number_active_subscriptions,
ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.data_url,
ComputedStringValue)
self.assertIsInstance(extended_data_product.computed.stored_data_size,
ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.recent_granules,
ComputedDictValue)
self.assertIsInstance(extended_data_product.computed.parameters,
ComputedListValue)
self.assertIsInstance(extended_data_product.computed.recent_events,
ComputedEventListValue)
self.assertIsInstance(extended_data_product.computed.provenance,
ComputedDictValue)
self.assertIsInstance(
extended_data_product.computed.user_notification_requests,
ComputedListValue)
self.assertIsInstance(
extended_data_product.computed.active_user_subscriptions,
ComputedListValue)
self.assertIsInstance(
extended_data_product.computed.past_user_subscriptions,
ComputedListValue)
self.assertIsInstance(extended_data_product.computed.last_granule,
ComputedDictValue)
self.assertIsInstance(extended_data_product.computed.is_persisted,
ComputedIntValue)
self.assertIsInstance(
extended_data_product.computed.data_contents_updated,
ComputedStringValue)
self.assertIsInstance(extended_data_product.computed.data_datetime,
ComputedListValue)
# exact text here keeps changing to fit UI capabilities. keep assertion general...
self.assertIn(
'ok',
extended_data_product.computed.last_granule.value['quality_flag'])
self.assertEqual(
2, len(extended_data_product.computed.data_datetime.value))
notifications = extended_data_product.computed.user_notification_requests.value
notification = notifications[0]
self.assertEqual(expected_data_product_id, notification.origin)
self.assertEqual("data product", notification.origin_type)
self.assertEqual('DetectionEvent', notification.event_type)
@attr('LOCOINT')
#@unittest.skip('refactoring')
@unittest.skipIf(not use_es, 'No ElasticSearch')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
@patch.dict(CFG, {'endpoint': {'receive': {'timeout': 90}}})
def test_activateInstrumentSample(self):
self.loggerpids = []
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
log.debug('new InstrumentModel id = %s ', instModel_id)
raw_config = StreamConfiguration(stream_name='raw',
parameter_dictionary_name='raw')
parsed_config = StreamConfiguration(
stream_name='parsed',
parameter_dictionary_name='ctd_parsed_param_dict')
# Create InstrumentAgent
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri=DRV_URI_GOOD,
stream_configurations=[raw_config, parsed_config])
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
log.debug('new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(
instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug(
'test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) '
)
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345")
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, instDevice_id)
log.debug(
"test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ",
instDevice_id)
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance,
name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config=port_agent_config,
alerts=[])
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubcli.create_stream_definition(
name='parsed', parameter_dictionary_id=parsed_pdict_id)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'raw', id_only=True)
raw_stream_def_id = self.pubsubcli.create_stream_definition(
name='raw', parameter_dictionary_id=raw_pdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id1 = self.dpclient.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug('new dp_id = %s', data_product_id1)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id1)
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1,
PRED.hasStream, None, True)
log.debug('Data product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id1,
PRED.hasDataset,
RT.Dataset, True)
log.debug('Data set for data_product_id1 = %s', dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
pid = self.create_logger('ctd_parsed', stream_ids[0])
self.loggerpids.append(pid)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id2 = self.dpclient.create_data_product(
data_product=dp_obj, stream_definition_id=raw_stream_def_id)
log.debug('new dp_id = %s', data_product_id2)
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id2)
# setup notifications for the device and parsed data product
user_id_1 = self._create_notification(user_name='user_1',
instrument_id=instDevice_id,
product_id=data_product_id1)
#---------- Create notifications for another user and verify that we see different computed subscriptions for the two users ---------
user_id_2 = self._create_notification(user_name='user_2',
instrument_id=instDevice_id,
product_id=data_product_id2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2,
PRED.hasStream, None, True)
log.debug('Data product streams2 = %s', str(stream_ids))
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id2,
PRED.hasDataset,
RT.Dataset, True)
log.debug('Data set for data_product_id2 = %s', dataset_ids[0])
self.raw_dataset = dataset_ids[0]
#elastic search debug
es_indexes, _ = self.container.resource_registry.find_resources(
restype='ElasticSearchIndex')
log.debug('ElasticSearch indexes: %s', [i.name for i in es_indexes])
log.debug('Bootstrap %s', CFG.bootstrap.use_es)
def start_instrument_agent():
self.imsclient.start_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id)
gevent.joinall([gevent.spawn(start_instrument_agent)])
#cleanup
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
#wait for start
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(
instAgentInstance_id)
gate = AgentProcessStateGate(self.processdispatchclient.read_process,
instDevice_id, ProcessStateEnum.RUNNING)
self.assertTrue(
gate. await (30),
"The instrument agent instance (%s) did not spawn in 30 seconds" %
gate.process_id)
#log.trace('Instrument agent instance obj: = %s' , str(inst_agent_instance_obj))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(instDevice_id,
to_name=gate.process_id,
process=FakeProcess())
log.debug("test_activateInstrumentSample: got ia client %s",
str(self._ia_client))
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: initialize %s", str(retval))
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.INACTIVE, state)
log.debug("(L4-CI-SA-RQ-334): Sending go_active command ")
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrument: return value from go_active %s",
str(reply))
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.IDLE, state)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug(
"(L4-CI-SA-RQ-334): current state after sending go_active command %s",
str(state))
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: run %s", str(reply))
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.COMMAND, state)
cmd = AgentCommand(command=ResourceAgentEvent.PAUSE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.STOPPED, state)
cmd = AgentCommand(command=ResourceAgentEvent.RESUME)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.COMMAND, state)
cmd = AgentCommand(command=ResourceAgentEvent.CLEAR)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.IDLE, state)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.COMMAND, state)
cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
for i in xrange(10):
retval = self._ia_client.execute_resource(cmd)
log.debug("test_activateInstrumentSample: return from sample %s",
str(retval))
log.debug("test_activateInstrumentSample: calling reset ")
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: return from reset %s",
str(reply))
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data_raw = self.dataretrieverclient.retrieve(self.raw_dataset)
self.assertIsInstance(replay_data_raw, Granule)
rdt_raw = RecordDictionaryTool.load_from_granule(replay_data_raw)
log.debug("RDT raw: %s", str(rdt_raw.pretty_print()))
self.assertIn('raw', rdt_raw)
raw_vals = rdt_raw['raw']
all_raw = "".join(raw_vals)
# look for 't' entered after a prompt -- ">t"
t_commands = all_raw.count(">t")
if 10 != t_commands:
log.error("%s raw_vals: ", len(raw_vals))
for i, r in enumerate(raw_vals):
log.error("raw val %s: %s", i, [r])
self.fail("Expected 10 't' strings in raw_vals, got %s" %
t_commands)
else:
log.debug("%s raw_vals: ", len(raw_vals))
for i, r in enumerate(raw_vals):
log.debug("raw val %s: %s", i, [r])
replay_data_parsed = self.dataretrieverclient.retrieve(
self.parsed_dataset)
self.assertIsInstance(replay_data_parsed, Granule)
rdt_parsed = RecordDictionaryTool.load_from_granule(replay_data_parsed)
log.debug("test_activateInstrumentSample: RDT parsed: %s",
str(rdt_parsed.pretty_print()))
self.assertIn('temp', rdt_parsed)
temp_vals = rdt_parsed['temp']
pressure_vals = rdt_parsed['pressure']
if 10 != len(temp_vals):
log.error("%s temp_vals: %s", len(temp_vals), temp_vals)
self.fail("Expected 10 temp_vals, got %s" % len(temp_vals))
log.debug("l4-ci-sa-rq-138")
"""
Physical resource control shall be subject to policy
Instrument management control capabilities shall be subject to policy
The actor accessing the control capabilities must be authorized to send commands.
note from maurice 2012-05-18: Talk to tim M to verify that this is policy. If it is then talk with Stephen to
get an example of a policy test and use that to create a test stub that will be
completed when we have instrument policies.
Tim M: The "actor", aka observatory operator, will access the instrument through ION.
"""
#--------------------------------------------------------------------------------
# Get the extended data product to see if it contains the granules
#--------------------------------------------------------------------------------
extended_product = self.dpclient.get_data_product_extension(
data_product_id=data_product_id1, user_id=user_id_1)
def poller(extended_product):
return len(extended_product.computed.user_notification_requests.
value) == 1
poll(poller, extended_product, timeout=30)
self._check_computed_attributes_of_extended_product(
expected_data_product_id=data_product_id1,
extended_data_product=extended_product)
#--------------------------------------------------------------------------------
# Get the extended instrument
#--------------------------------------------------------------------------------
extended_instrument = self.imsclient.get_instrument_device_extension(
instrument_device_id=instDevice_id, user_id=user_id_1)
#--------------------------------------------------------------------------------
# For the second user, check the extended data product and the extended intrument
#--------------------------------------------------------------------------------
extended_product = self.dpclient.get_data_product_extension(
data_product_id=data_product_id2, user_id=user_id_2)
self._check_computed_attributes_of_extended_product(
expected_data_product_id=data_product_id2,
extended_data_product=extended_product)
#--------------------------------------------------------------------------------
# Get the extended instrument
#--------------------------------------------------------------------------------
extended_instrument = self.imsclient.get_instrument_device_extension(
instrument_device_id=instDevice_id, user_id=user_id_2)
self._check_computed_attributes_of_extended_instrument(
expected_instrument_device_id=instDevice_id,
extended_instrument=extended_instrument)
#--------------------------------------------------------------------------------
# Deactivate loggers
#--------------------------------------------------------------------------------
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
self.dpclient.delete_data_product(data_product_id1)
self.dpclient.delete_data_product(data_product_id2)
class TestCTDTransformsIntegration(IonIntegrationTestCase):
pdict_id = None
def setUp(self):
# Start container
# print 'instantiating container'
self._start_container()
# container = Container()
# print 'starting container'
# container.start()
# print 'started container'
self.container.start_rel_from_url("res/deploy/r2deploy.yml")
print "started services"
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.dataset_management = self.datasetclient
def create_logger(self, name, stream_id=""):
# logger process
producer_definition = ProcessDefinition(name=name + "_logger")
producer_definition.executable = {
"module": "ion.processes.data.stream_granule_logger",
"class": "StreamGranuleLogger",
}
logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition)
configuration = {"process": {"stream_id": stream_id}}
pid = self.processdispatchclient.schedule_process(
process_definition_id=logger_procdef_id, configuration=configuration
)
return pid
def _create_instrument_model(self):
instModel_obj = IonObject(
RT.InstrumentModel,
name="SBE37IMModel",
description="SBE37IMModel",
stream_configuration={"raw": "ctd_raw_param_dict", "parsed": "ctd_parsed_param_dict"},
)
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
return instModel_id
def _create_instrument_agent(self, instModel_id):
instAgent_obj = IonObject(
RT.InstrumentAgent,
name="agent007",
description="SBE37IMAgent",
driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
driver_class="SBE37Driver",
)
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
return instAgent_id
def _create_instrument_device(self, instModel_id):
instDevice_obj = IonObject(
RT.InstrumentDevice, name="SBE37IMDevice", description="SBE37IMDevice", serial_number="12345"
)
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
log.debug(
"test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id
)
return instDevice_id
def _create_instrument_agent_instance(self, instAgent_id, instDevice_id):
port_agent_config = {
"device_addr": "sbe37-simulator.oceanobservatories.org",
"device_port": 4001,
"process_type": PortAgentProcessType.UNIX,
"binary_path": "port_agent",
"command_port": 4003,
"data_port": 4000,
"log_level": 5,
}
instAgentInstance_obj = IonObject(
RT.InstrumentAgentInstance,
name="SBE37IMAgentInstance",
description="SBE37IMAgentInstance",
port_agent_config=port_agent_config,
)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id
)
return instAgentInstance_id
def _create_param_dicts(self):
tdom, sdom = time_series_domain()
self.sdom = sdom.dump()
self.tdom = tdom.dump()
self.pdict_id = self.dataset_management.read_parameter_dictionary_by_name("ctd_parsed_param_dict", id_only=True)
def _create_input_data_products(self, ctd_stream_def_id, instDevice_id):
dp_obj = IonObject(
RT.DataProduct,
name="the parsed data",
description="ctd stream test",
temporal_domain=self.tdom,
spatial_domain=self.sdom,
)
ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product)
# ---------------------------------------------------------------------------
# Retrieve the id of the OUTPUT stream from the out Data Product
# ---------------------------------------------------------------------------
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True)
pid = self.create_logger("ctd_parsed", stream_ids[0])
self.loggerpids.append(pid)
# ---------------------------------------------------------------------------
# Create CTD Raw as the second data product
# ---------------------------------------------------------------------------
if not self.pdict_id:
self._create_param_dicts()
raw_stream_def_id = self.pubsubclient.create_stream_definition(
name="SBE37_RAW", parameter_dictionary_id=self.pdict_id
)
dp_obj = IonObject(
RT.DataProduct,
name="the raw data",
description="raw stream test",
temporal_domain=self.tdom,
spatial_domain=self.sdom,
)
ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_raw_data_product)
# ---------------------------------------------------------------------------
# Retrieve the id of the OUTPUT stream from the out Data Product
# ---------------------------------------------------------------------------
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True)
# ---------------------------------------------------------------------------
# Retrieve the id of the OUTPUT stream from the out Data Product
# ---------------------------------------------------------------------------
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True)
print "Data product streams2 = ", stream_ids
return ctd_parsed_data_product
def _create_data_process_definitions(self):
# -------------------------------------------------------------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
# -------------------------------------------------------------------------------------
dpd_obj = IonObject(
RT.DataProcessDefinition,
name="ctd_L0_all",
description="transform ctd package into three separate L0 streams",
module="ion.processes.data.transforms.ctd.ctd_L0_all",
class_name="ctd_L0_all",
)
self.ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
# -------------------------------------------------------------------------------------
# L1 Conductivity: Data Process Definition
# -------------------------------------------------------------------------------------
dpd_obj = IonObject(
RT.DataProcessDefinition,
name="ctd_L1_conductivity",
description="create the L1 conductivity data product",
module="ion.processes.data.transforms.ctd.ctd_L1_conductivity",
class_name="CTDL1ConductivityTransform",
)
self.ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
# -------------------------------------------------------------------------------------
# L1 Pressure: Data Process Definition
# -------------------------------------------------------------------------------------
dpd_obj = IonObject(
RT.DataProcessDefinition,
name="ctd_L1_pressure",
description="create the L1 pressure data product",
module="ion.processes.data.transforms.ctd.ctd_L1_pressure",
class_name="CTDL1PressureTransform",
)
self.ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
# -------------------------------------------------------------------------------------
# L1 Temperature: Data Process Definition
# -------------------------------------------------------------------------------------
dpd_obj = IonObject(
RT.DataProcessDefinition,
name="ctd_L1_temperature",
description="create the L1 temperature data product",
module="ion.processes.data.transforms.ctd.ctd_L1_temperature",
class_name="CTDL1TemperatureTransform",
)
self.ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
# -------------------------------------------------------------------------------------
# L2 Salinity: Data Process Definition
# -------------------------------------------------------------------------------------
dpd_obj = IonObject(
RT.DataProcessDefinition,
name="ctd_L2_salinity",
description="create the L1 temperature data product",
module="ion.processes.data.transforms.ctd.ctd_L2_salinity",
class_name="SalinityTransform",
)
self.ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
# -------------------------------------------------------------------------------------
# L2 Density: Data Process Definition
# -------------------------------------------------------------------------------------
dpd_obj = IonObject(
RT.DataProcessDefinition,
name="ctd_L2_density",
description="create the L1 temperature data product",
module="ion.processes.data.transforms.ctd.ctd_L2_density",
class_name="DensityTransform",
)
self.ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
return (
self.ctd_L0_all_dprocdef_id,
self.ctd_L1_conductivity_dprocdef_id,
self.ctd_L1_pressure_dprocdef_id,
self.ctd_L1_temperature_dprocdef_id,
self.ctd_L2_salinity_dprocdef_id,
self.ctd_L2_density_dprocdef_id,
)
def _create_stream_definitions(self):
if not self.pdict_id:
self._create_param_dicts()
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(
name="L0_Conductivity", parameter_dictionary_id=self.pdict_id
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_conductivity_id, self.ctd_L0_all_dprocdef_id, binding="conductivity"
)
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(
name="L0_Pressure", parameter_dictionary_id=self.pdict_id
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_pressure_id, self.ctd_L0_all_dprocdef_id, binding="pressure"
)
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(
name="L0_Temperature", parameter_dictionary_id=self.pdict_id
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_temperature_id, self.ctd_L0_all_dprocdef_id, binding="temperature"
)
return outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id, outgoing_stream_l0_temperature_id
def _create_l0_output_data_products(
self, outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id, outgoing_stream_l0_temperature_id
):
output_products = {}
ctd_l0_conductivity_output_dp_obj = IonObject(
RT.DataProduct,
name="L0_Conductivity",
description="transform output conductivity",
temporal_domain=self.tdom,
spatial_domain=self.sdom,
)
self.ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_conductivity_output_dp_obj, outgoing_stream_l0_conductivity_id
)
output_products["conductivity"] = self.ctd_l0_conductivity_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l0_conductivity_output_dp_id)
ctd_l0_pressure_output_dp_obj = IonObject(
RT.DataProduct,
name="L0_Pressure",
description="transform output pressure",
temporal_domain=self.tdom,
spatial_domain=self.sdom,
)
self.ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id
)
output_products["pressure"] = self.ctd_l0_pressure_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l0_pressure_output_dp_id)
ctd_l0_temperature_output_dp_obj = IonObject(
RT.DataProduct,
name="L0_Temperature",
description="transform output temperature",
temporal_domain=self.tdom,
spatial_domain=self.sdom,
)
self.ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id
)
output_products["temperature"] = self.ctd_l0_temperature_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l0_temperature_output_dp_id)
return output_products
def _create_l1_out_data_products(self):
ctd_l1_conductivity_output_dp_obj = IonObject(
RT.DataProduct,
name="L1_Conductivity",
description="transform output L1 conductivity",
temporal_domain=self.tdom,
spatial_domain=self.sdom,
)
self.ctd_l1_conductivity_output_dp_id = self.dataproductclient.create_data_product(
ctd_l1_conductivity_output_dp_obj, self.outgoing_stream_l1_conductivity_id
)
self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l1_conductivity_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(self.ctd_l1_conductivity_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger("ctd_l1_conductivity", stream_ids[0])
self.loggerpids.append(pid)
ctd_l1_pressure_output_dp_obj = IonObject(
RT.DataProduct,
name="L1_Pressure",
description="transform output L1 pressure",
temporal_domain=self.tdom,
spatial_domain=self.sdom,
)
self.ctd_l1_pressure_output_dp_id = self.dataproductclient.create_data_product(
ctd_l1_pressure_output_dp_obj, self.outgoing_stream_l1_pressure_id
)
self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l1_pressure_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(self.ctd_l1_pressure_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger("ctd_l1_pressure", stream_ids[0])
self.loggerpids.append(pid)
ctd_l1_temperature_output_dp_obj = IonObject(
RT.DataProduct,
name="L1_Temperature",
description="transform output L1 temperature",
temporal_domain=self.tdom,
spatial_domain=self.sdom,
)
self.ctd_l1_temperature_output_dp_id = self.dataproductclient.create_data_product(
ctd_l1_temperature_output_dp_obj, self.outgoing_stream_l1_temperature_id
)
self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l1_temperature_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(self.ctd_l1_temperature_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger("ctd_l1_temperature", stream_ids[0])
self.loggerpids.append(pid)
def _create_l2_out_data_products(self):
# -------------------------------
# L2 Salinity - Density: Output Data Products
# -------------------------------
if not self.pdict_id:
self._create_param_dicts()
outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(
name="L2_salinity", parameter_dictionary_id=self.pdict_id
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l2_salinity_id, self.ctd_L2_salinity_dprocdef_id, binding="salinity"
)
outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(
name="L2_Density", parameter_dictionary_id=self.pdict_id
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l2_density_id, self.ctd_L2_density_dprocdef_id, binding="density"
)
ctd_l2_salinity_output_dp_obj = IonObject(
RT.DataProduct,
name="L2_Salinity",
description="transform output L2 salinity",
temporal_domain=self.tdom,
spatial_domain=self.sdom,
)
self.ctd_l2_salinity_output_dp_id = self.dataproductclient.create_data_product(
ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id
)
self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l2_salinity_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(self.ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger("ctd_l2_salinity", stream_ids[0])
self.loggerpids.append(pid)
ctd_l2_density_output_dp_obj = IonObject(
RT.DataProduct,
name="L2_Density",
description="transform output pressure",
temporal_domain=self.tdom,
spatial_domain=self.sdom,
)
self.ctd_l2_density_output_dp_id = self.dataproductclient.create_data_product(
ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id
)
self.dataproductclient.activate_data_product_persistence(data_product_id=self.ctd_l2_density_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(self.ctd_l2_density_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger("ctd_l2_density", stream_ids[0])
self.loggerpids.append(pid)
def test_createTransformsThenActivateInstrument(self):
self.loggerpids = []
# -------------------------------------------------------------------------------------
# Create InstrumentModel
# -------------------------------------------------------------------------------------
instModel_id = self._create_instrument_model()
# -------------------------------------------------------------------------------------
# Create InstrumentAgent
# -------------------------------------------------------------------------------------
instAgent_id = self._create_instrument_agent(instModel_id)
# -------------------------------------------------------------------------------------
# Create InstrumentDevice
# -------------------------------------------------------------------------------------
instDevice_id = self._create_instrument_device(instModel_id)
# -------------------------------------------------------------------------------------
# Create Instrument Agent Instance
# -------------------------------------------------------------------------------------
instAgentInstance_id = self._create_instrument_agent_instance(instAgent_id, instDevice_id)
# -------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
# -------------------------------------------------------------------------------------
self._create_param_dicts()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(
name="SBE37_CDM", parameter_dictionary_id=self.pdict_id
)
# -------------------------------------------------------------------------------------
# Create two data products
# -------------------------------------------------------------------------------------
ctd_parsed_data_product = self._create_input_data_products(ctd_stream_def_id, instDevice_id)
# -------------------------------------------------------------------------------------
# Create data process definitions
# -------------------------------------------------------------------------------------
self._create_data_process_definitions()
# -------------------------------------------------------------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
# -------------------------------------------------------------------------------------
outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id, outgoing_stream_l0_temperature_id = (
self._create_stream_definitions()
)
self.output_products = {}
self.output_products = self._create_l0_output_data_products(
outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id, outgoing_stream_l0_temperature_id
)
self.outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(
name="L1_conductivity", parameter_dictionary_id=self.pdict_id
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
self.outgoing_stream_l1_conductivity_id, self.ctd_L1_conductivity_dprocdef_id, binding="conductivity"
)
self.outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(
name="L1_Pressure", parameter_dictionary_id=self.pdict_id
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
self.outgoing_stream_l1_pressure_id, self.ctd_L1_pressure_dprocdef_id, binding="pressure"
)
self.outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(
name="L1_Temperature", parameter_dictionary_id=self.pdict_id
)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
self.outgoing_stream_l1_temperature_id, self.ctd_L1_temperature_dprocdef_id, binding="temperature"
)
self._create_l1_out_data_products()
self._create_l2_out_data_products()
# -------------------------------------------------------------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
# -------------------------------------------------------------------------------------
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(
self.ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products
)
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
# -------------------------------------------------------------------------------------
# L1 Conductivity: Create the data process
# -------------------------------------------------------------------------------------
l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(
self.ctd_L1_conductivity_dprocdef_id,
[self.ctd_l0_conductivity_output_dp_id],
{"conductivity": self.ctd_l1_conductivity_output_dp_id},
)
self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id)
# -------------------------------------------------------------------------------------
# L1 Pressure: Create the data process
# -------------------------------------------------------------------------------------
l1_pressure_data_process_id = self.dataprocessclient.create_data_process(
self.ctd_L1_pressure_dprocdef_id,
[self.ctd_l0_pressure_output_dp_id],
{"pressure": self.ctd_l1_pressure_output_dp_id},
)
self.dataprocessclient.activate_data_process(l1_pressure_data_process_id)
# -------------------------------------------------------------------------------------
# L1 Temperature: Create the data process
# -------------------------------------------------------------------------------------
l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(
self.ctd_L1_temperature_dprocdef_id,
[self.ctd_l0_temperature_output_dp_id],
{"temperature": self.ctd_l1_temperature_output_dp_id},
)
self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id)
# -------------------------------------------------------------------------------------
# L2 Salinity: Create the data process
# -------------------------------------------------------------------------------------
l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(
self.ctd_L2_salinity_dprocdef_id, [ctd_parsed_data_product], {"salinity": self.ctd_l2_salinity_output_dp_id}
)
self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id)
# -------------------------------------------------------------------------------------
# L2 Density: Create the data process
# -------------------------------------------------------------------------------------
l2_density_all_data_process_id = self.dataprocessclient.create_data_process(
self.ctd_L2_density_dprocdef_id, [ctd_parsed_data_product], {"density": self.ctd_l2_density_output_dp_id}
)
self.dataprocessclient.activate_data_process(l2_density_all_data_process_id)
# -------------------------------------------------------------------------------------
# Launch InstrumentAgentInstance, connect to the resource agent client
# -------------------------------------------------------------------------------------
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
self.addCleanup(
self.imsclient.stop_instrument_agent_instance, instrument_agent_instance_id=instAgentInstance_id
)
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
# Wait for instrument agent to spawn
gate = ProcessStateGate(
self.processdispatchclient.read_process, inst_agent_instance_obj.agent_process_id, ProcessStateEnum.RUNNING
)
self.assertTrue(gate.await(15), "The instrument agent instance did not spawn in 15 seconds")
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(
instDevice_id, to_name=inst_agent_instance_obj.agent_process_id, process=FakeProcess()
)
# -------------------------------------------------------------------------------------
# Streaming
# -------------------------------------------------------------------------------------
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("(L4-CI-SA-RQ-334): current state after sending go_active command %s", str(state))
self.assertTrue(state, "DRIVER_STATE_COMMAND")
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
# todo ResourceAgentClient no longer has method set_param
# # Make sure the sampling rate and transmission are sane.
# params = {
# SBE37Parameter.NAVG : 1,
# SBE37Parameter.INTERVAL : 5,
# SBE37Parameter.TXREALTIME : True
# }
# self._ia_client.set_param(params)
# todo There is no ResourceAgentEvent attribute for go_streaming... so what should be the command for it?
cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE)
retval = self._ia_client.execute_resource(cmd)
gevent.sleep(15)
cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE)
retval = self._ia_client.execute_resource(cmd)
# todo There is no ResourceAgentEvent attribute for go_observatory... so what should be the command for it?
# log.debug("test_activateInstrumentStream: calling go_observatory")
# cmd = AgentCommand(command='go_observatory')
# reply = self._ia_client.execute_agent(cmd)
# cmd = AgentCommand(command='get_current_state')
# retval = self._ia_client.execute_agent(cmd)
# state = retval.result
# log.debug("test_activateInstrumentStream: return from go_observatory state %s", str(state))
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
# -------------------------------------------------------------------------------------------------
# Cleanup processes
# -------------------------------------------------------------------------------------------------
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
class TestPlatformLaunch(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient()
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
# Use the network definition provided by RSN OMS directly.
rsn_oms = CIOMSClientFactory.create_instance(DVR_CONFIG['oms_uri'])
self._network_definition = RsnOmsUtil.build_network_definition(rsn_oms)
# get serialized version for the configuration:
self._network_definition_ser = NetworkUtil.serialize_network_definition(
self._network_definition)
if log.isEnabledFor(logging.TRACE):
log.trace("NetworkDefinition serialization:\n%s",
self._network_definition_ser)
self._async_data_result = AsyncResult()
self._data_subscribers = []
self._samples_received = []
self.addCleanup(self._stop_data_subscribers)
self._async_event_result = AsyncResult()
self._event_subscribers = []
self._events_received = []
self.addCleanup(self._stop_event_subscribers)
self._start_event_subscriber()
def _start_data_subscriber(self, stream_name, stream_id):
"""
Starts data subscriber for the given stream_name and stream_config
"""
def consume_data(message, stream_route, stream_id):
# A callback for processing subscribed-to data.
log.info('Subscriber received data message: %s.', str(message))
self._samples_received.append(message)
self._async_data_result.set()
log.info('_start_data_subscriber stream_name=%r stream_id=%r',
stream_name, stream_id)
# Create subscription for the stream
exchange_name = '%s_queue' % stream_name
self.container.ex_manager.create_xn_queue(exchange_name).purge()
sub = StandaloneStreamSubscriber(exchange_name, consume_data)
sub.start()
self._data_subscribers.append(sub)
sub_id = self.PSC.create_subscription(name=exchange_name,
stream_ids=[stream_id])
self.PSC.activate_subscription(sub_id)
sub.subscription_id = sub_id
def _stop_data_subscribers(self):
"""
Stop the data subscribers on cleanup.
"""
try:
for sub in self._data_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self.PSC.deactivate_subscription(sub.subscription_id)
except:
pass
self.PSC.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._data_subscribers = []
def _start_event_subscriber(self,
event_type="DeviceEvent",
sub_type="platform_event"):
"""
Starts event subscriber for events of given event_type ("DeviceEvent"
by default) and given sub_type ("platform_event" by default).
"""
def consume_event(evt, *args, **kwargs):
# A callback for consuming events.
log.info('Event subscriber received evt: %s.', str(evt))
self._events_received.append(evt)
self._async_event_result.set(evt)
sub = EventSubscriber(event_type=event_type,
sub_type=sub_type,
callback=consume_event)
sub.start()
log.info("registered event subscriber for event_type=%r, sub_type=%r",
event_type, sub_type)
self._event_subscribers.append(sub)
sub._ready_event.wait(timeout=EVENT_TIMEOUT)
def _stop_event_subscribers(self):
"""
Stops the event subscribers on cleanup.
"""
try:
for sub in self._event_subscribers:
if hasattr(sub, 'subscription_id'):
try:
self.PSC.deactivate_subscription(sub.subscription_id)
except:
pass
self.PSC.delete_subscription(sub.subscription_id)
sub.stop()
finally:
self._event_subscribers = []
def _create_platform_configuration(self):
"""
Verify that agent configurations are being built properly
"""
#
# This method is an adaptation of test_agent_instance_config in
# test_instrument_management_service_integration.py
#
clients = DotDict()
clients.resource_registry = self.RR
clients.pubsub_management = self.PSC
clients.dataset_management = self.DSC
pconfig_builder = PlatformAgentConfigurationBuilder(clients)
iconfig_builder = InstrumentAgentConfigurationBuilder(clients)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
org_id = self.RR2.create(any_old(RT.Org))
inst_startup_config = {'startup': 'config'}
required_config_keys = [
'org_name', 'device_type', 'agent', 'driver_config',
'stream_config', 'startup_config', 'alarm_defs', 'children'
]
def verify_instrument_config(config, device_id):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual('Org_1', config['org_name'])
self.assertEqual(RT.InstrumentDevice, config['device_type'])
self.assertIn('driver_config', config)
driver_config = config['driver_config']
expected_driver_fields = {
'process_type': ('ZMQPyClassDriverLauncher', ),
}
for k, v in expected_driver_fields.iteritems():
self.assertIn(k, driver_config)
self.assertEqual(v, driver_config[k])
self.assertEqual
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertEqual(inst_startup_config, config['startup_config'])
self.assertIn('stream_config', config)
for key in ['alarm_defs', 'children']:
self.assertEqual({}, config[key])
def verify_child_config(config, device_id, inst_device_id=None):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual('Org_1', config['org_name'])
self.assertEqual(RT.PlatformDevice, config['device_type'])
self.assertEqual({'process_type': ('ZMQPyClassDriverLauncher', )},
config['driver_config'])
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertIn('stream_config', config)
if None is inst_device_id:
for key in ['alarm_defs', 'children', 'startup_config']:
self.assertEqual({}, config[key])
else:
for key in ['alarm_defs', 'startup_config']:
self.assertEqual({}, config[key])
self.assertIn(inst_device_id, config['children'])
verify_instrument_config(config['children'][inst_device_id],
inst_device_id)
def verify_parent_config(config,
parent_device_id,
child_device_id,
inst_device_id=None):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual('Org_1', config['org_name'])
self.assertEqual(RT.PlatformDevice, config['device_type'])
self.assertEqual({'process_type': ('ZMQPyClassDriverLauncher', )},
config['driver_config'])
self.assertEqual({'resource_id': parent_device_id},
config['agent'])
self.assertIn('stream_config', config)
for key in ['alarm_defs', 'startup_config']:
self.assertEqual({}, config[key])
self.assertIn(child_device_id, config['children'])
verify_child_config(config['children'][child_device_id],
child_device_id, inst_device_id)
parsed_rpdict_id = self.DSC.read_parameter_dictionary_by_name(
'platform_eng_parsed', id_only=True)
self.parsed_stream_def_id = self.PSC.create_stream_definition(
name='parsed', parameter_dictionary_id=parsed_rpdict_id)
rpdict_id = self.DSC.read_parameter_dictionary_by_name(
'ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.PSC.create_stream_definition(
name='raw', parameter_dictionary_id=rpdict_id)
#todo: create org and figure out which agent resource needs to get assigned to it
def _make_platform_agent_structure(agent_config=None):
if None is agent_config: agent_config = {}
# instance creation
platform_agent_instance_obj = any_old(RT.PlatformAgentInstance)
platform_agent_instance_obj.agent_config = agent_config
platform_agent_instance_id = self.IMS.create_platform_agent_instance(
platform_agent_instance_obj)
# agent creation
raw_config = StreamConfiguration(
stream_name='parsed',
parameter_dictionary_name='platform_eng_parsed',
records_per_granule=2,
granule_publish_rate=5)
platform_agent_obj = any_old(
RT.PlatformAgent, {"stream_configurations": [raw_config]})
platform_agent_id = self.IMS.create_platform_agent(
platform_agent_obj)
# device creation
platform_device_id = self.IMS.create_platform_device(
any_old(RT.PlatformDevice))
# data product creation
dp_obj = any_old(RT.DataProduct, {
"temporal_domain": tdom,
"spatial_domain": sdom
})
# dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id)
dp_id = self.DP.create_data_product(
data_product=dp_obj,
stream_definition_id=self.parsed_stream_def_id)
self.DAMS.assign_data_product(input_resource_id=platform_device_id,
data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
# assignments
self.RR2.assign_platform_agent_instance_to_platform_device(
platform_agent_instance_id, platform_device_id)
self.RR2.assign_platform_agent_to_platform_agent_instance(
platform_agent_id, platform_agent_instance_id)
self.RR2.assign_platform_device_to_org_with_has_resource(
platform_agent_instance_id, org_id)
return platform_agent_instance_id, platform_agent_id, platform_device_id
def _make_instrument_agent_structure(agent_config=None):
if None is agent_config: agent_config = {}
# instance creation
instrument_agent_instance_obj = any_old(
RT.InstrumentAgentInstance,
{"startup_config": inst_startup_config})
instrument_agent_instance_obj.agent_config = agent_config
instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(
instrument_agent_instance_obj)
# agent creation
raw_config = StreamConfiguration(
stream_name='raw',
parameter_dictionary_name='ctd_raw_param_dict',
records_per_granule=2,
granule_publish_rate=5)
instrument_agent_obj = any_old(
RT.InstrumentAgent, {"stream_configurations": [raw_config]})
instrument_agent_id = self.IMS.create_instrument_agent(
instrument_agent_obj)
# device creation
instrument_device_id = self.IMS.create_instrument_device(
any_old(RT.InstrumentDevice))
# data product creation
dp_obj = any_old(RT.DataProduct, {
"temporal_domain": tdom,
"spatial_domain": sdom
})
dp_id = self.DP.create_data_product(
data_product=dp_obj, stream_definition_id=raw_stream_def_id)
self.DAMS.assign_data_product(
input_resource_id=instrument_device_id, data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
# assignments
self.RR2.assign_instrument_agent_instance_to_instrument_device(
instrument_agent_instance_id, instrument_device_id)
self.RR2.assign_instrument_agent_to_instrument_agent_instance(
instrument_agent_id, instrument_agent_instance_id)
self.RR2.assign_instrument_device_to_org_with_has_resource(
instrument_agent_instance_id, org_id)
return instrument_agent_instance_id, instrument_agent_id, instrument_device_id
# can't do anything without an agent instance obj
log.debug(
"Testing that preparing a launcher without agent instance raises an error"
)
self.assertRaises(AssertionError,
pconfig_builder.prepare,
will_launch=False)
log.debug(
"Making the structure for a platform agent, which will be the child"
)
platform_agent_instance_child_id, _, platform_device_child_id = _make_platform_agent_structure(
)
platform_agent_instance_child_obj = self.RR2.read(
platform_agent_instance_child_id)
log.debug("Preparing a valid agent instance launch, for config only")
pconfig_builder.set_agent_instance_object(
platform_agent_instance_child_obj)
child_config = pconfig_builder.prepare(will_launch=False)
verify_child_config(child_config, platform_device_child_id)
log.debug(
"Making the structure for a platform agent, which will be the parent"
)
platform_agent_instance_parent_id, _, platform_device_parent_id = _make_platform_agent_structure(
)
platform_agent_instance_parent_obj = self.RR2.read(
platform_agent_instance_parent_id)
log.debug("Testing child-less parent as a child config")
pconfig_builder.set_agent_instance_object(
platform_agent_instance_parent_obj)
parent_config = pconfig_builder.prepare(will_launch=False)
verify_child_config(parent_config, platform_device_parent_id)
log.debug("assigning child platform to parent")
self.RR2.assign_platform_device_to_platform_device(
platform_device_child_id, platform_device_parent_id)
child_device_ids = self.RR2.find_platform_device_ids_of_device(
platform_device_parent_id)
self.assertNotEqual(0, len(child_device_ids))
log.debug("Testing parent + child as parent config")
pconfig_builder.set_agent_instance_object(
platform_agent_instance_parent_obj)
parent_config = pconfig_builder.prepare(will_launch=False)
verify_parent_config(parent_config, platform_device_parent_id,
platform_device_child_id)
log.debug("making the structure for an instrument agent")
instrument_agent_instance_id, _, instrument_device_id = _make_instrument_agent_structure(
)
instrument_agent_instance_obj = self.RR2.read(
instrument_agent_instance_id)
log.debug("Testing instrument config")
iconfig_builder.set_agent_instance_object(
instrument_agent_instance_obj)
instrument_config = iconfig_builder.prepare(will_launch=False)
verify_instrument_config(instrument_config, instrument_device_id)
log.debug("assigning instrument to platform")
self.RR2.assign_instrument_device_to_platform_device(
instrument_device_id, platform_device_child_id)
child_device_ids = self.RR2.find_instrument_device_ids_of_device(
platform_device_child_id)
self.assertNotEqual(0, len(child_device_ids))
log.debug("Testing entire config")
pconfig_builder.set_agent_instance_object(
platform_agent_instance_parent_obj)
full_config = pconfig_builder.prepare(will_launch=False)
verify_parent_config(full_config, platform_device_parent_id,
platform_device_child_id, instrument_device_id)
if log.isEnabledFor(logging.TRACE):
import pprint
pp = pprint.PrettyPrinter()
pp.pprint(full_config)
log.trace("full_config = %s", pp.pformat(full_config))
return full_config
def get_streamConfigs(self):
#
# This method is an adaptation of get_streamConfigs in
# test_driver_egg.py
#
return [
StreamConfiguration(
stream_name='parsed',
parameter_dictionary_name='platform_eng_parsed',
records_per_granule=2,
granule_publish_rate=5)
# TODO enable something like the following when also
# incorporating "raw" data:
#,
#StreamConfiguration(stream_name='raw',
# parameter_dictionary_name='ctd_raw_param_dict',
# records_per_granule=2,
# granule_publish_rate=5)
]
@skip("Still needs alignment with new configuration structure")
def test_hierarchy(self):
# TODO re-implement.
pass
def test_single_platform(self):
full_config = self._create_platform_configuration()
platform_id = 'LJ01D'
stream_configurations = self.get_streamConfigs()
agent__obj = IonObject(RT.PlatformAgent,
name='%s_PlatformAgent' % platform_id,
description='%s_PlatformAgent platform agent' %
platform_id,
stream_configurations=stream_configurations)
agent_id = self.IMS.create_platform_agent(agent__obj)
device__obj = IonObject(
RT.PlatformDevice,
name='%s_PlatformDevice' % platform_id,
description='%s_PlatformDevice platform device' % platform_id,
# ports=port_objs,
# platform_monitor_attributes = monitor_attribute_objs
)
self.device_id = self.IMS.create_platform_device(device__obj)
#######################################
# data product (adapted from test_instrument_management_service_integration)
tdom, sdom = time_series_domain()
tdom = tdom.dump()
sdom = sdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='DataProduct test',
processing_level_code='Parsed_Canonical',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id1 = self.DP.create_data_product(
data_product=dp_obj,
stream_definition_id=self.parsed_stream_def_id)
log.debug('data_product_id1 = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=self.device_id,
data_product_id=data_product_id1)
self.DP.activate_data_product_persistence(
data_product_id=data_product_id1)
#######################################
#######################################
# dataset
stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream,
None, True)
log.debug('Data product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.RR.find_objects(data_product_id1,
PRED.hasDataset, RT.Dataset,
True)
log.debug('Data set for data_product_id1 = %s', dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
#######################################
full_config['platform_config'] = {
'platform_id': platform_id,
'driver_config': DVR_CONFIG,
'network_definition': self._network_definition_ser
}
agent_instance_obj = IonObject(
RT.PlatformAgentInstance,
name='%s_PlatformAgentInstance' % platform_id,
description="%s_PlatformAgentInstance" % platform_id,
agent_config=full_config)
agent_instance_id = self.IMS.create_platform_agent_instance(
platform_agent_instance=agent_instance_obj,
platform_agent_id=agent_id,
platform_device_id=self.device_id)
stream_id = stream_ids[0]
self._start_data_subscriber(agent_instance_id, stream_id)
log.debug(
"about to call imsclient.start_platform_agent_instance with id=%s",
agent_instance_id)
pid = self.IMS.start_platform_agent_instance(
platform_agent_instance_id=agent_instance_id)
log.debug("start_platform_agent_instance returned pid=%s", pid)
#wait for start
instance_obj = self.IMS.read_platform_agent_instance(agent_instance_id)
gate = ProcessStateGate(self.PDC.read_process,
instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(
gate. await (90),
"The platform agent instance did not spawn in 90 seconds")
agent_instance_obj = self.IMS.read_instrument_agent_instance(
agent_instance_id)
log.debug('Platform agent instance obj')
# Start a resource agent client to talk with the instrument agent.
self._pa_client = ResourceAgentClient(
'paclient',
name=agent_instance_obj.agent_process_id,
process=FakeProcess())
log.debug("got platform agent client %s", str(self._pa_client))
# ping_agent can be issued before INITIALIZE
retval = self._pa_client.ping_agent(timeout=TIMEOUT)
log.debug('Base Platform ping_agent = %s', str(retval))
cmd = AgentCommand(command=PlatformAgentEvent.INITIALIZE)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform INITIALIZE = %s', str(retval))
# GO_ACTIVE
cmd = AgentCommand(command=PlatformAgentEvent.GO_ACTIVE)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform GO_ACTIVE = %s', str(retval))
# RUN:
cmd = AgentCommand(command=PlatformAgentEvent.RUN)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform RUN = %s', str(retval))
# START_MONITORING:
cmd = AgentCommand(command=PlatformAgentEvent.START_MONITORING)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform START_MONITORING = %s', str(retval))
# wait for data sample
# just wait for at least one -- see consume_data above
log.info("waiting for reception of a data sample...")
self._async_data_result.get(timeout=DATA_TIMEOUT)
self.assertTrue(len(self._samples_received) >= 1)
log.info("waiting a bit more for reception of more data samples...")
sleep(15)
log.info("Got data samples: %d", len(self._samples_received))
# wait for event
# just wait for at least one event -- see consume_event above
log.info("waiting for reception of an event...")
self._async_event_result.get(timeout=EVENT_TIMEOUT)
log.info("Received events: %s", len(self._events_received))
#get the extended platfrom which wil include platform aggreate status fields
# extended_platform = self.IMS.get_platform_device_extension(self.device_id)
# log.debug( 'test_single_platform extended_platform: %s', str(extended_platform) )
# log.debug( 'test_single_platform power_status_roll_up: %s', str(extended_platform.computed.power_status_roll_up.value) )
# log.debug( 'test_single_platform comms_status_roll_up: %s', str(extended_platform.computed.communications_status_roll_up.value) )
# STOP_MONITORING:
cmd = AgentCommand(command=PlatformAgentEvent.STOP_MONITORING)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform STOP_MONITORING = %s', str(retval))
# GO_INACTIVE
cmd = AgentCommand(command=PlatformAgentEvent.GO_INACTIVE)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform GO_INACTIVE = %s', str(retval))
# RESET: Resets the base platform agent, which includes termination of
# its sub-platforms processes:
cmd = AgentCommand(command=PlatformAgentEvent.RESET)
retval = self._pa_client.execute_agent(cmd, timeout=TIMEOUT)
log.debug('Base Platform RESET = %s', str(retval))
#-------------------------------
# Stop Base Platform AgentInstance
#-------------------------------
self.IMS.stop_platform_agent_instance(
platform_agent_instance_id=agent_instance_id)
class TestPlatformInstrument(BaseIntTestPlatform):
def setUp(self):
self._start_container()
self._pp = pprint.PrettyPrinter()
log.debug("oms_uri = %s", OMS_URI)
self.oms = CIOMSClientFactory.create_instance(OMS_URI)
#url = OmsTestMixin.start_http_server()
#log.debug("TestPlatformInstrument:setup http url %s", url)
#
#result = self.oms.event.register_event_listener(url)
#log.debug("TestPlatformInstrument:setup register_event_listener result %s", result)
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.RR2 = EnhancedResourceRegistryClient(self.rrclient)
self.org_id = self.RR2.create(any_old(RT.Org))
log.debug("Org created: %s", self.org_id)
# see _set_receive_timeout
self._receive_timeout = 300
self.instrument_device_id = ''
self.platform_device_id = ''
self.platform_site_id = ''
self.platform_agent_instance_id = ''
self._pa_client = ''
def done():
CIOMSClientFactory.destroy_instance(self.oms)
event_notifications = OmsTestMixin.stop_http_server()
log.info("event_notifications = %s" % str(event_notifications))
self.addCleanup(done)
@unittest.skip('Must be run locally...')
def test_platform_with_instrument_streaming(self):
#
# The following is with just a single platform and the single
# instrument "SBE37_SIM_08", which corresponds to the one on port 4008.
#
#load the paramaters and the param dicts necesssary for the VEL3D
log.debug( "load params------------------------------------------------------------------------------")
self._load_params()
log.debug( " _register_oms_listener------------------------------------------------------------------------------")
self._register_oms_listener()
#create the instrument device/agent/mode
log.debug( "---------- create_instrument_resources ----------" )
self._create_instrument_resources()
#create the platform device, agent and instance
log.debug( "---------- create_platform_configuration ----------" )
self._create_platform_configuration('LPJBox_CI_Ben_Hall')
self.rrclient.create_association(subject=self.platform_device_id, predicate=PRED.hasDevice, object=self.instrument_device_id)
log.debug( "---------- start_platform ----------" )
self._start_platform()
self.addCleanup(self._stop_platform)
# get everything in command mode:
self._ping_agent()
log.debug( " ---------- initialize ----------" )
self._initialize()
_ia_client = ResourceAgentClient(self.instrument_device_id, process=FakeProcess())
state = _ia_client.get_agent_state()
log.info("TestPlatformInstrument get_agent_state %s", state)
log.debug( " ---------- go_active ----------" )
self._go_active()
state = _ia_client.get_agent_state()
log.info("TestPlatformInstrument get_agent_state %s", state)
log.debug( "---------- run ----------" )
self._run()
gevent.sleep(2)
log.debug( " ---------- _start_resource_monitoring ----------" )
self._start_resource_monitoring()
gevent.sleep(2)
#
# # verify the instrument is command state:
# state = ia_client.get_agent_state()
# log.debug(" TestPlatformInstrument get_agent_state: %s", state)
# self.assertEqual(state, ResourceAgentState.COMMAND) _stop_resource_monitoring
log.debug( " ---------- _stop_resource_monitoring ----------" )
self._stop_resource_monitoring()
gevent.sleep(2)
log.debug( " ---------- go_inactive ----------" )
self._go_inactive()
state = _ia_client.get_agent_state()
log.info("TestPlatformInstrument get_agent_state %s", state)
self._reset()
self._shutdown()
def _get_platform_attributes(self):
log.debug( " ----------get_platform_attributes ----------")
attr_infos = self.oms.attr.get_platform_attributes('LPJBox_CI_Ben_Hall')
log.debug('_get_platform_attributes: %s', self._pp.pformat(attr_infos))
attrs = attr_infos['LPJBox_CI_Ben_Hall']
for attrid, arrinfo in attrs.iteritems():
arrinfo['attr_id'] = attrid
log.debug('_get_platform_attributes: %s', self._pp.pformat(attrs))
return attrs
def _load_params(self):
log.info(" ---------- load_params ----------")
# load_parameter_scenarios
self.container.spawn_process("Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=dict(
op="load",
scenario="BETA",
path="master",
categories="ParameterFunctions,ParameterDefs,ParameterDictionary,StreamDefinition",
clearcols="owner_id,org_ids",
assets="res/preload/r2_ioc/ooi_assets",
parseooi="True",
))
def _create_platform_configuration(self, platform_id, parent_platform_id=None):
"""
This method is an adaptation of test_agent_instance_config in
test_instrument_management_service_integration.py
@param platform_id
@param parent_platform_id
@return a DotDict with various of the constructed elements associated
to the platform.
"""
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
param_dict_name = 'platform_eng_parsed'
parsed_rpdict_id = self.dataset_management.read_parameter_dictionary_by_name(
param_dict_name,
id_only=True)
self.parsed_stream_def_id = self.pubsubclient.create_stream_definition(
name='parsed',
parameter_dictionary_id=parsed_rpdict_id)
driver_config = PLTFRM_DVR_CONFIG
driver_config['attributes'] = self._get_platform_attributes() #self._platform_attributes[platform_id]
#OMS returning an error for port.get_platform_ports
#driver_config['ports'] = self._platform_ports[platform_id]
log.debug("driver_config: %s", driver_config)
# instance creation
platform_agent_instance_obj = any_old(RT.PlatformAgentInstance, {
'driver_config': driver_config})
platform_agent_instance_obj.agent_config = {
'platform_config': { 'platform_id': 'LPJBox_CI_Ben_Hall', 'parent_platform_id': None }
}
self.platform_agent_instance_id = self.imsclient.create_platform_agent_instance(platform_agent_instance_obj)
# agent creation
platform_agent_obj = any_old(RT.PlatformAgent, {
"stream_configurations": self._get_platform_stream_configs(),
'driver_module': PLTFRM_DVR_MOD,
'driver_class': PLTFRM_DVR_CLS})
platform_agent_id = self.imsclient.create_platform_agent(platform_agent_obj)
# device creation
self.platform_device_id = self.imsclient.create_platform_device(any_old(RT.PlatformDevice))
# data product creation
dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom})
dp_id = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=self.parsed_stream_def_id)
self.damsclient.assign_data_product(input_resource_id=self.platform_device_id, data_product_id=dp_id)
self.dpclient.activate_data_product_persistence(data_product_id=dp_id)
self.addCleanup(self.dpclient.delete_data_product, dp_id)
# assignments
self.RR2.assign_platform_agent_instance_to_platform_device_with_has_agent_instance(self.platform_agent_instance_id, self.platform_device_id)
self.RR2.assign_platform_agent_to_platform_agent_instance_with_has_agent_definition(platform_agent_id, self.platform_agent_instance_id)
self.RR2.assign_platform_device_to_org_with_has_resource(self.platform_agent_instance_id, self.org_id)
#######################################
# dataset
log.debug('data product = %s', dp_id)
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, None, True)
log.debug('Data product stream_ids = %s', stream_ids)
stream_id = stream_ids[0]
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasDataset, RT.Dataset, True)
log.debug('Data set for data_product_id1 = %s', dataset_ids[0])
#######################################
log.debug('_create_platform_site_and_deployment platform_device_id: %s', self.platform_device_id)
site_object = IonObject(RT.PlatformSite, name='PlatformSite1')
self.platform_site_id = self.omsclient.create_platform_site(platform_site=site_object, parent_id='')
log.debug('_create_platform_site_and_deployment site id: %s', self.platform_site_id)
#create supporting objects for the Deployment resource
# 1. temporal constraint
# find current deployment using time constraints
current_time = int( calendar.timegm(time.gmtime()) )
# two years on either side of current time
start = current_time - 63115200
end = current_time + 63115200
temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=str(start), end_datetime=str(end))
# 2. PlatformPort object which defines device to port map
platform_port_obj= IonObject(OT.PlatformPort, reference_designator = 'GA01SUMO-FI003-01-CTDMO0999',
port_type=PortTypeEnum.UPLINK,
ip_address='0')
# now create the Deployment
deployment_obj = IonObject(RT.Deployment,
name='TestPlatformDeployment',
description='some new deployment',
context=IonObject(OT.CabledNodeDeploymentContext),
constraint_list=[temporal_bounds],
port_assignments={self.platform_device_id:platform_port_obj})
platform_deployment_id = self.omsclient.create_deployment(deployment=deployment_obj, site_id=self.platform_site_id, device_id=self.platform_device_id)
log.debug('_create_platform_site_and_deployment deployment_id: %s', platform_deployment_id)
deploy_obj2 = self.omsclient.read_deployment(platform_deployment_id)
log.debug('_create_platform_site_and_deployment deploy_obj2 : %s', deploy_obj2)
return self.platform_site_id, platform_deployment_id
def _create_instrument_resources(self):
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='VEL3D',
description="VEL3D")
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
log.debug( 'new InstrumentModel id = %s ', instModel_id)
raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='raw' )
vel3d_b_sample = StreamConfiguration(stream_name='vel3d_b_sample', parameter_dictionary_name='vel3d_b_sample')
vel3d_b_engineering = StreamConfiguration(stream_name='vel3d_b_engineering', parameter_dictionary_name='vel3d_b_engineering')
# Create InstrumentAgent
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri="http://sddevrepo.oceanobservatories.org/releases/nobska_mavs4_ooicore-0.0.7-py2.7.egg",
stream_configurations = [raw_config, vel3d_b_sample, vel3d_b_engineering])
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
log.debug('new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
instDevice_obj = IonObject(RT.InstrumentDevice,
name='VEL3DDevice',
description="VEL3DDevice",
serial_number="12345" )
self.instrument_device_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, self.instrument_device_id)
port_agent_config = {
'device_addr': '10.180.80.6',
'device_port': 2101,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': 1025,
'data_port': 1026,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='VEL3DAgentInstance',
description="VEL3DAgentInstance",
port_agent_config = port_agent_config,
alerts= [])
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj,
instAgent_id,
self.instrument_device_id)
self._start_port_agent(self.imsclient.read_instrument_agent_instance(instAgentInstance_id))
vel3d_b_sample_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('vel3d_b_sample', id_only=True)
vel3d_b_sample_stream_def_id = self.pubsubclient.create_stream_definition(name='vel3d_b_sample', parameter_dictionary_id=vel3d_b_sample_pdict_id)
vel3d_b_engineering_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('vel3d_b_engineering', id_only=True)
vel3d_b_engineering_stream_def_id = self.pubsubclient.create_stream_definition(name='vel3d_b_engineering', parameter_dictionary_id=vel3d_b_engineering_pdict_id)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('raw', id_only=True)
raw_stream_def_id = self.pubsubclient.create_stream_definition(name='raw', parameter_dictionary_id=raw_pdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='vel3d_b_sample',
description='vel3d_b_sample',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id1 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=vel3d_b_sample_stream_def_id)
self.damsclient.assign_data_product(input_resource_id=self.instrument_device_id, data_product_id=data_product_id1)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1)
dp_obj = IonObject(RT.DataProduct,
name='vel3d_b_engineering',
description='vel3d_b_engineering',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id2 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=vel3d_b_engineering_stream_def_id)
self.damsclient.assign_data_product(input_resource_id=self.instrument_device_id, data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id3 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id)
self.damsclient.assign_data_product(input_resource_id=self.instrument_device_id, data_product_id=data_product_id3)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id3)
#create instrument site and associated deployment
site_object = IonObject(RT.InstrumentSite, name='InstrumentSite1')
instrument_site_id = self.omsclient.create_instrument_site(instrument_site=site_object, parent_id=self.platform_site_id)
log.debug('_create_instrument_site_and_deployment site id: %s', instrument_site_id)
#create supporting objects for the Deployment resource
# 1. temporal constraint
# find current deployment using time constraints
current_time = int( calendar.timegm(time.gmtime()) )
# two years on either side of current time
start = current_time - 63115200
end = current_time + 63115200
temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=str(start), end_datetime=str(end))
# 2. PlatformPort object which defines device to port map
platform_port_obj= IonObject(OT.PlatformPort, reference_designator = 'GA01SUMO-FI003-03-CTDMO0999',
port_type=PortTypeEnum.PAYLOAD,
ip_address='0')
# now create the Deployment
deployment_obj = IonObject(RT.Deployment,
name='TestInstrumentDeployment',
description='some new deployment',
context=IonObject(OT.CabledInstrumentDeploymentContext),
constraint_list=[temporal_bounds],
port_assignments={self.instrument_device_id:platform_port_obj})
instrument_deployment_id = self.omsclient.create_deployment(deployment=deployment_obj, site_id=instrument_site_id, device_id=self.instrument_device_id)
log.debug('_create_instrument_site_and_deployment deployment_id: %s', instrument_deployment_id)
def _start_port_agent(self, instrument_agent_instance_obj=None):
"""
Construct and start the port agent, ONLY NEEDED FOR INSTRUMENT AGENTS.
"""
_port_agent_config = instrument_agent_instance_obj.port_agent_config
# It blocks until the port agent starts up or a timeout
_pagent = PortAgentProcess.launch_process(_port_agent_config, test_mode = True)
pid = _pagent.get_pid()
port = _pagent.get_data_port()
cmd_port = _pagent.get_command_port()
log.info("IMS:_start_pagent returned from PortAgentProcess.launch_process pid: %s ", pid)
# Hack to get ready for DEMO. Further though needs to be put int
# how we pass this config info around.
host = 'localhost'
driver_config = instrument_agent_instance_obj.driver_config
comms_config = driver_config.get('comms_config')
if comms_config:
host = comms_config.get('addr')
else:
log.warn("No comms_config specified, using '%s'" % host)
# Configure driver to use port agent port number.
instrument_agent_instance_obj.driver_config['comms_config'] = {
'addr' : host,
'cmd_port' : cmd_port,
'port' : port
}
instrument_agent_instance_obj.driver_config['pagent_pid'] = pid
self.imsclient.update_instrument_agent_instance(instrument_agent_instance_obj)
return self.imsclient.read_instrument_agent_instance(instrument_agent_instance_obj._id)
def _start_platform(self):
"""
Starts the given platform waiting for it to transition to the
UNINITIALIZED state (note that the agent starts in the LAUNCHING state).
More in concrete the sequence of steps here are:
- prepares subscriber to receive the UNINITIALIZED state transition
- launches the platform process
- waits for the start of the process
- waits for the transition to the UNINITIALIZED state
"""
##############################################################
# prepare to receive the UNINITIALIZED state transition:
async_res = AsyncResult()
def consume_event(evt, *args, **kwargs):
log.debug("Got ResourceAgentStateEvent %s from origin %r", evt.state, evt.origin)
if evt.state == PlatformAgentState.UNINITIALIZED:
async_res.set(evt)
# start subscriber:
sub = EventSubscriber(event_type="ResourceAgentStateEvent",
origin=self.platform_device_id,
callback=consume_event)
sub.start()
log.info("registered event subscriber to wait for state=%r from origin %r",
PlatformAgentState.UNINITIALIZED, self.platform_device_id)
#self._event_subscribers.append(sub)
sub._ready_event.wait(timeout=EVENT_TIMEOUT)
##############################################################
# now start the platform:
agent_instance_id = self.platform_agent_instance_id
log.debug("about to call start_platform_agent_instance with id=%s", agent_instance_id)
pid = self.imsclient.start_platform_agent_instance(platform_agent_instance_id=agent_instance_id)
log.debug("start_platform_agent_instance returned pid=%s", pid)
#wait for start
agent_instance_obj = self.imsclient.read_platform_agent_instance(agent_instance_id)
gate = AgentProcessStateGate(self.processdispatchclient.read_process,
self.platform_device_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(90), "The platform agent instance did not spawn in 90 seconds")
# Start a resource agent client to talk with the agent.
self._pa_client = ResourceAgentClient(self.platform_device_id,
name=gate.process_id,
process=FakeProcess())
log.debug("got platform agent client %s", str(self._pa_client))
##############################################################
# wait for the UNINITIALIZED event:
async_res.get(timeout=self._receive_timeout)
def _register_oms_listener(self):
#load the paramaters and the param dicts necesssary for the VEL3D
log.debug( "---------- connect_to_oms ---------- ")
log.debug("oms_uri = %s", OMS_URI)
self.oms = CIOMSClientFactory.create_instance(OMS_URI)
#buddha url
url = "http://10.22.88.168:5000/ion-service/oms_event"
log.info("test_oms_events_receive:setup http url %s", url)
result = self.oms.event.register_event_listener(url)
log.debug("_register_oms_listener register_event_listener result %s", result)
#-------------------------------------------------------------------------------------
# Set up the subscriber to catch the alert event
#-------------------------------------------------------------------------------------
def callback_for_alert(event, *args, **kwargs):
log.debug("caught an OMSDeviceStatusEvent: %s", event)
self.catch_alert.put(event)
self.event_subscriber = EventSubscriber(event_type='OMSDeviceStatusEvent',
callback=callback_for_alert)
self.event_subscriber.start()
self.addCleanup(self.event_subscriber.stop)
result = self.oms.event.generate_test_event({'platform_id': 'fake_platform_id', 'message': "fake event triggered from CI using OMS' generate_test_event", 'severity': '3', 'group ': 'power'})
log.debug("_register_oms_listener generate_test_event result %s", result)
def _stop_platform(self):
try:
self.IMS.stop_platform_agent_instance(self.platform_agent_instance_id)
except Exception:
log.warn(
"platform_id=%r: Exception in IMS.stop_platform_agent_instance with "
"platform_agent_instance_id = %r. Perhaps already dead.",
self.platform_device_id, self.platform_agent_instance_id)
class TestDriverEgg(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dpclient = DataProductManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataretrieverclient = DataRetrieverServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
#setup listerner vars
self._data_greenlets = []
self._no_samples = None
self._samples_received = []
self.event_publisher = EventPublisher()
def get_datastore(self, dataset_id):
dataset = self.datasetclient.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(
datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def get_streamConfigs(self):
raw_config = StreamConfiguration(
stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict')
parsed_config = StreamConfiguration(
stream_name='parsed',
parameter_dictionary_name='ctd_parsed_param_dict')
return raw_config, parsed_config
##########################
#
# The following tests generate different agent configs and pass them to a common base test script
#
###########################
@unittest.skip(
"this test can't be run from coi services. it is missing dependencies")
def test_driverLaunchModuleNoURI(self):
raw_config, parsed_config = self.get_streamConfigs()
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
driver_class="SBE37Driver",
stream_configurations=[raw_config, parsed_config])
self.base_activateInstrumentSample(instAgent_obj)
def test_driverLaunchModuleWithURI(self):
raw_config, parsed_config = self.get_streamConfigs()
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
driver_class="SBE37Driver",
driver_uri=DRV_URI_GOOD,
stream_configurations=[raw_config, parsed_config])
self.base_activateInstrumentSample(instAgent_obj)
def test_driverLaunchNoModuleOnlyURI(self):
raw_config, parsed_config = self.get_streamConfigs()
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
#driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
#driver_class="SBE37Driver",
driver_uri=DRV_URI_GOOD,
stream_configurations=[raw_config, parsed_config])
self.base_activateInstrumentSample(instAgent_obj)
def test_driverLaunchBogusModuleWithURI(self):
raw_config, parsed_config = self.get_streamConfigs()
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_module="bogus",
driver_class="Bogus",
driver_uri=DRV_URI_GOOD,
stream_configurations=[raw_config, parsed_config])
self.base_activateInstrumentSample(instAgent_obj)
@unittest.skip(
"Launches an egg 'process' even though the egg download should produce error 404"
)
def test_driverLaunchNoModule404URI(self):
raw_config, parsed_config = self.get_streamConfigs()
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
#driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
#driver_class="SBE37Driver",
driver_uri=DRV_URI_404,
stream_configurations=[raw_config, parsed_config])
self.base_activateInstrumentSample(instAgent_obj, False)
def test_driverLaunchNoModuleBadEggURI(self):
raw_config, parsed_config = self.get_streamConfigs()
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
#driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
#driver_class="SBE37Driver",
driver_uri=DRV_URI_BAD,
stream_configurations=[raw_config, parsed_config])
self.base_activateInstrumentSample(instAgent_obj, True, False)
def base_activateInstrumentSample(self,
instAgent_obj,
expect_launch=True,
expect_command=True):
"""
This method runs a test of launching a driver with a given agent configuration
"""
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
print 'new InstrumentModel id = %s ' % instModel_id
# Create InstrumentAgent
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
print 'new InstrumentAgent id = %s' % instAgent_id
self.imsclient.assign_instrument_model_to_instrument_agent(
instModel_id, instAgent_id)
# Create InstrumentDevice
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345")
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, instDevice_id)
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance,
name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config=port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_raw_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubcli.create_stream_definition(
name='parsed', parameter_dictionary_id=parsed_pdict_id)
raw_stream_def_id = self.pubsubcli.create_stream_definition(
name='raw', parameter_dictionary_id=raw_pdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id1 = self.dpclient.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
print 'new dp_id = %s' % data_product_id1
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id1)
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1,
PRED.hasStream, None, True)
print 'Data product streams1 = %s' % stream_ids
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id1,
PRED.hasDataset,
RT.Dataset, True)
print 'Data set for data_product_id1 = %s' % dataset_ids[0]
self.parsed_dataset = dataset_ids[0]
#create the datastore at the beginning of each int test that persists data
self.get_datastore(self.parsed_dataset)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id2 = self.dpclient.create_data_product(
data_product=dp_obj, stream_definition_id=raw_stream_def_id)
print 'new dp_id = %s' % str(data_product_id2)
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2,
PRED.hasStream, None, True)
print 'Data product streams2 = %s' % str(stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id2,
PRED.hasDataset,
RT.Dataset, True)
print 'Data set for data_product_id2 = %s' % dataset_ids[0]
self.raw_dataset = dataset_ids[0]
# add start/stop for instrument agent
gevent.joinall([
gevent.spawn(
lambda: self.imsclient.start_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id))
])
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
#wait for start
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(
instAgentInstance_id)
agent_process_id = ResourceAgentClient._get_agent_process_id(
instDevice_id)
print "Agent process id is '%s'" % str(agent_process_id)
self.assertTrue(agent_process_id)
gate = ProcessStateGate(self.processdispatchclient.read_process,
agent_process_id, ProcessStateEnum.RUNNING)
if not expect_launch:
self.assertFalse(
gate. await (30),
"The instance (%s) of bogus instrument agent spawned in 30 seconds ?!?"
% agent_process_id)
return
self.assertTrue(
gate. await (30),
"The instrument agent instance (%s) did not spawn in 30 seconds" %
agent_process_id)
print "Instrument Agent Instance successfully triggered ProcessStateGate as RUNNING"
#print 'Instrument agent instance obj: = %s' % str(inst_agent_instance_obj)
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(instDevice_id,
to_name=agent_process_id,
process=FakeProcess())
print "ResourceAgentClient created: %s" % str(self._ia_client)
print "Sending command=ResourceAgentEvent.INITIALIZE"
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
if not expect_command:
self.assertRaises(ServerError, self._ia_client.execute_agent, cmd)
return
retval = self._ia_client.execute_agent(cmd)
print "Result of INITIALIZE: %s" % str(retval)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertTrue(state, 'DRIVER_STATE_COMMAND')
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE)
retval = self._ia_client.execute_resource(cmd)
# This gevent sleep is there to test the autosample time, which will show something different from default
# only if the instrument runs for over a minute
gevent.sleep(90)
extended_instrument = self.imsclient.get_instrument_device_extension(
instrument_device_id=instDevice_id)
self.assertIsInstance(extended_instrument.computed.uptime,
ComputedStringValue)
autosample_string = extended_instrument.computed.uptime.value
autosampling_time = int(autosample_string.split()[4])
self.assertTrue(autosampling_time > 0)
cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE)
retval = self._ia_client.execute_resource(cmd)
print "Sending command=ResourceAgentEvent.RESET"
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client.execute_agent(cmd)
print "Result of RESET: %s" % str(reply)
class ExhaustiveParameterTest(IonIntegrationTestCase):
def setUp(self):
self.i=0
self._start_container()
self.container.start_rel_from_url('res/deploy/r2params.yml')
self.dataset_management = DatasetManagementServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
self.resource_registry = self.container.resource_registry
self.data_retriever = DataRetrieverServiceClient()
pdicts, _ = self.resource_registry.find_resources(restype='ParameterDictionary', id_only=False)
self.dp_ids = []
for pdict in pdicts:
stream_def_id = self.pubsub_management.create_stream_definition(pdict.name, parameter_dictionary_id=pdict._id)
dp_id = self.make_dp(stream_def_id)
if dp_id: self.dp_ids.append(dp_id)
def make_dp(self, stream_def_id):
stream_def = self.resource_registry.read(stream_def_id)
dp_obj = DataProduct(
name=stream_def.name,
description=stream_def.name,
processing_level_code='Parsed_Canonical')
data_product_id = self.data_product_management.create_data_product(dp_obj, stream_definition_id=stream_def_id)
self.data_product_management.activate_data_product_persistence(data_product_id)
return data_product_id
def fill_values(self, ptype, size):
if isinstance(ptype, ArrayType):
return ['blah'] * size
elif isinstance(ptype, QuantityType):
return np.sin(np.arange(size, dtype=ptype.value_encoding) * 2 * np.pi / 3)
elif isinstance(ptype, RecordType):
return [{'record': 'ok'}] * size
elif isinstance(ptype, ConstantRangeType):
return (1,1000)
elif isinstance(ptype, ConstantType):
return np.dtype(ptype.value_encoding).type(1)
elif isinstance(ptype, CategoryType):
return ptype.categories.keys()[0]
else:
return
def wait_until_we_have_enough_granules(self, dataset_id='',data_size=40):
'''
Loops until there is a sufficient amount of data in the dataset
'''
done = False
with gevent.Timeout(40):
while not done:
granule = self.data_retriever.retrieve_last_data_points(dataset_id, 1)
rdt = RecordDictionaryTool.load_from_granule(granule)
extents = self.dataset_management.dataset_extents(dataset_id, rdt._pdict.temporal_parameter_name)[0]
if rdt[rdt._pdict.temporal_parameter_name] and rdt[rdt._pdict.temporal_parameter_name][0] != rdt._pdict.get_context(rdt._pdict.temporal_parameter_name).fill_value and extents >= data_size:
done = True
else:
gevent.sleep(0.2)
def write_to_data_product(self,data_product_id):
dataset_ids, _ = self.resource_registry.find_objects(data_product_id, 'hasDataset', id_only=True)
dataset_id = dataset_ids.pop()
stream_ids , _ = self.resource_registry.find_objects(data_product_id, 'hasStream', id_only=True)
stream_id = stream_ids.pop()
stream_def_ids, _ = self.resource_registry.find_objects(stream_id, 'hasStreamDefinition', id_only=True)
stream_def_id = stream_def_ids.pop()
route = self.pubsub_management.read_stream_route(stream_id)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
time_param = rdt._pdict.temporal_parameter_name
if time_param is None:
print '%s has no temporal parameter' % self.resource_registry.read(data_product_id).name
return
rdt[time_param] = np.arange(40)
for field in rdt.fields:
if field == rdt._pdict.temporal_parameter_name:
continue
rdt[field] = self.fill_values(rdt._pdict.get_context(field).param_type,40)
publisher = StandaloneStreamPublisher(stream_id, route)
publisher.publish(rdt.to_granule())
self.wait_until_we_have_enough_granules(dataset_id,40)
granule = self.data_retriever.retrieve(dataset_id)
rdt_out = RecordDictionaryTool.load_from_granule(granule)
bad = []
for field in rdt.fields:
if not np.array_equal(rdt[field], rdt_out[field]):
print '%s' % field
print '%s != %s' % (rdt[field], rdt_out[field])
bad.append(field)
return bad
def test_data_products(self):
bad_data_products = {}
for dp_id in self.dp_ids:
try:
bad_fields = self.write_to_data_product(dp_id)
if bad_fields:
bad_data_products[dp_id] = "Couldn't write and retrieve %s." % bad_fields
except:
import traceback
bad_data_products[dp_id] = traceback.format_exc()
for dp_id, tb in bad_data_products.iteritems():
print '----------'
print 'Problem with %s' % self.resource_registry.read(dp_id).name
print tb
print '----------'
if bad_data_products:
raise AssertionError('There are bad parameter dictionaries.')
class BulkIngestBase(object):
"""
awkward, non-obvious test class! subclasses will implement data-specific methods and
this test class will parse sample file and assert data was read.
test_data_ingest: create resources and call...
start_agent: starts agent and then call...
start_listener: starts listeners for data, including one that when granule is received calls...
get_retrieve_client: asserts that callback had some data
See replacement TestPreloadThenLoadDataset. A little more declarative and straight-forward, but much slower (requires preload).
"""
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.pubsub_management = PubsubManagementServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
self.data_acquisition_management = DataAcquisitionManagementServiceClient()
self.data_retriever = DataRetrieverServiceClient()
self.process_dispatch_client = ProcessDispatcherServiceClient(node=self.container.node)
self.resource_registry = self.container.resource_registry
self.context_ids = self.build_param_contexts()
self.setup_resources()
def build_param_contexts(self):
raise NotImplementedError('build_param_contexts must be implemented in child classes')
def create_external_dataset(self):
raise NotImplementedError('create_external_dataset must be implemented in child classes')
def get_dvr_config(self):
raise NotImplementedError('get_dvr_config must be implemented in child classes')
def get_retrieve_client(self, dataset_id=''):
raise NotImplementedError('get_retrieve_client must be implemented in child classes')
def test_data_ingest(self):
self.pdict_id = self.create_parameter_dict(self.name)
self.stream_def_id = self.create_stream_def(self.name, self.pdict_id)
self.data_product_id = self.create_data_product(self.name, self.description, self.stream_def_id)
self.dataset_id = self.get_dataset_id(self.data_product_id)
self.stream_id, self.route = self.get_stream_id_and_route(self.data_product_id)
self.external_dataset_id = self.create_external_dataset()
self.data_producer_id = self.register_external_dataset(self.external_dataset_id)
self.start_agent()
def create_parameter_dict(self, name=''):
return self.dataset_management.create_parameter_dictionary(name=name, parameter_context_ids=self.context_ids, temporal_context='time')
def create_stream_def(self, name='', pdict_id=''):
return self.pubsub_management.create_stream_definition(name=name, parameter_dictionary_id=pdict_id)
def create_data_product(self, name='', description='', stream_def_id=''):
tdom, sdom = time_series_domain()
tdom = tdom.dump()
sdom = sdom.dump()
dp_obj = DataProduct(
name=name,
description=description,
processing_level_code='Parsed_Canonical',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id = self.data_product_management.create_data_product(data_product=dp_obj, stream_definition_id=stream_def_id)
self.data_product_management.activate_data_product_persistence(data_product_id)
self.addCleanup(self.data_product_management.suspend_data_product_persistence, data_product_id)
return data_product_id
def register_external_dataset(self, external_dataset_id=''):
return self.data_acquisition_management.register_external_data_set(external_dataset_id=external_dataset_id)
def get_dataset_id(self, data_product_id=''):
dataset_ids, assocs = self.resource_registry.find_objects(subject=data_product_id, predicate='hasDataset', id_only=True)
return dataset_ids[0]
def get_stream_id_and_route(self, data_product_id):
stream_ids, _ = self.resource_registry.find_objects(data_product_id, PRED.hasStream, RT.Stream, id_only=True)
stream_id = stream_ids[0]
route = self.pubsub_management.read_stream_route(stream_id)
#self.create_logger(self.name, stream_id)
return stream_id, route
def start_agent(self):
agent_config = {
'driver_config': self.get_dvr_config(),
'stream_config': {},
'agent': {'resource_id': self.external_dataset_id},
'test_mode': True
}
self._ia_pid = self.container.spawn_process(
name=self.EDA_NAME,
module=self.EDA_MOD,
cls=self.EDA_CLS,
config=agent_config)
self._ia_client = ResourceAgentClient(self.external_dataset_id, process=FakeProcess())
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command=DriverEvent.START_AUTOSAMPLE)
self._ia_client.execute_resource(command=cmd)
self.start_listener(self.dataset_id)
def stop_agent(self):
cmd = AgentCommand(command=DriverEvent.STOP_AUTOSAMPLE)
self._ia_client.execute_resource(cmd)
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
self._ia_client.execute_agent(cmd)
self.container.terminate_process(self._ia_pid)
def start_listener(self, dataset_id=''):
dataset_modified = Event()
#callback to use retrieve to get data from the coverage
def cb(*args, **kwargs):
self.get_retrieve_client(dataset_id=dataset_id)
#callback to keep execution going once dataset has been fully ingested
def cb2(*args, **kwargs):
dataset_modified.set()
es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id)
es.start()
es2 = EventSubscriber(event_type=OT.DeviceCommonLifecycleEvent, callback=cb2, origin='BaseDataHandler._acquire_sample')
es2.start()
self.addCleanup(es.stop)
self.addCleanup(es2.stop)
#let it go for up to 120 seconds, then stop the agent and reset it
dataset_modified.wait(120)
self.stop_agent()
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name+'_logger')
producer_definition.executable = {
'module':'ion.processes.data.stream_granule_logger',
'class':'StreamGranuleLogger'
}
logger_procdef_id = self.process_dispatch_client.create_process_definition(process_definition=producer_definition)
configuration = {
'process':{
'stream_id':stream_id,
}
}
pid = self.process_dispatch_client.schedule_process(process_definition_id=logger_procdef_id, configuration=configuration)
return pid
class TestInstrumentManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
# @unittest.skip('this test just for debugging setup')
# def test_just_the_setup(self):
# return
@attr('EXT')
def test_resources_associations_extensions(self):
"""
create one of each resource and association used by IMS
to guard against problems in ion-definitions
"""
#stuff we control
instrument_agent_instance_id, _ = self.RR.create(any_old(RT.InstrumentAgentInstance))
instrument_agent_id, _ = self.RR.create(any_old(RT.InstrumentAgent))
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel))
instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice))
platform_agent_instance_id, _ = self.RR.create(any_old(RT.PlatformAgentInstance))
platform_agent_id, _ = self.RR.create(any_old(RT.PlatformAgent))
platform_device_id, _ = self.RR.create(any_old(RT.PlatformDevice))
platform_model_id, _ = self.RR.create(any_old(RT.PlatformModel))
sensor_device_id, _ = self.RR.create(any_old(RT.SensorDevice))
sensor_model_id, _ = self.RR.create(any_old(RT.SensorModel))
#stuff we associate to
data_producer_id, _ = self.RR.create(any_old(RT.DataProducer))
org_id, _ = self.RR.create(any_old(RT.Org))
#instrument_agent_instance_id #is only a target
#instrument_agent
self.RR.create_association(instrument_agent_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(instrument_agent_instance_id, PRED.hasAgentDefinition, instrument_agent_id)
#instrument_device
self.RR.create_association(instrument_device_id, PRED.hasModel, instrument_model_id)
self.RR.create_association(instrument_device_id, PRED.hasAgentInstance, instrument_agent_instance_id)
self.RR.create_association(instrument_device_id, PRED.hasDataProducer, data_producer_id)
self.RR.create_association(instrument_device_id, PRED.hasDevice, sensor_device_id)
self.RR.create_association(org_id, PRED.hasResource, instrument_device_id)
instrument_model_id #is only a target
platform_agent_instance_id #is only a target
#platform_agent
self.RR.create_association(platform_agent_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_agent_instance_id, PRED.hasAgentDefinition, platform_agent_id)
#platform_device
self.RR.create_association(platform_device_id, PRED.hasModel, platform_model_id)
self.RR.create_association(platform_device_id, PRED.hasAgentInstance, platform_agent_instance_id)
self.RR.create_association(platform_device_id, PRED.hasDevice, instrument_device_id)
platform_model_id #is only a target
#sensor_device
self.RR.create_association(sensor_device_id, PRED.hasModel, sensor_model_id)
self.RR.create_association(sensor_device_id, PRED.hasDevice, instrument_device_id)
sensor_model_id #is only a target
#create a parsed product for this instrument output
tdom, sdom = time_series_domain()
tdom = tdom.dump()
sdom = sdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
processing_level_code='Parsed_Canonical',
temporal_domain = tdom,
spatial_domain = sdom)
pdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id)
data_product_id1 = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=data_product_id1)
def addInstOwner(inst_id, subject):
actor_identity_obj = any_old(RT.ActorIdentity, {"name": subject})
user_id = self.IDS.create_actor_identity(actor_identity_obj)
user_info_obj = any_old(RT.UserInfo)
user_info_id = self.IDS.create_user_info(user_id, user_info_obj)
self.RR.create_association(inst_id, PRED.hasOwner, user_id)
#Testing multiple instrument owners
addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254")
addInstOwner(instrument_device_id, "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Bob Cumbers A256")
extended_instrument = self.IMS.get_instrument_device_extension(instrument_device_id)
self.assertEqual(instrument_device_id, extended_instrument._id)
self.assertEqual(len(extended_instrument.owners), 2)
self.assertEqual(extended_instrument.instrument_model._id, instrument_model_id)
# Lifecycle
self.assertEquals(len(extended_instrument.lcstate_transitions), 7)
self.assertEquals(set(extended_instrument.lcstate_transitions.keys()), set(['enable', 'develop', 'deploy', 'retire', 'plan', 'integrate', 'announce']))
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(extended_instrument.computed.firmware_version, ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.last_data_received_datetime, ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.last_calibration_datetime, ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue)
self.assertIsInstance(extended_instrument.computed.power_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.communications_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.location_status_roll_up, ComputedIntValue)
log.debug("extended_instrument.computed: %s", extended_instrument.computed)
#check model
inst_model_obj = self.RR.read(instrument_model_id)
self.assertEqual(inst_model_obj.name, extended_instrument.instrument_model.name)
#check agent instance
inst_agent_instance_obj = self.RR.read(instrument_agent_instance_id)
self.assertEqual(inst_agent_instance_obj.name, extended_instrument.agent_instance.name)
#check agent
inst_agent_obj = self.RR.read(instrument_agent_id)
#compound assoc return list of lists so check the first element
self.assertEqual(inst_agent_obj.name, extended_instrument.instrument_agent.name)
#check platform device
plat_device_obj = self.RR.read(platform_device_id)
self.assertEqual(plat_device_obj.name, extended_instrument.platform_device.name)
extended_platform = self.IMS.get_platform_device_extension(platform_device_id)
self.assertEqual(1, len(extended_platform.instrument_devices))
self.assertEqual(instrument_device_id, extended_platform.instrument_devices[0]._id)
self.assertEqual(1, len(extended_platform.instrument_models))
self.assertEqual(instrument_model_id, extended_platform.instrument_models[0]._id)
self.assertEquals(extended_platform.platform_agent._id, platform_agent_id)
self.assertEquals(len(extended_platform.lcstate_transitions), 7)
self.assertEquals(set(extended_platform.lcstate_transitions.keys()), set(['enable', 'develop', 'deploy', 'retire', 'plan', 'integrate', 'announce']))
#check sensor devices
self.assertEqual(1, len(extended_instrument.sensor_devices))
#check data_product_parameters_set
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_instrument.computed.data_product_parameters_set.status)
self.assertTrue( 'Parsed_Canonical' in extended_instrument.computed.data_product_parameters_set.value)
# the ctd parameters should include 'temp'
self.assertTrue( 'temp' in extended_instrument.computed.data_product_parameters_set.value['Parsed_Canonical'])
#none of these will work because there is no agent
self.assertEqual(ComputedValueAvailability.NOTAVAILABLE,
extended_instrument.computed.firmware_version.status)
# self.assertEqual(ComputedValueAvailability.NOTAVAILABLE,
# extended_instrument.computed.operational_state.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.power_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.communications_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.data_status_roll_up.status)
# self.assertEqual(StatusType.STATUS_OK,
# extended_instrument.computed.data_status_roll_up.value)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.location_status_roll_up.status)
# self.assertEqual(ComputedValueAvailability.PROVIDED,
# extended_instrument.computed.recent_events.status)
# self.assertEqual([], extended_instrument.computed.recent_events.value)
# cleanup
c = DotDict()
c.resource_registry = self.RR
self.RR2.pluck(instrument_agent_id)
self.RR2.pluck(instrument_model_id)
self.RR2.pluck(instrument_device_id)
self.RR2.pluck(platform_agent_id)
self.RR2.pluck(sensor_device_id)
self.IMS.force_delete_instrument_agent(instrument_agent_id)
self.IMS.force_delete_instrument_model(instrument_model_id)
self.IMS.force_delete_instrument_device(instrument_device_id)
self.IMS.force_delete_platform_agent_instance(platform_agent_instance_id)
self.IMS.force_delete_platform_agent(platform_agent_id)
self.IMS.force_delete_platform_device(platform_device_id)
self.IMS.force_delete_platform_model(platform_model_id)
self.IMS.force_delete_sensor_device(sensor_device_id)
self.IMS.force_delete_sensor_model(sensor_model_id)
#stuff we associate to
self.RR.delete(data_producer_id)
self.RR.delete(org_id)
def test_custom_attributes(self):
"""
Test assignment of custom attributes
"""
instrument_model_id, _ = self.RR.create(any_old(RT.InstrumentModel,
{"custom_attributes":
{"favorite_color": "attr desc goes here"}
}))
instrument_device_id, _ = self.RR.create(any_old(RT.InstrumentDevice,
{"custom_attributes":
{"favorite_color": "red",
"bogus_attr": "should raise warning"
}
}))
self.IMS.assign_instrument_model_to_instrument_device(instrument_model_id, instrument_device_id)
# cleanup
self.IMS.force_delete_instrument_device(instrument_device_id)
self.IMS.force_delete_instrument_model(instrument_model_id)
def _get_datastore(self, dataset_id):
dataset = self.DSC.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def test_resource_state_save_restore(self):
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.IMS.create_instrument_model(instModel_obj)
log.debug( 'new InstrumentModel id = %s ', instModel_id)
# Create InstrumentAgent
raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 )
parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5 )
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg",
stream_configurations = [raw_config, parsed_config] )
instAgent_id = self.IMS.create_instrument_agent(instAgent_obj)
log.debug( 'new InstrumentAgent id = %s', instAgent_id)
self.IMS.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 '
+ '(SA Req: L4-CI-SA-RQ-241) ')
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345" )
instDevice_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj)
self.IMS.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ",
instDevice_id)
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config = port_agent_config)
instAgentInstance_id = self.IMS.create_instrument_agent_instance(instAgentInstance_obj,
instAgent_id,
instDevice_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
spdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=spdict_id)
rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id1 = self.DP.create_data_product(data_product=dp_obj,
stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1)
self.DP.activate_data_product_persistence(data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True)
log.debug( 'Data product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True)
log.debug( 'Data set for data_product_id1 = %s', dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
#create the datastore at the beginning of each int test that persists data
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id2 = self.DP.create_data_product(data_product=dp_obj,
stream_definition_id=raw_stream_def_id)
log.debug( 'new dp_id = %s', str(data_product_id2))
self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2)
self.DP.activate_data_product_persistence(data_product_id=data_product_id2)
# spin up agent
self.IMS.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
self.addCleanup(self.IMS.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
#wait for start
instance_obj = self.IMS.read_instrument_agent_instance(instAgentInstance_id)
gate = ProcessStateGate(self.PDC.read_process,
instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" %
instance_obj.agent_process_id)
# take snapshot of config
snap_id = self.IMS.save_resource_state(instDevice_id, "xyzzy snapshot")
snap_obj = self.RR.read_attachment(snap_id, include_content=True)
print "Saved config:"
print snap_obj.content
#modify config
instance_obj.driver_config["comms_config"] = "BAD_DATA"
self.RR.update(instance_obj)
#restore config
self.IMS.restore_resource_state(instDevice_id, snap_id)
instance_obj = self.RR.read(instAgentInstance_id)
self.assertNotEqual("BAD_DATA", instance_obj.driver_config["comms_config"])
self.DP.delete_data_product(data_product_id1)
self.DP.delete_data_product(data_product_id2)
def test_agent_instance_config(self):
"""
Verify that agent configurations are being built properly
"""
clients = DotDict()
clients.resource_registry = self.RR
clients.pubsub_management = self.PSC
clients.dataset_management = self.DSC
pconfig_builder = PlatformAgentConfigurationBuilder(clients)
iconfig_builder = InstrumentAgentConfigurationBuilder(clients)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
org_obj = any_old(RT.Org)
org_id = self.RR2.create(org_obj)
inst_startup_config = {'startup': 'config'}
generic_alerts_config = {'lvl1': {'lvl2': 'lvl3val'}}
required_config_keys = [
'org_name',
'device_type',
'agent',
'driver_config',
'stream_config',
'startup_config',
'aparam_alert_config',
'children']
def verify_instrument_config(config, device_id):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual(org_obj.name, config['org_name'])
self.assertEqual(RT.InstrumentDevice, config['device_type'])
self.assertIn('driver_config', config)
driver_config = config['driver_config']
expected_driver_fields = {'process_type': ('ZMQPyClassDriverLauncher',),
}
for k, v in expected_driver_fields.iteritems():
self.assertIn(k, driver_config)
self.assertEqual(v, driver_config[k])
self.assertEqual
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertEqual(inst_startup_config, config['startup_config'])
self.assertIn('aparam_alert_config', config)
self.assertEqual(generic_alerts_config, config['aparam_alert_config'])
self.assertIn('stream_config', config)
for key in ['children']:
self.assertEqual({}, config[key])
def verify_child_config(config, device_id, inst_device_id=None):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual(org_obj.name, config['org_name'])
self.assertEqual(RT.PlatformDevice, config['device_type'])
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertIn('aparam_alert_config', config)
self.assertEqual(generic_alerts_config, config['aparam_alert_config'])
self.assertIn('stream_config', config)
self.assertIn('driver_config', config)
self.assertIn('foo', config['driver_config'])
self.assertEqual('bar', config['driver_config']['foo'])
self.assertIn('process_type', config['driver_config'])
self.assertEqual(('ZMQPyClassDriverLauncher',), config['driver_config']['process_type'])
if None is inst_device_id:
for key in ['children', 'startup_config']:
self.assertEqual({}, config[key])
else:
for key in ['startup_config']:
self.assertEqual({}, config[key])
self.assertIn(inst_device_id, config['children'])
verify_instrument_config(config['children'][inst_device_id], inst_device_id)
def verify_parent_config(config, parent_device_id, child_device_id, inst_device_id=None):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual(org_obj.name, config['org_name'])
self.assertEqual(RT.PlatformDevice, config['device_type'])
self.assertIn('process_type', config['driver_config'])
self.assertEqual(('ZMQPyClassDriverLauncher',), config['driver_config']['process_type'])
self.assertEqual({'resource_id': parent_device_id}, config['agent'])
self.assertIn('aparam_alert_config', config)
self.assertEqual(generic_alerts_config, config['aparam_alert_config'])
self.assertIn('stream_config', config)
for key in ['startup_config']:
self.assertEqual({}, config[key])
self.assertIn(child_device_id, config['children'])
verify_child_config(config['children'][child_device_id], child_device_id, inst_device_id)
rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id)
#todo: create org and figure out which agent resource needs to get assigned to it
def _make_platform_agent_structure(agent_config=None):
if None is agent_config: agent_config = {}
# instance creation
platform_agent_instance_obj = any_old(RT.PlatformAgentInstance, {'driver_config': {'foo': 'bar'},
'alerts': generic_alerts_config})
platform_agent_instance_obj.agent_config = agent_config
platform_agent_instance_id = self.IMS.create_platform_agent_instance(platform_agent_instance_obj)
# agent creation
raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict', records_per_granule=2, granule_publish_rate=5 )
platform_agent_obj = any_old(RT.PlatformAgent, {"stream_configurations":[raw_config]})
platform_agent_id = self.IMS.create_platform_agent(platform_agent_obj)
# device creation
platform_device_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice))
# data product creation
dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom})
dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id)
self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
# assignments
self.RR2.assign_platform_agent_instance_to_platform_device(platform_agent_instance_id, platform_device_id)
self.RR2.assign_platform_agent_to_platform_agent_instance(platform_agent_id, platform_agent_instance_id)
self.RR2.assign_platform_device_to_org_with_has_resource(platform_agent_instance_id, org_id)
return platform_agent_instance_id, platform_agent_id, platform_device_id
def _make_instrument_agent_structure(agent_config=None):
if None is agent_config: agent_config = {}
# instance creation
instrument_agent_instance_obj = any_old(RT.InstrumentAgentInstance, {"startup_config": inst_startup_config,
'alerts': generic_alerts_config})
instrument_agent_instance_obj.agent_config = agent_config
instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(instrument_agent_instance_obj)
# agent creation
raw_config = StreamConfiguration(stream_name='raw',
parameter_dictionary_name='ctd_raw_param_dict',
records_per_granule=2,
granule_publish_rate=5 )
instrument_agent_obj = any_old(RT.InstrumentAgent, {"stream_configurations":[raw_config]})
instrument_agent_id = self.IMS.create_instrument_agent(instrument_agent_obj)
# device creation
instrument_device_id = self.IMS.create_instrument_device(any_old(RT.InstrumentDevice))
# data product creation
dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom})
dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
# assignments
self.RR2.assign_instrument_agent_instance_to_instrument_device(instrument_agent_instance_id, instrument_device_id)
self.RR2.assign_instrument_agent_to_instrument_agent_instance(instrument_agent_id, instrument_agent_instance_id)
self.RR2.assign_instrument_device_to_org_with_has_resource(instrument_agent_instance_id, org_id)
return instrument_agent_instance_id, instrument_agent_id, instrument_device_id
# can't do anything without an agent instance obj
log.debug("Testing that preparing a launcher without agent instance raises an error")
self.assertRaises(AssertionError, pconfig_builder.prepare, will_launch=False)
log.debug("Making the structure for a platform agent, which will be the child")
platform_agent_instance_child_id, _, platform_device_child_id = _make_platform_agent_structure()
platform_agent_instance_child_obj = self.RR2.read(platform_agent_instance_child_id)
log.debug("Preparing a valid agent instance launch, for config only")
pconfig_builder.set_agent_instance_object(platform_agent_instance_child_obj)
child_config = pconfig_builder.prepare(will_launch=False)
verify_child_config(child_config, platform_device_child_id)
log.debug("Making the structure for a platform agent, which will be the parent")
platform_agent_instance_parent_id, _, platform_device_parent_id = _make_platform_agent_structure()
platform_agent_instance_parent_obj = self.RR2.read(platform_agent_instance_parent_id)
log.debug("Testing child-less parent as a child config")
pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj)
parent_config = pconfig_builder.prepare(will_launch=False)
verify_child_config(parent_config, platform_device_parent_id)
log.debug("assigning child platform to parent")
self.RR2.assign_platform_device_to_platform_device(platform_device_child_id, platform_device_parent_id)
child_device_ids = self.RR2.find_platform_device_ids_of_device(platform_device_parent_id)
self.assertNotEqual(0, len(child_device_ids))
log.debug("Testing parent + child as parent config")
pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj)
parent_config = pconfig_builder.prepare(will_launch=False)
verify_parent_config(parent_config, platform_device_parent_id, platform_device_child_id)
log.debug("making the structure for an instrument agent")
instrument_agent_instance_id, _, instrument_device_id = _make_instrument_agent_structure()
instrument_agent_instance_obj = self.RR2.read(instrument_agent_instance_id)
log.debug("Testing instrument config")
iconfig_builder.set_agent_instance_object(instrument_agent_instance_obj)
instrument_config = iconfig_builder.prepare(will_launch=False)
verify_instrument_config(instrument_config, instrument_device_id)
log.debug("assigning instrument to platform")
self.RR2.assign_instrument_device_to_platform_device(instrument_device_id, platform_device_child_id)
child_device_ids = self.RR2.find_instrument_device_ids_of_device(platform_device_child_id)
self.assertNotEqual(0, len(child_device_ids))
log.debug("Testing entire config")
pconfig_builder.set_agent_instance_object(platform_agent_instance_parent_obj)
full_config = pconfig_builder.prepare(will_launch=False)
verify_parent_config(full_config, platform_device_parent_id, platform_device_child_id, instrument_device_id)
class TestBulkIngest(IonIntegrationTestCase):
EDA_MOD = 'ion.agents.data.external_dataset_agent'
EDA_CLS = 'ExternalDatasetAgent'
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataAcquisitionManagementService
self.client = DataAcquisitionManagementServiceClient(node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dams_client = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsub_client = PubsubManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.data_retriever = DataRetrieverServiceClient(node=self.container.node)
self._container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
# Data async and subscription TODO: Replace with new subscriber
self._finished_count = None
#TODO: Switch to gevent.queue.Queue
self._async_finished_result = AsyncResult()
self._finished_events_received = []
self._finished_event_subscriber = None
self._start_finished_event_subscriber()
self.addCleanup(self._stop_finished_event_subscriber)
self.DVR_CONFIG = {}
self.DVR_CONFIG = {
'dvr_mod' : 'ion.agents.data.handlers.slocum_data_handler',
'dvr_cls' : 'SlocumDataHandler',
}
self._setup_resources()
self.agent_config = {
'driver_config' : self.DVR_CONFIG,
'stream_config' : {},
'agent' : {'resource_id': self.EDA_RESOURCE_ID},
'test_mode' : True
}
datasetagent_instance_obj = IonObject(RT.ExternalDatasetAgentInstance, name='ExternalDatasetAgentInstance1', description='external data agent instance',
handler_module=self.EDA_MOD, handler_class=self.EDA_CLS,
dataset_driver_config=self.DVR_CONFIG, dataset_agent_config=self.agent_config )
self.dataset_agent_instance_id = self.dams_client.create_external_dataset_agent_instance(external_dataset_agent_instance=datasetagent_instance_obj,
external_dataset_agent_id=self.datasetagent_id, external_dataset_id=self.EDA_RESOURCE_ID)
#TG: Setup/configure the granule logger to log granules as they're published
pid = self.dams_client.start_external_dataset_agent_instance(self.dataset_agent_instance_id)
dataset_agent_instance_obj= self.dams_client.read_external_dataset_agent_instance(self.dataset_agent_instance_id)
print 'TestBulkIngest: Dataset agent instance obj: = ', dataset_agent_instance_obj
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient('datasetagentclient', name=pid, process=FakeProcess())
log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client))
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name+'_logger')
producer_definition.executable = {
'module':'ion.processes.data.stream_granule_logger',
'class':'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition)
configuration = {
'process':{
'stream_id':stream_id,
}
}
pid = self.processdispatchclient.schedule_process(process_definition_id= logger_procdef_id, configuration=configuration)
return pid
def _start_finished_event_subscriber(self):
def consume_event(*args,**kwargs):
log.debug('EventSubscriber event received: %s', str(args[0]) )
if args[0].description == 'TestingFinished':
log.debug('TestingFinished event received')
self._finished_events_received.append(args[0])
if self._finished_count and self._finished_count == len(self._finished_events_received):
log.debug('Finishing test...')
self._async_finished_result.set(len(self._finished_events_received))
log.debug('Called self._async_finished_result.set({0})'.format(len(self._finished_events_received)))
self._finished_event_subscriber = EventSubscriber(event_type='DeviceEvent', callback=consume_event)
self._finished_event_subscriber.start()
def _stop_finished_event_subscriber(self):
if self._finished_event_subscriber:
self._finished_event_subscriber.stop()
self._finished_event_subscriber = None
def tearDown(self):
pass
@unittest.skip('Update to agent refactor.')
def test_slocum_data_ingest(self):
HIST_CONSTRAINTS_1 = {}
# Test instrument driver execute interface to start and stop streaming mode.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Make sure the polling interval is appropriate for a test
params = {
'POLLING_INTERVAL': 3
}
self._ia_client.set_param(params)
self._finished_count = 1
cmd = AgentCommand(command='acquire_data')
self._ia_client.execute(cmd)
# Assert that data was received
self._async_finished_result.get(timeout=15)
self.assertTrue(len(self._finished_events_received) >= 1)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
#todo enable after Luke's mor to retrieve, right now must have the Time axis called 'time'
# replay_granule = self.data_retriever.retrieve_last_data_points(self.dataset_id, 10)
# rdt = RecordDictionaryTool.load_from_granule(replay_granule)
#
# comp = rdt['date_pattern'] == numpy.arange(10) + 10
#
# log.debug("TestBulkIngest: comp: %s", comp)
#
# self.assertTrue(comp.all())
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
def _setup_resources(self):
self.loggerpids = []
# Create DataProvider
dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation())
dprov.contact.name = 'Christopher Mueller'
dprov.contact.email = '*****@*****.**'
# Create DataSetModel
dataset_model = ExternalDatasetModel(name='slocum_model')
dataset_model.datset_type = 'SLOCUM'
dataset_model_id = self.dams_client.create_external_dataset_model(dataset_model)
# Create ExternalDataset
ds_name = 'slocum_test_dataset'
dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation())
dset.dataset_description.parameters['base_url'] = 'test_data/slocum/'
dset.dataset_description.parameters['list_pattern'] = 'ru05-2012-021-0-0-sbd.dat'
dset.dataset_description.parameters['date_pattern'] = '%Y %j'
dset.dataset_description.parameters['date_extraction_pattern'] = 'ru05-([\d]{4})-([\d]{3})-\d-\d-sbd.dat'
dset.dataset_description.parameters['temporal_dimension'] = None
dset.dataset_description.parameters['zonal_dimension'] = None
dset.dataset_description.parameters['meridional_dimension'] = None
dset.dataset_description.parameters['vertical_dimension'] = None
dset.dataset_description.parameters['variables'] = [
'c_wpt_y_lmc',
'sci_water_cond',
'm_y_lmc',
'u_hd_fin_ap_inflection_holdoff',
'sci_m_present_time',
'm_leakdetect_voltage_forward',
'sci_bb3slo_b660_scaled',
'c_science_send_all',
'm_gps_status',
'm_water_vx',
'm_water_vy',
'c_heading',
'sci_fl3slo_chlor_units',
'u_hd_fin_ap_gain',
'm_vacuum',
'u_min_water_depth',
'm_gps_lat',
'm_veh_temp',
'f_fin_offset',
'u_hd_fin_ap_hardover_holdoff',
'c_alt_time',
'm_present_time',
'm_heading',
'sci_bb3slo_b532_scaled',
'sci_fl3slo_cdom_units',
'm_fin',
'x_cycle_overrun_in_ms',
'sci_water_pressure',
'u_hd_fin_ap_igain',
'sci_fl3slo_phyco_units',
'm_battpos',
'sci_bb3slo_b470_scaled',
'm_lat',
'm_gps_lon',
'sci_ctd41cp_timestamp',
'm_pressure',
'c_wpt_x_lmc',
'c_ballast_pumped',
'x_lmc_xy_source',
'm_lon',
'm_avg_speed',
'sci_water_temp',
'u_pitch_ap_gain',
'm_roll',
'm_tot_num_inflections',
'm_x_lmc',
'u_pitch_ap_deadband',
'm_final_water_vy',
'm_final_water_vx',
'm_water_depth',
'm_leakdetect_voltage',
'u_pitch_max_delta_battpos',
'm_coulomb_amphr',
'm_pitch',
]
## Create the external dataset
ds_id = self.dams_client.create_external_dataset(external_dataset=dset, external_dataset_model_id=dataset_model_id)
ext_dprov_id = self.dams_client.create_external_data_provider(external_data_provider=dprov)
# Register the ExternalDataset
dproducer_id = self.dams_client.register_external_data_set(external_dataset_id=ds_id)
## Create the dataset agent
datasetagent_obj = IonObject(RT.ExternalDatasetAgent, name='ExternalDatasetAgent1', description='external data agent', handler_module=self.EDA_MOD, handler_class=self.EDA_CLS )
self.datasetagent_id = self.dams_client.create_external_dataset_agent(external_dataset_agent=datasetagent_obj, external_dataset_model_id=dataset_model_id)
# Generate the data product and associate it to the ExternalDataset
pdict = DatasetManagementService.get_parameter_dictionary_by_name('ctd_parsed_param_dict')
streamdef_id = self.pubsub_client.create_stream_definition(name="temp", parameter_dictionary_id=pdict.identifier)
tdom, sdom = time_series_domain()
tdom = tdom.dump()
sdom = sdom.dump()
dprod = IonObject(RT.DataProduct,
name='slocum_parsed_product',
description='parsed slocum product',
temporal_domain = tdom,
spatial_domain = sdom)
self.dproduct_id = self.dataproductclient.create_data_product(data_product=dprod,
stream_definition_id=streamdef_id)
self.dams_client.assign_data_product(input_resource_id=ds_id, data_product_id=self.dproduct_id)
#save the incoming slocum data
self.dataproductclient.activate_data_product_persistence(self.dproduct_id)
self.addCleanup(self.dataproductclient.suspend_data_product_persistence, self.dproduct_id)
stream_ids, assn = self.rrclient.find_objects(subject=self.dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True)
stream_id = stream_ids[0]
dataset_id, assn = self.rrclient.find_objects(subject=self.dproduct_id, predicate=PRED.hasDataset, object_type=RT.Dataset, id_only=True)
self.dataset_id = dataset_id[0]
pid = self.create_logger('slocum_parsed_product', stream_id )
self.loggerpids.append(pid)
self.DVR_CONFIG['dh_cfg'] = {
'TESTING':True,
'stream_id':stream_id,
'param_dictionary':pdict.dump(),
'data_producer_id':dproducer_id, #CBM: Should this be put in the main body of the config - with mod & cls?
'max_records':20,
}
# Create the logger for receiving publications
#self.create_stream_and_logger(name='slocum',stream_id=stream_id)
# Create agent config.
self.EDA_RESOURCE_ID = ds_id
self.EDA_NAME = ds_name
class VisStreamLauncher(ImmediateProcess):
"""
Class emulates a stream source from a NetCDF file. It emits a record of data every few seconds on a
stream identified by a routing key.
"""
def on_init(self):
log.debug("VizStreamProducer init. Self.id=%s" % self.id)
def on_start(self):
log.debug("VizStreamProducer start")
self.data_source_name = self.CFG.get('name')
self.dataset = self.CFG.get('dataset')
# create a pubsub client and a resource registry client
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
# Dummy instrument related clients
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.IngestClient = IngestionManagementServiceClient(node=self.container.node)
# create the pubsub client
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
# Additional code for creating a dummy instrument
"""
# Set up the preconditions. Look for an existing ingestion config
while True:
log.info("VisStreamLauncher:on_start: Waiting for an ingestion configuration to be available.")
ingestion_cfgs, _ = self.rrclient.find_resources(RT.IngestionConfiguration, None, None, True)
if len(ingestion_cfgs) > 0:
break
else:
gevent.sleep(1)
"""
# Check to see if the data_product already exists in the system (for e.g re launching the code after a crash)
dp_ids,_ = self.rrclient.find_resources(RT.DataProduct, None, self.data_source_name, True)
if len(dp_ids) > 0:
data_product_id = dp_ids[0]
print '>>>>>>>>>>>>> Found dp_id = ', data_product_id
else:
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel, name=self.data_source_name, description=self.data_source_name, model=self.data_source_name)
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
# Create InstrumentDevice
instDevice_obj = IonObject(RT.InstrumentDevice, name=self.data_source_name, description=self.data_source_name, serial_number="12345" )
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def)
print 'Creating new CDM data product with a stream definition'
dp_obj = IonObject(RT.DataProduct,name=self.data_source_name,description='ctd stream test')
data_product_id = self.dpclient.create_data_product(dp_obj, ctd_stream_def_id)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id, persist_data=True, persist_metadata=True)
print '>>>>>>>>>>>> New dp_id = ', data_product_id
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id, PRED.hasStream, None, True)
if self.dataset == 'sinusoidal':
pid = self.container.spawn_process(name='ctd_test.' + self.data_source_name ,
module='ion.processes.data.sinusoidal_stream_publisher',cls='SinusoidalCtdPublisher',config={'process':{'stream_id':stream_ids[0]}})
else:
pid = self.container.spawn_process(name='ctd_test.' + self.data_source_name ,
module='ion.processes.data.ctd_stream_publisher',cls='SimpleCtdPublisher',config={'process':{'stream_id':stream_ids[0]}})
def on_quit(self):
log.debug("VizStreamProducer quit")
class TestExternalDatasetAgentMgmt(IonIntegrationTestCase):
# DataHandler config
DVR_CONFIG = {
'dvr_mod': 'ion.agents.data.handlers.base_data_handler',
'dvr_cls': 'DummyDataHandler',
}
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
log.debug("TestExternalDatasetAgentMgmt: started services")
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(
node=self.container.node)
self.dpclient = DataProductManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
# @unittest.skip('not yet working. fix activate_data_product_persistence()')
#@unittest.skip()
@unittest.skip('not working')
def test_activateDatasetAgent(self):
# Create ExternalDatasetModel
datsetModel_obj = IonObject(RT.ExternalDatasetModel,
name='ExampleDatasetModel',
description="ExampleDatasetModel",
datset_type="FibSeries")
try:
datasetModel_id = self.damsclient.create_external_dataset_model(
datsetModel_obj)
except BadRequest as ex:
self.fail("failed to create new ExternalDatasetModel: %s" % ex)
log.debug(
"TestExternalDatasetAgentMgmt: new ExternalDatasetModel id = %s",
str(datasetModel_id))
# Create ExternalDatasetAgent
datasetAgent_obj = IonObject(RT.ExternalDatasetAgent,
name='datasetagent007',
description="datasetagent007",
handler_module=EDA_MOD,
handler_class=EDA_CLS)
try:
datasetAgent_id = self.damsclient.create_external_dataset_agent(
datasetAgent_obj, datasetModel_id)
except BadRequest as ex:
self.fail("failed to create new ExternalDatasetAgent: %s" % ex)
log.debug(
"TestExternalDatasetAgentMgmt: new ExternalDatasetAgent id = %s",
str(datasetAgent_id))
# Create ExternalDataset
log.debug(
'TestExternalDatasetAgentMgmt: Create external dataset resource ')
extDataset_obj = IonObject(RT.ExternalDataset,
name='ExtDataset',
description="ExtDataset")
try:
extDataset_id = self.damsclient.create_external_dataset(
extDataset_obj, datasetModel_id)
except BadRequest as ex:
self.fail("failed to create new external dataset resource: %s" %
ex)
log.debug(
"TestExternalDatasetAgentMgmt: new ExternalDataset id = %s ",
str(extDataset_id))
#register the dataset as a data producer
dproducer_id = self.damsclient.register_external_data_set(
extDataset_id)
# create a stream definition for the data from the ctd simulator
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(
name='SBE37_CDM', parameter_dictionary_id=pdict_id)
log.debug(
"TestExternalDatasetAgentMgmt: new Stream Definition id = %s",
str(ctd_stream_def_id))
log.debug(
"TestExternalDatasetAgentMgmt: Creating new data product with a stream definition"
)
dp_obj = IonObject(RT.DataProduct,
name='eoi dataset data',
description=' stream test')
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp')
data_product_id1 = self.dpclient.create_data_product(
dp_obj, ctd_stream_def_id)
log.debug("TestExternalDatasetAgentMgmt: new dp_id = %s",
str(data_product_id1))
self.damsclient.assign_data_product(input_resource_id=extDataset_id,
data_product_id=data_product_id1)
#todo fix the problem here....
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1,
PRED.hasStream, None, True)
log.debug("TestExternalDatasetAgentMgmt: Data product streams1 = %s",
str(stream_ids))
stream_id = stream_ids[0]
# Build a taxonomy for the dataset
tx = TaxyTool()
tx.add_taxonomy_set('data', 'external_data')
# Augment the DVR_CONFIG with the necessary pieces
self.DVR_CONFIG['dh_cfg'] = {
'TESTING': True,
'stream_id':
stream_id, #TODO: This should probably be a 'stream_config' dict with stream_name:stream_id members
'data_producer_id': dproducer_id,
# 'external_dataset_res':extDataset_obj, # Not needed - retrieved by EDA based on resource_id
'taxonomy': tx.dump(), #TODO: Currently does not support sets
'max_records': 4,
}
# Create agent config.
self._stream_config = {}
agent_config = {
'driver_config': self.DVR_CONFIG,
'stream_config': self._stream_config,
'agent': {
'resource_id': EDA_RESOURCE_ID
},
'test_mode': True
}
extDatasetAgentInstance_obj = IonObject(
RT.ExternalDatasetAgentInstance,
name='DatasetAgentInstance',
description="DatasetAgentInstance",
dataset_driver_config=self.DVR_CONFIG,
dataset_agent_config=agent_config)
extDatasetAgentInstance_id = self.damsclient.create_external_dataset_agent_instance(
external_dataset_agent_instance=extDatasetAgentInstance_obj,
external_dataset_agent_id=datasetAgent_id,
external_dataset_id=extDataset_id)
log.debug(
"TestExternalDatasetAgentMgmt: Dataset agent instance obj: = %s",
str(extDatasetAgentInstance_obj))
log.debug(
"TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s",
str(extDatasetAgentInstance_id))
#Check that the instance is currently not active
id, active = self.damsclient.retrieve_external_dataset_agent_instance(
extDataset_id)
log.debug(
"TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s active 1 = %s ",
str(id), str(active))
self.damsclient.start_external_dataset_agent_instance(
extDatasetAgentInstance_id)
dataset_agent_instance_obj = self.damsclient.read_external_dataset_agent_instance(
extDatasetAgentInstance_id)
log.debug(
"TestExternalDatasetAgentMgmt: Dataset agent instance obj: = %s",
str(dataset_agent_instance_obj))
# now the instance process should be active
id, active = self.damsclient.retrieve_external_dataset_agent_instance(
extDataset_id)
log.debug(
"TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s active 2 = %s ",
str(id), str(active))
# Start a resource agent client to talk with the instrument agent.
self._dsa_client = ResourceAgentClient(extDataset_id,
process=FakeProcess())
print 'TestExternalDatasetAgentMgmt: got ia client %s', self._dsa_client
log.debug("TestExternalDatasetAgentMgmt: got dataset client %s",
str(self._dsa_client))
# cmd=AgentCommand(command='initialize')
# _ = self._dsa_client.execute_agent(cmd)
#
# cmd = AgentCommand(command='go_active')
# _ = self._dsa_client.execute_agent(cmd)
#
# cmd = AgentCommand(command='run')
# _ = self._dsa_client.execute_agent(cmd)
#
# log.info('Send an unconstrained request for data (\'new data\')')
# cmd = AgentCommand(command='acquire_data')
# self._dsa_client.execute(cmd)
#
# log.info('Send a second unconstrained request for data (\'new data\'), should be rejected')
# cmd = AgentCommand(command='acquire_data')
# self._dsa_client.execute(cmd)
#
# cmd = AgentCommand(command='reset')
# _ = self._dsa_client.execute_agent(cmd)
# cmd = AgentCommand(command='get_current_state')
# retval = self._dsa_client.execute_agent(cmd)
# state = retval.result
# TODO: Think about what we really should be testing at this point
# The following is taken from ion.agents.data.test.test_external_dataset_agent.ExternalDatasetAgentTestBase.test_states()
# TODO: Do we also need to show data retrieval?
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='pause')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command='resume')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='clear')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='pause')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command='clear')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command='reset')
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
#-------------------------------
# Deactivate InstrumentAgentInstance
#-------------------------------
self.damsclient.stop_external_dataset_agent_instance(
extDatasetAgentInstance_id)
def test_dataset_agent_prepare_support(self):
eda_sup = self.damsclient.prepare_external_dataset_agent_support()
eda_obj = IonObject(RT.ExternalDatasetAgent,
name="ExternalDatasetAgent")
eda_id = self.damsclient.create_external_dataset_agent(eda_obj)
eda_sup = self.damsclient.prepare_external_dataset_agent_support(
external_dataset_agent_id=eda_id)
edai_sup = self.damsclient.prepare_external_dataset_agent_instance_support(
)
edai_obj = IonObject(RT.ExternalDatasetAgentInstance,
name="ExternalDatasetAgentInstance")
edai_id = self.damsclient.create_external_dataset_agent_instance(
edai_obj)
edai_sup = self.damsclient.prepare_external_dataset_agent_instance_support(
external_dataset_agent_instance_id=edai_id)
class TestActivateRSNVel3DInstrument(IonIntegrationTestCase):
def setUp(self):
# Start container
super(TestActivateRSNVel3DInstrument, self).setUp()
config = DotDict()
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml', config)
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataretrieverclient = DataRetrieverServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name+'_logger')
producer_definition.executable = {
'module':'ion.processes.data.stream_granule_logger',
'class':'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition)
configuration = {
'process':{
'stream_id':stream_id,
}
}
pid = self.processdispatchclient.schedule_process(process_definition_id=logger_procdef_id,
configuration=configuration)
return pid
@attr('LOCOINT')
@unittest.skip('under construction')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
@patch.dict(CFG, {'endpoint':{'receive':{'timeout': 180}}})
def test_activate_rsn_vel3d(self):
log.info("--------------------------------------------------------------------------------------------------------")
# load_parameter_scenarios
self.container.spawn_process("Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=dict(
op="load",
scenario="BETA",
path="master",
categories="ParameterFunctions,ParameterDefs,ParameterDictionary,StreamDefinition",
clearcols="owner_id,org_ids",
assets="res/preload/r2_ioc/ooi_assets",
parseooi="True",
))
self.loggerpids = []
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='Vel3DMModel',
description="Vel3DMModel")
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
log.debug( 'test_activate_rsn_vel3d new InstrumentModel id = %s ', instModel_id)
raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='raw' )
vel3d_b_sample = StreamConfiguration(stream_name='vel3d_b_sample', parameter_dictionary_name='vel3d_b_sample')
vel3d_b_engineering = StreamConfiguration(stream_name='vel3d_b_engineering', parameter_dictionary_name='vel3d_b_engineering')
RSN_VEL3D_01 = {
'DEV_ADDR' : "10.180.80.6",
'DEV_PORT' : 2101,
'DATA_PORT' : 1026,
'CMD_PORT' : 1025,
'PA_BINARY' : "port_agent"
}
# Create InstrumentAgent
instAgent_obj = IonObject(RT.InstrumentAgent,
name='Vel3DAgent',
description="Vel3DAgent",
driver_uri="http://sddevrepo.oceanobservatories.org/releases/nobska_mavs4_ooicore-0.0.7-py2.7.egg",
stream_configurations = [raw_config, vel3d_b_sample, vel3d_b_engineering])
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
log.debug('test_activate_rsn_vel3d new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug('test_activate_rsn_vel3d: Create instrument resource to represent the Vel3D ')
instDevice_obj = IonObject(RT.InstrumentDevice,
name='Vel3DDevice',
description="Vel3DDevice",
serial_number="12345" )
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
log.debug("test_activate_rsn_vel3d: new InstrumentDevice id = %s " , instDevice_id)
port_agent_config = {
'device_addr': '10.180.80.6',
'device_port': 2101,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': 1025,
'data_port': 1026,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='Vel3DAgentInstance',
description="Vel3DAgentInstance",
port_agent_config = port_agent_config,
alerts= [])
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj,
instAgent_id,
instDevice_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
parsed_sample_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('vel3d_b_sample', id_only=True)
parsed_sample_stream_def_id = self.pubsubcli.create_stream_definition(name='vel3d_b_sample', parameter_dictionary_id=parsed_sample_pdict_id)
parsed_eng_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('vel3d_b_engineering', id_only=True)
parsed_eng_stream_def_id = self.pubsubcli.create_stream_definition(name='vel3d_b_engineering', parameter_dictionary_id=parsed_eng_pdict_id)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('raw', id_only=True)
raw_stream_def_id = self.pubsubcli.create_stream_definition(name='raw', parameter_dictionary_id=raw_pdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='vel3d_b_sample',
description='vel3d_b_sample',
temporal_domain = tdom,
spatial_domain = sdom)
sample_data_product_id = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_sample_stream_def_id)
log.debug( 'new dp_id = %s' , sample_data_product_id)
self.dpclient.activate_data_product_persistence(data_product_id=sample_data_product_id)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=sample_data_product_id)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(sample_data_product_id, PRED.hasStream, None, True)
log.debug('sample_data_product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(sample_data_product_id, PRED.hasDataset, RT.Dataset, True)
log.debug('Data set for sample_data_product = %s' , dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
pid = self.create_logger('vel3d_b_sample', stream_ids[0] )
self.loggerpids.append(pid)
dp_obj = IonObject(RT.DataProduct,
name='vel3d_b_engineering',
description='vel3d_b_engineering',
temporal_domain = tdom,
spatial_domain = sdom)
eng_data_product_id = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_eng_stream_def_id)
log.debug( 'new dp_id = %s' , eng_data_product_id)
self.dpclient.activate_data_product_persistence(data_product_id=eng_data_product_id)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=eng_data_product_id)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id2 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id)
log.debug('new dp_id = %s', data_product_id2)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True)
log.debug('test_activate_rsn_vel3d Data product streams2 = %s' , str(stream_ids))
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True)
log.debug('test_activate_rsn_vel3d Data set for data_product_id2 = %s' , dataset_ids[0])
self.raw_dataset = dataset_ids[0]
def start_instrument_agent():
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
gevent.joinall([gevent.spawn(start_instrument_agent)])
#cleanup
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
#wait for start
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
gate = AgentProcessStateGate(self.processdispatchclient.read_process,
instDevice_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" %
gate.process_id)
#log.trace('Instrument agent instance obj: = %s' , str(inst_agent_instance_obj))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(instDevice_id,
to_name=gate.process_id,
process=FakeProcess())
def check_state(label, desired_state):
actual_state = self._ia_client.get_agent_state()
log.debug("%s instrument agent is in state '%s'", label, actual_state)
self.assertEqual(desired_state, actual_state)
log.debug("test_activate_rsn_vel3d: got ia client %s" , str(self._ia_client))
check_state("just-spawned", ResourceAgentState.UNINITIALIZED)
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
log.debug("test_activate_rsn_vel3d: initialize %s" , str(retval))
check_state("initialized", ResourceAgentState.INACTIVE)
log.debug("test_activate_rsn_vel3d Sending go_active command ")
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activate_rsn_vel3d: return value from go_active %s" , str(reply))
check_state("activated", ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("current state after sending go_active command %s" , str(state))
#
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activate_rsn_vel3d: run %s" , str(reply))
check_state("commanded", ResourceAgentState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("current state after sending run command %s" , str(state))
# cmd = AgentCommand(command=ProtocolEvent.START_AUTOSAMPLE)
# reply = self._ia_client.execute_agent(cmd)
# log.debug("test_activate_rsn_vel3d: run %s" , str(reply))
# state = self._ia_client.get_agent_state()
# self.assertEqual(ResourceAgentState.COMMAND, state)
#
# gevent.sleep(5)
#
# cmd = AgentCommand(command=ProtocolEvent.STOP_AUTOSAMPLE)
# reply = self._ia_client.execute_agent(cmd)
# log.debug("test_activate_rsn_vel3d: run %s" , str(reply))
# state = self._ia_client.get_agent_state()
# self.assertEqual(ResourceAgentState.COMMAND, state)
#
# cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
# retval = self._ia_client.execute_agent(cmd)
# state = retval.result
# log.debug("current state after sending STOP_AUTOSAMPLE command %s" , str(state))
#
# cmd = AgentCommand(command=ResourceAgentEvent.PAUSE)
# retval = self._ia_client.execute_agent(cmd)
# state = self._ia_client.get_agent_state()
# self.assertEqual(ResourceAgentState.STOPPED, state)
#
# cmd = AgentCommand(command=ResourceAgentEvent.RESUME)
# retval = self._ia_client.execute_agent(cmd)
# state = self._ia_client.get_agent_state()
# self.assertEqual(ResourceAgentState.COMMAND, state)
#
# cmd = AgentCommand(command=ResourceAgentEvent.CLEAR)
# retval = self._ia_client.execute_agent(cmd)
# state = self._ia_client.get_agent_state()
# self.assertEqual(ResourceAgentState.IDLE, state)
#
# cmd = AgentCommand(command=ResourceAgentEvent.RUN)
# retval = self._ia_client.execute_agent(cmd)
# state = self._ia_client.get_agent_state()
# self.assertEqual(ResourceAgentState.COMMAND, state)
log.debug( "test_activate_rsn_vel3d: calling reset ")
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activate_rsn_vel3d: return from reset %s" , str(reply))
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data_raw = self.dataretrieverclient.retrieve(self.raw_dataset)
self.assertIsInstance(replay_data_raw, Granule)
rdt_raw = RecordDictionaryTool.load_from_granule(replay_data_raw)
log.debug("RDT raw: %s", str(rdt_raw.pretty_print()) )
self.assertIn('raw', rdt_raw)
raw_vals = rdt_raw['raw']
#--------------------------------------------------------------------------------
# Deactivate loggers
#--------------------------------------------------------------------------------
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
self.dpclient.delete_data_product(sample_data_product_id)
self.dpclient.delete_data_product(eng_data_product_id)
self.dpclient.delete_data_product(data_product_id2)
class TestCTDTransformsIntegration(IonIntegrationTestCase):
pdict_id = None
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
print 'started services'
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(
node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.dataset_management = self.datasetclient
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name + '_logger')
producer_definition.executable = {
'module': 'ion.processes.data.stream_granule_logger',
'class': 'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(
process_definition=producer_definition)
configuration = {
'process': {
'stream_id': stream_id,
}
}
pid = self.processdispatchclient.schedule_process(
process_definition_id=logger_procdef_id,
configuration=configuration)
return pid
def _create_instrument_model(self):
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
return instModel_id
def _create_instrument_agent(self, instModel_id):
raw_config = StreamConfiguration(
stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict')
parsed_config = StreamConfiguration(
stream_name='parsed',
parameter_dictionary_name='ctd_parsed_param_dict')
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri=DRV_URI_GOOD,
stream_configurations=[raw_config, parsed_config])
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
self.imsclient.assign_instrument_model_to_instrument_agent(
instModel_id, instAgent_id)
return instAgent_id
def _create_instrument_device(self, instModel_id):
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345")
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, instDevice_id)
log.debug(
"test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ",
instDevice_id)
return instDevice_id
def _create_instrument_agent_instance(self, instAgent_id, instDevice_id):
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance,
name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config=port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id)
return instAgentInstance_id
def _create_param_dicts(self):
tdom, sdom = time_series_domain()
self.sdom = sdom.dump()
self.tdom = tdom.dump()
self.pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
def _create_input_data_products(
self,
ctd_stream_def_id,
instDevice_id,
):
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain=self.tdom,
spatial_domain=self.sdom)
ctd_parsed_data_product = self.dataproductclient.create_data_product(
dp_obj, ctd_stream_def_id)
self.damsclient.assign_data_product(
input_resource_id=instDevice_id,
data_product_id=ctd_parsed_data_product)
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_parsed_data_product)
#---------------------------------------------------------------------------
# Retrieve the id of the OUTPUT stream from the out Data Product
#---------------------------------------------------------------------------
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product,
PRED.hasStream, None, True)
pid = self.create_logger('ctd_parsed', stream_ids[0])
self.loggerpids.append(pid)
#---------------------------------------------------------------------------
# Create CTD Raw as the second data product
#---------------------------------------------------------------------------
if not self.pdict_id:
self._create_param_dicts()
raw_stream_def_id = self.pubsubclient.create_stream_definition(
name='SBE37_RAW', parameter_dictionary_id=self.pdict_id)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain=self.tdom,
spatial_domain=self.sdom)
ctd_raw_data_product = self.dataproductclient.create_data_product(
dp_obj, raw_stream_def_id)
self.damsclient.assign_data_product(
input_resource_id=instDevice_id,
data_product_id=ctd_raw_data_product)
self.dataproductclient.activate_data_product_persistence(
data_product_id=ctd_raw_data_product)
#---------------------------------------------------------------------------
# Retrieve the id of the OUTPUT stream from the out Data Product
#---------------------------------------------------------------------------
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product,
PRED.hasStream, None, True)
#---------------------------------------------------------------------------
# Retrieve the id of the OUTPUT stream from the out Data Product
#---------------------------------------------------------------------------
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product,
PRED.hasStream, None, True)
print 'Data product streams2 = ', stream_ids
return ctd_parsed_data_product
def _create_data_process_definitions(self):
#-------------------------------------------------------------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------------------------------------------------------------
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all')
self.ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
#-------------------------------------------------------------------------------------
# L1 Conductivity: Data Process Definition
#-------------------------------------------------------------------------------------
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L1_conductivity',
description='create the L1 conductivity data product',
module='ion.processes.data.transforms.ctd.ctd_L1_conductivity',
class_name='CTDL1ConductivityTransform')
self.ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
#-------------------------------------------------------------------------------------
# L1 Pressure: Data Process Definition
#-------------------------------------------------------------------------------------
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L1_pressure',
description='create the L1 pressure data product',
module='ion.processes.data.transforms.ctd.ctd_L1_pressure',
class_name='CTDL1PressureTransform')
self.ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
#-------------------------------------------------------------------------------------
# L1 Temperature: Data Process Definition
#-------------------------------------------------------------------------------------
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L1_temperature',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L1_temperature',
class_name='CTDL1TemperatureTransform')
self.ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
#-------------------------------------------------------------------------------------
# L2 Salinity: Data Process Definition
#-------------------------------------------------------------------------------------
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L2_salinity',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform')
self.ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
#-------------------------------------------------------------------------------------
# L2 Density: Data Process Definition
#-------------------------------------------------------------------------------------
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='ctd_L2_density',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_density',
class_name='DensityTransform')
self.ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(
dpd_obj)
return self.ctd_L0_all_dprocdef_id, self.ctd_L1_conductivity_dprocdef_id,\
self.ctd_L1_pressure_dprocdef_id,self.ctd_L1_temperature_dprocdef_id, \
self.ctd_L2_salinity_dprocdef_id, self.ctd_L2_density_dprocdef_id
def _create_stream_definitions(self):
if not self.pdict_id:
self._create_param_dicts()
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(
name='L0_Conductivity', parameter_dictionary_id=self.pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_conductivity_id,
self.ctd_L0_all_dprocdef_id,
binding='conductivity')
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(
name='L0_Pressure', parameter_dictionary_id=self.pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_pressure_id,
self.ctd_L0_all_dprocdef_id,
binding='pressure')
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(
name='L0_Temperature', parameter_dictionary_id=self.pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l0_temperature_id,
self.ctd_L0_all_dprocdef_id,
binding='temperature')
return outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id, outgoing_stream_l0_temperature_id
def _create_l0_output_data_products(self,
outgoing_stream_l0_conductivity_id,
outgoing_stream_l0_pressure_id,
outgoing_stream_l0_temperature_id):
out_data_prods = []
ctd_l0_conductivity_output_dp_obj = IonObject(
RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain=self.tdom,
spatial_domain=self.sdom)
self.ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id)
out_data_prods.append(self.ctd_l0_conductivity_output_dp_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=self.ctd_l0_conductivity_output_dp_id)
ctd_l0_pressure_output_dp_obj = IonObject(
RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain=self.tdom,
spatial_domain=self.sdom)
self.ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id)
out_data_prods.append(self.ctd_l0_pressure_output_dp_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=self.ctd_l0_pressure_output_dp_id)
ctd_l0_temperature_output_dp_obj = IonObject(
RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain=self.tdom,
spatial_domain=self.sdom)
self.ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(
ctd_l0_temperature_output_dp_obj,
outgoing_stream_l0_temperature_id)
out_data_prods.append(self.ctd_l0_temperature_output_dp_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=self.ctd_l0_temperature_output_dp_id)
return out_data_prods
def _create_l1_out_data_products(self):
ctd_l1_conductivity_output_dp_obj = IonObject(
RT.DataProduct,
name='L1_Conductivity',
description='transform output L1 conductivity',
temporal_domain=self.tdom,
spatial_domain=self.sdom)
self.ctd_l1_conductivity_output_dp_id = self.dataproductclient.create_data_product(
ctd_l1_conductivity_output_dp_obj,
self.outgoing_stream_l1_conductivity_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=self.ctd_l1_conductivity_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(
self.ctd_l1_conductivity_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger('ctd_l1_conductivity', stream_ids[0])
self.loggerpids.append(pid)
ctd_l1_pressure_output_dp_obj = IonObject(
RT.DataProduct,
name='L1_Pressure',
description='transform output L1 pressure',
temporal_domain=self.tdom,
spatial_domain=self.sdom)
self.ctd_l1_pressure_output_dp_id = self.dataproductclient.create_data_product(
ctd_l1_pressure_output_dp_obj, self.outgoing_stream_l1_pressure_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=self.ctd_l1_pressure_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(
self.ctd_l1_pressure_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger('ctd_l1_pressure', stream_ids[0])
self.loggerpids.append(pid)
ctd_l1_temperature_output_dp_obj = IonObject(
RT.DataProduct,
name='L1_Temperature',
description='transform output L1 temperature',
temporal_domain=self.tdom,
spatial_domain=self.sdom)
self.ctd_l1_temperature_output_dp_id = self.dataproductclient.create_data_product(
ctd_l1_temperature_output_dp_obj,
self.outgoing_stream_l1_temperature_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=self.ctd_l1_temperature_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(
self.ctd_l1_temperature_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger('ctd_l1_temperature', stream_ids[0])
self.loggerpids.append(pid)
def _create_l2_out_data_products(self):
#-------------------------------
# L2 Salinity - Density: Output Data Products
#-------------------------------
if not self.pdict_id: self._create_param_dicts()
outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(
name='L2_salinity', parameter_dictionary_id=self.pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l2_salinity_id,
self.ctd_L2_salinity_dprocdef_id,
binding='salinity')
outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(
name='L2_Density', parameter_dictionary_id=self.pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
outgoing_stream_l2_density_id,
self.ctd_L2_density_dprocdef_id,
binding='density')
ctd_l2_salinity_output_dp_obj = IonObject(
RT.DataProduct,
name='L2_Salinity',
description='transform output L2 salinity',
temporal_domain=self.tdom,
spatial_domain=self.sdom)
self.ctd_l2_salinity_output_dp_id = self.dataproductclient.create_data_product(
ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=self.ctd_l2_salinity_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(
self.ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger('ctd_l2_salinity', stream_ids[0])
self.loggerpids.append(pid)
ctd_l2_density_output_dp_obj = IonObject(
RT.DataProduct,
name='L2_Density',
description='transform output pressure',
temporal_domain=self.tdom,
spatial_domain=self.sdom)
self.ctd_l2_density_output_dp_id = self.dataproductclient.create_data_product(
ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id)
self.dataproductclient.activate_data_product_persistence(
data_product_id=self.ctd_l2_density_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(
self.ctd_l2_density_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger('ctd_l2_density', stream_ids[0])
self.loggerpids.append(pid)
@unittest.skip('This test errors on coi-nightly, may be OBE.')
def test_createTransformsThenActivateInstrument(self):
self.loggerpids = []
#-------------------------------------------------------------------------------------
# Create InstrumentModel
#-------------------------------------------------------------------------------------
instModel_id = self._create_instrument_model()
#-------------------------------------------------------------------------------------
# Create InstrumentAgent
#-------------------------------------------------------------------------------------
instAgent_id = self._create_instrument_agent(instModel_id)
#-------------------------------------------------------------------------------------
# Create InstrumentDevice
#-------------------------------------------------------------------------------------
instDevice_id = self._create_instrument_device(instModel_id)
#-------------------------------------------------------------------------------------
# Create Instrument Agent Instance
#-------------------------------------------------------------------------------------
instAgentInstance_id = self._create_instrument_agent_instance(
instAgent_id, instDevice_id)
#-------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#-------------------------------------------------------------------------------------
self._create_param_dicts()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(
name='SBE37_CDM', parameter_dictionary_id=self.pdict_id)
#-------------------------------------------------------------------------------------
# Create two data products
#-------------------------------------------------------------------------------------
ctd_parsed_data_product = self._create_input_data_products(
ctd_stream_def_id, instDevice_id)
#-------------------------------------------------------------------------------------
# Create data process definitions
#-------------------------------------------------------------------------------------
self._create_data_process_definitions()
#-------------------------------------------------------------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------------------------------------------------------------
outgoing_stream_l0_conductivity_id, \
outgoing_stream_l0_pressure_id, \
outgoing_stream_l0_temperature_id = self._create_stream_definitions()
self.out_prod_ids = self._create_l0_output_data_products(
outgoing_stream_l0_conductivity_id, outgoing_stream_l0_pressure_id,
outgoing_stream_l0_temperature_id)
self.outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(
name='L1_conductivity', parameter_dictionary_id=self.pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
self.outgoing_stream_l1_conductivity_id,
self.ctd_L1_conductivity_dprocdef_id,
binding='conductivity')
self.outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(
name='L1_Pressure', parameter_dictionary_id=self.pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
self.outgoing_stream_l1_pressure_id,
self.ctd_L1_pressure_dprocdef_id,
binding='pressure')
self.outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(
name='L1_Temperature', parameter_dictionary_id=self.pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
self.outgoing_stream_l1_temperature_id,
self.ctd_L1_temperature_dprocdef_id,
binding='temperature')
self._create_l1_out_data_products()
self._create_l2_out_data_products()
#-------------------------------------------------------------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------------------------------------------------------------
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id=self.ctd_L0_all_dprocdef_id,
in_data_product_ids=[ctd_parsed_data_product],
out_data_product_ids=self.out_prod_ids)
self.dataprocessclient.activate_data_process(
ctd_l0_all_data_process_id)
data_process = self.rrclient.read(ctd_l0_all_data_process_id)
process_ids, _ = self.rrclient.find_objects(
subject=ctd_l0_all_data_process_id,
predicate=PRED.hasProcess,
id_only=True)
self.addCleanup(self.processdispatchclient.cancel_process,
process_ids[0])
extended_process = self.dataprocessclient.get_data_process_extension(
ctd_l0_all_data_process_id)
self.assertEquals(extended_process.computed.operational_state.status,
ComputedValueAvailability.NOTAVAILABLE)
self.assertEquals(data_process.message_controllable, True)
#-------------------------------------------------------------------------------------
# L1 Conductivity: Create the data process
#-------------------------------------------------------------------------------------
l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(
self.ctd_L1_conductivity_dprocdef_id,
[self.ctd_l0_conductivity_output_dp_id],
[self.ctd_l1_conductivity_output_dp_id])
self.dataprocessclient.activate_data_process(
l1_conductivity_data_process_id)
data_process = self.rrclient.read(l1_conductivity_data_process_id)
process_ids, _ = self.rrclient.find_objects(
subject=l1_conductivity_data_process_id,
predicate=PRED.hasProcess,
id_only=True)
self.addCleanup(self.processdispatchclient.cancel_process,
process_ids[0])
#-------------------------------------------------------------------------------------
# L1 Pressure: Create the data process
#-------------------------------------------------------------------------------------
l1_pressure_data_process_id = self.dataprocessclient.create_data_process(
self.ctd_L1_pressure_dprocdef_id,
[self.ctd_l0_pressure_output_dp_id],
[self.ctd_l1_pressure_output_dp_id])
self.dataprocessclient.activate_data_process(
l1_pressure_data_process_id)
data_process = self.rrclient.read(l1_pressure_data_process_id)
process_ids, _ = self.rrclient.find_objects(
subject=l1_pressure_data_process_id,
predicate=PRED.hasProcess,
id_only=True)
self.addCleanup(self.processdispatchclient.cancel_process,
process_ids[0])
#-------------------------------------------------------------------------------------
# L1 Temperature: Create the data process
#-------------------------------------------------------------------------------------
l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(
self.ctd_L1_temperature_dprocdef_id,
[self.ctd_l0_temperature_output_dp_id],
[self.ctd_l1_temperature_output_dp_id])
self.dataprocessclient.activate_data_process(
l1_temperature_all_data_process_id)
data_process = self.rrclient.read(l1_temperature_all_data_process_id)
process_ids, _ = self.rrclient.find_objects(
subject=l1_temperature_all_data_process_id,
predicate=PRED.hasProcess,
id_only=True)
self.addCleanup(self.processdispatchclient.cancel_process,
process_ids[0])
#-------------------------------------------------------------------------------------
# L2 Salinity: Create the data process
#-------------------------------------------------------------------------------------
l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(
self.ctd_L2_salinity_dprocdef_id, [ctd_parsed_data_product],
[self.ctd_l2_salinity_output_dp_id])
self.dataprocessclient.activate_data_process(
l2_salinity_all_data_process_id)
data_process = self.rrclient.read(l2_salinity_all_data_process_id)
process_ids, _ = self.rrclient.find_objects(
subject=l2_salinity_all_data_process_id,
predicate=PRED.hasProcess,
id_only=True)
self.addCleanup(self.processdispatchclient.cancel_process,
process_ids[0])
#-------------------------------------------------------------------------------------
# L2 Density: Create the data process
#-------------------------------------------------------------------------------------
l2_density_all_data_process_id = self.dataprocessclient.create_data_process(
self.ctd_L2_density_dprocdef_id, [ctd_parsed_data_product],
[self.ctd_l2_density_output_dp_id])
self.dataprocessclient.activate_data_process(
l2_density_all_data_process_id)
data_process = self.rrclient.read(l2_density_all_data_process_id)
process_ids, _ = self.rrclient.find_objects(
subject=l2_density_all_data_process_id,
predicate=PRED.hasProcess,
id_only=True)
self.addCleanup(self.processdispatchclient.cancel_process,
process_ids[0])
#-------------------------------------------------------------------------------------
# Launch InstrumentAgentInstance, connect to the resource agent client
#-------------------------------------------------------------------------------------
self.imsclient.start_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id)
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(
instAgentInstance_id)
# Wait for instrument agent to spawn
gate = AgentProcessStateGate(self.processdispatchclient.read_process,
instDevice_id, ProcessStateEnum.RUNNING)
self.assertTrue(
gate. await (15),
"The instrument agent instance did not spawn in 15 seconds")
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(instDevice_id,
to_name=gate.process_id,
process=FakeProcess())
#-------------------------------------------------------------------------------------
# Streaming
#-------------------------------------------------------------------------------------
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug(
"(L4-CI-SA-RQ-334): current state after sending go_active command %s",
str(state))
self.assertTrue(state, 'DRIVER_STATE_COMMAND')
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
#todo ResourceAgentClient no longer has method set_param
# # Make sure the sampling rate and transmission are sane.
# params = {
# SBE37Parameter.NAVG : 1,
# SBE37Parameter.INTERVAL : 5,
# SBE37Parameter.TXREALTIME : True
# }
# self._ia_client.set_param(params)
#todo There is no ResourceAgentEvent attribute for go_streaming... so what should be the command for it?
cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE)
retval = self._ia_client.execute_resource(cmd)
# This gevent sleep is there to test the autosample time, which will show something different from default
# only if the instrument runs for over a minute
gevent.sleep(90)
extended_instrument = self.imsclient.get_instrument_device_extension(
instrument_device_id=instDevice_id)
self.assertIsInstance(extended_instrument.computed.uptime,
ComputedStringValue)
autosample_string = extended_instrument.computed.uptime.value
autosampling_time = int(autosample_string.split()[4])
self.assertTrue(autosampling_time > 0)
cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE)
retval = self._ia_client.execute_resource(cmd)
#todo There is no ResourceAgentEvent attribute for go_observatory... so what should be the command for it?
# log.debug("test_activateInstrumentStream: calling go_observatory")
# cmd = AgentCommand(command='go_observatory')
# reply = self._ia_client.execute_agent(cmd)
# cmd = AgentCommand(command='get_current_state')
# retval = self._ia_client.execute_agent(cmd)
# state = retval.result
# log.debug("test_activateInstrumentStream: return from go_observatory state %s", str(state))
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
#-------------------------------------------------------------------------------------------------
# Cleanup processes
#-------------------------------------------------------------------------------------------------
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
#--------------------------------------------------------------------------------
# Cleanup data products
#--------------------------------------------------------------------------------
dp_ids, _ = self.rrclient.find_resources(restype=RT.DataProduct,
id_only=True)
for dp_id in dp_ids:
self.dataproductclient.delete_data_product(dp_id)
class TestInstrumentAlerts(IonIntegrationTestCase):
pdict_id = None
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.catch_alert= gevent.queue.Queue()
def _create_instrument_model(self):
instModel_obj = IonObject( RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel" )
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
self.addCleanup(self.imsclient.delete_instrument_model, instModel_id)
return instModel_id
def _create_instrument_agent(self, instModel_id):
raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict')
parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict')
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri=DRV_URI_GOOD,
stream_configurations = [raw_config, parsed_config] )
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
self.addCleanup(self.imsclient.delete_instrument_agent, instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
return instAgent_id
def _create_instrument_device(self, instModel_id):
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
self.addCleanup(self.imsclient.delete_instrument_device, instDevice_id)
log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id)
return instDevice_id
def _create_instrument_stream_alarms(self, instDevice_id):
#Create stream alarms
"""
test_two_sided_interval
Test interval alarm and alarm event publishing for a closed
inteval.
"""
temp_alert_def1 = {
'name' : 'temperature_warning_interval temp below 25',
'stream_name' : 'parsed',
'description' : 'temperature_warning_interval temp below 25',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_DATA,
'value_id' : 'temp',
'resource_id' : instDevice_id,
'origin_type' : 'device',
#'lower_bound' : 0,
#'lower_rel_op' : '<',
# temp
'upper_rel_op' : '<=',
'upper_bound' : 25,
'alert_class' : 'IntervalAlert'
}
temp_alert_def2 = {
'name' : 'temperature_warning_interval temp below 50',
'stream_name' : 'parsed',
'description' : 'temperature_warning_interval temp below 50',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_DATA,
'value_id' : 'temp',
'resource_id' : instDevice_id,
'origin_type' : 'device',
#'lower_bound' : 25,
#'lower_rel_op' : '<',
# temp
'upper_rel_op' : '<=',
'upper_bound' : 50,
'alert_class' : 'IntervalAlert'
}
temp_alert_def3 = {
'name' : 'temperature_warning_interval temp below 75',
'stream_name' : 'parsed',
'description' : 'temperature_warning_interval temp below 75',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_DATA,
'value_id' : 'temp',
'resource_id' : instDevice_id,
'origin_type' : 'device',
#'lower_bound' : 50,
#'lower_rel_op' : '<',
# temp
'upper_rel_op' : '<=',
'upper_bound' : 75,
'alert_class' : 'IntervalAlert'
}
late_data_alert_def = {
'name' : 'late_data_warning',
'stream_name' : 'parsed',
'description' : 'Expected data has not arrived.',
'alert_type' : StreamAlertType.WARNING,
'aggregate_type' : AggregateStatusType.AGGREGATE_COMMS,
'value_id' : None,
'resource_id' : instDevice_id,
'origin_type' : 'device',
'time_delta' : 2,
'alert_class' : 'LateDataAlert'
}
return [temp_alert_def1, temp_alert_def2, temp_alert_def3, late_data_alert_def]
def _create_instrument_agent_instance(self, instAgent_id, instDevice_id):
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
self.all_alerts = self._create_instrument_stream_alarms(instDevice_id)
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config = port_agent_config,
alerts= self.all_alerts
)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj,
instAgent_id,
instDevice_id)
self.addCleanup(self.imsclient.delete_instrument_agent_instance, instAgentInstance_id)
return instAgentInstance_id
def test_alerts(self):
#
# test that with the 4009 sim we can get a late data alert
# as well as alerts for out of range for > 25, > 50, and > 75
# as well as the ALL_CLEAR alerts for each of them.
#
#-------------------------------------------------------------------------------------
# Create InstrumentModel
#-------------------------------------------------------------------------------------
instModel_id = self._create_instrument_model()
#-------------------------------------------------------------------------------------
# Create InstrumentAgent
#-------------------------------------------------------------------------------------
instAgent_id = self._create_instrument_agent(instModel_id)
#-------------------------------------------------------------------------------------
# Create InstrumentDevice
#-------------------------------------------------------------------------------------
instDevice_id = self._create_instrument_device(instModel_id)
# It is necessary for the instrument device to be associated with atleast one output data product
parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubclient.create_stream_definition(name='parsed', parameter_dictionary_id=parsed_pdict_id)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('raw', id_only=True)
raw_stream_def_id = self.pubsubclient.create_stream_definition(name='raw', parameter_dictionary_id=raw_pdict_id)
# We are creating two data products here, one for parsed and another raw
dp_obj_parsed = IonObject(RT.DataProduct,
name='parsed_data_product',
description='Parsed output data product for instrument')
dp_obj_raw = IonObject(RT.DataProduct,
name='raw_data_prod',
description='Raw output data product for instrument')
parsed_out_data_prod_id = self.dataproductclient.create_data_product(data_product=dp_obj_parsed, stream_definition_id=parsed_stream_def_id)
raw_out_data_prod_id = self.dataproductclient.create_data_product(data_product=dp_obj_raw, stream_definition_id=raw_stream_def_id)
self.addCleanup(self.dataproductclient.delete_data_product, parsed_out_data_prod_id)
self.addCleanup(self.dataproductclient.delete_data_product, raw_out_data_prod_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=parsed_out_data_prod_id)
self.dataproductclient.activate_data_product_persistence(data_product_id=raw_out_data_prod_id)
# todo: note that the generated config on the instruments will be done for both raw and parsed stream defs since these two data products constructed with each are associated as output data products with the instrument
# todo: if the config is not generated for a stream def, then the instrument agent will complain if the simulator generates data corresponding to a stream def that is not there in the stream config as a mentioned stream def
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=parsed_out_data_prod_id)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=raw_out_data_prod_id)
log.debug("assigned instdevice id: %s to data product: %s", instDevice_id, raw_out_data_prod_id)
#-------------------------------------------------------------------------------------
# Create Instrument Agent Instance
#-------------------------------------------------------------------------------------
instAgentInstance_id = self._create_instrument_agent_instance(instAgent_id,instDevice_id )
#-------------------------------------------------------------------------------------
# Launch InstrumentAgentInstance, connect to the resource agent client
#-------------------------------------------------------------------------------------
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
# Wait for instrument agent to spawn
gate = AgentProcessStateGate(self.processdispatchclient.read_process,
instDevice_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(15), "The instrument agent instance did not spawn in 15 seconds")
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(instDevice_id,
to_name=gate.process_id,
process=FakeProcess())
#-------------------------------------------------------------------------------------
# Set up the subscriber to catch the alert event
#-------------------------------------------------------------------------------------
def callback_for_alert(event, *args, **kwargs):
log.debug("caught an alert: %s", event)
self.catch_alert.put(event)
self.event_subscriber = EventSubscriber(event_type='DeviceStatusAlertEvent',
origin=instDevice_id,
callback=callback_for_alert)
self.event_subscriber.start()
self.addCleanup(self.event_subscriber.stop)
#-------------------------------------------------------------------------------------
# Running the instrument....
#-------------------------------------------------------------------------------------
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
# Prevent this test from hanging indefinitely until
# OOIION-1313 is resolved
timeout_val = 90
with gevent.Timeout(timeout_val, Exception('Agent failed to initialize after %fs' % timeout_val)):
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
with gevent.Timeout(timeout_val, Exception('Agent failed to go active after %fs' % timeout_val)):
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
with gevent.Timeout(timeout_val, Exception('Agent failed to get resource after %fs' % timeout_val)):
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("(L4-CI-SA-RQ-334): current state after sending go_active command %s", str(state))
self.assertTrue(state, 'DRIVER_STATE_COMMAND')
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
with gevent.Timeout(timeout_val, Exception('Agent failed to run after %fs' % timeout_val)):
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE)
with gevent.Timeout(timeout_val, Exception('Agent failed to start autosample after %fs' % timeout_val)):
retval = self._ia_client.execute_resource(cmd)
got_bad_temp = [False, False, False, False]
got_late_data = False
got_temp_all_clear = [False, False, False, False]
runtime = 0
starttime = time.time()
caught_events = []
while (got_bad_temp[0] == False or
got_bad_temp[1] == False or
got_bad_temp[2] == False or
got_temp_all_clear[0] == False or
got_temp_all_clear[1] == False or
got_temp_all_clear[2] == False or
got_late_data == False) and runtime < 120:
a = self.catch_alert.get(timeout=180)
caught_events.append(a)
if a.name == 'temperature_warning_interval temp below 25':
if a.sub_type == 'WARNING' and \
a.values[0] > 25:
got_bad_temp[0] = True
log.error(str(a.values[0]) + " should be above 25")
elif a.sub_type == 'ALL_CLEAR':
got_temp_all_clear[0] = True
log.debug("25 ALL_CLEAR")
if a.name == 'temperature_warning_interval temp below 50':
if a.sub_type == 'WARNING' and \
a.values[0] > 50:
got_bad_temp[1] = True
log.error(str(a.values[0]) + " should be above 50")
elif a.sub_type == 'ALL_CLEAR':
got_temp_all_clear[1] = True
log.debug("50 ALL_CLEAR" )
if a.name == 'temperature_warning_interval temp below 75':
if a.sub_type == 'WARNING' and \
a.values[0] > 75:
got_bad_temp[2] = True
log.error(str(a.values[0]) + " should be above 75")
elif a.sub_type == 'ALL_CLEAR':
got_temp_all_clear[2] = True
log.debug("75 ALL_CLEAR")
if a.name == 'late_data_warning' and \
a.description == 'Expected data has not arrived.':
got_late_data = True
log.debug("late value")
runtime = time.time() - starttime
log.debug("caught_events: %s", [c.name for c in caught_events])
for c in caught_events:
self.assertIn(c.name, ['temperature_warning_interval temp below 25',
'temperature_warning_interval temp below 50',
'temperature_warning_interval temp below 75',
'late_data_warning'])
self.assertEqual(c.origin, instDevice_id)
self.assertEqual(c.type_, 'DeviceStatusAlertEvent')
self.assertEqual(c.origin_type, 'InstrumentDevice')
self.assertTrue(got_bad_temp)
#simply log the state of the late data flag and the simulator does not seem to be consistent.
log.debug("test_alerts late data alert flag: %s", got_late_data)
class VisStreamLauncher(ImmediateProcess):
"""
Class emulates a stream source from a NetCDF file. It emits a record of data every few seconds on a
stream identified by a routing key.
"""
def on_init(self):
log.debug("VizStreamProducer init. Self.id=%s" % self.id)
def on_start(self):
log.debug("VizStreamProducer start")
self.data_source_name = self.CFG.get('name')
self.dataset = self.CFG.get('dataset')
# create a pubsub client and a resource registry client
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(
node=self.container.node)
# Dummy instrument related clients
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.dpclient = DataProductManagementServiceClient(
node=self.container.node)
self.IngestClient = IngestionManagementServiceClient(
node=self.container.node)
# create the pubsub client
self.pubsubclient = PubsubManagementServiceClient(
node=self.container.node)
# Additional code for creating a dummy instrument
"""
# Set up the preconditions. Look for an existing ingestion config
while True:
log.info("VisStreamLauncher:on_start: Waiting for an ingestion configuration to be available.")
ingestion_cfgs, _ = self.rrclient.find_resources(RT.IngestionConfiguration, None, None, True)
if len(ingestion_cfgs) > 0:
break
else:
gevent.sleep(1)
"""
# Check to see if the data_product already exists in the system (for e.g re launching the code after a crash)
dp_ids, _ = self.rrclient.find_resources(RT.DataProduct, None,
self.data_source_name, True)
if len(dp_ids) > 0:
data_product_id = dp_ids[0]
print '>>>>>>>>>>>>> Found dp_id = ', data_product_id
else:
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name=self.data_source_name,
description=self.data_source_name,
model_label=self.data_source_name)
instModel_id = self.imsclient.create_instrument_model(
instModel_obj)
# Create InstrumentDevice
instDevice_obj = IonObject(RT.InstrumentDevice,
name=self.data_source_name,
description=self.data_source_name,
serial_number="12345")
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, instDevice_id)
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(
container=ctd_stream_def)
print 'Creating new CDM data product with a stream definition'
dp_obj = IonObject(RT.DataProduct,
name=self.data_source_name,
description='ctd stream test')
data_product_id = self.dpclient.create_data_product(
dp_obj, ctd_stream_def_id)
self.damsclient.assign_data_product(
input_resource_id=instDevice_id,
data_product_id=data_product_id)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id,
persist_data=True,
persist_metadata=True)
print '>>>>>>>>>>>> New dp_id = ', data_product_id
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id,
PRED.hasStream, None, True)
if self.dataset == 'sinusoidal':
pid = self.container.spawn_process(
name='ctd_test.' + self.data_source_name,
module='ion.processes.data.sinusoidal_stream_publisher',
cls='SinusoidalCtdPublisher',
config={'process': {
'stream_id': stream_ids[0]
}})
else:
pid = self.container.spawn_process(
name='ctd_test.' + self.data_source_name,
module='ion.processes.data.ctd_stream_publisher',
cls='SimpleCtdPublisher',
config={'process': {
'stream_id': stream_ids[0]
}})
def on_quit(self):
log.debug("VizStreamProducer quit")
class TestCTDTransformsNoSim(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name+'_logger')
producer_definition.executable = {
'module':'ion.processes.data.stream_granule_logger',
'class':'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition)
configuration = {
'process':{
'stream_id':stream_id,
}
}
pid = self.processdispatchclient.schedule_process(process_definition_id= logger_procdef_id, configuration=configuration)
return pid
@unittest.skip('test not working')
def test_createTransformsThenPublishGranules(self):
# ctd simulator process
producer_definition = ProcessDefinition(name='Example Data Producer')
producer_definition.executable = {
'module':'ion.services.sa.test.simple_ctd_data_producer',
'class':'SimpleCtdDataProducer'
}
producer_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition)
#-------------------------------
# Create CTD Parsed as the first data product
#-------------------------------
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def)
print 'test_createTransformsThenActivateInstrument: new Stream Definition id = ', ctd_stream_def_id
print 'Creating new CDM data product with a stream definition'
craft = CoverageCraft
sdom, tdom = craft.create_domains()
sdom = sdom.dump()
tdom = tdom.dump()
parameter_dictionary = craft.create_parameters()
parameter_dictionary = parameter_dictionary.dump()
dp_obj = IonObject(RT.DataProduct,
name='ctd_parsed',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary)
print 'new ctd_parsed_data_product_id = ', ctd_parsed_data_product
#self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
#self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product))
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True)
print 'test_createTransformsThenActivateInstrument: Data product streams1 = ', stream_ids
self.parsed_stream_id = stream_ids[0]
#-------------------------------
# Create CTD Raw as the second data product
#-------------------------------
print 'test_createTransformsThenActivateInstrument: Creating new RAW data product with a stream definition'
raw_stream_def = SBE37_RAW_stream_definition()
raw_stream_def_id = self.pubsubclient.create_stream_definition(container=raw_stream_def)
dp_obj = IonObject(RT.DataProduct,
name='ctd_raw',
description='raw stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id, parameter_dictionary)
print 'new ctd_raw_data_product_id = ', ctd_raw_data_product
#self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_raw_data_product)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True)
print 'Data product streams2 = ', stream_ids
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition ctd_L0_all")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all',
process_source='some_source_reference')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" %ex)
#-------------------------------
# L1 Conductivity: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1ConductivityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_conductivity',
description='create the L1 conductivity data product',
module='ion.processes.data.transforms.ctd.ctd_L1_conductivity',
class_name='CTDL1ConductivityTransform',
process_source='CTDL1ConductivityTransform source code here...')
try:
ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1ConductivityTransform data process definition: %s" %ex)
#-------------------------------
# L1 Pressure: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1PressureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_pressure',
description='create the L1 pressure data product',
module='ion.processes.data.transforms.ctd.ctd_L1_pressure',
class_name='CTDL1PressureTransform',
process_source='CTDL1PressureTransform source code here...')
try:
ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1PressureTransform data process definition: %s" %ex)
#-------------------------------
# L1 Temperature: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1TemperatureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_temperature',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L1_temperature',
class_name='CTDL1TemperatureTransform',
process_source='CTDL1TemperatureTransform source code here...')
try:
ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1TemperatureTransform data process definition: %s" %ex)
#-------------------------------
# L2 Salinity: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition SalinityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_salinity',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform',
process_source='SalinityTransform source code here...')
try:
ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new SalinityTransform data process definition: %s" %ex)
#-------------------------------
# L2 Density: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition DensityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_density',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_density',
class_name='DensityTransform',
process_source='DensityTransform source code here...')
try:
ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new DensityTransform data process definition: %s" %ex)
self.loggerpids = []
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity = L0_conductivity_stream_definition()
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_conductivity, name='L0_Conductivity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id )
outgoing_stream_l0_pressure = L0_pressure_stream_definition()
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_pressure, name='L0_Pressure')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id )
outgoing_stream_l0_temperature = L0_temperature_stream_definition()
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_temperature, name='L0_Temperature')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id )
self.output_products={}
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject(RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id,
parameter_dictionary)
self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id)
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject(RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj, outgoing_stream_l0_pressure_id, parameter_dictionary)
self.output_products['pressure'] = ctd_l0_pressure_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id)
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject(RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj, outgoing_stream_l0_temperature_id, parameter_dictionary)
self.output_products['temperature'] = ctd_l0_temperature_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id)
#-------------------------------
# L1 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l1_conductivity = L1_conductivity_stream_definition()
outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_conductivity, name='L1_conductivity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id )
outgoing_stream_l1_pressure = L1_pressure_stream_definition()
outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_pressure, name='L1_Pressure')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id )
outgoing_stream_l1_temperature = L1_temperature_stream_definition()
outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_temperature, name='L1_Temperature')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id )
log.debug("test_createTransformsThenActivateInstrument: create output data product L1 conductivity")
ctd_l1_conductivity_output_dp_obj = IonObject(RT.DataProduct,
name='L1_Conductivity',
description='transform output L1 conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_conductivity_output_dp_obj, outgoing_stream_l1_conductivity_id, parameter_dictionary)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l1_conductivity_output_dp_id)
stream_ids, _ = self.rrclient.find_objects(ctd_l1_conductivity_output_dp_id, PRED.hasStream, None, True)
log.debug(" ctd_l1_conductivity stream id = %s", str(stream_ids) )
pid = self.create_logger(' ctd_l1_conductivity', stream_ids[0] )
self.loggerpids.append(pid)
log.debug("test_createTransformsThenActivateInstrument: create output data product L1 pressure")
ctd_l1_pressure_output_dp_obj = IonObject(RT.DataProduct,
name='L1_Pressure',
description='transform output L1 pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_pressure_output_dp_obj, outgoing_stream_l1_pressure_id, parameter_dictionary)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l1_pressure_output_dp_id)
stream_ids, _ = self.rrclient.find_objects(ctd_l1_pressure_output_dp_id, PRED.hasStream, None, True)
log.debug(" ctd_l1_pressure stream id = %s", str(stream_ids) )
pid = self.create_logger(' ctd_l1_pressure', stream_ids[0] )
self.loggerpids.append(pid)
log.debug("test_createTransformsThenActivateInstrument: create output data product L1 temperature")
ctd_l1_temperature_output_dp_obj = IonObject(RT.DataProduct,
name='L1_Temperature',
description='transform output L1 temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_temperature_output_dp_obj, outgoing_stream_l1_temperature_id, parameter_dictionary)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l1_temperature_output_dp_id)
stream_ids, _ = self.rrclient.find_objects(ctd_l1_temperature_output_dp_id, PRED.hasStream, None, True)
log.debug(" ctd_l1_temperature stream id = %s", str(stream_ids) )
pid = self.create_logger(' ctd_l1_temperature', stream_ids[0] )
self.loggerpids.append(pid)
#-------------------------------
# L2 Salinity - Density: Output Data Products
#-------------------------------
outgoing_stream_l2_salinity = L2_practical_salinity_stream_definition()
outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l2_salinity, name='L2_salinity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id )
outgoing_stream_l2_density = L2_density_stream_definition()
outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l2_density, name='L2_Density')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id )
log.debug("test_createTransformsThenActivateInstrument: create output data product L2 Salinity")
ctd_l2_salinity_output_dp_obj = IonObject(RT.DataProduct,
name='L2_Salinity',
description='transform output L2 salinity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l2_salinity_output_dp_id = self.dataproductclient.create_data_product(ctd_l2_salinity_output_dp_obj, outgoing_stream_l2_salinity_id, parameter_dictionary)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l2_salinity_output_dp_id)
log.debug("test_createTransformsThenActivateInstrument: create output data product L2 Density")
ctd_l2_density_output_dp_obj = IonObject(RT.DataProduct,
name='L2_Density',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l2_density_output_dp_id = self.dataproductclient.create_data_product(ctd_l2_density_output_dp_obj, outgoing_stream_l2_density_id, parameter_dictionary)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l2_density_output_dp_id)
# Set up subscribers/loggers to these streams
stream_ids, _ = self.rrclient.find_objects(ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True)
log.debug("L2 salinity stream id = %s", str(stream_ids) )
pid = self.create_logger('L2_salinity', stream_ids[0] )
self.loggerpids.append(pid)
stream_ids, _ = self.rrclient.find_objects(ctd_l2_density_output_dp_id, PRED.hasStream, None, True)
log.debug("L2 density stream id = %s", str(stream_ids) )
pid = self.create_logger('L2_density', stream_ids[0] )
self.loggerpids.append(pid)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L0 all data_process start")
try:
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products)
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L0 all data_process return")
#-------------------------------
# L1 Conductivity: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L1 Conductivity data_process start")
try:
l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_conductivity_dprocdef_id, [ctd_l0_conductivity_output_dp_id], {'output':ctd_l1_conductivity_output_dp_id})
self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L1 Conductivity data_process return")
#-------------------------------
# L1 Pressure: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process start")
try:
l1_pressure_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_pressure_dprocdef_id, [ctd_l0_pressure_output_dp_id], {'output':ctd_l1_pressure_output_dp_id})
self.dataprocessclient.activate_data_process(l1_pressure_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process return")
#-------------------------------
# L1 Temperature: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process start")
try:
l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_temperature_dprocdef_id, [ctd_l0_temperature_output_dp_id], {'output':ctd_l1_temperature_output_dp_id})
self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process return")
#-------------------------------
# L2 Salinity: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L2_salinity data_process start")
try:
l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_salinity_dprocdef_id, [ctd_parsed_data_product], {'output':ctd_l2_salinity_output_dp_id})
self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L2_salinity data_process return")
#-------------------------------
# L2 Density: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L2_Density data_process start")
try:
l2_density_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_density_dprocdef_id, [ctd_parsed_data_product], {'output':ctd_l2_density_output_dp_id})
self.dataprocessclient.activate_data_process(l2_density_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L2_Density data_process return")
#-------------------------------
# Streaming
#-------------------------------
# Start the ctd simulator to produce some data
configuration = {
'process':{
'stream_id':self.parsed_stream_id,
}
}
producer_pid = self.processdispatchclient.schedule_process(process_definition_id= producer_procdef_id, configuration=configuration)
time.sleep(2.0)
# clean up the launched processes
self.processdispatchclient.cancel_process(producer_pid)
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
class TestDataProductManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dpsc_cli = DataProductManagementServiceClient(node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.unsc = UserNotificationServiceClient()
self.data_retriever = DataRetrieverServiceClient()
#------------------------------------------
# Create the environment
#------------------------------------------
datastore_name = CACHE_DATASTORE_NAME
self.db = self.container.datastore_manager.get_datastore(datastore_name)
self.stream_def_id = self.pubsubcli.create_stream_definition(name='SBE37_CDM')
self.process_definitions = {}
ingestion_worker_definition = ProcessDefinition(name='ingestion worker')
ingestion_worker_definition.executable = {
'module':'ion.processes.data.ingestion.science_granule_ingestion_worker',
'class' :'ScienceGranuleIngestionWorker'
}
process_definition_id = self.process_dispatcher.create_process_definition(process_definition=ingestion_worker_definition)
self.process_definitions['ingestion_worker'] = process_definition_id
self.pids = []
self.exchange_points = []
self.exchange_names = []
#------------------------------------------------------------------------------------------------
# First launch the ingestors
#------------------------------------------------------------------------------------------------
self.exchange_space = 'science_granule_ingestion'
self.exchange_point = 'science_data'
config = DotDict()
config.process.datastore_name = 'datasets'
config.process.queue_name = self.exchange_space
self.exchange_names.append(self.exchange_space)
self.exchange_points.append(self.exchange_point)
pid = self.process_dispatcher.schedule_process(self.process_definitions['ingestion_worker'],configuration=config)
log.debug("the ingestion worker process id: %s", pid)
self.pids.append(pid)
self.addCleanup(self.cleaning_up)
def cleaning_up(self):
for pid in self.pids:
log.debug("number of pids to be terminated: %s", len(self.pids))
try:
self.process_dispatcher.cancel_process(pid)
log.debug("Terminated the process: %s", pid)
except:
log.debug("could not terminate the process id: %s" % pid)
IngestionManagementIntTest.clean_subscriptions()
for xn in self.exchange_names:
xni = self.container.ex_manager.create_xn_queue(xn)
xni.delete()
for xp in self.exchange_points:
xpi = self.container.ex_manager.create_xp(xp)
xpi.delete()
def get_datastore(self, dataset_id):
dataset = self.dataset_management.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def test_create_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
parameter_dictionary_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict')
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=parameter_dictionary_id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
# Generic time-series data domain creation
tdom, sdom = time_series_domain()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom.dump(),
spatial_domain = sdom.dump())
log.debug("Created an IonObject for a data product: %s" % dp_obj)
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product( data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
self.dpsc_cli.activate_data_product_persistence(dp_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
#------------------------------------------------------------------------------------------------
# test creating a new data product with a stream definition
#------------------------------------------------------------------------------------------------
log.debug('Creating new data product with a stream definition')
dp_obj = IonObject(RT.DataProduct,
name='DP2',
description='some new dp',
temporal_domain = tdom.dump(),
spatial_domain = sdom.dump())
dp_id2 = self.dpsc_cli.create_data_product(dp_obj, ctd_stream_def_id)
self.dpsc_cli.activate_data_product_persistence(dp_id2)
log.debug('new dp_id = %s' % dp_id2)
#------------------------------------------------------------------------------------------------
#make sure data product is associated with stream def
#------------------------------------------------------------------------------------------------
streamdefs = []
streams, _ = self.rrclient.find_objects(dp_id2, PRED.hasStream, RT.Stream, True)
for s in streams:
log.debug("Checking stream %s" % s)
sdefs, _ = self.rrclient.find_objects(s, PRED.hasStreamDefinition, RT.StreamDefinition, True)
for sd in sdefs:
log.debug("Checking streamdef %s" % sd)
streamdefs.append(sd)
self.assertIn(ctd_stream_def_id, streamdefs)
# test reading a non-existent data product
log.debug('reading non-existent data product')
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product('some_fake_id')
# update a data product (tests read also)
log.debug('Updating data product')
# first get the existing dp object
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# now tweak the object
dp_obj.description = 'the very first dp'
# now write the dp back to the registry
update_result = self.dpsc_cli.update_data_product(dp_obj)
# now get the dp back to see if it was updated
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertEquals(dp_obj.description,'the very first dp')
#test extension
extended_product = self.dpsc_cli.get_data_product_extension(dp_id)
self.assertEqual(dp_id, extended_product._id)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_product.computed.product_download_size_estimated.status)
self.assertEqual(0, extended_product.computed.product_download_size_estimated.value)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_product.computed.parameters.status)
#log.debug("test_create_data_product: parameters %s" % extended_product.computed.parameters.value)
# now 'delete' the data product
log.debug("deleting data product: %s" % dp_id)
self.dpsc_cli.delete_data_product(dp_id)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# Get the events corresponding to the data product
ret = self.unsc.get_recent_events(resource_id=dp_id)
events = ret.value
for event in events:
log.debug("event time: %s" % event.ts_created)
# self.assertTrue(len(events) > 0)
def test_data_product_stream_def(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
stream_def_id = self.dpsc_cli.get_data_product_stream_definition(dp_id)
self.assertEquals(ctd_stream_def_id, stream_def_id)
def test_activate_suspend_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
log.debug("Created an IonObject for a data product: %s" % dp_obj)
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test activate and suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.activate_data_product_persistence(dp_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True)
if not dataset_ids:
raise NotFound("Data Product %s dataset does not exist" % str(dp_id))
self.get_datastore(dataset_ids[0])
# Check that the streams associated with the data product are persisted with
stream_ids, _ = self.rrclient.find_objects(dp_id,PRED.hasStream,RT.Stream,True)
for stream_id in stream_ids:
self.assertTrue(self.ingestclient.is_persisted(stream_id))
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data = self.data_retriever.retrieve(dataset_ids[0])
self.assertIsInstance(replay_data, Granule)
log.debug("The data retriever was able to replay the dataset that was attached to the data product "
"we wanted to be persisted. Therefore the data product was indeed persisted with "
"otherwise we could not have retrieved its dataset using the data retriever. Therefore "
"this demonstration shows that L4-CI-SA-RQ-267 is satisfied: 'Data product management shall persist data products'")
data_product_object = self.rrclient.read(dp_id)
self.assertEquals(data_product_object.name,'DP1')
self.assertEquals(data_product_object.description,'some new dp')
log.debug("Towards L4-CI-SA-RQ-308: 'Data product management shall persist data product metadata'. "
" Attributes in create for the data product obj, name= '%s', description='%s', match those of object from the "
"resource registry, name='%s', desc='%s'" % (dp_obj.name, dp_obj.description,data_product_object.name,
data_product_object.description))
#------------------------------------------------------------------------------------------------
# test suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.suspend_data_product_persistence(dp_id)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
with self.assertRaises(NotFound):
dp_obj = self.rrclient.read(dp_id)
class TestExternalDatasetAgentMgmt(IonIntegrationTestCase):
# DataHandler config
DVR_CONFIG = {"dvr_mod": "ion.agents.data.handlers.base_data_handler", "dvr_cls": "DummyDataHandler"}
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url("res/deploy/r2deploy.yml")
log.debug("TestExternalDatasetAgentMgmt: started services")
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
# @unittest.skip('not yet working. fix activate_data_product_persistence()')
# @unittest.skip()
@unittest.skip("not working")
def test_activateDatasetAgent(self):
# Create ExternalDatasetModel
datsetModel_obj = IonObject(
RT.ExternalDatasetModel,
name="ExampleDatasetModel",
description="ExampleDatasetModel",
datset_type="FibSeries",
)
try:
datasetModel_id = self.damsclient.create_external_dataset_model(datsetModel_obj)
except BadRequest as ex:
self.fail("failed to create new ExternalDatasetModel: %s" % ex)
log.debug("TestExternalDatasetAgentMgmt: new ExternalDatasetModel id = %s", str(datasetModel_id))
# Create ExternalDatasetAgent
datasetAgent_obj = IonObject(
RT.ExternalDatasetAgent,
name="datasetagent007",
description="datasetagent007",
handler_module=EDA_MOD,
handler_class=EDA_CLS,
)
try:
datasetAgent_id = self.damsclient.create_external_dataset_agent(datasetAgent_obj, datasetModel_id)
except BadRequest as ex:
self.fail("failed to create new ExternalDatasetAgent: %s" % ex)
log.debug("TestExternalDatasetAgentMgmt: new ExternalDatasetAgent id = %s", str(datasetAgent_id))
# Create ExternalDataset
log.debug("TestExternalDatasetAgentMgmt: Create external dataset resource ")
extDataset_obj = IonObject(RT.ExternalDataset, name="ExtDataset", description="ExtDataset")
try:
extDataset_id = self.damsclient.create_external_dataset(extDataset_obj, datasetModel_id)
except BadRequest as ex:
self.fail("failed to create new external dataset resource: %s" % ex)
log.debug("TestExternalDatasetAgentMgmt: new ExternalDataset id = %s ", str(extDataset_id))
# register the dataset as a data producer
dproducer_id = self.damsclient.register_external_data_set(extDataset_id)
# create a stream definition for the data from the ctd simulator
pdict_id = self.dataset_management.read_parameter_dictionary_by_name("ctd_parsed_param_dict", id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name="SBE37_CDM", parameter_dictionary_id=pdict_id)
log.debug("TestExternalDatasetAgentMgmt: new Stream Definition id = %s", str(ctd_stream_def_id))
log.debug("TestExternalDatasetAgentMgmt: Creating new data product with a stream definition")
dp_obj = IonObject(RT.DataProduct, name="eoi dataset data", description=" stream test")
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(
RT.DataProduct, name="DP1", description="some new dp", temporal_domain=tdom, spatial_domain=sdom
)
data_product_id1 = self.dpclient.create_data_product(dp_obj, ctd_stream_def_id)
log.debug("TestExternalDatasetAgentMgmt: new dp_id = %s", str(data_product_id1))
self.damsclient.assign_data_product(input_resource_id=extDataset_id, data_product_id=data_product_id1)
# todo fix the problem here....
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True)
log.debug("TestExternalDatasetAgentMgmt: Data product streams1 = %s", str(stream_ids))
stream_id = stream_ids[0]
# Build a taxonomy for the dataset
tx = TaxyTool()
tx.add_taxonomy_set("data", "external_data")
# Augment the DVR_CONFIG with the necessary pieces
self.DVR_CONFIG["dh_cfg"] = {
"TESTING": True,
"stream_id": stream_id, # TODO: This should probably be a 'stream_config' dict with stream_name:stream_id members
"data_producer_id": dproducer_id,
# 'external_dataset_res':extDataset_obj, # Not needed - retrieved by EDA based on resource_id
"taxonomy": tx.dump(), # TODO: Currently does not support sets
"max_records": 4,
}
# Create agent config.
self._stream_config = {}
agent_config = {
"driver_config": self.DVR_CONFIG,
"stream_config": self._stream_config,
"agent": {"resource_id": EDA_RESOURCE_ID},
"test_mode": True,
}
extDatasetAgentInstance_obj = IonObject(
RT.ExternalDatasetAgentInstance,
name="DatasetAgentInstance",
description="DatasetAgentInstance",
dataset_driver_config=self.DVR_CONFIG,
dataset_agent_config=agent_config,
)
extDatasetAgentInstance_id = self.damsclient.create_external_dataset_agent_instance(
external_dataset_agent_instance=extDatasetAgentInstance_obj,
external_dataset_agent_id=datasetAgent_id,
external_dataset_id=extDataset_id,
)
log.debug("TestExternalDatasetAgentMgmt: Dataset agent instance obj: = %s", str(extDatasetAgentInstance_obj))
log.debug("TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s", str(extDatasetAgentInstance_id))
# Check that the instance is currently not active
id, active = self.damsclient.retrieve_external_dataset_agent_instance(extDataset_id)
log.debug(
"TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s active 1 = %s ", str(id), str(active)
)
self.damsclient.start_external_dataset_agent_instance(extDatasetAgentInstance_id)
dataset_agent_instance_obj = self.damsclient.read_external_dataset_agent_instance(extDatasetAgentInstance_id)
log.debug("TestExternalDatasetAgentMgmt: Dataset agent instance obj: = %s", str(dataset_agent_instance_obj))
# now the instance process should be active
id, active = self.damsclient.retrieve_external_dataset_agent_instance(extDataset_id)
log.debug(
"TestExternalDatasetAgentMgmt: Dataset agent instance id: = %s active 2 = %s ", str(id), str(active)
)
# Start a resource agent client to talk with the instrument agent.
self._dsa_client = ResourceAgentClient(extDataset_id, process=FakeProcess())
print "TestExternalDatasetAgentMgmt: got ia client %s", self._dsa_client
log.debug("TestExternalDatasetAgentMgmt: got dataset client %s", str(self._dsa_client))
# cmd=AgentCommand(command='initialize')
# _ = self._dsa_client.execute_agent(cmd)
#
# cmd = AgentCommand(command='go_active')
# _ = self._dsa_client.execute_agent(cmd)
#
# cmd = AgentCommand(command='run')
# _ = self._dsa_client.execute_agent(cmd)
#
# log.info('Send an unconstrained request for data (\'new data\')')
# cmd = AgentCommand(command='acquire_data')
# self._dsa_client.execute(cmd)
#
# log.info('Send a second unconstrained request for data (\'new data\'), should be rejected')
# cmd = AgentCommand(command='acquire_data')
# self._dsa_client.execute(cmd)
#
# cmd = AgentCommand(command='reset')
# _ = self._dsa_client.execute_agent(cmd)
# cmd = AgentCommand(command='get_current_state')
# retval = self._dsa_client.execute_agent(cmd)
# state = retval.result
# TODO: Think about what we really should be testing at this point
# The following is taken from ion.agents.data.test.test_external_dataset_agent.ExternalDatasetAgentTestBase.test_states()
# TODO: Do we also need to show data retrieval?
cmd = AgentCommand(command="get_current_state")
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command="initialize")
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command="get_current_state")
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command="go_active")
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command="get_current_state")
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command="run")
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command="get_current_state")
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command="pause")
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command="get_current_state")
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command="resume")
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command="get_current_state")
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command="clear")
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command="get_current_state")
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command="run")
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command="get_current_state")
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command="pause")
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command="get_current_state")
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.STOPPED)
cmd = AgentCommand(command="clear")
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command="get_current_state")
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command="run")
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command="get_current_state")
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
cmd = AgentCommand(command="reset")
retval = self._dsa_client.execute_agent(cmd)
cmd = AgentCommand(command="get_current_state")
retval = self._dsa_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
# -------------------------------
# Deactivate InstrumentAgentInstance
# -------------------------------
self.damsclient.stop_external_dataset_agent_instance(extDatasetAgentInstance_id)
def test_dataset_agent_prepare_support(self):
eda_sup = self.damsclient.prepare_external_dataset_agent_support()
eda_obj = IonObject(RT.ExternalDatasetAgent, name="ExternalDatasetAgent")
eda_id = self.damsclient.create_external_dataset_agent(eda_obj)
eda_sup = self.damsclient.prepare_external_dataset_agent_support(external_dataset_agent_id=eda_id)
edai_sup = self.damsclient.prepare_external_dataset_agent_instance_support()
edai_obj = IonObject(RT.ExternalDatasetAgentInstance, name="ExternalDatasetAgentInstance")
edai_id = self.damsclient.create_external_dataset_agent_instance(edai_obj)
edai_sup = self.damsclient.prepare_external_dataset_agent_instance_support(
external_dataset_agent_instance_id=edai_id
)
class TestDataProductManagementServiceCoverage(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
log.debug("Start rel from url")
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.DPMS = DataProductManagementServiceClient()
self.RR = ResourceRegistryServiceClient()
self.RR2 = EnhancedResourceRegistryClient(self.RR)
self.DAMS = DataAcquisitionManagementServiceClient()
self.PSMS = PubsubManagementServiceClient()
self.ingestclient = IngestionManagementServiceClient()
self.PD = ProcessDispatcherServiceClient()
self.DSMS = DatasetManagementServiceClient()
self.unsc = UserNotificationServiceClient()
self.data_retriever = DataRetrieverServiceClient()
#------------------------------------------
# Create the environment
#------------------------------------------
log.debug("get datastore")
datastore_name = CACHE_DATASTORE_NAME
self.db = self.container.datastore_manager.get_datastore(
datastore_name)
self.stream_def_id = self.PSMS.create_stream_definition(
name='SBE37_CDM')
self.process_definitions = {}
ingestion_worker_definition = ProcessDefinition(
name='ingestion worker')
ingestion_worker_definition.executable = {
'module':
'ion.processes.data.ingestion.science_granule_ingestion_worker',
'class': 'ScienceGranuleIngestionWorker'
}
process_definition_id = self.PD.create_process_definition(
process_definition=ingestion_worker_definition)
self.process_definitions['ingestion_worker'] = process_definition_id
self.pids = []
self.exchange_points = []
self.exchange_names = []
self.addCleanup(self.cleaning_up)
@staticmethod
def clean_subscriptions():
ingestion_management = IngestionManagementServiceClient()
pubsub = PubsubManagementServiceClient()
rr = ResourceRegistryServiceClient()
ingestion_config_ids = ingestion_management.list_ingestion_configurations(
id_only=True)
for ic in ingestion_config_ids:
subscription_ids, assocs = rr.find_objects(
subject=ic, predicate=PRED.hasSubscription, id_only=True)
for subscription_id, assoc in zip(subscription_ids, assocs):
rr.delete_association(assoc)
try:
pubsub.deactivate_subscription(subscription_id)
except:
log.exception("Unable to decativate subscription: %s",
subscription_id)
pubsub.delete_subscription(subscription_id)
def cleaning_up(self):
for pid in self.pids:
log.debug("number of pids to be terminated: %s", len(self.pids))
try:
self.PD.cancel_process(pid)
log.debug("Terminated the process: %s", pid)
except:
log.debug("could not terminate the process id: %s" % pid)
TestDataProductManagementServiceCoverage.clean_subscriptions()
for xn in self.exchange_names:
xni = self.container.ex_manager.create_xn_queue(xn)
xni.delete()
for xp in self.exchange_points:
xpi = self.container.ex_manager.create_xp(xp)
xpi.delete()
def test_CRUD_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
parameter_dictionary = self.DSMS.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict')
ctd_stream_def_id = self.PSMS.create_stream_definition(
name='Simulated CTD data',
parameter_dictionary_id=parameter_dictionary._id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
# Generic time-series data domain creation
tdom, sdom = time_series_domain()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain=tdom.dump(),
spatial_domain=sdom.dump())
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 10.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -10.0
dp_obj.ooi_product_name = "PRODNAME"
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
log.debug("create dataset")
dataset_id = self.RR2.create(any_old(RT.Dataset))
log.debug("dataset_id = %s", dataset_id)
log.debug("create data product 1")
dp_id = self.DPMS.create_data_product(
data_product=dp_obj,
stream_definition_id=ctd_stream_def_id,
dataset_id=dataset_id)
log.debug("dp_id = %s", dp_id)
# Assert that the data product has an associated stream at this stage
stream_ids, _ = self.RR.find_objects(dp_id, PRED.hasStream, RT.Stream,
True)
self.assertNotEquals(len(stream_ids), 0)
log.debug("read data product")
dp_obj = self.DPMS.read_data_product(dp_id)
log.debug("find data products")
self.assertIn(dp_id, [r._id for r in self.DPMS.find_data_products()])
log.debug("update data product")
dp_obj.name = "brand new"
self.DPMS.update_data_product(dp_obj)
self.assertEqual("brand new", self.DPMS.read_data_product(dp_id).name)
log.debug("activate/suspend persistence")
self.assertFalse(self.DPMS.is_persisted(dp_id))
self.DPMS.activate_data_product_persistence(dp_id)
self.addCleanup(self.DPMS.suspend_data_product_persistence,
dp_id) # delete op will do this for us
self.assertTrue(self.DPMS.is_persisted(dp_id))
log.debug("check error on checking persistence of nonexistent stream")
#with self.assertRaises(NotFound):
if True:
self.DPMS.is_persisted(self.RR2.create(any_old(RT.DataProduct)))
#log.debug("get extension")
#self.DPMS.get_data_product_extension(dp_id)
log.debug("getting last update")
lastupdate = self.DPMS.get_last_update(dp_id)
self.assertEqual(
{}, lastupdate) # should be no updates to a new data product
log.debug("prepare resource support")
support = self.DPMS.prepare_data_product_support(dp_id)
self.assertIsNotNone(support)
log.debug("delete data product")
self.DPMS.delete_data_product(dp_id)
log.debug("try to suspend again")
self.DPMS.suspend_data_product_persistence(dp_id)
# try basic create
log.debug("create without a dataset")
dp_id2 = self.DPMS.create_data_product_(any_old(RT.DataProduct))
self.assertIsNotNone(dp_id2)
log.debug("activate product %s", dp_id2)
self.assertRaises(BadRequest,
self.DPMS.activate_data_product_persistence, dp_id2)
#self.assertNotEqual(0, len(self.RR2.find_dataset_ids_of_data_product_using_has_dataset(dp_id2)))
log.debug("force delete data product")
self.DPMS.force_delete_data_product(dp_id2)
log.debug("test complete")
class TestActivateInstrumentIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
super(TestActivateInstrumentIntegration, self).setUp()
config = DotDict()
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml', config)
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataretrieverclient = DataRetrieverServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.usernotificationclient = UserNotificationServiceClient()
#setup listerner vars
self._data_greenlets = []
self._no_samples = None
self._samples_received = []
self.event_publisher = EventPublisher()
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name+'_logger')
producer_definition.executable = {
'module':'ion.processes.data.stream_granule_logger',
'class':'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition)
configuration = {
'process':{
'stream_id':stream_id,
}
}
pid = self.processdispatchclient.schedule_process(process_definition_id=logger_procdef_id,
configuration=configuration)
return pid
def _create_notification(self, user_name = '', instrument_id='', product_id=''):
#--------------------------------------------------------------------------------------
# Make notification request objects
#--------------------------------------------------------------------------------------
notification_request_1 = NotificationRequest( name= 'notification_1',
origin=instrument_id,
origin_type="instrument",
event_type='ResourceLifecycleEvent')
notification_request_2 = NotificationRequest( name='notification_2',
origin=product_id,
origin_type="data product",
event_type='DetectionEvent')
#--------------------------------------------------------------------------------------
# Create a user and get the user_id
#--------------------------------------------------------------------------------------
user = UserInfo()
user.name = user_name
user.contact.email = '*****@*****.**' % user_name
user_id, _ = self.rrclient.create(user)
#--------------------------------------------------------------------------------------
# Create notification
#--------------------------------------------------------------------------------------
self.usernotificationclient.create_notification(notification=notification_request_1, user_id=user_id)
self.usernotificationclient.create_notification(notification=notification_request_2, user_id=user_id)
log.debug( "test_activateInstrumentSample: create_user_notifications user_id %s", str(user_id) )
return user_id
def get_datastore(self, dataset_id):
dataset = self.datasetclient.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def _check_computed_attributes_of_extended_instrument(self, expected_instrument_device_id = '',extended_instrument = None):
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(extended_instrument.computed.last_data_received_datetime, ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue)
self.assertIsInstance(extended_instrument.computed.power_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.communications_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.data_status_roll_up, ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.location_status_roll_up, ComputedIntValue)
# the following assert will not work without elasticsearch.
#self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) )
# Verify the computed attribute for user notification requests
self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) )
notifications = extended_instrument.computed.user_notification_requests.value
notification = notifications[0]
self.assertEqual(expected_instrument_device_id, notification.origin)
self.assertEqual("instrument", notification.origin_type)
self.assertEqual('ResourceLifecycleEvent', notification.event_type)
def _check_computed_attributes_of_extended_product(self, expected_data_product_id = '', extended_data_product = None):
self.assertEqual(expected_data_product_id, extended_data_product._id)
log.debug("extended_data_product.computed: %s", extended_data_product.computed)
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(extended_data_product.computed.product_download_size_estimated, ComputedFloatValue)
self.assertIsInstance(extended_data_product.computed.number_active_subscriptions, ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.data_url, ComputedStringValue)
self.assertIsInstance(extended_data_product.computed.stored_data_size, ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.recent_granules, ComputedDictValue)
self.assertIsInstance(extended_data_product.computed.parameters, ComputedListValue)
self.assertIsInstance(extended_data_product.computed.recent_events, ComputedEventListValue)
self.assertIsInstance(extended_data_product.computed.provenance, ComputedDictValue)
self.assertIsInstance(extended_data_product.computed.user_notification_requests, ComputedListValue)
self.assertIsInstance(extended_data_product.computed.active_user_subscriptions, ComputedListValue)
self.assertIsInstance(extended_data_product.computed.past_user_subscriptions, ComputedListValue)
self.assertIsInstance(extended_data_product.computed.last_granule, ComputedDictValue)
self.assertIsInstance(extended_data_product.computed.is_persisted, ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.data_contents_updated, ComputedStringValue)
self.assertIsInstance(extended_data_product.computed.data_datetime, ComputedListValue)
# exact text here keeps changing to fit UI capabilities. keep assertion general...
self.assertEqual( 2, len(extended_data_product.computed.data_datetime.value) )
notifications = extended_data_product.computed.user_notification_requests.value
notification = notifications[0]
self.assertEqual(expected_data_product_id, notification.origin)
self.assertEqual("data product", notification.origin_type)
self.assertEqual('DetectionEvent', notification.event_type)
@attr('LOCOINT')
#@unittest.skip('refactoring')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
@patch.dict(CFG, {'endpoint':{'receive':{'timeout': 90}}})
def test_activateInstrumentSample(self):
self.loggerpids = []
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
log.debug( 'new InstrumentModel id = %s ', instModel_id)
raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='raw')
parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict')
# Create InstrumentAgent
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri=DRV_URI_GOOD,
stream_configurations = [raw_config, parsed_config])
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
log.debug('new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ')
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345" )
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) " , instDevice_id)
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config = port_agent_config,
alerts= [])
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj,
instAgent_id,
instDevice_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubcli.create_stream_definition(name='parsed', parameter_dictionary_id=parsed_pdict_id)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('raw', id_only=True)
raw_stream_def_id = self.pubsubcli.create_stream_definition(name='raw', parameter_dictionary_id=raw_pdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id1 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s' , data_product_id1)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True)
log.debug('Data product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True)
log.debug('Data set for data_product_id1 = %s' , dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
pid = self.create_logger('ctd_parsed', stream_ids[0] )
self.loggerpids.append(pid)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id2 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id)
log.debug('new dp_id = %s', data_product_id2)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2)
# setup notifications for the device and parsed data product
user_id_1 = self._create_notification( user_name='user_1', instrument_id=instDevice_id, product_id=data_product_id1)
#---------- Create notifications for another user and verify that we see different computed subscriptions for the two users ---------
user_id_2 = self._create_notification( user_name='user_2', instrument_id=instDevice_id, product_id=data_product_id2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True)
log.debug('Data product streams2 = %s' , str(stream_ids))
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True)
log.debug('Data set for data_product_id2 = %s' , dataset_ids[0])
self.raw_dataset = dataset_ids[0]
def start_instrument_agent():
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
gevent.joinall([gevent.spawn(start_instrument_agent)])
#cleanup
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
#wait for start
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
gate = AgentProcessStateGate(self.processdispatchclient.read_process,
instDevice_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" %
gate.process_id)
#log.trace('Instrument agent instance obj: = %s' , str(inst_agent_instance_obj))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(instDevice_id,
to_name=gate.process_id,
process=FakeProcess())
log.debug("test_activateInstrumentSample: got ia client %s" , str(self._ia_client))
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: initialize %s" , str(retval))
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.INACTIVE, state)
log.debug("(L4-CI-SA-RQ-334): Sending go_active command ")
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrument: return value from go_active %s" , str(reply))
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.IDLE, state)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("(L4-CI-SA-RQ-334): current state after sending go_active command %s" , str(state))
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: run %s" , str(reply))
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.COMMAND, state)
cmd = AgentCommand(command=ResourceAgentEvent.PAUSE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.STOPPED, state)
cmd = AgentCommand(command=ResourceAgentEvent.RESUME)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.COMMAND, state)
cmd = AgentCommand(command=ResourceAgentEvent.CLEAR)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.IDLE, state)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(ResourceAgentState.COMMAND, state)
for i in xrange(10):
monitor = DatasetMonitor(dataset_id=self.parsed_dataset)
self._ia_client.execute_resource(AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE))
if not monitor.wait():
raise AssertionError('Failed on the %ith granule' % i)
monitor.stop()
# cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
# for i in xrange(10):
# retval = self._ia_client.execute_resource(cmd)
# log.debug("test_activateInstrumentSample: return from sample %s" , str(retval))
log.debug( "test_activateInstrumentSample: calling reset ")
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: return from reset %s" , str(reply))
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data_raw = self.dataretrieverclient.retrieve(self.raw_dataset)
self.assertIsInstance(replay_data_raw, Granule)
rdt_raw = RecordDictionaryTool.load_from_granule(replay_data_raw)
log.debug("RDT raw: %s", str(rdt_raw.pretty_print()) )
self.assertIn('raw', rdt_raw)
raw_vals = rdt_raw['raw']
all_raw = "".join(raw_vals)
# look for 't' entered after a prompt -- ">t"
t_commands = all_raw.count(">t")
if 10 != t_commands:
log.error("%s raw_vals: ", len(raw_vals))
for i, r in enumerate(raw_vals): log.error("raw val %s: %s", i, [r])
self.fail("Expected 10 't' strings in raw_vals, got %s" % t_commands)
else:
log.debug("%s raw_vals: ", len(raw_vals))
for i, r in enumerate(raw_vals): log.debug("raw val %s: %s", i, [r])
replay_data_parsed = self.dataretrieverclient.retrieve(self.parsed_dataset)
self.assertIsInstance(replay_data_parsed, Granule)
rdt_parsed = RecordDictionaryTool.load_from_granule(replay_data_parsed)
log.debug("test_activateInstrumentSample: RDT parsed: %s", str(rdt_parsed.pretty_print()) )
self.assertIn('temp', rdt_parsed)
temp_vals = rdt_parsed['temp']
pressure_vals = rdt_parsed['pressure']
if 10 != len(temp_vals):
log.error("%s temp_vals: %s", len(temp_vals), temp_vals)
self.fail("Expected 10 temp_vals, got %s" % len(temp_vals))
log.debug("l4-ci-sa-rq-138")
"""
Physical resource control shall be subject to policy
Instrument management control capabilities shall be subject to policy
The actor accessing the control capabilities must be authorized to send commands.
note from maurice 2012-05-18: Talk to tim M to verify that this is policy. If it is then talk with Stephen to
get an example of a policy test and use that to create a test stub that will be
completed when we have instrument policies.
Tim M: The "actor", aka observatory operator, will access the instrument through ION.
"""
#--------------------------------------------------------------------------------
# Get the extended data product to see if it contains the granules
#--------------------------------------------------------------------------------
extended_product = self.dpclient.get_data_product_extension(data_product_id=data_product_id1, user_id=user_id_1)
def poller(extended_product):
return len(extended_product.computed.user_notification_requests.value) == 1
poll(poller, extended_product, timeout=30)
self._check_computed_attributes_of_extended_product( expected_data_product_id = data_product_id1, extended_data_product = extended_product)
#--------------------------------------------------------------------------------
# Get the extended instrument
#--------------------------------------------------------------------------------
extended_instrument = self.imsclient.get_instrument_device_extension(instrument_device_id=instDevice_id, user_id=user_id_1)
#--------------------------------------------------------------------------------
# For the second user, check the extended data product and the extended intrument
#--------------------------------------------------------------------------------
extended_product = self.dpclient.get_data_product_extension(data_product_id=data_product_id2, user_id=user_id_2)
self._check_computed_attributes_of_extended_product(expected_data_product_id = data_product_id2, extended_data_product = extended_product)
#--------------------------------------------------------------------------------
# Get the extended instrument
#--------------------------------------------------------------------------------
extended_instrument = self.imsclient.get_instrument_device_extension(instrument_device_id=instDevice_id, user_id=user_id_2)
self._check_computed_attributes_of_extended_instrument(expected_instrument_device_id = instDevice_id, extended_instrument = extended_instrument)
#--------------------------------------------------------------------------------
# Deactivate loggers
#--------------------------------------------------------------------------------
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
self.dpclient.delete_data_product(data_product_id1)
self.dpclient.delete_data_product(data_product_id2)
class TestDataProductProvenance(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.dpmsclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
# create missing data process definition
dpd_obj = IonObject(RT.DataProcessDefinition,
name=LOGICAL_TRANSFORM_DEFINITION_NAME,
description="normally in preload",
module='ion.processes.data.transforms.logical_transform',
class_name='logical_transform')
self.dataprocessclient.create_data_process_definition(dpd_obj)
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]:
self.dataprocessclient.deactivate_data_process(proc_id)
self.dataprocessclient.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
#@unittest.skip('not ready')
def test_get_provenance(self):
#create a deployment with metadata and an initial site and device
instrument_site_obj = IonObject(RT.InstrumentSite,
name='InstrumentSite1',
description='test instrument site')
instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, "")
log.debug( 'test_get_provenance: new instrument_site_id id = %s ', str(instrument_site_id))
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel" )
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" %ex)
log.debug( 'test_get_provenance: new InstrumentModel id = %s ', str(instModel_id))
self.omsclient.assign_instrument_model_to_instrument_site(instModel_id, instrument_site_id)
# Create InstrumentAgent
parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5 )
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg",
stream_configurations = [parsed_config] )
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" %ex)
log.debug( 'test_get_provenance:new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug('test_get_provenance: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ')
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
try:
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" %ex)
log.debug("test_get_provenance: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id)
#-------------------------------
# Create CTD Parsed data product
#-------------------------------
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubclient.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id)
log.debug( 'test_get_provenance:Creating new CDM data product with a stream definition')
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_parsed_data_product = self.dpmsclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s', ctd_parsed_data_product)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
self.dpmsclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product)
#-------------------------------
# create a data product for the site to pass the OMS check.... we need to remove this check
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
log_data_product_id = self.dpmsclient.create_data_product(dp_obj, parsed_stream_def_id)
self.omsclient.create_site_data_product(instrument_site_id, log_data_product_id)
#-------------------------------
# Deploy instrument device to instrument site
#-------------------------------
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment')
deployment_id = self.omsclient.create_deployment(deployment_obj)
self.omsclient.deploy_instrument_site(instrument_site_id, deployment_id)
self.imsclient.deploy_instrument_device(instDevice_id, deployment_id)
log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) )
self.omsclient.activate_deployment(deployment_id)
inst_device_objs, _ = self.rrclient.find_objects(subject=instrument_site_id, predicate=PRED.hasDevice, object_type=RT.InstrumetDevice, id_only=False)
log.debug("test_create_deployment: deployed device: %s ", str(inst_device_objs[0]) )
#-------------------------------
# Create the agent instance
#-------------------------------
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config = port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition ctd_L0_all")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" %ex)
#-------------------------------
# L1 Conductivity: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1ConductivityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_conductivity',
description='create the L1 conductivity data product',
module='ion.processes.data.transforms.ctd.ctd_L1_conductivity',
class_name='CTDL1ConductivityTransform')
try:
ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1ConductivityTransform data process definition: %s" %ex)
#-------------------------------
# L1 Pressure: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1PressureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_pressure',
description='create the L1 pressure data product',
module='ion.processes.data.transforms.ctd.ctd_L1_pressure',
class_name='CTDL1PressureTransform')
try:
ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1PressureTransform data process definition: %s" %ex)
#-------------------------------
# L1 Temperature: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1TemperatureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_temperature',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L1_temperature',
class_name='CTDL1TemperatureTransform')
try:
ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1TemperatureTransform data process definition: %s" %ex)
#-------------------------------
# L2 Salinity: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition SalinityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_salinity',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform')
try:
ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new SalinityTransform data process definition: %s" %ex)
#-------------------------------
# L2 Density: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition DensityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_density',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_density',
class_name='DensityTransform')
try:
ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new DensityTransform data process definition: %s" %ex)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' )
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' )
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' )
self.output_products={}
log.debug("TestDataProductProvenance: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id)
self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id
log.debug("TestDataProductProvenance: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_pressure_output_dp_obj,
outgoing_stream_l0_pressure_id)
self.output_products['pressure'] = ctd_l0_pressure_output_dp_id
log.debug("TestDataProductProvenance: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_temperature_output_dp_obj,
outgoing_stream_l0_temperature_id)
self.output_products['temperature'] = ctd_l0_temperature_output_dp_id
#-------------------------------
# L1 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(name='L1_conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id, binding='conductivity' )
outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(name='L1_Pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id, binding='pressure' )
outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(name='L1_Temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id, binding='temperature' )
log.debug("TestDataProductProvenance: create output data product L1 conductivity")
ctd_l1_conductivity_output_dp_obj = IonObject(RT.DataProduct,
name='L1_Conductivity',
description='transform output L1 conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_conductivity_output_dp_obj,
outgoing_stream_l1_conductivity_id)
log.debug("TestDataProductProvenance: create output data product L1 pressure")
ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Pressure',
description='transform output L1 pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_pressure_output_dp_obj,
outgoing_stream_l1_pressure_id)
log.debug("TestDataProductProvenance: create output data product L1 temperature")
ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Temperature',
description='transform output L1 temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_temperature_output_dp_obj,
outgoing_stream_l1_temperature_id)
#-------------------------------
# L2 Salinity - Density: Output Data Products
#-------------------------------
outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(name='L2_salinity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id, binding='salinity' )
outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(name='L2_Density', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id, binding='density' )
log.debug("TestDataProductProvenance: create output data product L2 Salinity")
ctd_l2_salinity_output_dp_obj = IonObject( RT.DataProduct,
name='L2_Salinity',
description='transform output L2 salinity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l2_salinity_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_salinity_output_dp_obj,
outgoing_stream_l2_salinity_id)
log.debug("TestDataProductProvenance: create output data product L2 Density")
# ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct,
# name='L2_Density',
# description='transform output pressure',
# temporal_domain = tdom,
# spatial_domain = sdom)
#
# ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj,
# outgoing_stream_l2_density_id,
# parameter_dictionary)
contactInfo = ContactInformation()
contactInfo.individual_names_given = "Bill"
contactInfo.individual_name_family = "Smith"
contactInfo.street_address = "111 First St"
contactInfo.city = "San Diego"
contactInfo.email = "*****@*****.**"
contactInfo.phones = ["858-555-6666"]
contactInfo.country = "USA"
contactInfo.postal_code = "92123"
ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct,
name='L2_Density',
description='transform output pressure',
contacts = [contactInfo],
iso_topic_category = "my_iso_topic_category_here",
quality_control_level = "1",
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj,
outgoing_stream_l2_density_id)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L0 all data_process start")
try:
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products)
#activate only this data process just for coverage
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
contents = "this is the lookup table contents, replace with a file..."
att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII)
deviceAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att)
log.info( 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = %s', deviceAttachment)
log.debug("TestDataProductProvenance: create L0 all data_process return")
#-------------------------------
# L1 Conductivity: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1 Conductivity data_process start")
try:
l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_conductivity_dprocdef_id, [ctd_l0_conductivity_output_dp_id], {'conductivity':ctd_l1_conductivity_output_dp_id})
self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L1 Pressure: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1_Pressure data_process start")
try:
l1_pressure_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_pressure_dprocdef_id, [ctd_l0_pressure_output_dp_id], {'pressure':ctd_l1_pressure_output_dp_id})
self.dataprocessclient.activate_data_process(l1_pressure_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L1 Temperature: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1_Pressure data_process start")
try:
l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_temperature_dprocdef_id, [ctd_l0_temperature_output_dp_id], {'temperature':ctd_l1_temperature_output_dp_id})
self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L2 Salinity: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L2_salinity data_process start")
try:
l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_salinity_dprocdef_id, [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id], {'salinity':ctd_l2_salinity_output_dp_id})
self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L2 Density: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L2_Density data_process start")
try:
l2_density_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_density_dprocdef_id, [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id], {'density':ctd_l2_density_output_dp_id})
self.dataprocessclient.activate_data_process(l2_density_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# Launch InstrumentAgentInstance, connect to the resource agent client
#-------------------------------
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
print 'TestDataProductProvenance: Instrument agent instance obj: = ', inst_agent_instance_obj
# Start a resource agent client to talk with the instrument agent.
# self._ia_client = ResourceAgentClient('iaclient', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess())
# print 'activate_instrument: got ia client %s', self._ia_client
# log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client))
#-------------------------------
# Deactivate InstrumentAgentInstance
#-------------------------------
self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
self.dataprocessclient.deactivate_data_process(l2_density_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l2_salinity_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_temperature_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_pressure_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_conductivity_data_process_id)
self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id)
#-------------------------------
# Retrieve the provenance info for the ctd density data product
#-------------------------------
provenance_dict = self.dpmsclient.get_data_product_provenance(ctd_l2_density_output_dp_id)
log.debug("TestDataProductProvenance: provenance_dict %s", str(provenance_dict))
#validate that products are represented
self.assertTrue (provenance_dict[str(ctd_l1_conductivity_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l0_conductivity_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l2_density_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l1_temperature_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l0_temperature_output_dp_id)])
density_dict = (provenance_dict[str(ctd_l2_density_output_dp_id)])
self.assertEquals(density_dict['producer'], [l2_density_all_data_process_id])
#-------------------------------
# Retrieve the extended resource for this data product
#-------------------------------
extended_product = self.dpmsclient.get_data_product_extension(ctd_l2_density_output_dp_id)
self.assertEqual(1, len(extended_product.data_processes) )
self.assertEqual(3, len(extended_product.process_input_data_products) )
# log.debug("TestDataProductProvenance: DataProduct provenance_product_list %s", str(extended_product.provenance_product_list))
# log.debug("TestDataProductProvenance: DataProduct data_processes %s", str(extended_product.data_processes))
# log.debug("TestDataProductProvenance: DataProduct process_input_data_products %s", str(extended_product.process_input_data_products))
# log.debug("TestDataProductProvenance: provenance %s", str(extended_product.computed.provenance.value))
#-------------------------------
# Retrieve the extended resource for this data process
#-------------------------------
extended_process_def = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id)
# log.debug("TestDataProductProvenance: DataProcess extended_process_def %s", str(extended_process_def))
# log.debug("TestDataProductProvenance: DataProcess data_processes %s", str(extended_process_def.data_processes))
# log.debug("TestDataProductProvenance: DataProcess data_products %s", str(extended_process_def.data_products))
self.assertEqual(1, len(extended_process_def.data_processes) )
self.assertEqual(3, len(extended_process_def.output_stream_definitions) )
self.assertEqual(3, len(extended_process_def.data_products) ) #one list because of one data process
#-------------------------------
# Request the xml report
#-------------------------------
results = self.dpmsclient.get_data_product_provenance_report(ctd_l2_density_output_dp_id)
#-------------------------------
# Cleanup
#-------------------------------
self.dpmsclient.delete_data_product(ctd_parsed_data_product)
self.dpmsclient.delete_data_product(log_data_product_id)
self.dpmsclient.delete_data_product(ctd_l0_conductivity_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l0_pressure_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l0_temperature_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l1_conductivity_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l1_pressure_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l1_temperature_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l2_salinity_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l2_density_output_dp_id)
class TestDriverEgg(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataretrieverclient = DataRetrieverServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
#setup listerner vars
self._data_greenlets = []
self._no_samples = None
self._samples_received = []
self.event_publisher = EventPublisher()
def get_streamConfigs(self):
raw_config = StreamConfiguration(stream_name='raw',
parameter_dictionary_name='ctd_raw_param_dict')
parsed_config = StreamConfiguration(stream_name='parsed',
parameter_dictionary_name='ctd_parsed_param_dict')
return raw_config, parsed_config
##########################
#
# The following tests generate different agent configs and pass them to a common base test script
#
###########################
@unittest.skip("this test can't be run from coi services. it is missing dependencies")
def test_driverLaunchModuleNoURI(self):
raw_config, parsed_config = self.get_streamConfigs()
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
driver_class="SBE37Driver",
stream_configurations = [raw_config, parsed_config])
self.base_activateInstrumentSample(instAgent_obj)
def test_driverLaunchModuleWithURI(self):
raw_config, parsed_config = self.get_streamConfigs()
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
driver_class="SBE37Driver",
driver_uri=DRV_URI_GOOD,
stream_configurations = [raw_config, parsed_config])
self.base_activateInstrumentSample(instAgent_obj)
def test_driverLaunchNoModuleOnlyURI(self):
raw_config, parsed_config = self.get_streamConfigs()
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
#driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
#driver_class="SBE37Driver",
driver_uri=DRV_URI_GOOD,
stream_configurations = [raw_config, parsed_config])
self.base_activateInstrumentSample(instAgent_obj)
def test_driverLaunchBogusModuleWithURI(self):
raw_config, parsed_config = self.get_streamConfigs()
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_module="bogus",
driver_class="Bogus",
driver_uri=DRV_URI_GOOD,
stream_configurations = [raw_config, parsed_config])
self.base_activateInstrumentSample(instAgent_obj)
@unittest.skip("Launches an egg 'process' even though the egg download should produce error 404")
def test_driverLaunchNoModule404URI(self):
raw_config, parsed_config = self.get_streamConfigs()
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
#driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
#driver_class="SBE37Driver",
driver_uri=DRV_URI_404,
stream_configurations = [raw_config, parsed_config])
self.base_activateInstrumentSample(instAgent_obj, False)
def test_driverLaunchNoModuleBadEggURI(self):
raw_config, parsed_config = self.get_streamConfigs()
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
#driver_module="mi.instrument.seabird.sbe37smb.ooicore.driver",
#driver_class="SBE37Driver",
driver_uri=DRV_URI_BAD,
stream_configurations = [raw_config, parsed_config])
self.base_activateInstrumentSample(instAgent_obj, True, False)
def base_activateInstrumentSample(self, instAgent_obj, expect_launch=True, expect_command=True):
"""
This method runs a test of launching a driver with a given agent configuration
"""
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
print 'new InstrumentModel id = %s ' % instModel_id
# Create InstrumentAgent
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
print 'new InstrumentAgent id = %s' % instAgent_id
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345" )
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config = port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj,
instAgent_id,
instDevice_id)
parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',
id_only=True)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_raw_param_dict',
id_only=True)
parsed_stream_def_id = self.pubsubcli.create_stream_definition(name='parsed',
parameter_dictionary_id=parsed_pdict_id)
raw_stream_def_id = self.pubsubcli.create_stream_definition(name='raw',
parameter_dictionary_id=raw_pdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test')
data_product_id1 = self.dpclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
print 'new dp_id = %s' % data_product_id1
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id1)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasStream, None, True)
print 'Data product streams1 = %s' % stream_ids
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True)
print 'Data set for data_product_id1 = %s' % dataset_ids[0]
self.parsed_dataset = dataset_ids[0]
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test')
data_product_id2 = self.dpclient.create_data_product(data_product=dp_obj,
stream_definition_id=raw_stream_def_id)
print 'new dp_id = %s' % str(data_product_id2)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(data_product_id=data_product_id2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasStream, None, True)
print 'Data product streams2 = %s' % str(stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id2, PRED.hasDataset, RT.Dataset, True)
print 'Data set for data_product_id2 = %s' % dataset_ids[0]
self.raw_dataset = dataset_ids[0]
# add start/stop for instrument agent
gevent.joinall([gevent.spawn(lambda:
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id))])
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
#wait for start
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
agent_process_id = ResourceAgentClient._get_agent_process_id(instDevice_id)
print "Agent process id is '%s'" % str(agent_process_id)
self.assertTrue(agent_process_id)
gate = ProcessStateGate(self.processdispatchclient.read_process,
agent_process_id,
ProcessStateEnum.RUNNING)
if not expect_launch:
self.assertFalse(gate.await(30), "The instance (%s) of bogus instrument agent spawned in 30 seconds ?!?" %
agent_process_id)
return
self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" %
agent_process_id)
print "Instrument Agent Instance successfully triggered ProcessStateGate as RUNNING"
#print 'Instrument agent instance obj: = %s' % str(inst_agent_instance_obj)
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(instDevice_id,
to_name=agent_process_id,
process=FakeProcess())
print "ResourceAgentClient created: %s" % str(self._ia_client)
print "Sending command=ResourceAgentEvent.INITIALIZE"
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
if not expect_command:
self.assertRaises(ServerError, self._ia_client.execute_agent, cmd)
return
retval = self._ia_client.execute_agent(cmd)
print "Result of INITIALIZE: %s" % str(retval)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertTrue(state, 'DRIVER_STATE_COMMAND')
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client.execute_agent(cmd)
self.assertTrue(reply.status == 0)
cmd = AgentCommand(command=SBE37ProtocolEvent.START_AUTOSAMPLE)
retval = self._ia_client.execute_resource(cmd)
# This gevent sleep is there to test the autosample time, which will show something different from default
# only if the instrument runs for over a minute
gevent.sleep(90)
extended_instrument = self.imsclient.get_instrument_device_extension(instrument_device_id=instDevice_id)
self.assertIsInstance(extended_instrument.computed.uptime, ComputedStringValue)
autosample_string = extended_instrument.computed.uptime.value
autosampling_time = int(autosample_string.split()[4])
self.assertTrue(autosampling_time > 0)
cmd = AgentCommand(command=SBE37ProtocolEvent.STOP_AUTOSAMPLE)
retval = self._ia_client.execute_resource(cmd)
print "Sending command=ResourceAgentEvent.RESET"
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client.execute_agent(cmd)
print "Result of RESET: %s" % str(reply)
class ExhaustiveParameterTest(IonIntegrationTestCase):
def setUp(self):
self.i = 0
self._start_container()
self.container.start_rel_from_url('res/deploy/r2params.yml')
self.dataset_management = DatasetManagementServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
self.resource_registry = self.container.resource_registry
self.data_retriever = DataRetrieverServiceClient()
pdicts, _ = self.resource_registry.find_resources(
restype='ParameterDictionary', id_only=False)
self.dp_ids = []
for pdict in pdicts:
stream_def_id = self.pubsub_management.create_stream_definition(
pdict.name, parameter_dictionary_id=pdict._id)
dp_id = self.make_dp(stream_def_id)
if dp_id: self.dp_ids.append(dp_id)
def make_dp(self, stream_def_id):
tdom, sdom = time_series_domain()
tdom = tdom.dump()
sdom = sdom.dump()
stream_def = self.resource_registry.read(stream_def_id)
dp_obj = DataProduct(name=stream_def.name,
description=stream_def.name,
processing_level_code='Parsed_Canonical',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id = self.data_product_management.create_data_product(
dp_obj, stream_definition_id=stream_def_id)
self.data_product_management.activate_data_product_persistence(
data_product_id)
return data_product_id
def fill_values(self, ptype, size):
if isinstance(ptype, ArrayType):
return ['blah'] * size
elif isinstance(ptype, QuantityType):
return np.sin(
np.arange(size, dtype=ptype.value_encoding) * 2 * np.pi / 3)
elif isinstance(ptype, RecordType):
return [{'record': 'ok'}] * size
elif isinstance(ptype, ConstantRangeType):
return (1, 1000)
elif isinstance(ptype, ConstantType):
return np.dtype(ptype.value_encoding).type(1)
elif isinstance(ptype, CategoryType):
return ptype.categories.keys()[0]
else:
return
def wait_until_we_have_enough_granules(self, dataset_id='', data_size=40):
'''
Loops until there is a sufficient amount of data in the dataset
'''
done = False
with gevent.Timeout(40):
while not done:
granule = self.data_retriever.retrieve_last_data_points(
dataset_id, 1)
rdt = RecordDictionaryTool.load_from_granule(granule)
extents = self.dataset_management.dataset_extents(
dataset_id, rdt._pdict.temporal_parameter_name)[0]
if rdt[rdt._pdict.temporal_parameter_name] and rdt[
rdt._pdict.
temporal_parameter_name][0] != rdt._pdict.get_context(
rdt._pdict.temporal_parameter_name
).fill_value and extents >= data_size:
done = True
else:
gevent.sleep(0.2)
def write_to_data_product(self, data_product_id):
dataset_ids, _ = self.resource_registry.find_objects(data_product_id,
'hasDataset',
id_only=True)
dataset_id = dataset_ids.pop()
stream_ids, _ = self.resource_registry.find_objects(data_product_id,
'hasStream',
id_only=True)
stream_id = stream_ids.pop()
stream_def_ids, _ = self.resource_registry.find_objects(
stream_id, 'hasStreamDefinition', id_only=True)
stream_def_id = stream_def_ids.pop()
route = self.pubsub_management.read_stream_route(stream_id)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
time_param = rdt._pdict.temporal_parameter_name
if time_param is None:
print '%s has no temporal parameter' % self.resource_registry.read(
data_product_id).name
return
rdt[time_param] = np.arange(40)
for field in rdt.fields:
if field == rdt._pdict.temporal_parameter_name:
continue
rdt[field] = self.fill_values(
rdt._pdict.get_context(field).param_type, 40)
publisher = StandaloneStreamPublisher(stream_id, route)
publisher.publish(rdt.to_granule())
self.wait_until_we_have_enough_granules(dataset_id, 40)
granule = self.data_retriever.retrieve(dataset_id)
rdt_out = RecordDictionaryTool.load_from_granule(granule)
bad = []
for field in rdt.fields:
if not np.array_equal(rdt[field], rdt_out[field]):
print '%s' % field
print '%s != %s' % (rdt[field], rdt_out[field])
bad.append(field)
return bad
def test_data_products(self):
bad_data_products = {}
for dp_id in self.dp_ids:
try:
bad_fields = self.write_to_data_product(dp_id)
if bad_fields:
bad_data_products[
dp_id] = "Couldn't write and retrieve %s." % bad_fields
except:
import traceback
bad_data_products[dp_id] = traceback.format_exc()
for dp_id, tb in bad_data_products.iteritems():
print '----------'
print 'Problem with %s' % self.resource_registry.read(dp_id).name
print tb
print '----------'
if bad_data_products:
raise AssertionError('There are bad parameter dictionaries.')
class TestAgentLaunchOps(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
unittest # suppress an pycharm inspector error if all unittest.skip references are commented out
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.OMS = ObservatoryManagementServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
# @unittest.skip('this test just for debugging setup')
# def test_just_the_setup(self):
# return
def test_get_agent_client_noprocess(self):
inst_device_id = self.RR2.create(any_old(RT.InstrumentDevice))
iap = ResourceAgentClient._get_agent_process_id(inst_device_id)
# should be no running agent
self.assertIsNone(iap)
# should raise NotFound
self.assertRaises(NotFound, ResourceAgentClient, inst_device_id)
def test_resource_state_save_restore(self):
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.IMS.create_instrument_model(instModel_obj)
log.debug( 'new InstrumentModel id = %s ', instModel_id)
# Create InstrumentAgent
raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict' )
parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict')
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri=DRV_URI_GOOD,
stream_configurations = [raw_config, parsed_config] )
instAgent_id = self.IMS.create_instrument_agent(instAgent_obj)
log.debug( 'new InstrumentAgent id = %s', instAgent_id)
self.IMS.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 '
+ '(SA Req: L4-CI-SA-RQ-241) ')
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345" )
instDevice_id = self.IMS.create_instrument_device(instrument_device=instDevice_obj)
self.IMS.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ",
instDevice_id)
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config = port_agent_config)
instAgentInstance_id = self.IMS.create_instrument_agent_instance(instAgentInstance_obj,
instAgent_id,
instDevice_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
spdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.PSC.create_stream_definition(name='parsed', parameter_dictionary_id=spdict_id)
rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id1 = self.DP.create_data_product(data_product=dp_obj,
stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s', data_product_id1)
self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id1)
self.DP.activate_data_product_persistence(data_product_id=data_product_id1)
self.addCleanup(self.DP.suspend_data_product_persistence, data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasStream, None, True)
log.debug( 'Data product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.RR.find_objects(data_product_id1, PRED.hasDataset, RT.Dataset, True)
log.debug( 'Data set for data_product_id1 = %s', dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
#create the datastore at the beginning of each int test that persists data
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain = tdom,
spatial_domain = sdom)
data_product_id2 = self.DP.create_data_product(data_product=dp_obj,
stream_definition_id=raw_stream_def_id)
log.debug( 'new dp_id = %s', str(data_product_id2))
self.DAMS.assign_data_product(input_resource_id=instDevice_id, data_product_id=data_product_id2)
self.DP.activate_data_product_persistence(data_product_id=data_product_id2)
self.addCleanup(self.DP.suspend_data_product_persistence, data_product_id2)
# spin up agent
self.IMS.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
self.addCleanup(self.IMS.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
#wait for start
instance_obj = self.IMS.read_instrument_agent_instance(instAgentInstance_id)
gate = AgentProcessStateGate(self.PDC.read_process,
instDevice_id,
ProcessStateEnum.RUNNING)
self.assertTrue(gate.await(30), "The instrument agent instance (%s) did not spawn in 30 seconds" %
gate.process_id)
# take snapshot of config
snap_id = self.IMS.save_resource_state(instDevice_id, "xyzzy snapshot")
snap_obj = self.RR.read_attachment(snap_id, include_content=True)
#modify config
instance_obj.driver_config["comms_config"] = "BAD_DATA"
self.RR.update(instance_obj)
#restore config
self.IMS.restore_resource_state(instDevice_id, snap_id)
instance_obj = self.RR.read(instAgentInstance_id)
if "BAD_DATA" == instance_obj.driver_config["comms_config"]:
print "Saved config:"
print snap_obj.content
self.fail("Saved config was not properly restored")
self.assertNotEqual("BAD_DATA", instance_obj.driver_config["comms_config"])
self.DP.delete_data_product(data_product_id1)
self.DP.delete_data_product(data_product_id2)
def test_agent_instance_config_hasDevice(self):
def assign_fn(child_device_id, parent_device_id):
self.RR2.create_association(parent_device_id, PRED.hasDevice, child_device_id)
def find_fn(parent_device_id):
ret, _ = self.RR.find_objects(subject=parent_device_id, predicate=PRED.hasDevice, id_only=True)
return ret
self.base_agent_instance_config(assign_fn, find_fn)
log.info("END test_agent_instance_config_hasDevice")
def test_agent_instance_config_hasNetworkParent(self):
def assign_fn(child_device_id, parent_device_id):
self.RR2.create_association(child_device_id, PRED.hasNetworkParent, parent_device_id)
def find_fn(parent_device_id):
ret, _ = self.RR.find_subjects(object=parent_device_id, predicate=PRED.hasNetworkParent, id_only=True)
return ret
self.base_agent_instance_config(assign_fn, find_fn)
log.info("END test_agent_instance_config_hasNetworkParent")
def base_agent_instance_config(self,
assign_child_platform_to_parent_platform_fn,
find_child_platform_ids_of_parent_platform_fn):
"""
Verify that agent configurations are being built properly
"""
clients = DotDict()
clients.resource_registry = self.RR
clients.pubsub_management = self.PSC
clients.dataset_management = self.DSC
config_builder = DotDict
config_builder.i = None
config_builder.p = None
def refresh_pconfig_builder_hack(config_builder):
"""
ugly hack to get around "idempotent" RR2 caching
remove after https://github.com/ooici/coi-services/pull/1190
"""
config_builder.p = PlatformAgentConfigurationBuilder(clients)
def refresh_iconfig_builder_hack(config_builder):
"""
ugly hack to get around "idempotent" RR2 caching
remove after https://github.com/ooici/coi-services/pull/1190
"""
config_builder.i = InstrumentAgentConfigurationBuilder(clients)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
org_obj = any_old(RT.Org)
org_id = self.RR2.create(org_obj)
inst_startup_config = {'startup': 'config'}
generic_alerts_config = [ {'lvl2': 'lvl3val'} ]
required_config_keys = [
'org_governance_name',
'device_type',
'agent',
'driver_config',
'stream_config',
'startup_config',
'aparam_alerts_config',
'children']
def verify_instrument_config(config, device_id):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual(org_obj.org_governance_name, config['org_governance_name'])
self.assertEqual(RT.InstrumentDevice, config['device_type'])
self.assertIn('driver_config', config)
driver_config = config['driver_config']
expected_driver_fields = {'process_type': ('ZMQPyClassDriverLauncher',),
}
for k, v in expected_driver_fields.iteritems():
self.assertIn(k, driver_config)
self.assertEqual(v, driver_config[k])
self.assertEqual
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertEqual(inst_startup_config, config['startup_config'])
self.assertIn('aparam_alerts_config', config)
self.assertEqual(generic_alerts_config, config['aparam_alerts_config'])
self.assertIn('stream_config', config)
for key in ['children']:
self.assertEqual({}, config[key])
def verify_child_config(config, device_id, inst_device_id=None):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual(org_obj.org_governance_name, config['org_governance_name'])
self.assertEqual(RT.PlatformDevice, config['device_type'])
self.assertEqual({'resource_id': device_id}, config['agent'])
self.assertIn('aparam_alerts_config', config)
self.assertEqual(generic_alerts_config, config['aparam_alerts_config'])
self.assertIn('stream_config', config)
self.assertIn('driver_config', config)
self.assertIn('foo', config['driver_config'])
self.assertIn('ports', config['driver_config'])
self.assertEqual('bar', config['driver_config']['foo'])
self.assertIn('process_type', config['driver_config'])
self.assertEqual(('ZMQPyClassDriverLauncher',), config['driver_config']['process_type'])
if None is inst_device_id:
for key in ['children', 'startup_config']:
self.assertEqual({}, config[key])
else:
for key in ['startup_config']:
self.assertEqual({}, config[key])
self.assertIn(inst_device_id, config['children'])
verify_instrument_config(config['children'][inst_device_id], inst_device_id)
if config['driver_config']['ports']:
self.assertTrue( isinstance(config['driver_config']['ports'], dict) )
def verify_parent_config(config, parent_device_id, child_device_id, inst_device_id=None):
for key in required_config_keys:
self.assertIn(key, config)
self.assertEqual(org_obj.org_governance_name, config['org_governance_name'])
self.assertEqual(RT.PlatformDevice, config['device_type'])
self.assertIn('process_type', config['driver_config'])
self.assertIn('ports', config['driver_config'])
self.assertEqual(('ZMQPyClassDriverLauncher',), config['driver_config']['process_type'])
self.assertEqual({'resource_id': parent_device_id}, config['agent'])
self.assertIn('aparam_alerts_config', config)
self.assertEqual(generic_alerts_config, config['aparam_alerts_config'])
self.assertIn('stream_config', config)
for key in ['startup_config']:
self.assertEqual({}, config[key])
if config['driver_config']['ports']:
self.assertTrue( isinstance(config['driver_config']['ports'], dict) )
self.assertIn(child_device_id, config['children'])
verify_child_config(config['children'][child_device_id], child_device_id, inst_device_id)
rpdict_id = self.DSC.read_parameter_dictionary_by_name('ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.PSC.create_stream_definition(name='raw', parameter_dictionary_id=rpdict_id)
#todo: create org and figure out which agent resource needs to get assigned to it
def _make_platform_agent_structure(name='', agent_config=None):
if None is agent_config: agent_config = {}
# instance creation
platform_agent_instance_obj = any_old(RT.PlatformAgentInstance, {'driver_config': {'foo': 'bar'},
'alerts': generic_alerts_config})
platform_agent_instance_obj.agent_config = agent_config
platform_agent_instance_id = self.IMS.create_platform_agent_instance(platform_agent_instance_obj)
# agent creation
raw_config = StreamConfiguration(stream_name='raw', parameter_dictionary_name='ctd_raw_param_dict' )
platform_agent_obj = any_old(RT.PlatformAgent, {"stream_configurations":[raw_config]})
platform_agent_id = self.IMS.create_platform_agent(platform_agent_obj)
# device creation
platform_device_id = self.IMS.create_platform_device(any_old(RT.PlatformDevice))
# data product creation
dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom})
dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id)
self.DAMS.assign_data_product(input_resource_id=platform_device_id, data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
self.addCleanup(self.DP.suspend_data_product_persistence, dp_id)
#deployment creation
site_obj = IonObject(RT.PlatformSite, name='sitePlatform')
site_id = self.OMS.create_platform_site(platform_site=site_obj)
# find current deployment using time constraints
current_time = int( calendar.timegm(time.gmtime()) )
# two years on either side of current time
start = current_time - 63115200
end = current_time + 63115200
temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=str(start), end_datetime=str(end))
platform_port_obj= IonObject(OT.PlatformPort, reference_designator = 'GA01SUMO-FI003-09-CTDMO0999',
port_type=PortTypeEnum.UPLINK,
ip_address=0)
deployment_obj = IonObject(RT.Deployment,
name='TestPlatformDeployment_' + name,
description='some new deployment',
context=IonObject(OT.CabledNodeDeploymentContext),
constraint_list=[temporal_bounds],
port_assignments={platform_device_id:platform_port_obj})
deploy_id = self.OMS.create_deployment(deployment=deployment_obj, site_id=site_id, device_id=platform_device_id)
# assignments
self.RR2.assign_platform_agent_instance_to_platform_device_with_has_agent_instance(platform_agent_instance_id, platform_device_id)
self.RR2.assign_platform_agent_to_platform_agent_instance_with_has_agent_definition(platform_agent_id, platform_agent_instance_id)
self.RR2.assign_platform_device_to_org_with_has_resource(platform_agent_instance_id, org_id)
return platform_agent_instance_id, platform_agent_id, platform_device_id
def _make_instrument_agent_structure(agent_config=None):
if None is agent_config: agent_config = {}
# instance creation
instrument_agent_instance_obj = any_old(RT.InstrumentAgentInstance, {"startup_config": inst_startup_config,
'alerts': generic_alerts_config})
instrument_agent_instance_obj.agent_config = agent_config
instrument_agent_instance_id = self.IMS.create_instrument_agent_instance(instrument_agent_instance_obj)
# agent creation
raw_config = StreamConfiguration(stream_name='raw',
parameter_dictionary_name='ctd_raw_param_dict' )
instrument_agent_obj = any_old(RT.InstrumentAgent, {"stream_configurations":[raw_config]})
instrument_agent_id = self.IMS.create_instrument_agent(instrument_agent_obj)
# device creation
instrument_device_id = self.IMS.create_instrument_device(any_old(RT.InstrumentDevice))
# data product creation
dp_obj = any_old(RT.DataProduct, {"temporal_domain":tdom, "spatial_domain": sdom})
dp_id = self.DP.create_data_product(data_product=dp_obj, stream_definition_id=raw_stream_def_id)
self.DAMS.assign_data_product(input_resource_id=instrument_device_id, data_product_id=dp_id)
self.DP.activate_data_product_persistence(data_product_id=dp_id)
self.addCleanup(self.DP.suspend_data_product_persistence, dp_id)
#deployment creation
site_obj = IonObject(RT.InstrumentSite, name='siteInstrument')
site_id = self.OMS.create_instrument_site(instrument_site =site_obj)
# find current deployment using time constraints
current_time = int( calendar.timegm(time.gmtime()) )
# two years on either side of current time
start = current_time - 63115200
end = current_time + 63115200
temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=str(start), end_datetime=str(end))
platform_port_obj= IonObject(OT.PlatformPort, reference_designator = 'GA01SUMO-FI003-08-CTDMO0888',
port_type=PortTypeEnum.PAYLOAD,
ip_address=0)
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment for Cabled Instrument',
description='some new deployment',
context=IonObject(OT.CabledInstrumentDeploymentContext),
constraint_list=[temporal_bounds],
port_assignments={instrument_device_id:platform_port_obj})
deploy_id = self.OMS.create_deployment(deployment=deployment_obj, site_id=site_id, device_id=instrument_device_id)
# assignments
self.RR2.assign_instrument_agent_instance_to_instrument_device_with_has_agent_instance(instrument_agent_instance_id, instrument_device_id)
self.RR2.assign_instrument_agent_to_instrument_agent_instance_with_has_agent_definition(instrument_agent_id, instrument_agent_instance_id)
self.RR2.assign_instrument_device_to_org_with_has_resource(instrument_agent_instance_id, org_id)
return instrument_agent_instance_id, instrument_agent_id, instrument_device_id
# can't do anything without an agent instance obj
log.debug("Testing that preparing a launcher without agent instance raises an error")
refresh_pconfig_builder_hack(config_builder) # associations have changed since builder was instantiated
self.assertRaises(AssertionError, config_builder.p.prepare, will_launch=False)
log.debug("Making the structure for a platform agent, which will be the child")
platform_agent_instance_child_id, _, platform_device_child_id = _make_platform_agent_structure(name='child')
platform_agent_instance_child_obj = self.RR2.read(platform_agent_instance_child_id)
log.debug("Preparing a valid agent instance launch, for config only")
refresh_pconfig_builder_hack(config_builder) # associations have changed since builder was instantiated
config_builder.p.set_agent_instance_object(platform_agent_instance_child_obj)
child_config = config_builder.p.prepare(will_launch=False)
verify_child_config(child_config, platform_device_child_id)
log.debug("Making the structure for a platform agent, which will be the parent")
platform_agent_instance_parent_id, _, platform_device_parent_id = _make_platform_agent_structure(name='parent')
platform_agent_instance_parent_obj = self.RR2.read(platform_agent_instance_parent_id)
log.debug("Testing child-less parent as a child config")
refresh_pconfig_builder_hack(config_builder) # associations have changed since builder was instantiated
config_builder.p.set_agent_instance_object(platform_agent_instance_parent_obj)
parent_config = config_builder.p.prepare(will_launch=False)
verify_child_config(parent_config, platform_device_parent_id)
log.debug("assigning child platform to parent")
assign_child_platform_to_parent_platform_fn(platform_device_child_id, platform_device_parent_id)
child_device_ids = find_child_platform_ids_of_parent_platform_fn(platform_device_parent_id)
self.assertNotEqual(0, len(child_device_ids))
log.debug("Testing parent + child as parent config")
refresh_pconfig_builder_hack(config_builder) # associations have changed since builder was instantiated
config_builder.p.set_agent_instance_object(platform_agent_instance_parent_obj)
parent_config = config_builder.p.prepare(will_launch=False)
verify_parent_config(parent_config, platform_device_parent_id, platform_device_child_id)
log.debug("making the structure for an instrument agent")
instrument_agent_instance_id, _, instrument_device_id = _make_instrument_agent_structure()
instrument_agent_instance_obj = self.RR2.read(instrument_agent_instance_id)
log.debug("Testing instrument config")
refresh_iconfig_builder_hack(config_builder) # associations have changed since builder was instantiated
config_builder.i.set_agent_instance_object(instrument_agent_instance_obj)
instrument_config = config_builder.i.prepare(will_launch=False)
verify_instrument_config(instrument_config, instrument_device_id)
log.debug("assigning instrument to platform")
self.RR2.assign_instrument_device_to_platform_device_with_has_device(instrument_device_id, platform_device_child_id)
child_device_ids = self.RR2.find_instrument_device_ids_of_platform_device_using_has_device(platform_device_child_id)
self.assertNotEqual(0, len(child_device_ids))
log.debug("Testing entire config")
refresh_pconfig_builder_hack(config_builder) # associations have changed since builder was instantiated
config_builder.p.set_agent_instance_object(platform_agent_instance_parent_obj)
full_config = config_builder.p.prepare(will_launch=False)
verify_parent_config(full_config, platform_device_parent_id, platform_device_child_id, instrument_device_id)
#self.fail(parent_config)
#plauncher.prepare(will_launch=False)
log.info("END base_agent_instance_config")
class RegistrationProcessTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dataset_management = DatasetManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.resource_registry = self.container.resource_registry
def _make_dataset(self):
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
parameter_dict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
dataset = Dataset(name='test_dataset')
dataset_id = self.dataset_management.create_dataset(
dataset, parameter_dictionary_id=parameter_dict_id)
return dataset_id
@unittest.skip("Array types temporarily unsupported")
def test_pydap(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_extended_parsed()
stream_def_id = self.pubsub_management.create_stream_definition(
'example', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition,
stream_def_id)
dp = DataProduct(name='example')
data_product_id = self.data_product_management.create_data_product(
dp, stream_def_id)
self.addCleanup(self.data_product_management.delete_data_product,
data_product_id)
self.data_product_management.activate_data_product_persistence(
data_product_id)
self.addCleanup(
self.data_product_management.suspend_data_product_persistence,
data_product_id)
dataset_id = self.resource_registry.find_objects(data_product_id,
PRED.hasDataset,
id_only=True)[0][0]
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
rdt = ph.get_rdt(stream_def_id)
ph.fill_rdt(rdt, 10)
ph.publish_rdt_to_data_product(data_product_id, rdt)
self.assertTrue(monitor.wait())
gevent.sleep(
1) # Yield to other greenlets, had an issue with connectivity
pydap_host = CFG.get_safe('server.pydap.host', 'localhost')
pydap_port = CFG.get_safe('server.pydap.port', 8001)
url = 'http://%s:%s/%s' % (pydap_host, pydap_port, data_product_id)
for i in xrange(
3
): # Do it three times to test that the cache doesn't corrupt the requests/responses
ds = open_url(url)
np.testing.assert_array_equal(list(ds['data']['time']),
np.arange(10))
untested = []
for k, v in rdt.iteritems():
if k == rdt.temporal_parameter:
continue
context = rdt.context(k)
if isinstance(context.param_type, QuantityType):
np.testing.assert_array_equal(list(ds['data'][k]), rdt[k])
elif isinstance(context.param_type, ArrayType):
if context.param_type.inner_encoding is None:
values = np.empty(rdt[k].shape, dtype='O')
for i, obj in enumerate(rdt[k]):
values[i] = str(obj)
np.testing.assert_array_equal(list(ds['data'][k]),
values)
elif len(rdt[k].shape) > 1:
values = np.empty(rdt[k].shape[0], dtype='O')
for i in xrange(rdt[k].shape[0]):
values[i] = ','.join(
map(lambda x: str(x), rdt[k][i].tolist()))
elif isinstance(context.param_type, ConstantType):
np.testing.assert_array_equal(list(ds['data'][k]), rdt[k])
elif isinstance(context.param_type, CategoryType):
np.testing.assert_array_equal(list(ds['data'][k]),
rdt[k].astype('|S'))
else:
untested.append('%s (%s)' % (k, context.param_type))
if untested:
raise AssertionError('Untested parameters: %s' % untested)
class TestDataProductProvenance(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.dpmsclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]:
self.dataprocessclient.deactivate_data_process(proc_id)
self.dataprocessclient.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
def test_get_data_product_provenance_report(self):
#Create a test device
device_obj = Device(name='Device1',
description='test instrument site')
device_id, _ = self.rrclient.create(device_obj)
self.addCleanup(self.rrclient.delete, device_id)
#Create a test DataProduct
data_product1_obj = DataProduct(name='DataProduct1',
description='test data product 1')
data_product1_id, _ = self.rrclient.create(data_product1_obj)
self.addCleanup(self.rrclient.delete, data_product1_id)
#Create a test DataProcess
data_process_obj = DataProcess(name='DataProcess',
description='test data process')
data_process_id, _ = self.rrclient.create(data_process_obj)
self.addCleanup(self.rrclient.delete, data_process_id)
#Create a second test DataProduct
data_product2_obj = DataProduct(name='DataProduct2',
description='test data product 2')
data_product2_id, _ = self.rrclient.create(data_product2_obj)
self.addCleanup(self.rrclient.delete, data_product2_id)
#Create a test DataProducer
data_producer_obj = DataProducer(name='DataProducer',
description='test data producer')
data_producer_id, rev = self.rrclient.create(data_producer_obj)
#Link the DataProcess to the second DataProduct manually
assoc_id, _ = self.rrclient.create_association(subject=data_process_id, predicate=PRED.hasInputProduct, object=data_product2_id)
self.addCleanup(self.rrclient.delete_association, assoc_id)
# Register the instrument and process. This links the device and the data process
# with their own producers
self.damsclient.register_instrument(device_id)
self.addCleanup(self.damsclient.unregister_instrument, device_id)
self.damsclient.register_process(data_process_id)
self.addCleanup(self.damsclient.unregister_process, data_process_id)
#Manually link the first DataProduct with the test DataProducer
assoc_id, _ = self.rrclient.create_association(subject=data_product1_id, predicate=PRED.hasDataProducer, object=data_producer_id)
#Get the DataProducer linked to the DataProcess (created in register_process above)
#Associate that with with DataProduct1's DataProducer
data_process_producer_ids, _ = self.rrclient.find_objects(subject=data_process_id, predicate=PRED.hasDataProducer, object_type=RT.DataProducer, id_only=True)
assoc_id, _ = self.rrclient.create_association(subject=data_process_producer_ids[0], predicate=PRED.hasParent, object=data_producer_id)
self.addCleanup(self.rrclient.delete_association, assoc_id)
#Get the DataProducer linked to the Device (created in register_instrument
#Associate that with the DataProcess's DataProducer
device_producer_ids, _ = self.rrclient.find_objects(subject=device_id, predicate=PRED.hasDataProducer, object_type=RT.DataProducer, id_only=True)
assoc_id, _ = self.rrclient.create_association(subject=data_producer_id, predicate=PRED.hasParent, object=device_producer_ids[0])
#Create the links between the Device, DataProducts, DataProcess, and all DataProducers
self.damsclient.assign_data_product(input_resource_id=device_id, data_product_id=data_product1_id)
self.addCleanup(self.damsclient.unassign_data_product, device_id, data_product1_id)
self.damsclient.assign_data_product(input_resource_id=data_process_id, data_product_id=data_product2_id)
self.addCleanup(self.damsclient.unassign_data_product, data_process_id, data_product2_id)
#Traverse through the relationships to get the links between objects
res = self.dpmsclient.get_data_product_provenance_report(data_product2_id)
#Make sure there are four keys
self.assertEqual(len(res.keys()), 4)
parent_count = 0
config_count = 0
for v in res.itervalues():
if 'parent' in v:
parent_count += 1
if 'config' in v:
config_count += 1
#Make sure there are three parents and four configs
self.assertEqual(parent_count, 3)
self.assertEqual(config_count, 4)
@unittest.skip('This test is obsolete with new framework')
def test_get_provenance(self):
#create a deployment with metadata and an initial site and device
instrument_site_obj = IonObject(RT.InstrumentSite,
name='InstrumentSite1',
description='test instrument site')
instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, "")
log.debug( 'test_get_provenance: new instrument_site_id id = %s ', str(instrument_site_id))
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel" )
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" %ex)
log.debug( 'test_get_provenance: new InstrumentModel id = %s ', str(instModel_id))
self.omsclient.assign_instrument_model_to_instrument_site(instModel_id, instrument_site_id)
# Create InstrumentAgent
parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict' )
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri=DRV_URI_GOOD,
stream_configurations = [parsed_config] )
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" %ex)
log.debug( 'test_get_provenance:new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug('test_get_provenance: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ')
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
try:
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" %ex)
log.debug("test_get_provenance: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id)
#-------------------------------
# Create CTD Parsed data product
#-------------------------------
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubclient.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id)
log.debug( 'test_get_provenance:Creating new CDM data product with a stream definition')
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_parsed_data_product = self.dpmsclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s', ctd_parsed_data_product)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
self.dpmsclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product)
#-------------------------------
# create a data product for the site to pass the OMS check.... we need to remove this check
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
log_data_product_id = self.dpmsclient.create_data_product(dp_obj, parsed_stream_def_id)
#-------------------------------
# Deploy instrument device to instrument site
#-------------------------------
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment')
deployment_id = self.omsclient.create_deployment(deployment_obj)
self.omsclient.deploy_instrument_site(instrument_site_id, deployment_id)
self.imsclient.deploy_instrument_device(instDevice_id, deployment_id)
log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) )
self.omsclient.activate_deployment(deployment_id)
inst_device_objs, _ = self.rrclient.find_objects(subject=instrument_site_id, predicate=PRED.hasDevice, object_type=RT.InstrumetDevice, id_only=False)
log.debug("test_create_deployment: deployed device: %s ", str(inst_device_objs[0]) )
#-------------------------------
# Create the agent instance
#-------------------------------
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config = port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition ctd_L0_all")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" %ex)
#-------------------------------
# L1 Conductivity: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1ConductivityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_conductivity',
description='create the L1 conductivity data product',
module='ion.processes.data.transforms.ctd.ctd_L1_conductivity',
class_name='CTDL1ConductivityTransform')
try:
ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1ConductivityTransform data process definition: %s" %ex)
#-------------------------------
# L1 Pressure: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1PressureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_pressure',
description='create the L1 pressure data product',
module='ion.processes.data.transforms.ctd.ctd_L1_pressure',
class_name='CTDL1PressureTransform')
try:
ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1PressureTransform data process definition: %s" %ex)
#-------------------------------
# L1 Temperature: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1TemperatureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_temperature',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L1_temperature',
class_name='CTDL1TemperatureTransform')
try:
ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1TemperatureTransform data process definition: %s" %ex)
#-------------------------------
# L2 Salinity: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition SalinityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_salinity',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform')
try:
ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new SalinityTransform data process definition: %s" %ex)
#-------------------------------
# L2 Density: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition DensityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_density',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_density',
class_name='DensityTransform')
try:
ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new DensityTransform data process definition: %s" %ex)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' )
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' )
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' )
log.debug("TestDataProductProvenance: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id)
log.debug("TestDataProductProvenance: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_pressure_output_dp_obj,
outgoing_stream_l0_pressure_id)
log.debug("TestDataProductProvenance: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_temperature_output_dp_obj,
outgoing_stream_l0_temperature_id)
#-------------------------------
# L1 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(name='L1_conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id, binding='conductivity' )
outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(name='L1_Pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id, binding='pressure' )
outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(name='L1_Temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id, binding='temperature' )
log.debug("TestDataProductProvenance: create output data product L1 conductivity")
ctd_l1_conductivity_output_dp_obj = IonObject(RT.DataProduct,
name='L1_Conductivity',
description='transform output L1 conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_conductivity_output_dp_obj,
outgoing_stream_l1_conductivity_id)
log.debug("TestDataProductProvenance: create output data product L1 pressure")
ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Pressure',
description='transform output L1 pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_pressure_output_dp_obj,
outgoing_stream_l1_pressure_id)
log.debug("TestDataProductProvenance: create output data product L1 temperature")
ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Temperature',
description='transform output L1 temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_temperature_output_dp_obj,
outgoing_stream_l1_temperature_id)
#-------------------------------
# L2 Salinity - Density: Output Data Products
#-------------------------------
outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(name='L2_salinity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id, binding='salinity' )
outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(name='L2_Density', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id, binding='density' )
log.debug("TestDataProductProvenance: create output data product L2 Salinity")
ctd_l2_salinity_output_dp_obj = IonObject( RT.DataProduct,
name='L2_Salinity',
description='transform output L2 salinity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l2_salinity_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_salinity_output_dp_obj,
outgoing_stream_l2_salinity_id)
log.debug("TestDataProductProvenance: create output data product L2 Density")
# ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct,
# name='L2_Density',
# description='transform output pressure',
# temporal_domain = tdom,
# spatial_domain = sdom)
#
# ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj,
# outgoing_stream_l2_density_id,
# parameter_dictionary)
contactInfo = ContactInformation()
contactInfo.individual_names_given = "Bill"
contactInfo.individual_name_family = "Smith"
contactInfo.street_address = "111 First St"
contactInfo.city = "San Diego"
contactInfo.email = "*****@*****.**"
contactInfo.phones = ["858-555-6666"]
contactInfo.country = "USA"
contactInfo.postal_code = "92123"
ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct,
name='L2_Density',
description='transform output pressure',
contacts = [contactInfo],
iso_topic_category = "my_iso_topic_category_here",
quality_control_level = "1",
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj,
outgoing_stream_l2_density_id)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L0 all data_process start")
try:
input_data_products = [ctd_parsed_data_product]
output_data_products = [ctd_l0_conductivity_output_dp_id, ctd_l0_pressure_output_dp_id, ctd_l0_temperature_output_dp_id]
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id = ctd_L0_all_dprocdef_id,
in_data_product_ids = input_data_products,
out_data_product_ids = output_data_products
)
#activate only this data process just for coverage
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
contents = "this is the lookup table contents, replace with a file..."
att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII)
deviceAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att)
log.info( 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = %s', deviceAttachment)
log.debug("TestDataProductProvenance: create L0 all data_process return")
#-------------------------------
# L1 Conductivity: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1 Conductivity data_process start")
try:
l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id = ctd_L1_conductivity_dprocdef_id,
in_data_product_ids = [ctd_l0_conductivity_output_dp_id],
out_data_product_ids = [ctd_l1_conductivity_output_dp_id])
self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L1 Pressure: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1_Pressure data_process start")
try:
l1_pressure_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id = ctd_L1_pressure_dprocdef_id,
in_data_product_ids = [ctd_l0_pressure_output_dp_id],
out_data_product_ids = [ctd_l1_pressure_output_dp_id])
self.dataprocessclient.activate_data_process(l1_pressure_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L1 Temperature: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1_Pressure data_process start")
try:
l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id = ctd_L1_temperature_dprocdef_id,
in_data_product_ids = [ctd_l0_temperature_output_dp_id],
out_data_product_ids = [ctd_l1_temperature_output_dp_id])
self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L2 Salinity: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L2_salinity data_process start")
try:
l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id = ctd_L2_salinity_dprocdef_id,
in_data_product_ids = [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id],
out_data_product_ids = [ctd_l2_salinity_output_dp_id])
self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L2 Density: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L2_Density data_process start")
try:
in_dp_ids = [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id]
out_dp_ids = [ctd_l2_density_output_dp_id]
l2_density_all_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id = ctd_L2_density_dprocdef_id,
in_data_product_ids = in_dp_ids,
out_data_product_ids = out_dp_ids)
self.dataprocessclient.activate_data_process(l2_density_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# Launch InstrumentAgentInstance, connect to the resource agent client
#-------------------------------
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
print 'TestDataProductProvenance: Instrument agent instance obj: = ', inst_agent_instance_obj
# Start a resource agent client to talk with the instrument agent.
# self._ia_client = ResourceAgentClient('iaclient', name=ResourceAgentClient._get_agent_process_id(instDevice_id, process=FakeProcess())
# print 'activate_instrument: got ia client %s', self._ia_client
# log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client))
#-------------------------------
# Deactivate InstrumentAgentInstance
#-------------------------------
self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
self.dataprocessclient.deactivate_data_process(l2_density_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l2_salinity_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_temperature_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_pressure_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_conductivity_data_process_id)
self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id)
#-------------------------------
# Retrieve the provenance info for the ctd density data product
#-------------------------------
provenance_dict = self.dpmsclient.get_data_product_provenance(ctd_l2_density_output_dp_id)
log.debug("TestDataProductProvenance: provenance_dict %s", str(provenance_dict))
#validate that products are represented
self.assertTrue (provenance_dict[str(ctd_l1_conductivity_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l0_conductivity_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l2_density_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l1_temperature_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l0_temperature_output_dp_id)])
density_dict = (provenance_dict[str(ctd_l2_density_output_dp_id)])
self.assertEquals(density_dict['producer'], [l2_density_all_data_process_id])
#-------------------------------
# Retrieve the extended resource for this data product
#-------------------------------
extended_product = self.dpmsclient.get_data_product_extension(ctd_l2_density_output_dp_id)
self.assertEqual(1, len(extended_product.data_processes) )
self.assertEqual(3, len(extended_product.process_input_data_products) )
# log.debug("TestDataProductProvenance: DataProduct provenance_product_list %s", str(extended_product.provenance_product_list))
# log.debug("TestDataProductProvenance: DataProduct data_processes %s", str(extended_product.data_processes))
# log.debug("TestDataProductProvenance: DataProduct process_input_data_products %s", str(extended_product.process_input_data_products))
# log.debug("TestDataProductProvenance: provenance %s", str(extended_product.computed.provenance.value))
#-------------------------------
# Retrieve the extended resource for this data process
#-------------------------------
extended_process_def = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id)
# log.debug("TestDataProductProvenance: DataProcess extended_process_def %s", str(extended_process_def))
# log.debug("TestDataProductProvenance: DataProcess data_processes %s", str(extended_process_def.data_processes))
# log.debug("TestDataProductProvenance: DataProcess data_products %s", str(extended_process_def.data_products))
self.assertEqual(1, len(extended_process_def.data_processes) )
self.assertEqual(3, len(extended_process_def.output_stream_definitions) )
self.assertEqual(3, len(extended_process_def.data_products) ) #one list because of one data process
#-------------------------------
# Request the xml report
#-------------------------------
results = self.dpmsclient.get_data_product_provenance_report(ctd_l2_density_output_dp_id)
print results
#-------------------------------
# Cleanup
#-------------------------------
self.dpmsclient.delete_data_product(ctd_parsed_data_product)
self.dpmsclient.delete_data_product(log_data_product_id)
self.dpmsclient.delete_data_product(ctd_l0_conductivity_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l0_pressure_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l0_temperature_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l1_conductivity_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l1_pressure_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l1_temperature_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l2_salinity_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l2_density_output_dp_id)
class TestDataProductVersions(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
print 'started services'
# Now create client to DataProductManagementService
self.client = DataProductManagementServiceClient(node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.rrclient)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
#@unittest.skip('not working')
@patch.dict(CFG, {'endpoint': {'receive': {'timeout': 60}}})
def test_createDataProductVersionSimple(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',
id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='test',
parameter_dictionary_id=pdict_id)
# test creating a new data product which will also create the initial/default version
log.debug('Creating new data product with a stream definition')
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='DP',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
dp_id = self.client.create_data_product(dp_obj, ctd_stream_def_id)
log.debug('new dp_id = %s', str(dp_id))
dpc_id = self.client.create_data_product_collection(data_product_id=dp_id,
collection_name='firstCollection',
collection_description='collection desc')
#test that the links exist
version_ids, _ = self.rrclient.find_objects(subject=dpc_id, predicate=PRED.hasVersion, id_only=True)
log.debug('version_ids = %s', str(version_ids))
self.assertTrue(version_ids, 'Failed to connect the data product to the version collection.')
self.assertTrue(version_ids[0] == dp_id, 'Failed to connect the data product to the version collection.')
# test creating a subsequent data product version which will update the data product pointers
dp2_obj = IonObject(RT.DataProduct,
name='DP2',
description='a second dp',
temporal_domain = tdom,
spatial_domain = sdom)
dp2_id = self.client.create_data_product(dp2_obj, ctd_stream_def_id)
log.debug('second dp_id = %s', dp2_id)
log.debug("adding product %s to collection %s", dp2_id, dpc_id)
self.client.add_data_product_version_to_collection(data_product_id=dp2_id,
data_product_collection_id=dpc_id,
version_name="second version",
version_description="a second version created" )
#test that the links exist
version_ids = self.RR2.find_data_product_ids_of_data_product_collection_using_has_version(dpc_id)
self.assertEqual(2, len(version_ids))
recent_version_id = self.client.get_current_version(dpc_id)
self.assertEquals(recent_version_id, dp2_id)
base_version_id = self.client.get_base_version(dpc_id)
self.assertEquals(base_version_id, dp_id)
#---------------------------------------------------------------------------------------------
# Now check that we can subscribe to the stream for the data product version
#---------------------------------------------------------------------------------------------
# Activating the data products contained in the versions held by the data product collection
data_product_collection_obj = self.rrclient.read(dpc_id)
version_list = data_product_collection_obj.version_list
self.assertEquals(len(version_list), 2)
for version in version_list:
data_product_id = version.data_product_id
self.client.activate_data_product_persistence(data_product_id)
streams = self.RR2.find_streams_of_data_product_using_has_stream(data_product_id)
self.assertNotEqual(0, len(streams))
for stream in streams:
self.assertTrue(stream.persisted)
log.debug("This satisfies L4-CI-DM-RQ-053: 'The dynamic data distribution services shall support multiple "
"versions of a given data topic.' This is true because we have shown above that we can persist the "
"streams associated to the data product versions, and therefore we can subscribe to them. In "
"test_oms_launch2.py, we have tested that activated data products like the ones we have here can be "
"used by data processes to gather streaming data and it all works together correctly. This therefore "
"completes the demonstration of req L4-CI-DM-RQ-053.")
#---------------------------------------------------------------------------------------------
# Now delete all the created stuff and look for possible problems in doing so
#---------------------------------------------------------------------------------------------
log.debug("deleting all data productgs")
self.client.delete_data_product(dp_id)
self.client.delete_data_product(dp2_id)
self.client.delete_data_product_collection(dpc_id)
self.client.force_delete_data_product_collection(dpc_id)
# now try to get the deleted dp object
self.assertRaises(NotFound, self.client.read_data_product_collection, dpc_id)
class TestBulkIngest(IonIntegrationTestCase):
EDA_MOD = 'ion.agents.data.external_dataset_agent'
EDA_CLS = 'ExternalDatasetAgent'
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataAcquisitionManagementService
self.client = DataAcquisitionManagementServiceClient(node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dams_client = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsub_client = PubsubManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.data_retriever = DataRetrieverServiceClient(node=self.container.node)
self._container_client = ContainerAgentClient(node=self.container.node, name=self.container.name)
# Data async and subscription TODO: Replace with new subscriber
self._finished_count = None
#TODO: Switch to gevent.queue.Queue
self._async_finished_result = AsyncResult()
self._finished_events_received = []
self._finished_event_subscriber = None
self._start_finished_event_subscriber()
self.addCleanup(self._stop_finished_event_subscriber)
self.DVR_CONFIG = {}
self.DVR_CONFIG = {
'dvr_mod' : 'ion.agents.data.handlers.slocum_data_handler',
'dvr_cls' : 'SlocumDataHandler',
}
self._setup_resources()
self.agent_config = {
'driver_config' : self.DVR_CONFIG,
'stream_config' : {},
'agent' : {'resource_id': self.EDA_RESOURCE_ID},
'test_mode' : True
}
datasetagent_instance_obj = IonObject(RT.ExternalDatasetAgentInstance, name='ExternalDatasetAgentInstance1', description='external data agent instance',
handler_module=self.EDA_MOD, handler_class=self.EDA_CLS,
dataset_driver_config=self.DVR_CONFIG, dataset_agent_config=self.agent_config )
self.dataset_agent_instance_id = self.dams_client.create_external_dataset_agent_instance(external_dataset_agent_instance=datasetagent_instance_obj,
external_dataset_agent_id=self.datasetagent_id, external_dataset_id=self.EDA_RESOURCE_ID)
#TG: Setup/configure the granule logger to log granules as they're published
pid = self.dams_client.start_external_dataset_agent_instance(self.dataset_agent_instance_id)
dataset_agent_instance_obj= self.dams_client.read_external_dataset_agent_instance(self.dataset_agent_instance_id)
print 'TestBulkIngest: Dataset agent instance obj: = ', dataset_agent_instance_obj
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient('datasetagentclient', name=pid, process=FakeProcess())
log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client))
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name+'_logger')
producer_definition.executable = {
'module':'ion.processes.data.stream_granule_logger',
'class':'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition)
configuration = {
'process':{
'stream_id':stream_id,
}
}
pid = self.processdispatchclient.schedule_process(process_definition_id= logger_procdef_id, configuration=configuration)
return pid
def _start_finished_event_subscriber(self):
def consume_event(*args,**kwargs):
log.debug('EventSubscriber event received: %s', str(args[0]) )
if args[0].description == 'TestingFinished':
log.debug('TestingFinished event received')
self._finished_events_received.append(args[0])
if self._finished_count and self._finished_count == len(self._finished_events_received):
log.debug('Finishing test...')
self._async_finished_result.set(len(self._finished_events_received))
log.debug('Called self._async_finished_result.set({0})'.format(len(self._finished_events_received)))
self._finished_event_subscriber = EventSubscriber(event_type='DeviceEvent', callback=consume_event)
self._finished_event_subscriber.start()
def _stop_finished_event_subscriber(self):
if self._finished_event_subscriber:
self._finished_event_subscriber.stop()
self._finished_event_subscriber = None
def tearDown(self):
pass
@unittest.skip('Update to agent refactor.')
def test_slocum_data_ingest(self):
HIST_CONSTRAINTS_1 = {}
# Test instrument driver execute interface to start and stop streaming mode.
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Make sure the polling interval is appropriate for a test
params = {
'POLLING_INTERVAL': 3
}
self._ia_client.set_param(params)
self._finished_count = 1
cmd = AgentCommand(command='acquire_data')
self._ia_client.execute(cmd)
# Assert that data was received
self._async_finished_result.get(timeout=15)
self.assertTrue(len(self._finished_events_received) >= 1)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
#todo enable after Luke's mor to retrieve, right now must have the Time axis called 'time'
# replay_granule = self.data_retriever.retrieve_last_data_points(self.dataset_id, 10)
# rdt = RecordDictionaryTool.load_from_granule(replay_granule)
#
# comp = rdt['date_pattern'] == numpy.arange(10) + 10
#
# log.debug("TestBulkIngest: comp: %s", comp)
#
# self.assertTrue(comp.all())
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
def _setup_resources(self):
self.loggerpids = []
# Create DataProvider
dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation())
dprov.contact.name = 'Christopher Mueller'
dprov.contact.email = '*****@*****.**'
# Create DataSetModel
dataset_model = ExternalDatasetModel(name='slocum_model')
dataset_model.datset_type = 'SLOCUM'
dataset_model_id = self.dams_client.create_external_dataset_model(dataset_model)
# Create ExternalDataset
ds_name = 'slocum_test_dataset'
dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation())
dset.dataset_description.parameters['base_url'] = 'test_data/slocum/'
dset.dataset_description.parameters['list_pattern'] = 'ru05-2012-021-0-0-sbd.dat'
dset.dataset_description.parameters['date_pattern'] = '%Y %j'
dset.dataset_description.parameters['date_extraction_pattern'] = 'ru05-([\d]{4})-([\d]{3})-\d-\d-sbd.dat'
dset.dataset_description.parameters['temporal_dimension'] = None
dset.dataset_description.parameters['zonal_dimension'] = None
dset.dataset_description.parameters['meridional_dimension'] = None
dset.dataset_description.parameters['vertical_dimension'] = None
dset.dataset_description.parameters['variables'] = [
'c_wpt_y_lmc',
'sci_water_cond',
'm_y_lmc',
'u_hd_fin_ap_inflection_holdoff',
'sci_m_present_time',
'm_leakdetect_voltage_forward',
'sci_bb3slo_b660_scaled',
'c_science_send_all',
'm_gps_status',
'm_water_vx',
'm_water_vy',
'c_heading',
'sci_fl3slo_chlor_units',
'u_hd_fin_ap_gain',
'm_vacuum',
'u_min_water_depth',
'm_gps_lat',
'm_veh_temp',
'f_fin_offset',
'u_hd_fin_ap_hardover_holdoff',
'c_alt_time',
'm_present_time',
'm_heading',
'sci_bb3slo_b532_scaled',
'sci_fl3slo_cdom_units',
'm_fin',
'x_cycle_overrun_in_ms',
'sci_water_pressure',
'u_hd_fin_ap_igain',
'sci_fl3slo_phyco_units',
'm_battpos',
'sci_bb3slo_b470_scaled',
'm_lat',
'm_gps_lon',
'sci_ctd41cp_timestamp',
'm_pressure',
'c_wpt_x_lmc',
'c_ballast_pumped',
'x_lmc_xy_source',
'm_lon',
'm_avg_speed',
'sci_water_temp',
'u_pitch_ap_gain',
'm_roll',
'm_tot_num_inflections',
'm_x_lmc',
'u_pitch_ap_deadband',
'm_final_water_vy',
'm_final_water_vx',
'm_water_depth',
'm_leakdetect_voltage',
'u_pitch_max_delta_battpos',
'm_coulomb_amphr',
'm_pitch',
]
## Create the external dataset
ds_id = self.dams_client.create_external_dataset(external_dataset=dset, external_dataset_model_id=dataset_model_id)
ext_dprov_id = self.dams_client.create_external_data_provider(external_data_provider=dprov)
# Register the ExternalDataset
dproducer_id = self.dams_client.register_external_data_set(external_dataset_id=ds_id)
## Create the dataset agent
datasetagent_obj = IonObject(RT.ExternalDatasetAgent, name='ExternalDatasetAgent1', description='external data agent', handler_module=self.EDA_MOD, handler_class=self.EDA_CLS )
self.datasetagent_id = self.dams_client.create_external_dataset_agent(external_dataset_agent=datasetagent_obj, external_dataset_model_id=dataset_model_id)
# Generate the data product and associate it to the ExternalDataset
pdict = DatasetManagementService.get_parameter_dictionary_by_name('ctd_parsed_param_dict')
streamdef_id = self.pubsub_client.create_stream_definition(name="temp", parameter_dictionary_id=pdict.identifier)
tdom, sdom = time_series_domain()
tdom = tdom.dump()
sdom = sdom.dump()
dprod = IonObject(RT.DataProduct,
name='slocum_parsed_product',
description='parsed slocum product',
temporal_domain = tdom,
spatial_domain = sdom)
self.dproduct_id = self.dataproductclient.create_data_product(data_product=dprod,
stream_definition_id=streamdef_id)
self.dams_client.assign_data_product(input_resource_id=ds_id, data_product_id=self.dproduct_id)
#save the incoming slocum data
self.dataproductclient.activate_data_product_persistence(self.dproduct_id)
stream_ids, assn = self.rrclient.find_objects(subject=self.dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True)
stream_id = stream_ids[0]
dataset_id, assn = self.rrclient.find_objects(subject=self.dproduct_id, predicate=PRED.hasDataset, object_type=RT.Dataset, id_only=True)
self.dataset_id = dataset_id[0]
pid = self.create_logger('slocum_parsed_product', stream_id )
self.loggerpids.append(pid)
self.DVR_CONFIG['dh_cfg'] = {
'TESTING':True,
'stream_id':stream_id,
'param_dictionary':pdict.dump(),
'data_producer_id':dproducer_id, #CBM: Should this be put in the main body of the config - with mod & cls?
'max_records':20,
}
# Create the logger for receiving publications
#self.create_stream_and_logger(name='slocum',stream_id=stream_id)
# Create agent config.
self.EDA_RESOURCE_ID = ds_id
self.EDA_NAME = ds_name
class TestDataProductManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dpsc_cli = DataProductManagementServiceClient()
self.rrclient = ResourceRegistryServiceClient()
self.damsclient = DataAcquisitionManagementServiceClient()
self.pubsubcli = PubsubManagementServiceClient()
self.ingestclient = IngestionManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.unsc = UserNotificationServiceClient()
self.data_retriever = DataRetrieverServiceClient()
#------------------------------------------
# Create the environment
#------------------------------------------
datastore_name = CACHE_DATASTORE_NAME
self.db = self.container.datastore_manager.get_datastore(datastore_name)
self.stream_def_id = self.pubsubcli.create_stream_definition(name='SBE37_CDM')
self.process_definitions = {}
ingestion_worker_definition = ProcessDefinition(name='ingestion worker')
ingestion_worker_definition.executable = {
'module':'ion.processes.data.ingestion.science_granule_ingestion_worker',
'class' :'ScienceGranuleIngestionWorker'
}
process_definition_id = self.process_dispatcher.create_process_definition(process_definition=ingestion_worker_definition)
self.process_definitions['ingestion_worker'] = process_definition_id
self.pids = []
self.exchange_points = []
self.exchange_names = []
#------------------------------------------------------------------------------------------------
# First launch the ingestors
#------------------------------------------------------------------------------------------------
self.exchange_space = 'science_granule_ingestion'
self.exchange_point = 'science_data'
config = DotDict()
config.process.datastore_name = 'datasets'
config.process.queue_name = self.exchange_space
self.exchange_names.append(self.exchange_space)
self.exchange_points.append(self.exchange_point)
pid = self.process_dispatcher.schedule_process(self.process_definitions['ingestion_worker'],configuration=config)
log.debug("the ingestion worker process id: %s", pid)
self.pids.append(pid)
self.addCleanup(self.cleaning_up)
def cleaning_up(self):
for pid in self.pids:
log.debug("number of pids to be terminated: %s", len(self.pids))
try:
self.process_dispatcher.cancel_process(pid)
log.debug("Terminated the process: %s", pid)
except:
log.debug("could not terminate the process id: %s" % pid)
IngestionManagementIntTest.clean_subscriptions()
for xn in self.exchange_names:
xni = self.container.ex_manager.create_xn_queue(xn)
xni.delete()
for xp in self.exchange_points:
xpi = self.container.ex_manager.create_xp(xp)
xpi.delete()
def get_datastore(self, dataset_id):
dataset = self.dataset_management.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
@attr('EXT')
@attr('PREP')
def test_create_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
parameter_dictionary = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict')
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=parameter_dictionary._id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
# Generic time-series data domain creation
tdom, sdom = time_series_domain()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom.dump(),
spatial_domain = sdom.dump())
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 10.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -10.0
dp_obj.ooi_product_name = "PRODNAME"
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product( data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
# Assert that the data product has an associated stream at this stage
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True)
self.assertNotEquals(len(stream_ids), 0)
# Assert that the data product has an associated stream def at this stage
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStreamDefinition, RT.StreamDefinition, True)
self.assertNotEquals(len(stream_ids), 0)
self.dpsc_cli.activate_data_product_persistence(dp_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0)
log.debug('Created data product %s', dp_obj)
#------------------------------------------------------------------------------------------------
# test creating a new data product with a stream definition
#------------------------------------------------------------------------------------------------
log.debug('Creating new data product with a stream definition')
dp_obj = IonObject(RT.DataProduct,
name='DP2',
description='some new dp',
temporal_domain = tdom.dump(),
spatial_domain = sdom.dump())
dp_id2 = self.dpsc_cli.create_data_product(dp_obj, ctd_stream_def_id)
self.dpsc_cli.activate_data_product_persistence(dp_id2)
log.debug('new dp_id = %s' % dp_id2)
#------------------------------------------------------------------------------------------------
#make sure data product is associated with stream def
#------------------------------------------------------------------------------------------------
streamdefs = []
streams, _ = self.rrclient.find_objects(dp_id2, PRED.hasStream, RT.Stream, True)
for s in streams:
log.debug("Checking stream %s" % s)
sdefs, _ = self.rrclient.find_objects(s, PRED.hasStreamDefinition, RT.StreamDefinition, True)
for sd in sdefs:
log.debug("Checking streamdef %s" % sd)
streamdefs.append(sd)
self.assertIn(ctd_stream_def_id, streamdefs)
group_names = self.dpsc_cli.get_data_product_group_list()
self.assertIn("PRODNAME", group_names)
# test reading a non-existent data product
log.debug('reading non-existent data product')
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product('some_fake_id')
# update a data product (tests read also)
log.debug('Updating data product')
# first get the existing dp object
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# now tweak the object
dp_obj.description = 'the very first dp'
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 20.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -20.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 20.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -20.0
# now write the dp back to the registry
update_result = self.dpsc_cli.update_data_product(dp_obj)
# now get the dp back to see if it was updated
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertEquals(dp_obj.description,'the very first dp')
self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0)
log.debug('Updated data product %s', dp_obj)
#test extension
extended_product = self.dpsc_cli.get_data_product_extension(dp_id)
self.assertEqual(dp_id, extended_product._id)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_product.computed.product_download_size_estimated.status)
self.assertEqual(0, extended_product.computed.product_download_size_estimated.value)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_product.computed.parameters.status)
#log.debug("test_create_data_product: parameters %s" % extended_product.computed.parameters.value)
def ion_object_encoder(obj):
return obj.__dict__
#test prepare for create
data_product_data = self.dpsc_cli.prepare_data_product_support()
#print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2)
self.assertEqual(data_product_data._id, "")
self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport)
self.assertEqual(len(data_product_data.associations['StreamDefinition'].resources), 2)
self.assertEqual(len(data_product_data.associations['Dataset'].resources), 0)
self.assertEqual(len(data_product_data.associations['StreamDefinition'].associated_resources), 0)
self.assertEqual(len(data_product_data.associations['Dataset'].associated_resources), 0)
#test prepare for update
data_product_data = self.dpsc_cli.prepare_data_product_support(dp_id)
#print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2)
self.assertEqual(data_product_data._id, dp_id)
self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport)
self.assertEqual(len(data_product_data.associations['StreamDefinition'].resources), 2)
self.assertEqual(len(data_product_data.associations['Dataset'].resources), 1)
self.assertEqual(len(data_product_data.associations['StreamDefinition'].associated_resources), 1)
self.assertEqual(data_product_data.associations['StreamDefinition'].associated_resources[0].s, dp_id)
self.assertEqual(len(data_product_data.associations['Dataset'].associated_resources), 1)
self.assertEqual(data_product_data.associations['Dataset'].associated_resources[0].s, dp_id)
# now 'delete' the data product
log.debug("deleting data product: %s" % dp_id)
self.dpsc_cli.delete_data_product(dp_id)
# Assert that there are no associated streams leftover after deleting the data product
stream_ids, assoc_ids = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True)
self.assertEquals(len(stream_ids), 0)
self.assertEquals(len(assoc_ids), 0)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# Get the events corresponding to the data product
ret = self.unsc.get_recent_events(resource_id=dp_id)
events = ret.value
for event in events:
log.debug("event time: %s" % event.ts_created)
self.assertTrue(len(events) > 0)
def test_data_product_stream_def(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
stream_def_id = self.dpsc_cli.get_data_product_stream_definition(dp_id)
self.assertEquals(ctd_stream_def_id, stream_def_id)
def test_derived_data_product(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='ctd parsed', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsubcli.delete_stream_definition, ctd_stream_def_id)
tdom, sdom = time_series_domain()
dp = DataProduct(name='Instrument DP', temporal_domain=tdom.dump(), spatial_domain=sdom.dump())
dp_id = self.dpsc_cli.create_data_product(dp, stream_definition_id=ctd_stream_def_id)
self.addCleanup(self.dpsc_cli.force_delete_data_product, dp_id)
self.dpsc_cli.activate_data_product_persistence(dp_id)
self.addCleanup(self.dpsc_cli.suspend_data_product_persistence, dp_id)
dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True)
if not dataset_ids:
raise NotFound("Data Product %s dataset does not exist" % str(dp_id))
dataset_id = dataset_ids[0]
# Make the derived data product
simple_stream_def_id = self.pubsubcli.create_stream_definition(name='TEMPWAT stream def', parameter_dictionary_id=pdict_id, available_fields=['time','temp'])
tempwat_dp = DataProduct(name='TEMPWAT')
tempwat_dp_id = self.dpsc_cli.create_data_product(tempwat_dp, stream_definition_id=simple_stream_def_id, parent_data_product_id=dp_id)
self.addCleanup(self.dpsc_cli.delete_data_product, tempwat_dp_id)
# Check that the streams associated with the data product are persisted with
stream_ids, _ = self.rrclient.find_objects(dp_id,PRED.hasStream,RT.Stream,True)
for stream_id in stream_ids:
self.assertTrue(self.ingestclient.is_persisted(stream_id))
stream_id = stream_ids[0]
route = self.pubsubcli.read_stream_route(stream_id=stream_id)
rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
rdt['pressure'] = np.arange(20)
publisher = StandaloneStreamPublisher(stream_id,route)
dataset_modified = Event()
def cb(*args, **kwargs):
dataset_modified.set()
es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id, auto_delete=True)
es.start()
self.addCleanup(es.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
tempwat_dataset_ids, _ = self.rrclient.find_objects(tempwat_dp_id, PRED.hasDataset, id_only=True)
tempwat_dataset_id = tempwat_dataset_ids[0]
granule = self.data_retriever.retrieve(tempwat_dataset_id, delivery_format=simple_stream_def_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt['time'], np.arange(20))
self.assertEquals(set(rdt.fields), set(['time','temp']))
def test_activate_suspend_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
log.debug("Created an IonObject for a data product: %s" % dp_obj)
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test activate and suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.activate_data_product_persistence(dp_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True)
if not dataset_ids:
raise NotFound("Data Product %s dataset does not exist" % str(dp_id))
dataset_id = dataset_ids[0]
# Check that the streams associated with the data product are persisted with
stream_ids, _ = self.rrclient.find_objects(dp_id,PRED.hasStream,RT.Stream,True)
for stream_id in stream_ids:
self.assertTrue(self.ingestclient.is_persisted(stream_id))
stream_id = stream_ids[0]
route = self.pubsubcli.read_stream_route(stream_id=stream_id)
rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
publisher = StandaloneStreamPublisher(stream_id,route)
dataset_modified = Event()
def cb(*args, **kwargs):
dataset_modified.set()
es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id, auto_delete=True)
es.start()
self.addCleanup(es.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data = self.data_retriever.retrieve(dataset_ids[0])
self.assertIsInstance(replay_data, Granule)
log.debug("The data retriever was able to replay the dataset that was attached to the data product "
"we wanted to be persisted. Therefore the data product was indeed persisted with "
"otherwise we could not have retrieved its dataset using the data retriever. Therefore "
"this demonstration shows that L4-CI-SA-RQ-267 is satisfied: 'Data product management shall persist data products'")
data_product_object = self.rrclient.read(dp_id)
self.assertEquals(data_product_object.name,'DP1')
self.assertEquals(data_product_object.description,'some new dp')
log.debug("Towards L4-CI-SA-RQ-308: 'Data product management shall persist data product metadata'. "
" Attributes in create for the data product obj, name= '%s', description='%s', match those of object from the "
"resource registry, name='%s', desc='%s'" % (dp_obj.name, dp_obj.description,data_product_object.name,
data_product_object.description))
#------------------------------------------------------------------------------------------------
# test suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.suspend_data_product_persistence(dp_id)
dataset_modified.clear()
rdt['time'] = np.arange(20,40)
publisher.publish(rdt.to_granule())
self.assertFalse(dataset_modified.wait(2))
self.dpsc_cli.activate_data_product_persistence(dp_id)
dataset_modified.clear()
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_almost_equal(rdt['time'], np.arange(40))
dataset_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasDataset, id_only=True)
self.assertEquals(len(dataset_ids), 1)
self.dpsc_cli.suspend_data_product_persistence(dp_id)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
with self.assertRaises(NotFound):
dp_obj = self.rrclient.read(dp_id)
def test_lookup_values(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_lookups()
stream_def_id = self.pubsubcli.create_stream_definition('lookup', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsubcli.delete_stream_definition, stream_def_id)
data_product = DataProduct(name='lookup data product')
tdom, sdom = time_series_domain()
data_product.temporal_domain = tdom.dump()
data_product.spatial_domain = sdom.dump()
data_product_id = self.dpsc_cli.create_data_product(data_product, stream_definition_id=stream_def_id)
self.addCleanup(self.dpsc_cli.delete_data_product, data_product_id)
data_producer = DataProducer(name='producer')
data_producer.producer_context = DataProcessProducerContext()
data_producer.producer_context.configuration['qc_keys'] = ['offset_document']
data_producer_id, _ = self.rrclient.create(data_producer)
self.addCleanup(self.rrclient.delete, data_producer_id)
assoc,_ = self.rrclient.create_association(subject=data_product_id, object=data_producer_id, predicate=PRED.hasDataProducer)
self.addCleanup(self.rrclient.delete_association, assoc)
document_keys = self.damsclient.list_qc_references(data_product_id)
self.assertEquals(document_keys, ['offset_document'])
svm = StoredValueManager(self.container)
svm.stored_value_cas('offset_document', {'offset_a':2.0})
self.dpsc_cli.activate_data_product_persistence(data_product_id)
dataset_ids, _ = self.rrclient.find_objects(subject=data_product_id, predicate=PRED.hasDataset, id_only=True)
dataset_id = dataset_ids[0]
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [0]
rdt['temp'] = [20.]
granule = rdt.to_granule()
stream_ids, _ = self.rrclient.find_objects(subject=data_product_id, predicate=PRED.hasStream, id_only=True)
stream_id = stream_ids[0]
route = self.pubsubcli.read_stream_route(stream_id=stream_id)
publisher = StandaloneStreamPublisher(stream_id, route)
publisher.publish(granule)
self.assertTrue(dataset_monitor.event.wait(10))
granule = self.data_retriever.retrieve(dataset_id)
rdt2 = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt['temp'], rdt2['temp'])
np.testing.assert_array_almost_equal(rdt2['calibrated'], np.array([22.0]))
svm.stored_value_cas('updated_document', {'offset_a':3.0})
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ep = EventPublisher(event_type=OT.ExternalReferencesUpdatedEvent)
ep.publish_event(origin=data_product_id, reference_keys=['updated_document'])
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [1]
rdt['temp'] = [20.]
granule = rdt.to_granule()
gevent.sleep(2) # Yield so that the event goes through
publisher.publish(granule)
self.assertTrue(dataset_monitor.event.wait(10))
granule = self.data_retriever.retrieve(dataset_id)
rdt2 = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt2['temp'],np.array([20.,20.]))
np.testing.assert_array_almost_equal(rdt2['calibrated'], np.array([22.0,23.0]))
class TestCTDTransformsIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
print 'started services'
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name+'_logger')
producer_definition.executable = {
'module':'ion.processes.data.stream_granule_logger',
'class':'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(process_definition=producer_definition)
configuration = {
'process':{
'stream_id':stream_id,
}
}
pid = self.processdispatchclient.schedule_process(process_definition_id= logger_procdef_id, configuration=configuration)
return pid
def test_createTransformsThenActivateInstrument(self):
self.loggerpids = []
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel, name='SBE37IMModel', description="SBE37IMModel", model="SBE37IMModel" )
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" %ex)
log.debug( 'new InstrumentModel id = %s ', instModel_id)
# Create InstrumentAgent
instAgent_obj = IonObject(RT.InstrumentAgent, name='agent007', description="SBE37IMAgent", driver_module="ion.agents.instrument.instrument_agent", driver_class="InstrumentAgent" )
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" %ex)
log.debug( 'new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug('test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ')
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
try:
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" %ex)
log.debug("test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id)
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance', description="SBE37IMAgentInstance",
driver_module='mi.instrument.seabird.sbe37smb.ooicore.driver', driver_class='SBE37Driver',
comms_device_address='sbe37-simulator.oceanobservatories.org', comms_device_port=4001, port_agent_work_dir='/tmp/', port_agent_delimeter=['<<','>>'] )
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id)
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubclient.create_stream_definition(container=ctd_stream_def)
log.debug( 'new Stream Definition id = %s', instDevice_id)
log.debug( 'Creating new CDM data product with a stream definition')
craft = CoverageCraft
sdom, tdom = craft.create_domains()
sdom = sdom.dump()
tdom = tdom.dump()
parameter_dictionary = craft.create_parameters()
parameter_dictionary = parameter_dictionary.dump()
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_parsed_data_product = self.dataproductclient.create_data_product(dp_obj, ctd_stream_def_id, parameter_dictionary)
log.debug( 'new dp_id = %s', ctd_parsed_data_product)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_parsed_data_product, PRED.hasStream, None, True)
log.debug( 'Data product streams1 = %s', stream_ids)
pid = self.create_logger('ctd_parsed', stream_ids[0] )
self.loggerpids.append(pid)
print 'test_createTransformsThenActivateInstrument: Data product streams1 = ', stream_ids
#-------------------------------
# Create CTD Raw as the second data product
#-------------------------------
log.debug( 'Creating new RAW data product with a stream definition')
raw_stream_def = SBE37_RAW_stream_definition()
raw_stream_def_id = self.pubsubclient.create_stream_definition(container=raw_stream_def)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_raw_data_product = self.dataproductclient.create_data_product(dp_obj, raw_stream_def_id, parameter_dictionary)
log.debug( 'new dp_id = %s', str(ctd_raw_data_product))
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_raw_data_product)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_raw_data_product)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True)
log.debug( 'Data product streams2 = %s', str(stream_ids))
#todo: attaching the taxonomy to the stream is a TEMPORARY measure
# Create taxonomies for both parsed and attach to the stream
RawTax = TaxyTool()
RawTax.add_taxonomy_set('raw_fixed','Fixed length bytes in an array of records')
RawTax.add_taxonomy_set('raw_blob','Unlimited length bytes in an array')
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(ctd_raw_data_product, PRED.hasStream, None, True)
print 'Data product streams2 = ', stream_ids
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition ctd_L0_all")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all',
process_source='some_source_reference')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" %ex)
#-------------------------------
# L1 Conductivity: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1ConductivityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_conductivity',
description='create the L1 conductivity data product',
module='ion.processes.data.transforms.ctd.ctd_L1_conductivity',
class_name='CTDL1ConductivityTransform',
process_source='CTDL1ConductivityTransform source code here...')
try:
ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1ConductivityTransform data process definition: %s" %ex)
#-------------------------------
# L1 Pressure: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1PressureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_pressure',
description='create the L1 pressure data product',
module='ion.processes.data.transforms.ctd.ctd_L1_pressure',
class_name='CTDL1PressureTransform',
process_source='CTDL1PressureTransform source code here...')
try:
ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1PressureTransform data process definition: %s" %ex)
#-------------------------------
# L1 Temperature: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition CTDL1TemperatureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_temperature',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L1_temperature',
class_name='CTDL1TemperatureTransform',
process_source='CTDL1TemperatureTransform source code here...')
try:
ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1TemperatureTransform data process definition: %s" %ex)
#-------------------------------
# L2 Salinity: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition SalinityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_salinity',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform',
process_source='SalinityTransform source code here...')
try:
ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new SalinityTransform data process definition: %s" %ex)
#-------------------------------
# L2 Density: Data Process Definition
#-------------------------------
log.debug("TestIntDataProcessMgmtServiceMultiOut: create data process definition DensityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_density',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_density',
class_name='DensityTransform',
process_source='DensityTransform source code here...')
try:
ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new DensityTransform data process definition: %s" %ex)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity = L0_conductivity_stream_definition()
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_conductivity, name='L0_Conductivity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id )
outgoing_stream_l0_pressure = L0_pressure_stream_definition()
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_pressure, name='L0_Pressure')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id )
outgoing_stream_l0_temperature = L0_temperature_stream_definition()
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l0_temperature, name='L0_Temperature')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id )
self.output_products={}
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id,
parameter_dictionary)
self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_conductivity_output_dp_id)
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_pressure_output_dp_obj,
outgoing_stream_l0_pressure_id,
parameter_dictionary)
self.output_products['pressure'] = ctd_l0_pressure_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_pressure_output_dp_id)
log.debug("test_createTransformsThenActivateInstrument: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l0_temperature_output_dp_obj,
outgoing_stream_l0_temperature_id,
parameter_dictionary)
self.output_products['temperature'] = ctd_l0_temperature_output_dp_id
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l0_temperature_output_dp_id)
#-------------------------------
# L1 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l1_conductivity = L1_conductivity_stream_definition()
outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_conductivity, name='L1_conductivity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id )
outgoing_stream_l1_pressure = L1_pressure_stream_definition()
outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_pressure, name='L1_Pressure')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id )
outgoing_stream_l1_temperature = L1_temperature_stream_definition()
outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l1_temperature, name='L1_Temperature')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id )
log.debug("test_createTransformsThenActivateInstrument: create output data product L1 conductivity")
ctd_l1_conductivity_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Conductivity',
description='transform output L1 conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_conductivity_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_conductivity_output_dp_obj,
outgoing_stream_l1_conductivity_id,
parameter_dictionary)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l1_conductivity_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(ctd_l1_conductivity_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger('ctd_l1_conductivity', stream_ids[0] )
self.loggerpids.append(pid)
log.debug("test_createTransformsThenActivateInstrument: create output data product L1 pressure")
ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Pressure',
description='transform output L1 pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_pressure_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_pressure_output_dp_obj,
outgoing_stream_l1_pressure_id,
parameter_dictionary
)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l1_pressure_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(ctd_l1_pressure_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger('ctd_l1_pressure', stream_ids[0] )
self.loggerpids.append(pid)
log.debug("test_createTransformsThenActivateInstrument: create output data product L1 temperature")
ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Temperature',
description='transform output L1 temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_temperature_output_dp_id = self.dataproductclient.create_data_product(ctd_l1_temperature_output_dp_obj,
outgoing_stream_l1_temperature_id,
parameter_dictionary)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l1_temperature_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(ctd_l1_temperature_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger('ctd_l1_temperature', stream_ids[0] )
self.loggerpids.append(pid)
#-------------------------------
# L2 Salinity - Density: Output Data Products
#-------------------------------
outgoing_stream_l2_salinity = L2_practical_salinity_stream_definition()
outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l2_salinity, name='L2_salinity')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id )
outgoing_stream_l2_density = L2_density_stream_definition()
outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(container=outgoing_stream_l2_density, name='L2_Density')
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id )
log.debug("test_createTransformsThenActivateInstrument: create output data product L2 Salinity")
ctd_l2_salinity_output_dp_obj = IonObject(RT.DataProduct,
name='L2_Salinity',
description='transform output L2 salinity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l2_salinity_output_dp_id = self.dataproductclient.create_data_product(ctd_l2_salinity_output_dp_obj,
outgoing_stream_l2_salinity_id,
parameter_dictionary)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l2_salinity_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(ctd_l2_salinity_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger('ctd_l2_salinity', stream_ids[0] )
self.loggerpids.append(pid)
log.debug("test_createTransformsThenActivateInstrument: create output data product L2 Density")
ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct,
name='L2_Density',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l2_density_output_dp_id = self.dataproductclient.create_data_product(ctd_l2_density_output_dp_obj,
outgoing_stream_l2_density_id,
parameter_dictionary)
self.dataproductclient.activate_data_product_persistence(data_product_id=ctd_l2_density_output_dp_id)
# Retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(ctd_l2_density_output_dp_id, PRED.hasStream, None, True)
pid = self.create_logger('ctd_l2_density', stream_ids[0] )
self.loggerpids.append(pid)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L0 all data_process start")
try:
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products)
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L0 all data_process return")
#-------------------------------
# L1 Conductivity: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L1 Conductivity data_process start")
try:
l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_conductivity_dprocdef_id, [ctd_l0_conductivity_output_dp_id], {'output':ctd_l1_conductivity_output_dp_id})
self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L1 Conductivity data_process return")
#-------------------------------
# L1 Pressure: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process start")
try:
l1_pressure_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_pressure_dprocdef_id, [ctd_l0_pressure_output_dp_id], {'output':ctd_l1_pressure_output_dp_id})
self.dataprocessclient.activate_data_process(l1_pressure_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process return")
#-------------------------------
# L1 Temperature: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process start")
try:
l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_temperature_dprocdef_id, [ctd_l0_temperature_output_dp_id], {'output':ctd_l1_temperature_output_dp_id})
self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L1_Pressure data_process return")
#-------------------------------
# L2 Salinity: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L2_salinity data_process start")
try:
l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_salinity_dprocdef_id, [ctd_parsed_data_product], {'output':ctd_l2_salinity_output_dp_id})
self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L2_salinity data_process return")
#-------------------------------
# L2 Density: Create the data process
#-------------------------------
log.debug("test_createTransformsThenActivateInstrument: create L2_Density data_process start")
try:
l2_density_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_density_dprocdef_id, [ctd_parsed_data_product], {'output':ctd_l2_density_output_dp_id})
self.dataprocessclient.activate_data_process(l2_density_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
log.debug("test_createTransformsThenActivateInstrument: create L2_Density data_process return")
#-------------------------------
# Launch InstrumentAgentInstance, connect to the resource agent client
#-------------------------------
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
print 'test_createTransformsThenActivateInstrument: Instrument agent instance obj: = ', inst_agent_instance_obj
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient('iaclient', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess())
print 'activate_instrument: got ia client %s', self._ia_client
log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client))
#-------------------------------
# Streaming
#-------------------------------
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentStream: initialize %s", str(retval))
log.debug("test_activateInstrumentStream: Sending go_active command (L4-CI-SA-RQ-334)")
cmd = AgentCommand(command='go_active')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentStream: return value from go_active %s", str(reply))
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("test_activateInstrumentStream: current state after sending go_active command %s (L4-CI-SA-RQ-334)", str(state))
cmd = AgentCommand(command='run')
reply = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("test_activateInstrumentStream: return from run state: %s", str(state))
# Make sure the sampling rate and transmission are sane.
params = {
SBE37Parameter.NAVG : 1,
SBE37Parameter.INTERVAL : 5,
SBE37Parameter.TXREALTIME : True
}
self._ia_client.set_param(params)
log.debug("test_activateInstrumentStream: calling go_streaming ")
cmd = AgentCommand(command='go_streaming')
reply = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("test_activateInstrumentStream: return from go_streaming state: %s", str(state))
time.sleep(7)
log.debug("test_activateInstrumentStream: calling go_observatory")
cmd = AgentCommand(command='go_observatory')
reply = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("test_activateInstrumentStream: return from go_observatory state %s", str(state))
log.debug("test_activateInstrumentStream: calling reset ")
cmd = AgentCommand(command='reset')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentStream: return from reset state:%s", str(reply.result))
time.sleep(2)
#-------------------------------
# Deactivate InstrumentAgentInstance
#-------------------------------
self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
class TestActivateInstrumentIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
print 'started services'
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(
node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dpclient = DataProductManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
#setup listerner vars
self._data_greenlets = []
self._no_samples = None
self._samples_received = []
@unittest.skip("TBD")
def test_activateInstrumentSample(self):
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel",
model_label="SBE37IMModel")
try:
instModel_id = self.imsclient.create_instrument_model(
instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" % ex)
print 'new InstrumentModel id = ', instModel_id
# Create InstrumentAgent
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_module="ion.agents.instrument.instrument_agent",
driver_class="InstrumentAgent")
try:
instAgent_id = self.imsclient.create_instrument_agent(
instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" % ex)
print 'new InstrumentAgent id = ', instAgent_id
self.imsclient.assign_instrument_model_to_instrument_agent(
instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug(
'test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) '
)
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345")
try:
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" % ex)
log.debug(
"test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ",
instDevice_id)
driver_config = {
'dvr_mod': 'ion.agents.instrument.drivers.sbe37.sbe37_driver',
'dvr_cls': 'SBE37Driver',
'workdir': '/tmp/',
}
instAgentInstance_obj = IonObject(
RT.InstrumentAgentInstance,
name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
driver_config=driver_config,
comms_device_address='sbe37-simulator.oceanobservatories.org',
comms_device_port=4001,
port_agent_work_dir='/tmp/',
port_agent_delimeter=['<<', '>>'])
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id)
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubcli.create_stream_definition(
container=ctd_stream_def)
print 'new Stream Definition id = ', instDevice_id
print 'Creating new CDM data product with a stream definition'
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test')
try:
data_product_id1 = self.dpclient.create_data_product(
dp_obj, ctd_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" % ex)
print 'new dp_id = ', data_product_id1
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id1)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id1,
persist_data=True,
persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1,
PRED.hasStream, None, True)
print 'Data product streams1 = ', stream_ids
print 'Creating new RAW data product with a stream definition'
raw_stream_def = SBE37_RAW_stream_definition()
raw_stream_def_id = self.pubsubcli.create_stream_definition(
container=raw_stream_def)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test')
try:
data_product_id2 = self.dpclient.create_data_product(
dp_obj, raw_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" % ex)
print 'new dp_id = ', data_product_id2
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id2,
persist_data=True,
persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2,
PRED.hasStream, None, True)
print 'Data product streams2 = ', stream_ids
self.imsclient.start_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(
instAgentInstance_id)
print 'Instrument agent instance obj: = ', inst_agent_instance_obj
# Start a resource agent client to talk with the instrument agent.
#self._ia_client = ResourceAgentClient('123xyz', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess())
self._ia_client = ResourceAgentClient(instDevice_id,
process=FakeProcess())
print 'activate_instrument: got ia client %s', self._ia_client
log.debug("test_activateInstrumentSample: got ia client %s",
str(self._ia_client))
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
print retval
log.debug("test_activateInstrumentSample: initialize %s", str(retval))
time.sleep(2)
log.debug(
"test_activateInstrumentSample: Sending go_active command (L4-CI-SA-RQ-334)"
)
cmd = AgentCommand(command='go_active')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrument: return value from go_active %s",
str(reply))
time.sleep(2)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug(
"test_activateInstrumentSample: current state after sending go_active command %s (L4-CI-SA-RQ-334)",
str(state))
cmd = AgentCommand(command='run')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: run %s", str(reply))
time.sleep(2)
log.debug("test_activateInstrumentSample: calling acquire_sample ")
cmd = AgentCommand(command='acquire_sample')
reply = self._ia_client.execute(cmd)
log.debug(
"test_activateInstrumentSample: return from acquire_sample %s",
str(reply))
time.sleep(2)
log.debug("test_activateInstrumentSample: calling acquire_sample 2")
cmd = AgentCommand(command='acquire_sample')
reply = self._ia_client.execute(cmd)
log.debug(
"test_activateInstrumentSample: return from acquire_sample 2 %s",
str(reply))
time.sleep(2)
log.debug("test_activateInstrumentSample: calling acquire_sample 3")
cmd = AgentCommand(command='acquire_sample')
reply = self._ia_client.execute(cmd)
log.debug(
"test_activateInstrumentSample: return from acquire_sample 3 %s",
str(reply))
time.sleep(2)
log.debug("test_activateInstrumentSample: calling reset ")
cmd = AgentCommand(command='reset')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: return from reset %s",
str(reply))
time.sleep(2)
#-------------------------------
# Deactivate InstrumentAgentInstance
#-------------------------------
self.imsclient.stop_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id)
def test_activateInstrumentStream(self):
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel",
model_label="SBE37IMModel")
try:
instModel_id = self.imsclient.create_instrument_model(
instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" % ex)
print 'new InstrumentModel id = ', instModel_id
# Create InstrumentAgent
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_module="ion.agents.instrument.instrument_agent",
driver_class="InstrumentAgent")
try:
instAgent_id = self.imsclient.create_instrument_agent(
instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" % ex)
print 'new InstrumentAgent id = ', instAgent_id
self.imsclient.assign_instrument_model_to_instrument_agent(
instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug(
'test_activateInstrumentStream: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) '
)
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345")
try:
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" % ex)
log.debug(
"test_activateInstrumentStream: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ",
instDevice_id)
driver_config = {
'dvr_mod': 'ion.agents.instrument.drivers.sbe37.sbe37_driver',
'dvr_cls': 'SBE37Driver',
'workdir': '/tmp/',
}
instAgentInstance_obj = IonObject(
RT.InstrumentAgentInstance,
name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
driver_config=driver_config,
comms_device_address='sbe37-simulator.oceanobservatories.org',
comms_device_port=4001,
port_agent_work_dir='/tmp/',
port_agent_delimeter=['<<', '>>'])
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id)
# create a stream definition for the data from the ctd simulator
ctd_stream_def = SBE37_CDM_stream_definition()
ctd_stream_def_id = self.pubsubcli.create_stream_definition(
container=ctd_stream_def)
log.debug(
'test_activateInstrumentStream new Stream Definition id = %s',
instDevice_id)
log.debug(
'test_activateInstrumentStream Creating new CDM data product with a stream definition'
)
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test')
try:
data_product_id1 = self.dpclient.create_data_product(
dp_obj, ctd_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" % ex)
log.debug('test_activateInstrumentStream new dp_id = %s',
str(data_product_id1))
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id1)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id1,
persist_data=True,
persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1,
PRED.hasStream, None, True)
log.debug('test_activateInstrumentStream Data product streams1 = %s',
str(stream_ids))
simdata_subscription_id = self.pubsubcli.create_subscription(
query=StreamQuery([stream_ids[0]]),
exchange_name='Sim_data_queue',
name='SimDataSubscription',
description='SimData SubscriptionDescription')
def simdata_message_received(message, headers):
input = str(message)
log.debug("test_activateInstrumentStream: granule received: %s",
input)
subscriber_registrar = StreamSubscriberRegistrar(
process=self.container, node=self.container.node)
simdata_subscriber = subscriber_registrar.create_subscriber(
exchange_name='Sim_data_queue', callback=simdata_message_received)
# Start subscribers
simdata_subscriber.start()
# Activate subscriptions
self.pubsubcli.activate_subscription(simdata_subscription_id)
log.debug(
'test_activateInstrumentStream Creating new RAW data product with a stream definition'
)
raw_stream_def = SBE37_RAW_stream_definition()
raw_stream_def_id = self.pubsubcli.create_stream_definition(
container=raw_stream_def)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test')
try:
data_product_id2 = self.dpclient.create_data_product(
dp_obj, raw_stream_def_id)
except BadRequest as ex:
self.fail("failed to create new data product: %s" % ex)
log.debug('test_activateInstrumentStream new dp_id = %s',
str(data_product_id2))
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id2,
persist_data=True,
persist_metadata=True)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2,
PRED.hasStream, None, True)
log.debug('test_activateInstrumentStream Data product streams2 = %s',
str(stream_ids))
self.imsclient.start_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(
instAgentInstance_id)
log.debug(
'test_activateInstrumentStream Instrument agent instance obj: = %s',
str(inst_agent_instance_obj))
# Start a resource agent client to talk with the instrument agent.
#self._ia_client = ResourceAgentClient('123xyz', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess())
self._ia_client = ResourceAgentClient(instDevice_id,
process=FakeProcess())
log.debug('test_activateInstrumentStream: got ia client %s',
str(self._ia_client))
log.debug("test_activateInstrumentStream: got ia client %s",
str(self._ia_client))
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentStream: initialize %s", str(retval))
time.sleep(2)
log.debug(
"test_activateInstrumentStream: Sending go_active command (L4-CI-SA-RQ-334)"
)
cmd = AgentCommand(command='go_active')
reply = self._ia_client.execute_agent(cmd)
log.debug(
"test_activateInstrumentStream: return value from go_active %s",
str(reply))
time.sleep(2)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug(
"test_activateInstrumentStream: current state after sending go_active command %s (L4-CI-SA-RQ-334)",
str(state))
cmd = AgentCommand(command='run')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrument: run %s", str(reply))
time.sleep(2)
log.debug("test_activateInstrumentStream: calling go_streaming ")
cmd = AgentCommand(command='go_streaming')
reply = self._ia_client.execute(cmd)
log.debug("test_activateInstrumentStream: return from go_streaming %s",
str(reply))
time.sleep(15)
log.debug("test_activateInstrumentStream: calling go_observatory")
cmd = AgentCommand(command='go_observatory')
reply = self._ia_client.execute(cmd)
log.debug(
"test_activateInstrumentStream: return from go_observatory %s",
str(reply))
time.sleep(2)
log.debug("test_activateInstrumentStream: calling reset ")
cmd = AgentCommand(command='reset')
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentStream: return from reset %s",
str(reply))
time.sleep(2)
#-------------------------------
# Deactivate InstrumentAgentInstance
#-------------------------------
self.imsclient.stop_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id)
