def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
streams = {
'parsed' : 'ctd_parsed_param_dict',
'raw' : 'ctd_raw_param_dict'
}
for (stream_name, param_dict_name) in streams.iteritems():
pd_id = dataset_management.read_parameter_dictionary_by_name(DEFAULT_PARAM_DICT, id_only=True)
if (not pd_id):
log.error("No pd_id found for param_dict '%s'" % DEFAULT_PARAM_DICT)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name,
parameter_dictionary_id=pd_id)
pd = None
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(stream_route=stream_route,
routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
self.stream_config[stream_name] = stream_config
def get_last_granule(cls, container, dataset_id):
dsm_cli = DatasetManagementServiceClient()
dataset = dsm_cli.read_dataset(dataset_id)
cc = container
datastore_name = dataset.datastore_name
view_name = dataset.view_name
datastore = cc.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
opts = dict(
start_key = [dataset_id, {}],
end_key = [dataset_id, 0],
descending = True,
limit = 1,
include_docs = True
)
results = datastore.query_view(view_name,opts=opts)
if not results:
raise NotFound('A granule could not be located.')
if results[0] is None:
raise NotFound('A granule could not be located.')
doc = results[0].get('doc')
if doc is None:
return None
ts = float(doc.get('ts_create',0))
coverage = DatasetManagementService._get_coverage(dataset_id)
rdt = cls._coverage_to_granule(coverage,tdoa=slice(cls.get_relative_time(coverage,ts),None))
coverage.close(timeout=5)
return rdt.to_granule()
def setUp(self):
# Start container
super(TestActivateInstrumentIntegration, self).setUp()
config = DotDict()
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml', config)
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataretrieverclient = DataRetrieverServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.usernotificationclient = UserNotificationServiceClient()
#setup listerner vars
self._data_greenlets = []
self._no_samples = None
self._samples_received = []
self.event_publisher = EventPublisher()
def load_data_product(self):
dset_i = 0
dataset_management = DatasetManagementServiceClient()
pubsub_management = PubsubManagementServiceClient()
data_product_management = DataProductManagementServiceClient()
resource_registry = self.container.instance.resource_registry
tdom, sdom = time_series_domain()
tdom = tdom.dump()
sdom = sdom.dump()
dp_obj = DataProduct(
name='instrument_data_product_%i' % dset_i,
description='ctd stream test',
processing_level_code='Parsed_Canonical',
temporal_domain = tdom,
spatial_domain = sdom)
pdict_id = dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
stream_def_id = pubsub_management.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id)
self.addCleanup(pubsub_management.delete_stream_definition, stream_def_id)
data_product_id = data_product_management.create_data_product(data_product=dp_obj, stream_definition_id=stream_def_id)
self.addCleanup(data_product_management.delete_data_product, data_product_id)
data_product_management.activate_data_product_persistence(data_product_id)
self.addCleanup(data_product_management.suspend_data_product_persistence, data_product_id)
stream_ids, assocs = resource_registry.find_objects(subject=data_product_id, predicate='hasStream', id_only=True)
stream_id = stream_ids[0]
route = pubsub_management.read_stream_route(stream_id)
dataset_ids, assocs = resource_registry.find_objects(subject=data_product_id, predicate='hasDataset', id_only=True)
dataset_id = dataset_ids[0]
return data_product_id, stream_id, route, stream_def_id, dataset_id
def get_last_granule(cls, container, dataset_id):
dsm_cli = DatasetManagementServiceClient()
dataset = dsm_cli.read_dataset(dataset_id)
cc = container
datastore_name = dataset.datastore_name
view_name = dataset.view_name
datastore = cc.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
opts = dict(
start_key = [dataset_id, {}],
end_key = [dataset_id, 0],
descending = True,
limit = 1,
include_docs = True
)
results = datastore.query_view(view_name,opts=opts)
if not results:
raise NotFound('A granule could not be located.')
if results[0] is None:
raise NotFound('A granule could not be located.')
doc = results[0].get('doc')
if doc is None:
return None
ts = float(doc.get('ts_create',0))
coverage = DatasetManagementService._get_coverage(dataset_id)
black_box = CoverageCraft(coverage)
black_box.sync_rdt_with_coverage(start_time=ts,end_time=None)
granule = black_box.to_granule()
return granule
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
encoder = IonObjectSerializer()
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
stream_name = 'ctdpf_parsed'
param_dict_name = 'ctdpf_parsed'
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
stream_def = pubsub_client.read_stream_definition(stream_def_id)
stream_def_dict = encoder.serialize(stream_def)
pd = stream_def.parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
parameter_dictionary=pd,
stream_def_dict=stream_def_dict)
self._stream_config[stream_name] = stream_config
def build_stream_config(streams):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=cc.node)
dataset_management = DatasetManagementServiceClient()
# Create streams and subscriptions for each stream named in driver.
agent_stream_config = {}
for (stream_name, param_dict_name) in streams.iteritems():
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(stream_route=stream_route,
routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
agent_stream_config[stream_name] = stream_config
return agent_stream_config
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dpclient = DataProductManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataretrieverclient = DataRetrieverServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
#setup listerner vars
self._data_greenlets = []
self._no_samples = None
self._samples_received = []
self.event_publisher = EventPublisher()
self.egg_url_good = "http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1a-py2.7.egg"
self.egg_url_bad = "http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.1a-py2.7.egg"
self.egg_url_404 = "http://sddevrepo.oceanobservatories.org/releases/completely_made_up_404.egg"
def get_parameter_function(cls, parameter_function_id=''):
'''
Preferred client-side class method for constructing a parameter function
'''
dms_cli = DatasetManagementServiceClient()
pf_res = dms_cli.read_parameter_function(parameter_function_id=parameter_function_id)
pf = AbstractFunction.load(pf_res.parameter_function)
pf._identifier = pf._id
return pf
class DatasetManagementIntTest(IonIntegrationTestCase):
def setUp(self):
import couchdb
super(DatasetManagementIntTest,self).setUp()
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.db = self.container.datastore_manager.get_datastore('scidata', DataStore.DS_PROFILE.SCIDATA)
self.db_raw = self.db.server
self.dataset_management_client = DatasetManagementServiceClient(node=self.container.node)
self.ingestion_client = IngestionManagementServiceClient(node=self.container.node)
def _random_data(self, entropy):
random_pressures = [(random.random()*100) for i in xrange(entropy)]
random_salinity = [(random.random()*28) for i in xrange(entropy)]
random_temperature = [(random.random()*10)+32 for i in xrange(entropy)]
random_times = [random.randrange(1328205227, 1328896395) for i in xrange(entropy)]
random_lat = [(random.random()*10)+30 for i in xrange(entropy)]
random_lon = [(random.random()*10)+70 for i in xrange(entropy)]
return [random_pressures, random_salinity, random_temperature, random_times, random_lat, random_lon]
def _generate_point(self, entropy=5):
points = []
random_values = self._random_data(entropy)
point = ctd_stream_packet(stream_id='test_data', p=random_values[0], c=random_values[1], t=random_values[2],time=random_values[3], lat=random_values[4], lon=random_values[5], create_hdf=False)
return point
def test_get_dataset_bounds(self):
for i in xrange(3):
point = self._generate_point()
self.db.create(point)
dataset_id = self.dataset_management_client.create_dataset(stream_id='test_data', datastore_name='scidata')
bounds = self.dataset_management_client.get_dataset_bounds(dataset_id=dataset_id)
self.assertTrue(bounds['latitude_bounds'][0] > 30.0)
self.assertTrue(bounds['latitude_bounds'][1] < 40.0)
self.assertTrue(bounds['longitude_bounds'][0] > 70.0)
self.assertTrue(bounds['longitude_bounds'][1] < 80.0)
self.dataset_management_client.delete_dataset(dataset_id)
@unittest.skip('not ready yet')
def test_dataset_ingestion(self):
couch_storage = { 'server':'localhost', 'database':'scidata'}
ingestion_configuration_id = self.ingestion_client.create_ingestion_configuration(
exchange_point_id='science_data',
couch_storage=couch_storage,
hdf_storage={},
number_of_workers=4,
default_policy={})
def _build_stream_config(self):
"""
"""
if not self.packet_config:
return
streams = self.packet_config
log.debug("Streams: %s", streams)
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
# Create streams and subscriptions for each stream named in driver.
self.stream_config = {}
for stream_name in streams:
pd_id = None
try:
pd_id = dataset_management.read_parameter_dictionary_by_name(stream_name, id_only=True)
except:
log.error("No pd_id found for param_dict '%s'" % stream_name)
if self.use_default_stream:
log.error("using default pd '%s'" % DEFAULT_STREAM_NAME)
pd_id = dataset_management.read_parameter_dictionary_by_name(DEFAULT_STREAM_NAME, id_only=True)
if not pd_id:
raise IDKException("Missing parameter dictionary for stream '%s'" % stream_name)
log.debug("parameter dictionary id: %s" % pd_id)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
# log.debug("Stream: %s (%s), stream_def_id %s" % (stream_name, type(stream_name), stream_def_id))
pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary
# log.debug("Parameter Dictionary: %s" % pd)
try:
stream_id, stream_route = pubsub_client.create_stream(
name=stream_name, exchange_point="science_data", stream_definition_id=stream_def_id
)
stream_config = dict(
stream_route=stream_route,
routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd,
)
self.stream_config[stream_name] = stream_config
# log.debug("Stream Config (%s): %s" % (stream_name, stream_config))
except Exception as e:
log.error("stream publisher exception: %s", e)
log.debug("Stream config setup complete.")
def get_parameter_dictionary(cls, parameter_dictionary_id=''):
"""
Class method to return a CoverageModel ParameterDictionary object from the
ION Resources. The object is built from the associated parameter contexts.
"""
dms_cli = DatasetManagementServiceClient()
pd = dms_cli.read_parameter_dictionary(parameter_dictionary_id)
pcs = dms_cli.read_parameter_contexts(parameter_dictionary_id=parameter_dictionary_id, id_only=False)
return cls.build_parameter_dictionary(pd, pcs)
def get_parameter_context(cls, parameter_context_id=''):
'''
Preferred client-side class method for constructing a parameter context
from a service call.
'''
dms_cli = DatasetManagementServiceClient()
pc_res = dms_cli.read_parameter_context(parameter_context_id=parameter_context_id)
pc = ParameterContext.load(pc_res.parameter_context)
pc._identifier = pc_res._id
return pc
def get_parameter_dictionary(cls, parameter_dictionary_id=''):
'''
Preferred client-side class method for constructing a parameter dictionary
from a service call.
'''
dms_cli = DatasetManagementServiceClient()
pd = dms_cli.read_parameter_dictionary(parameter_dictionary_id)
pcs = dms_cli.read_parameter_contexts(parameter_dictionary_id=parameter_dictionary_id, id_only=False)
return cls.build_parameter_dictionary(pd, pcs)
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
streams = {
'parsed': 'ctd_parsed_param_dict',
'raw': 'ctd_raw_param_dict'
}
for (stream_name, param_dict_name) in streams.iteritems():
pd_id = dataset_management.read_parameter_dictionary_by_name(
param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(
name=stream_name, parameter_dictionary_id=pd_id)
pd = pubsub_client.read_stream_definition(
stream_def_id).parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(
name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(
stream_route=stream_route,
routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
if stream_name == 'parsed':
type = 'IntervalAlarmDef'
kwargs = {
'name': 'test_sim_warning',
'stream_name': 'parsed',
'value_id': 'temp',
'message': 'Temperature is above test range of 5.0.',
'type': StreamAlarmType.WARNING,
'upper_bound': 5.0,
'upper_rel_op': '<'
}
alarm = {}
alarm['type'] = type
alarm['kwargs'] = kwargs
alarms = [alarm]
stream_config['alarms'] = alarms
self._stream_config[stream_name] = stream_config
def create_dataset(self, parameter_dict_id=''):
'''
Creates a time-series dataset
'''
dataset_management = DatasetManagementServiceClient()
if not parameter_dict_id:
parameter_dict_id = dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
dataset_id = dataset_management.create_dataset('test_dataset_', parameter_dictionary_id=parameter_dict_id)
self.addCleanup(dataset_management.delete_dataset, dataset_id)
return dataset_id
class RegistrationProcessTest(IonIntegrationTestCase):
def setUp(self):
#print >> sys.stderr, "setup"
self._start_container()
#print >> sys.stderr, "start container"
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
#print >> sys.stderr, "deploy"
self.dataset_management = DatasetManagementServiceClient()
#print >> sys.stderr, "dataset management"
#setup registry process and patch in CFG
def init(self):
super(RegistrationProcess, self).__init__()
self.CFG = CFG
RegistrationProcess.__init__ = init
self.rp = RegistrationProcess()
self.rp.on_start()
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Host requires file-system access to coverage files, CEI mode does not support.')
def test_get_dataset_to_xml(self):
dataset_id = self._make_dataset()
coverage_path = DatasetManagementService()._get_coverage_path(dataset_id)
cov = SimplexCoverage.load(coverage_path)
xml_str = self.rp.get_dataset_xml(coverage_path)
dom = parseString(xml_str)
node = dom.getElementsByTagName('addAttributes')
metadata = node[0]
for n in metadata.childNodes:
if n.nodeType != 3:
if n.attributes["name"].value == "title":
self.assertEquals(cov.name, n.childNodes[0].nodeValue)
if n.attributes["name"].value == "institution":
self.assertEquals('OOI', n.childNodes[0].nodeValue)
if n.attributes["name"].value == "infoUrl":
self.assertEquals(self.rp.pydap_url+cov.name, n.childNodes[0].nodeValue)
parameters = []
node = dom.getElementsByTagName('sourceName')
for n in node:
if n.nodeType != 3:
parameters.append(str(n.childNodes[0].nodeValue))
cov_params = [key for key in cov.list_parameters()]
self.assertEquals(parameters, cov_params)
cov.close()
def _make_dataset(self):
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
parameter_dict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
dataset_id = self.dataset_management.create_dataset('test_dataset', parameter_dictionary_id=parameter_dict_id, spatial_domain=sdom, temporal_domain=tdom)
return dataset_id
def on_start(self):
super(ReplayProcess,self).on_start()
dsm_cli = DatasetManagementServiceClient()
self.dataset_id = self.CFG.get_safe('process.dataset_id', None)
self.delivery_format = self.CFG.get_safe('process.delivery_format',{})
self.start_time = self.CFG.get_safe('process.delivery_format.start_time', None)
self.end_time = self.CFG.get_safe('process.delivery_format.end_time', None)
if self.dataset_id is None:
raise BadRequest('dataset_id not specified')
self.dataset = dsm_cli.read_dataset(self.dataset_id)
def get_parameter_dictionary(cls, parameter_dictionary_id=''):
'''
Preferred client-side class method for constructing a parameter dictionary
from a service call.
'''
dms_cli = DatasetManagementServiceClient()
pd = dms_cli.read_parameter_dictionary(parameter_dictionary_id)
pcs = dms_cli.read_parameter_contexts(parameter_dictionary_id=parameter_dictionary_id, id_only=False)
pdict = cls._merge_contexts([ParameterContext.load(i.parameter_context) for i in pcs], pd.temporal_context)
pdict._identifier = parameter_dictionary_id
return pdict
def create_dataset(self, parameter_dict_id=''):
'''
Creates a time-series dataset
'''
dataset_management = DatasetManagementServiceClient()
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
if not parameter_dict_id:
parameter_dict_id = dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
dataset_id = dataset_management.create_dataset('test_dataset_', parameter_dictionary_id=parameter_dict_id, spatial_domain=sdom, temporal_domain=tdom)
self.addCleanup(dataset_management.delete_dataset, dataset_id)
return dataset_id
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
streams = {"parsed": "ctd_parsed_param_dict", "raw": "ctd_raw_param_dict"}
for (stream_name, param_dict_name) in streams.iteritems():
pd_id = dataset_management.read_parameter_dictionary_by_name(param_dict_name, id_only=True)
stream_def_id = pubsub_client.create_stream_definition(name=stream_name, parameter_dictionary_id=pd_id)
pd = pubsub_client.read_stream_definition(stream_def_id).parameter_dictionary
stream_id, stream_route = pubsub_client.create_stream(
name=stream_name, exchange_point="science_data", stream_definition_id=stream_def_id
)
stream_config = dict(
stream_route=stream_route,
routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd,
)
if stream_name == "parsed":
type = "IntervalAlarmDef"
kwargs = {
"name": "test_sim_warning",
"stream_name": "parsed",
"value_id": "temp",
"message": "Temperature is above test range of 5.0.",
"type": StreamAlarmType.WARNING,
"upper_bound": 5.0,
"upper_rel_op": "<",
}
alarm = {}
alarm["type"] = type
alarm["kwargs"] = kwargs
alarms = [alarm]
stream_config["alarms"] = alarms
self._stream_config[stream_name] = stream_config
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient()
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
# Use the network definition provided by RSN OMS directly.
rsn_oms = CIOMSClientFactory.create_instance(DVR_CONFIG['oms_uri'])
self._network_definition = RsnOmsUtil.build_network_definition(rsn_oms)
# get serialized version for the configuration:
self._network_definition_ser = NetworkUtil.serialize_network_definition(self._network_definition)
if log.isEnabledFor(logging.TRACE):
log.trace("NetworkDefinition serialization:\n%s", self._network_definition_ser)
self._async_data_result = AsyncResult()
self._data_subscribers = []
self._samples_received = []
self.addCleanup(self._stop_data_subscribers)
self._async_event_result = AsyncResult()
self._event_subscribers = []
self._events_received = []
self.addCleanup(self._stop_event_subscribers)
self._start_event_subscriber()
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dmpsclient = DataProductManagementServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.psmsclient = PubsubManagementServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.c = DotDict()
self.c.resource_registry = self.rrclient
self.RR2 = EnhancedResourceRegistryClient(self.rrclient)
# create missing data process definition
self.dsmsclient = DataProcessManagementServiceClient(node=self.container.node)
dpd_obj = IonObject(RT.DataProcessDefinition,
name=LOGICAL_TRANSFORM_DEFINITION_NAME,
description="normally in preload",
module='ion.processes.data.transforms.logical_transform',
class_name='logical_transform')
self.dsmsclient.create_data_process_definition(dpd_obj)
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]:
self.dsmsclient.deactivate_data_process(proc_id)
self.dsmsclient.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dmpsclient = DataProductManagementServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.psmsclient = PubsubManagementServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.c = DotDict()
self.c.resource_registry = self.rrclient
self.RR2 = EnhancedResourceRegistryClient(self.rrclient)
self.dsmsclient = DataProcessManagementServiceClient(node=self.container.node)
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]:
self.dsmsclient.deactivate_data_process(proc_id)
self.dsmsclient.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dataset_management = DatasetManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.resource_registry = self.container.resource_registry
def setUp(self):
super(DataRetrieverIntTestAlpha,self).setUp()
self._start_container()
config = DotDict()
config.bootstrap.processes.ingestion.module = 'ion.processes.data.ingestion.ingestion_worker_a'
config.bootstrap.processes.replay.module = 'ion.processes.data.replay.replay_process_a'
self.container.start_rel_from_url('res/deploy/r2dm.yml', config)
self.datastore_name = 'test_datasets'
self.datastore = self.container.datastore_manager.get_datastore(self.datastore_name, profile=DataStore.DS_PROFILE.SCIDATA)
self.data_retriever = DataRetrieverServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.resource_registry = ResourceRegistryServiceClient()
xs_dot_xp = CFG.core_xps.science_data
try:
self.XS, xp_base = xs_dot_xp.split('.')
self.XP = '.'.join([get_sys_name(), xp_base])
except ValueError:
raise StandardError('Invalid CFG for core_xps.science_data: "%s"; must have "xs.xp" structure' % xs_dot_xp)
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.dpmsclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]:
self.dataprocessclient.deactivate_data_process(proc_id)
self.dataprocessclient.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
def setUp(self):
# Start container by calling parent's setUp
super(TestAssembly, self).setUp()
# Now create client to DataProductManagementService
self.client = DotDict()
self.client.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node)
self.client.DPMS = DataProductManagementServiceClient(node=self.container.node)
self.client.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.client.OMS = ObservatoryManagementServiceClient(node=self.container.node)
self.client.PSMS = PubsubManagementServiceClient(node=self.container.node)
self.client.DPRS = DataProcessManagementServiceClient(node=self.container.node)
self.client.RR = ResourceRegistryServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.client.RR)
self.dataset_management = DatasetManagementServiceClient()
dpd_obj = IonObject(RT.DataProcessDefinition,
name=LOGICAL_TRANSFORM_DEFINITION_NAME,
description="normally in preload",
module='ion.processes.data.transforms.logical_transform',
class_name='logical_transform')
self.client.DPRS.create_data_process_definition(dpd_obj)
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.client.RR.find_resources(RT.DataProcess, None, None, True)[0]:
self.client.DPRS.deactivate_data_process(proc_id)
self.client.DPRS.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Because hey why not?!
self.dataset_management = DatasetManagementServiceClient()
self.data_process_management = DataProcessManagementServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
def setUp(self):
self._start_container()
self.container.start_rel_from_url("res/deploy/r2deploy.yml")
self.pubsub_management = PubsubManagementServiceClient()
self.resource_registry = ResourceRegistryServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.queue_cleanup = list()
self.exchange_cleanup = list()
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
print 'started services'
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.client = DotDict()
self.client.IMS = InstrumentManagementServiceClient(
node=self.container.node)
self.client.RR = ResourceRegistryServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
#container = Container()
#print 'starting container'
#container.start()
#print 'started container'
unittest # suppress an pycharm inspector error if all unittest.skip references are commented out
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.DSC = DatasetManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.OMS = ObservatoryManagementServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dpclient = DataProductManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataretrieverclient = DataRetrieverServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
#setup listerner vars
self._data_greenlets = []
self._no_samples = None
self._samples_received = []
self.event_publisher = EventPublisher()
self.egg_url_good = "http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1a-py2.7.egg"
self.egg_url_bad = "http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.1a-py2.7.egg"
self.egg_url_404 = "http://sddevrepo.oceanobservatories.org/releases/completely_made_up_404.egg"
def _build_stream_config(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
dataset_management = DatasetManagementServiceClient()
# Create streams and subscriptions for each stream named in driver.
self.stream_config = {}
streams = {
'parsed': 'ctd_parsed_param_dict',
'raw': 'ctd_raw_param_dict'
}
for (stream_name, param_dict_name) in streams.iteritems():
pd_id = dataset_management.read_parameter_dictionary_by_name(
DEFAULT_PARAM_DICT, id_only=True)
if (not pd_id):
log.error("No pd_id found for param_dict '%s'" %
DEFAULT_PARAM_DICT)
stream_def_id = pubsub_client.create_stream_definition(
name=stream_name, parameter_dictionary_id=pd_id)
pd = None
stream_id, stream_route = pubsub_client.create_stream(
name=stream_name,
exchange_point='science_data',
stream_definition_id=stream_def_id)
stream_config = dict(stream_route=stream_route,
routing_key=stream_route.routing_key,
exchange_point=stream_route.exchange_point,
stream_id=stream_id,
stream_definition_ref=stream_def_id,
parameter_dictionary=pd)
self.stream_config[stream_name] = stream_config
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dataset_management_client = DatasetManagementServiceClient(
node=self.container.node)
self.pubsub_client = PubsubManagementServiceClient(
node=self.container.node)
self.time_dom, self.spatial_dom = time_series_domain()
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(
name='ctd_parsed_param_dict', id_only=True)
self.stream_def_id = self.pubsub_client.create_stream_definition(
name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
self.addCleanup(self.pubsub_client.delete_stream_definition,
self.stream_def_id)
self.stream_id, self.route_id = self.pubsub_client.create_stream(
name='parsed_stream',
stream_definition_id=self.stream_def_id,
exchange_point='science_data')
self.addCleanup(self.pubsub_client.delete_stream, self.stream_id)
self.subscription_id = self.pubsub_client.create_subscription(
name='parsed_subscription',
stream_ids=[self.stream_id],
exchange_name='parsed_subscription')
self.addCleanup(self.pubsub_client.delete_subscription,
self.subscription_id)
self.pubsub_client.activate_subscription(self.subscription_id)
self.addCleanup(self.pubsub_client.deactivate_subscription,
self.subscription_id)
self.publisher = StandaloneStreamPublisher(self.stream_id,
self.route_id)
def helper_create_highcharts_data_process_definition(container):
from interface.services.coi.iresource_registry_service import ResourceRegistryServiceClient
rrclient = ResourceRegistryServiceClient(node=container.node)
#First look to see if it exists and if not, then create it
dpd,_ = rrclient.find_resources(restype=RT.DataProcessDefinition, name='highcharts_transform')
if len(dpd) > 0:
return dpd[0]
# Data Process Definition
log.debug("Create data process definition for highcharts transform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='highcharts_transform',
description='Convert data streams to Highcharts data',
module='ion.processes.data.transforms.viz.highcharts',
class_name='VizTransformHighCharts')
from interface.services.sa.idata_process_management_service import DataProcessManagementServiceClient
dataprocessclient = DataProcessManagementServiceClient(node=container.node)
procdef_id = dataprocessclient.create_data_process_definition(dpd_obj)
from interface.services.dm.idataset_management_service import DatasetManagementServiceClient
datasetclient = DatasetManagementServiceClient(node=container.node)
pdict_id = datasetclient.read_parameter_dictionary_by_name('highcharts', id_only=True)
from interface.services.dm.ipubsub_management_service import PubsubManagementServiceClient
pubsubclient = PubsubManagementServiceClient(node=container.node)
# create a stream definition for the data from the
stream_def_id = pubsubclient.create_stream_definition(name='VizTransformHighCharts', parameter_dictionary_id=pdict_id)
dataprocessclient.assign_stream_definition_to_data_process_definition(stream_def_id, procdef_id, binding='highcharts' )
return procdef_id
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient(node=self.container.node)
self.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.DAMS = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.DP = DataProductManagementServiceClient(node=self.container.node)
self.PSC = PubsubManagementServiceClient(node=self.container.node)
self.PDC = ProcessDispatcherServiceClient(node=self.container.node)
self.DSC = DatasetManagementServiceClient()
self.IDS = IdentityManagementServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.RR)
# Use the network definition provided by RSN OMS directly.
rsn_oms = CIOMSClientFactory.create_instance(DVR_CONFIG['oms_uri'])
self._network_definition = RsnOmsUtil.build_network_definition(rsn_oms)
# get serialized version for the configuration:
self._network_definition_ser = NetworkUtil.serialize_network_definition(
self._network_definition)
if log.isEnabledFor(logging.TRACE):
log.trace("NetworkDefinition serialization:\n%s",
self._network_definition_ser)
self._async_data_result = AsyncResult()
self._data_subscribers = []
self._samples_received = []
self.addCleanup(self._stop_data_subscribers)
self._async_event_result = AsyncResult()
self._event_subscribers = []
self._events_received = []
self.addCleanup(self._stop_event_subscribers)
self._start_event_subscriber()
def setUp(self):
super(EventManagementIntTest, self).setUp()
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.event_management = EventManagementServiceClient()
self.rrc = ResourceRegistryServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.pubsub = PubsubManagementServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
self.queue_cleanup = []
self.exchange_cleanup = []
def setUp(self):
# Start container
self._start_container()
#self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
print 'started services'
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubclient = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
# create missing data process definition
dpd_obj = IonObject(RT.DataProcessDefinition,
name=LOGICAL_TRANSFORM_DEFINITION_NAME,
description="normally in preload",
module='ion.processes.data.transforms.logical_transform',
class_name='logical_transform')
self.dataprocessclient.create_data_process_definition(dpd_obj)
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]:
self.dataprocessclient.deactivate_data_process(proc_id)
self.dataprocessclient.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
def setUp(self):
# Start container by calling parent's setUp
super(TestAssembly, self).setUp()
# Now create client to DataProductManagementService
self.client = DotDict()
self.client.DAMS = DataAcquisitionManagementServiceClient(node=self.container.node)
self.client.DPMS = DataProductManagementServiceClient(node=self.container.node)
self.client.IMS = InstrumentManagementServiceClient(node=self.container.node)
self.client.OMS = ObservatoryManagementServiceClient(node=self.container.node)
self.client.PSMS = PubsubManagementServiceClient(node=self.container.node)
self.client.DPRS = DataProcessManagementServiceClient(node=self.container.node)
self.client.RR = ResourceRegistryServiceClient(node=self.container.node)
self.RR2 = EnhancedResourceRegistryClient(self.client.RR)
self.dataset_management = DatasetManagementServiceClient()
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.client.RR.find_resources(RT.DataProcess, None, None, True)[0]:
self.client.DPRS.deactivate_data_process(proc_id)
self.client.DPRS.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
def publish_loop(self):
sine_ampl = 2.0 # Amplitude in both directions
samples = 60
startTime = time.time()
count = samples #something other than zero
self.dataset_management = DatasetManagementServiceClient(
node=self.container.node)
while not self.finished.is_set():
count = time.time() - startTime
sine_curr_deg = (count % samples) * 360 / samples
c = numpy.array(
[sine_ampl * math.sin(math.radians(sine_curr_deg))])
t = numpy.array(
[sine_ampl * 2 * math.sin(math.radians(sine_curr_deg + 45))])
p = numpy.array(
[sine_ampl * 4 * math.sin(math.radians(sine_curr_deg + 60))])
lat = numpy.array([32.8])
lon = numpy.array([-119.6])
# convert time to ntp time. Standard notation in the system
tvar = numpy.array([ntplib.system_to_ntp_time(time.time())])
parameter_dictionary = self._create_parameter()
#parameter_dictionary = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict')
rdt = RecordDictionaryTool(param_dictionary=parameter_dictionary)
h = numpy.array([random.uniform(0.0, 360.0)])
rdt['time'] = tvar
rdt['lat'] = lat
rdt['lon'] = lon
rdt['temp'] = t
rdt['conductivity'] = c
rdt['pressure'] = p
g = rdt.to_granule(data_producer_id=self.id)
log.info('SinusoidalCtdPublisher sending 1 record!')
self.publisher.publish(g, self.stream_id)
time.sleep(1.0)
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
log.debug("TestExternalDatasetAgentMgmt: started services")
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(
node=self.container.node)
self.dpclient = DataProductManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.RR = ResourceRegistryServiceClient()
self.RR2 = EnhancedResourceRegistryClient(self.RR)
self.OMS = ObservatoryManagementServiceClient()
self.org_management_service = OrgManagementServiceClient()
self.IMS = InstrumentManagementServiceClient()
self.dpclient = DataProductManagementServiceClient()
self.pubsubcli = PubsubManagementServiceClient()
self.damsclient = DataAcquisitionManagementServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.data_retriever = DataRetrieverServiceClient()
self.data_product_management = DataProductManagementServiceClient()
self._load_stage = 0
self._resources = {}
def setUp(self):
self.username = CFG.get_safe('eoi.geoserver.user_name', 'admin')
self.PASSWORD = CFG.get_safe('eoi.geoserver.password', 'geoserver')
self.gs_host = CFG.get_safe('eoi.geoserver.server',
'http://localhost:8080')
self.gs_rest_url = ''.join([self.gs_host, '/geoserver/rest'])
self.gs_ows_url = ''.join([self.gs_host, '/geoserver/ows'])
IMPORTER_SERVICE_SERVER = CFG.get_safe('eoi.importer_service.server',
'http://localhost')
IMPORTER_SERVICE_PORT = str(
CFG.get_safe('eoi.importer_service.port', 8844))
self.importer_service_url = ''.join(
[IMPORTER_SERVICE_SERVER, ':', IMPORTER_SERVICE_PORT])
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dataset_management = DatasetManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.resource_registry = self.container.resource_registry
def setUp(self):
super(DataRetrieverIntTest, self).setUp()
self._start_container()
self.container.start_rel_from_url('res/deploy/r2dm.yml')
self.datastore_name = 'test_datasets'
self.datastore = self.container.datastore_manager.get_datastore(
self.datastore_name, profile=DataStore.DS_PROFILE.SCIDATA)
self.data_retriever = DataRetrieverServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.resource_registry = ResourceRegistryServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
xs_dot_xp = CFG.core_xps.science_data
try:
self.XS, xp_base = xs_dot_xp.split('.')
self.XP = '.'.join([get_sys_name(), xp_base])
except ValueError:
raise StandardError(
'Invalid CFG for core_xps.science_data: "%s"; must have "xs.xp" structure'
% xs_dot_xp)
class TestActivateRSNVel3DInstrument(IonIntegrationTestCase):
def setUp(self):
# Start container
super(TestActivateRSNVel3DInstrument, self).setUp()
config = DotDict()
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml', config)
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dpclient = DataProductManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataretrieverclient = DataRetrieverServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name + '_logger')
producer_definition.executable = {
'module': 'ion.processes.data.stream_granule_logger',
'class': 'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(
process_definition=producer_definition)
configuration = {
'process': {
'stream_id': stream_id,
}
}
pid = self.processdispatchclient.schedule_process(
process_definition_id=logger_procdef_id,
configuration=configuration)
return pid
@attr('LOCOINT')
@unittest.skip('under construction')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
@patch.dict(CFG, {'endpoint': {'receive': {'timeout': 180}}})
def test_activate_rsn_vel3d(self):
log.info(
"--------------------------------------------------------------------------------------------------------"
)
# load_parameter_scenarios
self.container.spawn_process(
"Loader",
"ion.processes.bootstrap.ion_loader",
"IONLoader",
config=dict(
op="load",
scenario="BETA",
path="master",
categories=
"ParameterFunctions,ParameterDefs,ParameterDictionary,StreamDefinition",
clearcols="owner_id,org_ids",
assets="res/preload/r2_ioc/ooi_assets",
parseooi="True",
))
self.loggerpids = []
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='Vel3DMModel',
description="Vel3DMModel")
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
log.debug('test_activate_rsn_vel3d new InstrumentModel id = %s ',
instModel_id)
raw_config = StreamConfiguration(stream_name='raw',
parameter_dictionary_name='raw')
vel3d_b_sample = StreamConfiguration(
stream_name='vel3d_b_sample',
parameter_dictionary_name='vel3d_b_sample')
vel3d_b_engineering = StreamConfiguration(
stream_name='vel3d_b_engineering',
parameter_dictionary_name='vel3d_b_engineering')
RSN_VEL3D_01 = {
'DEV_ADDR': "10.180.80.6",
'DEV_PORT': 2101,
'DATA_PORT': 1026,
'CMD_PORT': 1025,
'PA_BINARY': "port_agent"
}
# Create InstrumentAgent
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='Vel3DAgent',
description="Vel3DAgent",
driver_uri=
"http://sddevrepo.oceanobservatories.org/releases/nobska_mavs4_ooicore-0.0.7-py2.7.egg",
stream_configurations=[
raw_config, vel3d_b_sample, vel3d_b_engineering
])
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
log.debug('test_activate_rsn_vel3d new InstrumentAgent id = %s',
instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(
instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug(
'test_activate_rsn_vel3d: Create instrument resource to represent the Vel3D '
)
instDevice_obj = IonObject(RT.InstrumentDevice,
name='Vel3DDevice',
description="Vel3DDevice",
serial_number="12345")
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, instDevice_id)
log.debug("test_activate_rsn_vel3d: new InstrumentDevice id = %s ",
instDevice_id)
port_agent_config = {
'device_addr': '10.180.80.6',
'device_port': 2101,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': 1025,
'data_port': 1026,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance,
name='Vel3DAgentInstance',
description="Vel3DAgentInstance",
port_agent_config=port_agent_config,
alerts=[])
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id)
parsed_sample_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'vel3d_b_sample', id_only=True)
parsed_sample_stream_def_id = self.pubsubcli.create_stream_definition(
name='vel3d_b_sample',
parameter_dictionary_id=parsed_sample_pdict_id)
parsed_eng_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'vel3d_b_engineering', id_only=True)
parsed_eng_stream_def_id = self.pubsubcli.create_stream_definition(
name='vel3d_b_engineering',
parameter_dictionary_id=parsed_eng_pdict_id)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'raw', id_only=True)
raw_stream_def_id = self.pubsubcli.create_stream_definition(
name='raw', parameter_dictionary_id=raw_pdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='vel3d_b_sample',
description='vel3d_b_sample')
sample_data_product_id = self.dpclient.create_data_product(
data_product=dp_obj,
stream_definition_id=parsed_sample_stream_def_id)
log.debug('new dp_id = %s', sample_data_product_id)
self.dpclient.activate_data_product_persistence(
data_product_id=sample_data_product_id)
self.damsclient.assign_data_product(
input_resource_id=instDevice_id,
data_product_id=sample_data_product_id)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(sample_data_product_id,
PRED.hasStream, None, True)
log.debug('sample_data_product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(sample_data_product_id,
PRED.hasDataset,
RT.Dataset, True)
log.debug('Data set for sample_data_product = %s', dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
pid = self.create_logger('vel3d_b_sample', stream_ids[0])
self.loggerpids.append(pid)
dp_obj = IonObject(RT.DataProduct,
name='vel3d_b_engineering',
description='vel3d_b_engineering')
eng_data_product_id = self.dpclient.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_eng_stream_def_id)
log.debug('new dp_id = %s', eng_data_product_id)
self.dpclient.activate_data_product_persistence(
data_product_id=eng_data_product_id)
self.damsclient.assign_data_product(
input_resource_id=instDevice_id,
data_product_id=eng_data_product_id)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test')
data_product_id2 = self.dpclient.create_data_product(
data_product=dp_obj, stream_definition_id=raw_stream_def_id)
log.debug('new dp_id = %s', data_product_id2)
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2,
PRED.hasStream, None, True)
log.debug('test_activate_rsn_vel3d Data product streams2 = %s',
str(stream_ids))
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id2,
PRED.hasDataset,
RT.Dataset, True)
log.debug('test_activate_rsn_vel3d Data set for data_product_id2 = %s',
dataset_ids[0])
self.raw_dataset = dataset_ids[0]
def start_instrument_agent():
self.imsclient.start_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id)
gevent.joinall([gevent.spawn(start_instrument_agent)])
#cleanup
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
#wait for start
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(
instAgentInstance_id)
gate = AgentProcessStateGate(self.processdispatchclient.read_process,
instDevice_id, ProcessStateEnum.RUNNING)
self.assertTrue(
gate. await (30),
"The instrument agent instance (%s) did not spawn in 30 seconds" %
gate.process_id)
#log.trace('Instrument agent instance obj: = %s' , str(inst_agent_instance_obj))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(instDevice_id,
to_name=gate.process_id,
process=FakeProcess())
def check_state(label, desired_state):
actual_state = self._ia_client.get_agent_state()
log.debug("%s instrument agent is in state '%s'", label,
actual_state)
self.assertEqual(desired_state, actual_state)
log.debug("test_activate_rsn_vel3d: got ia client %s",
str(self._ia_client))
check_state("just-spawned", ResourceAgentState.UNINITIALIZED)
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
log.debug("test_activate_rsn_vel3d: initialize %s", str(retval))
check_state("initialized", ResourceAgentState.INACTIVE)
log.debug("test_activate_rsn_vel3d Sending go_active command ")
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activate_rsn_vel3d: return value from go_active %s",
str(reply))
check_state("activated", ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("current state after sending go_active command %s",
str(state))
#
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activate_rsn_vel3d: run %s", str(reply))
check_state("commanded", ResourceAgentState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug("current state after sending run command %s", str(state))
# cmd = AgentCommand(command=ProtocolEvent.START_AUTOSAMPLE)
# reply = self._ia_client.execute_agent(cmd)
# log.debug("test_activate_rsn_vel3d: run %s" , str(reply))
# state = self._ia_client.get_agent_state()
# self.assertEqual(ResourceAgentState.COMMAND, state)
#
# gevent.sleep(5)
#
# cmd = AgentCommand(command=ProtocolEvent.STOP_AUTOSAMPLE)
# reply = self._ia_client.execute_agent(cmd)
# log.debug("test_activate_rsn_vel3d: run %s" , str(reply))
# state = self._ia_client.get_agent_state()
# self.assertEqual(ResourceAgentState.COMMAND, state)
#
# cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
# retval = self._ia_client.execute_agent(cmd)
# state = retval.result
# log.debug("current state after sending STOP_AUTOSAMPLE command %s" , str(state))
#
# cmd = AgentCommand(command=ResourceAgentEvent.PAUSE)
# retval = self._ia_client.execute_agent(cmd)
# state = self._ia_client.get_agent_state()
# self.assertEqual(ResourceAgentState.STOPPED, state)
#
# cmd = AgentCommand(command=ResourceAgentEvent.RESUME)
# retval = self._ia_client.execute_agent(cmd)
# state = self._ia_client.get_agent_state()
# self.assertEqual(ResourceAgentState.COMMAND, state)
#
# cmd = AgentCommand(command=ResourceAgentEvent.CLEAR)
# retval = self._ia_client.execute_agent(cmd)
# state = self._ia_client.get_agent_state()
# self.assertEqual(ResourceAgentState.IDLE, state)
#
# cmd = AgentCommand(command=ResourceAgentEvent.RUN)
# retval = self._ia_client.execute_agent(cmd)
# state = self._ia_client.get_agent_state()
# self.assertEqual(ResourceAgentState.COMMAND, state)
log.debug("test_activate_rsn_vel3d: calling reset ")
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activate_rsn_vel3d: return from reset %s", str(reply))
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data_raw = self.dataretrieverclient.retrieve(self.raw_dataset)
self.assertIsInstance(replay_data_raw, Granule)
rdt_raw = RecordDictionaryTool.load_from_granule(replay_data_raw)
log.debug("RDT raw: %s", str(rdt_raw.pretty_print()))
self.assertIn('raw', rdt_raw)
raw_vals = rdt_raw['raw']
#--------------------------------------------------------------------------------
# Deactivate loggers
#--------------------------------------------------------------------------------
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
self.dpclient.delete_data_product(sample_data_product_id)
self.dpclient.delete_data_product(eng_data_product_id)
self.dpclient.delete_data_product(data_product_id2)
class TestDMEnd2End(IonIntegrationTestCase):
def setUp(self): # Love the non pep-8 convention
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.process_dispatcher = ProcessDispatcherServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.resource_registry = ResourceRegistryServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.ingestion_management = IngestionManagementServiceClient()
self.data_retriever = DataRetrieverServiceClient()
self.event = Event()
self.exchange_space_name = 'test_granules'
self.exchange_point_name = 'science_data'
self.i = 0
self.cci = 0
#--------------------------------------------------------------------------------
# Helper/Utility methods
#--------------------------------------------------------------------------------
def create_dataset(self, parameter_dict_id=''):
'''
Creates a time-series dataset
'''
if not parameter_dict_id:
parameter_dict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
dataset = Dataset('test_dataset_%i'%self.i)
dataset_id = self.dataset_management.create_dataset(dataset, parameter_dictionary_id=parameter_dict_id)
self.addCleanup(self.dataset_management.delete_dataset, dataset_id)
return dataset_id
def get_datastore(self, dataset_id):
'''
Gets an instance of the datastore
This method is primarily used to defeat a bug where integration tests in multiple containers may sometimes
delete a CouchDB datastore and the other containers are unaware of the new state of the datastore.
'''
dataset = self.dataset_management.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def get_ingestion_config(self):
'''
Grab the ingestion configuration from the resource registry
'''
# The ingestion configuration should have been created by the bootstrap service
# which is configured through r2deploy.yml
ingest_configs, _ = self.resource_registry.find_resources(restype=RT.IngestionConfiguration,id_only=True)
return ingest_configs[0]
def launch_producer(self, stream_id=''):
'''
Launch the producer
'''
pid = self.container.spawn_process('better_data_producer', 'ion.processes.data.example_data_producer', 'BetterDataProducer', {'process':{'stream_id':stream_id}})
self.addCleanup(self.container.terminate_process, pid)
def make_simple_dataset(self):
'''
Makes a stream, a stream definition and a dataset, the essentials for most of these tests
'''
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
stream_def_id = self.pubsub_management.create_stream_definition('ctd data %i' % self.i, parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
stream_id, route = self.pubsub_management.create_stream('ctd stream %i' % self.i, 'xp1', stream_definition_id=stream_def_id)
self.addCleanup(self.pubsub_management.delete_stream, stream_id)
dataset_id = self.create_dataset(pdict_id)
# self.get_datastore(dataset_id)
self.i += 1
return stream_id, route, stream_def_id, dataset_id
def publish_hifi(self,stream_id,stream_route,offset=0):
'''
Publish deterministic data
'''
pub = StandaloneStreamPublisher(stream_id, stream_route)
stream_def = self.pubsub_management.read_stream_definition(stream_id=stream_id)
stream_def_id = stream_def._id
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.arange(10) + (offset * 10)
rdt['temp'] = np.arange(10) + (offset * 10)
pub.publish(rdt.to_granule())
def publish_fake_data(self,stream_id, route):
'''
Make four granules
'''
for i in xrange(4):
self.publish_hifi(stream_id,route,i)
def start_ingestion(self, stream_id, dataset_id):
'''
Starts ingestion/persistence for a given dataset
'''
ingest_config_id = self.get_ingestion_config()
self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingest_config_id, dataset_id=dataset_id)
def stop_ingestion(self, stream_id):
ingest_config_id = self.get_ingestion_config()
self.ingestion_management.unpersist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingest_config_id)
def validate_granule_subscription(self, msg, route, stream_id):
'''
Validation for granule format
'''
if msg == {}:
return
rdt = RecordDictionaryTool.load_from_granule(msg)
log.info('%s', rdt.pretty_print())
self.assertIsInstance(msg,Granule,'Message is improperly formatted. (%s)' % type(msg))
self.event.set()
def wait_until_we_have_enough_granules(self, dataset_id='',data_size=40):
'''
Loops until there is a sufficient amount of data in the dataset
'''
done = False
with gevent.Timeout(40):
while not done:
extents = self.dataset_management.dataset_extents(dataset_id, 'time')
granule = self.data_retriever.retrieve_last_data_points(dataset_id, 1)
rdt = RecordDictionaryTool.load_from_granule(granule)
if rdt['time'] and rdt['time'][0] != rdt._pdict.get_context('time').fill_value and extents >= data_size:
done = True
else:
gevent.sleep(0.2)
#--------------------------------------------------------------------------------
# Test Methods
#--------------------------------------------------------------------------------
def test_dm_end_2_end(self):
#--------------------------------------------------------------------------------
# Set up a stream and have a mock instrument (producer) send data
#--------------------------------------------------------------------------------
self.event.clear()
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True)
# Add a field that supports binary data input.
bin_context = ParameterContext('binary', param_type=ArrayType())
context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump()))
# Add another field that supports dictionary elements.
rec_context = ParameterContext('records', param_type=RecordType())
context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time')
stream_definition = self.pubsub_management.create_stream_definition('ctd data', parameter_dictionary_id=pdict_id)
stream_id, route = self.pubsub_management.create_stream('producer', exchange_point=self.exchange_point_name, stream_definition_id=stream_definition)
#--------------------------------------------------------------------------------
# Start persisting the data on the stream
# - Get the ingestion configuration from the resource registry
# - Create the dataset
# - call persist_data_stream to setup the subscription for the ingestion workers
# on the stream that you specify which causes the data to be persisted
#--------------------------------------------------------------------------------
ingest_config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingest_config_id, dataset_id=dataset_id)
self.addCleanup(self.stop_ingestion, stream_id)
#--------------------------------------------------------------------------------
# Now the granules are ingesting and persisted
#--------------------------------------------------------------------------------
self.launch_producer(stream_id)
self.wait_until_we_have_enough_granules(dataset_id,40)
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data = self.data_retriever.retrieve(dataset_id)
self.assertIsInstance(replay_data, Granule)
rdt = RecordDictionaryTool.load_from_granule(replay_data)
self.assertTrue((rdt['time'][:10] == np.arange(10)).all(),'%s' % rdt['time'][:])
self.assertTrue((rdt['binary'][:10] == np.array(['hi']*10, dtype='object')).all())
#--------------------------------------------------------------------------------
# Now to try the streamed approach
#--------------------------------------------------------------------------------
replay_stream_id, replay_route = self.pubsub_management.create_stream('replay_out', exchange_point=self.exchange_point_name, stream_definition_id=stream_definition)
self.replay_id, process_id = self.data_retriever.define_replay(dataset_id=dataset_id, stream_id=replay_stream_id)
log.info('Process ID: %s', process_id)
replay_client = ReplayClient(process_id)
#--------------------------------------------------------------------------------
# Create the listening endpoint for the the retriever to talk to
#--------------------------------------------------------------------------------
sub_id = self.pubsub_management.create_subscription(self.exchange_space_name,stream_ids=[replay_stream_id])
self.addCleanup(self.pubsub_management.delete_subscription, sub_id)
self.pubsub_management.activate_subscription(sub_id)
self.addCleanup(self.pubsub_management.deactivate_subscription, sub_id)
subscriber = StandaloneStreamSubscriber(self.exchange_space_name, self.validate_granule_subscription)
subscriber.start()
self.addCleanup(subscriber.stop)
self.data_retriever.start_replay_agent(self.replay_id)
self.assertTrue(replay_client.await_agent_ready(5), 'The process never launched')
replay_client.start_replay()
self.assertTrue(self.event.wait(10))
self.data_retriever.cancel_replay_agent(self.replay_id)
#--------------------------------------------------------------------------------
# Test the slicing capabilities
#--------------------------------------------------------------------------------
granule = self.data_retriever.retrieve(dataset_id=dataset_id, query={'tdoa':slice(0,5)})
rdt = RecordDictionaryTool.load_from_granule(granule)
b = rdt['time'] == np.arange(5)
self.assertTrue(b.all() if not isinstance(b,bool) else b)
def test_coverage_transform(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_parsed()
stream_def_id = self.pubsub_management.create_stream_definition('ctd parsed', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
stream_id, route = self.pubsub_management.create_stream('example', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id)
self.addCleanup(self.pubsub_management.delete_stream, stream_id)
ingestion_config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=ingestion_config_id, dataset_id=dataset_id)
self.addCleanup(self.ingestion_management.unpersist_data_stream, stream_id, ingestion_config_id)
publisher = StandaloneStreamPublisher(stream_id, route)
rdt = ph.get_rdt(stream_def_id)
ph.fill_parsed_rdt(rdt)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_monitor.wait())
replay_granule = self.data_retriever.retrieve(dataset_id)
rdt_out = RecordDictionaryTool.load_from_granule(replay_granule)
np.testing.assert_array_almost_equal(rdt_out['time'], rdt['time'])
np.testing.assert_array_almost_equal(rdt_out['temp'], rdt['temp'])
np.testing.assert_allclose(rdt_out['conductivity_L1'], np.array([42.914]))
np.testing.assert_allclose(rdt_out['temp_L1'], np.array([20.]))
np.testing.assert_allclose(rdt_out['pressure_L1'], np.array([3.068]))
np.testing.assert_allclose(rdt_out['density'], np.array([1021.7144739593881], dtype='float32'))
np.testing.assert_allclose(rdt_out['salinity'], np.array([30.935132729668283], dtype='float32'))
def test_ingestion_pause(self):
ctd_stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset()
ingestion_config_id = self.get_ingestion_config()
self.start_ingestion(ctd_stream_id, dataset_id)
self.addCleanup(self.stop_ingestion, ctd_stream_id)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = np.arange(10)
publisher = StandaloneStreamPublisher(ctd_stream_id, route)
monitor = DatasetMonitor(dataset_id)
self.addCleanup(monitor.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(monitor.wait())
granule = self.data_retriever.retrieve(dataset_id)
self.ingestion_management.pause_data_stream(ctd_stream_id, ingestion_config_id)
monitor.event.clear()
rdt['time'] = np.arange(10,20)
publisher.publish(rdt.to_granule())
self.assertFalse(monitor.event.wait(1))
self.ingestion_management.resume_data_stream(ctd_stream_id, ingestion_config_id)
self.assertTrue(monitor.wait())
granule = self.data_retriever.retrieve(dataset_id)
rdt2 = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_almost_equal(rdt2['time'], np.arange(20))
def test_last_granule(self):
stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset()
self.start_ingestion(stream_id, dataset_id)
self.addCleanup(self.stop_ingestion, stream_id)
self.publish_hifi(stream_id,route, 0)
self.publish_hifi(stream_id,route, 1)
self.wait_until_we_have_enough_granules(dataset_id,20) # I just need two
success = False
def verifier():
replay_granule = self.data_retriever.retrieve_last_data_points(dataset_id, 10)
rdt = RecordDictionaryTool.load_from_granule(replay_granule)
comp = rdt['time'] == np.arange(10) + 10
if not isinstance(comp,bool):
return comp.all()
return False
success = poll(verifier)
self.assertTrue(success)
success = False
def verify_points():
replay_granule = self.data_retriever.retrieve_last_data_points(dataset_id,5)
rdt = RecordDictionaryTool.load_from_granule(replay_granule)
comp = rdt['time'] == np.arange(15,20)
if not isinstance(comp,bool):
return comp.all()
return False
success = poll(verify_points)
self.assertTrue(success)
def test_replay_with_parameters(self):
#--------------------------------------------------------------------------------
# Create the configurations and the dataset
#--------------------------------------------------------------------------------
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True)
# Add a field that supports binary data input.
bin_context = ParameterContext('binary', param_type=ArrayType())
context_ids.append(self.dataset_management.create_parameter_context('binary', bin_context.dump()))
# Add another field that supports dictionary elements.
rec_context = ParameterContext('records', param_type=RecordType())
context_ids.append(self.dataset_management.create_parameter_context('records', rec_context.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary('replay_pdict', parameter_context_ids=context_ids, temporal_context='time')
stream_def_id = self.pubsub_management.create_stream_definition('replay_stream', parameter_dictionary_id=pdict_id)
stream_id, route = self.pubsub_management.create_stream('replay_with_params', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id)
config_id = self.get_ingestion_config()
dataset_id = self.create_dataset(pdict_id)
self.ingestion_management.persist_data_stream(stream_id=stream_id, ingestion_configuration_id=config_id, dataset_id=dataset_id)
self.addCleanup(self.stop_ingestion, stream_id)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
self.publish_fake_data(stream_id, route)
self.assertTrue(dataset_monitor.wait())
query = {
'start_time': 0 - 2208988800,
'end_time': 19 - 2208988800,
'stride_time' : 2,
'parameters': ['time','temp']
}
retrieved_data = self.data_retriever.retrieve(dataset_id=dataset_id,query=query)
rdt = RecordDictionaryTool.load_from_granule(retrieved_data)
np.testing.assert_array_equal(rdt['time'], np.arange(0,20,2))
self.assertEquals(set(rdt.iterkeys()), set(['time','temp']))
extents = self.dataset_management.dataset_extents(dataset_id=dataset_id, parameters=['time','temp'])
self.assertTrue(extents['time']>=20)
self.assertTrue(extents['temp']>=20)
def test_repersist_data(self):
stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset()
self.start_ingestion(stream_id, dataset_id)
self.publish_hifi(stream_id,route,0)
self.publish_hifi(stream_id,route,1)
self.wait_until_we_have_enough_granules(dataset_id,20)
config_id = self.get_ingestion_config()
self.ingestion_management.unpersist_data_stream(stream_id=stream_id,ingestion_configuration_id=config_id)
self.ingestion_management.persist_data_stream(stream_id=stream_id,ingestion_configuration_id=config_id,dataset_id=dataset_id)
self.addCleanup(self.stop_ingestion, stream_id)
self.publish_hifi(stream_id,route,2)
self.publish_hifi(stream_id,route,3)
self.wait_until_we_have_enough_granules(dataset_id,40)
success = False
with gevent.timeout.Timeout(5):
while not success:
replay_granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(replay_granule)
comp = rdt['time'] == np.arange(0,40)
if not isinstance(comp,bool):
success = comp.all()
gevent.sleep(1)
self.assertTrue(success)
@unittest.skip('deprecated')
def test_correct_time(self):
# There are 2208988800 seconds between Jan 1 1900 and Jan 1 1970, i.e.
# the conversion factor between unix and NTP time
unix_now = np.floor(time.time())
ntp_now = unix_now + 2208988800
unix_ago = unix_now - 20
ntp_ago = unix_ago + 2208988800
stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset()
coverage = DatasetManagementService._get_simplex_coverage(dataset_id, mode='a')
coverage.insert_timesteps(20)
coverage.set_parameter_values('time', np.arange(ntp_ago,ntp_now))
temporal_bounds = self.dataset_management.dataset_temporal_bounds(dataset_id)
self.assertTrue( np.abs(temporal_bounds[0] - unix_ago) < 2)
self.assertTrue( np.abs(temporal_bounds[1] - unix_now) < 2)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Host requires file-system access to coverage files, CEI mode does not support.')
def test_out_of_band_retrieve(self):
# Setup the environemnt
stream_id, route, stream_def_id, dataset_id = self.make_simple_dataset()
self.start_ingestion(stream_id, dataset_id)
# Fill the dataset
self.publish_fake_data(stream_id, route)
self.wait_until_we_have_enough_granules(dataset_id,40)
# Retrieve the data
granule = DataRetrieverService.retrieve_oob(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
self.assertTrue((rdt['time'] == np.arange(40)).all())
def publish_and_wait(self, dataset_id, granule):
stream_ids, _ = self.resource_registry.find_objects(dataset_id, PRED.hasStream,id_only=True)
stream_id=stream_ids[0]
route = self.pubsub_management.read_stream_route(stream_id)
publisher = StandaloneStreamPublisher(stream_id,route)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
publisher.publish(granule)
self.assertTrue(dataset_monitor.wait())
def test_sparse_values(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_sparse()
stream_def_id = self.pubsub_management.create_stream_definition('sparse', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition, stream_def_id)
stream_id, route = self.pubsub_management.create_stream('example', exchange_point=self.exchange_point_name, stream_definition_id=stream_def_id)
self.addCleanup(self.pubsub_management.delete_stream, stream_id)
dataset_id = self.create_dataset(pdict_id)
self.start_ingestion(stream_id,dataset_id)
self.addCleanup(self.stop_ingestion, stream_id)
# Publish initial granule
# the first one has the sparse value set inside it, sets lat to 45 and lon to -71
ntp_now = time.time() + 2208988800
rdt = ph.get_rdt(stream_def_id)
rdt['time'] = [ntp_now]
rdt['internal_timestamp'] = [ntp_now]
rdt['temp'] = [300000]
rdt['preferred_timestamp'] = ['driver_timestamp']
rdt['port_timestamp'] = [ntp_now]
rdt['quality_flag'] = ['']
rdt['lat'] = [45]
rdt['conductivity'] = [4341400]
rdt['driver_timestamp'] = [ntp_now]
rdt['lon'] = [-71]
rdt['pressure'] = [256.8]
publisher = StandaloneStreamPublisher(stream_id, route)
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_monitor.wait())
dataset_monitor.reset()
replay_granule = self.data_retriever.retrieve(dataset_id)
rdt_out = RecordDictionaryTool.load_from_granule(replay_granule)
# Check the values and make sure they're correct
np.testing.assert_allclose(rdt_out['time'], rdt['time'])
np.testing.assert_allclose(rdt_out['temp'], rdt['temp'])
np.testing.assert_allclose(rdt_out['lat'], np.array([45]))
np.testing.assert_allclose(rdt_out['lon'], np.array([-71]))
np.testing.assert_allclose(rdt_out['conductivity_L1'], np.array([42.914]))
np.testing.assert_allclose(rdt_out['temp_L1'], np.array([20.]))
np.testing.assert_allclose(rdt_out['pressure_L1'], np.array([3.068]))
np.testing.assert_allclose(rdt_out['density'], np.array([1021.7144739593881], dtype='float32'))
np.testing.assert_allclose(rdt_out['salinity'], np.array([30.935132729668283], dtype='float32'))
# We're going to change the lat/lon
rdt = ph.get_rdt(stream_def_id)
rdt['time'] = time.time() + 2208988800
rdt['lat'] = [46]
rdt['lon'] = [-73]
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_monitor.wait())
dataset_monitor.reset()
replay_granule = self.data_retriever.retrieve(dataset_id)
rdt_out = RecordDictionaryTool.load_from_granule(replay_granule)
np.testing.assert_allclose(rdt_out['time'], rdt['time'])
for i in xrange(9):
ntp_now = time.time() + 2208988800
rdt['time'] = [ntp_now]
rdt['internal_timestamp'] = [ntp_now]
rdt['temp'] = [300000]
rdt['preferred_timestamp'] = ['driver_timestamp']
rdt['port_timestamp'] = [ntp_now]
rdt['quality_flag'] = [None]
rdt['conductivity'] = [4341400]
rdt['driver_timestamp'] = [ntp_now]
rdt['pressure'] = [256.8]
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_monitor.wait())
dataset_monitor.reset()
replay_granule = self.data_retriever.retrieve(dataset_id)
rdt_out = RecordDictionaryTool.load_from_granule(replay_granule)
np.testing.assert_allclose(rdt_out['pressure'], np.array([256.8] * 10))
np.testing.assert_allclose(rdt_out['lat'], np.array([45] + [46] * 9))
np.testing.assert_allclose(rdt_out['lon'], np.array([-71] + [-73] * 9))
class CtdbpTransformsIntTest(IonIntegrationTestCase):
def setUp(self):
super(CtdbpTransformsIntTest, self).setUp()
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.pubsub = PubsubManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.data_process_management = DataProcessManagementServiceClient()
self.dataproduct_management = DataProductManagementServiceClient()
self.resource_registry = ResourceRegistryServiceClient()
# This is for the time values inside the packets going into the transform
self.i = 0
# Cleanup of queue created by the subscriber
def _get_new_ctd_packet(self, stream_definition_id, length):
rdt = RecordDictionaryTool(stream_definition_id=stream_definition_id)
rdt['time'] = numpy.arange(self.i, self.i + length)
for field in rdt:
if isinstance(
rdt._pdict.get_context(field).param_type, QuantityType):
rdt[field] = numpy.array(
[random.uniform(0.0, 75.0) for i in xrange(length)])
g = rdt.to_granule()
self.i += length
return g
def _create_input_param_dict_for_test(self, parameter_dict_name=''):
pdict = ParameterDictionary()
t_ctxt = ParameterContext(
'time',
param_type=QuantityType(value_encoding=numpy.dtype('float64')))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1900'
pdict.add_context(t_ctxt)
cond_ctxt = ParameterContext(
'conductivity',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
cond_ctxt.uom = ''
pdict.add_context(cond_ctxt)
pres_ctxt = ParameterContext(
'pressure',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
pres_ctxt.uom = ''
pdict.add_context(pres_ctxt)
temp_ctxt = ParameterContext(
'temperature',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = ''
pdict.add_context(temp_ctxt)
dens_ctxt = ParameterContext(
'density',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
dens_ctxt.uom = ''
pdict.add_context(dens_ctxt)
sal_ctxt = ParameterContext(
'salinity',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
sal_ctxt.uom = ''
pdict.add_context(sal_ctxt)
#create temp streamdef so the data product can create the stream
pc_list = []
for pc_k, pc in pdict.iteritems():
ctxt_id = self.dataset_management.create_parameter_context(
pc_k, pc[1].dump())
pc_list.append(ctxt_id)
self.addCleanup(self.dataset_management.delete_parameter_context,
ctxt_id)
pdict_id = self.dataset_management.create_parameter_dictionary(
parameter_dict_name, pc_list)
self.addCleanup(self.dataset_management.delete_parameter_dictionary,
pdict_id)
return pdict_id
def test_ctdbp_L0_all(self):
"""
Test packets processed by the ctdbp_L0_all transform
"""
#----------- Data Process Definition --------------------------------
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='CTDBP_L0_all',
description=
'Take parsed stream and put the C, T and P into three separate L0 streams.',
module='ion.processes.data.transforms.ctdbp.ctdbp_L0',
class_name='CTDBP_L0_all')
dprocdef_id = self.data_process_management.create_data_process_definition(
dpd_obj)
self.addCleanup(
self.data_process_management.delete_data_process_definition,
dprocdef_id)
log.debug("created data process definition: id = %s", dprocdef_id)
#----------- Data Products --------------------------------
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
input_param_dict = self._create_input_param_dict_for_test(
parameter_dict_name='fictitious_ctdp_param_dict')
# Get the stream definition for the stream using the parameter dictionary
# input_param_dict = self.dataset_management.read_parameter_dictionary_by_name('ctdbp_cdef_sample', id_only=True)
input_stream_def_dict = self.pubsub.create_stream_definition(
name='parsed', parameter_dictionary_id=input_param_dict)
self.addCleanup(self.pubsub.delete_stream_definition,
input_stream_def_dict)
log.debug("Got the parsed parameter dictionary: id: %s",
input_param_dict)
log.debug("Got the stream def for parsed input: %s",
input_stream_def_dict)
# Input data product
parsed_stream_dp_obj = IonObject(
RT.DataProduct,
name='parsed_stream',
description='Parsed stream input to CTBP L0 transform',
temporal_domain=tdom,
spatial_domain=sdom)
input_dp_id = self.dataproduct_management.create_data_product(
data_product=parsed_stream_dp_obj,
stream_definition_id=input_stream_def_dict)
self.addCleanup(self.dataproduct_management.delete_data_product,
input_dp_id)
# output data product
L0_stream_dp_obj = IonObject(
RT.DataProduct,
name='L0_stream',
description='L0_stream output of CTBP L0 transform',
temporal_domain=tdom,
spatial_domain=sdom)
L0_stream_dp_id = self.dataproduct_management.create_data_product(
data_product=L0_stream_dp_obj,
stream_definition_id=input_stream_def_dict)
self.addCleanup(self.dataproduct_management.delete_data_product,
L0_stream_dp_id)
# We need the key name here to be "L0_stream", since when the data process is launched, this name goes into
# the config as in config.process.publish_streams.L0_stream when the config is used to launch the data process
out_stream_ids, _ = self.resource_registry.find_objects(
L0_stream_dp_id, PRED.hasStream, RT.Stream, True)
self.assertTrue(len(out_stream_ids))
output_stream_id = out_stream_ids[0]
dproc_id = self.data_process_management.create_data_process(
data_process_definition_id=dprocdef_id,
in_data_product_ids=[input_dp_id],
out_data_product_ids=[L0_stream_dp_id],
configuration=None)
self.addCleanup(self.data_process_management.delete_data_process,
dproc_id)
log.debug("Created a data process for ctdbp_L0. id: %s", dproc_id)
# Activate the data process
self.data_process_management.activate_data_process(dproc_id)
self.addCleanup(self.data_process_management.deactivate_data_process,
dproc_id)
#----------- Find the stream that is associated with the input data product when it was created by create_data_product() --------------------------------
stream_ids, _ = self.resource_registry.find_objects(
input_dp_id, PRED.hasStream, RT.Stream, True)
self.assertTrue(len(stream_ids))
input_stream_id = stream_ids[0]
stream_route = self.pubsub.read_stream_route(input_stream_id)
log.debug("The input stream for the L0 transform: %s", input_stream_id)
#----------- Create a subscriber that will listen to the transform's output --------------------------------
ar = gevent.event.AsyncResult()
def subscriber(m, r, s):
ar.set(m)
sub = StandaloneStreamSubscriber(exchange_name='sub',
callback=subscriber)
sub_id = self.pubsub.create_subscription('subscriber_to_transform',
stream_ids=[output_stream_id],
exchange_name='sub')
self.addCleanup(self.pubsub.delete_subscription, sub_id)
self.pubsub.activate_subscription(sub_id)
self.addCleanup(self.pubsub.deactivate_subscription, sub_id)
sub.start()
self.addCleanup(sub.stop)
#----------- Publish on that stream so that the transform can receive it --------------------------------
pub = StandaloneStreamPublisher(input_stream_id, stream_route)
publish_granule = self._get_new_ctd_packet(
stream_definition_id=input_stream_def_dict, length=5)
pub.publish(publish_granule)
log.debug("Published the following granule: %s", publish_granule)
granule_from_transform = ar.get(timeout=20)
log.debug("Got the following granule from the transform: %s",
granule_from_transform)
# Check that the granule published by the L0 transform has the right properties
self._check_granule_from_transform(granule_from_transform)
def _check_granule_from_transform(self, granule):
"""
An internal method to check if a granule has the right properties
"""
pass
class TestDeployment(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dmpsclient = DataProductManagementServiceClient(
node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.psmsclient = PubsubManagementServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.c = DotDict()
self.c.resource_registry = self.rrclient
self.RR2 = EnhancedResourceRegistryClient(self.rrclient)
# create missing data process definition
self.dsmsclient = DataProcessManagementServiceClient(
node=self.container.node)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name=LOGICAL_TRANSFORM_DEFINITION_NAME,
description="normally in preload",
module='ion.processes.data.transforms.logical_transform',
class_name='logical_transform')
self.dsmsclient.create_data_process_definition(dpd_obj)
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.rrclient.find_resources(
RT.DataProcess, None, None, True)[0]:
self.dsmsclient.deactivate_data_process(proc_id)
self.dsmsclient.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
#@unittest.skip("targeting")
def test_create_deployment(self):
#create a deployment with metadata and an initial site and device
platform_site__obj = IonObject(RT.PlatformSite,
name='PlatformSite1',
description='test platform site')
site_id = self.omsclient.create_platform_site(platform_site__obj)
platform_device__obj = IonObject(RT.PlatformDevice,
name='PlatformDevice1',
description='test platform device')
device_id = self.imsclient.create_platform_device(platform_device__obj)
start = IonTime(datetime.datetime(2013, 1, 1))
end = IonTime(datetime.datetime(2014, 1, 1))
temporal_bounds = IonObject(OT.TemporalBounds,
name='planned',
start_datetime=start.to_string(),
end_datetime=end.to_string())
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment',
constraint_list=[temporal_bounds])
deployment_id = self.omsclient.create_deployment(deployment_obj)
self.omsclient.deploy_platform_site(site_id, deployment_id)
self.imsclient.deploy_platform_device(device_id, deployment_id)
log.debug("test_create_deployment: created deployment id: %s ",
str(deployment_id))
#retrieve the deployment objects and check that the assoc site and device are attached
read_deployment_obj = self.omsclient.read_deployment(deployment_id)
log.debug("test_create_deployment: created deployment obj: %s ",
str(read_deployment_obj))
site_ids, _ = self.rrclient.find_subjects(RT.PlatformSite,
PRED.hasDeployment,
deployment_id, True)
self.assertEqual(len(site_ids), 1)
device_ids, _ = self.rrclient.find_subjects(RT.PlatformDevice,
PRED.hasDeployment,
deployment_id, True)
self.assertEqual(len(device_ids), 1)
#delete the deployment
self.RR2.pluck(deployment_id)
self.omsclient.force_delete_deployment(deployment_id)
# now try to get the deleted dp object
try:
self.omsclient.read_deployment(deployment_id)
except NotFound:
pass
else:
self.fail("deleted deployment was found during read")
#@unittest.skip("targeting")
def base_activate_deployment(self):
#-------------------------------------------------------------------------------------
# Create platform site, platform device, platform model
#-------------------------------------------------------------------------------------
platform_site__obj = IonObject(RT.PlatformSite,
name='PlatformSite1',
description='test platform site')
platform_site_id = self.omsclient.create_platform_site(
platform_site__obj)
platform_device_obj = IonObject(RT.PlatformDevice,
name='PlatformDevice1',
description='test platform device')
platform_device_id = self.imsclient.create_platform_device(
platform_device_obj)
platform_model__obj = IonObject(RT.PlatformModel,
name='PlatformModel1',
description='test platform model')
platform_model_id = self.imsclient.create_platform_model(
platform_model__obj)
#-------------------------------------------------------------------------------------
# Create instrument site
#-------------------------------------------------------------------------------------
instrument_site_obj = IonObject(RT.InstrumentSite,
name='InstrumentSite1',
description='test instrument site')
instrument_site_id = self.omsclient.create_instrument_site(
instrument_site_obj, platform_site_id)
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.psmsclient.create_stream_definition(
name='SBE37_CDM', parameter_dictionary_id=pdict_id)
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='Log Data Product',
description='some new dp',
temporal_domain=tdom,
spatial_domain=sdom)
out_log_data_product_id = self.dmpsclient.create_data_product(
dp_obj, ctd_stream_def_id)
#----------------------------------------------------------------------------------------------------
# Start the transform (a logical transform) that acts as an instrument site
#----------------------------------------------------------------------------------------------------
self.omsclient.create_site_data_product(
site_id=instrument_site_id,
data_product_id=out_log_data_product_id)
#----------------------------------------------------------------------------------------------------
# Create an instrument device
#----------------------------------------------------------------------------------------------------
instrument_device_obj = IonObject(RT.InstrumentDevice,
name='InstrumentDevice1',
description='test instrument device')
instrument_device_id = self.imsclient.create_instrument_device(
instrument_device_obj)
self.rrclient.create_association(platform_device_id, PRED.hasDevice,
instrument_device_id)
dp_obj = IonObject(RT.DataProduct,
name='Instrument Data Product',
description='some new dp',
temporal_domain=tdom,
spatial_domain=sdom)
inst_data_product_id = self.dmpsclient.create_data_product(
dp_obj, ctd_stream_def_id)
#assign data products appropriately
self.damsclient.assign_data_product(
input_resource_id=instrument_device_id,
data_product_id=inst_data_product_id)
#----------------------------------------------------------------------------------------------------
# Create an instrument model
#----------------------------------------------------------------------------------------------------
instrument_model_obj = IonObject(RT.InstrumentModel,
name='InstrumentModel1',
description='test instrument model')
instrument_model_id = self.imsclient.create_instrument_model(
instrument_model_obj)
#----------------------------------------------------------------------------------------------------
# Create a deployment object
#----------------------------------------------------------------------------------------------------
start = IonTime(datetime.datetime(2013, 1, 1))
end = IonTime(datetime.datetime(2014, 1, 1))
temporal_bounds = IonObject(OT.TemporalBounds,
name='planned',
start_datetime=start.to_string(),
end_datetime=end.to_string())
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment',
constraint_list=[temporal_bounds])
deployment_id = self.omsclient.create_deployment(deployment_obj)
log.debug("test_create_deployment: created deployment id: %s ",
str(deployment_id))
ret = DotDict(instrument_site_id=instrument_site_id,
instrument_device_id=instrument_device_id,
instrument_model_id=instrument_model_id,
platform_site_id=platform_site_id,
platform_device_id=platform_device_id,
platform_model_id=platform_model_id,
deployment_id=deployment_id)
return ret
#@unittest.skip("targeting")
def test_activate_deployment_normal(self):
res = self.base_activate_deployment()
log.debug("assigning platform and instrument models")
self.imsclient.assign_platform_model_to_platform_device(
res.platform_model_id, res.platform_device_id)
self.imsclient.assign_instrument_model_to_instrument_device(
res.instrument_model_id, res.instrument_device_id)
self.omsclient.assign_platform_model_to_platform_site(
res.platform_model_id, res.platform_site_id)
self.omsclient.assign_instrument_model_to_instrument_site(
res.instrument_model_id, res.instrument_site_id)
log.debug("adding instrument site and device to deployment")
self.omsclient.deploy_instrument_site(res.instrument_site_id,
res.deployment_id)
self.imsclient.deploy_instrument_device(res.instrument_device_id,
res.deployment_id)
log.debug("adding platform site and device to deployment")
self.omsclient.deploy_platform_site(res.platform_site_id,
res.deployment_id)
self.imsclient.deploy_platform_device(res.platform_device_id,
res.deployment_id)
log.debug("activating deployment, expecting success")
self.omsclient.activate_deployment(res.deployment_id)
#@unittest.skip("targeting")
def test_activate_deployment_nomodels(self):
res = self.base_activate_deployment()
self.omsclient.deploy_instrument_site(res.instrument_site_id,
res.deployment_id)
self.imsclient.deploy_instrument_device(res.instrument_device_id,
res.deployment_id)
log.debug(
"activating deployment without site+device models, expecting fail")
self.assert_deploy_fail(
res.deployment_id, "Expected at least 1 model for InstrumentSite")
log.debug("assigning instrument site model")
self.omsclient.assign_instrument_model_to_instrument_site(
res.instrument_model_id, res.instrument_site_id)
log.debug(
"activating deployment without device models, expecting fail")
self.assert_deploy_fail(res.deployment_id,
"Expected 1 model for InstrumentDevice")
#@unittest.skip("targeting")
def test_activate_deployment_nosite(self):
res = self.base_activate_deployment()
log.debug("assigning instrument models")
self.imsclient.assign_instrument_model_to_instrument_device(
res.instrument_model_id, res.instrument_device_id)
self.omsclient.assign_instrument_model_to_instrument_site(
res.instrument_model_id, res.instrument_site_id)
log.debug("deploying instrument device only")
self.imsclient.deploy_instrument_device(res.instrument_device_id,
res.deployment_id)
log.debug(
"activating deployment without device models, expecting fail")
self.assert_deploy_fail(res.deployment_id,
"No sites were found in the deployment")
#@unittest.skip("targeting")
def test_activate_deployment_nodevice(self):
res = self.base_activate_deployment()
log.debug("assigning platform and instrument models")
self.imsclient.assign_instrument_model_to_instrument_device(
res.instrument_model_id, res.instrument_device_id)
self.omsclient.assign_instrument_model_to_instrument_site(
res.instrument_model_id, res.instrument_site_id)
log.debug("deploying instrument site only")
self.omsclient.deploy_instrument_site(res.instrument_site_id,
res.deployment_id)
log.debug(
"activating deployment without device models, expecting fail")
self.assert_deploy_fail(
res.deployment_id,
"The set of devices could not be mapped to the set of sites")
def assert_deploy_fail(self,
deployment_id,
fail_message="did not specify fail_message"):
with self.assertRaises(BadRequest) as cm:
self.omsclient.activate_deployment(deployment_id)
self.assertIn(fail_message, cm.exception.message)
class TestTransformPrime(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url(
'res/deploy/r2deploy.yml') # Because hey why not?!
self.dataset_management = DatasetManagementServiceClient()
self.data_process_management = DataProcessManagementServiceClient()
self.pubsub_management = PubsubManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
def setup_streams(self):
in_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'sbe37_L0_test', id_only=True)
out_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'sbe37_L1_test', id_only=True)
in_stream_def_id = self.pubsub_management.create_stream_definition(
'L0 SBE37', parameter_dictionary_id=in_pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition,
in_stream_def_id)
out_stream_def_id = self.pubsub_management.create_stream_definition(
'L1 SBE37', parameter_dictionary_id=out_pdict_id)
self.addCleanup(self.pubsub_management.delete_stream_definition,
out_stream_def_id)
in_stream_id, in_route = self.pubsub_management.create_stream(
'L0 input',
stream_definition_id=in_stream_def_id,
exchange_point='test')
self.addCleanup(self.pubsub_management.delete_stream, in_stream_id)
out_stream_id, out_route = self.pubsub_management.create_stream(
'L0 output',
stream_definition_id=out_stream_def_id,
exchange_point='test')
self.addCleanup(self.pubsub_management.delete_stream, out_stream_id)
return [(in_stream_id, in_stream_def_id),
(out_stream_id, out_stream_def_id)]
def setup_advanced_streams(self):
in_pdict_id = out_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'sbe37_LC_TEST', id_only=True)
in_stream_def_id = self.pubsub_management.create_stream_definition(
'sbe37_instrument',
parameter_dictionary_id=in_pdict_id,
available_fields=[
'time', 'TEMPWAT_L0', 'CONDWAT_L0', 'PRESWAT_L0', 'lat', 'lon'
])
self.addCleanup(self.pubsub_management.delete_stream_definition,
in_stream_def_id)
out_stream_def_id = self.pubsub_management.create_stream_definition(
'sbe37_l2',
parameter_dictionary_id=out_pdict_id,
available_fields=['time', 'rho', 'PRACSAL_L2'])
self.addCleanup(self.pubsub_management.delete_stream_definition,
out_stream_def_id)
in_stream_id, in_route = self.pubsub_management.create_stream(
'instrument stream',
stream_definition_id=in_stream_def_id,
exchange_point='test')
self.addCleanup(self.pubsub_management.delete_stream, in_stream_id)
out_stream_id, out_route = self.pubsub_management.create_stream(
'data product stream',
stream_definition_id=out_stream_def_id,
exchange_point='test')
self.addCleanup(self.pubsub_management.delete_stream, out_stream_id)
return [(in_stream_id, in_stream_def_id),
(out_stream_id, out_stream_def_id)]
def preload(self):
config = DotDict()
config.op = 'load'
config.scenario = 'BASE,LC_TEST'
config.categories = 'ParameterFunctions,ParameterDefs,ParameterDictionary'
config.path = 'res/preload/r2_ioc'
self.container.spawn_process('preload',
'ion.processes.bootstrap.ion_loader',
'IONLoader', config)
def setup_advanced_transform(self):
self.preload()
queue_name = 'transform_prime'
stream_info = self.setup_advanced_streams()
in_stream_id, in_stream_def_id = stream_info[0]
out_stream_id, out_stream_def_id = stream_info[1]
routes = {}
routes[(in_stream_id, out_stream_id)] = None
config = DotDict()
config.process.queue_name = queue_name
config.process.routes = routes
config.process.publish_streams = {out_stream_id: out_stream_id}
sub_id = self.pubsub_management.create_subscription(
queue_name, stream_ids=[in_stream_id])
self.addCleanup(self.pubsub_management.delete_subscription, sub_id)
self.pubsub_management.activate_subscription(sub_id)
self.addCleanup(self.pubsub_management.deactivate_subscription, sub_id)
self.container.spawn_process(
'transform_prime', 'ion.processes.data.transforms.transform_prime',
'TransformPrime', config)
listen_sub_id = self.pubsub_management.create_subscription(
'listener', stream_ids=[out_stream_id])
self.addCleanup(self.pubsub_management.delete_subscription,
listen_sub_id)
self.pubsub_management.activate_subscription(listen_sub_id)
self.addCleanup(self.pubsub_management.deactivate_subscription,
listen_sub_id)
return [(in_stream_id, in_stream_def_id),
(out_stream_id, out_stream_def_id)]
def setup_transform(self):
self.preload()
queue_name = 'transform_prime'
stream_info = self.setup_streams()
in_stream_id, in_stream_def_id = stream_info[0]
out_stream_id, out_stream_def_id = stream_info[1]
routes = {}
routes[(in_stream_id, out_stream_id)] = None
config = DotDict()
config.process.queue_name = queue_name
config.process.routes = routes
config.process.publish_streams = {out_stream_id: out_stream_id}
sub_id = self.pubsub_management.create_subscription(
queue_name, stream_ids=[in_stream_id])
self.addCleanup(self.pubsub_management.delete_subscription, sub_id)
self.pubsub_management.activate_subscription(sub_id)
self.addCleanup(self.pubsub_management.deactivate_subscription, sub_id)
self.container.spawn_process(
'transform_prime', 'ion.processes.data.transforms.transform_prime',
'TransformPrime', config)
listen_sub_id = self.pubsub_management.create_subscription(
'listener', stream_ids=[out_stream_id])
self.addCleanup(self.pubsub_management.delete_subscription,
listen_sub_id)
self.pubsub_management.activate_subscription(listen_sub_id)
self.addCleanup(self.pubsub_management.deactivate_subscription,
listen_sub_id)
return [(in_stream_id, in_stream_def_id),
(out_stream_id, out_stream_def_id)]
def setup_validator(self, validator):
listener = StandaloneStreamSubscriber('listener', validator)
listener.start()
self.addCleanup(listener.stop)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_execute_advanced_transform(self):
# Runs a transform across L0-L2 with stream definitions including available fields
streams = self.setup_advanced_transform()
in_stream_id, in_stream_def_id = streams[0]
out_stream_id, out_stream_defs_id = streams[1]
validation_event = Event()
def validator(msg, route, stream_id):
rdt = RecordDictionaryTool.load_from_granule(msg)
if not np.allclose(rdt['rho'], np.array([1001.0055034])):
return
validation_event.set()
self.setup_validator(validator)
in_route = self.pubsub_management.read_stream_route(in_stream_id)
publisher = StandaloneStreamPublisher(in_stream_id, in_route)
outbound_rdt = RecordDictionaryTool(
stream_definition_id=in_stream_def_id)
outbound_rdt['time'] = [0]
outbound_rdt['TEMPWAT_L0'] = [280000]
outbound_rdt['CONDWAT_L0'] = [100000]
outbound_rdt['PRESWAT_L0'] = [2789]
outbound_rdt['lat'] = [45]
outbound_rdt['lon'] = [-71]
outbound_granule = outbound_rdt.to_granule()
publisher.publish(outbound_granule)
self.assertTrue(validation_event.wait(2))
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_execute_transform(self):
streams = self.setup_transform()
in_stream_id, in_stream_def_id = streams[0]
out_stream_id, out_stream_def_id = streams[1]
validation_event = Event()
def validator(msg, route, stream_id):
rdt = RecordDictionaryTool.load_from_granule(msg)
if not np.allclose(rdt['TEMPWAT_L1'], np.array([18.])):
return
if not np.allclose(rdt['CONDWAT_L1'], np.array([0.5])):
return
if not np.allclose(rdt['PRESWAT_L1'], np.array([0.04536611])):
return
validation_event.set()
self.setup_validator(validator)
in_route = self.pubsub_management.read_stream_route(in_stream_id)
publisher = StandaloneStreamPublisher(in_stream_id, in_route)
outbound_rdt = RecordDictionaryTool(
stream_definition_id=in_stream_def_id)
outbound_rdt['time'] = [0]
outbound_rdt['TEMPWAT_L0'] = [280000]
outbound_rdt['CONDWAT_L0'] = [100000]
outbound_rdt['PRESWAT_L0'] = [2789]
outbound_rdt['lat'] = [45]
outbound_rdt['lon'] = [-71]
outbound_granule = outbound_rdt.to_granule()
publisher.publish(outbound_granule)
self.assertTrue(validation_event.wait(2))
class TestCTDPChain(IonIntegrationTestCase):
def setUp(self):
super(TestCTDPChain, self).setUp()
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.pubsub = PubsubManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.data_process_management = DataProcessManagementServiceClient()
self.dataproduct_management = DataProductManagementServiceClient()
self.resource_registry = ResourceRegistryServiceClient()
# This is for the time values inside the packets going into the transform
self.i = 0
self.cnt = 0
# Cleanup of queue created by the subscriber
self.queue_cleanup = []
self.data_process_cleanup = []
def _get_new_ctd_L0_packet(self, stream_definition_id, length):
rdt = RecordDictionaryTool(stream_definition_id=stream_definition_id)
rdt['time'] = numpy.arange(self.i, self.i+length)
for field in rdt:
if isinstance(rdt._pdict.get_context(field).param_type, QuantityType):
rdt[field] = numpy.array([random.uniform(0.0,75.0) for i in xrange(length)])
g = rdt.to_granule()
self.i+=length
return g
def clean_queues(self):
for queue in self.queue_cleanup:
xn = self.container.ex_manager.create_xn_queue(queue)
xn.delete()
def cleaning_operations(self):
for dproc_id in self.data_process_cleanup:
self.data_process_management.delete_data_process(dproc_id)
def test_ctdp_chain(self):
"""
Test that packets are processed by a chain of CTDP transforms: L0, L1 and L2
"""
#-------------------------------------------------------------------------------------
# Prepare the stream def to be used for transform chain
#-------------------------------------------------------------------------------------
#todo Check whether the right parameter dictionary is being used
self._prepare_stream_def_for_transform_chain()
#-------------------------------------------------------------------------------------
# Prepare the data proc defs and in and out data products for the transforms
#-------------------------------------------------------------------------------------
# list_args_L0 = [data_proc_def_id, input_dpod_id, output_dpod_id]
list_args_L0 = self._prepare_things_you_need_to_launch_transform(name_of_transform='L0')
list_args_L1 = self._prepare_things_you_need_to_launch_transform(name_of_transform='L1')
list_args_L2_density = self._prepare_things_you_need_to_launch_transform(name_of_transform='L2_density')
list_args_L2_salinity = self._prepare_things_you_need_to_launch_transform(name_of_transform='L2_salinity')
log.debug("Got the following args: L0 = %s, L1 = %s, L2 density = %s, L2 salinity = %s", list_args_L0, list_args_L1, list_args_L2_density, list_args_L2_salinity )
#-------------------------------------------------------------------------------------
# Launch the CTDP transforms
#-------------------------------------------------------------------------------------
L0_data_proc_id = self._launch_transform('L0', *list_args_L0)
L1_data_proc_id = self._launch_transform('L1', *list_args_L1)
L2_density_data_proc_id = self._launch_transform('L2_density', *list_args_L2_density)
L2_salinity_data_proc_id = self._launch_transform('L2_salinity', *list_args_L2_salinity)
log.debug("Launched the transforms: L0 = %s, L1 = %s", L0_data_proc_id, L1_data_proc_id)
#-------------------------------------------------------------------------
# Start a subscriber listening to the output of each of the transforms
#-------------------------------------------------------------------------
ar_L0 = self.start_subscriber_listening_to_L0_transform(out_data_prod_id=list_args_L0[2])
ar_L1 = self.start_subscriber_listening_to_L1_transform(out_data_prod_id=list_args_L1[2])
ar_L2_density = self.start_subscriber_listening_to_L2_density_transform(out_data_prod_id=list_args_L2_density[2])
ar_L2_salinity = self.start_subscriber_listening_to_L2_density_transform(out_data_prod_id=list_args_L2_salinity[2])
#-------------------------------------------------------------------
# Publish the parsed packets that the L0 transform is listening for
#-------------------------------------------------------------------
stream_id, stream_route = self.get_stream_and_route_for_data_prod(data_prod_id= list_args_L0[1])
self._publish_for_L0_transform(stream_id, stream_route)
#-------------------------------------------------------------------
# Check the granules being outputted by the transforms
#-------------------------------------------------------------------
self._check_granule_from_L0_transform(ar_L0)
self._check_granule_from_L1_transform(ar_L1)
self._check_granule_from_L2_density_transform(ar_L2_density)
self._check_granule_from_L2_salinity_transform(ar_L2_salinity)
def _prepare_stream_def_for_transform_chain(self):
# Get the stream definition for the stream using the parameter dictionary
# pdict_id = self.dataset_management.read_parameter_dictionary_by_name(parameter_dict_name, id_only=True)
pdict_id = self._create_input_param_dict_for_test(parameter_dict_name = 'input_param_for_L0')
self.in_stream_def_id_for_L0 = self.pubsub.create_stream_definition(name='stream_def_for_L0', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub.delete_stream_definition, self.in_stream_def_id_for_L0)
pdict_id = self._create_input_param_dict_for_test(parameter_dict_name = 'params_for_other_transforms')
self.stream_def_id = self.pubsub.create_stream_definition(name='stream_def_for_CTDBP_transforms', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsub.delete_stream_definition, self.stream_def_id)
log.debug("Got the parsed parameter dictionary: id: %s", pdict_id)
log.debug("Got the stream def for parsed input to L0: %s", self.in_stream_def_id_for_L0)
log.debug("Got the stream def for other other streams: %s", self.stream_def_id)
def _prepare_things_you_need_to_launch_transform(self, name_of_transform = ''):
module, class_name = self._get_class_module(name_of_transform)
#-------------------------------------------------------------------------
# Data Process Definition
#-------------------------------------------------------------------------
dpd_obj = IonObject(RT.DataProcessDefinition,
name= 'CTDBP_%s_Transform' % name_of_transform,
description= 'Data Process Definition for the CTDBP %s transform.' % name_of_transform,
module= module,
class_name=class_name)
data_proc_def_id = self.data_process_management.create_data_process_definition(dpd_obj)
self.addCleanup(self.data_process_management.delete_data_process_definition, data_proc_def_id)
log.debug("created data process definition: id = %s", data_proc_def_id)
#-------------------------------------------------------------------------
# Construct temporal and spatial Coordinate Reference System objects for the data product objects
#-------------------------------------------------------------------------
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
#-------------------------------------------------------------------------
# Get the names of the input and output data products
#-------------------------------------------------------------------------
input_dpod_id = ''
output_dpod_id = ''
if name_of_transform == 'L0':
input_dpod_id = self._create_input_data_product('parsed', tdom, sdom)
output_dpod_id = self._create_output_data_product('L0', tdom, sdom)
self.in_prod_for_L1 = output_dpod_id
elif name_of_transform == 'L1':
input_dpod_id = self.in_prod_for_L1
output_dpod_id = self._create_output_data_product('L1', tdom, sdom)
self.in_prod_for_L2 = output_dpod_id
elif name_of_transform == 'L2_density':
input_dpod_id = self.in_prod_for_L2
output_dpod_id = self._create_output_data_product('L2_density', tdom, sdom)
elif name_of_transform == 'L2_salinity':
input_dpod_id = self.in_prod_for_L2
output_dpod_id = self._create_output_data_product('L2_salinity', tdom, sdom)
else:
self.fail("something bad happened")
return [data_proc_def_id, input_dpod_id, output_dpod_id]
def _get_class_module(self, name_of_transform):
options = {'L0' : self._class_module_L0,
'L1' : self._class_module_L1,
'L2_density' : self._class_module_L2_density,
'L2_salinity' : self._class_module_L2_salinity}
return options[name_of_transform]()
def _class_module_L0(self):
module = 'ion.processes.data.transforms.ctdbp.ctdbp_L0'
class_name = 'CTDBP_L0_all'
return module, class_name
def _class_module_L1(self):
module = 'ion.processes.data.transforms.ctdbp.ctdbp_L1'
class_name = 'CTDBP_L1_Transform'
return module, class_name
def _class_module_L2_density(self):
module = 'ion.processes.data.transforms.ctdbp.ctdbp_L2_density'
class_name = 'CTDBP_DensityTransform'
return module, class_name
def _class_module_L2_salinity(self):
module = 'ion.processes.data.transforms.ctdbp.ctdbp_L2_salinity'
class_name = 'CTDBP_SalinityTransform'
return module, class_name
def _create_input_data_product(self, name_of_transform = '', tdom = None, sdom = None):
dpod_obj = IonObject(RT.DataProduct,
name='dprod_%s' % name_of_transform,
description='for_%s' % name_of_transform,
temporal_domain = tdom,
spatial_domain = sdom)
log.debug("the stream def id: %s", self.stream_def_id)
if name_of_transform == 'L0':
stream_def_id = self.in_stream_def_id_for_L0
else:
stream_def_id = self.stream_def_id
dpod_id = self.dataproduct_management.create_data_product(data_product=dpod_obj,
stream_definition_id= stream_def_id
)
self.addCleanup(self.dataproduct_management.delete_data_product, dpod_id)
log.debug("got the data product out. id: %s", dpod_id)
return dpod_id
def _create_output_data_product(self, name_of_transform = '', tdom = None, sdom = None):
dpod_obj = IonObject(RT.DataProduct,
name='dprod_%s' % name_of_transform,
description='for_%s' % name_of_transform,
temporal_domain = tdom,
spatial_domain = sdom)
if name_of_transform == 'L0':
stream_def_id = self.in_stream_def_id_for_L0
else:
stream_def_id = self.stream_def_id
dpod_id = self.dataproduct_management.create_data_product(data_product=dpod_obj,
stream_definition_id=stream_def_id
)
self.addCleanup(self.dataproduct_management.delete_data_product, dpod_id)
return dpod_id
def _launch_transform(self, name_of_transform = '', data_proc_def_id = None, input_dpod_id = None, output_dpod_id = None):
# We need the key name here to be "L2_stream", since when the data process is launched, this name goes into
# the config as in config.process.publish_streams.L2_stream when the config is used to launch the data process
if name_of_transform in ['L0', 'L1']:
binding = '%s_stream' % name_of_transform
elif name_of_transform == 'L2_salinity':
binding = 'salinity'
elif name_of_transform == 'L2_density':
binding = 'density'
config = None
if name_of_transform == 'L1':
config = self._create_calibration_coefficients_dict()
elif name_of_transform == 'L2_density':
config = DotDict()
config.process = {'lat' : 32.7153, 'lon' : 117.1564}
log.debug("launching transform for name: %s",name_of_transform )
log.debug("launching transform for data_proc_def_id: %s\ninput_dpod_id: %s\noutput_dpod_id: %s", data_proc_def_id, input_dpod_id, output_dpod_id )
data_proc_id = self.data_process_management.create_data_process(
data_process_definition_id = data_proc_def_id,
in_data_product_ids= [input_dpod_id],
out_data_product_ids = [output_dpod_id],
configuration = config)
self.addCleanup(self.data_process_management.delete_data_process, data_proc_id)
self.data_process_management.activate_data_process(data_proc_id)
self.addCleanup(self.data_process_management.deactivate_data_process, data_proc_id)
log.debug("Created a data process for ctdbp %s transform: id = %s", name_of_transform, data_proc_id)
return data_proc_id
def get_stream_and_route_for_data_prod(self, data_prod_id = ''):
stream_ids, _ = self.resource_registry.find_objects(data_prod_id, PRED.hasStream, RT.Stream, True)
stream_id = stream_ids[0]
input_stream = self.resource_registry.read(stream_id)
stream_route = input_stream.stream_route
return stream_id, stream_route
def start_subscriber_listening_to_L0_transform(self, out_data_prod_id = ''):
#----------- Create subscribers to listen to the two transforms --------------------------------
stream_ids, _ = self.resource_registry.find_objects(out_data_prod_id, PRED.hasStream, RT.Stream, True)
output_stream_id_of_transform = stream_ids[0]
ar_L0 = self._start_subscriber_to_transform( name_of_transform = 'L0',stream_id=output_stream_id_of_transform)
return ar_L0
def start_subscriber_listening_to_L1_transform(self, out_data_prod_id = ''):
#----------- Create subscribers to listen to the two transforms --------------------------------
stream_ids, _ = self.resource_registry.find_objects(out_data_prod_id, PRED.hasStream, RT.Stream, True)
output_stream_id_of_transform = stream_ids[0]
ar_L1 = self._start_subscriber_to_transform( name_of_transform = 'L1',stream_id=output_stream_id_of_transform)
return ar_L1
def start_subscriber_listening_to_L2_density_transform(self, out_data_prod_id = ''):
#----------- Create subscribers to listen to the two transforms --------------------------------
stream_ids, _ = self.resource_registry.find_objects(out_data_prod_id, PRED.hasStream, RT.Stream, True)
output_stream_id_of_transform = stream_ids[0]
ar_L2_density = self._start_subscriber_to_transform( name_of_transform = 'L2_density', stream_id=output_stream_id_of_transform)
return ar_L2_density
def start_subscriber_listening_to_L2_salinity_transform(self, out_data_prod_id = ''):
#----------- Create subscribers to listen to the two transforms --------------------------------
stream_ids, _ = self.resource_registry.find_objects(out_data_prod_id, PRED.hasStream, RT.Stream, True)
output_stream_id_of_transform = stream_ids[0]
ar_L2_density = self._start_subscriber_to_transform( name_of_transform = 'L2_salinity',stream_id=output_stream_id_of_transform)
return ar_L2_density
def _start_subscriber_to_transform(self, name_of_transform = '', stream_id = ''):
ar = gevent.event.AsyncResult()
def subscriber(m,r,s):
ar.set(m)
sub = StandaloneStreamSubscriber(exchange_name='sub_%s' % name_of_transform, callback=subscriber)
# Note that this running the below line creates an exchange since none of that name exists before
sub_id = self.pubsub.create_subscription('subscriber_to_transform_%s' % name_of_transform,
stream_ids=[stream_id],
exchange_name='sub_%s' % name_of_transform)
self.addCleanup(self.pubsub.delete_subscription, sub_id)
self.pubsub.activate_subscription(sub_id)
self.addCleanup(self.pubsub.deactivate_subscription, sub_id)
sub.start()
self.addCleanup(sub.stop)
return ar
def _check_granule_from_L0_transform(self, ar = None):
granule_from_transform = ar.get(timeout=20)
log.debug("Got the following granule from the L0 transform: %s", granule_from_transform)
# Check the algorithm being applied
self._check_application_of_L0_algorithm(granule_from_transform)
def _check_granule_from_L1_transform(self, ar = None):
granule_from_transform = ar.get(timeout=20)
log.debug("Got the following granule from the L1 transform: %s", granule_from_transform)
# Check the algorithm being applied
self._check_application_of_L1_algorithm(granule_from_transform)
def _check_granule_from_L2_density_transform(self, ar = None):
granule_from_transform = ar.get(timeout=20)
log.debug("Got the following granule from the L2 transform: %s", granule_from_transform)
# Check the algorithm being applied
self._check_application_of_L2_density_algorithm(granule_from_transform)
def _check_granule_from_L2_salinity_transform(self, ar = None):
granule_from_transform = ar.get(timeout=20)
log.debug("Got the following granule from the L2 transform: %s", granule_from_transform)
# Check the algorithm being applied
self._check_application_of_L2_salinity_algorithm(granule_from_transform)
def _check_application_of_L0_algorithm(self, granule = None):
""" Check the algorithm applied by the L0 transform """
rdt = RecordDictionaryTool.load_from_granule(granule)
list_of_expected_keys = ['time', 'pressure', 'conductivity', 'temperature']
for key in list_of_expected_keys:
self.assertIn(key, rdt)
def _check_application_of_L1_algorithm(self, granule = None):
""" Check the algorithm applied by the L1 transform """
rdt = RecordDictionaryTool.load_from_granule(granule)
list_of_expected_keys = [ 'time', 'pressure', 'conductivity', 'temp']
for key in list_of_expected_keys:
self.assertIn(key, rdt)
def _check_application_of_L2_density_algorithm(self, granule = None):
""" Check the algorithm applied by the L2 transform """
rdt = RecordDictionaryTool.load_from_granule(granule)
list_of_expected_keys = ['time', 'density']
for key in list_of_expected_keys:
self.assertIn(key, rdt)
def _check_application_of_L2_salinity_algorithm(self, granule = None):
""" Check the algorithm applied by the L2 transform """
rdt = RecordDictionaryTool.load_from_granule(granule)
list_of_expected_keys = ['time', 'salinity']
for key in list_of_expected_keys:
self.assertIn(key, rdt)
def _publish_for_L0_transform(self, input_stream_id = None, stream_route = None):
#----------- Publish on that stream so that the transform can receive it --------------------------------
self._publish_to_transform(input_stream_id, stream_route )
def _publish_to_transform(self, stream_id = '', stream_route = None):
pub = StandaloneStreamPublisher(stream_id, stream_route)
publish_granule = self._get_new_ctd_L0_packet(stream_definition_id=self.in_stream_def_id_for_L0, length = 5)
pub.publish(publish_granule)
log.debug("Published the following granule: %s", publish_granule)
def _create_input_param_dict_for_test(self, parameter_dict_name = ''):
pdict = ParameterDictionary()
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=numpy.dtype('float64')))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1900'
pdict.add_context(t_ctxt)
cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
cond_ctxt.uom = 'Siemens_per_meter'
pdict.add_context(cond_ctxt)
pres_ctxt = ParameterContext('pressure', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
pres_ctxt.uom = 'Pascal'
pdict.add_context(pres_ctxt)
if parameter_dict_name == 'input_param_for_L0':
temp_ctxt = ParameterContext('temperature', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
else:
temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = 'degree_kelvin'
pdict.add_context(temp_ctxt)
dens_ctxt = ParameterContext('density', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
dens_ctxt.uom = 'g/m'
pdict.add_context(dens_ctxt)
sal_ctxt = ParameterContext('salinity', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
sal_ctxt.uom = 'PSU'
pdict.add_context(sal_ctxt)
#create temp streamdef so the data product can create the stream
pc_list = []
for pc_k, pc in pdict.iteritems():
ctxt_id = self.dataset_management.create_parameter_context(pc_k, pc[1].dump())
pc_list.append(ctxt_id)
if parameter_dict_name == 'input_param_for_L0':
self.addCleanup(self.dataset_management.delete_parameter_context,ctxt_id)
elif pc[1].name == 'temp':
self.addCleanup(self.dataset_management.delete_parameter_context,ctxt_id)
pdict_id = self.dataset_management.create_parameter_dictionary(parameter_dict_name, pc_list)
self.addCleanup(self.dataset_management.delete_parameter_dictionary, pdict_id)
return pdict_id
def _create_calibration_coefficients_dict(self):
config = DotDict()
config.process.calibration_coeffs = {
'temp_calibration_coeffs': {
'TA0' : 1.561342e-03,
'TA1' : 2.561486e-04,
'TA2' : 1.896537e-07,
'TA3' : 1.301189e-07,
'TOFFSET' : 0.000000e+00
},
'cond_calibration_coeffs': {
'G' : -9.896568e-01,
'H' : 1.316599e-01,
'I' : -2.213854e-04,
'J' : 3.292199e-05,
'CPCOR' : -9.570000e-08,
'CTCOR' : 3.250000e-06,
'CSLOPE' : 1.000000e+00
},
'pres_calibration_coeffs' : {
'PA0' : 4.960417e-02,
'PA1' : 4.883682e-04,
'PA2' : -5.687309e-12,
'PTCA0' : 5.249802e+05,
'PTCA1' : 7.595719e+00,
'PTCA2' : -1.322776e-01,
'PTCB0' : 2.503125e+01,
'PTCB1' : 5.000000e-05,
'PTCB2' : 0.000000e+00,
'PTEMPA0' : -6.431504e+01,
'PTEMPA1' : 5.168177e+01,
'PTEMPA2' : -2.847757e-01,
'POFFSET' : 0.000000e+00
}
}
return config
def get_parameter_dictionary_by_name(cls, name=''):
dms_cli = DatasetManagementServiceClient()
pd_res = dms_cli.read_parameter_dictionary_by_name(name=name,
id_only=True)
return cls.get_parameter_dictionary(pd_res)
class TestTransformWorker(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Instantiate a process to represent the test
process = TransformWorkerTestProcess()
self.dataset_management_client = DatasetManagementServiceClient(
node=self.container.node)
self.pubsub_client = PubsubManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceProcessClient(
node=self.container.node, process=process)
self.time_dom, self.spatial_dom = time_series_domain()
self.ph = ParameterHelper(self.dataset_management_client,
self.addCleanup)
self.wait_time = CFG.get_safe('endpoint.receive.timeout', 10)
def push_granule(self, data_product_id):
'''
Publishes and monitors that the granule arrived
'''
datasets, _ = self.rrclient.find_objects(data_product_id,
PRED.hasDataset,
id_only=True)
dataset_monitor = DatasetMonitor(datasets[0])
rdt = self.ph.rdt_for_data_product(data_product_id)
self.ph.fill_parsed_rdt(rdt)
self.ph.publish_rdt_to_data_product(data_product_id, rdt)
assert dataset_monitor.wait()
dataset_monitor.stop()
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_transform_worker(self):
# test that a data process (type: data-product-in / data-product-out) can be defined and launched.
# verify that the output granule fields are correctly populated
# test that the input and output data products are linked to facilitate provenance
self.dp_list = []
self.data_process_objs = []
self._output_stream_ids = []
self.granule_verified = Event()
self.worker_assigned_event_verified = Event()
self.dp_created_event_verified = Event()
self.heartbeat_event_verified = Event()
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(
name='ctd_parsed_param_dict', id_only=True)
# create the StreamDefinition
self.stream_def_id = self.pubsub_client.create_stream_definition(
name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
self.addCleanup(self.pubsub_client.delete_stream_definition,
self.stream_def_id)
# create the DataProduct that is the input to the data processes
input_dp_obj = IonObject(RT.DataProduct,
name='input_data_product',
description='input test stream')
self.input_dp_id = self.dataproductclient.create_data_product(
data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
# retrieve the Stream for this data product
stream_ids, assoc_ids = self.rrclient.find_objects(
self.input_dp_id, PRED.hasStream, RT.Stream, True)
self.stream_id = stream_ids[0]
self.start_event_listener()
# create the DPD, DataProcess and output DataProduct
dataprocessdef_id, dataprocess_id, dataproduct_id = self.create_data_process(
)
self.dp_list.append(dataprocess_id)
# validate the repository for data product algorithms persists the new resources NEW SA-1
# create_data_process call created one of each
dpd_ids, _ = self.rrclient.find_resources(
restype=OT.DataProcessDefinition, id_only=False)
# there will be more than one becuase of the DPDs that reperesent the PFs in the data product above
self.assertTrue(dpd_ids is not None)
dp_ids, _ = self.rrclient.find_resources(restype=OT.DataProcess,
id_only=False)
# only one DP becuase the PFs that are in the code dataproduct above are not activated yet.
self.assertEquals(len(dp_ids), 1)
# validate the name and version label NEW SA - 2
dataprocessdef_obj = self.dataprocessclient.read_data_process_definition(
dataprocessdef_id)
self.assertEqual(dataprocessdef_obj.version_label, '1.0a')
self.assertEqual(dataprocessdef_obj.name, 'add_arrays')
# validate that the DPD has an attachment NEW SA - 21
attachment_ids, assoc_ids = self.rrclient.find_objects(
dataprocessdef_id, PRED.hasAttachment, RT.Attachment, True)
self.assertEqual(len(attachment_ids), 1)
attachment_obj = self.rrclient.read_attachment(attachment_ids[0])
log.debug('attachment: %s', attachment_obj)
# validate that the data process resource has input and output data products associated
# L4-CI-SA-RQ-364 and NEW SA-3
outproduct_ids, assoc_ids = self.rrclient.find_objects(
dataprocess_id, PRED.hasOutputProduct, RT.DataProduct, True)
self.assertEqual(len(outproduct_ids), 1)
inproduct_ids, assoc_ids = self.rrclient.find_objects(
dataprocess_id, PRED.hasInputProduct, RT.DataProduct, True)
self.assertEqual(len(inproduct_ids), 1)
# Test for provenance. Get Data product produced by the data processes
output_data_product_id, _ = self.rrclient.find_objects(
subject=dataprocess_id,
object_type=RT.DataProduct,
predicate=PRED.hasOutputProduct,
id_only=True)
output_data_product_provenance = self.dataproductclient.get_data_product_provenance(
output_data_product_id[0])
# Do a basic check to see if there were 3 entries in the provenance graph. Parent and Child and the
# DataProcessDefinition creating the child from the parent.
self.assertTrue(len(output_data_product_provenance) == 2)
self.assertTrue(self.input_dp_id in output_data_product_provenance[
output_data_product_id[0]]['parents'])
self.assertTrue(output_data_product_provenance[
output_data_product_id[0]]['parents'][self.input_dp_id]
['data_process_definition_id'] == dataprocessdef_id)
# NEW SA - 4 | Data processing shall include the appropriate data product algorithm name and version number in
# the metadata of each output data product created by the data product algorithm.
output_data_product_obj, _ = self.rrclient.find_objects(
subject=dataprocess_id,
object_type=RT.DataProduct,
predicate=PRED.hasOutputProduct,
id_only=False)
self.assertTrue(output_data_product_obj[0].name != None)
self.assertTrue(output_data_product_obj[0]._rev != None)
# retrieve subscription from data process
subscription_objs, _ = self.rrclient.find_objects(
subject=dataprocess_id,
predicate=PRED.hasSubscription,
object_type=RT.Subscription,
id_only=False)
log.debug('test_transform_worker subscription_obj: %s',
subscription_objs[0])
# create a queue to catch the published granules
self.subscription_id = self.pubsub_client.create_subscription(
name='parsed_subscription',
stream_ids=[self.stream_id],
exchange_name=subscription_objs[0].exchange_name)
self.addCleanup(self.pubsub_client.delete_subscription,
self.subscription_id)
self.pubsub_client.activate_subscription(self.subscription_id)
self.addCleanup(self.pubsub_client.deactivate_subscription,
self.subscription_id)
stream_route = self.pubsub_client.read_stream_route(self.stream_id)
self.publisher = StandaloneStreamPublisher(stream_id=self.stream_id,
stream_route=stream_route)
for n in range(1, 101):
rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
rdt['time'] = [0] # time should always come first
rdt['conductivity'] = [1]
rdt['pressure'] = [2]
rdt['salinity'] = [8]
self.publisher.publish(rdt.to_granule())
# validate that the output granule is received and the updated value is correct
self.assertTrue(self.granule_verified.wait(self.wait_time))
# validate that the data process loaded into worker event is received (L4-CI-SA-RQ-182)
self.assertTrue(
self.worker_assigned_event_verified.wait(self.wait_time))
# validate that the data process create (with data product ids) event is received (NEW SA -42)
self.assertTrue(self.dp_created_event_verified.wait(self.wait_time))
# validate that the data process heartbeat event is received (for every hundred granules processed) (L4-CI-SA-RQ-182)
#this takes a while so set wait limit to large value
self.assertTrue(self.heartbeat_event_verified.wait(200))
# validate that the code from the transform function can be retrieve via inspect_data_process_definition
src = self.dataprocessclient.inspect_data_process_definition(
dataprocessdef_id)
self.assertIn('def add_arrays(a, b)', src)
# now delete the DPD and DP then verify that the resources are retired so that information required for provenance are still available
self.dataprocessclient.delete_data_process(dataprocess_id)
self.dataprocessclient.delete_data_process_definition(
dataprocessdef_id)
in_dp_objs, _ = self.rrclient.find_objects(
subject=dataprocess_id,
predicate=PRED.hasInputProduct,
object_type=RT.DataProduct,
id_only=True)
self.assertTrue(in_dp_objs is not None)
dpd_objs, _ = self.rrclient.find_subjects(
subject_type=RT.DataProcessDefinition,
predicate=PRED.hasDataProcess,
object=dataprocess_id,
id_only=True)
self.assertTrue(dpd_objs is not None)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_transform_worker_with_instrumentdevice(self):
# test that a data process (type: data-product-in / data-product-out) can be defined and launched.
# verify that the output granule fields are correctly populated
# test that the input and output data products are linked to facilitate provenance
self.data_process_objs = []
self._output_stream_ids = []
self.event_verified = Event()
# Create CTD Parsed as the initial data product
# create a stream definition for the data from the ctd simulator
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
self.stream_def_id = self.pubsub_client.create_stream_definition(
name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
# create the DataProduct that is the input to the data processes
input_dp_obj = IonObject(RT.DataProduct,
name='input_data_product',
description='input test stream')
self.input_dp_id = self.dataproductclient.create_data_product(
data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
# retrieve the Stream for this data product
stream_ids, assoc_ids = self.rrclient.find_objects(
self.input_dp_id, PRED.hasStream, RT.Stream, True)
self.stream_id = stream_ids[0]
log.debug('new ctd_parsed_data_product_id = %s' % self.input_dp_id)
# only ever need one device for testing purposes.
instDevice_obj, _ = self.rrclient.find_resources(
restype=RT.InstrumentDevice, name='test_ctd_device')
if instDevice_obj:
instDevice_id = instDevice_obj[0]._id
else:
instDevice_obj = IonObject(RT.InstrumentDevice,
name='test_ctd_device',
description="test_ctd_device",
serial_number="12345")
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=self.input_dp_id)
# create the DPD, DataProcess and output DataProduct
dataprocessdef_id, dataprocess_id, dataproduct_id = self.create_data_process(
)
self.addCleanup(self.dataprocessclient.delete_data_process,
dataprocess_id)
self.addCleanup(self.dataprocessclient.delete_data_process_definition,
dataprocessdef_id)
# Test for provenance. Get Data product produced by the data processes
output_data_product_id, _ = self.rrclient.find_objects(
subject=dataprocess_id,
object_type=RT.DataProduct,
predicate=PRED.hasOutputProduct,
id_only=True)
output_data_product_provenance = self.dataproductclient.get_data_product_provenance(
output_data_product_id[0])
# Do a basic check to see if there were 3 entries in the provenance graph. Parent and Child and the
# DataProcessDefinition creating the child from the parent.
self.assertTrue(len(output_data_product_provenance) == 3)
self.assertTrue(self.input_dp_id in output_data_product_provenance[
output_data_product_id[0]]['parents'])
self.assertTrue(instDevice_id in output_data_product_provenance[
self.input_dp_id]['parents'])
self.assertTrue(output_data_product_provenance[instDevice_id]['type']
== 'InstrumentDevice')
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_transform_worker_with_platformdevice(self):
# test that a data process (type: data-product-in / data-product-out) can be defined and launched.
# verify that the output granule fields are correctly populated
# test that the input and output data products are linked to facilitate provenance
self.data_process_objs = []
self._output_stream_ids = []
self.event_verified = Event()
# Create CTD Parsed as the initial data product
# create a stream definition for the data from the ctd simulator
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
self.stream_def_id = self.pubsub_client.create_stream_definition(
name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
# create the DataProduct that is the input to the data processes
input_dp_obj = IonObject(RT.DataProduct,
name='input_data_product',
description='input test stream')
self.input_dp_id = self.dataproductclient.create_data_product(
data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
# retrieve the Stream for this data product
stream_ids, assoc_ids = self.rrclient.find_objects(
self.input_dp_id, PRED.hasStream, RT.Stream, True)
self.stream_id = stream_ids[0]
log.debug('new ctd_parsed_data_product_id = %s' % self.input_dp_id)
# only ever need one device for testing purposes.
platform_device_obj, _ = self.rrclient.find_resources(
restype=RT.PlatformDevice, name='TestPlatform')
if platform_device_obj:
platform_device_id = platform_device_obj[0]._id
else:
platform_device_obj = IonObject(RT.PlatformDevice,
name='TestPlatform',
description="TestPlatform",
serial_number="12345")
platform_device_id = self.imsclient.create_platform_device(
platform_device=platform_device_obj)
self.damsclient.assign_data_product(
input_resource_id=platform_device_id,
data_product_id=self.input_dp_id)
# create the DPD, DataProcess and output DataProduct
dataprocessdef_id, dataprocess_id, dataproduct_id = self.create_data_process(
)
self.addCleanup(self.dataprocessclient.delete_data_process,
dataprocess_id)
self.addCleanup(self.dataprocessclient.delete_data_process_definition,
dataprocessdef_id)
# Test for provenance. Get Data product produced by the data processes
output_data_product_id, _ = self.rrclient.find_objects(
subject=dataprocess_id,
object_type=RT.DataProduct,
predicate=PRED.hasOutputProduct,
id_only=True)
output_data_product_provenance = self.dataproductclient.get_data_product_provenance(
output_data_product_id[0])
# Do a basic check to see if there were 3 entries in the provenance graph. Parent and Child and the
# DataProcessDefinition creating the child from the parent.
self.assertTrue(len(output_data_product_provenance) == 3)
self.assertTrue(self.input_dp_id in output_data_product_provenance[
output_data_product_id[0]]['parents'])
self.assertTrue(platform_device_id in output_data_product_provenance[
self.input_dp_id]['parents'])
self.assertTrue(output_data_product_provenance[platform_device_id]
['type'] == 'PlatformDevice')
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_event_transform_worker(self):
self.data_process_objs = []
self._output_stream_ids = []
self.event_verified = Event()
# test that a data process (type: data-product-in / event-out) can be defined and launched.
# verify that event fields are correctly populated
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(
name='ctd_parsed_param_dict', id_only=True)
# create the StreamDefinition
self.stream_def_id = self.pubsub_client.create_stream_definition(
name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
self.addCleanup(self.pubsub_client.delete_stream_definition,
self.stream_def_id)
# create the DataProduct
input_dp_obj = IonObject(RT.DataProduct,
name='input_data_product',
description='input test stream')
self.input_dp_id = self.dataproductclient.create_data_product(
data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
# retrieve the Stream for this data product
stream_ids, assoc_ids = self.rrclient.find_objects(
self.input_dp_id, PRED.hasStream, RT.Stream, True)
self.stream_id = stream_ids[0]
# create the DPD and two DPs
self.event_data_process_id = self.create_event_data_processes()
# retrieve subscription from data process
subscription_objs, _ = self.rrclient.find_objects(
subject=self.event_data_process_id,
predicate=PRED.hasSubscription,
object_type=RT.Subscription,
id_only=False)
log.debug('test_event_transform_worker subscription_obj: %s',
subscription_objs[0])
# create a queue to catch the published granules
self.subscription_id = self.pubsub_client.create_subscription(
name='parsed_subscription',
stream_ids=[self.stream_id],
exchange_name=subscription_objs[0].exchange_name)
self.addCleanup(self.pubsub_client.delete_subscription,
self.subscription_id)
self.pubsub_client.activate_subscription(self.subscription_id)
self.addCleanup(self.pubsub_client.deactivate_subscription,
self.subscription_id)
stream_route = self.pubsub_client.read_stream_route(self.stream_id)
self.publisher = StandaloneStreamPublisher(stream_id=self.stream_id,
stream_route=stream_route)
self.start_event_transform_listener()
self.data_modified = Event()
rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
rdt['time'] = [0] # time should always come first
rdt['conductivity'] = [1]
rdt['pressure'] = [2]
rdt['salinity'] = [8]
self.publisher.publish(rdt.to_granule())
self.assertTrue(self.event_verified.wait(self.wait_time))
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_bad_argument_map(self):
self._output_stream_ids = []
# test that a data process (type: data-product-in / data-product-out) parameter mapping it validated during
# data process creation and that the correct exception is raised for both input and output.
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(
name='ctd_parsed_param_dict', id_only=True)
# create the StreamDefinition
self.stream_def_id = self.pubsub_client.create_stream_definition(
name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
self.addCleanup(self.pubsub_client.delete_stream_definition,
self.stream_def_id)
# create the DataProduct that is the input to the data processes
input_dp_obj = IonObject(RT.DataProduct,
name='input_data_product',
description='input test stream')
self.input_dp_id = self.dataproductclient.create_data_product(
data_product=input_dp_obj, stream_definition_id=self.stream_def_id)
# two data processes using one transform and one DPD
dp1_func_output_dp_id = self.create_output_data_product()
# Set up DPD and DP #2 - array add function
tf_obj = IonObject(
RT.TransformFunction,
name='add_array_func',
description='adds values in an array',
function='add_arrays',
module="ion_example.add_arrays",
arguments=['arr1', 'arr2'],
function_type=TransformFunctionType.TRANSFORM,
uri=
'http://sddevrepo.oceanobservatories.org/releases/ion_example-0.1-py2.7.egg'
)
add_array_func_id, rev = self.rrclient.create(tf_obj)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='add_arrays',
description='adds the values of two arrays',
data_process_type=DataProcessTypeEnum.TRANSFORM_PROCESS)
add_array_dpd_id = self.dataprocessclient.create_data_process_definition(
data_process_definition=dpd_obj, function_id=add_array_func_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
self.stream_def_id, add_array_dpd_id, binding='add_array_func')
# create the data process with invalid argument map
argument_map = {"arr1": "foo", "arr2": "bar"}
output_param = "salinity"
with self.assertRaises(BadRequest) as cm:
dp1_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id=add_array_dpd_id,
inputs=[self.input_dp_id],
outputs=[dp1_func_output_dp_id],
argument_map=argument_map,
out_param_name=output_param)
ex = cm.exception
log.debug(' exception raised: %s', cm)
self.assertEqual(
ex.message,
"Input data product does not contain the parameters defined in argument map"
)
# create the data process with invalid output parameter name
argument_map = {"arr1": "conductivity", "arr2": "pressure"}
output_param = "foo"
with self.assertRaises(BadRequest) as cm:
dp1_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id=add_array_dpd_id,
inputs=[self.input_dp_id],
outputs=[dp1_func_output_dp_id],
argument_map=argument_map,
out_param_name=output_param)
ex = cm.exception
log.debug(' exception raised: %s', cm)
self.assertEqual(
ex.message,
"Output data product does not contain the output parameter name provided"
)
def create_event_data_processes(self):
# two data processes using one transform and one DPD
argument_map = {"a": "salinity"}
# set up DPD and DP #2 - array add function
tf_obj = IonObject(
RT.TransformFunction,
name='validate_salinity_array',
description='validate_salinity_array',
function='validate_salinity_array',
module="ion.processes.data.transforms.test.test_transform_worker",
arguments=['a'],
function_type=TransformFunctionType.TRANSFORM)
add_array_func_id, rev = self.rrclient.create(tf_obj)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='validate_salinity_array',
description='validate_salinity_array',
data_process_type=DataProcessTypeEnum.TRANSFORM_PROCESS,
)
add_array_dpd_id = self.dataprocessclient.create_data_process_definition(
data_process_definition=dpd_obj, function_id=add_array_func_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
self.stream_def_id,
add_array_dpd_id,
binding='validate_salinity_array')
# create the data process
dp1_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id=add_array_dpd_id,
inputs=[self.input_dp_id],
outputs=None,
argument_map=argument_map)
self.damsclient.register_process(dp1_data_process_id)
self.addCleanup(self.dataprocessclient.delete_data_process,
dp1_data_process_id)
return dp1_data_process_id
def create_data_process(self):
# two data processes using one transform and one DPD
dp1_func_output_dp_id = self.create_output_data_product()
argument_map = {"arr1": "conductivity", "arr2": "pressure"}
output_param = "salinity"
# set up DPD and DP #2 - array add function
tf_obj = IonObject(
RT.TransformFunction,
name='add_array_func',
description='adds values in an array',
function='add_arrays',
module="ion_example.add_arrays",
arguments=['arr1', 'arr2'],
function_type=TransformFunctionType.TRANSFORM,
uri=
'http://sddevrepo.oceanobservatories.org/releases/ion_example-0.1-py2.7.egg'
)
add_array_func_id, rev = self.rrclient.create(tf_obj)
dpd_obj = IonObject(
RT.DataProcessDefinition,
name='add_arrays',
description='adds the values of two arrays',
data_process_type=DataProcessTypeEnum.TRANSFORM_PROCESS,
version_label='1.0a')
add_array_dpd_id = self.dataprocessclient.create_data_process_definition(
data_process_definition=dpd_obj, function_id=add_array_func_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(
self.stream_def_id, add_array_dpd_id, binding='add_array_func')
# create the data process
dp1_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id=add_array_dpd_id,
inputs=[self.input_dp_id],
outputs=[dp1_func_output_dp_id],
argument_map=argument_map,
out_param_name=output_param)
self.damsclient.register_process(dp1_data_process_id)
#self.addCleanup(self.dataprocessclient.delete_data_process, dp1_data_process_id)
# add an attachment object to this DPD to test new SA-21
import msgpack
attachment_content = 'foo bar'
attachment_obj = IonObject(RT.Attachment,
name='test_attachment',
attachment_type=AttachmentType.ASCII,
content_type='text/plain',
content=msgpack.packb(attachment_content))
att_id = self.rrclient.create_attachment(add_array_dpd_id,
attachment_obj)
self.addCleanup(self.rrclient.delete_attachment, att_id)
return add_array_dpd_id, dp1_data_process_id, dp1_func_output_dp_id
def create_output_data_product(self):
dp1_outgoing_stream_id = self.pubsub_client.create_stream_definition(
name='dp1_stream', parameter_dictionary_id=self.parameter_dict_id)
dp1_output_dp_obj = IonObject(RT.DataProduct,
name='data_process1_data_product',
description='output of add array func')
dp1_func_output_dp_id = self.dataproductclient.create_data_product(
dp1_output_dp_obj, dp1_outgoing_stream_id)
self.addCleanup(self.dataproductclient.delete_data_product,
dp1_func_output_dp_id)
# retrieve the id of the OUTPUT stream from the out Data Product and add to granule logger
stream_ids, _ = self.rrclient.find_objects(dp1_func_output_dp_id,
PRED.hasStream, None, True)
self._output_stream_ids.append(stream_ids[0])
subscription_id = self.pubsub_client.create_subscription(
'validator', data_product_ids=[dp1_func_output_dp_id])
self.addCleanup(self.pubsub_client.delete_subscription,
subscription_id)
def on_granule(msg, route, stream_id):
log.debug('recv_packet stream_id: %s route: %s msg: %s',
stream_id, route, msg)
self.validate_output_granule(msg, route, stream_id)
self.granule_verified.set()
validator = StandaloneStreamSubscriber('validator',
callback=on_granule)
validator.start()
self.addCleanup(validator.stop)
self.pubsub_client.activate_subscription(subscription_id)
self.addCleanup(self.pubsub_client.deactivate_subscription,
subscription_id)
return dp1_func_output_dp_id
def validate_event(self, *args, **kwargs):
"""
This method is a callback function for receiving DataProcessStatusEvent.
"""
data_process_event = args[0]
log.debug("DataProcessStatusEvent: %s",
str(data_process_event.__dict__))
# if data process already created, check origin
if self.dp_list:
self.assertIn(data_process_event.origin, self.dp_list)
# if this is a heartbeat event then 100 granules have been processed
if 'data process status update.' in data_process_event.description:
self.heartbeat_event_verified.set()
else:
# else check that this is the assign event
if 'Data process assigned to transform worker' in data_process_event.description:
self.worker_assigned_event_verified.set()
elif 'Data process created for data product' in data_process_event.description:
self.dp_created_event_verified.set()
def validate_output_granule(self, msg, route, stream_id):
self.assertIn(stream_id, self._output_stream_ids)
rdt = RecordDictionaryTool.load_from_granule(msg)
log.debug('validate_output_granule rdt: %s', rdt)
sal_val = rdt['salinity']
np.testing.assert_array_equal(sal_val, np.array([3]))
def start_event_listener(self):
es = EventSubscriber(event_type=OT.DataProcessStatusEvent,
callback=self.validate_event)
es.start()
self.addCleanup(es.stop)
def validate_transform_event(self, *args, **kwargs):
"""
This method is a callback function for receiving DataProcessStatusEvent.
"""
status_alert_event = args[0]
np.testing.assert_array_equal(status_alert_event.origin,
self.stream_id)
np.testing.assert_array_equal(status_alert_event.values,
np.array([self.event_data_process_id]))
log.debug("DeviceStatusAlertEvent: %s",
str(status_alert_event.__dict__))
self.event_verified.set()
def start_event_transform_listener(self):
es = EventSubscriber(event_type=OT.DeviceStatusAlertEvent,
callback=self.validate_transform_event)
es.start()
self.addCleanup(es.stop)
def test_download(self):
egg_url = 'http://sddevrepo.oceanobservatories.org/releases/ion_example-0.1-py2.7.egg'
egg_path = TransformWorker.download_egg(egg_url)
import pkg_resources
pkg_resources.working_set.add_entry(egg_path)
from ion_example.add_arrays import add_arrays
a = add_arrays(1, 2)
self.assertEquals(a, 3)
def test_download(self):
# clients
dataset_management = DatasetManagementServiceClient()
# verify target object [REFDES01] do not exist in object_store
self.assertRaises(NotFound, dataset_management.read_qc_table,
'REFDES01')
# NOTE: time is again monkey patched in 'test_upload_qc' but should be static for that
# MONKEY PATCH time.time() for volatile ts_updated values in dict (set in POST below)
CONSTANT_TIME = time.time() # time value we'll use in assert tests
def new_time():
return CONSTANT_TIME
old_time = time.time
time.time = new_time
#upload some data
self.test_upload_qc()
# restore MONKEY PATCHed time
time.time = old_time
REFDES01 = dataset_management.read_qc_table('REFDES01')
RD01DP01 = REFDES01.get('RD01DP01', None)
self.assertEquals(
RD01DP01, {
'stuck_value': [{
'units': 'C',
'consecutive_values': 10,
'ts_created': CONSTANT_TIME,
'resolution': 0.005,
'author': 'Otter'
}],
'gradient_test': [{
'toldat': 0.1,
'xunits': 's',
'mindx': 30,
'author': 'Boon',
'startdat': None,
'ddatdx': [-0.01, 0.01],
'units': 'C',
'ts_created': CONSTANT_TIME
}],
'global_range': [{
'units': 'm/s',
'max_value': 1,
'min_value': -1,
'ts_created': CONSTANT_TIME,
'author': 'Douglas C. Neidermeyer'
}, {
'units': 'm/s',
'max_value': 10,
'min_value': -10,
'ts_created': CONSTANT_TIME,
'author': 'Bluto'
}],
'trend_test': [{
'author': 'Pinto',
'standard_deviation': 4.5,
'polynomial_order': 4,
'sample_length': 25,
'units': 'K',
'ts_created': CONSTANT_TIME
}],
'spike_test': [{
'author': 'Flounder',
'range_multiplier': 4,
'window_length': 15,
'units': 'degrees',
'ts_created': CONSTANT_TIME,
'accuracy': 0.0001
}]
})
class TestActivateInstrumentIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
super(TestActivateInstrumentIntegration, self).setUp()
config = DotDict()
config.bootstrap.use_es = True
self._start_container()
self.addCleanup(TestActivateInstrumentIntegration.es_cleanup)
self.container.start_rel_from_url('res/deploy/r2deploy.yml', config)
# Now create client to DataProductManagementService
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(
node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(
node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(
node=self.container.node)
self.dpclient = DataProductManagementServiceClient(
node=self.container.node)
self.datasetclient = DatasetManagementServiceClient(
node=self.container.node)
self.processdispatchclient = ProcessDispatcherServiceClient(
node=self.container.node)
self.dataprocessclient = DataProcessManagementServiceClient(
node=self.container.node)
self.dataproductclient = DataProductManagementServiceClient(
node=self.container.node)
self.dataretrieverclient = DataRetrieverServiceClient(
node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.usernotificationclient = UserNotificationServiceClient()
#setup listerner vars
self._data_greenlets = []
self._no_samples = None
self._samples_received = []
self.event_publisher = EventPublisher()
@staticmethod
def es_cleanup():
es_host = CFG.get_safe('server.elasticsearch.host', 'localhost')
es_port = CFG.get_safe('server.elasticsearch.port', '9200')
es = ep.ElasticSearch(host=es_host, port=es_port, timeout=10)
indexes = STD_INDEXES.keys()
indexes.append('%s_resources_index' % get_sys_name().lower())
indexes.append('%s_events_index' % get_sys_name().lower())
for index in indexes:
IndexManagementService._es_call(es.river_couchdb_delete, index)
IndexManagementService._es_call(es.index_delete, index)
def create_logger(self, name, stream_id=''):
# logger process
producer_definition = ProcessDefinition(name=name + '_logger')
producer_definition.executable = {
'module': 'ion.processes.data.stream_granule_logger',
'class': 'StreamGranuleLogger'
}
logger_procdef_id = self.processdispatchclient.create_process_definition(
process_definition=producer_definition)
configuration = {
'process': {
'stream_id': stream_id,
}
}
pid = self.processdispatchclient.schedule_process(
process_definition_id=logger_procdef_id,
configuration=configuration)
return pid
def _create_notification(self,
user_name='',
instrument_id='',
product_id=''):
#--------------------------------------------------------------------------------------
# Make notification request objects
#--------------------------------------------------------------------------------------
notification_request_1 = NotificationRequest(
name='notification_1',
origin=instrument_id,
origin_type="instrument",
event_type='ResourceLifecycleEvent')
notification_request_2 = NotificationRequest(
name='notification_2',
origin=product_id,
origin_type="data product",
event_type='DetectionEvent')
#--------------------------------------------------------------------------------------
# Create a user and get the user_id
#--------------------------------------------------------------------------------------
user = UserInfo()
user.name = user_name
user.contact.email = '*****@*****.**' % user_name
user_id, _ = self.rrclient.create(user)
#--------------------------------------------------------------------------------------
# Create notification
#--------------------------------------------------------------------------------------
self.usernotificationclient.create_notification(
notification=notification_request_1, user_id=user_id)
self.usernotificationclient.create_notification(
notification=notification_request_2, user_id=user_id)
log.debug(
"test_activateInstrumentSample: create_user_notifications user_id %s",
str(user_id))
return user_id
def get_datastore(self, dataset_id):
dataset = self.datasetclient.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(
datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def _check_computed_attributes_of_extended_instrument(
self, expected_instrument_device_id='', extended_instrument=None):
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(extended_instrument.computed.firmware_version,
ComputedFloatValue)
self.assertIsInstance(
extended_instrument.computed.last_data_received_datetime,
ComputedFloatValue)
self.assertIsInstance(
extended_instrument.computed.last_calibration_datetime,
ComputedFloatValue)
self.assertIsInstance(extended_instrument.computed.uptime,
ComputedStringValue)
self.assertIsInstance(
extended_instrument.computed.power_status_roll_up,
ComputedIntValue)
self.assertIsInstance(
extended_instrument.computed.communications_status_roll_up,
ComputedIntValue)
self.assertIsInstance(extended_instrument.computed.data_status_roll_up,
ComputedIntValue)
self.assertIsInstance(
extended_instrument.computed.location_status_roll_up,
ComputedIntValue)
# the following assert will not work without elasticsearch.
#self.assertEqual( 1, len(extended_instrument.computed.user_notification_requests.value) )
self.assertEqual(
extended_instrument.computed.communications_status_roll_up.value,
StatusType.STATUS_WARNING)
self.assertEqual(
extended_instrument.computed.data_status_roll_up.value,
StatusType.STATUS_OK)
self.assertEqual(
extended_instrument.computed.power_status_roll_up.value,
StatusType.STATUS_WARNING)
# Verify the computed attribute for user notification requests
self.assertEqual(
1,
len(extended_instrument.computed.user_notification_requests.value))
notifications = extended_instrument.computed.user_notification_requests.value
notification = notifications[0]
self.assertEqual(notification.origin, expected_instrument_device_id)
self.assertEqual(notification.origin_type, "instrument")
self.assertEqual(notification.event_type, 'ResourceLifecycleEvent')
def _check_computed_attributes_of_extended_product(
self, expected_data_product_id='', extended_data_product=None):
self.assertEqual(expected_data_product_id, extended_data_product._id)
log.debug("extended_data_product.computed: %s",
extended_data_product.computed)
# Verify that computed attributes exist for the extended instrument
self.assertIsInstance(
extended_data_product.computed.product_download_size_estimated,
ComputedIntValue)
self.assertIsInstance(
extended_data_product.computed.number_active_subscriptions,
ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.data_url,
ComputedStringValue)
self.assertIsInstance(extended_data_product.computed.stored_data_size,
ComputedIntValue)
self.assertIsInstance(extended_data_product.computed.recent_granules,
ComputedDictValue)
self.assertIsInstance(extended_data_product.computed.parameters,
ComputedListValue)
self.assertIsInstance(extended_data_product.computed.recent_events,
ComputedEventListValue)
self.assertIsInstance(extended_data_product.computed.provenance,
ComputedDictValue)
self.assertIsInstance(
extended_data_product.computed.user_notification_requests,
ComputedListValue)
self.assertIsInstance(
extended_data_product.computed.active_user_subscriptions,
ComputedListValue)
self.assertIsInstance(
extended_data_product.computed.past_user_subscriptions,
ComputedListValue)
self.assertIsInstance(extended_data_product.computed.last_granule,
ComputedDictValue)
self.assertIsInstance(extended_data_product.computed.is_persisted,
ComputedIntValue)
self.assertIsInstance(
extended_data_product.computed.data_contents_updated,
ComputedStringValue)
self.assertIsInstance(extended_data_product.computed.data_datetime,
ComputedListValue)
# exact text here keeps changing to fit UI capabilities. keep assertion general...
self.assertTrue('ok' in extended_data_product.computed.last_granule.
value['quality_flag'])
self.assertEqual(
2, len(extended_data_product.computed.data_datetime.value))
notifications = extended_data_product.computed.user_notification_requests.value
notification = notifications[0]
self.assertEqual(notification.origin, expected_data_product_id)
self.assertEqual(notification.origin_type, "data product")
self.assertEqual(notification.event_type, 'DetectionEvent')
@attr('LOCOINT')
@unittest.skipIf(not use_es, 'No ElasticSearch')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
@patch.dict(CFG, {'endpoint': {'receive': {'timeout': 60}}})
def test_activateInstrumentSample(self):
self.loggerpids = []
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel")
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
log.debug('new InstrumentModel id = %s ', instModel_id)
#Create stream alarms
"""
test_two_sided_interval
Test interval alarm and alarm event publishing for a closed
inteval.
"""
# kwargs = {
# 'name' : 'test_sim_warning',
# 'stream_name' : 'parsed',
# 'value_id' : 'temp',
# 'message' : 'Temperature is above test range of 5.0.',
# 'type' : StreamAlarmType.WARNING,
# 'upper_bound' : 5.0,
# 'upper_rel_op' : '<'
# }
kwargs = {
'name': 'temperature_warning_interval',
'stream_name': 'parsed',
'value_id': 'temp',
'message':
'Temperature is below the normal range of 50.0 and above.',
'type': StreamAlarmType.WARNING,
'lower_bound': 50.0,
'lower_rel_op': '<'
}
# Create alarm object.
alarm = {}
alarm['type'] = 'IntervalAlarmDef'
alarm['kwargs'] = kwargs
raw_config = StreamConfiguration(
stream_name='raw',
parameter_dictionary_name='ctd_raw_param_dict',
records_per_granule=2,
granule_publish_rate=5)
parsed_config = StreamConfiguration(
stream_name='parsed',
parameter_dictionary_name='ctd_parsed_param_dict',
records_per_granule=2,
granule_publish_rate=5,
alarms=[alarm])
# Create InstrumentAgent
instAgent_obj = IonObject(
RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri=
"http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1a-py2.7.egg",
stream_configurations=[raw_config, parsed_config])
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
log.debug('new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(
instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug(
'test_activateInstrumentSample: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) '
)
instDevice_obj = IonObject(RT.InstrumentDevice,
name='SBE37IMDevice',
description="SBE37IMDevice",
serial_number="12345")
instDevice_id = self.imsclient.create_instrument_device(
instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(
instModel_id, instDevice_id)
log.debug(
"test_activateInstrumentSample: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ",
instDevice_id)
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance,
name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config=port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(
instAgentInstance_obj, instAgent_id, instDevice_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
parsed_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubcli.create_stream_definition(
name='parsed', parameter_dictionary_id=parsed_pdict_id)
raw_pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_raw_param_dict', id_only=True)
raw_stream_def_id = self.pubsubcli.create_stream_definition(
name='raw', parameter_dictionary_id=raw_pdict_id)
#-------------------------------
# Create Raw and Parsed Data Products for the device
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id1 = self.dpclient.create_data_product(
data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug('new dp_id = %s', data_product_id1)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id1)
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id1)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id1,
PRED.hasStream, None, True)
log.debug('Data product streams1 = %s', stream_ids)
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id1,
PRED.hasDataset,
RT.Dataset, True)
log.debug('Data set for data_product_id1 = %s', dataset_ids[0])
self.parsed_dataset = dataset_ids[0]
pid = self.create_logger('ctd_parsed', stream_ids[0])
self.loggerpids.append(pid)
dp_obj = IonObject(RT.DataProduct,
name='the raw data',
description='raw stream test',
temporal_domain=tdom,
spatial_domain=sdom)
data_product_id2 = self.dpclient.create_data_product(
data_product=dp_obj, stream_definition_id=raw_stream_def_id)
log.debug('new dp_id = %s', data_product_id2)
self.damsclient.assign_data_product(input_resource_id=instDevice_id,
data_product_id=data_product_id2)
self.dpclient.activate_data_product_persistence(
data_product_id=data_product_id2)
# setup notifications for the device and parsed data product
user_id_1 = self._create_notification(user_name='user_1',
instrument_id=instDevice_id,
product_id=data_product_id1)
#---------- Create notifications for another user and verify that we see different computed subscriptions for the two users ---------
user_id_2 = self._create_notification(user_name='user_2',
instrument_id=instDevice_id,
product_id=data_product_id2)
# Retrieve the id of the OUTPUT stream from the out Data Product
stream_ids, _ = self.rrclient.find_objects(data_product_id2,
PRED.hasStream, None, True)
log.debug('Data product streams2 = %s', str(stream_ids))
# Retrieve the id of the OUTPUT stream from the out Data Product
dataset_ids, _ = self.rrclient.find_objects(data_product_id2,
PRED.hasDataset,
RT.Dataset, True)
log.debug('Data set for data_product_id2 = %s', dataset_ids[0])
self.raw_dataset = dataset_ids[0]
#elastic search debug
es_indexes, _ = self.container.resource_registry.find_resources(
restype='ElasticSearchIndex')
log.debug('ElasticSearch indexes: %s', [i.name for i in es_indexes])
log.debug('Bootstrap %s', CFG.bootstrap.use_es)
def start_instrument_agent():
self.imsclient.start_instrument_agent_instance(
instrument_agent_instance_id=instAgentInstance_id)
gevent.joinall([gevent.spawn(start_instrument_agent)])
#setup a subscriber to alarm events from the device
self._events_received = []
self._event_count = 0
self._samples_out_of_range = 0
self._samples_complete = False
self._async_sample_result = AsyncResult()
def consume_event(*args, **kwargs):
log.debug(
'TestActivateInstrument recieved ION event: args=%s, kwargs=%s, event=%s.',
str(args), str(kwargs), str(args[0]))
self._events_received.append(args[0])
self._event_count = len(self._events_received)
self._async_sample_result.set()
self._event_subscriber = EventSubscriber(
event_type=
'StreamWarningAlarmEvent', #'StreamWarningAlarmEvent', # StreamAlarmEvent
callback=consume_event,
origin=instDevice_id)
self._event_subscriber.start()
#cleanup
self.addCleanup(self.imsclient.stop_instrument_agent_instance,
instrument_agent_instance_id=instAgentInstance_id)
def stop_subscriber():
self._event_subscriber.stop()
self._event_subscriber = None
self.addCleanup(stop_subscriber)
#wait for start
inst_agent_instance_obj = self.imsclient.read_instrument_agent_instance(
instAgentInstance_id)
gate = ProcessStateGate(self.processdispatchclient.read_process,
inst_agent_instance_obj.agent_process_id,
ProcessStateEnum.RUNNING)
self.assertTrue(
gate. await (30),
"The instrument agent instance (%s) did not spawn in 30 seconds" %
inst_agent_instance_obj.agent_process_id)
log.debug('Instrument agent instance obj: = %s',
str(inst_agent_instance_obj))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = ResourceAgentClient(
instDevice_id,
to_name=inst_agent_instance_obj.agent_process_id,
process=FakeProcess())
log.debug("test_activateInstrumentSample: got ia client %s",
str(self._ia_client))
cmd = AgentCommand(command=ResourceAgentEvent.INITIALIZE)
retval = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: initialize %s", str(retval))
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.INACTIVE)
log.debug("(L4-CI-SA-RQ-334): Sending go_active command ")
cmd = AgentCommand(command=ResourceAgentEvent.GO_ACTIVE)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrument: return value from go_active %s",
str(reply))
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.GET_RESOURCE_STATE)
retval = self._ia_client.execute_agent(cmd)
state = retval.result
log.debug(
"(L4-CI-SA-RQ-334): current state after sending go_active command %s",
str(state))
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: run %s", str(reply))
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.PAUSE)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.STOPPED)
cmd = AgentCommand(command=ResourceAgentEvent.RESUME)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=ResourceAgentEvent.CLEAR)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.IDLE)
cmd = AgentCommand(command=ResourceAgentEvent.RUN)
retval = self._ia_client.execute_agent(cmd)
state = self._ia_client.get_agent_state()
self.assertEqual(state, ResourceAgentState.COMMAND)
cmd = AgentCommand(command=SBE37ProtocolEvent.ACQUIRE_SAMPLE)
for i in xrange(10):
retval = self._ia_client.execute_resource(cmd)
log.debug("test_activateInstrumentSample: return from sample %s",
str(retval))
log.debug("test_activateInstrumentSample: calling reset ")
cmd = AgentCommand(command=ResourceAgentEvent.RESET)
reply = self._ia_client.execute_agent(cmd)
log.debug("test_activateInstrumentSample: return from reset %s",
str(reply))
self._samples_complete = True
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data = self.dataretrieverclient.retrieve(self.parsed_dataset)
self.assertIsInstance(replay_data, Granule)
rdt = RecordDictionaryTool.load_from_granule(replay_data)
log.debug("test_activateInstrumentSample: RDT parsed: %s",
str(rdt.pretty_print()))
temp_vals = rdt['temp']
self.assertEquals(len(temp_vals), 10)
log.debug("test_activateInstrumentSample: all temp_vals: %s",
temp_vals)
#out_of_range_temp_vals = [i for i in temp_vals if i > 5]
out_of_range_temp_vals = [i for i in temp_vals if i < 50.0]
log.debug("test_activateInstrumentSample: Out_of_range_temp_vals: %s",
out_of_range_temp_vals)
self._samples_out_of_range = len(out_of_range_temp_vals)
# if no bad values were produced, then do not wait for an event
if self._samples_out_of_range == 0:
self._async_sample_result.set()
log.debug("test_activateInstrumentSample: _events_received: %s",
self._events_received)
log.debug("test_activateInstrumentSample: _event_count: %s",
self._event_count)
self._async_sample_result.get(timeout=CFG.endpoint.receive.timeout)
replay_data = self.dataretrieverclient.retrieve(self.raw_dataset)
self.assertIsInstance(replay_data, Granule)
rdt = RecordDictionaryTool.load_from_granule(replay_data)
log.debug("RDT raw: %s", str(rdt.pretty_print()))
raw_vals = rdt['raw']
self.assertEquals(len(raw_vals), 10)
log.debug("l4-ci-sa-rq-138")
"""
Physical resource control shall be subject to policy
Instrument management control capabilities shall be subject to policy
The actor accessing the control capabilities must be authorized to send commands.
note from maurice 2012-05-18: Talk to tim M to verify that this is policy. If it is then talk with Stephen to
get an example of a policy test and use that to create a test stub that will be
completed when we have instrument policies.
Tim M: The "actor", aka observatory operator, will access the instrument through ION.
"""
#--------------------------------------------------------------------------------
# Get the extended data product to see if it contains the granules
#--------------------------------------------------------------------------------
extended_product = self.dpclient.get_data_product_extension(
data_product_id=data_product_id1, user_id=user_id_1)
def poller(extended_product):
return len(extended_product.computed.user_notification_requests.
value) == 1
poll(poller, extended_product, timeout=30)
self._check_computed_attributes_of_extended_product(
expected_data_product_id=data_product_id1,
extended_data_product=extended_product)
#--------------------------------------------------------------------------------
#put some events into the eventsdb to test - this should set the comms and data status to WARNING
#--------------------------------------------------------------------------------
t = get_ion_ts()
self.event_publisher.publish_event(ts_created=t,
event_type='DeviceStatusEvent',
origin=instDevice_id,
state=DeviceStatusType.OUT_OF_RANGE,
values=[200])
self.event_publisher.publish_event(
ts_created=t,
event_type='DeviceCommsEvent',
origin=instDevice_id,
state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION,
lapse_interval_seconds=20)
#--------------------------------------------------------------------------------
# Get the extended instrument
#--------------------------------------------------------------------------------
extended_instrument = self.imsclient.get_instrument_device_extension(
instrument_device_id=instDevice_id, user_id=user_id_1)
self._check_computed_attributes_of_extended_instrument(
expected_instrument_device_id=instDevice_id,
extended_instrument=extended_instrument)
#--------------------------------------------------------------------------------
# For the second user, check the extended data product and the extended intrument
#--------------------------------------------------------------------------------
extended_product = self.dpclient.get_data_product_extension(
data_product_id=data_product_id2, user_id=user_id_2)
self._check_computed_attributes_of_extended_product(
expected_data_product_id=data_product_id2,
extended_data_product=extended_product)
#---------- Put some events into the eventsdb to test - this should set the comms and data status to WARNING ---------
t = get_ion_ts()
self.event_publisher.publish_event(ts_created=t,
event_type='DeviceStatusEvent',
origin=instDevice_id,
state=DeviceStatusType.OUT_OF_RANGE,
values=[200])
self.event_publisher.publish_event(
ts_created=t,
event_type='DeviceCommsEvent',
origin=instDevice_id,
state=DeviceCommsType.DATA_DELIVERY_INTERRUPTION,
lapse_interval_seconds=20)
#--------------------------------------------------------------------------------
# Get the extended instrument
#--------------------------------------------------------------------------------
extended_instrument = self.imsclient.get_instrument_device_extension(
instrument_device_id=instDevice_id, user_id=user_id_2)
self._check_computed_attributes_of_extended_instrument(
expected_instrument_device_id=instDevice_id,
extended_instrument=extended_instrument)
#--------------------------------------------------------------------------------
# Deactivate loggers
#--------------------------------------------------------------------------------
for pid in self.loggerpids:
self.processdispatchclient.cancel_process(pid)
self.dpclient.delete_data_product(data_product_id1)
self.dpclient.delete_data_product(data_product_id2)
def _setup_resources(self):
# TODO: some or all of this (or some variation) should move to DAMS'
# Build the test resources for the dataset
dms_cli = DatasetManagementServiceClient()
dams_cli = DataAcquisitionManagementServiceClient()
dpms_cli = DataProductManagementServiceClient()
rr_cli = ResourceRegistryServiceClient()
pubsub_cli = PubsubManagementServiceClient()
eda = ExternalDatasetAgent(handler_module=self.DVR_CONFIG['dvr_mod'],
handler_class=self.DVR_CONFIG['dvr_cls'])
eda_id = dams_cli.create_external_dataset_agent(eda)
eda_inst = ExternalDatasetAgentInstance()
eda_inst_id = dams_cli.create_external_dataset_agent_instance(eda_inst, external_dataset_agent_id=eda_id)
# Create and register the necessary resources/objects
# Create DataProvider
dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation())
dprov.contact.individual_names_given = 'Christopher Mueller'
dprov.contact.email = '*****@*****.**'
# Create DataSource
dsrc = DataSource(protocol_type='FILE', institution=Institution(), contact=ContactInformation())
dsrc.connection_params['base_data_url'] = ''
dsrc.contact.individual_names_given = 'Tim Giguere'
dsrc.contact.email = '*****@*****.**'
# Create ExternalDataset
ds_name = 'slocum_test_dataset'
dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation())
dset.dataset_description.parameters['base_url'] = 'test_data/slocum/'
dset.dataset_description.parameters['list_pattern'] = 'ru05-2012-021-0-0-sbd.dat'
dset.dataset_description.parameters['date_pattern'] = '%Y %j'
dset.dataset_description.parameters['date_extraction_pattern'] = 'ru05-([\d]{4})-([\d]{3})-\d-\d-sbd.dat'
dset.dataset_description.parameters['temporal_dimension'] = None
dset.dataset_description.parameters['zonal_dimension'] = None
dset.dataset_description.parameters['meridional_dimension'] = None
dset.dataset_description.parameters['vertical_dimension'] = None
dset.dataset_description.parameters['variables'] = [
'c_wpt_y_lmc',
'sci_water_cond',
'm_y_lmc',
'u_hd_fin_ap_inflection_holdoff',
'sci_m_present_time',
'm_leakdetect_voltage_forward',
'sci_bb3slo_b660_scaled',
'c_science_send_all',
'm_gps_status',
'm_water_vx',
'm_water_vy',
'c_heading',
'sci_fl3slo_chlor_units',
'u_hd_fin_ap_gain',
'm_vacuum',
'u_min_water_depth',
'm_gps_lat',
'm_veh_temp',
'f_fin_offset',
'u_hd_fin_ap_hardover_holdoff',
'c_alt_time',
'm_present_time',
'm_heading',
'sci_bb3slo_b532_scaled',
'sci_fl3slo_cdom_units',
'm_fin',
'x_cycle_overrun_in_ms',
'sci_water_pressure',
'u_hd_fin_ap_igain',
'sci_fl3slo_phyco_units',
'm_battpos',
'sci_bb3slo_b470_scaled',
'm_lat',
'm_gps_lon',
'sci_ctd41cp_timestamp',
'm_pressure',
'c_wpt_x_lmc',
'c_ballast_pumped',
'x_lmc_xy_source',
'm_lon',
'm_avg_speed',
'sci_water_temp',
'u_pitch_ap_gain',
'm_roll',
'm_tot_num_inflections',
'm_x_lmc',
'u_pitch_ap_deadband',
'm_final_water_vy',
'm_final_water_vx',
'm_water_depth',
'm_leakdetect_voltage',
'u_pitch_max_delta_battpos',
'm_coulomb_amphr',
'm_pitch', ]
# Create DataSourceModel
dsrc_model = DataSourceModel(name='slocum_model')
# dsrc_model.model = 'SLOCUM'
dsrc_model.data_handler_module = 'N/A'
dsrc_model.data_handler_class = 'N/A'
## Run everything through DAMS
ds_id = dams_cli.create_external_dataset(external_dataset=dset)
ext_dprov_id = dams_cli.create_external_data_provider(external_data_provider=dprov)
ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc)
ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model)
# Register the ExternalDataset
dproducer_id = dams_cli.register_external_data_set(external_dataset_id=ds_id)
# Or using each method
dams_cli.assign_data_source_to_external_data_provider(data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id)
dams_cli.assign_data_source_to_data_model(data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id)
dams_cli.assign_external_dataset_to_data_source(external_dataset_id=ds_id, data_source_id=ext_dsrc_id)
dams_cli.assign_external_dataset_to_agent_instance(external_dataset_id=ds_id, agent_instance_id=eda_inst_id)
# dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=self.eda_id, agent_instance_id=self.eda_inst_id)
#create temp streamdef so the data product can create the stream
pc_list = []
for pc_k, pc in self._create_parameter_dictionary().iteritems():
pc_list.append(dms_cli.create_parameter_context(pc_k, pc[1].dump()))
pdict_id = dms_cli.create_parameter_dictionary('slocum_param_dict', pc_list)
streamdef_id = pubsub_cli.create_stream_definition(name="slocum_stream_def", description="stream def for slocum testing", parameter_dictionary_id=pdict_id)
# dpms_cli.create_data_product()
# Generate the data product and associate it to the ExternalDataset
tdom, sdom = time_series_domain()
tdom, sdom = tdom.dump(), sdom.dump()
dprod = IonObject(RT.DataProduct,
name='slocum_parsed_product',
description='parsed slocum product',
temporal_domain=tdom,
spatial_domain=sdom)
dproduct_id = dpms_cli.create_data_product(data_product=dprod,
stream_definition_id=streamdef_id)
dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id)
stream_id, assn = rr_cli.find_objects(subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True)
stream_id = stream_id[0]
log.info('Created resources: {0}'.format({'ExternalDataset': ds_id, 'ExternalDataProvider': ext_dprov_id, 'DataSource': ext_dsrc_id, 'DataSourceModel': ext_dsrc_model_id, 'DataProducer': dproducer_id, 'DataProduct': dproduct_id, 'Stream': stream_id}))
# Create the logger for receiving publications
_, stream_route, _ = self.create_stream_and_logger(name='slocum', stream_id=stream_id)
self.EDA_RESOURCE_ID = ds_id
self.EDA_NAME = ds_name
self.DVR_CONFIG['dh_cfg'] = {
'TESTING': True,
'stream_id': stream_id,
'stream_route': stream_route,
'stream_def': streamdef_id,
'external_dataset_res': dset,
'data_producer_id': dproducer_id, # CBM: Should this be put in the main body of the config - with mod & cls?
'max_records': 20,
}
class TestDeployment(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.omsclient = ObservatoryManagementServiceClient(node=self.container.node)
self.imsclient = InstrumentManagementServiceClient(node=self.container.node)
self.dmpsclient = DataProductManagementServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.psmsclient = PubsubManagementServiceClient(node=self.container.node)
self.dataset_management = DatasetManagementServiceClient()
self.c = DotDict()
self.c.resource_registry = self.rrclient
self.RR2 = EnhancedResourceRegistryClient(self.rrclient)
self.dsmsclient = DataProcessManagementServiceClient(node=self.container.node)
# deactivate all data processes when tests are complete
def killAllDataProcesses():
for proc_id in self.rrclient.find_resources(RT.DataProcess, None, None, True)[0]:
self.dsmsclient.deactivate_data_process(proc_id)
self.dsmsclient.delete_data_process(proc_id)
self.addCleanup(killAllDataProcesses)
#@unittest.skip("targeting")
def test_create_deployment(self):
#create a deployment with metadata and an initial site and device
platform_site__obj = IonObject(RT.PlatformSite,
name='PlatformSite1',
description='test platform site')
site_id = self.omsclient.create_platform_site(platform_site__obj)
platform_device__obj = IonObject(RT.PlatformDevice,
name='PlatformDevice1',
description='test platform device')
device_id = self.imsclient.create_platform_device(platform_device__obj)
start = IonTime(datetime.datetime(2013,1,1))
end = IonTime(datetime.datetime(2014,1,1))
temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string())
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment',
constraint_list=[temporal_bounds])
deployment_id = self.omsclient.create_deployment(deployment_obj)
self.omsclient.deploy_platform_site(site_id, deployment_id)
self.imsclient.deploy_platform_device(device_id, deployment_id)
log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) )
#retrieve the deployment objects and check that the assoc site and device are attached
read_deployment_obj = self.omsclient.read_deployment(deployment_id)
log.debug("test_create_deployment: created deployment obj: %s ", str(read_deployment_obj) )
site_ids, _ = self.rrclient.find_subjects(RT.PlatformSite, PRED.hasDeployment, deployment_id, True)
self.assertEqual(len(site_ids), 1)
device_ids, _ = self.rrclient.find_subjects(RT.PlatformDevice, PRED.hasDeployment, deployment_id, True)
self.assertEqual(len(device_ids), 1)
#delete the deployment
self.RR2.pluck(deployment_id)
self.omsclient.force_delete_deployment(deployment_id)
# now try to get the deleted dp object
try:
self.omsclient.read_deployment(deployment_id)
except NotFound:
pass
else:
self.fail("deleted deployment was found during read")
#@unittest.skip("targeting")
def test_prepare_deployment_support(self):
deploy_sup = self.omsclient.prepare_deployment_support()
self.assertTrue(deploy_sup)
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].type_, "AssocDeploymentInstDevice")
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].associated_resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasPlatformDevice'].type_, "AssocDeploymentPlatDevice")
self.assertEquals(deploy_sup.associations['DeploymentHasPlatformDevice'].resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasPlatformDevice'].associated_resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].type_, "AssocDeploymentInstSite")
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].associated_resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasPlatformSite'].type_, "AssocDeploymentPlatSite")
self.assertEquals(deploy_sup.associations['DeploymentHasPlatformSite'].resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasPlatformSite'].associated_resources, [])
#create a deployment with metadata and an initial site and device
platform_site__obj = IonObject(RT.PlatformSite,
name='PlatformSite1',
description='test platform site')
site_id = self.omsclient.create_platform_site(platform_site__obj)
platform_device__obj = IonObject(RT.PlatformDevice,
name='PlatformDevice1',
description='test platform device')
device_id = self.imsclient.create_platform_device(platform_device__obj)
start = IonTime(datetime.datetime(2013,1,1))
end = IonTime(datetime.datetime(2014,1,1))
temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string())
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment',
constraint_list=[temporal_bounds])
deployment_id = self.omsclient.create_deployment(deployment_obj)
deploy_sup = self.omsclient.prepare_deployment_support(deployment_id)
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].associated_resources, [])
self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformDevice'].resources), 1)
self.assertEquals(deploy_sup.associations['DeploymentHasPlatformDevice'].associated_resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].associated_resources, [])
self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformSite'].resources), 1)
self.assertEquals(deploy_sup.associations['DeploymentHasPlatformSite'].associated_resources, [])
self.omsclient.assign_site_to_deployment(site_id, deployment_id)
self.omsclient.assign_device_to_deployment(device_id, deployment_id)
deploy_sup = self.omsclient.prepare_deployment_support(deployment_id)
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentDevice'].associated_resources, [])
self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformDevice'].resources), 1)
self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformDevice'].associated_resources), 1)
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].resources, [])
self.assertEquals(deploy_sup.associations['DeploymentHasInstrumentSite'].associated_resources, [])
self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformSite'].resources), 1)
self.assertEquals(len(deploy_sup.associations['DeploymentHasPlatformSite'].associated_resources), 1)
#delete the deployment
self.RR2.pluck(deployment_id)
self.omsclient.force_delete_deployment(deployment_id)
# now try to get the deleted dp object
try:
self.omsclient.read_deployment(deployment_id)
except NotFound:
pass
else:
self.fail("deleted deployment was found during read")
#@unittest.skip("targeting")
def base_activate_deployment(self):
#-------------------------------------------------------------------------------------
# Create platform site, platform device, platform model
#-------------------------------------------------------------------------------------
platform_site__obj = IonObject(RT.PlatformSite,
name='PlatformSite1',
description='test platform site')
platform_site_id = self.omsclient.create_platform_site(platform_site__obj)
platform_device_obj = IonObject(RT.PlatformDevice,
name='PlatformDevice1',
description='test platform device')
platform_device_id = self.imsclient.create_platform_device(platform_device_obj)
platform_model__obj = IonObject(RT.PlatformModel,
name='PlatformModel1',
description='test platform model')
platform_model_id = self.imsclient.create_platform_model(platform_model__obj)
#-------------------------------------------------------------------------------------
# Create instrument site
#-------------------------------------------------------------------------------------
instrument_site_obj = IonObject(RT.InstrumentSite,
name='InstrumentSite1',
description='test instrument site')
instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, platform_site_id)
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.psmsclient.create_stream_definition(name='SBE37_CDM', parameter_dictionary_id=pdict_id)
#----------------------------------------------------------------------------------------------------
# Create an instrument device
#----------------------------------------------------------------------------------------------------
instrument_device_obj = IonObject(RT.InstrumentDevice,
name='InstrumentDevice1',
description='test instrument device')
instrument_device_id = self.imsclient.create_instrument_device(instrument_device_obj)
self.rrclient.create_association(platform_device_id, PRED.hasDevice, instrument_device_id)
#----------------------------------------------------------------------------------------------------
# Create an instrument model
#----------------------------------------------------------------------------------------------------
instrument_model_obj = IonObject(RT.InstrumentModel,
name='InstrumentModel1',
description='test instrument model')
instrument_model_id = self.imsclient.create_instrument_model(instrument_model_obj)
#----------------------------------------------------------------------------------------------------
# Create a deployment object
#----------------------------------------------------------------------------------------------------
start = IonTime(datetime.datetime(2013,1,1))
end = IonTime(datetime.datetime(2014,1,1))
temporal_bounds = IonObject(OT.TemporalBounds, name='planned', start_datetime=start.to_string(), end_datetime=end.to_string())
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment',
context=IonObject(OT.CabledNodeDeploymentContext),
constraint_list=[temporal_bounds])
deployment_id = self.omsclient.create_deployment(deployment_obj)
log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) )
ret = DotDict(instrument_site_id=instrument_site_id,
instrument_device_id=instrument_device_id,
instrument_model_id=instrument_model_id,
platform_site_id=platform_site_id,
platform_device_id=platform_device_id,
platform_model_id=platform_model_id,
deployment_id=deployment_id)
return ret
#@unittest.skip("targeting")
def test_activate_deployment_normal(self):
res = self.base_activate_deployment()
log.debug("assigning platform and instrument models")
self.imsclient.assign_platform_model_to_platform_device(res.platform_model_id, res.platform_device_id)
self.imsclient.assign_instrument_model_to_instrument_device(res.instrument_model_id, res.instrument_device_id)
self.omsclient.assign_platform_model_to_platform_site(res.platform_model_id, res.platform_site_id)
self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id)
log.debug("adding instrument site and device to deployment")
self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id)
self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id)
log.debug("adding platform site and device to deployment")
self.omsclient.deploy_platform_site(res.platform_site_id, res.deployment_id)
self.imsclient.deploy_platform_device(res.platform_device_id, res.deployment_id)
log.debug("activating deployment, expecting success")
self.omsclient.activate_deployment(res.deployment_id)
log.debug("deactivatin deployment, expecting success")
self.omsclient.deactivate_deployment(res.deployment_id)
#@unittest.skip("targeting")
def test_activate_deployment_nomodels(self):
res = self.base_activate_deployment()
self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id)
self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id)
log.debug("activating deployment without site+device models, expecting fail")
self.assert_deploy_fail(res.deployment_id, NotFound, "Expected 1")
log.debug("assigning instrument site model")
self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id)
log.debug("activating deployment without device models, expecting fail")
self.assert_deploy_fail(res.deployment_id, NotFound, "Expected 1")
#@unittest.skip("targeting")
def test_activate_deployment_nosite(self):
res = self.base_activate_deployment()
log.debug("assigning instrument models")
self.imsclient.assign_instrument_model_to_instrument_device(res.instrument_model_id, res.instrument_device_id)
self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id)
log.debug("deploying instrument device only")
self.imsclient.deploy_instrument_device(res.instrument_device_id, res.deployment_id)
log.debug("activating deployment without instrument site, expecting fail")
self.assert_deploy_fail(res.deployment_id, BadRequest, "Devices in this deployment outnumber sites")
#@unittest.skip("targeting")
def test_activate_deployment_nodevice(self):
res = self.base_activate_deployment()
log.debug("assigning platform and instrument models")
self.imsclient.assign_instrument_model_to_instrument_device(res.instrument_model_id, res.instrument_device_id)
self.omsclient.assign_instrument_model_to_instrument_site(res.instrument_model_id, res.instrument_site_id)
log.debug("deploying instrument site only")
self.omsclient.deploy_instrument_site(res.instrument_site_id, res.deployment_id)
log.debug("activating deployment without device, expecting fail")
self.assert_deploy_fail(res.deployment_id, BadRequest, "No devices were found in the deployment")
def test_activate_deployment_asymmetric_children(self):
"""
P0
| \
P1 P2
|
I1
Complex deployment using CSP
P1, P2, and P3 share the same platform model. The CSP solver should be able to work this out
based on relationships to parents
"""
log.debug("create models")
imodel_id = self.RR2.create(any_old(RT.InstrumentModel))
pmodel_id = self.RR2.create(any_old(RT.PlatformModel))
log.debug("create devices")
idevice_id = self.RR2.create(any_old(RT.InstrumentDevice))
pdevice_id = [self.RR2.create(any_old(RT.PlatformDevice)) for _ in range(3)]
log.debug("create sites")
isite_id = self.RR2.create(any_old(RT.InstrumentSite))
psite_id = [self.RR2.create(any_old(RT.PlatformSite)) for _ in range(3)]
log.debug("assign models")
self.RR2.assign_instrument_model_to_instrument_device_with_has_model(imodel_id, idevice_id)
self.RR2.assign_instrument_model_to_instrument_site_with_has_model(imodel_id, isite_id)
for x in range(3):
self.RR2.assign_platform_model_to_platform_device_with_has_model(pmodel_id, pdevice_id[x])
self.RR2.assign_platform_model_to_platform_site_with_has_model(pmodel_id, psite_id[x])
log.debug("assign hierarchy")
self.RR2.assign_instrument_device_to_platform_device_with_has_device(idevice_id, pdevice_id[1])
self.RR2.assign_instrument_site_to_platform_site_with_has_site(isite_id, psite_id[1])
for x in range(1,3):
self.RR2.assign_platform_device_to_platform_device_with_has_device(pdevice_id[x], pdevice_id[0])
self.RR2.assign_platform_site_to_platform_site_with_has_site(psite_id[x], psite_id[0])
log.debug("create and activate deployment")
dep_id = self.RR2.create(any_old(RT.Deployment, {"context": IonObject(OT.RemotePlatformDeploymentContext)}))
self.RR2.assign_deployment_to_platform_device_with_has_deployment(dep_id, pdevice_id[0])
self.RR2.assign_deployment_to_platform_site_with_has_deployment(dep_id, psite_id[0])
self.omsclient.activate_deployment(dep_id)
log.debug("verifying deployment")
self.assertEqual(idevice_id, self.RR2.find_instrument_device_id_of_instrument_site_using_has_device(isite_id),
"The instrument device was not assigned to the instrument site")
for x in range(3):
self.assertEqual(pdevice_id[x], self.RR2.find_platform_device_id_of_platform_site_using_has_device(psite_id[x]),
"Platform device %d was not assigned to platform site %d" % (x, x))
def assert_deploy_fail(self, deployment_id, err_type=BadRequest, fail_message="did not specify fail_message"):
with self.assertRaises(err_type) as cm:
self.omsclient.activate_deployment(deployment_id)
self.assertIn(fail_message, cm.exception.message)
def test_3x3_matchups_remoteplatform(self):
self.base_3x3_matchups(IonObject(OT.RemotePlatformDeploymentContext))
def test_3x3_matchups_cabledinstrument(self):
self.base_3x3_matchups(IonObject(OT.CabledInstrumentDeploymentContext))
def test_3x3_matchups_cablednode(self):
self.base_3x3_matchups(IonObject(OT.CabledNodeDeploymentContext))
def base_3x3_matchups(self, deployment_context):
"""
This will be 1 root platform, 3 sub platforms (2 of one model, 1 of another) and 3 sub instruments each (2-to-1)
"""
deployment_context_type = type(deployment_context).__name__
instrument_model_id = [self.RR2.create(any_old(RT.InstrumentModel)) for _ in range(6)]
platform_model_id = [self.RR2.create(any_old(RT.PlatformModel)) for _ in range(3)]
instrument_device_id = [self.RR2.create(any_old(RT.InstrumentDevice)) for _ in range(9)]
platform_device_id = [self.RR2.create(any_old(RT.PlatformDevice)) for _ in range(4)]
instrument_site_id = [self.RR2.create(any_old(RT.InstrumentSite,
{"planned_uplink_port":
IonObject(OT.PlatformPort,
reference_designator="instport_%d" % (i+1))}))
for i in range(9)]
platform_site_id = [self.RR2.create(any_old(RT.PlatformSite,
{"planned_uplink_port":
IonObject(OT.PlatformPort,
reference_designator="platport_%d" % (i+1))}))
for i in range(4)]
def instrument_model_at(platform_idx, instrument_idx):
m = platform_idx * 2
if instrument_idx > 0:
m += 1
return m
def platform_model_at(platform_idx):
if platform_idx > 0:
return 1
return 0
def instrument_at(platform_idx, instrument_idx):
return platform_idx * 3 + instrument_idx
# set up the structure
for p in range(3):
m = platform_model_at(p)
self.RR2.assign_platform_model_to_platform_site_with_has_model(platform_model_id[m], platform_site_id[p])
self.RR2.assign_platform_model_to_platform_device_with_has_model(platform_model_id[m], platform_device_id[p])
self.RR2.assign_platform_device_to_platform_device_with_has_device(platform_device_id[p], platform_device_id[3])
self.RR2.assign_platform_site_to_platform_site_with_has_site(platform_site_id[p], platform_site_id[3])
for i in range(3):
m = instrument_model_at(p, i)
idx = instrument_at(p, i)
self.RR2.assign_instrument_model_to_instrument_site_with_has_model(instrument_model_id[m], instrument_site_id[idx])
self.RR2.assign_instrument_model_to_instrument_device_with_has_model(instrument_model_id[m], instrument_device_id[idx])
self.RR2.assign_instrument_device_to_platform_device_with_has_device(instrument_device_id[idx], platform_device_id[p])
self.RR2.assign_instrument_site_to_platform_site_with_has_site(instrument_site_id[idx], platform_site_id[p])
# top level models
self.RR2.assign_platform_model_to_platform_device_with_has_model(platform_model_id[2], platform_device_id[3])
self.RR2.assign_platform_model_to_platform_site_with_has_model(platform_model_id[2], platform_site_id[3])
# verify structure
for p in range(3):
parent_id = self.RR2.find_platform_device_id_by_platform_device_using_has_device(platform_device_id[p])
self.assertEqual(platform_device_id[3], parent_id)
parent_id = self.RR2.find_platform_site_id_by_platform_site_using_has_site(platform_site_id[p])
self.assertEqual(platform_site_id[3], parent_id)
for i in range(len(platform_site_id)):
self.assertEqual(self.RR2.find_platform_model_of_platform_device_using_has_model(platform_device_id[i]),
self.RR2.find_platform_model_of_platform_site_using_has_model(platform_site_id[i]))
for i in range(len(instrument_site_id)):
self.assertEqual(self.RR2.find_instrument_model_of_instrument_device_using_has_model(instrument_device_id[i]),
self.RR2.find_instrument_model_of_instrument_site_using_has_model(instrument_site_id[i]))
port_assignments = {}
for p in range(3):
port_assignments[platform_device_id[p]] = "platport_%d" % (p+1)
for i in range(3):
idx = instrument_at(p, i)
port_assignments[instrument_device_id[idx]] = "instport_%d" % (idx+1)
deployment_id = self.RR2.create(any_old(RT.Deployment,
{"context": deployment_context,
"port_assignments": port_assignments}))
log.debug("assigning device/site to %s deployment", deployment_context_type)
if OT.RemotePlatformDeploymentContext == deployment_context_type:
self.RR2.assign_deployment_to_platform_device_with_has_deployment(deployment_id, platform_device_id[3])
self.RR2.assign_deployment_to_platform_site_with_has_deployment(deployment_id, platform_site_id[3])
elif OT.CabledInstrumentDeploymentContext == deployment_context_type:
self.RR2.assign_deployment_to_instrument_device_with_has_deployment(deployment_id, instrument_device_id[1])
self.RR2.assign_deployment_to_instrument_site_with_has_deployment(deployment_id, instrument_site_id[1])
elif OT.CabledNodeDeploymentContext == deployment_context_type:
self.RR2.assign_deployment_to_platform_device_with_has_deployment(deployment_id, platform_device_id[1])
self.RR2.assign_deployment_to_platform_site_with_has_deployment(deployment_id, platform_site_id[1])
log.debug("activation of %s deployment", deployment_context_type)
self.omsclient.activate_deployment(deployment_id)
log.debug("validation of %s deployment", deployment_context_type)
if OT.RemotePlatformDeploymentContext == deployment_context_type:
# verify proper associations
for i, d in enumerate(platform_device_id):
self.assertEqual(d, self.RR2.find_platform_device_id_of_platform_site_using_has_device(platform_site_id[i]))
for i, d in enumerate(instrument_device_id):
self.assertEqual(d, self.RR2.find_instrument_device_id_of_instrument_site_using_has_device(instrument_site_id[i]))
elif OT.CabledInstrumentDeploymentContext == deployment_context_type:
self.assertEqual(instrument_device_id[1],
self.RR2.find_instrument_device_id_of_instrument_site_using_has_device(instrument_site_id[1]))
elif OT.CabledNodeDeploymentContext == deployment_context_type:
expected_platforms = [1]
expected_instruments = [3, 4, 5]
# verify proper associations
for i, d in enumerate(platform_device_id):
self.assertEqual(i in expected_platforms,
d in self.RR2.find_platform_device_ids_of_platform_site_using_has_device(platform_site_id[i]))
for i, d in enumerate(instrument_device_id):
self.assertEqual(i in expected_instruments,
d in self.RR2.find_instrument_device_ids_of_instrument_site_using_has_device(instrument_site_id[i]))
class EventManagementIntTest(IonIntegrationTestCase):
def setUp(self):
super(EventManagementIntTest, self).setUp()
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.event_management = EventManagementServiceClient()
self.rrc = ResourceRegistryServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.pubsub = PubsubManagementServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
self.queue_cleanup = []
self.exchange_cleanup = []
def tearDown(self):
for queue in self.queue_cleanup:
xn = self.container.ex_manager.create_xn_queue(queue)
xn.delete()
for exchange in self.exchange_cleanup:
xp = self.container.ex_manager.create_xp(exchange)
xp.delete()
def test_create_read_update_delete_event_type(self):
"""
Test that the CRUD method for event types work correctly
"""
event_type = EventType(name="an event type")
event_type.origin = 'instrument_1'
# create
event_type_id = self.event_management.create_event_type(event_type)
self.assertIsNotNone(event_type_id)
# read
read_event_type = self.event_management.read_event_type(event_type_id)
self.assertEquals(read_event_type.name, event_type.name)
self.assertEquals(read_event_type.origin, event_type.origin)
#update
read_event_type.origin = 'instrument_2'
read_event_type.producer = 'producer'
self.event_management.update_event_type(read_event_type)
updated_event_type = self.event_management.read_event_type(event_type_id)
self.assertEquals(updated_event_type.origin, 'instrument_2')
self.assertEquals(updated_event_type.producer, 'producer')
# delete
self.event_management.delete_event_type(event_type_id)
with self.assertRaises(NotFound):
self.event_management.read_event_type(event_type_id)
def test_create_read_update_delete_event_process_definition(self):
"""
Test that the CRUD methods for the event process definitions work correctly
"""
# Create
module = 'ion.processes.data.transforms.event_alert_transform'
class_name = 'EventAlertTransform'
procdef_id = self.event_management.create_event_process_definition(version='ver_1',
module=module,
class_name=class_name,
uri='http://hare.com',
arguments=['arg1', 'arg2'],
event_types=['ExampleDetectableEvent', 'type_2'],
sub_types=['sub_type_1'])
# Read
read_process_def = self.event_management.read_event_process_definition(procdef_id)
self.assertEquals(read_process_def.executable['module'], module)
self.assertEquals(read_process_def.executable['class'], class_name)
# Update
self.event_management.update_event_process_definition(event_process_definition_id=procdef_id,
class_name='StreamAlertTransform',
arguments=['arg3', 'arg4'],
event_types=['event_type_new'])
updated_event_process_def = self.event_management.read_event_process_definition(procdef_id)
self.assertEquals(updated_event_process_def.executable['class'], 'StreamAlertTransform')
self.assertEquals(updated_event_process_def.arguments, ['arg3', 'arg4'])
definition = updated_event_process_def.definition
self.assertEquals(updated_event_process_def.definition.event_types, ['event_type_new'])
# Delete
self.event_management.delete_event_process_definition(procdef_id)
with self.assertRaises(NotFound):
self.event_management.read_event_process_definition(procdef_id)
def test_event_in_stream_out_transform(self):
"""
Test the event-in/stream-out transform
"""
stream_id, _ = self.pubsub.create_stream('test_stream', exchange_point='science_data')
self.exchange_cleanup.append('science_data')
#---------------------------------------------------------------------------------------------
# Launch a ctd transform
#---------------------------------------------------------------------------------------------
# Create the process definition
process_definition = ProcessDefinition(
name='EventToStreamTransform',
description='For testing an event-in/stream-out transform')
process_definition.executable['module']= 'ion.processes.data.transforms.event_in_stream_out_transform'
process_definition.executable['class'] = 'EventToStreamTransform'
proc_def_id = self.process_dispatcher.create_process_definition(process_definition=process_definition)
# Build the config
config = DotDict()
config.process.queue_name = 'test_queue'
config.process.exchange_point = 'science_data'
config.process.publish_streams.output = stream_id
config.process.event_type = 'ExampleDetectableEvent'
config.process.variables = ['voltage', 'temperature' ]
# Schedule the process
pid = self.process_dispatcher.schedule_process(process_definition_id=proc_def_id, configuration=config)
self.addCleanup(self.process_dispatcher.cancel_process,pid)
#---------------------------------------------------------------------------------------------
# Create a subscriber for testing
#---------------------------------------------------------------------------------------------
ar_cond = gevent.event.AsyncResult()
def subscriber_callback(m, r, s):
ar_cond.set(m)
sub = StandaloneStreamSubscriber('sub', subscriber_callback)
self.addCleanup(sub.stop)
sub_id = self.pubsub.create_subscription('subscription_cond',
stream_ids=[stream_id],
exchange_name='sub')
self.pubsub.activate_subscription(sub_id)
self.queue_cleanup.append(sub.xn.queue)
sub.start()
gevent.sleep(4)
#---------------------------------------------------------------------------------------------
# Publish an event. The transform has been configured to receive this event
#---------------------------------------------------------------------------------------------
event_publisher = EventPublisher("ExampleDetectableEvent")
event_publisher.publish_event(origin = 'fake_origin', voltage = '5', temperature = '273')
# Assert that the transform processed the event and published data on the output stream
result_cond = ar_cond.get(timeout=10)
self.assertTrue(result_cond)
def test_create_read_delete_event_process(self):
"""
Test that the CRUD methods for the event processes work correctly
"""
#---------------------------------------------------------------------------------------------
# Create a process definition
#---------------------------------------------------------------------------------------------
# Create
module = 'ion.processes.data.transforms.event_alert_transform'
class_name = 'EventAlertTransform'
procdef_id = self.event_management.create_event_process_definition(version='ver_1',
module=module,
class_name=class_name,
uri='http://hare.com',
arguments=['arg1', 'arg2'],
event_types=['ExampleDetectableEvent', 'type_2'],
sub_types=['sub_type_1'])
# Read
read_process_def = self.event_management.read_event_process_definition(procdef_id)
self.assertEquals(read_process_def.arguments, ['arg1', 'arg2'])
#---------------------------------------------------------------------------------------------
# Use the process definition to create a process
#---------------------------------------------------------------------------------------------
# Create a stream
param_dict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True)
stream_def_id = self.pubsub.create_stream_definition('cond_stream_def', parameter_dictionary_id=param_dict_id)
tdom, sdom = time_series_domain()
tdom, sdom = tdom.dump(), sdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
# Create a data product
data_product_id = self.data_product_management.create_data_product(data_product=dp_obj, stream_definition_id=stream_def_id)
output_products = {}
output_products['conductivity'] = data_product_id
# Create an event process
event_process_id = self.event_management.create_event_process( process_definition_id=procdef_id,
event_types=['ExampleDetectableEvent','DetectionEvent'],
sub_types=['s1', 's2'],
origins=['or_1', 'or_2'],
origin_types=['or_t1', 'or_t2'],
out_data_products = output_products)
self.addCleanup(self.process_dispatcher.cancel_process, event_process_id)
#---------------------------------------------------------------------------------------------
# Read the event process object and make assertions
#--------------------------------------------------------------------------------------------- out_data_products={'conductivity': data_product_id})
event_process_obj = self.event_management.read_event_process(event_process_id=event_process_id)
# Get the stream associated with the data product for the sake of making assertions
stream_ids, _ = self.rrc.find_objects(data_product_id, PRED.hasStream, id_only=True)
stream_id = stream_ids[0]
# Assertions!
self.assertEquals(event_process_obj.detail.output_streams['conductivity'], stream_id)
self.assertEquals(event_process_obj.detail.event_types, ['ExampleDetectableEvent', 'DetectionEvent'])
self.assertEquals(event_process_obj.detail.sub_types, ['s1', 's2'])
self.assertEquals(event_process_obj.detail.origins, ['or_1', 'or_2'])
self.assertEquals(event_process_obj.detail.origin_types, ['or_t1', 'or_t2'])
