class TestDataProductManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dpsc_cli = DataProductManagementServiceClient(node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.unsc = UserNotificationServiceClient()
self.data_retriever = DataRetrieverServiceClient()
#------------------------------------------
# Create the environment
#------------------------------------------
datastore_name = CACHE_DATASTORE_NAME
self.db = self.container.datastore_manager.get_datastore(datastore_name)
self.stream_def_id = self.pubsubcli.create_stream_definition(name='SBE37_CDM')
self.process_definitions = {}
ingestion_worker_definition = ProcessDefinition(name='ingestion worker')
ingestion_worker_definition.executable = {
'module':'ion.processes.data.ingestion.science_granule_ingestion_worker',
'class' :'ScienceGranuleIngestionWorker'
}
process_definition_id = self.process_dispatcher.create_process_definition(process_definition=ingestion_worker_definition)
self.process_definitions['ingestion_worker'] = process_definition_id
self.pids = []
self.exchange_points = []
self.exchange_names = []
#------------------------------------------------------------------------------------------------
# First launch the ingestors
#------------------------------------------------------------------------------------------------
self.exchange_space = 'science_granule_ingestion'
self.exchange_point = 'science_data'
config = DotDict()
config.process.datastore_name = 'datasets'
config.process.queue_name = self.exchange_space
self.exchange_names.append(self.exchange_space)
self.exchange_points.append(self.exchange_point)
pid = self.process_dispatcher.schedule_process(self.process_definitions['ingestion_worker'],configuration=config)
log.debug("the ingestion worker process id: %s", pid)
self.pids.append(pid)
self.addCleanup(self.cleaning_up)
def cleaning_up(self):
for pid in self.pids:
log.debug("number of pids to be terminated: %s", len(self.pids))
try:
self.process_dispatcher.cancel_process(pid)
log.debug("Terminated the process: %s", pid)
except:
log.debug("could not terminate the process id: %s" % pid)
IngestionManagementIntTest.clean_subscriptions()
for xn in self.exchange_names:
xni = self.container.ex_manager.create_xn_queue(xn)
xni.delete()
for xp in self.exchange_points:
xpi = self.container.ex_manager.create_xp(xp)
xpi.delete()
def get_datastore(self, dataset_id):
dataset = self.dataset_management.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def test_create_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
parameter_dictionary_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict')
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=parameter_dictionary_id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
# Generic time-series data domain creation
tdom, sdom = time_series_domain()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom.dump(),
spatial_domain = sdom.dump())
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 200.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = 100.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 50.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = 100.0
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product( data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
self.dpsc_cli.activate_data_product_persistence(dp_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
self.assertEquals(dp_obj.geospatial_point_center.lat, 150.0)
log.debug('Created data product %s', dp_obj)
#------------------------------------------------------------------------------------------------
# test creating a new data product with a stream definition
#------------------------------------------------------------------------------------------------
log.debug('Creating new data product with a stream definition')
dp_obj = IonObject(RT.DataProduct,
name='DP2',
description='some new dp',
temporal_domain = tdom.dump(),
spatial_domain = sdom.dump())
dp_id2 = self.dpsc_cli.create_data_product(dp_obj, ctd_stream_def_id)
self.dpsc_cli.activate_data_product_persistence(dp_id2)
log.debug('new dp_id = %s' % dp_id2)
#------------------------------------------------------------------------------------------------
#make sure data product is associated with stream def
#------------------------------------------------------------------------------------------------
streamdefs = []
streams, _ = self.rrclient.find_objects(dp_id2, PRED.hasStream, RT.Stream, True)
for s in streams:
log.debug("Checking stream %s" % s)
sdefs, _ = self.rrclient.find_objects(s, PRED.hasStreamDefinition, RT.StreamDefinition, True)
for sd in sdefs:
log.debug("Checking streamdef %s" % sd)
streamdefs.append(sd)
self.assertIn(ctd_stream_def_id, streamdefs)
# test reading a non-existent data product
log.debug('reading non-existent data product')
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product('some_fake_id')
# update a data product (tests read also)
log.debug('Updating data product')
# first get the existing dp object
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# now tweak the object
dp_obj.description = 'the very first dp'
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 300.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = 200.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 150.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = 200.0
# now write the dp back to the registry
update_result = self.dpsc_cli.update_data_product(dp_obj)
# now get the dp back to see if it was updated
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertEquals(dp_obj.description,'the very first dp')
self.assertEquals(dp_obj.geospatial_point_center.lat, 250.0)
log.debug('Updated data product %s', dp_obj)
#test extension
extended_product = self.dpsc_cli.get_data_product_extension(dp_id)
self.assertEqual(dp_id, extended_product._id)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_product.computed.product_download_size_estimated.status)
self.assertEqual(0, extended_product.computed.product_download_size_estimated.value)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_product.computed.parameters.status)
#log.debug("test_create_data_product: parameters %s" % extended_product.computed.parameters.value)
# now 'delete' the data product
log.debug("deleting data product: %s" % dp_id)
self.dpsc_cli.delete_data_product(dp_id)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# Get the events corresponding to the data product
ret = self.unsc.get_recent_events(resource_id=dp_id)
events = ret.value
for event in events:
log.debug("event time: %s" % event.ts_created)
self.assertTrue(len(events) > 0)
def test_data_product_stream_def(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
stream_def_id = self.dpsc_cli.get_data_product_stream_definition(dp_id)
self.assertEquals(ctd_stream_def_id, stream_def_id)
def test_activate_suspend_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
log.debug("Created an IonObject for a data product: %s" % dp_obj)
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test activate and suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.activate_data_product_persistence(dp_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True)
if not dataset_ids:
raise NotFound("Data Product %s dataset does not exist" % str(dp_id))
self.get_datastore(dataset_ids[0])
# Check that the streams associated with the data product are persisted with
stream_ids, _ = self.rrclient.find_objects(dp_id,PRED.hasStream,RT.Stream,True)
for stream_id in stream_ids:
self.assertTrue(self.ingestclient.is_persisted(stream_id))
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data = self.data_retriever.retrieve(dataset_ids[0])
self.assertIsInstance(replay_data, Granule)
log.debug("The data retriever was able to replay the dataset that was attached to the data product "
"we wanted to be persisted. Therefore the data product was indeed persisted with "
"otherwise we could not have retrieved its dataset using the data retriever. Therefore "
"this demonstration shows that L4-CI-SA-RQ-267 is satisfied: 'Data product management shall persist data products'")
data_product_object = self.rrclient.read(dp_id)
self.assertEquals(data_product_object.name,'DP1')
self.assertEquals(data_product_object.description,'some new dp')
log.debug("Towards L4-CI-SA-RQ-308: 'Data product management shall persist data product metadata'. "
" Attributes in create for the data product obj, name= '%s', description='%s', match those of object from the "
"resource registry, name='%s', desc='%s'" % (dp_obj.name, dp_obj.description,data_product_object.name,
data_product_object.description))
#------------------------------------------------------------------------------------------------
# test suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.suspend_data_product_persistence(dp_id)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
with self.assertRaises(NotFound):
dp_obj = self.rrclient.read(dp_id)
class TestLoader(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.ingestion_management = IngestionManagementServiceClient()
self.rr = self.container.resource_registry
def _perform_preload(self, load_cfg):
#load_cfg["ui_path"] = "res/preload/r2_ioc/ui_assets"
#load_cfg["path"] = "R2PreloadedResources.xlsx"
#load_cfg["assetmappings"] = "OOIPreload.xlsx"
self.container.spawn_process("Loader",
"ion.processes.bootstrap.ion_loader",
"IONLoader",
config=load_cfg)
def _preload_instrument(self, inst_scenario):
load_cfg = dict(
op="load",
scenario=inst_scenario,
attachments="res/preload/r2_ioc/attachments",
assets='res/preload/r2_ioc/ooi_assets',
)
self._perform_preload(load_cfg)
def _preload_ui(self, ui_path="default"):
load_cfg = dict(
op="load",
loadui=True,
ui_path=ui_path,
)
self._perform_preload(load_cfg)
def _preload_cfg(self, cfg, path=TEST_PATH):
load_cfg = dict(cfg=cfg, path=path)
self._perform_preload(load_cfg)
def _preload_scenario(self,
scenario,
path=TEST_PATH,
idmap=False,
**kwargs):
load_cfg = dict(op="load",
scenario=scenario,
attachments="res/preload/r2_ioc/attachments",
path=path,
idmap=idmap)
load_cfg.update(kwargs)
self._perform_preload(load_cfg)
def _preload_ooi(self, path=TEST_PATH):
load_cfg = dict(
op="load",
loadooi=True,
assets="res/preload/r2_ioc/ooi_assets",
path=path,
ooiuntil="12/31/2013",
)
self._perform_preload(load_cfg)
# -------------------------------------------------------------------------
@attr('PRELOAD')
def test_ui_valid(self):
""" make sure UI assets are valid using DEFAULT_UI_ASSETS = 'http://userexperience.oceanobservatories.org/database-exports/Stable' """
self._preload_ui(ui_path='default')
obj_list, _ = self.rr.find_resources(restype=RT.UISpec,
name="ION UI Specs",
id_only=False)
self.assertEquals(len(obj_list), 1)
@attr('PRELOAD')
def test_ui_candidates_valid(self):
""" make sure UI assets are valid using DEFAULT_UI_ASSETS = 'http://userexperience.oceanobservatories.org/database-exports/Candidates' """
self._preload_ui(ui_path='candidate')
obj_list, _ = self.rr.find_resources(restype=RT.UISpec,
name="ION UI Specs",
id_only=False)
self.assertEquals(len(obj_list), 1)
@attr('PRELOAD')
def test_betademo_valid(self):
""" make sure can load asset DB """
self._preload_scenario("BETA,R2_DEMO,RSN_OMS", path=TEST_PATH)
self._preload_ooi(path=TEST_PATH)
# check that deployment port assignments subobject created correctly
#collect a set of deployments
deploy_list = []
#DEP3 of PDEV3
obj_list, _ = self.rr.find_resources(restype=RT.Deployment,
name="Platform Deployment",
id_only=False)
deploy_list.extend(obj_list)
log.debug('test_betademo_valid DEP3: %s ', obj_list)
#DEP4 of PDEV4
obj_list, _ = self.rr.find_resources(restype=RT.Deployment,
name="dep4",
id_only=False)
log.debug('test_betademo_valid DEP4: %s ', obj_list)
deploy_list.extend(obj_list)
self.assertEquals(len(deploy_list), 2)
for dply_obj in deploy_list:
for dev_id, platform_port in dply_obj.port_assignments.iteritems():
# all values in the port assignments dict should be PlatformPort objects
self.assertEquals(platform_port.type_, OT.PlatformPort)
@attr('PRELOAD')
def test_incremental(self):
""" make sure R2_DEMO scenario in master google doc
is valid and self-contained (doesn't rely on rows from other scenarios except BETA)
NOTE: test will pass/fail based on current google doc, not just code changes.
"""
self._preload_cfg("res/preload/r2_ioc/config/ooi_load_config.yml",
path=TEST_PATH)
self._preload_scenario("OOIR2_DEMO", path=TEST_PATH, idmap=True)
dp_list1, _ = self.rr.find_resources(restype=RT.DataProduct,
id_only=True)
ia_list1, _ = self.rr.find_resources(restype=RT.InstrumentAgent,
id_only=True)
self._preload_cfg("res/preload/r2_ioc/config/ooi_instruments.yml",
path=TEST_PATH)
ia_list2, _ = self.rr.find_resources(restype=RT.InstrumentAgent,
id_only=True)
self.assertGreater(len(ia_list2), len(ia_list1))
dp_list2, _ = self.rr.find_resources(restype=RT.DataProduct,
id_only=True)
self.assertGreater(len(dp_list2), len(dp_list1))
id_list2, _ = self.rr.find_resources(restype=RT.InstrumentDevice,
id_only=True)
self._preload_ooi(path=TEST_PATH)
dp_list3, _ = self.rr.find_resources(restype=RT.DataProduct,
id_only=True)
self.assertGreater(len(dp_list3), len(dp_list2))
id_list3, _ = self.rr.find_resources(restype=RT.InstrumentDevice,
id_only=True)
self.assertEquals(len(id_list3), len(id_list2))
self._preload_ooi(path=TEST_PATH)
dp_list4, _ = self.rr.find_resources(restype=RT.DataProduct,
id_only=True)
self.assertEquals(len(dp_list4), len(dp_list3))
id_list4, _ = self.rr.find_resources(restype=RT.InstrumentDevice,
id_only=True)
self.assertEquals(len(id_list4), len(id_list3))
def find_object_by_name(self, name, resource_type):
objects, _ = self.container.resource_registry.find_resources(
resource_type, name=name, id_only=False)
self.assertEquals(len(objects), 1)
return objects[0]
@attr('INT', group='loader')
@attr('SMOKE', group='loader')
def test_row_values(self):
""" use only rows from NOSE scenario for specific names and details included in this test.
rows in NOSE may rely on entries in BETA scenarios,
but should not specifically test values from those scenarios.
"""
# first make sure this scenario loads successfully
self._preload_scenario("BETA,NOSE")
# check for ExternalDataset
eds = self.find_object_by_name('Test External CTD Dataset',
RT.ExternalDataset)
edm1 = self.find_object_by_name('Test External CTD Dataset Model',
RT.ExternalDatasetModel)
edm2, _ = self.container.resource_registry.find_objects(
eds._id, PRED.hasModel, RT.ExternalDatasetModel, True)
self.assertEquals(edm1._id, edm2[0])
inst = self.find_object_by_name('Test External CTD Agent Instance',
RT.ExternalDatasetAgentInstance)
self.assertEquals('value1',
inst.driver_config['key1'],
msg='driver_config[key1] is not value1:\n%r' %
inst.driver_config)
# check for an Org
org = self.find_object_by_name('CASPER', RT.Org)
self.assertFalse(org.contacts is None)
self.assertEquals('Userbrough', org.contacts[0].individual_name_family)
self.assertEquals('primary', org.contacts[0].roles[0])
# check data product
dp = self.find_object_by_name('Test DP L0 CTD', RT.DataProduct)
# should be persisted
streams, _ = self.container.resource_registry.find_objects(
dp._id, PRED.hasStream, RT.Stream, True)
self.assertTrue(streams)
self.assertEquals(1, len(streams))
self.assertTrue(self.ingestion_management.is_persisted(streams[0]))
self.assertAlmostEqual(
32.88237,
dp.geospatial_bounds.geospatial_latitude_limit_north,
places=3)
# but L1 data product should not be persisted
dp = self.find_object_by_name('Test DP L1 conductivity',
RT.DataProduct)
streams, _ = self.container.resource_registry.find_objects(
dp._id, PRED.hasStream, RT.Stream, True)
self.assertEquals(1, len(streams))
self.assertTrue(streams)
self.assertFalse(self.ingestion_management.is_persisted(streams[0]))
site = self.find_object_by_name('Test Instrument Site',
RT.InstrumentSite)
self.assertFalse(site.constraint_list is None)
self.assertEquals(2, len(site.constraint_list))
con = site.constraint_list[0]
self.assertAlmostEqual(32.88237,
con.geospatial_latitude_limit_north,
places=3)
self.assertAlmostEqual(-117.23214,
con.geospatial_longitude_limit_east,
places=3)
con = site.constraint_list[1]
self.assertEquals('TemporalBounds', con.type_)
# check that coordinate system was loaded
self.assertFalse(site.coordinate_reference_system is None)
# check that InstrumentDevice contacts are loaded
dev = self.find_object_by_name('Unit Test SMB37', RT.InstrumentDevice)
self.assertTrue(len(dev.contacts) == 2)
self.assertEquals('Userbrough', dev.contacts[0].individual_name_family)
# check has attachments
attachments = self.container.resource_registry.find_attachments(
dev._id)
self.assertTrue(len(attachments) > 0)
# check for platform agents
agent = self.find_object_by_name('Unit Test Platform Agent',
RT.PlatformAgent)
self.assertEquals(2, len(agent.stream_configurations))
parsed = agent.stream_configurations[1]
# self.assertEquals('platform_eng_parsed', parsed.parameter_dictionary_name)
self.assertEquals('ctd_parsed_param_dict',
parsed.parameter_dictionary_name)
# OBSOLETE: check that alarm was added to StreamConfig
# self.assertEquals(1, len(parsed.alarms), msg='alarms: %r'%parsed.alarms)
# self.assertEquals('temp', parsed.alarms[0]['kwargs']['value_id'])
# check for platform agents
self.find_object_by_name('Unit Test Platform Agent Instance',
RT.PlatformAgentInstance)
# check for platform model boolean values
model = self.find_object_by_name('Nose Testing Platform Model',
RT.PlatformModel)
self.assertEquals(True, model.shore_networked)
self.assertNotEqual('str', model.shore_networked.__class__.__name__)
iai = self.find_object_by_name("Test InstrumentAgentInstance",
RT.InstrumentAgentInstance)
self.assertEqual(
{
'SCHEDULER': {
'VERSION': {
'number': 3.0
},
'CLOCK_SYNC': 48.2,
'ACQUIRE_STATUS': {}
},
'PARAMETERS': {
"TXWAVESTATS": False,
'TXWAVEBURST': 'false',
'TXREALTIME': True
}
}, iai.startup_config)
self.assertEqual(2, len(iai.alerts))
pai = self.find_object_by_name("Unit Test Platform Agent Instance",
RT.PlatformAgentInstance)
self.assertEqual(1, len(pai.alerts))
self.assertTrue(pai.agent_config.has_key('platform_config'))
log.debug('test_row_values PlatformAgentInstance driver_config: %s ',
pai.driver_config)
self.assertTrue(pai.driver_config.has_key('oms_uri'))
oms_uri = pai.driver_config['oms_uri']
log.debug('test_row_values PlatformAgentInstance oms_uri: %s ',
oms_uri)
self.assertEquals('http://*****:*****@10.180.80.10:9021/', oms_uri)
orgs, _ = self.container.resource_registry.find_subjects(
RT.Org, PRED.hasResource, iai._id, True)
self.assertEqual(1, len(orgs))
self.assertEqual(org._id, orgs[0])
entries, _ = self.container.resource_registry.find_resources(
RT.SchedulerEntry, id_only=False)
self.assertGreaterEqual(len(entries), 1)
@attr('PRELOAD')
def test_alpha_valid(self):
""" make sure R2_DEMO scenario in master google doc
is valid and self-contained (doesn't rely on rows from other scenarios except BETA)
NOTE: test will pass/fail based on current google doc, not just code changes.
"""
self._preload_cfg("res/preload/r2_ioc/config/ooi_alpha.yml",
path=TEST_PATH)
@attr('PRELOAD')
def test_beta_valid(self):
""" make sure R2_DEMO scenario in master google doc
is valid and self-contained (doesn't rely on rows from other scenarios except BETA)
NOTE: test will pass/fail based on current google doc, not just code changes.
"""
self._preload_cfg("res/preload/r2_ioc/config/ooi_beta.yml",
path=TEST_PATH)
failure_list = []
def add_failure(res_obj, msg):
fail_msg = "%s[%s/%s]: %s" % (res_obj.type_, res_obj._id,
res_obj.name, msg)
failure_list.append(fail_msg)
log.warn("Starting preload assertions now")
res_objs, res_keys = self.rr.find_resources_ext(alt_id_ns="PRE",
id_only=False)
log.info("Found %s preloaded resources", len(res_objs))
dp_objs = [res for res in res_objs if res.type_ == RT.DataProduct]
log.info("Checking %s DataProducts", len(dp_objs))
for dp in dp_objs:
pass
# Reenable this when we have geospatial coordinates for PNs
#if not all([dp.geospatial_bounds.geospatial_latitude_limit_north,
# dp.geospatial_bounds.geospatial_latitude_limit_south,
# dp.geospatial_bounds.geospatial_longitude_limit_east,
# dp.geospatial_bounds.geospatial_longitude_limit_west]):
# add_failure(dp, "geospatial_bounds location invalid: %s" % dp.geospatial_bounds)
#if not all([dp.geospatial_bounds.geospatial_vertical_min,
# dp.geospatial_bounds.geospatial_vertical_max]):
# add_failure(dp, "geospatial_bounds vertical invalid: %s" % dp.geospatial_bounds)
if failure_list:
fail_msg = "Preload assertions violated:\n" + "\n".join(
f for f in failure_list)
self.fail(fail_msg)
class TestDataProductManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dpsc_cli = DataProductManagementServiceClient()
self.rrclient = ResourceRegistryServiceClient()
self.damsclient = DataAcquisitionManagementServiceClient()
self.pubsubcli = PubsubManagementServiceClient()
self.ingestclient = IngestionManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.unsc = UserNotificationServiceClient()
self.data_retriever = DataRetrieverServiceClient()
self.identcli = IdentityManagementServiceClient()
#------------------------------------------
# Create the environment
#------------------------------------------
self.stream_def_id = self.pubsubcli.create_stream_definition(name='SBE37_CDM')
self.process_definitions = {}
ingestion_worker_definition = ProcessDefinition(name='ingestion worker')
ingestion_worker_definition.executable = {
'module':'ion.processes.data.ingestion.science_granule_ingestion_worker',
'class' :'ScienceGranuleIngestionWorker'
}
process_definition_id = self.process_dispatcher.create_process_definition(process_definition=ingestion_worker_definition)
self.process_definitions['ingestion_worker'] = process_definition_id
self.pids = []
self.exchange_points = []
self.exchange_names = []
#------------------------------------------------------------------------------------------------
# First launch the ingestors
#------------------------------------------------------------------------------------------------
self.exchange_space = 'science_granule_ingestion'
self.exchange_point = 'science_data'
config = DotDict()
config.process.datastore_name = 'datasets'
config.process.queue_name = self.exchange_space
self.exchange_names.append(self.exchange_space)
self.exchange_points.append(self.exchange_point)
pid = self.process_dispatcher.schedule_process(self.process_definitions['ingestion_worker'],configuration=config)
log.debug("the ingestion worker process id: %s", pid)
self.pids.append(pid)
self.addCleanup(self.cleaning_up)
def cleaning_up(self):
for pid in self.pids:
log.debug("number of pids to be terminated: %s", len(self.pids))
try:
self.process_dispatcher.cancel_process(pid)
log.debug("Terminated the process: %s", pid)
except:
log.debug("could not terminate the process id: %s" % pid)
IngestionManagementIntTest.clean_subscriptions()
for xn in self.exchange_names:
xni = self.container.ex_manager.create_xn_queue(xn)
xni.delete()
for xp in self.exchange_points:
xpi = self.container.ex_manager.create_xp(xp)
xpi.delete()
def get_datastore(self, dataset_id):
dataset = self.dataset_management.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
@attr('EXT')
@attr('PREP')
def test_create_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
parameter_dictionary = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict')
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=parameter_dictionary._id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp')
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 10.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -10.0
dp_obj.ooi_product_name = "PRODNAME"
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product( data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
# Assert that the data product has an associated stream at this stage
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True)
self.assertNotEquals(len(stream_ids), 0)
# Assert that the data product has an associated stream def at this stage
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStreamDefinition, RT.StreamDefinition, True)
self.assertNotEquals(len(stream_ids), 0)
self.dpsc_cli.activate_data_product_persistence(dp_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0)
log.debug('Created data product %s', dp_obj)
#------------------------------------------------------------------------------------------------
# test creating a new data product with a stream definition
#------------------------------------------------------------------------------------------------
log.debug('Creating new data product with a stream definition')
dp_obj = IonObject(RT.DataProduct,
name='DP2',
description='some new dp')
dp_id2 = self.dpsc_cli.create_data_product(dp_obj, ctd_stream_def_id)
self.dpsc_cli.activate_data_product_persistence(dp_id2)
log.debug('new dp_id = %s' % dp_id2)
#------------------------------------------------------------------------------------------------
#make sure data product is associated with stream def
#------------------------------------------------------------------------------------------------
streamdefs = []
streams, _ = self.rrclient.find_objects(dp_id2, PRED.hasStream, RT.Stream, True)
for s in streams:
log.debug("Checking stream %s" % s)
sdefs, _ = self.rrclient.find_objects(s, PRED.hasStreamDefinition, RT.StreamDefinition, True)
for sd in sdefs:
log.debug("Checking streamdef %s" % sd)
streamdefs.append(sd)
self.assertIn(ctd_stream_def_id, streamdefs)
group_names = self.dpsc_cli.get_data_product_group_list()
self.assertIn("PRODNAME", group_names)
#----------------------------------------------------------------------------------------
# Create users then notifications to this data product for each user
#----------------------------------------------------------------------------------------
# user_1
user_1 = UserInfo()
user_1.name = 'user_1'
user_1.contact.email = '*****@*****.**'
# user_2
user_2 = UserInfo()
user_2.name = 'user_2'
user_2.contact.email = '*****@*****.**'
#user1 is a complete user
self.subject = "/DC=org/DC=cilogon/C=US/O=ProtectNetwork/CN=Roger Unwin A254"
actor_identity_obj = IonObject("ActorIdentity", {"name": self.subject})
actor_id = self.identcli.create_actor_identity(actor_identity_obj)
user_credentials_obj = IonObject("UserCredentials", {"name": self.subject})
self.identcli.register_user_credentials(actor_id, user_credentials_obj)
user_id_1 = self.identcli.create_user_info(actor_id, user_1)
user_id_2, _ = self.rrclient.create(user_2)
delivery_config1a = IonObject(OT.DeliveryConfiguration, email='*****@*****.**', mode=DeliveryModeEnum.EMAIL, frequency=NotificationFrequencyEnum.BATCH)
delivery_config1b = IonObject(OT.DeliveryConfiguration, email='*****@*****.**', mode=DeliveryModeEnum.EMAIL, frequency=NotificationFrequencyEnum.BATCH)
notification_request_1 = NotificationRequest( name = "notification_1",
origin=dp_id,
origin_type="type_1",
event_type=OT.ResourceLifecycleEvent,
disabled_by_system = False,
delivery_configurations=[delivery_config1a, delivery_config1b])
delivery_config2a = IonObject(OT.DeliveryConfiguration, email='*****@*****.**', mode=DeliveryModeEnum.EMAIL, frequency=NotificationFrequencyEnum.BATCH)
delivery_config2b = IonObject(OT.DeliveryConfiguration, email='*****@*****.**', mode=DeliveryModeEnum.EMAIL, frequency=NotificationFrequencyEnum.BATCH)
notification_request_2 = NotificationRequest( name = "notification_2",
origin=dp_id,
origin_type="type_2",
disabled_by_system = False,
event_type=OT.DetectionEvent,
delivery_configurations=[delivery_config2a, delivery_config2b])
notification_request_1_id = self.unsc.create_notification(notification=notification_request_1, user_id=user_id_1)
notification_request_2_id = self.unsc.create_notification(notification=notification_request_2, user_id=user_id_2)
self.unsc.delete_notification(notification_request_1_id)
# test reading a non-existent data product
log.debug('reading non-existent data product')
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product('some_fake_id')
# update a data product (tests read also)
log.debug('Updating data product')
# first get the existing dp object
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# now tweak the object
dp_obj.description = 'the very first dp'
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 20.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -20.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 20.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -20.0
# now write the dp back to the registry
update_result = self.dpsc_cli.update_data_product(dp_obj)
# now get the dp back to see if it was updated
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertEquals(dp_obj.description,'the very first dp')
self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0)
log.debug('Updated data product %s', dp_obj)
#test extension
extended_product = self.dpsc_cli.get_data_product_extension(dp_id)
#validate that there is one active and one retired user notification for this data product
self.assertEqual(1, len(extended_product.computed.active_user_subscriptions.value))
self.assertEqual(1, len(extended_product.computed.past_user_subscriptions.value))
self.assertEqual(dp_id, extended_product._id)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_product.computed.product_download_size_estimated.status)
self.assertEqual(0, extended_product.computed.product_download_size_estimated.value)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_product.computed.parameters.status)
#log.debug("test_create_data_product: parameters %s" % extended_product.computed.parameters.value)
def ion_object_encoder(obj):
return obj.__dict__
#test prepare for create
data_product_data = self.dpsc_cli.prepare_data_product_support()
#print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2)
self.assertEqual(data_product_data._id, "")
self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport)
self.assertEqual(len(data_product_data.associations['StreamDefinition'].resources), 2)
self.assertEqual(len(data_product_data.associations['Dataset'].resources), 0)
self.assertEqual(len(data_product_data.associations['StreamDefinition'].associated_resources), 0)
self.assertEqual(len(data_product_data.associations['Dataset'].associated_resources), 0)
#test prepare for update
data_product_data = self.dpsc_cli.prepare_data_product_support(dp_id)
#print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2)
self.assertEqual(data_product_data._id, dp_id)
self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport)
self.assertEqual(len(data_product_data.associations['StreamDefinition'].resources), 2)
self.assertEqual(len(data_product_data.associations['Dataset'].resources), 1)
self.assertEqual(len(data_product_data.associations['StreamDefinition'].associated_resources), 1)
self.assertEqual(data_product_data.associations['StreamDefinition'].associated_resources[0].s, dp_id)
self.assertEqual(len(data_product_data.associations['Dataset'].associated_resources), 1)
self.assertEqual(data_product_data.associations['Dataset'].associated_resources[0].s, dp_id)
# now 'delete' the data product
log.debug("deleting data product: %s" % dp_id)
self.dpsc_cli.delete_data_product(dp_id)
# Assert that there are no associated streams leftover after deleting the data product
stream_ids, assoc_ids = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True)
self.assertEquals(len(stream_ids), 0)
self.assertEquals(len(assoc_ids), 0)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# Get the events corresponding to the data product
ret = self.unsc.get_recent_events(resource_id=dp_id)
events = ret.value
for event in events:
log.debug("event time: %s" % event.ts_created)
self.assertTrue(len(events) > 0)
def test_data_product_stream_def(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id)
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp')
dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
stream_def_id = self.dpsc_cli.get_data_product_stream_definition(dp_id)
self.assertEquals(ctd_stream_def_id, stream_def_id)
def test_derived_data_product(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='ctd parsed', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsubcli.delete_stream_definition, ctd_stream_def_id)
dp = DataProduct(name='Instrument DP')
dp_id = self.dpsc_cli.create_data_product(dp, stream_definition_id=ctd_stream_def_id)
self.addCleanup(self.dpsc_cli.force_delete_data_product, dp_id)
self.dpsc_cli.activate_data_product_persistence(dp_id)
self.addCleanup(self.dpsc_cli.suspend_data_product_persistence, dp_id)
dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True)
if not dataset_ids:
raise NotFound("Data Product %s dataset does not exist" % str(dp_id))
dataset_id = dataset_ids[0]
# Make the derived data product
simple_stream_def_id = self.pubsubcli.create_stream_definition(name='TEMPWAT stream def', parameter_dictionary_id=pdict_id, available_fields=['time','temp'])
tempwat_dp = DataProduct(name='TEMPWAT', category=DataProductTypeEnum.DERIVED)
tempwat_dp_id = self.dpsc_cli.create_data_product(tempwat_dp, stream_definition_id=simple_stream_def_id, parent_data_product_id=dp_id)
self.addCleanup(self.dpsc_cli.delete_data_product, tempwat_dp_id)
# Check that the streams associated with the data product are persisted with
stream_ids, _ = self.rrclient.find_objects(dp_id,PRED.hasStream,RT.Stream,True)
for stream_id in stream_ids:
self.assertTrue(self.ingestclient.is_persisted(stream_id))
stream_id = stream_ids[0]
route = self.pubsubcli.read_stream_route(stream_id=stream_id)
rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
rdt['pressure'] = np.arange(20)
publisher = StandaloneStreamPublisher(stream_id,route)
dataset_modified = Event()
def cb(*args, **kwargs):
dataset_modified.set()
es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id, auto_delete=True)
es.start()
self.addCleanup(es.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
tempwat_dataset_ids, _ = self.rrclient.find_objects(tempwat_dp_id, PRED.hasDataset, id_only=True)
tempwat_dataset_id = tempwat_dataset_ids[0]
granule = self.data_retriever.retrieve(tempwat_dataset_id, delivery_format=simple_stream_def_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt['time'], np.arange(20))
self.assertEquals(set(rdt.fields), set(['time','temp']))
def test_activate_suspend_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
# Construct temporal and spatial Coordinate Reference System objects
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp')
log.debug("Created an IonObject for a data product: %s" % dp_obj)
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# Subscribe to persist events
#------------------------------------------------------------------------------------------------
queue = gevent.queue.Queue()
def info_event_received(message, headers):
queue.put(message)
es = EventSubscriber(event_type=OT.InformationContentStatusEvent, callback=info_event_received, origin=dp_id, auto_delete=True)
es.start()
self.addCleanup(es.stop)
#------------------------------------------------------------------------------------------------
# test activate and suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.activate_data_product_persistence(dp_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True)
if not dataset_ids:
raise NotFound("Data Product %s dataset does not exist" % str(dp_id))
dataset_id = dataset_ids[0]
# Check that the streams associated with the data product are persisted with
stream_ids, _ = self.rrclient.find_objects(dp_id,PRED.hasStream,RT.Stream,True)
for stream_id in stream_ids:
self.assertTrue(self.ingestclient.is_persisted(stream_id))
stream_id = stream_ids[0]
route = self.pubsubcli.read_stream_route(stream_id=stream_id)
rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
publisher = StandaloneStreamPublisher(stream_id,route)
dataset_modified = Event()
def cb(*args, **kwargs):
dataset_modified.set()
es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id, auto_delete=True)
es.start()
self.addCleanup(es.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data = self.data_retriever.retrieve(dataset_ids[0])
self.assertIsInstance(replay_data, Granule)
log.debug("The data retriever was able to replay the dataset that was attached to the data product "
"we wanted to be persisted. Therefore the data product was indeed persisted with "
"otherwise we could not have retrieved its dataset using the data retriever. Therefore "
"this demonstration shows that L4-CI-SA-RQ-267 is satisfied: 'Data product management shall persist data products'")
data_product_object = self.rrclient.read(dp_id)
self.assertEquals(data_product_object.name,'DP1')
self.assertEquals(data_product_object.description,'some new dp')
log.debug("Towards L4-CI-SA-RQ-308: 'Data product management shall persist data product metadata'. "
" Attributes in create for the data product obj, name= '%s', description='%s', match those of object from the "
"resource registry, name='%s', desc='%s'" % (dp_obj.name, dp_obj.description,data_product_object.name,
data_product_object.description))
#------------------------------------------------------------------------------------------------
# test suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.suspend_data_product_persistence(dp_id)
dataset_modified.clear()
rdt['time'] = np.arange(20,40)
publisher.publish(rdt.to_granule())
self.assertFalse(dataset_modified.wait(2))
self.dpsc_cli.activate_data_product_persistence(dp_id)
dataset_modified.clear()
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_almost_equal(rdt['time'], np.arange(40))
dataset_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasDataset, id_only=True)
self.assertEquals(len(dataset_ids), 1)
self.dpsc_cli.suspend_data_product_persistence(dp_id)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
with self.assertRaises(NotFound):
dp_obj = self.rrclient.read(dp_id)
info_event_counter = 0
runtime = 0
starttime = time.time()
caught_events = []
#check that the four InfoStatusEvents were received
while info_event_counter < 4 and runtime < 60 :
a = queue.get(timeout=60)
caught_events.append(a)
info_event_counter += 1
runtime = time.time() - starttime
self.assertEquals(info_event_counter, 4)
class TestLoader(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.ingestion_management = IngestionManagementServiceClient()
def assert_can_load(self, scenarios, loadui=False, loadooi=False,
path=TESTED_DOC, ui_path='default'):
""" perform preload for given scenarios and raise exception if there is a problem with the data """
config = dict(op="load",
scenario=scenarios,
attachments="res/preload/r2_ioc/attachments",
loadui=loadui,
loadooi=loadooi,
path=path, ui_path=ui_path,
assets='res/preload/r2_ioc/ooi_assets',
bulk=loadooi,
ooiexclude='DataProduct,DataProductLink')
self.container.spawn_process("Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=config)
@attr('PRELOAD')
def test_ui_valid(self):
""" make sure UI assets are valid using DEFAULT_UI_ASSETS = 'https://userexperience.oceanobservatories.org/database-exports/' """
self.assert_can_load("BETA", loadui=True, ui_path='default')
@attr('PRELOAD')
def test_ui_candidates_valid(self):
""" make sure UI assets are valid using DEFAULT_UI_ASSETS = 'https://userexperience.oceanobservatories.org/database-exports/Candidates' """
self.assert_can_load("BETA", loadui=True, ui_path='candidate')
@attr('PRELOAD')
def test_assets_valid(self):
""" make sure can load asset DB """
self.assert_can_load("BETA,DEVS", path='master', loadooi=True)
@attr('PRELOAD')
def test_alpha_valid(self):
""" make sure R2_DEMO scenario in master google doc
is valid and self-contained (doesn't rely on rows from other scenarios except BETA)
NOTE: test will pass/fail based on current google doc, not just code changes.
"""
self.assert_can_load("BETA,ALPHA_SYS", path='master')
@attr('PRELOAD')
def test_beta_valid(self):
""" make sure R2_DEMO scenario in master google doc
is valid and self-contained (doesn't rely on rows from other scenarios except BETA)
NOTE: test will pass/fail based on current google doc, not just code changes.
"""
self.assert_can_load("BETA,BETA_SYS", path='master')
@attr('PRELOAD')
def test_devs_valid(self):
""" make sure DEVS scenario in master google doc
is valid and self-contained (doesn't rely on rows from other scenarios except BETA)
NOTE: test will pass/fail based on current google doc, not just code changes.
"""
self.assert_can_load("BETA,DEVS", path='master')
def find_object_by_name(self, name, resource_type):
objects,_ = self.container.resource_registry.find_resources(resource_type, id_only=False)
self.assertGreaterEqual(len(objects), 1)
filtered_objs = [obj for obj in objects if obj.name == name]
self.assertEquals(len(filtered_objs), 1)
return filtered_objs[0]
@attr('INT', group='loader')
@attr('SMOKE', group='loader')
def test_row_values(self):
""" use only rows from NOSE scenario for specific names and details included in this test.
rows in NOSE may rely on entries in BETA scenarios,
but should not specifically test values from those scenarios.
"""
# first make sure this scenario loads successfully
self.assert_can_load("BETA,NOSE")
# check for ExternalDataset
eds = self.find_object_by_name('Test External Dataset', RT.ExternalDataset)
edm1 = self.find_object_by_name('Test External Dataset Model', RT.ExternalDatasetModel)
edm2,_ = self.container.resource_registry.find_objects(eds._id, PRED.hasModel, RT.ExternalDatasetModel, True)
self.assertEquals(edm1._id, edm2[0])
inst = self.find_object_by_name('Test Instrument Agent Instance', RT.ExternalDatasetAgentInstance)
self.assertEquals('value1', inst.dataset_agent_config['key1'], msg='dataset_agent_config[key1] is not value1:\n%r'%inst.agent_config)
# check for an Org
org = self.find_object_by_name('CASPER', RT.Org)
self.assertFalse(org.contacts is None)
self.assertEquals('Userbrough', org.contacts[0].individual_name_family)
self.assertEquals('primary', org.contacts[0].roles[0])
# check data product
dp = self.find_object_by_name('Test DP L0 CTD', RT.DataProduct)
# should be persisted
streams, _ = self.container.resource_registry.find_objects(dp._id, PRED.hasStream, RT.Stream, True)
self.assertTrue(streams)
self.assertEquals(1, len(streams))
self.assertTrue(self.ingestion_management.is_persisted(streams[0]))
self.assertAlmostEqual(32.88237, dp.geospatial_bounds.geospatial_latitude_limit_north,places=3)
# but L1 data product should not be persisted
dp = self.find_object_by_name('Test DP L1 conductivity', RT.DataProduct)
streams, _ = self.container.resource_registry.find_objects(dp._id, PRED.hasStream, RT.Stream, True)
self.assertEquals(1, len(streams))
self.assertTrue(streams)
self.assertFalse(self.ingestion_management.is_persisted(streams[0]))
site = self.find_object_by_name('Test Instrument Site', RT.InstrumentSite)
self.assertFalse(site.constraint_list is None)
self.assertEquals(2, len(site.constraint_list))
con = site.constraint_list[0]
self.assertAlmostEqual( 32.88237, con.geospatial_latitude_limit_north, places=3)
self.assertAlmostEqual(-117.23214, con.geospatial_longitude_limit_east, places=3)
con = site.constraint_list[1]
self.assertEquals('TemporalBounds', con.type_)
# check that coordinate system was loaded
self.assertFalse(site.coordinate_reference_system is None)
# check that InstrumentDevice contacts are loaded
dev = self.find_object_by_name('Unit Test SMB37', RT.InstrumentDevice)
self.assertTrue(len(dev.contacts)==2)
self.assertEquals('Userbrough', dev.contacts[0].individual_name_family)
# check has attachments
attachments = self.container.resource_registry.find_attachments(dev._id)
self.assertTrue(len(attachments)>0)
# check for platform agents
agent = self.find_object_by_name('Unit Test Platform Agent', RT.PlatformAgent)
self.assertEquals(2, len(agent.stream_configurations))
parsed = agent.stream_configurations[1]
# self.assertEquals('platform_eng_parsed', parsed.parameter_dictionary_name)
self.assertEquals('ctd_parsed_param_dict', parsed.parameter_dictionary_name)
# OBSOLETE: check that alarm was added to StreamConfig
# self.assertEquals(1, len(parsed.alarms), msg='alarms: %r'%parsed.alarms)
# self.assertEquals('temp', parsed.alarms[0]['kwargs']['value_id'])
# check for platform agents
self.find_object_by_name('Unit Test Platform Agent Instance', RT.PlatformAgentInstance)
# check for platform model boolean values
model = self.find_object_by_name('Nose Testing Platform Model', RT.PlatformModel)
self.assertEquals(True, model.shore_networked)
self.assertNotEqual('str', model.shore_networked.__class__.__name__)
# check for data process definition
self.find_object_by_name("Logical Transform Definition", RT.DataProcessDefinition)
iai = self.find_object_by_name("Test InstrumentAgentInstance", RT.InstrumentAgentInstance)
self.assertEqual({'SCHEDULER': {'VERSION': {'number': 3.0}, 'CLOCK_SYNC': 48.2, 'ACQUIRE_STATUS': {}},
'PARAMETERS': {"TXWAVESTATS": False, 'TXWAVEBURST': 'false', 'TXREALTIME': True}},
iai.startup_config)
self.assertEqual(2, len(iai.alerts))
# self.assertEqual({'entry': 'foo'}, iai.alerts['complex'])
pai = self.find_object_by_name("Unit Test Platform Agent Instance", RT.PlatformAgentInstance)
self.assertEqual({'entry': 'foo'}, pai.alerts['complex'])
orgs, _ = self.container.resource_registry.find_subjects(RT.Org, PRED.hasResource, iai._id, True)
self.assertEqual(1, len(orgs))
self.assertEqual(org._id, orgs[0])
class TestDataProductManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dpsc_cli = DataProductManagementServiceClient()
self.rrclient = ResourceRegistryServiceClient()
self.damsclient = DataAcquisitionManagementServiceClient()
self.pubsubcli = PubsubManagementServiceClient()
self.ingestclient = IngestionManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.unsc = UserNotificationServiceClient()
self.data_retriever = DataRetrieverServiceClient()
#------------------------------------------
# Create the environment
#------------------------------------------
datastore_name = CACHE_DATASTORE_NAME
self.db = self.container.datastore_manager.get_datastore(datastore_name)
self.stream_def_id = self.pubsubcli.create_stream_definition(name='SBE37_CDM')
self.process_definitions = {}
ingestion_worker_definition = ProcessDefinition(name='ingestion worker')
ingestion_worker_definition.executable = {
'module':'ion.processes.data.ingestion.science_granule_ingestion_worker',
'class' :'ScienceGranuleIngestionWorker'
}
process_definition_id = self.process_dispatcher.create_process_definition(process_definition=ingestion_worker_definition)
self.process_definitions['ingestion_worker'] = process_definition_id
self.pids = []
self.exchange_points = []
self.exchange_names = []
#------------------------------------------------------------------------------------------------
# First launch the ingestors
#------------------------------------------------------------------------------------------------
self.exchange_space = 'science_granule_ingestion'
self.exchange_point = 'science_data'
config = DotDict()
config.process.datastore_name = 'datasets'
config.process.queue_name = self.exchange_space
self.exchange_names.append(self.exchange_space)
self.exchange_points.append(self.exchange_point)
pid = self.process_dispatcher.schedule_process(self.process_definitions['ingestion_worker'],configuration=config)
log.debug("the ingestion worker process id: %s", pid)
self.pids.append(pid)
self.addCleanup(self.cleaning_up)
def cleaning_up(self):
for pid in self.pids:
log.debug("number of pids to be terminated: %s", len(self.pids))
try:
self.process_dispatcher.cancel_process(pid)
log.debug("Terminated the process: %s", pid)
except:
log.debug("could not terminate the process id: %s" % pid)
IngestionManagementIntTest.clean_subscriptions()
for xn in self.exchange_names:
xni = self.container.ex_manager.create_xn_queue(xn)
xni.delete()
for xp in self.exchange_points:
xpi = self.container.ex_manager.create_xp(xp)
xpi.delete()
def get_datastore(self, dataset_id):
dataset = self.dataset_management.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
@attr('EXT')
@attr('PREP')
def test_create_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
parameter_dictionary = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict')
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=parameter_dictionary._id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
# Generic time-series data domain creation
tdom, sdom = time_series_domain()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom.dump(),
spatial_domain = sdom.dump())
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 10.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -10.0
dp_obj.ooi_product_name = "PRODNAME"
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product( data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
# Assert that the data product has an associated stream at this stage
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True)
self.assertNotEquals(len(stream_ids), 0)
# Assert that the data product has an associated stream def at this stage
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStreamDefinition, RT.StreamDefinition, True)
self.assertNotEquals(len(stream_ids), 0)
self.dpsc_cli.activate_data_product_persistence(dp_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0)
log.debug('Created data product %s', dp_obj)
#------------------------------------------------------------------------------------------------
# test creating a new data product with a stream definition
#------------------------------------------------------------------------------------------------
log.debug('Creating new data product with a stream definition')
dp_obj = IonObject(RT.DataProduct,
name='DP2',
description='some new dp',
temporal_domain = tdom.dump(),
spatial_domain = sdom.dump())
dp_id2 = self.dpsc_cli.create_data_product(dp_obj, ctd_stream_def_id)
self.dpsc_cli.activate_data_product_persistence(dp_id2)
log.debug('new dp_id = %s' % dp_id2)
#------------------------------------------------------------------------------------------------
#make sure data product is associated with stream def
#------------------------------------------------------------------------------------------------
streamdefs = []
streams, _ = self.rrclient.find_objects(dp_id2, PRED.hasStream, RT.Stream, True)
for s in streams:
log.debug("Checking stream %s" % s)
sdefs, _ = self.rrclient.find_objects(s, PRED.hasStreamDefinition, RT.StreamDefinition, True)
for sd in sdefs:
log.debug("Checking streamdef %s" % sd)
streamdefs.append(sd)
self.assertIn(ctd_stream_def_id, streamdefs)
group_names = self.dpsc_cli.get_data_product_group_list()
self.assertIn("PRODNAME", group_names)
# test reading a non-existent data product
log.debug('reading non-existent data product')
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product('some_fake_id')
# update a data product (tests read also)
log.debug('Updating data product')
# first get the existing dp object
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# now tweak the object
dp_obj.description = 'the very first dp'
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 20.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -20.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 20.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -20.0
# now write the dp back to the registry
update_result = self.dpsc_cli.update_data_product(dp_obj)
# now get the dp back to see if it was updated
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertEquals(dp_obj.description,'the very first dp')
self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0)
log.debug('Updated data product %s', dp_obj)
#test extension
extended_product = self.dpsc_cli.get_data_product_extension(dp_id)
self.assertEqual(dp_id, extended_product._id)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_product.computed.product_download_size_estimated.status)
self.assertEqual(0, extended_product.computed.product_download_size_estimated.value)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_product.computed.parameters.status)
#log.debug("test_create_data_product: parameters %s" % extended_product.computed.parameters.value)
def ion_object_encoder(obj):
return obj.__dict__
#test prepare for create
data_product_data = self.dpsc_cli.prepare_data_product_support()
#print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2)
self.assertEqual(data_product_data._id, "")
self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport)
self.assertEqual(len(data_product_data.associations['StreamDefinition'].resources), 2)
self.assertEqual(len(data_product_data.associations['Dataset'].resources), 0)
self.assertEqual(len(data_product_data.associations['StreamDefinition'].associated_resources), 0)
self.assertEqual(len(data_product_data.associations['Dataset'].associated_resources), 0)
#test prepare for update
data_product_data = self.dpsc_cli.prepare_data_product_support(dp_id)
#print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2)
self.assertEqual(data_product_data._id, dp_id)
self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport)
self.assertEqual(len(data_product_data.associations['StreamDefinition'].resources), 2)
self.assertEqual(len(data_product_data.associations['Dataset'].resources), 1)
self.assertEqual(len(data_product_data.associations['StreamDefinition'].associated_resources), 1)
self.assertEqual(data_product_data.associations['StreamDefinition'].associated_resources[0].s, dp_id)
self.assertEqual(len(data_product_data.associations['Dataset'].associated_resources), 1)
self.assertEqual(data_product_data.associations['Dataset'].associated_resources[0].s, dp_id)
# now 'delete' the data product
log.debug("deleting data product: %s" % dp_id)
self.dpsc_cli.delete_data_product(dp_id)
# Assert that there are no associated streams leftover after deleting the data product
stream_ids, assoc_ids = self.rrclient.find_objects(dp_id, PRED.hasStream, RT.Stream, True)
self.assertEquals(len(stream_ids), 0)
self.assertEquals(len(assoc_ids), 0)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# Get the events corresponding to the data product
ret = self.unsc.get_recent_events(resource_id=dp_id)
events = ret.value
for event in events:
log.debug("event time: %s" % event.ts_created)
self.assertTrue(len(events) > 0)
def test_data_product_stream_def(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
stream_def_id = self.dpsc_cli.get_data_product_stream_definition(dp_id)
self.assertEquals(ctd_stream_def_id, stream_def_id)
def test_derived_data_product(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='ctd parsed', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsubcli.delete_stream_definition, ctd_stream_def_id)
tdom, sdom = time_series_domain()
dp = DataProduct(name='Instrument DP', temporal_domain=tdom.dump(), spatial_domain=sdom.dump())
dp_id = self.dpsc_cli.create_data_product(dp, stream_definition_id=ctd_stream_def_id)
self.addCleanup(self.dpsc_cli.force_delete_data_product, dp_id)
self.dpsc_cli.activate_data_product_persistence(dp_id)
self.addCleanup(self.dpsc_cli.suspend_data_product_persistence, dp_id)
dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True)
if not dataset_ids:
raise NotFound("Data Product %s dataset does not exist" % str(dp_id))
dataset_id = dataset_ids[0]
# Make the derived data product
simple_stream_def_id = self.pubsubcli.create_stream_definition(name='TEMPWAT stream def', parameter_dictionary_id=pdict_id, available_fields=['time','temp'])
tempwat_dp = DataProduct(name='TEMPWAT')
tempwat_dp_id = self.dpsc_cli.create_data_product(tempwat_dp, stream_definition_id=simple_stream_def_id, parent_data_product_id=dp_id)
self.addCleanup(self.dpsc_cli.delete_data_product, tempwat_dp_id)
# Check that the streams associated with the data product are persisted with
stream_ids, _ = self.rrclient.find_objects(dp_id,PRED.hasStream,RT.Stream,True)
for stream_id in stream_ids:
self.assertTrue(self.ingestclient.is_persisted(stream_id))
stream_id = stream_ids[0]
route = self.pubsubcli.read_stream_route(stream_id=stream_id)
rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
rdt['pressure'] = np.arange(20)
publisher = StandaloneStreamPublisher(stream_id,route)
dataset_modified = Event()
def cb(*args, **kwargs):
dataset_modified.set()
es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id, auto_delete=True)
es.start()
self.addCleanup(es.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
tempwat_dataset_ids, _ = self.rrclient.find_objects(tempwat_dp_id, PRED.hasDataset, id_only=True)
tempwat_dataset_id = tempwat_dataset_ids[0]
granule = self.data_retriever.retrieve(tempwat_dataset_id, delivery_format=simple_stream_def_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt['time'], np.arange(20))
self.assertEquals(set(rdt.fields), set(['time','temp']))
def test_activate_suspend_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
log.debug("Created an IonObject for a data product: %s" % dp_obj)
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test activate and suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.activate_data_product_persistence(dp_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True)
if not dataset_ids:
raise NotFound("Data Product %s dataset does not exist" % str(dp_id))
dataset_id = dataset_ids[0]
# Check that the streams associated with the data product are persisted with
stream_ids, _ = self.rrclient.find_objects(dp_id,PRED.hasStream,RT.Stream,True)
for stream_id in stream_ids:
self.assertTrue(self.ingestclient.is_persisted(stream_id))
stream_id = stream_ids[0]
route = self.pubsubcli.read_stream_route(stream_id=stream_id)
rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
publisher = StandaloneStreamPublisher(stream_id,route)
dataset_modified = Event()
def cb(*args, **kwargs):
dataset_modified.set()
es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id, auto_delete=True)
es.start()
self.addCleanup(es.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data = self.data_retriever.retrieve(dataset_ids[0])
self.assertIsInstance(replay_data, Granule)
log.debug("The data retriever was able to replay the dataset that was attached to the data product "
"we wanted to be persisted. Therefore the data product was indeed persisted with "
"otherwise we could not have retrieved its dataset using the data retriever. Therefore "
"this demonstration shows that L4-CI-SA-RQ-267 is satisfied: 'Data product management shall persist data products'")
data_product_object = self.rrclient.read(dp_id)
self.assertEquals(data_product_object.name,'DP1')
self.assertEquals(data_product_object.description,'some new dp')
log.debug("Towards L4-CI-SA-RQ-308: 'Data product management shall persist data product metadata'. "
" Attributes in create for the data product obj, name= '%s', description='%s', match those of object from the "
"resource registry, name='%s', desc='%s'" % (dp_obj.name, dp_obj.description,data_product_object.name,
data_product_object.description))
#------------------------------------------------------------------------------------------------
# test suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.suspend_data_product_persistence(dp_id)
dataset_modified.clear()
rdt['time'] = np.arange(20,40)
publisher.publish(rdt.to_granule())
self.assertFalse(dataset_modified.wait(2))
self.dpsc_cli.activate_data_product_persistence(dp_id)
dataset_modified.clear()
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_almost_equal(rdt['time'], np.arange(40))
dataset_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasDataset, id_only=True)
self.assertEquals(len(dataset_ids), 1)
self.dpsc_cli.suspend_data_product_persistence(dp_id)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
with self.assertRaises(NotFound):
dp_obj = self.rrclient.read(dp_id)
def test_lookup_values(self):
ph = ParameterHelper(self.dataset_management, self.addCleanup)
pdict_id = ph.create_lookups()
stream_def_id = self.pubsubcli.create_stream_definition('lookup', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsubcli.delete_stream_definition, stream_def_id)
data_product = DataProduct(name='lookup data product')
tdom, sdom = time_series_domain()
data_product.temporal_domain = tdom.dump()
data_product.spatial_domain = sdom.dump()
data_product_id = self.dpsc_cli.create_data_product(data_product, stream_definition_id=stream_def_id)
self.addCleanup(self.dpsc_cli.delete_data_product, data_product_id)
data_producer = DataProducer(name='producer')
data_producer.producer_context = DataProcessProducerContext()
data_producer.producer_context.configuration['qc_keys'] = ['offset_document']
data_producer_id, _ = self.rrclient.create(data_producer)
self.addCleanup(self.rrclient.delete, data_producer_id)
assoc,_ = self.rrclient.create_association(subject=data_product_id, object=data_producer_id, predicate=PRED.hasDataProducer)
self.addCleanup(self.rrclient.delete_association, assoc)
document_keys = self.damsclient.list_qc_references(data_product_id)
self.assertEquals(document_keys, ['offset_document'])
svm = StoredValueManager(self.container)
svm.stored_value_cas('offset_document', {'offset_a':2.0})
self.dpsc_cli.activate_data_product_persistence(data_product_id)
dataset_ids, _ = self.rrclient.find_objects(subject=data_product_id, predicate=PRED.hasDataset, id_only=True)
dataset_id = dataset_ids[0]
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [0]
rdt['temp'] = [20.]
granule = rdt.to_granule()
stream_ids, _ = self.rrclient.find_objects(subject=data_product_id, predicate=PRED.hasStream, id_only=True)
stream_id = stream_ids[0]
route = self.pubsubcli.read_stream_route(stream_id=stream_id)
publisher = StandaloneStreamPublisher(stream_id, route)
publisher.publish(granule)
self.assertTrue(dataset_monitor.event.wait(10))
granule = self.data_retriever.retrieve(dataset_id)
rdt2 = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt['temp'], rdt2['temp'])
np.testing.assert_array_almost_equal(rdt2['calibrated'], np.array([22.0]))
svm.stored_value_cas('updated_document', {'offset_a':3.0})
dataset_monitor = DatasetMonitor(dataset_id)
self.addCleanup(dataset_monitor.stop)
ep = EventPublisher(event_type=OT.ExternalReferencesUpdatedEvent)
ep.publish_event(origin=data_product_id, reference_keys=['updated_document'])
rdt = RecordDictionaryTool(stream_definition_id=stream_def_id)
rdt['time'] = [1]
rdt['temp'] = [20.]
granule = rdt.to_granule()
gevent.sleep(2) # Yield so that the event goes through
publisher.publish(granule)
self.assertTrue(dataset_monitor.event.wait(10))
granule = self.data_retriever.retrieve(dataset_id)
rdt2 = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt2['temp'],np.array([20.,20.]))
np.testing.assert_array_almost_equal(rdt2['calibrated'], np.array([22.0,23.0]))
class TestLoader(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.ingestion_management = IngestionManagementServiceClient()
def assert_can_load(self, scenarios, loadui=False, loadooi=False,
path=TESTED_DOC, ui_path='default'):
""" perform preload for given scenarios and raise exception if there is a problem with the data """
config = dict(op="load",
scenario=scenarios,
attachments="res/preload/r2_ioc/attachments",
loadui=loadui,
loadooi=loadooi,
path=path, ui_path=ui_path,
assets='res/preload/r2_ioc/ooi_assets',
bulk=loadooi,
ooiexclude='DataProduct,DataProductLink')
self.container.spawn_process("Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=config)
@attr('PRELOAD')
def test_ui_valid(self):
""" make sure UI assets are valid using DEFAULT_UI_ASSETS = 'https://userexperience.oceanobservatories.org/database-exports/' """
self.assert_can_load("BASE,BETA", loadui=True, ui_path='default')
@attr('PRELOAD')
def test_ui_candidates_valid(self):
""" make sure UI assets are valid using DEFAULT_UI_ASSETS = 'https://userexperience.oceanobservatories.org/database-exports/Candidates' """
self.assert_can_load("BASE,BETA", loadui=True, ui_path='candidate')
@attr('PRELOAD')
def test_assets_valid(self):
""" make sure can load asset DB """
self.assert_can_load("BASE,BETA,DEVS", path='master', loadooi=True)
@attr('PRELOAD')
def test_alpha_valid(self):
""" make sure R2_DEMO scenario in master google doc
is valid and self-contained (doesn't rely on rows from other scenarios except BASE and BETA)
NOTE: test will pass/fail based on current google doc, not just code changes.
"""
self.assert_can_load("BASE,BETA,ALPHA_SYS", path='master')
@attr('PRELOAD')
def test_beta_valid(self):
""" make sure R2_DEMO scenario in master google doc
is valid and self-contained (doesn't rely on rows from other scenarios except BASE and BETA)
NOTE: test will pass/fail based on current google doc, not just code changes.
"""
self.assert_can_load("BASE,BETA,BETA_SYS", path='master')
@attr('PRELOAD')
def test_devs_valid(self):
""" make sure DEVS scenario in master google doc
is valid and self-contained (doesn't rely on rows from other scenarios except BASE and BETA)
NOTE: test will pass/fail based on current google doc, not just code changes.
"""
self.assert_can_load("BASE,BETA,DEVS", path='master')
def find_object_by_name(self, name, type):
objects,_ = self.container.resource_registry.find_resources(type, id_only=False)
self.assertTrue(len(objects)>=1)
found = None
for object in objects:
print object.name
if object.name==name:
self.assertFalse(found, msg='Found more than one %s "%s" (was expecting just one)'%(type,name))
found = object
self.assertTrue(found, msg='Did not find %s "%s"'%(type,name))
return found
@attr('INT', group='loader')
@attr('SMOKE', group='loader')
def test_row_values(self):
""" use only rows from NOSE scenario for specific names and details included in this test.
rows in NOSE may rely on entries in BASE and BETA scenarios,
but should not specifically test values from those scenarios.
"""
# first make sure this scenario loads successfully
self.assert_can_load("BASE,BETA,NOSE")
# check for an Org
org = self.find_object_by_name('CASPER', RT.Org)
self.assertFalse(org.contacts is None)
self.assertEquals('Userbrough', org.contacts[0].individual_name_family)
self.assertEquals('primary', org.contacts[0].roles[0])
# check data product
dp = self.find_object_by_name('Test DP L0 CTD', RT.DataProduct)
formats = dp.available_formats
self.assertEquals(2, len(formats))
self.assertEquals('csv', formats[0])
# should be persisted
streams, _ = self.container.resource_registry.find_objects(dp._id, PRED.hasStream, RT.Stream, True)
self.assertTrue(streams)
self.assertEquals(1, len(streams))
self.assertTrue(self.ingestion_management.is_persisted(streams[0]))
self.assertAlmostEqual(32.88237, dp.geospatial_bounds.geospatial_latitude_limit_north,places=3)
# but L1 data product should not be persisted
dp = self.find_object_by_name('Test DP L1 conductivity', RT.DataProduct)
streams, _ = self.container.resource_registry.find_objects(dp._id, PRED.hasStream, RT.Stream, True)
self.assertEquals(1, len(streams))
self.assertTrue(streams)
self.assertFalse(self.ingestion_management.is_persisted(streams[0]))
site = self.find_object_by_name('Test Instrument Site', RT.InstrumentSite)
self.assertFalse(site.constraint_list is None)
self.assertEquals(2, len(site.constraint_list))
con = site.constraint_list[0]
self.assertAlmostEqual( 32.88237, con.geospatial_latitude_limit_north, places=3)
self.assertAlmostEqual(-117.23214, con.geospatial_longitude_limit_east, places=3)
con = site.constraint_list[1]
self.assertEquals('TemporalBounds', con.type_)
# check that coordinate system was loaded
self.assertFalse(site.coordinate_reference_system is None)
# check that InstrumentDevice contacts are loaded
dev = self.find_object_by_name('Unit Test SMB37', RT.InstrumentDevice)
self.assertTrue(len(dev.contacts)==2)
self.assertEquals('Userbrough', dev.contacts[0].individual_name_family)
# check has attachments
attachments = self.container.resource_registry.find_attachments(dev._id)
self.assertTrue(len(attachments)>0)
# check for platform agents
agent = self.find_object_by_name('Unit Test Platform Agent', RT.PlatformAgent)
self.assertEquals(2, len(agent.stream_configurations))
parsed = agent.stream_configurations[1]
self.assertEquals('platform_eng_parsed', parsed.parameter_dictionary_name)
# check for platform agents
found_it = self.find_object_by_name('Unit Test Platform Agent Instance', RT.PlatformAgentInstance)
# check for platform model boolean values
model = self.find_object_by_name('Nose Testing Platform Model', RT.PlatformModel)
self.assertEquals(True, model.shore_networked)
self.assertNotEqual('str', model.shore_networked.__class__.__name__)
class TestDataProductManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dpsc_cli = DataProductManagementServiceClient(node=self.container.node)
self.rrclient = ResourceRegistryServiceClient(node=self.container.node)
self.damsclient = DataAcquisitionManagementServiceClient(node=self.container.node)
self.pubsubcli = PubsubManagementServiceClient(node=self.container.node)
self.ingestclient = IngestionManagementServiceClient(node=self.container.node)
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.unsc = UserNotificationServiceClient()
self.data_retriever = DataRetrieverServiceClient()
#------------------------------------------
# Create the environment
#------------------------------------------
datastore_name = CACHE_DATASTORE_NAME
self.db = self.container.datastore_manager.get_datastore(datastore_name)
self.stream_def_id = self.pubsubcli.create_stream_definition(name='SBE37_CDM')
self.process_definitions = {}
ingestion_worker_definition = ProcessDefinition(name='ingestion worker')
ingestion_worker_definition.executable = {
'module':'ion.processes.data.ingestion.science_granule_ingestion_worker',
'class' :'ScienceGranuleIngestionWorker'
}
process_definition_id = self.process_dispatcher.create_process_definition(process_definition=ingestion_worker_definition)
self.process_definitions['ingestion_worker'] = process_definition_id
self.pids = []
self.exchange_points = []
self.exchange_names = []
#------------------------------------------------------------------------------------------------
# First launch the ingestors
#------------------------------------------------------------------------------------------------
self.exchange_space = 'science_granule_ingestion'
self.exchange_point = 'science_data'
config = DotDict()
config.process.datastore_name = 'datasets'
config.process.queue_name = self.exchange_space
self.exchange_names.append(self.exchange_space)
self.exchange_points.append(self.exchange_point)
pid = self.process_dispatcher.schedule_process(self.process_definitions['ingestion_worker'],configuration=config)
log.debug("the ingestion worker process id: %s", pid)
self.pids.append(pid)
self.addCleanup(self.cleaning_up)
def cleaning_up(self):
for pid in self.pids:
log.debug("number of pids to be terminated: %s", len(self.pids))
try:
self.process_dispatcher.cancel_process(pid)
log.debug("Terminated the process: %s", pid)
except:
log.debug("could not terminate the process id: %s" % pid)
IngestionManagementIntTest.clean_subscriptions()
for xn in self.exchange_names:
xni = self.container.ex_manager.create_xn_queue(xn)
xni.delete()
for xp in self.exchange_points:
xpi = self.container.ex_manager.create_xp(xp)
xpi.delete()
def get_datastore(self, dataset_id):
dataset = self.dataset_management.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
def test_create_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
parameter_dictionary_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict')
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=parameter_dictionary_id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
# Generic time-series data domain creation
tdom, sdom = time_series_domain()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom.dump(),
spatial_domain = sdom.dump())
log.debug("Created an IonObject for a data product: %s" % dp_obj)
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product( data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
self.dpsc_cli.activate_data_product_persistence(dp_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
#------------------------------------------------------------------------------------------------
# test creating a new data product with a stream definition
#------------------------------------------------------------------------------------------------
log.debug('Creating new data product with a stream definition')
dp_obj = IonObject(RT.DataProduct,
name='DP2',
description='some new dp',
temporal_domain = tdom.dump(),
spatial_domain = sdom.dump())
dp_id2 = self.dpsc_cli.create_data_product(dp_obj, ctd_stream_def_id)
self.dpsc_cli.activate_data_product_persistence(dp_id2)
log.debug('new dp_id = %s' % dp_id2)
#------------------------------------------------------------------------------------------------
#make sure data product is associated with stream def
#------------------------------------------------------------------------------------------------
streamdefs = []
streams, _ = self.rrclient.find_objects(dp_id2, PRED.hasStream, RT.Stream, True)
for s in streams:
log.debug("Checking stream %s" % s)
sdefs, _ = self.rrclient.find_objects(s, PRED.hasStreamDefinition, RT.StreamDefinition, True)
for sd in sdefs:
log.debug("Checking streamdef %s" % sd)
streamdefs.append(sd)
self.assertIn(ctd_stream_def_id, streamdefs)
# test reading a non-existent data product
log.debug('reading non-existent data product')
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product('some_fake_id')
# update a data product (tests read also)
log.debug('Updating data product')
# first get the existing dp object
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# now tweak the object
dp_obj.description = 'the very first dp'
# now write the dp back to the registry
update_result = self.dpsc_cli.update_data_product(dp_obj)
# now get the dp back to see if it was updated
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertEquals(dp_obj.description,'the very first dp')
#test extension
extended_product = self.dpsc_cli.get_data_product_extension(dp_id)
self.assertEqual(dp_id, extended_product._id)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_product.computed.product_download_size_estimated.status)
self.assertEqual(0, extended_product.computed.product_download_size_estimated.value)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_product.computed.parameters.status)
#log.debug("test_create_data_product: parameters %s" % extended_product.computed.parameters.value)
# now 'delete' the data product
log.debug("deleting data product: %s" % dp_id)
self.dpsc_cli.delete_data_product(dp_id)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# Get the events corresponding to the data product
ret = self.unsc.get_recent_events(resource_id=dp_id)
events = ret.value
for event in events:
log.debug("event time: %s" % event.ts_created)
# self.assertTrue(len(events) > 0)
def test_data_product_stream_def(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id)
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
stream_def_id = self.dpsc_cli.get_data_product_stream_definition(dp_id)
self.assertEquals(ctd_stream_def_id, stream_def_id)
def test_activate_suspend_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(name='Simulated CTD data', parameter_dictionary_id=pdict_id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
# Construct temporal and spatial Coordinate Reference System objects
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
log.debug("Created an IonObject for a data product: %s" % dp_obj)
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product(data_product= dp_obj,
stream_definition_id=ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test activate and suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.activate_data_product_persistence(dp_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
dataset_ids, _ = self.rrclient.find_objects(subject=dp_id, predicate=PRED.hasDataset, id_only=True)
if not dataset_ids:
raise NotFound("Data Product %s dataset does not exist" % str(dp_id))
self.get_datastore(dataset_ids[0])
# Check that the streams associated with the data product are persisted with
stream_ids, _ = self.rrclient.find_objects(dp_id,PRED.hasStream,RT.Stream,True)
for stream_id in stream_ids:
self.assertTrue(self.ingestclient.is_persisted(stream_id))
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data = self.data_retriever.retrieve(dataset_ids[0])
self.assertIsInstance(replay_data, Granule)
log.debug("The data retriever was able to replay the dataset that was attached to the data product "
"we wanted to be persisted. Therefore the data product was indeed persisted with "
"otherwise we could not have retrieved its dataset using the data retriever. Therefore "
"this demonstration shows that L4-CI-SA-RQ-267 is satisfied: 'Data product management shall persist data products'")
data_product_object = self.rrclient.read(dp_id)
self.assertEquals(data_product_object.name,'DP1')
self.assertEquals(data_product_object.description,'some new dp')
log.debug("Towards L4-CI-SA-RQ-308: 'Data product management shall persist data product metadata'. "
" Attributes in create for the data product obj, name= '%s', description='%s', match those of object from the "
"resource registry, name='%s', desc='%s'" % (dp_obj.name, dp_obj.description,data_product_object.name,
data_product_object.description))
#------------------------------------------------------------------------------------------------
# test suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.suspend_data_product_persistence(dp_id)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
with self.assertRaises(NotFound):
dp_obj = self.rrclient.read(dp_id)
class TestLoader(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.ingestion_management = IngestionManagementServiceClient()
self.rr = self.container.resource_registry
def _perform_preload(self, load_cfg):
#load_cfg["ui_path"] = "res/preload/r2_ioc/ui_assets"
#load_cfg["path"] = "R2PreloadedResources.xlsx"
#load_cfg["assetmappings"] = "OOIPreload.xlsx"
self.container.spawn_process("Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=load_cfg)
def _preload_instrument(self, inst_scenario):
load_cfg = dict(op="load",
scenario=inst_scenario,
attachments="res/preload/r2_ioc/attachments",
assets='res/preload/r2_ioc/ooi_assets',
)
self._perform_preload(load_cfg)
def _preload_ui(self, ui_path="default"):
load_cfg = dict(op="load",
loadui=True,
ui_path=ui_path,
)
self._perform_preload(load_cfg)
def _preload_cfg(self, cfg, path=TEST_PATH):
load_cfg = dict(cfg=cfg,
path=path)
self._perform_preload(load_cfg)
def _preload_scenario(self, scenario, path=TEST_PATH, idmap=False, **kwargs):
load_cfg = dict(op="load",
scenario=scenario,
attachments="res/preload/r2_ioc/attachments",
path=path,
idmap=idmap)
load_cfg.update(kwargs)
self._perform_preload(load_cfg)
def _preload_ooi(self, path=TEST_PATH):
load_cfg = dict(op="load",
loadooi=True,
assets="res/preload/r2_ioc/ooi_assets",
path=path,
ooiuntil="12/31/2013",
)
self._perform_preload(load_cfg)
# -------------------------------------------------------------------------
@attr('PRELOAD')
def test_ui_valid(self):
""" make sure UI assets are valid using DEFAULT_UI_ASSETS = 'http://userexperience.oceanobservatories.org/database-exports/Stable' """
self._preload_ui(ui_path='default')
obj_list,_ = self.rr.find_resources(restype=RT.UISpec, name="ION UI Specs", id_only=False)
self.assertEquals(len(obj_list), 1)
@attr('PRELOAD')
def test_ui_candidates_valid(self):
""" make sure UI assets are valid using DEFAULT_UI_ASSETS = 'http://userexperience.oceanobservatories.org/database-exports/Candidates' """
self._preload_ui(ui_path='candidate')
obj_list,_ = self.rr.find_resources(restype=RT.UISpec, name="ION UI Specs", id_only=False)
self.assertEquals(len(obj_list), 1)
@attr('PRELOAD')
def test_betademo_valid(self):
""" make sure can load asset DB """
self._preload_scenario("BETA,R2_DEMO,RSN_OMS", path=TEST_PATH)
self._preload_ooi(path=TEST_PATH)
# check that deployment port assignments subobject created correctly
#collect a set of deployments
deploy_list = []
#DEP3 of PDEV3
obj_list,_ = self.rr.find_resources(restype=RT.Deployment, name="Platform Deployment", id_only=False)
deploy_list.extend(obj_list)
log.debug('test_betademo_valid DEP3: %s ', obj_list)
#DEP4 of PDEV4
obj_list,_ = self.rr.find_resources(restype=RT.Deployment, name="dep4", id_only=False)
log.debug('test_betademo_valid DEP4: %s ', obj_list)
deploy_list.extend(obj_list)
self.assertEquals(len(deploy_list), 2)
for dply_obj in deploy_list:
# CabledNode should have just one item in the assignments list
self.assertEquals(len(dply_obj.port_assignments), 1)
for dev_id, platform_port in dply_obj.port_assignments.iteritems():
# all values in the port assignments dict should be PlatformPort objects
self.assertEquals(platform_port.type_, OT.PlatformPort)
@attr('PRELOAD')
def test_incremental(self):
""" make sure R2_DEMO scenario in master google doc
is valid and self-contained (doesn't rely on rows from other scenarios except BETA)
NOTE: test will pass/fail based on current google doc, not just code changes.
"""
self._preload_cfg("res/preload/r2_ioc/config/ooi_load_config.yml", path=TEST_PATH)
self._preload_scenario("OOIR2_DEMO", path=TEST_PATH, idmap=True)
dp_list1,_ = self.rr.find_resources(restype=RT.DataProduct, id_only=True)
ia_list1,_ = self.rr.find_resources(restype=RT.InstrumentAgent, id_only=True)
self._preload_cfg("res/preload/r2_ioc/config/ooi_instruments.yml", path=TEST_PATH)
ia_list2,_ = self.rr.find_resources(restype=RT.InstrumentAgent, id_only=True)
self.assertGreater(len(ia_list2), len(ia_list1))
dp_list2,_ = self.rr.find_resources(restype=RT.DataProduct, id_only=True)
self.assertGreater(len(dp_list2), len(dp_list1))
id_list2,_ = self.rr.find_resources(restype=RT.InstrumentDevice, id_only=True)
self._preload_ooi(path=TEST_PATH)
dp_list3,_ = self.rr.find_resources(restype=RT.DataProduct, id_only=True)
self.assertGreater(len(dp_list3), len(dp_list2))
id_list3,_ = self.rr.find_resources(restype=RT.InstrumentDevice, id_only=True)
self.assertEquals(len(id_list3), len(id_list2))
self._preload_ooi(path=TEST_PATH)
dp_list4,_ = self.rr.find_resources(restype=RT.DataProduct, id_only=True)
self.assertEquals(len(dp_list4), len(dp_list3))
id_list4,_ = self.rr.find_resources(restype=RT.InstrumentDevice, id_only=True)
self.assertEquals(len(id_list4), len(id_list3))
def find_object_by_name(self, name, resource_type):
objects,_ = self.container.resource_registry.find_resources(resource_type, name=name, id_only=False)
self.assertEquals(len(objects), 1)
return objects[0]
@attr('INT', group='loader')
@attr('SMOKE', group='loader')
def test_row_values(self):
""" use only rows from NOSE scenario for specific names and details included in this test.
rows in NOSE may rely on entries in BETA scenarios,
but should not specifically test values from those scenarios.
"""
# first make sure this scenario loads successfully
self._preload_scenario("BETA,NOSE")
# check for ExternalDataset
eds = self.find_object_by_name('Test External CTD Dataset', RT.ExternalDataset)
edm1 = self.find_object_by_name('Test External CTD Dataset Model', RT.ExternalDatasetModel)
edm2,_ = self.container.resource_registry.find_objects(eds._id, PRED.hasModel, RT.ExternalDatasetModel, True)
self.assertEquals(edm1._id, edm2[0])
inst = self.find_object_by_name('Test External CTD Agent Instance', RT.ExternalDatasetAgentInstance)
self.assertEquals('value1', inst.driver_config['key1'], msg='driver_config[key1] is not value1:\n%r' % inst.driver_config)
# check for an Org
org = self.find_object_by_name('CASPER', RT.Org)
self.assertFalse(org.contacts is None)
self.assertEquals('Userbrough', org.contacts[0].individual_name_family)
self.assertEquals('primary', org.contacts[0].roles[0])
# check data product
dp = self.find_object_by_name('Test DP L0 CTD', RT.DataProduct)
# should be persisted
streams, _ = self.container.resource_registry.find_objects(dp._id, PRED.hasStream, RT.Stream, True)
self.assertTrue(streams)
self.assertEquals(1, len(streams))
self.assertTrue(self.ingestion_management.is_persisted(streams[0]))
self.assertAlmostEqual(32.88237, dp.geospatial_bounds.geospatial_latitude_limit_north,places=3)
# but L1 data product should not be persisted
dp = self.find_object_by_name('Test DP L1 conductivity', RT.DataProduct)
streams, _ = self.container.resource_registry.find_objects(dp._id, PRED.hasStream, RT.Stream, True)
self.assertEquals(1, len(streams))
self.assertTrue(streams)
self.assertFalse(self.ingestion_management.is_persisted(streams[0]))
site = self.find_object_by_name('Test Instrument Site', RT.InstrumentSite)
self.assertFalse(site.constraint_list is None)
self.assertEquals(2, len(site.constraint_list))
con = site.constraint_list[0]
self.assertAlmostEqual( 32.88237, con.geospatial_latitude_limit_north, places=3)
self.assertAlmostEqual(-117.23214, con.geospatial_longitude_limit_east, places=3)
con = site.constraint_list[1]
self.assertEquals('TemporalBounds', con.type_)
# check that coordinate system was loaded
self.assertFalse(site.coordinate_reference_system is None)
# check that InstrumentDevice contacts are loaded
dev = self.find_object_by_name('Unit Test SMB37', RT.InstrumentDevice)
self.assertTrue(len(dev.contacts)==2)
self.assertEquals('Userbrough', dev.contacts[0].individual_name_family)
# check has attachments
attachments = self.container.resource_registry.find_attachments(dev._id)
self.assertTrue(len(attachments)>0)
# check for platform agents
agent = self.find_object_by_name('Unit Test Platform Agent', RT.PlatformAgent)
self.assertEquals(2, len(agent.stream_configurations))
parsed = agent.stream_configurations[1]
# self.assertEquals('platform_eng_parsed', parsed.parameter_dictionary_name)
self.assertEquals('ctd_parsed_param_dict', parsed.parameter_dictionary_name)
# OBSOLETE: check that alarm was added to StreamConfig
# self.assertEquals(1, len(parsed.alarms), msg='alarms: %r'%parsed.alarms)
# self.assertEquals('temp', parsed.alarms[0]['kwargs']['value_id'])
# check for platform agents
self.find_object_by_name('Unit Test Platform Agent Instance', RT.PlatformAgentInstance)
# check for platform model boolean values
model = self.find_object_by_name('Nose Testing Platform Model', RT.PlatformModel)
self.assertEquals(True, model.shore_networked)
self.assertNotEqual('str', model.shore_networked.__class__.__name__)
iai = self.find_object_by_name("Test InstrumentAgentInstance", RT.InstrumentAgentInstance)
self.assertEqual({'SCHEDULER': {'VERSION': {'number': 3.0}, 'CLOCK_SYNC': 48.2, 'ACQUIRE_STATUS': {}},
'PARAMETERS': {"TXWAVESTATS": False, 'TXWAVEBURST': 'false', 'TXREALTIME': True}},
iai.startup_config)
self.assertEqual(2, len(iai.alerts))
pai = self.find_object_by_name("Unit Test Platform Agent Instance", RT.PlatformAgentInstance)
self.assertEqual(1, len(pai.alerts))
self.assertTrue(pai.agent_config.has_key('platform_config'))
log.debug('test_row_values PlatformAgentInstance driver_config: %s ', pai.driver_config)
self.assertTrue(pai.driver_config.has_key('oms_uri'))
oms_uri = pai.driver_config['oms_uri']
log.debug('test_row_values PlatformAgentInstance oms_uri: %s ', oms_uri)
self.assertEquals('http://*****:*****@10.180.80.10:9021/', oms_uri)
orgs, _ = self.container.resource_registry.find_subjects(RT.Org, PRED.hasResource, iai._id, True)
self.assertEqual(1, len(orgs))
self.assertEqual(org._id, orgs[0])
entries ,_ = self.container.resource_registry.find_resources(RT.SchedulerEntry, id_only=False)
self.assertGreaterEqual(len(entries), 1)
@attr('PRELOAD')
def test_alpha_valid(self):
""" make sure R2_DEMO scenario in master google doc
is valid and self-contained (doesn't rely on rows from other scenarios except BETA)
NOTE: test will pass/fail based on current google doc, not just code changes.
"""
self._preload_cfg("res/preload/r2_ioc/config/ooi_alpha.yml", path=TEST_PATH)
@attr('PRELOAD')
def test_beta_valid(self):
""" make sure R2_DEMO scenario in master google doc
is valid and self-contained (doesn't rely on rows from other scenarios except BETA)
NOTE: test will pass/fail based on current google doc, not just code changes.
"""
self._preload_cfg("res/preload/r2_ioc/config/ooi_beta.yml", path=TEST_PATH)
failure_list = []
def add_failure(res_obj, msg):
fail_msg = "%s[%s/%s]: %s" % (res_obj.type_, res_obj._id, res_obj.name, msg)
failure_list.append(fail_msg)
log.warn("Starting preload assertions now")
res_objs, res_keys = self.rr.find_resources_ext(alt_id_ns="PRE", id_only=False)
log.info("Found %s preloaded resources", len(res_objs))
dp_objs = [res for res in res_objs if res.type_ == RT.DataProduct]
log.info("Checking %s DataProducts", len(dp_objs))
for dp in dp_objs:
pass
# Reenable this when we have geospatial coordinates for PNs
#if not all([dp.geospatial_bounds.geospatial_latitude_limit_north,
# dp.geospatial_bounds.geospatial_latitude_limit_south,
# dp.geospatial_bounds.geospatial_longitude_limit_east,
# dp.geospatial_bounds.geospatial_longitude_limit_west]):
# add_failure(dp, "geospatial_bounds location invalid: %s" % dp.geospatial_bounds)
#if not all([dp.geospatial_bounds.geospatial_vertical_min,
# dp.geospatial_bounds.geospatial_vertical_max]):
# add_failure(dp, "geospatial_bounds vertical invalid: %s" % dp.geospatial_bounds)
if failure_list:
fail_msg = "Preload assertions violated:\n" + "\n".join(f for f in failure_list)
self.fail(fail_msg)
class TestLoader(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.ingestion_management = IngestionManagementServiceClient()
def test_lca_load(self):
config = dict(op="load", scenario="R2_DEMO", attachments="res/preload/r2_ioc/attachments")
self.container.spawn_process("Loader", "ion.processes.bootstrap.ion_loader", "IONLoader", config=config)
# make sure contact entries were created correctly
res,_ = self.container.resource_registry.find_resources(RT.Org, id_only=False)
self.assertTrue(len(res) > 1)
found = False
for org in res:
if org.name=='RSN':
self.assertFalse(found, msg='Found more than one Org "RSN" -- should have preloaded one')
found = True
self.assertFalse(org.contacts is None)
self.assertEquals('Delaney', org.contacts[0].individual_name_family)
self.assertEquals('primary', org.contacts[0].roles[0])
self.assertTrue(found, msg='Did not find Org "RSN" -- should have been preloaded')
# check data product
res,_ = self.container.resource_registry.find_resources(RT.DataProduct, name='CTDBP-1012-REC1 Raw', id_only=False)
self.assertEquals(1, len(res))
dp = res[0]
formats = dp.available_formats
self.assertEquals(2, len(formats))
self.assertEquals('csv', formats[0])
# should be persisted
streams, _ = self.container.resource_registry.find_objects(dp._id, PRED.hasStream, RT.Stream, True)
self.assertTrue(streams)
self.assertEquals(1, len(streams))
self.assertTrue(self.ingestion_management.is_persisted(streams[0]))
self.assertTrue(math.fabs(dp.geospatial_bounds.geospatial_latitude_limit_north-44.7)<.01)
# but L1 data product should not be persisted
res,_ = self.container.resource_registry.find_resources(RT.DataProduct, name='Conductivity L1', id_only=True)
self.assertEquals(1, len(res))
dpid = res[0]
streams, _ = self.container.resource_registry.find_objects(dpid, PRED.hasStream, RT.Stream, True)
self.assertEquals(1, len(streams))
self.assertTrue(streams)
self.assertFalse(self.ingestion_management.is_persisted(streams[0]))
res,_ = self.container.resource_registry.find_resources(RT.InstrumentSite, id_only=False)
self.assertTrue(len(res) > 1)
found = False
for site in res:
if site.name=='Logical instrument 1 Demo':
self.assertFalse(found, msg='Found more than one InstrumentSite "Logical instrument 1 Demo" -- should have preloaded one')
found = True
self.assertFalse(site.constraint_list is None)
self.assertEquals(2, len(site.constraint_list))
con = site.constraint_list[0]
self.assertTrue(math.fabs(con.geospatial_latitude_limit_north-32.88)<.01)
self.assertTrue(math.fabs(con.geospatial_longitude_limit_east+117.23)<.01)
con = site.constraint_list[1]
self.assertEquals('TemporalBounds', con.type_)
# check that coordinate system was loaded
self.assertFalse(site.coordinate_reference_system is None)
self.assertTrue(found, msg='Did not find InstrumentSite "Logical instrument 1 Demo" -- should have been preloaded')
# check that InstrumentDevice contacts are loaded
res,_ = self.container.resource_registry.find_resources(RT.InstrumentDevice, name='CTD Simulator 1 Demo', id_only=False)
self.assertTrue(len(res) == 1)
self.assertTrue(len(res[0].contacts)==2)
self.assertEquals('Ampe', res[0].contacts[0].individual_name_family)
# check has attachments
attachments,_ = self.container.resource_registry.find_attachments(res[0]._id)
self.assertTrue(len(attachments)>0)
# check for platform agents
res,_ = self.container.resource_registry.find_resources(RT.PlatformAgent, id_only=False)
self.assertTrue(len(res)>0)
# check for platform agents
res,_ = self.container.resource_registry.find_resources(RT.PlatformAgentInstance, id_only=False)
self.assertTrue(len(res)>0)
class TestDataProductManagementServiceIntegration(IonIntegrationTestCase):
def setUp(self):
# Start container
#print 'instantiating container'
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dpsc_cli = DataProductManagementServiceClient()
self.rrclient = ResourceRegistryServiceClient()
self.damsclient = DataAcquisitionManagementServiceClient()
self.pubsubcli = PubsubManagementServiceClient()
self.ingestclient = IngestionManagementServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.unsc = UserNotificationServiceClient()
self.data_retriever = DataRetrieverServiceClient()
#------------------------------------------
# Create the environment
#------------------------------------------
self.stream_def_id = self.pubsubcli.create_stream_definition(
name='SBE37_CDM')
self.process_definitions = {}
ingestion_worker_definition = ProcessDefinition(
name='ingestion worker')
ingestion_worker_definition.executable = {
'module':
'ion.processes.data.ingestion.science_granule_ingestion_worker',
'class': 'ScienceGranuleIngestionWorker'
}
process_definition_id = self.process_dispatcher.create_process_definition(
process_definition=ingestion_worker_definition)
self.process_definitions['ingestion_worker'] = process_definition_id
self.pids = []
self.exchange_points = []
self.exchange_names = []
#------------------------------------------------------------------------------------------------
# First launch the ingestors
#------------------------------------------------------------------------------------------------
self.exchange_space = 'science_granule_ingestion'
self.exchange_point = 'science_data'
config = DotDict()
config.process.datastore_name = 'datasets'
config.process.queue_name = self.exchange_space
self.exchange_names.append(self.exchange_space)
self.exchange_points.append(self.exchange_point)
pid = self.process_dispatcher.schedule_process(
self.process_definitions['ingestion_worker'], configuration=config)
log.debug("the ingestion worker process id: %s", pid)
self.pids.append(pid)
self.addCleanup(self.cleaning_up)
def cleaning_up(self):
for pid in self.pids:
log.debug("number of pids to be terminated: %s", len(self.pids))
try:
self.process_dispatcher.cancel_process(pid)
log.debug("Terminated the process: %s", pid)
except:
log.debug("could not terminate the process id: %s" % pid)
IngestionManagementIntTest.clean_subscriptions()
for xn in self.exchange_names:
xni = self.container.ex_manager.create_xn_queue(xn)
xni.delete()
for xp in self.exchange_points:
xpi = self.container.ex_manager.create_xp(xp)
xpi.delete()
def get_datastore(self, dataset_id):
dataset = self.dataset_management.read_dataset(dataset_id)
datastore_name = dataset.datastore_name
datastore = self.container.datastore_manager.get_datastore(
datastore_name, DataStore.DS_PROFILE.SCIDATA)
return datastore
@attr('EXT')
@attr('PREP')
def test_create_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
parameter_dictionary = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict')
ctd_stream_def_id = self.pubsubcli.create_stream_definition(
name='Simulated CTD data',
parameter_dictionary_id=parameter_dictionary._id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp')
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 10.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 10.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -10.0
dp_obj.ooi_product_name = "PRODNAME"
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product(
data_product=dp_obj, stream_definition_id=ctd_stream_def_id)
# Assert that the data product has an associated stream at this stage
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream,
RT.Stream, True)
self.assertNotEquals(len(stream_ids), 0)
# Assert that the data product has an associated stream def at this stage
stream_ids, _ = self.rrclient.find_objects(dp_id,
PRED.hasStreamDefinition,
RT.StreamDefinition, True)
self.assertNotEquals(len(stream_ids), 0)
self.dpsc_cli.activate_data_product_persistence(dp_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0)
log.debug('Created data product %s', dp_obj)
#------------------------------------------------------------------------------------------------
# test creating a new data product with a stream definition
#------------------------------------------------------------------------------------------------
log.debug('Creating new data product with a stream definition')
dp_obj = IonObject(RT.DataProduct,
name='DP2',
description='some new dp')
dp_id2 = self.dpsc_cli.create_data_product(dp_obj, ctd_stream_def_id)
self.dpsc_cli.activate_data_product_persistence(dp_id2)
log.debug('new dp_id = %s' % dp_id2)
#------------------------------------------------------------------------------------------------
#make sure data product is associated with stream def
#------------------------------------------------------------------------------------------------
streamdefs = []
streams, _ = self.rrclient.find_objects(dp_id2, PRED.hasStream,
RT.Stream, True)
for s in streams:
log.debug("Checking stream %s" % s)
sdefs, _ = self.rrclient.find_objects(s, PRED.hasStreamDefinition,
RT.StreamDefinition, True)
for sd in sdefs:
log.debug("Checking streamdef %s" % sd)
streamdefs.append(sd)
self.assertIn(ctd_stream_def_id, streamdefs)
group_names = self.dpsc_cli.get_data_product_group_list()
self.assertIn("PRODNAME", group_names)
# test reading a non-existent data product
log.debug('reading non-existent data product')
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product('some_fake_id')
# update a data product (tests read also)
log.debug('Updating data product')
# first get the existing dp object
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# now tweak the object
dp_obj.description = 'the very first dp'
dp_obj.geospatial_bounds.geospatial_latitude_limit_north = 20.0
dp_obj.geospatial_bounds.geospatial_latitude_limit_south = -20.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_east = 20.0
dp_obj.geospatial_bounds.geospatial_longitude_limit_west = -20.0
# now write the dp back to the registry
update_result = self.dpsc_cli.update_data_product(dp_obj)
# now get the dp back to see if it was updated
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertEquals(dp_obj.description, 'the very first dp')
self.assertEquals(dp_obj.geospatial_point_center.lat, 0.0)
log.debug('Updated data product %s', dp_obj)
#test extension
extended_product = self.dpsc_cli.get_data_product_extension(dp_id)
self.assertEqual(dp_id, extended_product._id)
self.assertEqual(
ComputedValueAvailability.PROVIDED,
extended_product.computed.product_download_size_estimated.status)
self.assertEqual(
0, extended_product.computed.product_download_size_estimated.value)
self.assertEqual(ComputedValueAvailability.PROVIDED,
extended_product.computed.parameters.status)
#log.debug("test_create_data_product: parameters %s" % extended_product.computed.parameters.value)
def ion_object_encoder(obj):
return obj.__dict__
#test prepare for create
data_product_data = self.dpsc_cli.prepare_data_product_support()
#print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2)
self.assertEqual(data_product_data._id, "")
self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport)
self.assertEqual(
len(data_product_data.associations['StreamDefinition'].resources),
2)
self.assertEqual(
len(data_product_data.associations['Dataset'].resources), 0)
self.assertEqual(
len(data_product_data.associations['StreamDefinition'].
associated_resources), 0)
self.assertEqual(
len(data_product_data.associations['Dataset'].associated_resources
), 0)
#test prepare for update
data_product_data = self.dpsc_cli.prepare_data_product_support(dp_id)
#print simplejson.dumps(data_product_data, default=ion_object_encoder, indent= 2)
self.assertEqual(data_product_data._id, dp_id)
self.assertEqual(data_product_data.type_, OT.DataProductPrepareSupport)
self.assertEqual(
len(data_product_data.associations['StreamDefinition'].resources),
2)
self.assertEqual(
len(data_product_data.associations['Dataset'].resources), 1)
self.assertEqual(
len(data_product_data.associations['StreamDefinition'].
associated_resources), 1)
self.assertEqual(
data_product_data.associations['StreamDefinition'].
associated_resources[0].s, dp_id)
self.assertEqual(
len(data_product_data.associations['Dataset'].associated_resources
), 1)
self.assertEqual(
data_product_data.associations['Dataset'].associated_resources[0].
s, dp_id)
# now 'delete' the data product
log.debug("deleting data product: %s" % dp_id)
self.dpsc_cli.delete_data_product(dp_id)
# Assert that there are no associated streams leftover after deleting the data product
stream_ids, assoc_ids = self.rrclient.find_objects(
dp_id, PRED.hasStream, RT.Stream, True)
self.assertEquals(len(stream_ids), 0)
self.assertEquals(len(assoc_ids), 0)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
with self.assertRaises(NotFound):
dp_obj = self.dpsc_cli.read_data_product(dp_id)
# Get the events corresponding to the data product
ret = self.unsc.get_recent_events(resource_id=dp_id)
events = ret.value
for event in events:
log.debug("event time: %s" % event.ts_created)
self.assertTrue(len(events) > 0)
def test_data_product_stream_def(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(
name='Simulated CTD data', parameter_dictionary_id=pdict_id)
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp')
dp_id = self.dpsc_cli.create_data_product(
data_product=dp_obj, stream_definition_id=ctd_stream_def_id)
stream_def_id = self.dpsc_cli.get_data_product_stream_definition(dp_id)
self.assertEquals(ctd_stream_def_id, stream_def_id)
def test_derived_data_product(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(
name='ctd parsed', parameter_dictionary_id=pdict_id)
self.addCleanup(self.pubsubcli.delete_stream_definition,
ctd_stream_def_id)
dp = DataProduct(name='Instrument DP')
dp_id = self.dpsc_cli.create_data_product(
dp, stream_definition_id=ctd_stream_def_id)
self.addCleanup(self.dpsc_cli.force_delete_data_product, dp_id)
self.dpsc_cli.activate_data_product_persistence(dp_id)
self.addCleanup(self.dpsc_cli.suspend_data_product_persistence, dp_id)
dataset_ids, _ = self.rrclient.find_objects(subject=dp_id,
predicate=PRED.hasDataset,
id_only=True)
if not dataset_ids:
raise NotFound("Data Product %s dataset does not exist" %
str(dp_id))
dataset_id = dataset_ids[0]
# Make the derived data product
simple_stream_def_id = self.pubsubcli.create_stream_definition(
name='TEMPWAT stream def',
parameter_dictionary_id=pdict_id,
available_fields=['time', 'temp'])
tempwat_dp = DataProduct(name='TEMPWAT',
category=DataProductTypeEnum.DERIVED)
tempwat_dp_id = self.dpsc_cli.create_data_product(
tempwat_dp,
stream_definition_id=simple_stream_def_id,
parent_data_product_id=dp_id)
self.addCleanup(self.dpsc_cli.delete_data_product, tempwat_dp_id)
# Check that the streams associated with the data product are persisted with
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream,
RT.Stream, True)
for stream_id in stream_ids:
self.assertTrue(self.ingestclient.is_persisted(stream_id))
stream_id = stream_ids[0]
route = self.pubsubcli.read_stream_route(stream_id=stream_id)
rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
rdt['pressure'] = np.arange(20)
publisher = StandaloneStreamPublisher(stream_id, route)
dataset_modified = Event()
def cb(*args, **kwargs):
dataset_modified.set()
es = EventSubscriber(event_type=OT.DatasetModified,
callback=cb,
origin=dataset_id,
auto_delete=True)
es.start()
self.addCleanup(es.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
tempwat_dataset_ids, _ = self.rrclient.find_objects(tempwat_dp_id,
PRED.hasDataset,
id_only=True)
tempwat_dataset_id = tempwat_dataset_ids[0]
granule = self.data_retriever.retrieve(
tempwat_dataset_id, delivery_format=simple_stream_def_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_equal(rdt['time'], np.arange(20))
self.assertEquals(set(rdt.fields), set(['time', 'temp']))
def test_activate_suspend_data_product(self):
#------------------------------------------------------------------------------------------------
# create a stream definition for the data from the ctd simulator
#------------------------------------------------------------------------------------------------
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
ctd_stream_def_id = self.pubsubcli.create_stream_definition(
name='Simulated CTD data', parameter_dictionary_id=pdict_id)
log.debug("Created stream def id %s" % ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# test creating a new data product w/o a stream definition
#------------------------------------------------------------------------------------------------
# Construct temporal and spatial Coordinate Reference System objects
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp')
log.debug("Created an IonObject for a data product: %s" % dp_obj)
#------------------------------------------------------------------------------------------------
# Create a set of ParameterContext objects to define the parameters in the coverage, add each to the ParameterDictionary
#------------------------------------------------------------------------------------------------
dp_id = self.dpsc_cli.create_data_product(
data_product=dp_obj, stream_definition_id=ctd_stream_def_id)
#------------------------------------------------------------------------------------------------
# Subscribe to persist events
#------------------------------------------------------------------------------------------------
queue = gevent.queue.Queue()
def info_event_received(message, headers):
queue.put(message)
es = EventSubscriber(event_type=OT.InformationContentStatusEvent,
callback=info_event_received,
origin=dp_id,
auto_delete=True)
es.start()
self.addCleanup(es.stop)
#------------------------------------------------------------------------------------------------
# test activate and suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.activate_data_product_persistence(dp_id)
dp_obj = self.dpsc_cli.read_data_product(dp_id)
self.assertIsNotNone(dp_obj)
dataset_ids, _ = self.rrclient.find_objects(subject=dp_id,
predicate=PRED.hasDataset,
id_only=True)
if not dataset_ids:
raise NotFound("Data Product %s dataset does not exist" %
str(dp_id))
dataset_id = dataset_ids[0]
# Check that the streams associated with the data product are persisted with
stream_ids, _ = self.rrclient.find_objects(dp_id, PRED.hasStream,
RT.Stream, True)
for stream_id in stream_ids:
self.assertTrue(self.ingestclient.is_persisted(stream_id))
stream_id = stream_ids[0]
route = self.pubsubcli.read_stream_route(stream_id=stream_id)
rdt = RecordDictionaryTool(stream_definition_id=ctd_stream_def_id)
rdt['time'] = np.arange(20)
rdt['temp'] = np.arange(20)
publisher = StandaloneStreamPublisher(stream_id, route)
dataset_modified = Event()
def cb(*args, **kwargs):
dataset_modified.set()
es = EventSubscriber(event_type=OT.DatasetModified,
callback=cb,
origin=dataset_id,
auto_delete=True)
es.start()
self.addCleanup(es.stop)
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
#--------------------------------------------------------------------------------
# Now get the data in one chunk using an RPC Call to start_retreive
#--------------------------------------------------------------------------------
replay_data = self.data_retriever.retrieve(dataset_ids[0])
self.assertIsInstance(replay_data, Granule)
log.debug(
"The data retriever was able to replay the dataset that was attached to the data product "
"we wanted to be persisted. Therefore the data product was indeed persisted with "
"otherwise we could not have retrieved its dataset using the data retriever. Therefore "
"this demonstration shows that L4-CI-SA-RQ-267 is satisfied: 'Data product management shall persist data products'"
)
data_product_object = self.rrclient.read(dp_id)
self.assertEquals(data_product_object.name, 'DP1')
self.assertEquals(data_product_object.description, 'some new dp')
log.debug(
"Towards L4-CI-SA-RQ-308: 'Data product management shall persist data product metadata'. "
" Attributes in create for the data product obj, name= '%s', description='%s', match those of object from the "
"resource registry, name='%s', desc='%s'" %
(dp_obj.name, dp_obj.description, data_product_object.name,
data_product_object.description))
#------------------------------------------------------------------------------------------------
# test suspend data product persistence
#------------------------------------------------------------------------------------------------
self.dpsc_cli.suspend_data_product_persistence(dp_id)
dataset_modified.clear()
rdt['time'] = np.arange(20, 40)
publisher.publish(rdt.to_granule())
self.assertFalse(dataset_modified.wait(2))
self.dpsc_cli.activate_data_product_persistence(dp_id)
dataset_modified.clear()
publisher.publish(rdt.to_granule())
self.assertTrue(dataset_modified.wait(30))
granule = self.data_retriever.retrieve(dataset_id)
rdt = RecordDictionaryTool.load_from_granule(granule)
np.testing.assert_array_almost_equal(rdt['time'], np.arange(40))
dataset_ids, _ = self.rrclient.find_objects(dp_id,
PRED.hasDataset,
id_only=True)
self.assertEquals(len(dataset_ids), 1)
self.dpsc_cli.suspend_data_product_persistence(dp_id)
self.dpsc_cli.force_delete_data_product(dp_id)
# now try to get the deleted dp object
with self.assertRaises(NotFound):
dp_obj = self.rrclient.read(dp_id)
info_event_counter = 0
runtime = 0
starttime = time.time()
caught_events = []
#check that the four InfoStatusEvents were received
while info_event_counter < 4 and runtime < 60:
a = queue.get(timeout=60)
caught_events.append(a)
info_event_counter += 1
runtime = time.time() - starttime
self.assertEquals(info_event_counter, 4)
class TestInstrumentDataIngestion(IonIntegrationTestCase):
"""
Tests for data ingestion from an instrument. It is basically the
test_poll from test_instrument_agent (as of 7/23/12) with
the additional preparation of the ingestion management service and the
verification that the published granules are persisted.
"""
def setUp(self):
self.resource_registry = ResourceRegistryServiceClient()
self.ingestion_management = IngestionManagementServiceClient()
self._support = DriverIntegrationTestSupport(DRV_MOD,
DRV_CLS,
DEV_ADDR,
DEV_PORT,
DELIM,
WORK_DIR)
# Start port agent, add stop to cleanup.
self._pagent = None
self._start_pagent()
self.addCleanup(self._support.stop_pagent)
# Start container.
self._start_container()
# Bring up services in a deploy file (no need to message)
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
# Start data suscribers, add stop to cleanup.
# Define stream_config.
self._no_samples = None
self._async_data_result = AsyncResult()
self._data_greenlets = []
self._stream_config = {}
self._samples_received = []
self._data_subscribers = []
self._start_data_subscribers()
self.addCleanup(self._stop_data_subscribers)
self.prepare_ingestion()
# Start event subscribers, add stop to cleanup.
self._no_events = None
self._async_event_result = AsyncResult()
self._events_received = []
# Create agent config.
agent_config = {
'driver_config' : DVR_CONFIG,
'stream_config' : self._stream_config,
'agent' : {'resource_id': IA_RESOURCE_ID},
'test_mode' : True
}
# Start instrument agent.
self._ia_pid = None
log.debug("TestInstrumentDataIngestion.setup(): starting IA.")
container_client = ContainerAgentClient(node=self.container.node,
name=self.container.name)
self._ia_pid = container_client.spawn_process(name=IA_NAME,
module=IA_MOD,
cls=IA_CLS,
config=agent_config)
log.info('Agent pid=%s.', str(self._ia_pid))
# Start a resource agent client to talk with the instrument agent.
self._ia_client = None
self._ia_client = ResourceAgentClient(IA_RESOURCE_ID, process=FakeProcess())
log.info('Got ia client %s.', str(self._ia_client))
def _start_pagent(self):
"""
Construct and start the port agent.
"""
port = self._support.start_pagent()
# Configure driver to use port agent port number.
DVR_CONFIG['comms_config'] = {
'addr' : 'localhost',
'port' : port
}
def _start_data_subscribers(self):
"""
"""
# Create a pubsub client to create streams.
pubsub_client = PubsubManagementServiceClient(node=self.container.node)
# A callback for processing subscribed-to data.
def consume_data(message, headers):
log.info('Subscriber received data message: type(message)=%s.', str(type(message)))
log.info('Subscriber received data message: %s.', str(message))
self._samples_received.append(message)
if self._no_samples and self._no_samples == len(self._samples_received):
self._async_data_result.set()
# Create a stream subscriber registrar to create subscribers.
subscriber_registrar = StreamSubscriberRegistrar(process=self.container,
container=self.container)
# Create streams and subscriptions for each stream named in driver.
self._stream_config = {}
self._data_subscribers = []
for stream_name in PACKET_CONFIG:
stream_def = ctd_stream_definition(stream_id=None)
stream_def_id = pubsub_client.create_stream_definition(
container=stream_def)
stream_id = pubsub_client.create_stream(
name=stream_name,
stream_definition_id=stream_def_id,
original=True,
encoding='ION R2')
taxy = get_taxonomy(stream_name)
stream_config = dict(
id=stream_id,
taxonomy=taxy.dump()
)
self._stream_config[stream_name] = stream_config
# Create subscriptions for each stream.
exchange_name = '%s_queue' % stream_name
sub = subscriber_registrar.create_subscriber(exchange_name=exchange_name,
callback=consume_data)
self._listen(sub)
self._data_subscribers.append(sub)
query = StreamQuery(stream_ids=[stream_id])
sub_id = pubsub_client.create_subscription(
query=query, exchange_name=exchange_name, exchange_point='science_data')
pubsub_client.activate_subscription(sub_id)
def _listen(self, sub):
"""
Pass in a subscriber here, this will make it listen in a background greenlet.
"""
gl = spawn(sub.listen)
self._data_greenlets.append(gl)
sub._ready_event.wait(timeout=5)
return gl
def _stop_data_subscribers(self):
"""
Stop the data subscribers on cleanup.
"""
for sub in self._data_subscribers:
sub.stop()
for gl in self._data_greenlets:
gl.kill()
def assertRawSampleDict(self, val):
"""
Verify the value is a raw sample dictionary for the sbe37.
"""
#{'stream_name': 'raw', 'blob': ['20.8074,80.04590, 761.394, 33.5658, 1506.076, 01 Feb 2001, 01:01:00'], 'time': [1342392781.61211]}
self.assertTrue(isinstance(val, dict))
self.assertTrue(val.has_key('blob'))
self.assertTrue(val.has_key('time'))
blob = val['blob'][0]
time = val['time'][0]
self.assertTrue(isinstance(blob, str))
self.assertTrue(isinstance(time, float))
def assertParsedSampleDict(self, val):
"""
Verify the value is a parsed sample dictionary for the sbe37.
"""
#{'p': [-6.945], 'c': [0.08707], 't': [20.002], 'time': [1333752198.450622]}
self.assertTrue(isinstance(val, dict))
self.assertTrue(val.has_key('c'))
self.assertTrue(val.has_key('t'))
self.assertTrue(val.has_key('p'))
self.assertTrue(val.has_key('time'))
c = val['c'][0]
t = val['t'][0]
p = val['p'][0]
time = val['time'][0]
self.assertTrue(isinstance(c, float))
self.assertTrue(isinstance(t, float))
self.assertTrue(isinstance(p, float))
self.assertTrue(isinstance(time, float))
def assertSample(self, sample):
"""
Verify a sample retrieved from the sbe37, which is expected to be a
dict of dicts {'raw':raw_sample, 'parsed':parsed_sample}.
"""
self.assertTrue(isinstance(sample, dict))
raw_sample = sample['raw']
parsed_sample = sample['parsed']
self.assertParsedSampleDict(parsed_sample)
self.assertRawSampleDict(raw_sample)
def get_ingestion_config(self):
#
# From test_dm_end_2_end.py as of 7/23/12
#
#--------------------------------------------------------------------------------
# Grab the ingestion configuration from the resource registry
#--------------------------------------------------------------------------------
# The ingestion configuration should have been created by the bootstrap service
# which is configured through r2deploy.yml
ingest_configs, _ = self.resource_registry.find_resources(
restype=RT.IngestionConfiguration,id_only=True)
return ingest_configs[0]
def prepare_ingestion(self):
#
# Takes pieces from test_dm_end_2_end.py as of 7/23/12
#
# Get the ingestion configuration from the resource registry
self.ingest_config_id = ingest_config_id = self.get_ingestion_config()
# to keep the (stream_id, dataset_id) associated with each stream_name
self.dataset_ids = {}
for stream_name, stream_config in self._stream_config.iteritems():
stream_id = stream_config['id']
dataset_id = self.ingestion_management.persist_data_stream(
stream_id=stream_id,
ingestion_configuration_id=ingest_config_id)
log.info("persisting stream_name=%s (stream_id=%s): dataset_id=%s" % (
stream_name, stream_id, dataset_id))
self.assertTrue(self.ingestion_management.is_persisted(stream_id))
self.dataset_ids[stream_name] = (stream_id, dataset_id)
def verify_granules_persisted(self):
#
# takes elements from ingestion_management_test.py as of 7/23/12
#
ingest_config_id = self.ingest_config_id
for stream_name, (stream_id, dataset_id) in self.dataset_ids.iteritems():
assoc = self.resource_registry.find_associations(
subject=ingest_config_id, predicate=PRED.hasSubscription)
sub = self.resource_registry.read(assoc[0].o)
self.assertTrue(sub.is_active)
dataset = self.resource_registry.read(dataset_id)
self.assertIsInstance(dataset, DataSet)
log.info("Data persisted for stream_name=%s (stream_id=%s, "
"dataset_id=%s) dataset=%s" % (stream_name, stream_id, dataset_id, dataset))
def test_poll_and_verify_granules_persisted(self):
#
# As test_instrument_agent.py:TestInstrumentAgent.test_poll with
# verification that data are persisted.
#
self._test_poll()
self.verify_granules_persisted()
def _test_poll(self):
#
# Same as test_instrument_agent.py:TestInstrumentAgent.test_poll as of 7/23/12
#
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
cmd = AgentCommand(command='initialize')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.INACTIVE)
cmd = AgentCommand(command='go_active')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.IDLE)
cmd = AgentCommand(command='run')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.OBSERVATORY)
# Lets call acquire_sample 3 times, so we should get 6 samples =
# 3 raw samples + 3 parsed samples
self._no_samples = 6
# Poll for a few samples.
cmd = AgentCommand(command='acquire_sample')
reply = self._ia_client.execute(cmd)
log.debug("acquire_sample reply.result=%s" % reply.result)
self.assertSample(reply.result)
cmd = AgentCommand(command='acquire_sample')
reply = self._ia_client.execute(cmd)
self.assertSample(reply.result)
cmd = AgentCommand(command='acquire_sample')
reply = self._ia_client.execute(cmd)
self.assertSample(reply.result)
# Assert we got 6 samples.
self._async_data_result.get(timeout=10)
self.assertEquals(len(self._samples_received), 6)
cmd = AgentCommand(command='reset')
retval = self._ia_client.execute_agent(cmd)
cmd = AgentCommand(command='get_current_state')
retval = self._ia_client.execute_agent(cmd)
state = retval.result
self.assertEqual(state, InstrumentAgentState.UNINITIALIZED)
class TestLoader(IonIntegrationTestCase):
def setUp(self):
# Start container
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.ingestion_management = IngestionManagementServiceClient()
def assert_can_load(self,
scenarios,
loadui=False,
loadooi=False,
path=TESTED_DOC,
ui_path='default'):
""" perform preload for given scenarios and raise exception if there is a problem with the data """
config = dict(op="load",
scenario=scenarios,
attachments="res/preload/r2_ioc/attachments",
loadui=loadui,
loadooi=loadooi,
path=path,
ui_path=ui_path,
assets='res/preload/r2_ioc/ooi_assets',
bulk=loadooi,
ooiexclude='DataProduct,DataProductLink')
self.container.spawn_process("Loader",
"ion.processes.bootstrap.ion_loader",
"IONLoader",
config=config)
@attr('PRELOAD')
def test_ui_valid(self):
""" make sure UI assets are valid using DEFAULT_UI_ASSETS = 'https://userexperience.oceanobservatories.org/database-exports/' """
self.assert_can_load("BASE,BETA", loadui=True, ui_path='default')
@attr('PRELOAD')
def test_ui_candidates_valid(self):
""" make sure UI assets are valid using DEFAULT_UI_ASSETS = 'https://userexperience.oceanobservatories.org/database-exports/Candidates' """
self.assert_can_load("BASE,BETA", loadui=True, ui_path='candidate')
@attr('PRELOAD')
def test_assets_valid(self):
""" make sure can load asset DB """
self.assert_can_load("BASE,BETA,DEVS", path='master', loadooi=True)
@attr('PRELOAD')
def test_alpha_valid(self):
""" make sure R2_DEMO scenario in master google doc
is valid and self-contained (doesn't rely on rows from other scenarios except BASE and BETA)
NOTE: test will pass/fail based on current google doc, not just code changes.
"""
self.assert_can_load("BASE,BETA,ALPHA_SYS", path='master')
@attr('PRELOAD')
def test_beta_valid(self):
""" make sure R2_DEMO scenario in master google doc
is valid and self-contained (doesn't rely on rows from other scenarios except BASE and BETA)
NOTE: test will pass/fail based on current google doc, not just code changes.
"""
self.assert_can_load("BASE,BETA,BETA_SYS", path='master')
@attr('PRELOAD')
def test_devs_valid(self):
""" make sure DEVS scenario in master google doc
is valid and self-contained (doesn't rely on rows from other scenarios except BASE and BETA)
NOTE: test will pass/fail based on current google doc, not just code changes.
"""
self.assert_can_load("BASE,BETA,DEVS", path='master')
def find_object_by_name(self, name, resource_type):
objects, _ = self.container.resource_registry.find_resources(
resource_type, id_only=False)
self.assertGreaterEqual(len(objects), 1)
filtered_objs = [obj for obj in objects if obj.name == name]
self.assertEquals(len(filtered_objs), 1)
return filtered_objs[0]
@attr('UNIT', group='loader')
def test_parse_alarms(self):
loader = IONLoader()
out = loader._parse_alarm('5<temp<10')
self.assertEqual('<', out['lower_rel_op'])
self.assertEqual(5, out['lower_bound'])
self.assertEqual('<', out['upper_rel_op'])
self.assertEqual(10, out['upper_bound'])
self.assertEqual('temp', out['value_id'])
out = loader._parse_alarm('5<=temp<10')
self.assertEqual('<=', out['lower_rel_op'])
self.assertEqual(5, out['lower_bound'])
self.assertEqual('<', out['upper_rel_op'])
self.assertEqual(10, out['upper_bound'])
self.assertEqual('temp', out['value_id'])
out = loader._parse_alarm('5<temp<=10')
self.assertEqual('<', out['lower_rel_op'])
self.assertEqual(5, out['lower_bound'])
self.assertEqual('<=', out['upper_rel_op'])
self.assertEqual(10, out['upper_bound'])
self.assertEqual('temp', out['value_id'])
out = loader._parse_alarm('5<=temp<=10')
self.assertEqual('<=', out['lower_rel_op'])
self.assertEqual(5, out['lower_bound'])
self.assertEqual('<=', out['upper_rel_op'])
self.assertEqual(10, out['upper_bound'])
self.assertEqual('temp', out['value_id'])
out = loader._parse_alarm('5<temp')
self.assertEqual('<', out['lower_rel_op'])
self.assertEqual(5, out['lower_bound'])
self.assertEqual(3, len(out), msg='value: %r' % out)
self.assertEqual('temp', out['value_id'])
out = loader._parse_alarm('5<=temp')
self.assertEqual('<=', out['lower_rel_op'])
self.assertEqual(5, out['lower_bound'])
self.assertEqual('temp', out['value_id'])
self.assertEqual(3, len(out))
out = loader._parse_alarm('temp<10')
self.assertEqual('<', out['upper_rel_op'])
self.assertEqual(10, out['upper_bound'])
self.assertEqual('temp', out['value_id'])
self.assertEqual(3, len(out))
out = loader._parse_alarm('temp<=10')
self.assertEqual('<=', out['upper_rel_op'])
self.assertEqual(10, out['upper_bound'])
self.assertEqual('temp', out['value_id'])
self.assertEqual(3, len(out))
@attr('INT', group='loader')
@attr('SMOKE', group='loader')
def test_row_values(self):
""" use only rows from NOSE scenario for specific names and details included in this test.
rows in NOSE may rely on entries in BASE and BETA scenarios,
but should not specifically test values from those scenarios.
"""
# first make sure this scenario loads successfully
self.assert_can_load("BASE,BETA,NOSE")
# check for an Org
org = self.find_object_by_name('CASPER', RT.Org)
self.assertFalse(org.contacts is None)
self.assertEquals('Userbrough', org.contacts[0].individual_name_family)
self.assertEquals('primary', org.contacts[0].roles[0])
# check data product
dp = self.find_object_by_name('Test DP L0 CTD', RT.DataProduct)
formats = dp.available_formats
self.assertEquals(2, len(formats))
self.assertEquals('csv', formats[0])
# should be persisted
streams, _ = self.container.resource_registry.find_objects(
dp._id, PRED.hasStream, RT.Stream, True)
self.assertTrue(streams)
self.assertEquals(1, len(streams))
self.assertTrue(self.ingestion_management.is_persisted(streams[0]))
self.assertAlmostEqual(
32.88237,
dp.geospatial_bounds.geospatial_latitude_limit_north,
places=3)
# but L1 data product should not be persisted
dp = self.find_object_by_name('Test DP L1 conductivity',
RT.DataProduct)
streams, _ = self.container.resource_registry.find_objects(
dp._id, PRED.hasStream, RT.Stream, True)
self.assertEquals(1, len(streams))
self.assertTrue(streams)
self.assertFalse(self.ingestion_management.is_persisted(streams[0]))
site = self.find_object_by_name('Test Instrument Site',
RT.InstrumentSite)
self.assertFalse(site.constraint_list is None)
self.assertEquals(2, len(site.constraint_list))
con = site.constraint_list[0]
self.assertAlmostEqual(32.88237,
con.geospatial_latitude_limit_north,
places=3)
self.assertAlmostEqual(-117.23214,
con.geospatial_longitude_limit_east,
places=3)
con = site.constraint_list[1]
self.assertEquals('TemporalBounds', con.type_)
# check that coordinate system was loaded
self.assertFalse(site.coordinate_reference_system is None)
# check that InstrumentDevice contacts are loaded
dev = self.find_object_by_name('Unit Test SMB37', RT.InstrumentDevice)
self.assertTrue(len(dev.contacts) == 2)
self.assertEquals('Userbrough', dev.contacts[0].individual_name_family)
# check has attachments
attachments = self.container.resource_registry.find_attachments(
dev._id)
self.assertTrue(len(attachments) > 0)
# check for platform agents
agent = self.find_object_by_name('Unit Test Platform Agent',
RT.PlatformAgent)
self.assertEquals(2, len(agent.stream_configurations))
parsed = agent.stream_configurations[1]
# self.assertEquals('platform_eng_parsed', parsed.parameter_dictionary_name)
self.assertEquals('ctd_parsed_param_dict',
parsed.parameter_dictionary_name)
# check that alarm was added to StreamConfig
self.assertEquals(1,
len(parsed.alarms),
msg='alarms: %r' % parsed.alarms)
self.assertEquals('temp', parsed.alarms[0]['kwargs']['value_id'])
# check for platform agents
self.find_object_by_name('Unit Test Platform Agent Instance',
RT.PlatformAgentInstance)
# check for platform model boolean values
model = self.find_object_by_name('Nose Testing Platform Model',
RT.PlatformModel)
self.assertEquals(True, model.shore_networked)
self.assertNotEqual('str', model.shore_networked.__class__.__name__)
# check for data process definition
self.find_object_by_name("Logical Transform Definition",
RT.DataProcessDefinition)
iai = self.find_object_by_name("Test InstrumentAgentInstance",
RT.InstrumentAgentInstance)
self.assertEqual(
iai.startup_config, {
'SCHEDULER': {
'VERSION': 3.0,
'CLOCK_SYNC': 48.2
},
'PARAMETERS': {
'TXWAVEBURST': 'false',
'TXREALTIME': True
}
})
orgs, _ = self.container.resource_registry.find_subjects(
RT.Org, PRED.hasResource, iai._id, True)
self.assertEqual(1, len(orgs))
self.assertEqual(org._id, orgs[0])
