def test_subobject_search(self):
contact = ContactInformation()
contact.email = '*****@*****.**'
contact.individual_name_family = 'Tester'
contact.individual_names_given = 'Intern'
dp = DataProduct(name='example')
dp.contacts.append(contact)
dp_id,_ = self.rr.create(dp)
#--------------------------------------------------------------------------------
# Example using the full field name
#--------------------------------------------------------------------------------
search_string = 'search "contacts.email" is "*****@*****.**" from "data_products"'
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
self.assertEquals(results[0]['_source'].name, 'example')
#--------------------------------------------------------------------------------
# Example using a sub-object's field name (ambiguous searching)
#--------------------------------------------------------------------------------
search_string = 'search "individual_names_given" is "Intern" from "data_products"'
results = self.poll(9, self.discovery.parse, search_string)
self.assertIsNotNone(results, 'Results not found')
self.assertTrue(results[0]['_id'] == dp_id)
self.assertEquals(results[0]['_source'].name, 'example')
def test__constraints_for_new_request_no_new_data(self,
list_file_info_mock):
edres = ExternalDataset(name='test_ed_res',
dataset_description=DatasetDescription(),
update_description=UpdateDescription(),
contact=ContactInformation())
old_list = [
('test_data/ruv/RDLm_SEAB_2012_06_06_1200.ruv', 1339006638.0,
119066),
('test_data/ruv/RDLm_SEAB_2012_06_06_1300.ruv', 1339006629.0,
109316),
('test_data/ruv/RDLm_SEAB_2012_06_06_1400.ruv', 1339006521.0,
113411),
]
config = {
'external_dataset_res': edres,
'new_data_check': old_list,
'ds_params': {
'base_url':
'test_data/ruv',
'list_pattern':
'RDLm_SEAB_*.ruv',
'date_pattern':
'%Y %m %d %H %M',
'date_extraction_pattern':
'RDLm_SEAB_([\d]{4})_([\d]{2})_([\d]{2})_([\d]{2})([\d]{2}).ruv'
}
}
list_file_info_mock.return_value = old_list
with self.assertRaises(NoNewDataWarning):
RuvDataHandler._constraints_for_new_request(config)
def test__init_acquisition_cycle_ext_ds_res(self):
edres = ExternalDataset(name='test_ed_res',
dataset_description=DatasetDescription(),
update_description=UpdateDescription(),
contact=ContactInformation())
edres.dataset_description.parameters[
'dataset_path'] = 'test_data/usgs.nc'
config = {'external_dataset_res': edres}
NetcdfDataHandler._init_acquisition_cycle(config)
self.assertIn('dataset_object', config)
self.assertTrue(isinstance(config['dataset_object'], Dataset))
def create_external_dataset(self):
ds_name = 'hypm_01_wfp_eng_dataset'
dset = ExternalDataset(name=ds_name,
dataset_description=DatasetDescription(),
update_description=UpdateDescription(),
contact=ContactInformation())
dset.dataset_description.parameters['base_url'] = 'test_data'
dset.dataset_description.parameters['list_pattern'] = 'E*.HEX'
return self.data_acquisition_management.create_external_dataset(
external_dataset=dset)
def _create_tst_data_set_handler(self, key="DS_BASE"):
th, tn = tempfile.mkstemp(prefix=(key + "_"), suffix='.nc')
self._create_tst_data_set(th, tn, key)
ext_ds = ExternalDataset(name="test",
dataset_description=DatasetDescription(),
update_description=UpdateDescription(),
contact=ContactInformation())
ext_ds.dataset_description.parameters["dataset_path"] = tn
ext_ds.dataset_description.parameters["temporal_dimension"] = 'time'
dsh = DapExternalDataHandler(ext_dataset=ext_ds)
self._dsh_list[key] = dsh, tn
def test__init_acquisition_cycle_ext_ds_res(self, SlocumParser_mock):
edres = ExternalDataset(name='test_ed_res',
dataset_description=DatasetDescription(),
update_description=UpdateDescription(),
contact=ContactInformation())
edres.dataset_description.parameters[
'dataset_path'] = 'test_data/ru05-2012-021-0-0-sbd.dat'
config = {'external_dataset_res': edres}
SlocumDataHandler._init_acquisition_cycle(config)
SlocumParser_mock.assert_called_once_with(
edres.dataset_description.parameters['dataset_path'])
self.assertIn('parser', config)
self.assertEquals(config['parser'], SlocumParser_mock())
def test__init_acquisition_cycle_ext_ds_res(self, RuvParser_mock):
edres = ExternalDataset(name='test_ed_res',
dataset_description=DatasetDescription(),
update_description=UpdateDescription(),
contact=ContactInformation())
edres.dataset_description.parameters[
'dataset_path'] = 'test_data/RDLi_SEAB_2011_08_24_1600.ruv'
config = {'external_dataset_res': edres}
RuvDataHandler._init_acquisition_cycle(config)
RuvParser_mock.assert_called_once_with(
edres.dataset_description.parameters['dataset_path'])
self.assertIn('parser', config)
self.assertTrue(config['parser'], RuvParser_mock())
def create_external_dataset(self):
ds_name = 'slocum_dataset'
dset = ExternalDataset(name=ds_name,
dataset_description=DatasetDescription(),
update_description=UpdateDescription(),
contact=ContactInformation())
dset.dataset_description.parameters['base_url'] = 'test_data/slocum'
dset.dataset_description.parameters['list_pattern'] = 'ru05-*-sbd.dat'
dset.dataset_description.parameters['date_pattern'] = '%Y %j'
dset.dataset_description.parameters[
'date_extraction_pattern'] = 'ru05-([\d]{4})-([\d]{3})-\d-\d-sbd.dat'
return self.data_acquisition_management.create_external_dataset(
external_dataset=dset)
def test__init_acquisition_cycle_ext_ds_res(self):
edres = ExternalDataset(name='test_ed_res',
dataset_description=DatasetDescription(),
update_description=UpdateDescription(),
contact=ContactInformation())
edres.dataset_description.parameters['base_url'] = 'test_data/dir'
edres.dataset_description.parameters['list_pattern'] = 'test_filter'
config = {'external_dataset_res': edres}
RuvDataHandler._init_acquisition_cycle(config)
self.assertIn('ds_params', config)
ds_params = config['ds_params']
self.assertIn('base_url', ds_params)
self.assertEquals(ds_params['base_url'], 'test_data/dir')
self.assertIn('list_pattern', ds_params)
self.assertEquals(ds_params['list_pattern'], 'test_filter')
def test__get_data(self, RecordDictionaryTool_mock):
edres = ExternalDataset(name='test_ed_res',
dataset_description=DatasetDescription(),
update_description=UpdateDescription(),
contact=ContactInformation())
edres.dataset_description.parameters[
'dataset_path'] = 'test_data/usgs.nc'
edres.dataset_description.parameters['temporal_dimension'] = 'time'
edres.dataset_description.parameters['zonal_dimension'] = 'lat'
edres.dataset_description.parameters['meridional_dimension'] = 'lon'
edres.dataset_description.parameters['vertical_dimension'] = 'z'
edres.dataset_description.parameters['variables'] = [
'water_temperature'
]
config = {
'external_dataset_res':
edres,
'dataset_object':
Dataset(edres.dataset_description.parameters['dataset_path']),
'constraints': {
'count': 6
},
'max_records':
4,
'data_producer_id':
sentinel.dprod_id,
'param_dictionary':
sentinel.pdict,
'stream_def':
sentinel.stream_def_id
}
my_dict = {}
def setitem(name, val):
my_dict[name] = val
retval = MagicMock(spec=RecordDictionaryTool)
retval.__setitem__ = Mock(side_effect=setitem)
retval.to_granule.return_value = MagicMock(spec=Granule)
RecordDictionaryTool_mock.return_value = retval
for x in NetcdfDataHandler._get_data(config):
self.assertTrue(isinstance(x, Granule))
retval.to_granule.assert_any_call()
def read_external_data_provider(self, ds_id=''):
dprov = ExternalDataProvider(institution=Institution(),
contact=ContactInformation())
if ds_id == HFR or ds_id == HFR_LOCAL:
# dprov.institution.name = "HFRNET UCSD"
pass
elif ds_id == KOKAGG:
# dprov.institution.name = "University of Hawaii"
pass
elif ds_id == AST2:
# dprov.institution.name = "OOI CGSN"
dprov.contact.name = "Robert Weller"
dprov.contact.email = "*****@*****.**"
elif ds_id == SSTA:
# dprov.institution.name = "Remote Sensing Systems"
dprov.contact.email = "*****@*****.**"
else:
return None
return dprov
def test_constructor(self):
ext_ds = ExternalDataset(name="test",
dataset_description=DatasetDescription(),
update_description=UpdateDescription(),
contact=ContactInformation())
ext_ds.dataset_description.parameters["dataset_path"] = self._dsh_list[
"DS_BASE"][1]
ext_ds.dataset_description.parameters["temporal_dimension"] = 'time'
dsh = DapExternalDataHandler(ext_dataset=ext_ds)
self.assertTrue(type(dsh), DapExternalDataHandler)
dsrc = DataSource(name="test")
dsrc.connection_params["base_data_url"] = ""
dsh1 = DapExternalDataHandler(data_source=dsrc, ext_dataset=ext_ds)
self.assertTrue(type(dsh1), DapExternalDataHandler)
dsh2 = DapExternalDataHandler(data_source=dsrc,
ext_dataset=ext_ds,
BLOCK_SIZE=10)
self.assertEqual(dsh2._block_size, 10)
def read_data_source(self, ds_id=''):
dsrc = DataSource(protocol_type="DAP",
institution=Institution(),
contact=ContactInformation())
if ds_id == HFR:
dsrc.connection_params[
"base_data_url"] = "http://hfrnet.ucsd.edu:8080/thredds/dodsC/"
elif ds_id == HFR_LOCAL:
pass
elif ds_id == KOKAGG:
dsrc.connection_params[
"base_data_url"] = "http://oos.soest.hawaii.edu/thredds/dodsC/"
elif ds_id == AST2:
dsrc.connection_params[
"base_data_url"] = "http://ooi.whoi.edu/thredds/dodsC/"
dsrc.contact.name = "Rich Signell"
dsrc.contact.email = "*****@*****.**"
elif ds_id == SSTA:
dsrc.connection_params[
"base_data_url"] = "http://thredds1.pfeg.noaa.gov/thredds/dodsC/"
elif ds_id == GHPM:
pass
elif ds_id == COMP1:
pass
elif ds_id == COMP2:
pass
elif ds_id == COADS:
pass
elif ds_id == NCOM:
pass
elif ds_id == NAVO_GLIDER_LOCAL:
pass
elif ds_id == USGS:
pass
else:
return None
return dsrc
def test__constraints_for_new_request(self):
# ret = SlocumDataHandler._constraints_for_new_request({})
# self.assertIsInstance(ret, dict)
old_list = [
('test_data/slocum/ru05-2012-021-0-0-sbd.dat', 1337261358.0,
521081),
('test_data/slocum/ru05-2012-022-0-0-sbd.dat', 1337261358.0,
521081),
]
# old_list = None
edres = ExternalDataset(name='test_ed_res',
dataset_description=DatasetDescription(),
update_description=UpdateDescription(),
contact=ContactInformation())
config = {
'external_dataset_res': edres,
'new_data_check': old_list,
'ds_params': {
# These would be extracted from the dataset_description.parameters during _init_acquisition_cycle, but since that isn't called, just add them here
'base_url': 'test_data/slocum/',
'list_pattern':
'ru05-*-sbd.dat', # Appended to base to filter files; Either a shell style pattern (for filesystem) or regex (for http/ftp)
'date_pattern': '%Y %j',
'date_extraction_pattern':
'ru05-([\d]{4})-([\d]{3})-\d-\d-sbd.dat'
}
}
ret = SlocumDataHandler._constraints_for_new_request(config)
log.debug('test__constraints_for_new_request: {0}'.format(
ret['new_files']))
self.assertEqual(
ret['new_files'],
list_file_info(config['ds_params']['base_url'],
config['ds_params']['list_pattern']))
def _setup_resources(self):
# TODO: some or all of this (or some variation) should move to DAMS'
# Build the test resources for the dataset
dms_cli = DatasetManagementServiceClient()
dams_cli = DataAcquisitionManagementServiceClient()
dpms_cli = DataProductManagementServiceClient()
rr_cli = ResourceRegistryServiceClient()
pubsub_cli = PubsubManagementServiceClient()
eda = ExternalDatasetAgent(handler_module=self.DVR_CONFIG['dvr_mod'],
handler_class=self.DVR_CONFIG['dvr_cls'])
eda_id = dams_cli.create_external_dataset_agent(eda)
eda_inst = ExternalDatasetAgentInstance()
eda_inst_id = dams_cli.create_external_dataset_agent_instance(eda_inst, external_dataset_agent_id=eda_id)
# Create and register the necessary resources/objects
# Create DataProvider
dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation())
dprov.contact.individual_names_given = 'Christopher Mueller'
dprov.contact.email = '*****@*****.**'
# Create DataSource
dsrc = DataSource(protocol_type='FILE', institution=Institution(), contact=ContactInformation())
dsrc.connection_params['base_data_url'] = ''
dsrc.contact.individual_names_given = 'Tim Giguere'
dsrc.contact.email = '*****@*****.**'
# Create ExternalDataset
ds_name = 'slocum_test_dataset'
dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation())
dset.dataset_description.parameters['base_url'] = 'test_data/slocum/'
dset.dataset_description.parameters['list_pattern'] = 'ru05-2012-021-0-0-sbd.dat'
dset.dataset_description.parameters['date_pattern'] = '%Y %j'
dset.dataset_description.parameters['date_extraction_pattern'] = 'ru05-([\d]{4})-([\d]{3})-\d-\d-sbd.dat'
dset.dataset_description.parameters['temporal_dimension'] = None
dset.dataset_description.parameters['zonal_dimension'] = None
dset.dataset_description.parameters['meridional_dimension'] = None
dset.dataset_description.parameters['vertical_dimension'] = None
dset.dataset_description.parameters['variables'] = [
'c_wpt_y_lmc',
'sci_water_cond',
'm_y_lmc',
'u_hd_fin_ap_inflection_holdoff',
'sci_m_present_time',
'm_leakdetect_voltage_forward',
'sci_bb3slo_b660_scaled',
'c_science_send_all',
'm_gps_status',
'm_water_vx',
'm_water_vy',
'c_heading',
'sci_fl3slo_chlor_units',
'u_hd_fin_ap_gain',
'm_vacuum',
'u_min_water_depth',
'm_gps_lat',
'm_veh_temp',
'f_fin_offset',
'u_hd_fin_ap_hardover_holdoff',
'c_alt_time',
'm_present_time',
'm_heading',
'sci_bb3slo_b532_scaled',
'sci_fl3slo_cdom_units',
'm_fin',
'x_cycle_overrun_in_ms',
'sci_water_pressure',
'u_hd_fin_ap_igain',
'sci_fl3slo_phyco_units',
'm_battpos',
'sci_bb3slo_b470_scaled',
'm_lat',
'm_gps_lon',
'sci_ctd41cp_timestamp',
'm_pressure',
'c_wpt_x_lmc',
'c_ballast_pumped',
'x_lmc_xy_source',
'm_lon',
'm_avg_speed',
'sci_water_temp',
'u_pitch_ap_gain',
'm_roll',
'm_tot_num_inflections',
'm_x_lmc',
'u_pitch_ap_deadband',
'm_final_water_vy',
'm_final_water_vx',
'm_water_depth',
'm_leakdetect_voltage',
'u_pitch_max_delta_battpos',
'm_coulomb_amphr',
'm_pitch', ]
# Create DataSourceModel
dsrc_model = DataSourceModel(name='slocum_model')
# dsrc_model.model = 'SLOCUM'
dsrc_model.data_handler_module = 'N/A'
dsrc_model.data_handler_class = 'N/A'
## Run everything through DAMS
ds_id = dams_cli.create_external_dataset(external_dataset=dset)
ext_dprov_id = dams_cli.create_external_data_provider(external_data_provider=dprov)
ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc)
ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model)
# Register the ExternalDataset
dproducer_id = dams_cli.register_external_data_set(external_dataset_id=ds_id)
# Or using each method
dams_cli.assign_data_source_to_external_data_provider(data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id)
dams_cli.assign_data_source_to_data_model(data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id)
dams_cli.assign_external_dataset_to_data_source(external_dataset_id=ds_id, data_source_id=ext_dsrc_id)
dams_cli.assign_external_dataset_to_agent_instance(external_dataset_id=ds_id, agent_instance_id=eda_inst_id)
# dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=self.eda_id, agent_instance_id=self.eda_inst_id)
#create temp streamdef so the data product can create the stream
pc_list = []
for pc_k, pc in self._create_parameter_dictionary().iteritems():
pc_list.append(dms_cli.create_parameter_context(pc_k, pc[1].dump()))
pdict_id = dms_cli.create_parameter_dictionary('slocum_param_dict', pc_list)
streamdef_id = pubsub_cli.create_stream_definition(name="slocum_stream_def", description="stream def for slocum testing", parameter_dictionary_id=pdict_id)
# dpms_cli.create_data_product()
# Generate the data product and associate it to the ExternalDataset
tdom, sdom = time_series_domain()
tdom, sdom = tdom.dump(), sdom.dump()
dprod = IonObject(RT.DataProduct,
name='slocum_parsed_product',
description='parsed slocum product',
temporal_domain=tdom,
spatial_domain=sdom)
dproduct_id = dpms_cli.create_data_product(data_product=dprod,
stream_definition_id=streamdef_id)
dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id)
stream_id, assn = rr_cli.find_objects(subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True)
stream_id = stream_id[0]
log.info('Created resources: {0}'.format({'ExternalDataset': ds_id, 'ExternalDataProvider': ext_dprov_id, 'DataSource': ext_dsrc_id, 'DataSourceModel': ext_dsrc_model_id, 'DataProducer': dproducer_id, 'DataProduct': dproduct_id, 'Stream': stream_id}))
# Create the logger for receiving publications
_, stream_route, _ = self.create_stream_and_logger(name='slocum', stream_id=stream_id)
self.EDA_RESOURCE_ID = ds_id
self.EDA_NAME = ds_name
self.DVR_CONFIG['dh_cfg'] = {
'TESTING': True,
'stream_id': stream_id,
'stream_route': stream_route,
'stream_def': streamdef_id,
'external_dataset_res': dset,
'data_producer_id': dproducer_id, # CBM: Should this be put in the main body of the config - with mod & cls?
'max_records': 20,
}
def _setup_resources(self):
# TODO: some or all of this (or some variation) should move to DAMS'
# Build the test resources for the dataset
dms_cli = DatasetManagementServiceClient()
dams_cli = DataAcquisitionManagementServiceClient()
dpms_cli = DataProductManagementServiceClient()
rr_cli = ResourceRegistryServiceClient()
pubsub_cli = PubsubManagementServiceClient()
eda = ExternalDatasetAgent(name='example eda',handler_module=self.DVR_CONFIG['dvr_mod'],
handler_class=self.DVR_CONFIG['dvr_cls'])
eda_id = dams_cli.create_external_dataset_agent(eda)
eda_inst = ExternalDatasetAgentInstance(name='example eda instance')
eda_inst_id = dams_cli.create_external_dataset_agent_instance(eda_inst,
external_dataset_agent_id=eda_id)
# Create and register the necessary resources/objects
# Create DataProvider
dprov = ExternalDataProvider(name='example data provider', institution=Institution(),
contact=ContactInformation())
dprov.contact.individual_names_given = 'Christopher Mueller'
dprov.contact.email = '*****@*****.**'
# Create DataSource
dsrc = DataSource(name='example datasource', protocol_type='DAP', institution=Institution(),
contact=ContactInformation())
dsrc.connection_params['base_data_url'] = ''
dsrc.contact.individual_names_given = 'Tim Giguere'
dsrc.contact.email = '*****@*****.**'
# Create ExternalDataset
ds_name = 'usgs_test_dataset'
dset = ExternalDataset(name=ds_name,
dataset_description=DatasetDescription(),
update_description=UpdateDescription(),
contact=ContactInformation())
# The usgs.nc test dataset is a download of the R1 dataset found here:
# http://thredds-test.oceanobservatories.org/thredds/dodsC/ooiciData/E66B1A74-A684-454A-9ADE-8388C2C634E5.ncml
dset.dataset_description.parameters['dataset_path'] = 'test_data/usgs.nc'
dset.dataset_description.parameters['temporal_dimension'] = 'time'
dset.dataset_description.parameters['zonal_dimension'] = 'lon'
dset.dataset_description.parameters['meridional_dimension'] = 'lat'
dset.dataset_description.parameters['vertical_dimension'] = 'z'
dset.dataset_description.parameters['variables'] = [
'water_temperature',
'streamflow',
'water_temperature_bottom',
'water_temperature_middle',
'specific_conductance',
'data_qualifier', ]
# Create DataSourceModel
dsrc_model = DataSourceModel(name='dap_model')
#dsrc_model.model = 'DAP'
dsrc_model.data_handler_module = 'N/A'
dsrc_model.data_handler_class = 'N/A'
## Run everything through DAMS
ds_id = dams_cli.create_external_dataset(external_dataset=dset)
ext_dprov_id = dams_cli.create_external_data_provider(external_data_provider=dprov)
ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc)
ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model)
# Register the ExternalDataset
dproducer_id = dams_cli.register_external_data_set(external_dataset_id=ds_id)
# Or using each method
dams_cli.assign_data_source_to_external_data_provider(data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id)
dams_cli.assign_data_source_to_data_model(data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id)
dams_cli.assign_external_dataset_to_data_source(external_dataset_id=ds_id, data_source_id=ext_dsrc_id)
dams_cli.assign_external_dataset_to_agent_instance(external_dataset_id=ds_id, agent_instance_id=eda_inst_id)
# dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=self.eda_id, agent_instance_id=self.eda_inst_id)
#create temp streamdef so the data product can create the stream
pc_list = []
#Get 'time' parameter context
pc_list.append(dms_cli.read_parameter_context_by_name('time', id_only=True))
for pc_k, pc in self._create_parameter_dictionary().iteritems():
pc_list.append(dms_cli.create_parameter_context(pc_k, pc[1].dump()))
pdict_id = dms_cli.create_parameter_dictionary('netcdf_param_dict', pc_list)
#create temp streamdef so the data product can create the stream
streamdef_id = pubsub_cli.create_stream_definition(name="netcdf", description="netcdf", parameter_dictionary_id=pdict_id)
tdom, sdom = time_series_domain()
tdom, sdom = tdom.dump(), sdom.dump()
dprod = IonObject(RT.DataProduct,
name='usgs_parsed_product',
description='parsed usgs product',
temporal_domain=tdom,
spatial_domain=sdom)
# Generate the data product and associate it to the ExternalDataset
dproduct_id = dpms_cli.create_data_product(data_product=dprod,
stream_definition_id=streamdef_id)
dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id)
stream_id, assn = rr_cli.find_objects(subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True)
stream_id = stream_id[0]
log.info('Created resources: {0}'.format({'ExternalDataset': ds_id, 'ExternalDataProvider': ext_dprov_id, 'DataSource': ext_dsrc_id, 'DataSourceModel': ext_dsrc_model_id, 'DataProducer': dproducer_id, 'DataProduct': dproduct_id, 'Stream': stream_id}))
# Create the logger for receiving publications
_, stream_route, _ = self.create_stream_and_logger(name='usgs', stream_id=stream_id)
self.EDA_RESOURCE_ID = ds_id
self.EDA_NAME = ds_name
self.DVR_CONFIG['dh_cfg'] = {
'TESTING': True,
'stream_id': stream_id,
'stream_route': stream_route,
'stream_def': streamdef_id,
'data_producer_id': dproducer_id, # CBM: Should this be put in the main body of the config - with mod & cls?
'max_records': 1,
}
def _setup_resources(self):
# TODO: some or all of this (or some variation) should move to DAMS'
# Build the test resources for the dataset
dams_cli = DataAcquisitionManagementServiceClient()
dpms_cli = DataProductManagementServiceClient()
rr_cli = ResourceRegistryServiceClient()
eda = ExternalDatasetAgent()
eda_id = dams_cli.create_external_dataset_agent(eda)
eda_inst = ExternalDatasetAgentInstance()
eda_inst_id = dams_cli.create_external_dataset_agent_instance(
eda_inst, external_dataset_agent_id=eda_id)
# Create and register the necessary resources/objects
# Create DataProvider
dprov = ExternalDataProvider(institution=Institution(),
contact=ContactInformation())
dprov.contact.name = 'Christopher Mueller'
dprov.contact.email = '*****@*****.**'
# Create DataSource
dsrc = DataSource(protocol_type='DAP',
institution=Institution(),
contact=ContactInformation())
dsrc.connection_params['base_data_url'] = ''
dsrc.contact.name = 'Tim Giguere'
dsrc.contact.email = '*****@*****.**'
# Create ExternalDataset
ds_name = 'usgs_test_dataset'
dset = ExternalDataset(name=ds_name,
dataset_description=DatasetDescription(),
update_description=UpdateDescription(),
contact=ContactInformation())
# The usgs.nc test dataset is a download of the R1 dataset found here:
# http://thredds-test.oceanobservatories.org/thredds/dodsC/ooiciData/E66B1A74-A684-454A-9ADE-8388C2C634E5.ncml
dset.dataset_description.parameters[
'dataset_path'] = 'test_data/usgs.nc'
dset.dataset_description.parameters['temporal_dimension'] = 'time'
dset.dataset_description.parameters['zonal_dimension'] = 'lon'
dset.dataset_description.parameters['meridional_dimension'] = 'lat'
dset.dataset_description.parameters['vertical_dimension'] = 'z'
dset.dataset_description.parameters['variables'] = [
'water_temperature',
'streamflow',
'water_temperature_bottom',
'water_temperature_middle',
'specific_conductance',
'data_qualifier',
]
# Create DataSourceModel
dsrc_model = DataSourceModel(name='dap_model')
dsrc_model.model = 'DAP'
dsrc_model.data_handler_module = 'N/A'
dsrc_model.data_handler_class = 'N/A'
## Run everything through DAMS
ds_id = dams_cli.create_external_dataset(external_dataset=dset)
ext_dprov_id = dams_cli.create_external_data_provider(
external_data_provider=dprov)
ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc)
ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model)
# Register the ExternalDataset
dproducer_id = dams_cli.register_external_data_set(
external_dataset_id=ds_id)
# Or using each method
dams_cli.assign_data_source_to_external_data_provider(
data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id)
dams_cli.assign_data_source_to_data_model(
data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id)
dams_cli.assign_external_dataset_to_data_source(
external_dataset_id=ds_id, data_source_id=ext_dsrc_id)
dams_cli.assign_external_dataset_to_agent_instance(
external_dataset_id=ds_id, agent_instance_id=eda_inst_id)
# dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=self.eda_id, agent_instance_id=self.eda_inst_id)
# Generate the data product and associate it to the ExternalDataset
dprod = DataProduct(name='usgs_parsed_product',
description='parsed usgs product')
dproduct_id = dpms_cli.create_data_product(data_product=dprod)
dams_cli.assign_data_product(input_resource_id=ds_id,
data_product_id=dproduct_id,
create_stream=True)
stream_id, assn = rr_cli.find_objects(subject=dproduct_id,
predicate=PRED.hasStream,
object_type=RT.Stream,
id_only=True)
stream_id = stream_id[0]
log.info('Created resources: {0}'.format({
'ExternalDataset': ds_id,
'ExternalDataProvider': ext_dprov_id,
'DataSource': ext_dsrc_id,
'DataSourceModel': ext_dsrc_model_id,
'DataProducer': dproducer_id,
'DataProduct': dproduct_id,
'Stream': stream_id
}))
#CBM: Use CF standard_names
ttool = TaxyTool()
ttool.add_taxonomy_set('time', 'time')
ttool.add_taxonomy_set('lon', 'longitude')
ttool.add_taxonomy_set('lat', 'latitude')
ttool.add_taxonomy_set('z', 'water depth')
ttool.add_taxonomy_set('water_temperature',
'average water temperature')
ttool.add_taxonomy_set('water_temperature_bottom',
'water temperature at bottom of water column')
ttool.add_taxonomy_set('water_temperature_middle',
'water temperature at middle of water column')
ttool.add_taxonomy_set('streamflow', 'flow velocity of stream')
ttool.add_taxonomy_set('specific_conductance',
'specific conductance of water')
ttool.add_taxonomy_set('data_qualifier', 'data qualifier flag')
ttool.add_taxonomy_set('coords',
'This group contains coordinate parameters')
ttool.add_taxonomy_set('data', 'This group contains data parameters')
# Create the logger for receiving publications
self.create_stream_and_logger(name='usgs', stream_id=stream_id)
self.EDA_RESOURCE_ID = ds_id
self.EDA_NAME = ds_name
self.DVR_CONFIG['dh_cfg'] = {
'TESTING': True,
'stream_id': stream_id,
'taxonomy': ttool.dump(),
'data_producer_id':
dproducer_id, #CBM: Should this be put in the main body of the config - with mod & cls?
'max_records': 4,
}
def test_get_provenance(self):
#create a deployment with metadata and an initial site and device
instrument_site_obj = IonObject(RT.InstrumentSite,
name='InstrumentSite1',
description='test instrument site')
instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, "")
log.debug( 'test_get_provenance: new instrument_site_id id = %s ', str(instrument_site_id))
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel" )
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" %ex)
log.debug( 'test_get_provenance: new InstrumentModel id = %s ', str(instModel_id))
self.omsclient.assign_instrument_model_to_instrument_site(instModel_id, instrument_site_id)
# Create InstrumentAgent
parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict', records_per_granule=2, granule_publish_rate=5 )
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri="http://sddevrepo.oceanobservatories.org/releases/seabird_sbe37smb_ooicore-0.0.1-py2.7.egg",
stream_configurations = [parsed_config] )
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" %ex)
log.debug( 'test_get_provenance:new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug('test_get_provenance: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ')
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
try:
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" %ex)
log.debug("test_get_provenance: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id)
#-------------------------------
# Create CTD Parsed data product
#-------------------------------
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubclient.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id)
log.debug( 'test_get_provenance:Creating new CDM data product with a stream definition')
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_parsed_data_product = self.dpmsclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s', ctd_parsed_data_product)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
self.dpmsclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product)
#-------------------------------
# create a data product for the site to pass the OMS check.... we need to remove this check
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
log_data_product_id = self.dpmsclient.create_data_product(dp_obj, parsed_stream_def_id)
self.omsclient.create_site_data_product(instrument_site_id, log_data_product_id)
#-------------------------------
# Deploy instrument device to instrument site
#-------------------------------
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment')
deployment_id = self.omsclient.create_deployment(deployment_obj)
self.omsclient.deploy_instrument_site(instrument_site_id, deployment_id)
self.imsclient.deploy_instrument_device(instDevice_id, deployment_id)
log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) )
self.omsclient.activate_deployment(deployment_id)
inst_device_objs, _ = self.rrclient.find_objects(subject=instrument_site_id, predicate=PRED.hasDevice, object_type=RT.InstrumetDevice, id_only=False)
log.debug("test_create_deployment: deployed device: %s ", str(inst_device_objs[0]) )
#-------------------------------
# Create the agent instance
#-------------------------------
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config = port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition ctd_L0_all")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" %ex)
#-------------------------------
# L1 Conductivity: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1ConductivityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_conductivity',
description='create the L1 conductivity data product',
module='ion.processes.data.transforms.ctd.ctd_L1_conductivity',
class_name='CTDL1ConductivityTransform')
try:
ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1ConductivityTransform data process definition: %s" %ex)
#-------------------------------
# L1 Pressure: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1PressureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_pressure',
description='create the L1 pressure data product',
module='ion.processes.data.transforms.ctd.ctd_L1_pressure',
class_name='CTDL1PressureTransform')
try:
ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1PressureTransform data process definition: %s" %ex)
#-------------------------------
# L1 Temperature: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1TemperatureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_temperature',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L1_temperature',
class_name='CTDL1TemperatureTransform')
try:
ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1TemperatureTransform data process definition: %s" %ex)
#-------------------------------
# L2 Salinity: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition SalinityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_salinity',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform')
try:
ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new SalinityTransform data process definition: %s" %ex)
#-------------------------------
# L2 Density: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition DensityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_density',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_density',
class_name='DensityTransform')
try:
ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new DensityTransform data process definition: %s" %ex)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' )
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' )
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' )
self.output_products={}
log.debug("TestDataProductProvenance: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id)
self.output_products['conductivity'] = ctd_l0_conductivity_output_dp_id
log.debug("TestDataProductProvenance: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_pressure_output_dp_obj,
outgoing_stream_l0_pressure_id)
self.output_products['pressure'] = ctd_l0_pressure_output_dp_id
log.debug("TestDataProductProvenance: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_temperature_output_dp_obj,
outgoing_stream_l0_temperature_id)
self.output_products['temperature'] = ctd_l0_temperature_output_dp_id
#-------------------------------
# L1 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(name='L1_conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id, binding='conductivity' )
outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(name='L1_Pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id, binding='pressure' )
outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(name='L1_Temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id, binding='temperature' )
log.debug("TestDataProductProvenance: create output data product L1 conductivity")
ctd_l1_conductivity_output_dp_obj = IonObject(RT.DataProduct,
name='L1_Conductivity',
description='transform output L1 conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_conductivity_output_dp_obj,
outgoing_stream_l1_conductivity_id)
log.debug("TestDataProductProvenance: create output data product L1 pressure")
ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Pressure',
description='transform output L1 pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_pressure_output_dp_obj,
outgoing_stream_l1_pressure_id)
log.debug("TestDataProductProvenance: create output data product L1 temperature")
ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Temperature',
description='transform output L1 temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_temperature_output_dp_obj,
outgoing_stream_l1_temperature_id)
#-------------------------------
# L2 Salinity - Density: Output Data Products
#-------------------------------
outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(name='L2_salinity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id, binding='salinity' )
outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(name='L2_Density', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id, binding='density' )
log.debug("TestDataProductProvenance: create output data product L2 Salinity")
ctd_l2_salinity_output_dp_obj = IonObject( RT.DataProduct,
name='L2_Salinity',
description='transform output L2 salinity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l2_salinity_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_salinity_output_dp_obj,
outgoing_stream_l2_salinity_id)
log.debug("TestDataProductProvenance: create output data product L2 Density")
# ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct,
# name='L2_Density',
# description='transform output pressure',
# temporal_domain = tdom,
# spatial_domain = sdom)
#
# ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj,
# outgoing_stream_l2_density_id,
# parameter_dictionary)
contactInfo = ContactInformation()
contactInfo.individual_names_given = "Bill"
contactInfo.individual_name_family = "Smith"
contactInfo.street_address = "111 First St"
contactInfo.city = "San Diego"
contactInfo.email = "*****@*****.**"
contactInfo.phones = ["858-555-6666"]
contactInfo.country = "USA"
contactInfo.postal_code = "92123"
ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct,
name='L2_Density',
description='transform output pressure',
contacts = [contactInfo],
iso_topic_category = "my_iso_topic_category_here",
quality_control_level = "1",
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj,
outgoing_stream_l2_density_id)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L0 all data_process start")
try:
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L0_all_dprocdef_id, [ctd_parsed_data_product], self.output_products)
#activate only this data process just for coverage
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
contents = "this is the lookup table contents, replace with a file..."
att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII)
deviceAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att)
log.info( 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = %s', deviceAttachment)
log.debug("TestDataProductProvenance: create L0 all data_process return")
#-------------------------------
# L1 Conductivity: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1 Conductivity data_process start")
try:
l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_conductivity_dprocdef_id, [ctd_l0_conductivity_output_dp_id], {'conductivity':ctd_l1_conductivity_output_dp_id})
self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L1 Pressure: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1_Pressure data_process start")
try:
l1_pressure_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_pressure_dprocdef_id, [ctd_l0_pressure_output_dp_id], {'pressure':ctd_l1_pressure_output_dp_id})
self.dataprocessclient.activate_data_process(l1_pressure_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L1 Temperature: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1_Pressure data_process start")
try:
l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L1_temperature_dprocdef_id, [ctd_l0_temperature_output_dp_id], {'temperature':ctd_l1_temperature_output_dp_id})
self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L2 Salinity: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L2_salinity data_process start")
try:
l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_salinity_dprocdef_id, [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id], {'salinity':ctd_l2_salinity_output_dp_id})
self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L2 Density: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L2_Density data_process start")
try:
l2_density_all_data_process_id = self.dataprocessclient.create_data_process(ctd_L2_density_dprocdef_id, [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id], {'density':ctd_l2_density_output_dp_id})
self.dataprocessclient.activate_data_process(l2_density_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# Launch InstrumentAgentInstance, connect to the resource agent client
#-------------------------------
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
print 'TestDataProductProvenance: Instrument agent instance obj: = ', inst_agent_instance_obj
# Start a resource agent client to talk with the instrument agent.
# self._ia_client = ResourceAgentClient('iaclient', name=inst_agent_instance_obj.agent_process_id, process=FakeProcess())
# print 'activate_instrument: got ia client %s', self._ia_client
# log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client))
#-------------------------------
# Deactivate InstrumentAgentInstance
#-------------------------------
self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
self.dataprocessclient.deactivate_data_process(l2_density_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l2_salinity_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_temperature_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_pressure_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_conductivity_data_process_id)
self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id)
#-------------------------------
# Retrieve the provenance info for the ctd density data product
#-------------------------------
provenance_dict = self.dpmsclient.get_data_product_provenance(ctd_l2_density_output_dp_id)
log.debug("TestDataProductProvenance: provenance_dict %s", str(provenance_dict))
#validate that products are represented
self.assertTrue (provenance_dict[str(ctd_l1_conductivity_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l0_conductivity_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l2_density_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l1_temperature_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l0_temperature_output_dp_id)])
density_dict = (provenance_dict[str(ctd_l2_density_output_dp_id)])
self.assertEquals(density_dict['producer'], [l2_density_all_data_process_id])
#-------------------------------
# Retrieve the extended resource for this data product
#-------------------------------
extended_product = self.dpmsclient.get_data_product_extension(ctd_l2_density_output_dp_id)
self.assertEqual(1, len(extended_product.data_processes) )
self.assertEqual(3, len(extended_product.process_input_data_products) )
# log.debug("TestDataProductProvenance: DataProduct provenance_product_list %s", str(extended_product.provenance_product_list))
# log.debug("TestDataProductProvenance: DataProduct data_processes %s", str(extended_product.data_processes))
# log.debug("TestDataProductProvenance: DataProduct process_input_data_products %s", str(extended_product.process_input_data_products))
# log.debug("TestDataProductProvenance: provenance %s", str(extended_product.computed.provenance.value))
#-------------------------------
# Retrieve the extended resource for this data process
#-------------------------------
extended_process_def = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id)
# log.debug("TestDataProductProvenance: DataProcess extended_process_def %s", str(extended_process_def))
# log.debug("TestDataProductProvenance: DataProcess data_processes %s", str(extended_process_def.data_processes))
# log.debug("TestDataProductProvenance: DataProcess data_products %s", str(extended_process_def.data_products))
self.assertEqual(1, len(extended_process_def.data_processes) )
self.assertEqual(3, len(extended_process_def.output_stream_definitions) )
self.assertEqual(3, len(extended_process_def.data_products) ) #one list because of one data process
#-------------------------------
# Request the xml report
#-------------------------------
results = self.dpmsclient.get_data_product_provenance_report(ctd_l2_density_output_dp_id)
#-------------------------------
# Cleanup
#-------------------------------
self.dpmsclient.delete_data_product(ctd_parsed_data_product)
self.dpmsclient.delete_data_product(log_data_product_id)
self.dpmsclient.delete_data_product(ctd_l0_conductivity_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l0_pressure_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l0_temperature_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l1_conductivity_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l1_pressure_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l1_temperature_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l2_salinity_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l2_density_output_dp_id)
def _setup_resources(self):
# TODO: some or all of this (or some variation) should move to DAMS'
# Build the test resources for the dataset
dms_cli = DatasetManagementServiceClient()
dams_cli = DataAcquisitionManagementServiceClient()
dpms_cli = DataProductManagementServiceClient()
rr_cli = ResourceRegistryServiceClient()
pubsub_cli = PubsubManagementServiceClient()
eda = ExternalDatasetAgent(name='example data agent',
handler_module=self.DVR_CONFIG['dvr_mod'],
handler_class=self.DVR_CONFIG['dvr_cls'])
eda_id = dams_cli.create_external_dataset_agent(eda)
eda_inst = ExternalDatasetAgentInstance(
name='example dataset agent instance')
eda_inst_id = dams_cli.create_external_dataset_agent_instance(
eda_inst, external_dataset_agent_id=eda_id)
# Create and register the necessary resources/objects
# Create DataProvider
dprov = ExternalDataProvider(name='example data provider',
institution=Institution(),
contact=ContactInformation())
dprov.contact.individual_names_given = 'Christopher Mueller'
dprov.contact.email = '*****@*****.**'
# Create DataSource
dsrc = DataSource(name='example datasource',
protocol_type='FILE',
institution=Institution(),
contact=ContactInformation())
dsrc.connection_params['base_data_url'] = ''
dsrc.contact.individual_names_given = 'Tim Giguere'
dsrc.contact.email = '*****@*****.**'
# Create ExternalDataset
ds_name = 'ruv_test_dataset'
dset = ExternalDataset(name=ds_name,
dataset_description=DatasetDescription(),
update_description=UpdateDescription(),
contact=ContactInformation())
dset.dataset_description.parameters['base_url'] = 'test_data/ruv/'
dset.dataset_description.parameters[
'list_pattern'] = 'RDLi_SEAB_2011_08_24_1600.ruv'
dset.dataset_description.parameters['date_pattern'] = '%Y %m %d %H %M'
dset.dataset_description.parameters[
'date_extraction_pattern'] = 'RDLi_SEAB_([\d]{4})_([\d]{2})_([\d]{2})_([\d]{2})([\d]{2}).ruv'
dset.dataset_description.parameters['temporal_dimension'] = None
dset.dataset_description.parameters['zonal_dimension'] = None
dset.dataset_description.parameters['meridional_dimension'] = None
dset.dataset_description.parameters['vertical_dimension'] = None
dset.dataset_description.parameters['variables'] = []
# Create DataSourceModel
dsrc_model = DataSourceModel(name='ruv_model')
#dsrc_model.model = 'RUV'
dsrc_model.data_handler_module = 'N/A'
dsrc_model.data_handler_class = 'N/A'
## Run everything through DAMS
ds_id = dams_cli.create_external_dataset(external_dataset=dset)
ext_dprov_id = dams_cli.create_external_data_provider(
external_data_provider=dprov)
ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc)
ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model)
# Register the ExternalDataset
dproducer_id = dams_cli.register_external_data_set(
external_dataset_id=ds_id)
# Or using each method
dams_cli.assign_data_source_to_external_data_provider(
data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id)
dams_cli.assign_data_source_to_data_model(
data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id)
dams_cli.assign_external_dataset_to_data_source(
external_dataset_id=ds_id, data_source_id=ext_dsrc_id)
dams_cli.assign_external_dataset_to_agent_instance(
external_dataset_id=ds_id, agent_instance_id=eda_inst_id)
pdict = ParameterDictionary()
t_ctxt = ParameterContext(
'data',
param_type=QuantityType(value_encoding=numpy.dtype('int64')))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1970'
pdict.add_context(t_ctxt)
#create temp streamdef so the data product can create the stream
pc_list = []
for pc_k, pc in pdict.iteritems():
pc_list.append(dms_cli.create_parameter_context(
pc_k, pc[1].dump()))
pdict_id = dms_cli.create_parameter_dictionary('ruv_param_dict',
pc_list)
streamdef_id = pubsub_cli.create_stream_definition(
name="ruv",
description="stream def for ruv testing",
parameter_dictionary_id=pdict_id)
dprod = IonObject(RT.DataProduct,
name='ruv_parsed_product',
description='parsed ruv product')
# Generate the data product and associate it to the ExternalDataset
dproduct_id = dpms_cli.create_data_product(
data_product=dprod, stream_definition_id=streamdef_id)
dams_cli.assign_data_product(input_resource_id=ds_id,
data_product_id=dproduct_id)
stream_id, assn = rr_cli.find_objects(subject=dproduct_id,
predicate=PRED.hasStream,
object_type=RT.Stream,
id_only=True)
stream_id = stream_id[0]
log.info('Created resources: {0}'.format({
'ExternalDataset': ds_id,
'ExternalDataProvider': ext_dprov_id,
'DataSource': ext_dsrc_id,
'DataSourceModel': ext_dsrc_model_id,
'DataProducer': dproducer_id,
'DataProduct': dproduct_id,
'Stream': stream_id
}))
#CBM: Eventually, probably want to group this crap somehow - not sure how yet...
# Create the logger for receiving publications
_, stream_route, _ = self.create_stream_and_logger(name='ruv',
stream_id=stream_id)
self.EDA_RESOURCE_ID = ds_id
self.EDA_NAME = ds_name
self.DVR_CONFIG['dh_cfg'] = {
'TESTING': True,
'stream_id': stream_id,
'stream_route': stream_route,
'external_dataset_res': dset,
'param_dictionary': pdict.dump(),
'data_producer_id':
dproducer_id, # CBM: Should this be put in the main body of the config - with mod & cls?
'max_records': 20,
}
def _setup_resources(self):
self.loggerpids = []
# Create DataProvider
dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation())
dprov.contact.name = 'Christopher Mueller'
dprov.contact.email = '*****@*****.**'
# Create DataSetModel
dataset_model = ExternalDatasetModel(name='slocum_model')
dataset_model.datset_type = 'SLOCUM'
dataset_model_id = self.dams_client.create_external_dataset_model(dataset_model)
# Create ExternalDataset
ds_name = 'slocum_test_dataset'
dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation())
dset.dataset_description.parameters['base_url'] = 'test_data/slocum/'
dset.dataset_description.parameters['list_pattern'] = 'ru05-2012-021-0-0-sbd.dat'
dset.dataset_description.parameters['date_pattern'] = '%Y %j'
dset.dataset_description.parameters['date_extraction_pattern'] = 'ru05-([\d]{4})-([\d]{3})-\d-\d-sbd.dat'
dset.dataset_description.parameters['temporal_dimension'] = None
dset.dataset_description.parameters['zonal_dimension'] = None
dset.dataset_description.parameters['meridional_dimension'] = None
dset.dataset_description.parameters['vertical_dimension'] = None
dset.dataset_description.parameters['variables'] = [
'c_wpt_y_lmc',
'sci_water_cond',
'm_y_lmc',
'u_hd_fin_ap_inflection_holdoff',
'sci_m_present_time',
'm_leakdetect_voltage_forward',
'sci_bb3slo_b660_scaled',
'c_science_send_all',
'm_gps_status',
'm_water_vx',
'm_water_vy',
'c_heading',
'sci_fl3slo_chlor_units',
'u_hd_fin_ap_gain',
'm_vacuum',
'u_min_water_depth',
'm_gps_lat',
'm_veh_temp',
'f_fin_offset',
'u_hd_fin_ap_hardover_holdoff',
'c_alt_time',
'm_present_time',
'm_heading',
'sci_bb3slo_b532_scaled',
'sci_fl3slo_cdom_units',
'm_fin',
'x_cycle_overrun_in_ms',
'sci_water_pressure',
'u_hd_fin_ap_igain',
'sci_fl3slo_phyco_units',
'm_battpos',
'sci_bb3slo_b470_scaled',
'm_lat',
'm_gps_lon',
'sci_ctd41cp_timestamp',
'm_pressure',
'c_wpt_x_lmc',
'c_ballast_pumped',
'x_lmc_xy_source',
'm_lon',
'm_avg_speed',
'sci_water_temp',
'u_pitch_ap_gain',
'm_roll',
'm_tot_num_inflections',
'm_x_lmc',
'u_pitch_ap_deadband',
'm_final_water_vy',
'm_final_water_vx',
'm_water_depth',
'm_leakdetect_voltage',
'u_pitch_max_delta_battpos',
'm_coulomb_amphr',
'm_pitch',
]
## Create the external dataset
ds_id = self.dams_client.create_external_dataset(external_dataset=dset, external_dataset_model_id=dataset_model_id)
ext_dprov_id = self.dams_client.create_external_data_provider(external_data_provider=dprov)
# Register the ExternalDataset
dproducer_id = self.dams_client.register_external_data_set(external_dataset_id=ds_id)
## Create the dataset agent
datasetagent_obj = IonObject(RT.ExternalDatasetAgent, name='ExternalDatasetAgent1', description='external data agent', handler_module=self.EDA_MOD, handler_class=self.EDA_CLS )
self.datasetagent_id = self.dams_client.create_external_dataset_agent(external_dataset_agent=datasetagent_obj, external_dataset_model_id=dataset_model_id)
# Generate the data product and associate it to the ExternalDataset
pdict = DatasetManagementService.get_parameter_dictionary_by_name('ctd_parsed_param_dict')
streamdef_id = self.pubsub_client.create_stream_definition(name="temp", parameter_dictionary_id=pdict.identifier)
tdom, sdom = time_series_domain()
tdom = tdom.dump()
sdom = sdom.dump()
dprod = IonObject(RT.DataProduct,
name='slocum_parsed_product',
description='parsed slocum product',
temporal_domain = tdom,
spatial_domain = sdom)
self.dproduct_id = self.dataproductclient.create_data_product(data_product=dprod,
stream_definition_id=streamdef_id)
self.dams_client.assign_data_product(input_resource_id=ds_id, data_product_id=self.dproduct_id)
#save the incoming slocum data
self.dataproductclient.activate_data_product_persistence(self.dproduct_id)
self.addCleanup(self.dataproductclient.suspend_data_product_persistence, self.dproduct_id)
stream_ids, assn = self.rrclient.find_objects(subject=self.dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True)
stream_id = stream_ids[0]
dataset_id, assn = self.rrclient.find_objects(subject=self.dproduct_id, predicate=PRED.hasDataset, object_type=RT.Dataset, id_only=True)
self.dataset_id = dataset_id[0]
pid = self.create_logger('slocum_parsed_product', stream_id )
self.loggerpids.append(pid)
self.DVR_CONFIG['dh_cfg'] = {
'TESTING':True,
'stream_id':stream_id,
'param_dictionary':pdict.dump(),
'data_producer_id':dproducer_id, #CBM: Should this be put in the main body of the config - with mod & cls?
'max_records':20,
}
# Create the logger for receiving publications
#self.create_stream_and_logger(name='slocum',stream_id=stream_id)
# Create agent config.
self.EDA_RESOURCE_ID = ds_id
self.EDA_NAME = ds_name
def _setup_hfr(self):
# TODO: some or all of this (or some variation) should move to DAMS
# Create and register the necessary resources/objects
eda = ExternalDatasetAgent()
eda_id = self.dams_cli.create_external_dataset_agent(eda)
eda_inst = ExternalDatasetAgentInstance()
eda_inst_id = self.dams_cli.create_external_dataset_agent_instance(
eda_inst, external_dataset_agent_id=eda_id)
# Create DataProvider
dprov = ExternalDataProvider(institution=Institution(),
contact=ContactInformation())
# dprov.institution.name = "HFR UCSD"
# Create DataSource
dsrc = DataSource(protocol_type="DAP",
institution=Institution(),
contact=ContactInformation())
dsrc.connection_params[
"base_data_url"] = "http://hfrnet.ucsd.edu:8080/thredds/dodsC/"
# Create ExternalDataset
dset = ExternalDataset(name="UCSD HFR",
dataset_description=DatasetDescription(),
update_description=UpdateDescription(),
contact=ContactInformation())
dset.dataset_description.parameters[
"dataset_path"] = "HFRNet/USEGC/6km/hourly/RTV"
# dset.dataset_description.parameters["dataset_path"] = "test_data/hfr.nc"
dset.dataset_description.parameters["temporal_dimension"] = "time"
dset.dataset_description.parameters["zonal_dimension"] = "lon"
dset.dataset_description.parameters["meridional_dimension"] = "lat"
# Create DataSourceModel
dsrc_model = DataSourceModel(name="dap_model")
dsrc_model.model = "DAP"
dsrc_model.data_handler_module = "eoi.agent.handler.dap_external_data_handler"
dsrc_model.data_handler_class = "DapExternalDataHandler"
## Run everything through DAMS
ds_id = self.hfr_ds_id = self.dams_cli.create_external_dataset(
external_dataset=dset)
ext_dprov_id = self.dams_cli.create_external_data_provider(
external_data_provider=dprov)
ext_dsrc_id = self.dams_cli.create_data_source(data_source=dsrc)
ext_dsrc_model_id = self.dams_cli.create_data_source_model(dsrc_model)
# Register the ExternalDataset
dproducer_id = self.dams_cli.register_external_data_set(
external_dataset_id=ds_id)
## Associate everything
# Convenience method
# self.dams_cli.assign_eoi_resources(external_data_provider_id=ext_dprov_id, data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id, external_dataset_id=ds_id, external_data_agent_id=eda_id, agent_instance_id=eda_inst_id)
# Or using each method
self.dams_cli.assign_data_source_to_external_data_provider(
data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id)
self.dams_cli.assign_data_source_to_data_model(
data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id)
self.dams_cli.assign_external_dataset_to_data_source(
external_dataset_id=ds_id, data_source_id=ext_dsrc_id)
self.dams_cli.assign_external_dataset_to_agent_instance(
external_dataset_id=ds_id, agent_instance_id=eda_inst_id)
# self.dams_cli.assign_external_dataset_agent_to_data_model(external_data_agent_id=eda_id, data_source_model_id=ext_dsrc_model_id)
# self.dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=eda_id, agent_instance_id=eda_inst_id)
# Generate the data product and associate it to the ExternalDataset
dprod = DataProduct(name='hfr_product', description='raw hfr product')
dproduct_id = self.dpms_cli.create_data_product(data_product=dprod)
self.dams_cli.assign_data_product(input_resource_id=ds_id,
data_product_id=dproduct_id,
create_stream=True)
def _setup_ncom(self):
# TODO: some or all of this (or some variation) should move to DAMS
# Create and register the necessary resources/objects
eda = ExternalDatasetAgent()
eda_id = self.dams_cli.create_external_dataset_agent(eda)
eda_inst = ExternalDatasetAgentInstance()
eda_inst_id = self.dams_cli.create_external_dataset_agent_instance(
eda_inst, external_dataset_agent_id=eda_id)
# Create DataProvider
dprov = ExternalDataProvider(institution=Institution(),
contact=ContactInformation())
# dprov.institution.name = "OOI CGSN"
dprov.contact.name = "Robert Weller"
dprov.contact.email = "*****@*****.**"
# Create DataSource
dsrc = DataSource(protocol_type="DAP",
institution=Institution(),
contact=ContactInformation())
# dsrc.connection_params["base_data_url"] = "http://ooi.whoi.edu/thredds/dodsC/"
dsrc.connection_params["base_data_url"] = ""
dsrc.contact.name = "Rich Signell"
dsrc.contact.email = "*****@*****.**"
# Create ExternalDataset
dset = ExternalDataset(name="test",
dataset_description=DatasetDescription(),
update_description=UpdateDescription(),
contact=ContactInformation())
# dset.dataset_description.parameters["dataset_path"] = "ooi/AS02CPSM_R_M.nc"
dset.dataset_description.parameters[
"dataset_path"] = "test_data/ncom.nc"
dset.dataset_description.parameters["temporal_dimension"] = "time"
dset.dataset_description.parameters["zonal_dimension"] = "lon"
dset.dataset_description.parameters["meridional_dimension"] = "lat"
# Create DataSourceModel
dsrc_model = DataSourceModel(name="dap_model")
dsrc_model.model = "DAP"
dsrc_model.data_handler_module = "eoi.agent.handler.dap_external_data_handler"
dsrc_model.data_handler_class = "DapExternalDataHandler"
## Run everything through DAMS
ds_id = self.ncom_ds_id = self.dams_cli.create_external_dataset(
external_dataset=dset)
ext_dprov_id = self.dams_cli.create_external_data_provider(
external_data_provider=dprov)
ext_dsrc_id = self.dams_cli.create_data_source(data_source=dsrc)
ext_dsrc_model_id = self.dams_cli.create_data_source_model(dsrc_model)
# Register the ExternalDataset
dproducer_id = self.dams_cli.register_external_data_set(
external_dataset_id=ds_id)
## Associate everything
# Convenience method
# self.dams_cli.assign_eoi_resources(external_data_provider_id=ext_dprov_id, data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id, external_dataset_id=ds_id, external_data_agent_id=eda_id, agent_instance_id=eda_inst_id)
# Or using each method
self.dams_cli.assign_data_source_to_external_data_provider(
data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id)
self.dams_cli.assign_data_source_to_data_model(
data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id)
self.dams_cli.assign_external_dataset_to_data_source(
external_dataset_id=ds_id, data_source_id=ext_dsrc_id)
self.dams_cli.assign_external_dataset_to_agent_instance(
external_dataset_id=ds_id, agent_instance_id=eda_inst_id)
# self.dams_cli.assign_external_dataset_agent_to_data_model(external_data_agent_id=eda_id, data_source_model_id=ext_dsrc_model_id)
# self.dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=eda_id, agent_instance_id=eda_inst_id)
# Generate the data product and associate it to the ExternalDataset
dprod = DataProduct(name='ncom_product',
description='raw ncom product')
dproduct_id = self.dpms_cli.create_data_product(data_product=dprod)
self.dams_cli.assign_data_product(input_resource_id=ds_id,
data_product_id=dproduct_id,
create_stream=True)
def test__constraints_for_new_request(self):
edres = ExternalDataset(name='test_ed_res',
dataset_description=DatasetDescription(),
update_description=UpdateDescription(),
contact=ContactInformation())
# old_list = [
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0000.ruv',
# '04-Jun-2012 20:43',
# '136K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0100.ruv',
# '04-Jun-2012 21:43',
# '135K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0200.ruv',
# '04-Jun-2012 22:42',
# '137K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0300.ruv',
# '04-Jun-2012 23:41',
# '136K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0400.ruv',
# '05-Jun-2012 00:41',
# '150K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0500.ruv',
# '05-Jun-2012 01:41',
# '142K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0600.ruv',
# '05-Jun-2012 02:41',
# '138K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0700.ruv',
# '05-Jun-2012 03:41',
# '136K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0800.ruv',
# '05-Jun-2012 04:41',
# '138K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0900.ruv',
# '05-Jun-2012 05:40',
# '147K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1000.ruv',
# '05-Jun-2012 06:40',
# '143K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1100.ruv',
# '05-Jun-2012 07:40',
# '148K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1200.ruv',
# '05-Jun-2012 08:40',
# '147K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1300.ruv',
# '05-Jun-2012 09:39',
# '148K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1400.ruv',
# '05-Jun-2012 10:38',
# '143K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1500.ruv',
# '05-Jun-2012 11:43',
# '143K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1600.ruv',
# '05-Jun-2012 12:43',
# '146K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1700.ruv',
# '05-Jun-2012 13:42',
# '134K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1800.ruv',
# '05-Jun-2012 14:42',
# '143K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1900.ruv',
# '05-Jun-2012 15:42',
# '148K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_2000.ruv',
# '05-Jun-2012 16:41',
# '157K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_2100.ruv',
# '05-Jun-2012 17:41',
# '160K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_2200.ruv',
# '05-Jun-2012 18:41',
# '158K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_2300.ruv',
# '05-Jun-2012 19:41',
# '148K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0000.ruv',
# '05-Jun-2012 20:40',
# '140K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0100.ruv',
# '05-Jun-2012 21:40',
# '133K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0200.ruv',
# '05-Jun-2012 22:40',
# '143K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0300.ruv',
# '05-Jun-2012 23:39',
# '156K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0400.ruv',
# '06-Jun-2012 00:39',
# '146K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0500.ruv',
# '06-Jun-2012 01:39',
# '147K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0600.ruv',
# '06-Jun-2012 02:39',
# '147K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0700.ruv',
# '06-Jun-2012 03:43',
# '148K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0800.ruv',
# '06-Jun-2012 04:42',
# '137K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0900.ruv',
# '06-Jun-2012 05:42',
# '130K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_1000.ruv',
# '06-Jun-2012 06:42',
# '129K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_1100.ruv',
# '06-Jun-2012 07:42',
# '136K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_1200.ruv',
# '06-Jun-2012 08:42',
# '137K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_1300.ruv',
# '06-Jun-2012 09:41',
# '151K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_1400.ruv',
# '06-Jun-2012 10:41',
# '153K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_1500.ruv',
# '06-Jun-2012 11:41',
# '156K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_1600.ruv',
# '06-Jun-2012 12:41',
# '157K'),
# ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_1700.ruv',
# '06-Jun-2012 13:40',
# '161K'),]
old_list = [
('test_data/ruv/RDLm_SEAB_2012_06_06_1200.ruv', 1339006638.0,
119066),
('test_data/ruv/RDLm_SEAB_2012_06_06_1300.ruv', 1339006629.0,
109316),
('test_data/ruv/RDLm_SEAB_2012_06_06_1400.ruv', 1339006521.0,
113411),
]
# old_list = None
config = {
'external_dataset_res': edres,
# 'new_data_check':None,
'new_data_check': old_list,
'ds_params': {
# These would be extracted from the dataset_description.parameters during _init_acquisition_cycle, but since that isn't called, just add them here
# 'base_url':'http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/',
# CBM: What's the difference between RDLi and RDLm???
# 'pattern':'<a href="(RDLm.*\.ruv)">.*(\d{2}-[a-zA-Z]{3}-\d{4} \d{2}:\d{2})\s*(\d{1,5}\w)',# Appended to base to filter files; Either a shell style pattern (for filesystem) or regex (for http/ftp)
'base_url':
'test_data/ruv',
'list_pattern':
'RDLm_SEAB_*.ruv',
'date_pattern':
'%Y %m %d %H %M',
'date_extraction_pattern':
'RDLm_SEAB_([\d]{4})_([\d]{2})_([\d]{2})_([\d]{2})([\d]{2}).ruv'
}
}
ret = RuvDataHandler._constraints_for_new_request(config)
log.debug('test__new_data_constraints: {0}'.format(config))
self.assertEqual(
ret['new_files'],
list_file_info(config['ds_params']['base_url'],
config['ds_params']['list_pattern']))
def read_external_dataset(self, ds_id=''):
CWD = os.getcwd()
ext_ds = ExternalDataset(name="test",
dataset_description=DatasetDescription(),
update_description=UpdateDescription(),
contact=ContactInformation())
if ds_id == HFR:
ext_ds.dataset_description.parameters
ext_ds.dataset_description.parameters[
"dataset_path"] = "HFRNet/USEGC/6km/hourly/RTV"
ext_ds.dataset_description.parameters[
"temporal_dimension"] = "time"
ext_ds.dataset_description.parameters["zonal_dimension"] = "lon"
ext_ds.dataset_description.parameters[
"meridional_dimension"] = "lat"
elif ds_id == HFR_LOCAL:
ext_ds.dataset_description.parameters[
"dataset_path"] = CWD + "/test_data/hfr.nc"
ext_ds.dataset_description.parameters[
"temporal_dimension"] = "time"
ext_ds.dataset_description.parameters["zonal_dimension"] = "lon"
ext_ds.dataset_description.parameters[
"meridional_dimension"] = "lat"
elif ds_id == KOKAGG:
ext_ds.dataset_description.parameters[
"dataset_path"] = "hioos/hfr/kokagg"
ext_ds.dataset_description.parameters[
"temporal_dimension"] = "time"
ext_ds.dataset_description.parameters["zonal_dimension"] = "lon"
ext_ds.dataset_description.parameters[
"meridional_dimension"] = "lat"
ext_ds.contact.name = "Pierre Flament"
ext_ds.contact.email = "*****@*****.**"
elif ds_id == AST2:
ext_ds.dataset_description.parameters[
"dataset_path"] = "ooi/AS02CPSM_R_M.nc"
ext_ds.dataset_description.parameters[
"temporal_dimension"] = "time"
ext_ds.dataset_description.parameters["zonal_dimension"] = "lon"
ext_ds.dataset_description.parameters[
"meridional_dimension"] = "lat"
elif ds_id == SSTA:
ext_ds.dataset_description.parameters[
"dataset_path"] = "satellite/GR/ssta/1day"
ext_ds.dataset_description.parameters[
"temporal_dimension"] = "time"
ext_ds.dataset_description.parameters["zonal_dimension"] = "lon"
ext_ds.dataset_description.parameters[
"meridional_dimension"] = "lat"
elif ds_id == GHPM:
ext_ds.dataset_description.parameters[
"dataset_path"] = CWD + "/test_data/ast2_ghpm_spp_ctd_1.nc"
ext_ds.dataset_description.parameters[
"temporal_dimension"] = "time"
ext_ds.dataset_description.parameters["zonal_dimension"] = "lon"
ext_ds.dataset_description.parameters[
"meridional_dimension"] = "lat"
elif ds_id == COMP1:
ext_ds.dataset_description.parameters[
"dataset_path"] = CWD + "/test_data/ast2_ghpm_spp_ctd_1.nc"
ext_ds.dataset_description.parameters[
"temporal_dimension"] = "time"
ext_ds.dataset_description.parameters["zonal_dimension"] = "lon"
ext_ds.dataset_description.parameters[
"meridional_dimension"] = "lat"
elif ds_id == COMP2:
ext_ds.dataset_description.parameters[
"dataset_path"] = CWD + "/test_data/ast2_ghpm_spp_ctd_2.nc"
ext_ds.dataset_description.parameters[
"temporal_dimension"] = "time"
ext_ds.dataset_description.parameters["zonal_dimension"] = "lon"
ext_ds.dataset_description.parameters[
"meridional_dimension"] = "lat"
elif ds_id == COADS:
ext_ds.dataset_description.parameters[
"dataset_path"] = CWD + "/test_data/coads.nc"
ext_ds.dataset_description.parameters[
"temporal_dimension"] = "time"
ext_ds.dataset_description.parameters["zonal_dimension"] = "lon"
ext_ds.dataset_description.parameters[
"meridional_dimension"] = "lat"
elif ds_id == NCOM:
ext_ds.dataset_description.parameters[
"dataset_path"] = CWD + "/test_data/ncom.nc"
ext_ds.dataset_description.parameters[
"temporal_dimension"] = "time"
ext_ds.dataset_description.parameters["zonal_dimension"] = "lon"
ext_ds.dataset_description.parameters[
"meridional_dimension"] = "lat"
elif ds_id == NAVO_GLIDER_LOCAL:
ext_ds.dataset_description.parameters[
"dataset_path"] = CWD + "/test_data/p5150001.nc"
ext_ds.dataset_description.parameters[
"temporal_dimension"] = "time"
ext_ds.dataset_description.parameters["zonal_dimension"] = "lon"
ext_ds.dataset_description.parameters[
"meridional_dimension"] = "lat"
elif ds_id == USGS:
ext_ds.dataset_description.parameters[
"dataset_path"] = CWD + "/test_data/usgs.nc"
ext_ds.dataset_description.parameters[
"temporal_dimension"] = "time"
ext_ds.dataset_description.parameters["zonal_dimension"] = "lon"
ext_ds.dataset_description.parameters[
"meridional_dimension"] = "lat"
else:
return None
return ext_ds
def define_user(self,
user_id='',
first_name='',
last_name='',
username='',
password='',
email='',
attributes=None):
if user_id:
raise NotImplementedError("Update not supported: user_id=%s" %
user_id)
if not email:
raise BadRequest('Email is required')
username = username or email
user = self._get_user_by_email(email)
if user:
raise BadRequest("Email already taken")
if not username or not is_valid_identifier(username,
valid_chars=EMAIL_VALID):
raise BadRequest("Argument username invalid: %s" % username)
if attributes and type(attributes) is not dict:
raise BadRequest("Argument attributes invalid type")
if not first_name:
first_name = username
attributes = attributes or {}
full_name = ("%s %s" %
(first_name, last_name)) if last_name else first_name
IdentityUtils.check_password_policy(password)
contact = ContactInformation(individual_names_given=first_name,
individual_name_family=last_name,
email=email)
user_profile = UserIdentityDetails(contact=contact, profile=attributes)
actor_obj = ActorIdentity(name=full_name, details=user_profile)
# Support fast setting of credentials without expensive compute of bcrypt hash, for quick preload
pwd_salt, pwd_hash = None, None
if attributes and "scion_init_pwdsalt" in attributes and "scion_init_pwdhash" in attributes:
pwd_salt, pwd_hash = attributes.pop(
"scion_init_pwdsalt"), attributes.pop("scion_init_pwdhash")
user_exists = self.idm_client.is_user_existing(username)
if user_exists:
raise BadRequest("Username already taken")
actor_id = self.idm_client.create_actor_identity(actor_obj)
if pwd_salt and pwd_hash:
# Add to credentials
actor_obj1 = self.rr.read(actor_id)
cred_obj = None
for cred in actor_obj1.credentials:
if cred.username == username:
cred_obj = cred
break
if not cred_obj:
cred_obj = Credentials()
cred_obj.username = username
actor_obj1.credentials.append(cred_obj)
actor_obj1.alt_ids.append("UNAME:" + username)
cred_obj.identity_provider = "SciON"
cred_obj.authentication_service = "SciON IdM"
cred_obj.password_salt = pwd_salt
cred_obj.password_hash = pwd_hash
self.rr.update(actor_obj1)
else:
self.idm_client.set_actor_credentials(actor_id, username, password)
return actor_id
def setUp(self):
self._datasets = {}
file = self._create_tst_data_set('base')
ext_ds = ExternalDataset(name="base", dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation())
ext_ds.dataset_description.parameters["dataset_path"] = file
ext_ds.dataset_description.parameters["temporal_dimension"] = 'time'
self._datasets['base'] = HfrRadialDataHandler(data_source=file, ext_dataset=ext_ds)
file = self._create_tst_data_set('mod_gatt')
ext_ds = ExternalDataset(name="mod_gatt", dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation())
ext_ds.dataset_description.parameters["dataset_path"] = file
ext_ds.dataset_description.parameters["temporal_dimension"] = 'time'
self._datasets['mod_gatt'] = HfrRadialDataHandler(data_source=file, ext_dataset=ext_ds)
file = self._create_tst_data_set('new_gatt')
ext_ds = ExternalDataset(name="new_gatt", dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation())
ext_ds.dataset_description.parameters["dataset_path"] = file
ext_ds.dataset_description.parameters["temporal_dimension"] = 'time'
self._datasets['new_gatt'] = HfrRadialDataHandler(data_source=file, ext_dataset=ext_ds)
file = self._create_tst_data_set('mod_var')
ext_ds = ExternalDataset(name="mod_var", dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation())
ext_ds.dataset_description.parameters["dataset_path"] = file
ext_ds.dataset_description.parameters["temporal_dimension"] = 'time'
self._datasets['mod_var'] = HfrRadialDataHandler(data_source=file, ext_dataset=ext_ds)
file = self._create_tst_data_set('new_var')
ext_ds = ExternalDataset(name="new_var", dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation())
ext_ds.dataset_description.parameters["dataset_path"] = file
ext_ds.dataset_description.parameters["temporal_dimension"] = 'time'
self._datasets['new_var'] = HfrRadialDataHandler(data_source=file, ext_dataset=ext_ds)
file = self._create_tst_data_set('mod_varatt')
ext_ds = ExternalDataset(name="mod_varatt", dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation())
ext_ds.dataset_description.parameters["dataset_path"] = file
ext_ds.dataset_description.parameters["temporal_dimension"] = 'time'
self._datasets['mod_varatt'] = HfrRadialDataHandler(data_source=file, ext_dataset=ext_ds)
#file = 'http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/ASSA/RDLi_ASSA_2012_02_09_1100.ruv '
#ext_ds = ExternalDataset(name="remote", dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation())
#ext_ds.dataset_description.parameters["dataset_path"] = file
#ext_ds.dataset_description.parameters["temporal_dimension"] = 'time'
#self._datasets['remote'] = HfrRadialDataHandler(data_source=file, ext_dataset=ext_ds)
pass
def _setup_resources(self):
# TODO: some or all of this (or some variation) should move to DAMS'
# Build the test resources for the dataset
dams_cli = DataAcquisitionManagementServiceClient()
dpms_cli = DataProductManagementServiceClient()
rr_cli = ResourceRegistryServiceClient()
eda = ExternalDatasetAgent()
eda_id = dams_cli.create_external_dataset_agent(eda)
eda_inst = ExternalDatasetAgentInstance()
eda_inst_id = dams_cli.create_external_dataset_agent_instance(eda_inst, external_dataset_agent_id=eda_id)
# Create and register the necessary resources/objects
# Create DataProvider
dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation())
dprov.contact.name = 'Christopher Mueller'
dprov.contact.email = '*****@*****.**'
# Create DataSource
dsrc = DataSource(protocol_type='FILE', institution=Institution(), contact=ContactInformation())
dsrc.connection_params['base_data_url'] = ''
dsrc.contact.name='Tim Giguere'
dsrc.contact.email = '*****@*****.**'
# Create ExternalDataset
ds_name = 'slocum_test_dataset'
dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation())
dset.dataset_description.parameters['dataset_path'] = 'test_data/ru05-2012-021-0-0-sbd.dat'
dset.dataset_description.parameters['temporal_dimension'] = None
dset.dataset_description.parameters['zonal_dimension'] = None
dset.dataset_description.parameters['meridional_dimension'] = None
dset.dataset_description.parameters['vertical_dimension'] = None
dset.dataset_description.parameters['variables'] = [
'c_wpt_y_lmc',
'sci_water_cond',
'm_y_lmc',
'u_hd_fin_ap_inflection_holdoff',
'sci_m_present_time',
'm_leakdetect_voltage_forward',
'sci_bb3slo_b660_scaled',
'c_science_send_all',
'm_gps_status',
'm_water_vx',
'm_water_vy',
'c_heading',
'sci_fl3slo_chlor_units',
'u_hd_fin_ap_gain',
'm_vacuum',
'u_min_water_depth',
'm_gps_lat',
'm_veh_temp',
'f_fin_offset',
'u_hd_fin_ap_hardover_holdoff',
'c_alt_time',
'm_present_time',
'm_heading',
'sci_bb3slo_b532_scaled',
'sci_fl3slo_cdom_units',
'm_fin',
'x_cycle_overrun_in_ms',
'sci_water_pressure',
'u_hd_fin_ap_igain',
'sci_fl3slo_phyco_units',
'm_battpos',
'sci_bb3slo_b470_scaled',
'm_lat',
'm_gps_lon',
'sci_ctd41cp_timestamp',
'm_pressure',
'c_wpt_x_lmc',
'c_ballast_pumped',
'x_lmc_xy_source',
'm_lon',
'm_avg_speed',
'sci_water_temp',
'u_pitch_ap_gain',
'm_roll',
'm_tot_num_inflections',
'm_x_lmc',
'u_pitch_ap_deadband',
'm_final_water_vy',
'm_final_water_vx',
'm_water_depth',
'm_leakdetect_voltage',
'u_pitch_max_delta_battpos',
'm_coulomb_amphr',
'm_pitch',
]
# Create DataSourceModel
dsrc_model = DataSourceModel(name='slocum_model')
dsrc_model.model = 'SLOCUM'
dsrc_model.data_handler_module = 'N/A'
dsrc_model.data_handler_class = 'N/A'
## Run everything through DAMS
ds_id = dams_cli.create_external_dataset(external_dataset=dset)
ext_dprov_id = dams_cli.create_external_data_provider(external_data_provider=dprov)
ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc)
ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model)
# Register the ExternalDataset
dproducer_id = dams_cli.register_external_data_set(external_dataset_id=ds_id)
# Or using each method
dams_cli.assign_data_source_to_external_data_provider(data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id)
dams_cli.assign_data_source_to_data_model(data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id)
dams_cli.assign_external_dataset_to_data_source(external_dataset_id=ds_id, data_source_id=ext_dsrc_id)
dams_cli.assign_external_dataset_to_agent_instance(external_dataset_id=ds_id, agent_instance_id=eda_inst_id)
# dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=self.eda_id, agent_instance_id=self.eda_inst_id)
# Generate the data product and associate it to the ExternalDataset
dprod = DataProduct(name='slocum_parsed_product', description='parsed slocum product')
dproduct_id = dpms_cli.create_data_product(data_product=dprod)
dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id, create_stream=True)
stream_id, assn = rr_cli.find_objects(subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True)
stream_id = stream_id[0]
log.info('Created resources: {0}'.format({'ExternalDataset':ds_id, 'ExternalDataProvider':ext_dprov_id, 'DataSource':ext_dsrc_id, 'DataSourceModel':ext_dsrc_model_id, 'DataProducer':dproducer_id, 'DataProduct':dproduct_id, 'Stream':stream_id}))
#CBM: Use CF standard_names
ttool = TaxyTool()
ttool.add_taxonomy_set('c_wpt_y_lmc'),
ttool.add_taxonomy_set('sci_water_cond'),
ttool.add_taxonomy_set('m_y_lmc'),
ttool.add_taxonomy_set('u_hd_fin_ap_inflection_holdoff'),
ttool.add_taxonomy_set('sci_m_present_time'),
ttool.add_taxonomy_set('m_leakdetect_voltage_forward'),
ttool.add_taxonomy_set('sci_bb3slo_b660_scaled'),
ttool.add_taxonomy_set('c_science_send_all'),
ttool.add_taxonomy_set('m_gps_status'),
ttool.add_taxonomy_set('m_water_vx'),
ttool.add_taxonomy_set('m_water_vy'),
ttool.add_taxonomy_set('c_heading'),
ttool.add_taxonomy_set('sci_fl3slo_chlor_units'),
ttool.add_taxonomy_set('u_hd_fin_ap_gain'),
ttool.add_taxonomy_set('m_vacuum'),
ttool.add_taxonomy_set('u_min_water_depth'),
ttool.add_taxonomy_set('m_gps_lat'),
ttool.add_taxonomy_set('m_veh_temp'),
ttool.add_taxonomy_set('f_fin_offset'),
ttool.add_taxonomy_set('u_hd_fin_ap_hardover_holdoff'),
ttool.add_taxonomy_set('c_alt_time'),
ttool.add_taxonomy_set('m_present_time'),
ttool.add_taxonomy_set('m_heading'),
ttool.add_taxonomy_set('sci_bb3slo_b532_scaled'),
ttool.add_taxonomy_set('sci_fl3slo_cdom_units'),
ttool.add_taxonomy_set('m_fin'),
ttool.add_taxonomy_set('x_cycle_overrun_in_ms'),
ttool.add_taxonomy_set('sci_water_pressure'),
ttool.add_taxonomy_set('u_hd_fin_ap_igain'),
ttool.add_taxonomy_set('sci_fl3slo_phyco_units'),
ttool.add_taxonomy_set('m_battpos'),
ttool.add_taxonomy_set('sci_bb3slo_b470_scaled'),
ttool.add_taxonomy_set('m_lat'),
ttool.add_taxonomy_set('m_gps_lon'),
ttool.add_taxonomy_set('sci_ctd41cp_timestamp'),
ttool.add_taxonomy_set('m_pressure'),
ttool.add_taxonomy_set('c_wpt_x_lmc'),
ttool.add_taxonomy_set('c_ballast_pumped'),
ttool.add_taxonomy_set('x_lmc_xy_source'),
ttool.add_taxonomy_set('m_lon'),
ttool.add_taxonomy_set('m_avg_speed'),
ttool.add_taxonomy_set('sci_water_temp'),
ttool.add_taxonomy_set('u_pitch_ap_gain'),
ttool.add_taxonomy_set('m_roll'),
ttool.add_taxonomy_set('m_tot_num_inflections'),
ttool.add_taxonomy_set('m_x_lmc'),
ttool.add_taxonomy_set('u_pitch_ap_deadband'),
ttool.add_taxonomy_set('m_final_water_vy'),
ttool.add_taxonomy_set('m_final_water_vx'),
ttool.add_taxonomy_set('m_water_depth'),
ttool.add_taxonomy_set('m_leakdetect_voltage'),
ttool.add_taxonomy_set('u_pitch_max_delta_battpos'),
ttool.add_taxonomy_set('m_coulomb_amphr'),
ttool.add_taxonomy_set('m_pitch'),
#CBM: Eventually, probably want to group this crap somehow - not sure how yet...
# Create the logger for receiving publications
self.create_stream_and_logger(name='slocum',stream_id=stream_id)
self.EDA_RESOURCE_ID = ds_id
self.EDA_NAME = ds_name
self.DVR_CONFIG['dh_cfg'] = {
'TESTING':True,
'stream_id':stream_id,
'external_dataset_res':dset,
'taxonomy':ttool.dump(),
'data_producer_id':dproducer_id,#CBM: Should this be put in the main body of the config - with mod & cls?
'max_records':20,
}
def test_get_provenance(self):
#create a deployment with metadata and an initial site and device
instrument_site_obj = IonObject(RT.InstrumentSite,
name='InstrumentSite1',
description='test instrument site')
instrument_site_id = self.omsclient.create_instrument_site(instrument_site_obj, "")
log.debug( 'test_get_provenance: new instrument_site_id id = %s ', str(instrument_site_id))
# Create InstrumentModel
instModel_obj = IonObject(RT.InstrumentModel,
name='SBE37IMModel',
description="SBE37IMModel" )
try:
instModel_id = self.imsclient.create_instrument_model(instModel_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentModel: %s" %ex)
log.debug( 'test_get_provenance: new InstrumentModel id = %s ', str(instModel_id))
self.omsclient.assign_instrument_model_to_instrument_site(instModel_id, instrument_site_id)
# Create InstrumentAgent
parsed_config = StreamConfiguration(stream_name='parsed', parameter_dictionary_name='ctd_parsed_param_dict' )
instAgent_obj = IonObject(RT.InstrumentAgent,
name='agent007',
description="SBE37IMAgent",
driver_uri=DRV_URI_GOOD,
stream_configurations = [parsed_config] )
try:
instAgent_id = self.imsclient.create_instrument_agent(instAgent_obj)
except BadRequest as ex:
self.fail("failed to create new InstrumentAgent: %s" %ex)
log.debug( 'test_get_provenance:new InstrumentAgent id = %s', instAgent_id)
self.imsclient.assign_instrument_model_to_instrument_agent(instModel_id, instAgent_id)
# Create InstrumentDevice
log.debug('test_get_provenance: Create instrument resource to represent the SBE37 (SA Req: L4-CI-SA-RQ-241) ')
instDevice_obj = IonObject(RT.InstrumentDevice, name='SBE37IMDevice', description="SBE37IMDevice", serial_number="12345" )
try:
instDevice_id = self.imsclient.create_instrument_device(instrument_device=instDevice_obj)
self.imsclient.assign_instrument_model_to_instrument_device(instModel_id, instDevice_id)
except BadRequest as ex:
self.fail("failed to create new InstrumentDevice: %s" %ex)
log.debug("test_get_provenance: new InstrumentDevice id = %s (SA Req: L4-CI-SA-RQ-241) ", instDevice_id)
#-------------------------------
# Create CTD Parsed data product
#-------------------------------
tdom, sdom = time_series_domain()
sdom = sdom.dump()
tdom = tdom.dump()
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
parsed_stream_def_id = self.pubsubclient.create_stream_definition(name='parsed', parameter_dictionary_id=pdict_id)
log.debug( 'test_get_provenance:Creating new CDM data product with a stream definition')
dp_obj = IonObject(RT.DataProduct,
name='the parsed data',
description='ctd stream test',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_parsed_data_product = self.dpmsclient.create_data_product(data_product=dp_obj, stream_definition_id=parsed_stream_def_id)
log.debug( 'new dp_id = %s', ctd_parsed_data_product)
self.damsclient.assign_data_product(input_resource_id=instDevice_id, data_product_id=ctd_parsed_data_product)
self.dpmsclient.activate_data_product_persistence(data_product_id=ctd_parsed_data_product)
#-------------------------------
# create a data product for the site to pass the OMS check.... we need to remove this check
#-------------------------------
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
log_data_product_id = self.dpmsclient.create_data_product(dp_obj, parsed_stream_def_id)
#-------------------------------
# Deploy instrument device to instrument site
#-------------------------------
deployment_obj = IonObject(RT.Deployment,
name='TestDeployment',
description='some new deployment')
deployment_id = self.omsclient.create_deployment(deployment_obj)
self.omsclient.deploy_instrument_site(instrument_site_id, deployment_id)
self.imsclient.deploy_instrument_device(instDevice_id, deployment_id)
log.debug("test_create_deployment: created deployment id: %s ", str(deployment_id) )
self.omsclient.activate_deployment(deployment_id)
inst_device_objs, _ = self.rrclient.find_objects(subject=instrument_site_id, predicate=PRED.hasDevice, object_type=RT.InstrumetDevice, id_only=False)
log.debug("test_create_deployment: deployed device: %s ", str(inst_device_objs[0]) )
#-------------------------------
# Create the agent instance
#-------------------------------
port_agent_config = {
'device_addr': CFG.device.sbe37.host,
'device_port': CFG.device.sbe37.port,
'process_type': PortAgentProcessType.UNIX,
'binary_path': "port_agent",
'port_agent_addr': 'localhost',
'command_port': CFG.device.sbe37.port_agent_cmd_port,
'data_port': CFG.device.sbe37.port_agent_data_port,
'log_level': 5,
'type': PortAgentType.ETHERNET
}
instAgentInstance_obj = IonObject(RT.InstrumentAgentInstance, name='SBE37IMAgentInstance',
description="SBE37IMAgentInstance",
port_agent_config = port_agent_config)
instAgentInstance_id = self.imsclient.create_instrument_agent_instance(instAgentInstance_obj, instAgent_id, instDevice_id)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition ctd_L0_all")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L0_all',
description='transform ctd package into three separate L0 streams',
module='ion.processes.data.transforms.ctd.ctd_L0_all',
class_name='ctd_L0_all')
try:
ctd_L0_all_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new ctd_L0_all data process definition: %s" %ex)
#-------------------------------
# L1 Conductivity: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1ConductivityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_conductivity',
description='create the L1 conductivity data product',
module='ion.processes.data.transforms.ctd.ctd_L1_conductivity',
class_name='CTDL1ConductivityTransform')
try:
ctd_L1_conductivity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1ConductivityTransform data process definition: %s" %ex)
#-------------------------------
# L1 Pressure: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1PressureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_pressure',
description='create the L1 pressure data product',
module='ion.processes.data.transforms.ctd.ctd_L1_pressure',
class_name='CTDL1PressureTransform')
try:
ctd_L1_pressure_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1PressureTransform data process definition: %s" %ex)
#-------------------------------
# L1 Temperature: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition CTDL1TemperatureTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L1_temperature',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L1_temperature',
class_name='CTDL1TemperatureTransform')
try:
ctd_L1_temperature_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new CTDL1TemperatureTransform data process definition: %s" %ex)
#-------------------------------
# L2 Salinity: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition SalinityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_salinity',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_salinity',
class_name='SalinityTransform')
try:
ctd_L2_salinity_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new SalinityTransform data process definition: %s" %ex)
#-------------------------------
# L2 Density: Data Process Definition
#-------------------------------
log.debug("TestDataProductProvenance: create data process definition DensityTransform")
dpd_obj = IonObject(RT.DataProcessDefinition,
name='ctd_L2_density',
description='create the L1 temperature data product',
module='ion.processes.data.transforms.ctd.ctd_L2_density',
class_name='DensityTransform')
try:
ctd_L2_density_dprocdef_id = self.dataprocessclient.create_data_process_definition(dpd_obj)
except BadRequest as ex:
self.fail("failed to create new DensityTransform data process definition: %s" %ex)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l0_conductivity_id = self.pubsubclient.create_stream_definition(name='L0_Conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_conductivity_id, ctd_L0_all_dprocdef_id, binding='conductivity' )
outgoing_stream_l0_pressure_id = self.pubsubclient.create_stream_definition(name='L0_Pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_pressure_id, ctd_L0_all_dprocdef_id, binding='pressure' )
outgoing_stream_l0_temperature_id = self.pubsubclient.create_stream_definition(name='L0_Temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l0_temperature_id, ctd_L0_all_dprocdef_id, binding='temperature' )
log.debug("TestDataProductProvenance: create output data product L0 conductivity")
ctd_l0_conductivity_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Conductivity',
description='transform output conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_conductivity_output_dp_obj,
outgoing_stream_l0_conductivity_id)
log.debug("TestDataProductProvenance: create output data product L0 pressure")
ctd_l0_pressure_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Pressure',
description='transform output pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_pressure_output_dp_obj,
outgoing_stream_l0_pressure_id)
log.debug("TestDataProductProvenance: create output data product L0 temperature")
ctd_l0_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L0_Temperature',
description='transform output temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l0_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l0_temperature_output_dp_obj,
outgoing_stream_l0_temperature_id)
#-------------------------------
# L1 Conductivity - Temperature - Pressure: Output Data Products
#-------------------------------
outgoing_stream_l1_conductivity_id = self.pubsubclient.create_stream_definition(name='L1_conductivity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_conductivity_id, ctd_L1_conductivity_dprocdef_id, binding='conductivity' )
outgoing_stream_l1_pressure_id = self.pubsubclient.create_stream_definition(name='L1_Pressure', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_pressure_id, ctd_L1_pressure_dprocdef_id, binding='pressure' )
outgoing_stream_l1_temperature_id = self.pubsubclient.create_stream_definition(name='L1_Temperature', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l1_temperature_id, ctd_L1_temperature_dprocdef_id, binding='temperature' )
log.debug("TestDataProductProvenance: create output data product L1 conductivity")
ctd_l1_conductivity_output_dp_obj = IonObject(RT.DataProduct,
name='L1_Conductivity',
description='transform output L1 conductivity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_conductivity_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_conductivity_output_dp_obj,
outgoing_stream_l1_conductivity_id)
log.debug("TestDataProductProvenance: create output data product L1 pressure")
ctd_l1_pressure_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Pressure',
description='transform output L1 pressure',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_pressure_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_pressure_output_dp_obj,
outgoing_stream_l1_pressure_id)
log.debug("TestDataProductProvenance: create output data product L1 temperature")
ctd_l1_temperature_output_dp_obj = IonObject( RT.DataProduct,
name='L1_Temperature',
description='transform output L1 temperature',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l1_temperature_output_dp_id = self.dpmsclient.create_data_product(ctd_l1_temperature_output_dp_obj,
outgoing_stream_l1_temperature_id)
#-------------------------------
# L2 Salinity - Density: Output Data Products
#-------------------------------
outgoing_stream_l2_salinity_id = self.pubsubclient.create_stream_definition(name='L2_salinity', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_salinity_id, ctd_L2_salinity_dprocdef_id, binding='salinity' )
outgoing_stream_l2_density_id = self.pubsubclient.create_stream_definition(name='L2_Density', parameter_dictionary_id=pdict_id)
self.dataprocessclient.assign_stream_definition_to_data_process_definition(outgoing_stream_l2_density_id, ctd_L2_density_dprocdef_id, binding='density' )
log.debug("TestDataProductProvenance: create output data product L2 Salinity")
ctd_l2_salinity_output_dp_obj = IonObject( RT.DataProduct,
name='L2_Salinity',
description='transform output L2 salinity',
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l2_salinity_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_salinity_output_dp_obj,
outgoing_stream_l2_salinity_id)
log.debug("TestDataProductProvenance: create output data product L2 Density")
# ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct,
# name='L2_Density',
# description='transform output pressure',
# temporal_domain = tdom,
# spatial_domain = sdom)
#
# ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj,
# outgoing_stream_l2_density_id,
# parameter_dictionary)
contactInfo = ContactInformation()
contactInfo.individual_names_given = "Bill"
contactInfo.individual_name_family = "Smith"
contactInfo.street_address = "111 First St"
contactInfo.city = "San Diego"
contactInfo.email = "*****@*****.**"
contactInfo.phones = ["858-555-6666"]
contactInfo.country = "USA"
contactInfo.postal_code = "92123"
ctd_l2_density_output_dp_obj = IonObject( RT.DataProduct,
name='L2_Density',
description='transform output pressure',
contacts = [contactInfo],
iso_topic_category = "my_iso_topic_category_here",
quality_control_level = "1",
temporal_domain = tdom,
spatial_domain = sdom)
ctd_l2_density_output_dp_id = self.dpmsclient.create_data_product(ctd_l2_density_output_dp_obj,
outgoing_stream_l2_density_id)
#-------------------------------
# L0 Conductivity - Temperature - Pressure: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L0 all data_process start")
try:
input_data_products = [ctd_parsed_data_product]
output_data_products = [ctd_l0_conductivity_output_dp_id, ctd_l0_pressure_output_dp_id, ctd_l0_temperature_output_dp_id]
ctd_l0_all_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id = ctd_L0_all_dprocdef_id,
in_data_product_ids = input_data_products,
out_data_product_ids = output_data_products
)
#activate only this data process just for coverage
self.dataprocessclient.activate_data_process(ctd_l0_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
contents = "this is the lookup table contents, replace with a file..."
att = IonObject(RT.Attachment, name='deviceLookupTable', content=base64.encodestring(contents), keywords=['DataProcessInput'], attachment_type=AttachmentType.ASCII)
deviceAttachment = self.rrclient.create_attachment(ctd_l0_all_data_process_id, att)
log.info( 'test_createTransformsThenActivateInstrument: InstrumentDevice attachment id = %s', deviceAttachment)
log.debug("TestDataProductProvenance: create L0 all data_process return")
#-------------------------------
# L1 Conductivity: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1 Conductivity data_process start")
try:
l1_conductivity_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id = ctd_L1_conductivity_dprocdef_id,
in_data_product_ids = [ctd_l0_conductivity_output_dp_id],
out_data_product_ids = [ctd_l1_conductivity_output_dp_id])
self.dataprocessclient.activate_data_process(l1_conductivity_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L1 Pressure: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1_Pressure data_process start")
try:
l1_pressure_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id = ctd_L1_pressure_dprocdef_id,
in_data_product_ids = [ctd_l0_pressure_output_dp_id],
out_data_product_ids = [ctd_l1_pressure_output_dp_id])
self.dataprocessclient.activate_data_process(l1_pressure_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L1 Temperature: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L1_Pressure data_process start")
try:
l1_temperature_all_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id = ctd_L1_temperature_dprocdef_id,
in_data_product_ids = [ctd_l0_temperature_output_dp_id],
out_data_product_ids = [ctd_l1_temperature_output_dp_id])
self.dataprocessclient.activate_data_process(l1_temperature_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L2 Salinity: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L2_salinity data_process start")
try:
l2_salinity_all_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id = ctd_L2_salinity_dprocdef_id,
in_data_product_ids = [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id],
out_data_product_ids = [ctd_l2_salinity_output_dp_id])
self.dataprocessclient.activate_data_process(l2_salinity_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# L2 Density: Create the data process
#-------------------------------
log.debug("TestDataProductProvenance: create L2_Density data_process start")
try:
in_dp_ids = [ctd_l1_conductivity_output_dp_id, ctd_l1_pressure_output_dp_id, ctd_l1_temperature_output_dp_id]
out_dp_ids = [ctd_l2_density_output_dp_id]
l2_density_all_data_process_id = self.dataprocessclient.create_data_process(
data_process_definition_id = ctd_L2_density_dprocdef_id,
in_data_product_ids = in_dp_ids,
out_data_product_ids = out_dp_ids)
self.dataprocessclient.activate_data_process(l2_density_all_data_process_id)
except BadRequest as ex:
self.fail("failed to create new data process: %s" %ex)
#-------------------------------
# Launch InstrumentAgentInstance, connect to the resource agent client
#-------------------------------
self.imsclient.start_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
inst_agent_instance_obj= self.imsclient.read_instrument_agent_instance(instAgentInstance_id)
print 'TestDataProductProvenance: Instrument agent instance obj: = ', inst_agent_instance_obj
# Start a resource agent client to talk with the instrument agent.
# self._ia_client = ResourceAgentClient('iaclient', name=ResourceAgentClient._get_agent_process_id(instDevice_id, process=FakeProcess())
# print 'activate_instrument: got ia client %s', self._ia_client
# log.debug(" test_createTransformsThenActivateInstrument:: got ia client %s", str(self._ia_client))
#-------------------------------
# Deactivate InstrumentAgentInstance
#-------------------------------
self.imsclient.stop_instrument_agent_instance(instrument_agent_instance_id=instAgentInstance_id)
self.dataprocessclient.deactivate_data_process(l2_density_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l2_salinity_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_temperature_all_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_pressure_data_process_id)
self.dataprocessclient.deactivate_data_process(l1_conductivity_data_process_id)
self.dataprocessclient.deactivate_data_process(ctd_l0_all_data_process_id)
#-------------------------------
# Retrieve the provenance info for the ctd density data product
#-------------------------------
provenance_dict = self.dpmsclient.get_data_product_provenance(ctd_l2_density_output_dp_id)
log.debug("TestDataProductProvenance: provenance_dict %s", str(provenance_dict))
#validate that products are represented
self.assertTrue (provenance_dict[str(ctd_l1_conductivity_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l0_conductivity_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l2_density_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l1_temperature_output_dp_id)])
self.assertTrue (provenance_dict[str(ctd_l0_temperature_output_dp_id)])
density_dict = (provenance_dict[str(ctd_l2_density_output_dp_id)])
self.assertEquals(density_dict['producer'], [l2_density_all_data_process_id])
#-------------------------------
# Retrieve the extended resource for this data product
#-------------------------------
extended_product = self.dpmsclient.get_data_product_extension(ctd_l2_density_output_dp_id)
self.assertEqual(1, len(extended_product.data_processes) )
self.assertEqual(3, len(extended_product.process_input_data_products) )
# log.debug("TestDataProductProvenance: DataProduct provenance_product_list %s", str(extended_product.provenance_product_list))
# log.debug("TestDataProductProvenance: DataProduct data_processes %s", str(extended_product.data_processes))
# log.debug("TestDataProductProvenance: DataProduct process_input_data_products %s", str(extended_product.process_input_data_products))
# log.debug("TestDataProductProvenance: provenance %s", str(extended_product.computed.provenance.value))
#-------------------------------
# Retrieve the extended resource for this data process
#-------------------------------
extended_process_def = self.dataprocessclient.get_data_process_definition_extension(ctd_L0_all_dprocdef_id)
# log.debug("TestDataProductProvenance: DataProcess extended_process_def %s", str(extended_process_def))
# log.debug("TestDataProductProvenance: DataProcess data_processes %s", str(extended_process_def.data_processes))
# log.debug("TestDataProductProvenance: DataProcess data_products %s", str(extended_process_def.data_products))
self.assertEqual(1, len(extended_process_def.data_processes) )
self.assertEqual(3, len(extended_process_def.output_stream_definitions) )
self.assertEqual(3, len(extended_process_def.data_products) ) #one list because of one data process
#-------------------------------
# Request the xml report
#-------------------------------
results = self.dpmsclient.get_data_product_provenance_report(ctd_l2_density_output_dp_id)
print results
#-------------------------------
# Cleanup
#-------------------------------
self.dpmsclient.delete_data_product(ctd_parsed_data_product)
self.dpmsclient.delete_data_product(log_data_product_id)
self.dpmsclient.delete_data_product(ctd_l0_conductivity_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l0_pressure_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l0_temperature_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l1_conductivity_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l1_pressure_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l1_temperature_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l2_salinity_output_dp_id)
self.dpmsclient.delete_data_product(ctd_l2_density_output_dp_id)
def test__constraints_for_new_request(self):
old_list = [
1302393600, 1302480000, 1302566400, 1302652800, 1302739200,
1302825600, 1302912000, 1302998400, 1303084800, 1303171200,
1303257600, 1303344000, 1303430400, 1303516800, 1303603200,
1303689600, 1303776000, 1303862400, 1303948800, 1304035200,
1304121600, 1304208000, 1304294400, 1304380800, 1304467200,
1304553600, 1304640000, 1304726400, 1304812800, 1304899200,
1304985600, 1305072000, 1305158400, 1305244800, 1305331200,
1305417600, 1305504000, 1305590400, 1305676800, 1305763200,
1305849600, 1305936000, 1306022400, 1306108800, 1306195200,
1306281600, 1306368000, 1306454400, 1306540800, 1306627200,
1306713600, 1306800000, 1306886400, 1306972800, 1307059200,
1307145600, 1307232000, 1307318400, 1307404800, 1307491200,
1307577600, 1307664000, 1307750400, 1307836800, 1307923200,
1308009600, 1308096000, 1308182400, 1308268800, 1308355200,
1308441600, 1308528000, 1308614400, 1308700800, 1308787200,
1308873600, 1308960000, 1309046400, 1309132800, 1309219200,
1309305600, 1309392000, 1309478400, 1309564800, 1309651200,
1309737600, 1309824000, 1309910400, 1309996800, 1310083200,
1310169600, 1310256000, 1310342400, 1310428800, 1310515200,
1310601600, 1310688000, 1310774400, 1310860800, 1310947200,
1311033600, 1311120000, 1311206400, 1311292800, 1311379200,
1311465600, 1311552000, 1311638400, 1311724800, 1311811200,
1311897600, 1311984000, 1312070400, 1312156800, 1312243200,
1312329600, 1312416000, 1312502400, 1312588800, 1312675200,
1312761600, 1312848000, 1312934400, 1313020800, 1313107200,
1313193600, 1313280000, 1313366400, 1313452800, 1313539200,
1313625600, 1313712000, 1313798400, 1313884800, 1313971200,
1314057600, 1314144000, 1314230400, 1314316800, 1314403200,
1314489600, 1314576000, 1314662400, 1314748800, 1314835200,
1314921600, 1315008000, 1315094400, 1315180800, 1315267200,
1315353600, 1315440000, 1315526400, 1315612800, 1315699200,
1315785600, 1315872000, 1315958400, 1316044800, 1316131200,
1316217600, 1316304000, 1316390400, 1316476800, 1316563200,
1316649600, 1316736000, 1316822400, 1316908800, 1316995200,
1317081600, 1317168000, 1317254400, 1317340800, 1317427200,
1317513600, 1317600000, 1317686400, 1317772800, 1317859200,
1317945600, 1318032000, 1318118400, 1318204800, 1318291200,
1318377600, 1318464000, 1318550400, 1318636800, 1318723200,
1318809600, 1318896000, 1318982400, 1319068800, 1319155200,
1319241600, 1319328000, 1319414400, 1319500800, 1319587200,
1319673600, 1319760000, 1319846400, 1319932800, 1320019200,
1320105600, 1320192000, 1320278400, 1320364800, 1320451200,
1320537600, 1320624000, 1320710400, 1320796800, 1320883200,
1320969600, 1321056000, 1321142400, 1321228800, 1321315200,
1321401600, 1321488000, 1321574400, 1321660800, 1321747200,
1321833600, 1321920000, 1322006400, 1322092800, 1322179200,
1322265600, 1322352000, 1322438400, 1322524800, 1322611200,
1322697600, 1322784000, 1322870400, 1322956800, 1323043200,
1323129600, 1323216000, 1323302400, 1323388800, 1323475200,
1323561600, 1323648000, 1323734400, 1323820800, 1323907200,
1323993600, 1324080000, 1324166400, 1324252800, 1324339200,
1324425600, 1324512000, 1324598400, 1324684800, 1324771200,
1324857600, 1324944000, 1325030400, 1325116800, 1325203200,
1325289600, 1325376000, 1325462400, 1325548800, 1325635200,
1325721600, 1325808000, 1325894400, 1325980800, 1326067200,
1326153600, 1326240000, 1326326400, 1326412800, 1326499200,
1326585600
]
edres = ExternalDataset(name='test_ed_res',
dataset_description=DatasetDescription(),
update_description=UpdateDescription(),
contact=ContactInformation())
edres.dataset_description.parameters[
'dataset_path'] = 'test_data/usgs.nc'
edres.dataset_description.parameters['temporal_dimension'] = 'time'
edres.update_description.parameters['new_data_check'] = msgpack.packb(
old_list, default=encode_ion)
config = {
'external_dataset_res':
edres,
'dataset_object':
Dataset(edres.dataset_description.parameters['dataset_path'])
}
ret = NetcdfDataHandler._constraints_for_new_request(config)
#log.debug('test__constraints_for_new_request: {0}'.format(ret['temporal_slice']))
self.assertEqual(ret['temporal_slice'], slice(281, 295, None))
