def test__constraints_for_new_request_no_new_data(self, list_file_info_mock): edres = ExternalDataset(name='test_ed_res', dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) old_list = [ ('test_data/ruv/RDLm_SEAB_2012_06_06_1200.ruv', 1339006638.0, 119066), ('test_data/ruv/RDLm_SEAB_2012_06_06_1300.ruv', 1339006629.0, 109316), ('test_data/ruv/RDLm_SEAB_2012_06_06_1400.ruv', 1339006521.0, 113411), ] config = { 'external_dataset_res': edres, 'new_data_check': old_list, 'ds_params': { 'base_url': 'test_data/ruv', 'list_pattern': 'RDLm_SEAB_*.ruv', 'date_pattern': '%Y %m %d %H %M', 'date_extraction_pattern': 'RDLm_SEAB_([\d]{4})_([\d]{2})_([\d]{2})_([\d]{2})([\d]{2}).ruv' } } list_file_info_mock.return_value = old_list with self.assertRaises(NoNewDataWarning): RuvDataHandler._constraints_for_new_request(config)
def test__init_acquisition_cycle_ext_ds_res(self): edres = ExternalDataset(name='test_ed_res', dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) edres.dataset_description.parameters[ 'dataset_path'] = 'test_data/usgs.nc' config = {'external_dataset_res': edres} NetcdfDataHandler._init_acquisition_cycle(config) self.assertIn('dataset_object', config) self.assertTrue(isinstance(config['dataset_object'], Dataset))
def create_external_dataset(self): ds_name = 'hypm_01_wfp_eng_dataset' dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) dset.dataset_description.parameters['base_url'] = 'test_data' dset.dataset_description.parameters['list_pattern'] = 'E*.HEX' return self.data_acquisition_management.create_external_dataset( external_dataset=dset)
def test__init_acquisition_cycle_ext_ds_res(self, SlocumParser_mock): edres = ExternalDataset(name='test_ed_res', dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) edres.dataset_description.parameters[ 'dataset_path'] = 'test_data/ru05-2012-021-0-0-sbd.dat' config = {'external_dataset_res': edres} SlocumDataHandler._init_acquisition_cycle(config) SlocumParser_mock.assert_called_once_with( edres.dataset_description.parameters['dataset_path']) self.assertIn('parser', config) self.assertEquals(config['parser'], SlocumParser_mock())
def test__init_acquisition_cycle_ext_ds_res(self, RuvParser_mock): edres = ExternalDataset(name='test_ed_res', dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) edres.dataset_description.parameters[ 'dataset_path'] = 'test_data/RDLi_SEAB_2011_08_24_1600.ruv' config = {'external_dataset_res': edres} RuvDataHandler._init_acquisition_cycle(config) RuvParser_mock.assert_called_once_with( edres.dataset_description.parameters['dataset_path']) self.assertIn('parser', config) self.assertTrue(config['parser'], RuvParser_mock())
def _create_tst_data_set_handler(self, key="DS_BASE"): th, tn = tempfile.mkstemp(prefix=(key + "_"), suffix='.nc') self._create_tst_data_set(th, tn, key) ext_ds = ExternalDataset(name="test", dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) ext_ds.dataset_description.parameters["dataset_path"] = tn ext_ds.dataset_description.parameters["temporal_dimension"] = 'time' dsh = DapExternalDataHandler(ext_dataset=ext_ds) self._dsh_list[key] = dsh, tn
def create_external_dataset(self): ds_name = 'slocum_dataset' dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) dset.dataset_description.parameters['base_url'] = 'test_data/slocum' dset.dataset_description.parameters['list_pattern'] = 'ru05-*-sbd.dat' dset.dataset_description.parameters['date_pattern'] = '%Y %j' dset.dataset_description.parameters[ 'date_extraction_pattern'] = 'ru05-([\d]{4})-([\d]{3})-\d-\d-sbd.dat' return self.data_acquisition_management.create_external_dataset( external_dataset=dset)
def test__init_acquisition_cycle_ext_ds_res(self): edres = ExternalDataset(name='test_ed_res', dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) edres.dataset_description.parameters['base_url'] = 'test_data/dir' edres.dataset_description.parameters['list_pattern'] = 'test_filter' config = {'external_dataset_res': edres} RuvDataHandler._init_acquisition_cycle(config) self.assertIn('ds_params', config) ds_params = config['ds_params'] self.assertIn('base_url', ds_params) self.assertEquals(ds_params['base_url'], 'test_data/dir') self.assertIn('list_pattern', ds_params) self.assertEquals(ds_params['list_pattern'], 'test_filter')
def test__get_data(self, RecordDictionaryTool_mock): edres = ExternalDataset(name='test_ed_res', dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) edres.dataset_description.parameters[ 'dataset_path'] = 'test_data/usgs.nc' edres.dataset_description.parameters['temporal_dimension'] = 'time' edres.dataset_description.parameters['zonal_dimension'] = 'lat' edres.dataset_description.parameters['meridional_dimension'] = 'lon' edres.dataset_description.parameters['vertical_dimension'] = 'z' edres.dataset_description.parameters['variables'] = [ 'water_temperature' ] config = { 'external_dataset_res': edres, 'dataset_object': Dataset(edres.dataset_description.parameters['dataset_path']), 'constraints': { 'count': 6 }, 'max_records': 4, 'data_producer_id': sentinel.dprod_id, 'param_dictionary': sentinel.pdict, 'stream_def': sentinel.stream_def_id } my_dict = {} def setitem(name, val): my_dict[name] = val retval = MagicMock(spec=RecordDictionaryTool) retval.__setitem__ = Mock(side_effect=setitem) retval.to_granule.return_value = MagicMock(spec=Granule) RecordDictionaryTool_mock.return_value = retval for x in NetcdfDataHandler._get_data(config): self.assertTrue(isinstance(x, Granule)) retval.to_granule.assert_any_call()
def test_constructor(self): ext_ds = ExternalDataset(name="test", dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) ext_ds.dataset_description.parameters["dataset_path"] = self._dsh_list[ "DS_BASE"][1] ext_ds.dataset_description.parameters["temporal_dimension"] = 'time' dsh = DapExternalDataHandler(ext_dataset=ext_ds) self.assertTrue(type(dsh), DapExternalDataHandler) dsrc = DataSource(name="test") dsrc.connection_params["base_data_url"] = "" dsh1 = DapExternalDataHandler(data_source=dsrc, ext_dataset=ext_ds) self.assertTrue(type(dsh1), DapExternalDataHandler) dsh2 = DapExternalDataHandler(data_source=dsrc, ext_dataset=ext_ds, BLOCK_SIZE=10) self.assertEqual(dsh2._block_size, 10)
def test__constraints_for_new_request(self): # ret = SlocumDataHandler._constraints_for_new_request({}) # self.assertIsInstance(ret, dict) old_list = [ ('test_data/slocum/ru05-2012-021-0-0-sbd.dat', 1337261358.0, 521081), ('test_data/slocum/ru05-2012-022-0-0-sbd.dat', 1337261358.0, 521081), ] # old_list = None edres = ExternalDataset(name='test_ed_res', dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) config = { 'external_dataset_res': edres, 'new_data_check': old_list, 'ds_params': { # These would be extracted from the dataset_description.parameters during _init_acquisition_cycle, but since that isn't called, just add them here 'base_url': 'test_data/slocum/', 'list_pattern': 'ru05-*-sbd.dat', # Appended to base to filter files; Either a shell style pattern (for filesystem) or regex (for http/ftp) 'date_pattern': '%Y %j', 'date_extraction_pattern': 'ru05-([\d]{4})-([\d]{3})-\d-\d-sbd.dat' } } ret = SlocumDataHandler._constraints_for_new_request(config) log.debug('test__constraints_for_new_request: {0}'.format( ret['new_files'])) self.assertEqual( ret['new_files'], list_file_info(config['ds_params']['base_url'], config['ds_params']['list_pattern']))
def _setup_resources(self): # TODO: some or all of this (or some variation) should move to DAMS' # Build the test resources for the dataset dms_cli = DatasetManagementServiceClient() dams_cli = DataAcquisitionManagementServiceClient() dpms_cli = DataProductManagementServiceClient() rr_cli = ResourceRegistryServiceClient() pubsub_cli = PubsubManagementServiceClient() eda = ExternalDatasetAgent(name='example eda',handler_module=self.DVR_CONFIG['dvr_mod'], handler_class=self.DVR_CONFIG['dvr_cls']) eda_id = dams_cli.create_external_dataset_agent(eda) eda_inst = ExternalDatasetAgentInstance(name='example eda instance') eda_inst_id = dams_cli.create_external_dataset_agent_instance(eda_inst, external_dataset_agent_id=eda_id) # Create and register the necessary resources/objects # Create DataProvider dprov = ExternalDataProvider(name='example data provider', institution=Institution(), contact=ContactInformation()) dprov.contact.individual_names_given = 'Christopher Mueller' dprov.contact.email = '*****@*****.**' # Create DataSource dsrc = DataSource(name='example datasource', protocol_type='DAP', institution=Institution(), contact=ContactInformation()) dsrc.connection_params['base_data_url'] = '' dsrc.contact.individual_names_given = 'Tim Giguere' dsrc.contact.email = '*****@*****.**' # Create ExternalDataset ds_name = 'usgs_test_dataset' dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) # The usgs.nc test dataset is a download of the R1 dataset found here: # http://thredds-test.oceanobservatories.org/thredds/dodsC/ooiciData/E66B1A74-A684-454A-9ADE-8388C2C634E5.ncml dset.dataset_description.parameters['dataset_path'] = 'test_data/usgs.nc' dset.dataset_description.parameters['temporal_dimension'] = 'time' dset.dataset_description.parameters['zonal_dimension'] = 'lon' dset.dataset_description.parameters['meridional_dimension'] = 'lat' dset.dataset_description.parameters['vertical_dimension'] = 'z' dset.dataset_description.parameters['variables'] = [ 'water_temperature', 'streamflow', 'water_temperature_bottom', 'water_temperature_middle', 'specific_conductance', 'data_qualifier', ] # Create DataSourceModel dsrc_model = DataSourceModel(name='dap_model') #dsrc_model.model = 'DAP' dsrc_model.data_handler_module = 'N/A' dsrc_model.data_handler_class = 'N/A' ## Run everything through DAMS ds_id = dams_cli.create_external_dataset(external_dataset=dset) ext_dprov_id = dams_cli.create_external_data_provider(external_data_provider=dprov) ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc) ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model) # Register the ExternalDataset dproducer_id = dams_cli.register_external_data_set(external_dataset_id=ds_id) # Or using each method dams_cli.assign_data_source_to_external_data_provider(data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id) dams_cli.assign_data_source_to_data_model(data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id) dams_cli.assign_external_dataset_to_data_source(external_dataset_id=ds_id, data_source_id=ext_dsrc_id) dams_cli.assign_external_dataset_to_agent_instance(external_dataset_id=ds_id, agent_instance_id=eda_inst_id) # dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=self.eda_id, agent_instance_id=self.eda_inst_id) #create temp streamdef so the data product can create the stream pc_list = [] #Get 'time' parameter context pc_list.append(dms_cli.read_parameter_context_by_name('time', id_only=True)) for pc_k, pc in self._create_parameter_dictionary().iteritems(): pc_list.append(dms_cli.create_parameter_context(pc_k, pc[1].dump())) pdict_id = dms_cli.create_parameter_dictionary('netcdf_param_dict', pc_list) #create temp streamdef so the data product can create the stream streamdef_id = pubsub_cli.create_stream_definition(name="netcdf", description="netcdf", parameter_dictionary_id=pdict_id) tdom, sdom = time_series_domain() tdom, sdom = tdom.dump(), sdom.dump() dprod = IonObject(RT.DataProduct, name='usgs_parsed_product', description='parsed usgs product', temporal_domain=tdom, spatial_domain=sdom) # Generate the data product and associate it to the ExternalDataset dproduct_id = dpms_cli.create_data_product(data_product=dprod, stream_definition_id=streamdef_id) dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id) stream_id, assn = rr_cli.find_objects(subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True) stream_id = stream_id[0] log.info('Created resources: {0}'.format({'ExternalDataset': ds_id, 'ExternalDataProvider': ext_dprov_id, 'DataSource': ext_dsrc_id, 'DataSourceModel': ext_dsrc_model_id, 'DataProducer': dproducer_id, 'DataProduct': dproduct_id, 'Stream': stream_id})) # Create the logger for receiving publications _, stream_route, _ = self.create_stream_and_logger(name='usgs', stream_id=stream_id) self.EDA_RESOURCE_ID = ds_id self.EDA_NAME = ds_name self.DVR_CONFIG['dh_cfg'] = { 'TESTING': True, 'stream_id': stream_id, 'stream_route': stream_route, 'stream_def': streamdef_id, 'data_producer_id': dproducer_id, # CBM: Should this be put in the main body of the config - with mod & cls? 'max_records': 1, }
def _setup_resources(self): # TODO: some or all of this (or some variation) should move to DAMS' # Build the test resources for the dataset dms_cli = DatasetManagementServiceClient() dams_cli = DataAcquisitionManagementServiceClient() dpms_cli = DataProductManagementServiceClient() rr_cli = ResourceRegistryServiceClient() pubsub_cli = PubsubManagementServiceClient() eda = ExternalDatasetAgent(name='example data agent', handler_module=self.DVR_CONFIG['dvr_mod'], handler_class=self.DVR_CONFIG['dvr_cls']) eda_id = dams_cli.create_external_dataset_agent(eda) eda_inst = ExternalDatasetAgentInstance( name='example dataset agent instance') eda_inst_id = dams_cli.create_external_dataset_agent_instance( eda_inst, external_dataset_agent_id=eda_id) # Create and register the necessary resources/objects # Create DataProvider dprov = ExternalDataProvider(name='example data provider', institution=Institution(), contact=ContactInformation()) dprov.contact.individual_names_given = 'Christopher Mueller' dprov.contact.email = '*****@*****.**' # Create DataSource dsrc = DataSource(name='example datasource', protocol_type='FILE', institution=Institution(), contact=ContactInformation()) dsrc.connection_params['base_data_url'] = '' dsrc.contact.individual_names_given = 'Tim Giguere' dsrc.contact.email = '*****@*****.**' # Create ExternalDataset ds_name = 'ruv_test_dataset' dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) dset.dataset_description.parameters['base_url'] = 'test_data/ruv/' dset.dataset_description.parameters[ 'list_pattern'] = 'RDLi_SEAB_2011_08_24_1600.ruv' dset.dataset_description.parameters['date_pattern'] = '%Y %m %d %H %M' dset.dataset_description.parameters[ 'date_extraction_pattern'] = 'RDLi_SEAB_([\d]{4})_([\d]{2})_([\d]{2})_([\d]{2})([\d]{2}).ruv' dset.dataset_description.parameters['temporal_dimension'] = None dset.dataset_description.parameters['zonal_dimension'] = None dset.dataset_description.parameters['meridional_dimension'] = None dset.dataset_description.parameters['vertical_dimension'] = None dset.dataset_description.parameters['variables'] = [] # Create DataSourceModel dsrc_model = DataSourceModel(name='ruv_model') #dsrc_model.model = 'RUV' dsrc_model.data_handler_module = 'N/A' dsrc_model.data_handler_class = 'N/A' ## Run everything through DAMS ds_id = dams_cli.create_external_dataset(external_dataset=dset) ext_dprov_id = dams_cli.create_external_data_provider( external_data_provider=dprov) ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc) ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model) # Register the ExternalDataset dproducer_id = dams_cli.register_external_data_set( external_dataset_id=ds_id) # Or using each method dams_cli.assign_data_source_to_external_data_provider( data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id) dams_cli.assign_data_source_to_data_model( data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id) dams_cli.assign_external_dataset_to_data_source( external_dataset_id=ds_id, data_source_id=ext_dsrc_id) dams_cli.assign_external_dataset_to_agent_instance( external_dataset_id=ds_id, agent_instance_id=eda_inst_id) pdict = ParameterDictionary() t_ctxt = ParameterContext( 'data', param_type=QuantityType(value_encoding=numpy.dtype('int64'))) t_ctxt.axis = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 01-01-1970' pdict.add_context(t_ctxt) #create temp streamdef so the data product can create the stream pc_list = [] for pc_k, pc in pdict.iteritems(): pc_list.append(dms_cli.create_parameter_context( pc_k, pc[1].dump())) pdict_id = dms_cli.create_parameter_dictionary('ruv_param_dict', pc_list) streamdef_id = pubsub_cli.create_stream_definition( name="ruv", description="stream def for ruv testing", parameter_dictionary_id=pdict_id) dprod = IonObject(RT.DataProduct, name='ruv_parsed_product', description='parsed ruv product') # Generate the data product and associate it to the ExternalDataset dproduct_id = dpms_cli.create_data_product( data_product=dprod, stream_definition_id=streamdef_id) dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id) stream_id, assn = rr_cli.find_objects(subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True) stream_id = stream_id[0] log.info('Created resources: {0}'.format({ 'ExternalDataset': ds_id, 'ExternalDataProvider': ext_dprov_id, 'DataSource': ext_dsrc_id, 'DataSourceModel': ext_dsrc_model_id, 'DataProducer': dproducer_id, 'DataProduct': dproduct_id, 'Stream': stream_id })) #CBM: Eventually, probably want to group this crap somehow - not sure how yet... # Create the logger for receiving publications _, stream_route, _ = self.create_stream_and_logger(name='ruv', stream_id=stream_id) self.EDA_RESOURCE_ID = ds_id self.EDA_NAME = ds_name self.DVR_CONFIG['dh_cfg'] = { 'TESTING': True, 'stream_id': stream_id, 'stream_route': stream_route, 'external_dataset_res': dset, 'param_dictionary': pdict.dump(), 'data_producer_id': dproducer_id, # CBM: Should this be put in the main body of the config - with mod & cls? 'max_records': 20, }
def _setup_resources(self): self.loggerpids = [] # Create DataProvider dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation()) dprov.contact.name = 'Christopher Mueller' dprov.contact.email = '*****@*****.**' # Create DataSetModel dataset_model = ExternalDatasetModel(name='slocum_model') dataset_model.datset_type = 'SLOCUM' dataset_model_id = self.dams_client.create_external_dataset_model(dataset_model) # Create ExternalDataset ds_name = 'slocum_test_dataset' dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) dset.dataset_description.parameters['base_url'] = 'test_data/slocum/' dset.dataset_description.parameters['list_pattern'] = 'ru05-2012-021-0-0-sbd.dat' dset.dataset_description.parameters['date_pattern'] = '%Y %j' dset.dataset_description.parameters['date_extraction_pattern'] = 'ru05-([\d]{4})-([\d]{3})-\d-\d-sbd.dat' dset.dataset_description.parameters['temporal_dimension'] = None dset.dataset_description.parameters['zonal_dimension'] = None dset.dataset_description.parameters['meridional_dimension'] = None dset.dataset_description.parameters['vertical_dimension'] = None dset.dataset_description.parameters['variables'] = [ 'c_wpt_y_lmc', 'sci_water_cond', 'm_y_lmc', 'u_hd_fin_ap_inflection_holdoff', 'sci_m_present_time', 'm_leakdetect_voltage_forward', 'sci_bb3slo_b660_scaled', 'c_science_send_all', 'm_gps_status', 'm_water_vx', 'm_water_vy', 'c_heading', 'sci_fl3slo_chlor_units', 'u_hd_fin_ap_gain', 'm_vacuum', 'u_min_water_depth', 'm_gps_lat', 'm_veh_temp', 'f_fin_offset', 'u_hd_fin_ap_hardover_holdoff', 'c_alt_time', 'm_present_time', 'm_heading', 'sci_bb3slo_b532_scaled', 'sci_fl3slo_cdom_units', 'm_fin', 'x_cycle_overrun_in_ms', 'sci_water_pressure', 'u_hd_fin_ap_igain', 'sci_fl3slo_phyco_units', 'm_battpos', 'sci_bb3slo_b470_scaled', 'm_lat', 'm_gps_lon', 'sci_ctd41cp_timestamp', 'm_pressure', 'c_wpt_x_lmc', 'c_ballast_pumped', 'x_lmc_xy_source', 'm_lon', 'm_avg_speed', 'sci_water_temp', 'u_pitch_ap_gain', 'm_roll', 'm_tot_num_inflections', 'm_x_lmc', 'u_pitch_ap_deadband', 'm_final_water_vy', 'm_final_water_vx', 'm_water_depth', 'm_leakdetect_voltage', 'u_pitch_max_delta_battpos', 'm_coulomb_amphr', 'm_pitch', ] ## Create the external dataset ds_id = self.dams_client.create_external_dataset(external_dataset=dset, external_dataset_model_id=dataset_model_id) ext_dprov_id = self.dams_client.create_external_data_provider(external_data_provider=dprov) # Register the ExternalDataset dproducer_id = self.dams_client.register_external_data_set(external_dataset_id=ds_id) ## Create the dataset agent datasetagent_obj = IonObject(RT.ExternalDatasetAgent, name='ExternalDatasetAgent1', description='external data agent', handler_module=self.EDA_MOD, handler_class=self.EDA_CLS ) self.datasetagent_id = self.dams_client.create_external_dataset_agent(external_dataset_agent=datasetagent_obj, external_dataset_model_id=dataset_model_id) # Generate the data product and associate it to the ExternalDataset pdict = DatasetManagementService.get_parameter_dictionary_by_name('ctd_parsed_param_dict') streamdef_id = self.pubsub_client.create_stream_definition(name="temp", parameter_dictionary_id=pdict.identifier) tdom, sdom = time_series_domain() tdom = tdom.dump() sdom = sdom.dump() dprod = IonObject(RT.DataProduct, name='slocum_parsed_product', description='parsed slocum product', temporal_domain = tdom, spatial_domain = sdom) self.dproduct_id = self.dataproductclient.create_data_product(data_product=dprod, stream_definition_id=streamdef_id) self.dams_client.assign_data_product(input_resource_id=ds_id, data_product_id=self.dproduct_id) #save the incoming slocum data self.dataproductclient.activate_data_product_persistence(self.dproduct_id) self.addCleanup(self.dataproductclient.suspend_data_product_persistence, self.dproduct_id) stream_ids, assn = self.rrclient.find_objects(subject=self.dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True) stream_id = stream_ids[0] dataset_id, assn = self.rrclient.find_objects(subject=self.dproduct_id, predicate=PRED.hasDataset, object_type=RT.Dataset, id_only=True) self.dataset_id = dataset_id[0] pid = self.create_logger('slocum_parsed_product', stream_id ) self.loggerpids.append(pid) self.DVR_CONFIG['dh_cfg'] = { 'TESTING':True, 'stream_id':stream_id, 'param_dictionary':pdict.dump(), 'data_producer_id':dproducer_id, #CBM: Should this be put in the main body of the config - with mod & cls? 'max_records':20, } # Create the logger for receiving publications #self.create_stream_and_logger(name='slocum',stream_id=stream_id) # Create agent config. self.EDA_RESOURCE_ID = ds_id self.EDA_NAME = ds_name
def _setup_hfr(self): # TODO: some or all of this (or some variation) should move to DAMS # Create and register the necessary resources/objects eda = ExternalDatasetAgent() eda_id = self.dams_cli.create_external_dataset_agent(eda) eda_inst = ExternalDatasetAgentInstance() eda_inst_id = self.dams_cli.create_external_dataset_agent_instance( eda_inst, external_dataset_agent_id=eda_id) # Create DataProvider dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation()) # dprov.institution.name = "HFR UCSD" # Create DataSource dsrc = DataSource(protocol_type="DAP", institution=Institution(), contact=ContactInformation()) dsrc.connection_params[ "base_data_url"] = "http://hfrnet.ucsd.edu:8080/thredds/dodsC/" # Create ExternalDataset dset = ExternalDataset(name="UCSD HFR", dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) dset.dataset_description.parameters[ "dataset_path"] = "HFRNet/USEGC/6km/hourly/RTV" # dset.dataset_description.parameters["dataset_path"] = "test_data/hfr.nc" dset.dataset_description.parameters["temporal_dimension"] = "time" dset.dataset_description.parameters["zonal_dimension"] = "lon" dset.dataset_description.parameters["meridional_dimension"] = "lat" # Create DataSourceModel dsrc_model = DataSourceModel(name="dap_model") dsrc_model.model = "DAP" dsrc_model.data_handler_module = "eoi.agent.handler.dap_external_data_handler" dsrc_model.data_handler_class = "DapExternalDataHandler" ## Run everything through DAMS ds_id = self.hfr_ds_id = self.dams_cli.create_external_dataset( external_dataset=dset) ext_dprov_id = self.dams_cli.create_external_data_provider( external_data_provider=dprov) ext_dsrc_id = self.dams_cli.create_data_source(data_source=dsrc) ext_dsrc_model_id = self.dams_cli.create_data_source_model(dsrc_model) # Register the ExternalDataset dproducer_id = self.dams_cli.register_external_data_set( external_dataset_id=ds_id) ## Associate everything # Convenience method # self.dams_cli.assign_eoi_resources(external_data_provider_id=ext_dprov_id, data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id, external_dataset_id=ds_id, external_data_agent_id=eda_id, agent_instance_id=eda_inst_id) # Or using each method self.dams_cli.assign_data_source_to_external_data_provider( data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id) self.dams_cli.assign_data_source_to_data_model( data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id) self.dams_cli.assign_external_dataset_to_data_source( external_dataset_id=ds_id, data_source_id=ext_dsrc_id) self.dams_cli.assign_external_dataset_to_agent_instance( external_dataset_id=ds_id, agent_instance_id=eda_inst_id) # self.dams_cli.assign_external_dataset_agent_to_data_model(external_data_agent_id=eda_id, data_source_model_id=ext_dsrc_model_id) # self.dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=eda_id, agent_instance_id=eda_inst_id) # Generate the data product and associate it to the ExternalDataset dprod = DataProduct(name='hfr_product', description='raw hfr product') dproduct_id = self.dpms_cli.create_data_product(data_product=dprod) self.dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id, create_stream=True)
def _setup_ncom(self): # TODO: some or all of this (or some variation) should move to DAMS # Create and register the necessary resources/objects eda = ExternalDatasetAgent() eda_id = self.dams_cli.create_external_dataset_agent(eda) eda_inst = ExternalDatasetAgentInstance() eda_inst_id = self.dams_cli.create_external_dataset_agent_instance( eda_inst, external_dataset_agent_id=eda_id) # Create DataProvider dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation()) # dprov.institution.name = "OOI CGSN" dprov.contact.name = "Robert Weller" dprov.contact.email = "*****@*****.**" # Create DataSource dsrc = DataSource(protocol_type="DAP", institution=Institution(), contact=ContactInformation()) # dsrc.connection_params["base_data_url"] = "http://ooi.whoi.edu/thredds/dodsC/" dsrc.connection_params["base_data_url"] = "" dsrc.contact.name = "Rich Signell" dsrc.contact.email = "*****@*****.**" # Create ExternalDataset dset = ExternalDataset(name="test", dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) # dset.dataset_description.parameters["dataset_path"] = "ooi/AS02CPSM_R_M.nc" dset.dataset_description.parameters[ "dataset_path"] = "test_data/ncom.nc" dset.dataset_description.parameters["temporal_dimension"] = "time" dset.dataset_description.parameters["zonal_dimension"] = "lon" dset.dataset_description.parameters["meridional_dimension"] = "lat" # Create DataSourceModel dsrc_model = DataSourceModel(name="dap_model") dsrc_model.model = "DAP" dsrc_model.data_handler_module = "eoi.agent.handler.dap_external_data_handler" dsrc_model.data_handler_class = "DapExternalDataHandler" ## Run everything through DAMS ds_id = self.ncom_ds_id = self.dams_cli.create_external_dataset( external_dataset=dset) ext_dprov_id = self.dams_cli.create_external_data_provider( external_data_provider=dprov) ext_dsrc_id = self.dams_cli.create_data_source(data_source=dsrc) ext_dsrc_model_id = self.dams_cli.create_data_source_model(dsrc_model) # Register the ExternalDataset dproducer_id = self.dams_cli.register_external_data_set( external_dataset_id=ds_id) ## Associate everything # Convenience method # self.dams_cli.assign_eoi_resources(external_data_provider_id=ext_dprov_id, data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id, external_dataset_id=ds_id, external_data_agent_id=eda_id, agent_instance_id=eda_inst_id) # Or using each method self.dams_cli.assign_data_source_to_external_data_provider( data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id) self.dams_cli.assign_data_source_to_data_model( data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id) self.dams_cli.assign_external_dataset_to_data_source( external_dataset_id=ds_id, data_source_id=ext_dsrc_id) self.dams_cli.assign_external_dataset_to_agent_instance( external_dataset_id=ds_id, agent_instance_id=eda_inst_id) # self.dams_cli.assign_external_dataset_agent_to_data_model(external_data_agent_id=eda_id, data_source_model_id=ext_dsrc_model_id) # self.dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=eda_id, agent_instance_id=eda_inst_id) # Generate the data product and associate it to the ExternalDataset dprod = DataProduct(name='ncom_product', description='raw ncom product') dproduct_id = self.dpms_cli.create_data_product(data_product=dprod) self.dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id, create_stream=True)
def read_external_dataset(self, ds_id=''): CWD = os.getcwd() ext_ds = ExternalDataset(name="test", dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) if ds_id == HFR: ext_ds.dataset_description.parameters ext_ds.dataset_description.parameters[ "dataset_path"] = "HFRNet/USEGC/6km/hourly/RTV" ext_ds.dataset_description.parameters[ "temporal_dimension"] = "time" ext_ds.dataset_description.parameters["zonal_dimension"] = "lon" ext_ds.dataset_description.parameters[ "meridional_dimension"] = "lat" elif ds_id == HFR_LOCAL: ext_ds.dataset_description.parameters[ "dataset_path"] = CWD + "/test_data/hfr.nc" ext_ds.dataset_description.parameters[ "temporal_dimension"] = "time" ext_ds.dataset_description.parameters["zonal_dimension"] = "lon" ext_ds.dataset_description.parameters[ "meridional_dimension"] = "lat" elif ds_id == KOKAGG: ext_ds.dataset_description.parameters[ "dataset_path"] = "hioos/hfr/kokagg" ext_ds.dataset_description.parameters[ "temporal_dimension"] = "time" ext_ds.dataset_description.parameters["zonal_dimension"] = "lon" ext_ds.dataset_description.parameters[ "meridional_dimension"] = "lat" ext_ds.contact.name = "Pierre Flament" ext_ds.contact.email = "*****@*****.**" elif ds_id == AST2: ext_ds.dataset_description.parameters[ "dataset_path"] = "ooi/AS02CPSM_R_M.nc" ext_ds.dataset_description.parameters[ "temporal_dimension"] = "time" ext_ds.dataset_description.parameters["zonal_dimension"] = "lon" ext_ds.dataset_description.parameters[ "meridional_dimension"] = "lat" elif ds_id == SSTA: ext_ds.dataset_description.parameters[ "dataset_path"] = "satellite/GR/ssta/1day" ext_ds.dataset_description.parameters[ "temporal_dimension"] = "time" ext_ds.dataset_description.parameters["zonal_dimension"] = "lon" ext_ds.dataset_description.parameters[ "meridional_dimension"] = "lat" elif ds_id == GHPM: ext_ds.dataset_description.parameters[ "dataset_path"] = CWD + "/test_data/ast2_ghpm_spp_ctd_1.nc" ext_ds.dataset_description.parameters[ "temporal_dimension"] = "time" ext_ds.dataset_description.parameters["zonal_dimension"] = "lon" ext_ds.dataset_description.parameters[ "meridional_dimension"] = "lat" elif ds_id == COMP1: ext_ds.dataset_description.parameters[ "dataset_path"] = CWD + "/test_data/ast2_ghpm_spp_ctd_1.nc" ext_ds.dataset_description.parameters[ "temporal_dimension"] = "time" ext_ds.dataset_description.parameters["zonal_dimension"] = "lon" ext_ds.dataset_description.parameters[ "meridional_dimension"] = "lat" elif ds_id == COMP2: ext_ds.dataset_description.parameters[ "dataset_path"] = CWD + "/test_data/ast2_ghpm_spp_ctd_2.nc" ext_ds.dataset_description.parameters[ "temporal_dimension"] = "time" ext_ds.dataset_description.parameters["zonal_dimension"] = "lon" ext_ds.dataset_description.parameters[ "meridional_dimension"] = "lat" elif ds_id == COADS: ext_ds.dataset_description.parameters[ "dataset_path"] = CWD + "/test_data/coads.nc" ext_ds.dataset_description.parameters[ "temporal_dimension"] = "time" ext_ds.dataset_description.parameters["zonal_dimension"] = "lon" ext_ds.dataset_description.parameters[ "meridional_dimension"] = "lat" elif ds_id == NCOM: ext_ds.dataset_description.parameters[ "dataset_path"] = CWD + "/test_data/ncom.nc" ext_ds.dataset_description.parameters[ "temporal_dimension"] = "time" ext_ds.dataset_description.parameters["zonal_dimension"] = "lon" ext_ds.dataset_description.parameters[ "meridional_dimension"] = "lat" elif ds_id == NAVO_GLIDER_LOCAL: ext_ds.dataset_description.parameters[ "dataset_path"] = CWD + "/test_data/p5150001.nc" ext_ds.dataset_description.parameters[ "temporal_dimension"] = "time" ext_ds.dataset_description.parameters["zonal_dimension"] = "lon" ext_ds.dataset_description.parameters[ "meridional_dimension"] = "lat" elif ds_id == USGS: ext_ds.dataset_description.parameters[ "dataset_path"] = CWD + "/test_data/usgs.nc" ext_ds.dataset_description.parameters[ "temporal_dimension"] = "time" ext_ds.dataset_description.parameters["zonal_dimension"] = "lon" ext_ds.dataset_description.parameters[ "meridional_dimension"] = "lat" else: return None return ext_ds
def setUp(self): self._datasets = {} file = self._create_tst_data_set('base') ext_ds = ExternalDataset(name="base", dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) ext_ds.dataset_description.parameters["dataset_path"] = file ext_ds.dataset_description.parameters["temporal_dimension"] = 'time' self._datasets['base'] = HfrRadialDataHandler(data_source=file, ext_dataset=ext_ds) file = self._create_tst_data_set('mod_gatt') ext_ds = ExternalDataset(name="mod_gatt", dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) ext_ds.dataset_description.parameters["dataset_path"] = file ext_ds.dataset_description.parameters["temporal_dimension"] = 'time' self._datasets['mod_gatt'] = HfrRadialDataHandler(data_source=file, ext_dataset=ext_ds) file = self._create_tst_data_set('new_gatt') ext_ds = ExternalDataset(name="new_gatt", dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) ext_ds.dataset_description.parameters["dataset_path"] = file ext_ds.dataset_description.parameters["temporal_dimension"] = 'time' self._datasets['new_gatt'] = HfrRadialDataHandler(data_source=file, ext_dataset=ext_ds) file = self._create_tst_data_set('mod_var') ext_ds = ExternalDataset(name="mod_var", dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) ext_ds.dataset_description.parameters["dataset_path"] = file ext_ds.dataset_description.parameters["temporal_dimension"] = 'time' self._datasets['mod_var'] = HfrRadialDataHandler(data_source=file, ext_dataset=ext_ds) file = self._create_tst_data_set('new_var') ext_ds = ExternalDataset(name="new_var", dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) ext_ds.dataset_description.parameters["dataset_path"] = file ext_ds.dataset_description.parameters["temporal_dimension"] = 'time' self._datasets['new_var'] = HfrRadialDataHandler(data_source=file, ext_dataset=ext_ds) file = self._create_tst_data_set('mod_varatt') ext_ds = ExternalDataset(name="mod_varatt", dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) ext_ds.dataset_description.parameters["dataset_path"] = file ext_ds.dataset_description.parameters["temporal_dimension"] = 'time' self._datasets['mod_varatt'] = HfrRadialDataHandler(data_source=file, ext_dataset=ext_ds) #file = 'http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/ASSA/RDLi_ASSA_2012_02_09_1100.ruv ' #ext_ds = ExternalDataset(name="remote", dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) #ext_ds.dataset_description.parameters["dataset_path"] = file #ext_ds.dataset_description.parameters["temporal_dimension"] = 'time' #self._datasets['remote'] = HfrRadialDataHandler(data_source=file, ext_dataset=ext_ds) pass
def test__constraints_for_new_request(self): old_list = [ 1302393600, 1302480000, 1302566400, 1302652800, 1302739200, 1302825600, 1302912000, 1302998400, 1303084800, 1303171200, 1303257600, 1303344000, 1303430400, 1303516800, 1303603200, 1303689600, 1303776000, 1303862400, 1303948800, 1304035200, 1304121600, 1304208000, 1304294400, 1304380800, 1304467200, 1304553600, 1304640000, 1304726400, 1304812800, 1304899200, 1304985600, 1305072000, 1305158400, 1305244800, 1305331200, 1305417600, 1305504000, 1305590400, 1305676800, 1305763200, 1305849600, 1305936000, 1306022400, 1306108800, 1306195200, 1306281600, 1306368000, 1306454400, 1306540800, 1306627200, 1306713600, 1306800000, 1306886400, 1306972800, 1307059200, 1307145600, 1307232000, 1307318400, 1307404800, 1307491200, 1307577600, 1307664000, 1307750400, 1307836800, 1307923200, 1308009600, 1308096000, 1308182400, 1308268800, 1308355200, 1308441600, 1308528000, 1308614400, 1308700800, 1308787200, 1308873600, 1308960000, 1309046400, 1309132800, 1309219200, 1309305600, 1309392000, 1309478400, 1309564800, 1309651200, 1309737600, 1309824000, 1309910400, 1309996800, 1310083200, 1310169600, 1310256000, 1310342400, 1310428800, 1310515200, 1310601600, 1310688000, 1310774400, 1310860800, 1310947200, 1311033600, 1311120000, 1311206400, 1311292800, 1311379200, 1311465600, 1311552000, 1311638400, 1311724800, 1311811200, 1311897600, 1311984000, 1312070400, 1312156800, 1312243200, 1312329600, 1312416000, 1312502400, 1312588800, 1312675200, 1312761600, 1312848000, 1312934400, 1313020800, 1313107200, 1313193600, 1313280000, 1313366400, 1313452800, 1313539200, 1313625600, 1313712000, 1313798400, 1313884800, 1313971200, 1314057600, 1314144000, 1314230400, 1314316800, 1314403200, 1314489600, 1314576000, 1314662400, 1314748800, 1314835200, 1314921600, 1315008000, 1315094400, 1315180800, 1315267200, 1315353600, 1315440000, 1315526400, 1315612800, 1315699200, 1315785600, 1315872000, 1315958400, 1316044800, 1316131200, 1316217600, 1316304000, 1316390400, 1316476800, 1316563200, 1316649600, 1316736000, 1316822400, 1316908800, 1316995200, 1317081600, 1317168000, 1317254400, 1317340800, 1317427200, 1317513600, 1317600000, 1317686400, 1317772800, 1317859200, 1317945600, 1318032000, 1318118400, 1318204800, 1318291200, 1318377600, 1318464000, 1318550400, 1318636800, 1318723200, 1318809600, 1318896000, 1318982400, 1319068800, 1319155200, 1319241600, 1319328000, 1319414400, 1319500800, 1319587200, 1319673600, 1319760000, 1319846400, 1319932800, 1320019200, 1320105600, 1320192000, 1320278400, 1320364800, 1320451200, 1320537600, 1320624000, 1320710400, 1320796800, 1320883200, 1320969600, 1321056000, 1321142400, 1321228800, 1321315200, 1321401600, 1321488000, 1321574400, 1321660800, 1321747200, 1321833600, 1321920000, 1322006400, 1322092800, 1322179200, 1322265600, 1322352000, 1322438400, 1322524800, 1322611200, 1322697600, 1322784000, 1322870400, 1322956800, 1323043200, 1323129600, 1323216000, 1323302400, 1323388800, 1323475200, 1323561600, 1323648000, 1323734400, 1323820800, 1323907200, 1323993600, 1324080000, 1324166400, 1324252800, 1324339200, 1324425600, 1324512000, 1324598400, 1324684800, 1324771200, 1324857600, 1324944000, 1325030400, 1325116800, 1325203200, 1325289600, 1325376000, 1325462400, 1325548800, 1325635200, 1325721600, 1325808000, 1325894400, 1325980800, 1326067200, 1326153600, 1326240000, 1326326400, 1326412800, 1326499200, 1326585600 ] edres = ExternalDataset(name='test_ed_res', dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) edres.dataset_description.parameters[ 'dataset_path'] = 'test_data/usgs.nc' edres.dataset_description.parameters['temporal_dimension'] = 'time' edres.update_description.parameters['new_data_check'] = msgpack.packb( old_list, default=encode_ion) config = { 'external_dataset_res': edres, 'dataset_object': Dataset(edres.dataset_description.parameters['dataset_path']) } ret = NetcdfDataHandler._constraints_for_new_request(config) #log.debug('test__constraints_for_new_request: {0}'.format(ret['temporal_slice'])) self.assertEqual(ret['temporal_slice'], slice(281, 295, None))
def _setup_resources(self): # TODO: some or all of this (or some variation) should move to DAMS' # Build the test resources for the dataset dams_cli = DataAcquisitionManagementServiceClient() dpms_cli = DataProductManagementServiceClient() rr_cli = ResourceRegistryServiceClient() eda = ExternalDatasetAgent() eda_id = dams_cli.create_external_dataset_agent(eda) eda_inst = ExternalDatasetAgentInstance() eda_inst_id = dams_cli.create_external_dataset_agent_instance( eda_inst, external_dataset_agent_id=eda_id) # Create and register the necessary resources/objects # Create DataProvider dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation()) dprov.contact.name = 'Christopher Mueller' dprov.contact.email = '*****@*****.**' # Create DataSource dsrc = DataSource(protocol_type='DAP', institution=Institution(), contact=ContactInformation()) dsrc.connection_params['base_data_url'] = '' dsrc.contact.name = 'Tim Giguere' dsrc.contact.email = '*****@*****.**' # Create ExternalDataset ds_name = 'usgs_test_dataset' dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) # The usgs.nc test dataset is a download of the R1 dataset found here: # http://thredds-test.oceanobservatories.org/thredds/dodsC/ooiciData/E66B1A74-A684-454A-9ADE-8388C2C634E5.ncml dset.dataset_description.parameters[ 'dataset_path'] = 'test_data/usgs.nc' dset.dataset_description.parameters['temporal_dimension'] = 'time' dset.dataset_description.parameters['zonal_dimension'] = 'lon' dset.dataset_description.parameters['meridional_dimension'] = 'lat' dset.dataset_description.parameters['vertical_dimension'] = 'z' dset.dataset_description.parameters['variables'] = [ 'water_temperature', 'streamflow', 'water_temperature_bottom', 'water_temperature_middle', 'specific_conductance', 'data_qualifier', ] # Create DataSourceModel dsrc_model = DataSourceModel(name='dap_model') dsrc_model.model = 'DAP' dsrc_model.data_handler_module = 'N/A' dsrc_model.data_handler_class = 'N/A' ## Run everything through DAMS ds_id = dams_cli.create_external_dataset(external_dataset=dset) ext_dprov_id = dams_cli.create_external_data_provider( external_data_provider=dprov) ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc) ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model) # Register the ExternalDataset dproducer_id = dams_cli.register_external_data_set( external_dataset_id=ds_id) # Or using each method dams_cli.assign_data_source_to_external_data_provider( data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id) dams_cli.assign_data_source_to_data_model( data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id) dams_cli.assign_external_dataset_to_data_source( external_dataset_id=ds_id, data_source_id=ext_dsrc_id) dams_cli.assign_external_dataset_to_agent_instance( external_dataset_id=ds_id, agent_instance_id=eda_inst_id) # dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=self.eda_id, agent_instance_id=self.eda_inst_id) # Generate the data product and associate it to the ExternalDataset dprod = DataProduct(name='usgs_parsed_product', description='parsed usgs product') dproduct_id = dpms_cli.create_data_product(data_product=dprod) dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id, create_stream=True) stream_id, assn = rr_cli.find_objects(subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True) stream_id = stream_id[0] log.info('Created resources: {0}'.format({ 'ExternalDataset': ds_id, 'ExternalDataProvider': ext_dprov_id, 'DataSource': ext_dsrc_id, 'DataSourceModel': ext_dsrc_model_id, 'DataProducer': dproducer_id, 'DataProduct': dproduct_id, 'Stream': stream_id })) #CBM: Use CF standard_names ttool = TaxyTool() ttool.add_taxonomy_set('time', 'time') ttool.add_taxonomy_set('lon', 'longitude') ttool.add_taxonomy_set('lat', 'latitude') ttool.add_taxonomy_set('z', 'water depth') ttool.add_taxonomy_set('water_temperature', 'average water temperature') ttool.add_taxonomy_set('water_temperature_bottom', 'water temperature at bottom of water column') ttool.add_taxonomy_set('water_temperature_middle', 'water temperature at middle of water column') ttool.add_taxonomy_set('streamflow', 'flow velocity of stream') ttool.add_taxonomy_set('specific_conductance', 'specific conductance of water') ttool.add_taxonomy_set('data_qualifier', 'data qualifier flag') ttool.add_taxonomy_set('coords', 'This group contains coordinate parameters') ttool.add_taxonomy_set('data', 'This group contains data parameters') # Create the logger for receiving publications self.create_stream_and_logger(name='usgs', stream_id=stream_id) self.EDA_RESOURCE_ID = ds_id self.EDA_NAME = ds_name self.DVR_CONFIG['dh_cfg'] = { 'TESTING': True, 'stream_id': stream_id, 'taxonomy': ttool.dump(), 'data_producer_id': dproducer_id, #CBM: Should this be put in the main body of the config - with mod & cls? 'max_records': 4, }
def _setup_resources(self): # TODO: some or all of this (or some variation) should move to DAMS' # Build the test resources for the dataset dms_cli = DatasetManagementServiceClient() dams_cli = DataAcquisitionManagementServiceClient() dpms_cli = DataProductManagementServiceClient() rr_cli = ResourceRegistryServiceClient() pubsub_cli = PubsubManagementServiceClient() eda = ExternalDatasetAgent(handler_module=self.DVR_CONFIG['dvr_mod'], handler_class=self.DVR_CONFIG['dvr_cls']) eda_id = dams_cli.create_external_dataset_agent(eda) eda_inst = ExternalDatasetAgentInstance() eda_inst_id = dams_cli.create_external_dataset_agent_instance(eda_inst, external_dataset_agent_id=eda_id) # Create and register the necessary resources/objects # Create DataProvider dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation()) dprov.contact.individual_names_given = 'Christopher Mueller' dprov.contact.email = '*****@*****.**' # Create DataSource dsrc = DataSource(protocol_type='FILE', institution=Institution(), contact=ContactInformation()) dsrc.connection_params['base_data_url'] = '' dsrc.contact.individual_names_given = 'Tim Giguere' dsrc.contact.email = '*****@*****.**' # Create ExternalDataset ds_name = 'slocum_test_dataset' dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) dset.dataset_description.parameters['base_url'] = 'test_data/slocum/' dset.dataset_description.parameters['list_pattern'] = 'ru05-2012-021-0-0-sbd.dat' dset.dataset_description.parameters['date_pattern'] = '%Y %j' dset.dataset_description.parameters['date_extraction_pattern'] = 'ru05-([\d]{4})-([\d]{3})-\d-\d-sbd.dat' dset.dataset_description.parameters['temporal_dimension'] = None dset.dataset_description.parameters['zonal_dimension'] = None dset.dataset_description.parameters['meridional_dimension'] = None dset.dataset_description.parameters['vertical_dimension'] = None dset.dataset_description.parameters['variables'] = [ 'c_wpt_y_lmc', 'sci_water_cond', 'm_y_lmc', 'u_hd_fin_ap_inflection_holdoff', 'sci_m_present_time', 'm_leakdetect_voltage_forward', 'sci_bb3slo_b660_scaled', 'c_science_send_all', 'm_gps_status', 'm_water_vx', 'm_water_vy', 'c_heading', 'sci_fl3slo_chlor_units', 'u_hd_fin_ap_gain', 'm_vacuum', 'u_min_water_depth', 'm_gps_lat', 'm_veh_temp', 'f_fin_offset', 'u_hd_fin_ap_hardover_holdoff', 'c_alt_time', 'm_present_time', 'm_heading', 'sci_bb3slo_b532_scaled', 'sci_fl3slo_cdom_units', 'm_fin', 'x_cycle_overrun_in_ms', 'sci_water_pressure', 'u_hd_fin_ap_igain', 'sci_fl3slo_phyco_units', 'm_battpos', 'sci_bb3slo_b470_scaled', 'm_lat', 'm_gps_lon', 'sci_ctd41cp_timestamp', 'm_pressure', 'c_wpt_x_lmc', 'c_ballast_pumped', 'x_lmc_xy_source', 'm_lon', 'm_avg_speed', 'sci_water_temp', 'u_pitch_ap_gain', 'm_roll', 'm_tot_num_inflections', 'm_x_lmc', 'u_pitch_ap_deadband', 'm_final_water_vy', 'm_final_water_vx', 'm_water_depth', 'm_leakdetect_voltage', 'u_pitch_max_delta_battpos', 'm_coulomb_amphr', 'm_pitch', ] # Create DataSourceModel dsrc_model = DataSourceModel(name='slocum_model') # dsrc_model.model = 'SLOCUM' dsrc_model.data_handler_module = 'N/A' dsrc_model.data_handler_class = 'N/A' ## Run everything through DAMS ds_id = dams_cli.create_external_dataset(external_dataset=dset) ext_dprov_id = dams_cli.create_external_data_provider(external_data_provider=dprov) ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc) ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model) # Register the ExternalDataset dproducer_id = dams_cli.register_external_data_set(external_dataset_id=ds_id) # Or using each method dams_cli.assign_data_source_to_external_data_provider(data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id) dams_cli.assign_data_source_to_data_model(data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id) dams_cli.assign_external_dataset_to_data_source(external_dataset_id=ds_id, data_source_id=ext_dsrc_id) dams_cli.assign_external_dataset_to_agent_instance(external_dataset_id=ds_id, agent_instance_id=eda_inst_id) # dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=self.eda_id, agent_instance_id=self.eda_inst_id) #create temp streamdef so the data product can create the stream pc_list = [] for pc_k, pc in self._create_parameter_dictionary().iteritems(): pc_list.append(dms_cli.create_parameter_context(pc_k, pc[1].dump())) pdict_id = dms_cli.create_parameter_dictionary('slocum_param_dict', pc_list) streamdef_id = pubsub_cli.create_stream_definition(name="slocum_stream_def", description="stream def for slocum testing", parameter_dictionary_id=pdict_id) # dpms_cli.create_data_product() # Generate the data product and associate it to the ExternalDataset tdom, sdom = time_series_domain() tdom, sdom = tdom.dump(), sdom.dump() dprod = IonObject(RT.DataProduct, name='slocum_parsed_product', description='parsed slocum product', temporal_domain=tdom, spatial_domain=sdom) dproduct_id = dpms_cli.create_data_product(data_product=dprod, stream_definition_id=streamdef_id) dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id) stream_id, assn = rr_cli.find_objects(subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True) stream_id = stream_id[0] log.info('Created resources: {0}'.format({'ExternalDataset': ds_id, 'ExternalDataProvider': ext_dprov_id, 'DataSource': ext_dsrc_id, 'DataSourceModel': ext_dsrc_model_id, 'DataProducer': dproducer_id, 'DataProduct': dproduct_id, 'Stream': stream_id})) # Create the logger for receiving publications _, stream_route, _ = self.create_stream_and_logger(name='slocum', stream_id=stream_id) self.EDA_RESOURCE_ID = ds_id self.EDA_NAME = ds_name self.DVR_CONFIG['dh_cfg'] = { 'TESTING': True, 'stream_id': stream_id, 'stream_route': stream_route, 'stream_def': streamdef_id, 'external_dataset_res': dset, 'data_producer_id': dproducer_id, # CBM: Should this be put in the main body of the config - with mod & cls? 'max_records': 20, }
def test__constraints_for_new_request(self): edres = ExternalDataset(name='test_ed_res', dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) # old_list = [ # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0000.ruv', # '04-Jun-2012 20:43', # '136K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0100.ruv', # '04-Jun-2012 21:43', # '135K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0200.ruv', # '04-Jun-2012 22:42', # '137K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0300.ruv', # '04-Jun-2012 23:41', # '136K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0400.ruv', # '05-Jun-2012 00:41', # '150K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0500.ruv', # '05-Jun-2012 01:41', # '142K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0600.ruv', # '05-Jun-2012 02:41', # '138K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0700.ruv', # '05-Jun-2012 03:41', # '136K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0800.ruv', # '05-Jun-2012 04:41', # '138K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_0900.ruv', # '05-Jun-2012 05:40', # '147K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1000.ruv', # '05-Jun-2012 06:40', # '143K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1100.ruv', # '05-Jun-2012 07:40', # '148K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1200.ruv', # '05-Jun-2012 08:40', # '147K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1300.ruv', # '05-Jun-2012 09:39', # '148K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1400.ruv', # '05-Jun-2012 10:38', # '143K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1500.ruv', # '05-Jun-2012 11:43', # '143K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1600.ruv', # '05-Jun-2012 12:43', # '146K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1700.ruv', # '05-Jun-2012 13:42', # '134K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1800.ruv', # '05-Jun-2012 14:42', # '143K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_1900.ruv', # '05-Jun-2012 15:42', # '148K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_2000.ruv', # '05-Jun-2012 16:41', # '157K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_2100.ruv', # '05-Jun-2012 17:41', # '160K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_2200.ruv', # '05-Jun-2012 18:41', # '158K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_05_2300.ruv', # '05-Jun-2012 19:41', # '148K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0000.ruv', # '05-Jun-2012 20:40', # '140K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0100.ruv', # '05-Jun-2012 21:40', # '133K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0200.ruv', # '05-Jun-2012 22:40', # '143K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0300.ruv', # '05-Jun-2012 23:39', # '156K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0400.ruv', # '06-Jun-2012 00:39', # '146K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0500.ruv', # '06-Jun-2012 01:39', # '147K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0600.ruv', # '06-Jun-2012 02:39', # '147K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0700.ruv', # '06-Jun-2012 03:43', # '148K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0800.ruv', # '06-Jun-2012 04:42', # '137K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_0900.ruv', # '06-Jun-2012 05:42', # '130K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_1000.ruv', # '06-Jun-2012 06:42', # '129K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_1100.ruv', # '06-Jun-2012 07:42', # '136K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_1200.ruv', # '06-Jun-2012 08:42', # '137K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_1300.ruv', # '06-Jun-2012 09:41', # '151K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_1400.ruv', # '06-Jun-2012 10:41', # '153K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_1500.ruv', # '06-Jun-2012 11:41', # '156K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_1600.ruv', # '06-Jun-2012 12:41', # '157K'), # ('http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/RDLi_BELM_2012_06_06_1700.ruv', # '06-Jun-2012 13:40', # '161K'),] old_list = [ ('test_data/ruv/RDLm_SEAB_2012_06_06_1200.ruv', 1339006638.0, 119066), ('test_data/ruv/RDLm_SEAB_2012_06_06_1300.ruv', 1339006629.0, 109316), ('test_data/ruv/RDLm_SEAB_2012_06_06_1400.ruv', 1339006521.0, 113411), ] # old_list = None config = { 'external_dataset_res': edres, # 'new_data_check':None, 'new_data_check': old_list, 'ds_params': { # These would be extracted from the dataset_description.parameters during _init_acquisition_cycle, but since that isn't called, just add them here # 'base_url':'http://marine.rutgers.edu/cool/maracoos/codar/ooi/radials/BELM/', # CBM: What's the difference between RDLi and RDLm??? # 'pattern':'<a href="(RDLm.*\.ruv)">.*(\d{2}-[a-zA-Z]{3}-\d{4} \d{2}:\d{2})\s*(\d{1,5}\w)',# Appended to base to filter files; Either a shell style pattern (for filesystem) or regex (for http/ftp) 'base_url': 'test_data/ruv', 'list_pattern': 'RDLm_SEAB_*.ruv', 'date_pattern': '%Y %m %d %H %M', 'date_extraction_pattern': 'RDLm_SEAB_([\d]{4})_([\d]{2})_([\d]{2})_([\d]{2})([\d]{2}).ruv' } } ret = RuvDataHandler._constraints_for_new_request(config) log.debug('test__new_data_constraints: {0}'.format(config)) self.assertEqual( ret['new_files'], list_file_info(config['ds_params']['base_url'], config['ds_params']['list_pattern']))
def _setup_resources(self): # TODO: some or all of this (or some variation) should move to DAMS' # Build the test resources for the dataset dams_cli = DataAcquisitionManagementServiceClient() dpms_cli = DataProductManagementServiceClient() rr_cli = ResourceRegistryServiceClient() eda = ExternalDatasetAgent() eda_id = dams_cli.create_external_dataset_agent(eda) eda_inst = ExternalDatasetAgentInstance() eda_inst_id = dams_cli.create_external_dataset_agent_instance(eda_inst, external_dataset_agent_id=eda_id) # Create and register the necessary resources/objects # Create DataProvider dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation()) dprov.contact.name = 'Christopher Mueller' dprov.contact.email = '*****@*****.**' # Create DataSource dsrc = DataSource(protocol_type='FILE', institution=Institution(), contact=ContactInformation()) dsrc.connection_params['base_data_url'] = '' dsrc.contact.name='Tim Giguere' dsrc.contact.email = '*****@*****.**' # Create ExternalDataset ds_name = 'slocum_test_dataset' dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) dset.dataset_description.parameters['dataset_path'] = 'test_data/ru05-2012-021-0-0-sbd.dat' dset.dataset_description.parameters['temporal_dimension'] = None dset.dataset_description.parameters['zonal_dimension'] = None dset.dataset_description.parameters['meridional_dimension'] = None dset.dataset_description.parameters['vertical_dimension'] = None dset.dataset_description.parameters['variables'] = [ 'c_wpt_y_lmc', 'sci_water_cond', 'm_y_lmc', 'u_hd_fin_ap_inflection_holdoff', 'sci_m_present_time', 'm_leakdetect_voltage_forward', 'sci_bb3slo_b660_scaled', 'c_science_send_all', 'm_gps_status', 'm_water_vx', 'm_water_vy', 'c_heading', 'sci_fl3slo_chlor_units', 'u_hd_fin_ap_gain', 'm_vacuum', 'u_min_water_depth', 'm_gps_lat', 'm_veh_temp', 'f_fin_offset', 'u_hd_fin_ap_hardover_holdoff', 'c_alt_time', 'm_present_time', 'm_heading', 'sci_bb3slo_b532_scaled', 'sci_fl3slo_cdom_units', 'm_fin', 'x_cycle_overrun_in_ms', 'sci_water_pressure', 'u_hd_fin_ap_igain', 'sci_fl3slo_phyco_units', 'm_battpos', 'sci_bb3slo_b470_scaled', 'm_lat', 'm_gps_lon', 'sci_ctd41cp_timestamp', 'm_pressure', 'c_wpt_x_lmc', 'c_ballast_pumped', 'x_lmc_xy_source', 'm_lon', 'm_avg_speed', 'sci_water_temp', 'u_pitch_ap_gain', 'm_roll', 'm_tot_num_inflections', 'm_x_lmc', 'u_pitch_ap_deadband', 'm_final_water_vy', 'm_final_water_vx', 'm_water_depth', 'm_leakdetect_voltage', 'u_pitch_max_delta_battpos', 'm_coulomb_amphr', 'm_pitch', ] # Create DataSourceModel dsrc_model = DataSourceModel(name='slocum_model') dsrc_model.model = 'SLOCUM' dsrc_model.data_handler_module = 'N/A' dsrc_model.data_handler_class = 'N/A' ## Run everything through DAMS ds_id = dams_cli.create_external_dataset(external_dataset=dset) ext_dprov_id = dams_cli.create_external_data_provider(external_data_provider=dprov) ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc) ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model) # Register the ExternalDataset dproducer_id = dams_cli.register_external_data_set(external_dataset_id=ds_id) # Or using each method dams_cli.assign_data_source_to_external_data_provider(data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id) dams_cli.assign_data_source_to_data_model(data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id) dams_cli.assign_external_dataset_to_data_source(external_dataset_id=ds_id, data_source_id=ext_dsrc_id) dams_cli.assign_external_dataset_to_agent_instance(external_dataset_id=ds_id, agent_instance_id=eda_inst_id) # dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=self.eda_id, agent_instance_id=self.eda_inst_id) # Generate the data product and associate it to the ExternalDataset dprod = DataProduct(name='slocum_parsed_product', description='parsed slocum product') dproduct_id = dpms_cli.create_data_product(data_product=dprod) dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id, create_stream=True) stream_id, assn = rr_cli.find_objects(subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True) stream_id = stream_id[0] log.info('Created resources: {0}'.format({'ExternalDataset':ds_id, 'ExternalDataProvider':ext_dprov_id, 'DataSource':ext_dsrc_id, 'DataSourceModel':ext_dsrc_model_id, 'DataProducer':dproducer_id, 'DataProduct':dproduct_id, 'Stream':stream_id})) #CBM: Use CF standard_names ttool = TaxyTool() ttool.add_taxonomy_set('c_wpt_y_lmc'), ttool.add_taxonomy_set('sci_water_cond'), ttool.add_taxonomy_set('m_y_lmc'), ttool.add_taxonomy_set('u_hd_fin_ap_inflection_holdoff'), ttool.add_taxonomy_set('sci_m_present_time'), ttool.add_taxonomy_set('m_leakdetect_voltage_forward'), ttool.add_taxonomy_set('sci_bb3slo_b660_scaled'), ttool.add_taxonomy_set('c_science_send_all'), ttool.add_taxonomy_set('m_gps_status'), ttool.add_taxonomy_set('m_water_vx'), ttool.add_taxonomy_set('m_water_vy'), ttool.add_taxonomy_set('c_heading'), ttool.add_taxonomy_set('sci_fl3slo_chlor_units'), ttool.add_taxonomy_set('u_hd_fin_ap_gain'), ttool.add_taxonomy_set('m_vacuum'), ttool.add_taxonomy_set('u_min_water_depth'), ttool.add_taxonomy_set('m_gps_lat'), ttool.add_taxonomy_set('m_veh_temp'), ttool.add_taxonomy_set('f_fin_offset'), ttool.add_taxonomy_set('u_hd_fin_ap_hardover_holdoff'), ttool.add_taxonomy_set('c_alt_time'), ttool.add_taxonomy_set('m_present_time'), ttool.add_taxonomy_set('m_heading'), ttool.add_taxonomy_set('sci_bb3slo_b532_scaled'), ttool.add_taxonomy_set('sci_fl3slo_cdom_units'), ttool.add_taxonomy_set('m_fin'), ttool.add_taxonomy_set('x_cycle_overrun_in_ms'), ttool.add_taxonomy_set('sci_water_pressure'), ttool.add_taxonomy_set('u_hd_fin_ap_igain'), ttool.add_taxonomy_set('sci_fl3slo_phyco_units'), ttool.add_taxonomy_set('m_battpos'), ttool.add_taxonomy_set('sci_bb3slo_b470_scaled'), ttool.add_taxonomy_set('m_lat'), ttool.add_taxonomy_set('m_gps_lon'), ttool.add_taxonomy_set('sci_ctd41cp_timestamp'), ttool.add_taxonomy_set('m_pressure'), ttool.add_taxonomy_set('c_wpt_x_lmc'), ttool.add_taxonomy_set('c_ballast_pumped'), ttool.add_taxonomy_set('x_lmc_xy_source'), ttool.add_taxonomy_set('m_lon'), ttool.add_taxonomy_set('m_avg_speed'), ttool.add_taxonomy_set('sci_water_temp'), ttool.add_taxonomy_set('u_pitch_ap_gain'), ttool.add_taxonomy_set('m_roll'), ttool.add_taxonomy_set('m_tot_num_inflections'), ttool.add_taxonomy_set('m_x_lmc'), ttool.add_taxonomy_set('u_pitch_ap_deadband'), ttool.add_taxonomy_set('m_final_water_vy'), ttool.add_taxonomy_set('m_final_water_vx'), ttool.add_taxonomy_set('m_water_depth'), ttool.add_taxonomy_set('m_leakdetect_voltage'), ttool.add_taxonomy_set('u_pitch_max_delta_battpos'), ttool.add_taxonomy_set('m_coulomb_amphr'), ttool.add_taxonomy_set('m_pitch'), #CBM: Eventually, probably want to group this crap somehow - not sure how yet... # Create the logger for receiving publications self.create_stream_and_logger(name='slocum',stream_id=stream_id) self.EDA_RESOURCE_ID = ds_id self.EDA_NAME = ds_name self.DVR_CONFIG['dh_cfg'] = { 'TESTING':True, 'stream_id':stream_id, 'external_dataset_res':dset, 'taxonomy':ttool.dump(), 'data_producer_id':dproducer_id,#CBM: Should this be put in the main body of the config - with mod & cls? 'max_records':20, }