def _setup_resources(self): # TODO: some or all of this (or some variation) should move to DAMS' # Build the test resources for the dataset dms_cli = DatasetManagementServiceClient() dams_cli = DataAcquisitionManagementServiceClient() dpms_cli = DataProductManagementServiceClient() rr_cli = ResourceRegistryServiceClient() pubsub_cli = PubsubManagementServiceClient() eda = ExternalDatasetAgent(handler_module=self.DVR_CONFIG['dvr_mod'], handler_class=self.DVR_CONFIG['dvr_cls']) eda_id = dams_cli.create_external_dataset_agent(eda) eda_inst = ExternalDatasetAgentInstance() eda_inst_id = dams_cli.create_external_dataset_agent_instance(eda_inst, external_dataset_agent_id=eda_id) # Create and register the necessary resources/objects # Create DataProvider dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation()) dprov.contact.individual_names_given = 'Christopher Mueller' dprov.contact.email = '*****@*****.**' # Create DataSource dsrc = DataSource(protocol_type='FILE', institution=Institution(), contact=ContactInformation()) dsrc.connection_params['base_data_url'] = '' dsrc.contact.individual_names_given = 'Tim Giguere' dsrc.contact.email = '*****@*****.**' # Create ExternalDataset ds_name = 'slocum_test_dataset' dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) dset.dataset_description.parameters['base_url'] = 'test_data/slocum/' dset.dataset_description.parameters['list_pattern'] = 'ru05-2012-021-0-0-sbd.dat' dset.dataset_description.parameters['date_pattern'] = '%Y %j' dset.dataset_description.parameters['date_extraction_pattern'] = 'ru05-([\d]{4})-([\d]{3})-\d-\d-sbd.dat' dset.dataset_description.parameters['temporal_dimension'] = None dset.dataset_description.parameters['zonal_dimension'] = None dset.dataset_description.parameters['meridional_dimension'] = None dset.dataset_description.parameters['vertical_dimension'] = None dset.dataset_description.parameters['variables'] = [ 'c_wpt_y_lmc', 'sci_water_cond', 'm_y_lmc', 'u_hd_fin_ap_inflection_holdoff', 'sci_m_present_time', 'm_leakdetect_voltage_forward', 'sci_bb3slo_b660_scaled', 'c_science_send_all', 'm_gps_status', 'm_water_vx', 'm_water_vy', 'c_heading', 'sci_fl3slo_chlor_units', 'u_hd_fin_ap_gain', 'm_vacuum', 'u_min_water_depth', 'm_gps_lat', 'm_veh_temp', 'f_fin_offset', 'u_hd_fin_ap_hardover_holdoff', 'c_alt_time', 'm_present_time', 'm_heading', 'sci_bb3slo_b532_scaled', 'sci_fl3slo_cdom_units', 'm_fin', 'x_cycle_overrun_in_ms', 'sci_water_pressure', 'u_hd_fin_ap_igain', 'sci_fl3slo_phyco_units', 'm_battpos', 'sci_bb3slo_b470_scaled', 'm_lat', 'm_gps_lon', 'sci_ctd41cp_timestamp', 'm_pressure', 'c_wpt_x_lmc', 'c_ballast_pumped', 'x_lmc_xy_source', 'm_lon', 'm_avg_speed', 'sci_water_temp', 'u_pitch_ap_gain', 'm_roll', 'm_tot_num_inflections', 'm_x_lmc', 'u_pitch_ap_deadband', 'm_final_water_vy', 'm_final_water_vx', 'm_water_depth', 'm_leakdetect_voltage', 'u_pitch_max_delta_battpos', 'm_coulomb_amphr', 'm_pitch', ] # Create DataSourceModel dsrc_model = DataSourceModel(name='slocum_model') # dsrc_model.model = 'SLOCUM' dsrc_model.data_handler_module = 'N/A' dsrc_model.data_handler_class = 'N/A' ## Run everything through DAMS ds_id = dams_cli.create_external_dataset(external_dataset=dset) ext_dprov_id = dams_cli.create_external_data_provider(external_data_provider=dprov) ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc) ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model) # Register the ExternalDataset dproducer_id = dams_cli.register_external_data_set(external_dataset_id=ds_id) # Or using each method dams_cli.assign_data_source_to_external_data_provider(data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id) dams_cli.assign_data_source_to_data_model(data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id) dams_cli.assign_external_dataset_to_data_source(external_dataset_id=ds_id, data_source_id=ext_dsrc_id) dams_cli.assign_external_dataset_to_agent_instance(external_dataset_id=ds_id, agent_instance_id=eda_inst_id) # dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=self.eda_id, agent_instance_id=self.eda_inst_id) #create temp streamdef so the data product can create the stream pc_list = [] for pc_k, pc in self._create_parameter_dictionary().iteritems(): pc_list.append(dms_cli.create_parameter_context(pc_k, pc[1].dump())) pdict_id = dms_cli.create_parameter_dictionary('slocum_param_dict', pc_list) streamdef_id = pubsub_cli.create_stream_definition(name="slocum_stream_def", description="stream def for slocum testing", parameter_dictionary_id=pdict_id) # dpms_cli.create_data_product() # Generate the data product and associate it to the ExternalDataset tdom, sdom = time_series_domain() tdom, sdom = tdom.dump(), sdom.dump() dprod = IonObject(RT.DataProduct, name='slocum_parsed_product', description='parsed slocum product', temporal_domain=tdom, spatial_domain=sdom) dproduct_id = dpms_cli.create_data_product(data_product=dprod, stream_definition_id=streamdef_id) dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id) stream_id, assn = rr_cli.find_objects(subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True) stream_id = stream_id[0] log.info('Created resources: {0}'.format({'ExternalDataset': ds_id, 'ExternalDataProvider': ext_dprov_id, 'DataSource': ext_dsrc_id, 'DataSourceModel': ext_dsrc_model_id, 'DataProducer': dproducer_id, 'DataProduct': dproduct_id, 'Stream': stream_id})) # Create the logger for receiving publications _, stream_route, _ = self.create_stream_and_logger(name='slocum', stream_id=stream_id) self.EDA_RESOURCE_ID = ds_id self.EDA_NAME = ds_name self.DVR_CONFIG['dh_cfg'] = { 'TESTING': True, 'stream_id': stream_id, 'stream_route': stream_route, 'stream_def': streamdef_id, 'external_dataset_res': dset, 'data_producer_id': dproducer_id, # CBM: Should this be put in the main body of the config - with mod & cls? 'max_records': 20, }
def _setup_resources(self): # TODO: some or all of this (or some variation) should move to DAMS' # Build the test resources for the dataset dms_cli = DatasetManagementServiceClient() dams_cli = DataAcquisitionManagementServiceClient() dpms_cli = DataProductManagementServiceClient() rr_cli = ResourceRegistryServiceClient() pubsub_cli = PubsubManagementServiceClient() eda = ExternalDatasetAgent(name='example eda',handler_module=self.DVR_CONFIG['dvr_mod'], handler_class=self.DVR_CONFIG['dvr_cls']) eda_id = dams_cli.create_external_dataset_agent(eda) eda_inst = ExternalDatasetAgentInstance(name='example eda instance') eda_inst_id = dams_cli.create_external_dataset_agent_instance(eda_inst, external_dataset_agent_id=eda_id) # Create and register the necessary resources/objects # Create DataProvider dprov = ExternalDataProvider(name='example data provider', institution=Institution(), contact=ContactInformation()) dprov.contact.individual_names_given = 'Christopher Mueller' dprov.contact.email = '*****@*****.**' # Create DataSource dsrc = DataSource(name='example datasource', protocol_type='DAP', institution=Institution(), contact=ContactInformation()) dsrc.connection_params['base_data_url'] = '' dsrc.contact.individual_names_given = 'Tim Giguere' dsrc.contact.email = '*****@*****.**' # Create ExternalDataset ds_name = 'usgs_test_dataset' dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) # The usgs.nc test dataset is a download of the R1 dataset found here: # http://thredds-test.oceanobservatories.org/thredds/dodsC/ooiciData/E66B1A74-A684-454A-9ADE-8388C2C634E5.ncml dset.dataset_description.parameters['dataset_path'] = 'test_data/usgs.nc' dset.dataset_description.parameters['temporal_dimension'] = 'time' dset.dataset_description.parameters['zonal_dimension'] = 'lon' dset.dataset_description.parameters['meridional_dimension'] = 'lat' dset.dataset_description.parameters['vertical_dimension'] = 'z' dset.dataset_description.parameters['variables'] = [ 'water_temperature', 'streamflow', 'water_temperature_bottom', 'water_temperature_middle', 'specific_conductance', 'data_qualifier', ] # Create DataSourceModel dsrc_model = DataSourceModel(name='dap_model') #dsrc_model.model = 'DAP' dsrc_model.data_handler_module = 'N/A' dsrc_model.data_handler_class = 'N/A' ## Run everything through DAMS ds_id = dams_cli.create_external_dataset(external_dataset=dset) ext_dprov_id = dams_cli.create_external_data_provider(external_data_provider=dprov) ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc) ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model) # Register the ExternalDataset dproducer_id = dams_cli.register_external_data_set(external_dataset_id=ds_id) # Or using each method dams_cli.assign_data_source_to_external_data_provider(data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id) dams_cli.assign_data_source_to_data_model(data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id) dams_cli.assign_external_dataset_to_data_source(external_dataset_id=ds_id, data_source_id=ext_dsrc_id) dams_cli.assign_external_dataset_to_agent_instance(external_dataset_id=ds_id, agent_instance_id=eda_inst_id) # dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=self.eda_id, agent_instance_id=self.eda_inst_id) #create temp streamdef so the data product can create the stream pc_list = [] #Get 'time' parameter context pc_list.append(dms_cli.read_parameter_context_by_name('time', id_only=True)) for pc_k, pc in self._create_parameter_dictionary().iteritems(): pc_list.append(dms_cli.create_parameter_context(pc_k, pc[1].dump())) pdict_id = dms_cli.create_parameter_dictionary('netcdf_param_dict', pc_list) #create temp streamdef so the data product can create the stream streamdef_id = pubsub_cli.create_stream_definition(name="netcdf", description="netcdf", parameter_dictionary_id=pdict_id) tdom, sdom = time_series_domain() tdom, sdom = tdom.dump(), sdom.dump() dprod = IonObject(RT.DataProduct, name='usgs_parsed_product', description='parsed usgs product', temporal_domain=tdom, spatial_domain=sdom) # Generate the data product and associate it to the ExternalDataset dproduct_id = dpms_cli.create_data_product(data_product=dprod, stream_definition_id=streamdef_id) dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id) stream_id, assn = rr_cli.find_objects(subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True) stream_id = stream_id[0] log.info('Created resources: {0}'.format({'ExternalDataset': ds_id, 'ExternalDataProvider': ext_dprov_id, 'DataSource': ext_dsrc_id, 'DataSourceModel': ext_dsrc_model_id, 'DataProducer': dproducer_id, 'DataProduct': dproduct_id, 'Stream': stream_id})) # Create the logger for receiving publications _, stream_route, _ = self.create_stream_and_logger(name='usgs', stream_id=stream_id) self.EDA_RESOURCE_ID = ds_id self.EDA_NAME = ds_name self.DVR_CONFIG['dh_cfg'] = { 'TESTING': True, 'stream_id': stream_id, 'stream_route': stream_route, 'stream_def': streamdef_id, 'data_producer_id': dproducer_id, # CBM: Should this be put in the main body of the config - with mod & cls? 'max_records': 1, }
def _setup_resources(self): # TODO: some or all of this (or some variation) should move to DAMS' # Build the test resources for the dataset dams_cli = DataAcquisitionManagementServiceClient() dpms_cli = DataProductManagementServiceClient() rr_cli = ResourceRegistryServiceClient() pubsub_cli = PubsubManagementServiceClient() eda = ExternalDatasetAgent() eda_id = dams_cli.create_external_dataset_agent(eda) eda_inst = ExternalDatasetAgentInstance() eda_inst_id = dams_cli.create_external_dataset_agent_instance(eda_inst, external_dataset_agent_id=eda_id) # Create and register the necessary resources/objects # Create DataProvider dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation()) dprov.contact.name = 'Christopher Mueller' dprov.contact.email = '*****@*****.**' # Create DataSource dsrc = DataSource(protocol_type='DAP', institution=Institution(), contact=ContactInformation()) dsrc.connection_params['base_data_url'] = '' dsrc.contact.name='Tim Giguere' dsrc.contact.email = '*****@*****.**' # Create ExternalDataset ds_name = 'usgs_test_dataset' dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) # The usgs.nc test dataset is a download of the R1 dataset found here: # http://thredds-test.oceanobservatories.org/thredds/dodsC/ooiciData/E66B1A74-A684-454A-9ADE-8388C2C634E5.ncml dset.dataset_description.parameters['dataset_path'] = 'test_data/usgs.nc' dset.dataset_description.parameters['temporal_dimension'] = 'time' dset.dataset_description.parameters['zonal_dimension'] = 'lon' dset.dataset_description.parameters['meridional_dimension'] = 'lat' dset.dataset_description.parameters['vertical_dimension'] = 'z' dset.dataset_description.parameters['variables'] = [ 'water_temperature', 'streamflow', 'water_temperature_bottom', 'water_temperature_middle', 'specific_conductance', 'data_qualifier', ] # Create DataSourceModel dsrc_model = DataSourceModel(name='dap_model') dsrc_model.model = 'DAP' dsrc_model.data_handler_module = 'N/A' dsrc_model.data_handler_class = 'N/A' ## Run everything through DAMS ds_id = dams_cli.create_external_dataset(external_dataset=dset) ext_dprov_id = dams_cli.create_external_data_provider(external_data_provider=dprov) ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc) ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model) # Register the ExternalDataset dproducer_id = dams_cli.register_external_data_set(external_dataset_id=ds_id) # Or using each method dams_cli.assign_data_source_to_external_data_provider(data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id) dams_cli.assign_data_source_to_data_model(data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id) dams_cli.assign_external_dataset_to_data_source(external_dataset_id=ds_id, data_source_id=ext_dsrc_id) dams_cli.assign_external_dataset_to_agent_instance(external_dataset_id=ds_id, agent_instance_id=eda_inst_id) # dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=self.eda_id, agent_instance_id=self.eda_inst_id) #create temp streamdef so the data product can create the stream streamdef_id = pubsub_cli.create_stream_definition(name="temp", description="temp") craft = CoverageCraft sdom, tdom = craft.create_domains() sdom = sdom.dump() tdom = tdom.dump() parameter_dictionary = craft.create_parameters() parameter_dictionary = parameter_dictionary.dump() dprod = IonObject(RT.DataProduct, name='usgs_parsed_product', description='parsed usgs product', temporal_domain = tdom, spatial_domain = sdom) # Generate the data product and associate it to the ExternalDataset dproduct_id = dpms_cli.create_data_product(data_product=dprod, stream_definition_id=streamdef_id, parameter_dictionary=parameter_dictionary) dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id) stream_id, assn = rr_cli.find_objects(subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True) stream_id = stream_id[0] log.info('Created resources: {0}'.format({'ExternalDataset':ds_id, 'ExternalDataProvider':ext_dprov_id, 'DataSource':ext_dsrc_id, 'DataSourceModel':ext_dsrc_model_id, 'DataProducer':dproducer_id, 'DataProduct':dproduct_id, 'Stream':stream_id})) #CBM: Use CF standard_names # ttool = TaxyTool() # ttool.add_taxonomy_set('time','time') # ttool.add_taxonomy_set('lon','longitude') # ttool.add_taxonomy_set('lat','latitude') # ttool.add_taxonomy_set('z','water depth') # ttool.add_taxonomy_set('water_temperature', 'average water temperature') # ttool.add_taxonomy_set('water_temperature_bottom','water temperature at bottom of water column') # ttool.add_taxonomy_set('water_temperature_middle', 'water temperature at middle of water column') # ttool.add_taxonomy_set('streamflow', 'flow velocity of stream') # ttool.add_taxonomy_set('specific_conductance', 'specific conductance of water') # ttool.add_taxonomy_set('data_qualifier','data qualifier flag') # # ttool.add_taxonomy_set('coords','This group contains coordinate parameters') # ttool.add_taxonomy_set('data','This group contains data parameters') # Create the logger for receiving publications self.create_stream_and_logger(name='usgs',stream_id=stream_id) pdict = ParameterDictionary() t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=numpy.dtype('int64'))) t_ctxt.reference_frame = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 01-01-1970' pdict.add_context(t_ctxt) lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) lat_ctxt.reference_frame = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' pdict.add_context(lat_ctxt) lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) lon_ctxt.reference_frame = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' pdict.add_context(lon_ctxt) temp_ctxt = ParameterContext('water_temperature', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) temp_ctxt.uom = 'degree_Celsius' pdict.add_context(temp_ctxt) temp_ctxt = ParameterContext('water_temperature_bottom', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) temp_ctxt.uom = 'degree_Celsius' pdict.add_context(temp_ctxt) temp_ctxt = ParameterContext('water_temperature_middle', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) temp_ctxt.uom = 'degree_Celsius' pdict.add_context(temp_ctxt) temp_ctxt = ParameterContext('z', param_type=QuantityType(value_encoding = numpy.dtype('float32'))) temp_ctxt.uom = 'meters' pdict.add_context(temp_ctxt) cond_ctxt = ParameterContext('streamflow', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) cond_ctxt.uom = 'unknown' pdict.add_context(cond_ctxt) pres_ctxt = ParameterContext('specific_conductance', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) pres_ctxt.uom = 'unknown' pdict.add_context(pres_ctxt) pres_ctxt = ParameterContext('data_qualifier', param_type=QuantityType(value_encoding=numpy.dtype('bool'))) pres_ctxt.uom = 'unknown' pdict.add_context(pres_ctxt) self.EDA_RESOURCE_ID = ds_id self.EDA_NAME = ds_name self.DVR_CONFIG['dh_cfg'] = { 'TESTING':True, 'stream_id':stream_id, #'taxonomy':ttool.dump(), 'param_dictionary':pdict.dump(), 'data_producer_id':dproducer_id,#CBM: Should this be put in the main body of the config - with mod & cls? 'max_records':4, }
class TestIntDataAcquisitionManagementService(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataAcquisitionManagementService self.client = DataAcquisitionManagementServiceClient(node=self.container.node) self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) def tearDown(self): pass #@unittest.skip('Not done yet.') def test_data_source_ops(self): # test creating a new data source print 'Creating new data source' datasource_obj = IonObject(RT.DataSource, name='DataSource1', description='instrument based new source' , data_source_type='sbe37') try: ds_id = self.client.create_data_source(datasource_obj) except BadRequest as ex: self.fail("failed to create new data source: %s" %ex) print 'new data source id = ', ds_id # test reading a non-existent data source print 'reading non-existent data source' try: dp_obj = self.client.read_data_source('some_fake_id') except NotFound as ex: pass else: self.fail("non-existing data source was found during read: %s" %dp_obj) # update a data source (tests read also) print 'Updating data source' # first get the existing data source object try: datasource_obj = self.client.read_data_source(ds_id) except NotFound as ex: self.fail("existing data source was not found during read") else: pass # now tweak the object datasource_obj.description = 'the very first data source' # now write the dp back to the registry try: update_result = self.client.update_data_source(datasource_obj) except NotFound as ex: self.fail("existing data source was not found during update") except Conflict as ex: self.fail("revision conflict exception during data source update") #else: # self.assertTrue(update_result == True) # now get the data source back to see if it was updated try: datasource_obj = self.client.read_data_source(ds_id) except NotFound as ex: self.fail("existing data source was not found during read") else: pass self.assertTrue(datasource_obj.description == 'the very first data source') # now 'delete' the data source print "deleting data source" try: delete_result = self.client.force_delete_data_source(ds_id) except NotFound as ex: self.fail("existing data source was not found during delete") #self.assertTrue(delete_result == True) # now try to get the deleted dp object try: dp_obj = self.client.read_data_source(ds_id) except NotFound as ex: pass else: self.fail("deleted data source was found during read") # now try to delete the already deleted data source object print "deleting non-existing data source" try: delete_result = self.client.delete_data_source(ds_id) except NotFound as ex: pass else: self.fail("non-existing data source was found during delete") def test_register_instrument(self): # set up initial instrument to register instrument_obj = IonObject(RT.InstrumentDevice, name='Inst1',description='an instrument that is creating the data product') instrument_id, rev = self.rrclient.create(instrument_obj) self.base_register_instrument(instrument_id) def test_register_platform(self): # set up initial instrument to register platform_obj = IonObject(RT.PlatformDevice, name='Plat1',description='a platform that is creating the data product') platform_id, rev = self.rrclient.create(platform_obj) #@unittest.skip('Not done yet.') def base_register_instrument(self, instrument_id): # Register an instrument as a data producer in coordination with DM PubSub: create stream, register and create producer object dataproduct_obj = IonObject(RT.DataProduct, name='DataProduct1',description='sample data product') dataproduct_id, rev = self.rrclient.create(dataproduct_obj) # test registering a new data producer try: ds_id = self.client.register_instrument(instrument_id) except BadRequest as ex: self.fail("failed to create new data producer: %s" %ex) print 'new data producer id = ', ds_id # test assigning a data product to an instrument, creating the stream for the product try: self.client.assign_data_product(instrument_id, dataproduct_id) self.client.assign_data_product_source(dataproduct_id, instrument_id) except BadRequest as ex: self.fail("failed to assign data product to data producer: %s" %ex) except NotFound as ex: self.fail("failed to assign data product to data producer: %s" %ex) assocs = self.rrclient.find_associations(dataproduct_id, PRED.hasSource, instrument_id) if not assocs or len(assocs) == 0: self.fail("failed to assign data product to data producer") # test UNassigning a data product from instrument, deleting the stream for the product try: self.client.unassign_data_product(instrument_id, dataproduct_id) self.client.unassign_data_product_source(dataproduct_id, instrument_id) except BadRequest as ex: self.fail("failed to failed to UNassign data product to data producer data producer: %s" %ex) except NotFound as ex: self.fail("failed to failed to UNassign data product to data producer data producer: %s" %ex) assocs = self.rrclient.find_associations(dataproduct_id, PRED.hasSource, instrument_id) if assocs: self.fail("failed to unassign data product to data producer") # test UNregistering a new data producer try: ds_id = self.client.unregister_instrument(instrument_id) except NotFound as ex: self.fail("failed to unregister instrument producer: %s" %ex) def test_register_external_data_set(self): # Register an external data set as a data producer in coordination with DM PubSub: create stream, register and create producer object # set up initial instrument to register ext_dataset_obj = IonObject(RT.ExternalDataset, name='DataSet1',description='an external data feed') ext_dataset_id, rev = self.rrclient.create(ext_dataset_obj) dataproduct_obj = IonObject(RT.DataProduct, name='DataProduct1',description='sample data product') dataproduct_id, rev = self.rrclient.create(dataproduct_obj) # test registering a new external data set try: ds_id = self.client.register_external_data_set(ext_dataset_id) except BadRequest as ex: self.fail("failed to create new data producer: %s" %ex) print 'new data producer id = ', ds_id # test assigning a data product to an ext_dataset_id, creating the stream for the product try: self.client.assign_data_product(ext_dataset_id, dataproduct_id) except BadRequest as ex: self.fail("failed to assign data product to data producer: %s" %ex) except NotFound as ex: self.fail("failed to assign data product to data producer: %s" %ex) # test UNassigning a data product from ext_dataset_id, deleting the stream for the product try: self.client.unassign_data_product(ext_dataset_id, dataproduct_id) except BadRequest as ex: self.fail("failed to failed to UNassign data product to data producer data producer: %s" %ex) except NotFound as ex: self.fail("failed to failed to UNassign data product to data producer data producer: %s" %ex) # test UNregistering a external data set try: ds_id = self.client.unregister_external_data_set(ext_dataset_id) except NotFound as ex: self.fail("failed to unregister instrument producer: %s" %ex) #@unittest.skip('not ready') def test_eoi_resources(self): # # test creating a new data provider # print 'Creating new external_data_provider' dataprovider_obj = IonObject(RT.ExternalDataProvider, name='ExtDataProvider1', description='external data provider ') try: dataprovider_id = self.client.create_external_data_provider(dataprovider_obj) except BadRequest as ex: self.fail("failed to create new data provider: %s" %ex) print 'new data provider id = ', dataprovider_id # # test creating a new data source # print 'Creating new data source' datasource_obj = IonObject(RT.DataSource, name='DataSource1', description='data source ', data_source_type='DAP') try: datasource_id = self.client.create_data_source(datasource_obj) except BadRequest as ex: self.fail("failed to create new data source: %s" %ex) print 'new data source id = ', datasource_id # # test creating a new data source model # print 'Creating new data source model' datamodel_obj = IonObject(RT.DataSourceModel, name='DataSourceModel1', description='data source model') try: datamodel_id = self.client.create_data_source_model(datamodel_obj) except BadRequest as ex: self.fail("failed to create new data source model: %s" %ex) print 'new data source model id = ', datamodel_id # # test creating a new external data set # print 'Creating new external data set' dataset_obj = IonObject(RT.ExternalDataset, name='ExternalDataSet1', description='external data set ') try: extdataset_id = self.client.create_external_dataset(dataset_obj) except BadRequest as ex: self.fail("failed to create new external data set: %s" %ex) print 'new external data set id = ', extdataset_id # # test creating a new dataset agent instance # print 'Creating new external data agent ' datasetagent_obj = IonObject(RT.ExternalDatasetAgent, name='ExternalDatasetAgent1', description='external data agent ') try: datasetagent_id = self.client.create_external_dataset_agent(datasetagent_obj) except BadRequest as ex: self.fail("failed to create new external dataset agent: %s" %ex) print 'new external data agent id = ', datasetagent_id # # test creating a new datasource agent instance # print 'Creating new data source agent ' datasourceagent_obj = IonObject(RT.DataSourceAgent, name='DataSourceAgent1', description=' DataSource agent ') try: datasource_agent_id = self.client.create_data_source_agent(datasourceagent_obj) except BadRequest as ex: self.fail("failed to create new external datasource agent: %s" %ex) print 'new external data agent id = ', datasource_agent_id # # test creating a new dataset agent instance # print 'Creating new external dataset agent instance' datasetagentinstance_obj = IonObject(RT.ExternalDatasetAgentInstance, name='ExternalDatasetAgentInstance1', description='external dataset agent instance ') try: datasetagentinstance_id = self.client.create_external_dataset_agent_instance(datasetagentinstance_obj, datasetagent_id) except BadRequest as ex: self.fail("failed to create new external dataset agent instance: %s" %ex) print 'new external data agent instance id = ', datasetagentinstance_id # # test creating a new datasource agent instance # print 'Creating new data source agent ' datasourceagentinstance_obj = IonObject(RT.DataSourceAgentInstance, name='ExternalDataSourceAgentInstance1', description='external DataSource agent instance ') try: datasource_agent_instance_id = self.client.create_data_source_agent_instance(datasourceagentinstance_obj) except BadRequest as ex: self.fail("failed to create new external datasource agent instance: %s" %ex) print 'new external data agent id = ', datasource_agent_instance_id # # test assign / unassign # self.client.unassign_data_source_from_external_data_provider(datasource_id, dataprovider_id) self.client.unassign_data_source_from_data_model(datasource_id, datamodel_id) self.client.unassign_external_dataset_from_data_source(extdataset_id, datasource_id) # # test read # try: dp_obj = self.client.read_external_data_provider(dataprovider_id) except NotFound as ex: self.fail("existing data provicer was not found during read") else: pass try: dp_obj = self.client.read_data_source(datasource_id) except NotFound as ex: self.fail("existing data source was not found during read") else: pass # # test delete # try: self.client.delete_external_data_provider(dataprovider_id) self.client.delete_data_source(datasource_id) self.client.delete_external_dataset(extdataset_id) self.client.delete_data_source_model(datamodel_id) self.client.delete_external_dataset_agent(datasetagent_id) self.client.delete_data_source_agent_instance(datasource_agent_instance_id) self.client.force_delete_external_data_provider(dataprovider_id) self.client.force_delete_data_source(datasource_id) self.client.force_delete_external_dataset(extdataset_id) self.client.force_delete_data_source_model(datamodel_id) self.client.force_delete_external_dataset_agent(datasetagent_id) self.client.force_delete_data_source_agent_instance(datasource_agent_instance_id) except NotFound as ex: self.fail("existing data product was not found during delete") # test reading a non-existent data product print 'reading non-existent data product' try: bad_obj = self.client.read_external_data_provider('some_fake_id') except NotFound as ex: pass else: self.fail("non-existing data product was found during read: %s" %bad_obj) def make_grt_parser(self): return self.client.create_parser(Parser(name='grt', description='', module='ion.util.parsers.global_range_test', method='grt_parser', config=None)) @unittest.skip("Deprecated") def test_qc_attachment(self): instrument_device = InstrumentDevice(name='whatever') instrument_device_id,_ = self.rrclient.create(instrument_device) self.addCleanup(self.rrclient.delete, instrument_device_id) self.client.register_instrument(instrument_device_id) self.addCleanup(self.client.unregister_instrument, instrument_device_id) dp = DataProduct(name='instrument output') dp_id,_ = self.rrclient.create(dp) self.addCleanup(self.rrclient.delete, dp_id) parser_id = self.make_grt_parser() attachment = Attachment(name='qc ref', attachment_type=AttachmentType.REFERENCE,content=global_range_test_document, context=ReferenceAttachmentContext(parser_id=parser_id)) att_id = self.rrclient.create_attachment(dp_id, attachment) self.addCleanup(self.rrclient.delete_attachment, att_id) attachment2 = Attachment(name='qc ref2', attachment_type=AttachmentType.REFERENCE, content=global_range_test_document2, context=ReferenceAttachmentContext(parser_id=parser_id)) att2_id = self.rrclient.create_attachment(dp_id, attachment2) self.addCleanup(self.rrclient.delete_attachment, att2_id) self.client.assign_data_product(instrument_device_id, dp_id) self.addCleanup(self.client.unassign_data_product, instrument_device_id, dp_id) svm = StoredValueManager(self.container) doc = svm.read_value('grt_CE01ISSM-MF005-01-CTDBPC999_TEMPWAT') np.testing.assert_array_almost_equal(doc['grt_min_value'], -2.)
class TestIntExternalObservatoryAgent(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url(rel_url='res/deploy/r2eoi.yml') self.dams_cli = DataAcquisitionManagementServiceClient() self.dpms_cli = DataProductManagementServiceClient() eda = ExternalDatasetAgent() self.eda_id = self.dams_cli.create_external_dataset_agent(eda) eda_inst = ExternalDatasetAgentInstance() self.eda_inst_id = self.dams_cli.create_external_dataset_agent_instance(eda_inst, external_dataset_agent_id=self.eda_id) self._setup_ncom() proc_name = self.ncom_ds_id+'_worker' config = {} config['process']={'name':proc_name,'type':'agent'} config['process']['eoa']={'dataset_id':self.ncom_ds_id} pid = self.container.spawn_process(name=proc_name, module='ion.agents.eoi.external_observatory_agent', cls='ExternalObservatoryAgent', config=config) queue_id = "%s.%s" % (self.container.id, pid) log.debug("Spawned worker process ==> proc_name: %s\tproc_id: %s\tqueue_id: %s" % (proc_name, pid, queue_id)) self._agent_cli = ResourceAgentClient(self.ncom_ds_id, name=pid, process=FakeProcess()) log.debug("Got a ResourceAgentClient: res_id=%s" % self._agent_cli.resource_id) def _setup_ncom(self): # TODO: some or all of this (or some variation) should move to DAMS # Create and register the necessary resources/objects # Create DataProvider dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation()) # dprov.institution.name = "OOI CGSN" dprov.contact.name = "Robert Weller" dprov.contact.email = "*****@*****.**" # Create DataSource dsrc = DataSource(protocol_type="DAP", institution=Institution(), contact=ContactInformation()) # dsrc.connection_params["base_data_url"] = "http://ooi.whoi.edu/thredds/dodsC/" dsrc.connection_params["base_data_url"] = "" dsrc.contact.name="Rich Signell" dsrc.contact.email = "*****@*****.**" # Create ExternalDataset dset = ExternalDataset(name="test", dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) # dset.dataset_description.parameters["dataset_path"] = "ooi/AS02CPSM_R_M.nc" dset.dataset_description.parameters["dataset_path"] = "test_data/ncom.nc" dset.dataset_description.parameters["temporal_dimension"] = "time" dset.dataset_description.parameters["zonal_dimension"] = "lon" dset.dataset_description.parameters["meridional_dimension"] = "lat" # Create DataSourceModel dsrc_model = DataSourceModel(name="dap_model") dsrc_model.model = "DAP" dsrc_model.data_handler_module = "ion.agents.eoi.handler.dap_external_data_handler" dsrc_model.data_handler_class = "DapExternalDataHandler" ## Run everything through DAMS ds_id = self.ncom_ds_id = self.dams_cli.create_external_dataset(external_dataset=dset) ext_dprov_id = self.dams_cli.create_external_data_provider(external_data_provider=dprov) ext_dsrc_id = self.dams_cli.create_data_source(data_source=dsrc) ext_dsrc_model_id = self.dams_cli.create_data_source_model(dsrc_model) # Register the ExternalDataset dproducer_id = self.dams_cli.register_external_data_set(external_dataset_id=ds_id) ## Associate everything # Convenience method # self.dams_cli.assign_eoi_resources(external_data_provider_id=ext_dprov_id, data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id, external_dataset_id=ds_id, external_data_agent_id=self.eda_id, agent_instance_id=self.eda_inst_id) # Or using each method self.dams_cli.assign_data_source_to_external_data_provider(data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id) self.dams_cli.assign_data_source_to_data_model(data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id) self.dams_cli.assign_external_dataset_to_data_source(external_dataset_id=ds_id, data_source_id=ext_dsrc_id) self.dams_cli.assign_external_dataset_to_agent_instance(external_dataset_id=ds_id, agent_instance_id=self.eda_inst_id) # self.dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=self.eda_id, agent_instance_id=self.eda_inst_id) # Generate the data product and associate it to the ExternalDataset dprod = DataProduct(name='ncom_product', description='raw ncom product') dproduct_id = self.dpms_cli.create_data_product(data_product=dprod) self.dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id, create_stream=True) ########## Tests ########## # @unittest.skip("Currently broken due to resource/agent refactorings") def test_get_capabilities(self): # Get all the capabilities caps = self._agent_cli.get_capabilities() log.debug("all capabilities: %s" % caps) lst=[['RES_CMD', 'acquire_data'], ['RES_CMD', 'acquire_data_by_request'], ['RES_CMD', 'acquire_new_data'], ['RES_CMD', 'close'], ['RES_CMD', 'compare'], ['RES_CMD', 'get_attributes'], ['RES_CMD', 'get_fingerprint'], ['RES_CMD', 'get_status'], ['RES_CMD', 'has_new_data']] self.assertEquals(caps, lst) caps = self._agent_cli.get_capabilities(capability_types=['RES_CMD']) log.debug("resource commands: %s" % caps) lst=[['RES_CMD', 'acquire_data'], ['RES_CMD', 'acquire_data_by_request'], ['RES_CMD', 'acquire_new_data'], ['RES_CMD', 'close'], ['RES_CMD', 'compare'], ['RES_CMD', 'get_attributes'], ['RES_CMD', 'get_fingerprint'], ['RES_CMD', 'get_status'], ['RES_CMD', 'has_new_data']] self.assertEquals(caps, lst) caps = self._agent_cli.get_capabilities(capability_types=['RES_PAR']) log.debug("resource commands: %s" % caps) self.assertEqual(type(caps), list) caps = self._agent_cli.get_capabilities(capability_types=['AGT_CMD']) log.debug("resource commands: %s" % caps) self.assertEqual(type(caps), list) caps = self._agent_cli.get_capabilities(capability_types=['AGT_PAR']) log.debug("resource commands: %s" % caps) self.assertEqual(type(caps), list) @unittest.skip("Currently broken due to resource/agent refactorings") def test_execute_get_attrs(self): cmd = AgentCommand(command_id="111", command="get_attributes") log.debug("Execute AgentCommand: %s" % cmd) ret = self._agent_cli.execute(cmd) log.debug("Returned: %s" % ret) self.assertEquals(ret.status, 0) self.assertTrue(type(ret.result), dict)
class TestIntExternalObservatoryAgentService(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url(rel_url='res/deploy/r2eoi.yml') # self.eoas_cli = ExternalObservatoryAgentServiceClient() # self.rr_cli = ResourceRegistryServiceClient() self.dams_cli = DataAcquisitionManagementServiceClient() self.dpms_cli = DataProductManagementServiceClient() self._setup_ncom() self._setup_hfr() # eoas_proc = self.container.proc_manager.procs_by_name['external_data_agent_management'] # log.debug("Got EOAS Process: %s" % eoas_proc) self._ncom_agt_cli = ResourceAgentClient(resource_id=self.ncom_ds_id, name='external_observatory_agent', process=FakeProcess()) log.debug("Got a ResourceAgentClient: res_id=%s" % self._ncom_agt_cli.resource_id) self._hfr_agt_cli = ResourceAgentClient(resource_id=self.hfr_ds_id, name='external_observatory_agent', process=FakeProcess()) log.debug("Got a ResourceAgentClient: res_id=%s" % self._hfr_agt_cli.resource_id) def _setup_ncom(self): # TODO: some or all of this (or some variation) should move to DAMS # Create and register the necessary resources/objects eda = ExternalDatasetAgent() eda_id = self.dams_cli.create_external_dataset_agent(eda) eda_inst = ExternalDatasetAgentInstance() eda_inst_id = self.dams_cli.create_external_dataset_agent_instance(eda_inst, external_dataset_agent_id=eda_id) # Create DataProvider dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation()) # dprov.institution.name = "OOI CGSN" dprov.contact.name = "Robert Weller" dprov.contact.email = "*****@*****.**" # Create DataSource dsrc = DataSource(protocol_type="DAP", institution=Institution(), contact=ContactInformation()) # dsrc.connection_params["base_data_url"] = "http://ooi.whoi.edu/thredds/dodsC/" dsrc.connection_params["base_data_url"] = "" dsrc.contact.name="Rich Signell" dsrc.contact.email = "*****@*****.**" # Create ExternalDataset dset = ExternalDataset(name="test", dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) # dset.dataset_description.parameters["dataset_path"] = "ooi/AS02CPSM_R_M.nc" dset.dataset_description.parameters["dataset_path"] = "test_data/ncom.nc" dset.dataset_description.parameters["temporal_dimension"] = "time" dset.dataset_description.parameters["zonal_dimension"] = "lon" dset.dataset_description.parameters["meridional_dimension"] = "lat" # Create DataSourceModel dsrc_model = DataSourceModel(name="dap_model") dsrc_model.model = "DAP" dsrc_model.data_handler_module = "eoi.agent.handler.dap_external_data_handler" dsrc_model.data_handler_class = "DapExternalDataHandler" ## Run everything through DAMS ds_id = self.ncom_ds_id = self.dams_cli.create_external_dataset(external_dataset=dset) ext_dprov_id = self.dams_cli.create_external_data_provider(external_data_provider=dprov) ext_dsrc_id = self.dams_cli.create_data_source(data_source=dsrc) ext_dsrc_model_id = self.dams_cli.create_data_source_model(dsrc_model) # Register the ExternalDataset dproducer_id = self.dams_cli.register_external_data_set(external_dataset_id=ds_id) ## Associate everything # Convenience method # self.dams_cli.assign_eoi_resources(external_data_provider_id=ext_dprov_id, data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id, external_dataset_id=ds_id, external_data_agent_id=eda_id, agent_instance_id=eda_inst_id) # Or using each method self.dams_cli.assign_data_source_to_external_data_provider(data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id) self.dams_cli.assign_data_source_to_data_model(data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id) self.dams_cli.assign_external_dataset_to_data_source(external_dataset_id=ds_id, data_source_id=ext_dsrc_id) self.dams_cli.assign_external_dataset_to_agent_instance(external_dataset_id=ds_id, agent_instance_id=eda_inst_id) # self.dams_cli.assign_external_dataset_agent_to_data_model(external_data_agent_id=eda_id, data_source_model_id=ext_dsrc_model_id) # self.dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=eda_id, agent_instance_id=eda_inst_id) # Generate the data product and associate it to the ExternalDataset dprod = DataProduct(name='ncom_product', description='raw ncom product') dproduct_id = self.dpms_cli.create_data_product(data_product=dprod) self.dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id, create_stream=True) def _setup_hfr(self): # TODO: some or all of this (or some variation) should move to DAMS # Create and register the necessary resources/objects eda = ExternalDatasetAgent() eda_id = self.dams_cli.create_external_dataset_agent(eda) eda_inst = ExternalDatasetAgentInstance() eda_inst_id = self.dams_cli.create_external_dataset_agent_instance(eda_inst, external_dataset_agent_id=eda_id) # Create DataProvider dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation()) # dprov.institution.name = "HFR UCSD" # Create DataSource dsrc = DataSource(protocol_type="DAP", institution=Institution(), contact=ContactInformation()) dsrc.connection_params["base_data_url"] = "http://hfrnet.ucsd.edu:8080/thredds/dodsC/" # Create ExternalDataset dset = ExternalDataset(name="UCSD HFR", dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) dset.dataset_description.parameters["dataset_path"] = "HFRNet/USEGC/6km/hourly/RTV" # dset.dataset_description.parameters["dataset_path"] = "test_data/hfr.nc" dset.dataset_description.parameters["temporal_dimension"] = "time" dset.dataset_description.parameters["zonal_dimension"] = "lon" dset.dataset_description.parameters["meridional_dimension"] = "lat" # Create DataSourceModel dsrc_model = DataSourceModel(name="dap_model") dsrc_model.model = "DAP" dsrc_model.data_handler_module = "eoi.agent.handler.dap_external_data_handler" dsrc_model.data_handler_class = "DapExternalDataHandler" ## Run everything through DAMS ds_id = self.hfr_ds_id = self.dams_cli.create_external_dataset(external_dataset=dset) ext_dprov_id = self.dams_cli.create_external_data_provider(external_data_provider=dprov) ext_dsrc_id = self.dams_cli.create_data_source(data_source=dsrc) ext_dsrc_model_id = self.dams_cli.create_data_source_model(dsrc_model) # Register the ExternalDataset dproducer_id = self.dams_cli.register_external_data_set(external_dataset_id=ds_id) ## Associate everything # Convenience method # self.dams_cli.assign_eoi_resources(external_data_provider_id=ext_dprov_id, data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id, external_dataset_id=ds_id, external_data_agent_id=eda_id, agent_instance_id=eda_inst_id) # Or using each method self.dams_cli.assign_data_source_to_external_data_provider(data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id) self.dams_cli.assign_data_source_to_data_model(data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id) self.dams_cli.assign_external_dataset_to_data_source(external_dataset_id=ds_id, data_source_id=ext_dsrc_id) self.dams_cli.assign_external_dataset_to_agent_instance(external_dataset_id=ds_id, agent_instance_id=eda_inst_id) # self.dams_cli.assign_external_dataset_agent_to_data_model(external_data_agent_id=eda_id, data_source_model_id=ext_dsrc_model_id) # self.dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=eda_id, agent_instance_id=eda_inst_id) # Generate the data product and associate it to the ExternalDataset dprod = DataProduct(name='hfr_product', description='raw hfr product') dproduct_id = self.dpms_cli.create_data_product(data_product=dprod) self.dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id, create_stream=True) ########## Tests ########## @unittest.skip("Currently broken due to resource/agent refactorings") def test_get_capabilities(self): # Get all the capabilities caps = self._ncom_agt_cli.get_capabilities() log.debug("all capabilities: %s" % caps) lst=[['RES_CMD', 'acquire_data'], ['RES_CMD', 'acquire_data_by_request'], ['RES_CMD', 'acquire_new_data'], ['RES_CMD', 'close'], ['RES_CMD', 'compare'], ['RES_CMD', 'get_attributes'], ['RES_CMD', 'get_fingerprint'], ['RES_CMD', 'get_status'], ['RES_CMD', 'has_new_data']] self.assertEquals(caps, lst) caps = self._ncom_agt_cli.get_capabilities(capability_types=['RES_CMD']) log.debug("resource commands: %s" % caps) lst=[['RES_CMD', 'acquire_data'], ['RES_CMD', 'acquire_data_by_request'], ['RES_CMD', 'acquire_new_data'], ['RES_CMD', 'close'], ['RES_CMD', 'compare'], ['RES_CMD', 'get_attributes'], ['RES_CMD', 'get_fingerprint'], ['RES_CMD', 'get_status'], ['RES_CMD', 'has_new_data']] self.assertEquals(caps, lst) caps = self._ncom_agt_cli.get_capabilities(capability_types=['RES_PAR']) log.debug("resource commands: %s" % caps) self.assertEqual(type(caps), list) caps = self._ncom_agt_cli.get_capabilities(capability_types=['AGT_CMD']) log.debug("resource commands: %s" % caps) self.assertEqual(type(caps), list) caps = self._ncom_agt_cli.get_capabilities(capability_types=['AGT_PAR']) log.debug("resource commands: %s" % caps) self.assertEqual(type(caps), list) @unittest.skip("Currently broken due to resource/agent refactorings") def test_execute_get_attrs(self): cmd = AgentCommand(command_id="111", command="get_attributes") log.debug("Execute AgentCommand: %s" % cmd) ret = self._ncom_agt_cli.execute(cmd) log.debug("Returned: %s" % ret) self.assertEquals(ret.status, 0) self.assertTrue(type(ret.result), dict) @unittest.skip("Currently broken due to resource/agent refactorings") def test_execute_get_fingerprint(self): cmd = AgentCommand(command_id="111", command="get_fingerprint") log.debug("Execute AgentCommand: %s" % cmd) ret = self._ncom_agt_cli.execute(cmd) log.debug("Returned: %s" % ret) self.assertEquals(ret.status, 0) self.assertTrue(type(ret.result), dict) @unittest.skip("Currently broken due to resource/agent refactorings") def test_execute_single_worker(self): cmd = AgentCommand(command_id="111", command="get_attributes") log.debug("Execute AgentCommand: %s" % cmd) ret = self._ncom_agt_cli.execute(cmd) log.debug("Returned: %s" % ret) self.assertEquals(ret.status, 0) self.assertTrue(type(ret.result), dict) cmd = AgentCommand(command_id="112", command="get_fingerprint") log.debug("Execute AgentCommand: %s" % cmd) ret = self._ncom_agt_cli.execute(cmd) log.debug("Returned: %s" % ret) self.assertEquals(ret.status, 0) self.assertTrue(type(ret.result), dict) @unittest.skip("Currently broken due to resource/agent refactorings") def test_execute_multi_worker(self): cmd = AgentCommand(command_id="111", command="has_new_data") log.debug("Execute AgentCommand: %s" % cmd) ret = self._ncom_agt_cli.execute(cmd) log.debug("Returned: %s" % ret) self.assertEquals(ret.status, 0) self.assertTrue(type(ret.result), dict) cmd = AgentCommand(command_id="111", command="has_new_data") log.debug("Execute AgentCommand: %s" % cmd) ret = self._hfr_agt_cli.execute(cmd) log.debug("Returned: %s" % ret) self.assertEquals(ret.status, 0) self.assertTrue(type(ret.result), dict) cmd = AgentCommand(command_id="112", command="get_fingerprint") log.debug("Execute AgentCommand: %s" % cmd) ret = self._ncom_agt_cli.execute(cmd) log.debug("Returned: %s" % ret) self.assertEquals(ret.status, 0) self.assertTrue(type(ret.result), dict) cmd = AgentCommand(command_id="112", command="get_fingerprint") log.debug("Execute AgentCommand: %s" % cmd) ret = self._hfr_agt_cli.execute(cmd) log.debug("Returned: %s" % ret) self.assertEquals(ret.status, 0) self.assertTrue(type(ret.result), dict) @unittest.skip("Currently broken due to resource/agent refactorings") def test_execute_acquire_data(self): cmd = AgentCommand(command_id="113", command="acquire_data") log.debug("Execute AgentCommand: %s" % cmd) ret = self._ncom_agt_cli.execute(cmd) log.debug("Returned: %s" % ret) self.assertEquals(ret.status, 0) @unittest.skip("Currently broken due to resource/agent refactorings") def test_execute_acquire_new_data(self): cmd = AgentCommand(command_id="113", command="acquire_new_data") log.debug("Execute AgentCommand: %s" % cmd) ret = self._ncom_agt_cli.execute(cmd) log.debug("Returned: %s" % ret) self.assertEquals(ret.status, 0) @unittest.skip("Underlying method not yet implemented") def test_set_param(self): res_id = self.ncom_ds_id log.debug("test_get_worker with res_id: %s" % res_id) res = self.eoas_cli.get_worker(res_id) with self.assertRaises(IonException): self.eoas_cli.set_param(resource_id=res_id, name="param", value="value") @unittest.skip("Underlying method not yet implemented") def test_get_param(self): res_id = self.ncom_ds_id log.debug("test_get_worker with res_id: %s" % res_id) res = self.eoas_cli.get_worker(res_id) with self.assertRaises(IonException): self.eoas_cli.get_param(resource_id=res_id, name="param") @unittest.skip("Underlying method not yet implemented") def test_execute_agent(self): res_id = self.ncom_ds_id log.debug("test_get_worker with res_id: %s" % res_id) res = self.eoas_cli.get_worker(res_id) with self.assertRaises(IonException): self.eoas_cli.execute_agent(resource_id=res_id) @unittest.skip("Underlying method not yet implemented") def test_set_agent_param(self): res_id = self.ncom_ds_id log.debug("test_get_worker with res_id: %s" % res_id) res = self.eoas_cli.get_worker(res_id) with self.assertRaises(IonException): self.eoas_cli.set_agent_param(resource_id=res_id, name="param", value="value") @unittest.skip("Underlying method not yet implemented") def test_get_agent_param(self): res_id = self.ncom_ds_id log.debug("test_get_worker with res_id: %s" % res_id) res = self.eoas_cli.get_worker(res_id) with self.assertRaises(IonException): self.eoas_cli.get_agent_param(resource_id=res_id, name="param")
def _setup_resources(self): # TODO: some or all of this (or some variation) should move to DAMS' # Build the test resources for the dataset dms_cli = DatasetManagementServiceClient() dams_cli = DataAcquisitionManagementServiceClient() dpms_cli = DataProductManagementServiceClient() rr_cli = ResourceRegistryServiceClient() pubsub_cli = PubsubManagementServiceClient() eda = ExternalDatasetAgent(name='example data agent', handler_module=self.DVR_CONFIG['dvr_mod'], handler_class=self.DVR_CONFIG['dvr_cls']) eda_id = dams_cli.create_external_dataset_agent(eda) eda_inst = ExternalDatasetAgentInstance( name='example dataset agent instance') eda_inst_id = dams_cli.create_external_dataset_agent_instance( eda_inst, external_dataset_agent_id=eda_id) # Create and register the necessary resources/objects # Create DataProvider dprov = ExternalDataProvider(name='example data provider', institution=Institution(), contact=ContactInformation()) dprov.contact.individual_names_given = 'Christopher Mueller' dprov.contact.email = '*****@*****.**' # Create DataSource dsrc = DataSource(name='example datasource', protocol_type='FILE', institution=Institution(), contact=ContactInformation()) dsrc.connection_params['base_data_url'] = '' dsrc.contact.individual_names_given = 'Tim Giguere' dsrc.contact.email = '*****@*****.**' # Create ExternalDataset ds_name = 'ruv_test_dataset' dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) dset.dataset_description.parameters['base_url'] = 'test_data/ruv/' dset.dataset_description.parameters[ 'list_pattern'] = 'RDLi_SEAB_2011_08_24_1600.ruv' dset.dataset_description.parameters['date_pattern'] = '%Y %m %d %H %M' dset.dataset_description.parameters[ 'date_extraction_pattern'] = 'RDLi_SEAB_([\d]{4})_([\d]{2})_([\d]{2})_([\d]{2})([\d]{2}).ruv' dset.dataset_description.parameters['temporal_dimension'] = None dset.dataset_description.parameters['zonal_dimension'] = None dset.dataset_description.parameters['meridional_dimension'] = None dset.dataset_description.parameters['vertical_dimension'] = None dset.dataset_description.parameters['variables'] = [] # Create DataSourceModel dsrc_model = DataSourceModel(name='ruv_model') #dsrc_model.model = 'RUV' dsrc_model.data_handler_module = 'N/A' dsrc_model.data_handler_class = 'N/A' ## Run everything through DAMS ds_id = dams_cli.create_external_dataset(external_dataset=dset) ext_dprov_id = dams_cli.create_external_data_provider( external_data_provider=dprov) ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc) ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model) # Register the ExternalDataset dproducer_id = dams_cli.register_external_data_set( external_dataset_id=ds_id) # Or using each method dams_cli.assign_data_source_to_external_data_provider( data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id) dams_cli.assign_data_source_to_data_model( data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id) dams_cli.assign_external_dataset_to_data_source( external_dataset_id=ds_id, data_source_id=ext_dsrc_id) dams_cli.assign_external_dataset_to_agent_instance( external_dataset_id=ds_id, agent_instance_id=eda_inst_id) pdict = ParameterDictionary() t_ctxt = ParameterContext( 'data', param_type=QuantityType(value_encoding=numpy.dtype('int64'))) t_ctxt.axis = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 01-01-1970' pdict.add_context(t_ctxt) #create temp streamdef so the data product can create the stream pc_list = [] for pc_k, pc in pdict.iteritems(): pc_list.append(dms_cli.create_parameter_context( pc_k, pc[1].dump())) pdict_id = dms_cli.create_parameter_dictionary('ruv_param_dict', pc_list) streamdef_id = pubsub_cli.create_stream_definition( name="ruv", description="stream def for ruv testing", parameter_dictionary_id=pdict_id) dprod = IonObject(RT.DataProduct, name='ruv_parsed_product', description='parsed ruv product') # Generate the data product and associate it to the ExternalDataset dproduct_id = dpms_cli.create_data_product( data_product=dprod, stream_definition_id=streamdef_id) dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id) stream_id, assn = rr_cli.find_objects(subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True) stream_id = stream_id[0] log.info('Created resources: {0}'.format({ 'ExternalDataset': ds_id, 'ExternalDataProvider': ext_dprov_id, 'DataSource': ext_dsrc_id, 'DataSourceModel': ext_dsrc_model_id, 'DataProducer': dproducer_id, 'DataProduct': dproduct_id, 'Stream': stream_id })) #CBM: Eventually, probably want to group this crap somehow - not sure how yet... # Create the logger for receiving publications _, stream_route, _ = self.create_stream_and_logger(name='ruv', stream_id=stream_id) self.EDA_RESOURCE_ID = ds_id self.EDA_NAME = ds_name self.DVR_CONFIG['dh_cfg'] = { 'TESTING': True, 'stream_id': stream_id, 'stream_route': stream_route, 'external_dataset_res': dset, 'param_dictionary': pdict.dump(), 'data_producer_id': dproducer_id, # CBM: Should this be put in the main body of the config - with mod & cls? 'max_records': 20, }
class TestIntDataAcquisitionManagementService(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataAcquisitionManagementService self.client = DataAcquisitionManagementServiceClient(node=self.container.node) self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) self.dataprocessclient = DataProcessManagementServiceClient(node=self.container.node) def tearDown(self): pass #@unittest.skip('Not done yet.') def test_data_source_ops(self): # test creating a new data source print 'Creating new data source' datasource_obj = IonObject(RT.DataSource, name='DataSource1', description='instrument based new source' , type='sbe37') try: ds_id = self.client.create_data_source(datasource_obj) except BadRequest as ex: self.fail("failed to create new data source: %s" %ex) print 'new data source id = ', ds_id # test reading a non-existent data source print 'reading non-existent data source' try: dp_obj = self.client.read_data_source('some_fake_id') except NotFound as ex: pass else: self.fail("non-existing data source was found during read: %s" %dp_obj) # update a data source (tests read also) print 'Updating data source' # first get the existing data source object try: datasource_obj = self.client.read_data_source(ds_id) except NotFound as ex: self.fail("existing data source was not found during read") else: pass # now tweak the object datasource_obj.description = 'the very first data source' # now write the dp back to the registry try: update_result = self.client.update_data_source(datasource_obj) except NotFound as ex: self.fail("existing data source was not found during update") except Conflict as ex: self.fail("revision conflict exception during data source update") #else: # self.assertTrue(update_result == True) # now get the data source back to see if it was updated try: datasource_obj = self.client.read_data_source(ds_id) except NotFound as ex: self.fail("existing data source was not found during read") else: pass self.assertTrue(datasource_obj.description == 'the very first data source') # now 'delete' the data source print "deleting data source" try: delete_result = self.client.force_delete_data_source(ds_id) except NotFound as ex: self.fail("existing data source was not found during delete") #self.assertTrue(delete_result == True) # now try to get the deleted dp object try: dp_obj = self.client.read_data_source(ds_id) except NotFound as ex: pass else: self.fail("deleted data source was found during read") # now try to delete the already deleted data source object print "deleting non-existing data source" try: delete_result = self.client.delete_data_source(ds_id) except NotFound as ex: pass else: self.fail("non-existing data source was found during delete") #@unittest.skip('Not done yet.') def test_register_instrument(self): # Register an instrument as a data producer in coordination with DM PubSub: create stream, register and create producer object # set up initial instrument to register instrument_obj = IonObject(RT.InstrumentDevice, name='Inst1',description='an instrument that is creating the data product') instrument_id, rev = self.rrclient.create(instrument_obj) dataproduct_obj = IonObject(RT.DataProduct, name='DataProduct1',description='sample data product') dataproduct_id, rev = self.rrclient.create(dataproduct_obj) # test registering a new data producer try: ds_id = self.client.register_instrument(instrument_id) except BadRequest as ex: self.fail("failed to create new data producer: %s" %ex) print 'new data producer id = ', ds_id # test assigning a data product to an instrument, creating the stream for the product try: self.client.assign_data_product(instrument_id, dataproduct_id) except BadRequest as ex: self.fail("failed to assign data product to data producer: %s" %ex) except NotFound as ex: self.fail("failed to assign data product to data producer: %s" %ex) # test UNassigning a data product from instrument, deleting the stream for the product try: self.client.unassign_data_product(instrument_id, dataproduct_id) except BadRequest as ex: self.fail("failed to failed to UNassign data product to data producer data producer: %s" %ex) except NotFound as ex: self.fail("failed to failed to UNassign data product to data producer data producer: %s" %ex) # test UNregistering a new data producer try: ds_id = self.client.unregister_instrument(instrument_id) except NotFound as ex: self.fail("failed to unregister instrument producer: %s" %ex) def test_register_external_data_set(self): # Register an external data set as a data producer in coordination with DM PubSub: create stream, register and create producer object # set up initial instrument to register ext_dataset_obj = IonObject(RT.ExternalDataset, name='DataSet1',description='an external data feed') ext_dataset_id, rev = self.rrclient.create(ext_dataset_obj) dataproduct_obj = IonObject(RT.DataProduct, name='DataProduct1',description='sample data product') dataproduct_id, rev = self.rrclient.create(dataproduct_obj) # test registering a new external data set try: ds_id = self.client.register_external_data_set(ext_dataset_id) except BadRequest as ex: self.fail("failed to create new data producer: %s" %ex) print 'new data producer id = ', ds_id # test assigning a data product to an ext_dataset_id, creating the stream for the product try: self.client.assign_data_product(ext_dataset_id, dataproduct_id) except BadRequest as ex: self.fail("failed to assign data product to data producer: %s" %ex) except NotFound as ex: self.fail("failed to assign data product to data producer: %s" %ex) # test UNassigning a data product from ext_dataset_id, deleting the stream for the product try: self.client.unassign_data_product(ext_dataset_id, dataproduct_id) except BadRequest as ex: self.fail("failed to failed to UNassign data product to data producer data producer: %s" %ex) except NotFound as ex: self.fail("failed to failed to UNassign data product to data producer data producer: %s" %ex) # test UNregistering a external data set try: ds_id = self.client.unregister_external_data_set(ext_dataset_id) except NotFound as ex: self.fail("failed to unregister instrument producer: %s" %ex) #@unittest.skip('not ready') def test_eoi_resources(self): # # test creating a new data provider # print 'Creating new external_data_provider' dataprovider_obj = IonObject(RT.ExternalDataProvider, name='ExtDataProvider1', description='external data provider ') try: dataprovider_id = self.client.create_external_data_provider(dataprovider_obj) except BadRequest as ex: self.fail("failed to create new data provider: %s" %ex) print 'new data provider id = ', dataprovider_id # # test creating a new data source # print 'Creating new data source' datasource_obj = IonObject(RT.DataSource, name='DataSource1', description='data source ', type='DAP') try: datasource_id = self.client.create_data_source(datasource_obj) except BadRequest as ex: self.fail("failed to create new data source: %s" %ex) print 'new data source id = ', datasource_id # # test creating a new data source model # print 'Creating new data source model' datamodel_obj = IonObject(RT.DataSourceModel, name='DataSourceModel1', description='data source model') try: datamodel_id = self.client.create_data_source_model(datamodel_obj) except BadRequest as ex: self.fail("failed to create new data source model: %s" %ex) print 'new data source model id = ', datamodel_id # # test creating a new external data set # print 'Creating new external data set' dataset_obj = IonObject(RT.ExternalDataset, name='ExternalDataSet1', description='external data set ') try: extdataset_id = self.client.create_external_dataset(dataset_obj) except BadRequest as ex: self.fail("failed to create new external data set: %s" %ex) print 'new external data set id = ', extdataset_id # # test creating a new dataset agent instance # print 'Creating new external data agent ' datasetagent_obj = IonObject(RT.ExternalDatasetAgent, name='ExternalDatasetAgent1', description='external data agent ', handler_module = 'module_name', handler_class = 'class_name') try: datasetagent_id = self.client.create_external_dataset_agent(datasetagent_obj) except BadRequest as ex: self.fail("failed to create new external dataset agent: %s" %ex) print 'new external data agent id = ', datasetagent_id # # test creating a new datasource agent instance # print 'Creating new data source agent ' datasourceagent_obj = IonObject(RT.DataSourceAgent, name='DataSourceAgent1', description=' DataSource agent ') try: datasource_agent_id = self.client.create_data_source_agent(datasourceagent_obj) except BadRequest as ex: self.fail("failed to create new external datasource agent: %s" %ex) print 'new external data agent id = ', datasource_agent_id # # test creating a new dataset agent instance # print 'Creating new external dataset agent instance' datasetagentinstance_obj = IonObject(RT.ExternalDatasetAgentInstance, name='ExternalDatasetAgentInstance1', description='external dataset agent instance ') try: datasetagentinstance_id = self.client.create_external_dataset_agent_instance(datasetagentinstance_obj, datasetagent_id) except BadRequest as ex: self.fail("failed to create new external dataset agent instance: %s" %ex) print 'new external data agent instance id = ', datasetagentinstance_id # # test creating a new datasource agent instance # print 'Creating new data source agent ' datasourceagentinstance_obj = IonObject(RT.DataSourceAgentInstance, name='ExternalDataSourceAgentInstance1', description='external DataSource agent instance ') try: datasource_agent_instance_id = self.client.create_data_source_agent_instance(datasourceagentinstance_obj) except BadRequest as ex: self.fail("failed to create new external datasource agent instance: %s" %ex) print 'new external data agent id = ', datasource_agent_instance_id # # test assign / unassign # self.client.unassign_data_source_from_external_data_provider(datasource_id, dataprovider_id) self.client.unassign_data_source_from_data_model(datasource_id, datamodel_id) self.client.unassign_external_dataset_from_data_source(extdataset_id, datasource_id) # # test read # try: dp_obj = self.client.read_external_data_provider(dataprovider_id) except NotFound as ex: self.fail("existing data provicer was not found during read") else: pass try: dp_obj = self.client.read_data_source(datasource_id) except NotFound as ex: self.fail("existing data source was not found during read") else: pass # # test delete # try: self.client.delete_external_data_provider(dataprovider_id) self.client.delete_data_source(datasource_id) self.client.delete_external_dataset(extdataset_id) self.client.delete_data_source_model(datamodel_id) self.client.delete_external_dataset_agent(datasetagent_id) self.client.delete_data_source_agent(datasource_agent_instance_id) self.client.force_delete_external_data_provider(dataprovider_id) self.client.force_delete_data_source(datasource_id) self.client.force_delete_external_dataset(extdataset_id) self.client.force_delete_data_source_model(datamodel_id) self.client.force_delete_external_dataset_agent(datasetagent_id) self.client.force_delete_data_source_agent(datasource_agent_instance_id) except NotFound as ex: self.fail("existing data product was not found during delete") # test reading a non-existent data product print 'reading non-existent data product' try: bad_obj = self.client.read_external_data_provider('some_fake_id') except NotFound as ex: pass else: self.fail("non-existing data product was found during read: %s" %bad_obj)
class TestIntExternalObservatoryAgent(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url(rel_url='res/deploy/r2eoi.yml') self.dams_cli = DataAcquisitionManagementServiceClient() self.dpms_cli = DataProductManagementServiceClient() eda = ExternalDatasetAgent() self.eda_id = self.dams_cli.create_external_dataset_agent(eda) eda_inst = ExternalDatasetAgentInstance() self.eda_inst_id = self.dams_cli.create_external_dataset_agent_instance( eda_inst, external_dataset_agent_id=self.eda_id) self._setup_ncom() proc_name = self.ncom_ds_id + '_worker' config = {} config['process'] = {'name': proc_name, 'type': 'agent'} config['process']['eoa'] = {'dataset_id': self.ncom_ds_id} pid = self.container.spawn_process( name=proc_name, module='eoi.agent.external_observatory_agent', cls='ExternalObservatoryAgent', config=config) queue_id = "%s.%s" % (self.container.id, pid) log.debug( "Spawned worker process ==> proc_name: %s\tproc_id: %s\tqueue_id: %s" % (proc_name, pid, queue_id)) self._agent_cli = ResourceAgentClient(self.ncom_ds_id, name=pid, process=FakeProcess()) log.debug("Got a ResourceAgentClient: res_id=%s" % self._agent_cli.resource_id) def _setup_ncom(self): # TODO: some or all of this (or some variation) should move to DAMS # Create and register the necessary resources/objects # Create DataProvider dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation()) # dprov.institution.name = "OOI CGSN" dprov.contact.name = "Robert Weller" dprov.contact.email = "*****@*****.**" # Create DataSource dsrc = DataSource(protocol_type="DAP", institution=Institution(), contact=ContactInformation()) # dsrc.connection_params["base_data_url"] = "http://ooi.whoi.edu/thredds/dodsC/" dsrc.connection_params["base_data_url"] = "" dsrc.contact.name = "Rich Signell" dsrc.contact.email = "*****@*****.**" # Create ExternalDataset dset = ExternalDataset(name="test", dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) # dset.dataset_description.parameters["dataset_path"] = "ooi/AS02CPSM_R_M.nc" dset.dataset_description.parameters[ "dataset_path"] = "test_data/ncom.nc" dset.dataset_description.parameters["temporal_dimension"] = "time" dset.dataset_description.parameters["zonal_dimension"] = "lon" dset.dataset_description.parameters["meridional_dimension"] = "lat" # Create DataSourceModel dsrc_model = DataSourceModel(name="dap_model") dsrc_model.model = "DAP" dsrc_model.data_handler_module = "eoi.agent.handler.dap_external_data_handler" dsrc_model.data_handler_class = "DapExternalDataHandler" ## Run everything through DAMS ds_id = self.ncom_ds_id = self.dams_cli.create_external_dataset( external_dataset=dset) ext_dprov_id = self.dams_cli.create_external_data_provider( external_data_provider=dprov) ext_dsrc_id = self.dams_cli.create_data_source(data_source=dsrc) ext_dsrc_model_id = self.dams_cli.create_data_source_model(dsrc_model) # Register the ExternalDataset dproducer_id = self.dams_cli.register_external_data_set( external_dataset_id=ds_id) ## Associate everything # Convenience method # self.dams_cli.assign_eoi_resources(external_data_provider_id=ext_dprov_id, data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id, external_dataset_id=ds_id, external_data_agent_id=self.eda_id, agent_instance_id=self.eda_inst_id) # Or using each method self.dams_cli.assign_data_source_to_external_data_provider( data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id) self.dams_cli.assign_data_source_to_data_model( data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id) self.dams_cli.assign_external_dataset_to_data_source( external_dataset_id=ds_id, data_source_id=ext_dsrc_id) self.dams_cli.assign_external_dataset_to_agent_instance( external_dataset_id=ds_id, agent_instance_id=self.eda_inst_id) # self.dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=self.eda_id, agent_instance_id=self.eda_inst_id) # Generate the data product and associate it to the ExternalDataset dprod = DataProduct(name='ncom_product', description='raw ncom product') dproduct_id = self.dpms_cli.create_data_product(data_product=dprod) self.dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id, create_stream=True) ########## Tests ########## # @unittest.skip("Currently broken due to resource/agent refactorings") def test_get_capabilities(self): # Get all the capabilities caps = self._agent_cli.get_capabilities() log.debug("all capabilities: %s" % caps) lst = [['RES_CMD', 'acquire_data'], ['RES_CMD', 'acquire_data_by_request'], ['RES_CMD', 'acquire_new_data'], ['RES_CMD', 'close'], ['RES_CMD', 'compare'], ['RES_CMD', 'get_attributes'], ['RES_CMD', 'get_fingerprint'], ['RES_CMD', 'get_status'], ['RES_CMD', 'has_new_data']] self.assertEquals(caps, lst) caps = self._agent_cli.get_capabilities(capability_types=['RES_CMD']) log.debug("resource commands: %s" % caps) lst = [['RES_CMD', 'acquire_data'], ['RES_CMD', 'acquire_data_by_request'], ['RES_CMD', 'acquire_new_data'], ['RES_CMD', 'close'], ['RES_CMD', 'compare'], ['RES_CMD', 'get_attributes'], ['RES_CMD', 'get_fingerprint'], ['RES_CMD', 'get_status'], ['RES_CMD', 'has_new_data']] self.assertEquals(caps, lst) caps = self._agent_cli.get_capabilities(capability_types=['RES_PAR']) log.debug("resource commands: %s" % caps) self.assertEqual(type(caps), list) caps = self._agent_cli.get_capabilities(capability_types=['AGT_CMD']) log.debug("resource commands: %s" % caps) self.assertEqual(type(caps), list) caps = self._agent_cli.get_capabilities(capability_types=['AGT_PAR']) log.debug("resource commands: %s" % caps) self.assertEqual(type(caps), list) @unittest.skip("Currently broken due to resource/agent refactorings") def test_execute_get_attrs(self): cmd = AgentCommand(command_id="111", command="get_attributes") log.debug("Execute AgentCommand: %s" % cmd) ret = self._agent_cli.execute(cmd) log.debug("Returned: %s" % ret) self.assertEquals(ret.status, 0) self.assertTrue(type(ret.result), dict)
class TestIntExternalObservatoryAgentService(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url(rel_url='res/deploy/r2eoi.yml') # self.eoas_cli = ExternalObservatoryAgentServiceClient() # self.rr_cli = ResourceRegistryServiceClient() self.dams_cli = DataAcquisitionManagementServiceClient() self.dpms_cli = DataProductManagementServiceClient() self._setup_ncom() self._setup_hfr() # eoas_proc = self.container.proc_manager.procs_by_name['external_data_agent_management'] # log.debug("Got EOAS Process: %s" % eoas_proc) self._ncom_agt_cli = ResourceAgentClient( resource_id=self.ncom_ds_id, name='external_observatory_agent', process=FakeProcess()) log.debug("Got a ResourceAgentClient: res_id=%s" % self._ncom_agt_cli.resource_id) self._hfr_agt_cli = ResourceAgentClient( resource_id=self.hfr_ds_id, name='external_observatory_agent', process=FakeProcess()) log.debug("Got a ResourceAgentClient: res_id=%s" % self._hfr_agt_cli.resource_id) def _setup_ncom(self): # TODO: some or all of this (or some variation) should move to DAMS # Create and register the necessary resources/objects eda = ExternalDatasetAgent() eda_id = self.dams_cli.create_external_dataset_agent(eda) eda_inst = ExternalDatasetAgentInstance() eda_inst_id = self.dams_cli.create_external_dataset_agent_instance( eda_inst, external_dataset_agent_id=eda_id) # Create DataProvider dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation()) # dprov.institution.name = "OOI CGSN" dprov.contact.name = "Robert Weller" dprov.contact.email = "*****@*****.**" # Create DataSource dsrc = DataSource(protocol_type="DAP", institution=Institution(), contact=ContactInformation()) # dsrc.connection_params["base_data_url"] = "http://ooi.whoi.edu/thredds/dodsC/" dsrc.connection_params["base_data_url"] = "" dsrc.contact.name = "Rich Signell" dsrc.contact.email = "*****@*****.**" # Create ExternalDataset dset = ExternalDataset(name="test", dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) # dset.dataset_description.parameters["dataset_path"] = "ooi/AS02CPSM_R_M.nc" dset.dataset_description.parameters[ "dataset_path"] = "test_data/ncom.nc" dset.dataset_description.parameters["temporal_dimension"] = "time" dset.dataset_description.parameters["zonal_dimension"] = "lon" dset.dataset_description.parameters["meridional_dimension"] = "lat" # Create DataSourceModel dsrc_model = DataSourceModel(name="dap_model") dsrc_model.model = "DAP" dsrc_model.data_handler_module = "eoi.agent.handler.dap_external_data_handler" dsrc_model.data_handler_class = "DapExternalDataHandler" ## Run everything through DAMS ds_id = self.ncom_ds_id = self.dams_cli.create_external_dataset( external_dataset=dset) ext_dprov_id = self.dams_cli.create_external_data_provider( external_data_provider=dprov) ext_dsrc_id = self.dams_cli.create_data_source(data_source=dsrc) ext_dsrc_model_id = self.dams_cli.create_data_source_model(dsrc_model) # Register the ExternalDataset dproducer_id = self.dams_cli.register_external_data_set( external_dataset_id=ds_id) ## Associate everything # Convenience method # self.dams_cli.assign_eoi_resources(external_data_provider_id=ext_dprov_id, data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id, external_dataset_id=ds_id, external_data_agent_id=eda_id, agent_instance_id=eda_inst_id) # Or using each method self.dams_cli.assign_data_source_to_external_data_provider( data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id) self.dams_cli.assign_data_source_to_data_model( data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id) self.dams_cli.assign_external_dataset_to_data_source( external_dataset_id=ds_id, data_source_id=ext_dsrc_id) self.dams_cli.assign_external_dataset_to_agent_instance( external_dataset_id=ds_id, agent_instance_id=eda_inst_id) # self.dams_cli.assign_external_dataset_agent_to_data_model(external_data_agent_id=eda_id, data_source_model_id=ext_dsrc_model_id) # self.dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=eda_id, agent_instance_id=eda_inst_id) # Generate the data product and associate it to the ExternalDataset dprod = DataProduct(name='ncom_product', description='raw ncom product') dproduct_id = self.dpms_cli.create_data_product(data_product=dprod) self.dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id, create_stream=True) def _setup_hfr(self): # TODO: some or all of this (or some variation) should move to DAMS # Create and register the necessary resources/objects eda = ExternalDatasetAgent() eda_id = self.dams_cli.create_external_dataset_agent(eda) eda_inst = ExternalDatasetAgentInstance() eda_inst_id = self.dams_cli.create_external_dataset_agent_instance( eda_inst, external_dataset_agent_id=eda_id) # Create DataProvider dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation()) # dprov.institution.name = "HFR UCSD" # Create DataSource dsrc = DataSource(protocol_type="DAP", institution=Institution(), contact=ContactInformation()) dsrc.connection_params[ "base_data_url"] = "http://hfrnet.ucsd.edu:8080/thredds/dodsC/" # Create ExternalDataset dset = ExternalDataset(name="UCSD HFR", dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) dset.dataset_description.parameters[ "dataset_path"] = "HFRNet/USEGC/6km/hourly/RTV" # dset.dataset_description.parameters["dataset_path"] = "test_data/hfr.nc" dset.dataset_description.parameters["temporal_dimension"] = "time" dset.dataset_description.parameters["zonal_dimension"] = "lon" dset.dataset_description.parameters["meridional_dimension"] = "lat" # Create DataSourceModel dsrc_model = DataSourceModel(name="dap_model") dsrc_model.model = "DAP" dsrc_model.data_handler_module = "eoi.agent.handler.dap_external_data_handler" dsrc_model.data_handler_class = "DapExternalDataHandler" ## Run everything through DAMS ds_id = self.hfr_ds_id = self.dams_cli.create_external_dataset( external_dataset=dset) ext_dprov_id = self.dams_cli.create_external_data_provider( external_data_provider=dprov) ext_dsrc_id = self.dams_cli.create_data_source(data_source=dsrc) ext_dsrc_model_id = self.dams_cli.create_data_source_model(dsrc_model) # Register the ExternalDataset dproducer_id = self.dams_cli.register_external_data_set( external_dataset_id=ds_id) ## Associate everything # Convenience method # self.dams_cli.assign_eoi_resources(external_data_provider_id=ext_dprov_id, data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id, external_dataset_id=ds_id, external_data_agent_id=eda_id, agent_instance_id=eda_inst_id) # Or using each method self.dams_cli.assign_data_source_to_external_data_provider( data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id) self.dams_cli.assign_data_source_to_data_model( data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id) self.dams_cli.assign_external_dataset_to_data_source( external_dataset_id=ds_id, data_source_id=ext_dsrc_id) self.dams_cli.assign_external_dataset_to_agent_instance( external_dataset_id=ds_id, agent_instance_id=eda_inst_id) # self.dams_cli.assign_external_dataset_agent_to_data_model(external_data_agent_id=eda_id, data_source_model_id=ext_dsrc_model_id) # self.dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=eda_id, agent_instance_id=eda_inst_id) # Generate the data product and associate it to the ExternalDataset dprod = DataProduct(name='hfr_product', description='raw hfr product') dproduct_id = self.dpms_cli.create_data_product(data_product=dprod) self.dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id, create_stream=True) ########## Tests ########## @unittest.skip("Currently broken due to resource/agent refactorings") def test_get_capabilities(self): # Get all the capabilities caps = self._ncom_agt_cli.get_capabilities() log.debug("all capabilities: %s" % caps) lst = [['RES_CMD', 'acquire_data'], ['RES_CMD', 'acquire_data_by_request'], ['RES_CMD', 'acquire_new_data'], ['RES_CMD', 'close'], ['RES_CMD', 'compare'], ['RES_CMD', 'get_attributes'], ['RES_CMD', 'get_fingerprint'], ['RES_CMD', 'get_status'], ['RES_CMD', 'has_new_data']] self.assertEquals(caps, lst) caps = self._ncom_agt_cli.get_capabilities( capability_types=['RES_CMD']) log.debug("resource commands: %s" % caps) lst = [['RES_CMD', 'acquire_data'], ['RES_CMD', 'acquire_data_by_request'], ['RES_CMD', 'acquire_new_data'], ['RES_CMD', 'close'], ['RES_CMD', 'compare'], ['RES_CMD', 'get_attributes'], ['RES_CMD', 'get_fingerprint'], ['RES_CMD', 'get_status'], ['RES_CMD', 'has_new_data']] self.assertEquals(caps, lst) caps = self._ncom_agt_cli.get_capabilities( capability_types=['RES_PAR']) log.debug("resource commands: %s" % caps) self.assertEqual(type(caps), list) caps = self._ncom_agt_cli.get_capabilities( capability_types=['AGT_CMD']) log.debug("resource commands: %s" % caps) self.assertEqual(type(caps), list) caps = self._ncom_agt_cli.get_capabilities( capability_types=['AGT_PAR']) log.debug("resource commands: %s" % caps) self.assertEqual(type(caps), list) @unittest.skip("Currently broken due to resource/agent refactorings") def test_execute_get_attrs(self): cmd = AgentCommand(command_id="111", command="get_attributes") log.debug("Execute AgentCommand: %s" % cmd) ret = self._ncom_agt_cli.execute(cmd) log.debug("Returned: %s" % ret) self.assertEquals(ret.status, 0) self.assertTrue(type(ret.result), dict) @unittest.skip("Currently broken due to resource/agent refactorings") def test_execute_get_fingerprint(self): cmd = AgentCommand(command_id="111", command="get_fingerprint") log.debug("Execute AgentCommand: %s" % cmd) ret = self._ncom_agt_cli.execute(cmd) log.debug("Returned: %s" % ret) self.assertEquals(ret.status, 0) self.assertTrue(type(ret.result), dict) @unittest.skip("Currently broken due to resource/agent refactorings") def test_execute_single_worker(self): cmd = AgentCommand(command_id="111", command="get_attributes") log.debug("Execute AgentCommand: %s" % cmd) ret = self._ncom_agt_cli.execute(cmd) log.debug("Returned: %s" % ret) self.assertEquals(ret.status, 0) self.assertTrue(type(ret.result), dict) cmd = AgentCommand(command_id="112", command="get_fingerprint") log.debug("Execute AgentCommand: %s" % cmd) ret = self._ncom_agt_cli.execute(cmd) log.debug("Returned: %s" % ret) self.assertEquals(ret.status, 0) self.assertTrue(type(ret.result), dict) @unittest.skip("Currently broken due to resource/agent refactorings") def test_execute_multi_worker(self): cmd = AgentCommand(command_id="111", command="has_new_data") log.debug("Execute AgentCommand: %s" % cmd) ret = self._ncom_agt_cli.execute(cmd) log.debug("Returned: %s" % ret) self.assertEquals(ret.status, 0) self.assertTrue(type(ret.result), dict) cmd = AgentCommand(command_id="111", command="has_new_data") log.debug("Execute AgentCommand: %s" % cmd) ret = self._hfr_agt_cli.execute(cmd) log.debug("Returned: %s" % ret) self.assertEquals(ret.status, 0) self.assertTrue(type(ret.result), dict) cmd = AgentCommand(command_id="112", command="get_fingerprint") log.debug("Execute AgentCommand: %s" % cmd) ret = self._ncom_agt_cli.execute(cmd) log.debug("Returned: %s" % ret) self.assertEquals(ret.status, 0) self.assertTrue(type(ret.result), dict) cmd = AgentCommand(command_id="112", command="get_fingerprint") log.debug("Execute AgentCommand: %s" % cmd) ret = self._hfr_agt_cli.execute(cmd) log.debug("Returned: %s" % ret) self.assertEquals(ret.status, 0) self.assertTrue(type(ret.result), dict) @unittest.skip("Currently broken due to resource/agent refactorings") def test_execute_acquire_data(self): cmd = AgentCommand(command_id="113", command="acquire_data") log.debug("Execute AgentCommand: %s" % cmd) ret = self._ncom_agt_cli.execute(cmd) log.debug("Returned: %s" % ret) self.assertEquals(ret.status, 0) @unittest.skip("Currently broken due to resource/agent refactorings") def test_execute_acquire_new_data(self): cmd = AgentCommand(command_id="113", command="acquire_new_data") log.debug("Execute AgentCommand: %s" % cmd) ret = self._ncom_agt_cli.execute(cmd) log.debug("Returned: %s" % ret) self.assertEquals(ret.status, 0) @unittest.skip("Underlying method not yet implemented") def test_set_param(self): res_id = self.ncom_ds_id log.debug("test_get_worker with res_id: %s" % res_id) res = self.eoas_cli.get_worker(res_id) with self.assertRaises(IonException): self.eoas_cli.set_param(resource_id=res_id, name="param", value="value") @unittest.skip("Underlying method not yet implemented") def test_get_param(self): res_id = self.ncom_ds_id log.debug("test_get_worker with res_id: %s" % res_id) res = self.eoas_cli.get_worker(res_id) with self.assertRaises(IonException): self.eoas_cli.get_param(resource_id=res_id, name="param") @unittest.skip("Underlying method not yet implemented") def test_execute_agent(self): res_id = self.ncom_ds_id log.debug("test_get_worker with res_id: %s" % res_id) res = self.eoas_cli.get_worker(res_id) with self.assertRaises(IonException): self.eoas_cli.execute_agent(resource_id=res_id) @unittest.skip("Underlying method not yet implemented") def test_set_agent_param(self): res_id = self.ncom_ds_id log.debug("test_get_worker with res_id: %s" % res_id) res = self.eoas_cli.get_worker(res_id) with self.assertRaises(IonException): self.eoas_cli.set_agent_param(resource_id=res_id, name="param", value="value") @unittest.skip("Underlying method not yet implemented") def test_get_agent_param(self): res_id = self.ncom_ds_id log.debug("test_get_worker with res_id: %s" % res_id) res = self.eoas_cli.get_worker(res_id) with self.assertRaises(IonException): self.eoas_cli.get_agent_param(resource_id=res_id, name="param")
def _setup_resources(self): # TODO: some or all of this (or some variation) should move to DAMS' # Build the test resources for the dataset dams_cli = DataAcquisitionManagementServiceClient() dpms_cli = DataProductManagementServiceClient() rr_cli = ResourceRegistryServiceClient() pubsub_cli = PubsubManagementServiceClient() eda = ExternalDatasetAgent() eda_id = dams_cli.create_external_dataset_agent(eda) eda_inst = ExternalDatasetAgentInstance() eda_inst_id = dams_cli.create_external_dataset_agent_instance(eda_inst, external_dataset_agent_id=eda_id) # Create and register the necessary resources/objects # Create DataProvider dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation()) dprov.contact.name = "Christopher Mueller" dprov.contact.email = "*****@*****.**" # Create DataSource dsrc = DataSource(protocol_type="FILE", institution=Institution(), contact=ContactInformation()) dsrc.connection_params["base_data_url"] = "" dsrc.contact.name = "Tim Giguere" dsrc.contact.email = "*****@*****.**" # Create ExternalDataset ds_name = "slocum_test_dataset" dset = ExternalDataset( name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation(), ) dset.dataset_description.parameters["base_url"] = "test_data/slocum/" dset.dataset_description.parameters["list_pattern"] = "ru05-2012-021-0-0-sbd.dat" dset.dataset_description.parameters["date_pattern"] = "%Y %j" dset.dataset_description.parameters["date_extraction_pattern"] = "ru05-([\d]{4})-([\d]{3})-\d-\d-sbd.dat" dset.dataset_description.parameters["temporal_dimension"] = None dset.dataset_description.parameters["zonal_dimension"] = None dset.dataset_description.parameters["meridional_dimension"] = None dset.dataset_description.parameters["vertical_dimension"] = None dset.dataset_description.parameters["variables"] = [ "c_wpt_y_lmc", "sci_water_cond", "m_y_lmc", "u_hd_fin_ap_inflection_holdoff", "sci_m_present_time", "m_leakdetect_voltage_forward", "sci_bb3slo_b660_scaled", "c_science_send_all", "m_gps_status", "m_water_vx", "m_water_vy", "c_heading", "sci_fl3slo_chlor_units", "u_hd_fin_ap_gain", "m_vacuum", "u_min_water_depth", "m_gps_lat", "m_veh_temp", "f_fin_offset", "u_hd_fin_ap_hardover_holdoff", "c_alt_time", "m_present_time", "m_heading", "sci_bb3slo_b532_scaled", "sci_fl3slo_cdom_units", "m_fin", "x_cycle_overrun_in_ms", "sci_water_pressure", "u_hd_fin_ap_igain", "sci_fl3slo_phyco_units", "m_battpos", "sci_bb3slo_b470_scaled", "m_lat", "m_gps_lon", "sci_ctd41cp_timestamp", "m_pressure", "c_wpt_x_lmc", "c_ballast_pumped", "x_lmc_xy_source", "m_lon", "m_avg_speed", "sci_water_temp", "u_pitch_ap_gain", "m_roll", "m_tot_num_inflections", "m_x_lmc", "u_pitch_ap_deadband", "m_final_water_vy", "m_final_water_vx", "m_water_depth", "m_leakdetect_voltage", "u_pitch_max_delta_battpos", "m_coulomb_amphr", "m_pitch", ] # Create DataSourceModel dsrc_model = DataSourceModel(name="slocum_model") dsrc_model.model = "SLOCUM" dsrc_model.data_handler_module = "N/A" dsrc_model.data_handler_class = "N/A" ## Run everything through DAMS ds_id = dams_cli.create_external_dataset(external_dataset=dset) ext_dprov_id = dams_cli.create_external_data_provider(external_data_provider=dprov) ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc) ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model) # Register the ExternalDataset dproducer_id = dams_cli.register_external_data_set(external_dataset_id=ds_id) # Or using each method dams_cli.assign_data_source_to_external_data_provider( data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id ) dams_cli.assign_data_source_to_data_model(data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id) dams_cli.assign_external_dataset_to_data_source(external_dataset_id=ds_id, data_source_id=ext_dsrc_id) dams_cli.assign_external_dataset_to_agent_instance(external_dataset_id=ds_id, agent_instance_id=eda_inst_id) # dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=self.eda_id, agent_instance_id=self.eda_inst_id) # create temp streamdef so the data product can create the stream streamdef_id = pubsub_cli.create_stream_definition(name="temp", description="temp") # Generate the data product and associate it to the ExternalDataset craft = CoverageCraft sdom, tdom = craft.create_domains() sdom = sdom.dump() tdom = tdom.dump() parameter_dictionary = craft.create_parameters() parameter_dictionary = parameter_dictionary.dump() dprod = IonObject( RT.DataProduct, name="slocum_parsed_product", description="parsed slocum product", temporal_domain=tdom, spatial_domain=sdom, ) dproduct_id = dpms_cli.create_data_product( data_product=dprod, stream_definition_id=streamdef_id, parameter_dictionary=parameter_dictionary ) dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id) stream_id, assn = rr_cli.find_objects( subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True ) stream_id = stream_id[0] log.info( "Created resources: {0}".format( { "ExternalDataset": ds_id, "ExternalDataProvider": ext_dprov_id, "DataSource": ext_dsrc_id, "DataSourceModel": ext_dsrc_model_id, "DataProducer": dproducer_id, "DataProduct": dproduct_id, "Stream": stream_id, } ) ) # CBM: Use CF standard_names # ttool = TaxyTool() # # ttool.add_taxonomy_set('c_wpt_y_lmc'), # ttool.add_taxonomy_set('sci_water_cond'), # ttool.add_taxonomy_set('m_y_lmc'), # ttool.add_taxonomy_set('u_hd_fin_ap_inflection_holdoff'), # ttool.add_taxonomy_set('sci_m_present_time'), # ttool.add_taxonomy_set('m_leakdetect_voltage_forward'), # ttool.add_taxonomy_set('sci_bb3slo_b660_scaled'), # ttool.add_taxonomy_set('c_science_send_all'), # ttool.add_taxonomy_set('m_gps_status'), # ttool.add_taxonomy_set('m_water_vx'), # ttool.add_taxonomy_set('m_water_vy'), # ttool.add_taxonomy_set('c_heading'), # ttool.add_taxonomy_set('sci_fl3slo_chlor_units'), # ttool.add_taxonomy_set('u_hd_fin_ap_gain'), # ttool.add_taxonomy_set('m_vacuum'), # ttool.add_taxonomy_set('u_min_water_depth'), # ttool.add_taxonomy_set('m_gps_lat'), # ttool.add_taxonomy_set('m_veh_temp'), # ttool.add_taxonomy_set('f_fin_offset'), # ttool.add_taxonomy_set('u_hd_fin_ap_hardover_holdoff'), # ttool.add_taxonomy_set('c_alt_time'), # ttool.add_taxonomy_set('m_present_time'), # ttool.add_taxonomy_set('m_heading'), # ttool.add_taxonomy_set('sci_bb3slo_b532_scaled'), # ttool.add_taxonomy_set('sci_fl3slo_cdom_units'), # ttool.add_taxonomy_set('m_fin'), # ttool.add_taxonomy_set('x_cycle_overrun_in_ms'), # ttool.add_taxonomy_set('sci_water_pressure'), # ttool.add_taxonomy_set('u_hd_fin_ap_igain'), # ttool.add_taxonomy_set('sci_fl3slo_phyco_units'), # ttool.add_taxonomy_set('m_battpos'), # ttool.add_taxonomy_set('sci_bb3slo_b470_scaled'), # ttool.add_taxonomy_set('m_lat'), # ttool.add_taxonomy_set('m_gps_lon'), # ttool.add_taxonomy_set('sci_ctd41cp_timestamp'), # ttool.add_taxonomy_set('m_pressure'), # ttool.add_taxonomy_set('c_wpt_x_lmc'), # ttool.add_taxonomy_set('c_ballast_pumped'), # ttool.add_taxonomy_set('x_lmc_xy_source'), # ttool.add_taxonomy_set('m_lon'), # ttool.add_taxonomy_set('m_avg_speed'), # ttool.add_taxonomy_set('sci_water_temp'), # ttool.add_taxonomy_set('u_pitch_ap_gain'), # ttool.add_taxonomy_set('m_roll'), # ttool.add_taxonomy_set('m_tot_num_inflections'), # ttool.add_taxonomy_set('m_x_lmc'), # ttool.add_taxonomy_set('u_pitch_ap_deadband'), # ttool.add_taxonomy_set('m_final_water_vy'), # ttool.add_taxonomy_set('m_final_water_vx'), # ttool.add_taxonomy_set('m_water_depth'), # ttool.add_taxonomy_set('m_leakdetect_voltage'), # ttool.add_taxonomy_set('u_pitch_max_delta_battpos'), # ttool.add_taxonomy_set('m_coulomb_amphr'), # ttool.add_taxonomy_set('m_pitch'), pdict = ParameterDictionary() t_ctxt = ParameterContext("c_wpt_y_lmc", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("sci_water_cond", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_y_lmc", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext( "u_hd_fin_ap_inflection_holdoff", param_type=QuantityType(value_encoding=numpy.dtype("float32")) ) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("sci_m_present_time", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext( "m_leakdetect_voltage_forward", param_type=QuantityType(value_encoding=numpy.dtype("float32")) ) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext( "sci_bb3slo_b660_scaled", param_type=QuantityType(value_encoding=numpy.dtype("float32")) ) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("c_science_send_all", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_gps_status", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_water_vx", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_water_vy", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("c_heading", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext( "sci_fl3slo_chlor_units", param_type=QuantityType(value_encoding=numpy.dtype("float32")) ) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("u_hd_fin_ap_gain", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_vacuum", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("u_min_water_depth", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_gps_lat", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_veh_temp", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("f_fin_offset", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext( "u_hd_fin_ap_hardover_holdoff", param_type=QuantityType(value_encoding=numpy.dtype("float32")) ) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("c_alt_time", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_present_time", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_heading", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext( "sci_bb3slo_b532_scaled", param_type=QuantityType(value_encoding=numpy.dtype("float32")) ) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext( "sci_fl3slo_cdom_units", param_type=QuantityType(value_encoding=numpy.dtype("float32")) ) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_fin", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext( "x_cycle_overrun_in_ms", param_type=QuantityType(value_encoding=numpy.dtype("float32")) ) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("sci_water_pressure", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("u_hd_fin_ap_igain", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext( "sci_fl3slo_phyco_units", param_type=QuantityType(value_encoding=numpy.dtype("float32")) ) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_battpos", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext( "sci_bb3slo_b470_scaled", param_type=QuantityType(value_encoding=numpy.dtype("float32")) ) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_lat", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_gps_lon", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext( "sci_ctd41cp_timestamp", param_type=QuantityType(value_encoding=numpy.dtype("float32")) ) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_pressure", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("c_wpt_x_lmc", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("c_ballast_pumped", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("x_lmc_xy_source", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_lon", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_avg_speed", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("sci_water_temp", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("u_pitch_ap_gain", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_roll", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext( "m_tot_num_inflections", param_type=QuantityType(value_encoding=numpy.dtype("float32")) ) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_x_lmc", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("u_pitch_ap_deadband", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_final_water_vy", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_final_water_vx", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_water_depth", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext( "m_leakdetect_voltage", param_type=QuantityType(value_encoding=numpy.dtype("float32")) ) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext( "u_pitch_max_delta_battpos", param_type=QuantityType(value_encoding=numpy.dtype("float32")) ) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_coulomb_amphr", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) t_ctxt = ParameterContext("m_pitch", param_type=QuantityType(value_encoding=numpy.dtype("float32"))) t_ctxt.uom = "unknown" pdict.add_context(t_ctxt) # CBM: Eventually, probably want to group this crap somehow - not sure how yet... # Create the logger for receiving publications self.create_stream_and_logger(name="slocum", stream_id=stream_id) self.EDA_RESOURCE_ID = ds_id self.EDA_NAME = ds_name self.DVR_CONFIG["dh_cfg"] = { "TESTING": True, "stream_id": stream_id, "external_dataset_res": dset, "param_dictionary": pdict.dump(), "data_producer_id": dproducer_id, # CBM: Should this be put in the main body of the config - with mod & cls? "max_records": 20, }
def _setup_resources(self): # TODO: some or all of this (or some variation) should move to DAMS' # Build the test resources for the dataset dms_cli = DatasetManagementServiceClient() dams_cli = DataAcquisitionManagementServiceClient() dpms_cli = DataProductManagementServiceClient() rr_cli = ResourceRegistryServiceClient() pubsub_cli = PubsubManagementServiceClient() eda = ExternalDatasetAgent(handler_module=self.DVR_CONFIG['dvr_mod'], handler_class=self.DVR_CONFIG['dvr_cls']) eda_id = dams_cli.create_external_dataset_agent(eda) eda_inst = ExternalDatasetAgentInstance() eda_inst_id = dams_cli.create_external_dataset_agent_instance(eda_inst, external_dataset_agent_id=eda_id) # Create and register the necessary resources/objects # Create DataProvider dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation()) dprov.contact.individual_names_given = 'Christopher Mueller' dprov.contact.email = '*****@*****.**' # Create DataSource dsrc = DataSource(protocol_type='DAP', institution=Institution(), contact=ContactInformation()) dsrc.connection_params['base_data_url'] = '' dsrc.contact.individual_names_given = 'Tim Giguere' dsrc.contact.email = '*****@*****.**' # Create ExternalDataset ds_name = 'dummy_dataset' dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) # The usgs.nc test dataset is a download of the R1 dataset found here: # http://thredds-test.oceanobservatories.org/thredds/dodsC/ooiciData/E66B1A74-A684-454A-9ADE-8388C2C634E5.ncml dset.dataset_description.parameters['base_url'] = 'test_data/dummy' dset.dataset_description.parameters['list_pattern'] = 'test*.dum' dset.dataset_description.parameters['date_pattern'] = '%Y %m %d %H' dset.dataset_description.parameters['date_extraction_pattern'] = 'test([\d]{4})-([\d]{2})-([\d]{2})-([\d]{2}).dum' dset.dataset_description.parameters['temporal_dimension'] = 'time' dset.dataset_description.parameters['zonal_dimension'] = 'lon' dset.dataset_description.parameters['meridional_dimension'] = 'lat' dset.dataset_description.parameters['variables'] = [ 'dummy', ] # Create DataSourceModel dsrc_model = DataSourceModel(name='dap_model') #dsrc_model.model = 'DAP' dsrc_model.data_handler_module = 'N/A' dsrc_model.data_handler_class = 'N/A' ## Run everything through DAMS ds_id = dams_cli.create_external_dataset(external_dataset=dset) ext_dprov_id = dams_cli.create_external_data_provider(external_data_provider=dprov) ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc) ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model) # Register the ExternalDataset dproducer_id = dams_cli.register_external_data_set(external_dataset_id=ds_id) # Or using each method dams_cli.assign_data_source_to_external_data_provider(data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id) dams_cli.assign_data_source_to_data_model(data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id) dams_cli.assign_external_dataset_to_data_source(external_dataset_id=ds_id, data_source_id=ext_dsrc_id) dams_cli.assign_external_dataset_to_agent_instance(external_dataset_id=ds_id, agent_instance_id=eda_inst_id) #create temp streamdef so the data product can create the stream pc_list = [] #Get 'time' parameter context pc_list.append(dms_cli.read_parameter_context_by_name('time', id_only=True)) pc_list.append(dms_cli.read_parameter_context_by_name('dummy', id_only=True)) for pc_k, pc in self._create_parameter_dictionary().iteritems(): pc_list.append(dms_cli.create_parameter_context(pc_k, pc[1].dump())) pdict_id = dms_cli.create_parameter_dictionary('dummy_param_dict', pc_list) streamdef_id = pubsub_cli.create_stream_definition(name="dummy", parameter_dictionary_id=pdict_id, description="dummy") tdom, sdom = time_series_domain() tdom, sdom = tdom.dump(), sdom.dump() dprod = DataProduct( name='dummy_dataset', description='dummy data product', temporal_domain=tdom, spatial_domain=sdom) dproduct_id = dpms_cli.create_data_product(data_product=dprod, stream_definition_id=streamdef_id) dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id) # , create_stream=True) stream_id, assn = rr_cli.find_objects(subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True) stream_id = stream_id[0] log.info('Created resources: {0}'.format({'ExternalDataset': ds_id, 'ExternalDataProvider': ext_dprov_id, 'DataSource': ext_dsrc_id, 'DataSourceModel': ext_dsrc_model_id, 'DataProducer': dproducer_id, 'DataProduct': dproduct_id, 'Stream': stream_id})) # Create the logger for receiving publications _, stream_route, stream_def = self.create_stream_and_logger(name='dummy', stream_id=stream_id) self.EDA_RESOURCE_ID = ds_id self.EDA_NAME = ds_name self.DVR_CONFIG['dh_cfg'] = { 'TESTING': True, 'stream_id': stream_id, 'stream_route': stream_route, 'stream_def': stream_def, 'data_producer_id': dproducer_id, # CBM: Should this be put in the main body of the config - with mod & cls? 'max_records': 4, } folder = 'test_data/dummy' if os.path.isdir(folder): for the_file in os.listdir(folder): file_path = os.path.join(folder, the_file) try: if os.path.isfile(file_path): os.unlink(file_path) except Exception as e: log.debug('_setup_resources error: {0}'.format(e)) if not os.path.exists(folder): os.makedirs(folder) self.add_dummy_file('test_data/dummy/test2012-02-01-12.dum') self.add_dummy_file('test_data/dummy/test2012-02-01-13.dum') self.add_dummy_file('test_data/dummy/test2012-02-01-14.dum') self.add_dummy_file('test_data/dummy/test2012-02-01-15.dum')
def _setup_resources(self): # TODO: some or all of this (or some variation) should move to DAMS' # Build the test resources for the dataset dams_cli = DataAcquisitionManagementServiceClient() dpms_cli = DataProductManagementServiceClient() rr_cli = ResourceRegistryServiceClient() eda = ExternalDatasetAgent() eda_id = dams_cli.create_external_dataset_agent(eda) eda_inst = ExternalDatasetAgentInstance() eda_inst_id = dams_cli.create_external_dataset_agent_instance(eda_inst, external_dataset_agent_id=eda_id) # Create and register the necessary resources/objects # Create DataProvider dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation()) dprov.contact.name = 'Christopher Mueller' dprov.contact.email = '*****@*****.**' # Create DataSource dsrc = DataSource(protocol_type='FILE', institution=Institution(), contact=ContactInformation()) dsrc.connection_params['base_data_url'] = '' dsrc.contact.name='Tim Giguere' dsrc.contact.email = '*****@*****.**' # Create ExternalDataset ds_name = 'slocum_test_dataset' dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) dset.dataset_description.parameters['dataset_path'] = 'test_data/ru05-2012-021-0-0-sbd.dat' dset.dataset_description.parameters['temporal_dimension'] = None dset.dataset_description.parameters['zonal_dimension'] = None dset.dataset_description.parameters['meridional_dimension'] = None dset.dataset_description.parameters['vertical_dimension'] = None dset.dataset_description.parameters['variables'] = [ 'c_wpt_y_lmc', 'sci_water_cond', 'm_y_lmc', 'u_hd_fin_ap_inflection_holdoff', 'sci_m_present_time', 'm_leakdetect_voltage_forward', 'sci_bb3slo_b660_scaled', 'c_science_send_all', 'm_gps_status', 'm_water_vx', 'm_water_vy', 'c_heading', 'sci_fl3slo_chlor_units', 'u_hd_fin_ap_gain', 'm_vacuum', 'u_min_water_depth', 'm_gps_lat', 'm_veh_temp', 'f_fin_offset', 'u_hd_fin_ap_hardover_holdoff', 'c_alt_time', 'm_present_time', 'm_heading', 'sci_bb3slo_b532_scaled', 'sci_fl3slo_cdom_units', 'm_fin', 'x_cycle_overrun_in_ms', 'sci_water_pressure', 'u_hd_fin_ap_igain', 'sci_fl3slo_phyco_units', 'm_battpos', 'sci_bb3slo_b470_scaled', 'm_lat', 'm_gps_lon', 'sci_ctd41cp_timestamp', 'm_pressure', 'c_wpt_x_lmc', 'c_ballast_pumped', 'x_lmc_xy_source', 'm_lon', 'm_avg_speed', 'sci_water_temp', 'u_pitch_ap_gain', 'm_roll', 'm_tot_num_inflections', 'm_x_lmc', 'u_pitch_ap_deadband', 'm_final_water_vy', 'm_final_water_vx', 'm_water_depth', 'm_leakdetect_voltage', 'u_pitch_max_delta_battpos', 'm_coulomb_amphr', 'm_pitch', ] # Create DataSourceModel dsrc_model = DataSourceModel(name='slocum_model') dsrc_model.model = 'SLOCUM' dsrc_model.data_handler_module = 'N/A' dsrc_model.data_handler_class = 'N/A' ## Run everything through DAMS ds_id = dams_cli.create_external_dataset(external_dataset=dset) ext_dprov_id = dams_cli.create_external_data_provider(external_data_provider=dprov) ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc) ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model) # Register the ExternalDataset dproducer_id = dams_cli.register_external_data_set(external_dataset_id=ds_id) # Or using each method dams_cli.assign_data_source_to_external_data_provider(data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id) dams_cli.assign_data_source_to_data_model(data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id) dams_cli.assign_external_dataset_to_data_source(external_dataset_id=ds_id, data_source_id=ext_dsrc_id) dams_cli.assign_external_dataset_to_agent_instance(external_dataset_id=ds_id, agent_instance_id=eda_inst_id) # dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=self.eda_id, agent_instance_id=self.eda_inst_id) # Generate the data product and associate it to the ExternalDataset dprod = DataProduct(name='slocum_parsed_product', description='parsed slocum product') dproduct_id = dpms_cli.create_data_product(data_product=dprod) dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id, create_stream=True) stream_id, assn = rr_cli.find_objects(subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True) stream_id = stream_id[0] log.info('Created resources: {0}'.format({'ExternalDataset':ds_id, 'ExternalDataProvider':ext_dprov_id, 'DataSource':ext_dsrc_id, 'DataSourceModel':ext_dsrc_model_id, 'DataProducer':dproducer_id, 'DataProduct':dproduct_id, 'Stream':stream_id})) #CBM: Use CF standard_names ttool = TaxyTool() ttool.add_taxonomy_set('c_wpt_y_lmc'), ttool.add_taxonomy_set('sci_water_cond'), ttool.add_taxonomy_set('m_y_lmc'), ttool.add_taxonomy_set('u_hd_fin_ap_inflection_holdoff'), ttool.add_taxonomy_set('sci_m_present_time'), ttool.add_taxonomy_set('m_leakdetect_voltage_forward'), ttool.add_taxonomy_set('sci_bb3slo_b660_scaled'), ttool.add_taxonomy_set('c_science_send_all'), ttool.add_taxonomy_set('m_gps_status'), ttool.add_taxonomy_set('m_water_vx'), ttool.add_taxonomy_set('m_water_vy'), ttool.add_taxonomy_set('c_heading'), ttool.add_taxonomy_set('sci_fl3slo_chlor_units'), ttool.add_taxonomy_set('u_hd_fin_ap_gain'), ttool.add_taxonomy_set('m_vacuum'), ttool.add_taxonomy_set('u_min_water_depth'), ttool.add_taxonomy_set('m_gps_lat'), ttool.add_taxonomy_set('m_veh_temp'), ttool.add_taxonomy_set('f_fin_offset'), ttool.add_taxonomy_set('u_hd_fin_ap_hardover_holdoff'), ttool.add_taxonomy_set('c_alt_time'), ttool.add_taxonomy_set('m_present_time'), ttool.add_taxonomy_set('m_heading'), ttool.add_taxonomy_set('sci_bb3slo_b532_scaled'), ttool.add_taxonomy_set('sci_fl3slo_cdom_units'), ttool.add_taxonomy_set('m_fin'), ttool.add_taxonomy_set('x_cycle_overrun_in_ms'), ttool.add_taxonomy_set('sci_water_pressure'), ttool.add_taxonomy_set('u_hd_fin_ap_igain'), ttool.add_taxonomy_set('sci_fl3slo_phyco_units'), ttool.add_taxonomy_set('m_battpos'), ttool.add_taxonomy_set('sci_bb3slo_b470_scaled'), ttool.add_taxonomy_set('m_lat'), ttool.add_taxonomy_set('m_gps_lon'), ttool.add_taxonomy_set('sci_ctd41cp_timestamp'), ttool.add_taxonomy_set('m_pressure'), ttool.add_taxonomy_set('c_wpt_x_lmc'), ttool.add_taxonomy_set('c_ballast_pumped'), ttool.add_taxonomy_set('x_lmc_xy_source'), ttool.add_taxonomy_set('m_lon'), ttool.add_taxonomy_set('m_avg_speed'), ttool.add_taxonomy_set('sci_water_temp'), ttool.add_taxonomy_set('u_pitch_ap_gain'), ttool.add_taxonomy_set('m_roll'), ttool.add_taxonomy_set('m_tot_num_inflections'), ttool.add_taxonomy_set('m_x_lmc'), ttool.add_taxonomy_set('u_pitch_ap_deadband'), ttool.add_taxonomy_set('m_final_water_vy'), ttool.add_taxonomy_set('m_final_water_vx'), ttool.add_taxonomy_set('m_water_depth'), ttool.add_taxonomy_set('m_leakdetect_voltage'), ttool.add_taxonomy_set('u_pitch_max_delta_battpos'), ttool.add_taxonomy_set('m_coulomb_amphr'), ttool.add_taxonomy_set('m_pitch'), #CBM: Eventually, probably want to group this crap somehow - not sure how yet... # Create the logger for receiving publications self.create_stream_and_logger(name='slocum',stream_id=stream_id) self.EDA_RESOURCE_ID = ds_id self.EDA_NAME = ds_name self.DVR_CONFIG['dh_cfg'] = { 'TESTING':True, 'stream_id':stream_id, 'external_dataset_res':dset, 'taxonomy':ttool.dump(), 'data_producer_id':dproducer_id,#CBM: Should this be put in the main body of the config - with mod & cls? 'max_records':20, }
def _setup_resources(self): # TODO: some or all of this (or some variation) should move to DAMS' # Build the test resources for the dataset dams_cli = DataAcquisitionManagementServiceClient() dpms_cli = DataProductManagementServiceClient() rr_cli = ResourceRegistryServiceClient() eda = ExternalDatasetAgent() eda_id = dams_cli.create_external_dataset_agent(eda) eda_inst = ExternalDatasetAgentInstance() eda_inst_id = dams_cli.create_external_dataset_agent_instance(eda_inst, external_dataset_agent_id=eda_id) # Create and register the necessary resources/objects # Create DataProvider dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation()) dprov.contact.name = 'Christopher Mueller' dprov.contact.email = '*****@*****.**' # Create DataSource dsrc = DataSource(protocol_type='FILE', institution=Institution(), contact=ContactInformation()) dsrc.connection_params['base_data_url'] = '' dsrc.contact.name='Tim Giguere' dsrc.contact.email = '*****@*****.**' # Create ExternalDataset ds_name = 'ruv_test_dataset' dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) dset.dataset_description.parameters['dataset_path'] = 'test_data/RDLi_SEAB_2011_08_24_1600.ruv' dset.dataset_description.parameters['temporal_dimension'] = None dset.dataset_description.parameters['zonal_dimension'] = None dset.dataset_description.parameters['meridional_dimension'] = None dset.dataset_description.parameters['vertical_dimension'] = None dset.dataset_description.parameters['variables'] = [ ] # Create DataSourceModel dsrc_model = DataSourceModel(name='ruv_model') dsrc_model.model = 'RUV' dsrc_model.data_handler_module = 'N/A' dsrc_model.data_handler_class = 'N/A' ## Run everything through DAMS ds_id = dams_cli.create_external_dataset(external_dataset=dset) ext_dprov_id = dams_cli.create_external_data_provider(external_data_provider=dprov) ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc) ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model) # Register the ExternalDataset dproducer_id = dams_cli.register_external_data_set(external_dataset_id=ds_id) # Or using each method dams_cli.assign_data_source_to_external_data_provider(data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id) dams_cli.assign_data_source_to_data_model(data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id) dams_cli.assign_external_dataset_to_data_source(external_dataset_id=ds_id, data_source_id=ext_dsrc_id) dams_cli.assign_external_dataset_to_agent_instance(external_dataset_id=ds_id, agent_instance_id=eda_inst_id) # dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=self.eda_id, agent_instance_id=self.eda_inst_id) # Generate the data product and associate it to the ExternalDataset dprod = DataProduct(name='ruv_parsed_product', description='parsed ruv product') dproduct_id = dpms_cli.create_data_product(data_product=dprod) dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id, create_stream=True) stream_id, assn = rr_cli.find_objects(subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True) stream_id = stream_id[0] log.info('Created resources: {0}'.format({'ExternalDataset':ds_id, 'ExternalDataProvider':ext_dprov_id, 'DataSource':ext_dsrc_id, 'DataSourceModel':ext_dsrc_model_id, 'DataProducer':dproducer_id, 'DataProduct':dproduct_id, 'Stream':stream_id})) #CBM: Use CF standard_names ttool = TaxyTool() ttool.add_taxonomy_set('data','test data') #CBM: Eventually, probably want to group this crap somehow - not sure how yet... # Create the logger for receiving publications self.create_stream_and_logger(name='ruv',stream_id=stream_id) self.EDA_RESOURCE_ID = ds_id self.EDA_NAME = ds_name self.DVR_CONFIG['dh_cfg'] = { 'TESTING':True, 'stream_id':stream_id, 'external_dataset_res':dset, 'taxonomy':ttool.dump(), 'data_producer_id':dproducer_id,#CBM: Should this be put in the main body of the config - with mod & cls? 'max_records':20, }
def _setup_resources(self): # TODO: some or all of this (or some variation) should move to DAMS' # Build the test resources for the dataset dams_cli = DataAcquisitionManagementServiceClient() dpms_cli = DataProductManagementServiceClient() rr_cli = ResourceRegistryServiceClient() pubsub_cli = PubsubManagementServiceClient() eda = ExternalDatasetAgent() eda_id = dams_cli.create_external_dataset_agent(eda) eda_inst = ExternalDatasetAgentInstance() eda_inst_id = dams_cli.create_external_dataset_agent_instance(eda_inst, external_dataset_agent_id=eda_id) # Create and register the necessary resources/objects # Create DataProvider dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation()) dprov.contact.name = 'Christopher Mueller' dprov.contact.email = '*****@*****.**' # Create DataSource dsrc = DataSource(protocol_type='FILE', institution=Institution(), contact=ContactInformation()) dsrc.connection_params['base_data_url'] = '' dsrc.contact.name='Tim Giguere' dsrc.contact.email = '*****@*****.**' # Create ExternalDataset ds_name = 'ruv_test_dataset' dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) dset.dataset_description.parameters['base_url'] = 'test_data/ruv/' dset.dataset_description.parameters['list_pattern'] = 'RDLi_SEAB_2011_08_24_1600.ruv' dset.dataset_description.parameters['date_pattern'] = '%Y %m %d %H %M' dset.dataset_description.parameters['date_extraction_pattern'] = 'RDLi_SEAB_([\d]{4})_([\d]{2})_([\d]{2})_([\d]{2})([\d]{2}).ruv' dset.dataset_description.parameters['temporal_dimension'] = None dset.dataset_description.parameters['zonal_dimension'] = None dset.dataset_description.parameters['meridional_dimension'] = None dset.dataset_description.parameters['vertical_dimension'] = None dset.dataset_description.parameters['variables'] = [ ] # Create DataSourceModel dsrc_model = DataSourceModel(name='ruv_model') dsrc_model.model = 'RUV' dsrc_model.data_handler_module = 'N/A' dsrc_model.data_handler_class = 'N/A' ## Run everything through DAMS ds_id = dams_cli.create_external_dataset(external_dataset=dset) ext_dprov_id = dams_cli.create_external_data_provider(external_data_provider=dprov) ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc) ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model) # Register the ExternalDataset dproducer_id = dams_cli.register_external_data_set(external_dataset_id=ds_id) # Or using each method dams_cli.assign_data_source_to_external_data_provider(data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id) dams_cli.assign_data_source_to_data_model(data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id) dams_cli.assign_external_dataset_to_data_source(external_dataset_id=ds_id, data_source_id=ext_dsrc_id) dams_cli.assign_external_dataset_to_agent_instance(external_dataset_id=ds_id, agent_instance_id=eda_inst_id) # dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=self.eda_id, agent_instance_id=self.eda_inst_id) #create temp streamdef so the data product can create the stream craft = CoverageCraft sdom, tdom = craft.create_domains() sdom = sdom.dump() tdom = tdom.dump() parameter_dictionary = craft.create_parameters() parameter_dictionary = parameter_dictionary.dump() dprod = IonObject(RT.DataProduct, name='ruv_parsed_product', description='parsed ruv product', temporal_domain = tdom, spatial_domain = sdom) streamdef_id = pubsub_cli.create_stream_definition(name="temp", description="temp") # Generate the data product and associate it to the ExternalDataset dproduct_id = dpms_cli.create_data_product(data_product=dprod, stream_definition_id=streamdef_id, parameter_dictionary=parameter_dictionary) dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id) stream_id, assn = rr_cli.find_objects(subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True) stream_id = stream_id[0] log.info('Created resources: {0}'.format({'ExternalDataset':ds_id, 'ExternalDataProvider':ext_dprov_id, 'DataSource':ext_dsrc_id, 'DataSourceModel':ext_dsrc_model_id, 'DataProducer':dproducer_id, 'DataProduct':dproduct_id, 'Stream':stream_id})) #CBM: Use CF standard_names #ttool = TaxyTool() # #ttool.add_taxonomy_set('data','test data') pdict = ParameterDictionary() t_ctxt = ParameterContext('data', param_type=QuantityType(value_encoding=numpy.dtype('int64'))) t_ctxt.axis = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 01-01-1970' pdict.add_context(t_ctxt) #CBM: Eventually, probably want to group this crap somehow - not sure how yet... # Create the logger for receiving publications self.create_stream_and_logger(name='ruv',stream_id=stream_id) self.EDA_RESOURCE_ID = ds_id self.EDA_NAME = ds_name self.DVR_CONFIG['dh_cfg'] = { 'TESTING':True, 'stream_id':stream_id, 'external_dataset_res':dset, 'param_dictionary':pdict.dump(), 'data_producer_id':dproducer_id,#CBM: Should this be put in the main body of the config - with mod & cls? 'max_records':20, }
class TestIntDataAcquisitionManagementService(IonIntegrationTestCase): def setUp(self): # Start container self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') # Now create client to DataAcquisitionManagementService self.client = DataAcquisitionManagementServiceClient(node=self.container.node) self.rrclient = ResourceRegistryServiceClient(node=self.container.node) self.dataproductclient = DataProductManagementServiceClient(node=self.container.node) def tearDown(self): pass #@unittest.skip('Not done yet.') def test_data_source_ops(self): # test creating a new data source print 'Creating new data source' datasource_obj = IonObject(RT.DataSource, name='DataSource1', description='instrument based new source' , type='sbe37') try: ds_id = self.client.create_data_source(datasource_obj) except BadRequest as ex: self.fail("failed to create new data source: %s" %ex) print 'new data source id = ', ds_id # test reading a non-existent data source print 'reading non-existent data source' try: dp_obj = self.client.read_data_source('some_fake_id') except NotFound as ex: pass else: self.fail("non-existing data source was found during read: %s" %dp_obj) # update a data source (tests read also) print 'Updating data source' # first get the existing data source object try: datasource_obj = self.client.read_data_source(ds_id) except NotFound as ex: self.fail("existing data source was not found during read") else: pass # now tweak the object datasource_obj.description = 'the very first data source' # now write the dp back to the registry try: update_result = self.client.update_data_source(datasource_obj) except NotFound as ex: self.fail("existing data source was not found during update") except Conflict as ex: self.fail("revision conflict exception during data source update") #else: # self.assertTrue(update_result == True) # now get the data source back to see if it was updated try: datasource_obj = self.client.read_data_source(ds_id) except NotFound as ex: self.fail("existing data source was not found during read") else: pass self.assertTrue(datasource_obj.description == 'the very first data source') # now 'delete' the data source print "deleting data source" try: delete_result = self.client.delete_data_source(ds_id) except NotFound as ex: self.fail("existing data source was not found during delete") #self.assertTrue(delete_result == True) # now try to get the deleted dp object try: dp_obj = self.client.read_data_source(ds_id) except NotFound as ex: pass else: self.fail("deleted data source was found during read") # now try to delete the already deleted data source object print "deleting non-existing data source" try: delete_result = self.client.delete_data_source(ds_id) except NotFound as ex: pass else: self.fail("non-existing data source was found during delete") #@unittest.skip('Not done yet.') def test_register_instrument(self): # Register an instrument as a data producer in coordination with DM PubSub: create stream, register and create producer object # set up initial instrument to register instrument_obj = IonObject(RT.InstrumentDevice, name='Inst1',description='an instrument that is creating the data product') instrument_id, rev = self.rrclient.create(instrument_obj) dataproduct_obj = IonObject(RT.DataProduct, name='DataProduct1',description='sample data product') dataproduct_id, rev = self.rrclient.create(dataproduct_obj) # test registering a new data producer try: ds_id = self.client.register_instrument(instrument_id) except BadRequest as ex: self.fail("failed to create new data producer: %s" %ex) print 'new data producer id = ', ds_id # test assigning a data product to an instrument, creating the stream for the product try: self.client.assign_data_product(instrument_id, dataproduct_id, True) except BadRequest as ex: self.fail("failed to assign data product to data producer: %s" %ex) except NotFound as ex: self.fail("failed to assign data product to data producer: %s" %ex) # test UNassigning a data product from instrument, deleting the stream for the product try: self.client.unassign_data_product(instrument_id, dataproduct_id, True) except BadRequest as ex: self.fail("failed to failed to UNassign data product to data producer data producer: %s" %ex) except NotFound as ex: self.fail("failed to failed to UNassign data product to data producer data producer: %s" %ex) # test UNregistering a new data producer try: ds_id = self.client.unregister_instrument(instrument_id) except NotFound as ex: self.fail("failed to unregister instrument producer: %s" %ex) def test_register_external_data_set(self): # Register an external data set as a data producer in coordination with DM PubSub: create stream, register and create producer object # set up initial instrument to register ext_dataset_obj = IonObject(RT.ExternalDataset, name='DataSet1',description='an external data feed') ext_dataset_id, rev = self.rrclient.create(ext_dataset_obj) dataproduct_obj = IonObject(RT.DataProduct, name='DataProduct1',description='sample data product') dataproduct_id, rev = self.rrclient.create(dataproduct_obj) # test registering a new external data set try: ds_id = self.client.register_external_data_set(ext_dataset_id) except BadRequest as ex: self.fail("failed to create new data producer: %s" %ex) print 'new data producer id = ', ds_id # test assigning a data product to an ext_dataset_id, creating the stream for the product try: self.client.assign_data_product(ext_dataset_id, dataproduct_id, True) except BadRequest as ex: self.fail("failed to assign data product to data producer: %s" %ex) except NotFound as ex: self.fail("failed to assign data product to data producer: %s" %ex) # test UNassigning a data product from ext_dataset_id, deleting the stream for the product try: self.client.unassign_data_product(ext_dataset_id, dataproduct_id, True) except BadRequest as ex: self.fail("failed to failed to UNassign data product to data producer data producer: %s" %ex) except NotFound as ex: self.fail("failed to failed to UNassign data product to data producer data producer: %s" %ex) # test UNregistering a external data set try: ds_id = self.client.unregister_external_data_set(ext_dataset_id) except NotFound as ex: self.fail("failed to unregister instrument producer: %s" %ex) def test_register_process(self): # Register a data process as a data producer in coordination with DM PubSub: create stream, register and create producer object # set up initial instrument to register process_obj = IonObject(RT.DataProcess, name='Proc1',description='a data process transform') process_id, rev = self.rrclient.create(process_obj) dataproduct_obj = IonObject(RT.DataProduct, name='DataProduct1',description='sample data product') dataproduct_id, rev = self.rrclient.create(dataproduct_obj) # test registering a new process try: ds_id = self.client.register_process(process_id) except BadRequest as ex: self.fail("failed to create new data producer: %s" %ex) print 'new data producer id = ', ds_id # test assigning a data product to a process, no stream create try: self.client.assign_data_product(process_id, dataproduct_id, False) except BadRequest as ex: self.fail("failed to create new data producer: %s" %ex) except NotFound as ex: self.fail("failed to create new data producer: %s" %ex) # test UNassigning a data product from the data process, deleting the stream for the product try: self.client.unassign_data_product(process_id, dataproduct_id, False) except BadRequest as ex: self.fail("failed to failed to UNassign data product to data producer data producer: %s" %ex) except NotFound as ex: self.fail("failed to failed to UNassign data product to data producer data producer: %s" %ex) # test UNregistering a process try: ds_id = self.client.unregister_process(process_id) except NotFound as ex: self.fail("failed to unregister instrument producer: %s" %ex) #@unittest.skip('not ready') def test_eoi_resources(self): # # test creating a new data provider # print 'Creating new external_data_provider' dataprovider_obj = IonObject(RT.ExternalDataProvider, name='ExtDataProvider1', description='external data provider ') try: dataprovider_id = self.client.create_external_data_provider(dataprovider_obj) except BadRequest as ex: self.fail("failed to create new data provider: %s" %ex) print 'new data provider id = ', dataprovider_id # # test creating a new data source # print 'Creating new data source' datasource_obj = IonObject(RT.DataSource, name='DataSource1', description='data source ', type='DAP') try: datasource_id = self.client.create_data_source(datasource_obj) except BadRequest as ex: self.fail("failed to create new data source: %s" %ex) print 'new data source id = ', datasource_id # # test creating a new data source model # print 'Creating new data source model' datamodel_obj = IonObject(RT.DataSourceModel, name='DataSourceModel1', description='data source model', model='model1') try: datamodel_id = self.client.create_data_source_model(datamodel_obj) except BadRequest as ex: self.fail("failed to create new data source model: %s" %ex) print 'new data source model id = ', datamodel_id # # test creating a new external data set # print 'Creating new external data set' dataset_obj = IonObject(RT.ExternalDataset, name='ExternalDataSet1', description='external data set ') try: extdataset_id = self.client.create_external_dataset(dataset_obj) except BadRequest as ex: self.fail("failed to create new external data set: %s" %ex) print 'new external data set id = ', extdataset_id # # test creating a new dataset agent instance # print 'Creating new external data agent ' datasetagent_obj = IonObject(RT.ExternalDatasetAgent, name='ExternalDatasetAgent1', description='external data agent ') try: datasetagent_id = self.client.create_external_dataset_agent(datasetagent_obj) except BadRequest as ex: self.fail("failed to create new external dataset agent: %s" %ex) print 'new external data agent id = ', datasetagent_id # # test creating a new datasource agent instance # print 'Creating new data source agent ' datasourceagent_obj = IonObject(RT.DataSourceAgent, name='DataSourceAgent1', description=' DataSource agent ') try: datasource_agent_id = self.client.create_data_source_agent(datasourceagent_obj) except BadRequest as ex: self.fail("failed to create new external datasource agent: %s" %ex) print 'new external data agent id = ', datasource_agent_id # # test creating a new dataset agent instance # print 'Creating new external dataset agent instance' datasetagentinstance_obj = IonObject(RT.ExternalDatasetAgentInstance, name='ExternalDatasetAgentInstance1', description='external dataset agent instance ') try: datasetagentinstance_id = self.client.create_external_dataset_agent_instance(datasetagentinstance_obj, datasetagent_id) except BadRequest as ex: self.fail("failed to create new external dataset agent instance: %s" %ex) print 'new external data agent instance id = ', datasetagentinstance_id # # test creating a new datasource agent instance # print 'Creating new data source agent ' datasourceagentinstance_obj = IonObject(RT.DataSourceAgentInstance, name='ExternalDataSourceAgentInstance1', description='external DataSource agent instance ') try: datasource_agent_instance_id = self.client.create_data_source_agent_instance(datasourceagentinstance_obj) except BadRequest as ex: self.fail("failed to create new external datasource agent instance: %s" %ex) print 'new external data agent id = ', datasource_agent_instance_id # # test assign / unassign # self.client.unassign_data_source_from_external_data_provider(datasource_id, dataprovider_id) self.client.unassign_data_source_from_data_model(datasource_id, datamodel_id) self.client.unassign_external_dataset_from_data_source(extdataset_id, datasource_id) # # test read # try: dp_obj = self.client.read_external_data_provider(dataprovider_id) except NotFound as ex: self.fail("existing data provicer was not found during read") else: pass try: dp_obj = self.client.read_data_source(datasource_id) except NotFound as ex: self.fail("existing data source was not found during read") else: pass # # test delete # try: self.client.delete_external_data_provider(dataprovider_id) self.client.delete_data_source(datasource_id) self.client.delete_external_dataset(extdataset_id) except NotFound as ex: self.fail("existing data product was not found during delete") # test reading a non-existent data product print 'reading non-existent data product' try: bad_obj = self.client.read_external_data_provider('some_fake_id') except NotFound as ex: pass else: self.fail("non-existing data product was found during read: %s" %bad_obj)
def _setup_resources(self): # TODO: some or all of this (or some variation) should move to DAMS' # Build the test resources for the dataset dams_cli = DataAcquisitionManagementServiceClient() dpms_cli = DataProductManagementServiceClient() rr_cli = ResourceRegistryServiceClient() eda = ExternalDatasetAgent() eda_id = dams_cli.create_external_dataset_agent(eda) eda_inst = ExternalDatasetAgentInstance() eda_inst_id = dams_cli.create_external_dataset_agent_instance( eda_inst, external_dataset_agent_id=eda_id) # Create and register the necessary resources/objects # Create DataProvider dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation()) dprov.contact.name = 'Christopher Mueller' dprov.contact.email = '*****@*****.**' # Create DataSource dsrc = DataSource(protocol_type='DAP', institution=Institution(), contact=ContactInformation()) dsrc.connection_params['base_data_url'] = '' dsrc.contact.name = 'Tim Giguere' dsrc.contact.email = '*****@*****.**' # Create ExternalDataset ds_name = 'usgs_test_dataset' dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) # The usgs.nc test dataset is a download of the R1 dataset found here: # http://thredds-test.oceanobservatories.org/thredds/dodsC/ooiciData/E66B1A74-A684-454A-9ADE-8388C2C634E5.ncml dset.dataset_description.parameters[ 'dataset_path'] = 'test_data/usgs.nc' dset.dataset_description.parameters['temporal_dimension'] = 'time' dset.dataset_description.parameters['zonal_dimension'] = 'lon' dset.dataset_description.parameters['meridional_dimension'] = 'lat' dset.dataset_description.parameters['vertical_dimension'] = 'z' dset.dataset_description.parameters['variables'] = [ 'water_temperature', 'streamflow', 'water_temperature_bottom', 'water_temperature_middle', 'specific_conductance', 'data_qualifier', ] # Create DataSourceModel dsrc_model = DataSourceModel(name='dap_model') dsrc_model.model = 'DAP' dsrc_model.data_handler_module = 'N/A' dsrc_model.data_handler_class = 'N/A' ## Run everything through DAMS ds_id = dams_cli.create_external_dataset(external_dataset=dset) ext_dprov_id = dams_cli.create_external_data_provider( external_data_provider=dprov) ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc) ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model) # Register the ExternalDataset dproducer_id = dams_cli.register_external_data_set( external_dataset_id=ds_id) # Or using each method dams_cli.assign_data_source_to_external_data_provider( data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id) dams_cli.assign_data_source_to_data_model( data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id) dams_cli.assign_external_dataset_to_data_source( external_dataset_id=ds_id, data_source_id=ext_dsrc_id) dams_cli.assign_external_dataset_to_agent_instance( external_dataset_id=ds_id, agent_instance_id=eda_inst_id) # dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=self.eda_id, agent_instance_id=self.eda_inst_id) # Generate the data product and associate it to the ExternalDataset dprod = DataProduct(name='usgs_parsed_product', description='parsed usgs product') dproduct_id = dpms_cli.create_data_product(data_product=dprod) dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id, create_stream=True) stream_id, assn = rr_cli.find_objects(subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True) stream_id = stream_id[0] log.info('Created resources: {0}'.format({ 'ExternalDataset': ds_id, 'ExternalDataProvider': ext_dprov_id, 'DataSource': ext_dsrc_id, 'DataSourceModel': ext_dsrc_model_id, 'DataProducer': dproducer_id, 'DataProduct': dproduct_id, 'Stream': stream_id })) #CBM: Use CF standard_names ttool = TaxyTool() ttool.add_taxonomy_set('time', 'time') ttool.add_taxonomy_set('lon', 'longitude') ttool.add_taxonomy_set('lat', 'latitude') ttool.add_taxonomy_set('z', 'water depth') ttool.add_taxonomy_set('water_temperature', 'average water temperature') ttool.add_taxonomy_set('water_temperature_bottom', 'water temperature at bottom of water column') ttool.add_taxonomy_set('water_temperature_middle', 'water temperature at middle of water column') ttool.add_taxonomy_set('streamflow', 'flow velocity of stream') ttool.add_taxonomy_set('specific_conductance', 'specific conductance of water') ttool.add_taxonomy_set('data_qualifier', 'data qualifier flag') ttool.add_taxonomy_set('coords', 'This group contains coordinate parameters') ttool.add_taxonomy_set('data', 'This group contains data parameters') # Create the logger for receiving publications self.create_stream_and_logger(name='usgs', stream_id=stream_id) self.EDA_RESOURCE_ID = ds_id self.EDA_NAME = ds_name self.DVR_CONFIG['dh_cfg'] = { 'TESTING': True, 'stream_id': stream_id, 'taxonomy': ttool.dump(), 'data_producer_id': dproducer_id, #CBM: Should this be put in the main body of the config - with mod & cls? 'max_records': 4, }