def get_param_dict(param_dict_name=None): raise NotImplementedError( 'This method has been replaced by DatasetManagementService, please use read_parameter_dictionary_by_name instead' ) # read the file just once, not every time needed global _PARAMETER_DICTIONARIES global _PARAMETER_CONTEXTS if not _PARAMETER_DICTIONARIES: param_dict_defs_file = "res/config/param_dict_defs.yml" with open(param_dict_defs_file, "r") as f_dict: dict_string = f_dict.read() _PARAMETER_DICTIONARIES = yaml.load(dict_string) param_context_defs_file = "res/config/param_context_defs.yml" with open(param_context_defs_file, "r") as f_ctxt: ctxt_string = f_ctxt.read() _PARAMETER_CONTEXTS = yaml.load(ctxt_string) # make sure we have the one requested context_names = _PARAMETER_DICTIONARIES[param_dict_name] for name in context_names: if not _PARAMETER_CONTEXTS.has_key(name): raise AssertionError( 'The parameter dict has a context that does not exist in the parameter context defs specified in yml: %s' % name) # package and ship pdict = ParameterDictionary() for ctxt_name in context_names: param_context = ParameterContext.load(_PARAMETER_CONTEXTS[ctxt_name]) pdict.add_context(param_context) return pdict
def _create_parameter_dictionary(self): pdict = ParameterDictionary() lat_ctxt = ParameterContext( 'lat', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' pdict.add_context(lat_ctxt) lon_ctxt = ParameterContext( 'lon', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' pdict.add_context(lon_ctxt) temp_ctxt = ParameterContext( 'water_temperature', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) temp_ctxt.uom = 'degree_Celsius' pdict.add_context(temp_ctxt) temp_ctxt = ParameterContext( 'water_temperature_bottom', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) temp_ctxt.uom = 'degree_Celsius' pdict.add_context(temp_ctxt) temp_ctxt = ParameterContext( 'water_temperature_middle', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) temp_ctxt.uom = 'degree_Celsius' pdict.add_context(temp_ctxt) temp_ctxt = ParameterContext( 'z', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) temp_ctxt.uom = 'meters' pdict.add_context(temp_ctxt) cond_ctxt = ParameterContext( 'streamflow', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) cond_ctxt.uom = 'unknown' pdict.add_context(cond_ctxt) pres_ctxt = ParameterContext( 'specific_conductance', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) pres_ctxt.uom = 'unknown' pdict.add_context(pres_ctxt) pres_ctxt = ParameterContext( 'data_qualifier', param_type=QuantityType(value_encoding=numpy.dtype('bool'))) pres_ctxt.uom = 'unknown' pdict.add_context(pres_ctxt) return pdict
def _create_parameter(self): pdict = ParameterDictionary() pdict = self._add_location_time_ctxt(pdict) pres_ctxt = ParameterContext( 'pressure', param_type=QuantityType(value_encoding=numpy.float32)) pres_ctxt.uom = 'Pascal' pres_ctxt.fill_value = 0x0 pdict.add_context(pres_ctxt) temp_ctxt = ParameterContext( 'temp', param_type=QuantityType(value_encoding=numpy.float32)) temp_ctxt.uom = 'degree_Celsius' temp_ctxt.fill_value = 0e0 pdict.add_context(temp_ctxt) cond_ctxt = ParameterContext( 'conductivity', param_type=QuantityType(value_encoding=numpy.float32)) cond_ctxt.uom = 'unknown' cond_ctxt.fill_value = 0e0 pdict.add_context(cond_ctxt) return pdict
def _create_input_param_dict_for_test(self, parameter_dict_name=''): pdict = ParameterDictionary() t_ctxt = ParameterContext( 'time', param_type=QuantityType(value_encoding=numpy.dtype('float64'))) t_ctxt.axis = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 01-01-1900' pdict.add_context(t_ctxt) cond_ctxt = ParameterContext( 'conductivity', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) cond_ctxt.uom = '' pdict.add_context(cond_ctxt) pres_ctxt = ParameterContext( 'pressure', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) pres_ctxt.uom = '' pdict.add_context(pres_ctxt) temp_ctxt = ParameterContext( 'temperature', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) temp_ctxt.uom = '' pdict.add_context(temp_ctxt) dens_ctxt = ParameterContext( 'density', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) dens_ctxt.uom = '' pdict.add_context(dens_ctxt) sal_ctxt = ParameterContext( 'salinity', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) sal_ctxt.uom = '' pdict.add_context(sal_ctxt) #create temp streamdef so the data product can create the stream pc_list = [] for pc_k, pc in pdict.iteritems(): ctxt_id = self.dataset_management.create_parameter_context( pc_k, pc[1].dump()) pc_list.append(ctxt_id) self.addCleanup(self.dataset_management.delete_parameter_context, ctxt_id) pdict_id = self.dataset_management.create_parameter_dictionary( parameter_dict_name, pc_list) self.addCleanup(self.dataset_management.delete_parameter_dictionary, pdict_id) return pdict_id
def rdt_to_granule(self, context, value_array, comp_val=None): pdict = ParameterDictionary() pdict.add_context(context) rdt = RecordDictionaryTool(param_dictionary=pdict) rdt['test'] = value_array granule = rdt.to_granule() rdt2 = RecordDictionaryTool.load_from_granule(granule) testval = comp_val if comp_val is not None else value_array actual = rdt2['test'] if isinstance(testval, basestring): self.assertEquals(testval, actual) else: np.testing.assert_array_equal(testval, actual)
def get_param_dict(self): pdict = ParameterDictionary() cond_ctxt = ParameterContext( 'conductivity', param_type=QuantityType(value_encoding=np.float64)) cond_ctxt.uom = 'unknown' cond_ctxt.fill_value = 0e0 pdict.add_context(cond_ctxt) pres_ctxt = ParameterContext( 'pressure', param_type=QuantityType(value_encoding=np.float64)) pres_ctxt.uom = 'unknown' pres_ctxt.fill_value = 0x0 pdict.add_context(pres_ctxt) temp_ctxt = ParameterContext( 'temperature', param_type=QuantityType(value_encoding=np.float64)) temp_ctxt.uom = 'unknown' temp_ctxt.fill_value = 0x0 pdict.add_context(temp_ctxt) oxy_ctxt = ParameterContext( 'oxygen', param_type=QuantityType(value_encoding=np.float64)) oxy_ctxt.uom = 'unknown' oxy_ctxt.fill_value = 0x0 pdict.add_context(oxy_ctxt) internal_ts_ctxt = ParameterContext( name='internal_timestamp', param_type=QuantityType(value_encoding=np.float64)) internal_ts_ctxt._derived_from_name = 'time' internal_ts_ctxt.uom = 'seconds' internal_ts_ctxt.fill_value = -1 pdict.add_context(internal_ts_ctxt, is_temporal=True) driver_ts_ctxt = ParameterContext( name='driver_timestamp', param_type=QuantityType(value_encoding=np.float64)) driver_ts_ctxt._derived_from_name = 'time' driver_ts_ctxt.uom = 'seconds' driver_ts_ctxt.fill_value = -1 pdict.add_context(driver_ts_ctxt) return pdict
def rdt_to_granule(self, context, value_array, comp_val=None): time = ParameterContext( name='time', param_type=QuantityType(value_encoding=np.float64)) pdict = ParameterDictionary() pdict.add_context(time, is_temporal=True) pdict.add_context(context) rdt = RecordDictionaryTool(param_dictionary=pdict) rdt['time'] = np.arange(len(value_array)) rdt['test'] = value_array granule = rdt.to_granule() rdt2 = RecordDictionaryTool.load_from_granule(granule) testval = comp_val if comp_val is not None else value_array actual = rdt2['test'] if isinstance(testval, basestring): self.assertEquals(testval, actual) else: np.testing.assert_array_equal(testval, actual)
def adhoc_get_parameter_dictionary(stream_name): """ @param stream_name IGNORED in this adhoc function; it returns the same ParameterDictionary definition always. @retval corresponding ParameterDictionary. """ #@TODO Luke - Maybe we can make this a bit more versatile, we could make this a standard pdict... pdict = ParameterDictionary() # ctxt = ParameterContext('value', param_type=QuantityType(value_encoding=numpy.float32)) ctxt = ParameterContext('value', param_type=QuantityType(value_encoding=numpy.dtype('float64'))) ctxt.uom = 'unknown' ctxt.fill_value = 0e0 pdict.add_context(ctxt) ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=numpy.dtype('int64'))) ctxt.axis = AxisTypeEnum.TIME ctxt.uom = 'seconds since 01-01-1970' pdict.add_context(ctxt) ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) ctxt.axis = AxisTypeEnum.LON ctxt.uom = 'degree_east' pdict.add_context(ctxt) ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) ctxt.axis = AxisTypeEnum.LAT ctxt.uom = 'degree_north' pdict.add_context(ctxt) ctxt = ParameterContext('height', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) ctxt.axis = AxisTypeEnum.HEIGHT ctxt.uom = 'unknown' pdict.add_context(ctxt) return pdict
def cov_io(self, context, value_array, comp_val=None): pdict = ParameterDictionary() time = ParameterContext( name='time', param_type=QuantityType(value_encoding=np.float64)) pdict.add_context(context) pdict.add_context(time, True) # Construct temporal and spatial Coordinate Reference System objects tcrs = CRS([AxisTypeEnum.TIME]) scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT]) # Construct temporal and spatial Domain objects tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline) sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE ) # 0d spatial topology (station/trajectory) # Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain cov = SimplexCoverage('test_data', create_guid(), 'sample coverage_model', parameter_dictionary=pdict, temporal_domain=tdom, spatial_domain=sdom) self.addCleanup(shutil.rmtree, cov.persistence_dir) cov.insert_timesteps(len(value_array)) cov.set_parameter_values('test', tdoa=slice(0, len(value_array)), value=value_array) comp_val = comp_val if comp_val is not None else value_array testval = cov.get_parameter_values('test') try: np.testing.assert_array_equal(testval, comp_val) except: print repr(value_array) raise
def _create_parameter_dictionary(self): pdict = ParameterDictionary() t_ctxt = ParameterContext('c_wpt_y_lmc', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_water_cond', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_y_lmc', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('u_hd_fin_ap_inflection_holdoff', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_m_present_time', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_leakdetect_voltage_forward', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_bb3slo_b660_scaled', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('c_science_send_all', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_gps_status', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_water_vx', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_water_vy', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('c_heading', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_fl3slo_chlor_units', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('u_hd_fin_ap_gain', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_vacuum', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('u_min_water_depth', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_gps_lat', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_veh_temp', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('f_fin_offset', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('u_hd_fin_ap_hardover_holdoff', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('c_alt_time', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_present_time', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_heading', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_bb3slo_b532_scaled', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_fl3slo_cdom_units', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_fin', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('x_cycle_overrun_in_ms', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_water_pressure', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('u_hd_fin_ap_igain', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_fl3slo_phyco_units', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_battpos', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_bb3slo_b470_scaled', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_lat', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_gps_lon', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_ctd41cp_timestamp', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_pressure', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('c_wpt_x_lmc', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('c_ballast_pumped', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('x_lmc_xy_source', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_lon', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_avg_speed', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_water_temp', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('u_pitch_ap_gain', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_roll', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_tot_num_inflections', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_x_lmc', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('u_pitch_ap_deadband', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_final_water_vy', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_final_water_vx', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_water_depth', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_leakdetect_voltage', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('u_pitch_max_delta_battpos', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_coulomb_amphr', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_pitch', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) return pdict
def _setup_resources(self): # TODO: some or all of this (or some variation) should move to DAMS' # Build the test resources for the dataset dms_cli = DatasetManagementServiceClient() dams_cli = DataAcquisitionManagementServiceClient() dpms_cli = DataProductManagementServiceClient() rr_cli = ResourceRegistryServiceClient() pubsub_cli = PubsubManagementServiceClient() eda = ExternalDatasetAgent(name='example data agent', handler_module=self.DVR_CONFIG['dvr_mod'], handler_class=self.DVR_CONFIG['dvr_cls']) eda_id = dams_cli.create_external_dataset_agent(eda) eda_inst = ExternalDatasetAgentInstance( name='example dataset agent instance') eda_inst_id = dams_cli.create_external_dataset_agent_instance( eda_inst, external_dataset_agent_id=eda_id) # Create and register the necessary resources/objects # Create DataProvider dprov = ExternalDataProvider(name='example data provider', institution=Institution(), contact=ContactInformation()) dprov.contact.individual_names_given = 'Christopher Mueller' dprov.contact.email = '*****@*****.**' # Create DataSource dsrc = DataSource(name='example datasource', protocol_type='FILE', institution=Institution(), contact=ContactInformation()) dsrc.connection_params['base_data_url'] = '' dsrc.contact.individual_names_given = 'Tim Giguere' dsrc.contact.email = '*****@*****.**' # Create ExternalDataset ds_name = 'ruv_test_dataset' dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) dset.dataset_description.parameters['base_url'] = 'test_data/ruv/' dset.dataset_description.parameters[ 'list_pattern'] = 'RDLi_SEAB_2011_08_24_1600.ruv' dset.dataset_description.parameters['date_pattern'] = '%Y %m %d %H %M' dset.dataset_description.parameters[ 'date_extraction_pattern'] = 'RDLi_SEAB_([\d]{4})_([\d]{2})_([\d]{2})_([\d]{2})([\d]{2}).ruv' dset.dataset_description.parameters['temporal_dimension'] = None dset.dataset_description.parameters['zonal_dimension'] = None dset.dataset_description.parameters['meridional_dimension'] = None dset.dataset_description.parameters['vertical_dimension'] = None dset.dataset_description.parameters['variables'] = [] # Create DataSourceModel dsrc_model = DataSourceModel(name='ruv_model') #dsrc_model.model = 'RUV' dsrc_model.data_handler_module = 'N/A' dsrc_model.data_handler_class = 'N/A' ## Run everything through DAMS ds_id = dams_cli.create_external_dataset(external_dataset=dset) ext_dprov_id = dams_cli.create_external_data_provider( external_data_provider=dprov) ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc) ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model) # Register the ExternalDataset dproducer_id = dams_cli.register_external_data_set( external_dataset_id=ds_id) # Or using each method dams_cli.assign_data_source_to_external_data_provider( data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id) dams_cli.assign_data_source_to_data_model( data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id) dams_cli.assign_external_dataset_to_data_source( external_dataset_id=ds_id, data_source_id=ext_dsrc_id) dams_cli.assign_external_dataset_to_agent_instance( external_dataset_id=ds_id, agent_instance_id=eda_inst_id) pdict = ParameterDictionary() t_ctxt = ParameterContext( 'data', param_type=QuantityType(value_encoding=numpy.dtype('int64'))) t_ctxt.axis = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 01-01-1970' pdict.add_context(t_ctxt) #create temp streamdef so the data product can create the stream pc_list = [] for pc_k, pc in pdict.iteritems(): pc_list.append(dms_cli.create_parameter_context( pc_k, pc[1].dump())) pdict_id = dms_cli.create_parameter_dictionary('ruv_param_dict', pc_list) streamdef_id = pubsub_cli.create_stream_definition( name="ruv", description="stream def for ruv testing", parameter_dictionary_id=pdict_id) dprod = IonObject(RT.DataProduct, name='ruv_parsed_product', description='parsed ruv product') # Generate the data product and associate it to the ExternalDataset dproduct_id = dpms_cli.create_data_product( data_product=dprod, stream_definition_id=streamdef_id) dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id) stream_id, assn = rr_cli.find_objects(subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True) stream_id = stream_id[0] log.info('Created resources: {0}'.format({ 'ExternalDataset': ds_id, 'ExternalDataProvider': ext_dprov_id, 'DataSource': ext_dsrc_id, 'DataSourceModel': ext_dsrc_model_id, 'DataProducer': dproducer_id, 'DataProduct': dproduct_id, 'Stream': stream_id })) #CBM: Eventually, probably want to group this crap somehow - not sure how yet... # Create the logger for receiving publications _, stream_route, _ = self.create_stream_and_logger(name='ruv', stream_id=stream_id) self.EDA_RESOURCE_ID = ds_id self.EDA_NAME = ds_name self.DVR_CONFIG['dh_cfg'] = { 'TESTING': True, 'stream_id': stream_id, 'stream_route': stream_route, 'external_dataset_res': dset, 'param_dictionary': pdict.dump(), 'data_producer_id': dproducer_id, # CBM: Should this be put in the main body of the config - with mod & cls? 'max_records': 20, }
def sync_rdt_with_coverage(self, coverage=None, tdoa=None, start_time=None, end_time=None, stride_time=None, parameters=None): ''' Builds a granule based on the coverage ''' if coverage is None: coverage = self.coverage slice_ = slice(None) # Defaults to all values if tdoa is not None and isinstance(tdoa, slice): slice_ = tdoa elif stride_time is not None: validate_is_instance(start_time, Number, 'start_time must be a number for striding.') validate_is_instance(end_time, Number, 'end_time must be a number for striding.') validate_is_instance(stride_time, Number, 'stride_time must be a number for striding.') ugly_range = np.arange(start_time, end_time, stride_time) idx_values = [ TimeUtils.get_relative_time(coverage, i) for i in ugly_range ] slice_ = [idx_values] elif not (start_time is None and end_time is None): time_var = coverage._temporal_param_name uom = coverage.get_parameter_context(time_var).uom if start_time is not None: start_units = TimeUtils.ts_to_units(uom, start_time) log.info('Units: %s', start_units) start_idx = TimeUtils.get_relative_time(coverage, start_units) log.info('Start Index: %s', start_idx) start_time = start_idx if end_time is not None: end_units = TimeUtils.ts_to_units(uom, end_time) log.info('End units: %s', end_units) end_idx = TimeUtils.get_relative_time(coverage, end_units) log.info('End index: %s', end_idx) end_time = end_idx slice_ = slice(start_time, end_time, stride_time) log.info('Slice: %s', slice_) if parameters is not None: pdict = ParameterDictionary() params = set(coverage.list_parameters()).intersection(parameters) for param in params: pdict.add_context(coverage.get_parameter_context(param)) rdt = RecordDictionaryTool(param_dictionary=pdict) self.pdict = pdict else: rdt = RecordDictionaryTool( param_dictionary=coverage.parameter_dictionary) fields = coverage.list_parameters() if parameters is not None: fields = set(fields).intersection(parameters) for d in fields: rdt[d] = coverage.get_parameter_values(d, tdoa=slice_) self.rdt = rdt # Sync
def create_parameters(cls): ''' WARNING: This method is a wrapper intended only for tests, it should not be used in production code. It probably will not align to most datasets. ''' pdict = ParameterDictionary() t_ctxt = ParameterContext( 'time', param_type=QuantityType(value_encoding=np.int64)) t_ctxt.axis = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 1970-01-01' t_ctxt.fill_value = 0x0 pdict.add_context(t_ctxt) lat_ctxt = ParameterContext( 'lat', param_type=QuantityType(value_encoding=np.float32)) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' lat_ctxt.fill_value = 0e0 pdict.add_context(lat_ctxt) lon_ctxt = ParameterContext( 'lon', param_type=QuantityType(value_encoding=np.float32)) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' lon_ctxt.fill_value = 0e0 pdict.add_context(lon_ctxt) temp_ctxt = ParameterContext( 'temp', param_type=QuantityType(value_encoding=np.float32)) temp_ctxt.uom = 'degree_Celsius' temp_ctxt.fill_value = 0e0 pdict.add_context(temp_ctxt) cond_ctxt = ParameterContext( 'conductivity', param_type=QuantityType(value_encoding=np.float32)) cond_ctxt.uom = 'unknown' cond_ctxt.fill_value = 0e0 pdict.add_context(cond_ctxt) data_ctxt = ParameterContext( 'data', param_type=QuantityType(value_encoding=np.int8)) data_ctxt.uom = 'byte' data_ctxt.fill_value = 0x0 pdict.add_context(data_ctxt) pres_ctxt = ParameterContext( 'pressure', param_type=QuantityType(value_encoding=np.float32)) pres_ctxt.uom = 'Pascal' pres_ctxt.fill_value = 0x0 pdict.add_context(pres_ctxt) sal_ctxt = ParameterContext( 'salinity', param_type=QuantityType(value_encoding=np.float32)) sal_ctxt.uom = 'PSU' sal_ctxt.fill_value = 0x0 pdict.add_context(sal_ctxt) dens_ctxt = ParameterContext( 'density', param_type=QuantityType(value_encoding=np.float32)) dens_ctxt.uom = 'unknown' dens_ctxt.fill_value = 0x0 pdict.add_context(dens_ctxt) return pdict