def _create_parameter(self): pdict = ParameterDictionary() pdict = self._add_location_time_ctxt(pdict) pres_ctxt = ParameterContext( 'pressure', param_type=QuantityType(value_encoding=numpy.float32)) pres_ctxt.uom = 'Pascal' pres_ctxt.fill_value = 0x0 pdict.add_context(pres_ctxt) temp_ctxt = ParameterContext( 'temp', param_type=QuantityType(value_encoding=numpy.float32)) temp_ctxt.uom = 'degree_Celsius' temp_ctxt.fill_value = 0e0 pdict.add_context(temp_ctxt) cond_ctxt = ParameterContext( 'conductivity', param_type=QuantityType(value_encoding=numpy.float32)) cond_ctxt.uom = 'unknown' cond_ctxt.fill_value = 0e0 pdict.add_context(cond_ctxt) return pdict
def _create_parameter_dictionary(self): pdict = ParameterDictionary() lat_ctxt = ParameterContext( 'lat', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' pdict.add_context(lat_ctxt) lon_ctxt = ParameterContext( 'lon', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' pdict.add_context(lon_ctxt) temp_ctxt = ParameterContext( 'water_temperature', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) temp_ctxt.uom = 'degree_Celsius' pdict.add_context(temp_ctxt) temp_ctxt = ParameterContext( 'water_temperature_bottom', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) temp_ctxt.uom = 'degree_Celsius' pdict.add_context(temp_ctxt) temp_ctxt = ParameterContext( 'water_temperature_middle', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) temp_ctxt.uom = 'degree_Celsius' pdict.add_context(temp_ctxt) temp_ctxt = ParameterContext( 'z', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) temp_ctxt.uom = 'meters' pdict.add_context(temp_ctxt) cond_ctxt = ParameterContext( 'streamflow', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) cond_ctxt.uom = 'unknown' pdict.add_context(cond_ctxt) pres_ctxt = ParameterContext( 'specific_conductance', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) pres_ctxt.uom = 'unknown' pdict.add_context(pres_ctxt) pres_ctxt = ParameterContext( 'data_qualifier', param_type=QuantityType(value_encoding=numpy.dtype('bool'))) pres_ctxt.uom = 'unknown' pdict.add_context(pres_ctxt) return pdict
def get_quantity_type(parameter_type, encoding): if encoding[0] == 'S': slen = encoding[1:] try: int(slen) except ValueError: raise TypeError('Improper String Length: %s' % slen) param_type = QuantityType(value_encoding=np.dtype('|%s' % encoding)) else: param_type = QuantityType(value_encoding=np.dtype(encoding)) return param_type
def param_type(self, s): if s == 'record': return RecordType() elif s == 'array': return ArrayType() else: return QuantityType(value_encoding=np.dtype(s))
def build_param_contexts(self): context_ids = [] t_ctxt = ParameterContext( 'time', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.uom = 'seconds since 01-01-1970' context_ids.append( self.dataset_management.create_parameter_context( name='time', parameter_context=t_ctxt.dump())) cnd_ctxt = ParameterContext('conductivity', param_type=ArrayType()) cnd_ctxt.uom = 'mmho/cm' context_ids.append( self.dataset_management.create_parameter_context( name='conductivity', parameter_context=cnd_ctxt.dump())) temp_ctxt = ParameterContext('temperature', param_type=ArrayType()) temp_ctxt.uom = 'degC' context_ids.append( self.dataset_management.create_parameter_context( name='temperature', parameter_context=temp_ctxt.dump())) press_ctxt = ParameterContext('pressure', param_type=ArrayType()) press_ctxt.uom = 'decibars' context_ids.append( self.dataset_management.create_parameter_context( name='pressure', parameter_context=press_ctxt.dump())) oxy_ctxt = ParameterContext('oxygen', param_type=ArrayType()) oxy_ctxt.uom = 'Hz' context_ids.append( self.dataset_management.create_parameter_context( name='oxygen', parameter_context=oxy_ctxt.dump())) return context_ids
def adhoc_get_parameter_dictionary(stream_name): """ @param stream_name IGNORED in this adhoc function; it returns the same ParameterDictionary definition always. @retval corresponding ParameterDictionary. """ #@TODO Luke - Maybe we can make this a bit more versatile, we could make this a standard pdict... pdict = ParameterDictionary() # ctxt = ParameterContext('value', param_type=QuantityType(value_encoding=numpy.float32)) ctxt = ParameterContext('value', param_type=QuantityType(value_encoding=numpy.dtype('float64'))) ctxt.uom = 'unknown' ctxt.fill_value = 0e0 pdict.add_context(ctxt) ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=numpy.dtype('int64'))) ctxt.axis = AxisTypeEnum.TIME ctxt.uom = 'seconds since 01-01-1970' pdict.add_context(ctxt) ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) ctxt.axis = AxisTypeEnum.LON ctxt.uom = 'degree_east' pdict.add_context(ctxt) ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) ctxt.axis = AxisTypeEnum.LAT ctxt.uom = 'degree_north' pdict.add_context(ctxt) ctxt = ParameterContext('height', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) ctxt.axis = AxisTypeEnum.HEIGHT ctxt.uom = 'unknown' pdict.add_context(ctxt) return pdict
def _add_location_time_ctxt(self, pdict): t_ctxt = ParameterContext( 'time', param_type=QuantityType(value_encoding=numpy.int64)) t_ctxt.axis = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 1970-01-01' t_ctxt.fill_value = 0x0 pdict.add_context(t_ctxt) lat_ctxt = ParameterContext( 'lat', param_type=QuantityType(value_encoding=numpy.float32)) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' lat_ctxt.fill_value = 0e0 pdict.add_context(lat_ctxt) lon_ctxt = ParameterContext( 'lon', param_type=QuantityType(value_encoding=numpy.float32)) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' lon_ctxt.fill_value = 0e0 pdict.add_context(lon_ctxt) return pdict
def get_constant_type(parameter_type, encoding, code_set): groups = re.match(r'(constant)(<)(.*)(>)', parameter_type).groups() if groups[2] == 'str' or groups[2] == 'string': if encoding[0] != 'S' or len(encoding) < 2: raise TypeError( 'Constant strings need to be specified with a valid length (e.g. S8): %s' % encoding) slen = encoding[1:] try: int(slen) except ValueError: raise TypeError('Improper String Length: %s' % slen) parameter_type = QuantityType(value_encoding=np.dtype('|%s' % encoding)) else: parameter_type = get_parameter_type(groups[2], encoding, code_set) return ConstantType(parameter_type)
def _create_param_contexts(self): context_ids = [] t_ctxt = ParameterContext( 'ingestion_timestamp', param_type=QuantityType(value_encoding=numpy.dtype('float64'))) t_ctxt.uom = 'seconds since 1900-01-01' t_ctxt.fill_value = -9999 t_ctxt_id = self.dataset_management_client.create_parameter_context( name='ingestion_timestamp', parameter_context=t_ctxt.dump()) context_ids.append(t_ctxt_id) raw_ctxt = ParameterContext('raw', param_type=ArrayType()) raw_ctxt.uom = '' context_ids.append( self.dataset_management_client.create_parameter_context( name='raw', parameter_context=raw_ctxt.dump())) return context_ids, t_ctxt_id
def get_param_dict(self): pdict = ParameterDictionary() cond_ctxt = ParameterContext( 'conductivity', param_type=QuantityType(value_encoding=np.float64)) cond_ctxt.uom = 'unknown' cond_ctxt.fill_value = 0e0 pdict.add_context(cond_ctxt) pres_ctxt = ParameterContext( 'pressure', param_type=QuantityType(value_encoding=np.float64)) pres_ctxt.uom = 'unknown' pres_ctxt.fill_value = 0x0 pdict.add_context(pres_ctxt) temp_ctxt = ParameterContext( 'temperature', param_type=QuantityType(value_encoding=np.float64)) temp_ctxt.uom = 'unknown' temp_ctxt.fill_value = 0x0 pdict.add_context(temp_ctxt) oxy_ctxt = ParameterContext( 'oxygen', param_type=QuantityType(value_encoding=np.float64)) oxy_ctxt.uom = 'unknown' oxy_ctxt.fill_value = 0x0 pdict.add_context(oxy_ctxt) internal_ts_ctxt = ParameterContext( name='internal_timestamp', param_type=QuantityType(value_encoding=np.float64)) internal_ts_ctxt._derived_from_name = 'time' internal_ts_ctxt.uom = 'seconds' internal_ts_ctxt.fill_value = -1 pdict.add_context(internal_ts_ctxt, is_temporal=True) driver_ts_ctxt = ParameterContext( name='driver_timestamp', param_type=QuantityType(value_encoding=np.float64)) driver_ts_ctxt._derived_from_name = 'time' driver_ts_ctxt.uom = 'seconds' driver_ts_ctxt.fill_value = -1 pdict.add_context(driver_ts_ctxt) return pdict
def build_param_contexts(self): context_ids = [] t_ctxt = ParameterContext( 'Time_Time', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.uom = 'seconds since 01-01-1970' context_ids.append( self.dataset_management.create_parameter_context( name='Time_Time', parameter_context=t_ctxt.dump())) core_current_ctxt = ParameterContext( 'Core_Current', param_type=QuantityType(value_encoding=np.dtype('float32'))) core_current_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Core_Current', parameter_context=core_current_ctxt.dump())) core_voltage_ctxt = ParameterContext( 'Core_Voltage', param_type=QuantityType(value_encoding=np.dtype('float32'))) core_voltage_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Core_Voltage', parameter_context=core_voltage_ctxt.dump())) core_pressure_ctxt = ParameterContext( 'Core_Pressure', param_type=QuantityType(value_encoding=np.dtype('float32'))) core_pressure_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Core_Pressure', parameter_context=core_pressure_ctxt.dump())) fluorometer_value_ctxt = ParameterContext( 'Fluorometer_Value', param_type=QuantityType(value_encoding=np.dtype('float32'))) fluorometer_value_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Fluorometer_Value', parameter_context=fluorometer_value_ctxt.dump())) fluorometer_gain_ctxt = ParameterContext( 'Fluorometer_Gain', param_type=QuantityType(value_encoding=np.dtype('int32'))) fluorometer_gain_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Fluorometer_Gain', parameter_context=fluorometer_gain_ctxt.dump())) turbidity_value_ctxt = ParameterContext( 'Turbidity_Value', param_type=QuantityType(value_encoding=np.dtype('float32'))) turbidity_value_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Turbidity_Value', parameter_context=turbidity_value_ctxt.dump())) turbidity_gain_ctxt = ParameterContext( 'Turbidity_Gain', param_type=QuantityType(value_encoding=np.dtype('int32'))) turbidity_gain_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Turbidity_Gain', parameter_context=turbidity_gain_ctxt.dump())) optode_oxygen_ctxt = ParameterContext( 'Optode_Oxygen', param_type=QuantityType(value_encoding=np.dtype('float32'))) optode_oxygen_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Optode_Oxygen', parameter_context=optode_oxygen_ctxt.dump())) optode_temp_ctxt = ParameterContext( 'Optode_Temp', param_type=QuantityType(value_encoding=np.dtype('float32'))) optode_temp_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Optode_Temp', parameter_context=optode_temp_ctxt.dump())) par_value_ctxt = ParameterContext( 'Par_Value', param_type=QuantityType(value_encoding=np.dtype('float32'))) par_value_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Par_Value', parameter_context=par_value_ctxt.dump())) puck_scatter_ctxt = ParameterContext( 'Puck_Scatter', param_type=QuantityType(value_encoding=np.dtype('int16'))) puck_scatter_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Puck_Scatter', parameter_context=puck_scatter_ctxt.dump())) puck_chla_ctxt = ParameterContext( 'Puck_Chla', param_type=QuantityType(value_encoding=np.dtype('int16'))) puck_chla_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Puck_Chla', parameter_context=puck_chla_ctxt.dump())) puck_cdom_ctxt = ParameterContext( 'Puck_CDOM', param_type=QuantityType(value_encoding=np.dtype('int16'))) puck_cdom_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Puck_CDOM', parameter_context=puck_cdom_ctxt.dump())) biosuite_scatter_ctxt = ParameterContext( 'BioSuite_Scatter', param_type=QuantityType(value_encoding=np.dtype('int16'))) biosuite_scatter_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='BioSuite_Scatter', parameter_context=biosuite_scatter_ctxt.dump())) biosuite_chla_ctxt = ParameterContext( 'BioSuite_Chla', param_type=QuantityType(value_encoding=np.dtype('int16'))) biosuite_chla_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='BioSuite_Chla', parameter_context=biosuite_chla_ctxt.dump())) biosuite_cdom_ctxt = ParameterContext( 'BioSuite_CDOM', param_type=QuantityType(value_encoding=np.dtype('int16'))) biosuite_cdom_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='BioSuite_CDOM', parameter_context=biosuite_cdom_ctxt.dump())) biosuite_temp_ctxt = ParameterContext( 'BioSuite_Temp', param_type=QuantityType(value_encoding=np.dtype('int16'))) biosuite_temp_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='BioSuite_Temp', parameter_context=biosuite_temp_ctxt.dump())) biosuite_par_ctxt = ParameterContext( 'BioSuite_Par', param_type=QuantityType(value_encoding=np.dtype('int16'))) biosuite_par_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='BioSuite_Par', parameter_context=biosuite_par_ctxt.dump())) flbb_chla_ctxt = ParameterContext( 'FLBB_Chla', param_type=QuantityType(value_encoding=np.dtype('int16'))) flbb_chla_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='FLBB_Chla', parameter_context=flbb_chla_ctxt.dump())) flbb_turb_ctxt = ParameterContext( 'FLBB_Turb', param_type=QuantityType(value_encoding=np.dtype('int16'))) flbb_turb_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='FLBB_Turb', parameter_context=flbb_turb_ctxt.dump())) flbb_temp_ctxt = ParameterContext( 'FLBB_Temp', param_type=QuantityType(value_encoding=np.dtype('int16'))) flbb_temp_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='FLBB_Temp', parameter_context=flbb_temp_ctxt.dump())) return context_ids
def build_param_contexts(self): context_ids = [] t_ctxt = ParameterContext( 'time', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.uom = 'seconds since 01-01-1970' context_ids.append( self.dataset_management.create_parameter_context( name='time', parameter_context=t_ctxt.dump())) ut_ctxt = ParameterContext( 'upload_time', param_type=QuantityType(value_encoding=np.dtype('int64'))) ut_ctxt.uom = 'seconds since 01-01-1970' context_ids.append( self.dataset_management.create_parameter_context( name='upload_time', parameter_context=ut_ctxt.dump())) vela_ctxt = ParameterContext('VelA', param_type=ArrayType()) vela_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='VelA', parameter_context=vela_ctxt.dump())) velb_ctxt = ParameterContext('VelB', param_type=ArrayType()) velb_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='VelB', parameter_context=velb_ctxt.dump())) velc_ctxt = ParameterContext('VelC', param_type=ArrayType()) velc_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='VelC', parameter_context=velc_ctxt.dump())) veld_ctxt = ParameterContext('VelD', param_type=ArrayType()) veld_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='VelD', parameter_context=veld_ctxt.dump())) mx_ctxt = ParameterContext('Mx', param_type=ArrayType()) mx_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Mx', parameter_context=mx_ctxt.dump())) my_ctxt = ParameterContext('My', param_type=ArrayType()) my_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='My', parameter_context=my_ctxt.dump())) mz_ctxt = ParameterContext('Mz', param_type=ArrayType()) mz_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Mz', parameter_context=mz_ctxt.dump())) pitch_ctxt = ParameterContext('Pitch', param_type=ArrayType()) pitch_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Pitch', parameter_context=pitch_ctxt.dump())) roll_ctxt = ParameterContext('Roll', param_type=ArrayType()) roll_ctxt.uom = 'unknown' context_ids.append( self.dataset_management.create_parameter_context( name='Roll', parameter_context=roll_ctxt.dump())) return context_ids
def _create_parameter_dictionary(self): pdict = ParameterDictionary() t_ctxt = ParameterContext('c_wpt_y_lmc', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_water_cond', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_y_lmc', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('u_hd_fin_ap_inflection_holdoff', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_m_present_time', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_leakdetect_voltage_forward', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_bb3slo_b660_scaled', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('c_science_send_all', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_gps_status', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_water_vx', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_water_vy', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('c_heading', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_fl3slo_chlor_units', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('u_hd_fin_ap_gain', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_vacuum', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('u_min_water_depth', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_gps_lat', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_veh_temp', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('f_fin_offset', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('u_hd_fin_ap_hardover_holdoff', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('c_alt_time', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_present_time', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_heading', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_bb3slo_b532_scaled', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_fl3slo_cdom_units', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_fin', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('x_cycle_overrun_in_ms', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_water_pressure', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('u_hd_fin_ap_igain', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_fl3slo_phyco_units', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_battpos', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_bb3slo_b470_scaled', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_lat', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_gps_lon', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_ctd41cp_timestamp', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_pressure', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('c_wpt_x_lmc', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('c_ballast_pumped', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('x_lmc_xy_source', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_lon', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_avg_speed', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('sci_water_temp', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('u_pitch_ap_gain', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_roll', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_tot_num_inflections', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_x_lmc', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('u_pitch_ap_deadband', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_final_water_vy', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_final_water_vx', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_water_depth', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_leakdetect_voltage', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('u_pitch_max_delta_battpos', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_coulomb_amphr', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) t_ctxt = ParameterContext('m_pitch', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) t_ctxt.uom = 'unknown' pdict.add_context(t_ctxt) return pdict
def _setup_resources(self): # TODO: some or all of this (or some variation) should move to DAMS' # Build the test resources for the dataset dms_cli = DatasetManagementServiceClient() dams_cli = DataAcquisitionManagementServiceClient() dpms_cli = DataProductManagementServiceClient() rr_cli = ResourceRegistryServiceClient() pubsub_cli = PubsubManagementServiceClient() eda = ExternalDatasetAgent(name='example data agent', handler_module=self.DVR_CONFIG['dvr_mod'], handler_class=self.DVR_CONFIG['dvr_cls']) eda_id = dams_cli.create_external_dataset_agent(eda) eda_inst = ExternalDatasetAgentInstance( name='example dataset agent instance') eda_inst_id = dams_cli.create_external_dataset_agent_instance( eda_inst, external_dataset_agent_id=eda_id) # Create and register the necessary resources/objects # Create DataProvider dprov = ExternalDataProvider(name='example data provider', institution=Institution(), contact=ContactInformation()) dprov.contact.individual_names_given = 'Christopher Mueller' dprov.contact.email = '*****@*****.**' # Create DataSource dsrc = DataSource(name='example datasource', protocol_type='FILE', institution=Institution(), contact=ContactInformation()) dsrc.connection_params['base_data_url'] = '' dsrc.contact.individual_names_given = 'Tim Giguere' dsrc.contact.email = '*****@*****.**' # Create ExternalDataset ds_name = 'ruv_test_dataset' dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation()) dset.dataset_description.parameters['base_url'] = 'test_data/ruv/' dset.dataset_description.parameters[ 'list_pattern'] = 'RDLi_SEAB_2011_08_24_1600.ruv' dset.dataset_description.parameters['date_pattern'] = '%Y %m %d %H %M' dset.dataset_description.parameters[ 'date_extraction_pattern'] = 'RDLi_SEAB_([\d]{4})_([\d]{2})_([\d]{2})_([\d]{2})([\d]{2}).ruv' dset.dataset_description.parameters['temporal_dimension'] = None dset.dataset_description.parameters['zonal_dimension'] = None dset.dataset_description.parameters['meridional_dimension'] = None dset.dataset_description.parameters['vertical_dimension'] = None dset.dataset_description.parameters['variables'] = [] # Create DataSourceModel dsrc_model = DataSourceModel(name='ruv_model') #dsrc_model.model = 'RUV' dsrc_model.data_handler_module = 'N/A' dsrc_model.data_handler_class = 'N/A' ## Run everything through DAMS ds_id = dams_cli.create_external_dataset(external_dataset=dset) ext_dprov_id = dams_cli.create_external_data_provider( external_data_provider=dprov) ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc) ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model) # Register the ExternalDataset dproducer_id = dams_cli.register_external_data_set( external_dataset_id=ds_id) # Or using each method dams_cli.assign_data_source_to_external_data_provider( data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id) dams_cli.assign_data_source_to_data_model( data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id) dams_cli.assign_external_dataset_to_data_source( external_dataset_id=ds_id, data_source_id=ext_dsrc_id) dams_cli.assign_external_dataset_to_agent_instance( external_dataset_id=ds_id, agent_instance_id=eda_inst_id) pdict = ParameterDictionary() t_ctxt = ParameterContext( 'data', param_type=QuantityType(value_encoding=numpy.dtype('int64'))) t_ctxt.axis = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 01-01-1970' pdict.add_context(t_ctxt) #create temp streamdef so the data product can create the stream pc_list = [] for pc_k, pc in pdict.iteritems(): pc_list.append(dms_cli.create_parameter_context( pc_k, pc[1].dump())) pdict_id = dms_cli.create_parameter_dictionary('ruv_param_dict', pc_list) streamdef_id = pubsub_cli.create_stream_definition( name="ruv", description="stream def for ruv testing", parameter_dictionary_id=pdict_id) dprod = IonObject(RT.DataProduct, name='ruv_parsed_product', description='parsed ruv product') # Generate the data product and associate it to the ExternalDataset dproduct_id = dpms_cli.create_data_product( data_product=dprod, stream_definition_id=streamdef_id) dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id) stream_id, assn = rr_cli.find_objects(subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True) stream_id = stream_id[0] log.info('Created resources: {0}'.format({ 'ExternalDataset': ds_id, 'ExternalDataProvider': ext_dprov_id, 'DataSource': ext_dsrc_id, 'DataSourceModel': ext_dsrc_model_id, 'DataProducer': dproducer_id, 'DataProduct': dproduct_id, 'Stream': stream_id })) #CBM: Eventually, probably want to group this crap somehow - not sure how yet... # Create the logger for receiving publications _, stream_route, _ = self.create_stream_and_logger(name='ruv', stream_id=stream_id) self.EDA_RESOURCE_ID = ds_id self.EDA_NAME = ds_name self.DVR_CONFIG['dh_cfg'] = { 'TESTING': True, 'stream_id': stream_id, 'stream_route': stream_route, 'external_dataset_res': dset, 'param_dictionary': pdict.dump(), 'data_producer_id': dproducer_id, # CBM: Should this be put in the main body of the config - with mod & cls? 'max_records': 20, }
def build_contexts(): ''' Builds the relevant parameter context objects ''' contexts = [] cond_ctxt = ParameterContext( 'conductivity', param_type=QuantityType(value_encoding=np.float32)) cond_ctxt.uom = 'unknown' cond_ctxt.fill_value = 0e0 contexts.append(cond_ctxt) pres_ctxt = ParameterContext( 'pressure', param_type=QuantityType(value_encoding=np.float32)) pres_ctxt.uom = 'Pascal' pres_ctxt.fill_value = 0x0 contexts.append(pres_ctxt) sal_ctxt = ParameterContext( 'salinity', param_type=QuantityType(value_encoding=np.float32)) sal_ctxt.uom = 'PSU' sal_ctxt.fill_value = 0x0 contexts.append(sal_ctxt) den_ctxt = ParameterContext( 'density', param_type=QuantityType(value_encoding=np.float32)) den_ctxt.uom = 'kg/m3' den_ctxt.fill_value = 0x0 contexts.append(den_ctxt) temp_ctxt = ParameterContext( 'temp', param_type=QuantityType(value_encoding=np.float32)) temp_ctxt.uom = 'degree_Celsius' temp_ctxt.fill_value = 0e0 contexts.append(temp_ctxt) t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.int64)) t_ctxt.uom = 'seconds since 1970-01-01' t_ctxt.fill_value = 0x0 contexts.append(t_ctxt) lat_ctxt = ParameterContext( 'lat', param_type=QuantityType(value_encoding=np.float32)) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' lat_ctxt.fill_value = 0e0 contexts.append(lat_ctxt) lon_ctxt = ParameterContext( 'lon', param_type=QuantityType(value_encoding=np.float32)) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' lon_ctxt.fill_value = 0e0 contexts.append(lon_ctxt) raw_ctxt = ParameterContext('raw', param_type=ArrayType()) raw_ctxt.description = 'raw binary string values' raw_ctxt.uom = 'utf-8 byte string' raw_ctxt.fill_value = 0x0 contexts.append(raw_ctxt) port_ts_ctxt = ParameterContext( name='port_timestamp', param_type=QuantityType(value_encoding=np.float64)) port_ts_ctxt._derived_from_name = 'time' port_ts_ctxt.uom = 'seconds' port_ts_ctxt.fill_value = -1 contexts.append(port_ts_ctxt) driver_ts_ctxt = ParameterContext( name='driver_timestamp', param_type=QuantityType(value_encoding=np.float64)) driver_ts_ctxt._derived_from_name = 'time' driver_ts_ctxt.uom = 'seconds' driver_ts_ctxt.fill_value = -1 contexts.append(driver_ts_ctxt) internal_ts_ctxt = ParameterContext( name='internal_timestamp', param_type=QuantityType(value_encoding=np.float64)) internal_ts_ctxt._derived_from_name = 'time' internal_ts_ctxt.uom = 'seconds' internal_ts_ctxt.fill_value = -1 contexts.append(internal_ts_ctxt) timer_num_ctxt = ParameterContext( name='timer', param_type=QuantityType(value_encoding=np.float64)) timer_num_ctxt.fill_value = -1 contexts.append(timer_num_ctxt) serial_num_ctxt = ParameterContext( name='serial_num', param_type=QuantityType(value_encoding=np.int32)) serial_num_ctxt.fill_value = -1 contexts.append(serial_num_ctxt) count_ctxt = ParameterContext( name='counts', param_type=QuantityType(value_encoding=np.uint64)) count_ctxt.fill_value = -1 contexts.append(count_ctxt) checksum_ctxt = ParameterContext( name='checksum', param_type=QuantityType(value_encoding=np.int32)) checksum_ctxt.fill_value = -1 contexts.append(checksum_ctxt) pref_ts_ctxt = ParameterContext( name='preferred_timestamp', param_type=QuantityType(value_encoding=np.uint64)) pref_ts_ctxt.description = 'name of preferred timestamp' pref_ts_ctxt.fill_value = 0x0 contexts.append(pref_ts_ctxt) # TODO: This should probably be of type CategoryType when implemented qual_flag_ctxt = ParameterContext(name='quality_flag', param_type=ArrayType()) qual_flag_ctxt.description = 'flag indicating quality' qual_flag_ctxt.fill_value = None contexts.append(qual_flag_ctxt) viz_ts_ctxt = ParameterContext( name='viz_timestamp', param_type=QuantityType(value_encoding=np.float64)) viz_ts_ctxt._derived_from_name = 'time' viz_ts_ctxt.uom = 'seconds' viz_ts_ctxt.fill_value = -1 contexts.append(viz_ts_ctxt) viz_prod_type_ctxt = ParameterContext(name='viz_product_type', param_type=ArrayType()) viz_prod_type_ctxt.fill_value = None contexts.append(viz_prod_type_ctxt) image_obj_ctxt = ParameterContext(name='image_obj', param_type=ArrayType()) image_obj_ctxt.fill_value = None contexts.append(image_obj_ctxt) image_name_ctxt = ParameterContext(name='image_name', param_type=ArrayType()) contexts.append(image_name_ctxt) content_type_ctxt = ParameterContext(name='content_type', param_type=ArrayType()) contexts.append(content_type_ctxt) gdt_ctxt = ParameterContext(name='google_dt_components', param_type=RecordType()) contexts.append(gdt_ctxt) mpl_ctxt = ParameterContext(name='mpl_graph', param_type=RecordType()) contexts.append(mpl_ctxt) dummy_ctxt = ParameterContext( name='dummy', param_type=QuantityType(value_encoding=np.int64)) contexts.append(dummy_ctxt) return contexts
def create_parameters(cls): ''' WARNING: This method is a wrapper intended only for tests, it should not be used in production code. It probably will not align to most datasets. ''' pdict = ParameterDictionary() t_ctxt = ParameterContext( 'time', param_type=QuantityType(value_encoding=np.int64)) t_ctxt.axis = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 1970-01-01' t_ctxt.fill_value = 0x0 pdict.add_context(t_ctxt) lat_ctxt = ParameterContext( 'lat', param_type=QuantityType(value_encoding=np.float32)) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' lat_ctxt.fill_value = 0e0 pdict.add_context(lat_ctxt) lon_ctxt = ParameterContext( 'lon', param_type=QuantityType(value_encoding=np.float32)) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' lon_ctxt.fill_value = 0e0 pdict.add_context(lon_ctxt) temp_ctxt = ParameterContext( 'temp', param_type=QuantityType(value_encoding=np.float32)) temp_ctxt.uom = 'degree_Celsius' temp_ctxt.fill_value = 0e0 pdict.add_context(temp_ctxt) cond_ctxt = ParameterContext( 'conductivity', param_type=QuantityType(value_encoding=np.float32)) cond_ctxt.uom = 'unknown' cond_ctxt.fill_value = 0e0 pdict.add_context(cond_ctxt) data_ctxt = ParameterContext( 'data', param_type=QuantityType(value_encoding=np.int8)) data_ctxt.uom = 'byte' data_ctxt.fill_value = 0x0 pdict.add_context(data_ctxt) pres_ctxt = ParameterContext( 'pressure', param_type=QuantityType(value_encoding=np.float32)) pres_ctxt.uom = 'Pascal' pres_ctxt.fill_value = 0x0 pdict.add_context(pres_ctxt) sal_ctxt = ParameterContext( 'salinity', param_type=QuantityType(value_encoding=np.float32)) sal_ctxt.uom = 'PSU' sal_ctxt.fill_value = 0x0 pdict.add_context(sal_ctxt) dens_ctxt = ParameterContext( 'density', param_type=QuantityType(value_encoding=np.float32)) dens_ctxt.uom = 'unknown' dens_ctxt.fill_value = 0x0 pdict.add_context(dens_ctxt) return pdict
def get_boolean_type(self): return QuantityType(value_encoding=np.dtype('int8'))
def __init__(self, name, param_type=None, axis=None, fill_value=None, variability=None, uom=None, **kwargs): """ Construct a new ParameterContext object Must provide the 'name' argument. It can be either a string indicating the name of the parameter, or an exising ParameterContext object that should be used as a template. If 'name' is a ParameterContext, a keyword argument 'new_name' can be provided which will be used as the name for the new ParameterContext. If 'new_name' is not provided, the name will be the same as the template. When 'name' is a ParameterContext - the provided ParameterContext is utilized as a 'template' and it's attributes are copied into the new ParameterContext. If additional constructor arguments are provided (i.e. axis, fill_value, etc), they will be used preferentially over those in the 'template' ParameterContext. If param_type is specified, it must be compatible (i.e. equivalent) to the param_type in the 'template' ParameterContext. @param name The local name OR a 'template' ParameterContext. @param param_type The concrete AbstractParameterType; defaults to QuantityType if not provided @param axis The axis, typically a member of AxisTypeEnum; if not None, associated with the appropriate Domain @param fill_value The default fill value @param variability Indicates if the parameter is a function of time, space, both, or neither; Default is VariabilityEnum.BOTH @param **kwargs Keyword arguments matching members of ParameterContext.ATTRS are applied. Additional keyword arguments are copied and the copy is passed up to AbstractIdentifiable; see documentation for that class for details """ kwc=kwargs.copy() my_kwargs = {x:kwc.pop(x) for x in self.ATTRS if x in kwc} new_name = kwc.pop('new_name') if 'new_name' in kwc else None param_context = None if not isinstance(name, (basestring, ParameterContext)): raise SystemError('\'name\' must be an instance of either basestring or ParameterContext') if isinstance(name, ParameterContext): param_context = name name = new_name or param_context.name # TODO: If additional required constructor parameters are added, make sure they're dealt with in _load if param_context is None and name is None: raise SystemError('Must specify \'name\', which can be either a string or ParameterContext.') AbstractIdentifiable.__init__(self, **kwc) if not param_context is None: self._derived_from_name = param_context._derived_from_name self.name = name or self._derived_from_name # CBM TODO: Is this right? If param_type is provided, AND is equivalent to the clone's param_type, use it if not param_type is None and (param_type == param_context.param_type): self.param_type = param_type else: self.param_type = copy.deepcopy(param_context.param_type) # Ensure the param_type's name is the same as yours self.param_type.name = self.name self.axis = axis or param_context.axis if uom is not None: self.uom = uom elif param_context.uom is not None: self.uom = param_context.uom self.fill_value = fill_value or param_context.fill_value self.variability = variability or param_context.variability for a in self.ATTRS: setattr(self, a, kwargs[a] if a in my_kwargs else getattr(param_context, a)) else: # TODO: Should this be None? potential for mismatches if the self-given name happens to match a "blessed" name... self._derived_from_name = name self.name = name if param_type and not isinstance(param_type, AbstractParameterType): raise SystemError('\'param_type\' must be a concrete subclass of AbstractParameterType') self.param_type = param_type or QuantityType() # Ensure the param_type's name is the same as yours self.param_type.name = self.name self.axis = axis or None if uom is not None: self.uom = uom if fill_value is not None: # Provided by ParameterType self.fill_value = fill_value self.variability = variability or VariabilityEnum.BOTH for a in self.ATTRS: setattr(self, a, kwargs[a] if a in my_kwargs else None)