def get_param_dict(param_dict_name=None):
raise NotImplementedError(
'This method has been replaced by DatasetManagementService, please use read_parameter_dictionary_by_name instead'
)
# read the file just once, not every time needed
global _PARAMETER_DICTIONARIES
global _PARAMETER_CONTEXTS
if not _PARAMETER_DICTIONARIES:
param_dict_defs_file = "res/config/param_dict_defs.yml"
with open(param_dict_defs_file, "r") as f_dict:
dict_string = f_dict.read()
_PARAMETER_DICTIONARIES = yaml.load(dict_string)
param_context_defs_file = "res/config/param_context_defs.yml"
with open(param_context_defs_file, "r") as f_ctxt:
ctxt_string = f_ctxt.read()
_PARAMETER_CONTEXTS = yaml.load(ctxt_string)
# make sure we have the one requested
context_names = _PARAMETER_DICTIONARIES[param_dict_name]
for name in context_names:
if not _PARAMETER_CONTEXTS.has_key(name):
raise AssertionError(
'The parameter dict has a context that does not exist in the parameter context defs specified in yml: %s'
% name)
# package and ship
pdict = ParameterDictionary()
for ctxt_name in context_names:
param_context = ParameterContext.load(_PARAMETER_CONTEXTS[ctxt_name])
pdict.add_context(param_context)
return pdict
def _create_parameter_dictionary(self):
pdict = ParameterDictionary()
lat_ctxt = ParameterContext(
'lat',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
pdict.add_context(lat_ctxt)
lon_ctxt = ParameterContext(
'lon',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
pdict.add_context(lon_ctxt)
temp_ctxt = ParameterContext(
'water_temperature',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = 'degree_Celsius'
pdict.add_context(temp_ctxt)
temp_ctxt = ParameterContext(
'water_temperature_bottom',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = 'degree_Celsius'
pdict.add_context(temp_ctxt)
temp_ctxt = ParameterContext(
'water_temperature_middle',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = 'degree_Celsius'
pdict.add_context(temp_ctxt)
temp_ctxt = ParameterContext(
'z',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = 'meters'
pdict.add_context(temp_ctxt)
cond_ctxt = ParameterContext(
'streamflow',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
cond_ctxt.uom = 'unknown'
pdict.add_context(cond_ctxt)
pres_ctxt = ParameterContext(
'specific_conductance',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
pres_ctxt.uom = 'unknown'
pdict.add_context(pres_ctxt)
pres_ctxt = ParameterContext(
'data_qualifier',
param_type=QuantityType(value_encoding=numpy.dtype('bool')))
pres_ctxt.uom = 'unknown'
pdict.add_context(pres_ctxt)
return pdict
def _create_parameter(self):
pdict = ParameterDictionary()
pdict = self._add_location_time_ctxt(pdict)
pres_ctxt = ParameterContext(
'pressure', param_type=QuantityType(value_encoding=numpy.float32))
pres_ctxt.uom = 'Pascal'
pres_ctxt.fill_value = 0x0
pdict.add_context(pres_ctxt)
temp_ctxt = ParameterContext(
'temp', param_type=QuantityType(value_encoding=numpy.float32))
temp_ctxt.uom = 'degree_Celsius'
temp_ctxt.fill_value = 0e0
pdict.add_context(temp_ctxt)
cond_ctxt = ParameterContext(
'conductivity',
param_type=QuantityType(value_encoding=numpy.float32))
cond_ctxt.uom = 'unknown'
cond_ctxt.fill_value = 0e0
pdict.add_context(cond_ctxt)
return pdict
def _create_input_param_dict_for_test(self, parameter_dict_name=''):
pdict = ParameterDictionary()
t_ctxt = ParameterContext(
'time',
param_type=QuantityType(value_encoding=numpy.dtype('float64')))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1900'
pdict.add_context(t_ctxt)
cond_ctxt = ParameterContext(
'conductivity',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
cond_ctxt.uom = ''
pdict.add_context(cond_ctxt)
pres_ctxt = ParameterContext(
'pressure',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
pres_ctxt.uom = ''
pdict.add_context(pres_ctxt)
temp_ctxt = ParameterContext(
'temperature',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = ''
pdict.add_context(temp_ctxt)
dens_ctxt = ParameterContext(
'density',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
dens_ctxt.uom = ''
pdict.add_context(dens_ctxt)
sal_ctxt = ParameterContext(
'salinity',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
sal_ctxt.uom = ''
pdict.add_context(sal_ctxt)
#create temp streamdef so the data product can create the stream
pc_list = []
for pc_k, pc in pdict.iteritems():
ctxt_id = self.dataset_management.create_parameter_context(
pc_k, pc[1].dump())
pc_list.append(ctxt_id)
self.addCleanup(self.dataset_management.delete_parameter_context,
ctxt_id)
pdict_id = self.dataset_management.create_parameter_dictionary(
parameter_dict_name, pc_list)
self.addCleanup(self.dataset_management.delete_parameter_dictionary,
pdict_id)
return pdict_id
def rdt_to_granule(self, context, value_array, comp_val=None):
pdict = ParameterDictionary()
pdict.add_context(context)
rdt = RecordDictionaryTool(param_dictionary=pdict)
rdt['test'] = value_array
granule = rdt.to_granule()
rdt2 = RecordDictionaryTool.load_from_granule(granule)
testval = comp_val if comp_val is not None else value_array
actual = rdt2['test']
if isinstance(testval, basestring):
self.assertEquals(testval, actual)
else:
np.testing.assert_array_equal(testval, actual)
def get_param_dict(self):
pdict = ParameterDictionary()
cond_ctxt = ParameterContext(
'conductivity', param_type=QuantityType(value_encoding=np.float64))
cond_ctxt.uom = 'unknown'
cond_ctxt.fill_value = 0e0
pdict.add_context(cond_ctxt)
pres_ctxt = ParameterContext(
'pressure', param_type=QuantityType(value_encoding=np.float64))
pres_ctxt.uom = 'unknown'
pres_ctxt.fill_value = 0x0
pdict.add_context(pres_ctxt)
temp_ctxt = ParameterContext(
'temperature', param_type=QuantityType(value_encoding=np.float64))
temp_ctxt.uom = 'unknown'
temp_ctxt.fill_value = 0x0
pdict.add_context(temp_ctxt)
oxy_ctxt = ParameterContext(
'oxygen', param_type=QuantityType(value_encoding=np.float64))
oxy_ctxt.uom = 'unknown'
oxy_ctxt.fill_value = 0x0
pdict.add_context(oxy_ctxt)
internal_ts_ctxt = ParameterContext(
name='internal_timestamp',
param_type=QuantityType(value_encoding=np.float64))
internal_ts_ctxt._derived_from_name = 'time'
internal_ts_ctxt.uom = 'seconds'
internal_ts_ctxt.fill_value = -1
pdict.add_context(internal_ts_ctxt, is_temporal=True)
driver_ts_ctxt = ParameterContext(
name='driver_timestamp',
param_type=QuantityType(value_encoding=np.float64))
driver_ts_ctxt._derived_from_name = 'time'
driver_ts_ctxt.uom = 'seconds'
driver_ts_ctxt.fill_value = -1
pdict.add_context(driver_ts_ctxt)
return pdict
def rdt_to_granule(self, context, value_array, comp_val=None):
time = ParameterContext(
name='time', param_type=QuantityType(value_encoding=np.float64))
pdict = ParameterDictionary()
pdict.add_context(time, is_temporal=True)
pdict.add_context(context)
rdt = RecordDictionaryTool(param_dictionary=pdict)
rdt['time'] = np.arange(len(value_array))
rdt['test'] = value_array
granule = rdt.to_granule()
rdt2 = RecordDictionaryTool.load_from_granule(granule)
testval = comp_val if comp_val is not None else value_array
actual = rdt2['test']
if isinstance(testval, basestring):
self.assertEquals(testval, actual)
else:
np.testing.assert_array_equal(testval, actual)
def adhoc_get_parameter_dictionary(stream_name):
"""
@param stream_name IGNORED in this adhoc function; it returns the same
ParameterDictionary definition always.
@retval corresponding ParameterDictionary.
"""
#@TODO Luke - Maybe we can make this a bit more versatile, we could make this a standard pdict...
pdict = ParameterDictionary()
# ctxt = ParameterContext('value', param_type=QuantityType(value_encoding=numpy.float32))
ctxt = ParameterContext('value', param_type=QuantityType(value_encoding=numpy.dtype('float64')))
ctxt.uom = 'unknown'
ctxt.fill_value = 0e0
pdict.add_context(ctxt)
ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=numpy.dtype('int64')))
ctxt.axis = AxisTypeEnum.TIME
ctxt.uom = 'seconds since 01-01-1970'
pdict.add_context(ctxt)
ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
ctxt.axis = AxisTypeEnum.LON
ctxt.uom = 'degree_east'
pdict.add_context(ctxt)
ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
ctxt.axis = AxisTypeEnum.LAT
ctxt.uom = 'degree_north'
pdict.add_context(ctxt)
ctxt = ParameterContext('height', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
ctxt.axis = AxisTypeEnum.HEIGHT
ctxt.uom = 'unknown'
pdict.add_context(ctxt)
return pdict
def cov_io(self, context, value_array, comp_val=None):
pdict = ParameterDictionary()
time = ParameterContext(
name='time', param_type=QuantityType(value_encoding=np.float64))
pdict.add_context(context)
pdict.add_context(time, True)
# Construct temporal and spatial Coordinate Reference System objects
tcrs = CRS([AxisTypeEnum.TIME])
scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT])
# Construct temporal and spatial Domain objects
tdom = GridDomain(GridShape('temporal', [0]), tcrs,
MutabilityEnum.EXTENSIBLE) # 1d (timeline)
sdom = GridDomain(GridShape('spatial',
[0]), scrs, MutabilityEnum.IMMUTABLE
) # 0d spatial topology (station/trajectory)
# Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain
cov = SimplexCoverage('test_data',
create_guid(),
'sample coverage_model',
parameter_dictionary=pdict,
temporal_domain=tdom,
spatial_domain=sdom)
self.addCleanup(shutil.rmtree, cov.persistence_dir)
cov.insert_timesteps(len(value_array))
cov.set_parameter_values('test',
tdoa=slice(0, len(value_array)),
value=value_array)
comp_val = comp_val if comp_val is not None else value_array
testval = cov.get_parameter_values('test')
try:
np.testing.assert_array_equal(testval, comp_val)
except:
print repr(value_array)
raise
def _create_parameter_dictionary(self):
pdict = ParameterDictionary()
t_ctxt = ParameterContext('c_wpt_y_lmc', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('sci_water_cond', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_y_lmc', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('u_hd_fin_ap_inflection_holdoff', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('sci_m_present_time', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_leakdetect_voltage_forward', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('sci_bb3slo_b660_scaled', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('c_science_send_all', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_gps_status', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_water_vx', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_water_vy', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('c_heading', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('sci_fl3slo_chlor_units', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('u_hd_fin_ap_gain', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_vacuum', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('u_min_water_depth', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_gps_lat', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_veh_temp', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('f_fin_offset', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('u_hd_fin_ap_hardover_holdoff', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('c_alt_time', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_present_time', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_heading', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('sci_bb3slo_b532_scaled', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('sci_fl3slo_cdom_units', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_fin', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('x_cycle_overrun_in_ms', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('sci_water_pressure', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('u_hd_fin_ap_igain', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('sci_fl3slo_phyco_units', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_battpos', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('sci_bb3slo_b470_scaled', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_lat', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_gps_lon', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('sci_ctd41cp_timestamp', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_pressure', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('c_wpt_x_lmc', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('c_ballast_pumped', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('x_lmc_xy_source', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_lon', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_avg_speed', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('sci_water_temp', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('u_pitch_ap_gain', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_roll', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_tot_num_inflections', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_x_lmc', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('u_pitch_ap_deadband', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_final_water_vy', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_final_water_vx', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_water_depth', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_leakdetect_voltage', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('u_pitch_max_delta_battpos', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_coulomb_amphr', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
t_ctxt = ParameterContext('m_pitch', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
t_ctxt.uom = 'unknown'
pdict.add_context(t_ctxt)
return pdict
def _setup_resources(self):
# TODO: some or all of this (or some variation) should move to DAMS'
# Build the test resources for the dataset
dms_cli = DatasetManagementServiceClient()
dams_cli = DataAcquisitionManagementServiceClient()
dpms_cli = DataProductManagementServiceClient()
rr_cli = ResourceRegistryServiceClient()
pubsub_cli = PubsubManagementServiceClient()
eda = ExternalDatasetAgent(name='example data agent',
handler_module=self.DVR_CONFIG['dvr_mod'],
handler_class=self.DVR_CONFIG['dvr_cls'])
eda_id = dams_cli.create_external_dataset_agent(eda)
eda_inst = ExternalDatasetAgentInstance(
name='example dataset agent instance')
eda_inst_id = dams_cli.create_external_dataset_agent_instance(
eda_inst, external_dataset_agent_id=eda_id)
# Create and register the necessary resources/objects
# Create DataProvider
dprov = ExternalDataProvider(name='example data provider',
institution=Institution(),
contact=ContactInformation())
dprov.contact.individual_names_given = 'Christopher Mueller'
dprov.contact.email = '*****@*****.**'
# Create DataSource
dsrc = DataSource(name='example datasource',
protocol_type='FILE',
institution=Institution(),
contact=ContactInformation())
dsrc.connection_params['base_data_url'] = ''
dsrc.contact.individual_names_given = 'Tim Giguere'
dsrc.contact.email = '*****@*****.**'
# Create ExternalDataset
ds_name = 'ruv_test_dataset'
dset = ExternalDataset(name=ds_name,
dataset_description=DatasetDescription(),
update_description=UpdateDescription(),
contact=ContactInformation())
dset.dataset_description.parameters['base_url'] = 'test_data/ruv/'
dset.dataset_description.parameters[
'list_pattern'] = 'RDLi_SEAB_2011_08_24_1600.ruv'
dset.dataset_description.parameters['date_pattern'] = '%Y %m %d %H %M'
dset.dataset_description.parameters[
'date_extraction_pattern'] = 'RDLi_SEAB_([\d]{4})_([\d]{2})_([\d]{2})_([\d]{2})([\d]{2}).ruv'
dset.dataset_description.parameters['temporal_dimension'] = None
dset.dataset_description.parameters['zonal_dimension'] = None
dset.dataset_description.parameters['meridional_dimension'] = None
dset.dataset_description.parameters['vertical_dimension'] = None
dset.dataset_description.parameters['variables'] = []
# Create DataSourceModel
dsrc_model = DataSourceModel(name='ruv_model')
#dsrc_model.model = 'RUV'
dsrc_model.data_handler_module = 'N/A'
dsrc_model.data_handler_class = 'N/A'
## Run everything through DAMS
ds_id = dams_cli.create_external_dataset(external_dataset=dset)
ext_dprov_id = dams_cli.create_external_data_provider(
external_data_provider=dprov)
ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc)
ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model)
# Register the ExternalDataset
dproducer_id = dams_cli.register_external_data_set(
external_dataset_id=ds_id)
# Or using each method
dams_cli.assign_data_source_to_external_data_provider(
data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id)
dams_cli.assign_data_source_to_data_model(
data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id)
dams_cli.assign_external_dataset_to_data_source(
external_dataset_id=ds_id, data_source_id=ext_dsrc_id)
dams_cli.assign_external_dataset_to_agent_instance(
external_dataset_id=ds_id, agent_instance_id=eda_inst_id)
pdict = ParameterDictionary()
t_ctxt = ParameterContext(
'data',
param_type=QuantityType(value_encoding=numpy.dtype('int64')))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1970'
pdict.add_context(t_ctxt)
#create temp streamdef so the data product can create the stream
pc_list = []
for pc_k, pc in pdict.iteritems():
pc_list.append(dms_cli.create_parameter_context(
pc_k, pc[1].dump()))
pdict_id = dms_cli.create_parameter_dictionary('ruv_param_dict',
pc_list)
streamdef_id = pubsub_cli.create_stream_definition(
name="ruv",
description="stream def for ruv testing",
parameter_dictionary_id=pdict_id)
dprod = IonObject(RT.DataProduct,
name='ruv_parsed_product',
description='parsed ruv product')
# Generate the data product and associate it to the ExternalDataset
dproduct_id = dpms_cli.create_data_product(
data_product=dprod, stream_definition_id=streamdef_id)
dams_cli.assign_data_product(input_resource_id=ds_id,
data_product_id=dproduct_id)
stream_id, assn = rr_cli.find_objects(subject=dproduct_id,
predicate=PRED.hasStream,
object_type=RT.Stream,
id_only=True)
stream_id = stream_id[0]
log.info('Created resources: {0}'.format({
'ExternalDataset': ds_id,
'ExternalDataProvider': ext_dprov_id,
'DataSource': ext_dsrc_id,
'DataSourceModel': ext_dsrc_model_id,
'DataProducer': dproducer_id,
'DataProduct': dproduct_id,
'Stream': stream_id
}))
#CBM: Eventually, probably want to group this crap somehow - not sure how yet...
# Create the logger for receiving publications
_, stream_route, _ = self.create_stream_and_logger(name='ruv',
stream_id=stream_id)
self.EDA_RESOURCE_ID = ds_id
self.EDA_NAME = ds_name
self.DVR_CONFIG['dh_cfg'] = {
'TESTING': True,
'stream_id': stream_id,
'stream_route': stream_route,
'external_dataset_res': dset,
'param_dictionary': pdict.dump(),
'data_producer_id':
dproducer_id, # CBM: Should this be put in the main body of the config - with mod & cls?
'max_records': 20,
}
def sync_rdt_with_coverage(self,
coverage=None,
tdoa=None,
start_time=None,
end_time=None,
stride_time=None,
parameters=None):
'''
Builds a granule based on the coverage
'''
if coverage is None:
coverage = self.coverage
slice_ = slice(None) # Defaults to all values
if tdoa is not None and isinstance(tdoa, slice):
slice_ = tdoa
elif stride_time is not None:
validate_is_instance(start_time, Number,
'start_time must be a number for striding.')
validate_is_instance(end_time, Number,
'end_time must be a number for striding.')
validate_is_instance(stride_time, Number,
'stride_time must be a number for striding.')
ugly_range = np.arange(start_time, end_time, stride_time)
idx_values = [
TimeUtils.get_relative_time(coverage, i) for i in ugly_range
]
slice_ = [idx_values]
elif not (start_time is None and end_time is None):
time_var = coverage._temporal_param_name
uom = coverage.get_parameter_context(time_var).uom
if start_time is not None:
start_units = TimeUtils.ts_to_units(uom, start_time)
log.info('Units: %s', start_units)
start_idx = TimeUtils.get_relative_time(coverage, start_units)
log.info('Start Index: %s', start_idx)
start_time = start_idx
if end_time is not None:
end_units = TimeUtils.ts_to_units(uom, end_time)
log.info('End units: %s', end_units)
end_idx = TimeUtils.get_relative_time(coverage, end_units)
log.info('End index: %s', end_idx)
end_time = end_idx
slice_ = slice(start_time, end_time, stride_time)
log.info('Slice: %s', slice_)
if parameters is not None:
pdict = ParameterDictionary()
params = set(coverage.list_parameters()).intersection(parameters)
for param in params:
pdict.add_context(coverage.get_parameter_context(param))
rdt = RecordDictionaryTool(param_dictionary=pdict)
self.pdict = pdict
else:
rdt = RecordDictionaryTool(
param_dictionary=coverage.parameter_dictionary)
fields = coverage.list_parameters()
if parameters is not None:
fields = set(fields).intersection(parameters)
for d in fields:
rdt[d] = coverage.get_parameter_values(d, tdoa=slice_)
self.rdt = rdt # Sync
def create_parameters(cls):
'''
WARNING: This method is a wrapper intended only for tests, it should not be used in production code.
It probably will not align to most datasets.
'''
pdict = ParameterDictionary()
t_ctxt = ParameterContext(
'time', param_type=QuantityType(value_encoding=np.int64))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 1970-01-01'
t_ctxt.fill_value = 0x0
pdict.add_context(t_ctxt)
lat_ctxt = ParameterContext(
'lat', param_type=QuantityType(value_encoding=np.float32))
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt.fill_value = 0e0
pdict.add_context(lat_ctxt)
lon_ctxt = ParameterContext(
'lon', param_type=QuantityType(value_encoding=np.float32))
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt.fill_value = 0e0
pdict.add_context(lon_ctxt)
temp_ctxt = ParameterContext(
'temp', param_type=QuantityType(value_encoding=np.float32))
temp_ctxt.uom = 'degree_Celsius'
temp_ctxt.fill_value = 0e0
pdict.add_context(temp_ctxt)
cond_ctxt = ParameterContext(
'conductivity', param_type=QuantityType(value_encoding=np.float32))
cond_ctxt.uom = 'unknown'
cond_ctxt.fill_value = 0e0
pdict.add_context(cond_ctxt)
data_ctxt = ParameterContext(
'data', param_type=QuantityType(value_encoding=np.int8))
data_ctxt.uom = 'byte'
data_ctxt.fill_value = 0x0
pdict.add_context(data_ctxt)
pres_ctxt = ParameterContext(
'pressure', param_type=QuantityType(value_encoding=np.float32))
pres_ctxt.uom = 'Pascal'
pres_ctxt.fill_value = 0x0
pdict.add_context(pres_ctxt)
sal_ctxt = ParameterContext(
'salinity', param_type=QuantityType(value_encoding=np.float32))
sal_ctxt.uom = 'PSU'
sal_ctxt.fill_value = 0x0
pdict.add_context(sal_ctxt)
dens_ctxt = ParameterContext(
'density', param_type=QuantityType(value_encoding=np.float32))
dens_ctxt.uom = 'unknown'
dens_ctxt.fill_value = 0x0
pdict.add_context(dens_ctxt)
return pdict
