def defining_parameter_dictionary(self):
# Define the parameter context objects
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.int64))
t_ctxt.reference_frame = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 1970-01-01'
t_ctxt.fill_value = 0x0
lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=np.float32))
lat_ctxt.reference_frame = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt.fill_value = 0e0
lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=np.float32))
lon_ctxt.reference_frame = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt.fill_value = 0e0
height_ctxt = ParameterContext('height', param_type=QuantityType(value_encoding=np.float32))
height_ctxt.reference_frame = AxisTypeEnum.HEIGHT
height_ctxt.uom = 'meters'
height_ctxt.fill_value = 0e0
temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=np.float32))
temp_ctxt.uom = 'degree_Celsius'
temp_ctxt.fill_value = 0e0
data_ctxt = ParameterContext('data', param_type=QuantityType(value_encoding=np.int8))
data_ctxt.uom = 'byte'
data_ctxt.fill_value = 0x0
# Define the parameter dictionary objects
self.temp = ParameterDictionary()
self.temp.add_context(t_ctxt)
self.temp.add_context(lat_ctxt)
self.temp.add_context(lon_ctxt)
self.temp.add_context(height_ctxt)
self.temp.add_context(temp_ctxt)
self.temp.add_context(data_ctxt)
def setUp(self):
self._tx = TaxyTool()
self._tx.add_taxonomy_set('temp', 'long_temp_name')
self._tx.add_taxonomy_set('cond', 'long_cond_name')
self._tx.add_taxonomy_set('pres', 'long_pres_name')
self._tx.add_taxonomy_set('rdt')
self._tx.add_taxonomy_set('rdt2')
# map is {<local name>: <granule name or path>}
self._rdt = RecordDictionaryTool(taxonomy=self._tx)
self._pdict = ParameterDictionary()
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=numpy.dtype('int64')))
t_ctxt.reference_frame = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1970'
self._pdict.add_context(t_ctxt)
lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
lat_ctxt.reference_frame = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
self._pdict.add_context(lat_ctxt)
lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
lon_ctxt.reference_frame = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
self._pdict.add_context(lon_ctxt)
temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = 'degree_Celsius'
self._pdict.add_context(temp_ctxt)
cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
cond_ctxt.uom = 'unknown'
self._pdict.add_context(cond_ctxt)
pres_ctxt = ParameterContext('pres', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
pres_ctxt.uom = 'unknown'
self._pdict.add_context(pres_ctxt)
self._rdt_pdict = RecordDictionaryTool(param_dictionary=self._pdict)
def create_parameters(cls):
pdict = ParameterDictionary()
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.int64))
t_ctxt.reference_frame = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 1970-01-01'
t_ctxt.fill_value = 0x0
pdict.add_context(t_ctxt)
lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=np.float32))
lat_ctxt.reference_frame = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt.fill_value = 0e0
pdict.add_context(lat_ctxt)
lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=np.float32))
lon_ctxt.reference_frame = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt.fill_value = 0e0
pdict.add_context(lon_ctxt)
depth_ctxt = ParameterContext('depth', param_type=QuantityType(value_encoding=np.float32))
depth_ctxt.reference_frame = AxisTypeEnum.HEIGHT
depth_ctxt.uom = 'meters'
depth_ctxt.fill_value = 0e0
pdict.add_context(depth_ctxt)
temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=np.float32))
temp_ctxt.uom = 'degree_Celsius'
temp_ctxt.fill_value = 0e0
pdict.add_context(temp_ctxt)
cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=np.float32))
cond_ctxt.uom = 'unknown'
cond_ctxt.fill_value = 0e0
pdict.add_context(cond_ctxt)
data_ctxt = ParameterContext('data', param_type=QuantityType(value_encoding=np.int8))
data_ctxt.uom = 'byte'
data_ctxt.fill_value = 0x0
pdict.add_context(data_ctxt)
return pdict
def _setup_resources(self):
# TODO: some or all of this (or some variation) should move to DAMS'
# Build the test resources for the dataset
dams_cli = DataAcquisitionManagementServiceClient()
dpms_cli = DataProductManagementServiceClient()
rr_cli = ResourceRegistryServiceClient()
pubsub_cli = PubsubManagementServiceClient()
eda = ExternalDatasetAgent()
eda_id = dams_cli.create_external_dataset_agent(eda)
eda_inst = ExternalDatasetAgentInstance()
eda_inst_id = dams_cli.create_external_dataset_agent_instance(eda_inst, external_dataset_agent_id=eda_id)
# Create and register the necessary resources/objects
# Create DataProvider
dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation())
dprov.contact.name = 'Christopher Mueller'
dprov.contact.email = '*****@*****.**'
# Create DataSource
dsrc = DataSource(protocol_type='DAP', institution=Institution(), contact=ContactInformation())
dsrc.connection_params['base_data_url'] = ''
dsrc.contact.name='Tim Giguere'
dsrc.contact.email = '*****@*****.**'
# Create ExternalDataset
ds_name = 'usgs_test_dataset'
dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation())
# The usgs.nc test dataset is a download of the R1 dataset found here:
# http://thredds-test.oceanobservatories.org/thredds/dodsC/ooiciData/E66B1A74-A684-454A-9ADE-8388C2C634E5.ncml
dset.dataset_description.parameters['dataset_path'] = 'test_data/usgs.nc'
dset.dataset_description.parameters['temporal_dimension'] = 'time'
dset.dataset_description.parameters['zonal_dimension'] = 'lon'
dset.dataset_description.parameters['meridional_dimension'] = 'lat'
dset.dataset_description.parameters['vertical_dimension'] = 'z'
dset.dataset_description.parameters['variables'] = [
'water_temperature',
'streamflow',
'water_temperature_bottom',
'water_temperature_middle',
'specific_conductance',
'data_qualifier',
]
# Create DataSourceModel
dsrc_model = DataSourceModel(name='dap_model')
dsrc_model.model = 'DAP'
dsrc_model.data_handler_module = 'N/A'
dsrc_model.data_handler_class = 'N/A'
## Run everything through DAMS
ds_id = dams_cli.create_external_dataset(external_dataset=dset)
ext_dprov_id = dams_cli.create_external_data_provider(external_data_provider=dprov)
ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc)
ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model)
# Register the ExternalDataset
dproducer_id = dams_cli.register_external_data_set(external_dataset_id=ds_id)
# Or using each method
dams_cli.assign_data_source_to_external_data_provider(data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id)
dams_cli.assign_data_source_to_data_model(data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id)
dams_cli.assign_external_dataset_to_data_source(external_dataset_id=ds_id, data_source_id=ext_dsrc_id)
dams_cli.assign_external_dataset_to_agent_instance(external_dataset_id=ds_id, agent_instance_id=eda_inst_id)
# dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=self.eda_id, agent_instance_id=self.eda_inst_id)
#create temp streamdef so the data product can create the stream
streamdef_id = pubsub_cli.create_stream_definition(name="temp", description="temp")
craft = CoverageCraft
sdom, tdom = craft.create_domains()
sdom = sdom.dump()
tdom = tdom.dump()
parameter_dictionary = craft.create_parameters()
parameter_dictionary = parameter_dictionary.dump()
dprod = IonObject(RT.DataProduct,
name='usgs_parsed_product',
description='parsed usgs product',
temporal_domain = tdom,
spatial_domain = sdom)
# Generate the data product and associate it to the ExternalDataset
dproduct_id = dpms_cli.create_data_product(data_product=dprod,
stream_definition_id=streamdef_id,
parameter_dictionary=parameter_dictionary)
dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id)
stream_id, assn = rr_cli.find_objects(subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True)
stream_id = stream_id[0]
log.info('Created resources: {0}'.format({'ExternalDataset':ds_id, 'ExternalDataProvider':ext_dprov_id, 'DataSource':ext_dsrc_id, 'DataSourceModel':ext_dsrc_model_id, 'DataProducer':dproducer_id, 'DataProduct':dproduct_id, 'Stream':stream_id}))
#CBM: Use CF standard_names
# ttool = TaxyTool()
# ttool.add_taxonomy_set('time','time')
# ttool.add_taxonomy_set('lon','longitude')
# ttool.add_taxonomy_set('lat','latitude')
# ttool.add_taxonomy_set('z','water depth')
# ttool.add_taxonomy_set('water_temperature', 'average water temperature')
# ttool.add_taxonomy_set('water_temperature_bottom','water temperature at bottom of water column')
# ttool.add_taxonomy_set('water_temperature_middle', 'water temperature at middle of water column')
# ttool.add_taxonomy_set('streamflow', 'flow velocity of stream')
# ttool.add_taxonomy_set('specific_conductance', 'specific conductance of water')
# ttool.add_taxonomy_set('data_qualifier','data qualifier flag')
#
# ttool.add_taxonomy_set('coords','This group contains coordinate parameters')
# ttool.add_taxonomy_set('data','This group contains data parameters')
# Create the logger for receiving publications
self.create_stream_and_logger(name='usgs',stream_id=stream_id)
pdict = ParameterDictionary()
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=numpy.dtype('int64')))
t_ctxt.reference_frame = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1970'
pdict.add_context(t_ctxt)
lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
lat_ctxt.reference_frame = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
pdict.add_context(lat_ctxt)
lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
lon_ctxt.reference_frame = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
pdict.add_context(lon_ctxt)
temp_ctxt = ParameterContext('water_temperature', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = 'degree_Celsius'
pdict.add_context(temp_ctxt)
temp_ctxt = ParameterContext('water_temperature_bottom', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = 'degree_Celsius'
pdict.add_context(temp_ctxt)
temp_ctxt = ParameterContext('water_temperature_middle', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = 'degree_Celsius'
pdict.add_context(temp_ctxt)
temp_ctxt = ParameterContext('z', param_type=QuantityType(value_encoding = numpy.dtype('float32')))
temp_ctxt.uom = 'meters'
pdict.add_context(temp_ctxt)
cond_ctxt = ParameterContext('streamflow', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
cond_ctxt.uom = 'unknown'
pdict.add_context(cond_ctxt)
pres_ctxt = ParameterContext('specific_conductance', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
pres_ctxt.uom = 'unknown'
pdict.add_context(pres_ctxt)
pres_ctxt = ParameterContext('data_qualifier', param_type=QuantityType(value_encoding=numpy.dtype('bool')))
pres_ctxt.uom = 'unknown'
pdict.add_context(pres_ctxt)
self.EDA_RESOURCE_ID = ds_id
self.EDA_NAME = ds_name
self.DVR_CONFIG['dh_cfg'] = {
'TESTING':True,
'stream_id':stream_id,
#'taxonomy':ttool.dump(),
'param_dictionary':pdict.dump(),
'data_producer_id':dproducer_id,#CBM: Should this be put in the main body of the config - with mod & cls?
'max_records':4,
}
def load():
from coverage_model.parameter import ParameterContext
from coverage_model.parameter_types import QuantityType, ArrayType, RecordType
from coverage_model.basic_types import AxisTypeEnum
import numpy as np
contexts = []
cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=np.float32))
cond_ctxt.uom = 'unknown'
cond_ctxt.fill_value = 0e0
contexts.append(cond_ctxt)
pres_ctxt = ParameterContext('pressure', param_type=QuantityType(value_encoding=np.float32))
pres_ctxt.uom = 'Pascal'
pres_ctxt.fill_value = 0x0
contexts.append(pres_ctxt)
sal_ctxt = ParameterContext('salinity', param_type=QuantityType(value_encoding=np.float32))
sal_ctxt.uom = 'PSU'
sal_ctxt.fill_value = 0x0
contexts.append(sal_ctxt)
den_ctxt = ParameterContext('density', param_type=QuantityType(value_encoding=np.float32))
den_ctxt.uom = 'kg/m3'
den_ctxt.fill_value = 0x0
contexts.append(den_ctxt)
temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=np.float32))
temp_ctxt.uom = 'degree_Celsius'
temp_ctxt.fill_value = 0e0
contexts.append(temp_ctxt)
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.int64))
t_ctxt.uom = 'seconds since 1970-01-01'
t_ctxt.fill_value = 0x0
contexts.append(t_ctxt)
lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=np.float32))
lat_ctxt.reference_frame = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt.fill_value = 0e0
contexts.append(lat_ctxt)
lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=np.float32))
lon_ctxt.reference_frame = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt.fill_value = 0e0
contexts.append(lon_ctxt)
raw_ctxt = ParameterContext('raw', param_type=ArrayType())
raw_ctxt.description = 'raw binary string values'
raw_ctxt.uom = 'utf-8 byte string'
contexts.append(raw_ctxt)
port_ts_ctxt = ParameterContext(name='port_timestamp', param_type=QuantityType(value_encoding=np.float64))
port_ts_ctxt._derived_from_name = 'time'
port_ts_ctxt.uom = 'seconds'
port_ts_ctxt.fill_value = -1
contexts.append(port_ts_ctxt)
driver_ts_ctxt = ParameterContext(name='driver_timestamp', param_type=QuantityType(value_encoding=np.float64))
driver_ts_ctxt._derived_from_name = 'time'
driver_ts_ctxt.uom = 'seconds'
driver_ts_ctxt.fill_value = -1
contexts.append(driver_ts_ctxt)
internal_ts_ctxt = ParameterContext(name='internal_timestamp', param_type=QuantityType(value_encoding=np.float64))
internal_ts_ctxt._derived_from_name = 'time'
internal_ts_ctxt.uom = 'seconds'
internal_ts_ctxt.fill_value = -1
contexts.append(internal_ts_ctxt)
timer_num_ctxt = ParameterContext(name='timer', param_type=QuantityType(value_encoding=np.float64))
timer_num_ctxt.fill_value = -1
contexts.append(timer_num_ctxt)
serial_num_ctxt = ParameterContext(name='serial_num', param_type=QuantityType(value_encoding=np.int32))
serial_num_ctxt.fill_value = -1
contexts.append(serial_num_ctxt)
count_ctxt = ParameterContext(name='counts', param_type=QuantityType(value_encoding=np.uint64))
count_ctxt.fill_value = -1
contexts.append(count_ctxt)
checksum_ctxt = ParameterContext(name='checksum', param_type=QuantityType(value_encoding=np.int32))
checksum_ctxt.fill_value = -1
contexts.append(checksum_ctxt)
pref_ts_ctxt = ParameterContext(name='preferred_timestamp', param_type=ArrayType())
pref_ts_ctxt.description = 'name of preferred timestamp'
contexts.append(pref_ts_ctxt)
# TODO: This should probably be of type CategoryType when implemented
qual_flag_ctxt = ParameterContext(name='quality_flag', param_type=ArrayType())
qual_flag_ctxt.description = 'flag indicating quality'
contexts.append(qual_flag_ctxt)
viz_ts_ctxt = ParameterContext(name='viz_timestamp', param_type=QuantityType(value_encoding=np.float64))
viz_ts_ctxt._derived_from_name = 'time'
viz_ts_ctxt.uom = 'seconds'
viz_ts_ctxt.fill_value = -1
contexts.append(viz_ts_ctxt)
viz_prod_type_ctxt = ParameterContext(name='viz_product_type', param_type=ArrayType())
contexts.append(viz_prod_type_ctxt)
image_obj_ctxt = ParameterContext(name='image_obj', param_type=ArrayType())
contexts.append(image_obj_ctxt)
image_name_ctxt = ParameterContext(name='image_name', param_type=ArrayType())
contexts.append(image_name_ctxt)
content_type_ctxt = ParameterContext(name='content_type', param_type=ArrayType())
contexts.append(content_type_ctxt)
gdt_ctxt = ParameterContext(name='google_dt_components', param_type=RecordType())
contexts.append(gdt_ctxt)
mpl_ctxt = ParameterContext(name='mpl_graph', param_type=RecordType())
contexts.append(mpl_ctxt)
dummy_ctxt = ParameterContext(name='dummy', param_type=QuantityType(value_encoding=np.int64))
contexts.append(dummy_ctxt)
return contexts
def _setup_resources(self):
# TODO: some or all of this (or some variation) should move to DAMS'
# Build the test resources for the dataset
dams_cli = DataAcquisitionManagementServiceClient()
dpms_cli = DataProductManagementServiceClient()
rr_cli = ResourceRegistryServiceClient()
pubsub_cli = PubsubManagementServiceClient()
eda = ExternalDatasetAgent()
eda_id = dams_cli.create_external_dataset_agent(eda)
eda_inst = ExternalDatasetAgentInstance()
eda_inst_id = dams_cli.create_external_dataset_agent_instance(eda_inst, external_dataset_agent_id=eda_id)
# Create and register the necessary resources/objects
# Create DataProvider
dprov = ExternalDataProvider(institution=Institution(), contact=ContactInformation())
dprov.contact.name = 'Christopher Mueller'
dprov.contact.email = '*****@*****.**'
# Create DataSource
dsrc = DataSource(protocol_type='FILE', institution=Institution(), contact=ContactInformation())
dsrc.connection_params['base_data_url'] = ''
dsrc.contact.name='Tim Giguere'
dsrc.contact.email = '*****@*****.**'
# Create ExternalDataset
ds_name = 'ruv_test_dataset'
dset = ExternalDataset(name=ds_name, dataset_description=DatasetDescription(), update_description=UpdateDescription(), contact=ContactInformation())
dset.dataset_description.parameters['base_url'] = 'test_data/ruv/'
dset.dataset_description.parameters['list_pattern'] = 'RDLi_SEAB_2011_08_24_1600.ruv'
dset.dataset_description.parameters['date_pattern'] = '%Y %m %d %H %M'
dset.dataset_description.parameters['date_extraction_pattern'] = 'RDLi_SEAB_([\d]{4})_([\d]{2})_([\d]{2})_([\d]{2})([\d]{2}).ruv'
dset.dataset_description.parameters['temporal_dimension'] = None
dset.dataset_description.parameters['zonal_dimension'] = None
dset.dataset_description.parameters['meridional_dimension'] = None
dset.dataset_description.parameters['vertical_dimension'] = None
dset.dataset_description.parameters['variables'] = [
]
# Create DataSourceModel
dsrc_model = DataSourceModel(name='ruv_model')
dsrc_model.model = 'RUV'
dsrc_model.data_handler_module = 'N/A'
dsrc_model.data_handler_class = 'N/A'
## Run everything through DAMS
ds_id = dams_cli.create_external_dataset(external_dataset=dset)
ext_dprov_id = dams_cli.create_external_data_provider(external_data_provider=dprov)
ext_dsrc_id = dams_cli.create_data_source(data_source=dsrc)
ext_dsrc_model_id = dams_cli.create_data_source_model(dsrc_model)
# Register the ExternalDataset
dproducer_id = dams_cli.register_external_data_set(external_dataset_id=ds_id)
# Or using each method
dams_cli.assign_data_source_to_external_data_provider(data_source_id=ext_dsrc_id, external_data_provider_id=ext_dprov_id)
dams_cli.assign_data_source_to_data_model(data_source_id=ext_dsrc_id, data_source_model_id=ext_dsrc_model_id)
dams_cli.assign_external_dataset_to_data_source(external_dataset_id=ds_id, data_source_id=ext_dsrc_id)
dams_cli.assign_external_dataset_to_agent_instance(external_dataset_id=ds_id, agent_instance_id=eda_inst_id)
# dams_cli.assign_external_data_agent_to_agent_instance(external_data_agent_id=self.eda_id, agent_instance_id=self.eda_inst_id)
#create temp streamdef so the data product can create the stream
craft = CoverageCraft
sdom, tdom = craft.create_domains()
sdom = sdom.dump()
tdom = tdom.dump()
parameter_dictionary = craft.create_parameters()
parameter_dictionary = parameter_dictionary.dump()
dprod = IonObject(RT.DataProduct,
name='ruv_parsed_product',
description='parsed ruv product',
temporal_domain = tdom,
spatial_domain = sdom)
streamdef_id = pubsub_cli.create_stream_definition(name="temp", description="temp")
# Generate the data product and associate it to the ExternalDataset
dproduct_id = dpms_cli.create_data_product(data_product=dprod,
stream_definition_id=streamdef_id,
parameter_dictionary=parameter_dictionary)
dams_cli.assign_data_product(input_resource_id=ds_id, data_product_id=dproduct_id)
stream_id, assn = rr_cli.find_objects(subject=dproduct_id, predicate=PRED.hasStream, object_type=RT.Stream, id_only=True)
stream_id = stream_id[0]
log.info('Created resources: {0}'.format({'ExternalDataset':ds_id, 'ExternalDataProvider':ext_dprov_id, 'DataSource':ext_dsrc_id, 'DataSourceModel':ext_dsrc_model_id, 'DataProducer':dproducer_id, 'DataProduct':dproduct_id, 'Stream':stream_id}))
#CBM: Use CF standard_names
#ttool = TaxyTool()
#
#ttool.add_taxonomy_set('data','test data')
pdict = ParameterDictionary()
t_ctxt = ParameterContext('data', param_type=QuantityType(value_encoding=numpy.dtype('int64')))
t_ctxt.reference_frame = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1970'
pdict.add_context(t_ctxt)
#CBM: Eventually, probably want to group this crap somehow - not sure how yet...
# Create the logger for receiving publications
self.create_stream_and_logger(name='ruv',stream_id=stream_id)
self.EDA_RESOURCE_ID = ds_id
self.EDA_NAME = ds_name
self.DVR_CONFIG['dh_cfg'] = {
'TESTING':True,
'stream_id':stream_id,
'external_dataset_res':dset,
'param_dictionary':pdict.dump(),
'data_producer_id':dproducer_id,#CBM: Should this be put in the main body of the config - with mod & cls?
'max_records':20,
}
def test_build_granule_and_load_from_granule(self):
pdict = ParameterDictionary()
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.reference_frame = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1970'
pdict.add_context(t_ctxt)
lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=np.dtype('float32')))
lat_ctxt.reference_frame = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
pdict.add_context(lat_ctxt)
lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=np.dtype('float32')))
lon_ctxt.reference_frame = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
pdict.add_context(lon_ctxt)
temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=np.dtype('float32')))
temp_ctxt.uom = 'degree_Celsius'
pdict.add_context(temp_ctxt)
cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=np.dtype('float32')))
cond_ctxt.uom = 'unknown'
pdict.add_context(cond_ctxt)
pres_ctxt = ParameterContext('pres', param_type=QuantityType(value_encoding=np.dtype('float32')))
pres_ctxt.uom = 'unknown'
pdict.add_context(pres_ctxt)
rdt = RecordDictionaryTool(param_dictionary=pdict)
#Create some arrays and fill them with random values
temp_array = np.random.standard_normal(100)
cond_array = np.random.standard_normal(100)
pres_array = np.random.standard_normal(100)
time_array = np.random.standard_normal(100)
lat_array = np.random.standard_normal(100)
lon_array = np.random.standard_normal(100)
#Use the RecordDictionaryTool to add the values. This also would work if you used long_temp_name, etc.
rdt['temp'] = temp_array
rdt['conductivity'] = cond_array
rdt['pres'] = pres_array
rdt['time'] = time_array
rdt['lat'] = lat_array
rdt['lon'] = lon_array
g = build_granule(data_producer_id='john', record_dictionary=rdt, param_dictionary=pdict)
l_pd = ParameterDictionary.load(g.param_dictionary)
#l_tx = TaxyTool.load_from_granule(g)
l_rd = RecordDictionaryTool.load_from_granule(g)
# Make sure we got back the same Taxonomy Object
#self.assertEquals(l_pd, pdict)
self.assertEquals(l_pd.ord_from_key('temp'), pdict.ord_from_key('temp'))
self.assertEquals(l_pd.ord_from_key('conductivity'), pdict.ord_from_key('conductivity'))
# Now test the record dictionary object
self.assertEquals(l_rd._rd, rdt._rd)
#self.assertEquals(l_rd._param_dict, rdt._param_dict)
for k, v in l_rd.iteritems():
self.assertIn(k, rdt)
if isinstance(v, np.ndarray):
self.assertTrue( (v == rdt[k]).all())
else:
self.assertEquals(v._rd, rdt[k]._rd)
