def make_cal_data_product(self):
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(
pdict_id, id_only=True)
# Add a handful of Calibration Coefficient parameters
for cc in ['cc_ta0', 'cc_ta1', 'cc_ta2', 'cc_ta3', 'cc_toffset']:
c = ParameterContext(cc,
param_type=SparseConstantType(
value_encoding='float32',
fill_value=-9999))
c.uom = '1'
context_ids.append(
self.dataset_management.create_parameter_context(cc, c.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary(
'calcoeff_dict', context_ids, temporal_context='time')
data_product_id = self.create_data_product('calcoeff_dp',
pdict_id=pdict_id)
self.activate_data_product(data_product_id)
dataset_id, _ = self.resource_registry.find_objects(data_product_id,
PRED.hasDataset,
id_only=True)
return data_product_id, dataset_id[0]
def make_manual_upload_data_product(self):
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(
pdict_id, id_only=True)
# Add a handful of Calibration Coefficient parameters
for cc in ['temp_hitl_qc', 'cond_hitl_qc']:
c = ParameterContext(cc, param_type=BooleanType())
c.uom = '1'
context_ids.append(
self.dataset_management.create_parameter_context(cc, c.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary(
'manup_dict', context_ids, temporal_context='time')
data_product_id = self.create_data_product('manup_dp',
pdict_id=pdict_id)
self.activate_data_product(data_product_id)
dataset_id, _ = self.resource_registry.find_objects(data_product_id,
PRED.hasDataset,
id_only=True)
return data_product_id, dataset_id[0]
def get_cc_value(self, value):
placeholder = value.lower()
# Check to see if this coefficient exists already
hits, _ = Container.instance.resource_registry.find_resources(name=placeholder, restype=RT.ParameterContext, id_only=True)
if hits:
return hits[0], placeholder
pc = ParameterContext(name=placeholder, param_type=SparseConstantType(value_encoding='float64'), fill_value=-9999.)
pc.uom = '1'
ctxt_id = self.dataset_management.create_parameter_context(name=placeholder, parameter_context=pc.dump())
return ctxt_id, placeholder
def get_lookup_value(self,value):
placeholder = value.replace('LV_','')
document_key = ''
if '||' in placeholder:
document_key, placeholder = placeholder.split('||')
document_val = placeholder
placeholder = '%s_%s' % (placeholder, uuid4().hex)
pc = ParameterContext(name=placeholder, param_type=SparseConstantType(base_type=ConstantType(value_encoding='float64'), fill_value=-9999.))
pc.lookup_value = document_val
pc.document_key = document_key
ctxt_id = self.dataset_management.create_parameter_context(name=placeholder, parameter_context=pc.dump())
return ctxt_id, placeholder
def create_simple_cc(self):
contexts = {}
types_manager = TypesManager(self.dataset_management, None, None)
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('float64')))
t_ctxt.uom = 'seconds since 1900-01-01'
t_ctxt_id = self.dataset_management.create_parameter_context(name='time', parameter_context=t_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, t_ctxt_id)
contexts['time'] = t_ctxt, t_ctxt_id
temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=fill_value)
temp_ctxt.uom = 'deg_C'
temp_ctxt.ooi_short_name = 'TEMPWAT'
temp_ctxt_id = self.dataset_management.create_parameter_context(name='temp', parameter_context=temp_ctxt.dump(), ooi_short_name='TEMPWAT')
self.addCleanup(self.dataset_management.delete_parameter_context, temp_ctxt_id)
contexts['temp'] = temp_ctxt, temp_ctxt_id
func = NumexprFunction('offset', 'temp + offset', ['temp','offset'])
types_manager.get_pfunc = lambda pfid : func
func = types_manager.evaluate_pmap('pfid', {'temp':'temp', 'offset':'CC_coefficient'})
func_id = self.dataset_management.create_parameter_function('offset', func.dump())
self.addCleanup(self.dataset_management.delete_parameter_function, func_id)
offset_ctxt = ParameterContext('offset', param_type=ParameterFunctionType(func), fill_value=fill_value)
offset_ctxt.uom = '1'
offset_ctxt_id = self.dataset_management.create_parameter_context('offset', offset_ctxt.dump(), parameter_function_id=func_id)
self.addCleanup(self.dataset_management.delete_parameter_context, offset_ctxt_id)
contexts['offset'] = offset_ctxt, offset_ctxt_id
return contexts
def create_illegal_char(self):
contexts = {}
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('float64')))
t_ctxt.uom = 'seconds since 1900-01-01'
t_ctxt_id = self.dataset_management.create_parameter_context(name='time', parameter_context=t_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, t_ctxt_id)
contexts['time'] = (t_ctxt, t_ctxt_id)
i_ctxt = ParameterContext('ice-cream', param_type=QuantityType(value_encoding=np.dtype('float32')))
i_ctxt.uom = '1'
i_ctxt_id = self.dataset_management.create_parameter_context(name='ice-cream', parameter_context=i_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, i_ctxt_id)
contexts['ice-cream'] = (i_ctxt, i_ctxt_id)
return contexts
def get_lookup_value(self, value):
return self._placeholder(
value, lambda placeholder: ParameterContext(
name=placeholder,
param_type=SparseConstantType(base_type=ConstantType(
value_encoding='float64'),
fill_value=-9999.)))
def make_grt_qc(self, name, data_product):
pfunc_id, pfunc = self.find_grt()
grt_min_id, grt_min_name = self.get_lookup_value('LV_grt_$designator_%s||grt_min_value' % data_product)
grt_max_id, grt_max_name = self.get_lookup_value('LV_grt_$designator_%s||grt_max_value' % data_product)
pmap = {'dat':name, 'dat_min':grt_min_name,'dat_max':grt_max_name}
pfunc.param_map = pmap
pfunc.lookup_values = [grt_min_id, grt_max_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_glblrng_qc' % dp_name.lower(), param_type=ParameterFunctionType(pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_GLBLRNG_QC' % dp_name
pc.display_name = '%s Global Range Test Quality Control Flag' % dp_name
pc.description = "The OOI Global Range quality control algorithm generates a QC flag for the input data point indicating whether it falls within a given range."
ctxt_id = self.dataset_management.create_parameter_context(name='%s_glblrng_qc' % dp_name.lower(), parameter_type='function', parameter_context=pc.dump(), parameter_function_id=pfunc_id, ooi_short_name=pc.ooi_short_name, units='1', value_encoding='int8', display_name=pc.display_name, description=pc.description)
return ctxt_id, pc
def get_cc_value(self, value):
placeholder = value.lower()
# Check to see if this coefficient exists already
hits, _ = Container.instance.resource_registry.find_resources(
name=placeholder, restype=RT.ParameterContext, id_only=True)
if hits:
return hits[0], placeholder
pc = ParameterContext(
name=placeholder,
param_type=SparseConstantType(value_encoding='float64'),
fill_value=-9999.)
pc.uom = '1'
ctxt_id = self.dataset_management.create_parameter_context(
name=placeholder, parameter_context=pc.dump())
return ctxt_id, placeholder
def create_lookup_contexts(self):
contexts = {}
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('float64')))
t_ctxt.uom = 'seconds since 01-01-1900'
t_ctxt_id = self.dataset_management.create_parameter_context(name='time', parameter_context=t_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, t_ctxt_id)
contexts['time'] = (t_ctxt, t_ctxt_id)
temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999)
temp_ctxt.uom = 'deg_C'
temp_ctxt_id = self.dataset_management.create_parameter_context(name='temp', parameter_context=temp_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, temp_ctxt_id)
contexts['temp'] = temp_ctxt, temp_ctxt_id
offset_ctxt = ParameterContext('offset_a', param_type=QuantityType(value_encoding='float32'), fill_value=-9999)
offset_ctxt.lookup_value = True
offset_ctxt_id = self.dataset_management.create_parameter_context(name='offset_a', parameter_context=offset_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, offset_ctxt_id)
contexts['offset_a'] = offset_ctxt, offset_ctxt_id
func = NumexprFunction('calibrated', 'temp + offset', ['temp','offset'], param_map={'temp':'temp', 'offset':'offset_a'})
func.lookup_values = ['LV_offset']
calibrated = ParameterContext('calibrated', param_type=ParameterFunctionType(func, value_encoding='float32'), fill_value=-9999)
calibrated_id = self.dataset_management.create_parameter_context(name='calibrated', parameter_context=calibrated.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, calibrated_id)
contexts['calibrated'] = calibrated, calibrated_id
return contexts
def make_grt_qc(self, name, data_product):
pfunc_id, pfunc = self.find_grt()
grt_min_id, grt_min_name = self.get_lookup_value(
'LV_grt_$designator_%s||grt_min_value' % data_product)
grt_max_id, grt_max_name = self.get_lookup_value(
'LV_grt_$designator_%s||grt_max_value' % data_product)
pmap = {'dat': name, 'dat_min': grt_min_name, 'dat_max': grt_max_name}
pfunc.param_map = pmap
pfunc.lookup_values = [grt_min_id, grt_max_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_glblrng_qc' % dp_name.lower(),
param_type=ParameterFunctionType(
pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_GLBLRNG_QC' % dp_name
pc.display_name = '%s Global Range Test Quality Control Flag' % dp_name
pc.description = "The OOI Global Range quality control algorithm generates a QC flag for the input data point indicating whether it falls within a given range."
ctxt_id = self.dataset_management.create_parameter_context(
name='%s_glblrng_qc' % dp_name.lower(),
parameter_type='function',
parameter_context=pc.dump(),
parameter_function_id=pfunc_id,
ooi_short_name=pc.ooi_short_name,
units='1',
value_encoding='int8',
display_name=pc.display_name,
description=pc.description)
return ctxt_id, pc
def make_stuckvalue_qc(self, name, data_product):
pfunc_id, pfunc = self.find_stuck_value()
reso_id, reso_name = self.get_lookup_value(
'LV_svt_$designator_%s||svt_resolution' % data_product)
n_id, n_name = self.get_lookup_value('LV_svt_$designator_%s||svt_n' %
data_product)
pmap = {'x': name, 'reso': reso_name, 'num': n_name}
pfunc.param_map = pmap
pfunc.lookup_values = [reso_id, n_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_stuckvl_qc' % dp_name.lower(),
param_type=ParameterFunctionType(
pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_STUCKVL_QC' % dp_name
pc.display_name = '%s Stuck Value Test Quality Control Flag' % dp_name
pc.description = 'The OOI Stuck Value Test quality control algorithm generates a flag for repeated occurrence of one value in a time series.'
ctxt_id = self.dataset_management.create_parameter_context(
name='%s_stuckvl_qc' % dp_name.lower(),
parameter_type='function',
parameter_context=pc.dump(),
parameter_function_id=pfunc_id,
ooi_short_name=pc.ooi_short_name,
units='1',
value_encoding='int8',
display_name=pc.display_name,
description=pc.description)
return ctxt_id, pc
def make_propagate_qc(self, inputs):
pfunc_id, pfunc = self.find_propagate_test()
pmap = {"strict_validation": False}
arg_list = ['strict_validation']
for i, val in enumerate(inputs):
if i >= 100:
break
pmap['array%s' % i] = val
arg_list.append('array%s' % i)
pfunc.param_map = pmap
pfunc.arg_list = arg_list
pc = ParameterContext(name='cmbnflg_qc',
param_type=ParameterFunctionType(
pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = 'CMBNFLG_QC'
pc.display_name = 'Combined Data Quality Control Flag'
pc.description = 'The purpose of this computation is to produce a single merged QC flag from a set of potentially many flags.'
ctxt_id = self.dataset_management.create_parameter_context(
name='cmbnflg_qc',
parameter_type='function',
parameter_context=pc.dump(),
parameter_function_id=pfunc_id,
ooi_short_name=pc.ooi_short_name,
units='1',
value_encoding='int8',
display_name=pc.display_name,
description=pc.description)
return ctxt_id, pc
def make_manual_upload_data_product(self):
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True)
# Add a handful of Calibration Coefficient parameters
for cc in ['temp_hitl_qc', 'cond_hitl_qc']:
c = ParameterContext(cc, param_type=BooleanType())
c.uom = '1'
context_ids.append(self.dataset_management.create_parameter_context(cc, c.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary('manup_dict', context_ids, temporal_context='time')
data_product_id = self.create_data_product('manup_dp', pdict_id=pdict_id)
self.activate_data_product(data_product_id)
dataset_id, _ = self.resource_registry.find_objects(data_product_id, PRED.hasDataset, id_only=True)
return data_product_id, dataset_id[0]
def make_stuckvalue_qc(self, name, data_product):
pfunc_id, pfunc = self.find_stuck_value()
reso_id, reso_name = self.get_lookup_value('LV_svt_$designator_%s||svt_resolution' % data_product)
n_id, n_name = self.get_lookup_value('LV_svt_$designator_%s||svt_n' % data_product)
pmap = {'x' : name, 'reso': reso_name, 'num': n_name}
pfunc.param_map = pmap
pfunc.lookup_values = [reso_id, n_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_stuckvl_qc' % dp_name.lower(), param_type=ParameterFunctionType(pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_STUCKVL_QC' % dp_name
pc.display_name = '%s Stuck Value Test Quality Control Flag' % dp_name
pc.description = 'The OOI Stuck Value Test quality control algorithm generates a flag for repeated occurrence of one value in a time series.'
ctxt_id = self.dataset_management.create_parameter_context(name='%s_stuckvl_qc' % dp_name.lower(), parameter_type='function', parameter_context=pc.dump(), parameter_function_id=pfunc_id, ooi_short_name=pc.ooi_short_name, units='1', value_encoding='int8', display_name=pc.display_name, description=pc.description)
return ctxt_id, pc
def make_cal_data_product(self):
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(pdict_id, id_only=True)
# Add a handful of Calibration Coefficient parameters
for cc in ['cc_ta0', 'cc_ta1', 'cc_ta2', 'cc_ta3', 'cc_toffset']:
c = ParameterContext(cc, param_type=SparseConstantType(value_encoding='float32', fill_value=-9999))
c.uom = '1'
context_ids.append(self.dataset_management.create_parameter_context(cc, c.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary('calcoeff_dict', context_ids, temporal_context='time')
data_product_id = self.create_data_product('calcoeff_dp', pdict_id=pdict_id)
self.activate_data_product(data_product_id)
dataset_id, _ = self.resource_registry.find_objects(data_product_id, PRED.hasDataset, id_only=True)
return data_product_id, dataset_id[0]
def make_propagate_qc(self,inputs):
pfunc_id, pfunc = self.find_propagate_test()
pmap = {"strict_validation":False}
arg_list = ['strict_validation']
for i,val in enumerate(inputs):
if i >= 100:
break
pmap['array%s' % i] = val
arg_list.append('array%s' % i)
pfunc.param_map = pmap
pfunc.arg_list = arg_list
pc = ParameterContext(name='cmbnflg_qc', param_type=ParameterFunctionType(pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = 'CMBNFLG_QC'
pc.display_name = 'Combined Data Quality Control Flag'
pc.description = 'The purpose of this computation is to produce a single merged QC flag from a set of potentially many flags.'
ctxt_id = self.dataset_management.create_parameter_context(name='cmbnflg_qc', parameter_type='function', parameter_context=pc.dump(), parameter_function_id=pfunc_id, ooi_short_name=pc.ooi_short_name, units='1', value_encoding='int8', display_name=pc.display_name, description=pc.description)
return ctxt_id, pc
def make_spike_qc(self, name, data_product):
pfunc_id, pfunc = self.find_spike()
spike_acc_id, spike_acc_name = self.get_lookup_value('LV_spike_$designator_%s||acc' % data_product)
spike_n_id, spike_n_name = self.get_lookup_value('LV_spike_$designator_%s||spike_n' % data_product)
spike_l_id, spike_l_name = self.get_lookup_value('LV_spike_$designator_%s||spike_l' % data_product)
pmap = {'dat':name, 'acc':spike_acc_name, 'N':spike_n_name, 'L':spike_l_name}
pfunc.param_map = pmap
pfunc.lookup_values = [spike_acc_id, spike_n_id, spike_l_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_spketst_qc' % dp_name.lower(), param_type=ParameterFunctionType(pfunc, value_encoding='|i1'))
pc.uom='1'
pc.ooi_short_name = '%s_SPKETST_QC' % dp_name
pc.display_name = '%s Spike Test Quality Control Flag' % dp_name
pc.description = "The OOI Spike Test quality control algorithm generates a flag for individual data values that deviate significantly from surrounding data values."
ctxt_id = self.dataset_management.create_parameter_context(name='%s_spketst_qc' % dp_name.lower(), parameter_type='function', parameter_context=pc.dump(), parameter_function_id=pfunc_id, ooi_short_name=pc.ooi_short_name, units='1', value_encoding='int8', display_name=pc.display_name, description=pc.description)
return ctxt_id, pc
def make_trendtest_qc(self, name, data_product):
pfunc_id, pfunc = self.find_trend_test()
order_id, order_name = self.get_lookup_value('LV_trend_$designator_%s||polynomial_order' % data_product)
dev_id, dev_name = self.get_lookup_value('LV_trend_$designator_%s||standard_deviation' % data_product)
pmap = {"dat":name ,"t":'time',"ord_n":order_name,"ntsd":dev_name}
pfunc.param_map = pmap
pfunc.lookup_values = [order_id, dev_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_trndtst_qc' % dp_name.lower(), param_type=ParameterFunctionType(pfunc,value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_TRNDTST_QC' % dp_name
pc.display_name = '%s Trend Test Test Quality Control Flag' % dp_name
pc.description = 'The OOI Trend Test quality control algorithm generates flags on data values within a time series where a significant fraction of the variability in the time series can be explained by a drift, where the drift is assumed to be a polynomial of specified order.'
ctxt_id = self.dataset_management.create_parameter_context(name='%s_trndtst_qc' % dp_name.lower(), parameter_type='function', parameter_context=pc.dump(), parameter_function_id=pfunc_id, ooi_short_name=pc.ooi_short_name, units='1', value_encoding='int8', display_name=pc.display_name, description=pc.description)
return ctxt_id, pc
def make_localrange_qc(self, name, data_product):
pfunc_id, pfunc = self.find_localrange_test()
datlim_id, datlim = self.get_array_lookup_value('LV_lrt_$designator_%s||datlim' % data_product)
datlimz_id, datlimz = self.get_array_lookup_value('LV_lrt_$designator_%s||datlimz' % data_product)
dims_id, dims = self.get_string_array_lookup_value('LV_lrt_$designator_%s||dims' % data_product)
pmap = {"dat":name, "dims*":dims, "datlim*":datlim, "datlimz*":datlimz}
pfunc.param_map = pmap
pfunc.lookup_values = [datlim_id, datlimz_id, dims_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_loclrng_qc' % dp_name.lower(), param_type=ParameterFunctionType(pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_LOCLRNG_QC' % dp_name
pc.display_name = '%s Local Range Test Quality Control Flag' % dp_name
pc.description = 'The OOI Local Range Test is the computation to test whether a given data point falls within pre-defined ranges.'
ctxt_id = self.dataset_management.create_parameter_context(name='%s_loclrng_qc' % dp_name.lower(), parameter_type='function', parameter_context=pc.dump(), parameter_function_id=pfunc_id, ooi_short_name=pc.ooi_short_name, units='1', value_encoding='int8', display_name=pc.display_name, description=pc.description)
return ctxt_id, pc
def make_gradienttest_qc(self, name, data_product):
pfunc_id, pfunc = self.find_gradient_test()
ddatdx_id, ddatdx = self.get_lookup_value('LV_grad_$designator_%s_time||d_dat_dx' % data_product)
mindx_id, mindx = self.get_lookup_value('LV_grad_$designator_%s_time||min_dx' % data_product)
startdat_id, startdat = self.get_lookup_value('LV_grad_$designator_%s_time||start_dat' % data_product)
toldat_id, toldat = self.get_lookup_value('LV_grad_$designator_%s_time||tol_dat' % data_product)
pmap = {"dat":name, "x": 'time', 'ddatdx': ddatdx, 'mindx':mindx, 'startdat': startdat, 'toldat':toldat}
pfunc.param_map = pmap
pfunc.lookup_values = [ddatdx_id, mindx_id, startdat_id, toldat_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_gradtst_qc' % dp_name.lower(), param_type=ParameterFunctionType(pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_GRADTST_QC' % dp_name
pc.display_name = '%s Gradient Test Quality Control Flag' % dp_name
pc.description = 'The OOI Gradient Test is an automated quality control algorithm used on various OOI data products. This automated algorithm generates flags for data points according to whether changes between successive points are within a pre-determined range.'
ctxt_id = self.dataset_management.create_parameter_context(name='%s_gradtst_qc' % dp_name.lower(), parameter_type='function', parameter_context=pc.dump(), parameter_function_id=pfunc_id, ooi_short_name=pc.ooi_short_name, units='1', value_encoding='int8', display_name=pc.display_name, description=pc.description)
return ctxt_id, pc
def create_qc_contexts(self):
contexts = {}
qc_whatever_ctxt = ParameterContext('qc_whatever', param_type=ArrayType())
qc_whatever_ctxt.uom = '1'
qc_whatever_ctxt_id = self.dataset_management.create_parameter_context(name='qc_whatever', parameter_context=qc_whatever_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, qc_whatever_ctxt_id)
contexts['qc_whatever'] = qc_whatever_ctxt, qc_whatever_ctxt_id
nexpr = NumexprFunction('range_qc', 'min < var > max', ['min','max','var'])
expr_id = self.dataset_management.create_parameter_function(name='range_qc', parameter_function=nexpr.dump())
self.addCleanup(self.dataset_management.delete_parameter_function, expr_id)
pmap = {'min':0, 'max':20, 'var':'temp'}
nexpr.param_map = pmap
temp_qc_ctxt = ParameterContext('temp_qc', param_type=ParameterFunctionType(function=nexpr), variability=VariabilityEnum.TEMPORAL)
temp_qc_ctxt.uom = '1'
temp_qc_ctxt_id = self.dataset_management.create_parameter_context(name='temp_qc', parameter_context=temp_qc_ctxt.dump(), parameter_function_id=expr_id)
self.addCleanup(self.dataset_management.delete_parameter_context, temp_qc_ctxt_id)
contexts['temp_qc'] = temp_qc_ctxt, temp_qc_ctxt_id
return contexts
def __init__(self,
total_domain=(10, 10),
brick_size=5,
use_hdf=False,
root_dir='test_data/multi_dim_trials',
guid=None,
dtype='int16'):
self.total_domain = total_domain
self.brick_sizes = tuple(brick_size for x in total_domain)
self.use_hdf = use_hdf
self.dtype = np.dtype(dtype).name
if self.use_hdf:
self.guid = guid or create_guid()
name = '%s_%s' % (self.guid, self.dtype)
self.root_dir = root_dir
if not os.path.exists(self.root_dir):
os.makedirs(self.root_dir)
if os.path.exists(os.path.join(self.root_dir, name)):
shutil.rmtree(os.path.join(self.root_dir, name))
self.master_manager = MasterManager(
self.root_dir, name, name='md_test_{0}'.format(name))
self.master_manager.flush()
pc = ParameterContext('test_param',
param_type=QuantityType(self.dtype),
fill_value=-1)
self.param_manager = ParameterManager(
os.path.join(self.root_dir, name, pc.name), pc.name)
self.param_manager.parameter_context = pc
self.master_manager.create_group(pc.name)
self.param_manager.flush()
self.bricks = {}
self.brick_origins = bricking_utils.calc_brick_origins(
self.total_domain, self.brick_sizes)
self.brick_extents, self.rtree_extents = bricking_utils.calc_brick_and_rtree_extents(
self.brick_origins, self.brick_sizes)
self.build_bricks()
self.rtree = RTreeProxy()
for x in BrickingAssessor.rtree_populator(self.rtree_extents,
self.brick_extents):
self.rtree.insert(*x)
def make_gradienttest_qc(self, name, data_product):
pfunc_id, pfunc = self.find_gradient_test()
ddatdx_id, ddatdx = self.get_lookup_value(
'LV_grad_$designator_%s_time||d_dat_dx' % data_product)
mindx_id, mindx = self.get_lookup_value(
'LV_grad_$designator_%s_time||min_dx' % data_product)
startdat_id, startdat = self.get_lookup_value(
'LV_grad_$designator_%s_time||start_dat' % data_product)
toldat_id, toldat = self.get_lookup_value(
'LV_grad_$designator_%s_time||tol_dat' % data_product)
pmap = {
"dat": name,
"x": 'time',
'ddatdx': ddatdx,
'mindx': mindx,
'startdat': startdat,
'toldat': toldat
}
pfunc.param_map = pmap
pfunc.lookup_values = [ddatdx_id, mindx_id, startdat_id, toldat_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_gradtst_qc' % dp_name.lower(),
param_type=ParameterFunctionType(
pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_GRADTST_QC' % dp_name
pc.display_name = '%s Gradient Test Quality Control Flag' % dp_name
pc.description = 'The OOI Gradient Test is an automated quality control algorithm used on various OOI data products. This automated algorithm generates flags for data points according to whether changes between successive points are within a pre-determined range.'
ctxt_id = self.dataset_management.create_parameter_context(
name='%s_gradtst_qc' % dp_name.lower(),
parameter_type='function',
parameter_context=pc.dump(),
parameter_function_id=pfunc_id,
ooi_short_name=pc.ooi_short_name,
units='1',
value_encoding='int8',
display_name=pc.display_name,
description=pc.description)
return ctxt_id, pc
def create_simple_qc(self):
contexts = {}
types_manager = TypesManager(self.dataset_management,None,None)
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('float64')))
t_ctxt.uom = 'seconds since 1900-01-01'
t_ctxt_id = self.dataset_management.create_parameter_context(name='time', parameter_context=t_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, t_ctxt_id)
contexts['time'] = (t_ctxt, t_ctxt_id)
temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999)
temp_ctxt.uom = 'deg_C'
temp_ctxt.ooi_short_name = 'TEMPWAT'
temp_ctxt.qc_contexts = types_manager.make_qc_functions('temp','TEMPWAT',lambda *args, **kwargs : None)
temp_ctxt_id = self.dataset_management.create_parameter_context(name='temp', parameter_context=temp_ctxt.dump(), ooi_short_name='TEMPWAT')
self.addCleanup(self.dataset_management.delete_parameter_context, temp_ctxt_id)
contexts['temp'] = temp_ctxt, temp_ctxt_id
return contexts
def make_localrange_qc(self, name, data_product):
pfunc_id, pfunc = self.find_localrange_test()
datlim_id, datlim = self.get_array_lookup_value(
'LV_lrt_$designator_%s||datlim' % data_product)
datlimz_id, datlimz = self.get_array_lookup_value(
'LV_lrt_$designator_%s||datlimz' % data_product)
dims_id, dims = self.get_string_array_lookup_value(
'LV_lrt_$designator_%s||dims' % data_product)
pmap = {
"dat": name,
"dims*": dims,
"datlim*": datlim,
"datlimz*": datlimz
}
pfunc.param_map = pmap
pfunc.lookup_values = [datlim_id, datlimz_id, dims_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_loclrng_qc' % dp_name.lower(),
param_type=ParameterFunctionType(
pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_LOCLRNG_QC' % dp_name
pc.display_name = '%s Local Range Test Quality Control Flag' % dp_name
pc.description = 'The OOI Local Range Test is the computation to test whether a given data point falls within pre-defined ranges.'
ctxt_id = self.dataset_management.create_parameter_context(
name='%s_loclrng_qc' % dp_name.lower(),
parameter_type='function',
parameter_context=pc.dump(),
parameter_function_id=pfunc_id,
ooi_short_name=pc.ooi_short_name,
units='1',
value_encoding='int8',
display_name=pc.display_name,
description=pc.description)
return ctxt_id, pc
def make_spike_qc(self, name, data_product):
pfunc_id, pfunc = self.find_spike()
spike_acc_id, spike_acc_name = self.get_lookup_value(
'LV_spike_$designator_%s||acc' % data_product)
spike_n_id, spike_n_name = self.get_lookup_value(
'LV_spike_$designator_%s||spike_n' % data_product)
spike_l_id, spike_l_name = self.get_lookup_value(
'LV_spike_$designator_%s||spike_l' % data_product)
pmap = {
'dat': name,
'acc': spike_acc_name,
'N': spike_n_name,
'L': spike_l_name
}
pfunc.param_map = pmap
pfunc.lookup_values = [spike_acc_id, spike_n_id, spike_l_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_spketst_qc' % dp_name.lower(),
param_type=ParameterFunctionType(
pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_SPKETST_QC' % dp_name
pc.display_name = '%s Spike Test Quality Control Flag' % dp_name
pc.description = "The OOI Spike Test quality control algorithm generates a flag for individual data values that deviate significantly from surrounding data values."
ctxt_id = self.dataset_management.create_parameter_context(
name='%s_spketst_qc' % dp_name.lower(),
parameter_type='function',
parameter_context=pc.dump(),
parameter_function_id=pfunc_id,
ooi_short_name=pc.ooi_short_name,
units='1',
value_encoding='int8',
display_name=pc.display_name,
description=pc.description)
return ctxt_id, pc
def make_trendtest_qc(self, name, data_product):
pfunc_id, pfunc = self.find_trend_test()
order_id, order_name = self.get_lookup_value(
'LV_trend_$designator_%s||polynomial_order' % data_product)
dev_id, dev_name = self.get_lookup_value(
'LV_trend_$designator_%s||standard_deviation' % data_product)
pmap = {
"dat": name,
"t": 'time',
"ord_n": order_name,
"ntsd": dev_name
}
pfunc.param_map = pmap
pfunc.lookup_values = [order_id, dev_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_trndtst_qc' % dp_name.lower(),
param_type=ParameterFunctionType(
pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_TRNDTST_QC' % dp_name
pc.display_name = '%s Trend Test Test Quality Control Flag' % dp_name
pc.description = 'The OOI Trend Test quality control algorithm generates flags on data values within a time series where a significant fraction of the variability in the time series can be explained by a drift, where the drift is assumed to be a polynomial of specified order.'
ctxt_id = self.dataset_management.create_parameter_context(
name='%s_trndtst_qc' % dp_name.lower(),
parameter_type='function',
parameter_context=pc.dump(),
parameter_function_id=pfunc_id,
ooi_short_name=pc.ooi_short_name,
units='1',
value_encoding='int8',
display_name=pc.display_name,
description=pc.description)
return ctxt_id, pc
def create_contexts(self):
context_ids = []
cond_ctxt = ParameterContext(
'conductivity_test',
param_type=QuantityType(value_encoding=np.float32))
cond_ctxt.uom = 'unknown'
cond_ctxt.fill_value = 0e0
context_ids.append(
self.dataset_management.create_parameter_context(
name='conductivity_test', parameter_context=cond_ctxt.dump()))
pres_ctxt = ParameterContext(
'pressure_test',
param_type=QuantityType(value_encoding=np.float32))
pres_ctxt.uom = 'Pascal'
pres_ctxt.fill_value = 0x0
context_ids.append(
self.dataset_management.create_parameter_context(
name='pressure_test', parameter_context=pres_ctxt.dump()))
sal_ctxt = ParameterContext(
'salinity_test',
param_type=QuantityType(value_encoding=np.float32))
sal_ctxt.uom = 'PSU'
sal_ctxt.fill_value = 0x0
context_ids.append(
self.dataset_management.create_parameter_context(
name='salinity_test', parameter_context=sal_ctxt.dump()))
temp_ctxt = ParameterContext(
'temp_test', param_type=QuantityType(value_encoding=np.float32))
temp_ctxt.uom = 'degree_Celsius'
temp_ctxt.fill_value = 0e0
context_ids.append(
self.dataset_management.create_parameter_context(
name='temp_test', parameter_context=temp_ctxt.dump()))
t_ctxt = ParameterContext(
'time_test', param_type=QuantityType(value_encoding=np.int64))
t_ctxt.uom = 'seconds since 1970-01-01'
t_ctxt.fill_value = 0x0
context_ids.append(
self.dataset_management.create_parameter_context(
name='time_test', parameter_context=t_ctxt.dump()))
return context_ids
def create_extended_parsed_contexts(self):
contexts, funcs = self.create_parsed_params()
expr, expr_id = funcs['density_L2']
density_lookup_map = {'conductivity':'conductivity_L1', 'temp':'temp_L1', 'pressure':'pressure_L1', 'lat':'lat_lookup', 'lon':'lon_lookup'}
expr.param_map = density_lookup_map
density_lookup_ctxt = ParameterContext('density_lookup', param_type=ParameterFunctionType(expr), variability=VariabilityEnum.TEMPORAL)
density_lookup_ctxt.uom = 'kg m-3'
density_lookup_ctxt_id = self.dataset_management.create_parameter_context(name='density_lookup', parameter_context=density_lookup_ctxt.dump(), parameter_function_id=expr_id)
self.addCleanup(self.dataset_management.delete_parameter_context, density_lookup_ctxt_id)
contexts['density_lookup'] = density_lookup_ctxt, density_lookup_ctxt_id
lat_lookup_ctxt = ParameterContext('lat_lookup', param_type=ConstantType(QuantityType(value_encoding=np.dtype('float32'))), fill_value=fill_value)
lat_lookup_ctxt.uom = 'degree_north'
lat_lookup_ctxt.lookup_value = 'lat'
lat_lookup_ctxt.document_key = ''
lat_lookup_ctxt_id = self.dataset_management.create_parameter_context(name='lat_lookup', parameter_context=lat_lookup_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, lat_lookup_ctxt_id)
contexts['lat_lookup'] = lat_lookup_ctxt, lat_lookup_ctxt_id
lon_lookup_ctxt = ParameterContext('lon_lookup', param_type=ConstantType(QuantityType(value_encoding=np.dtype('float32'))), fill_value=fill_value)
lon_lookup_ctxt.uom = 'degree_east'
lon_lookup_ctxt.lookup_value = 'lon'
lon_lookup_ctxt.document_key = ''
lon_lookup_ctxt_id = self.dataset_management.create_parameter_context(name='lon_lookup', parameter_context=lon_lookup_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, lon_lookup_ctxt_id)
contexts['lon_lookup'] = lon_lookup_ctxt, lon_lookup_ctxt_id
beam_samples = ParameterContext('beam_samples', param_type=ArrayType(inner_encoding='float64'))
beam_samples.uom = 'db'
beam_samples_id = self.dataset_management.create_parameter_context(name='beam_samples', parameter_context=beam_samples.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, beam_samples_id)
contexts['beam_samples'] = beam_samples, beam_samples_id
return contexts, funcs
def create_pfuncs(self):
contexts = {}
funcs = {}
t_ctxt = ParameterContext(
'TIME', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 01-01-1900'
t_ctxt_id = self.dataset_management.create_parameter_context(
name='test_TIME', parameter_context=t_ctxt.dump())
contexts['TIME'] = (t_ctxt, t_ctxt_id)
lat_ctxt = ParameterContext(
'LAT',
param_type=ConstantType(
QuantityType(value_encoding=np.dtype('float32'))),
fill_value=-9999)
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt_id = self.dataset_management.create_parameter_context(
name='test_LAT', parameter_context=lat_ctxt.dump())
contexts['LAT'] = lat_ctxt, lat_ctxt_id
lon_ctxt = ParameterContext(
'LON',
param_type=ConstantType(
QuantityType(value_encoding=np.dtype('float32'))),
fill_value=-9999)
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt_id = self.dataset_management.create_parameter_context(
name='test_LON', parameter_context=lon_ctxt.dump())
contexts['LON'] = lon_ctxt, lon_ctxt_id
# Independent Parameters
# Temperature - values expected to be the decimal results of conversion from hex
temp_ctxt = ParameterContext(
'TEMPWAT_L0',
param_type=QuantityType(value_encoding=np.dtype('float32')),
fill_value=-9999)
temp_ctxt.uom = 'deg_C'
temp_ctxt_id = self.dataset_management.create_parameter_context(
name='test_TEMPWAT_L0', parameter_context=temp_ctxt.dump())
contexts['TEMPWAT_L0'] = temp_ctxt, temp_ctxt_id
# Conductivity - values expected to be the decimal results of conversion from hex
cond_ctxt = ParameterContext(
'CONDWAT_L0',
param_type=QuantityType(value_encoding=np.dtype('float32')),
fill_value=-9999)
cond_ctxt.uom = 'S m-1'
cond_ctxt_id = self.dataset_management.create_parameter_context(
name='test_CONDWAT_L0', parameter_context=cond_ctxt.dump())
contexts['CONDWAT_L0'] = cond_ctxt, cond_ctxt_id
# Pressure - values expected to be the decimal results of conversion from hex
press_ctxt = ParameterContext(
'PRESWAT_L0',
param_type=QuantityType(value_encoding=np.dtype('float32')),
fill_value=-9999)
press_ctxt.uom = 'dbar'
press_ctxt_id = self.dataset_management.create_parameter_context(
name='test_PRESWAT_L0', parameter_context=press_ctxt.dump())
contexts['PRESWAT_L0'] = press_ctxt, press_ctxt_id
# Dependent Parameters
# TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10
tl1_func = '(T / 10000) - 10'
expr = NumexprFunction('TEMPWAT_L1', tl1_func, ['T'])
expr_id = self.dataset_management.create_parameter_function(
name='test_TEMPWAT_L1', parameter_function=expr.dump())
funcs['TEMPWAT_L1'] = expr, expr_id
tl1_pmap = {'T': 'TEMPWAT_L0'}
expr.param_map = tl1_pmap
tempL1_ctxt = ParameterContext(
'TEMPWAT_L1',
param_type=ParameterFunctionType(function=expr),
variability=VariabilityEnum.TEMPORAL)
tempL1_ctxt.uom = 'deg_C'
tempL1_ctxt_id = self.dataset_management.create_parameter_context(
name='test_TEMPWAT_L1',
parameter_context=tempL1_ctxt.dump(),
parameter_function_id=expr_id)
contexts['TEMPWAT_L1'] = tempL1_ctxt, tempL1_ctxt_id
# CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5
cl1_func = '(C / 100000) - 0.5'
expr = NumexprFunction('CONDWAT_L1', cl1_func, ['C'])
expr_id = self.dataset_management.create_parameter_function(
name='test_CONDWAT_L1', parameter_function=expr.dump())
funcs['CONDWAT_L1'] = expr, expr_id
cl1_pmap = {'C': 'CONDWAT_L0'}
expr.param_map = cl1_pmap
condL1_ctxt = ParameterContext(
'CONDWAT_L1',
param_type=ParameterFunctionType(function=expr),
variability=VariabilityEnum.TEMPORAL)
condL1_ctxt.uom = 'S m-1'
condL1_ctxt_id = self.dataset_management.create_parameter_context(
name='test_CONDWAT_L1',
parameter_context=condL1_ctxt.dump(),
parameter_function_id=expr_id)
contexts['CONDWAT_L1'] = condL1_ctxt, condL1_ctxt_id
# Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721
# PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range)
pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)'
expr = NumexprFunction('PRESWAT_L1', pl1_func, ['P', 'p_range'])
expr_id = self.dataset_management.create_parameter_function(
name='test_PRESWAT_L1', parameter_function=expr.dump())
funcs['PRESWAT_L1'] = expr, expr_id
pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721}
expr.param_map = pl1_pmap
presL1_ctxt = ParameterContext(
'PRESWAT_L1',
param_type=ParameterFunctionType(function=expr),
variability=VariabilityEnum.TEMPORAL)
presL1_ctxt.uom = 'S m-1'
presL1_ctxt_id = self.dataset_management.create_parameter_context(
name='test_CONDWAT_L1',
parameter_context=presL1_ctxt.dump(),
parameter_function_id=expr_id)
contexts['PRESWAT_L1'] = presL1_ctxt, presL1_ctxt_id
# Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project:
# https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1
# PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1)
owner = 'gsw'
sal_func = 'SP_from_C'
sal_arglist = ['C', 't', 'p']
expr = PythonFunction('PRACSAL', owner, sal_func, sal_arglist)
expr_id = self.dataset_management.create_parameter_function(
name='test_PRACSAL', parameter_function=expr.dump())
funcs['PRACSAL'] = expr, expr_id
# A magic function that may or may not exist actually forms the line below at runtime.
sal_pmap = {
'C':
NumexprFunction('CONDWAT_L1*10',
'C*10', ['C'],
param_map={'C': 'CONDWAT_L1'}),
't':
'TEMPWAT_L1',
'p':
'PRESWAT_L1'
}
expr.param_map = sal_pmap
sal_ctxt = ParameterContext('PRACSAL',
param_type=ParameterFunctionType(expr),
variability=VariabilityEnum.TEMPORAL)
sal_ctxt.uom = 'g kg-1'
sal_ctxt_id = self.dataset_management.create_parameter_context(
name='test_PRACSAL',
parameter_context=sal_ctxt.dump(),
parameter_function_id=expr_id)
contexts['PRACSAL'] = sal_ctxt, sal_ctxt_id
# absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude)
# conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1)
# DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1)
owner = 'gsw'
abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP',
['PRACSAL', 'PRESWAT_L1', 'LON', 'LAT'])
cons_temp_expr = PythonFunction(
'cons_temp', owner, 'CT_from_t',
[abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1'])
dens_expr = PythonFunction(
'DENSITY', owner, 'rho',
[abs_sal_expr, cons_temp_expr, 'PRESWAT_L1'])
dens_ctxt = ParameterContext(
'DENSITY',
param_type=ParameterFunctionType(dens_expr),
variability=VariabilityEnum.TEMPORAL)
dens_ctxt.uom = 'kg m-3'
dens_ctxt_id = self.dataset_management.create_parameter_context(
name='test_DENSITY', parameter_context=dens_ctxt.dump())
contexts['DENSITY'] = dens_ctxt, dens_ctxt_id
return contexts, funcs
def _L1_pdict(self):
pdict_id = self._L0_pdict()
param_context_ids = self.dataset_management.read_parameter_contexts(pdict_id,id_only=True)
# TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10
tl1_func = '(TEMPWAT_L0 / 10000) - 10'
tl1_pmap = {'TEMPWAT_L0':'TEMPWAT_L0'}
func = NumexprFunction('TEMPWAT_L1', tl1_func, tl1_pmap)
tempL1_ctxt = ParameterContext('TEMPWAT_L1', param_type=ParameterFunctionType(function=func), variability=VariabilityEnum.TEMPORAL)
tempL1_ctxt.uom = 'deg_C'
tempL1_ctxt_id = self.dataset_management.create_parameter_context(name=tempL1_ctxt.name, parameter_context=tempL1_ctxt.dump(), parameter_type='pfunc', unit_of_measure=tempL1_ctxt.uom)
param_context_ids.append(tempL1_ctxt_id)
# CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5
cl1_func = '(CONDWAT_L0 / 100000) - 0.5'
cl1_pmap = {'CONDWAT_L0':'CONDWAT_L0'}
func = NumexprFunction('CONDWAT_L1', cl1_func, cl1_pmap)
condL1_ctxt = ParameterContext('CONDWAT_L1', param_type=ParameterFunctionType(function=func), variability=VariabilityEnum.TEMPORAL)
condL1_ctxt.uom = 'S m-1'
condL1_ctxt_id = self.dataset_management.create_parameter_context(name=condL1_ctxt.name, parameter_context=condL1_ctxt.dump(), parameter_type='pfunc', unit_of_measure=condL1_ctxt.uom)
param_context_ids.append(condL1_ctxt_id)
# Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721
# PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range)
pl1_func = '(PRESWAT_L0 * 679.34040721 / (0.85 * 65536)) - (0.05 * 679.34040721)'
pl1_pmap = {'PRESWAT_L0':'PRESWAT_L0'}
func = NumexprFunction('PRESWAT_L1', pl1_func, pl1_pmap)
presL1_ctxt = ParameterContext('PRESWAT_L1', param_type=ParameterFunctionType(function=func), variability=VariabilityEnum.TEMPORAL)
presL1_ctxt.uom = 'S m-1'
presL1_ctxt_id = self.dataset_management.create_parameter_context(name=presL1_ctxt.name, parameter_context=presL1_ctxt.dump(), parameter_type='pfunc', unit_of_measure=presL1_ctxt.uom)
param_context_ids.append(presL1_ctxt_id)
# Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project:
# https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1
# PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1)
owner = 'gsw'
sal_func = 'SP_from_C'
sal_arglist = [NumexprFunction('CONDWAT_L1*10', 'C*10', {'C':'CONDWAT_L1'}), 'TEMPWAT_L1', 'PRESWAT_L1']
sal_kwargmap = None
func = PythonFunction('PRACSAL', owner, sal_func, sal_arglist, sal_kwargmap)
sal_ctxt = ParameterContext('PRACSAL', param_type=ParameterFunctionType(func), variability=VariabilityEnum.TEMPORAL)
sal_ctxt.uom = 'g kg-1'
sal_ctxt_id = self.dataset_management.create_parameter_context(name=sal_ctxt.name, parameter_context=sal_ctxt.dump(), parameter_type='pfunc', unit_of_measure=sal_ctxt.uom)
param_context_ids.append(sal_ctxt_id)
# absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude)
# conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1)
# DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1)
owner = 'gsw'
abs_sal_func = PythonFunction('abs_sal', owner, 'SA_from_SP', ['PRACSAL', 'PRESWAT_L1', 'lon','lat'], None)
#abs_sal_func = PythonFunction('abs_sal', owner, 'SA_from_SP', ['lon','lat'], None)
cons_temp_func = PythonFunction('cons_temp', owner, 'CT_from_t', [abs_sal_func, 'TEMPWAT_L1', 'PRESWAT_L1'], None)
dens_func = PythonFunction('DENSITY', owner, 'rho', [abs_sal_func, cons_temp_func, 'PRESWAT_L1'], None)
dens_ctxt = ParameterContext('DENSITY', param_type=ParameterFunctionType(dens_func), variability=VariabilityEnum.TEMPORAL)
dens_ctxt.uom = 'kg m-3'
dens_ctxt_id = self.dataset_management.create_parameter_context(name=dens_ctxt.name, parameter_context=dens_ctxt.dump(), parameter_type='pfunc', unit_of_measure=dens_ctxt.uom)
param_context_ids.append(dens_ctxt_id)
pdict_id = self.dataset_management.create_parameter_dictionary('L1_SBE37', parameter_context_ids=param_context_ids, temporal_context='time')
return pdict_id
def _create_input_param_dict_for_test(self, parameter_dict_name=''):
pdict = ParameterDictionary()
t_ctxt = ParameterContext(
'time',
param_type=QuantityType(value_encoding=numpy.dtype('float64')))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1900'
pdict.add_context(t_ctxt)
cond_ctxt = ParameterContext(
'conductivity',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
cond_ctxt.uom = ''
pdict.add_context(cond_ctxt)
pres_ctxt = ParameterContext(
'pressure',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
pres_ctxt.uom = ''
pdict.add_context(pres_ctxt)
temp_ctxt = ParameterContext(
'temperature',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = ''
pdict.add_context(temp_ctxt)
dens_ctxt = ParameterContext(
'density',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
dens_ctxt.uom = ''
pdict.add_context(dens_ctxt)
sal_ctxt = ParameterContext(
'salinity',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
sal_ctxt.uom = ''
pdict.add_context(sal_ctxt)
#create temp streamdef so the data product can create the stream
pc_list = []
for pc_k, pc in pdict.iteritems():
ctxt_id = self.dataset_management.create_parameter_context(
pc_k, pc[1].dump())
pc_list.append(ctxt_id)
self.addCleanup(self.dataset_management.delete_parameter_context,
ctxt_id)
pdict_id = self.dataset_management.create_parameter_dictionary(
parameter_dict_name, pc_list)
self.addCleanup(self.dataset_management.delete_parameter_dictionary,
pdict_id)
return pdict_id
def _create_input_param_dict_for_test(self, parameter_dict_name = ''):
pdict = ParameterDictionary()
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=numpy.dtype('float64')))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1900'
pdict.add_context(t_ctxt)
cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
cond_ctxt.uom = 'Siemens_per_meter'
pdict.add_context(cond_ctxt)
pres_ctxt = ParameterContext('pressure', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
pres_ctxt.uom = 'Pascal'
pdict.add_context(pres_ctxt)
if parameter_dict_name == 'input_param_for_L0':
temp_ctxt = ParameterContext('temperature', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
else:
temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = 'degree_kelvin'
pdict.add_context(temp_ctxt)
dens_ctxt = ParameterContext('density', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
dens_ctxt.uom = 'g/m'
pdict.add_context(dens_ctxt)
sal_ctxt = ParameterContext('salinity', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
sal_ctxt.uom = 'PSU'
pdict.add_context(sal_ctxt)
#create temp streamdef so the data product can create the stream
pc_list = []
for pc_k, pc in pdict.iteritems():
ctxt_id = self.dataset_management.create_parameter_context(pc_k, pc[1].dump())
pc_list.append(ctxt_id)
if parameter_dict_name == 'input_param_for_L0':
self.addCleanup(self.dataset_management.delete_parameter_context,ctxt_id)
elif pc[1].name == 'temp':
self.addCleanup(self.dataset_management.delete_parameter_context,ctxt_id)
pdict_id = self.dataset_management.create_parameter_dictionary(parameter_dict_name, pc_list)
self.addCleanup(self.dataset_management.delete_parameter_dictionary, pdict_id)
return pdict_id
def get_string_array_lookup_value(self, value):
return self._placeholder(
value, lambda placeholder: ParameterContext(
name=placeholder,
param_type=SparseConstantType(base_type=ArrayType())))
def create_simple_qc(self):
contexts = {}
types_manager = TypesManager(self.dataset_management,None,None)
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('float64')))
t_ctxt.uom = 'seconds since 1900-01-01'
t_ctxt_id = self.dataset_management.create_parameter_context(name='time', parameter_context=t_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, t_ctxt_id)
contexts['time'] = (t_ctxt, t_ctxt_id)
temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=fill_value)
temp_ctxt.uom = 'deg_C'
temp_ctxt.ooi_short_name = 'TEMPWAT'
temp_ctxt.qc_contexts = types_manager.make_qc_functions('temp','TEMPWAT',lambda *args, **kwargs : None)
temp_ctxt_id = self.dataset_management.create_parameter_context(name='temp', parameter_context=temp_ctxt.dump(), ooi_short_name='TEMPWAT')
self.addCleanup(self.dataset_management.delete_parameter_context, temp_ctxt_id)
contexts['temp'] = temp_ctxt, temp_ctxt_id
press_ctxt = ParameterContext('pressure', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=fill_value)
press_ctxt.uom = 'dbar'
press_ctxt.ooi_short_name = 'PRESWAT'
press_ctxt.qc_contexts = types_manager.make_qc_functions('pressure', 'PRESWAT', lambda *args, **kwargs : None)
press_ctxt_id = self.dataset_management.create_parameter_context(name='pressure', parameter_context=press_ctxt.dump(), ooi_short_name='PRESWAT')
self.addCleanup(self.dataset_management.delete_parameter_context, press_ctxt_id)
contexts['pressure'] = press_ctxt, press_ctxt_id
lat_ctxt = ParameterContext('lat', param_type=SparseConstantType(base_type=ConstantType(value_encoding='float64'), fill_value=fill_value), fill_value=fill_value)
lat_ctxt.uom = 'degree_north'
lat_ctxt_id = self.dataset_management.create_parameter_context(name='lat', parameter_context=lat_ctxt.dump())
contexts['lat'] = lat_ctxt, lat_ctxt_id
lon_ctxt = ParameterContext('lon', param_type=SparseConstantType(base_type=ConstantType(value_encoding='float64'), fill_value=fill_value), fill_value=fill_value)
lon_ctxt.uom = 'degree_east'
lon_ctxt_id = self.dataset_management.create_parameter_context(name='lon', parameter_context=lon_ctxt.dump())
contexts['lon'] = lon_ctxt, lon_ctxt_id
return contexts
def create_lookup_contexts(self):
contexts = {}
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('float64')))
t_ctxt.uom = 'seconds since 1900-01-01'
t_ctxt_id = self.dataset_management.create_parameter_context(name='time', parameter_context=t_ctxt.dump())
contexts['time'] = (t_ctxt, t_ctxt_id)
temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=fill_value)
temp_ctxt.uom = 'deg_C'
temp_ctxt_id = self.dataset_management.create_parameter_context(name='temp', parameter_context=temp_ctxt.dump())
contexts['temp'] = temp_ctxt, temp_ctxt_id
offset_ctxt = ParameterContext(name='offset_a', param_type=SparseConstantType(base_type=ConstantType(value_encoding='float64'), fill_value=fill_value))
offset_ctxt.uom = ''
offset_ctxt.lookup_value = 'offset_a'
offset_ctxt.document_key = ''
offset_ctxt_id = self.dataset_management.create_parameter_context(name='offset_a', parameter_context=offset_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, offset_ctxt_id)
contexts['offset_a'] = offset_ctxt, offset_ctxt_id
offsetb_ctxt = ParameterContext('offset_b', param_type=SparseConstantType(base_type=ConstantType(value_encoding='float64'), fill_value=fill_value))
offsetb_ctxt.uom = ''
offsetb_ctxt.lookup_value = 'offset_b'
offsetb_ctxt.document_key = 'coefficient_document'
offsetb_ctxt_id = self.dataset_management.create_parameter_context(name='offset_b', parameter_context=offsetb_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, offsetb_ctxt_id)
contexts['offset_b'] = offsetb_ctxt, offsetb_ctxt_id
offsetc_ctxt = ParameterContext('offset_c', param_type=SparseConstantType(base_type=ConstantType(value_encoding='float64'), fill_value=fill_value))
offsetc_ctxt.uom = ''
offsetc_ctxt.lookup_value = 'offset_c'
offsetc_ctxt.document_key = '$designator_OFFSETC'
offsetc_ctxt_id = self.dataset_management.create_parameter_context(name='offset_c', parameter_context=offsetc_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, offsetc_ctxt_id)
contexts['offset_c'] = offsetc_ctxt, offsetc_ctxt_id
func = NumexprFunction('calibrated', 'temp + offset', ['temp','offset'], param_map={'temp':'temp', 'offset':'offset_a'})
func.lookup_values = ['LV_offset']
calibrated = ParameterContext('calibrated', param_type=ParameterFunctionType(func, value_encoding='float32'), fill_value=fill_value)
calibrated.uom = 'deg_C'
calibrated_id = self.dataset_management.create_parameter_context(name='calibrated', parameter_context=calibrated.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, calibrated_id)
contexts['calibrated'] = calibrated, calibrated_id
func = NumexprFunction('calibrated_b', 'temp + offset_a + offset_b', ['temp','offset_a', 'offset_b'], param_map={'temp':'temp', 'offset_a':'offset_a', 'offset_b':'offset_b'})
func.lookup_values = ['LV_offset_a', 'LV_offset_b']
calibrated_b = ParameterContext('calibrated_b', param_type=ParameterFunctionType(func, value_encoding='float32'), fill_value=fill_value)
calibrated_b.uom = 'deg_C'
calibrated_b_id = self.dataset_management.create_parameter_context(name='calibrated_b', parameter_context=calibrated_b.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, calibrated_b_id)
contexts['calibrated_b'] = calibrated_b, calibrated_b_id
return contexts
def _get_pdict(self, filter_values):
t_ctxt = ParameterContext(
'TIME', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 01-01-1900'
t_ctxt_id = self.dataset_management.create_parameter_context(
name='TIME',
parameter_context=t_ctxt.dump(),
parameter_type='quantity<int64>',
unit_of_measure=t_ctxt.uom)
lat_ctxt = ParameterContext(
'LAT',
param_type=ConstantType(
QuantityType(value_encoding=np.dtype('float32'))),
fill_value=-9999)
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt_id = self.dataset_management.create_parameter_context(
name='LAT',
parameter_context=lat_ctxt.dump(),
parameter_type='quantity<float32>',
unit_of_measure=lat_ctxt.uom)
lon_ctxt = ParameterContext(
'LON',
param_type=ConstantType(
QuantityType(value_encoding=np.dtype('float32'))),
fill_value=-9999)
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt_id = self.dataset_management.create_parameter_context(
name='LON',
parameter_context=lon_ctxt.dump(),
parameter_type='quantity<float32>',
unit_of_measure=lon_ctxt.uom)
# Independent Parameters
# Temperature - values expected to be the decimal results of conversion from hex
temp_ctxt = ParameterContext(
'TEMPWAT_L0',
param_type=QuantityType(value_encoding=np.dtype('float32')),
fill_value=-9999)
temp_ctxt.uom = 'deg_C'
temp_ctxt_id = self.dataset_management.create_parameter_context(
name='TEMPWAT_L0',
parameter_context=temp_ctxt.dump(),
parameter_type='quantity<float32>',
unit_of_measure=temp_ctxt.uom)
# Conductivity - values expected to be the decimal results of conversion from hex
cond_ctxt = ParameterContext(
'CONDWAT_L0',
param_type=QuantityType(value_encoding=np.dtype('float32')),
fill_value=-9999)
cond_ctxt.uom = 'S m-1'
cond_ctxt_id = self.dataset_management.create_parameter_context(
name='CONDWAT_L0',
parameter_context=cond_ctxt.dump(),
parameter_type='quantity<float32>',
unit_of_measure=cond_ctxt.uom)
# Pressure - values expected to be the decimal results of conversion from hex
press_ctxt = ParameterContext(
'PRESWAT_L0',
param_type=QuantityType(value_encoding=np.dtype('float32')),
fill_value=-9999)
press_ctxt.uom = 'dbar'
press_ctxt_id = self.dataset_management.create_parameter_context(
name='PRESWAT_L0',
parameter_context=press_ctxt.dump(),
parameter_type='quantity<float32>',
unit_of_measure=press_ctxt.uom)
# Dependent Parameters
# TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10
tl1_func = '(T / 10000) - 10'
tl1_pmap = {'T': 'TEMPWAT_L0'}
expr = NumexprFunction('TEMPWAT_L1',
tl1_func, ['T'],
param_map=tl1_pmap)
tempL1_ctxt = ParameterContext(
'TEMPWAT_L1',
param_type=ParameterFunctionType(function=expr),
variability=VariabilityEnum.TEMPORAL)
tempL1_ctxt.uom = 'deg_C'
tempL1_ctxt_id = self.dataset_management.create_parameter_context(
name=tempL1_ctxt.name,
parameter_context=tempL1_ctxt.dump(),
parameter_type='pfunc',
unit_of_measure=tempL1_ctxt.uom)
# CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5
cl1_func = '(C / 100000) - 0.5'
cl1_pmap = {'C': 'CONDWAT_L0'}
expr = NumexprFunction('CONDWAT_L1',
cl1_func, ['C'],
param_map=cl1_pmap)
condL1_ctxt = ParameterContext(
'CONDWAT_L1',
param_type=ParameterFunctionType(function=expr),
variability=VariabilityEnum.TEMPORAL)
condL1_ctxt.uom = 'S m-1'
condL1_ctxt_id = self.dataset_management.create_parameter_context(
name=condL1_ctxt.name,
parameter_context=condL1_ctxt.dump(),
parameter_type='pfunc',
unit_of_measure=condL1_ctxt.uom)
# Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721
# PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range)
pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)'
pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721}
expr = NumexprFunction('PRESWAT_L1',
pl1_func, ['P', 'p_range'],
param_map=pl1_pmap)
presL1_ctxt = ParameterContext(
'PRESWAT_L1',
param_type=ParameterFunctionType(function=expr),
variability=VariabilityEnum.TEMPORAL)
presL1_ctxt.uom = 'S m-1'
presL1_ctxt_id = self.dataset_management.create_parameter_context(
name=presL1_ctxt.name,
parameter_context=presL1_ctxt.dump(),
parameter_type='pfunc',
unit_of_measure=presL1_ctxt.uom)
# Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project:
# https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1
# PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1)
owner = 'gsw'
sal_func = 'SP_from_C'
sal_arglist = ['C', 't', 'p']
sal_pmap = {
'C':
NumexprFunction('CONDWAT_L1*10',
'C*10', ['C'],
param_map={'C': 'CONDWAT_L1'}),
't':
'TEMPWAT_L1',
'p':
'PRESWAT_L1'
}
sal_kwargmap = None
expr = PythonFunction('PRACSAL', owner, sal_func, sal_arglist,
sal_kwargmap, sal_pmap)
sal_ctxt = ParameterContext('PRACSAL',
param_type=ParameterFunctionType(expr),
variability=VariabilityEnum.TEMPORAL)
sal_ctxt.uom = 'g kg-1'
sal_ctxt_id = self.dataset_management.create_parameter_context(
name=sal_ctxt.name,
parameter_context=sal_ctxt.dump(),
parameter_type='pfunc',
unit_of_measure=sal_ctxt.uom)
# absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude)
# conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1)
# DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1)
owner = 'gsw'
abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP',
['PRACSAL', 'PRESWAT_L1', 'LON', 'LAT'])
cons_temp_expr = PythonFunction(
'cons_temp', owner, 'CT_from_t',
[abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1'])
dens_expr = PythonFunction(
'DENSITY', owner, 'rho',
[abs_sal_expr, cons_temp_expr, 'PRESWAT_L1'])
dens_ctxt = ParameterContext(
'DENSITY',
param_type=ParameterFunctionType(dens_expr),
variability=VariabilityEnum.TEMPORAL)
dens_ctxt.uom = 'kg m-3'
dens_ctxt_id = self.dataset_management.create_parameter_context(
name=dens_ctxt.name,
parameter_context=dens_ctxt.dump(),
parameter_type='pfunc',
unit_of_measure=dens_ctxt.uom)
ids = [
t_ctxt_id, lat_ctxt_id, lon_ctxt_id, temp_ctxt_id, cond_ctxt_id,
press_ctxt_id, tempL1_ctxt_id, condL1_ctxt_id, presL1_ctxt_id,
sal_ctxt_id, dens_ctxt_id
]
contexts = [
t_ctxt, lat_ctxt, lon_ctxt, temp_ctxt, cond_ctxt, press_ctxt,
tempL1_ctxt, condL1_ctxt, presL1_ctxt, sal_ctxt, dens_ctxt
]
context_ids = [
ids[i] for i, ctxt in enumerate(contexts)
if ctxt.name in filter_values
]
pdict_name = '_'.join(
[ctxt.name for ctxt in contexts if ctxt.name in filter_values])
try:
self.pdicts[pdict_name]
return self.pdicts[pdict_name]
except KeyError:
pdict_id = self.dataset_management.create_parameter_dictionary(
pdict_name,
parameter_context_ids=context_ids,
temporal_context='time')
self.pdicts[pdict_name] = pdict_id
return pdict_id
def create_pfuncs(self):
contexts = {}
funcs = {}
t_ctxt = ParameterContext(name='TIME',
parameter_type='quantity',
value_encoding='float64',
units='seconds since 1900-01-01')
t_ctxt_id = self.dataset_management.create_parameter(t_ctxt)
contexts['TIME'] = t_ctxt_id
lat_ctxt = ParameterContext(name='LAT',
parameter_type="sparse",
value_encoding='float32',
units='degrees_north')
lat_ctxt_id = self.dataset_management.create_parameter(lat_ctxt)
contexts['LAT'] = lat_ctxt_id
lon_ctxt = ParameterContext(name='LON',
parameter_type='sparse',
value_encoding='float32',
units='degrees_east')
lon_ctxt_id = self.dataset_management.create_parameter(lon_ctxt)
contexts['LON'] = lon_ctxt_id
# Independent Parameters
# Temperature - values expected to be the decimal results of conversion from hex
temp_ctxt = ParameterContext(name='TEMPWAT_L0',
parameter_type='quantity',
value_encoding='float32',
units='deg_C')
temp_ctxt_id = self.dataset_management.create_parameter(temp_ctxt)
contexts['TEMPWAT_L0'] = temp_ctxt_id
# Conductivity - values expected to be the decimal results of conversion from hex
cond_ctxt = ParameterContext(name='CONDWAT_L0',
parameter_type='quantity',
value_encoding='float32',
units='S m-1')
cond_ctxt_id = self.dataset_management.create_parameter(cond_ctxt)
contexts['CONDWAT_L0'] = cond_ctxt_id
# Pressure - values expected to be the decimal results of conversion from hex
press_ctxt = ParameterContext(name='PRESWAT_L0',
parameter_type='quantity',
value_encoding='float32',
units='dbar')
press_ctxt_id = self.dataset_management.create_parameter(press_ctxt)
contexts['PRESWAT_L0'] = press_ctxt_id
# Dependent Parameters
# TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10
tl1_func = '(T / 10000) - 10'
expr = ParameterFunction(name='TEMPWAT_L1',
function_type=PFT.NUMEXPR,
function=tl1_func,
args=['T'])
expr_id = self.dataset_management.create_parameter_function(expr)
funcs['TEMPWAT_L1'] = expr_id
tl1_pmap = {'T': 'TEMPWAT_L0'}
tempL1_ctxt = ParameterContext(name='TEMPWAT_L1',
parameter_type='function',
parameter_function_id=expr_id,
parameter_function_map=tl1_pmap,
value_encoding='float32',
units='deg_C')
tempL1_ctxt_id = self.dataset_management.create_parameter(tempL1_ctxt)
contexts['TEMPWAT_L1'] = tempL1_ctxt_id
# CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5
cl1_func = '(C / 100000) - 0.5'
expr = ParameterFunction(name='CONDWAT_L1',
function_type=PFT.NUMEXPR,
function=cl1_func,
args=['C'])
expr_id = self.dataset_management.create_parameter_function(expr)
funcs['CONDWAT_L1'] = expr_id
cl1_pmap = {'C': 'CONDWAT_L0'}
condL1_ctxt = ParameterContext(name='CONDWAT_L1',
parameter_type='function',
parameter_function_id=expr_id,
parameter_function_map=cl1_pmap,
value_encoding='float32',
units='S m-1')
condL1_ctxt_id = self.dataset_management.create_parameter(condL1_ctxt)
contexts['CONDWAT_L1'] = condL1_ctxt_id
# Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721
# PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range)
pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)'
expr = ParameterFunction(name='PRESWAT_L1',function=pl1_func,function_type=PFT.NUMEXPR,args=['P','p_range'])
expr_id = self.dataset_management.create_parameter_function(expr)
funcs['PRESWAT_L1'] = expr_id
pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721}
presL1_ctxt = ParameterContext(name='PRESWAT_L1',
parameter_type='function',
parameter_function_id=expr_id,
parameter_function_map=pl1_pmap,
value_encoding='float32',
units='S m-1')
presL1_ctxt_id = self.dataset_management.create_parameter(presL1_ctxt)
contexts['PRESWAT_L1'] = presL1_ctxt_id
# Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project:
# https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1
# PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1)
owner = 'gsw'
sal_func = 'SP_from_C'
sal_arglist = ['C', 't', 'p']
expr = ParameterFunction(name='PRACSAL',function_type=PFT.PYTHON,function=sal_func,owner=owner,args=sal_arglist)
expr_id = self.dataset_management.create_parameter_function(expr)
funcs['PRACSAL'] = expr_id
c10_f = ParameterFunction(name='c10', function_type=PFT.NUMEXPR, function='C*10', args=['C'])
expr_id = self.dataset_management.create_parameter_function(c10_f)
c10 = ParameterContext(name='c10',
parameter_type='function',
parameter_function_id=expr_id,
parameter_function_map={'C':'CONDWAT_L1'},
value_encoding='float32',
units='1')
c10_id = self.dataset_management.create_parameter(c10)
contexts['c10'] = c10_id
# A magic function that may or may not exist actually forms the line below at runtime.
sal_pmap = {'C': 'c10', 't': 'TEMPWAT_L1', 'p': 'PRESWAT_L1'}
sal_ctxt = ParameterContext(name='PRACSAL',
parameter_type='function',
parameter_function_id=expr_id,
parameter_function_map=sal_pmap,
value_encoding='float32',
units='g kg-1')
sal_ctxt_id = self.dataset_management.create_parameter(sal_ctxt)
contexts['PRACSAL'] = sal_ctxt_id
# absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude)
# conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1)
# DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1)
owner = 'gsw'
abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP', ['PRACSAL', 'PRESWAT_L1', 'LON','LAT'])
cons_temp_expr = PythonFunction('cons_temp', owner, 'CT_from_t', [abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1'])
dens_expr = PythonFunction('DENSITY', owner, 'rho', [abs_sal_expr, cons_temp_expr, 'PRESWAT_L1'])
dens_ctxt = CoverageParameterContext('DENSITY', param_type=ParameterFunctionType(dens_expr), variability=VariabilityEnum.TEMPORAL)
dens_ctxt.uom = 'kg m-3'
dens_ctxt_id = self.dataset_management.create_parameter_context(name='DENSITY', parameter_context=dens_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, dens_ctxt_id)
contexts['DENSITY'] = dens_ctxt_id
return contexts, funcs
def _get_pdict(self, filter_values):
t_ctxt = ParameterContext('TIME', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 01-01-1900'
t_ctxt_id = self.dataset_management.create_parameter_context(name='TIME', parameter_context=t_ctxt.dump(), parameter_type='quantity<int64>', units=t_ctxt.uom)
self.add_context_to_cleanup(t_ctxt_id)
lat_ctxt = ParameterContext('LAT', param_type=ConstantType(QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999)
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt_id = self.dataset_management.create_parameter_context(name='LAT', parameter_context=lat_ctxt.dump(), parameter_type='quantity<float32>', units=lat_ctxt.uom)
self.add_context_to_cleanup(lat_ctxt_id)
lon_ctxt = ParameterContext('LON', param_type=ConstantType(QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999)
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt_id = self.dataset_management.create_parameter_context(name='LON', parameter_context=lon_ctxt.dump(), parameter_type='quantity<float32>', units=lon_ctxt.uom)
self.add_context_to_cleanup(lon_ctxt_id)
# Independent Parameters
# Temperature - values expected to be the decimal results of conversion from hex
temp_ctxt = ParameterContext('TEMPWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999)
temp_ctxt.uom = 'deg_C'
temp_ctxt_id = self.dataset_management.create_parameter_context(name='TEMPWAT_L0', parameter_context=temp_ctxt.dump(), parameter_type='quantity<float32>', units=temp_ctxt.uom)
self.add_context_to_cleanup(temp_ctxt_id)
# Conductivity - values expected to be the decimal results of conversion from hex
cond_ctxt = ParameterContext('CONDWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999)
cond_ctxt.uom = 'S m-1'
cond_ctxt_id = self.dataset_management.create_parameter_context(name='CONDWAT_L0', parameter_context=cond_ctxt.dump(), parameter_type='quantity<float32>', units=cond_ctxt.uom)
self.add_context_to_cleanup(cond_ctxt_id)
# Pressure - values expected to be the decimal results of conversion from hex
press_ctxt = ParameterContext('PRESWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999)
press_ctxt.uom = 'dbar'
press_ctxt_id = self.dataset_management.create_parameter_context(name='PRESWAT_L0', parameter_context=press_ctxt.dump(), parameter_type='quantity<float32>', units=press_ctxt.uom)
self.add_context_to_cleanup(press_ctxt_id)
# Dependent Parameters
# TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10
tl1_func = '(T / 10000) - 10'
tl1_pmap = {'T': 'TEMPWAT_L0'}
expr = NumexprFunction('TEMPWAT_L1', tl1_func, ['T'], param_map=tl1_pmap)
tempL1_ctxt = ParameterContext('TEMPWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL)
tempL1_ctxt.uom = 'deg_C'
tempL1_ctxt_id = self.dataset_management.create_parameter_context(name=tempL1_ctxt.name, parameter_context=tempL1_ctxt.dump(), parameter_type='pfunc', units=tempL1_ctxt.uom)
self.add_context_to_cleanup(tempL1_ctxt_id)
# CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5
cl1_func = '(C / 100000) - 0.5'
cl1_pmap = {'C': 'CONDWAT_L0'}
expr = NumexprFunction('CONDWAT_L1', cl1_func, ['C'], param_map=cl1_pmap)
condL1_ctxt = ParameterContext('CONDWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL)
condL1_ctxt.uom = 'S m-1'
condL1_ctxt_id = self.dataset_management.create_parameter_context(name=condL1_ctxt.name, parameter_context=condL1_ctxt.dump(), parameter_type='pfunc', units=condL1_ctxt.uom)
self.add_context_to_cleanup(condL1_ctxt_id)
# Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721
# PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range)
pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)'
pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721}
expr = NumexprFunction('PRESWAT_L1', pl1_func, ['P', 'p_range'], param_map=pl1_pmap)
presL1_ctxt = ParameterContext('PRESWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL)
presL1_ctxt.uom = 'S m-1'
presL1_ctxt_id = self.dataset_management.create_parameter_context(name=presL1_ctxt.name, parameter_context=presL1_ctxt.dump(), parameter_type='pfunc', units=presL1_ctxt.uom)
self.add_context_to_cleanup(presL1_ctxt_id)
# Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project:
# https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1
# PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1)
owner = 'gsw'
sal_func = 'SP_from_C'
sal_arglist = ['C', 't', 'p']
sal_pmap = {'C': NumexprFunction('CONDWAT_L1*10', 'C*10', ['C'], param_map={'C': 'CONDWAT_L1'}), 't': 'TEMPWAT_L1', 'p': 'PRESWAT_L1'}
sal_kwargmap = None
expr = PythonFunction('PRACSAL', owner, sal_func, sal_arglist, sal_kwargmap, sal_pmap)
sal_ctxt = ParameterContext('PRACSAL', param_type=ParameterFunctionType(expr), variability=VariabilityEnum.TEMPORAL)
sal_ctxt.uom = 'g kg-1'
sal_ctxt_id = self.dataset_management.create_parameter_context(name=sal_ctxt.name, parameter_context=sal_ctxt.dump(), parameter_type='pfunc', units=sal_ctxt.uom)
self.add_context_to_cleanup(sal_ctxt_id)
# absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude)
# conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1)
# DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1)
owner = 'gsw'
abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP', ['PRACSAL', 'PRESWAT_L1', 'LON','LAT'])
cons_temp_expr = PythonFunction('cons_temp', owner, 'CT_from_t', [abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1'])
dens_expr = PythonFunction('DENSITY', owner, 'rho', [abs_sal_expr, cons_temp_expr, 'PRESWAT_L1'])
dens_ctxt = ParameterContext('DENSITY', param_type=ParameterFunctionType(dens_expr), variability=VariabilityEnum.TEMPORAL)
dens_ctxt.uom = 'kg m-3'
dens_ctxt_id = self.dataset_management.create_parameter_context(name=dens_ctxt.name, parameter_context=dens_ctxt.dump(), parameter_type='pfunc', units=dens_ctxt.uom)
self.add_context_to_cleanup(dens_ctxt_id)
ids = [t_ctxt_id, lat_ctxt_id, lon_ctxt_id, temp_ctxt_id, cond_ctxt_id, press_ctxt_id, tempL1_ctxt_id, condL1_ctxt_id, presL1_ctxt_id, sal_ctxt_id, dens_ctxt_id]
contexts = [t_ctxt, lat_ctxt, lon_ctxt, temp_ctxt, cond_ctxt, press_ctxt, tempL1_ctxt, condL1_ctxt, presL1_ctxt, sal_ctxt, dens_ctxt]
context_ids = [ids[i] for i,ctxt in enumerate(contexts) if ctxt.name in filter_values]
pdict_name = '_'.join([ctxt.name for ctxt in contexts if ctxt.name in filter_values])
try:
self.pdicts[pdict_name]
return self.pdicts[pdict_name]
except KeyError:
pdict_id = self.dataset_management.create_parameter_dictionary(pdict_name, parameter_context_ids=context_ids, temporal_context='time')
self.addCleanup(self.dataset_management.delete_parameter_dictionary, pdict_id)
self.pdicts[pdict_name] = pdict_id
return pdict_id
def create_contexts(self):
context_ids = []
cond_ctxt = ParameterContext('conductivity_test', param_type=QuantityType(value_encoding=np.float32))
cond_ctxt.uom = 'unknown'
cond_ctxt.fill_value = 0e0
context_ids.append(self.dataset_management.create_parameter_context(name='conductivity_test', parameter_context=cond_ctxt.dump()))
pres_ctxt = ParameterContext('pressure_test', param_type=QuantityType(value_encoding=np.float32))
pres_ctxt.uom = 'Pascal'
pres_ctxt.fill_value = 0x0
context_ids.append(self.dataset_management.create_parameter_context(name='pressure_test', parameter_context=pres_ctxt.dump()))
sal_ctxt = ParameterContext('salinity_test', param_type=QuantityType(value_encoding=np.float32))
sal_ctxt.uom = 'PSU'
sal_ctxt.fill_value = 0x0
context_ids.append(self.dataset_management.create_parameter_context(name='salinity_test', parameter_context=sal_ctxt.dump()))
temp_ctxt = ParameterContext('temp_test', param_type=QuantityType(value_encoding=np.float32))
temp_ctxt.uom = 'degree_Celsius'
temp_ctxt.fill_value = 0e0
context_ids.append(self.dataset_management.create_parameter_context(name='temp_test', parameter_context=temp_ctxt.dump()))
t_ctxt = ParameterContext('time_test', param_type=QuantityType(value_encoding=np.int64))
t_ctxt.uom = 'seconds since 1970-01-01'
t_ctxt.fill_value = 0x0
context_ids.append(self.dataset_management.create_parameter_context(name='time_test', parameter_context=t_ctxt.dump()))
return context_ids
def create_parsed_params(self):
contexts = {}
funcs = {}
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('float64')))
t_ctxt.uom = 'seconds since 1900-01-01'
t_ctxt_id = self.dataset_management.create_parameter_context(name='time', parameter_context=t_ctxt.dump())
contexts['time'] = (t_ctxt, t_ctxt_id)
lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=fill_value)
lat_ctxt.uom = 'degree_north'
lat_ctxt_id = self.dataset_management.create_parameter_context(name='lat', parameter_context=lat_ctxt.dump())
contexts['lat'] = lat_ctxt, lat_ctxt_id
lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=fill_value)
lon_ctxt.uom = 'degree_east'
lon_ctxt_id = self.dataset_management.create_parameter_context(name='lon', parameter_context=lon_ctxt.dump())
contexts['lon'] = lon_ctxt, lon_ctxt_id
# Independent Parameters
# Temperature - values expected to be the decimal results of conversion from hex
temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=fill_value)
temp_ctxt.uom = 'deg_C'
temp_ctxt_id = self.dataset_management.create_parameter_context(name='temp', parameter_context=temp_ctxt.dump())
contexts['temp'] = temp_ctxt, temp_ctxt_id
# Conductivity - values expected to be the decimal results of conversion from hex
cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=fill_value)
cond_ctxt.uom = 'S m-1'
cond_ctxt_id = self.dataset_management.create_parameter_context(name='conductivity', parameter_context=cond_ctxt.dump())
contexts['conductivity'] = cond_ctxt, cond_ctxt_id
# Pressure - values expected to be the decimal results of conversion from hex
press_ctxt = ParameterContext('pressure', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=fill_value)
press_ctxt.uom = 'dbar'
press_ctxt_id = self.dataset_management.create_parameter_context(name='pressure', parameter_context=press_ctxt.dump())
contexts['pressure'] = press_ctxt, press_ctxt_id
preffered_ctxt = ParameterContext('preferred_timestamp', param_type=CategoryType(categories={0:'port_timestamp', 1:'driver_timestamp', 2:'internal_timestamp', 3:'time', -99:'empty'}), fill_value=-99)
preffered_ctxt.uom = ''
preffered_ctxt_id = self.dataset_management.create_parameter_context(name='preferred_timestamp', parameter_context=preffered_ctxt.dump())
contexts['preferred_timestamp'] = preffered_ctxt, preffered_ctxt_id
port_ctxt = ParameterContext('port_timestamp', param_type=QuantityType(value_encoding=np.dtype('float64')), fill_value=fill_value)
port_ctxt.uom = 'seconds since 1900-01-01'
port_ctxt_id = self.dataset_management.create_parameter_context(name='port_timestamp', parameter_context=port_ctxt.dump())
contexts['port_timestamp'] = port_ctxt, port_ctxt_id
driver_ctxt = ParameterContext('driver_timestamp', param_type=QuantityType(value_encoding=np.dtype('float64')), fill_value=fill_value)
driver_ctxt.uom = 'seconds since 1900-01-01'
driver_ctxt_id = self.dataset_management.create_parameter_context(name='driver_timestamp', parameter_context=driver_ctxt.dump())
contexts['driver_timestamp'] = driver_ctxt, driver_ctxt_id
internal_ctxt = ParameterContext('internal_timestamp', param_type=QuantityType(value_encoding=np.dtype('float64')), fill_value=fill_value)
internal_ctxt.uom = 'seconds since 1900-01-01'
internal_ctxt_id = self.dataset_management.create_parameter_context(name='internal_timestamp', parameter_context=internal_ctxt.dump())
contexts['internal_timestamp'] = internal_ctxt, internal_ctxt_id
quality_ctxt = ParameterContext('quality_flag', param_type=ArrayType())
quality_ctxt.uom = ''
quality_ctxt_id = self.dataset_management.create_parameter_context(name='quality_flag', parameter_context=quality_ctxt.dump())
contexts['quality_flag'] = quality_ctxt, quality_ctxt_id
# Dependent Parameters
# TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10
tl1_func = '(temperature / 10000.0) - 10'
expr = NumexprFunction('temp_L1', tl1_func, ['temperature'])
expr_id = self.dataset_management.create_parameter_function(name='temp_L1', parameter_function=expr.dump())
self.addCleanup(self.dataset_management.delete_parameter_function, expr_id)
funcs['temp_L1'] = expr, expr_id
tl1_pmap = {'temperature':'temp'}
expr.param_map = tl1_pmap
tempL1_ctxt = ParameterContext('temp_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL)
tempL1_ctxt.uom = 'deg_C'
tempL1_ctxt_id = self.dataset_management.create_parameter_context(name='temp_L1', parameter_context=tempL1_ctxt.dump(), parameter_function_id=expr_id)
self.addCleanup(self.dataset_management.delete_parameter_context, tempL1_ctxt_id)
contexts['temp_L1'] = tempL1_ctxt, tempL1_ctxt_id
# CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5
cl1_func = '(conductivity / 100000.0) - 0.5'
expr = NumexprFunction('conductivity_L1', cl1_func, ['conductivity'])
expr_id = self.dataset_management.create_parameter_function(name='conductivity_L1', parameter_function=expr.dump())
self.addCleanup(self.dataset_management.delete_parameter_function, expr_id)
funcs['conductivity_L1'] = expr, expr_id
cl1_pmap = {'conductivity':'conductivity'}
expr.param_map = cl1_pmap
condL1_ctxt = ParameterContext('conductivity_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL)
condL1_ctxt.uom = 'S m-1'
condL1_ctxt_id = self.dataset_management.create_parameter_context(name='conductivity_L1', parameter_context=condL1_ctxt.dump(), parameter_function_id=expr_id)
self.addCleanup(self.dataset_management.delete_parameter_context, condL1_ctxt_id)
contexts['conductivity_L1'] = condL1_ctxt, condL1_ctxt_id
# Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721
# PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range)
pl1_func = '(pressure / 100.0) + 0.5'
expr = NumexprFunction('pressure_L1', pl1_func, ['pressure'])
expr_id = self.dataset_management.create_parameter_function(name='pressure_L1', parameter_function=expr.dump())
self.addCleanup(self.dataset_management.delete_parameter_function, expr_id)
funcs['pressure_L1'] = expr, expr_id
pl1_pmap = {'pressure':'pressure'}
expr.param_map = pl1_pmap
presL1_ctxt = ParameterContext('pressure_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL)
presL1_ctxt.uom = 'S m-1'
presL1_ctxt_id = self.dataset_management.create_parameter_context(name='pressure_L1', parameter_context=presL1_ctxt.dump(), parameter_function_id=expr_id)
self.addCleanup(self.dataset_management.delete_parameter_context, presL1_ctxt_id)
contexts['pressure_L1'] = presL1_ctxt, presL1_ctxt_id
# Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project:
# https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1
# PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1)
owner = 'ion_functions.workflow_tests.fake_data'
sal_func = 'data_l2_salinity'
sal_arglist = ['conductivity', 'temp', 'pressure']
expr = PythonFunction('salinity_L2', owner, sal_func, sal_arglist)
expr_id = self.dataset_management.create_parameter_function(name='salinity_L2', parameter_function=expr.dump())
self.addCleanup(self.dataset_management.delete_parameter_function, expr_id)
funcs['salinity_L2'] = expr, expr_id
# A magic function that may or may not exist actually forms the line below at runtime.
sal_pmap = {'conductivity':'conductivity_L1', 'temp':'temp_L1', 'pressure':'pressure_L1'}
expr.param_map = sal_pmap
sal_ctxt = ParameterContext('salinity', param_type=ParameterFunctionType(expr), variability=VariabilityEnum.TEMPORAL)
sal_ctxt.uom = 'g kg-1'
sal_ctxt_id = self.dataset_management.create_parameter_context(name='salinity', parameter_context=sal_ctxt.dump(), parameter_function_id=expr_id)
self.addCleanup(self.dataset_management.delete_parameter_context, sal_ctxt_id)
contexts['salinity'] = sal_ctxt, sal_ctxt_id
# absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude)
# conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1)
# DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1)
owner = 'ion_functions.workflow_tests.fake_data'
dens_func = 'data_l2_density'
dens_arglist =['conductivity', 'temp', 'pressure', 'lat', 'lon']
expr = PythonFunction('density_L2', owner, dens_func, dens_arglist)
expr_id = self.dataset_management.create_parameter_function(name='density_L2', parameter_function=expr.dump())
self.addCleanup(self.dataset_management.delete_parameter_function, expr_id)
funcs['density_L2'] = expr, expr_id
dens_pmap = {'conductivity':'conductivity_L1', 'temp':'temp_L1', 'pressure':'pressure_L1', 'lat':'lat', 'lon':'lon'}
expr.param_map = dens_pmap
dens_ctxt = ParameterContext('density', param_type=ParameterFunctionType(expr), variability=VariabilityEnum.TEMPORAL)
dens_ctxt.uom = 'kg m-3'
dens_ctxt_id = self.dataset_management.create_parameter_context(name='density', parameter_context=dens_ctxt.dump(), parameter_function_id=expr_id)
self.addCleanup(self.dataset_management.delete_parameter_context, dens_ctxt_id)
contexts['density'] = dens_ctxt, dens_ctxt_id
return contexts, funcs
def _get_pdict(self, filter_values):
t_ctxt = ParameterContext("time", param_type=QuantityType(value_encoding=np.dtype("int64")))
t_ctxt.uom = "seconds since 01-01-1900"
t_ctxt.fill_value = -9999
t_ctxt_id = self.dataset_management.create_parameter_context(
name="time", parameter_context=t_ctxt.dump(), parameter_type="quantity<int64>", unit_of_measure=t_ctxt.uom
)
lat_ctxt = ParameterContext("lat", param_type=ConstantType(QuantityType(value_encoding=np.dtype("float32"))))
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = "degree_north"
lat_ctxt.fill_value = -9999
lat_ctxt_id = self.dataset_management.create_parameter_context(
name="lat",
parameter_context=lat_ctxt.dump(),
parameter_type="quantity<float32>",
unit_of_measure=lat_ctxt.uom,
)
lon_ctxt = ParameterContext("lon", param_type=ConstantType(QuantityType(value_encoding=np.dtype("float32"))))
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = "degree_east"
lon_ctxt.fill_value = -9999
lon_ctxt_id = self.dataset_management.create_parameter_context(
name="lon",
parameter_context=lon_ctxt.dump(),
parameter_type="quantity<float32>",
unit_of_measure=lon_ctxt.uom,
)
temp_ctxt = ParameterContext("TEMPWAT_L0", param_type=QuantityType(value_encoding=np.dtype("float32")))
temp_ctxt.uom = "deg_C"
temp_ctxt.fill_value = -9999
temp_ctxt_id = self.dataset_management.create_parameter_context(
name="TEMPWAT_L0",
parameter_context=temp_ctxt.dump(),
parameter_type="quantity<float32>",
unit_of_measure=temp_ctxt.uom,
)
# Conductivity - values expected to be the decimal results of conversion from hex
cond_ctxt = ParameterContext("CONDWAT_L0", param_type=QuantityType(value_encoding=np.dtype("float32")))
cond_ctxt.uom = "S m-1"
cond_ctxt.fill_value = -9999
cond_ctxt_id = self.dataset_management.create_parameter_context(
name="CONDWAT_L0",
parameter_context=cond_ctxt.dump(),
parameter_type="quantity<float32>",
unit_of_measure=cond_ctxt.uom,
)
# Pressure - values expected to be the decimal results of conversion from hex
press_ctxt = ParameterContext("PRESWAT_L0", param_type=QuantityType(value_encoding=np.dtype("float32")))
press_ctxt.uom = "dbar"
press_ctxt.fill_value = -9999
press_ctxt_id = self.dataset_management.create_parameter_context(
name="PRESWAT_L0",
parameter_context=press_ctxt.dump(),
parameter_type="quantity<float32>",
unit_of_measure=press_ctxt.uom,
)
# TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10
tl1_func = "(TEMPWAT_L0 / 10000) - 10"
tl1_pmap = {"TEMPWAT_L0": "TEMPWAT_L0"}
func = NumexprFunction("TEMPWAT_L1", tl1_func, tl1_pmap)
tempL1_ctxt = ParameterContext(
"TEMPWAT_L1", param_type=ParameterFunctionType(function=func), variability=VariabilityEnum.TEMPORAL
)
tempL1_ctxt.uom = "deg_C"
tempL1_ctxt_id = self.dataset_management.create_parameter_context(
name=tempL1_ctxt.name,
parameter_context=tempL1_ctxt.dump(),
parameter_type="pfunc",
unit_of_measure=tempL1_ctxt.uom,
)
# CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5
cl1_func = "(CONDWAT_L0 / 100000) - 0.5"
cl1_pmap = {"CONDWAT_L0": "CONDWAT_L0"}
func = NumexprFunction("CONDWAT_L1", cl1_func, cl1_pmap)
condL1_ctxt = ParameterContext(
"CONDWAT_L1", param_type=ParameterFunctionType(function=func), variability=VariabilityEnum.TEMPORAL
)
condL1_ctxt.uom = "S m-1"
condL1_ctxt_id = self.dataset_management.create_parameter_context(
name=condL1_ctxt.name,
parameter_context=condL1_ctxt.dump(),
parameter_type="pfunc",
unit_of_measure=condL1_ctxt.uom,
)
# Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721
# PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range)
pl1_func = "(PRESWAT_L0 * 679.34040721 / (0.85 * 65536)) - (0.05 * 679.34040721)"
pl1_pmap = {"PRESWAT_L0": "PRESWAT_L0"}
func = NumexprFunction("PRESWAT_L1", pl1_func, pl1_pmap)
presL1_ctxt = ParameterContext(
"PRESWAT_L1", param_type=ParameterFunctionType(function=func), variability=VariabilityEnum.TEMPORAL
)
presL1_ctxt.uom = "S m-1"
presL1_ctxt_id = self.dataset_management.create_parameter_context(
name=presL1_ctxt.name,
parameter_context=presL1_ctxt.dump(),
parameter_type="pfunc",
unit_of_measure=presL1_ctxt.uom,
)
# Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project:
# https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1
# PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1)
owner = "gsw"
sal_func = "SP_from_C"
sal_arglist = [NumexprFunction("CONDWAT_L1*10", "C*10", {"C": "CONDWAT_L1"}), "TEMPWAT_L1", "PRESWAT_L1"]
sal_kwargmap = None
func = PythonFunction("PRACSAL", owner, sal_func, sal_arglist, sal_kwargmap)
sal_ctxt = ParameterContext(
"PRACSAL", param_type=ParameterFunctionType(func), variability=VariabilityEnum.TEMPORAL
)
sal_ctxt.uom = "g kg-1"
sal_ctxt_id = self.dataset_management.create_parameter_context(
name=sal_ctxt.name, parameter_context=sal_ctxt.dump(), parameter_type="pfunc", unit_of_measure=sal_ctxt.uom
)
# absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude)
# conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1)
# DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1)
owner = "gsw"
abs_sal_func = PythonFunction("abs_sal", owner, "SA_from_SP", ["PRACSAL", "PRESWAT_L1", "lon", "lat"], None)
# abs_sal_func = PythonFunction('abs_sal', owner, 'SA_from_SP', ['lon','lat'], None)
cons_temp_func = PythonFunction(
"cons_temp", owner, "CT_from_t", [abs_sal_func, "TEMPWAT_L1", "PRESWAT_L1"], None
)
dens_func = PythonFunction("DENSITY", owner, "rho", [abs_sal_func, cons_temp_func, "PRESWAT_L1"], None)
dens_ctxt = ParameterContext(
"DENSITY", param_type=ParameterFunctionType(dens_func), variability=VariabilityEnum.TEMPORAL
)
dens_ctxt.uom = "kg m-3"
dens_ctxt_id = self.dataset_management.create_parameter_context(
name=dens_ctxt.name,
parameter_context=dens_ctxt.dump(),
parameter_type="pfunc",
unit_of_measure=dens_ctxt.uom,
)
ids = [
t_ctxt_id,
lat_ctxt_id,
lon_ctxt_id,
temp_ctxt_id,
cond_ctxt_id,
press_ctxt_id,
tempL1_ctxt_id,
condL1_ctxt_id,
presL1_ctxt_id,
sal_ctxt_id,
dens_ctxt_id,
]
contexts = [
t_ctxt,
lat_ctxt,
lon_ctxt,
temp_ctxt,
cond_ctxt,
press_ctxt,
tempL1_ctxt,
condL1_ctxt,
presL1_ctxt,
sal_ctxt,
dens_ctxt,
]
context_ids = [ids[i] for i, ctxt in enumerate(contexts) if ctxt.name in filter_values]
pdict_name = "_".join([ctxt.name for ctxt in contexts if ctxt.name in filter_values])
pdict_id = self.dataset_management.create_parameter_dictionary(
pdict_name, parameter_context_ids=context_ids, temporal_context="time"
)
return pdict_id
def create_pfuncs(self):
contexts = {}
funcs = {}
t_ctxt = ParameterContext('TIME', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 1900-01-01'
t_ctxt_id = self.dataset_management.create_parameter_context(name='test_TIME', parameter_context=t_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, t_ctxt_id)
contexts['TIME'] = (t_ctxt, t_ctxt_id)
lat_ctxt = ParameterContext('LAT', param_type=ConstantType(QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999)
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt_id = self.dataset_management.create_parameter_context(name='test_LAT', parameter_context=lat_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, lat_ctxt_id)
contexts['LAT'] = lat_ctxt, lat_ctxt_id
lon_ctxt = ParameterContext('LON', param_type=ConstantType(QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999)
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt_id = self.dataset_management.create_parameter_context(name='test_LON', parameter_context=lon_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, lon_ctxt_id)
contexts['LON'] = lon_ctxt, lon_ctxt_id
# Independent Parameters
# Temperature - values expected to be the decimal results of conversion from hex
temp_ctxt = ParameterContext('TEMPWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999)
temp_ctxt.uom = 'deg_C'
temp_ctxt_id = self.dataset_management.create_parameter_context(name='test_TEMPWAT_L0', parameter_context=temp_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, temp_ctxt_id)
contexts['TEMPWAT_L0'] = temp_ctxt, temp_ctxt_id
# Conductivity - values expected to be the decimal results of conversion from hex
cond_ctxt = ParameterContext('CONDWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999)
cond_ctxt.uom = 'S m-1'
cond_ctxt_id = self.dataset_management.create_parameter_context(name='test_CONDWAT_L0', parameter_context=cond_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, cond_ctxt_id)
contexts['CONDWAT_L0'] = cond_ctxt, cond_ctxt_id
# Pressure - values expected to be the decimal results of conversion from hex
press_ctxt = ParameterContext('PRESWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999)
press_ctxt.uom = 'dbar'
press_ctxt_id = self.dataset_management.create_parameter_context(name='test_PRESWAT_L0', parameter_context=press_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, press_ctxt_id)
contexts['PRESWAT_L0'] = press_ctxt, press_ctxt_id
# Dependent Parameters
# TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10
tl1_func = '(T / 10000) - 10'
expr = NumexprFunction('TEMPWAT_L1', tl1_func, ['T'])
expr_id = self.dataset_management.create_parameter_function(name='test_TEMPWAT_L1', parameter_function=expr.dump())
self.addCleanup(self.dataset_management.delete_parameter_function, expr_id)
funcs['TEMPWAT_L1'] = expr, expr_id
tl1_pmap = {'T': 'TEMPWAT_L0'}
expr.param_map = tl1_pmap
tempL1_ctxt = ParameterContext('TEMPWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL)
tempL1_ctxt.uom = 'deg_C'
tempL1_ctxt_id = self.dataset_management.create_parameter_context(name='test_TEMPWAT_L1', parameter_context=tempL1_ctxt.dump(), parameter_function_id=expr_id)
self.addCleanup(self.dataset_management.delete_parameter_context, tempL1_ctxt_id)
contexts['TEMPWAT_L1'] = tempL1_ctxt, tempL1_ctxt_id
# CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5
cl1_func = '(C / 100000) - 0.5'
expr = NumexprFunction('CONDWAT_L1', cl1_func, ['C'])
expr_id = self.dataset_management.create_parameter_function(name='test_CONDWAT_L1', parameter_function=expr.dump())
self.addCleanup(self.dataset_management.delete_parameter_function, expr_id)
funcs['CONDWAT_L1'] = expr, expr_id
cl1_pmap = {'C': 'CONDWAT_L0'}
expr.param_map = cl1_pmap
condL1_ctxt = ParameterContext('CONDWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL)
condL1_ctxt.uom = 'S m-1'
condL1_ctxt_id = self.dataset_management.create_parameter_context(name='test_CONDWAT_L1', parameter_context=condL1_ctxt.dump(), parameter_function_id=expr_id)
self.addCleanup(self.dataset_management.delete_parameter_context, condL1_ctxt_id)
contexts['CONDWAT_L1'] = condL1_ctxt, condL1_ctxt_id
# Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721
# PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range)
pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)'
expr = NumexprFunction('PRESWAT_L1', pl1_func, ['P', 'p_range'])
expr_id = self.dataset_management.create_parameter_function(name='test_PRESWAT_L1', parameter_function=expr.dump())
self.addCleanup(self.dataset_management.delete_parameter_function, expr_id)
funcs['PRESWAT_L1'] = expr, expr_id
pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721}
expr.param_map = pl1_pmap
presL1_ctxt = ParameterContext('PRESWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL)
presL1_ctxt.uom = 'S m-1'
presL1_ctxt_id = self.dataset_management.create_parameter_context(name='test_CONDWAT_L1', parameter_context=presL1_ctxt.dump(), parameter_function_id=expr_id)
self.addCleanup(self.dataset_management.delete_parameter_context, presL1_ctxt_id)
contexts['PRESWAT_L1'] = presL1_ctxt, presL1_ctxt_id
# Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project:
# https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1
# PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1)
owner = 'gsw'
sal_func = 'SP_from_C'
sal_arglist = ['C', 't', 'p']
expr = PythonFunction('PRACSAL', owner, sal_func, sal_arglist)
expr_id = self.dataset_management.create_parameter_function(name='test_PRACSAL', parameter_function=expr.dump())
self.addCleanup(self.dataset_management.delete_parameter_function, expr_id)
funcs['PRACSAL'] = expr, expr_id
# A magic function that may or may not exist actually forms the line below at runtime.
sal_pmap = {'C': NumexprFunction('CONDWAT_L1*10', 'C*10', ['C'], param_map={'C': 'CONDWAT_L1'}), 't': 'TEMPWAT_L1', 'p': 'PRESWAT_L1'}
expr.param_map = sal_pmap
sal_ctxt = ParameterContext('PRACSAL', param_type=ParameterFunctionType(expr), variability=VariabilityEnum.TEMPORAL)
sal_ctxt.uom = 'g kg-1'
sal_ctxt_id = self.dataset_management.create_parameter_context(name='test_PRACSAL', parameter_context=sal_ctxt.dump(), parameter_function_id=expr_id)
self.addCleanup(self.dataset_management.delete_parameter_context, sal_ctxt_id)
contexts['PRACSAL'] = sal_ctxt, sal_ctxt_id
# absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude)
# conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1)
# DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1)
owner = 'gsw'
abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP', ['PRACSAL', 'PRESWAT_L1', 'LON','LAT'])
cons_temp_expr = PythonFunction('cons_temp', owner, 'CT_from_t', [abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1'])
dens_expr = PythonFunction('DENSITY', owner, 'rho', [abs_sal_expr, cons_temp_expr, 'PRESWAT_L1'])
dens_ctxt = ParameterContext('DENSITY', param_type=ParameterFunctionType(dens_expr), variability=VariabilityEnum.TEMPORAL)
dens_ctxt.uom = 'kg m-3'
dens_ctxt_id = self.dataset_management.create_parameter_context(name='test_DENSITY', parameter_context=dens_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, dens_ctxt_id)
contexts['DENSITY'] = dens_ctxt, dens_ctxt_id
return contexts, funcs
def _L0_pdict(self):
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 01-01-1900'
t_ctxt.fill_value = -9999
t_ctxt_id = self.dataset_management.create_parameter_context(name='time', parameter_context=t_ctxt.dump(), parameter_type='quantity<int64>', unit_of_measure=t_ctxt.uom)
lat_ctxt = ParameterContext('lat', param_type=ConstantType(QuantityType(value_encoding=np.dtype('float32'))))
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt.fill_value = -9999
lat_ctxt_id = self.dataset_management.create_parameter_context(name='lat', parameter_context=lat_ctxt.dump(), parameter_type='quantity<float32>', unit_of_measure=lat_ctxt.uom)
lon_ctxt = ParameterContext('lon', param_type=ConstantType(QuantityType(value_encoding=np.dtype('float32'))))
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt.fill_value = -9999
lon_ctxt_id = self.dataset_management.create_parameter_context(name='lon', parameter_context=lon_ctxt.dump(), parameter_type='quantity<float32>', unit_of_measure=lon_ctxt.uom)
temp_ctxt = ParameterContext('TEMPWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')))
temp_ctxt.uom = 'deg_C'
temp_ctxt.fill_value = -9999
temp_ctxt_id = self.dataset_management.create_parameter_context(name='TEMPWAT_L0', parameter_context=temp_ctxt.dump(), parameter_type='quantity<float32>', unit_of_measure=temp_ctxt.uom)
# Conductivity - values expected to be the decimal results of conversion from hex
cond_ctxt = ParameterContext('CONDWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')))
cond_ctxt.uom = 'S m-1'
cond_ctxt.fill_value = -9999
cond_ctxt_id = self.dataset_management.create_parameter_context(name='CONDWAT_L0', parameter_context=cond_ctxt.dump(), parameter_type='quantity<float32>', unit_of_measure=cond_ctxt.uom)
# Pressure - values expected to be the decimal results of conversion from hex
press_ctxt = ParameterContext('PRESWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')))
press_ctxt.uom = 'dbar'
press_ctxt.fill_value = -9999
press_ctxt_id = self.dataset_management.create_parameter_context(name='PRESWAT_L0', parameter_context=press_ctxt.dump(), parameter_type='quantity<float32>', unit_of_measure=press_ctxt.uom)
context_ids = [t_ctxt_id, lat_ctxt_id, lon_ctxt_id, temp_ctxt_id, cond_ctxt_id, press_ctxt_id]
pdict_id = self.dataset_management.create_parameter_dictionary('L0 SBE37', parameter_context_ids=context_ids, temporal_context='time')
return pdict_id
def create_simple_array(self):
contexts = {}
types_manager = TypesManager(self.dataset_management,None,None)
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('float64')))
t_ctxt.uom = 'seconds since 1900-01-01'
t_ctxt_id = self.dataset_management.create_parameter_context(name='time', parameter_context=t_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, t_ctxt_id)
contexts['time'] = (t_ctxt, t_ctxt_id)
temp_ctxt = ParameterContext('temp_sample', param_type=ArrayType(inner_encoding='float32'))
temp_ctxt.uom = 'deg_C'
temp_ctxt.ooi_short_name = 'TEMPWAT'
temp_ctxt.display_name = 'Temperature'
temp_ctxt_id = self.dataset_management.create_parameter_context(name='temp', parameter_context=temp_ctxt.dump(), ooi_short_name='TEMPWAT')
self.addCleanup(self.dataset_management.delete_parameter_context, temp_ctxt_id)
contexts['temp'] = temp_ctxt, temp_ctxt_id
cond_ctxt = ParameterContext('cond_sample', param_type=ArrayType(inner_encoding='float64'))
cond_ctxt.uom = 'deg_C'
cond_ctxt.ooi_short_name = 'CONDWAT'
cond_ctxt.display_anme = 'Conductivity'
cond_ctxt_id = self.dataset_management.create_parameter_context(name='cond', parameter_context=cond_ctxt.dump(), ooi_short_name='CONDWAT')
self.addCleanup(self.dataset_management.delete_parameter_context, cond_ctxt_id)
contexts['cond'] = cond_ctxt, cond_ctxt_id
func = self.create_matrix_offset_function()
func.param_map = {'x':'temp_sample', 'y':'cond_sample'}
temp_offset = ParameterContext('temp_offset', param_type=ParameterFunctionType(func))
temp_offset.uom = '1'
temp_offset_id = self.dataset_management.create_parameter_context(name='temp_offset', parameter_context=temp_offset.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, temp_offset_id)
contexts['temp_offset'] = temp_offset, temp_offset_id
return contexts
def _create_input_param_dict_for_test(self, parameter_dict_name=""):
pdict = ParameterDictionary()
t_ctxt = ParameterContext("time", param_type=QuantityType(value_encoding=numpy.dtype("float64")))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = "seconds since 01-01-1900"
pdict.add_context(t_ctxt)
cond_ctxt = ParameterContext("conductivity", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
cond_ctxt.uom = ""
pdict.add_context(cond_ctxt)
pres_ctxt = ParameterContext("pressure", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
pres_ctxt.uom = ""
pdict.add_context(pres_ctxt)
if parameter_dict_name == "input_param_dict":
temp_ctxt = ParameterContext("temperature", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
else:
temp_ctxt = ParameterContext("temp", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
temp_ctxt.uom = ""
pdict.add_context(temp_ctxt)
dens_ctxt = ParameterContext("density", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
dens_ctxt.uom = ""
pdict.add_context(dens_ctxt)
sal_ctxt = ParameterContext("salinity", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
sal_ctxt.uom = ""
pdict.add_context(sal_ctxt)
# create temp streamdef so the data product can create the stream
pc_list = []
for pc_k, pc in pdict.iteritems():
ctxt_id = self.dataset_management.create_parameter_context(pc_k, pc[1].dump())
pc_list.append(ctxt_id)
if parameter_dict_name == "input_param_dict":
self.addCleanup(self.dataset_management.delete_parameter_context, ctxt_id)
elif parameter_dict_name == "output_param_dict" and pc[1].name == "temp":
self.addCleanup(self.dataset_management.delete_parameter_context, ctxt_id)
pdict_id = self.dataset_management.create_parameter_dictionary(parameter_dict_name, pc_list)
self.addCleanup(self.dataset_management.delete_parameter_dictionary, pdict_id)
return pdict_id
