def make_grt_qc(self, name, data_product):
pfunc_id, pfunc = self.find_grt()
grt_min_id, grt_min_name = self.get_lookup_value(
'LV_grt_$designator_%s||grt_min_value' % data_product)
grt_max_id, grt_max_name = self.get_lookup_value(
'LV_grt_$designator_%s||grt_max_value' % data_product)
pmap = {'dat': name, 'dat_min': grt_min_name, 'dat_max': grt_max_name}
pfunc.param_map = pmap
pfunc.lookup_values = [grt_min_id, grt_max_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_glblrng_qc' % dp_name.lower(),
param_type=ParameterFunctionType(
pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_GLBLRNG_QC' % dp_name
pc.display_name = '%s Global Range Test Quality Control Flag' % dp_name
pc.description = "The OOI Global Range quality control algorithm generates a QC flag for the input data point indicating whether it falls within a given range."
ctxt_id = self.dataset_management.create_parameter_context(
name='%s_glblrng_qc' % dp_name.lower(),
parameter_type='function',
parameter_context=pc.dump(),
parameter_function_id=pfunc_id,
ooi_short_name=pc.ooi_short_name,
units='1',
value_encoding='int8',
display_name=pc.display_name,
description=pc.description)
return ctxt_id, pc
def get_lookup_value(self, value):
return self._placeholder(
value, lambda placeholder: ParameterContext(
name=placeholder,
param_type=SparseConstantType(base_type=ConstantType(
value_encoding='float64'),
fill_value=-9999.)))
def make_stuckvalue_qc(self, name, data_product):
pfunc_id, pfunc = self.find_stuck_value()
reso_id, reso_name = self.get_lookup_value(
'LV_svt_$designator_%s||svt_resolution' % data_product)
n_id, n_name = self.get_lookup_value('LV_svt_$designator_%s||svt_n' %
data_product)
pmap = {'x': name, 'reso': reso_name, 'num': n_name}
pfunc.param_map = pmap
pfunc.lookup_values = [reso_id, n_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_stuckvl_qc' % dp_name.lower(),
param_type=ParameterFunctionType(
pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_STUCKVL_QC' % dp_name
pc.display_name = '%s Stuck Value Test Quality Control Flag' % dp_name
pc.description = 'The OOI Stuck Value Test quality control algorithm generates a flag for repeated occurrence of one value in a time series.'
ctxt_id = self.dataset_management.create_parameter_context(
name='%s_stuckvl_qc' % dp_name.lower(),
parameter_type='function',
parameter_context=pc.dump(),
parameter_function_id=pfunc_id,
ooi_short_name=pc.ooi_short_name,
units='1',
value_encoding='int8',
display_name=pc.display_name,
description=pc.description)
return ctxt_id, pc
def make_propagate_qc(self, inputs):
pfunc_id, pfunc = self.find_propagate_test()
pmap = {"strict_validation": False}
arg_list = ['strict_validation']
for i, val in enumerate(inputs):
if i >= 100:
break
pmap['array%s' % i] = val
arg_list.append('array%s' % i)
pfunc.param_map = pmap
pfunc.arg_list = arg_list
pc = ParameterContext(name='cmbnflg_qc',
param_type=ParameterFunctionType(
pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = 'CMBNFLG_QC'
pc.display_name = 'Combined Data Quality Control Flag'
pc.description = 'The purpose of this computation is to produce a single merged QC flag from a set of potentially many flags.'
ctxt_id = self.dataset_management.create_parameter_context(
name='cmbnflg_qc',
parameter_type='function',
parameter_context=pc.dump(),
parameter_function_id=pfunc_id,
ooi_short_name=pc.ooi_short_name,
units='1',
value_encoding='int8',
display_name=pc.display_name,
description=pc.description)
return ctxt_id, pc
def make_manual_upload_data_product(self):
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(
pdict_id, id_only=True)
# Add a handful of Calibration Coefficient parameters
for cc in ['temp_hitl_qc', 'cond_hitl_qc']:
c = ParameterContext(cc, param_type=BooleanType())
c.uom = '1'
context_ids.append(
self.dataset_management.create_parameter_context(cc, c.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary(
'manup_dict', context_ids, temporal_context='time')
data_product_id = self.create_data_product('manup_dp',
pdict_id=pdict_id)
self.activate_data_product(data_product_id)
dataset_id, _ = self.resource_registry.find_objects(data_product_id,
PRED.hasDataset,
id_only=True)
return data_product_id, dataset_id[0]
def make_cal_data_product(self):
# Get a precompiled parameter dictionary with basic ctd fields
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
context_ids = self.dataset_management.read_parameter_contexts(
pdict_id, id_only=True)
# Add a handful of Calibration Coefficient parameters
for cc in ['cc_ta0', 'cc_ta1', 'cc_ta2', 'cc_ta3', 'cc_toffset']:
c = ParameterContext(cc,
param_type=SparseConstantType(
value_encoding='float32',
fill_value=-9999))
c.uom = '1'
context_ids.append(
self.dataset_management.create_parameter_context(cc, c.dump()))
pdict_id = self.dataset_management.create_parameter_dictionary(
'calcoeff_dict', context_ids, temporal_context='time')
data_product_id = self.create_data_product('calcoeff_dp',
pdict_id=pdict_id)
self.activate_data_product(data_product_id)
dataset_id, _ = self.resource_registry.find_objects(data_product_id,
PRED.hasDataset,
id_only=True)
return data_product_id, dataset_id[0]
def create_contexts(self):
context_ids = []
cond_ctxt = ParameterContext(
'conductivity_test',
param_type=QuantityType(value_encoding=np.float32))
cond_ctxt.uom = 'unknown'
cond_ctxt.fill_value = 0e0
context_ids.append(
self.dataset_management.create_parameter_context(
name='conductivity_test', parameter_context=cond_ctxt.dump()))
pres_ctxt = ParameterContext(
'pressure_test',
param_type=QuantityType(value_encoding=np.float32))
pres_ctxt.uom = 'Pascal'
pres_ctxt.fill_value = 0x0
context_ids.append(
self.dataset_management.create_parameter_context(
name='pressure_test', parameter_context=pres_ctxt.dump()))
sal_ctxt = ParameterContext(
'salinity_test',
param_type=QuantityType(value_encoding=np.float32))
sal_ctxt.uom = 'PSU'
sal_ctxt.fill_value = 0x0
context_ids.append(
self.dataset_management.create_parameter_context(
name='salinity_test', parameter_context=sal_ctxt.dump()))
temp_ctxt = ParameterContext(
'temp_test', param_type=QuantityType(value_encoding=np.float32))
temp_ctxt.uom = 'degree_Celsius'
temp_ctxt.fill_value = 0e0
context_ids.append(
self.dataset_management.create_parameter_context(
name='temp_test', parameter_context=temp_ctxt.dump()))
t_ctxt = ParameterContext(
'time_test', param_type=QuantityType(value_encoding=np.int64))
t_ctxt.uom = 'seconds since 1970-01-01'
t_ctxt.fill_value = 0x0
context_ids.append(
self.dataset_management.create_parameter_context(
name='time_test', parameter_context=t_ctxt.dump()))
return context_ids
def _create_input_param_dict_for_test(self, parameter_dict_name=''):
pdict = ParameterDictionary()
t_ctxt = ParameterContext(
'time',
param_type=QuantityType(value_encoding=numpy.dtype('float64')))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1900'
pdict.add_context(t_ctxt)
cond_ctxt = ParameterContext(
'conductivity',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
cond_ctxt.uom = ''
pdict.add_context(cond_ctxt)
pres_ctxt = ParameterContext(
'pressure',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
pres_ctxt.uom = ''
pdict.add_context(pres_ctxt)
temp_ctxt = ParameterContext(
'temperature',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = ''
pdict.add_context(temp_ctxt)
dens_ctxt = ParameterContext(
'density',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
dens_ctxt.uom = ''
pdict.add_context(dens_ctxt)
sal_ctxt = ParameterContext(
'salinity',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
sal_ctxt.uom = ''
pdict.add_context(sal_ctxt)
#create temp streamdef so the data product can create the stream
pc_list = []
for pc_k, pc in pdict.iteritems():
ctxt_id = self.dataset_management.create_parameter_context(
pc_k, pc[1].dump())
pc_list.append(ctxt_id)
self.addCleanup(self.dataset_management.delete_parameter_context,
ctxt_id)
pdict_id = self.dataset_management.create_parameter_dictionary(
parameter_dict_name, pc_list)
self.addCleanup(self.dataset_management.delete_parameter_dictionary,
pdict_id)
return pdict_id
def __init__(self,
total_domain=(10, 10),
brick_size=5,
use_hdf=False,
root_dir='test_data/multi_dim_trials',
guid=None,
dtype='int16'):
self.total_domain = total_domain
self.brick_sizes = tuple(brick_size for x in total_domain)
self.use_hdf = use_hdf
self.dtype = np.dtype(dtype).name
if self.use_hdf:
self.guid = guid or create_guid()
name = '%s_%s' % (self.guid, self.dtype)
self.root_dir = root_dir
if not os.path.exists(self.root_dir):
os.makedirs(self.root_dir)
if os.path.exists(os.path.join(self.root_dir, name)):
shutil.rmtree(os.path.join(self.root_dir, name))
self.master_manager = MasterManager(
self.root_dir, name, name='md_test_{0}'.format(name))
self.master_manager.flush()
pc = ParameterContext('test_param',
param_type=QuantityType(self.dtype),
fill_value=-1)
self.param_manager = ParameterManager(
os.path.join(self.root_dir, name, pc.name), pc.name)
self.param_manager.parameter_context = pc
self.master_manager.create_group(pc.name)
self.param_manager.flush()
self.bricks = {}
self.brick_origins = bricking_utils.calc_brick_origins(
self.total_domain, self.brick_sizes)
self.brick_extents, self.rtree_extents = bricking_utils.calc_brick_and_rtree_extents(
self.brick_origins, self.brick_sizes)
self.build_bricks()
self.rtree = RTreeProxy()
for x in BrickingAssessor.rtree_populator(self.rtree_extents,
self.brick_extents):
self.rtree.insert(*x)
def get_cc_value(self, value):
placeholder = value.lower()
# Check to see if this coefficient exists already
hits, _ = Container.instance.resource_registry.find_resources(
name=placeholder, restype=RT.ParameterContext, id_only=True)
if hits:
return hits[0], placeholder
pc = ParameterContext(
name=placeholder,
param_type=SparseConstantType(value_encoding='float64'),
fill_value=-9999.)
pc.uom = '1'
ctxt_id = self.dataset_management.create_parameter_context(
name=placeholder, parameter_context=pc.dump())
return ctxt_id, placeholder
def make_gradienttest_qc(self, name, data_product):
pfunc_id, pfunc = self.find_gradient_test()
ddatdx_id, ddatdx = self.get_lookup_value(
'LV_grad_$designator_%s_time||d_dat_dx' % data_product)
mindx_id, mindx = self.get_lookup_value(
'LV_grad_$designator_%s_time||min_dx' % data_product)
startdat_id, startdat = self.get_lookup_value(
'LV_grad_$designator_%s_time||start_dat' % data_product)
toldat_id, toldat = self.get_lookup_value(
'LV_grad_$designator_%s_time||tol_dat' % data_product)
pmap = {
"dat": name,
"x": 'time',
'ddatdx': ddatdx,
'mindx': mindx,
'startdat': startdat,
'toldat': toldat
}
pfunc.param_map = pmap
pfunc.lookup_values = [ddatdx_id, mindx_id, startdat_id, toldat_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_gradtst_qc' % dp_name.lower(),
param_type=ParameterFunctionType(
pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_GRADTST_QC' % dp_name
pc.display_name = '%s Gradient Test Quality Control Flag' % dp_name
pc.description = 'The OOI Gradient Test is an automated quality control algorithm used on various OOI data products. This automated algorithm generates flags for data points according to whether changes between successive points are within a pre-determined range.'
ctxt_id = self.dataset_management.create_parameter_context(
name='%s_gradtst_qc' % dp_name.lower(),
parameter_type='function',
parameter_context=pc.dump(),
parameter_function_id=pfunc_id,
ooi_short_name=pc.ooi_short_name,
units='1',
value_encoding='int8',
display_name=pc.display_name,
description=pc.description)
return ctxt_id, pc
def make_localrange_qc(self, name, data_product):
pfunc_id, pfunc = self.find_localrange_test()
datlim_id, datlim = self.get_array_lookup_value(
'LV_lrt_$designator_%s||datlim' % data_product)
datlimz_id, datlimz = self.get_array_lookup_value(
'LV_lrt_$designator_%s||datlimz' % data_product)
dims_id, dims = self.get_string_array_lookup_value(
'LV_lrt_$designator_%s||dims' % data_product)
pmap = {
"dat": name,
"dims*": dims,
"datlim*": datlim,
"datlimz*": datlimz
}
pfunc.param_map = pmap
pfunc.lookup_values = [datlim_id, datlimz_id, dims_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_loclrng_qc' % dp_name.lower(),
param_type=ParameterFunctionType(
pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_LOCLRNG_QC' % dp_name
pc.display_name = '%s Local Range Test Quality Control Flag' % dp_name
pc.description = 'The OOI Local Range Test is the computation to test whether a given data point falls within pre-defined ranges.'
ctxt_id = self.dataset_management.create_parameter_context(
name='%s_loclrng_qc' % dp_name.lower(),
parameter_type='function',
parameter_context=pc.dump(),
parameter_function_id=pfunc_id,
ooi_short_name=pc.ooi_short_name,
units='1',
value_encoding='int8',
display_name=pc.display_name,
description=pc.description)
return ctxt_id, pc
def make_spike_qc(self, name, data_product):
pfunc_id, pfunc = self.find_spike()
spike_acc_id, spike_acc_name = self.get_lookup_value(
'LV_spike_$designator_%s||acc' % data_product)
spike_n_id, spike_n_name = self.get_lookup_value(
'LV_spike_$designator_%s||spike_n' % data_product)
spike_l_id, spike_l_name = self.get_lookup_value(
'LV_spike_$designator_%s||spike_l' % data_product)
pmap = {
'dat': name,
'acc': spike_acc_name,
'N': spike_n_name,
'L': spike_l_name
}
pfunc.param_map = pmap
pfunc.lookup_values = [spike_acc_id, spike_n_id, spike_l_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_spketst_qc' % dp_name.lower(),
param_type=ParameterFunctionType(
pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_SPKETST_QC' % dp_name
pc.display_name = '%s Spike Test Quality Control Flag' % dp_name
pc.description = "The OOI Spike Test quality control algorithm generates a flag for individual data values that deviate significantly from surrounding data values."
ctxt_id = self.dataset_management.create_parameter_context(
name='%s_spketst_qc' % dp_name.lower(),
parameter_type='function',
parameter_context=pc.dump(),
parameter_function_id=pfunc_id,
ooi_short_name=pc.ooi_short_name,
units='1',
value_encoding='int8',
display_name=pc.display_name,
description=pc.description)
return ctxt_id, pc
def make_trendtest_qc(self, name, data_product):
pfunc_id, pfunc = self.find_trend_test()
order_id, order_name = self.get_lookup_value(
'LV_trend_$designator_%s||polynomial_order' % data_product)
dev_id, dev_name = self.get_lookup_value(
'LV_trend_$designator_%s||standard_deviation' % data_product)
pmap = {
"dat": name,
"t": 'time',
"ord_n": order_name,
"ntsd": dev_name
}
pfunc.param_map = pmap
pfunc.lookup_values = [order_id, dev_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_trndtst_qc' % dp_name.lower(),
param_type=ParameterFunctionType(
pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_TRNDTST_QC' % dp_name
pc.display_name = '%s Trend Test Test Quality Control Flag' % dp_name
pc.description = 'The OOI Trend Test quality control algorithm generates flags on data values within a time series where a significant fraction of the variability in the time series can be explained by a drift, where the drift is assumed to be a polynomial of specified order.'
ctxt_id = self.dataset_management.create_parameter_context(
name='%s_trndtst_qc' % dp_name.lower(),
parameter_type='function',
parameter_context=pc.dump(),
parameter_function_id=pfunc_id,
ooi_short_name=pc.ooi_short_name,
units='1',
value_encoding='int8',
display_name=pc.display_name,
description=pc.description)
return ctxt_id, pc
def create_pfuncs(self):
contexts = {}
funcs = {}
t_ctxt = ParameterContext(
'TIME', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 01-01-1900'
t_ctxt_id = self.dataset_management.create_parameter_context(
name='test_TIME', parameter_context=t_ctxt.dump())
contexts['TIME'] = (t_ctxt, t_ctxt_id)
lat_ctxt = ParameterContext(
'LAT',
param_type=ConstantType(
QuantityType(value_encoding=np.dtype('float32'))),
fill_value=-9999)
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt_id = self.dataset_management.create_parameter_context(
name='test_LAT', parameter_context=lat_ctxt.dump())
contexts['LAT'] = lat_ctxt, lat_ctxt_id
lon_ctxt = ParameterContext(
'LON',
param_type=ConstantType(
QuantityType(value_encoding=np.dtype('float32'))),
fill_value=-9999)
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt_id = self.dataset_management.create_parameter_context(
name='test_LON', parameter_context=lon_ctxt.dump())
contexts['LON'] = lon_ctxt, lon_ctxt_id
# Independent Parameters
# Temperature - values expected to be the decimal results of conversion from hex
temp_ctxt = ParameterContext(
'TEMPWAT_L0',
param_type=QuantityType(value_encoding=np.dtype('float32')),
fill_value=-9999)
temp_ctxt.uom = 'deg_C'
temp_ctxt_id = self.dataset_management.create_parameter_context(
name='test_TEMPWAT_L0', parameter_context=temp_ctxt.dump())
contexts['TEMPWAT_L0'] = temp_ctxt, temp_ctxt_id
# Conductivity - values expected to be the decimal results of conversion from hex
cond_ctxt = ParameterContext(
'CONDWAT_L0',
param_type=QuantityType(value_encoding=np.dtype('float32')),
fill_value=-9999)
cond_ctxt.uom = 'S m-1'
cond_ctxt_id = self.dataset_management.create_parameter_context(
name='test_CONDWAT_L0', parameter_context=cond_ctxt.dump())
contexts['CONDWAT_L0'] = cond_ctxt, cond_ctxt_id
# Pressure - values expected to be the decimal results of conversion from hex
press_ctxt = ParameterContext(
'PRESWAT_L0',
param_type=QuantityType(value_encoding=np.dtype('float32')),
fill_value=-9999)
press_ctxt.uom = 'dbar'
press_ctxt_id = self.dataset_management.create_parameter_context(
name='test_PRESWAT_L0', parameter_context=press_ctxt.dump())
contexts['PRESWAT_L0'] = press_ctxt, press_ctxt_id
# Dependent Parameters
# TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10
tl1_func = '(T / 10000) - 10'
expr = NumexprFunction('TEMPWAT_L1', tl1_func, ['T'])
expr_id = self.dataset_management.create_parameter_function(
name='test_TEMPWAT_L1', parameter_function=expr.dump())
funcs['TEMPWAT_L1'] = expr, expr_id
tl1_pmap = {'T': 'TEMPWAT_L0'}
expr.param_map = tl1_pmap
tempL1_ctxt = ParameterContext(
'TEMPWAT_L1',
param_type=ParameterFunctionType(function=expr),
variability=VariabilityEnum.TEMPORAL)
tempL1_ctxt.uom = 'deg_C'
tempL1_ctxt_id = self.dataset_management.create_parameter_context(
name='test_TEMPWAT_L1',
parameter_context=tempL1_ctxt.dump(),
parameter_function_id=expr_id)
contexts['TEMPWAT_L1'] = tempL1_ctxt, tempL1_ctxt_id
# CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5
cl1_func = '(C / 100000) - 0.5'
expr = NumexprFunction('CONDWAT_L1', cl1_func, ['C'])
expr_id = self.dataset_management.create_parameter_function(
name='test_CONDWAT_L1', parameter_function=expr.dump())
funcs['CONDWAT_L1'] = expr, expr_id
cl1_pmap = {'C': 'CONDWAT_L0'}
expr.param_map = cl1_pmap
condL1_ctxt = ParameterContext(
'CONDWAT_L1',
param_type=ParameterFunctionType(function=expr),
variability=VariabilityEnum.TEMPORAL)
condL1_ctxt.uom = 'S m-1'
condL1_ctxt_id = self.dataset_management.create_parameter_context(
name='test_CONDWAT_L1',
parameter_context=condL1_ctxt.dump(),
parameter_function_id=expr_id)
contexts['CONDWAT_L1'] = condL1_ctxt, condL1_ctxt_id
# Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721
# PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range)
pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)'
expr = NumexprFunction('PRESWAT_L1', pl1_func, ['P', 'p_range'])
expr_id = self.dataset_management.create_parameter_function(
name='test_PRESWAT_L1', parameter_function=expr.dump())
funcs['PRESWAT_L1'] = expr, expr_id
pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721}
expr.param_map = pl1_pmap
presL1_ctxt = ParameterContext(
'PRESWAT_L1',
param_type=ParameterFunctionType(function=expr),
variability=VariabilityEnum.TEMPORAL)
presL1_ctxt.uom = 'S m-1'
presL1_ctxt_id = self.dataset_management.create_parameter_context(
name='test_CONDWAT_L1',
parameter_context=presL1_ctxt.dump(),
parameter_function_id=expr_id)
contexts['PRESWAT_L1'] = presL1_ctxt, presL1_ctxt_id
# Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project:
# https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1
# PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1)
owner = 'gsw'
sal_func = 'SP_from_C'
sal_arglist = ['C', 't', 'p']
expr = PythonFunction('PRACSAL', owner, sal_func, sal_arglist)
expr_id = self.dataset_management.create_parameter_function(
name='test_PRACSAL', parameter_function=expr.dump())
funcs['PRACSAL'] = expr, expr_id
# A magic function that may or may not exist actually forms the line below at runtime.
sal_pmap = {
'C':
NumexprFunction('CONDWAT_L1*10',
'C*10', ['C'],
param_map={'C': 'CONDWAT_L1'}),
't':
'TEMPWAT_L1',
'p':
'PRESWAT_L1'
}
expr.param_map = sal_pmap
sal_ctxt = ParameterContext('PRACSAL',
param_type=ParameterFunctionType(expr),
variability=VariabilityEnum.TEMPORAL)
sal_ctxt.uom = 'g kg-1'
sal_ctxt_id = self.dataset_management.create_parameter_context(
name='test_PRACSAL',
parameter_context=sal_ctxt.dump(),
parameter_function_id=expr_id)
contexts['PRACSAL'] = sal_ctxt, sal_ctxt_id
# absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude)
# conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1)
# DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1)
owner = 'gsw'
abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP',
['PRACSAL', 'PRESWAT_L1', 'LON', 'LAT'])
cons_temp_expr = PythonFunction(
'cons_temp', owner, 'CT_from_t',
[abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1'])
dens_expr = PythonFunction(
'DENSITY', owner, 'rho',
[abs_sal_expr, cons_temp_expr, 'PRESWAT_L1'])
dens_ctxt = ParameterContext(
'DENSITY',
param_type=ParameterFunctionType(dens_expr),
variability=VariabilityEnum.TEMPORAL)
dens_ctxt.uom = 'kg m-3'
dens_ctxt_id = self.dataset_management.create_parameter_context(
name='test_DENSITY', parameter_context=dens_ctxt.dump())
contexts['DENSITY'] = dens_ctxt, dens_ctxt_id
return contexts, funcs
def _get_pdict(self, filter_values):
t_ctxt = ParameterContext(
'TIME', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 01-01-1900'
t_ctxt_id = self.dataset_management.create_parameter_context(
name='TIME',
parameter_context=t_ctxt.dump(),
parameter_type='quantity<int64>',
unit_of_measure=t_ctxt.uom)
lat_ctxt = ParameterContext(
'LAT',
param_type=ConstantType(
QuantityType(value_encoding=np.dtype('float32'))),
fill_value=-9999)
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt_id = self.dataset_management.create_parameter_context(
name='LAT',
parameter_context=lat_ctxt.dump(),
parameter_type='quantity<float32>',
unit_of_measure=lat_ctxt.uom)
lon_ctxt = ParameterContext(
'LON',
param_type=ConstantType(
QuantityType(value_encoding=np.dtype('float32'))),
fill_value=-9999)
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt_id = self.dataset_management.create_parameter_context(
name='LON',
parameter_context=lon_ctxt.dump(),
parameter_type='quantity<float32>',
unit_of_measure=lon_ctxt.uom)
# Independent Parameters
# Temperature - values expected to be the decimal results of conversion from hex
temp_ctxt = ParameterContext(
'TEMPWAT_L0',
param_type=QuantityType(value_encoding=np.dtype('float32')),
fill_value=-9999)
temp_ctxt.uom = 'deg_C'
temp_ctxt_id = self.dataset_management.create_parameter_context(
name='TEMPWAT_L0',
parameter_context=temp_ctxt.dump(),
parameter_type='quantity<float32>',
unit_of_measure=temp_ctxt.uom)
# Conductivity - values expected to be the decimal results of conversion from hex
cond_ctxt = ParameterContext(
'CONDWAT_L0',
param_type=QuantityType(value_encoding=np.dtype('float32')),
fill_value=-9999)
cond_ctxt.uom = 'S m-1'
cond_ctxt_id = self.dataset_management.create_parameter_context(
name='CONDWAT_L0',
parameter_context=cond_ctxt.dump(),
parameter_type='quantity<float32>',
unit_of_measure=cond_ctxt.uom)
# Pressure - values expected to be the decimal results of conversion from hex
press_ctxt = ParameterContext(
'PRESWAT_L0',
param_type=QuantityType(value_encoding=np.dtype('float32')),
fill_value=-9999)
press_ctxt.uom = 'dbar'
press_ctxt_id = self.dataset_management.create_parameter_context(
name='PRESWAT_L0',
parameter_context=press_ctxt.dump(),
parameter_type='quantity<float32>',
unit_of_measure=press_ctxt.uom)
# Dependent Parameters
# TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10
tl1_func = '(T / 10000) - 10'
tl1_pmap = {'T': 'TEMPWAT_L0'}
expr = NumexprFunction('TEMPWAT_L1',
tl1_func, ['T'],
param_map=tl1_pmap)
tempL1_ctxt = ParameterContext(
'TEMPWAT_L1',
param_type=ParameterFunctionType(function=expr),
variability=VariabilityEnum.TEMPORAL)
tempL1_ctxt.uom = 'deg_C'
tempL1_ctxt_id = self.dataset_management.create_parameter_context(
name=tempL1_ctxt.name,
parameter_context=tempL1_ctxt.dump(),
parameter_type='pfunc',
unit_of_measure=tempL1_ctxt.uom)
# CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5
cl1_func = '(C / 100000) - 0.5'
cl1_pmap = {'C': 'CONDWAT_L0'}
expr = NumexprFunction('CONDWAT_L1',
cl1_func, ['C'],
param_map=cl1_pmap)
condL1_ctxt = ParameterContext(
'CONDWAT_L1',
param_type=ParameterFunctionType(function=expr),
variability=VariabilityEnum.TEMPORAL)
condL1_ctxt.uom = 'S m-1'
condL1_ctxt_id = self.dataset_management.create_parameter_context(
name=condL1_ctxt.name,
parameter_context=condL1_ctxt.dump(),
parameter_type='pfunc',
unit_of_measure=condL1_ctxt.uom)
# Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721
# PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range)
pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)'
pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721}
expr = NumexprFunction('PRESWAT_L1',
pl1_func, ['P', 'p_range'],
param_map=pl1_pmap)
presL1_ctxt = ParameterContext(
'PRESWAT_L1',
param_type=ParameterFunctionType(function=expr),
variability=VariabilityEnum.TEMPORAL)
presL1_ctxt.uom = 'S m-1'
presL1_ctxt_id = self.dataset_management.create_parameter_context(
name=presL1_ctxt.name,
parameter_context=presL1_ctxt.dump(),
parameter_type='pfunc',
unit_of_measure=presL1_ctxt.uom)
# Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project:
# https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1
# PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1)
owner = 'gsw'
sal_func = 'SP_from_C'
sal_arglist = ['C', 't', 'p']
sal_pmap = {
'C':
NumexprFunction('CONDWAT_L1*10',
'C*10', ['C'],
param_map={'C': 'CONDWAT_L1'}),
't':
'TEMPWAT_L1',
'p':
'PRESWAT_L1'
}
sal_kwargmap = None
expr = PythonFunction('PRACSAL', owner, sal_func, sal_arglist,
sal_kwargmap, sal_pmap)
sal_ctxt = ParameterContext('PRACSAL',
param_type=ParameterFunctionType(expr),
variability=VariabilityEnum.TEMPORAL)
sal_ctxt.uom = 'g kg-1'
sal_ctxt_id = self.dataset_management.create_parameter_context(
name=sal_ctxt.name,
parameter_context=sal_ctxt.dump(),
parameter_type='pfunc',
unit_of_measure=sal_ctxt.uom)
# absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude)
# conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1)
# DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1)
owner = 'gsw'
abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP',
['PRACSAL', 'PRESWAT_L1', 'LON', 'LAT'])
cons_temp_expr = PythonFunction(
'cons_temp', owner, 'CT_from_t',
[abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1'])
dens_expr = PythonFunction(
'DENSITY', owner, 'rho',
[abs_sal_expr, cons_temp_expr, 'PRESWAT_L1'])
dens_ctxt = ParameterContext(
'DENSITY',
param_type=ParameterFunctionType(dens_expr),
variability=VariabilityEnum.TEMPORAL)
dens_ctxt.uom = 'kg m-3'
dens_ctxt_id = self.dataset_management.create_parameter_context(
name=dens_ctxt.name,
parameter_context=dens_ctxt.dump(),
parameter_type='pfunc',
unit_of_measure=dens_ctxt.uom)
ids = [
t_ctxt_id, lat_ctxt_id, lon_ctxt_id, temp_ctxt_id, cond_ctxt_id,
press_ctxt_id, tempL1_ctxt_id, condL1_ctxt_id, presL1_ctxt_id,
sal_ctxt_id, dens_ctxt_id
]
contexts = [
t_ctxt, lat_ctxt, lon_ctxt, temp_ctxt, cond_ctxt, press_ctxt,
tempL1_ctxt, condL1_ctxt, presL1_ctxt, sal_ctxt, dens_ctxt
]
context_ids = [
ids[i] for i, ctxt in enumerate(contexts)
if ctxt.name in filter_values
]
pdict_name = '_'.join(
[ctxt.name for ctxt in contexts if ctxt.name in filter_values])
try:
self.pdicts[pdict_name]
return self.pdicts[pdict_name]
except KeyError:
pdict_id = self.dataset_management.create_parameter_dictionary(
pdict_name,
parameter_context_ids=context_ids,
temporal_context='time')
self.pdicts[pdict_name] = pdict_id
return pdict_id
def get_string_array_lookup_value(self, value):
return self._placeholder(
value, lambda placeholder: ParameterContext(
name=placeholder,
param_type=SparseConstantType(base_type=ArrayType())))
