def make_grt_qc(self, name, data_product):
pfunc_id, pfunc = self.find_grt()
grt_min_id, grt_min_name = self.get_lookup_value(
'LV_grt_$designator_%s||grt_min_value' % data_product)
grt_max_id, grt_max_name = self.get_lookup_value(
'LV_grt_$designator_%s||grt_max_value' % data_product)
pmap = {'dat': name, 'dat_min': grt_min_name, 'dat_max': grt_max_name}
pfunc.param_map = pmap
pfunc.lookup_values = [grt_min_id, grt_max_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_glblrng_qc' % dp_name.lower(),
param_type=ParameterFunctionType(
pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_GLBLRNG_QC' % dp_name
pc.display_name = '%s Global Range Test Quality Control Flag' % dp_name
pc.description = "The OOI Global Range quality control algorithm generates a QC flag for the input data point indicating whether it falls within a given range."
ctxt_id = self.dataset_management.create_parameter_context(
name='%s_glblrng_qc' % dp_name.lower(),
parameter_type='function',
parameter_context=pc.dump(),
parameter_function_id=pfunc_id,
ooi_short_name=pc.ooi_short_name,
units='1',
value_encoding='int8',
display_name=pc.display_name,
description=pc.description)
return ctxt_id, pc
def make_propagate_qc(self, inputs):
pfunc_id, pfunc = self.find_propagate_test()
pmap = {"strict_validation": False}
arg_list = ['strict_validation']
for i, val in enumerate(inputs):
if i >= 100:
break
pmap['array%s' % i] = val
arg_list.append('array%s' % i)
pfunc.param_map = pmap
pfunc.arg_list = arg_list
pc = ParameterContext(name='cmbnflg_qc',
param_type=ParameterFunctionType(
pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = 'CMBNFLG_QC'
pc.display_name = 'Combined Data Quality Control Flag'
pc.description = 'The purpose of this computation is to produce a single merged QC flag from a set of potentially many flags.'
ctxt_id = self.dataset_management.create_parameter_context(
name='cmbnflg_qc',
parameter_type='function',
parameter_context=pc.dump(),
parameter_function_id=pfunc_id,
ooi_short_name=pc.ooi_short_name,
units='1',
value_encoding='int8',
display_name=pc.display_name,
description=pc.description)
return ctxt_id, pc
def make_stuckvalue_qc(self, name, data_product):
pfunc_id, pfunc = self.find_stuck_value()
reso_id, reso_name = self.get_lookup_value(
'LV_svt_$designator_%s||svt_resolution' % data_product)
n_id, n_name = self.get_lookup_value('LV_svt_$designator_%s||svt_n' %
data_product)
pmap = {'x': name, 'reso': reso_name, 'num': n_name}
pfunc.param_map = pmap
pfunc.lookup_values = [reso_id, n_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_stuckvl_qc' % dp_name.lower(),
param_type=ParameterFunctionType(
pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_STUCKVL_QC' % dp_name
pc.display_name = '%s Stuck Value Test Quality Control Flag' % dp_name
pc.description = 'The OOI Stuck Value Test quality control algorithm generates a flag for repeated occurrence of one value in a time series.'
ctxt_id = self.dataset_management.create_parameter_context(
name='%s_stuckvl_qc' % dp_name.lower(),
parameter_type='function',
parameter_context=pc.dump(),
parameter_function_id=pfunc_id,
ooi_short_name=pc.ooi_short_name,
units='1',
value_encoding='int8',
display_name=pc.display_name,
description=pc.description)
return ctxt_id, pc
def get_function_type(parameter_type, encoding, pfid, pmap):
if pfid is None or pmap is None:
raise TypeError('Function Types require proper function IDs and maps')
try:
pmap = ast.literal_eval(pmap)
except:
raise TypeError('Invalid Parameter Map Syntax')
func = evaluate_pmap(pfid, pmap) # Parse out nested PFIDs and such
param_type = ParameterFunctionType(func)
return param_type
def test_lookup_value_check(self):
func = NumexprFunction('f',
'coeff_a * x', ['coeff_a', 'x'],
param_map={
'x': 'x',
'coeff_a': 'coeff_a'
})
func.lookup_values = ['abc123']
test_context = ParameterContext('test',
param_type=ParameterFunctionType(func))
tm = TypesManager(None, None, None)
self.assertTrue(tm.has_lookup_value(test_context))
self.assertEquals(tm.get_lookup_value_ids(test_context), ['abc123'])
def make_gradienttest_qc(self, name, data_product):
pfunc_id, pfunc = self.find_gradient_test()
ddatdx_id, ddatdx = self.get_lookup_value(
'LV_grad_$designator_%s_time||d_dat_dx' % data_product)
mindx_id, mindx = self.get_lookup_value(
'LV_grad_$designator_%s_time||min_dx' % data_product)
startdat_id, startdat = self.get_lookup_value(
'LV_grad_$designator_%s_time||start_dat' % data_product)
toldat_id, toldat = self.get_lookup_value(
'LV_grad_$designator_%s_time||tol_dat' % data_product)
pmap = {
"dat": name,
"x": 'time',
'ddatdx': ddatdx,
'mindx': mindx,
'startdat': startdat,
'toldat': toldat
}
pfunc.param_map = pmap
pfunc.lookup_values = [ddatdx_id, mindx_id, startdat_id, toldat_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_gradtst_qc' % dp_name.lower(),
param_type=ParameterFunctionType(
pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_GRADTST_QC' % dp_name
pc.display_name = '%s Gradient Test Quality Control Flag' % dp_name
pc.description = 'The OOI Gradient Test is an automated quality control algorithm used on various OOI data products. This automated algorithm generates flags for data points according to whether changes between successive points are within a pre-determined range.'
ctxt_id = self.dataset_management.create_parameter_context(
name='%s_gradtst_qc' % dp_name.lower(),
parameter_type='function',
parameter_context=pc.dump(),
parameter_function_id=pfunc_id,
ooi_short_name=pc.ooi_short_name,
units='1',
value_encoding='int8',
display_name=pc.display_name,
description=pc.description)
return ctxt_id, pc
def make_localrange_qc(self, name, data_product):
pfunc_id, pfunc = self.find_localrange_test()
datlim_id, datlim = self.get_array_lookup_value(
'LV_lrt_$designator_%s||datlim' % data_product)
datlimz_id, datlimz = self.get_array_lookup_value(
'LV_lrt_$designator_%s||datlimz' % data_product)
dims_id, dims = self.get_string_array_lookup_value(
'LV_lrt_$designator_%s||dims' % data_product)
pmap = {
"dat": name,
"dims*": dims,
"datlim*": datlim,
"datlimz*": datlimz
}
pfunc.param_map = pmap
pfunc.lookup_values = [datlim_id, datlimz_id, dims_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_loclrng_qc' % dp_name.lower(),
param_type=ParameterFunctionType(
pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_LOCLRNG_QC' % dp_name
pc.display_name = '%s Local Range Test Quality Control Flag' % dp_name
pc.description = 'The OOI Local Range Test is the computation to test whether a given data point falls within pre-defined ranges.'
ctxt_id = self.dataset_management.create_parameter_context(
name='%s_loclrng_qc' % dp_name.lower(),
parameter_type='function',
parameter_context=pc.dump(),
parameter_function_id=pfunc_id,
ooi_short_name=pc.ooi_short_name,
units='1',
value_encoding='int8',
display_name=pc.display_name,
description=pc.description)
return ctxt_id, pc
def make_spike_qc(self, name, data_product):
pfunc_id, pfunc = self.find_spike()
spike_acc_id, spike_acc_name = self.get_lookup_value(
'LV_spike_$designator_%s||acc' % data_product)
spike_n_id, spike_n_name = self.get_lookup_value(
'LV_spike_$designator_%s||spike_n' % data_product)
spike_l_id, spike_l_name = self.get_lookup_value(
'LV_spike_$designator_%s||spike_l' % data_product)
pmap = {
'dat': name,
'acc': spike_acc_name,
'N': spike_n_name,
'L': spike_l_name
}
pfunc.param_map = pmap
pfunc.lookup_values = [spike_acc_id, spike_n_id, spike_l_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_spketst_qc' % dp_name.lower(),
param_type=ParameterFunctionType(
pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_SPKETST_QC' % dp_name
pc.display_name = '%s Spike Test Quality Control Flag' % dp_name
pc.description = "The OOI Spike Test quality control algorithm generates a flag for individual data values that deviate significantly from surrounding data values."
ctxt_id = self.dataset_management.create_parameter_context(
name='%s_spketst_qc' % dp_name.lower(),
parameter_type='function',
parameter_context=pc.dump(),
parameter_function_id=pfunc_id,
ooi_short_name=pc.ooi_short_name,
units='1',
value_encoding='int8',
display_name=pc.display_name,
description=pc.description)
return ctxt_id, pc
def make_trendtest_qc(self, name, data_product):
pfunc_id, pfunc = self.find_trend_test()
order_id, order_name = self.get_lookup_value(
'LV_trend_$designator_%s||polynomial_order' % data_product)
dev_id, dev_name = self.get_lookup_value(
'LV_trend_$designator_%s||standard_deviation' % data_product)
pmap = {
"dat": name,
"t": 'time',
"ord_n": order_name,
"ntsd": dev_name
}
pfunc.param_map = pmap
pfunc.lookup_values = [order_id, dev_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_trndtst_qc' % dp_name.lower(),
param_type=ParameterFunctionType(
pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_TRNDTST_QC' % dp_name
pc.display_name = '%s Trend Test Test Quality Control Flag' % dp_name
pc.description = 'The OOI Trend Test quality control algorithm generates flags on data values within a time series where a significant fraction of the variability in the time series can be explained by a drift, where the drift is assumed to be a polynomial of specified order.'
ctxt_id = self.dataset_management.create_parameter_context(
name='%s_trndtst_qc' % dp_name.lower(),
parameter_type='function',
parameter_context=pc.dump(),
parameter_function_id=pfunc_id,
ooi_short_name=pc.ooi_short_name,
units='1',
value_encoding='int8',
display_name=pc.display_name,
description=pc.description)
return ctxt_id, pc
def create_pfuncs(self):
contexts = {}
funcs = {}
t_ctxt = ParameterContext(name='TIME',
parameter_type='quantity',
value_encoding='float64',
units='seconds since 1900-01-01')
t_ctxt_id = self.dataset_management.create_parameter(t_ctxt)
contexts['TIME'] = t_ctxt_id
lat_ctxt = ParameterContext(name='LAT',
parameter_type="sparse",
value_encoding='float32',
units='degrees_north')
lat_ctxt_id = self.dataset_management.create_parameter(lat_ctxt)
contexts['LAT'] = lat_ctxt_id
lon_ctxt = ParameterContext(name='LON',
parameter_type='sparse',
value_encoding='float32',
units='degrees_east')
lon_ctxt_id = self.dataset_management.create_parameter(lon_ctxt)
contexts['LON'] = lon_ctxt_id
# Independent Parameters
# Temperature - values expected to be the decimal results of conversion from hex
temp_ctxt = ParameterContext(name='TEMPWAT_L0',
parameter_type='quantity',
value_encoding='float32',
units='deg_C')
temp_ctxt_id = self.dataset_management.create_parameter(temp_ctxt)
contexts['TEMPWAT_L0'] = temp_ctxt_id
# Conductivity - values expected to be the decimal results of conversion from hex
cond_ctxt = ParameterContext(name='CONDWAT_L0',
parameter_type='quantity',
value_encoding='float32',
units='S m-1')
cond_ctxt_id = self.dataset_management.create_parameter(cond_ctxt)
contexts['CONDWAT_L0'] = cond_ctxt_id
# Pressure - values expected to be the decimal results of conversion from hex
press_ctxt = ParameterContext(name='PRESWAT_L0',
parameter_type='quantity',
value_encoding='float32',
units='dbar')
press_ctxt_id = self.dataset_management.create_parameter(press_ctxt)
contexts['PRESWAT_L0'] = press_ctxt_id
# Dependent Parameters
# TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10
tl1_func = '(T / 10000) - 10'
expr = ParameterFunction(name='TEMPWAT_L1',
function_type=PFT.NUMEXPR,
function=tl1_func,
args=['T'])
expr_id = self.dataset_management.create_parameter_function(expr)
funcs['TEMPWAT_L1'] = expr_id
tl1_pmap = {'T': 'TEMPWAT_L0'}
tempL1_ctxt = ParameterContext(name='TEMPWAT_L1',
parameter_type='function',
parameter_function_id=expr_id,
parameter_function_map=tl1_pmap,
value_encoding='float32',
units='deg_C')
tempL1_ctxt_id = self.dataset_management.create_parameter(tempL1_ctxt)
contexts['TEMPWAT_L1'] = tempL1_ctxt_id
# CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5
cl1_func = '(C / 100000) - 0.5'
expr = ParameterFunction(name='CONDWAT_L1',
function_type=PFT.NUMEXPR,
function=cl1_func,
args=['C'])
expr_id = self.dataset_management.create_parameter_function(expr)
funcs['CONDWAT_L1'] = expr_id
cl1_pmap = {'C': 'CONDWAT_L0'}
condL1_ctxt = ParameterContext(name='CONDWAT_L1',
parameter_type='function',
parameter_function_id=expr_id,
parameter_function_map=cl1_pmap,
value_encoding='float32',
units='S m-1')
condL1_ctxt_id = self.dataset_management.create_parameter(condL1_ctxt)
contexts['CONDWAT_L1'] = condL1_ctxt_id
# Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721
# PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range)
pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)'
expr = ParameterFunction(name='PRESWAT_L1',function=pl1_func,function_type=PFT.NUMEXPR,args=['P','p_range'])
expr_id = self.dataset_management.create_parameter_function(expr)
funcs['PRESWAT_L1'] = expr_id
pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721}
presL1_ctxt = ParameterContext(name='PRESWAT_L1',
parameter_type='function',
parameter_function_id=expr_id,
parameter_function_map=pl1_pmap,
value_encoding='float32',
units='S m-1')
presL1_ctxt_id = self.dataset_management.create_parameter(presL1_ctxt)
contexts['PRESWAT_L1'] = presL1_ctxt_id
# Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project:
# https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1
# PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1)
owner = 'gsw'
sal_func = 'SP_from_C'
sal_arglist = ['C', 't', 'p']
expr = ParameterFunction(name='PRACSAL',function_type=PFT.PYTHON,function=sal_func,owner=owner,args=sal_arglist)
expr_id = self.dataset_management.create_parameter_function(expr)
funcs['PRACSAL'] = expr_id
c10_f = ParameterFunction(name='c10', function_type=PFT.NUMEXPR, function='C*10', args=['C'])
expr_id = self.dataset_management.create_parameter_function(c10_f)
c10 = ParameterContext(name='c10',
parameter_type='function',
parameter_function_id=expr_id,
parameter_function_map={'C':'CONDWAT_L1'},
value_encoding='float32',
units='1')
c10_id = self.dataset_management.create_parameter(c10)
contexts['c10'] = c10_id
# A magic function that may or may not exist actually forms the line below at runtime.
sal_pmap = {'C': 'c10', 't': 'TEMPWAT_L1', 'p': 'PRESWAT_L1'}
sal_ctxt = ParameterContext(name='PRACSAL',
parameter_type='function',
parameter_function_id=expr_id,
parameter_function_map=sal_pmap,
value_encoding='float32',
units='g kg-1')
sal_ctxt_id = self.dataset_management.create_parameter(sal_ctxt)
contexts['PRACSAL'] = sal_ctxt_id
# absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude)
# conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1)
# DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1)
owner = 'gsw'
abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP', ['PRACSAL', 'PRESWAT_L1', 'LON','LAT'])
cons_temp_expr = PythonFunction('cons_temp', owner, 'CT_from_t', [abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1'])
dens_expr = PythonFunction('DENSITY', owner, 'rho', [abs_sal_expr, cons_temp_expr, 'PRESWAT_L1'])
dens_ctxt = CoverageParameterContext('DENSITY', param_type=ParameterFunctionType(dens_expr), variability=VariabilityEnum.TEMPORAL)
dens_ctxt.uom = 'kg m-3'
dens_ctxt_id = self.dataset_management.create_parameter_context(name='DENSITY', parameter_context=dens_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, dens_ctxt_id)
contexts['DENSITY'] = dens_ctxt_id
return contexts, funcs
def _get_pdict(self, filter_values):
t_ctxt = ParameterContext(
'TIME', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 01-01-1900'
t_ctxt_id = self.dataset_management.create_parameter_context(
name='TIME',
parameter_context=t_ctxt.dump(),
parameter_type='quantity<int64>',
unit_of_measure=t_ctxt.uom)
lat_ctxt = ParameterContext(
'LAT',
param_type=ConstantType(
QuantityType(value_encoding=np.dtype('float32'))),
fill_value=-9999)
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt_id = self.dataset_management.create_parameter_context(
name='LAT',
parameter_context=lat_ctxt.dump(),
parameter_type='quantity<float32>',
unit_of_measure=lat_ctxt.uom)
lon_ctxt = ParameterContext(
'LON',
param_type=ConstantType(
QuantityType(value_encoding=np.dtype('float32'))),
fill_value=-9999)
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt_id = self.dataset_management.create_parameter_context(
name='LON',
parameter_context=lon_ctxt.dump(),
parameter_type='quantity<float32>',
unit_of_measure=lon_ctxt.uom)
# Independent Parameters
# Temperature - values expected to be the decimal results of conversion from hex
temp_ctxt = ParameterContext(
'TEMPWAT_L0',
param_type=QuantityType(value_encoding=np.dtype('float32')),
fill_value=-9999)
temp_ctxt.uom = 'deg_C'
temp_ctxt_id = self.dataset_management.create_parameter_context(
name='TEMPWAT_L0',
parameter_context=temp_ctxt.dump(),
parameter_type='quantity<float32>',
unit_of_measure=temp_ctxt.uom)
# Conductivity - values expected to be the decimal results of conversion from hex
cond_ctxt = ParameterContext(
'CONDWAT_L0',
param_type=QuantityType(value_encoding=np.dtype('float32')),
fill_value=-9999)
cond_ctxt.uom = 'S m-1'
cond_ctxt_id = self.dataset_management.create_parameter_context(
name='CONDWAT_L0',
parameter_context=cond_ctxt.dump(),
parameter_type='quantity<float32>',
unit_of_measure=cond_ctxt.uom)
# Pressure - values expected to be the decimal results of conversion from hex
press_ctxt = ParameterContext(
'PRESWAT_L0',
param_type=QuantityType(value_encoding=np.dtype('float32')),
fill_value=-9999)
press_ctxt.uom = 'dbar'
press_ctxt_id = self.dataset_management.create_parameter_context(
name='PRESWAT_L0',
parameter_context=press_ctxt.dump(),
parameter_type='quantity<float32>',
unit_of_measure=press_ctxt.uom)
# Dependent Parameters
# TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10
tl1_func = '(T / 10000) - 10'
tl1_pmap = {'T': 'TEMPWAT_L0'}
expr = NumexprFunction('TEMPWAT_L1',
tl1_func, ['T'],
param_map=tl1_pmap)
tempL1_ctxt = ParameterContext(
'TEMPWAT_L1',
param_type=ParameterFunctionType(function=expr),
variability=VariabilityEnum.TEMPORAL)
tempL1_ctxt.uom = 'deg_C'
tempL1_ctxt_id = self.dataset_management.create_parameter_context(
name=tempL1_ctxt.name,
parameter_context=tempL1_ctxt.dump(),
parameter_type='pfunc',
unit_of_measure=tempL1_ctxt.uom)
# CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5
cl1_func = '(C / 100000) - 0.5'
cl1_pmap = {'C': 'CONDWAT_L0'}
expr = NumexprFunction('CONDWAT_L1',
cl1_func, ['C'],
param_map=cl1_pmap)
condL1_ctxt = ParameterContext(
'CONDWAT_L1',
param_type=ParameterFunctionType(function=expr),
variability=VariabilityEnum.TEMPORAL)
condL1_ctxt.uom = 'S m-1'
condL1_ctxt_id = self.dataset_management.create_parameter_context(
name=condL1_ctxt.name,
parameter_context=condL1_ctxt.dump(),
parameter_type='pfunc',
unit_of_measure=condL1_ctxt.uom)
# Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721
# PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range)
pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)'
pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721}
expr = NumexprFunction('PRESWAT_L1',
pl1_func, ['P', 'p_range'],
param_map=pl1_pmap)
presL1_ctxt = ParameterContext(
'PRESWAT_L1',
param_type=ParameterFunctionType(function=expr),
variability=VariabilityEnum.TEMPORAL)
presL1_ctxt.uom = 'S m-1'
presL1_ctxt_id = self.dataset_management.create_parameter_context(
name=presL1_ctxt.name,
parameter_context=presL1_ctxt.dump(),
parameter_type='pfunc',
unit_of_measure=presL1_ctxt.uom)
# Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project:
# https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1
# PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1)
owner = 'gsw'
sal_func = 'SP_from_C'
sal_arglist = ['C', 't', 'p']
sal_pmap = {
'C':
NumexprFunction('CONDWAT_L1*10',
'C*10', ['C'],
param_map={'C': 'CONDWAT_L1'}),
't':
'TEMPWAT_L1',
'p':
'PRESWAT_L1'
}
sal_kwargmap = None
expr = PythonFunction('PRACSAL', owner, sal_func, sal_arglist,
sal_kwargmap, sal_pmap)
sal_ctxt = ParameterContext('PRACSAL',
param_type=ParameterFunctionType(expr),
variability=VariabilityEnum.TEMPORAL)
sal_ctxt.uom = 'g kg-1'
sal_ctxt_id = self.dataset_management.create_parameter_context(
name=sal_ctxt.name,
parameter_context=sal_ctxt.dump(),
parameter_type='pfunc',
unit_of_measure=sal_ctxt.uom)
# absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude)
# conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1)
# DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1)
owner = 'gsw'
abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP',
['PRACSAL', 'PRESWAT_L1', 'LON', 'LAT'])
cons_temp_expr = PythonFunction(
'cons_temp', owner, 'CT_from_t',
[abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1'])
dens_expr = PythonFunction(
'DENSITY', owner, 'rho',
[abs_sal_expr, cons_temp_expr, 'PRESWAT_L1'])
dens_ctxt = ParameterContext(
'DENSITY',
param_type=ParameterFunctionType(dens_expr),
variability=VariabilityEnum.TEMPORAL)
dens_ctxt.uom = 'kg m-3'
dens_ctxt_id = self.dataset_management.create_parameter_context(
name=dens_ctxt.name,
parameter_context=dens_ctxt.dump(),
parameter_type='pfunc',
unit_of_measure=dens_ctxt.uom)
ids = [
t_ctxt_id, lat_ctxt_id, lon_ctxt_id, temp_ctxt_id, cond_ctxt_id,
press_ctxt_id, tempL1_ctxt_id, condL1_ctxt_id, presL1_ctxt_id,
sal_ctxt_id, dens_ctxt_id
]
contexts = [
t_ctxt, lat_ctxt, lon_ctxt, temp_ctxt, cond_ctxt, press_ctxt,
tempL1_ctxt, condL1_ctxt, presL1_ctxt, sal_ctxt, dens_ctxt
]
context_ids = [
ids[i] for i, ctxt in enumerate(contexts)
if ctxt.name in filter_values
]
pdict_name = '_'.join(
[ctxt.name for ctxt in contexts if ctxt.name in filter_values])
try:
self.pdicts[pdict_name]
return self.pdicts[pdict_name]
except KeyError:
pdict_id = self.dataset_management.create_parameter_dictionary(
pdict_name,
parameter_context_ids=context_ids,
temporal_context='time')
self.pdicts[pdict_name] = pdict_id
return pdict_id
def create_pfuncs(self):
contexts = {}
funcs = {}
t_ctxt = ParameterContext(
'TIME', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 01-01-1900'
t_ctxt_id = self.dataset_management.create_parameter_context(
name='test_TIME', parameter_context=t_ctxt.dump())
contexts['TIME'] = (t_ctxt, t_ctxt_id)
lat_ctxt = ParameterContext(
'LAT',
param_type=ConstantType(
QuantityType(value_encoding=np.dtype('float32'))),
fill_value=-9999)
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt_id = self.dataset_management.create_parameter_context(
name='test_LAT', parameter_context=lat_ctxt.dump())
contexts['LAT'] = lat_ctxt, lat_ctxt_id
lon_ctxt = ParameterContext(
'LON',
param_type=ConstantType(
QuantityType(value_encoding=np.dtype('float32'))),
fill_value=-9999)
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt_id = self.dataset_management.create_parameter_context(
name='test_LON', parameter_context=lon_ctxt.dump())
contexts['LON'] = lon_ctxt, lon_ctxt_id
# Independent Parameters
# Temperature - values expected to be the decimal results of conversion from hex
temp_ctxt = ParameterContext(
'TEMPWAT_L0',
param_type=QuantityType(value_encoding=np.dtype('float32')),
fill_value=-9999)
temp_ctxt.uom = 'deg_C'
temp_ctxt_id = self.dataset_management.create_parameter_context(
name='test_TEMPWAT_L0', parameter_context=temp_ctxt.dump())
contexts['TEMPWAT_L0'] = temp_ctxt, temp_ctxt_id
# Conductivity - values expected to be the decimal results of conversion from hex
cond_ctxt = ParameterContext(
'CONDWAT_L0',
param_type=QuantityType(value_encoding=np.dtype('float32')),
fill_value=-9999)
cond_ctxt.uom = 'S m-1'
cond_ctxt_id = self.dataset_management.create_parameter_context(
name='test_CONDWAT_L0', parameter_context=cond_ctxt.dump())
contexts['CONDWAT_L0'] = cond_ctxt, cond_ctxt_id
# Pressure - values expected to be the decimal results of conversion from hex
press_ctxt = ParameterContext(
'PRESWAT_L0',
param_type=QuantityType(value_encoding=np.dtype('float32')),
fill_value=-9999)
press_ctxt.uom = 'dbar'
press_ctxt_id = self.dataset_management.create_parameter_context(
name='test_PRESWAT_L0', parameter_context=press_ctxt.dump())
contexts['PRESWAT_L0'] = press_ctxt, press_ctxt_id
# Dependent Parameters
# TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10
tl1_func = '(T / 10000) - 10'
expr = NumexprFunction('TEMPWAT_L1', tl1_func, ['T'])
expr_id = self.dataset_management.create_parameter_function(
name='test_TEMPWAT_L1', parameter_function=expr.dump())
funcs['TEMPWAT_L1'] = expr, expr_id
tl1_pmap = {'T': 'TEMPWAT_L0'}
expr.param_map = tl1_pmap
tempL1_ctxt = ParameterContext(
'TEMPWAT_L1',
param_type=ParameterFunctionType(function=expr),
variability=VariabilityEnum.TEMPORAL)
tempL1_ctxt.uom = 'deg_C'
tempL1_ctxt_id = self.dataset_management.create_parameter_context(
name='test_TEMPWAT_L1',
parameter_context=tempL1_ctxt.dump(),
parameter_function_id=expr_id)
contexts['TEMPWAT_L1'] = tempL1_ctxt, tempL1_ctxt_id
# CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5
cl1_func = '(C / 100000) - 0.5'
expr = NumexprFunction('CONDWAT_L1', cl1_func, ['C'])
expr_id = self.dataset_management.create_parameter_function(
name='test_CONDWAT_L1', parameter_function=expr.dump())
funcs['CONDWAT_L1'] = expr, expr_id
cl1_pmap = {'C': 'CONDWAT_L0'}
expr.param_map = cl1_pmap
condL1_ctxt = ParameterContext(
'CONDWAT_L1',
param_type=ParameterFunctionType(function=expr),
variability=VariabilityEnum.TEMPORAL)
condL1_ctxt.uom = 'S m-1'
condL1_ctxt_id = self.dataset_management.create_parameter_context(
name='test_CONDWAT_L1',
parameter_context=condL1_ctxt.dump(),
parameter_function_id=expr_id)
contexts['CONDWAT_L1'] = condL1_ctxt, condL1_ctxt_id
# Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721
# PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range)
pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)'
expr = NumexprFunction('PRESWAT_L1', pl1_func, ['P', 'p_range'])
expr_id = self.dataset_management.create_parameter_function(
name='test_PRESWAT_L1', parameter_function=expr.dump())
funcs['PRESWAT_L1'] = expr, expr_id
pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721}
expr.param_map = pl1_pmap
presL1_ctxt = ParameterContext(
'PRESWAT_L1',
param_type=ParameterFunctionType(function=expr),
variability=VariabilityEnum.TEMPORAL)
presL1_ctxt.uom = 'S m-1'
presL1_ctxt_id = self.dataset_management.create_parameter_context(
name='test_CONDWAT_L1',
parameter_context=presL1_ctxt.dump(),
parameter_function_id=expr_id)
contexts['PRESWAT_L1'] = presL1_ctxt, presL1_ctxt_id
# Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project:
# https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1
# PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1)
owner = 'gsw'
sal_func = 'SP_from_C'
sal_arglist = ['C', 't', 'p']
expr = PythonFunction('PRACSAL', owner, sal_func, sal_arglist)
expr_id = self.dataset_management.create_parameter_function(
name='test_PRACSAL', parameter_function=expr.dump())
funcs['PRACSAL'] = expr, expr_id
# A magic function that may or may not exist actually forms the line below at runtime.
sal_pmap = {
'C':
NumexprFunction('CONDWAT_L1*10',
'C*10', ['C'],
param_map={'C': 'CONDWAT_L1'}),
't':
'TEMPWAT_L1',
'p':
'PRESWAT_L1'
}
expr.param_map = sal_pmap
sal_ctxt = ParameterContext('PRACSAL',
param_type=ParameterFunctionType(expr),
variability=VariabilityEnum.TEMPORAL)
sal_ctxt.uom = 'g kg-1'
sal_ctxt_id = self.dataset_management.create_parameter_context(
name='test_PRACSAL',
parameter_context=sal_ctxt.dump(),
parameter_function_id=expr_id)
contexts['PRACSAL'] = sal_ctxt, sal_ctxt_id
# absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude)
# conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1)
# DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1)
owner = 'gsw'
abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP',
['PRACSAL', 'PRESWAT_L1', 'LON', 'LAT'])
cons_temp_expr = PythonFunction(
'cons_temp', owner, 'CT_from_t',
[abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1'])
dens_expr = PythonFunction(
'DENSITY', owner, 'rho',
[abs_sal_expr, cons_temp_expr, 'PRESWAT_L1'])
dens_ctxt = ParameterContext(
'DENSITY',
param_type=ParameterFunctionType(dens_expr),
variability=VariabilityEnum.TEMPORAL)
dens_ctxt.uom = 'kg m-3'
dens_ctxt_id = self.dataset_management.create_parameter_context(
name='test_DENSITY', parameter_context=dens_ctxt.dump())
contexts['DENSITY'] = dens_ctxt, dens_ctxt_id
return contexts, funcs
def get_external_type(self, parameter_type, encoding, target_info):
pfunc = ExternalFunction(name=target_info['name'],
external_guid=target_info['target_dataset'],
external_name=target_info['target_name'])
param_type = ParameterFunctionType(pfunc)
return param_type