def make_stuckvalue_qc(self, name, data_product):
pfunc_id, pfunc = self.find_stuck_value()
reso_id, reso_name = self.get_lookup_value(
'LV_svt_$designator_%s||svt_resolution' % data_product)
n_id, n_name = self.get_lookup_value('LV_svt_$designator_%s||svt_n' %
data_product)
pmap = {'x': name, 'reso': reso_name, 'num': n_name}
pfunc.param_map = pmap
pfunc.lookup_values = [reso_id, n_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_stuckvl_qc' % dp_name.lower(),
param_type=ParameterFunctionType(
pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_STUCKVL_QC' % dp_name
pc.display_name = '%s Stuck Value Test Quality Control Flag' % dp_name
pc.description = 'The OOI Stuck Value Test quality control algorithm generates a flag for repeated occurrence of one value in a time series.'
ctxt_id = self.dataset_management.create_parameter_context(
name='%s_stuckvl_qc' % dp_name.lower(),
parameter_type='function',
parameter_context=pc.dump(),
parameter_function_id=pfunc_id,
ooi_short_name=pc.ooi_short_name,
units='1',
value_encoding='int8',
display_name=pc.display_name,
description=pc.description)
return ctxt_id, pc
def make_grt_qc(self, name, data_product):
pfunc_id, pfunc = self.find_grt()
grt_min_id, grt_min_name = self.get_lookup_value(
'LV_grt_$designator_%s||grt_min_value' % data_product)
grt_max_id, grt_max_name = self.get_lookup_value(
'LV_grt_$designator_%s||grt_max_value' % data_product)
pmap = {'dat': name, 'dat_min': grt_min_name, 'dat_max': grt_max_name}
pfunc.param_map = pmap
pfunc.lookup_values = [grt_min_id, grt_max_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_glblrng_qc' % dp_name.lower(),
param_type=ParameterFunctionType(
pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_GLBLRNG_QC' % dp_name
pc.display_name = '%s Global Range Test Quality Control Flag' % dp_name
pc.description = "The OOI Global Range quality control algorithm generates a QC flag for the input data point indicating whether it falls within a given range."
ctxt_id = self.dataset_management.create_parameter_context(
name='%s_glblrng_qc' % dp_name.lower(),
parameter_type='function',
parameter_context=pc.dump(),
parameter_function_id=pfunc_id,
ooi_short_name=pc.ooi_short_name,
units='1',
value_encoding='int8',
display_name=pc.display_name,
description=pc.description)
return ctxt_id, pc
def create_simple_array(self):
contexts = {}
types_manager = TypesManager(self.dataset_management,None,None)
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('float64')))
t_ctxt.uom = 'seconds since 1900-01-01'
t_ctxt_id = self.dataset_management.create_parameter_context(name='time', parameter_context=t_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, t_ctxt_id)
contexts['time'] = (t_ctxt, t_ctxt_id)
temp_ctxt = ParameterContext('temp_sample', param_type=ArrayType(inner_encoding='float32'))
temp_ctxt.uom = 'deg_C'
temp_ctxt.ooi_short_name = 'TEMPWAT'
temp_ctxt.display_name = 'Temperature'
temp_ctxt_id = self.dataset_management.create_parameter_context(name='temp', parameter_context=temp_ctxt.dump(), ooi_short_name='TEMPWAT')
self.addCleanup(self.dataset_management.delete_parameter_context, temp_ctxt_id)
contexts['temp'] = temp_ctxt, temp_ctxt_id
cond_ctxt = ParameterContext('cond_sample', param_type=ArrayType(inner_encoding='float64'))
cond_ctxt.uom = 'deg_C'
cond_ctxt.ooi_short_name = 'CONDWAT'
cond_ctxt.display_anme = 'Conductivity'
cond_ctxt_id = self.dataset_management.create_parameter_context(name='cond', parameter_context=cond_ctxt.dump(), ooi_short_name='CONDWAT')
self.addCleanup(self.dataset_management.delete_parameter_context, cond_ctxt_id)
contexts['cond'] = cond_ctxt, cond_ctxt_id
func = self.create_matrix_offset_function()
func.param_map = {'x':'temp_sample', 'y':'cond_sample'}
temp_offset = ParameterContext('temp_offset', param_type=ParameterFunctionType(func))
temp_offset.uom = '1'
temp_offset_id = self.dataset_management.create_parameter_context(name='temp_offset', parameter_context=temp_offset.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, temp_offset_id)
contexts['temp_offset'] = temp_offset, temp_offset_id
return contexts
def make_propagate_qc(self, inputs):
pfunc_id, pfunc = self.find_propagate_test()
pmap = {"strict_validation": False}
arg_list = ['strict_validation']
for i, val in enumerate(inputs):
if i >= 100:
break
pmap['array%s' % i] = val
arg_list.append('array%s' % i)
pfunc.param_map = pmap
pfunc.arg_list = arg_list
pc = ParameterContext(name='cmbnflg_qc',
param_type=ParameterFunctionType(
pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = 'CMBNFLG_QC'
pc.display_name = 'Combined Data Quality Control Flag'
pc.description = 'The purpose of this computation is to produce a single merged QC flag from a set of potentially many flags.'
ctxt_id = self.dataset_management.create_parameter_context(
name='cmbnflg_qc',
parameter_type='function',
parameter_context=pc.dump(),
parameter_function_id=pfunc_id,
ooi_short_name=pc.ooi_short_name,
units='1',
value_encoding='int8',
display_name=pc.display_name,
description=pc.description)
return ctxt_id, pc
def get_cc_value(self, value):
placeholder = value.lower()
# Check to see if this coefficient exists already
hits, _ = Container.instance.resource_registry.find_resources(name=placeholder, restype=RT.ParameterContext, id_only=True)
if hits:
return hits[0], placeholder
pc = ParameterContext(name=placeholder, param_type=SparseConstantType(value_encoding='float64'), fill_value=-9999.)
pc.uom = '1'
pc.display_name = 'Calibration Coefficient ' + placeholder
ctxt_id = self.dataset_management.create_parameter_context(name=placeholder, parameter_context=pc.dump())
return ctxt_id, placeholder
def make_grt_qc(self, name, data_product):
pfunc_id, pfunc = self.find_grt()
grt_min_id, grt_min_name = self.get_lookup_value('LV_grt_$designator_%s||grt_min_value' % data_product)
grt_max_id, grt_max_name = self.get_lookup_value('LV_grt_$designator_%s||grt_max_value' % data_product)
pmap = {'dat':name, 'dat_min':grt_min_name,'dat_max':grt_max_name}
pfunc.param_map = pmap
pfunc.lookup_values = [grt_min_id, grt_max_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_glblrng_qc' % dp_name.lower(), param_type=ParameterFunctionType(pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_GLBLRNG_QC' % dp_name
pc.display_name = '%s Global Range Test Quality Control Flag' % dp_name
pc.description = "The OOI Global Range quality control algorithm generates a QC flag for the input data point indicating whether it falls within a given range."
ctxt_id = self.dataset_management.create_parameter_context(name='%s_glblrng_qc' % dp_name.lower(), parameter_type='function', parameter_context=pc.dump(), parameter_function_id=pfunc_id, ooi_short_name=pc.ooi_short_name, units='1', value_encoding='int8', display_name=pc.display_name, description=pc.description)
return ctxt_id, pc
def make_stuckvalue_qc(self, name, data_product):
pfunc_id, pfunc = self.find_stuck_value()
reso_id, reso_name = self.get_lookup_value('LV_svt_$designator_%s||svt_resolution' % data_product)
n_id, n_name = self.get_lookup_value('LV_svt_$designator_%s||svt_n' % data_product)
pmap = {'x' : name, 'reso': reso_name, 'num': n_name}
pfunc.param_map = pmap
pfunc.lookup_values = [reso_id, n_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_stuckvl_qc' % dp_name.lower(), param_type=ParameterFunctionType(pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_STUCKVL_QC' % dp_name
pc.display_name = '%s Stuck Value Test Quality Control Flag' % dp_name
pc.description = 'The OOI Stuck Value Test quality control algorithm generates a flag for repeated occurrence of one value in a time series.'
ctxt_id = self.dataset_management.create_parameter_context(name='%s_stuckvl_qc' % dp_name.lower(), parameter_type='function', parameter_context=pc.dump(), parameter_function_id=pfunc_id, ooi_short_name=pc.ooi_short_name, units='1', value_encoding='int8', display_name=pc.display_name, description=pc.description)
return ctxt_id, pc
def get_cc_value(self, value):
placeholder = value.lower()
# Check to see if this coefficient exists already
hits, _ = Container.instance.resource_registry.find_resources(
name=placeholder, restype=RT.ParameterContext, id_only=True)
if hits:
return hits[0], placeholder
pc = ParameterContext(
name=placeholder,
param_type=SparseConstantType(value_encoding='float64'),
fill_value=-9999.)
pc.uom = '1'
pc.display_name = 'Calibration Coefficient ' + placeholder
ctxt_id = self.dataset_management.create_parameter_context(
name=placeholder, parameter_context=pc.dump())
return ctxt_id, placeholder
def make_gradienttest_qc(self, name, data_product):
pfunc_id, pfunc = self.find_gradient_test()
ddatdx_id, ddatdx = self.get_lookup_value(
'LV_grad_$designator_%s_time||d_dat_dx' % data_product)
mindx_id, mindx = self.get_lookup_value(
'LV_grad_$designator_%s_time||min_dx' % data_product)
startdat_id, startdat = self.get_lookup_value(
'LV_grad_$designator_%s_time||start_dat' % data_product)
toldat_id, toldat = self.get_lookup_value(
'LV_grad_$designator_%s_time||tol_dat' % data_product)
pmap = {
"dat": name,
"x": 'time',
'ddatdx': ddatdx,
'mindx': mindx,
'startdat': startdat,
'toldat': toldat
}
pfunc.param_map = pmap
pfunc.lookup_values = [ddatdx_id, mindx_id, startdat_id, toldat_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_gradtst_qc' % dp_name.lower(),
param_type=ParameterFunctionType(
pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_GRADTST_QC' % dp_name
pc.display_name = '%s Gradient Test Quality Control Flag' % dp_name
pc.description = 'The OOI Gradient Test is an automated quality control algorithm used on various OOI data products. This automated algorithm generates flags for data points according to whether changes between successive points are within a pre-determined range.'
ctxt_id = self.dataset_management.create_parameter_context(
name='%s_gradtst_qc' % dp_name.lower(),
parameter_type='function',
parameter_context=pc.dump(),
parameter_function_id=pfunc_id,
ooi_short_name=pc.ooi_short_name,
units='1',
value_encoding='int8',
display_name=pc.display_name,
description=pc.description)
return ctxt_id, pc
def make_propagate_qc(self,inputs):
pfunc_id, pfunc = self.find_propagate_test()
pmap = {"strict_validation":False}
arg_list = ['strict_validation']
for i,val in enumerate(inputs):
if i >= 100:
break
pmap['array%s' % i] = val
arg_list.append('array%s' % i)
pfunc.param_map = pmap
pfunc.arg_list = arg_list
pc = ParameterContext(name='cmbnflg_qc', param_type=ParameterFunctionType(pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = 'CMBNFLG_QC'
pc.display_name = 'Combined Data Quality Control Flag'
pc.description = 'The purpose of this computation is to produce a single merged QC flag from a set of potentially many flags.'
ctxt_id = self.dataset_management.create_parameter_context(name='cmbnflg_qc', parameter_type='function', parameter_context=pc.dump(), parameter_function_id=pfunc_id, ooi_short_name=pc.ooi_short_name, units='1', value_encoding='int8', display_name=pc.display_name, description=pc.description)
return ctxt_id, pc
def make_trendtest_qc(self, name, data_product):
pfunc_id, pfunc = self.find_trend_test()
order_id, order_name = self.get_lookup_value('LV_trend_$designator_%s||polynomial_order' % data_product)
dev_id, dev_name = self.get_lookup_value('LV_trend_$designator_%s||standard_deviation' % data_product)
pmap = {"dat":name ,"t":'time',"ord_n":order_name,"ntsd":dev_name}
pfunc.param_map = pmap
pfunc.lookup_values = [order_id, dev_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_trndtst_qc' % dp_name.lower(), param_type=ParameterFunctionType(pfunc,value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_TRNDTST_QC' % dp_name
pc.display_name = '%s Trend Test Test Quality Control Flag' % dp_name
pc.description = 'The OOI Trend Test quality control algorithm generates flags on data values within a time series where a significant fraction of the variability in the time series can be explained by a drift, where the drift is assumed to be a polynomial of specified order.'
ctxt_id = self.dataset_management.create_parameter_context(name='%s_trndtst_qc' % dp_name.lower(), parameter_type='function', parameter_context=pc.dump(), parameter_function_id=pfunc_id, ooi_short_name=pc.ooi_short_name, units='1', value_encoding='int8', display_name=pc.display_name, description=pc.description)
return ctxt_id, pc
def make_spike_qc(self, name, data_product):
pfunc_id, pfunc = self.find_spike()
spike_acc_id, spike_acc_name = self.get_lookup_value('LV_spike_$designator_%s||acc' % data_product)
spike_n_id, spike_n_name = self.get_lookup_value('LV_spike_$designator_%s||spike_n' % data_product)
spike_l_id, spike_l_name = self.get_lookup_value('LV_spike_$designator_%s||spike_l' % data_product)
pmap = {'dat':name, 'acc':spike_acc_name, 'N':spike_n_name, 'L':spike_l_name}
pfunc.param_map = pmap
pfunc.lookup_values = [spike_acc_id, spike_n_id, spike_l_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_spketst_qc' % dp_name.lower(), param_type=ParameterFunctionType(pfunc, value_encoding='|i1'))
pc.uom='1'
pc.ooi_short_name = '%s_SPKETST_QC' % dp_name
pc.display_name = '%s Spike Test Quality Control Flag' % dp_name
pc.description = "The OOI Spike Test quality control algorithm generates a flag for individual data values that deviate significantly from surrounding data values."
ctxt_id = self.dataset_management.create_parameter_context(name='%s_spketst_qc' % dp_name.lower(), parameter_type='function', parameter_context=pc.dump(), parameter_function_id=pfunc_id, ooi_short_name=pc.ooi_short_name, units='1', value_encoding='int8', display_name=pc.display_name, description=pc.description)
return ctxt_id, pc
def make_localrange_qc(self, name, data_product):
pfunc_id, pfunc = self.find_localrange_test()
datlim_id, datlim = self.get_array_lookup_value('LV_lrt_$designator_%s||datlim' % data_product)
datlimz_id, datlimz = self.get_array_lookup_value('LV_lrt_$designator_%s||datlimz' % data_product)
dims_id, dims = self.get_string_array_lookup_value('LV_lrt_$designator_%s||dims' % data_product)
pmap = {"dat":name, "dims*":dims, "datlim*":datlim, "datlimz*":datlimz}
pfunc.param_map = pmap
pfunc.lookup_values = [datlim_id, datlimz_id, dims_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_loclrng_qc' % dp_name.lower(), param_type=ParameterFunctionType(pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_LOCLRNG_QC' % dp_name
pc.display_name = '%s Local Range Test Quality Control Flag' % dp_name
pc.description = 'The OOI Local Range Test is the computation to test whether a given data point falls within pre-defined ranges.'
ctxt_id = self.dataset_management.create_parameter_context(name='%s_loclrng_qc' % dp_name.lower(), parameter_type='function', parameter_context=pc.dump(), parameter_function_id=pfunc_id, ooi_short_name=pc.ooi_short_name, units='1', value_encoding='int8', display_name=pc.display_name, description=pc.description)
return ctxt_id, pc
def make_gradienttest_qc(self, name, data_product):
pfunc_id, pfunc = self.find_gradient_test()
ddatdx_id, ddatdx = self.get_lookup_value('LV_grad_$designator_%s_time||d_dat_dx' % data_product)
mindx_id, mindx = self.get_lookup_value('LV_grad_$designator_%s_time||min_dx' % data_product)
startdat_id, startdat = self.get_lookup_value('LV_grad_$designator_%s_time||start_dat' % data_product)
toldat_id, toldat = self.get_lookup_value('LV_grad_$designator_%s_time||tol_dat' % data_product)
pmap = {"dat":name, "x": 'time', 'ddatdx': ddatdx, 'mindx':mindx, 'startdat': startdat, 'toldat':toldat}
pfunc.param_map = pmap
pfunc.lookup_values = [ddatdx_id, mindx_id, startdat_id, toldat_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_gradtst_qc' % dp_name.lower(), param_type=ParameterFunctionType(pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_GRADTST_QC' % dp_name
pc.display_name = '%s Gradient Test Quality Control Flag' % dp_name
pc.description = 'The OOI Gradient Test is an automated quality control algorithm used on various OOI data products. This automated algorithm generates flags for data points according to whether changes between successive points are within a pre-determined range.'
ctxt_id = self.dataset_management.create_parameter_context(name='%s_gradtst_qc' % dp_name.lower(), parameter_type='function', parameter_context=pc.dump(), parameter_function_id=pfunc_id, ooi_short_name=pc.ooi_short_name, units='1', value_encoding='int8', display_name=pc.display_name, description=pc.description)
return ctxt_id, pc
def make_localrange_qc(self, name, data_product):
pfunc_id, pfunc = self.find_localrange_test()
datlim_id, datlim = self.get_array_lookup_value(
'LV_lrt_$designator_%s||datlim' % data_product)
datlimz_id, datlimz = self.get_array_lookup_value(
'LV_lrt_$designator_%s||datlimz' % data_product)
dims_id, dims = self.get_string_array_lookup_value(
'LV_lrt_$designator_%s||dims' % data_product)
pmap = {
"dat": name,
"dims*": dims,
"datlim*": datlim,
"datlimz*": datlimz
}
pfunc.param_map = pmap
pfunc.lookup_values = [datlim_id, datlimz_id, dims_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_loclrng_qc' % dp_name.lower(),
param_type=ParameterFunctionType(
pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_LOCLRNG_QC' % dp_name
pc.display_name = '%s Local Range Test Quality Control Flag' % dp_name
pc.description = 'The OOI Local Range Test is the computation to test whether a given data point falls within pre-defined ranges.'
ctxt_id = self.dataset_management.create_parameter_context(
name='%s_loclrng_qc' % dp_name.lower(),
parameter_type='function',
parameter_context=pc.dump(),
parameter_function_id=pfunc_id,
ooi_short_name=pc.ooi_short_name,
units='1',
value_encoding='int8',
display_name=pc.display_name,
description=pc.description)
return ctxt_id, pc
def make_spike_qc(self, name, data_product):
pfunc_id, pfunc = self.find_spike()
spike_acc_id, spike_acc_name = self.get_lookup_value(
'LV_spike_$designator_%s||acc' % data_product)
spike_n_id, spike_n_name = self.get_lookup_value(
'LV_spike_$designator_%s||spike_n' % data_product)
spike_l_id, spike_l_name = self.get_lookup_value(
'LV_spike_$designator_%s||spike_l' % data_product)
pmap = {
'dat': name,
'acc': spike_acc_name,
'N': spike_n_name,
'L': spike_l_name
}
pfunc.param_map = pmap
pfunc.lookup_values = [spike_acc_id, spike_n_id, spike_l_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_spketst_qc' % dp_name.lower(),
param_type=ParameterFunctionType(
pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_SPKETST_QC' % dp_name
pc.display_name = '%s Spike Test Quality Control Flag' % dp_name
pc.description = "The OOI Spike Test quality control algorithm generates a flag for individual data values that deviate significantly from surrounding data values."
ctxt_id = self.dataset_management.create_parameter_context(
name='%s_spketst_qc' % dp_name.lower(),
parameter_type='function',
parameter_context=pc.dump(),
parameter_function_id=pfunc_id,
ooi_short_name=pc.ooi_short_name,
units='1',
value_encoding='int8',
display_name=pc.display_name,
description=pc.description)
return ctxt_id, pc
def make_trendtest_qc(self, name, data_product):
pfunc_id, pfunc = self.find_trend_test()
order_id, order_name = self.get_lookup_value(
'LV_trend_$designator_%s||polynomial_order' % data_product)
dev_id, dev_name = self.get_lookup_value(
'LV_trend_$designator_%s||standard_deviation' % data_product)
pmap = {
"dat": name,
"t": 'time',
"ord_n": order_name,
"ntsd": dev_name
}
pfunc.param_map = pmap
pfunc.lookup_values = [order_id, dev_id]
dp_name = self.dp_name(data_product)
pc = ParameterContext(name='%s_trndtst_qc' % dp_name.lower(),
param_type=ParameterFunctionType(
pfunc, value_encoding='|i1'))
pc.uom = '1'
pc.ooi_short_name = '%s_TRNDTST_QC' % dp_name
pc.display_name = '%s Trend Test Test Quality Control Flag' % dp_name
pc.description = 'The OOI Trend Test quality control algorithm generates flags on data values within a time series where a significant fraction of the variability in the time series can be explained by a drift, where the drift is assumed to be a polynomial of specified order.'
ctxt_id = self.dataset_management.create_parameter_context(
name='%s_trndtst_qc' % dp_name.lower(),
parameter_type='function',
parameter_context=pc.dump(),
parameter_function_id=pfunc_id,
ooi_short_name=pc.ooi_short_name,
units='1',
value_encoding='int8',
display_name=pc.display_name,
description=pc.description)
return ctxt_id, pc
