def create_extended_parsed_contexts(self):
contexts, funcs = self.create_parsed_params()
expr, expr_id = funcs['density_L2']
density_lookup_map = {'conductivity':'conductivity_L1', 'temp':'temp_L1', 'pressure':'pressure_L1', 'lat':'lat_lookup', 'lon':'lon_lookup'}
expr.param_map = density_lookup_map
density_lookup_ctxt = ParameterContext('density_lookup', param_type=ParameterFunctionType(expr), variability=VariabilityEnum.TEMPORAL)
density_lookup_ctxt.uom = 'kg m-3'
density_lookup_ctxt_id = self.dataset_management.create_parameter_context(name='density_lookup', parameter_context=density_lookup_ctxt.dump(), parameter_function_id=expr_id)
self.addCleanup(self.dataset_management.delete_parameter_context, density_lookup_ctxt_id)
contexts['density_lookup'] = density_lookup_ctxt, density_lookup_ctxt_id
lat_lookup_ctxt = ParameterContext('lat_lookup', param_type=ConstantType(QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999)
lat_lookup_ctxt.axis = AxisTypeEnum.LAT
lat_lookup_ctxt.uom = 'degree_north'
lat_lookup_ctxt.lookup_value = 'lat'
lat_lookup_ctxt.document_key = ''
lat_lookup_ctxt_id = self.dataset_management.create_parameter_context(name='lat_lookup', parameter_context=lat_lookup_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, lat_lookup_ctxt_id)
contexts['lat_lookup'] = lat_lookup_ctxt, lat_lookup_ctxt_id
lon_lookup_ctxt = ParameterContext('lon_lookup', param_type=ConstantType(QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999)
lon_lookup_ctxt.axis = AxisTypeEnum.LON
lon_lookup_ctxt.uom = 'degree_east'
lon_lookup_ctxt.lookup_value = 'lon'
lon_lookup_ctxt.document_key = ''
lon_lookup_ctxt_id = self.dataset_management.create_parameter_context(name='lon_lookup', parameter_context=lon_lookup_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, lon_lookup_ctxt_id)
contexts['lon_lookup'] = lon_lookup_ctxt, lon_lookup_ctxt_id
return contexts, funcs
def _create_input_param_dict_for_test(self, parameter_dict_name=''):
pdict = ParameterDictionary()
t_ctxt = ParameterContext(
'time',
param_type=QuantityType(value_encoding=numpy.dtype('float64')))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1900'
pdict.add_context(t_ctxt)
cond_ctxt = ParameterContext(
'conductivity',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
cond_ctxt.uom = ''
pdict.add_context(cond_ctxt)
pres_ctxt = ParameterContext(
'pressure',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
pres_ctxt.uom = ''
pdict.add_context(pres_ctxt)
temp_ctxt = ParameterContext(
'temperature',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = ''
pdict.add_context(temp_ctxt)
dens_ctxt = ParameterContext(
'density',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
dens_ctxt.uom = ''
pdict.add_context(dens_ctxt)
sal_ctxt = ParameterContext(
'salinity',
param_type=QuantityType(value_encoding=numpy.dtype('float32')))
sal_ctxt.uom = ''
pdict.add_context(sal_ctxt)
#create temp streamdef so the data product can create the stream
pc_list = []
for pc_k, pc in pdict.iteritems():
ctxt_id = self.dataset_management.create_parameter_context(
pc_k, pc[1].dump())
pc_list.append(ctxt_id)
self.addCleanup(self.dataset_management.delete_parameter_context,
ctxt_id)
pdict_id = self.dataset_management.create_parameter_dictionary(
parameter_dict_name, pc_list)
self.addCleanup(self.dataset_management.delete_parameter_dictionary,
pdict_id)
return pdict_id
def _L0_pdict(self):
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 01-01-1900'
t_ctxt.fill_value = -9999
t_ctxt_id = self.dataset_management.create_parameter_context(name='time', parameter_context=t_ctxt.dump(), parameter_type='quantity<int64>', unit_of_measure=t_ctxt.uom)
lat_ctxt = ParameterContext('lat', param_type=ConstantType(QuantityType(value_encoding=np.dtype('float32'))))
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt.fill_value = -9999
lat_ctxt_id = self.dataset_management.create_parameter_context(name='lat', parameter_context=lat_ctxt.dump(), parameter_type='quantity<float32>', unit_of_measure=lat_ctxt.uom)
lon_ctxt = ParameterContext('lon', param_type=ConstantType(QuantityType(value_encoding=np.dtype('float32'))))
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt.fill_value = -9999
lon_ctxt_id = self.dataset_management.create_parameter_context(name='lon', parameter_context=lon_ctxt.dump(), parameter_type='quantity<float32>', unit_of_measure=lon_ctxt.uom)
temp_ctxt = ParameterContext('TEMPWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')))
temp_ctxt.uom = 'deg_C'
temp_ctxt.fill_value = -9999
temp_ctxt_id = self.dataset_management.create_parameter_context(name='TEMPWAT_L0', parameter_context=temp_ctxt.dump(), parameter_type='quantity<float32>', unit_of_measure=temp_ctxt.uom)
# Conductivity - values expected to be the decimal results of conversion from hex
cond_ctxt = ParameterContext('CONDWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')))
cond_ctxt.uom = 'S m-1'
cond_ctxt.fill_value = -9999
cond_ctxt_id = self.dataset_management.create_parameter_context(name='CONDWAT_L0', parameter_context=cond_ctxt.dump(), parameter_type='quantity<float32>', unit_of_measure=cond_ctxt.uom)
# Pressure - values expected to be the decimal results of conversion from hex
press_ctxt = ParameterContext('PRESWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')))
press_ctxt.uom = 'dbar'
press_ctxt.fill_value = -9999
press_ctxt_id = self.dataset_management.create_parameter_context(name='PRESWAT_L0', parameter_context=press_ctxt.dump(), parameter_type='quantity<float32>', unit_of_measure=press_ctxt.uom)
context_ids = [t_ctxt_id, lat_ctxt_id, lon_ctxt_id, temp_ctxt_id, cond_ctxt_id, press_ctxt_id]
pdict_id = self.dataset_management.create_parameter_dictionary('L0 SBE37', parameter_context_ids=context_ids, temporal_context='time')
return pdict_id
def _create_input_param_dict_for_test(self, parameter_dict_name = ''):
pdict = ParameterDictionary()
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=numpy.dtype('float64')))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = 'seconds since 01-01-1900'
pdict.add_context(t_ctxt)
cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
cond_ctxt.uom = 'Siemens_per_meter'
pdict.add_context(cond_ctxt)
pres_ctxt = ParameterContext('pressure', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
pres_ctxt.uom = 'Pascal'
pdict.add_context(pres_ctxt)
if parameter_dict_name == 'input_param_for_L0':
temp_ctxt = ParameterContext('temperature', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
else:
temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
temp_ctxt.uom = 'degree_kelvin'
pdict.add_context(temp_ctxt)
dens_ctxt = ParameterContext('density', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
dens_ctxt.uom = 'g/m'
pdict.add_context(dens_ctxt)
sal_ctxt = ParameterContext('salinity', param_type=QuantityType(value_encoding=numpy.dtype('float32')))
sal_ctxt.uom = 'PSU'
pdict.add_context(sal_ctxt)
#create temp streamdef so the data product can create the stream
pc_list = []
for pc_k, pc in pdict.iteritems():
ctxt_id = self.dataset_management.create_parameter_context(pc_k, pc[1].dump())
pc_list.append(ctxt_id)
if parameter_dict_name == 'input_param_for_L0':
self.addCleanup(self.dataset_management.delete_parameter_context,ctxt_id)
elif pc[1].name == 'temp':
self.addCleanup(self.dataset_management.delete_parameter_context,ctxt_id)
pdict_id = self.dataset_management.create_parameter_dictionary(parameter_dict_name, pc_list)
self.addCleanup(self.dataset_management.delete_parameter_dictionary, pdict_id)
return pdict_id
def _create_input_param_dict_for_test(self, parameter_dict_name=""):
pdict = ParameterDictionary()
t_ctxt = ParameterContext("time", param_type=QuantityType(value_encoding=numpy.dtype("float64")))
t_ctxt.axis = AxisTypeEnum.TIME
t_ctxt.uom = "seconds since 01-01-1900"
pdict.add_context(t_ctxt)
cond_ctxt = ParameterContext("conductivity", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
cond_ctxt.uom = ""
pdict.add_context(cond_ctxt)
pres_ctxt = ParameterContext("pressure", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
pres_ctxt.uom = ""
pdict.add_context(pres_ctxt)
if parameter_dict_name == "input_param_dict":
temp_ctxt = ParameterContext("temperature", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
else:
temp_ctxt = ParameterContext("temp", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
temp_ctxt.uom = ""
pdict.add_context(temp_ctxt)
dens_ctxt = ParameterContext("density", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
dens_ctxt.uom = ""
pdict.add_context(dens_ctxt)
sal_ctxt = ParameterContext("salinity", param_type=QuantityType(value_encoding=numpy.dtype("float32")))
sal_ctxt.uom = ""
pdict.add_context(sal_ctxt)
# create temp streamdef so the data product can create the stream
pc_list = []
for pc_k, pc in pdict.iteritems():
ctxt_id = self.dataset_management.create_parameter_context(pc_k, pc[1].dump())
pc_list.append(ctxt_id)
if parameter_dict_name == "input_param_dict":
self.addCleanup(self.dataset_management.delete_parameter_context, ctxt_id)
elif parameter_dict_name == "output_param_dict" and pc[1].name == "temp":
self.addCleanup(self.dataset_management.delete_parameter_context, ctxt_id)
pdict_id = self.dataset_management.create_parameter_dictionary(parameter_dict_name, pc_list)
self.addCleanup(self.dataset_management.delete_parameter_dictionary, pdict_id)
return pdict_id
def _get_pdict(self, filter_values):
t_ctxt = ParameterContext('TIME', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 01-01-1900'
t_ctxt_id = self.dataset_management.create_parameter_context(name='TIME', parameter_context=t_ctxt.dump(), parameter_type='quantity<int64>', units=t_ctxt.uom)
self.add_context_to_cleanup(t_ctxt_id)
lat_ctxt = ParameterContext('LAT', param_type=ConstantType(QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999)
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt_id = self.dataset_management.create_parameter_context(name='LAT', parameter_context=lat_ctxt.dump(), parameter_type='quantity<float32>', units=lat_ctxt.uom)
self.add_context_to_cleanup(lat_ctxt_id)
lon_ctxt = ParameterContext('LON', param_type=ConstantType(QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999)
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt_id = self.dataset_management.create_parameter_context(name='LON', parameter_context=lon_ctxt.dump(), parameter_type='quantity<float32>', units=lon_ctxt.uom)
self.add_context_to_cleanup(lon_ctxt_id)
# Independent Parameters
# Temperature - values expected to be the decimal results of conversion from hex
temp_ctxt = ParameterContext('TEMPWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999)
temp_ctxt.uom = 'deg_C'
temp_ctxt_id = self.dataset_management.create_parameter_context(name='TEMPWAT_L0', parameter_context=temp_ctxt.dump(), parameter_type='quantity<float32>', units=temp_ctxt.uom)
self.add_context_to_cleanup(temp_ctxt_id)
# Conductivity - values expected to be the decimal results of conversion from hex
cond_ctxt = ParameterContext('CONDWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999)
cond_ctxt.uom = 'S m-1'
cond_ctxt_id = self.dataset_management.create_parameter_context(name='CONDWAT_L0', parameter_context=cond_ctxt.dump(), parameter_type='quantity<float32>', units=cond_ctxt.uom)
self.add_context_to_cleanup(cond_ctxt_id)
# Pressure - values expected to be the decimal results of conversion from hex
press_ctxt = ParameterContext('PRESWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999)
press_ctxt.uom = 'dbar'
press_ctxt_id = self.dataset_management.create_parameter_context(name='PRESWAT_L0', parameter_context=press_ctxt.dump(), parameter_type='quantity<float32>', units=press_ctxt.uom)
self.add_context_to_cleanup(press_ctxt_id)
# Dependent Parameters
# TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10
tl1_func = '(T / 10000) - 10'
tl1_pmap = {'T': 'TEMPWAT_L0'}
expr = NumexprFunction('TEMPWAT_L1', tl1_func, ['T'], param_map=tl1_pmap)
tempL1_ctxt = ParameterContext('TEMPWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL)
tempL1_ctxt.uom = 'deg_C'
tempL1_ctxt_id = self.dataset_management.create_parameter_context(name=tempL1_ctxt.name, parameter_context=tempL1_ctxt.dump(), parameter_type='pfunc', units=tempL1_ctxt.uom)
self.add_context_to_cleanup(tempL1_ctxt_id)
# CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5
cl1_func = '(C / 100000) - 0.5'
cl1_pmap = {'C': 'CONDWAT_L0'}
expr = NumexprFunction('CONDWAT_L1', cl1_func, ['C'], param_map=cl1_pmap)
condL1_ctxt = ParameterContext('CONDWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL)
condL1_ctxt.uom = 'S m-1'
condL1_ctxt_id = self.dataset_management.create_parameter_context(name=condL1_ctxt.name, parameter_context=condL1_ctxt.dump(), parameter_type='pfunc', units=condL1_ctxt.uom)
self.add_context_to_cleanup(condL1_ctxt_id)
# Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721
# PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range)
pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)'
pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721}
expr = NumexprFunction('PRESWAT_L1', pl1_func, ['P', 'p_range'], param_map=pl1_pmap)
presL1_ctxt = ParameterContext('PRESWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL)
presL1_ctxt.uom = 'S m-1'
presL1_ctxt_id = self.dataset_management.create_parameter_context(name=presL1_ctxt.name, parameter_context=presL1_ctxt.dump(), parameter_type='pfunc', units=presL1_ctxt.uom)
self.add_context_to_cleanup(presL1_ctxt_id)
# Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project:
# https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1
# PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1)
owner = 'gsw'
sal_func = 'SP_from_C'
sal_arglist = ['C', 't', 'p']
sal_pmap = {'C': NumexprFunction('CONDWAT_L1*10', 'C*10', ['C'], param_map={'C': 'CONDWAT_L1'}), 't': 'TEMPWAT_L1', 'p': 'PRESWAT_L1'}
sal_kwargmap = None
expr = PythonFunction('PRACSAL', owner, sal_func, sal_arglist, sal_kwargmap, sal_pmap)
sal_ctxt = ParameterContext('PRACSAL', param_type=ParameterFunctionType(expr), variability=VariabilityEnum.TEMPORAL)
sal_ctxt.uom = 'g kg-1'
sal_ctxt_id = self.dataset_management.create_parameter_context(name=sal_ctxt.name, parameter_context=sal_ctxt.dump(), parameter_type='pfunc', units=sal_ctxt.uom)
self.add_context_to_cleanup(sal_ctxt_id)
# absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude)
# conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1)
# DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1)
owner = 'gsw'
abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP', ['PRACSAL', 'PRESWAT_L1', 'LON','LAT'])
cons_temp_expr = PythonFunction('cons_temp', owner, 'CT_from_t', [abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1'])
dens_expr = PythonFunction('DENSITY', owner, 'rho', [abs_sal_expr, cons_temp_expr, 'PRESWAT_L1'])
dens_ctxt = ParameterContext('DENSITY', param_type=ParameterFunctionType(dens_expr), variability=VariabilityEnum.TEMPORAL)
dens_ctxt.uom = 'kg m-3'
dens_ctxt_id = self.dataset_management.create_parameter_context(name=dens_ctxt.name, parameter_context=dens_ctxt.dump(), parameter_type='pfunc', units=dens_ctxt.uom)
self.add_context_to_cleanup(dens_ctxt_id)
ids = [t_ctxt_id, lat_ctxt_id, lon_ctxt_id, temp_ctxt_id, cond_ctxt_id, press_ctxt_id, tempL1_ctxt_id, condL1_ctxt_id, presL1_ctxt_id, sal_ctxt_id, dens_ctxt_id]
contexts = [t_ctxt, lat_ctxt, lon_ctxt, temp_ctxt, cond_ctxt, press_ctxt, tempL1_ctxt, condL1_ctxt, presL1_ctxt, sal_ctxt, dens_ctxt]
context_ids = [ids[i] for i,ctxt in enumerate(contexts) if ctxt.name in filter_values]
pdict_name = '_'.join([ctxt.name for ctxt in contexts if ctxt.name in filter_values])
try:
self.pdicts[pdict_name]
return self.pdicts[pdict_name]
except KeyError:
pdict_id = self.dataset_management.create_parameter_dictionary(pdict_name, parameter_context_ids=context_ids, temporal_context='time')
self.addCleanup(self.dataset_management.delete_parameter_dictionary, pdict_id)
self.pdicts[pdict_name] = pdict_id
return pdict_id
def create_pfuncs(self):
contexts = {}
funcs = {}
t_ctxt = ParameterContext('TIME', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 1900-01-01'
t_ctxt_id = self.dataset_management.create_parameter_context(name='test_TIME', parameter_context=t_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, t_ctxt_id)
contexts['TIME'] = (t_ctxt, t_ctxt_id)
lat_ctxt = ParameterContext('LAT', param_type=ConstantType(QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999)
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt_id = self.dataset_management.create_parameter_context(name='test_LAT', parameter_context=lat_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, lat_ctxt_id)
contexts['LAT'] = lat_ctxt, lat_ctxt_id
lon_ctxt = ParameterContext('LON', param_type=ConstantType(QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999)
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt_id = self.dataset_management.create_parameter_context(name='test_LON', parameter_context=lon_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, lon_ctxt_id)
contexts['LON'] = lon_ctxt, lon_ctxt_id
# Independent Parameters
# Temperature - values expected to be the decimal results of conversion from hex
temp_ctxt = ParameterContext('TEMPWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999)
temp_ctxt.uom = 'deg_C'
temp_ctxt_id = self.dataset_management.create_parameter_context(name='test_TEMPWAT_L0', parameter_context=temp_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, temp_ctxt_id)
contexts['TEMPWAT_L0'] = temp_ctxt, temp_ctxt_id
# Conductivity - values expected to be the decimal results of conversion from hex
cond_ctxt = ParameterContext('CONDWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999)
cond_ctxt.uom = 'S m-1'
cond_ctxt_id = self.dataset_management.create_parameter_context(name='test_CONDWAT_L0', parameter_context=cond_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, cond_ctxt_id)
contexts['CONDWAT_L0'] = cond_ctxt, cond_ctxt_id
# Pressure - values expected to be the decimal results of conversion from hex
press_ctxt = ParameterContext('PRESWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999)
press_ctxt.uom = 'dbar'
press_ctxt_id = self.dataset_management.create_parameter_context(name='test_PRESWAT_L0', parameter_context=press_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, press_ctxt_id)
contexts['PRESWAT_L0'] = press_ctxt, press_ctxt_id
# Dependent Parameters
# TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10
tl1_func = '(T / 10000) - 10'
expr = NumexprFunction('TEMPWAT_L1', tl1_func, ['T'])
expr_id = self.dataset_management.create_parameter_function(name='test_TEMPWAT_L1', parameter_function=expr.dump())
self.addCleanup(self.dataset_management.delete_parameter_function, expr_id)
funcs['TEMPWAT_L1'] = expr, expr_id
tl1_pmap = {'T': 'TEMPWAT_L0'}
expr.param_map = tl1_pmap
tempL1_ctxt = ParameterContext('TEMPWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL)
tempL1_ctxt.uom = 'deg_C'
tempL1_ctxt_id = self.dataset_management.create_parameter_context(name='test_TEMPWAT_L1', parameter_context=tempL1_ctxt.dump(), parameter_function_id=expr_id)
self.addCleanup(self.dataset_management.delete_parameter_context, tempL1_ctxt_id)
contexts['TEMPWAT_L1'] = tempL1_ctxt, tempL1_ctxt_id
# CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5
cl1_func = '(C / 100000) - 0.5'
expr = NumexprFunction('CONDWAT_L1', cl1_func, ['C'])
expr_id = self.dataset_management.create_parameter_function(name='test_CONDWAT_L1', parameter_function=expr.dump())
self.addCleanup(self.dataset_management.delete_parameter_function, expr_id)
funcs['CONDWAT_L1'] = expr, expr_id
cl1_pmap = {'C': 'CONDWAT_L0'}
expr.param_map = cl1_pmap
condL1_ctxt = ParameterContext('CONDWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL)
condL1_ctxt.uom = 'S m-1'
condL1_ctxt_id = self.dataset_management.create_parameter_context(name='test_CONDWAT_L1', parameter_context=condL1_ctxt.dump(), parameter_function_id=expr_id)
self.addCleanup(self.dataset_management.delete_parameter_context, condL1_ctxt_id)
contexts['CONDWAT_L1'] = condL1_ctxt, condL1_ctxt_id
# Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721
# PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range)
pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)'
expr = NumexprFunction('PRESWAT_L1', pl1_func, ['P', 'p_range'])
expr_id = self.dataset_management.create_parameter_function(name='test_PRESWAT_L1', parameter_function=expr.dump())
self.addCleanup(self.dataset_management.delete_parameter_function, expr_id)
funcs['PRESWAT_L1'] = expr, expr_id
pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721}
expr.param_map = pl1_pmap
presL1_ctxt = ParameterContext('PRESWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL)
presL1_ctxt.uom = 'S m-1'
presL1_ctxt_id = self.dataset_management.create_parameter_context(name='test_CONDWAT_L1', parameter_context=presL1_ctxt.dump(), parameter_function_id=expr_id)
self.addCleanup(self.dataset_management.delete_parameter_context, presL1_ctxt_id)
contexts['PRESWAT_L1'] = presL1_ctxt, presL1_ctxt_id
# Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project:
# https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1
# PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1)
owner = 'gsw'
sal_func = 'SP_from_C'
sal_arglist = ['C', 't', 'p']
expr = PythonFunction('PRACSAL', owner, sal_func, sal_arglist)
expr_id = self.dataset_management.create_parameter_function(name='test_PRACSAL', parameter_function=expr.dump())
self.addCleanup(self.dataset_management.delete_parameter_function, expr_id)
funcs['PRACSAL'] = expr, expr_id
# A magic function that may or may not exist actually forms the line below at runtime.
sal_pmap = {'C': NumexprFunction('CONDWAT_L1*10', 'C*10', ['C'], param_map={'C': 'CONDWAT_L1'}), 't': 'TEMPWAT_L1', 'p': 'PRESWAT_L1'}
expr.param_map = sal_pmap
sal_ctxt = ParameterContext('PRACSAL', param_type=ParameterFunctionType(expr), variability=VariabilityEnum.TEMPORAL)
sal_ctxt.uom = 'g kg-1'
sal_ctxt_id = self.dataset_management.create_parameter_context(name='test_PRACSAL', parameter_context=sal_ctxt.dump(), parameter_function_id=expr_id)
self.addCleanup(self.dataset_management.delete_parameter_context, sal_ctxt_id)
contexts['PRACSAL'] = sal_ctxt, sal_ctxt_id
# absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude)
# conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1)
# DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1)
owner = 'gsw'
abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP', ['PRACSAL', 'PRESWAT_L1', 'LON','LAT'])
cons_temp_expr = PythonFunction('cons_temp', owner, 'CT_from_t', [abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1'])
dens_expr = PythonFunction('DENSITY', owner, 'rho', [abs_sal_expr, cons_temp_expr, 'PRESWAT_L1'])
dens_ctxt = ParameterContext('DENSITY', param_type=ParameterFunctionType(dens_expr), variability=VariabilityEnum.TEMPORAL)
dens_ctxt.uom = 'kg m-3'
dens_ctxt_id = self.dataset_management.create_parameter_context(name='test_DENSITY', parameter_context=dens_ctxt.dump())
self.addCleanup(self.dataset_management.delete_parameter_context, dens_ctxt_id)
contexts['DENSITY'] = dens_ctxt, dens_ctxt_id
return contexts, funcs
def _get_pdict(self, filter_values):
t_ctxt = ParameterContext(
'TIME', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 01-01-1900'
t_ctxt_id = self.dataset_management.create_parameter_context(
name='TIME',
parameter_context=t_ctxt.dump(),
parameter_type='quantity<int64>',
unit_of_measure=t_ctxt.uom)
lat_ctxt = ParameterContext(
'LAT',
param_type=ConstantType(
QuantityType(value_encoding=np.dtype('float32'))),
fill_value=-9999)
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt_id = self.dataset_management.create_parameter_context(
name='LAT',
parameter_context=lat_ctxt.dump(),
parameter_type='quantity<float32>',
unit_of_measure=lat_ctxt.uom)
lon_ctxt = ParameterContext(
'LON',
param_type=ConstantType(
QuantityType(value_encoding=np.dtype('float32'))),
fill_value=-9999)
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt_id = self.dataset_management.create_parameter_context(
name='LON',
parameter_context=lon_ctxt.dump(),
parameter_type='quantity<float32>',
unit_of_measure=lon_ctxt.uom)
# Independent Parameters
# Temperature - values expected to be the decimal results of conversion from hex
temp_ctxt = ParameterContext(
'TEMPWAT_L0',
param_type=QuantityType(value_encoding=np.dtype('float32')),
fill_value=-9999)
temp_ctxt.uom = 'deg_C'
temp_ctxt_id = self.dataset_management.create_parameter_context(
name='TEMPWAT_L0',
parameter_context=temp_ctxt.dump(),
parameter_type='quantity<float32>',
unit_of_measure=temp_ctxt.uom)
# Conductivity - values expected to be the decimal results of conversion from hex
cond_ctxt = ParameterContext(
'CONDWAT_L0',
param_type=QuantityType(value_encoding=np.dtype('float32')),
fill_value=-9999)
cond_ctxt.uom = 'S m-1'
cond_ctxt_id = self.dataset_management.create_parameter_context(
name='CONDWAT_L0',
parameter_context=cond_ctxt.dump(),
parameter_type='quantity<float32>',
unit_of_measure=cond_ctxt.uom)
# Pressure - values expected to be the decimal results of conversion from hex
press_ctxt = ParameterContext(
'PRESWAT_L0',
param_type=QuantityType(value_encoding=np.dtype('float32')),
fill_value=-9999)
press_ctxt.uom = 'dbar'
press_ctxt_id = self.dataset_management.create_parameter_context(
name='PRESWAT_L0',
parameter_context=press_ctxt.dump(),
parameter_type='quantity<float32>',
unit_of_measure=press_ctxt.uom)
# Dependent Parameters
# TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10
tl1_func = '(T / 10000) - 10'
tl1_pmap = {'T': 'TEMPWAT_L0'}
expr = NumexprFunction('TEMPWAT_L1',
tl1_func, ['T'],
param_map=tl1_pmap)
tempL1_ctxt = ParameterContext(
'TEMPWAT_L1',
param_type=ParameterFunctionType(function=expr),
variability=VariabilityEnum.TEMPORAL)
tempL1_ctxt.uom = 'deg_C'
tempL1_ctxt_id = self.dataset_management.create_parameter_context(
name=tempL1_ctxt.name,
parameter_context=tempL1_ctxt.dump(),
parameter_type='pfunc',
unit_of_measure=tempL1_ctxt.uom)
# CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5
cl1_func = '(C / 100000) - 0.5'
cl1_pmap = {'C': 'CONDWAT_L0'}
expr = NumexprFunction('CONDWAT_L1',
cl1_func, ['C'],
param_map=cl1_pmap)
condL1_ctxt = ParameterContext(
'CONDWAT_L1',
param_type=ParameterFunctionType(function=expr),
variability=VariabilityEnum.TEMPORAL)
condL1_ctxt.uom = 'S m-1'
condL1_ctxt_id = self.dataset_management.create_parameter_context(
name=condL1_ctxt.name,
parameter_context=condL1_ctxt.dump(),
parameter_type='pfunc',
unit_of_measure=condL1_ctxt.uom)
# Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721
# PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range)
pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)'
pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721}
expr = NumexprFunction('PRESWAT_L1',
pl1_func, ['P', 'p_range'],
param_map=pl1_pmap)
presL1_ctxt = ParameterContext(
'PRESWAT_L1',
param_type=ParameterFunctionType(function=expr),
variability=VariabilityEnum.TEMPORAL)
presL1_ctxt.uom = 'S m-1'
presL1_ctxt_id = self.dataset_management.create_parameter_context(
name=presL1_ctxt.name,
parameter_context=presL1_ctxt.dump(),
parameter_type='pfunc',
unit_of_measure=presL1_ctxt.uom)
# Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project:
# https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1
# PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1)
owner = 'gsw'
sal_func = 'SP_from_C'
sal_arglist = ['C', 't', 'p']
sal_pmap = {
'C':
NumexprFunction('CONDWAT_L1*10',
'C*10', ['C'],
param_map={'C': 'CONDWAT_L1'}),
't':
'TEMPWAT_L1',
'p':
'PRESWAT_L1'
}
sal_kwargmap = None
expr = PythonFunction('PRACSAL', owner, sal_func, sal_arglist,
sal_kwargmap, sal_pmap)
sal_ctxt = ParameterContext('PRACSAL',
param_type=ParameterFunctionType(expr),
variability=VariabilityEnum.TEMPORAL)
sal_ctxt.uom = 'g kg-1'
sal_ctxt_id = self.dataset_management.create_parameter_context(
name=sal_ctxt.name,
parameter_context=sal_ctxt.dump(),
parameter_type='pfunc',
unit_of_measure=sal_ctxt.uom)
# absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude)
# conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1)
# DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1)
owner = 'gsw'
abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP',
['PRACSAL', 'PRESWAT_L1', 'LON', 'LAT'])
cons_temp_expr = PythonFunction(
'cons_temp', owner, 'CT_from_t',
[abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1'])
dens_expr = PythonFunction(
'DENSITY', owner, 'rho',
[abs_sal_expr, cons_temp_expr, 'PRESWAT_L1'])
dens_ctxt = ParameterContext(
'DENSITY',
param_type=ParameterFunctionType(dens_expr),
variability=VariabilityEnum.TEMPORAL)
dens_ctxt.uom = 'kg m-3'
dens_ctxt_id = self.dataset_management.create_parameter_context(
name=dens_ctxt.name,
parameter_context=dens_ctxt.dump(),
parameter_type='pfunc',
unit_of_measure=dens_ctxt.uom)
ids = [
t_ctxt_id, lat_ctxt_id, lon_ctxt_id, temp_ctxt_id, cond_ctxt_id,
press_ctxt_id, tempL1_ctxt_id, condL1_ctxt_id, presL1_ctxt_id,
sal_ctxt_id, dens_ctxt_id
]
contexts = [
t_ctxt, lat_ctxt, lon_ctxt, temp_ctxt, cond_ctxt, press_ctxt,
tempL1_ctxt, condL1_ctxt, presL1_ctxt, sal_ctxt, dens_ctxt
]
context_ids = [
ids[i] for i, ctxt in enumerate(contexts)
if ctxt.name in filter_values
]
pdict_name = '_'.join(
[ctxt.name for ctxt in contexts if ctxt.name in filter_values])
try:
self.pdicts[pdict_name]
return self.pdicts[pdict_name]
except KeyError:
pdict_id = self.dataset_management.create_parameter_dictionary(
pdict_name,
parameter_context_ids=context_ids,
temporal_context='time')
self.pdicts[pdict_name] = pdict_id
return pdict_id
def _get_pdict(self, filter_values):
t_ctxt = ParameterContext("time", param_type=QuantityType(value_encoding=np.dtype("int64")))
t_ctxt.uom = "seconds since 01-01-1900"
t_ctxt.fill_value = -9999
t_ctxt_id = self.dataset_management.create_parameter_context(
name="time", parameter_context=t_ctxt.dump(), parameter_type="quantity<int64>", unit_of_measure=t_ctxt.uom
)
lat_ctxt = ParameterContext("lat", param_type=ConstantType(QuantityType(value_encoding=np.dtype("float32"))))
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = "degree_north"
lat_ctxt.fill_value = -9999
lat_ctxt_id = self.dataset_management.create_parameter_context(
name="lat",
parameter_context=lat_ctxt.dump(),
parameter_type="quantity<float32>",
unit_of_measure=lat_ctxt.uom,
)
lon_ctxt = ParameterContext("lon", param_type=ConstantType(QuantityType(value_encoding=np.dtype("float32"))))
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = "degree_east"
lon_ctxt.fill_value = -9999
lon_ctxt_id = self.dataset_management.create_parameter_context(
name="lon",
parameter_context=lon_ctxt.dump(),
parameter_type="quantity<float32>",
unit_of_measure=lon_ctxt.uom,
)
temp_ctxt = ParameterContext("TEMPWAT_L0", param_type=QuantityType(value_encoding=np.dtype("float32")))
temp_ctxt.uom = "deg_C"
temp_ctxt.fill_value = -9999
temp_ctxt_id = self.dataset_management.create_parameter_context(
name="TEMPWAT_L0",
parameter_context=temp_ctxt.dump(),
parameter_type="quantity<float32>",
unit_of_measure=temp_ctxt.uom,
)
# Conductivity - values expected to be the decimal results of conversion from hex
cond_ctxt = ParameterContext("CONDWAT_L0", param_type=QuantityType(value_encoding=np.dtype("float32")))
cond_ctxt.uom = "S m-1"
cond_ctxt.fill_value = -9999
cond_ctxt_id = self.dataset_management.create_parameter_context(
name="CONDWAT_L0",
parameter_context=cond_ctxt.dump(),
parameter_type="quantity<float32>",
unit_of_measure=cond_ctxt.uom,
)
# Pressure - values expected to be the decimal results of conversion from hex
press_ctxt = ParameterContext("PRESWAT_L0", param_type=QuantityType(value_encoding=np.dtype("float32")))
press_ctxt.uom = "dbar"
press_ctxt.fill_value = -9999
press_ctxt_id = self.dataset_management.create_parameter_context(
name="PRESWAT_L0",
parameter_context=press_ctxt.dump(),
parameter_type="quantity<float32>",
unit_of_measure=press_ctxt.uom,
)
# TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10
tl1_func = "(TEMPWAT_L0 / 10000) - 10"
tl1_pmap = {"TEMPWAT_L0": "TEMPWAT_L0"}
func = NumexprFunction("TEMPWAT_L1", tl1_func, tl1_pmap)
tempL1_ctxt = ParameterContext(
"TEMPWAT_L1", param_type=ParameterFunctionType(function=func), variability=VariabilityEnum.TEMPORAL
)
tempL1_ctxt.uom = "deg_C"
tempL1_ctxt_id = self.dataset_management.create_parameter_context(
name=tempL1_ctxt.name,
parameter_context=tempL1_ctxt.dump(),
parameter_type="pfunc",
unit_of_measure=tempL1_ctxt.uom,
)
# CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5
cl1_func = "(CONDWAT_L0 / 100000) - 0.5"
cl1_pmap = {"CONDWAT_L0": "CONDWAT_L0"}
func = NumexprFunction("CONDWAT_L1", cl1_func, cl1_pmap)
condL1_ctxt = ParameterContext(
"CONDWAT_L1", param_type=ParameterFunctionType(function=func), variability=VariabilityEnum.TEMPORAL
)
condL1_ctxt.uom = "S m-1"
condL1_ctxt_id = self.dataset_management.create_parameter_context(
name=condL1_ctxt.name,
parameter_context=condL1_ctxt.dump(),
parameter_type="pfunc",
unit_of_measure=condL1_ctxt.uom,
)
# Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721
# PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range)
pl1_func = "(PRESWAT_L0 * 679.34040721 / (0.85 * 65536)) - (0.05 * 679.34040721)"
pl1_pmap = {"PRESWAT_L0": "PRESWAT_L0"}
func = NumexprFunction("PRESWAT_L1", pl1_func, pl1_pmap)
presL1_ctxt = ParameterContext(
"PRESWAT_L1", param_type=ParameterFunctionType(function=func), variability=VariabilityEnum.TEMPORAL
)
presL1_ctxt.uom = "S m-1"
presL1_ctxt_id = self.dataset_management.create_parameter_context(
name=presL1_ctxt.name,
parameter_context=presL1_ctxt.dump(),
parameter_type="pfunc",
unit_of_measure=presL1_ctxt.uom,
)
# Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project:
# https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1
# PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1)
owner = "gsw"
sal_func = "SP_from_C"
sal_arglist = [NumexprFunction("CONDWAT_L1*10", "C*10", {"C": "CONDWAT_L1"}), "TEMPWAT_L1", "PRESWAT_L1"]
sal_kwargmap = None
func = PythonFunction("PRACSAL", owner, sal_func, sal_arglist, sal_kwargmap)
sal_ctxt = ParameterContext(
"PRACSAL", param_type=ParameterFunctionType(func), variability=VariabilityEnum.TEMPORAL
)
sal_ctxt.uom = "g kg-1"
sal_ctxt_id = self.dataset_management.create_parameter_context(
name=sal_ctxt.name, parameter_context=sal_ctxt.dump(), parameter_type="pfunc", unit_of_measure=sal_ctxt.uom
)
# absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude)
# conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1)
# DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1)
owner = "gsw"
abs_sal_func = PythonFunction("abs_sal", owner, "SA_from_SP", ["PRACSAL", "PRESWAT_L1", "lon", "lat"], None)
# abs_sal_func = PythonFunction('abs_sal', owner, 'SA_from_SP', ['lon','lat'], None)
cons_temp_func = PythonFunction(
"cons_temp", owner, "CT_from_t", [abs_sal_func, "TEMPWAT_L1", "PRESWAT_L1"], None
)
dens_func = PythonFunction("DENSITY", owner, "rho", [abs_sal_func, cons_temp_func, "PRESWAT_L1"], None)
dens_ctxt = ParameterContext(
"DENSITY", param_type=ParameterFunctionType(dens_func), variability=VariabilityEnum.TEMPORAL
)
dens_ctxt.uom = "kg m-3"
dens_ctxt_id = self.dataset_management.create_parameter_context(
name=dens_ctxt.name,
parameter_context=dens_ctxt.dump(),
parameter_type="pfunc",
unit_of_measure=dens_ctxt.uom,
)
ids = [
t_ctxt_id,
lat_ctxt_id,
lon_ctxt_id,
temp_ctxt_id,
cond_ctxt_id,
press_ctxt_id,
tempL1_ctxt_id,
condL1_ctxt_id,
presL1_ctxt_id,
sal_ctxt_id,
dens_ctxt_id,
]
contexts = [
t_ctxt,
lat_ctxt,
lon_ctxt,
temp_ctxt,
cond_ctxt,
press_ctxt,
tempL1_ctxt,
condL1_ctxt,
presL1_ctxt,
sal_ctxt,
dens_ctxt,
]
context_ids = [ids[i] for i, ctxt in enumerate(contexts) if ctxt.name in filter_values]
pdict_name = "_".join([ctxt.name for ctxt in contexts if ctxt.name in filter_values])
pdict_id = self.dataset_management.create_parameter_dictionary(
pdict_name, parameter_context_ids=context_ids, temporal_context="time"
)
return pdict_id
def create_pfuncs(self):
contexts = {}
funcs = {}
t_ctxt = ParameterContext(
'TIME', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 01-01-1900'
t_ctxt_id = self.dataset_management.create_parameter_context(
name='test_TIME', parameter_context=t_ctxt.dump())
contexts['TIME'] = (t_ctxt, t_ctxt_id)
lat_ctxt = ParameterContext(
'LAT',
param_type=ConstantType(
QuantityType(value_encoding=np.dtype('float32'))),
fill_value=-9999)
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt_id = self.dataset_management.create_parameter_context(
name='test_LAT', parameter_context=lat_ctxt.dump())
contexts['LAT'] = lat_ctxt, lat_ctxt_id
lon_ctxt = ParameterContext(
'LON',
param_type=ConstantType(
QuantityType(value_encoding=np.dtype('float32'))),
fill_value=-9999)
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt_id = self.dataset_management.create_parameter_context(
name='test_LON', parameter_context=lon_ctxt.dump())
contexts['LON'] = lon_ctxt, lon_ctxt_id
# Independent Parameters
# Temperature - values expected to be the decimal results of conversion from hex
temp_ctxt = ParameterContext(
'TEMPWAT_L0',
param_type=QuantityType(value_encoding=np.dtype('float32')),
fill_value=-9999)
temp_ctxt.uom = 'deg_C'
temp_ctxt_id = self.dataset_management.create_parameter_context(
name='test_TEMPWAT_L0', parameter_context=temp_ctxt.dump())
contexts['TEMPWAT_L0'] = temp_ctxt, temp_ctxt_id
# Conductivity - values expected to be the decimal results of conversion from hex
cond_ctxt = ParameterContext(
'CONDWAT_L0',
param_type=QuantityType(value_encoding=np.dtype('float32')),
fill_value=-9999)
cond_ctxt.uom = 'S m-1'
cond_ctxt_id = self.dataset_management.create_parameter_context(
name='test_CONDWAT_L0', parameter_context=cond_ctxt.dump())
contexts['CONDWAT_L0'] = cond_ctxt, cond_ctxt_id
# Pressure - values expected to be the decimal results of conversion from hex
press_ctxt = ParameterContext(
'PRESWAT_L0',
param_type=QuantityType(value_encoding=np.dtype('float32')),
fill_value=-9999)
press_ctxt.uom = 'dbar'
press_ctxt_id = self.dataset_management.create_parameter_context(
name='test_PRESWAT_L0', parameter_context=press_ctxt.dump())
contexts['PRESWAT_L0'] = press_ctxt, press_ctxt_id
# Dependent Parameters
# TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10
tl1_func = '(T / 10000) - 10'
expr = NumexprFunction('TEMPWAT_L1', tl1_func, ['T'])
expr_id = self.dataset_management.create_parameter_function(
name='test_TEMPWAT_L1', parameter_function=expr.dump())
funcs['TEMPWAT_L1'] = expr, expr_id
tl1_pmap = {'T': 'TEMPWAT_L0'}
expr.param_map = tl1_pmap
tempL1_ctxt = ParameterContext(
'TEMPWAT_L1',
param_type=ParameterFunctionType(function=expr),
variability=VariabilityEnum.TEMPORAL)
tempL1_ctxt.uom = 'deg_C'
tempL1_ctxt_id = self.dataset_management.create_parameter_context(
name='test_TEMPWAT_L1',
parameter_context=tempL1_ctxt.dump(),
parameter_function_id=expr_id)
contexts['TEMPWAT_L1'] = tempL1_ctxt, tempL1_ctxt_id
# CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5
cl1_func = '(C / 100000) - 0.5'
expr = NumexprFunction('CONDWAT_L1', cl1_func, ['C'])
expr_id = self.dataset_management.create_parameter_function(
name='test_CONDWAT_L1', parameter_function=expr.dump())
funcs['CONDWAT_L1'] = expr, expr_id
cl1_pmap = {'C': 'CONDWAT_L0'}
expr.param_map = cl1_pmap
condL1_ctxt = ParameterContext(
'CONDWAT_L1',
param_type=ParameterFunctionType(function=expr),
variability=VariabilityEnum.TEMPORAL)
condL1_ctxt.uom = 'S m-1'
condL1_ctxt_id = self.dataset_management.create_parameter_context(
name='test_CONDWAT_L1',
parameter_context=condL1_ctxt.dump(),
parameter_function_id=expr_id)
contexts['CONDWAT_L1'] = condL1_ctxt, condL1_ctxt_id
# Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721
# PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range)
pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)'
expr = NumexprFunction('PRESWAT_L1', pl1_func, ['P', 'p_range'])
expr_id = self.dataset_management.create_parameter_function(
name='test_PRESWAT_L1', parameter_function=expr.dump())
funcs['PRESWAT_L1'] = expr, expr_id
pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721}
expr.param_map = pl1_pmap
presL1_ctxt = ParameterContext(
'PRESWAT_L1',
param_type=ParameterFunctionType(function=expr),
variability=VariabilityEnum.TEMPORAL)
presL1_ctxt.uom = 'S m-1'
presL1_ctxt_id = self.dataset_management.create_parameter_context(
name='test_CONDWAT_L1',
parameter_context=presL1_ctxt.dump(),
parameter_function_id=expr_id)
contexts['PRESWAT_L1'] = presL1_ctxt, presL1_ctxt_id
# Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project:
# https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1
# PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1)
owner = 'gsw'
sal_func = 'SP_from_C'
sal_arglist = ['C', 't', 'p']
expr = PythonFunction('PRACSAL', owner, sal_func, sal_arglist)
expr_id = self.dataset_management.create_parameter_function(
name='test_PRACSAL', parameter_function=expr.dump())
funcs['PRACSAL'] = expr, expr_id
# A magic function that may or may not exist actually forms the line below at runtime.
sal_pmap = {
'C':
NumexprFunction('CONDWAT_L1*10',
'C*10', ['C'],
param_map={'C': 'CONDWAT_L1'}),
't':
'TEMPWAT_L1',
'p':
'PRESWAT_L1'
}
expr.param_map = sal_pmap
sal_ctxt = ParameterContext('PRACSAL',
param_type=ParameterFunctionType(expr),
variability=VariabilityEnum.TEMPORAL)
sal_ctxt.uom = 'g kg-1'
sal_ctxt_id = self.dataset_management.create_parameter_context(
name='test_PRACSAL',
parameter_context=sal_ctxt.dump(),
parameter_function_id=expr_id)
contexts['PRACSAL'] = sal_ctxt, sal_ctxt_id
# absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude)
# conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1)
# DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1)
owner = 'gsw'
abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP',
['PRACSAL', 'PRESWAT_L1', 'LON', 'LAT'])
cons_temp_expr = PythonFunction(
'cons_temp', owner, 'CT_from_t',
[abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1'])
dens_expr = PythonFunction(
'DENSITY', owner, 'rho',
[abs_sal_expr, cons_temp_expr, 'PRESWAT_L1'])
dens_ctxt = ParameterContext(
'DENSITY',
param_type=ParameterFunctionType(dens_expr),
variability=VariabilityEnum.TEMPORAL)
dens_ctxt.uom = 'kg m-3'
dens_ctxt_id = self.dataset_management.create_parameter_context(
name='test_DENSITY', parameter_context=dens_ctxt.dump())
contexts['DENSITY'] = dens_ctxt, dens_ctxt_id
return contexts, funcs
def create_parsed_params(self):
contexts = {}
funcs = {}
t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('float64')))
t_ctxt.uom = 'seconds since 1900-01-01'
t_ctxt_id = self.dataset_management.create_parameter_context(name='time', parameter_context=t_ctxt.dump())
contexts['time'] = (t_ctxt, t_ctxt_id)
lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999)
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt_id = self.dataset_management.create_parameter_context(name='lat', parameter_context=lat_ctxt.dump())
contexts['lat'] = lat_ctxt, lat_ctxt_id
lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999)
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt_id = self.dataset_management.create_parameter_context(name='lon', parameter_context=lon_ctxt.dump())
contexts['lon'] = lon_ctxt, lon_ctxt_id
# Independent Parameters
# Temperature - values expected to be the decimal results of conversion from hex
temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999)
temp_ctxt.uom = 'deg_C'
temp_ctxt_id = self.dataset_management.create_parameter_context(name='temp', parameter_context=temp_ctxt.dump())
contexts['temp'] = temp_ctxt, temp_ctxt_id
# Conductivity - values expected to be the decimal results of conversion from hex
cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999)
cond_ctxt.uom = 'S m-1'
cond_ctxt_id = self.dataset_management.create_parameter_context(name='conductivity', parameter_context=cond_ctxt.dump())
contexts['conductivity'] = cond_ctxt, cond_ctxt_id
# Pressure - values expected to be the decimal results of conversion from hex
press_ctxt = ParameterContext('pressure', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999)
press_ctxt.uom = 'dbar'
press_ctxt_id = self.dataset_management.create_parameter_context(name='pressure', parameter_context=press_ctxt.dump())
contexts['pressure'] = press_ctxt, press_ctxt_id
preffered_ctxt = ParameterContext('preferred_timestamp', param_type=CategoryType(categories={0:'port_timestamp', 1:'driver_timestamp', 2:'internal_timestamp', 3:'time', -99:'empty'}), fill_value=-99)
preffered_ctxt.uom = ''
preffered_ctxt_id = self.dataset_management.create_parameter_context(name='preferred_timestamp', parameter_context=preffered_ctxt.dump())
contexts['preferred_timestamp'] = preffered_ctxt, preffered_ctxt_id
port_ctxt = ParameterContext('port_timestamp', param_type=QuantityType(value_encoding=np.dtype('float64')), fill_value=-9999)
port_ctxt.uom = 'seconds since 1900-01-01'
port_ctxt_id = self.dataset_management.create_parameter_context(name='port_timestamp', parameter_context=port_ctxt.dump())
contexts['port_timestamp'] = port_ctxt, port_ctxt_id
driver_ctxt = ParameterContext('driver_timestamp', param_type=QuantityType(value_encoding=np.dtype('float64')), fill_value=-9999)
driver_ctxt.uom = 'seconds since 1900-01-01'
driver_ctxt_id = self.dataset_management.create_parameter_context(name='driver_timestamp', parameter_context=driver_ctxt.dump())
contexts['driver_timestamp'] = driver_ctxt, driver_ctxt_id
internal_ctxt = ParameterContext('internal_timestamp', param_type=QuantityType(value_encoding=np.dtype('float64')), fill_value=-9999)
internal_ctxt.uom = 'seconds since 1900-01-01'
internal_ctxt_id = self.dataset_management.create_parameter_context(name='internal_timestamp', parameter_context=internal_ctxt.dump())
contexts['internal_timestamp'] = internal_ctxt, internal_ctxt_id
quality_ctxt = ParameterContext('quality_flag', param_type=ArrayType())
quality_ctxt.uom = ''
quality_ctxt_id = self.dataset_management.create_parameter_context(name='quality_flag', parameter_context=quality_ctxt.dump())
contexts['quality_flag'] = quality_ctxt, quality_ctxt_id
# Dependent Parameters
# TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10
tl1_func = '(temperature / 10000.0) - 10'
expr = NumexprFunction('temp_L1', tl1_func, ['temperature'])
expr_id = self.dataset_management.create_parameter_function(name='temp_L1', parameter_function=expr.dump())
self.addCleanup(self.dataset_management.delete_parameter_function, expr_id)
funcs['temp_L1'] = expr, expr_id
tl1_pmap = {'temperature':'temp'}
expr.param_map = tl1_pmap
tempL1_ctxt = ParameterContext('temp_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL)
tempL1_ctxt.uom = 'deg_C'
tempL1_ctxt_id = self.dataset_management.create_parameter_context(name='temp_L1', parameter_context=tempL1_ctxt.dump(), parameter_function_id=expr_id)
self.addCleanup(self.dataset_management.delete_parameter_context, tempL1_ctxt_id)
contexts['temp_L1'] = tempL1_ctxt, tempL1_ctxt_id
# CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5
cl1_func = '(conductivity / 100000.0) - 0.5'
expr = NumexprFunction('conductivity_L1', cl1_func, ['conductivity'])
expr_id = self.dataset_management.create_parameter_function(name='conductivity_L1', parameter_function=expr.dump())
self.addCleanup(self.dataset_management.delete_parameter_function, expr_id)
funcs['conductivity_L1'] = expr, expr_id
cl1_pmap = {'conductivity':'conductivity'}
expr.param_map = cl1_pmap
condL1_ctxt = ParameterContext('conductivity_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL)
condL1_ctxt.uom = 'S m-1'
condL1_ctxt_id = self.dataset_management.create_parameter_context(name='conductivity_L1', parameter_context=condL1_ctxt.dump(), parameter_function_id=expr_id)
self.addCleanup(self.dataset_management.delete_parameter_context, condL1_ctxt_id)
contexts['conductivity_L1'] = condL1_ctxt, condL1_ctxt_id
# Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721
# PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range)
pl1_func = '(pressure / 100.0) + 0.5'
expr = NumexprFunction('pressure_L1', pl1_func, ['pressure'])
expr_id = self.dataset_management.create_parameter_function(name='pressure_L1', parameter_function=expr.dump())
self.addCleanup(self.dataset_management.delete_parameter_function, expr_id)
funcs['pressure_L1'] = expr, expr_id
pl1_pmap = {'pressure':'pressure'}
expr.param_map = pl1_pmap
presL1_ctxt = ParameterContext('pressure_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL)
presL1_ctxt.uom = 'S m-1'
presL1_ctxt_id = self.dataset_management.create_parameter_context(name='pressure_L1', parameter_context=presL1_ctxt.dump(), parameter_function_id=expr_id)
self.addCleanup(self.dataset_management.delete_parameter_context, presL1_ctxt_id)
contexts['pressure_L1'] = presL1_ctxt, presL1_ctxt_id
# Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project:
# https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1
# PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1)
owner = 'ion_functions.workflow_tests.fake_data'
sal_func = 'data_l2_salinity'
sal_arglist = ['conductivity', 'temp', 'pressure']
expr = PythonFunction('salinity_L2', owner, sal_func, sal_arglist)
expr_id = self.dataset_management.create_parameter_function(name='salinity_L2', parameter_function=expr.dump())
self.addCleanup(self.dataset_management.delete_parameter_function, expr_id)
funcs['salinity_L2'] = expr, expr_id
# A magic function that may or may not exist actually forms the line below at runtime.
sal_pmap = {'conductivity':'conductivity_L1', 'temp':'temp_L1', 'pressure':'pressure_L1'}
expr.param_map = sal_pmap
sal_ctxt = ParameterContext('salinity', param_type=ParameterFunctionType(expr), variability=VariabilityEnum.TEMPORAL)
sal_ctxt.uom = 'g kg-1'
sal_ctxt_id = self.dataset_management.create_parameter_context(name='salinity', parameter_context=sal_ctxt.dump(), parameter_function_id=expr_id)
self.addCleanup(self.dataset_management.delete_parameter_context, sal_ctxt_id)
contexts['salinity'] = sal_ctxt, sal_ctxt_id
# absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude)
# conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1)
# DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1)
owner = 'ion_functions.workflow_tests.fake_data'
dens_func = 'data_l2_density'
dens_arglist =['conductivity', 'temp', 'pressure', 'lat', 'lon']
expr = PythonFunction('density_L2', owner, dens_func, dens_arglist)
expr_id = self.dataset_management.create_parameter_function(name='density_L2', parameter_function=expr.dump())
self.addCleanup(self.dataset_management.delete_parameter_function, expr_id)
funcs['density_L2'] = expr, expr_id
dens_pmap = {'conductivity':'conductivity_L1', 'temp':'temp_L1', 'pressure':'pressure_L1', 'lat':'lat', 'lon':'lon'}
expr.param_map = dens_pmap
dens_ctxt = ParameterContext('density', param_type=ParameterFunctionType(expr), variability=VariabilityEnum.TEMPORAL)
dens_ctxt.uom = 'kg m-3'
dens_ctxt_id = self.dataset_management.create_parameter_context(name='density', parameter_context=dens_ctxt.dump(), parameter_function_id=expr_id)
self.addCleanup(self.dataset_management.delete_parameter_context, dens_ctxt_id)
contexts['density'] = dens_ctxt, dens_ctxt_id
return contexts, funcs
