def create_lookup_contexts(self): contexts = {} t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('float64'))) t_ctxt.uom = 'seconds since 01-01-1900' t_ctxt_id = self.dataset_management.create_parameter_context(name='time', parameter_context=t_ctxt.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, t_ctxt_id) contexts['time'] = (t_ctxt, t_ctxt_id) temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) temp_ctxt.uom = 'deg_C' temp_ctxt_id = self.dataset_management.create_parameter_context(name='temp', parameter_context=temp_ctxt.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, temp_ctxt_id) contexts['temp'] = temp_ctxt, temp_ctxt_id offset_ctxt = ParameterContext('offset_a', param_type=QuantityType(value_encoding='float32'), fill_value=-9999) offset_ctxt.lookup_value = True offset_ctxt_id = self.dataset_management.create_parameter_context(name='offset_a', parameter_context=offset_ctxt.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, offset_ctxt_id) contexts['offset_a'] = offset_ctxt, offset_ctxt_id func = NumexprFunction('calibrated', 'temp + offset', ['temp','offset'], param_map={'temp':'temp', 'offset':'offset_a'}) func.lookup_values = ['LV_offset'] calibrated = ParameterContext('calibrated', param_type=ParameterFunctionType(func, value_encoding='float32'), fill_value=-9999) calibrated_id = self.dataset_management.create_parameter_context(name='calibrated', parameter_context=calibrated.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, calibrated_id) contexts['calibrated'] = calibrated, calibrated_id return contexts
def create_simple_cc(self): contexts = {} types_manager = TypesManager(self.dataset_management, None, None) t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('float64'))) t_ctxt.uom = 'seconds since 1900-01-01' t_ctxt_id = self.dataset_management.create_parameter_context(name='time', parameter_context=t_ctxt.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, t_ctxt_id) contexts['time'] = t_ctxt, t_ctxt_id temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=fill_value) temp_ctxt.uom = 'deg_C' temp_ctxt.ooi_short_name = 'TEMPWAT' temp_ctxt_id = self.dataset_management.create_parameter_context(name='temp', parameter_context=temp_ctxt.dump(), ooi_short_name='TEMPWAT') self.addCleanup(self.dataset_management.delete_parameter_context, temp_ctxt_id) contexts['temp'] = temp_ctxt, temp_ctxt_id func = NumexprFunction('offset', 'temp + offset', ['temp','offset']) types_manager.get_pfunc = lambda pfid : func func = types_manager.evaluate_pmap('pfid', {'temp':'temp', 'offset':'CC_coefficient'}) func_id = self.dataset_management.create_parameter_function('offset', func.dump()) self.addCleanup(self.dataset_management.delete_parameter_function, func_id) offset_ctxt = ParameterContext('offset', param_type=ParameterFunctionType(func), fill_value=fill_value) offset_ctxt.uom = '1' offset_ctxt_id = self.dataset_management.create_parameter_context('offset', offset_ctxt.dump(), parameter_function_id=func_id) self.addCleanup(self.dataset_management.delete_parameter_context, offset_ctxt_id) contexts['offset'] = offset_ctxt, offset_ctxt_id return contexts
def test_lookup_value_check(self): func = NumexprFunction('f', 'coeff_a * x', ['coeff_a','x'], param_map={'x':'x', 'coeff_a':'coeff_a'}) func.lookup_values = ['abc123'] test_context = ParameterContext('test', param_type=ParameterFunctionType(func)) tm = TypesManager(None,None,None) self.assertTrue(tm.has_lookup_value(test_context)) self.assertEquals(tm.get_lookup_value_ids(test_context), ['abc123'])
def test_lookup_value_check(self): func = NumexprFunction("f", "coeff_a * x", ["coeff_a", "x"], param_map={"x": "x", "coeff_a": "coeff_a"}) func.lookup_values = ["abc123"] test_context = ParameterContext("test", param_type=ParameterFunctionType(func)) tm = TypesManager(None, None, None) self.assertTrue(tm.has_lookup_value(test_context)) self.assertEquals(tm.get_lookup_value_ids(test_context), ["abc123"])
def create_lookup_contexts(self): contexts = {} t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('float64'))) t_ctxt.uom = 'seconds since 1900-01-01' t_ctxt_id = self.dataset_management.create_parameter_context(name='time', parameter_context=t_ctxt.dump()) contexts['time'] = (t_ctxt, t_ctxt_id) temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=fill_value) temp_ctxt.uom = 'deg_C' temp_ctxt_id = self.dataset_management.create_parameter_context(name='temp', parameter_context=temp_ctxt.dump()) contexts['temp'] = temp_ctxt, temp_ctxt_id offset_ctxt = ParameterContext(name='offset_a', param_type=SparseConstantType(base_type=ConstantType(value_encoding='float64'), fill_value=fill_value)) offset_ctxt.uom = '' offset_ctxt.lookup_value = 'offset_a' offset_ctxt.document_key = '' offset_ctxt_id = self.dataset_management.create_parameter_context(name='offset_a', parameter_context=offset_ctxt.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, offset_ctxt_id) contexts['offset_a'] = offset_ctxt, offset_ctxt_id offsetb_ctxt = ParameterContext('offset_b', param_type=SparseConstantType(base_type=ConstantType(value_encoding='float64'), fill_value=fill_value)) offsetb_ctxt.uom = '' offsetb_ctxt.lookup_value = 'offset_b' offsetb_ctxt.document_key = 'coefficient_document' offsetb_ctxt_id = self.dataset_management.create_parameter_context(name='offset_b', parameter_context=offsetb_ctxt.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, offsetb_ctxt_id) contexts['offset_b'] = offsetb_ctxt, offsetb_ctxt_id offsetc_ctxt = ParameterContext('offset_c', param_type=SparseConstantType(base_type=ConstantType(value_encoding='float64'), fill_value=fill_value)) offsetc_ctxt.uom = '' offsetc_ctxt.lookup_value = 'offset_c' offsetc_ctxt.document_key = '$designator_OFFSETC' offsetc_ctxt_id = self.dataset_management.create_parameter_context(name='offset_c', parameter_context=offsetc_ctxt.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, offsetc_ctxt_id) contexts['offset_c'] = offsetc_ctxt, offsetc_ctxt_id func = NumexprFunction('calibrated', 'temp + offset', ['temp','offset'], param_map={'temp':'temp', 'offset':'offset_a'}) func.lookup_values = ['LV_offset'] calibrated = ParameterContext('calibrated', param_type=ParameterFunctionType(func, value_encoding='float32'), fill_value=fill_value) calibrated.uom = 'deg_C' calibrated_id = self.dataset_management.create_parameter_context(name='calibrated', parameter_context=calibrated.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, calibrated_id) contexts['calibrated'] = calibrated, calibrated_id func = NumexprFunction('calibrated_b', 'temp + offset_a + offset_b', ['temp','offset_a', 'offset_b'], param_map={'temp':'temp', 'offset_a':'offset_a', 'offset_b':'offset_b'}) func.lookup_values = ['LV_offset_a', 'LV_offset_b'] calibrated_b = ParameterContext('calibrated_b', param_type=ParameterFunctionType(func, value_encoding='float32'), fill_value=fill_value) calibrated_b.uom = 'deg_C' calibrated_b_id = self.dataset_management.create_parameter_context(name='calibrated_b', parameter_context=calibrated_b.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, calibrated_b_id) contexts['calibrated_b'] = calibrated_b, calibrated_b_id return contexts
def test_lookup_value_check(self): func = NumexprFunction('f', 'coeff_a * x', ['coeff_a', 'x'], param_map={ 'x': 'x', 'coeff_a': 'coeff_a' }) func.lookup_values = ['abc123'] test_context = ParameterContext('test', param_type=ParameterFunctionType(func)) tm = TypesManager(None, None, None) self.assertTrue(tm.has_lookup_value(test_context)) self.assertEquals(tm.get_lookup_value_ids(test_context), ['abc123'])
def create_qc_contexts(self): contexts = {} qc_whatever_ctxt = ParameterContext('qc_whatever', param_type=ArrayType()) qc_whatever_ctxt.uom = '1' qc_whatever_ctxt_id = self.dataset_management.create_parameter_context(name='qc_whatever', parameter_context=qc_whatever_ctxt.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, qc_whatever_ctxt_id) contexts['qc_whatever'] = qc_whatever_ctxt, qc_whatever_ctxt_id nexpr = NumexprFunction('range_qc', 'min < var > max', ['min','max','var']) expr_id = self.dataset_management.create_parameter_function(name='range_qc', parameter_function=nexpr.dump()) self.addCleanup(self.dataset_management.delete_parameter_function, expr_id) pmap = {'min':0, 'max':20, 'var':'temp'} nexpr.param_map = pmap temp_qc_ctxt = ParameterContext('temp_qc', param_type=ParameterFunctionType(function=nexpr), variability=VariabilityEnum.TEMPORAL) temp_qc_ctxt.uom = '1' temp_qc_ctxt_id = self.dataset_management.create_parameter_context(name='temp_qc', parameter_context=temp_qc_ctxt.dump(), parameter_function_id=expr_id) self.addCleanup(self.dataset_management.delete_parameter_context, temp_qc_ctxt_id) contexts['temp_qc'] = temp_qc_ctxt, temp_qc_ctxt_id return contexts
def get_coverage_function(self, parameter_function): func = None if parameter_function.function_type == PFT.PYTHON: func = PythonFunction(name=parameter_function.name, owner=parameter_function.owner, func_name=parameter_function.function, arg_list=parameter_function.args, kwarg_map=None, param_map=None, egg_uri=parameter_function.egg_uri) elif parameter_function.function_type == PFT.NUMEXPR: func = NumexprFunction(name=parameter_function.name, expression=parameter_function.function, arg_list=parameter_function.args) if not isinstance(func, AbstractFunction): raise Conflict("Incompatible parameter function loaded: %s" % parameter_function._id) return func
def create_parsed_params(self): contexts = {} funcs = {} t_ctxt = ParameterContext('time', param_type=QuantityType(value_encoding=np.dtype('float64'))) t_ctxt.uom = 'seconds since 1900-01-01' t_ctxt_id = self.dataset_management.create_parameter_context(name='time', parameter_context=t_ctxt.dump()) contexts['time'] = (t_ctxt, t_ctxt_id) lat_ctxt = ParameterContext('lat', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=fill_value) lat_ctxt.uom = 'degree_north' lat_ctxt_id = self.dataset_management.create_parameter_context(name='lat', parameter_context=lat_ctxt.dump()) contexts['lat'] = lat_ctxt, lat_ctxt_id lon_ctxt = ParameterContext('lon', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=fill_value) lon_ctxt.uom = 'degree_east' lon_ctxt_id = self.dataset_management.create_parameter_context(name='lon', parameter_context=lon_ctxt.dump()) contexts['lon'] = lon_ctxt, lon_ctxt_id # Independent Parameters # Temperature - values expected to be the decimal results of conversion from hex temp_ctxt = ParameterContext('temp', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=fill_value) temp_ctxt.uom = 'deg_C' temp_ctxt_id = self.dataset_management.create_parameter_context(name='temp', parameter_context=temp_ctxt.dump()) contexts['temp'] = temp_ctxt, temp_ctxt_id # Conductivity - values expected to be the decimal results of conversion from hex cond_ctxt = ParameterContext('conductivity', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=fill_value) cond_ctxt.uom = 'S m-1' cond_ctxt_id = self.dataset_management.create_parameter_context(name='conductivity', parameter_context=cond_ctxt.dump()) contexts['conductivity'] = cond_ctxt, cond_ctxt_id # Pressure - values expected to be the decimal results of conversion from hex press_ctxt = ParameterContext('pressure', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=fill_value) press_ctxt.uom = 'dbar' press_ctxt_id = self.dataset_management.create_parameter_context(name='pressure', parameter_context=press_ctxt.dump()) contexts['pressure'] = press_ctxt, press_ctxt_id preffered_ctxt = ParameterContext('preferred_timestamp', param_type=CategoryType(categories={0:'port_timestamp', 1:'driver_timestamp', 2:'internal_timestamp', 3:'time', -99:'empty'}), fill_value=-99) preffered_ctxt.uom = '' preffered_ctxt_id = self.dataset_management.create_parameter_context(name='preferred_timestamp', parameter_context=preffered_ctxt.dump()) contexts['preferred_timestamp'] = preffered_ctxt, preffered_ctxt_id port_ctxt = ParameterContext('port_timestamp', param_type=QuantityType(value_encoding=np.dtype('float64')), fill_value=fill_value) port_ctxt.uom = 'seconds since 1900-01-01' port_ctxt_id = self.dataset_management.create_parameter_context(name='port_timestamp', parameter_context=port_ctxt.dump()) contexts['port_timestamp'] = port_ctxt, port_ctxt_id driver_ctxt = ParameterContext('driver_timestamp', param_type=QuantityType(value_encoding=np.dtype('float64')), fill_value=fill_value) driver_ctxt.uom = 'seconds since 1900-01-01' driver_ctxt_id = self.dataset_management.create_parameter_context(name='driver_timestamp', parameter_context=driver_ctxt.dump()) contexts['driver_timestamp'] = driver_ctxt, driver_ctxt_id internal_ctxt = ParameterContext('internal_timestamp', param_type=QuantityType(value_encoding=np.dtype('float64')), fill_value=fill_value) internal_ctxt.uom = 'seconds since 1900-01-01' internal_ctxt_id = self.dataset_management.create_parameter_context(name='internal_timestamp', parameter_context=internal_ctxt.dump()) contexts['internal_timestamp'] = internal_ctxt, internal_ctxt_id quality_ctxt = ParameterContext('quality_flag', param_type=ArrayType()) quality_ctxt.uom = '' quality_ctxt_id = self.dataset_management.create_parameter_context(name='quality_flag', parameter_context=quality_ctxt.dump()) contexts['quality_flag'] = quality_ctxt, quality_ctxt_id # Dependent Parameters # TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10 tl1_func = '(temperature / 10000.0) - 10' expr = NumexprFunction('temp_L1', tl1_func, ['temperature']) expr_id = self.dataset_management.create_parameter_function(name='temp_L1', parameter_function=expr.dump()) self.addCleanup(self.dataset_management.delete_parameter_function, expr_id) funcs['temp_L1'] = expr, expr_id tl1_pmap = {'temperature':'temp'} expr.param_map = tl1_pmap tempL1_ctxt = ParameterContext('temp_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) tempL1_ctxt.uom = 'deg_C' tempL1_ctxt_id = self.dataset_management.create_parameter_context(name='temp_L1', parameter_context=tempL1_ctxt.dump(), parameter_function_id=expr_id) self.addCleanup(self.dataset_management.delete_parameter_context, tempL1_ctxt_id) contexts['temp_L1'] = tempL1_ctxt, tempL1_ctxt_id # CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5 cl1_func = '(conductivity / 100000.0) - 0.5' expr = NumexprFunction('conductivity_L1', cl1_func, ['conductivity']) expr_id = self.dataset_management.create_parameter_function(name='conductivity_L1', parameter_function=expr.dump()) self.addCleanup(self.dataset_management.delete_parameter_function, expr_id) funcs['conductivity_L1'] = expr, expr_id cl1_pmap = {'conductivity':'conductivity'} expr.param_map = cl1_pmap condL1_ctxt = ParameterContext('conductivity_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) condL1_ctxt.uom = 'S m-1' condL1_ctxt_id = self.dataset_management.create_parameter_context(name='conductivity_L1', parameter_context=condL1_ctxt.dump(), parameter_function_id=expr_id) self.addCleanup(self.dataset_management.delete_parameter_context, condL1_ctxt_id) contexts['conductivity_L1'] = condL1_ctxt, condL1_ctxt_id # Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721 # PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range) pl1_func = '(pressure / 100.0) + 0.5' expr = NumexprFunction('pressure_L1', pl1_func, ['pressure']) expr_id = self.dataset_management.create_parameter_function(name='pressure_L1', parameter_function=expr.dump()) self.addCleanup(self.dataset_management.delete_parameter_function, expr_id) funcs['pressure_L1'] = expr, expr_id pl1_pmap = {'pressure':'pressure'} expr.param_map = pl1_pmap presL1_ctxt = ParameterContext('pressure_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) presL1_ctxt.uom = 'S m-1' presL1_ctxt_id = self.dataset_management.create_parameter_context(name='pressure_L1', parameter_context=presL1_ctxt.dump(), parameter_function_id=expr_id) self.addCleanup(self.dataset_management.delete_parameter_context, presL1_ctxt_id) contexts['pressure_L1'] = presL1_ctxt, presL1_ctxt_id # Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project: # https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1 # PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1) owner = 'ion_functions.workflow_tests.fake_data' sal_func = 'data_l2_salinity' sal_arglist = ['conductivity', 'temp', 'pressure'] expr = PythonFunction('salinity_L2', owner, sal_func, sal_arglist) expr_id = self.dataset_management.create_parameter_function(name='salinity_L2', parameter_function=expr.dump()) self.addCleanup(self.dataset_management.delete_parameter_function, expr_id) funcs['salinity_L2'] = expr, expr_id # A magic function that may or may not exist actually forms the line below at runtime. sal_pmap = {'conductivity':'conductivity_L1', 'temp':'temp_L1', 'pressure':'pressure_L1'} expr.param_map = sal_pmap sal_ctxt = ParameterContext('salinity', param_type=ParameterFunctionType(expr), variability=VariabilityEnum.TEMPORAL) sal_ctxt.uom = 'g kg-1' sal_ctxt_id = self.dataset_management.create_parameter_context(name='salinity', parameter_context=sal_ctxt.dump(), parameter_function_id=expr_id) self.addCleanup(self.dataset_management.delete_parameter_context, sal_ctxt_id) contexts['salinity'] = sal_ctxt, sal_ctxt_id # absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude) # conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1) # DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1) owner = 'ion_functions.workflow_tests.fake_data' dens_func = 'data_l2_density' dens_arglist =['conductivity', 'temp', 'pressure', 'lat', 'lon'] expr = PythonFunction('density_L2', owner, dens_func, dens_arglist) expr_id = self.dataset_management.create_parameter_function(name='density_L2', parameter_function=expr.dump()) self.addCleanup(self.dataset_management.delete_parameter_function, expr_id) funcs['density_L2'] = expr, expr_id dens_pmap = {'conductivity':'conductivity_L1', 'temp':'temp_L1', 'pressure':'pressure_L1', 'lat':'lat', 'lon':'lon'} expr.param_map = dens_pmap dens_ctxt = ParameterContext('density', param_type=ParameterFunctionType(expr), variability=VariabilityEnum.TEMPORAL) dens_ctxt.uom = 'kg m-3' dens_ctxt_id = self.dataset_management.create_parameter_context(name='density', parameter_context=dens_ctxt.dump(), parameter_function_id=expr_id) self.addCleanup(self.dataset_management.delete_parameter_context, dens_ctxt_id) contexts['density'] = dens_ctxt, dens_ctxt_id return contexts, funcs
def create_pfuncs(self): contexts = {} funcs = {} t_ctxt = ParameterContext('TIME', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.uom = 'seconds since 1900-01-01' t_ctxt_id = self.dataset_management.create_parameter_context(name='test_TIME', parameter_context=t_ctxt.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, t_ctxt_id) contexts['TIME'] = (t_ctxt, t_ctxt_id) lat_ctxt = ParameterContext('LAT', param_type=ConstantType(QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' lat_ctxt_id = self.dataset_management.create_parameter_context(name='test_LAT', parameter_context=lat_ctxt.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, lat_ctxt_id) contexts['LAT'] = lat_ctxt, lat_ctxt_id lon_ctxt = ParameterContext('LON', param_type=ConstantType(QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' lon_ctxt_id = self.dataset_management.create_parameter_context(name='test_LON', parameter_context=lon_ctxt.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, lon_ctxt_id) contexts['LON'] = lon_ctxt, lon_ctxt_id # Independent Parameters # Temperature - values expected to be the decimal results of conversion from hex temp_ctxt = ParameterContext('TEMPWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) temp_ctxt.uom = 'deg_C' temp_ctxt_id = self.dataset_management.create_parameter_context(name='test_TEMPWAT_L0', parameter_context=temp_ctxt.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, temp_ctxt_id) contexts['TEMPWAT_L0'] = temp_ctxt, temp_ctxt_id # Conductivity - values expected to be the decimal results of conversion from hex cond_ctxt = ParameterContext('CONDWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) cond_ctxt.uom = 'S m-1' cond_ctxt_id = self.dataset_management.create_parameter_context(name='test_CONDWAT_L0', parameter_context=cond_ctxt.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, cond_ctxt_id) contexts['CONDWAT_L0'] = cond_ctxt, cond_ctxt_id # Pressure - values expected to be the decimal results of conversion from hex press_ctxt = ParameterContext('PRESWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) press_ctxt.uom = 'dbar' press_ctxt_id = self.dataset_management.create_parameter_context(name='test_PRESWAT_L0', parameter_context=press_ctxt.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, press_ctxt_id) contexts['PRESWAT_L0'] = press_ctxt, press_ctxt_id # Dependent Parameters # TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10 tl1_func = '(T / 10000) - 10' expr = NumexprFunction('TEMPWAT_L1', tl1_func, ['T']) expr_id = self.dataset_management.create_parameter_function(name='test_TEMPWAT_L1', parameter_function=expr.dump()) self.addCleanup(self.dataset_management.delete_parameter_function, expr_id) funcs['TEMPWAT_L1'] = expr, expr_id tl1_pmap = {'T': 'TEMPWAT_L0'} expr.param_map = tl1_pmap tempL1_ctxt = ParameterContext('TEMPWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) tempL1_ctxt.uom = 'deg_C' tempL1_ctxt_id = self.dataset_management.create_parameter_context(name='test_TEMPWAT_L1', parameter_context=tempL1_ctxt.dump(), parameter_function_id=expr_id) self.addCleanup(self.dataset_management.delete_parameter_context, tempL1_ctxt_id) contexts['TEMPWAT_L1'] = tempL1_ctxt, tempL1_ctxt_id # CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5 cl1_func = '(C / 100000) - 0.5' expr = NumexprFunction('CONDWAT_L1', cl1_func, ['C']) expr_id = self.dataset_management.create_parameter_function(name='test_CONDWAT_L1', parameter_function=expr.dump()) self.addCleanup(self.dataset_management.delete_parameter_function, expr_id) funcs['CONDWAT_L1'] = expr, expr_id cl1_pmap = {'C': 'CONDWAT_L0'} expr.param_map = cl1_pmap condL1_ctxt = ParameterContext('CONDWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) condL1_ctxt.uom = 'S m-1' condL1_ctxt_id = self.dataset_management.create_parameter_context(name='test_CONDWAT_L1', parameter_context=condL1_ctxt.dump(), parameter_function_id=expr_id) self.addCleanup(self.dataset_management.delete_parameter_context, condL1_ctxt_id) contexts['CONDWAT_L1'] = condL1_ctxt, condL1_ctxt_id # Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721 # PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range) pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)' expr = NumexprFunction('PRESWAT_L1', pl1_func, ['P', 'p_range']) expr_id = self.dataset_management.create_parameter_function(name='test_PRESWAT_L1', parameter_function=expr.dump()) self.addCleanup(self.dataset_management.delete_parameter_function, expr_id) funcs['PRESWAT_L1'] = expr, expr_id pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721} expr.param_map = pl1_pmap presL1_ctxt = ParameterContext('PRESWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) presL1_ctxt.uom = 'S m-1' presL1_ctxt_id = self.dataset_management.create_parameter_context(name='test_CONDWAT_L1', parameter_context=presL1_ctxt.dump(), parameter_function_id=expr_id) self.addCleanup(self.dataset_management.delete_parameter_context, presL1_ctxt_id) contexts['PRESWAT_L1'] = presL1_ctxt, presL1_ctxt_id # Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project: # https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1 # PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1) owner = 'gsw' sal_func = 'SP_from_C' sal_arglist = ['C', 't', 'p'] expr = PythonFunction('PRACSAL', owner, sal_func, sal_arglist) expr_id = self.dataset_management.create_parameter_function(name='test_PRACSAL', parameter_function=expr.dump()) self.addCleanup(self.dataset_management.delete_parameter_function, expr_id) funcs['PRACSAL'] = expr, expr_id # A magic function that may or may not exist actually forms the line below at runtime. sal_pmap = {'C': NumexprFunction('CONDWAT_L1*10', 'C*10', ['C'], param_map={'C': 'CONDWAT_L1'}), 't': 'TEMPWAT_L1', 'p': 'PRESWAT_L1'} expr.param_map = sal_pmap sal_ctxt = ParameterContext('PRACSAL', param_type=ParameterFunctionType(expr), variability=VariabilityEnum.TEMPORAL) sal_ctxt.uom = 'g kg-1' sal_ctxt_id = self.dataset_management.create_parameter_context(name='test_PRACSAL', parameter_context=sal_ctxt.dump(), parameter_function_id=expr_id) self.addCleanup(self.dataset_management.delete_parameter_context, sal_ctxt_id) contexts['PRACSAL'] = sal_ctxt, sal_ctxt_id # absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude) # conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1) # DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1) owner = 'gsw' abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP', ['PRACSAL', 'PRESWAT_L1', 'LON','LAT']) cons_temp_expr = PythonFunction('cons_temp', owner, 'CT_from_t', [abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1']) dens_expr = PythonFunction('DENSITY', owner, 'rho', [abs_sal_expr, cons_temp_expr, 'PRESWAT_L1']) dens_ctxt = ParameterContext('DENSITY', param_type=ParameterFunctionType(dens_expr), variability=VariabilityEnum.TEMPORAL) dens_ctxt.uom = 'kg m-3' dens_ctxt_id = self.dataset_management.create_parameter_context(name='test_DENSITY', parameter_context=dens_ctxt.dump()) self.addCleanup(self.dataset_management.delete_parameter_context, dens_ctxt_id) contexts['DENSITY'] = dens_ctxt, dens_ctxt_id return contexts, funcs
def _get_pdict(self, filter_values): t_ctxt = ParameterContext( 'TIME', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.uom = 'seconds since 01-01-1900' t_ctxt_id = self.dataset_management.create_parameter_context( name='TIME', parameter_context=t_ctxt.dump(), parameter_type='quantity<int64>', unit_of_measure=t_ctxt.uom) lat_ctxt = ParameterContext( 'LAT', param_type=ConstantType( QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' lat_ctxt_id = self.dataset_management.create_parameter_context( name='LAT', parameter_context=lat_ctxt.dump(), parameter_type='quantity<float32>', unit_of_measure=lat_ctxt.uom) lon_ctxt = ParameterContext( 'LON', param_type=ConstantType( QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' lon_ctxt_id = self.dataset_management.create_parameter_context( name='LON', parameter_context=lon_ctxt.dump(), parameter_type='quantity<float32>', unit_of_measure=lon_ctxt.uom) # Independent Parameters # Temperature - values expected to be the decimal results of conversion from hex temp_ctxt = ParameterContext( 'TEMPWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) temp_ctxt.uom = 'deg_C' temp_ctxt_id = self.dataset_management.create_parameter_context( name='TEMPWAT_L0', parameter_context=temp_ctxt.dump(), parameter_type='quantity<float32>', unit_of_measure=temp_ctxt.uom) # Conductivity - values expected to be the decimal results of conversion from hex cond_ctxt = ParameterContext( 'CONDWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) cond_ctxt.uom = 'S m-1' cond_ctxt_id = self.dataset_management.create_parameter_context( name='CONDWAT_L0', parameter_context=cond_ctxt.dump(), parameter_type='quantity<float32>', unit_of_measure=cond_ctxt.uom) # Pressure - values expected to be the decimal results of conversion from hex press_ctxt = ParameterContext( 'PRESWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) press_ctxt.uom = 'dbar' press_ctxt_id = self.dataset_management.create_parameter_context( name='PRESWAT_L0', parameter_context=press_ctxt.dump(), parameter_type='quantity<float32>', unit_of_measure=press_ctxt.uom) # Dependent Parameters # TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10 tl1_func = '(T / 10000) - 10' tl1_pmap = {'T': 'TEMPWAT_L0'} expr = NumexprFunction('TEMPWAT_L1', tl1_func, ['T'], param_map=tl1_pmap) tempL1_ctxt = ParameterContext( 'TEMPWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) tempL1_ctxt.uom = 'deg_C' tempL1_ctxt_id = self.dataset_management.create_parameter_context( name=tempL1_ctxt.name, parameter_context=tempL1_ctxt.dump(), parameter_type='pfunc', unit_of_measure=tempL1_ctxt.uom) # CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5 cl1_func = '(C / 100000) - 0.5' cl1_pmap = {'C': 'CONDWAT_L0'} expr = NumexprFunction('CONDWAT_L1', cl1_func, ['C'], param_map=cl1_pmap) condL1_ctxt = ParameterContext( 'CONDWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) condL1_ctxt.uom = 'S m-1' condL1_ctxt_id = self.dataset_management.create_parameter_context( name=condL1_ctxt.name, parameter_context=condL1_ctxt.dump(), parameter_type='pfunc', unit_of_measure=condL1_ctxt.uom) # Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721 # PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range) pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)' pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721} expr = NumexprFunction('PRESWAT_L1', pl1_func, ['P', 'p_range'], param_map=pl1_pmap) presL1_ctxt = ParameterContext( 'PRESWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) presL1_ctxt.uom = 'S m-1' presL1_ctxt_id = self.dataset_management.create_parameter_context( name=presL1_ctxt.name, parameter_context=presL1_ctxt.dump(), parameter_type='pfunc', unit_of_measure=presL1_ctxt.uom) # Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project: # https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1 # PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1) owner = 'gsw' sal_func = 'SP_from_C' sal_arglist = ['C', 't', 'p'] sal_pmap = { 'C': NumexprFunction('CONDWAT_L1*10', 'C*10', ['C'], param_map={'C': 'CONDWAT_L1'}), 't': 'TEMPWAT_L1', 'p': 'PRESWAT_L1' } sal_kwargmap = None expr = PythonFunction('PRACSAL', owner, sal_func, sal_arglist, sal_kwargmap, sal_pmap) sal_ctxt = ParameterContext('PRACSAL', param_type=ParameterFunctionType(expr), variability=VariabilityEnum.TEMPORAL) sal_ctxt.uom = 'g kg-1' sal_ctxt_id = self.dataset_management.create_parameter_context( name=sal_ctxt.name, parameter_context=sal_ctxt.dump(), parameter_type='pfunc', unit_of_measure=sal_ctxt.uom) # absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude) # conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1) # DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1) owner = 'gsw' abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP', ['PRACSAL', 'PRESWAT_L1', 'LON', 'LAT']) cons_temp_expr = PythonFunction( 'cons_temp', owner, 'CT_from_t', [abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1']) dens_expr = PythonFunction( 'DENSITY', owner, 'rho', [abs_sal_expr, cons_temp_expr, 'PRESWAT_L1']) dens_ctxt = ParameterContext( 'DENSITY', param_type=ParameterFunctionType(dens_expr), variability=VariabilityEnum.TEMPORAL) dens_ctxt.uom = 'kg m-3' dens_ctxt_id = self.dataset_management.create_parameter_context( name=dens_ctxt.name, parameter_context=dens_ctxt.dump(), parameter_type='pfunc', unit_of_measure=dens_ctxt.uom) ids = [ t_ctxt_id, lat_ctxt_id, lon_ctxt_id, temp_ctxt_id, cond_ctxt_id, press_ctxt_id, tempL1_ctxt_id, condL1_ctxt_id, presL1_ctxt_id, sal_ctxt_id, dens_ctxt_id ] contexts = [ t_ctxt, lat_ctxt, lon_ctxt, temp_ctxt, cond_ctxt, press_ctxt, tempL1_ctxt, condL1_ctxt, presL1_ctxt, sal_ctxt, dens_ctxt ] context_ids = [ ids[i] for i, ctxt in enumerate(contexts) if ctxt.name in filter_values ] pdict_name = '_'.join( [ctxt.name for ctxt in contexts if ctxt.name in filter_values]) try: self.pdicts[pdict_name] return self.pdicts[pdict_name] except KeyError: pdict_id = self.dataset_management.create_parameter_dictionary( pdict_name, parameter_context_ids=context_ids, temporal_context='time') self.pdicts[pdict_name] = pdict_id return pdict_id
def create_pfuncs(self): contexts = {} funcs = {} t_ctxt = ParameterContext( 'TIME', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.uom = 'seconds since 01-01-1900' t_ctxt_id = self.dataset_management.create_parameter_context( name='test_TIME', parameter_context=t_ctxt.dump()) contexts['TIME'] = (t_ctxt, t_ctxt_id) lat_ctxt = ParameterContext( 'LAT', param_type=ConstantType( QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' lat_ctxt_id = self.dataset_management.create_parameter_context( name='test_LAT', parameter_context=lat_ctxt.dump()) contexts['LAT'] = lat_ctxt, lat_ctxt_id lon_ctxt = ParameterContext( 'LON', param_type=ConstantType( QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' lon_ctxt_id = self.dataset_management.create_parameter_context( name='test_LON', parameter_context=lon_ctxt.dump()) contexts['LON'] = lon_ctxt, lon_ctxt_id # Independent Parameters # Temperature - values expected to be the decimal results of conversion from hex temp_ctxt = ParameterContext( 'TEMPWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) temp_ctxt.uom = 'deg_C' temp_ctxt_id = self.dataset_management.create_parameter_context( name='test_TEMPWAT_L0', parameter_context=temp_ctxt.dump()) contexts['TEMPWAT_L0'] = temp_ctxt, temp_ctxt_id # Conductivity - values expected to be the decimal results of conversion from hex cond_ctxt = ParameterContext( 'CONDWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) cond_ctxt.uom = 'S m-1' cond_ctxt_id = self.dataset_management.create_parameter_context( name='test_CONDWAT_L0', parameter_context=cond_ctxt.dump()) contexts['CONDWAT_L0'] = cond_ctxt, cond_ctxt_id # Pressure - values expected to be the decimal results of conversion from hex press_ctxt = ParameterContext( 'PRESWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) press_ctxt.uom = 'dbar' press_ctxt_id = self.dataset_management.create_parameter_context( name='test_PRESWAT_L0', parameter_context=press_ctxt.dump()) contexts['PRESWAT_L0'] = press_ctxt, press_ctxt_id # Dependent Parameters # TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10 tl1_func = '(T / 10000) - 10' expr = NumexprFunction('TEMPWAT_L1', tl1_func, ['T']) expr_id = self.dataset_management.create_parameter_function( name='test_TEMPWAT_L1', parameter_function=expr.dump()) funcs['TEMPWAT_L1'] = expr, expr_id tl1_pmap = {'T': 'TEMPWAT_L0'} expr.param_map = tl1_pmap tempL1_ctxt = ParameterContext( 'TEMPWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) tempL1_ctxt.uom = 'deg_C' tempL1_ctxt_id = self.dataset_management.create_parameter_context( name='test_TEMPWAT_L1', parameter_context=tempL1_ctxt.dump(), parameter_function_id=expr_id) contexts['TEMPWAT_L1'] = tempL1_ctxt, tempL1_ctxt_id # CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5 cl1_func = '(C / 100000) - 0.5' expr = NumexprFunction('CONDWAT_L1', cl1_func, ['C']) expr_id = self.dataset_management.create_parameter_function( name='test_CONDWAT_L1', parameter_function=expr.dump()) funcs['CONDWAT_L1'] = expr, expr_id cl1_pmap = {'C': 'CONDWAT_L0'} expr.param_map = cl1_pmap condL1_ctxt = ParameterContext( 'CONDWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) condL1_ctxt.uom = 'S m-1' condL1_ctxt_id = self.dataset_management.create_parameter_context( name='test_CONDWAT_L1', parameter_context=condL1_ctxt.dump(), parameter_function_id=expr_id) contexts['CONDWAT_L1'] = condL1_ctxt, condL1_ctxt_id # Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721 # PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range) pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)' expr = NumexprFunction('PRESWAT_L1', pl1_func, ['P', 'p_range']) expr_id = self.dataset_management.create_parameter_function( name='test_PRESWAT_L1', parameter_function=expr.dump()) funcs['PRESWAT_L1'] = expr, expr_id pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721} expr.param_map = pl1_pmap presL1_ctxt = ParameterContext( 'PRESWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) presL1_ctxt.uom = 'S m-1' presL1_ctxt_id = self.dataset_management.create_parameter_context( name='test_CONDWAT_L1', parameter_context=presL1_ctxt.dump(), parameter_function_id=expr_id) contexts['PRESWAT_L1'] = presL1_ctxt, presL1_ctxt_id # Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project: # https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1 # PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1) owner = 'gsw' sal_func = 'SP_from_C' sal_arglist = ['C', 't', 'p'] expr = PythonFunction('PRACSAL', owner, sal_func, sal_arglist) expr_id = self.dataset_management.create_parameter_function( name='test_PRACSAL', parameter_function=expr.dump()) funcs['PRACSAL'] = expr, expr_id # A magic function that may or may not exist actually forms the line below at runtime. sal_pmap = { 'C': NumexprFunction('CONDWAT_L1*10', 'C*10', ['C'], param_map={'C': 'CONDWAT_L1'}), 't': 'TEMPWAT_L1', 'p': 'PRESWAT_L1' } expr.param_map = sal_pmap sal_ctxt = ParameterContext('PRACSAL', param_type=ParameterFunctionType(expr), variability=VariabilityEnum.TEMPORAL) sal_ctxt.uom = 'g kg-1' sal_ctxt_id = self.dataset_management.create_parameter_context( name='test_PRACSAL', parameter_context=sal_ctxt.dump(), parameter_function_id=expr_id) contexts['PRACSAL'] = sal_ctxt, sal_ctxt_id # absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude) # conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1) # DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1) owner = 'gsw' abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP', ['PRACSAL', 'PRESWAT_L1', 'LON', 'LAT']) cons_temp_expr = PythonFunction( 'cons_temp', owner, 'CT_from_t', [abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1']) dens_expr = PythonFunction( 'DENSITY', owner, 'rho', [abs_sal_expr, cons_temp_expr, 'PRESWAT_L1']) dens_ctxt = ParameterContext( 'DENSITY', param_type=ParameterFunctionType(dens_expr), variability=VariabilityEnum.TEMPORAL) dens_ctxt.uom = 'kg m-3' dens_ctxt_id = self.dataset_management.create_parameter_context( name='test_DENSITY', parameter_context=dens_ctxt.dump()) contexts['DENSITY'] = dens_ctxt, dens_ctxt_id return contexts, funcs