Exemple #1
0
    def make_grt_qc(self, name, data_product):
        pfunc_id, pfunc = self.find_grt()
        grt_min_id, grt_min_name = self.get_lookup_value(
            'LV_grt_$designator_%s||grt_min_value' % data_product)
        grt_max_id, grt_max_name = self.get_lookup_value(
            'LV_grt_$designator_%s||grt_max_value' % data_product)

        pmap = {'dat': name, 'dat_min': grt_min_name, 'dat_max': grt_max_name}
        pfunc.param_map = pmap
        pfunc.lookup_values = [grt_min_id, grt_max_id]
        dp_name = self.dp_name(data_product)
        pc = ParameterContext(name='%s_glblrng_qc' % dp_name.lower(),
                              param_type=ParameterFunctionType(
                                  pfunc, value_encoding='|i1'))
        pc.uom = '1'
        pc.ooi_short_name = '%s_GLBLRNG_QC' % dp_name
        pc.display_name = '%s Global Range Test Quality Control Flag' % dp_name
        pc.description = "The OOI Global Range quality control algorithm generates a QC flag for the input data point indicating whether it falls within a given range."
        ctxt_id = self.dataset_management.create_parameter_context(
            name='%s_glblrng_qc' % dp_name.lower(),
            parameter_type='function',
            parameter_context=pc.dump(),
            parameter_function_id=pfunc_id,
            ooi_short_name=pc.ooi_short_name,
            units='1',
            value_encoding='int8',
            display_name=pc.display_name,
            description=pc.description)
        return ctxt_id, pc
Exemple #2
0
    def make_propagate_qc(self, inputs):

        pfunc_id, pfunc = self.find_propagate_test()
        pmap = {"strict_validation": False}
        arg_list = ['strict_validation']
        for i, val in enumerate(inputs):
            if i >= 100:
                break
            pmap['array%s' % i] = val
            arg_list.append('array%s' % i)
        pfunc.param_map = pmap
        pfunc.arg_list = arg_list
        pc = ParameterContext(name='cmbnflg_qc',
                              param_type=ParameterFunctionType(
                                  pfunc, value_encoding='|i1'))
        pc.uom = '1'
        pc.ooi_short_name = 'CMBNFLG_QC'
        pc.display_name = 'Combined Data Quality Control Flag'
        pc.description = 'The purpose of this computation is to produce a single merged QC flag from a set of potentially many flags.'
        ctxt_id = self.dataset_management.create_parameter_context(
            name='cmbnflg_qc',
            parameter_type='function',
            parameter_context=pc.dump(),
            parameter_function_id=pfunc_id,
            ooi_short_name=pc.ooi_short_name,
            units='1',
            value_encoding='int8',
            display_name=pc.display_name,
            description=pc.description)
        return ctxt_id, pc
Exemple #3
0
    def make_stuckvalue_qc(self, name, data_product):
        pfunc_id, pfunc = self.find_stuck_value()

        reso_id, reso_name = self.get_lookup_value(
            'LV_svt_$designator_%s||svt_resolution' % data_product)
        n_id, n_name = self.get_lookup_value('LV_svt_$designator_%s||svt_n' %
                                             data_product)

        pmap = {'x': name, 'reso': reso_name, 'num': n_name}
        pfunc.param_map = pmap
        pfunc.lookup_values = [reso_id, n_id]
        dp_name = self.dp_name(data_product)
        pc = ParameterContext(name='%s_stuckvl_qc' % dp_name.lower(),
                              param_type=ParameterFunctionType(
                                  pfunc, value_encoding='|i1'))
        pc.uom = '1'
        pc.ooi_short_name = '%s_STUCKVL_QC' % dp_name
        pc.display_name = '%s Stuck Value Test Quality Control Flag' % dp_name
        pc.description = 'The OOI Stuck Value Test quality control algorithm generates a flag for repeated occurrence of one value in a time series.'

        ctxt_id = self.dataset_management.create_parameter_context(
            name='%s_stuckvl_qc' % dp_name.lower(),
            parameter_type='function',
            parameter_context=pc.dump(),
            parameter_function_id=pfunc_id,
            ooi_short_name=pc.ooi_short_name,
            units='1',
            value_encoding='int8',
            display_name=pc.display_name,
            description=pc.description)
        return ctxt_id, pc
Exemple #4
0
def get_function_type(parameter_type, encoding, pfid, pmap):
    if pfid is None or pmap is None:
        raise TypeError('Function Types require proper function IDs and maps')
    try:
        pmap = ast.literal_eval(pmap)
    except:
        raise TypeError('Invalid Parameter Map Syntax')
    func = evaluate_pmap(pfid, pmap)  # Parse out nested PFIDs and such
    param_type = ParameterFunctionType(func)
    return param_type
Exemple #5
0
    def test_lookup_value_check(self):
        func = NumexprFunction('f',
                               'coeff_a * x', ['coeff_a', 'x'],
                               param_map={
                                   'x': 'x',
                                   'coeff_a': 'coeff_a'
                               })
        func.lookup_values = ['abc123']
        test_context = ParameterContext('test',
                                        param_type=ParameterFunctionType(func))

        tm = TypesManager(None, None, None)
        self.assertTrue(tm.has_lookup_value(test_context))
        self.assertEquals(tm.get_lookup_value_ids(test_context), ['abc123'])
Exemple #6
0
    def make_gradienttest_qc(self, name, data_product):

        pfunc_id, pfunc = self.find_gradient_test()

        ddatdx_id, ddatdx = self.get_lookup_value(
            'LV_grad_$designator_%s_time||d_dat_dx' % data_product)
        mindx_id, mindx = self.get_lookup_value(
            'LV_grad_$designator_%s_time||min_dx' % data_product)
        startdat_id, startdat = self.get_lookup_value(
            'LV_grad_$designator_%s_time||start_dat' % data_product)
        toldat_id, toldat = self.get_lookup_value(
            'LV_grad_$designator_%s_time||tol_dat' % data_product)

        pmap = {
            "dat": name,
            "x": 'time',
            'ddatdx': ddatdx,
            'mindx': mindx,
            'startdat': startdat,
            'toldat': toldat
        }
        pfunc.param_map = pmap
        pfunc.lookup_values = [ddatdx_id, mindx_id, startdat_id, toldat_id]
        dp_name = self.dp_name(data_product)

        pc = ParameterContext(name='%s_gradtst_qc' % dp_name.lower(),
                              param_type=ParameterFunctionType(
                                  pfunc, value_encoding='|i1'))
        pc.uom = '1'
        pc.ooi_short_name = '%s_GRADTST_QC' % dp_name
        pc.display_name = '%s Gradient Test Quality Control Flag' % dp_name
        pc.description = 'The OOI Gradient Test is an automated quality control algorithm used on various OOI data products. This automated algorithm generates flags for data points according to whether changes between successive points are within a pre-determined range.'
        ctxt_id = self.dataset_management.create_parameter_context(
            name='%s_gradtst_qc' % dp_name.lower(),
            parameter_type='function',
            parameter_context=pc.dump(),
            parameter_function_id=pfunc_id,
            ooi_short_name=pc.ooi_short_name,
            units='1',
            value_encoding='int8',
            display_name=pc.display_name,
            description=pc.description)
        return ctxt_id, pc
Exemple #7
0
    def make_localrange_qc(self, name, data_product):

        pfunc_id, pfunc = self.find_localrange_test()

        datlim_id, datlim = self.get_array_lookup_value(
            'LV_lrt_$designator_%s||datlim' % data_product)
        datlimz_id, datlimz = self.get_array_lookup_value(
            'LV_lrt_$designator_%s||datlimz' % data_product)
        dims_id, dims = self.get_string_array_lookup_value(
            'LV_lrt_$designator_%s||dims' % data_product)

        pmap = {
            "dat": name,
            "dims*": dims,
            "datlim*": datlim,
            "datlimz*": datlimz
        }
        pfunc.param_map = pmap
        pfunc.lookup_values = [datlim_id, datlimz_id, dims_id]
        dp_name = self.dp_name(data_product)

        pc = ParameterContext(name='%s_loclrng_qc' % dp_name.lower(),
                              param_type=ParameterFunctionType(
                                  pfunc, value_encoding='|i1'))
        pc.uom = '1'
        pc.ooi_short_name = '%s_LOCLRNG_QC' % dp_name
        pc.display_name = '%s Local Range Test Quality Control Flag' % dp_name
        pc.description = 'The OOI Local Range Test is the computation to test whether a given data point falls within pre-defined ranges.'
        ctxt_id = self.dataset_management.create_parameter_context(
            name='%s_loclrng_qc' % dp_name.lower(),
            parameter_type='function',
            parameter_context=pc.dump(),
            parameter_function_id=pfunc_id,
            ooi_short_name=pc.ooi_short_name,
            units='1',
            value_encoding='int8',
            display_name=pc.display_name,
            description=pc.description)
        return ctxt_id, pc
Exemple #8
0
    def make_spike_qc(self, name, data_product):
        pfunc_id, pfunc = self.find_spike()
        spike_acc_id, spike_acc_name = self.get_lookup_value(
            'LV_spike_$designator_%s||acc' % data_product)
        spike_n_id, spike_n_name = self.get_lookup_value(
            'LV_spike_$designator_%s||spike_n' % data_product)
        spike_l_id, spike_l_name = self.get_lookup_value(
            'LV_spike_$designator_%s||spike_l' % data_product)

        pmap = {
            'dat': name,
            'acc': spike_acc_name,
            'N': spike_n_name,
            'L': spike_l_name
        }
        pfunc.param_map = pmap
        pfunc.lookup_values = [spike_acc_id, spike_n_id, spike_l_id]
        dp_name = self.dp_name(data_product)
        pc = ParameterContext(name='%s_spketst_qc' % dp_name.lower(),
                              param_type=ParameterFunctionType(
                                  pfunc, value_encoding='|i1'))
        pc.uom = '1'
        pc.ooi_short_name = '%s_SPKETST_QC' % dp_name
        pc.display_name = '%s Spike Test Quality Control Flag' % dp_name

        pc.description = "The OOI Spike Test quality control algorithm generates a flag for individual data values that deviate significantly from surrounding data values."

        ctxt_id = self.dataset_management.create_parameter_context(
            name='%s_spketst_qc' % dp_name.lower(),
            parameter_type='function',
            parameter_context=pc.dump(),
            parameter_function_id=pfunc_id,
            ooi_short_name=pc.ooi_short_name,
            units='1',
            value_encoding='int8',
            display_name=pc.display_name,
            description=pc.description)
        return ctxt_id, pc
Exemple #9
0
    def make_trendtest_qc(self, name, data_product):

        pfunc_id, pfunc = self.find_trend_test()

        order_id, order_name = self.get_lookup_value(
            'LV_trend_$designator_%s||polynomial_order' % data_product)
        dev_id, dev_name = self.get_lookup_value(
            'LV_trend_$designator_%s||standard_deviation' % data_product)

        pmap = {
            "dat": name,
            "t": 'time',
            "ord_n": order_name,
            "ntsd": dev_name
        }

        pfunc.param_map = pmap
        pfunc.lookup_values = [order_id, dev_id]
        dp_name = self.dp_name(data_product)
        pc = ParameterContext(name='%s_trndtst_qc' % dp_name.lower(),
                              param_type=ParameterFunctionType(
                                  pfunc, value_encoding='|i1'))
        pc.uom = '1'
        pc.ooi_short_name = '%s_TRNDTST_QC' % dp_name
        pc.display_name = '%s Trend Test Test Quality Control Flag' % dp_name
        pc.description = 'The OOI Trend Test quality control algorithm generates flags on data values within a time series where a significant fraction of the variability in the time series can be explained by a drift, where the drift is assumed to be a polynomial of specified order.'
        ctxt_id = self.dataset_management.create_parameter_context(
            name='%s_trndtst_qc' % dp_name.lower(),
            parameter_type='function',
            parameter_context=pc.dump(),
            parameter_function_id=pfunc_id,
            ooi_short_name=pc.ooi_short_name,
            units='1',
            value_encoding='int8',
            display_name=pc.display_name,
            description=pc.description)
        return ctxt_id, pc
    def create_pfuncs(self):
        
        contexts = {}
        funcs = {}

        t_ctxt = ParameterContext(name='TIME', 
                                  parameter_type='quantity',
                                  value_encoding='float64',
                                  units='seconds since 1900-01-01')
        t_ctxt_id = self.dataset_management.create_parameter(t_ctxt)
        contexts['TIME'] = t_ctxt_id

        lat_ctxt = ParameterContext(name='LAT', 
                                    parameter_type="sparse",
                                    value_encoding='float32',
                                    units='degrees_north')
        lat_ctxt_id = self.dataset_management.create_parameter(lat_ctxt)
        contexts['LAT'] = lat_ctxt_id

        lon_ctxt = ParameterContext(name='LON', 
                                    parameter_type='sparse',
                                    value_encoding='float32',
                                    units='degrees_east')
        lon_ctxt_id = self.dataset_management.create_parameter(lon_ctxt)
        contexts['LON'] = lon_ctxt_id

        # Independent Parameters

        # Temperature - values expected to be the decimal results of conversion from hex
        temp_ctxt = ParameterContext(name='TEMPWAT_L0', 
                parameter_type='quantity',
                value_encoding='float32',
                units='deg_C')
        temp_ctxt_id = self.dataset_management.create_parameter(temp_ctxt)
        contexts['TEMPWAT_L0'] = temp_ctxt_id

        # Conductivity - values expected to be the decimal results of conversion from hex
        cond_ctxt = ParameterContext(name='CONDWAT_L0', 
                parameter_type='quantity',
                value_encoding='float32',
                units='S m-1')
        cond_ctxt_id = self.dataset_management.create_parameter(cond_ctxt)
        contexts['CONDWAT_L0'] = cond_ctxt_id

        # Pressure - values expected to be the decimal results of conversion from hex
        press_ctxt = ParameterContext(name='PRESWAT_L0', 
                parameter_type='quantity',
                value_encoding='float32',
                units='dbar')
        press_ctxt_id = self.dataset_management.create_parameter(press_ctxt)
        contexts['PRESWAT_L0'] = press_ctxt_id


        # Dependent Parameters

        # TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10
        tl1_func = '(T / 10000) - 10'
        expr = ParameterFunction(name='TEMPWAT_L1',
                function_type=PFT.NUMEXPR,
                function=tl1_func,
                args=['T'])
        expr_id = self.dataset_management.create_parameter_function(expr)
        funcs['TEMPWAT_L1'] = expr_id

        tl1_pmap = {'T': 'TEMPWAT_L0'}
        tempL1_ctxt = ParameterContext(name='TEMPWAT_L1', 
                parameter_type='function',
                parameter_function_id=expr_id,
                parameter_function_map=tl1_pmap,
                value_encoding='float32',
                units='deg_C')
        tempL1_ctxt_id = self.dataset_management.create_parameter(tempL1_ctxt)
        contexts['TEMPWAT_L1'] = tempL1_ctxt_id

        # CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5
        cl1_func = '(C / 100000) - 0.5'
        expr = ParameterFunction(name='CONDWAT_L1',
                function_type=PFT.NUMEXPR,
                function=cl1_func,
                args=['C'])
        expr_id = self.dataset_management.create_parameter_function(expr)
        funcs['CONDWAT_L1'] = expr_id

        cl1_pmap = {'C': 'CONDWAT_L0'}
        condL1_ctxt = ParameterContext(name='CONDWAT_L1', 
                parameter_type='function',
                parameter_function_id=expr_id,
                parameter_function_map=cl1_pmap,
                value_encoding='float32',
                units='S m-1')
        condL1_ctxt_id = self.dataset_management.create_parameter(condL1_ctxt)
        contexts['CONDWAT_L1'] = condL1_ctxt_id

        # Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721
        #   PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range)
        pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)'
        expr = ParameterFunction(name='PRESWAT_L1',function=pl1_func,function_type=PFT.NUMEXPR,args=['P','p_range'])
        expr_id = self.dataset_management.create_parameter_function(expr)
        funcs['PRESWAT_L1'] = expr_id
        
        pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721}
        presL1_ctxt = ParameterContext(name='PRESWAT_L1',
                parameter_type='function',
                parameter_function_id=expr_id,
                parameter_function_map=pl1_pmap,
                value_encoding='float32',
                units='S m-1')
        presL1_ctxt_id = self.dataset_management.create_parameter(presL1_ctxt)
        contexts['PRESWAT_L1'] = presL1_ctxt_id

        # Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project:
        #       https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1

        # PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1)
        owner = 'gsw'
        sal_func = 'SP_from_C'
        sal_arglist = ['C', 't', 'p']
        expr = ParameterFunction(name='PRACSAL',function_type=PFT.PYTHON,function=sal_func,owner=owner,args=sal_arglist)
        expr_id = self.dataset_management.create_parameter_function(expr)
        funcs['PRACSAL'] = expr_id
        
        c10_f = ParameterFunction(name='c10', function_type=PFT.NUMEXPR, function='C*10', args=['C'])
        expr_id = self.dataset_management.create_parameter_function(c10_f)
        c10 = ParameterContext(name='c10', 
                parameter_type='function',
                parameter_function_id=expr_id,
                parameter_function_map={'C':'CONDWAT_L1'},
                value_encoding='float32',
                units='1')
        c10_id = self.dataset_management.create_parameter(c10)
        contexts['c10'] = c10_id

        # A magic function that may or may not exist actually forms the line below at runtime.
        sal_pmap = {'C': 'c10', 't': 'TEMPWAT_L1', 'p': 'PRESWAT_L1'}
        sal_ctxt = ParameterContext(name='PRACSAL', 
                parameter_type='function',
                parameter_function_id=expr_id,
                parameter_function_map=sal_pmap,
                value_encoding='float32',
                units='g kg-1')

        sal_ctxt_id = self.dataset_management.create_parameter(sal_ctxt)
        contexts['PRACSAL'] = sal_ctxt_id

        # absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude)
        # conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1)
        # DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1)
        owner = 'gsw'
        abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP', ['PRACSAL', 'PRESWAT_L1', 'LON','LAT'])
        cons_temp_expr = PythonFunction('cons_temp', owner, 'CT_from_t', [abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1'])
        dens_expr = PythonFunction('DENSITY', owner, 'rho', [abs_sal_expr, cons_temp_expr, 'PRESWAT_L1'])
        dens_ctxt = CoverageParameterContext('DENSITY', param_type=ParameterFunctionType(dens_expr), variability=VariabilityEnum.TEMPORAL)
        dens_ctxt.uom = 'kg m-3'
        dens_ctxt_id = self.dataset_management.create_parameter_context(name='DENSITY', parameter_context=dens_ctxt.dump())
        self.addCleanup(self.dataset_management.delete_parameter_context, dens_ctxt_id)
        contexts['DENSITY'] = dens_ctxt_id
        return contexts, funcs
Exemple #11
0
    def _get_pdict(self, filter_values):
        t_ctxt = ParameterContext(
            'TIME', param_type=QuantityType(value_encoding=np.dtype('int64')))
        t_ctxt.uom = 'seconds since 01-01-1900'
        t_ctxt_id = self.dataset_management.create_parameter_context(
            name='TIME',
            parameter_context=t_ctxt.dump(),
            parameter_type='quantity<int64>',
            unit_of_measure=t_ctxt.uom)

        lat_ctxt = ParameterContext(
            'LAT',
            param_type=ConstantType(
                QuantityType(value_encoding=np.dtype('float32'))),
            fill_value=-9999)
        lat_ctxt.axis = AxisTypeEnum.LAT
        lat_ctxt.uom = 'degree_north'
        lat_ctxt_id = self.dataset_management.create_parameter_context(
            name='LAT',
            parameter_context=lat_ctxt.dump(),
            parameter_type='quantity<float32>',
            unit_of_measure=lat_ctxt.uom)

        lon_ctxt = ParameterContext(
            'LON',
            param_type=ConstantType(
                QuantityType(value_encoding=np.dtype('float32'))),
            fill_value=-9999)
        lon_ctxt.axis = AxisTypeEnum.LON
        lon_ctxt.uom = 'degree_east'
        lon_ctxt_id = self.dataset_management.create_parameter_context(
            name='LON',
            parameter_context=lon_ctxt.dump(),
            parameter_type='quantity<float32>',
            unit_of_measure=lon_ctxt.uom)

        # Independent Parameters
        # Temperature - values expected to be the decimal results of conversion from hex
        temp_ctxt = ParameterContext(
            'TEMPWAT_L0',
            param_type=QuantityType(value_encoding=np.dtype('float32')),
            fill_value=-9999)
        temp_ctxt.uom = 'deg_C'
        temp_ctxt_id = self.dataset_management.create_parameter_context(
            name='TEMPWAT_L0',
            parameter_context=temp_ctxt.dump(),
            parameter_type='quantity<float32>',
            unit_of_measure=temp_ctxt.uom)

        # Conductivity - values expected to be the decimal results of conversion from hex
        cond_ctxt = ParameterContext(
            'CONDWAT_L0',
            param_type=QuantityType(value_encoding=np.dtype('float32')),
            fill_value=-9999)
        cond_ctxt.uom = 'S m-1'
        cond_ctxt_id = self.dataset_management.create_parameter_context(
            name='CONDWAT_L0',
            parameter_context=cond_ctxt.dump(),
            parameter_type='quantity<float32>',
            unit_of_measure=cond_ctxt.uom)

        # Pressure - values expected to be the decimal results of conversion from hex
        press_ctxt = ParameterContext(
            'PRESWAT_L0',
            param_type=QuantityType(value_encoding=np.dtype('float32')),
            fill_value=-9999)
        press_ctxt.uom = 'dbar'
        press_ctxt_id = self.dataset_management.create_parameter_context(
            name='PRESWAT_L0',
            parameter_context=press_ctxt.dump(),
            parameter_type='quantity<float32>',
            unit_of_measure=press_ctxt.uom)

        # Dependent Parameters

        # TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10
        tl1_func = '(T / 10000) - 10'
        tl1_pmap = {'T': 'TEMPWAT_L0'}
        expr = NumexprFunction('TEMPWAT_L1',
                               tl1_func, ['T'],
                               param_map=tl1_pmap)
        tempL1_ctxt = ParameterContext(
            'TEMPWAT_L1',
            param_type=ParameterFunctionType(function=expr),
            variability=VariabilityEnum.TEMPORAL)
        tempL1_ctxt.uom = 'deg_C'
        tempL1_ctxt_id = self.dataset_management.create_parameter_context(
            name=tempL1_ctxt.name,
            parameter_context=tempL1_ctxt.dump(),
            parameter_type='pfunc',
            unit_of_measure=tempL1_ctxt.uom)

        # CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5
        cl1_func = '(C / 100000) - 0.5'
        cl1_pmap = {'C': 'CONDWAT_L0'}
        expr = NumexprFunction('CONDWAT_L1',
                               cl1_func, ['C'],
                               param_map=cl1_pmap)
        condL1_ctxt = ParameterContext(
            'CONDWAT_L1',
            param_type=ParameterFunctionType(function=expr),
            variability=VariabilityEnum.TEMPORAL)
        condL1_ctxt.uom = 'S m-1'
        condL1_ctxt_id = self.dataset_management.create_parameter_context(
            name=condL1_ctxt.name,
            parameter_context=condL1_ctxt.dump(),
            parameter_type='pfunc',
            unit_of_measure=condL1_ctxt.uom)

        # Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721
        #   PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range)
        pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)'
        pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721}
        expr = NumexprFunction('PRESWAT_L1',
                               pl1_func, ['P', 'p_range'],
                               param_map=pl1_pmap)
        presL1_ctxt = ParameterContext(
            'PRESWAT_L1',
            param_type=ParameterFunctionType(function=expr),
            variability=VariabilityEnum.TEMPORAL)
        presL1_ctxt.uom = 'S m-1'
        presL1_ctxt_id = self.dataset_management.create_parameter_context(
            name=presL1_ctxt.name,
            parameter_context=presL1_ctxt.dump(),
            parameter_type='pfunc',
            unit_of_measure=presL1_ctxt.uom)

        # Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project:
        #       https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1

        # PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1)
        owner = 'gsw'
        sal_func = 'SP_from_C'
        sal_arglist = ['C', 't', 'p']
        sal_pmap = {
            'C':
            NumexprFunction('CONDWAT_L1*10',
                            'C*10', ['C'],
                            param_map={'C': 'CONDWAT_L1'}),
            't':
            'TEMPWAT_L1',
            'p':
            'PRESWAT_L1'
        }
        sal_kwargmap = None
        expr = PythonFunction('PRACSAL', owner, sal_func, sal_arglist,
                              sal_kwargmap, sal_pmap)
        sal_ctxt = ParameterContext('PRACSAL',
                                    param_type=ParameterFunctionType(expr),
                                    variability=VariabilityEnum.TEMPORAL)
        sal_ctxt.uom = 'g kg-1'
        sal_ctxt_id = self.dataset_management.create_parameter_context(
            name=sal_ctxt.name,
            parameter_context=sal_ctxt.dump(),
            parameter_type='pfunc',
            unit_of_measure=sal_ctxt.uom)

        # absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude)
        # conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1)
        # DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1)
        owner = 'gsw'
        abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP',
                                      ['PRACSAL', 'PRESWAT_L1', 'LON', 'LAT'])
        cons_temp_expr = PythonFunction(
            'cons_temp', owner, 'CT_from_t',
            [abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1'])
        dens_expr = PythonFunction(
            'DENSITY', owner, 'rho',
            [abs_sal_expr, cons_temp_expr, 'PRESWAT_L1'])
        dens_ctxt = ParameterContext(
            'DENSITY',
            param_type=ParameterFunctionType(dens_expr),
            variability=VariabilityEnum.TEMPORAL)
        dens_ctxt.uom = 'kg m-3'
        dens_ctxt_id = self.dataset_management.create_parameter_context(
            name=dens_ctxt.name,
            parameter_context=dens_ctxt.dump(),
            parameter_type='pfunc',
            unit_of_measure=dens_ctxt.uom)

        ids = [
            t_ctxt_id, lat_ctxt_id, lon_ctxt_id, temp_ctxt_id, cond_ctxt_id,
            press_ctxt_id, tempL1_ctxt_id, condL1_ctxt_id, presL1_ctxt_id,
            sal_ctxt_id, dens_ctxt_id
        ]
        contexts = [
            t_ctxt, lat_ctxt, lon_ctxt, temp_ctxt, cond_ctxt, press_ctxt,
            tempL1_ctxt, condL1_ctxt, presL1_ctxt, sal_ctxt, dens_ctxt
        ]
        context_ids = [
            ids[i] for i, ctxt in enumerate(contexts)
            if ctxt.name in filter_values
        ]
        pdict_name = '_'.join(
            [ctxt.name for ctxt in contexts if ctxt.name in filter_values])

        try:
            self.pdicts[pdict_name]
            return self.pdicts[pdict_name]
        except KeyError:
            pdict_id = self.dataset_management.create_parameter_dictionary(
                pdict_name,
                parameter_context_ids=context_ids,
                temporal_context='time')
            self.pdicts[pdict_name] = pdict_id
            return pdict_id
Exemple #12
0
    def create_pfuncs(self):
        contexts = {}
        funcs = {}

        t_ctxt = ParameterContext(
            'TIME', param_type=QuantityType(value_encoding=np.dtype('int64')))
        t_ctxt.uom = 'seconds since 01-01-1900'
        t_ctxt_id = self.dataset_management.create_parameter_context(
            name='test_TIME', parameter_context=t_ctxt.dump())
        contexts['TIME'] = (t_ctxt, t_ctxt_id)

        lat_ctxt = ParameterContext(
            'LAT',
            param_type=ConstantType(
                QuantityType(value_encoding=np.dtype('float32'))),
            fill_value=-9999)
        lat_ctxt.axis = AxisTypeEnum.LAT
        lat_ctxt.uom = 'degree_north'
        lat_ctxt_id = self.dataset_management.create_parameter_context(
            name='test_LAT', parameter_context=lat_ctxt.dump())
        contexts['LAT'] = lat_ctxt, lat_ctxt_id

        lon_ctxt = ParameterContext(
            'LON',
            param_type=ConstantType(
                QuantityType(value_encoding=np.dtype('float32'))),
            fill_value=-9999)
        lon_ctxt.axis = AxisTypeEnum.LON
        lon_ctxt.uom = 'degree_east'
        lon_ctxt_id = self.dataset_management.create_parameter_context(
            name='test_LON', parameter_context=lon_ctxt.dump())
        contexts['LON'] = lon_ctxt, lon_ctxt_id

        # Independent Parameters

        # Temperature - values expected to be the decimal results of conversion from hex
        temp_ctxt = ParameterContext(
            'TEMPWAT_L0',
            param_type=QuantityType(value_encoding=np.dtype('float32')),
            fill_value=-9999)
        temp_ctxt.uom = 'deg_C'
        temp_ctxt_id = self.dataset_management.create_parameter_context(
            name='test_TEMPWAT_L0', parameter_context=temp_ctxt.dump())
        contexts['TEMPWAT_L0'] = temp_ctxt, temp_ctxt_id

        # Conductivity - values expected to be the decimal results of conversion from hex
        cond_ctxt = ParameterContext(
            'CONDWAT_L0',
            param_type=QuantityType(value_encoding=np.dtype('float32')),
            fill_value=-9999)
        cond_ctxt.uom = 'S m-1'
        cond_ctxt_id = self.dataset_management.create_parameter_context(
            name='test_CONDWAT_L0', parameter_context=cond_ctxt.dump())
        contexts['CONDWAT_L0'] = cond_ctxt, cond_ctxt_id

        # Pressure - values expected to be the decimal results of conversion from hex
        press_ctxt = ParameterContext(
            'PRESWAT_L0',
            param_type=QuantityType(value_encoding=np.dtype('float32')),
            fill_value=-9999)
        press_ctxt.uom = 'dbar'
        press_ctxt_id = self.dataset_management.create_parameter_context(
            name='test_PRESWAT_L0', parameter_context=press_ctxt.dump())
        contexts['PRESWAT_L0'] = press_ctxt, press_ctxt_id

        # Dependent Parameters

        # TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10
        tl1_func = '(T / 10000) - 10'
        expr = NumexprFunction('TEMPWAT_L1', tl1_func, ['T'])
        expr_id = self.dataset_management.create_parameter_function(
            name='test_TEMPWAT_L1', parameter_function=expr.dump())
        funcs['TEMPWAT_L1'] = expr, expr_id

        tl1_pmap = {'T': 'TEMPWAT_L0'}
        expr.param_map = tl1_pmap
        tempL1_ctxt = ParameterContext(
            'TEMPWAT_L1',
            param_type=ParameterFunctionType(function=expr),
            variability=VariabilityEnum.TEMPORAL)
        tempL1_ctxt.uom = 'deg_C'
        tempL1_ctxt_id = self.dataset_management.create_parameter_context(
            name='test_TEMPWAT_L1',
            parameter_context=tempL1_ctxt.dump(),
            parameter_function_id=expr_id)
        contexts['TEMPWAT_L1'] = tempL1_ctxt, tempL1_ctxt_id

        # CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5
        cl1_func = '(C / 100000) - 0.5'
        expr = NumexprFunction('CONDWAT_L1', cl1_func, ['C'])
        expr_id = self.dataset_management.create_parameter_function(
            name='test_CONDWAT_L1', parameter_function=expr.dump())
        funcs['CONDWAT_L1'] = expr, expr_id

        cl1_pmap = {'C': 'CONDWAT_L0'}
        expr.param_map = cl1_pmap
        condL1_ctxt = ParameterContext(
            'CONDWAT_L1',
            param_type=ParameterFunctionType(function=expr),
            variability=VariabilityEnum.TEMPORAL)
        condL1_ctxt.uom = 'S m-1'
        condL1_ctxt_id = self.dataset_management.create_parameter_context(
            name='test_CONDWAT_L1',
            parameter_context=condL1_ctxt.dump(),
            parameter_function_id=expr_id)
        contexts['CONDWAT_L1'] = condL1_ctxt, condL1_ctxt_id

        # Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721
        #   PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range)
        pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)'
        expr = NumexprFunction('PRESWAT_L1', pl1_func, ['P', 'p_range'])
        expr_id = self.dataset_management.create_parameter_function(
            name='test_PRESWAT_L1', parameter_function=expr.dump())
        funcs['PRESWAT_L1'] = expr, expr_id

        pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721}
        expr.param_map = pl1_pmap
        presL1_ctxt = ParameterContext(
            'PRESWAT_L1',
            param_type=ParameterFunctionType(function=expr),
            variability=VariabilityEnum.TEMPORAL)
        presL1_ctxt.uom = 'S m-1'
        presL1_ctxt_id = self.dataset_management.create_parameter_context(
            name='test_CONDWAT_L1',
            parameter_context=presL1_ctxt.dump(),
            parameter_function_id=expr_id)
        contexts['PRESWAT_L1'] = presL1_ctxt, presL1_ctxt_id

        # Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project:
        #       https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1

        # PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1)
        owner = 'gsw'
        sal_func = 'SP_from_C'
        sal_arglist = ['C', 't', 'p']
        expr = PythonFunction('PRACSAL', owner, sal_func, sal_arglist)
        expr_id = self.dataset_management.create_parameter_function(
            name='test_PRACSAL', parameter_function=expr.dump())
        funcs['PRACSAL'] = expr, expr_id

        # A magic function that may or may not exist actually forms the line below at runtime.
        sal_pmap = {
            'C':
            NumexprFunction('CONDWAT_L1*10',
                            'C*10', ['C'],
                            param_map={'C': 'CONDWAT_L1'}),
            't':
            'TEMPWAT_L1',
            'p':
            'PRESWAT_L1'
        }
        expr.param_map = sal_pmap
        sal_ctxt = ParameterContext('PRACSAL',
                                    param_type=ParameterFunctionType(expr),
                                    variability=VariabilityEnum.TEMPORAL)
        sal_ctxt.uom = 'g kg-1'
        sal_ctxt_id = self.dataset_management.create_parameter_context(
            name='test_PRACSAL',
            parameter_context=sal_ctxt.dump(),
            parameter_function_id=expr_id)
        contexts['PRACSAL'] = sal_ctxt, sal_ctxt_id

        # absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude)
        # conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1)
        # DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1)
        owner = 'gsw'
        abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP',
                                      ['PRACSAL', 'PRESWAT_L1', 'LON', 'LAT'])
        cons_temp_expr = PythonFunction(
            'cons_temp', owner, 'CT_from_t',
            [abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1'])
        dens_expr = PythonFunction(
            'DENSITY', owner, 'rho',
            [abs_sal_expr, cons_temp_expr, 'PRESWAT_L1'])
        dens_ctxt = ParameterContext(
            'DENSITY',
            param_type=ParameterFunctionType(dens_expr),
            variability=VariabilityEnum.TEMPORAL)
        dens_ctxt.uom = 'kg m-3'
        dens_ctxt_id = self.dataset_management.create_parameter_context(
            name='test_DENSITY', parameter_context=dens_ctxt.dump())
        contexts['DENSITY'] = dens_ctxt, dens_ctxt_id
        return contexts, funcs
Exemple #13
0
 def get_external_type(self, parameter_type, encoding, target_info):
     pfunc = ExternalFunction(name=target_info['name'],
                              external_guid=target_info['target_dataset'],
                              external_name=target_info['target_name'])
     param_type = ParameterFunctionType(pfunc)
     return param_type