def make_grt_qc(self, name, data_product): pfunc_id, pfunc = self.find_grt() grt_min_id, grt_min_name = self.get_lookup_value( 'LV_grt_$designator_%s||grt_min_value' % data_product) grt_max_id, grt_max_name = self.get_lookup_value( 'LV_grt_$designator_%s||grt_max_value' % data_product) pmap = {'dat': name, 'dat_min': grt_min_name, 'dat_max': grt_max_name} pfunc.param_map = pmap pfunc.lookup_values = [grt_min_id, grt_max_id] dp_name = self.dp_name(data_product) pc = ParameterContext(name='%s_glblrng_qc' % dp_name.lower(), param_type=ParameterFunctionType( pfunc, value_encoding='|i1')) pc.uom = '1' pc.ooi_short_name = '%s_GLBLRNG_QC' % dp_name pc.display_name = '%s Global Range Test Quality Control Flag' % dp_name pc.description = "The OOI Global Range quality control algorithm generates a QC flag for the input data point indicating whether it falls within a given range." ctxt_id = self.dataset_management.create_parameter_context( name='%s_glblrng_qc' % dp_name.lower(), parameter_type='function', parameter_context=pc.dump(), parameter_function_id=pfunc_id, ooi_short_name=pc.ooi_short_name, units='1', value_encoding='int8', display_name=pc.display_name, description=pc.description) return ctxt_id, pc
def get_lookup_value(self, value): return self._placeholder( value, lambda placeholder: ParameterContext( name=placeholder, param_type=SparseConstantType(base_type=ConstantType( value_encoding='float64'), fill_value=-9999.)))
def make_stuckvalue_qc(self, name, data_product): pfunc_id, pfunc = self.find_stuck_value() reso_id, reso_name = self.get_lookup_value( 'LV_svt_$designator_%s||svt_resolution' % data_product) n_id, n_name = self.get_lookup_value('LV_svt_$designator_%s||svt_n' % data_product) pmap = {'x': name, 'reso': reso_name, 'num': n_name} pfunc.param_map = pmap pfunc.lookup_values = [reso_id, n_id] dp_name = self.dp_name(data_product) pc = ParameterContext(name='%s_stuckvl_qc' % dp_name.lower(), param_type=ParameterFunctionType( pfunc, value_encoding='|i1')) pc.uom = '1' pc.ooi_short_name = '%s_STUCKVL_QC' % dp_name pc.display_name = '%s Stuck Value Test Quality Control Flag' % dp_name pc.description = 'The OOI Stuck Value Test quality control algorithm generates a flag for repeated occurrence of one value in a time series.' ctxt_id = self.dataset_management.create_parameter_context( name='%s_stuckvl_qc' % dp_name.lower(), parameter_type='function', parameter_context=pc.dump(), parameter_function_id=pfunc_id, ooi_short_name=pc.ooi_short_name, units='1', value_encoding='int8', display_name=pc.display_name, description=pc.description) return ctxt_id, pc
def make_propagate_qc(self, inputs): pfunc_id, pfunc = self.find_propagate_test() pmap = {"strict_validation": False} arg_list = ['strict_validation'] for i, val in enumerate(inputs): if i >= 100: break pmap['array%s' % i] = val arg_list.append('array%s' % i) pfunc.param_map = pmap pfunc.arg_list = arg_list pc = ParameterContext(name='cmbnflg_qc', param_type=ParameterFunctionType( pfunc, value_encoding='|i1')) pc.uom = '1' pc.ooi_short_name = 'CMBNFLG_QC' pc.display_name = 'Combined Data Quality Control Flag' pc.description = 'The purpose of this computation is to produce a single merged QC flag from a set of potentially many flags.' ctxt_id = self.dataset_management.create_parameter_context( name='cmbnflg_qc', parameter_type='function', parameter_context=pc.dump(), parameter_function_id=pfunc_id, ooi_short_name=pc.ooi_short_name, units='1', value_encoding='int8', display_name=pc.display_name, description=pc.description) return ctxt_id, pc
def make_manual_upload_data_product(self): # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) context_ids = self.dataset_management.read_parameter_contexts( pdict_id, id_only=True) # Add a handful of Calibration Coefficient parameters for cc in ['temp_hitl_qc', 'cond_hitl_qc']: c = ParameterContext(cc, param_type=BooleanType()) c.uom = '1' context_ids.append( self.dataset_management.create_parameter_context(cc, c.dump())) pdict_id = self.dataset_management.create_parameter_dictionary( 'manup_dict', context_ids, temporal_context='time') data_product_id = self.create_data_product('manup_dp', pdict_id=pdict_id) self.activate_data_product(data_product_id) dataset_id, _ = self.resource_registry.find_objects(data_product_id, PRED.hasDataset, id_only=True) return data_product_id, dataset_id[0]
def make_cal_data_product(self): # Get a precompiled parameter dictionary with basic ctd fields pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) context_ids = self.dataset_management.read_parameter_contexts( pdict_id, id_only=True) # Add a handful of Calibration Coefficient parameters for cc in ['cc_ta0', 'cc_ta1', 'cc_ta2', 'cc_ta3', 'cc_toffset']: c = ParameterContext(cc, param_type=SparseConstantType( value_encoding='float32', fill_value=-9999)) c.uom = '1' context_ids.append( self.dataset_management.create_parameter_context(cc, c.dump())) pdict_id = self.dataset_management.create_parameter_dictionary( 'calcoeff_dict', context_ids, temporal_context='time') data_product_id = self.create_data_product('calcoeff_dp', pdict_id=pdict_id) self.activate_data_product(data_product_id) dataset_id, _ = self.resource_registry.find_objects(data_product_id, PRED.hasDataset, id_only=True) return data_product_id, dataset_id[0]
def create_contexts(self): context_ids = [] cond_ctxt = ParameterContext( 'conductivity_test', param_type=QuantityType(value_encoding=np.float32)) cond_ctxt.uom = 'unknown' cond_ctxt.fill_value = 0e0 context_ids.append( self.dataset_management.create_parameter_context( name='conductivity_test', parameter_context=cond_ctxt.dump())) pres_ctxt = ParameterContext( 'pressure_test', param_type=QuantityType(value_encoding=np.float32)) pres_ctxt.uom = 'Pascal' pres_ctxt.fill_value = 0x0 context_ids.append( self.dataset_management.create_parameter_context( name='pressure_test', parameter_context=pres_ctxt.dump())) sal_ctxt = ParameterContext( 'salinity_test', param_type=QuantityType(value_encoding=np.float32)) sal_ctxt.uom = 'PSU' sal_ctxt.fill_value = 0x0 context_ids.append( self.dataset_management.create_parameter_context( name='salinity_test', parameter_context=sal_ctxt.dump())) temp_ctxt = ParameterContext( 'temp_test', param_type=QuantityType(value_encoding=np.float32)) temp_ctxt.uom = 'degree_Celsius' temp_ctxt.fill_value = 0e0 context_ids.append( self.dataset_management.create_parameter_context( name='temp_test', parameter_context=temp_ctxt.dump())) t_ctxt = ParameterContext( 'time_test', param_type=QuantityType(value_encoding=np.int64)) t_ctxt.uom = 'seconds since 1970-01-01' t_ctxt.fill_value = 0x0 context_ids.append( self.dataset_management.create_parameter_context( name='time_test', parameter_context=t_ctxt.dump())) return context_ids
def _create_input_param_dict_for_test(self, parameter_dict_name=''): pdict = ParameterDictionary() t_ctxt = ParameterContext( 'time', param_type=QuantityType(value_encoding=numpy.dtype('float64'))) t_ctxt.axis = AxisTypeEnum.TIME t_ctxt.uom = 'seconds since 01-01-1900' pdict.add_context(t_ctxt) cond_ctxt = ParameterContext( 'conductivity', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) cond_ctxt.uom = '' pdict.add_context(cond_ctxt) pres_ctxt = ParameterContext( 'pressure', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) pres_ctxt.uom = '' pdict.add_context(pres_ctxt) temp_ctxt = ParameterContext( 'temperature', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) temp_ctxt.uom = '' pdict.add_context(temp_ctxt) dens_ctxt = ParameterContext( 'density', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) dens_ctxt.uom = '' pdict.add_context(dens_ctxt) sal_ctxt = ParameterContext( 'salinity', param_type=QuantityType(value_encoding=numpy.dtype('float32'))) sal_ctxt.uom = '' pdict.add_context(sal_ctxt) #create temp streamdef so the data product can create the stream pc_list = [] for pc_k, pc in pdict.iteritems(): ctxt_id = self.dataset_management.create_parameter_context( pc_k, pc[1].dump()) pc_list.append(ctxt_id) self.addCleanup(self.dataset_management.delete_parameter_context, ctxt_id) pdict_id = self.dataset_management.create_parameter_dictionary( parameter_dict_name, pc_list) self.addCleanup(self.dataset_management.delete_parameter_dictionary, pdict_id) return pdict_id
def __init__(self, total_domain=(10, 10), brick_size=5, use_hdf=False, root_dir='test_data/multi_dim_trials', guid=None, dtype='int16'): self.total_domain = total_domain self.brick_sizes = tuple(brick_size for x in total_domain) self.use_hdf = use_hdf self.dtype = np.dtype(dtype).name if self.use_hdf: self.guid = guid or create_guid() name = '%s_%s' % (self.guid, self.dtype) self.root_dir = root_dir if not os.path.exists(self.root_dir): os.makedirs(self.root_dir) if os.path.exists(os.path.join(self.root_dir, name)): shutil.rmtree(os.path.join(self.root_dir, name)) self.master_manager = MasterManager( self.root_dir, name, name='md_test_{0}'.format(name)) self.master_manager.flush() pc = ParameterContext('test_param', param_type=QuantityType(self.dtype), fill_value=-1) self.param_manager = ParameterManager( os.path.join(self.root_dir, name, pc.name), pc.name) self.param_manager.parameter_context = pc self.master_manager.create_group(pc.name) self.param_manager.flush() self.bricks = {} self.brick_origins = bricking_utils.calc_brick_origins( self.total_domain, self.brick_sizes) self.brick_extents, self.rtree_extents = bricking_utils.calc_brick_and_rtree_extents( self.brick_origins, self.brick_sizes) self.build_bricks() self.rtree = RTreeProxy() for x in BrickingAssessor.rtree_populator(self.rtree_extents, self.brick_extents): self.rtree.insert(*x)
def get_cc_value(self, value): placeholder = value.lower() # Check to see if this coefficient exists already hits, _ = Container.instance.resource_registry.find_resources( name=placeholder, restype=RT.ParameterContext, id_only=True) if hits: return hits[0], placeholder pc = ParameterContext( name=placeholder, param_type=SparseConstantType(value_encoding='float64'), fill_value=-9999.) pc.uom = '1' ctxt_id = self.dataset_management.create_parameter_context( name=placeholder, parameter_context=pc.dump()) return ctxt_id, placeholder
def make_gradienttest_qc(self, name, data_product): pfunc_id, pfunc = self.find_gradient_test() ddatdx_id, ddatdx = self.get_lookup_value( 'LV_grad_$designator_%s_time||d_dat_dx' % data_product) mindx_id, mindx = self.get_lookup_value( 'LV_grad_$designator_%s_time||min_dx' % data_product) startdat_id, startdat = self.get_lookup_value( 'LV_grad_$designator_%s_time||start_dat' % data_product) toldat_id, toldat = self.get_lookup_value( 'LV_grad_$designator_%s_time||tol_dat' % data_product) pmap = { "dat": name, "x": 'time', 'ddatdx': ddatdx, 'mindx': mindx, 'startdat': startdat, 'toldat': toldat } pfunc.param_map = pmap pfunc.lookup_values = [ddatdx_id, mindx_id, startdat_id, toldat_id] dp_name = self.dp_name(data_product) pc = ParameterContext(name='%s_gradtst_qc' % dp_name.lower(), param_type=ParameterFunctionType( pfunc, value_encoding='|i1')) pc.uom = '1' pc.ooi_short_name = '%s_GRADTST_QC' % dp_name pc.display_name = '%s Gradient Test Quality Control Flag' % dp_name pc.description = 'The OOI Gradient Test is an automated quality control algorithm used on various OOI data products. This automated algorithm generates flags for data points according to whether changes between successive points are within a pre-determined range.' ctxt_id = self.dataset_management.create_parameter_context( name='%s_gradtst_qc' % dp_name.lower(), parameter_type='function', parameter_context=pc.dump(), parameter_function_id=pfunc_id, ooi_short_name=pc.ooi_short_name, units='1', value_encoding='int8', display_name=pc.display_name, description=pc.description) return ctxt_id, pc
def make_localrange_qc(self, name, data_product): pfunc_id, pfunc = self.find_localrange_test() datlim_id, datlim = self.get_array_lookup_value( 'LV_lrt_$designator_%s||datlim' % data_product) datlimz_id, datlimz = self.get_array_lookup_value( 'LV_lrt_$designator_%s||datlimz' % data_product) dims_id, dims = self.get_string_array_lookup_value( 'LV_lrt_$designator_%s||dims' % data_product) pmap = { "dat": name, "dims*": dims, "datlim*": datlim, "datlimz*": datlimz } pfunc.param_map = pmap pfunc.lookup_values = [datlim_id, datlimz_id, dims_id] dp_name = self.dp_name(data_product) pc = ParameterContext(name='%s_loclrng_qc' % dp_name.lower(), param_type=ParameterFunctionType( pfunc, value_encoding='|i1')) pc.uom = '1' pc.ooi_short_name = '%s_LOCLRNG_QC' % dp_name pc.display_name = '%s Local Range Test Quality Control Flag' % dp_name pc.description = 'The OOI Local Range Test is the computation to test whether a given data point falls within pre-defined ranges.' ctxt_id = self.dataset_management.create_parameter_context( name='%s_loclrng_qc' % dp_name.lower(), parameter_type='function', parameter_context=pc.dump(), parameter_function_id=pfunc_id, ooi_short_name=pc.ooi_short_name, units='1', value_encoding='int8', display_name=pc.display_name, description=pc.description) return ctxt_id, pc
def make_spike_qc(self, name, data_product): pfunc_id, pfunc = self.find_spike() spike_acc_id, spike_acc_name = self.get_lookup_value( 'LV_spike_$designator_%s||acc' % data_product) spike_n_id, spike_n_name = self.get_lookup_value( 'LV_spike_$designator_%s||spike_n' % data_product) spike_l_id, spike_l_name = self.get_lookup_value( 'LV_spike_$designator_%s||spike_l' % data_product) pmap = { 'dat': name, 'acc': spike_acc_name, 'N': spike_n_name, 'L': spike_l_name } pfunc.param_map = pmap pfunc.lookup_values = [spike_acc_id, spike_n_id, spike_l_id] dp_name = self.dp_name(data_product) pc = ParameterContext(name='%s_spketst_qc' % dp_name.lower(), param_type=ParameterFunctionType( pfunc, value_encoding='|i1')) pc.uom = '1' pc.ooi_short_name = '%s_SPKETST_QC' % dp_name pc.display_name = '%s Spike Test Quality Control Flag' % dp_name pc.description = "The OOI Spike Test quality control algorithm generates a flag for individual data values that deviate significantly from surrounding data values." ctxt_id = self.dataset_management.create_parameter_context( name='%s_spketst_qc' % dp_name.lower(), parameter_type='function', parameter_context=pc.dump(), parameter_function_id=pfunc_id, ooi_short_name=pc.ooi_short_name, units='1', value_encoding='int8', display_name=pc.display_name, description=pc.description) return ctxt_id, pc
def make_trendtest_qc(self, name, data_product): pfunc_id, pfunc = self.find_trend_test() order_id, order_name = self.get_lookup_value( 'LV_trend_$designator_%s||polynomial_order' % data_product) dev_id, dev_name = self.get_lookup_value( 'LV_trend_$designator_%s||standard_deviation' % data_product) pmap = { "dat": name, "t": 'time', "ord_n": order_name, "ntsd": dev_name } pfunc.param_map = pmap pfunc.lookup_values = [order_id, dev_id] dp_name = self.dp_name(data_product) pc = ParameterContext(name='%s_trndtst_qc' % dp_name.lower(), param_type=ParameterFunctionType( pfunc, value_encoding='|i1')) pc.uom = '1' pc.ooi_short_name = '%s_TRNDTST_QC' % dp_name pc.display_name = '%s Trend Test Test Quality Control Flag' % dp_name pc.description = 'The OOI Trend Test quality control algorithm generates flags on data values within a time series where a significant fraction of the variability in the time series can be explained by a drift, where the drift is assumed to be a polynomial of specified order.' ctxt_id = self.dataset_management.create_parameter_context( name='%s_trndtst_qc' % dp_name.lower(), parameter_type='function', parameter_context=pc.dump(), parameter_function_id=pfunc_id, ooi_short_name=pc.ooi_short_name, units='1', value_encoding='int8', display_name=pc.display_name, description=pc.description) return ctxt_id, pc
def create_pfuncs(self): contexts = {} funcs = {} t_ctxt = ParameterContext( 'TIME', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.uom = 'seconds since 01-01-1900' t_ctxt_id = self.dataset_management.create_parameter_context( name='test_TIME', parameter_context=t_ctxt.dump()) contexts['TIME'] = (t_ctxt, t_ctxt_id) lat_ctxt = ParameterContext( 'LAT', param_type=ConstantType( QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' lat_ctxt_id = self.dataset_management.create_parameter_context( name='test_LAT', parameter_context=lat_ctxt.dump()) contexts['LAT'] = lat_ctxt, lat_ctxt_id lon_ctxt = ParameterContext( 'LON', param_type=ConstantType( QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' lon_ctxt_id = self.dataset_management.create_parameter_context( name='test_LON', parameter_context=lon_ctxt.dump()) contexts['LON'] = lon_ctxt, lon_ctxt_id # Independent Parameters # Temperature - values expected to be the decimal results of conversion from hex temp_ctxt = ParameterContext( 'TEMPWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) temp_ctxt.uom = 'deg_C' temp_ctxt_id = self.dataset_management.create_parameter_context( name='test_TEMPWAT_L0', parameter_context=temp_ctxt.dump()) contexts['TEMPWAT_L0'] = temp_ctxt, temp_ctxt_id # Conductivity - values expected to be the decimal results of conversion from hex cond_ctxt = ParameterContext( 'CONDWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) cond_ctxt.uom = 'S m-1' cond_ctxt_id = self.dataset_management.create_parameter_context( name='test_CONDWAT_L0', parameter_context=cond_ctxt.dump()) contexts['CONDWAT_L0'] = cond_ctxt, cond_ctxt_id # Pressure - values expected to be the decimal results of conversion from hex press_ctxt = ParameterContext( 'PRESWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) press_ctxt.uom = 'dbar' press_ctxt_id = self.dataset_management.create_parameter_context( name='test_PRESWAT_L0', parameter_context=press_ctxt.dump()) contexts['PRESWAT_L0'] = press_ctxt, press_ctxt_id # Dependent Parameters # TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10 tl1_func = '(T / 10000) - 10' expr = NumexprFunction('TEMPWAT_L1', tl1_func, ['T']) expr_id = self.dataset_management.create_parameter_function( name='test_TEMPWAT_L1', parameter_function=expr.dump()) funcs['TEMPWAT_L1'] = expr, expr_id tl1_pmap = {'T': 'TEMPWAT_L0'} expr.param_map = tl1_pmap tempL1_ctxt = ParameterContext( 'TEMPWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) tempL1_ctxt.uom = 'deg_C' tempL1_ctxt_id = self.dataset_management.create_parameter_context( name='test_TEMPWAT_L1', parameter_context=tempL1_ctxt.dump(), parameter_function_id=expr_id) contexts['TEMPWAT_L1'] = tempL1_ctxt, tempL1_ctxt_id # CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5 cl1_func = '(C / 100000) - 0.5' expr = NumexprFunction('CONDWAT_L1', cl1_func, ['C']) expr_id = self.dataset_management.create_parameter_function( name='test_CONDWAT_L1', parameter_function=expr.dump()) funcs['CONDWAT_L1'] = expr, expr_id cl1_pmap = {'C': 'CONDWAT_L0'} expr.param_map = cl1_pmap condL1_ctxt = ParameterContext( 'CONDWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) condL1_ctxt.uom = 'S m-1' condL1_ctxt_id = self.dataset_management.create_parameter_context( name='test_CONDWAT_L1', parameter_context=condL1_ctxt.dump(), parameter_function_id=expr_id) contexts['CONDWAT_L1'] = condL1_ctxt, condL1_ctxt_id # Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721 # PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range) pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)' expr = NumexprFunction('PRESWAT_L1', pl1_func, ['P', 'p_range']) expr_id = self.dataset_management.create_parameter_function( name='test_PRESWAT_L1', parameter_function=expr.dump()) funcs['PRESWAT_L1'] = expr, expr_id pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721} expr.param_map = pl1_pmap presL1_ctxt = ParameterContext( 'PRESWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) presL1_ctxt.uom = 'S m-1' presL1_ctxt_id = self.dataset_management.create_parameter_context( name='test_CONDWAT_L1', parameter_context=presL1_ctxt.dump(), parameter_function_id=expr_id) contexts['PRESWAT_L1'] = presL1_ctxt, presL1_ctxt_id # Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project: # https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1 # PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1) owner = 'gsw' sal_func = 'SP_from_C' sal_arglist = ['C', 't', 'p'] expr = PythonFunction('PRACSAL', owner, sal_func, sal_arglist) expr_id = self.dataset_management.create_parameter_function( name='test_PRACSAL', parameter_function=expr.dump()) funcs['PRACSAL'] = expr, expr_id # A magic function that may or may not exist actually forms the line below at runtime. sal_pmap = { 'C': NumexprFunction('CONDWAT_L1*10', 'C*10', ['C'], param_map={'C': 'CONDWAT_L1'}), 't': 'TEMPWAT_L1', 'p': 'PRESWAT_L1' } expr.param_map = sal_pmap sal_ctxt = ParameterContext('PRACSAL', param_type=ParameterFunctionType(expr), variability=VariabilityEnum.TEMPORAL) sal_ctxt.uom = 'g kg-1' sal_ctxt_id = self.dataset_management.create_parameter_context( name='test_PRACSAL', parameter_context=sal_ctxt.dump(), parameter_function_id=expr_id) contexts['PRACSAL'] = sal_ctxt, sal_ctxt_id # absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude) # conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1) # DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1) owner = 'gsw' abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP', ['PRACSAL', 'PRESWAT_L1', 'LON', 'LAT']) cons_temp_expr = PythonFunction( 'cons_temp', owner, 'CT_from_t', [abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1']) dens_expr = PythonFunction( 'DENSITY', owner, 'rho', [abs_sal_expr, cons_temp_expr, 'PRESWAT_L1']) dens_ctxt = ParameterContext( 'DENSITY', param_type=ParameterFunctionType(dens_expr), variability=VariabilityEnum.TEMPORAL) dens_ctxt.uom = 'kg m-3' dens_ctxt_id = self.dataset_management.create_parameter_context( name='test_DENSITY', parameter_context=dens_ctxt.dump()) contexts['DENSITY'] = dens_ctxt, dens_ctxt_id return contexts, funcs
def _get_pdict(self, filter_values): t_ctxt = ParameterContext( 'TIME', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.uom = 'seconds since 01-01-1900' t_ctxt_id = self.dataset_management.create_parameter_context( name='TIME', parameter_context=t_ctxt.dump(), parameter_type='quantity<int64>', unit_of_measure=t_ctxt.uom) lat_ctxt = ParameterContext( 'LAT', param_type=ConstantType( QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' lat_ctxt_id = self.dataset_management.create_parameter_context( name='LAT', parameter_context=lat_ctxt.dump(), parameter_type='quantity<float32>', unit_of_measure=lat_ctxt.uom) lon_ctxt = ParameterContext( 'LON', param_type=ConstantType( QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' lon_ctxt_id = self.dataset_management.create_parameter_context( name='LON', parameter_context=lon_ctxt.dump(), parameter_type='quantity<float32>', unit_of_measure=lon_ctxt.uom) # Independent Parameters # Temperature - values expected to be the decimal results of conversion from hex temp_ctxt = ParameterContext( 'TEMPWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) temp_ctxt.uom = 'deg_C' temp_ctxt_id = self.dataset_management.create_parameter_context( name='TEMPWAT_L0', parameter_context=temp_ctxt.dump(), parameter_type='quantity<float32>', unit_of_measure=temp_ctxt.uom) # Conductivity - values expected to be the decimal results of conversion from hex cond_ctxt = ParameterContext( 'CONDWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) cond_ctxt.uom = 'S m-1' cond_ctxt_id = self.dataset_management.create_parameter_context( name='CONDWAT_L0', parameter_context=cond_ctxt.dump(), parameter_type='quantity<float32>', unit_of_measure=cond_ctxt.uom) # Pressure - values expected to be the decimal results of conversion from hex press_ctxt = ParameterContext( 'PRESWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) press_ctxt.uom = 'dbar' press_ctxt_id = self.dataset_management.create_parameter_context( name='PRESWAT_L0', parameter_context=press_ctxt.dump(), parameter_type='quantity<float32>', unit_of_measure=press_ctxt.uom) # Dependent Parameters # TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10 tl1_func = '(T / 10000) - 10' tl1_pmap = {'T': 'TEMPWAT_L0'} expr = NumexprFunction('TEMPWAT_L1', tl1_func, ['T'], param_map=tl1_pmap) tempL1_ctxt = ParameterContext( 'TEMPWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) tempL1_ctxt.uom = 'deg_C' tempL1_ctxt_id = self.dataset_management.create_parameter_context( name=tempL1_ctxt.name, parameter_context=tempL1_ctxt.dump(), parameter_type='pfunc', unit_of_measure=tempL1_ctxt.uom) # CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5 cl1_func = '(C / 100000) - 0.5' cl1_pmap = {'C': 'CONDWAT_L0'} expr = NumexprFunction('CONDWAT_L1', cl1_func, ['C'], param_map=cl1_pmap) condL1_ctxt = ParameterContext( 'CONDWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) condL1_ctxt.uom = 'S m-1' condL1_ctxt_id = self.dataset_management.create_parameter_context( name=condL1_ctxt.name, parameter_context=condL1_ctxt.dump(), parameter_type='pfunc', unit_of_measure=condL1_ctxt.uom) # Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721 # PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range) pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)' pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721} expr = NumexprFunction('PRESWAT_L1', pl1_func, ['P', 'p_range'], param_map=pl1_pmap) presL1_ctxt = ParameterContext( 'PRESWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) presL1_ctxt.uom = 'S m-1' presL1_ctxt_id = self.dataset_management.create_parameter_context( name=presL1_ctxt.name, parameter_context=presL1_ctxt.dump(), parameter_type='pfunc', unit_of_measure=presL1_ctxt.uom) # Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project: # https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1 # PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1) owner = 'gsw' sal_func = 'SP_from_C' sal_arglist = ['C', 't', 'p'] sal_pmap = { 'C': NumexprFunction('CONDWAT_L1*10', 'C*10', ['C'], param_map={'C': 'CONDWAT_L1'}), 't': 'TEMPWAT_L1', 'p': 'PRESWAT_L1' } sal_kwargmap = None expr = PythonFunction('PRACSAL', owner, sal_func, sal_arglist, sal_kwargmap, sal_pmap) sal_ctxt = ParameterContext('PRACSAL', param_type=ParameterFunctionType(expr), variability=VariabilityEnum.TEMPORAL) sal_ctxt.uom = 'g kg-1' sal_ctxt_id = self.dataset_management.create_parameter_context( name=sal_ctxt.name, parameter_context=sal_ctxt.dump(), parameter_type='pfunc', unit_of_measure=sal_ctxt.uom) # absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude) # conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1) # DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1) owner = 'gsw' abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP', ['PRACSAL', 'PRESWAT_L1', 'LON', 'LAT']) cons_temp_expr = PythonFunction( 'cons_temp', owner, 'CT_from_t', [abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1']) dens_expr = PythonFunction( 'DENSITY', owner, 'rho', [abs_sal_expr, cons_temp_expr, 'PRESWAT_L1']) dens_ctxt = ParameterContext( 'DENSITY', param_type=ParameterFunctionType(dens_expr), variability=VariabilityEnum.TEMPORAL) dens_ctxt.uom = 'kg m-3' dens_ctxt_id = self.dataset_management.create_parameter_context( name=dens_ctxt.name, parameter_context=dens_ctxt.dump(), parameter_type='pfunc', unit_of_measure=dens_ctxt.uom) ids = [ t_ctxt_id, lat_ctxt_id, lon_ctxt_id, temp_ctxt_id, cond_ctxt_id, press_ctxt_id, tempL1_ctxt_id, condL1_ctxt_id, presL1_ctxt_id, sal_ctxt_id, dens_ctxt_id ] contexts = [ t_ctxt, lat_ctxt, lon_ctxt, temp_ctxt, cond_ctxt, press_ctxt, tempL1_ctxt, condL1_ctxt, presL1_ctxt, sal_ctxt, dens_ctxt ] context_ids = [ ids[i] for i, ctxt in enumerate(contexts) if ctxt.name in filter_values ] pdict_name = '_'.join( [ctxt.name for ctxt in contexts if ctxt.name in filter_values]) try: self.pdicts[pdict_name] return self.pdicts[pdict_name] except KeyError: pdict_id = self.dataset_management.create_parameter_dictionary( pdict_name, parameter_context_ids=context_ids, temporal_context='time') self.pdicts[pdict_name] = pdict_id return pdict_id
def get_string_array_lookup_value(self, value): return self._placeholder( value, lambda placeholder: ParameterContext( name=placeholder, param_type=SparseConstantType(base_type=ArrayType())))