def get_parameter_dictionary(cls, parameter_dictionary_id=''): """ Class method to return a CoverageModel ParameterDictionary object from the ION Resources. The object is built from the associated parameter contexts. """ dms_cli = DatasetManagementServiceClient() pd = dms_cli.read_parameter_dictionary(parameter_dictionary_id) pcs = dms_cli.read_parameter_contexts(parameter_dictionary_id=parameter_dictionary_id, id_only=False) return cls.build_parameter_dictionary(pd, pcs)
def get_parameter_dictionary(cls, parameter_dictionary_id=''): ''' Preferred client-side class method for constructing a parameter dictionary from a service call. ''' dms_cli = DatasetManagementServiceClient() pd = dms_cli.read_parameter_dictionary(parameter_dictionary_id) pcs = dms_cli.read_parameter_contexts(parameter_dictionary_id=parameter_dictionary_id, id_only=False) return cls.build_parameter_dictionary(pd, pcs)
def get_parameter_dictionary(cls, parameter_dictionary_id=''): ''' Preferred client-side class method for constructing a parameter dictionary from a service call. ''' dms_cli = DatasetManagementServiceClient() pd = dms_cli.read_parameter_dictionary(parameter_dictionary_id) pcs = dms_cli.read_parameter_contexts( parameter_dictionary_id=parameter_dictionary_id, id_only=False) return cls.build_parameter_dictionary(pd, pcs)
def get_parameter_dictionary(cls, parameter_dictionary_id=''): """ Class method to return a CoverageModel ParameterDictionary object from the ION Resources. The object is built from the associated parameter contexts. """ dms_cli = DatasetManagementServiceClient() pd = dms_cli.read_parameter_dictionary(parameter_dictionary_id) pcs = dms_cli.read_parameter_contexts( parameter_dictionary_id=parameter_dictionary_id, id_only=False) return cls.build_parameter_dictionary(pd, pcs)
def get_parameter_dictionary(cls, parameter_dictionary_id=''): ''' Preferred client-side class method for constructing a parameter dictionary from a service call. ''' dms_cli = DatasetManagementServiceClient() pd = dms_cli.read_parameter_dictionary(parameter_dictionary_id) pcs = dms_cli.read_parameter_contexts(parameter_dictionary_id=parameter_dictionary_id, id_only=False) pdict = cls._merge_contexts([ParameterContext.load(i.parameter_context) for i in pcs], pd.temporal_context) pdict._identifier = parameter_dictionary_id return pdict
def get_parameter_dictionary(cls, parameter_dictionary_id=''): ''' Preferred client-side class method for constructing a parameter dictionary from a service call. ''' dms_cli = DatasetManagementServiceClient() pd = dms_cli.read_parameter_dictionary(parameter_dictionary_id) pcs = dms_cli.read_parameter_contexts( parameter_dictionary_id=parameter_dictionary_id, id_only=False) pdict = cls._merge_contexts( [ParameterContext.load(i.parameter_context) for i in pcs], pd.temporal_context) pdict._identifier = parameter_dictionary_id return pdict
class DatasetManagementIntTest(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.resource_registry = ResourceRegistryServiceClient() self.dataset_management = DatasetManagementServiceClient() def test_dataset_crud(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) dataset = Dataset(name='ctd_dataset') dataset_id = self.dataset_management.create_dataset( dataset, parameter_dictionary_id=pdict_id) ds_obj = self.dataset_management.read_dataset(dataset_id) self.assertEquals(ds_obj.name, 'ctd_dataset') ds_obj.name = 'something different' self.dataset_management.update_dataset(ds_obj) ds_obj2 = self.dataset_management.read_dataset(dataset_id) self.assertEquals(ds_obj.name, ds_obj2.name) def test_context_crud(self): context_ids = self.create_contexts() context_id = context_ids.pop() ctxt = self.dataset_management.read_parameter_context(context_id) context = DatasetManagementService.get_coverage_parameter(ctxt) self.assertIsInstance(context, CoverageParameterContext) self.dataset_management.delete_parameter_context(context_id) with self.assertRaises(NotFound): self.dataset_management.read_parameter_context(context_id) def test_pfunc_crud(self): contexts, funcs = self.create_pfuncs() context_ids = [context_id for context_id in contexts.itervalues()] pdict_id = self.dataset_management.create_parameter_dictionary( name='functional_pdict', parameter_context_ids=context_ids, temporal_context='time') self.addCleanup(self.dataset_management.delete_parameter_dictionary, pdict_id) expr_id = funcs['CONDWAT_L1'] expr = self.dataset_management.read_parameter_function(expr_id) func_class = DatasetManagementService.get_coverage_function(expr) self.assertIsInstance(func_class, NumexprFunction) def test_pdict_crud(self): context_ids = self.create_contexts() pdict_res_id = self.dataset_management.create_parameter_dictionary( name='pdict1', parameter_context_ids=context_ids, temporal_context='time') pdict_contexts = self.dataset_management.read_parameter_contexts( parameter_dictionary_id=pdict_res_id, id_only=True) pdict = DatasetManagementService.get_parameter_dictionary(pdict_res_id) self.assertIsInstance(pdict, ParameterDictionary) self.assertTrue('time_test' in pdict) self.assertEquals(pdict.identifier, pdict_res_id) self.assertEquals(set(pdict_contexts), set(context_ids)) self.dataset_management.delete_parameter_dictionary( parameter_dictionary_id=pdict_res_id) with self.assertRaises(NotFound): self.dataset_management.read_parameter_dictionary( parameter_dictionary_id=pdict_res_id) def create_contexts(self): context_ids = [] cond = ParameterContext(name='condictivity_test', parameter_type='quantity', value_encoding='float32', units='1', fill_value=0) context_ids.append(self.dataset_management.create_parameter(cond)) pres = ParameterContext(name='pressure_test', parameter_type='quantity', value_encoding='float32', units='Pa', fill_value=0) context_ids.append(self.dataset_management.create_parameter(pres)) sal = ParameterContext(name='salinity_test', parameter_type='quantity', value_encoding='float32', units='psu', fill_value=0) context_ids.append(self.dataset_management.create_parameter(sal)) temp = ParameterContext(name='temp_test', parameter_type='quantity', value_encoding='float32', units='degree_C', fill_value=0) context_ids.append(self.dataset_management.create_parameter(temp)) time_test = ParameterContext(name='time_test', parameter_type='quantity', value_encoding='float32', units='seconds since 1970-01-01', fill_value=0) context_ids.append(self.dataset_management.create_parameter(time_test)) return context_ids def create_pfuncs(self): contexts = {} funcs = {} time_ = ParameterContext(name='TIME', parameter_type='quantity', value_encoding='float32', units='seconds since 1900-01-01', fill_value=0) t_ctxt_id = self.dataset_management.create_parameter(time_) contexts['TIME'] = t_ctxt_id lat = ParameterContext(name='LAT', parameter_type='sparse', value_encoding='float32', units='degrees_north', fill_value=-9999.) lat_ctxt_id = self.dataset_management.create_parameter(lat) contexts['LAT'] = lat_ctxt_id lon = ParameterContext(name='LON', parameter_type="sparse", value_encoding='float32', units='degrees_east', fill_value=-9999) lon_ctxt_id = self.dataset_management.create_parameter(lon) contexts['LON'] = lon_ctxt_id # Independent Parameters # Temperature - values expected to be the decimal results of conversion from hex temp = ParameterContext(name='TEMPWAT_L0', parameter_type='quantity', value_encoding='float32', units='deg_C') temp_ctxt_id = self.dataset_management.create_parameter(temp) contexts['TEMPWAT_L0'] = temp_ctxt_id # Conductivity - values expected to be the decimal results of conversion from hex cond = ParameterContext(name='CONDWAT_L0', parameter_type='quantity', value_encoding='float32', units='S m-1') cond_ctxt_id = self.dataset_management.create_parameter(cond) contexts['CONDWAT_L0'] = cond_ctxt_id # Pressure - values expected to be the decimal results of conversion from hex press = ParameterContext(name='PRESWAT_L0', parameter_type='quantity', value_encoding='float32', units='dbar') press_ctxt_id = self.dataset_management.create_parameter(press) contexts['PRESWAT_L0'] = press_ctxt_id # Dependent Parameters # TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10 tl1_func = '(T / 10000) - 10' tempwat_f = ParameterFunction(name='TEMPWAT_L1', function_type=PFT.NUMEXPR, function=tl1_func, args=['T']) expr_id = self.dataset_management.create_parameter_function(tempwat_f) funcs['TEMPWAT_L1'] = expr_id tl1_pmap = {'T': 'TEMPWAT_L0'} tempL1 = ParameterContext(name='TEMPWAT_L1', parameter_type='function', parameter_function_id=expr_id, parameter_function_map=tl1_pmap, value_encoding='float32', units='deg_C') tempL1_ctxt_id = self.dataset_management.create_parameter(tempL1) contexts['TEMPWAT_L1'] = tempL1_ctxt_id # CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5 cl1_func = '(C / 100000) - 0.5' condwat_f = ParameterFunction(name='CONDWAT_L1', function_type=PFT.NUMEXPR, function=cl1_func, args=['C']) expr_id = self.dataset_management.create_parameter_function(condwat_f) funcs['CONDWAT_L1'] = expr_id cl1_pmap = {'C': 'CONDWAT_L0'} condL1 = ParameterContext(name='CONDWAT_L1', parameter_type='function', parameter_function_id=expr_id, parameter_function_map=cl1_pmap, value_encoding='float32', units='S m-1') condL1_ctxt_id = self.dataset_management.create_parameter(condL1) contexts['CONDWAT_L1'] = condL1_ctxt_id # Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721 # PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range) pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)' preswat_f = ParameterFunction(name='PRESWAT_L1', function_type=PFT.NUMEXPR, function=pl1_func, args=['P', 'p_range']) expr_id = self.dataset_management.create_parameter_function(preswat_f) funcs['PRESWAT_L1'] = expr_id pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721} presL1 = ParameterContext(name='PRESWAT_L1', parameter_type='function', parameter_function_id=expr_id, parameter_function_map=pl1_pmap, value_encoding='float32', units='dbar') presL1_ctxt_id = self.dataset_management.create_parameter(presL1) contexts['PRESWAT_L1'] = presL1_ctxt_id # A magic function that may or may not exist actually forms the line below at runtime. cond_f = ParameterFunction(name='condwat10', function_type=PFT.NUMEXPR, function='C*10', args=['C']) expr_id = self.dataset_management.create_parameter_function(cond_f) cond10 = ParameterContext(name='c10', parameter_type='function', parameter_function_id=expr_id, parameter_function_map={'C': 'CONDWAT_L1'}, value_encoding='float32', units='1') cond10_id = self.dataset_management.create_parameter(cond10) contexts['C10'] = cond10_id # Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project: # https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1 # PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1) owner = 'gsw' sal_func = 'SP_from_C' sal_arglist = ['C', 't', 'p'] pracsal_f = ParameterFunction(name='PRACSAL', function_type=PFT.PYTHON, owner=owner, function=sal_func, args=sal_arglist) expr_id = self.dataset_management.create_parameter_function(pracsal_f) funcs['PRACSAL'] = expr_id sal_pmap = {'C': 'c10', 't': 'TEMPWAT_L1', 'p': 'PRESWAT_L1'} sal_ctxt = ParameterContext(name='PRACSAL', parameter_type='function', parameter_function_id=expr_id, parameter_function_map=sal_pmap, value_encoding='float32', units='g kg-1') sal_ctxt_id = self.dataset_management.create_parameter(sal_ctxt) contexts['PRACSAL'] = sal_ctxt_id # absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude) # conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1) # DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1) return contexts, funcs def test_verify_contexts(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name( name='ctd_parsed_param_dict', id_only=True) pcontexts = self.dataset_management.read_parameter_contexts( parameter_dictionary_id=pdict_id) for pcontext in pcontexts: self.assertTrue('fill_value' in pcontext) self.assertTrue('reference_urls' in pcontext) self.assertTrue('internal_name' in pcontext) self.assertTrue('display_name' in pcontext) self.assertTrue('standard_name' in pcontext) self.assertTrue('ooi_short_name' in pcontext) self.assertTrue('description' in pcontext) self.assertTrue('precision' in pcontext)
class DatasetManagementIntTest(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.resource_registry = ResourceRegistryServiceClient() self.dataset_management = DatasetManagementServiceClient() def test_dataset_crud(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) tdom, sdom = time_series_domain() dataset_id = self.dataset_management.create_dataset(name='ctd_dataset', parameter_dictionary_id=pdict_id, spatial_domain=sdom.dump(), temporal_domain=tdom.dump()) ds_obj = self.dataset_management.read_dataset(dataset_id) self.assertEquals(ds_obj.name, 'ctd_dataset') ds_obj.name = 'something different' self.dataset_management.update_dataset(ds_obj) ds_obj2 = self.dataset_management.read_dataset(dataset_id) self.assertEquals(ds_obj.name, ds_obj2.name) def test_context_crud(self): context_ids = self.create_contexts() context_id = context_ids.pop() ctxt = self.dataset_management.read_parameter_context(context_id) context = DatasetManagementService.get_coverage_parameter(ctxt) self.assertIsInstance(context, CoverageParameterContext) self.dataset_management.delete_parameter_context(context_id) with self.assertRaises(NotFound): self.dataset_management.read_parameter_context(context_id) def test_pfunc_crud(self): contexts, funcs = self.create_pfuncs() context_ids = [context_id for context_id in contexts.itervalues()] pdict_id = self.dataset_management.create_parameter_dictionary(name='functional_pdict', parameter_context_ids=context_ids, temporal_context='time') self.addCleanup(self.dataset_management.delete_parameter_dictionary, pdict_id) expr_id = funcs['CONDWAT_L1'] expr = self.dataset_management.read_parameter_function(expr_id) func_class = DatasetManagementService.get_coverage_function(expr) self.assertIsInstance(func_class, NumexprFunction) def test_pdict_crud(self): context_ids = self.create_contexts() pdict_res_id = self.dataset_management.create_parameter_dictionary(name='pdict1', parameter_context_ids=context_ids, temporal_context='time') pdict_contexts = self.dataset_management.read_parameter_contexts(parameter_dictionary_id=pdict_res_id, id_only=True) pdict = DatasetManagementService.get_parameter_dictionary(pdict_res_id) self.assertIsInstance(pdict, ParameterDictionary) self.assertTrue('time_test' in pdict) self.assertEquals(pdict.identifier, pdict_res_id) self.assertEquals(set(pdict_contexts), set(context_ids)) self.dataset_management.delete_parameter_dictionary(parameter_dictionary_id=pdict_res_id) with self.assertRaises(NotFound): self.dataset_management.read_parameter_dictionary(parameter_dictionary_id=pdict_res_id) def create_contexts(self): context_ids = [] cond = ParameterContext(name='condictivity_test', parameter_type='quantity', value_encoding='float32', units='1', fill_value=0) context_ids.append(self.dataset_management.create_parameter(cond)) pres = ParameterContext(name='pressure_test', parameter_type='quantity', value_encoding='float32', units='Pa', fill_value=0) context_ids.append(self.dataset_management.create_parameter(pres)) sal = ParameterContext(name='salinity_test', parameter_type='quantity', value_encoding='float32', units='psu', fill_value=0) context_ids.append(self.dataset_management.create_parameter(sal)) temp = ParameterContext(name='temp_test', parameter_type='quantity', value_encoding='float32', units='degree_C', fill_value=0) context_ids.append(self.dataset_management.create_parameter(temp)) time_test = ParameterContext(name='time_test', parameter_type='quantity', value_encoding='float32', units='seconds since 1970-01-01', fill_value=0) context_ids.append(self.dataset_management.create_parameter(time_test)) return context_ids def create_pfuncs(self): contexts = {} funcs = {} time_ = ParameterContext(name='TIME', parameter_type='quantity', value_encoding='float32', units='seconds since 1900-01-01', fill_value=0) t_ctxt_id = self.dataset_management.create_parameter(time_) contexts['TIME'] = t_ctxt_id lat = ParameterContext(name='LAT', parameter_type='sparse', value_encoding='float32', units='degrees_north', fill_value=-9999.) lat_ctxt_id = self.dataset_management.create_parameter(lat) contexts['LAT'] = lat_ctxt_id lon = ParameterContext(name='LON', parameter_type="sparse", value_encoding='float32', units='degrees_east', fill_value=-9999) lon_ctxt_id = self.dataset_management.create_parameter(lon) contexts['LON'] = lon_ctxt_id # Independent Parameters # Temperature - values expected to be the decimal results of conversion from hex temp = ParameterContext(name='TEMPWAT_L0', parameter_type='quantity', value_encoding='float32', units='deg_C') temp_ctxt_id = self.dataset_management.create_parameter(temp) contexts['TEMPWAT_L0'] = temp_ctxt_id # Conductivity - values expected to be the decimal results of conversion from hex cond = ParameterContext(name='CONDWAT_L0', parameter_type='quantity', value_encoding='float32', units='S m-1') cond_ctxt_id = self.dataset_management.create_parameter(cond) contexts['CONDWAT_L0'] = cond_ctxt_id # Pressure - values expected to be the decimal results of conversion from hex press = ParameterContext(name='PRESWAT_L0', parameter_type='quantity', value_encoding='float32', units='dbar') press_ctxt_id = self.dataset_management.create_parameter(press) contexts['PRESWAT_L0'] = press_ctxt_id # Dependent Parameters # TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10 tl1_func = '(T / 10000) - 10' tempwat_f = ParameterFunction(name='TEMPWAT_L1', function_type=PFT.NUMEXPR, function=tl1_func, args=['T']) expr_id = self.dataset_management.create_parameter_function(tempwat_f) funcs['TEMPWAT_L1'] = expr_id tl1_pmap = {'T': 'TEMPWAT_L0'} tempL1 = ParameterContext(name='TEMPWAT_L1', parameter_type='function', parameter_function_id=expr_id, parameter_function_map=tl1_pmap, value_encoding='float32', units='deg_C') tempL1_ctxt_id = self.dataset_management.create_parameter(tempL1) contexts['TEMPWAT_L1'] = tempL1_ctxt_id # CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5 cl1_func = '(C / 100000) - 0.5' condwat_f = ParameterFunction(name='CONDWAT_L1', function_type=PFT.NUMEXPR, function=cl1_func, args=['C']) expr_id = self.dataset_management.create_parameter_function(condwat_f) funcs['CONDWAT_L1'] = expr_id cl1_pmap = {'C': 'CONDWAT_L0'} condL1 = ParameterContext(name='CONDWAT_L1', parameter_type='function', parameter_function_id=expr_id, parameter_function_map=cl1_pmap, value_encoding='float32', units='S m-1') condL1_ctxt_id = self.dataset_management.create_parameter(condL1) contexts['CONDWAT_L1'] = condL1_ctxt_id # Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721 # PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range) pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)' preswat_f = ParameterFunction(name='PRESWAT_L1', function_type=PFT.NUMEXPR, function=pl1_func, args=['P', 'p_range']) expr_id = self.dataset_management.create_parameter_function(preswat_f) funcs['PRESWAT_L1'] = expr_id pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721} presL1 = ParameterContext(name='PRESWAT_L1', parameter_type='function', parameter_function_id=expr_id, parameter_function_map=pl1_pmap, value_encoding='float32', units='dbar') presL1_ctxt_id = self.dataset_management.create_parameter(presL1) contexts['PRESWAT_L1'] = presL1_ctxt_id # A magic function that may or may not exist actually forms the line below at runtime. cond_f = ParameterFunction(name='condwat10', function_type=PFT.NUMEXPR, function='C*10', args=['C']) expr_id = self.dataset_management.create_parameter_function(cond_f) cond10 = ParameterContext(name='c10', parameter_type='function', parameter_function_id=expr_id, parameter_function_map={'C':'CONDWAT_L1'}, value_encoding='float32', units='1') cond10_id = self.dataset_management.create_parameter(cond10) contexts['C10'] = cond10_id # Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project: # https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1 # PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1) owner = 'gsw' sal_func = 'SP_from_C' sal_arglist = ['C', 't', 'p'] pracsal_f = ParameterFunction(name='PRACSAL', function_type=PFT.PYTHON, owner=owner, function=sal_func, args=sal_arglist) expr_id = self.dataset_management.create_parameter_function(pracsal_f) funcs['PRACSAL'] = expr_id sal_pmap = {'C': 'c10', 't': 'TEMPWAT_L1', 'p': 'PRESWAT_L1'} sal_ctxt = ParameterContext(name='PRACSAL', parameter_type='function', parameter_function_id=expr_id, parameter_function_map=sal_pmap, value_encoding='float32', units='g kg-1') sal_ctxt_id = self.dataset_management.create_parameter(sal_ctxt) contexts['PRACSAL'] = sal_ctxt_id # absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude) # conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1) # DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1) return contexts, funcs def test_verify_contexts(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name(name='ctd_parsed_param_dict', id_only=True) pcontexts = self.dataset_management.read_parameter_contexts(parameter_dictionary_id=pdict_id) for pcontext in pcontexts: self.assertTrue('fill_value' in pcontext) self.assertTrue('reference_urls' in pcontext) self.assertTrue('internal_name' in pcontext) self.assertTrue('display_name' in pcontext) self.assertTrue('standard_name' in pcontext) self.assertTrue('ooi_short_name' in pcontext) self.assertTrue('description' in pcontext) self.assertTrue('precision' in pcontext)
class DatasetManagementIntTest(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.resource_registry = ResourceRegistryServiceClient() self.dataset_management = DatasetManagementServiceClient() def test_dataset_crud(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) tdom, sdom = time_series_domain() dataset_id = self.dataset_management.create_dataset(name='ctd_dataset', parameter_dictionary_id=pdict_id, spatial_domain=sdom.dump(), temporal_domain=tdom.dump()) ds_obj = self.dataset_management.read_dataset(dataset_id) self.assertEquals(ds_obj.name, 'ctd_dataset') ds_obj.name = 'something different' self.dataset_management.update_dataset(ds_obj) self.dataset_management.register_dataset(dataset_id) ds_obj2 = self.dataset_management.read_dataset(dataset_id) self.assertEquals(ds_obj.name, ds_obj2.name) self.assertTrue(ds_obj2.registered) def test_context_crud(self): context_ids = self.create_contexts() context_id = context_ids.pop() context = DatasetManagementService.get_parameter_context(context_id) self.assertIsInstance(context, ParameterContext) self.assertEquals(context.identifier, context_id) self.dataset_management.delete_parameter_context(context_id) with self.assertRaises(NotFound): self.dataset_management.read_parameter_context(context_id) def test_pfunc_crud(self): contexts, funcs = self.create_pfuncs() context_ids = [context_id for ctxt,context_id in contexts.itervalues()] pdict_id = self.dataset_management.create_parameter_dictionary(name='functional_pdict', parameter_context_ids=context_ids, temporal_context='time') self.addCleanup(self.dataset_management.delete_parameter_dictionary, pdict_id) expr, expr_id = funcs['CONDWAT_L1'] func_class = DatasetManagementService.get_parameter_function(expr_id) self.assertIsInstance(func_class, NumexprFunction) def test_pdict_crud(self): context_ids = self.create_contexts() pdict_res_id = self.dataset_management.create_parameter_dictionary(name='pdict1', parameter_context_ids=context_ids, temporal_context='time') pdict_contexts = self.dataset_management.read_parameter_contexts(parameter_dictionary_id=pdict_res_id, id_only=True) pdict = DatasetManagementService.get_parameter_dictionary(pdict_res_id) self.assertIsInstance(pdict, ParameterDictionary) self.assertTrue('time_test' in pdict) self.assertEquals(pdict.identifier, pdict_res_id) self.assertEquals(set(pdict_contexts), set(context_ids)) self.dataset_management.delete_parameter_dictionary(parameter_dictionary_id=pdict_res_id) with self.assertRaises(NotFound): self.dataset_management.read_parameter_dictionary(parameter_dictionary_id=pdict_res_id) def create_contexts(self): context_ids = [] cond_ctxt = ParameterContext('conductivity_test', param_type=QuantityType(value_encoding=np.float32)) cond_ctxt.uom = 'unknown' cond_ctxt.fill_value = 0e0 context_ids.append(self.dataset_management.create_parameter_context(name='conductivity_test', parameter_context=cond_ctxt.dump())) pres_ctxt = ParameterContext('pressure_test', param_type=QuantityType(value_encoding=np.float32)) pres_ctxt.uom = 'Pascal' pres_ctxt.fill_value = 0x0 context_ids.append(self.dataset_management.create_parameter_context(name='pressure_test', parameter_context=pres_ctxt.dump())) sal_ctxt = ParameterContext('salinity_test', param_type=QuantityType(value_encoding=np.float32)) sal_ctxt.uom = 'PSU' sal_ctxt.fill_value = 0x0 context_ids.append(self.dataset_management.create_parameter_context(name='salinity_test', parameter_context=sal_ctxt.dump())) temp_ctxt = ParameterContext('temp_test', param_type=QuantityType(value_encoding=np.float32)) temp_ctxt.uom = 'degree_Celsius' temp_ctxt.fill_value = 0e0 context_ids.append(self.dataset_management.create_parameter_context(name='temp_test', parameter_context=temp_ctxt.dump())) t_ctxt = ParameterContext('time_test', param_type=QuantityType(value_encoding=np.int64)) t_ctxt.uom = 'seconds since 1970-01-01' t_ctxt.fill_value = 0x0 context_ids.append(self.dataset_management.create_parameter_context(name='time_test', parameter_context=t_ctxt.dump())) return context_ids def create_pfuncs(self): contexts = {} funcs = {} t_ctxt = ParameterContext('TIME', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.uom = 'seconds since 01-01-1900' t_ctxt_id = self.dataset_management.create_parameter_context(name='test_TIME', parameter_context=t_ctxt.dump()) contexts['TIME'] = (t_ctxt, t_ctxt_id) lat_ctxt = ParameterContext('LAT', param_type=ConstantType(QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' lat_ctxt_id = self.dataset_management.create_parameter_context(name='test_LAT', parameter_context=lat_ctxt.dump()) contexts['LAT'] = lat_ctxt, lat_ctxt_id lon_ctxt = ParameterContext('LON', param_type=ConstantType(QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' lon_ctxt_id = self.dataset_management.create_parameter_context(name='test_LON', parameter_context=lon_ctxt.dump()) contexts['LON'] = lon_ctxt, lon_ctxt_id # Independent Parameters # Temperature - values expected to be the decimal results of conversion from hex temp_ctxt = ParameterContext('TEMPWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) temp_ctxt.uom = 'deg_C' temp_ctxt_id = self.dataset_management.create_parameter_context(name='test_TEMPWAT_L0', parameter_context=temp_ctxt.dump()) contexts['TEMPWAT_L0'] = temp_ctxt, temp_ctxt_id # Conductivity - values expected to be the decimal results of conversion from hex cond_ctxt = ParameterContext('CONDWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) cond_ctxt.uom = 'S m-1' cond_ctxt_id = self.dataset_management.create_parameter_context(name='test_CONDWAT_L0', parameter_context=cond_ctxt.dump()) contexts['CONDWAT_L0'] = cond_ctxt, cond_ctxt_id # Pressure - values expected to be the decimal results of conversion from hex press_ctxt = ParameterContext('PRESWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) press_ctxt.uom = 'dbar' press_ctxt_id = self.dataset_management.create_parameter_context(name='test_PRESWAT_L0', parameter_context=press_ctxt.dump()) contexts['PRESWAT_L0'] = press_ctxt, press_ctxt_id # Dependent Parameters # TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10 tl1_func = '(T / 10000) - 10' expr = NumexprFunction('TEMPWAT_L1', tl1_func, ['T']) expr_id = self.dataset_management.create_parameter_function(name='test_TEMPWAT_L1', parameter_function=expr.dump()) funcs['TEMPWAT_L1'] = expr, expr_id tl1_pmap = {'T': 'TEMPWAT_L0'} expr.param_map = tl1_pmap tempL1_ctxt = ParameterContext('TEMPWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) tempL1_ctxt.uom = 'deg_C' tempL1_ctxt_id = self.dataset_management.create_parameter_context(name='test_TEMPWAT_L1', parameter_context=tempL1_ctxt.dump(), parameter_function_id=expr_id) contexts['TEMPWAT_L1'] = tempL1_ctxt, tempL1_ctxt_id # CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5 cl1_func = '(C / 100000) - 0.5' expr = NumexprFunction('CONDWAT_L1', cl1_func, ['C']) expr_id = self.dataset_management.create_parameter_function(name='test_CONDWAT_L1', parameter_function=expr.dump()) funcs['CONDWAT_L1'] = expr, expr_id cl1_pmap = {'C': 'CONDWAT_L0'} expr.param_map = cl1_pmap condL1_ctxt = ParameterContext('CONDWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) condL1_ctxt.uom = 'S m-1' condL1_ctxt_id = self.dataset_management.create_parameter_context(name='test_CONDWAT_L1', parameter_context=condL1_ctxt.dump(), parameter_function_id=expr_id) contexts['CONDWAT_L1'] = condL1_ctxt, condL1_ctxt_id # Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721 # PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range) pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)' expr = NumexprFunction('PRESWAT_L1', pl1_func, ['P', 'p_range']) expr_id = self.dataset_management.create_parameter_function(name='test_PRESWAT_L1', parameter_function=expr.dump()) funcs['PRESWAT_L1'] = expr, expr_id pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721} expr.param_map = pl1_pmap presL1_ctxt = ParameterContext('PRESWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) presL1_ctxt.uom = 'S m-1' presL1_ctxt_id = self.dataset_management.create_parameter_context(name='test_CONDWAT_L1', parameter_context=presL1_ctxt.dump(), parameter_function_id=expr_id) contexts['PRESWAT_L1'] = presL1_ctxt, presL1_ctxt_id # Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project: # https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1 # PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1) owner = 'gsw' sal_func = 'SP_from_C' sal_arglist = ['C', 't', 'p'] expr = PythonFunction('PRACSAL', owner, sal_func, sal_arglist) expr_id = self.dataset_management.create_parameter_function(name='test_PRACSAL', parameter_function=expr.dump()) funcs['PRACSAL'] = expr, expr_id # A magic function that may or may not exist actually forms the line below at runtime. sal_pmap = {'C': NumexprFunction('CONDWAT_L1*10', 'C*10', ['C'], param_map={'C': 'CONDWAT_L1'}), 't': 'TEMPWAT_L1', 'p': 'PRESWAT_L1'} expr.param_map = sal_pmap sal_ctxt = ParameterContext('PRACSAL', param_type=ParameterFunctionType(expr), variability=VariabilityEnum.TEMPORAL) sal_ctxt.uom = 'g kg-1' sal_ctxt_id = self.dataset_management.create_parameter_context(name='test_PRACSAL', parameter_context=sal_ctxt.dump(), parameter_function_id=expr_id) contexts['PRACSAL'] = sal_ctxt, sal_ctxt_id # absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude) # conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1) # DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1) owner = 'gsw' abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP', ['PRACSAL', 'PRESWAT_L1', 'LON','LAT']) cons_temp_expr = PythonFunction('cons_temp', owner, 'CT_from_t', [abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1']) dens_expr = PythonFunction('DENSITY', owner, 'rho', [abs_sal_expr, cons_temp_expr, 'PRESWAT_L1']) dens_ctxt = ParameterContext('DENSITY', param_type=ParameterFunctionType(dens_expr), variability=VariabilityEnum.TEMPORAL) dens_ctxt.uom = 'kg m-3' dens_ctxt_id = self.dataset_management.create_parameter_context(name='test_DENSITY', parameter_context=dens_ctxt.dump()) contexts['DENSITY'] = dens_ctxt, dens_ctxt_id return contexts, funcs def test_verify_contexts(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name(name='ctd_parsed_param_dict', id_only=True) pcontexts = self.dataset_management.read_parameter_contexts(parameter_dictionary_id=pdict_id) for pcontext in pcontexts: self.assertTrue('fill_value' in pcontext) self.assertTrue('reference_urls' in pcontext) self.assertTrue('internal_name' in pcontext) self.assertTrue('display_name' in pcontext) self.assertTrue('standard_name' in pcontext) self.assertTrue('ooi_short_name' in pcontext) self.assertTrue('description' in pcontext) self.assertTrue('precision' in pcontext)
class EventManagementIntTest(IonIntegrationTestCase): def setUp(self): super(EventManagementIntTest, self).setUp() self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.event_management = EventManagementServiceClient() self.rrc = ResourceRegistryServiceClient() self.process_dispatcher = ProcessDispatcherServiceClient() self.pubsub = PubsubManagementServiceClient() self.dataset_management = DatasetManagementServiceClient() self.data_product_management = DataProductManagementServiceClient() self.queue_cleanup = [] self.exchange_cleanup = [] def tearDown(self): for queue in self.queue_cleanup: xn = self.container.ex_manager.create_xn_queue(queue) xn.delete() for exchange in self.exchange_cleanup: xp = self.container.ex_manager.create_xp(exchange) xp.delete() def test_create_read_update_delete_event_type(self): """ Test that the CRUD method for event types work correctly """ event_type = EventType(name="an event type") event_type.origin = 'instrument_1' # create event_type_id = self.event_management.create_event_type(event_type) self.assertIsNotNone(event_type_id) # read read_event_type = self.event_management.read_event_type(event_type_id) self.assertEquals(read_event_type.name, event_type.name) self.assertEquals(read_event_type.origin, event_type.origin) #update read_event_type.origin = 'instrument_2' read_event_type.producer = 'producer' self.event_management.update_event_type(read_event_type) updated_event_type = self.event_management.read_event_type(event_type_id) self.assertEquals(updated_event_type.origin, 'instrument_2') self.assertEquals(updated_event_type.producer, 'producer') # delete self.event_management.delete_event_type(event_type_id) with self.assertRaises(NotFound): self.event_management.read_event_type(event_type_id) def test_create_read_update_delete_event_process_definition(self): """ Test that the CRUD methods for the event process definitions work correctly """ # Create module = 'ion.processes.data.transforms.event_alert_transform' class_name = 'EventAlertTransform' procdef_id = self.event_management.create_event_process_definition(version='ver_1', module=module, class_name=class_name, uri='http://hare.com', arguments=['arg1', 'arg2'], event_types=['ExampleDetectableEvent', 'type_2'], sub_types=['sub_type_1']) # Read read_process_def = self.event_management.read_event_process_definition(procdef_id) self.assertEquals(read_process_def.executable['module'], module) self.assertEquals(read_process_def.executable['class'], class_name) # Update self.event_management.update_event_process_definition(event_process_definition_id=procdef_id, class_name='StreamAlertTransform', arguments=['arg3', 'arg4'], event_types=['event_type_new']) updated_event_process_def = self.event_management.read_event_process_definition(procdef_id) self.assertEquals(updated_event_process_def.executable['class'], 'StreamAlertTransform') self.assertEquals(updated_event_process_def.arguments, ['arg3', 'arg4']) self.assertEquals(updated_event_process_def.definition.event_types, ['event_type_new']) # Delete self.event_management.delete_event_process_definition(procdef_id) with self.assertRaises(NotFound): self.event_management.read_event_process_definition(procdef_id) def test_event_in_stream_out_transform(self): """ Test the event-in/stream-out transform """ stream_id, _ = self.pubsub.create_stream('test_stream', exchange_point='science_data') self.exchange_cleanup.append('science_data') #--------------------------------------------------------------------------------------------- # Launch a ctd transform #--------------------------------------------------------------------------------------------- # Create the process definition process_definition = ProcessDefinition( name='EventToStreamTransform', description='For testing an event-in/stream-out transform') process_definition.executable['module']= 'ion.processes.data.transforms.event_in_stream_out_transform' process_definition.executable['class'] = 'EventToStreamTransform' proc_def_id = self.process_dispatcher.create_process_definition(process_definition=process_definition) # Build the config config = DotDict() config.process.queue_name = 'test_queue' config.process.exchange_point = 'science_data' config.process.publish_streams.output = stream_id config.process.event_type = 'ExampleDetectableEvent' config.process.variables = ['voltage', 'temperature' ] # Schedule the process self.process_dispatcher.schedule_process(process_definition_id=proc_def_id, configuration=config) #--------------------------------------------------------------------------------------------- # Create a subscriber for testing #--------------------------------------------------------------------------------------------- ar_cond = gevent.event.AsyncResult() def subscriber_callback(m, r, s): ar_cond.set(m) sub = StandaloneStreamSubscriber('sub', subscriber_callback) self.addCleanup(sub.stop) sub_id = self.pubsub.create_subscription('subscription_cond', stream_ids=[stream_id], exchange_name='sub') self.pubsub.activate_subscription(sub_id) self.queue_cleanup.append(sub.xn.queue) sub.start() gevent.sleep(4) #--------------------------------------------------------------------------------------------- # Publish an event. The transform has been configured to receive this event #--------------------------------------------------------------------------------------------- event_publisher = EventPublisher("ExampleDetectableEvent") event_publisher.publish_event(origin = 'fake_origin', voltage = '5', temperature = '273') # Assert that the transform processed the event and published data on the output stream result_cond = ar_cond.get(timeout=10) self.assertTrue(result_cond) def test_create_read_delete_event_process(self): """ Test that the CRUD methods for the event processes work correctly """ #--------------------------------------------------------------------------------------------- # Create a process definition #--------------------------------------------------------------------------------------------- # Create module = 'ion.processes.data.transforms.event_alert_transform' class_name = 'EventAlertTransform' procdef_id = self.event_management.create_event_process_definition(version='ver_1', module=module, class_name=class_name, uri='http://hare.com', arguments=['arg1', 'arg2'], event_types=['ExampleDetectableEvent', 'type_2'], sub_types=['sub_type_1']) # Read read_process_def = self.event_management.read_event_process_definition(procdef_id) self.assertEquals(read_process_def.arguments, ['arg1', 'arg2']) #--------------------------------------------------------------------------------------------- # Use the process definition to create a process #--------------------------------------------------------------------------------------------- # Create a stream param_dict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True) parameter_dictionary = self.dataset_management.read_parameter_dictionary(parameter_dictionary_id=param_dict_id) stream_def_id = self.pubsub.create_stream_definition('cond_stream_def', parameter_dictionary_id=param_dict_id) tdom, sdom = time_series_domain() tdom, sdom = tdom.dump(), sdom.dump() dp_obj = IonObject(RT.DataProduct, name='DP1', description='some new dp', temporal_domain = tdom, spatial_domain = sdom) # Create a data product data_product_id = self.data_product_management.create_data_product(data_product=dp_obj, stream_definition_id=stream_def_id, parameter_dictionary=parameter_dictionary) output_products = {} output_products['conductivity'] = data_product_id # Create an event process event_process_id = self.event_management.create_event_process( process_definition_id=procdef_id, event_types=['ExampleDetectableEvent','DetectionEvent'], sub_types=['s1', 's2'], origins=['or_1', 'or_2'], origin_types=['or_t1', 'or_t2'], out_data_products = output_products) #--------------------------------------------------------------------------------------------- # Read the event process object and make assertions #--------------------------------------------------------------------------------------------- out_data_products={'conductivity': data_product_id}) event_process_obj = self.event_management.read_event_process(event_process_id=event_process_id) # Get the stream associated with the data product for the sake of making assertions stream_ids, _ = self.rrc.find_objects(data_product_id, PRED.hasStream, id_only=True) stream_id = stream_ids[0] # Assertions! self.assertEquals(event_process_obj.detail.output_streams['conductivity'], stream_id) self.assertEquals(event_process_obj.detail.event_types, ['ExampleDetectableEvent', 'DetectionEvent']) self.assertEquals(event_process_obj.detail.sub_types, ['s1', 's2']) self.assertEquals(event_process_obj.detail.origins, ['or_1', 'or_2']) self.assertEquals(event_process_obj.detail.origin_types, ['or_t1', 'or_t2']) @unittest.skip("Not yet implemented in R 2.0") def test_activate_deactivate_data_process(self): """ Test that the activation and deactivation of event processes happens correctly """ pass @unittest.skip("Not yet implemented in R 2.0") def test_update_event_process_inputs(self): """ Test that the event process inputs can be correctly updated """ pass
class TestStreamIngestionWorker(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.dataset_management_client = DatasetManagementServiceClient(node=self.container.node) self.pubsub_client = PubsubManagementServiceClient(node=self.container.node) self.time_dom, self.spatial_dom = time_series_domain() self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(name='ctd_parsed_param_dict', id_only=True) self.stream_def_id = self.pubsub_client.create_stream_definition(name='stream_def', parameter_dictionary_id=self.parameter_dict_id) self.addCleanup(self.pubsub_client.delete_stream_definition, self.stream_def_id) self.stream_id, self.route_id = self.pubsub_client.create_stream(name='parsed_stream', stream_definition_id=self.stream_def_id, exchange_point='science_data') self.addCleanup(self.pubsub_client.delete_stream, self.stream_id) self.subscription_id = self.pubsub_client.create_subscription(name='parsed_subscription', stream_ids=[self.stream_id], exchange_name='parsed_subscription') self.addCleanup(self.pubsub_client.delete_subscription, self.subscription_id) self.pubsub_client.activate_subscription(self.subscription_id) self.addCleanup(self.pubsub_client.deactivate_subscription, self.subscription_id) self.publisher = StandaloneStreamPublisher(self.stream_id, self.route_id) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_stream_ingestion_worker(self): self.start_ingestion_worker() context_ids, time_ctxt = self._create_param_contexts() pdict_id = self.dataset_management_client.create_parameter_dictionary(name='stream_ingestion_pdict', parameter_context_ids=context_ids, temporal_context='ingestion_timestamp') self.addCleanup(self.dataset_management_client.delete_parameter_dictionary, pdict_id) dataset_id = self.dataset_management_client.create_dataset(name='fake_dataset', description='fake_dataset', stream_id=self.stream_id, parameter_dictionary_id=pdict_id) self.addCleanup(self.dataset_management_client.delete_dataset, dataset_id) self.cov = self._create_coverage(dataset_id=dataset_id, parameter_dict_id=pdict_id, time_dom=self.time_dom, spatial_dom=self.spatial_dom) self.addCleanup(self.cov.close) rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id) rdt['conductivity'] = 1 rdt['pressure'] = 2 rdt['salinity'] = 3 self.start_listener(dataset_id) self.publisher.publish(rdt.to_granule()) self.data_modified = Event() self.data_modified.wait(30) cov = self.get_coverage(dataset_id) self.assertIsNotNone(cov.get_parameter_values('raw')) deserializer = IonObjectDeserializer(obj_registry=get_obj_registry()) granule = retrieve_stream(dataset_id) rdt_complex = RecordDictionaryTool.load_from_granule(granule) rdt_complex['raw'] = [deserializer.deserialize(i) for i in rdt_complex['raw']] for gran in rdt_complex['raw']: rdt_new = RecordDictionaryTool.load_from_granule(gran) self.assertIn(1, rdt_new['conductivity']) self.assertIn(2, rdt_new['pressure']) self.assertIn(3, rdt_new['salinity']) cov.close() def start_ingestion_worker(self): config = DotDict() config.process.queue_name = 'parsed_subscription' self.container.spawn_process( name='stream_ingestion_worker', module='ion.processes.data.ingestion.stream_ingestion_worker', cls='StreamIngestionWorker', config=config ) def start_listener(self, dataset_id): def cb(*args, **kwargs): self.data_modified.set() es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id) es.start() self.addCleanup(es.stop) def _create_param_contexts(self): context_ids = [] t_ctxt = ParameterContext('ingestion_timestamp', param_type=QuantityType(value_encoding=numpy.dtype('float64'))) t_ctxt.uom = 'seconds since 1900-01-01' t_ctxt.fill_value = -9999 t_ctxt_id = self.dataset_management_client.create_parameter_context(name='ingestion_timestamp', parameter_context=t_ctxt.dump()) context_ids.append(t_ctxt_id) raw_ctxt = ParameterContext('raw', param_type=ArrayType()) raw_ctxt.uom = '' context_ids.append(self.dataset_management_client.create_parameter_context(name='raw', parameter_context=raw_ctxt.dump())) return context_ids, t_ctxt_id def _create_coverage(self, dataset_id, parameter_dict_id, time_dom, spatial_dom): pd = self.dataset_management_client.read_parameter_dictionary(parameter_dict_id) pdict = ParameterDictionary.load(pd) sdom = GridDomain.load(spatial_dom.dump()) tdom = GridDomain.load(time_dom.dump()) file_root = FileSystem.get_url(FS.CACHE,'datasets') scov = SimplexCoverage(file_root, dataset_id, dataset_id, parameter_dictionary=pdict, temporal_domain=tdom, spatial_domain=sdom) return scov def get_coverage(self, dataset_id,mode='w'): file_root = FileSystem.get_url(FS.CACHE,'datasets') coverage = AbstractCoverage.load(file_root, dataset_id, mode=mode) return coverage
class TestStreamIngestionWorker(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.dataset_management_client = DatasetManagementServiceClient( node=self.container.node) self.pubsub_client = PubsubManagementServiceClient( node=self.container.node) self.time_dom, self.spatial_dom = time_series_domain() self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name( name='ctd_parsed_param_dict', id_only=True) self.stream_def_id = self.pubsub_client.create_stream_definition( name='stream_def', parameter_dictionary_id=self.parameter_dict_id) self.addCleanup(self.pubsub_client.delete_stream_definition, self.stream_def_id) self.stream_id, self.route_id = self.pubsub_client.create_stream( name='parsed_stream', stream_definition_id=self.stream_def_id, exchange_point='science_data') self.addCleanup(self.pubsub_client.delete_stream, self.stream_id) self.subscription_id = self.pubsub_client.create_subscription( name='parsed_subscription', stream_ids=[self.stream_id], exchange_name='parsed_subscription') self.addCleanup(self.pubsub_client.delete_subscription, self.subscription_id) self.pubsub_client.activate_subscription(self.subscription_id) self.addCleanup(self.pubsub_client.deactivate_subscription, self.subscription_id) self.publisher = StandaloneStreamPublisher(self.stream_id, self.route_id) @attr('LOCOINT') @unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode') def test_stream_ingestion_worker(self): self.start_ingestion_worker() context_ids, time_ctxt = self._create_param_contexts() pdict_id = self.dataset_management_client.create_parameter_dictionary( name='stream_ingestion_pdict', parameter_context_ids=context_ids, temporal_context='ingestion_timestamp') self.addCleanup( self.dataset_management_client.delete_parameter_dictionary, pdict_id) dataset_id = self.dataset_management_client.create_dataset( name='fake_dataset', description='fake_dataset', stream_id=self.stream_id, spatial_domain=self.spatial_dom.dump(), temporal_domain=self.time_dom.dump(), parameter_dictionary_id=pdict_id) self.addCleanup(self.dataset_management_client.delete_dataset, dataset_id) self.cov = self._create_coverage(dataset_id=dataset_id, parameter_dict_id=pdict_id, time_dom=self.time_dom, spatial_dom=self.spatial_dom) self.addCleanup(self.cov.close) rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id) rdt['conductivity'] = 1 rdt['pressure'] = 2 rdt['salinity'] = 3 self.start_listener(dataset_id) self.publisher.publish(rdt.to_granule()) self.data_modified = Event() self.data_modified.wait(30) cov = self.get_coverage(dataset_id) self.assertIsNotNone(cov.get_parameter_values('raw')) deserializer = IonObjectDeserializer(obj_registry=get_obj_registry()) granule = retrieve_stream(dataset_id) rdt_complex = RecordDictionaryTool.load_from_granule(granule) rdt_complex['raw'] = [ deserializer.deserialize(i) for i in rdt_complex['raw'] ] for gran in rdt_complex['raw']: rdt_new = RecordDictionaryTool.load_from_granule(gran) self.assertIn(1, rdt_new['conductivity']) self.assertIn(2, rdt_new['pressure']) self.assertIn(3, rdt_new['salinity']) cov.close() def start_ingestion_worker(self): config = DotDict() config.process.queue_name = 'parsed_subscription' self.container.spawn_process( name='stream_ingestion_worker', module='ion.processes.data.ingestion.stream_ingestion_worker', cls='StreamIngestionWorker', config=config) def start_listener(self, dataset_id): def cb(*args, **kwargs): self.data_modified.set() es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id) es.start() self.addCleanup(es.stop) def _create_param_contexts(self): context_ids = [] t_ctxt = ParameterContext( 'ingestion_timestamp', param_type=QuantityType(value_encoding=numpy.dtype('float64'))) t_ctxt.uom = 'seconds since 1900-01-01' t_ctxt.fill_value = -9999 t_ctxt_id = self.dataset_management_client.create_parameter_context( name='ingestion_timestamp', parameter_context=t_ctxt.dump()) context_ids.append(t_ctxt_id) raw_ctxt = ParameterContext('raw', param_type=ArrayType()) raw_ctxt.uom = '' context_ids.append( self.dataset_management_client.create_parameter_context( name='raw', parameter_context=raw_ctxt.dump())) return context_ids, t_ctxt_id def _create_coverage(self, dataset_id, parameter_dict_id, time_dom, spatial_dom): pd = self.dataset_management_client.read_parameter_dictionary( parameter_dict_id) pdict = ParameterDictionary.load(pd) sdom = GridDomain.load(spatial_dom.dump()) tdom = GridDomain.load(time_dom.dump()) file_root = FileSystem.get_url(FS.CACHE, 'datasets') scov = SimplexCoverage(file_root, dataset_id, dataset_id, parameter_dictionary=pdict, temporal_domain=tdom, spatial_domain=sdom) return scov def get_coverage(self, dataset_id, mode='w'): file_root = FileSystem.get_url(FS.CACHE, 'datasets') coverage = AbstractCoverage.load(file_root, dataset_id, mode=mode) return coverage
class DatasetManagementIntTest(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.resource_registry = ResourceRegistryServiceClient() self.dataset_management = DatasetManagementServiceClient() def test_dataset_crud(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name( 'ctd_parsed_param_dict', id_only=True) tdom, sdom = time_series_domain() dataset_id = self.dataset_management.create_dataset( name='ctd_dataset', parameter_dictionary_id=pdict_id, spatial_domain=sdom.dump(), temporal_domain=tdom.dump()) ds_obj = self.dataset_management.read_dataset(dataset_id) self.assertEquals(ds_obj.name, 'ctd_dataset') ds_obj.name = 'something different' self.dataset_management.update_dataset(ds_obj) self.dataset_management.register_dataset(dataset_id) ds_obj2 = self.dataset_management.read_dataset(dataset_id) self.assertEquals(ds_obj.name, ds_obj2.name) self.assertTrue(ds_obj2.registered) def test_context_crud(self): context_ids = self.create_contexts() context_id = context_ids.pop() context = DatasetManagementService.get_parameter_context(context_id) self.assertIsInstance(context, ParameterContext) self.assertEquals(context.identifier, context_id) self.dataset_management.delete_parameter_context(context_id) with self.assertRaises(NotFound): self.dataset_management.read_parameter_context(context_id) def test_pfunc_crud(self): contexts, funcs = self.create_pfuncs() context_ids = [ context_id for ctxt, context_id in contexts.itervalues() ] pdict_id = self.dataset_management.create_parameter_dictionary( name='functional_pdict', parameter_context_ids=context_ids, temporal_context='time') self.addCleanup(self.dataset_management.delete_parameter_dictionary, pdict_id) expr, expr_id = funcs['CONDWAT_L1'] func_class = DatasetManagementService.get_parameter_function(expr_id) self.assertIsInstance(func_class, NumexprFunction) def test_pdict_crud(self): context_ids = self.create_contexts() pdict_res_id = self.dataset_management.create_parameter_dictionary( name='pdict1', parameter_context_ids=context_ids, temporal_context='time') pdict_contexts = self.dataset_management.read_parameter_contexts( parameter_dictionary_id=pdict_res_id, id_only=True) pdict = DatasetManagementService.get_parameter_dictionary(pdict_res_id) self.assertIsInstance(pdict, ParameterDictionary) self.assertTrue('time_test' in pdict) self.assertEquals(pdict.identifier, pdict_res_id) self.assertEquals(set(pdict_contexts), set(context_ids)) self.dataset_management.delete_parameter_dictionary( parameter_dictionary_id=pdict_res_id) with self.assertRaises(NotFound): self.dataset_management.read_parameter_dictionary( parameter_dictionary_id=pdict_res_id) def create_contexts(self): context_ids = [] cond_ctxt = ParameterContext( 'conductivity_test', param_type=QuantityType(value_encoding=np.float32)) cond_ctxt.uom = 'unknown' cond_ctxt.fill_value = 0e0 context_ids.append( self.dataset_management.create_parameter_context( name='conductivity_test', parameter_context=cond_ctxt.dump())) pres_ctxt = ParameterContext( 'pressure_test', param_type=QuantityType(value_encoding=np.float32)) pres_ctxt.uom = 'Pascal' pres_ctxt.fill_value = 0x0 context_ids.append( self.dataset_management.create_parameter_context( name='pressure_test', parameter_context=pres_ctxt.dump())) sal_ctxt = ParameterContext( 'salinity_test', param_type=QuantityType(value_encoding=np.float32)) sal_ctxt.uom = 'PSU' sal_ctxt.fill_value = 0x0 context_ids.append( self.dataset_management.create_parameter_context( name='salinity_test', parameter_context=sal_ctxt.dump())) temp_ctxt = ParameterContext( 'temp_test', param_type=QuantityType(value_encoding=np.float32)) temp_ctxt.uom = 'degree_Celsius' temp_ctxt.fill_value = 0e0 context_ids.append( self.dataset_management.create_parameter_context( name='temp_test', parameter_context=temp_ctxt.dump())) t_ctxt = ParameterContext( 'time_test', param_type=QuantityType(value_encoding=np.int64)) t_ctxt.uom = 'seconds since 1970-01-01' t_ctxt.fill_value = 0x0 context_ids.append( self.dataset_management.create_parameter_context( name='time_test', parameter_context=t_ctxt.dump())) return context_ids def create_pfuncs(self): contexts = {} funcs = {} t_ctxt = ParameterContext( 'TIME', param_type=QuantityType(value_encoding=np.dtype('int64'))) t_ctxt.uom = 'seconds since 01-01-1900' t_ctxt_id = self.dataset_management.create_parameter_context( name='test_TIME', parameter_context=t_ctxt.dump()) contexts['TIME'] = (t_ctxt, t_ctxt_id) lat_ctxt = ParameterContext( 'LAT', param_type=ConstantType( QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999) lat_ctxt.axis = AxisTypeEnum.LAT lat_ctxt.uom = 'degree_north' lat_ctxt_id = self.dataset_management.create_parameter_context( name='test_LAT', parameter_context=lat_ctxt.dump()) contexts['LAT'] = lat_ctxt, lat_ctxt_id lon_ctxt = ParameterContext( 'LON', param_type=ConstantType( QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999) lon_ctxt.axis = AxisTypeEnum.LON lon_ctxt.uom = 'degree_east' lon_ctxt_id = self.dataset_management.create_parameter_context( name='test_LON', parameter_context=lon_ctxt.dump()) contexts['LON'] = lon_ctxt, lon_ctxt_id # Independent Parameters # Temperature - values expected to be the decimal results of conversion from hex temp_ctxt = ParameterContext( 'TEMPWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) temp_ctxt.uom = 'deg_C' temp_ctxt_id = self.dataset_management.create_parameter_context( name='test_TEMPWAT_L0', parameter_context=temp_ctxt.dump()) contexts['TEMPWAT_L0'] = temp_ctxt, temp_ctxt_id # Conductivity - values expected to be the decimal results of conversion from hex cond_ctxt = ParameterContext( 'CONDWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) cond_ctxt.uom = 'S m-1' cond_ctxt_id = self.dataset_management.create_parameter_context( name='test_CONDWAT_L0', parameter_context=cond_ctxt.dump()) contexts['CONDWAT_L0'] = cond_ctxt, cond_ctxt_id # Pressure - values expected to be the decimal results of conversion from hex press_ctxt = ParameterContext( 'PRESWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999) press_ctxt.uom = 'dbar' press_ctxt_id = self.dataset_management.create_parameter_context( name='test_PRESWAT_L0', parameter_context=press_ctxt.dump()) contexts['PRESWAT_L0'] = press_ctxt, press_ctxt_id # Dependent Parameters # TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10 tl1_func = '(T / 10000) - 10' expr = NumexprFunction('TEMPWAT_L1', tl1_func, ['T']) expr_id = self.dataset_management.create_parameter_function( name='test_TEMPWAT_L1', parameter_function=expr.dump()) funcs['TEMPWAT_L1'] = expr, expr_id tl1_pmap = {'T': 'TEMPWAT_L0'} expr.param_map = tl1_pmap tempL1_ctxt = ParameterContext( 'TEMPWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) tempL1_ctxt.uom = 'deg_C' tempL1_ctxt_id = self.dataset_management.create_parameter_context( name='test_TEMPWAT_L1', parameter_context=tempL1_ctxt.dump(), parameter_function_id=expr_id) contexts['TEMPWAT_L1'] = tempL1_ctxt, tempL1_ctxt_id # CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5 cl1_func = '(C / 100000) - 0.5' expr = NumexprFunction('CONDWAT_L1', cl1_func, ['C']) expr_id = self.dataset_management.create_parameter_function( name='test_CONDWAT_L1', parameter_function=expr.dump()) funcs['CONDWAT_L1'] = expr, expr_id cl1_pmap = {'C': 'CONDWAT_L0'} expr.param_map = cl1_pmap condL1_ctxt = ParameterContext( 'CONDWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) condL1_ctxt.uom = 'S m-1' condL1_ctxt_id = self.dataset_management.create_parameter_context( name='test_CONDWAT_L1', parameter_context=condL1_ctxt.dump(), parameter_function_id=expr_id) contexts['CONDWAT_L1'] = condL1_ctxt, condL1_ctxt_id # Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721 # PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range) pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)' expr = NumexprFunction('PRESWAT_L1', pl1_func, ['P', 'p_range']) expr_id = self.dataset_management.create_parameter_function( name='test_PRESWAT_L1', parameter_function=expr.dump()) funcs['PRESWAT_L1'] = expr, expr_id pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721} expr.param_map = pl1_pmap presL1_ctxt = ParameterContext( 'PRESWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL) presL1_ctxt.uom = 'S m-1' presL1_ctxt_id = self.dataset_management.create_parameter_context( name='test_CONDWAT_L1', parameter_context=presL1_ctxt.dump(), parameter_function_id=expr_id) contexts['PRESWAT_L1'] = presL1_ctxt, presL1_ctxt_id # Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project: # https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1 # PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1) owner = 'gsw' sal_func = 'SP_from_C' sal_arglist = ['C', 't', 'p'] expr = PythonFunction('PRACSAL', owner, sal_func, sal_arglist) expr_id = self.dataset_management.create_parameter_function( name='test_PRACSAL', parameter_function=expr.dump()) funcs['PRACSAL'] = expr, expr_id # A magic function that may or may not exist actually forms the line below at runtime. sal_pmap = { 'C': NumexprFunction('CONDWAT_L1*10', 'C*10', ['C'], param_map={'C': 'CONDWAT_L1'}), 't': 'TEMPWAT_L1', 'p': 'PRESWAT_L1' } expr.param_map = sal_pmap sal_ctxt = ParameterContext('PRACSAL', param_type=ParameterFunctionType(expr), variability=VariabilityEnum.TEMPORAL) sal_ctxt.uom = 'g kg-1' sal_ctxt_id = self.dataset_management.create_parameter_context( name='test_PRACSAL', parameter_context=sal_ctxt.dump(), parameter_function_id=expr_id) contexts['PRACSAL'] = sal_ctxt, sal_ctxt_id # absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude) # conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1) # DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1) owner = 'gsw' abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP', ['PRACSAL', 'PRESWAT_L1', 'LON', 'LAT']) cons_temp_expr = PythonFunction( 'cons_temp', owner, 'CT_from_t', [abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1']) dens_expr = PythonFunction( 'DENSITY', owner, 'rho', [abs_sal_expr, cons_temp_expr, 'PRESWAT_L1']) dens_ctxt = ParameterContext( 'DENSITY', param_type=ParameterFunctionType(dens_expr), variability=VariabilityEnum.TEMPORAL) dens_ctxt.uom = 'kg m-3' dens_ctxt_id = self.dataset_management.create_parameter_context( name='test_DENSITY', parameter_context=dens_ctxt.dump()) contexts['DENSITY'] = dens_ctxt, dens_ctxt_id return contexts, funcs def test_verify_contexts(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name( name='ctd_parsed_param_dict', id_only=True) pcontexts = self.dataset_management.read_parameter_contexts( parameter_dictionary_id=pdict_id) for pcontext in pcontexts: self.assertTrue('fill_value' in pcontext) self.assertTrue('reference_urls' in pcontext) self.assertTrue('internal_name' in pcontext) self.assertTrue('display_name' in pcontext) self.assertTrue('standard_name' in pcontext) self.assertTrue('ooi_short_name' in pcontext) self.assertTrue('description' in pcontext) self.assertTrue('precision' in pcontext)
class DatasetManagementIntTest(IonIntegrationTestCase): def setUp(self): self._start_container() self.container.start_rel_from_url('res/deploy/r2deploy.yml') self.resource_registry = ResourceRegistryServiceClient() self.dataset_management = DatasetManagementServiceClient() def test_dataset_crud(self): pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True) tdom, sdom = time_series_domain() dataset_id = self.dataset_management.create_dataset(name='ctd_dataset', parameter_dictionary_id=pdict_id, spatial_domain=sdom.dump(), temporal_domain=tdom.dump()) ds_obj = self.dataset_management.read_dataset(dataset_id) self.assertEquals(ds_obj.name, 'ctd_dataset') ds_obj.name = 'something different' self.dataset_management.update_dataset(ds_obj) self.dataset_management.register_dataset(dataset_id) ds_obj2 = self.dataset_management.read_dataset(dataset_id) self.assertEquals(ds_obj.name, ds_obj2.name) self.assertTrue(ds_obj2.registered) def test_context_crud(self): context_ids = self.create_contexts() context_id = context_ids.pop() context = DatasetManagementService.get_parameter_context(context_id) self.assertIsInstance(context, ParameterContext) self.assertEquals(context.identifier, context_id) self.dataset_management.delete_parameter_context(context_id) with self.assertRaises(NotFound): self.dataset_management.read_parameter_context(context_id) def test_pdict_crud(self): context_ids = self.create_contexts() pdict_res_id = self.dataset_management.create_parameter_dictionary(name='pdict1', parameter_context_ids=context_ids, temporal_context='time') pdict_contexts = self.dataset_management.read_parameter_contexts(parameter_dictionary_id=pdict_res_id, id_only=True) pdict = DatasetManagementService.get_parameter_dictionary(pdict_res_id) self.assertIsInstance(pdict, ParameterDictionary) self.assertTrue('time_test' in pdict) self.assertEquals(pdict.identifier, pdict_res_id) self.assertEquals(set(pdict_contexts), set(context_ids)) self.dataset_management.delete_parameter_dictionary(parameter_dictionary_id=pdict_res_id) with self.assertRaises(NotFound): self.dataset_management.read_parameter_dictionary(parameter_dictionary_id=pdict_res_id) def create_contexts(self): context_ids = [] cond_ctxt = ParameterContext('conductivity_test', param_type=QuantityType(value_encoding=np.float32)) cond_ctxt.uom = 'unknown' cond_ctxt.fill_value = 0e0 context_ids.append(self.dataset_management.create_parameter_context(name='conductivity_test', parameter_context=cond_ctxt.dump())) pres_ctxt = ParameterContext('pressure_test', param_type=QuantityType(value_encoding=np.float32)) pres_ctxt.uom = 'Pascal' pres_ctxt.fill_value = 0x0 context_ids.append(self.dataset_management.create_parameter_context(name='pressure_test', parameter_context=pres_ctxt.dump())) sal_ctxt = ParameterContext('salinity_test', param_type=QuantityType(value_encoding=np.float32)) sal_ctxt.uom = 'PSU' sal_ctxt.fill_value = 0x0 context_ids.append(self.dataset_management.create_parameter_context(name='salinity_test', parameter_context=sal_ctxt.dump())) temp_ctxt = ParameterContext('temp_test', param_type=QuantityType(value_encoding=np.float32)) temp_ctxt.uom = 'degree_Celsius' temp_ctxt.fill_value = 0e0 context_ids.append(self.dataset_management.create_parameter_context(name='temp_test', parameter_context=temp_ctxt.dump())) t_ctxt = ParameterContext('time_test', param_type=QuantityType(value_encoding=np.int64)) t_ctxt.uom = 'seconds since 1970-01-01' t_ctxt.fill_value = 0x0 context_ids.append(self.dataset_management.create_parameter_context(name='time_test', parameter_context=t_ctxt.dump())) return context_ids