def get_parameter_dictionary(cls, parameter_dictionary_id=''):
"""
Class method to return a CoverageModel ParameterDictionary object from the
ION Resources. The object is built from the associated parameter contexts.
"""
dms_cli = DatasetManagementServiceClient()
pd = dms_cli.read_parameter_dictionary(parameter_dictionary_id)
pcs = dms_cli.read_parameter_contexts(parameter_dictionary_id=parameter_dictionary_id, id_only=False)
return cls.build_parameter_dictionary(pd, pcs)
def get_parameter_dictionary(cls, parameter_dictionary_id=''):
'''
Preferred client-side class method for constructing a parameter dictionary
from a service call.
'''
dms_cli = DatasetManagementServiceClient()
pd = dms_cli.read_parameter_dictionary(parameter_dictionary_id)
pcs = dms_cli.read_parameter_contexts(parameter_dictionary_id=parameter_dictionary_id, id_only=False)
return cls.build_parameter_dictionary(pd, pcs)
def get_parameter_dictionary(cls, parameter_dictionary_id=''):
'''
Preferred client-side class method for constructing a parameter dictionary
from a service call.
'''
dms_cli = DatasetManagementServiceClient()
pd = dms_cli.read_parameter_dictionary(parameter_dictionary_id)
pcs = dms_cli.read_parameter_contexts(
parameter_dictionary_id=parameter_dictionary_id, id_only=False)
return cls.build_parameter_dictionary(pd, pcs)
def get_parameter_dictionary(cls, parameter_dictionary_id=''):
"""
Class method to return a CoverageModel ParameterDictionary object from the
ION Resources. The object is built from the associated parameter contexts.
"""
dms_cli = DatasetManagementServiceClient()
pd = dms_cli.read_parameter_dictionary(parameter_dictionary_id)
pcs = dms_cli.read_parameter_contexts(
parameter_dictionary_id=parameter_dictionary_id, id_only=False)
return cls.build_parameter_dictionary(pd, pcs)
def get_parameter_dictionary(cls, parameter_dictionary_id=''):
'''
Preferred client-side class method for constructing a parameter dictionary
from a service call.
'''
dms_cli = DatasetManagementServiceClient()
pd = dms_cli.read_parameter_dictionary(parameter_dictionary_id)
pcs = dms_cli.read_parameter_contexts(parameter_dictionary_id=parameter_dictionary_id, id_only=False)
pdict = cls._merge_contexts([ParameterContext.load(i.parameter_context) for i in pcs], pd.temporal_context)
pdict._identifier = parameter_dictionary_id
return pdict
def get_parameter_dictionary(cls, parameter_dictionary_id=''):
'''
Preferred client-side class method for constructing a parameter dictionary
from a service call.
'''
dms_cli = DatasetManagementServiceClient()
pd = dms_cli.read_parameter_dictionary(parameter_dictionary_id)
pcs = dms_cli.read_parameter_contexts(
parameter_dictionary_id=parameter_dictionary_id, id_only=False)
pdict = cls._merge_contexts(
[ParameterContext.load(i.parameter_context) for i in pcs],
pd.temporal_context)
pdict._identifier = parameter_dictionary_id
return pdict
class DatasetManagementIntTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.resource_registry = ResourceRegistryServiceClient()
self.dataset_management = DatasetManagementServiceClient()
def test_dataset_crud(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
dataset = Dataset(name='ctd_dataset')
dataset_id = self.dataset_management.create_dataset(
dataset, parameter_dictionary_id=pdict_id)
ds_obj = self.dataset_management.read_dataset(dataset_id)
self.assertEquals(ds_obj.name, 'ctd_dataset')
ds_obj.name = 'something different'
self.dataset_management.update_dataset(ds_obj)
ds_obj2 = self.dataset_management.read_dataset(dataset_id)
self.assertEquals(ds_obj.name, ds_obj2.name)
def test_context_crud(self):
context_ids = self.create_contexts()
context_id = context_ids.pop()
ctxt = self.dataset_management.read_parameter_context(context_id)
context = DatasetManagementService.get_coverage_parameter(ctxt)
self.assertIsInstance(context, CoverageParameterContext)
self.dataset_management.delete_parameter_context(context_id)
with self.assertRaises(NotFound):
self.dataset_management.read_parameter_context(context_id)
def test_pfunc_crud(self):
contexts, funcs = self.create_pfuncs()
context_ids = [context_id for context_id in contexts.itervalues()]
pdict_id = self.dataset_management.create_parameter_dictionary(
name='functional_pdict',
parameter_context_ids=context_ids,
temporal_context='time')
self.addCleanup(self.dataset_management.delete_parameter_dictionary,
pdict_id)
expr_id = funcs['CONDWAT_L1']
expr = self.dataset_management.read_parameter_function(expr_id)
func_class = DatasetManagementService.get_coverage_function(expr)
self.assertIsInstance(func_class, NumexprFunction)
def test_pdict_crud(self):
context_ids = self.create_contexts()
pdict_res_id = self.dataset_management.create_parameter_dictionary(
name='pdict1',
parameter_context_ids=context_ids,
temporal_context='time')
pdict_contexts = self.dataset_management.read_parameter_contexts(
parameter_dictionary_id=pdict_res_id, id_only=True)
pdict = DatasetManagementService.get_parameter_dictionary(pdict_res_id)
self.assertIsInstance(pdict, ParameterDictionary)
self.assertTrue('time_test' in pdict)
self.assertEquals(pdict.identifier, pdict_res_id)
self.assertEquals(set(pdict_contexts), set(context_ids))
self.dataset_management.delete_parameter_dictionary(
parameter_dictionary_id=pdict_res_id)
with self.assertRaises(NotFound):
self.dataset_management.read_parameter_dictionary(
parameter_dictionary_id=pdict_res_id)
def create_contexts(self):
context_ids = []
cond = ParameterContext(name='condictivity_test',
parameter_type='quantity',
value_encoding='float32',
units='1',
fill_value=0)
context_ids.append(self.dataset_management.create_parameter(cond))
pres = ParameterContext(name='pressure_test',
parameter_type='quantity',
value_encoding='float32',
units='Pa',
fill_value=0)
context_ids.append(self.dataset_management.create_parameter(pres))
sal = ParameterContext(name='salinity_test',
parameter_type='quantity',
value_encoding='float32',
units='psu',
fill_value=0)
context_ids.append(self.dataset_management.create_parameter(sal))
temp = ParameterContext(name='temp_test',
parameter_type='quantity',
value_encoding='float32',
units='degree_C',
fill_value=0)
context_ids.append(self.dataset_management.create_parameter(temp))
time_test = ParameterContext(name='time_test',
parameter_type='quantity',
value_encoding='float32',
units='seconds since 1970-01-01',
fill_value=0)
context_ids.append(self.dataset_management.create_parameter(time_test))
return context_ids
def create_pfuncs(self):
contexts = {}
funcs = {}
time_ = ParameterContext(name='TIME',
parameter_type='quantity',
value_encoding='float32',
units='seconds since 1900-01-01',
fill_value=0)
t_ctxt_id = self.dataset_management.create_parameter(time_)
contexts['TIME'] = t_ctxt_id
lat = ParameterContext(name='LAT',
parameter_type='sparse',
value_encoding='float32',
units='degrees_north',
fill_value=-9999.)
lat_ctxt_id = self.dataset_management.create_parameter(lat)
contexts['LAT'] = lat_ctxt_id
lon = ParameterContext(name='LON',
parameter_type="sparse",
value_encoding='float32',
units='degrees_east',
fill_value=-9999)
lon_ctxt_id = self.dataset_management.create_parameter(lon)
contexts['LON'] = lon_ctxt_id
# Independent Parameters
# Temperature - values expected to be the decimal results of conversion from hex
temp = ParameterContext(name='TEMPWAT_L0',
parameter_type='quantity',
value_encoding='float32',
units='deg_C')
temp_ctxt_id = self.dataset_management.create_parameter(temp)
contexts['TEMPWAT_L0'] = temp_ctxt_id
# Conductivity - values expected to be the decimal results of conversion from hex
cond = ParameterContext(name='CONDWAT_L0',
parameter_type='quantity',
value_encoding='float32',
units='S m-1')
cond_ctxt_id = self.dataset_management.create_parameter(cond)
contexts['CONDWAT_L0'] = cond_ctxt_id
# Pressure - values expected to be the decimal results of conversion from hex
press = ParameterContext(name='PRESWAT_L0',
parameter_type='quantity',
value_encoding='float32',
units='dbar')
press_ctxt_id = self.dataset_management.create_parameter(press)
contexts['PRESWAT_L0'] = press_ctxt_id
# Dependent Parameters
# TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10
tl1_func = '(T / 10000) - 10'
tempwat_f = ParameterFunction(name='TEMPWAT_L1',
function_type=PFT.NUMEXPR,
function=tl1_func,
args=['T'])
expr_id = self.dataset_management.create_parameter_function(tempwat_f)
funcs['TEMPWAT_L1'] = expr_id
tl1_pmap = {'T': 'TEMPWAT_L0'}
tempL1 = ParameterContext(name='TEMPWAT_L1',
parameter_type='function',
parameter_function_id=expr_id,
parameter_function_map=tl1_pmap,
value_encoding='float32',
units='deg_C')
tempL1_ctxt_id = self.dataset_management.create_parameter(tempL1)
contexts['TEMPWAT_L1'] = tempL1_ctxt_id
# CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5
cl1_func = '(C / 100000) - 0.5'
condwat_f = ParameterFunction(name='CONDWAT_L1',
function_type=PFT.NUMEXPR,
function=cl1_func,
args=['C'])
expr_id = self.dataset_management.create_parameter_function(condwat_f)
funcs['CONDWAT_L1'] = expr_id
cl1_pmap = {'C': 'CONDWAT_L0'}
condL1 = ParameterContext(name='CONDWAT_L1',
parameter_type='function',
parameter_function_id=expr_id,
parameter_function_map=cl1_pmap,
value_encoding='float32',
units='S m-1')
condL1_ctxt_id = self.dataset_management.create_parameter(condL1)
contexts['CONDWAT_L1'] = condL1_ctxt_id
# Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721
# PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range)
pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)'
preswat_f = ParameterFunction(name='PRESWAT_L1',
function_type=PFT.NUMEXPR,
function=pl1_func,
args=['P', 'p_range'])
expr_id = self.dataset_management.create_parameter_function(preswat_f)
funcs['PRESWAT_L1'] = expr_id
pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721}
presL1 = ParameterContext(name='PRESWAT_L1',
parameter_type='function',
parameter_function_id=expr_id,
parameter_function_map=pl1_pmap,
value_encoding='float32',
units='dbar')
presL1_ctxt_id = self.dataset_management.create_parameter(presL1)
contexts['PRESWAT_L1'] = presL1_ctxt_id
# A magic function that may or may not exist actually forms the line below at runtime.
cond_f = ParameterFunction(name='condwat10',
function_type=PFT.NUMEXPR,
function='C*10',
args=['C'])
expr_id = self.dataset_management.create_parameter_function(cond_f)
cond10 = ParameterContext(name='c10',
parameter_type='function',
parameter_function_id=expr_id,
parameter_function_map={'C': 'CONDWAT_L1'},
value_encoding='float32',
units='1')
cond10_id = self.dataset_management.create_parameter(cond10)
contexts['C10'] = cond10_id
# Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project:
# https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1
# PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1)
owner = 'gsw'
sal_func = 'SP_from_C'
sal_arglist = ['C', 't', 'p']
pracsal_f = ParameterFunction(name='PRACSAL',
function_type=PFT.PYTHON,
owner=owner,
function=sal_func,
args=sal_arglist)
expr_id = self.dataset_management.create_parameter_function(pracsal_f)
funcs['PRACSAL'] = expr_id
sal_pmap = {'C': 'c10', 't': 'TEMPWAT_L1', 'p': 'PRESWAT_L1'}
sal_ctxt = ParameterContext(name='PRACSAL',
parameter_type='function',
parameter_function_id=expr_id,
parameter_function_map=sal_pmap,
value_encoding='float32',
units='g kg-1')
sal_ctxt_id = self.dataset_management.create_parameter(sal_ctxt)
contexts['PRACSAL'] = sal_ctxt_id
# absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude)
# conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1)
# DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1)
return contexts, funcs
def test_verify_contexts(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
name='ctd_parsed_param_dict', id_only=True)
pcontexts = self.dataset_management.read_parameter_contexts(
parameter_dictionary_id=pdict_id)
for pcontext in pcontexts:
self.assertTrue('fill_value' in pcontext)
self.assertTrue('reference_urls' in pcontext)
self.assertTrue('internal_name' in pcontext)
self.assertTrue('display_name' in pcontext)
self.assertTrue('standard_name' in pcontext)
self.assertTrue('ooi_short_name' in pcontext)
self.assertTrue('description' in pcontext)
self.assertTrue('precision' in pcontext)
class DatasetManagementIntTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.resource_registry = ResourceRegistryServiceClient()
self.dataset_management = DatasetManagementServiceClient()
def test_dataset_crud(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
tdom, sdom = time_series_domain()
dataset_id = self.dataset_management.create_dataset(name='ctd_dataset', parameter_dictionary_id=pdict_id, spatial_domain=sdom.dump(), temporal_domain=tdom.dump())
ds_obj = self.dataset_management.read_dataset(dataset_id)
self.assertEquals(ds_obj.name, 'ctd_dataset')
ds_obj.name = 'something different'
self.dataset_management.update_dataset(ds_obj)
ds_obj2 = self.dataset_management.read_dataset(dataset_id)
self.assertEquals(ds_obj.name, ds_obj2.name)
def test_context_crud(self):
context_ids = self.create_contexts()
context_id = context_ids.pop()
ctxt = self.dataset_management.read_parameter_context(context_id)
context = DatasetManagementService.get_coverage_parameter(ctxt)
self.assertIsInstance(context, CoverageParameterContext)
self.dataset_management.delete_parameter_context(context_id)
with self.assertRaises(NotFound):
self.dataset_management.read_parameter_context(context_id)
def test_pfunc_crud(self):
contexts, funcs = self.create_pfuncs()
context_ids = [context_id for context_id in contexts.itervalues()]
pdict_id = self.dataset_management.create_parameter_dictionary(name='functional_pdict', parameter_context_ids=context_ids, temporal_context='time')
self.addCleanup(self.dataset_management.delete_parameter_dictionary, pdict_id)
expr_id = funcs['CONDWAT_L1']
expr = self.dataset_management.read_parameter_function(expr_id)
func_class = DatasetManagementService.get_coverage_function(expr)
self.assertIsInstance(func_class, NumexprFunction)
def test_pdict_crud(self):
context_ids = self.create_contexts()
pdict_res_id = self.dataset_management.create_parameter_dictionary(name='pdict1', parameter_context_ids=context_ids, temporal_context='time')
pdict_contexts = self.dataset_management.read_parameter_contexts(parameter_dictionary_id=pdict_res_id, id_only=True)
pdict = DatasetManagementService.get_parameter_dictionary(pdict_res_id)
self.assertIsInstance(pdict, ParameterDictionary)
self.assertTrue('time_test' in pdict)
self.assertEquals(pdict.identifier, pdict_res_id)
self.assertEquals(set(pdict_contexts), set(context_ids))
self.dataset_management.delete_parameter_dictionary(parameter_dictionary_id=pdict_res_id)
with self.assertRaises(NotFound):
self.dataset_management.read_parameter_dictionary(parameter_dictionary_id=pdict_res_id)
def create_contexts(self):
context_ids = []
cond = ParameterContext(name='condictivity_test',
parameter_type='quantity',
value_encoding='float32',
units='1',
fill_value=0)
context_ids.append(self.dataset_management.create_parameter(cond))
pres = ParameterContext(name='pressure_test',
parameter_type='quantity',
value_encoding='float32',
units='Pa',
fill_value=0)
context_ids.append(self.dataset_management.create_parameter(pres))
sal = ParameterContext(name='salinity_test',
parameter_type='quantity',
value_encoding='float32',
units='psu',
fill_value=0)
context_ids.append(self.dataset_management.create_parameter(sal))
temp = ParameterContext(name='temp_test',
parameter_type='quantity',
value_encoding='float32',
units='degree_C',
fill_value=0)
context_ids.append(self.dataset_management.create_parameter(temp))
time_test = ParameterContext(name='time_test',
parameter_type='quantity',
value_encoding='float32',
units='seconds since 1970-01-01',
fill_value=0)
context_ids.append(self.dataset_management.create_parameter(time_test))
return context_ids
def create_pfuncs(self):
contexts = {}
funcs = {}
time_ = ParameterContext(name='TIME',
parameter_type='quantity',
value_encoding='float32',
units='seconds since 1900-01-01',
fill_value=0)
t_ctxt_id = self.dataset_management.create_parameter(time_)
contexts['TIME'] = t_ctxt_id
lat = ParameterContext(name='LAT',
parameter_type='sparse',
value_encoding='float32',
units='degrees_north',
fill_value=-9999.)
lat_ctxt_id = self.dataset_management.create_parameter(lat)
contexts['LAT'] = lat_ctxt_id
lon = ParameterContext(name='LON',
parameter_type="sparse",
value_encoding='float32',
units='degrees_east',
fill_value=-9999)
lon_ctxt_id = self.dataset_management.create_parameter(lon)
contexts['LON'] = lon_ctxt_id
# Independent Parameters
# Temperature - values expected to be the decimal results of conversion from hex
temp = ParameterContext(name='TEMPWAT_L0',
parameter_type='quantity',
value_encoding='float32',
units='deg_C')
temp_ctxt_id = self.dataset_management.create_parameter(temp)
contexts['TEMPWAT_L0'] = temp_ctxt_id
# Conductivity - values expected to be the decimal results of conversion from hex
cond = ParameterContext(name='CONDWAT_L0',
parameter_type='quantity',
value_encoding='float32',
units='S m-1')
cond_ctxt_id = self.dataset_management.create_parameter(cond)
contexts['CONDWAT_L0'] = cond_ctxt_id
# Pressure - values expected to be the decimal results of conversion from hex
press = ParameterContext(name='PRESWAT_L0',
parameter_type='quantity',
value_encoding='float32',
units='dbar')
press_ctxt_id = self.dataset_management.create_parameter(press)
contexts['PRESWAT_L0'] = press_ctxt_id
# Dependent Parameters
# TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10
tl1_func = '(T / 10000) - 10'
tempwat_f = ParameterFunction(name='TEMPWAT_L1',
function_type=PFT.NUMEXPR,
function=tl1_func,
args=['T'])
expr_id = self.dataset_management.create_parameter_function(tempwat_f)
funcs['TEMPWAT_L1'] = expr_id
tl1_pmap = {'T': 'TEMPWAT_L0'}
tempL1 = ParameterContext(name='TEMPWAT_L1',
parameter_type='function',
parameter_function_id=expr_id,
parameter_function_map=tl1_pmap,
value_encoding='float32',
units='deg_C')
tempL1_ctxt_id = self.dataset_management.create_parameter(tempL1)
contexts['TEMPWAT_L1'] = tempL1_ctxt_id
# CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5
cl1_func = '(C / 100000) - 0.5'
condwat_f = ParameterFunction(name='CONDWAT_L1',
function_type=PFT.NUMEXPR,
function=cl1_func,
args=['C'])
expr_id = self.dataset_management.create_parameter_function(condwat_f)
funcs['CONDWAT_L1'] = expr_id
cl1_pmap = {'C': 'CONDWAT_L0'}
condL1 = ParameterContext(name='CONDWAT_L1',
parameter_type='function',
parameter_function_id=expr_id,
parameter_function_map=cl1_pmap,
value_encoding='float32',
units='S m-1')
condL1_ctxt_id = self.dataset_management.create_parameter(condL1)
contexts['CONDWAT_L1'] = condL1_ctxt_id
# Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721
# PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range)
pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)'
preswat_f = ParameterFunction(name='PRESWAT_L1',
function_type=PFT.NUMEXPR,
function=pl1_func,
args=['P', 'p_range'])
expr_id = self.dataset_management.create_parameter_function(preswat_f)
funcs['PRESWAT_L1'] = expr_id
pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721}
presL1 = ParameterContext(name='PRESWAT_L1',
parameter_type='function',
parameter_function_id=expr_id,
parameter_function_map=pl1_pmap,
value_encoding='float32',
units='dbar')
presL1_ctxt_id = self.dataset_management.create_parameter(presL1)
contexts['PRESWAT_L1'] = presL1_ctxt_id
# A magic function that may or may not exist actually forms the line below at runtime.
cond_f = ParameterFunction(name='condwat10',
function_type=PFT.NUMEXPR,
function='C*10',
args=['C'])
expr_id = self.dataset_management.create_parameter_function(cond_f)
cond10 = ParameterContext(name='c10',
parameter_type='function',
parameter_function_id=expr_id,
parameter_function_map={'C':'CONDWAT_L1'},
value_encoding='float32',
units='1')
cond10_id = self.dataset_management.create_parameter(cond10)
contexts['C10'] = cond10_id
# Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project:
# https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1
# PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1)
owner = 'gsw'
sal_func = 'SP_from_C'
sal_arglist = ['C', 't', 'p']
pracsal_f = ParameterFunction(name='PRACSAL',
function_type=PFT.PYTHON,
owner=owner,
function=sal_func,
args=sal_arglist)
expr_id = self.dataset_management.create_parameter_function(pracsal_f)
funcs['PRACSAL'] = expr_id
sal_pmap = {'C': 'c10', 't': 'TEMPWAT_L1', 'p': 'PRESWAT_L1'}
sal_ctxt = ParameterContext(name='PRACSAL',
parameter_type='function',
parameter_function_id=expr_id,
parameter_function_map=sal_pmap,
value_encoding='float32',
units='g kg-1')
sal_ctxt_id = self.dataset_management.create_parameter(sal_ctxt)
contexts['PRACSAL'] = sal_ctxt_id
# absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude)
# conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1)
# DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1)
return contexts, funcs
def test_verify_contexts(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(name='ctd_parsed_param_dict', id_only=True)
pcontexts = self.dataset_management.read_parameter_contexts(parameter_dictionary_id=pdict_id)
for pcontext in pcontexts:
self.assertTrue('fill_value' in pcontext)
self.assertTrue('reference_urls' in pcontext)
self.assertTrue('internal_name' in pcontext)
self.assertTrue('display_name' in pcontext)
self.assertTrue('standard_name' in pcontext)
self.assertTrue('ooi_short_name' in pcontext)
self.assertTrue('description' in pcontext)
self.assertTrue('precision' in pcontext)
class DatasetManagementIntTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.resource_registry = ResourceRegistryServiceClient()
self.dataset_management = DatasetManagementServiceClient()
def test_dataset_crud(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
tdom, sdom = time_series_domain()
dataset_id = self.dataset_management.create_dataset(name='ctd_dataset', parameter_dictionary_id=pdict_id, spatial_domain=sdom.dump(), temporal_domain=tdom.dump())
ds_obj = self.dataset_management.read_dataset(dataset_id)
self.assertEquals(ds_obj.name, 'ctd_dataset')
ds_obj.name = 'something different'
self.dataset_management.update_dataset(ds_obj)
self.dataset_management.register_dataset(dataset_id)
ds_obj2 = self.dataset_management.read_dataset(dataset_id)
self.assertEquals(ds_obj.name, ds_obj2.name)
self.assertTrue(ds_obj2.registered)
def test_context_crud(self):
context_ids = self.create_contexts()
context_id = context_ids.pop()
context = DatasetManagementService.get_parameter_context(context_id)
self.assertIsInstance(context, ParameterContext)
self.assertEquals(context.identifier, context_id)
self.dataset_management.delete_parameter_context(context_id)
with self.assertRaises(NotFound):
self.dataset_management.read_parameter_context(context_id)
def test_pfunc_crud(self):
contexts, funcs = self.create_pfuncs()
context_ids = [context_id for ctxt,context_id in contexts.itervalues()]
pdict_id = self.dataset_management.create_parameter_dictionary(name='functional_pdict', parameter_context_ids=context_ids, temporal_context='time')
self.addCleanup(self.dataset_management.delete_parameter_dictionary, pdict_id)
expr, expr_id = funcs['CONDWAT_L1']
func_class = DatasetManagementService.get_parameter_function(expr_id)
self.assertIsInstance(func_class, NumexprFunction)
def test_pdict_crud(self):
context_ids = self.create_contexts()
pdict_res_id = self.dataset_management.create_parameter_dictionary(name='pdict1', parameter_context_ids=context_ids, temporal_context='time')
pdict_contexts = self.dataset_management.read_parameter_contexts(parameter_dictionary_id=pdict_res_id, id_only=True)
pdict = DatasetManagementService.get_parameter_dictionary(pdict_res_id)
self.assertIsInstance(pdict, ParameterDictionary)
self.assertTrue('time_test' in pdict)
self.assertEquals(pdict.identifier, pdict_res_id)
self.assertEquals(set(pdict_contexts), set(context_ids))
self.dataset_management.delete_parameter_dictionary(parameter_dictionary_id=pdict_res_id)
with self.assertRaises(NotFound):
self.dataset_management.read_parameter_dictionary(parameter_dictionary_id=pdict_res_id)
def create_contexts(self):
context_ids = []
cond_ctxt = ParameterContext('conductivity_test', param_type=QuantityType(value_encoding=np.float32))
cond_ctxt.uom = 'unknown'
cond_ctxt.fill_value = 0e0
context_ids.append(self.dataset_management.create_parameter_context(name='conductivity_test', parameter_context=cond_ctxt.dump()))
pres_ctxt = ParameterContext('pressure_test', param_type=QuantityType(value_encoding=np.float32))
pres_ctxt.uom = 'Pascal'
pres_ctxt.fill_value = 0x0
context_ids.append(self.dataset_management.create_parameter_context(name='pressure_test', parameter_context=pres_ctxt.dump()))
sal_ctxt = ParameterContext('salinity_test', param_type=QuantityType(value_encoding=np.float32))
sal_ctxt.uom = 'PSU'
sal_ctxt.fill_value = 0x0
context_ids.append(self.dataset_management.create_parameter_context(name='salinity_test', parameter_context=sal_ctxt.dump()))
temp_ctxt = ParameterContext('temp_test', param_type=QuantityType(value_encoding=np.float32))
temp_ctxt.uom = 'degree_Celsius'
temp_ctxt.fill_value = 0e0
context_ids.append(self.dataset_management.create_parameter_context(name='temp_test', parameter_context=temp_ctxt.dump()))
t_ctxt = ParameterContext('time_test', param_type=QuantityType(value_encoding=np.int64))
t_ctxt.uom = 'seconds since 1970-01-01'
t_ctxt.fill_value = 0x0
context_ids.append(self.dataset_management.create_parameter_context(name='time_test', parameter_context=t_ctxt.dump()))
return context_ids
def create_pfuncs(self):
contexts = {}
funcs = {}
t_ctxt = ParameterContext('TIME', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 01-01-1900'
t_ctxt_id = self.dataset_management.create_parameter_context(name='test_TIME', parameter_context=t_ctxt.dump())
contexts['TIME'] = (t_ctxt, t_ctxt_id)
lat_ctxt = ParameterContext('LAT', param_type=ConstantType(QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999)
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt_id = self.dataset_management.create_parameter_context(name='test_LAT', parameter_context=lat_ctxt.dump())
contexts['LAT'] = lat_ctxt, lat_ctxt_id
lon_ctxt = ParameterContext('LON', param_type=ConstantType(QuantityType(value_encoding=np.dtype('float32'))), fill_value=-9999)
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt_id = self.dataset_management.create_parameter_context(name='test_LON', parameter_context=lon_ctxt.dump())
contexts['LON'] = lon_ctxt, lon_ctxt_id
# Independent Parameters
# Temperature - values expected to be the decimal results of conversion from hex
temp_ctxt = ParameterContext('TEMPWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999)
temp_ctxt.uom = 'deg_C'
temp_ctxt_id = self.dataset_management.create_parameter_context(name='test_TEMPWAT_L0', parameter_context=temp_ctxt.dump())
contexts['TEMPWAT_L0'] = temp_ctxt, temp_ctxt_id
# Conductivity - values expected to be the decimal results of conversion from hex
cond_ctxt = ParameterContext('CONDWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999)
cond_ctxt.uom = 'S m-1'
cond_ctxt_id = self.dataset_management.create_parameter_context(name='test_CONDWAT_L0', parameter_context=cond_ctxt.dump())
contexts['CONDWAT_L0'] = cond_ctxt, cond_ctxt_id
# Pressure - values expected to be the decimal results of conversion from hex
press_ctxt = ParameterContext('PRESWAT_L0', param_type=QuantityType(value_encoding=np.dtype('float32')), fill_value=-9999)
press_ctxt.uom = 'dbar'
press_ctxt_id = self.dataset_management.create_parameter_context(name='test_PRESWAT_L0', parameter_context=press_ctxt.dump())
contexts['PRESWAT_L0'] = press_ctxt, press_ctxt_id
# Dependent Parameters
# TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10
tl1_func = '(T / 10000) - 10'
expr = NumexprFunction('TEMPWAT_L1', tl1_func, ['T'])
expr_id = self.dataset_management.create_parameter_function(name='test_TEMPWAT_L1', parameter_function=expr.dump())
funcs['TEMPWAT_L1'] = expr, expr_id
tl1_pmap = {'T': 'TEMPWAT_L0'}
expr.param_map = tl1_pmap
tempL1_ctxt = ParameterContext('TEMPWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL)
tempL1_ctxt.uom = 'deg_C'
tempL1_ctxt_id = self.dataset_management.create_parameter_context(name='test_TEMPWAT_L1', parameter_context=tempL1_ctxt.dump(), parameter_function_id=expr_id)
contexts['TEMPWAT_L1'] = tempL1_ctxt, tempL1_ctxt_id
# CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5
cl1_func = '(C / 100000) - 0.5'
expr = NumexprFunction('CONDWAT_L1', cl1_func, ['C'])
expr_id = self.dataset_management.create_parameter_function(name='test_CONDWAT_L1', parameter_function=expr.dump())
funcs['CONDWAT_L1'] = expr, expr_id
cl1_pmap = {'C': 'CONDWAT_L0'}
expr.param_map = cl1_pmap
condL1_ctxt = ParameterContext('CONDWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL)
condL1_ctxt.uom = 'S m-1'
condL1_ctxt_id = self.dataset_management.create_parameter_context(name='test_CONDWAT_L1', parameter_context=condL1_ctxt.dump(), parameter_function_id=expr_id)
contexts['CONDWAT_L1'] = condL1_ctxt, condL1_ctxt_id
# Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721
# PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range)
pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)'
expr = NumexprFunction('PRESWAT_L1', pl1_func, ['P', 'p_range'])
expr_id = self.dataset_management.create_parameter_function(name='test_PRESWAT_L1', parameter_function=expr.dump())
funcs['PRESWAT_L1'] = expr, expr_id
pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721}
expr.param_map = pl1_pmap
presL1_ctxt = ParameterContext('PRESWAT_L1', param_type=ParameterFunctionType(function=expr), variability=VariabilityEnum.TEMPORAL)
presL1_ctxt.uom = 'S m-1'
presL1_ctxt_id = self.dataset_management.create_parameter_context(name='test_CONDWAT_L1', parameter_context=presL1_ctxt.dump(), parameter_function_id=expr_id)
contexts['PRESWAT_L1'] = presL1_ctxt, presL1_ctxt_id
# Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project:
# https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1
# PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1)
owner = 'gsw'
sal_func = 'SP_from_C'
sal_arglist = ['C', 't', 'p']
expr = PythonFunction('PRACSAL', owner, sal_func, sal_arglist)
expr_id = self.dataset_management.create_parameter_function(name='test_PRACSAL', parameter_function=expr.dump())
funcs['PRACSAL'] = expr, expr_id
# A magic function that may or may not exist actually forms the line below at runtime.
sal_pmap = {'C': NumexprFunction('CONDWAT_L1*10', 'C*10', ['C'], param_map={'C': 'CONDWAT_L1'}), 't': 'TEMPWAT_L1', 'p': 'PRESWAT_L1'}
expr.param_map = sal_pmap
sal_ctxt = ParameterContext('PRACSAL', param_type=ParameterFunctionType(expr), variability=VariabilityEnum.TEMPORAL)
sal_ctxt.uom = 'g kg-1'
sal_ctxt_id = self.dataset_management.create_parameter_context(name='test_PRACSAL', parameter_context=sal_ctxt.dump(), parameter_function_id=expr_id)
contexts['PRACSAL'] = sal_ctxt, sal_ctxt_id
# absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude)
# conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1)
# DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1)
owner = 'gsw'
abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP', ['PRACSAL', 'PRESWAT_L1', 'LON','LAT'])
cons_temp_expr = PythonFunction('cons_temp', owner, 'CT_from_t', [abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1'])
dens_expr = PythonFunction('DENSITY', owner, 'rho', [abs_sal_expr, cons_temp_expr, 'PRESWAT_L1'])
dens_ctxt = ParameterContext('DENSITY', param_type=ParameterFunctionType(dens_expr), variability=VariabilityEnum.TEMPORAL)
dens_ctxt.uom = 'kg m-3'
dens_ctxt_id = self.dataset_management.create_parameter_context(name='test_DENSITY', parameter_context=dens_ctxt.dump())
contexts['DENSITY'] = dens_ctxt, dens_ctxt_id
return contexts, funcs
def test_verify_contexts(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(name='ctd_parsed_param_dict', id_only=True)
pcontexts = self.dataset_management.read_parameter_contexts(parameter_dictionary_id=pdict_id)
for pcontext in pcontexts:
self.assertTrue('fill_value' in pcontext)
self.assertTrue('reference_urls' in pcontext)
self.assertTrue('internal_name' in pcontext)
self.assertTrue('display_name' in pcontext)
self.assertTrue('standard_name' in pcontext)
self.assertTrue('ooi_short_name' in pcontext)
self.assertTrue('description' in pcontext)
self.assertTrue('precision' in pcontext)
class EventManagementIntTest(IonIntegrationTestCase):
def setUp(self):
super(EventManagementIntTest, self).setUp()
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.event_management = EventManagementServiceClient()
self.rrc = ResourceRegistryServiceClient()
self.process_dispatcher = ProcessDispatcherServiceClient()
self.pubsub = PubsubManagementServiceClient()
self.dataset_management = DatasetManagementServiceClient()
self.data_product_management = DataProductManagementServiceClient()
self.queue_cleanup = []
self.exchange_cleanup = []
def tearDown(self):
for queue in self.queue_cleanup:
xn = self.container.ex_manager.create_xn_queue(queue)
xn.delete()
for exchange in self.exchange_cleanup:
xp = self.container.ex_manager.create_xp(exchange)
xp.delete()
def test_create_read_update_delete_event_type(self):
"""
Test that the CRUD method for event types work correctly
"""
event_type = EventType(name="an event type")
event_type.origin = 'instrument_1'
# create
event_type_id = self.event_management.create_event_type(event_type)
self.assertIsNotNone(event_type_id)
# read
read_event_type = self.event_management.read_event_type(event_type_id)
self.assertEquals(read_event_type.name, event_type.name)
self.assertEquals(read_event_type.origin, event_type.origin)
#update
read_event_type.origin = 'instrument_2'
read_event_type.producer = 'producer'
self.event_management.update_event_type(read_event_type)
updated_event_type = self.event_management.read_event_type(event_type_id)
self.assertEquals(updated_event_type.origin, 'instrument_2')
self.assertEquals(updated_event_type.producer, 'producer')
# delete
self.event_management.delete_event_type(event_type_id)
with self.assertRaises(NotFound):
self.event_management.read_event_type(event_type_id)
def test_create_read_update_delete_event_process_definition(self):
"""
Test that the CRUD methods for the event process definitions work correctly
"""
# Create
module = 'ion.processes.data.transforms.event_alert_transform'
class_name = 'EventAlertTransform'
procdef_id = self.event_management.create_event_process_definition(version='ver_1',
module=module,
class_name=class_name,
uri='http://hare.com',
arguments=['arg1', 'arg2'],
event_types=['ExampleDetectableEvent', 'type_2'],
sub_types=['sub_type_1'])
# Read
read_process_def = self.event_management.read_event_process_definition(procdef_id)
self.assertEquals(read_process_def.executable['module'], module)
self.assertEquals(read_process_def.executable['class'], class_name)
# Update
self.event_management.update_event_process_definition(event_process_definition_id=procdef_id,
class_name='StreamAlertTransform',
arguments=['arg3', 'arg4'],
event_types=['event_type_new'])
updated_event_process_def = self.event_management.read_event_process_definition(procdef_id)
self.assertEquals(updated_event_process_def.executable['class'], 'StreamAlertTransform')
self.assertEquals(updated_event_process_def.arguments, ['arg3', 'arg4'])
self.assertEquals(updated_event_process_def.definition.event_types, ['event_type_new'])
# Delete
self.event_management.delete_event_process_definition(procdef_id)
with self.assertRaises(NotFound):
self.event_management.read_event_process_definition(procdef_id)
def test_event_in_stream_out_transform(self):
"""
Test the event-in/stream-out transform
"""
stream_id, _ = self.pubsub.create_stream('test_stream', exchange_point='science_data')
self.exchange_cleanup.append('science_data')
#---------------------------------------------------------------------------------------------
# Launch a ctd transform
#---------------------------------------------------------------------------------------------
# Create the process definition
process_definition = ProcessDefinition(
name='EventToStreamTransform',
description='For testing an event-in/stream-out transform')
process_definition.executable['module']= 'ion.processes.data.transforms.event_in_stream_out_transform'
process_definition.executable['class'] = 'EventToStreamTransform'
proc_def_id = self.process_dispatcher.create_process_definition(process_definition=process_definition)
# Build the config
config = DotDict()
config.process.queue_name = 'test_queue'
config.process.exchange_point = 'science_data'
config.process.publish_streams.output = stream_id
config.process.event_type = 'ExampleDetectableEvent'
config.process.variables = ['voltage', 'temperature' ]
# Schedule the process
self.process_dispatcher.schedule_process(process_definition_id=proc_def_id, configuration=config)
#---------------------------------------------------------------------------------------------
# Create a subscriber for testing
#---------------------------------------------------------------------------------------------
ar_cond = gevent.event.AsyncResult()
def subscriber_callback(m, r, s):
ar_cond.set(m)
sub = StandaloneStreamSubscriber('sub', subscriber_callback)
self.addCleanup(sub.stop)
sub_id = self.pubsub.create_subscription('subscription_cond',
stream_ids=[stream_id],
exchange_name='sub')
self.pubsub.activate_subscription(sub_id)
self.queue_cleanup.append(sub.xn.queue)
sub.start()
gevent.sleep(4)
#---------------------------------------------------------------------------------------------
# Publish an event. The transform has been configured to receive this event
#---------------------------------------------------------------------------------------------
event_publisher = EventPublisher("ExampleDetectableEvent")
event_publisher.publish_event(origin = 'fake_origin', voltage = '5', temperature = '273')
# Assert that the transform processed the event and published data on the output stream
result_cond = ar_cond.get(timeout=10)
self.assertTrue(result_cond)
def test_create_read_delete_event_process(self):
"""
Test that the CRUD methods for the event processes work correctly
"""
#---------------------------------------------------------------------------------------------
# Create a process definition
#---------------------------------------------------------------------------------------------
# Create
module = 'ion.processes.data.transforms.event_alert_transform'
class_name = 'EventAlertTransform'
procdef_id = self.event_management.create_event_process_definition(version='ver_1',
module=module,
class_name=class_name,
uri='http://hare.com',
arguments=['arg1', 'arg2'],
event_types=['ExampleDetectableEvent', 'type_2'],
sub_types=['sub_type_1'])
# Read
read_process_def = self.event_management.read_event_process_definition(procdef_id)
self.assertEquals(read_process_def.arguments, ['arg1', 'arg2'])
#---------------------------------------------------------------------------------------------
# Use the process definition to create a process
#---------------------------------------------------------------------------------------------
# Create a stream
param_dict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict',id_only=True)
parameter_dictionary = self.dataset_management.read_parameter_dictionary(parameter_dictionary_id=param_dict_id)
stream_def_id = self.pubsub.create_stream_definition('cond_stream_def', parameter_dictionary_id=param_dict_id)
tdom, sdom = time_series_domain()
tdom, sdom = tdom.dump(), sdom.dump()
dp_obj = IonObject(RT.DataProduct,
name='DP1',
description='some new dp',
temporal_domain = tdom,
spatial_domain = sdom)
# Create a data product
data_product_id = self.data_product_management.create_data_product(data_product=dp_obj, stream_definition_id=stream_def_id, parameter_dictionary=parameter_dictionary)
output_products = {}
output_products['conductivity'] = data_product_id
# Create an event process
event_process_id = self.event_management.create_event_process( process_definition_id=procdef_id,
event_types=['ExampleDetectableEvent','DetectionEvent'],
sub_types=['s1', 's2'],
origins=['or_1', 'or_2'],
origin_types=['or_t1', 'or_t2'],
out_data_products = output_products)
#---------------------------------------------------------------------------------------------
# Read the event process object and make assertions
#--------------------------------------------------------------------------------------------- out_data_products={'conductivity': data_product_id})
event_process_obj = self.event_management.read_event_process(event_process_id=event_process_id)
# Get the stream associated with the data product for the sake of making assertions
stream_ids, _ = self.rrc.find_objects(data_product_id, PRED.hasStream, id_only=True)
stream_id = stream_ids[0]
# Assertions!
self.assertEquals(event_process_obj.detail.output_streams['conductivity'], stream_id)
self.assertEquals(event_process_obj.detail.event_types, ['ExampleDetectableEvent', 'DetectionEvent'])
self.assertEquals(event_process_obj.detail.sub_types, ['s1', 's2'])
self.assertEquals(event_process_obj.detail.origins, ['or_1', 'or_2'])
self.assertEquals(event_process_obj.detail.origin_types, ['or_t1', 'or_t2'])
@unittest.skip("Not yet implemented in R 2.0")
def test_activate_deactivate_data_process(self):
"""
Test that the activation and deactivation of event processes happens correctly
"""
pass
@unittest.skip("Not yet implemented in R 2.0")
def test_update_event_process_inputs(self):
"""
Test that the event process inputs can be correctly updated
"""
pass
class TestStreamIngestionWorker(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dataset_management_client = DatasetManagementServiceClient(node=self.container.node)
self.pubsub_client = PubsubManagementServiceClient(node=self.container.node)
self.time_dom, self.spatial_dom = time_series_domain()
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(name='ctd_parsed_param_dict', id_only=True)
self.stream_def_id = self.pubsub_client.create_stream_definition(name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
self.addCleanup(self.pubsub_client.delete_stream_definition, self.stream_def_id)
self.stream_id, self.route_id = self.pubsub_client.create_stream(name='parsed_stream', stream_definition_id=self.stream_def_id, exchange_point='science_data')
self.addCleanup(self.pubsub_client.delete_stream, self.stream_id)
self.subscription_id = self.pubsub_client.create_subscription(name='parsed_subscription', stream_ids=[self.stream_id], exchange_name='parsed_subscription')
self.addCleanup(self.pubsub_client.delete_subscription, self.subscription_id)
self.pubsub_client.activate_subscription(self.subscription_id)
self.addCleanup(self.pubsub_client.deactivate_subscription, self.subscription_id)
self.publisher = StandaloneStreamPublisher(self.stream_id, self.route_id)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False), 'Skip test while in CEI LAUNCH mode')
def test_stream_ingestion_worker(self):
self.start_ingestion_worker()
context_ids, time_ctxt = self._create_param_contexts()
pdict_id = self.dataset_management_client.create_parameter_dictionary(name='stream_ingestion_pdict', parameter_context_ids=context_ids, temporal_context='ingestion_timestamp')
self.addCleanup(self.dataset_management_client.delete_parameter_dictionary, pdict_id)
dataset_id = self.dataset_management_client.create_dataset(name='fake_dataset', description='fake_dataset', stream_id=self.stream_id, parameter_dictionary_id=pdict_id)
self.addCleanup(self.dataset_management_client.delete_dataset, dataset_id)
self.cov = self._create_coverage(dataset_id=dataset_id, parameter_dict_id=pdict_id, time_dom=self.time_dom, spatial_dom=self.spatial_dom)
self.addCleanup(self.cov.close)
rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
rdt['conductivity'] = 1
rdt['pressure'] = 2
rdt['salinity'] = 3
self.start_listener(dataset_id)
self.publisher.publish(rdt.to_granule())
self.data_modified = Event()
self.data_modified.wait(30)
cov = self.get_coverage(dataset_id)
self.assertIsNotNone(cov.get_parameter_values('raw'))
deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())
granule = retrieve_stream(dataset_id)
rdt_complex = RecordDictionaryTool.load_from_granule(granule)
rdt_complex['raw'] = [deserializer.deserialize(i) for i in rdt_complex['raw']]
for gran in rdt_complex['raw']:
rdt_new = RecordDictionaryTool.load_from_granule(gran)
self.assertIn(1, rdt_new['conductivity'])
self.assertIn(2, rdt_new['pressure'])
self.assertIn(3, rdt_new['salinity'])
cov.close()
def start_ingestion_worker(self):
config = DotDict()
config.process.queue_name = 'parsed_subscription'
self.container.spawn_process(
name='stream_ingestion_worker',
module='ion.processes.data.ingestion.stream_ingestion_worker',
cls='StreamIngestionWorker',
config=config
)
def start_listener(self, dataset_id):
def cb(*args, **kwargs):
self.data_modified.set()
es = EventSubscriber(event_type=OT.DatasetModified, callback=cb, origin=dataset_id)
es.start()
self.addCleanup(es.stop)
def _create_param_contexts(self):
context_ids = []
t_ctxt = ParameterContext('ingestion_timestamp', param_type=QuantityType(value_encoding=numpy.dtype('float64')))
t_ctxt.uom = 'seconds since 1900-01-01'
t_ctxt.fill_value = -9999
t_ctxt_id = self.dataset_management_client.create_parameter_context(name='ingestion_timestamp', parameter_context=t_ctxt.dump())
context_ids.append(t_ctxt_id)
raw_ctxt = ParameterContext('raw', param_type=ArrayType())
raw_ctxt.uom = ''
context_ids.append(self.dataset_management_client.create_parameter_context(name='raw', parameter_context=raw_ctxt.dump()))
return context_ids, t_ctxt_id
def _create_coverage(self, dataset_id, parameter_dict_id, time_dom, spatial_dom):
pd = self.dataset_management_client.read_parameter_dictionary(parameter_dict_id)
pdict = ParameterDictionary.load(pd)
sdom = GridDomain.load(spatial_dom.dump())
tdom = GridDomain.load(time_dom.dump())
file_root = FileSystem.get_url(FS.CACHE,'datasets')
scov = SimplexCoverage(file_root, dataset_id, dataset_id, parameter_dictionary=pdict, temporal_domain=tdom, spatial_domain=sdom)
return scov
def get_coverage(self, dataset_id,mode='w'):
file_root = FileSystem.get_url(FS.CACHE,'datasets')
coverage = AbstractCoverage.load(file_root, dataset_id, mode=mode)
return coverage
class TestStreamIngestionWorker(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.dataset_management_client = DatasetManagementServiceClient(
node=self.container.node)
self.pubsub_client = PubsubManagementServiceClient(
node=self.container.node)
self.time_dom, self.spatial_dom = time_series_domain()
self.parameter_dict_id = self.dataset_management_client.read_parameter_dictionary_by_name(
name='ctd_parsed_param_dict', id_only=True)
self.stream_def_id = self.pubsub_client.create_stream_definition(
name='stream_def', parameter_dictionary_id=self.parameter_dict_id)
self.addCleanup(self.pubsub_client.delete_stream_definition,
self.stream_def_id)
self.stream_id, self.route_id = self.pubsub_client.create_stream(
name='parsed_stream',
stream_definition_id=self.stream_def_id,
exchange_point='science_data')
self.addCleanup(self.pubsub_client.delete_stream, self.stream_id)
self.subscription_id = self.pubsub_client.create_subscription(
name='parsed_subscription',
stream_ids=[self.stream_id],
exchange_name='parsed_subscription')
self.addCleanup(self.pubsub_client.delete_subscription,
self.subscription_id)
self.pubsub_client.activate_subscription(self.subscription_id)
self.addCleanup(self.pubsub_client.deactivate_subscription,
self.subscription_id)
self.publisher = StandaloneStreamPublisher(self.stream_id,
self.route_id)
@attr('LOCOINT')
@unittest.skipIf(os.getenv('CEI_LAUNCH_TEST', False),
'Skip test while in CEI LAUNCH mode')
def test_stream_ingestion_worker(self):
self.start_ingestion_worker()
context_ids, time_ctxt = self._create_param_contexts()
pdict_id = self.dataset_management_client.create_parameter_dictionary(
name='stream_ingestion_pdict',
parameter_context_ids=context_ids,
temporal_context='ingestion_timestamp')
self.addCleanup(
self.dataset_management_client.delete_parameter_dictionary,
pdict_id)
dataset_id = self.dataset_management_client.create_dataset(
name='fake_dataset',
description='fake_dataset',
stream_id=self.stream_id,
spatial_domain=self.spatial_dom.dump(),
temporal_domain=self.time_dom.dump(),
parameter_dictionary_id=pdict_id)
self.addCleanup(self.dataset_management_client.delete_dataset,
dataset_id)
self.cov = self._create_coverage(dataset_id=dataset_id,
parameter_dict_id=pdict_id,
time_dom=self.time_dom,
spatial_dom=self.spatial_dom)
self.addCleanup(self.cov.close)
rdt = RecordDictionaryTool(stream_definition_id=self.stream_def_id)
rdt['conductivity'] = 1
rdt['pressure'] = 2
rdt['salinity'] = 3
self.start_listener(dataset_id)
self.publisher.publish(rdt.to_granule())
self.data_modified = Event()
self.data_modified.wait(30)
cov = self.get_coverage(dataset_id)
self.assertIsNotNone(cov.get_parameter_values('raw'))
deserializer = IonObjectDeserializer(obj_registry=get_obj_registry())
granule = retrieve_stream(dataset_id)
rdt_complex = RecordDictionaryTool.load_from_granule(granule)
rdt_complex['raw'] = [
deserializer.deserialize(i) for i in rdt_complex['raw']
]
for gran in rdt_complex['raw']:
rdt_new = RecordDictionaryTool.load_from_granule(gran)
self.assertIn(1, rdt_new['conductivity'])
self.assertIn(2, rdt_new['pressure'])
self.assertIn(3, rdt_new['salinity'])
cov.close()
def start_ingestion_worker(self):
config = DotDict()
config.process.queue_name = 'parsed_subscription'
self.container.spawn_process(
name='stream_ingestion_worker',
module='ion.processes.data.ingestion.stream_ingestion_worker',
cls='StreamIngestionWorker',
config=config)
def start_listener(self, dataset_id):
def cb(*args, **kwargs):
self.data_modified.set()
es = EventSubscriber(event_type=OT.DatasetModified,
callback=cb,
origin=dataset_id)
es.start()
self.addCleanup(es.stop)
def _create_param_contexts(self):
context_ids = []
t_ctxt = ParameterContext(
'ingestion_timestamp',
param_type=QuantityType(value_encoding=numpy.dtype('float64')))
t_ctxt.uom = 'seconds since 1900-01-01'
t_ctxt.fill_value = -9999
t_ctxt_id = self.dataset_management_client.create_parameter_context(
name='ingestion_timestamp', parameter_context=t_ctxt.dump())
context_ids.append(t_ctxt_id)
raw_ctxt = ParameterContext('raw', param_type=ArrayType())
raw_ctxt.uom = ''
context_ids.append(
self.dataset_management_client.create_parameter_context(
name='raw', parameter_context=raw_ctxt.dump()))
return context_ids, t_ctxt_id
def _create_coverage(self, dataset_id, parameter_dict_id, time_dom,
spatial_dom):
pd = self.dataset_management_client.read_parameter_dictionary(
parameter_dict_id)
pdict = ParameterDictionary.load(pd)
sdom = GridDomain.load(spatial_dom.dump())
tdom = GridDomain.load(time_dom.dump())
file_root = FileSystem.get_url(FS.CACHE, 'datasets')
scov = SimplexCoverage(file_root,
dataset_id,
dataset_id,
parameter_dictionary=pdict,
temporal_domain=tdom,
spatial_domain=sdom)
return scov
def get_coverage(self, dataset_id, mode='w'):
file_root = FileSystem.get_url(FS.CACHE, 'datasets')
coverage = AbstractCoverage.load(file_root, dataset_id, mode=mode)
return coverage
class DatasetManagementIntTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.resource_registry = ResourceRegistryServiceClient()
self.dataset_management = DatasetManagementServiceClient()
def test_dataset_crud(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
'ctd_parsed_param_dict', id_only=True)
tdom, sdom = time_series_domain()
dataset_id = self.dataset_management.create_dataset(
name='ctd_dataset',
parameter_dictionary_id=pdict_id,
spatial_domain=sdom.dump(),
temporal_domain=tdom.dump())
ds_obj = self.dataset_management.read_dataset(dataset_id)
self.assertEquals(ds_obj.name, 'ctd_dataset')
ds_obj.name = 'something different'
self.dataset_management.update_dataset(ds_obj)
self.dataset_management.register_dataset(dataset_id)
ds_obj2 = self.dataset_management.read_dataset(dataset_id)
self.assertEquals(ds_obj.name, ds_obj2.name)
self.assertTrue(ds_obj2.registered)
def test_context_crud(self):
context_ids = self.create_contexts()
context_id = context_ids.pop()
context = DatasetManagementService.get_parameter_context(context_id)
self.assertIsInstance(context, ParameterContext)
self.assertEquals(context.identifier, context_id)
self.dataset_management.delete_parameter_context(context_id)
with self.assertRaises(NotFound):
self.dataset_management.read_parameter_context(context_id)
def test_pfunc_crud(self):
contexts, funcs = self.create_pfuncs()
context_ids = [
context_id for ctxt, context_id in contexts.itervalues()
]
pdict_id = self.dataset_management.create_parameter_dictionary(
name='functional_pdict',
parameter_context_ids=context_ids,
temporal_context='time')
self.addCleanup(self.dataset_management.delete_parameter_dictionary,
pdict_id)
expr, expr_id = funcs['CONDWAT_L1']
func_class = DatasetManagementService.get_parameter_function(expr_id)
self.assertIsInstance(func_class, NumexprFunction)
def test_pdict_crud(self):
context_ids = self.create_contexts()
pdict_res_id = self.dataset_management.create_parameter_dictionary(
name='pdict1',
parameter_context_ids=context_ids,
temporal_context='time')
pdict_contexts = self.dataset_management.read_parameter_contexts(
parameter_dictionary_id=pdict_res_id, id_only=True)
pdict = DatasetManagementService.get_parameter_dictionary(pdict_res_id)
self.assertIsInstance(pdict, ParameterDictionary)
self.assertTrue('time_test' in pdict)
self.assertEquals(pdict.identifier, pdict_res_id)
self.assertEquals(set(pdict_contexts), set(context_ids))
self.dataset_management.delete_parameter_dictionary(
parameter_dictionary_id=pdict_res_id)
with self.assertRaises(NotFound):
self.dataset_management.read_parameter_dictionary(
parameter_dictionary_id=pdict_res_id)
def create_contexts(self):
context_ids = []
cond_ctxt = ParameterContext(
'conductivity_test',
param_type=QuantityType(value_encoding=np.float32))
cond_ctxt.uom = 'unknown'
cond_ctxt.fill_value = 0e0
context_ids.append(
self.dataset_management.create_parameter_context(
name='conductivity_test', parameter_context=cond_ctxt.dump()))
pres_ctxt = ParameterContext(
'pressure_test',
param_type=QuantityType(value_encoding=np.float32))
pres_ctxt.uom = 'Pascal'
pres_ctxt.fill_value = 0x0
context_ids.append(
self.dataset_management.create_parameter_context(
name='pressure_test', parameter_context=pres_ctxt.dump()))
sal_ctxt = ParameterContext(
'salinity_test',
param_type=QuantityType(value_encoding=np.float32))
sal_ctxt.uom = 'PSU'
sal_ctxt.fill_value = 0x0
context_ids.append(
self.dataset_management.create_parameter_context(
name='salinity_test', parameter_context=sal_ctxt.dump()))
temp_ctxt = ParameterContext(
'temp_test', param_type=QuantityType(value_encoding=np.float32))
temp_ctxt.uom = 'degree_Celsius'
temp_ctxt.fill_value = 0e0
context_ids.append(
self.dataset_management.create_parameter_context(
name='temp_test', parameter_context=temp_ctxt.dump()))
t_ctxt = ParameterContext(
'time_test', param_type=QuantityType(value_encoding=np.int64))
t_ctxt.uom = 'seconds since 1970-01-01'
t_ctxt.fill_value = 0x0
context_ids.append(
self.dataset_management.create_parameter_context(
name='time_test', parameter_context=t_ctxt.dump()))
return context_ids
def create_pfuncs(self):
contexts = {}
funcs = {}
t_ctxt = ParameterContext(
'TIME', param_type=QuantityType(value_encoding=np.dtype('int64')))
t_ctxt.uom = 'seconds since 01-01-1900'
t_ctxt_id = self.dataset_management.create_parameter_context(
name='test_TIME', parameter_context=t_ctxt.dump())
contexts['TIME'] = (t_ctxt, t_ctxt_id)
lat_ctxt = ParameterContext(
'LAT',
param_type=ConstantType(
QuantityType(value_encoding=np.dtype('float32'))),
fill_value=-9999)
lat_ctxt.axis = AxisTypeEnum.LAT
lat_ctxt.uom = 'degree_north'
lat_ctxt_id = self.dataset_management.create_parameter_context(
name='test_LAT', parameter_context=lat_ctxt.dump())
contexts['LAT'] = lat_ctxt, lat_ctxt_id
lon_ctxt = ParameterContext(
'LON',
param_type=ConstantType(
QuantityType(value_encoding=np.dtype('float32'))),
fill_value=-9999)
lon_ctxt.axis = AxisTypeEnum.LON
lon_ctxt.uom = 'degree_east'
lon_ctxt_id = self.dataset_management.create_parameter_context(
name='test_LON', parameter_context=lon_ctxt.dump())
contexts['LON'] = lon_ctxt, lon_ctxt_id
# Independent Parameters
# Temperature - values expected to be the decimal results of conversion from hex
temp_ctxt = ParameterContext(
'TEMPWAT_L0',
param_type=QuantityType(value_encoding=np.dtype('float32')),
fill_value=-9999)
temp_ctxt.uom = 'deg_C'
temp_ctxt_id = self.dataset_management.create_parameter_context(
name='test_TEMPWAT_L0', parameter_context=temp_ctxt.dump())
contexts['TEMPWAT_L0'] = temp_ctxt, temp_ctxt_id
# Conductivity - values expected to be the decimal results of conversion from hex
cond_ctxt = ParameterContext(
'CONDWAT_L0',
param_type=QuantityType(value_encoding=np.dtype('float32')),
fill_value=-9999)
cond_ctxt.uom = 'S m-1'
cond_ctxt_id = self.dataset_management.create_parameter_context(
name='test_CONDWAT_L0', parameter_context=cond_ctxt.dump())
contexts['CONDWAT_L0'] = cond_ctxt, cond_ctxt_id
# Pressure - values expected to be the decimal results of conversion from hex
press_ctxt = ParameterContext(
'PRESWAT_L0',
param_type=QuantityType(value_encoding=np.dtype('float32')),
fill_value=-9999)
press_ctxt.uom = 'dbar'
press_ctxt_id = self.dataset_management.create_parameter_context(
name='test_PRESWAT_L0', parameter_context=press_ctxt.dump())
contexts['PRESWAT_L0'] = press_ctxt, press_ctxt_id
# Dependent Parameters
# TEMPWAT_L1 = (TEMPWAT_L0 / 10000) - 10
tl1_func = '(T / 10000) - 10'
expr = NumexprFunction('TEMPWAT_L1', tl1_func, ['T'])
expr_id = self.dataset_management.create_parameter_function(
name='test_TEMPWAT_L1', parameter_function=expr.dump())
funcs['TEMPWAT_L1'] = expr, expr_id
tl1_pmap = {'T': 'TEMPWAT_L0'}
expr.param_map = tl1_pmap
tempL1_ctxt = ParameterContext(
'TEMPWAT_L1',
param_type=ParameterFunctionType(function=expr),
variability=VariabilityEnum.TEMPORAL)
tempL1_ctxt.uom = 'deg_C'
tempL1_ctxt_id = self.dataset_management.create_parameter_context(
name='test_TEMPWAT_L1',
parameter_context=tempL1_ctxt.dump(),
parameter_function_id=expr_id)
contexts['TEMPWAT_L1'] = tempL1_ctxt, tempL1_ctxt_id
# CONDWAT_L1 = (CONDWAT_L0 / 100000) - 0.5
cl1_func = '(C / 100000) - 0.5'
expr = NumexprFunction('CONDWAT_L1', cl1_func, ['C'])
expr_id = self.dataset_management.create_parameter_function(
name='test_CONDWAT_L1', parameter_function=expr.dump())
funcs['CONDWAT_L1'] = expr, expr_id
cl1_pmap = {'C': 'CONDWAT_L0'}
expr.param_map = cl1_pmap
condL1_ctxt = ParameterContext(
'CONDWAT_L1',
param_type=ParameterFunctionType(function=expr),
variability=VariabilityEnum.TEMPORAL)
condL1_ctxt.uom = 'S m-1'
condL1_ctxt_id = self.dataset_management.create_parameter_context(
name='test_CONDWAT_L1',
parameter_context=condL1_ctxt.dump(),
parameter_function_id=expr_id)
contexts['CONDWAT_L1'] = condL1_ctxt, condL1_ctxt_id
# Equation uses p_range, which is a calibration coefficient - Fixing to 679.34040721
# PRESWAT_L1 = (PRESWAT_L0 * p_range / (0.85 * 65536)) - (0.05 * p_range)
pl1_func = '(P * p_range / (0.85 * 65536)) - (0.05 * p_range)'
expr = NumexprFunction('PRESWAT_L1', pl1_func, ['P', 'p_range'])
expr_id = self.dataset_management.create_parameter_function(
name='test_PRESWAT_L1', parameter_function=expr.dump())
funcs['PRESWAT_L1'] = expr, expr_id
pl1_pmap = {'P': 'PRESWAT_L0', 'p_range': 679.34040721}
expr.param_map = pl1_pmap
presL1_ctxt = ParameterContext(
'PRESWAT_L1',
param_type=ParameterFunctionType(function=expr),
variability=VariabilityEnum.TEMPORAL)
presL1_ctxt.uom = 'S m-1'
presL1_ctxt_id = self.dataset_management.create_parameter_context(
name='test_CONDWAT_L1',
parameter_context=presL1_ctxt.dump(),
parameter_function_id=expr_id)
contexts['PRESWAT_L1'] = presL1_ctxt, presL1_ctxt_id
# Density & practical salinity calucluated using the Gibbs Seawater library - available via python-gsw project:
# https://code.google.com/p/python-gsw/ & http://pypi.python.org/pypi/gsw/3.0.1
# PRACSAL = gsw.SP_from_C((CONDWAT_L1 * 10), TEMPWAT_L1, PRESWAT_L1)
owner = 'gsw'
sal_func = 'SP_from_C'
sal_arglist = ['C', 't', 'p']
expr = PythonFunction('PRACSAL', owner, sal_func, sal_arglist)
expr_id = self.dataset_management.create_parameter_function(
name='test_PRACSAL', parameter_function=expr.dump())
funcs['PRACSAL'] = expr, expr_id
# A magic function that may or may not exist actually forms the line below at runtime.
sal_pmap = {
'C':
NumexprFunction('CONDWAT_L1*10',
'C*10', ['C'],
param_map={'C': 'CONDWAT_L1'}),
't':
'TEMPWAT_L1',
'p':
'PRESWAT_L1'
}
expr.param_map = sal_pmap
sal_ctxt = ParameterContext('PRACSAL',
param_type=ParameterFunctionType(expr),
variability=VariabilityEnum.TEMPORAL)
sal_ctxt.uom = 'g kg-1'
sal_ctxt_id = self.dataset_management.create_parameter_context(
name='test_PRACSAL',
parameter_context=sal_ctxt.dump(),
parameter_function_id=expr_id)
contexts['PRACSAL'] = sal_ctxt, sal_ctxt_id
# absolute_salinity = gsw.SA_from_SP(PRACSAL, PRESWAT_L1, longitude, latitude)
# conservative_temperature = gsw.CT_from_t(absolute_salinity, TEMPWAT_L1, PRESWAT_L1)
# DENSITY = gsw.rho(absolute_salinity, conservative_temperature, PRESWAT_L1)
owner = 'gsw'
abs_sal_expr = PythonFunction('abs_sal', owner, 'SA_from_SP',
['PRACSAL', 'PRESWAT_L1', 'LON', 'LAT'])
cons_temp_expr = PythonFunction(
'cons_temp', owner, 'CT_from_t',
[abs_sal_expr, 'TEMPWAT_L1', 'PRESWAT_L1'])
dens_expr = PythonFunction(
'DENSITY', owner, 'rho',
[abs_sal_expr, cons_temp_expr, 'PRESWAT_L1'])
dens_ctxt = ParameterContext(
'DENSITY',
param_type=ParameterFunctionType(dens_expr),
variability=VariabilityEnum.TEMPORAL)
dens_ctxt.uom = 'kg m-3'
dens_ctxt_id = self.dataset_management.create_parameter_context(
name='test_DENSITY', parameter_context=dens_ctxt.dump())
contexts['DENSITY'] = dens_ctxt, dens_ctxt_id
return contexts, funcs
def test_verify_contexts(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name(
name='ctd_parsed_param_dict', id_only=True)
pcontexts = self.dataset_management.read_parameter_contexts(
parameter_dictionary_id=pdict_id)
for pcontext in pcontexts:
self.assertTrue('fill_value' in pcontext)
self.assertTrue('reference_urls' in pcontext)
self.assertTrue('internal_name' in pcontext)
self.assertTrue('display_name' in pcontext)
self.assertTrue('standard_name' in pcontext)
self.assertTrue('ooi_short_name' in pcontext)
self.assertTrue('description' in pcontext)
self.assertTrue('precision' in pcontext)
class DatasetManagementIntTest(IonIntegrationTestCase):
def setUp(self):
self._start_container()
self.container.start_rel_from_url('res/deploy/r2deploy.yml')
self.resource_registry = ResourceRegistryServiceClient()
self.dataset_management = DatasetManagementServiceClient()
def test_dataset_crud(self):
pdict_id = self.dataset_management.read_parameter_dictionary_by_name('ctd_parsed_param_dict', id_only=True)
tdom, sdom = time_series_domain()
dataset_id = self.dataset_management.create_dataset(name='ctd_dataset', parameter_dictionary_id=pdict_id, spatial_domain=sdom.dump(), temporal_domain=tdom.dump())
ds_obj = self.dataset_management.read_dataset(dataset_id)
self.assertEquals(ds_obj.name, 'ctd_dataset')
ds_obj.name = 'something different'
self.dataset_management.update_dataset(ds_obj)
self.dataset_management.register_dataset(dataset_id)
ds_obj2 = self.dataset_management.read_dataset(dataset_id)
self.assertEquals(ds_obj.name, ds_obj2.name)
self.assertTrue(ds_obj2.registered)
def test_context_crud(self):
context_ids = self.create_contexts()
context_id = context_ids.pop()
context = DatasetManagementService.get_parameter_context(context_id)
self.assertIsInstance(context, ParameterContext)
self.assertEquals(context.identifier, context_id)
self.dataset_management.delete_parameter_context(context_id)
with self.assertRaises(NotFound):
self.dataset_management.read_parameter_context(context_id)
def test_pdict_crud(self):
context_ids = self.create_contexts()
pdict_res_id = self.dataset_management.create_parameter_dictionary(name='pdict1', parameter_context_ids=context_ids, temporal_context='time')
pdict_contexts = self.dataset_management.read_parameter_contexts(parameter_dictionary_id=pdict_res_id, id_only=True)
pdict = DatasetManagementService.get_parameter_dictionary(pdict_res_id)
self.assertIsInstance(pdict, ParameterDictionary)
self.assertTrue('time_test' in pdict)
self.assertEquals(pdict.identifier, pdict_res_id)
self.assertEquals(set(pdict_contexts), set(context_ids))
self.dataset_management.delete_parameter_dictionary(parameter_dictionary_id=pdict_res_id)
with self.assertRaises(NotFound):
self.dataset_management.read_parameter_dictionary(parameter_dictionary_id=pdict_res_id)
def create_contexts(self):
context_ids = []
cond_ctxt = ParameterContext('conductivity_test', param_type=QuantityType(value_encoding=np.float32))
cond_ctxt.uom = 'unknown'
cond_ctxt.fill_value = 0e0
context_ids.append(self.dataset_management.create_parameter_context(name='conductivity_test', parameter_context=cond_ctxt.dump()))
pres_ctxt = ParameterContext('pressure_test', param_type=QuantityType(value_encoding=np.float32))
pres_ctxt.uom = 'Pascal'
pres_ctxt.fill_value = 0x0
context_ids.append(self.dataset_management.create_parameter_context(name='pressure_test', parameter_context=pres_ctxt.dump()))
sal_ctxt = ParameterContext('salinity_test', param_type=QuantityType(value_encoding=np.float32))
sal_ctxt.uom = 'PSU'
sal_ctxt.fill_value = 0x0
context_ids.append(self.dataset_management.create_parameter_context(name='salinity_test', parameter_context=sal_ctxt.dump()))
temp_ctxt = ParameterContext('temp_test', param_type=QuantityType(value_encoding=np.float32))
temp_ctxt.uom = 'degree_Celsius'
temp_ctxt.fill_value = 0e0
context_ids.append(self.dataset_management.create_parameter_context(name='temp_test', parameter_context=temp_ctxt.dump()))
t_ctxt = ParameterContext('time_test', param_type=QuantityType(value_encoding=np.int64))
t_ctxt.uom = 'seconds since 1970-01-01'
t_ctxt.fill_value = 0x0
context_ids.append(self.dataset_management.create_parameter_context(name='time_test', parameter_context=t_ctxt.dump()))
return context_ids
