def get_coverage_parameter(cls, parameter_context): """ Creates a Coverage Model based Parameter Context given the ParameterContext IonObject. Note: If the parameter is a parameter function and depends on dynamically created calibrations, this will fail. """ # Only CF and netCDF compliant variable names parameter_context.name = re.sub(r'[^a-zA-Z0-9_]', '_', parameter_context.name) from ion.services.dm.utility.types import TypesManager # The TypesManager does all the parsing and converting to the coverage model instances tm = TypesManager(None, {}, {}) # First thing to do is create the parameter type param_type = tm.get_parameter_type( parameter_context.parameter_type, parameter_context.value_encoding, parameter_context.code_report, parameter_context.parameter_function_id, parameter_context.parameter_function_map, { 'name': parameter_context.name, 'target_dataset': parameter_context.target_dataset, 'target_name': parameter_context.target_name }) # Ugh, I hate it but I did copy this section from # ion/processes/bootstrap/ion_loader.py context = ParameterContext(name=parameter_context.name, param_type=param_type) # Now copy over all the attrs context.uom = parameter_context.units try: if isinstance(context.uom, basestring): tm.get_unit(context.uom) except UdunitsError: log.warning('Parameter %s has invalid units: %s', parameter_context.name, context.uom) # Fill values can be a bit tricky... context.fill_value = tm.get_fill_value( parameter_context.fill_value, parameter_context.value_encoding, param_type) context.reference_urls = parameter_context.reference_urls context.internal_name = parameter_context.name context.display_name = parameter_context.display_name context.standard_name = parameter_context.standard_name context.ooi_short_name = parameter_context.ooi_short_name context.description = parameter_context.description context.precision = parameter_context.precision context.visible = parameter_context.visible return context
def get_coverage_parameter(cls, parameter_context): """ Creates a Coverage Model based Parameter Context given the ParameterContext IonObject. Note: If the parameter is a parameter function and depends on dynamically created calibrations, this will fail. """ # Only CF and netCDF compliant variable names parameter_context.name = re.sub(r'[^a-zA-Z0-9_]', '_', parameter_context.name) from ion.services.dm.utility.types import TypesManager # The TypesManager does all the parsing and converting to the coverage model instances tm = TypesManager(None, {}, {}) # First thing to do is create the parameter type param_type = tm.get_parameter_type( parameter_context.parameter_type, parameter_context.value_encoding, parameter_context.code_report, parameter_context.parameter_function_id, parameter_context.parameter_function_map) # Ugh, I hate it but I did copy this section from # ion/processes/bootstrap/ion_loader.py context = ParameterContext(name=parameter_context.name, param_type=param_type) # Now copy over all the attrs context.uom = parameter_context.units try: if isinstance(context.uom, basestring): tm.get_unit(context.uom) except UdunitsError: log.warning('Parameter %s has invalid units: %s', parameter_context.name, context.uom) # Fill values can be a bit tricky... context.fill_value = tm.get_fill_value(parameter_context.fill_value, parameter_context.value_encoding, param_type) context.reference_urls = parameter_context.reference_urls context.internal_name = parameter_context.name context.display_name = parameter_context.display_name context.standard_name = parameter_context.standard_name context.ooi_short_name = parameter_context.ooi_short_name context.description = parameter_context.description context.precision = parameter_context.precision context.visible = parameter_context.visible return context
class TestTypes(PyonTestCase): def setUp(self): PyonTestCase.setUp(self) self.types_manager = TypesManager(None,None,None) def get_context(self, ptype, encoding, fill_value, codeset=None): ptype = self.types_manager.get_parameter_type(ptype, encoding, codeset) context = ParameterContext(name='test', param_type=ptype) context.fill_value = self.types_manager.get_fill_value(fill_value, encoding, ptype) return context def get_pval(self, context): return get_value_class(context.param_type, SimpleDomainSet((20,))) def rdt_to_granule(self, context, value_array, comp_val=None): time = ParameterContext(name='time', param_type=QuantityType(value_encoding=np.float64)) pdict = ParameterDictionary() pdict.add_context(time, is_temporal=True) pdict.add_context(context) rdt = RecordDictionaryTool(param_dictionary=pdict) rdt['time'] = np.arange(len(value_array)) rdt['test'] = value_array granule = rdt.to_granule() rdt2 = RecordDictionaryTool.load_from_granule(granule) testval = comp_val if comp_val is not None else value_array actual = rdt2['test'] if isinstance(testval, basestring): self.assertEquals(testval, actual) else: np.testing.assert_array_equal(testval, actual) def cov_io(self, context, value_array, comp_val=None): cov = _make_cov('test_data', [context], nt=0) self.addCleanup(shutil.rmtree, cov.persistence_dir) value_array = np.asanyarray(value_array) data_dict = { 'time' : NumpyParameterData('time', np.arange(len(value_array))), 'test' : NumpyParameterData('test', value_array) } cov.set_parameter_values(data_dict) comp_val = comp_val if comp_val is not None else value_array testval = cov.get_parameter_values(['test']).get_data()['test'] if isinstance(testval, np.recarray): testval = testval.view(np.ndarray) try: np.testing.assert_array_equal(testval, comp_val) except: print repr(value_array) raise def test_quantity_type(self): ptype = 'quantity' encodings = ['int8', 'int16', 'int32', 'int64', 'uint8', 'uint16', 'uint32', 'uint64', 'float32', 'float64'] fill_value = '9' for encoding in encodings: context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) paramval[:] = np.arange(20) self.assertTrue((paramval[:] == np.arange(20)).all()) self.rdt_to_granule(context, np.arange(20)) self.cov_io(context, np.arange(20)) def test_string_type(self): ptype = 'string' encoding = 'S8' fill_value = 'empty' context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) paramval[:] = [context.fill_value] * 20 [self.assertEquals(paramval[i], context.fill_value) for i in xrange(20)] paramval[:] = ['hi'] * 20 [self.assertEquals(paramval[i], 'hi') for i in xrange(20)] ptype = 'str' encoding = 'vlen' fill_value = 'empty' self.rdt_to_granule(context, ['hi'] * 20) self.cov_io(context, ['hi'] * 20) def test_string_arrays(self): ptype = 'string' encoding = 'str' fill_value = 'none' context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) paramval[:] = [context.fill_value] * 20 [self.assertEquals(paramval[i], context.fill_value) for i in xrange(20)] paramval[:] = ['hi'] * 20 [self.assertEquals(paramval[i], 'hi') for i in xrange(20)] self.rdt_to_granule(context,['hi'] * 20) self.cov_io(context, ['hi'] * 20) @unittest.skip("Array types are incomplete") def test_array_type(self): ptype = 'array<quantity>' encoding = 'int32' fill_value = '-9999' context = self.get_context(ptype, encoding, fill_value) self.assertEquals(context.param_type.inner_encoding, np.dtype('int32').str) testval = np.array([1,2,3]*20).reshape(20,3) self.rdt_to_granule(context, [[1,2,3]] * 20, testval) self.cov_io(context, testval) @unittest.skip("Category types are incomplete") def test_category_type(self): ptype = 'category<int8:str>' encoding = 'int8' codeset = '{0: "off", 1: "on"}' fill_value = '0' context = self.get_context(ptype, encoding, fill_value, codeset) paramval = self.get_pval(context) for key in context.param_type.categories: self.assertIsInstance(key,np.int8) paramval[:] = [None] * 20 [self.assertEquals(paramval[i], 'off') for i in xrange(20)] paramval[:] = [context.fill_value] * 20 [self.assertEquals(paramval[i], 'off') for i in xrange(20)] paramval[:] = [1] * 20 [self.assertEquals(paramval[i], 'on') for i in xrange(20)] self.rdt_to_granule(context, [1] * 20, ['on'] * 20) self.cov_io(context, [1]* 20, ['on'] * 20) def test_const_str(self): ptype = 'constant<str>' encoding = 'S12' fill_value = 'empty' context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) paramval[:] = context.fill_value [self.assertEquals(paramval[i], context.fill_value) for i in xrange(20)] paramval[:] = 'hi' [self.assertEquals(paramval[i], 'hi') for i in xrange(20)] long_str = 'this string is too long' paramval[0] = long_str [self.assertEquals(paramval[i], long_str[:12]) for i in xrange(20)] self.rdt_to_granule(context, ['hi']) self.cov_io(context, ['hi']) def test_const(self): ptype = 'constant<quantity>' encoding = 'uint8' fill_value = '12' context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) [self.assertEquals(paramval[i], context.fill_value) for i in xrange(20)] paramval[:] = 2 [self.assertEquals(paramval[i], 2) for i in xrange(20)] self.rdt_to_granule(context, [2]) self.cov_io(context, [2]) def test_boolean(self): ptype = 'boolean' encoding = 'int8' fill_value = 'false' context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) paramval[:] = context.fill_value [self.assertEquals(paramval[i], context.fill_value) for i in xrange(20)] paramval[:] = [1] * 20 [self.assertTrue(paramval[i]) for i in xrange(20)] self.assertEquals(self.types_manager.get_fill_value('true',encoding), 1, ptype) self.rdt_to_granule(context, [1] * 20, [True] * 20) self.cov_io(context, [1] * 20, [True] * 20) def test_bad_codeset(self): ptype = 'category<int8:str>' encoding = 'int8' codeset = '{{0:"hi"}' self.assertRaises(TypeError, self.types_manager.get_parameter_type, ptype, encoding, codeset) def test_invalid_str_len(self): ptype = 'constant<str>' encoding = 'Sfour' self.assertRaises(TypeError, self.types_manager.get_parameter_type, ptype, encoding) ptype = 'constant<str>' encoding = 'int8' self.assertRaises(TypeError, self.types_manager.get_parameter_type, ptype, encoding) ptype = 'quantity' encoding = 'Sfour' self.assertRaises(TypeError, self.types_manager.get_parameter_type, ptype, encoding) def test_str_fill(self): ptype = 'quantity' encoding = 'S8' fill_value = 'now' context = self.get_context(ptype,encoding,fill_value) paramval = self.get_pval(context) [self.assertEquals(paramval[i], 'now') for i in xrange(20)] def test_invalid_fill(self): encoding = 'opaque' fill_value = 'a' self.assertRaises(TypeError, self.types_manager.get_fill_value,fill_value, encoding) def test_record_type(self): ptype = 'record<>' encoding = '' fill_value = '' context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) [self.assertEquals(paramval[i], None) for i in xrange(20)] paramval[:] = [{0:'a'}] * 20 [self.assertEquals(paramval[i], {0:'a'}) for i in xrange(20)] testval = [{0:'a'}] * 20 self.rdt_to_granule(context, [{0:'a'}] * 20) self.cov_io(context, testval) def test_bad_ptype(self): self.assertRaises(TypeError, self.types_manager.get_parameter_type, 'flimsy','','') def test_lookup_value_check(self): func = NumexprFunction('f', 'coeff_a * x', ['coeff_a','x'], param_map={'x':'x', 'coeff_a':'coeff_a'}) func.lookup_values = ['abc123'] test_context = ParameterContext('test', param_type=ParameterFunctionType(func)) tm = TypesManager(None,None,None) self.assertTrue(tm.has_lookup_value(test_context)) self.assertEquals(tm.get_lookup_value_ids(test_context), ['abc123']) def test_bad_units(self): tm = TypesManager(None,None,None) self.assertRaises(UdunitsError,tm.get_unit, 'something')
class TestTypes(PyonTestCase): def setUp(self): PyonTestCase.setUp(self) self.types_manager = TypesManager(None, None, None) def get_context(self, ptype, encoding, fill_value, codeset=None): ptype = self.types_manager.get_parameter_type(ptype, encoding, codeset) context = ParameterContext(name='test', param_type=ptype) context.fill_value = self.types_manager.get_fill_value( fill_value, encoding, ptype) return context def get_pval(self, context): return get_value_class(context.param_type, SimpleDomainSet((20, ))) def rdt_to_granule(self, context, value_array, comp_val=None): pdict = ParameterDictionary() pdict.add_context(context) rdt = RecordDictionaryTool(param_dictionary=pdict) rdt['test'] = value_array granule = rdt.to_granule() rdt2 = RecordDictionaryTool.load_from_granule(granule) testval = comp_val if comp_val is not None else value_array actual = rdt2['test'] if isinstance(testval, basestring): self.assertEquals(testval, actual) else: np.testing.assert_array_equal(testval, actual) def cov_io(self, context, value_array, comp_val=None): pdict = ParameterDictionary() time = ParameterContext( name='time', param_type=QuantityType(value_encoding=np.float64)) pdict.add_context(context) pdict.add_context(time, True) # Construct temporal and spatial Coordinate Reference System objects tcrs = CRS([AxisTypeEnum.TIME]) scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT]) # Construct temporal and spatial Domain objects tdom = GridDomain(GridShape('temporal', [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline) sdom = GridDomain(GridShape('spatial', [0]), scrs, MutabilityEnum.IMMUTABLE ) # 0d spatial topology (station/trajectory) # Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain cov = SimplexCoverage('test_data', create_guid(), 'sample coverage_model', parameter_dictionary=pdict, temporal_domain=tdom, spatial_domain=sdom) self.addCleanup(shutil.rmtree, cov.persistence_dir) cov.insert_timesteps(len(value_array)) cov.set_parameter_values('test', tdoa=slice(0, len(value_array)), value=value_array) comp_val = comp_val if comp_val is not None else value_array testval = cov.get_parameter_values('test') try: np.testing.assert_array_equal(testval, comp_val) except: print repr(value_array) raise def test_quantity_type(self): ptype = 'quantity' encodings = [ 'int8', 'int16', 'int32', 'int64', 'uint8', 'uint16', 'uint32', 'uint64', 'float32', 'float64' ] fill_value = '9' for encoding in encodings: context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) paramval[:] = np.arange(20) self.assertTrue((paramval[:] == np.arange(20)).all()) self.rdt_to_granule(context, np.arange(20)) self.cov_io(context, np.arange(20)) def test_string_type(self): ptype = 'quantity' encoding = 'S8' fill_value = 'empty' context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) paramval[:] = [context.fill_value] * 20 [ self.assertEquals(paramval[i], context.fill_value) for i in xrange(20) ] paramval[:] = ['hi'] * 20 [self.assertEquals(paramval[i], 'hi') for i in xrange(20)] ptype = 'str' encoding = 'vlen' fill_value = 'empty' context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) paramval[:] = [context.fill_value] * 20 [ self.assertEquals(paramval[i], context.fill_value) for i in xrange(20) ] paramval[:] = ['hi'] * 20 [self.assertEquals(paramval[i], 'hi') for i in xrange(20)] self.rdt_to_granule(context, ['hi'] * 20) self.cov_io(context, ['hi'] * 20) def test_string_arrays(self): ptype = 'array<quantity>' encoding = 'str' fill_value = 'none' context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) paramval[:] = [context.fill_value] * 20 [ self.assertEquals(paramval[i], context.fill_value) for i in xrange(20) ] paramval[:] = ['hi'] * 20 [self.assertEquals(paramval[i], 'hi') for i in xrange(20)] self.rdt_to_granule(context, ['hi'] * 20) self.cov_io(context, ['hi'] * 20) def test_plain_array_type(self): ptype = 'array<quantity>' encoding = '' fill_value = 'empty' context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) paramval[:] = [context.fill_value] * 20 [ self.assertEquals(paramval[i], context.fill_value) for i in xrange(20) ] paramval[:] = [[1, 2, 3]] * 20 [ np.testing.assert_array_equal(paramval[i], [1, 2, 3]) for i in xrange(20) ] testval = np.array([None] * 20) for i in xrange(20): testval[i] = [1, 2, 3] self.rdt_to_granule(context, [[1, 2, 3]] * 20, testval) self.cov_io(context, testval) def test_array_type(self): ptype = 'array<quantity>' encoding = 'int32' fill_value = '-9999' context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) self.assertEquals(context.param_type.inner_encoding, np.dtype('int32').str) #TODO: Fill value not supported #-------------------------------------------------------------------------------- #paramval[:] = [context.fill_value] * 20 #[self.assertEquals(paramval[i], context.fill_value) for i in xrange(20)] #-------------------------------------------------------------------------------- paramval[:] = [[1, 2, 3]] * 20 np.testing.assert_array_equal(paramval[:], np.array([1, 2, 3] * 20).reshape(20, 3)) # None not supported #testval = np.array([None] * 20) #for i in xrange(20): # testval[i] = [1,2,3] testval = np.array([1, 2, 3] * 20).reshape(20, 3) self.rdt_to_granule(context, [[1, 2, 3]] * 20, testval) self.cov_io(context, testval) def test_category_type(self): ptype = 'category<int8:str>' encoding = 'int8' codeset = '{0: "off", 1: "on"}' fill_value = '0' context = self.get_context(ptype, encoding, fill_value, codeset) paramval = self.get_pval(context) for key in context.param_type.categories: self.assertIsInstance(key, np.int8) paramval[:] = [None] * 20 [self.assertEquals(paramval[i], 'off') for i in xrange(20)] paramval[:] = [context.fill_value] * 20 [self.assertEquals(paramval[i], 'off') for i in xrange(20)] paramval[:] = [1] * 20 [self.assertEquals(paramval[i], 'on') for i in xrange(20)] self.rdt_to_granule(context, [1] * 20, ['on'] * 20) self.cov_io(context, [1] * 20, ['on'] * 20) def test_const_str(self): ptype = 'constant<str>' encoding = 'S12' fill_value = 'empty' context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) paramval[:] = context.fill_value [ self.assertEquals(paramval[i], context.fill_value) for i in xrange(20) ] paramval[:] = 'hi' [self.assertEquals(paramval[i], 'hi') for i in xrange(20)] long_str = 'this string is too long' paramval[0] = long_str [self.assertEquals(paramval[i], long_str[:12]) for i in xrange(20)] self.rdt_to_granule(context, 'hi') self.cov_io(context, 'hi') def test_const(self): ptype = 'constant<quantity>' encoding = 'uint8' fill_value = '12' context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) [ self.assertEquals(paramval[i], context.fill_value) for i in xrange(20) ] paramval[:] = 2 [self.assertEquals(paramval[i], 2) for i in xrange(20)] self.rdt_to_granule(context, 2) self.cov_io(context, [2]) def test_boolean(self): ptype = 'boolean' encoding = 'int8' fill_value = 'false' context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) paramval[:] = context.fill_value [ self.assertEquals(paramval[i], context.fill_value) for i in xrange(20) ] paramval[:] = [1] * 20 [self.assertTrue(paramval[i]) for i in xrange(20)] self.assertEquals(self.types_manager.get_fill_value('true', encoding), 1, ptype) self.rdt_to_granule(context, [1] * 20, [True] * 20) self.cov_io(context, [1] * 20, [True] * 20) def test_range_type(self): ptype = 'range<quantity>' encoding = 'int32' fill_value = '(-9999, -9998)' context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) [ self.assertEquals(paramval[i], context.fill_value) for i in xrange(20) ] paramval[:] = (0, 1000) [self.assertEquals(paramval[i], (0, 1000)) for i in xrange(20)] testval = np.array([None]) testval[0] = (0, 1000) self.rdt_to_granule(context, (0, 1000), testval) testval = np.array([None] * 20) for i in xrange(20): testval[i] = (0, 1000) self.cov_io(context, paramval, testval) def test_bad_codeset(self): ptype = 'category<int8:str>' encoding = 'int8' codeset = '{{0:"hi"}' self.assertRaises(TypeError, self.types_manager.get_parameter_type, ptype, encoding, codeset) def test_invalid_str_len(self): ptype = 'constant<str>' encoding = 'Sfour' self.assertRaises(TypeError, self.types_manager.get_parameter_type, ptype, encoding) ptype = 'constant<str>' encoding = 'int8' self.assertRaises(TypeError, self.types_manager.get_parameter_type, ptype, encoding) ptype = 'quantity' encoding = 'Sfour' self.assertRaises(TypeError, self.types_manager.get_parameter_type, ptype, encoding) def test_invalid_range(self): ptype = 'range<str>' encoding = '' self.assertRaises(TypeError, self.types_manager.get_parameter_type, ptype, encoding) def test_str_fill(self): ptype = 'quantity' encoding = 'S8' fill_value = 'now' context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) [self.assertEquals(paramval[i], 'now') for i in xrange(20)] def test_invalid_fill(self): encoding = 'opaque' fill_value = 'a' self.assertRaises(TypeError, self.types_manager.get_fill_value, fill_value, encoding) def test_invalid_range_fill(self): ptype = 'range<quantity>' encoding = 'int32' fill_value = '-9999' ptype = self.types_manager.get_parameter_type(ptype, encoding) self.assertRaises(TypeError, self.types_manager.get_fill_value, fill_value, encoding, ptype) def test_record_type(self): ptype = 'record<>' encoding = '' fill_value = '' context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) [self.assertEquals(paramval[i], None) for i in xrange(20)] paramval[:] = [{0: 'a'}] * 20 [self.assertEquals(paramval[i], {0: 'a'}) for i in xrange(20)] testval = [{0: 'a'}] * 20 self.rdt_to_granule(context, [{0: 'a'}] * 20) self.cov_io(context, testval) def test_bad_ptype(self): self.assertRaises(TypeError, self.types_manager.get_parameter_type, 'flimsy', '', '') def test_lookup_value_check(self): func = NumexprFunction('f', 'coeff_a * x', ['coeff_a', 'x'], param_map={ 'x': 'x', 'coeff_a': 'coeff_a' }) func.lookup_values = ['abc123'] test_context = ParameterContext('test', param_type=ParameterFunctionType(func)) tm = TypesManager(None, None, None) self.assertTrue(tm.has_lookup_value(test_context)) self.assertEquals(tm.get_lookup_value_ids(test_context), ['abc123']) def test_bad_units(self): tm = TypesManager(None, None, None) self.assertRaises(UdunitsError, tm.get_unit, 'something')
class TestTypes(PyonTestCase): def setUp(self): PyonTestCase.setUp(self) self.types_manager = TypesManager(None, None, None) def get_context(self, ptype, encoding, fill_value, codeset=None): ptype = self.types_manager.get_parameter_type(ptype, encoding, codeset) context = ParameterContext(name="test", param_type=ptype) context.fill_value = self.types_manager.get_fill_value(fill_value, encoding, ptype) return context def get_pval(self, context): return get_value_class(context.param_type, SimpleDomainSet((20,))) def rdt_to_granule(self, context, value_array, comp_val=None): pdict = ParameterDictionary() pdict.add_context(context) rdt = RecordDictionaryTool(param_dictionary=pdict) rdt["test"] = value_array granule = rdt.to_granule() rdt2 = RecordDictionaryTool.load_from_granule(granule) testval = comp_val if comp_val is not None else value_array actual = rdt2["test"] if isinstance(testval, basestring): self.assertEquals(testval, actual) else: np.testing.assert_array_equal(testval, actual) def cov_io(self, context, value_array, comp_val=None): pdict = ParameterDictionary() time = ParameterContext(name="time", param_type=QuantityType(value_encoding=np.float64)) pdict.add_context(context) pdict.add_context(time, True) # Construct temporal and spatial Coordinate Reference System objects tcrs = CRS([AxisTypeEnum.TIME]) scrs = CRS([AxisTypeEnum.LON, AxisTypeEnum.LAT]) # Construct temporal and spatial Domain objects tdom = GridDomain(GridShape("temporal", [0]), tcrs, MutabilityEnum.EXTENSIBLE) # 1d (timeline) sdom = GridDomain( GridShape("spatial", [0]), scrs, MutabilityEnum.IMMUTABLE ) # 0d spatial topology (station/trajectory) # Instantiate the SimplexCoverage providing the ParameterDictionary, spatial Domain and temporal Domain cov = SimplexCoverage( "test_data", create_guid(), "sample coverage_model", parameter_dictionary=pdict, temporal_domain=tdom, spatial_domain=sdom, ) self.addCleanup(shutil.rmtree, cov.persistence_dir) cov.insert_timesteps(len(value_array)) cov.set_parameter_values("test", tdoa=slice(0, len(value_array)), value=value_array) comp_val = comp_val if comp_val is not None else value_array testval = cov.get_parameter_values("test") try: np.testing.assert_array_equal(testval, comp_val) except: print repr(value_array) raise def test_quantity_type(self): ptype = "quantity" encodings = ["int8", "int16", "int32", "int64", "uint8", "uint16", "uint32", "uint64", "float32", "float64"] fill_value = "9" for encoding in encodings: context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) paramval[:] = np.arange(20) self.assertTrue((paramval[:] == np.arange(20)).all()) self.rdt_to_granule(context, np.arange(20)) self.cov_io(context, np.arange(20)) def test_string_type(self): ptype = "quantity" encoding = "S8" fill_value = "empty" context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) paramval[:] = [context.fill_value] * 20 [self.assertEquals(paramval[i], context.fill_value) for i in xrange(20)] paramval[:] = ["hi"] * 20 [self.assertEquals(paramval[i], "hi") for i in xrange(20)] ptype = "str" encoding = "vlen" fill_value = "empty" context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) paramval[:] = [context.fill_value] * 20 [self.assertEquals(paramval[i], context.fill_value) for i in xrange(20)] paramval[:] = ["hi"] * 20 [self.assertEquals(paramval[i], "hi") for i in xrange(20)] self.rdt_to_granule(context, ["hi"] * 20) self.cov_io(context, ["hi"] * 20) def test_string_arrays(self): ptype = "array<quantity>" encoding = "str" fill_value = "none" context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) paramval[:] = [context.fill_value] * 20 [self.assertEquals(paramval[i], context.fill_value) for i in xrange(20)] paramval[:] = ["hi"] * 20 [self.assertEquals(paramval[i], "hi") for i in xrange(20)] self.rdt_to_granule(context, ["hi"] * 20) self.cov_io(context, ["hi"] * 20) def test_array_type(self): ptype = "array<quantity>" encoding = "int32" fill_value = "empty" context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) paramval[:] = [context.fill_value] * 20 [self.assertEquals(paramval[i], context.fill_value) for i in xrange(20)] paramval[:] = [[1, 2, 3]] * 20 [np.testing.assert_array_equal(paramval[i], [1, 2, 3]) for i in xrange(20)] testval = np.array([None] * 20) for i in xrange(20): testval[i] = [1, 2, 3] self.rdt_to_granule(context, [[1, 2, 3]] * 20, testval) self.cov_io(context, testval) def test_category_type(self): ptype = "category<int8:str>" encoding = "int8" codeset = '{0: "off", 1: "on"}' fill_value = "0" context = self.get_context(ptype, encoding, fill_value, codeset) paramval = self.get_pval(context) for key in context.param_type.categories: self.assertIsInstance(key, np.int8) paramval[:] = [None] * 20 [self.assertEquals(paramval[i], "off") for i in xrange(20)] paramval[:] = [context.fill_value] * 20 [self.assertEquals(paramval[i], "off") for i in xrange(20)] paramval[:] = [1] * 20 [self.assertEquals(paramval[i], "on") for i in xrange(20)] self.rdt_to_granule(context, [1] * 20, ["on"] * 20) self.cov_io(context, [1] * 20, ["on"] * 20) def test_const_str(self): ptype = "constant<str>" encoding = "S12" fill_value = "empty" context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) paramval[:] = context.fill_value [self.assertEquals(paramval[i], context.fill_value) for i in xrange(20)] paramval[:] = "hi" [self.assertEquals(paramval[i], "hi") for i in xrange(20)] long_str = "this string is too long" paramval[0] = long_str [self.assertEquals(paramval[i], long_str[:12]) for i in xrange(20)] self.rdt_to_granule(context, "hi") self.cov_io(context, "hi") def test_const(self): ptype = "constant<quantity>" encoding = "uint8" fill_value = "12" context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) [self.assertEquals(paramval[i], context.fill_value) for i in xrange(20)] paramval[:] = 2 [self.assertEquals(paramval[i], 2) for i in xrange(20)] self.rdt_to_granule(context, 2) self.cov_io(context, [2]) def test_boolean(self): ptype = "boolean" encoding = "int8" fill_value = "false" context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) paramval[:] = context.fill_value [self.assertEquals(paramval[i], context.fill_value) for i in xrange(20)] paramval[:] = [1] * 20 [self.assertTrue(paramval[i]) for i in xrange(20)] self.assertEquals(self.types_manager.get_fill_value("true", encoding), 1, ptype) self.rdt_to_granule(context, [1] * 20, [True] * 20) self.cov_io(context, [1] * 20, [True] * 20) def test_range_type(self): ptype = "range<quantity>" encoding = "int32" fill_value = "(-9999, -9998)" context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) [self.assertEquals(paramval[i], context.fill_value) for i in xrange(20)] paramval[:] = (0, 1000) [self.assertEquals(paramval[i], (0, 1000)) for i in xrange(20)] testval = np.array([None]) testval[0] = (0, 1000) self.rdt_to_granule(context, (0, 1000), testval) testval = np.array([None] * 20) for i in xrange(20): testval[i] = (0, 1000) self.cov_io(context, paramval, testval) def test_bad_codeset(self): ptype = "category<int8:str>" encoding = "int8" codeset = '{{0:"hi"}' self.assertRaises(TypeError, self.types_manager.get_parameter_type, ptype, encoding, codeset) def test_invalid_str_len(self): ptype = "constant<str>" encoding = "Sfour" self.assertRaises(TypeError, self.types_manager.get_parameter_type, ptype, encoding) ptype = "constant<str>" encoding = "int8" self.assertRaises(TypeError, self.types_manager.get_parameter_type, ptype, encoding) ptype = "quantity" encoding = "Sfour" self.assertRaises(TypeError, self.types_manager.get_parameter_type, ptype, encoding) def test_invalid_range(self): ptype = "range<str>" encoding = "" self.assertRaises(TypeError, self.types_manager.get_parameter_type, ptype, encoding) def test_str_fill(self): ptype = "quantity" encoding = "S8" fill_value = "now" context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) [self.assertEquals(paramval[i], "now") for i in xrange(20)] def test_invalid_fill(self): encoding = "opaque" fill_value = "a" self.assertRaises(TypeError, self.types_manager.get_fill_value, fill_value, encoding) def test_invalid_range_fill(self): ptype = "range<quantity>" encoding = "int32" fill_value = "-9999" ptype = self.types_manager.get_parameter_type(ptype, encoding) self.assertRaises(TypeError, self.types_manager.get_fill_value, fill_value, encoding, ptype) def test_record_type(self): ptype = "record<>" encoding = "" fill_value = "" context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) [self.assertEquals(paramval[i], None) for i in xrange(20)] paramval[:] = [{0: "a"}] * 20 [self.assertEquals(paramval[i], {0: "a"}) for i in xrange(20)] testval = [{0: "a"}] * 20 self.rdt_to_granule(context, [{0: "a"}] * 20) self.cov_io(context, testval) def test_bad_ptype(self): self.assertRaises(TypeError, self.types_manager.get_parameter_type, "flimsy", "", "") def test_lookup_value_check(self): func = NumexprFunction("f", "coeff_a * x", ["coeff_a", "x"], param_map={"x": "x", "coeff_a": "coeff_a"}) func.lookup_values = ["abc123"] test_context = ParameterContext("test", param_type=ParameterFunctionType(func)) tm = TypesManager(None, None, None) self.assertTrue(tm.has_lookup_value(test_context)) self.assertEquals(tm.get_lookup_value_ids(test_context), ["abc123"]) def test_bad_units(self): tm = TypesManager(None, None, None) self.assertRaises(UdunitsError, tm.get_unit, "something")
class TestTypes(PyonTestCase): def setUp(self): PyonTestCase.setUp(self) self.types_manager = TypesManager(None, None, None) def get_context(self, ptype, encoding, fill_value, codeset=None): ptype = self.types_manager.get_parameter_type(ptype, encoding, codeset) context = ParameterContext(name='test', param_type=ptype) context.fill_value = self.types_manager.get_fill_value( fill_value, encoding, ptype) return context def get_pval(self, context): return get_value_class(context.param_type, SimpleDomainSet((20, ))) def rdt_to_granule(self, context, value_array, comp_val=None): time = ParameterContext( name='time', param_type=QuantityType(value_encoding=np.float64)) pdict = ParameterDictionary() pdict.add_context(time, is_temporal=True) pdict.add_context(context) rdt = RecordDictionaryTool(param_dictionary=pdict) rdt['time'] = np.arange(len(value_array)) rdt['test'] = value_array granule = rdt.to_granule() rdt2 = RecordDictionaryTool.load_from_granule(granule) testval = comp_val if comp_val is not None else value_array actual = rdt2['test'] if isinstance(testval, basestring): self.assertEquals(testval, actual) else: np.testing.assert_array_equal(testval, actual) def cov_io(self, context, value_array, comp_val=None): cov = _make_cov('test_data', [context], nt=0) self.addCleanup(shutil.rmtree, cov.persistence_dir) value_array = np.asanyarray(value_array) data_dict = { 'time': NumpyParameterData('time', np.arange(len(value_array))), 'test': NumpyParameterData('test', value_array) } cov.set_parameter_values(data_dict) comp_val = comp_val if comp_val is not None else value_array testval = cov.get_parameter_values(['test']).get_data()['test'] if isinstance(testval, np.recarray): testval = testval.view(np.ndarray) try: np.testing.assert_array_equal(testval, comp_val) except: print repr(value_array) raise def test_quantity_type(self): ptype = 'quantity' encodings = [ 'int8', 'int16', 'int32', 'int64', 'uint8', 'uint16', 'uint32', 'uint64', 'float32', 'float64' ] fill_value = '9' for encoding in encodings: context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) paramval[:] = np.arange(20) self.assertTrue((paramval[:] == np.arange(20)).all()) self.rdt_to_granule(context, np.arange(20)) self.cov_io(context, np.arange(20)) def test_string_type(self): ptype = 'string' encoding = 'S8' fill_value = 'empty' context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) paramval[:] = [context.fill_value] * 20 [ self.assertEquals(paramval[i], context.fill_value) for i in xrange(20) ] paramval[:] = ['hi'] * 20 [self.assertEquals(paramval[i], 'hi') for i in xrange(20)] ptype = 'str' encoding = 'vlen' fill_value = 'empty' self.rdt_to_granule(context, ['hi'] * 20) self.cov_io(context, ['hi'] * 20) def test_string_arrays(self): ptype = 'string' encoding = 'str' fill_value = 'none' context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) paramval[:] = [context.fill_value] * 20 [ self.assertEquals(paramval[i], context.fill_value) for i in xrange(20) ] paramval[:] = ['hi'] * 20 [self.assertEquals(paramval[i], 'hi') for i in xrange(20)] self.rdt_to_granule(context, ['hi'] * 20) self.cov_io(context, ['hi'] * 20) @unittest.skip("Array types are incomplete") def test_array_type(self): ptype = 'array<quantity>' encoding = 'int32' fill_value = '-9999' context = self.get_context(ptype, encoding, fill_value) self.assertEquals(context.param_type.inner_encoding, np.dtype('int32').str) testval = np.array([1, 2, 3] * 20).reshape(20, 3) self.rdt_to_granule(context, [[1, 2, 3]] * 20, testval) self.cov_io(context, testval) @unittest.skip("Category types are incomplete") def test_category_type(self): ptype = 'category<int8:str>' encoding = 'int8' codeset = '{0: "off", 1: "on"}' fill_value = '0' context = self.get_context(ptype, encoding, fill_value, codeset) paramval = self.get_pval(context) for key in context.param_type.categories: self.assertIsInstance(key, np.int8) paramval[:] = [None] * 20 [self.assertEquals(paramval[i], 'off') for i in xrange(20)] paramval[:] = [context.fill_value] * 20 [self.assertEquals(paramval[i], 'off') for i in xrange(20)] paramval[:] = [1] * 20 [self.assertEquals(paramval[i], 'on') for i in xrange(20)] self.rdt_to_granule(context, [1] * 20, ['on'] * 20) self.cov_io(context, [1] * 20, ['on'] * 20) def test_const_str(self): ptype = 'constant<str>' encoding = 'S12' fill_value = 'empty' context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) paramval[:] = context.fill_value [ self.assertEquals(paramval[i], context.fill_value) for i in xrange(20) ] paramval[:] = 'hi' [self.assertEquals(paramval[i], 'hi') for i in xrange(20)] long_str = 'this string is too long' paramval[0] = long_str [self.assertEquals(paramval[i], long_str[:12]) for i in xrange(20)] self.rdt_to_granule(context, ['hi']) self.cov_io(context, ['hi']) def test_const(self): ptype = 'constant<quantity>' encoding = 'uint8' fill_value = '12' context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) [ self.assertEquals(paramval[i], context.fill_value) for i in xrange(20) ] paramval[:] = 2 [self.assertEquals(paramval[i], 2) for i in xrange(20)] self.rdt_to_granule(context, [2]) self.cov_io(context, [2]) def test_boolean(self): ptype = 'boolean' encoding = 'int8' fill_value = 'false' context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) paramval[:] = context.fill_value [ self.assertEquals(paramval[i], context.fill_value) for i in xrange(20) ] paramval[:] = [1] * 20 [self.assertTrue(paramval[i]) for i in xrange(20)] self.assertEquals(self.types_manager.get_fill_value('true', encoding), 1, ptype) self.rdt_to_granule(context, [1] * 20, [True] * 20) self.cov_io(context, [1] * 20, [True] * 20) def test_bad_codeset(self): ptype = 'category<int8:str>' encoding = 'int8' codeset = '{{0:"hi"}' self.assertRaises(TypeError, self.types_manager.get_parameter_type, ptype, encoding, codeset) def test_invalid_str_len(self): ptype = 'constant<str>' encoding = 'Sfour' self.assertRaises(TypeError, self.types_manager.get_parameter_type, ptype, encoding) ptype = 'constant<str>' encoding = 'int8' self.assertRaises(TypeError, self.types_manager.get_parameter_type, ptype, encoding) ptype = 'quantity' encoding = 'Sfour' self.assertRaises(TypeError, self.types_manager.get_parameter_type, ptype, encoding) def test_str_fill(self): ptype = 'quantity' encoding = 'S8' fill_value = 'now' context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) [self.assertEquals(paramval[i], 'now') for i in xrange(20)] def test_invalid_fill(self): encoding = 'opaque' fill_value = 'a' self.assertRaises(TypeError, self.types_manager.get_fill_value, fill_value, encoding) def test_record_type(self): ptype = 'record<>' encoding = '' fill_value = '' context = self.get_context(ptype, encoding, fill_value) paramval = self.get_pval(context) [self.assertEquals(paramval[i], None) for i in xrange(20)] paramval[:] = [{0: 'a'}] * 20 [self.assertEquals(paramval[i], {0: 'a'}) for i in xrange(20)] testval = [{0: 'a'}] * 20 self.rdt_to_granule(context, [{0: 'a'}] * 20) self.cov_io(context, testval) def test_bad_ptype(self): self.assertRaises(TypeError, self.types_manager.get_parameter_type, 'flimsy', '', '') def test_lookup_value_check(self): func = NumexprFunction('f', 'coeff_a * x', ['coeff_a', 'x'], param_map={ 'x': 'x', 'coeff_a': 'coeff_a' }) func.lookup_values = ['abc123'] test_context = ParameterContext('test', param_type=ParameterFunctionType(func)) tm = TypesManager(None, None, None) self.assertTrue(tm.has_lookup_value(test_context)) self.assertEquals(tm.get_lookup_value_ids(test_context), ['abc123']) def test_bad_units(self): tm = TypesManager(None, None, None) self.assertRaises(UdunitsError, tm.get_unit, 'something')