def guess_dtype(data): """ Attempt to guess an appropriate dtype for the object, returning None if nothing is appropriate (or if it should be left up the the array constructor to figure out) """ if isinstance(data, h5r.RegionReference): return h5t.special_dtype(ref=h5r.RegionReference) if isinstance(data, h5r.Reference): return h5t.special_dtype(ref=h5r.Reference) return None
from unittest import TestCase import numpy as np from h5py import h5t import ctypes strings = ["Hi", "Hello", "This is a string", "HDF5 is awesome!"] vlen_dtype = h5t.special_dtype(vlen=str) vlen_htype = h5t.py_create(vlen_dtype, logical=1) obj_htype = h5t.py_create(vlen_dtype) class TestVlenObject(TestCase): """ Test conversion routines between string vlens and object pointers """ def test_obj2vlen_simple(self): """ Object to vlen (contiguous) """ objarr = np.array(strings, dtype=vlen_dtype) destbuffer = np.ndarray(objarr.shape, dtype=np.uintp, buffer=objarr).copy() h5t.convert(obj_htype, vlen_htype, len(strings), destbuffer) for idx, val in enumerate(destbuffer): self.assertEqual(ctypes.string_at(int(val)), strings[idx])
def test_vlen(self): a = np.array([np.arange(3), np.arange(4)], dtype=h5t.special_dtype(vlen=int)) self.f.attrs['a'] = a self.assertArrayEqual(self.f.attrs['a'][0], a[0])
def createBaseDataType(typeItem): dtRet = None if type(typeItem) == str or type(typeItem) == unicode: # should be one of the predefined types dtName = getNumpyTypename(typeItem) dtRet = np.dtype(dtName) return dtRet # return predefined type if type(typeItem) != dict: raise TypeError("Type Error: invalid type") if 'class' not in typeItem: raise KeyError("'class' not provided") typeClass = typeItem['class'] dims = '' if 'dims' in typeItem: dims = None if type(typeItem['dims']) == int: dims = (typeItem['dims']) # make into a tuple elif type(typeItem['dims']) not in (list, tuple): raise TypeError("expected list or integer for dims") else: dims = typeItem['dims'] dims = str(tuple(dims)) if typeClass == 'H5T_INTEGER': if 'base' not in typeItem: raise KeyError("'base' not provided") baseType = getNumpyTypename(typeItem['base'], typeClass='H5T_INTEGER') dtRet = np.dtype(dims + baseType) elif typeClass == 'H5T_FLOAT': if 'base' not in typeItem: raise KeyError("'base' not provided") baseType = getNumpyTypename(typeItem['base'], typeClass='H5T_FLOAT') dtRet = np.dtype(dims + baseType) elif typeClass == 'H5T_STRING': if 'length' not in typeItem: raise KeyError("'length' not provided") if 'charSet' not in typeItem: raise KeyError("'charSet' not provided") if typeItem['length'] == 'H5T_VARIABLE': if dims: raise TypeError("ArrayType is not supported for variable len types") if typeItem['charSet'] == 'H5T_CSET_ASCII': dtRet = special_dtype(vlen=str) elif typeItem['charSet'] == 'H5T_CSET_UTF8': dtRet = special_dtype(vlen=unicode) else: raise TypeError("unexpected 'charSet' value") else: nStrSize = typeItem['length'] if type(nStrSize) != int: raise TypeError("expecting integer value for 'length'") type_code = None if typeItem['charSet'] == 'H5T_CSET_ASCII': type_code = 'S' elif typeItem['charSet'] == 'H5T_CSET_UTF8': raise TypeError("fixed-width unicode strings are not supported") else: raise TypeError("unexpected 'charSet' value") dtRet = np.dtype(dims + type_code + str(nStrSize)) # fixed size string elif typeClass == 'H5T_VLEN': if dims: raise TypeError("ArrayType is not supported for variable len types") if 'base' not in typeItem: raise KeyError("'base' not provided") baseType = createBaseDataType(typeItem['base']) dtRet = special_dtype(vlen=np.dtype(baseType)) elif typeClass == 'H5T_OPAQUE': if dims: raise TypeError("Opaque Type is not supported for variable len types") if 'size' not in typeItem: raise KeyError("'size' not provided") nSize = int(typeItem['size']) if nSize <= 0: raise TypeError("'size' must be non-negative") dtRet = np.dtype('V' + str(nSize)) elif typeClass == 'H5T_ARRAY': if not dims: raise KeyError("'dims' must be provided for array types") if 'base' not in typeItem: raise KeyError("'base' not provided") arrayBaseType = typeItem['base'] if type(arrayBaseType) is dict: if "class" not in arrayBaseType: raise KeyError("'class' not provided for array base type") if arrayBaseType["class"] not in ('H5T_INTEGER', 'H5T_FLOAT', 'H5T_STRING'): raise TypeError("Array Type base type must be integer, float, or string") baseType = createDataType(arrayBaseType) dtRet = np.dtype(dims+baseType.str) return dtRet # return predefined type elif typeClass == 'H5T_REFERENCE': if 'base' not in typeItem: raise KeyError("'base' not provided") if typeItem['base'] == 'H5T_STD_REF_OBJ': dtRet = special_dtype(ref=Reference) elif typeItem['base'] == 'H5T_STD_REF_DSETREG': dtRet = special_dtype(ref=RegionReference) else: raise TypeError("Invalid base type for reference type") else: raise TypeError("Invalid type class") return dtRet
def createBaseDataType(typeItem): dtRet = None if type(typeItem) in (str, unicode): # should be one of the predefined types dtName = getNumpyTypename(typeItem) dtRet = np.dtype(dtName) return dtRet # return predefined type if type(typeItem) != dict: raise TypeError("Type Error: invalid type") if 'class' not in typeItem: raise KeyError("'class' not provided") typeClass = typeItem['class'] if typeClass == 'H5T_INTEGER': if 'base' not in typeItem: raise KeyError("'base' not provided") if 'dims' in typeItem: raise TypeError("'dims' not supported for integer types") baseType = getNumpyTypename(typeItem['base'], typeClass='H5T_INTEGER') dtRet = np.dtype(baseType) elif typeClass == 'H5T_FLOAT': if 'base' not in typeItem: raise KeyError("'base' not provided") if 'dims' in typeItem: raise TypeError("'dims' not supported for floating point types") baseType = getNumpyTypename(typeItem['base'], typeClass='H5T_FLOAT') dtRet = np.dtype(baseType) elif typeClass == 'H5T_STRING': if 'length' not in typeItem: raise KeyError("'length' not provided") if 'charSet' not in typeItem: raise KeyError("'charSet' not provided") if typeItem['length'] == 'H5T_VARIABLE': if 'dims' in typeItem: raise TypeError("'dims' not supported for variable types") if typeItem['charSet'] == 'H5T_CSET_ASCII': dtRet = special_dtype(vlen=bytes) elif typeItem['charSet'] == 'H5T_CSET_UTF8': dtRet = special_dtype(vlen=unicode) else: raise TypeError("unexpected 'charSet' value") else: nStrSize = typeItem['length'] if type(nStrSize) != int: raise TypeError("expecting integer value for 'length'") type_code = None if typeItem['charSet'] == 'H5T_CSET_ASCII': type_code = 'S' elif typeItem['charSet'] == 'H5T_CSET_UTF8': raise TypeError("fixed-width unicode strings are not supported") else: raise TypeError("unexpected 'charSet' value") dtRet = np.dtype(type_code + str(nStrSize)) # fixed size string elif typeClass == 'H5T_VLEN': if 'dims' in typeItem: raise TypeError("'dims' not supported for vlen types") if 'base' not in typeItem: raise KeyError("'base' not provided") baseType = createBaseDataType(typeItem['base']) dtRet = special_dtype(vlen=np.dtype(baseType)) elif typeClass == 'H5T_OPAQUE': if 'dims' in typeItem: raise TypeError("'dims' not supported for opaque types") if 'size' not in typeItem: raise KeyError("'size' not provided") nSize = int(typeItem['size']) if nSize <= 0: raise TypeError("'size' must be non-negative") dtRet = np.dtype('V' + str(nSize)) elif typeClass == 'H5T_ARRAY': if not 'dims' in typeItem: raise KeyError("'dims' must be provided for array types") if 'base' not in typeItem: raise KeyError("'base' not provided") arrayBaseType = typeItem['base'] if type(arrayBaseType) is dict: if "class" not in arrayBaseType: raise KeyError("'class' not provided for array base type") if arrayBaseType["class"] not in ('H5T_INTEGER', 'H5T_FLOAT', 'H5T_STRING'): raise TypeError("Array Type base type must be integer, float, or string") dt_base = createDataType(arrayBaseType) if type(typeItem['dims']) == int: dims = (typeItem['dims']) # make into a tuple elif type(typeItem['dims']) not in (list, tuple): raise TypeError("expected list or integer for dims") else: dims = typeItem['dims'] # create an array type of the base type dtRet = np.dtype((dt_base, dims)) elif typeClass == 'H5T_REFERENCE': if 'dims' in typeItem: raise TypeError("'dims' not supported for reference types") if 'base' not in typeItem: raise KeyError("'base' not provided") if typeItem['base'] == 'H5T_STD_REF_OBJ': dtRet = special_dtype(ref=Reference) elif typeItem['base'] == 'H5T_STD_REF_DSETREG': dtRet = special_dtype(ref=RegionReference) else: raise TypeError("Invalid base type for reference type") elif typeClass == 'H5T_ENUM': if 'base' not in typeItem: raise KeyError("Expected 'base' to be provided for enum type") base_json = typeItem["base"] if 'class' not in base_json: raise KeyError("Expected class field in base type") if base_json['class'] != 'H5T_INTEGER': raise TypeError("Only integer base types can be used with enum type") if 'mapping' not in typeItem: raise KeyError("'mapping' not provided for enum type") mapping = typeItem["mapping"] if len(mapping) == 0: raise KeyError("empty enum map") dt = createBaseDataType(base_json) if dt.kind == 'i' and dt.name=='int8' and len(mapping) == 2 and 'TRUE' in mapping and 'FALSE' in mapping: # convert to numpy boolean type dtRet = np.dtype("bool") else: # not a boolean enum, use h5py special dtype dtRet = special_dtype(enum=(dt, mapping)) else: raise TypeError("Invalid type class") return dtRet
def test_enum_lit(self): """ (Types) Enum literal is HDF5 integer """ dt = h5t.special_dtype(enum=('i', {'a': 1, 'b': 2})) htype = h5t.py_create(dt) self.assertIsInstance(htype, h5t.TypeIntegerID)
def test_regref_log(self): """ (Types) Region reference logical is HDF5 dset reference """ dt = h5t.special_dtype(ref=h5r.RegionReference) htype = h5t.py_create(dt, logical=True) self.assertEqual(htype, h5t.STD_REF_DSETREG)
def test_regref(self): """ (Types) Region reference literal is Python object """ dt = h5t.special_dtype(ref=h5r.RegionReference) htype = h5t.py_create(dt) self.assertEqual(htype, h5t.PYTHON_OBJECT)
def test_objref_log(self): """ (Types) Object reference logical is HDF5 reference """ dt = h5t.special_dtype(ref=h5r.Reference) htype = h5t.py_create(dt, logical=True) self.assertEqual(htype, h5t.STD_REF_OBJ)
def createBaseDataType(typeItem): dtRet = None if type(typeItem) in (str, unicode): # should be one of the predefined types dtName = getNumpyTypename(typeItem) dtRet = np.dtype(dtName) return dtRet # return predefined type if type(typeItem) != dict: raise TypeError("Type Error: invalid type") if 'class' not in typeItem: raise KeyError("'class' not provided") typeClass = typeItem['class'] if typeClass == 'H5T_INTEGER': if 'base' not in typeItem: raise KeyError("'base' not provided") if 'dims' in typeItem: raise TypeError("'dims' not supported for integer types") baseType = getNumpyTypename(typeItem['base'], typeClass='H5T_INTEGER') dtRet = np.dtype(baseType) elif typeClass == 'H5T_FLOAT': if 'base' not in typeItem: raise KeyError("'base' not provided") if 'dims' in typeItem: raise TypeError("'dims' not supported for floating point types") baseType = getNumpyTypename(typeItem['base'], typeClass='H5T_FLOAT') dtRet = np.dtype(baseType) elif typeClass == 'H5T_STRING': if 'length' not in typeItem: raise KeyError("'length' not provided") if 'charSet' not in typeItem: raise KeyError("'charSet' not provided") if typeItem['length'] == 'H5T_VARIABLE': if 'dims' in typeItem: raise TypeError("'dims' not supported for variable types") if typeItem['charSet'] == 'H5T_CSET_ASCII': dtRet = special_dtype(vlen=bytes) elif typeItem['charSet'] == 'H5T_CSET_UTF8': dtRet = special_dtype(vlen=unicode) else: raise TypeError("unexpected 'charSet' value") else: nStrSize = typeItem['length'] if type(nStrSize) != int: raise TypeError("expecting integer value for 'length'") type_code = None if typeItem['charSet'] == 'H5T_CSET_ASCII': type_code = 'S' elif typeItem['charSet'] == 'H5T_CSET_UTF8': raise TypeError( "fixed-width unicode strings are not supported") else: raise TypeError("unexpected 'charSet' value") dtRet = np.dtype(type_code + str(nStrSize)) # fixed size string elif typeClass == 'H5T_VLEN': if 'dims' in typeItem: raise TypeError("'dims' not supported for vlen types") if 'base' not in typeItem: raise KeyError("'base' not provided") baseType = createBaseDataType(typeItem['base']) dtRet = special_dtype(vlen=np.dtype(baseType)) elif typeClass == 'H5T_OPAQUE': if 'dims' in typeItem: raise TypeError("'dims' not supported for opaque types") if 'size' not in typeItem: raise KeyError("'size' not provided") nSize = int(typeItem['size']) if nSize <= 0: raise TypeError("'size' must be non-negative") dtRet = np.dtype('V' + str(nSize)) elif typeClass == 'H5T_ARRAY': if not 'dims' in typeItem: raise KeyError("'dims' must be provided for array types") if 'base' not in typeItem: raise KeyError("'base' not provided") arrayBaseType = typeItem['base'] if type(arrayBaseType) is dict: if "class" not in arrayBaseType: raise KeyError("'class' not provided for array base type") if arrayBaseType["class"] not in ('H5T_INTEGER', 'H5T_FLOAT', 'H5T_STRING'): raise TypeError( "Array Type base type must be integer, float, or string") dt_base = createDataType(arrayBaseType) if type(typeItem['dims']) == int: dims = (typeItem['dims']) # make into a tuple elif type(typeItem['dims']) not in (list, tuple): raise TypeError("expected list or integer for dims") else: dims = typeItem['dims'] # create an array type of the base type dtRet = np.dtype((dt_base, dims)) elif typeClass == 'H5T_REFERENCE': if 'dims' in typeItem: raise TypeError("'dims' not supported for reference types") if 'base' not in typeItem: raise KeyError("'base' not provided") if typeItem['base'] == 'H5T_STD_REF_OBJ': dtRet = special_dtype(ref=Reference) elif typeItem['base'] == 'H5T_STD_REF_DSETREG': dtRet = special_dtype(ref=RegionReference) else: raise TypeError("Invalid base type for reference type") elif typeClass == 'H5T_ENUM': if 'base' not in typeItem: raise KeyError("Expected 'base' to be provided for enum type") base_json = typeItem["base"] if 'class' not in base_json: raise KeyError("Expected class field in base type") if base_json['class'] != 'H5T_INTEGER': raise TypeError( "Only integer base types can be used with enum type") if 'mapping' not in typeItem: raise KeyError("'mapping' not provided for enum type") mapping = typeItem["mapping"] if len(mapping) == 0: raise KeyError("empty enum map") dt = createBaseDataType(base_json) if dt.kind == 'i' and dt.name == 'int8' and len( mapping) == 2 and 'TRUE' in mapping and 'FALSE' in mapping: # convert to numpy boolean type dtRet = np.dtype("bool") else: # not a boolean enum, use h5py special dtype dtRet = special_dtype(enum=(dt, mapping)) else: raise TypeError("Invalid type class") return dtRet