def guess_dtype(data):
""" Attempt to guess an appropriate dtype for the object, returning None
if nothing is appropriate (or if it should be left up the the array
constructor to figure out)
"""
if isinstance(data, h5r.RegionReference):
return h5t.special_dtype(ref=h5r.RegionReference)
if isinstance(data, h5r.Reference):
return h5t.special_dtype(ref=h5r.Reference)
return None
from unittest import TestCase
import numpy as np
from h5py import h5t
import ctypes
strings = ["Hi", "Hello", "This is a string", "HDF5 is awesome!"]
vlen_dtype = h5t.special_dtype(vlen=str)
vlen_htype = h5t.py_create(vlen_dtype, logical=1)
obj_htype = h5t.py_create(vlen_dtype)
class TestVlenObject(TestCase):
"""
Test conversion routines between string vlens and object pointers
"""
def test_obj2vlen_simple(self):
""" Object to vlen (contiguous) """
objarr = np.array(strings, dtype=vlen_dtype)
destbuffer = np.ndarray(objarr.shape, dtype=np.uintp, buffer=objarr).copy()
h5t.convert(obj_htype, vlen_htype, len(strings), destbuffer)
for idx, val in enumerate(destbuffer):
self.assertEqual(ctypes.string_at(int(val)), strings[idx])
def test_vlen(self):
a = np.array([np.arange(3), np.arange(4)],
dtype=h5t.special_dtype(vlen=int))
self.f.attrs['a'] = a
self.assertArrayEqual(self.f.attrs['a'][0], a[0])
def createBaseDataType(typeItem):
dtRet = None
if type(typeItem) == str or type(typeItem) == unicode:
# should be one of the predefined types
dtName = getNumpyTypename(typeItem)
dtRet = np.dtype(dtName)
return dtRet # return predefined type
if type(typeItem) != dict:
raise TypeError("Type Error: invalid type")
if 'class' not in typeItem:
raise KeyError("'class' not provided")
typeClass = typeItem['class']
dims = ''
if 'dims' in typeItem:
dims = None
if type(typeItem['dims']) == int:
dims = (typeItem['dims']) # make into a tuple
elif type(typeItem['dims']) not in (list, tuple):
raise TypeError("expected list or integer for dims")
else:
dims = typeItem['dims']
dims = str(tuple(dims))
if typeClass == 'H5T_INTEGER':
if 'base' not in typeItem:
raise KeyError("'base' not provided")
baseType = getNumpyTypename(typeItem['base'], typeClass='H5T_INTEGER')
dtRet = np.dtype(dims + baseType)
elif typeClass == 'H5T_FLOAT':
if 'base' not in typeItem:
raise KeyError("'base' not provided")
baseType = getNumpyTypename(typeItem['base'], typeClass='H5T_FLOAT')
dtRet = np.dtype(dims + baseType)
elif typeClass == 'H5T_STRING':
if 'length' not in typeItem:
raise KeyError("'length' not provided")
if 'charSet' not in typeItem:
raise KeyError("'charSet' not provided")
if typeItem['length'] == 'H5T_VARIABLE':
if dims:
raise TypeError("ArrayType is not supported for variable len types")
if typeItem['charSet'] == 'H5T_CSET_ASCII':
dtRet = special_dtype(vlen=str)
elif typeItem['charSet'] == 'H5T_CSET_UTF8':
dtRet = special_dtype(vlen=unicode)
else:
raise TypeError("unexpected 'charSet' value")
else:
nStrSize = typeItem['length']
if type(nStrSize) != int:
raise TypeError("expecting integer value for 'length'")
type_code = None
if typeItem['charSet'] == 'H5T_CSET_ASCII':
type_code = 'S'
elif typeItem['charSet'] == 'H5T_CSET_UTF8':
raise TypeError("fixed-width unicode strings are not supported")
else:
raise TypeError("unexpected 'charSet' value")
dtRet = np.dtype(dims + type_code + str(nStrSize)) # fixed size string
elif typeClass == 'H5T_VLEN':
if dims:
raise TypeError("ArrayType is not supported for variable len types")
if 'base' not in typeItem:
raise KeyError("'base' not provided")
baseType = createBaseDataType(typeItem['base'])
dtRet = special_dtype(vlen=np.dtype(baseType))
elif typeClass == 'H5T_OPAQUE':
if dims:
raise TypeError("Opaque Type is not supported for variable len types")
if 'size' not in typeItem:
raise KeyError("'size' not provided")
nSize = int(typeItem['size'])
if nSize <= 0:
raise TypeError("'size' must be non-negative")
dtRet = np.dtype('V' + str(nSize))
elif typeClass == 'H5T_ARRAY':
if not dims:
raise KeyError("'dims' must be provided for array types")
if 'base' not in typeItem:
raise KeyError("'base' not provided")
arrayBaseType = typeItem['base']
if type(arrayBaseType) is dict:
if "class" not in arrayBaseType:
raise KeyError("'class' not provided for array base type")
if arrayBaseType["class"] not in ('H5T_INTEGER', 'H5T_FLOAT', 'H5T_STRING'):
raise TypeError("Array Type base type must be integer, float, or string")
baseType = createDataType(arrayBaseType)
dtRet = np.dtype(dims+baseType.str)
return dtRet # return predefined type
elif typeClass == 'H5T_REFERENCE':
if 'base' not in typeItem:
raise KeyError("'base' not provided")
if typeItem['base'] == 'H5T_STD_REF_OBJ':
dtRet = special_dtype(ref=Reference)
elif typeItem['base'] == 'H5T_STD_REF_DSETREG':
dtRet = special_dtype(ref=RegionReference)
else:
raise TypeError("Invalid base type for reference type")
else:
raise TypeError("Invalid type class")
return dtRet
def createBaseDataType(typeItem):
dtRet = None
if type(typeItem) in (str, unicode):
# should be one of the predefined types
dtName = getNumpyTypename(typeItem)
dtRet = np.dtype(dtName)
return dtRet # return predefined type
if type(typeItem) != dict:
raise TypeError("Type Error: invalid type")
if 'class' not in typeItem:
raise KeyError("'class' not provided")
typeClass = typeItem['class']
if typeClass == 'H5T_INTEGER':
if 'base' not in typeItem:
raise KeyError("'base' not provided")
if 'dims' in typeItem:
raise TypeError("'dims' not supported for integer types")
baseType = getNumpyTypename(typeItem['base'], typeClass='H5T_INTEGER')
dtRet = np.dtype(baseType)
elif typeClass == 'H5T_FLOAT':
if 'base' not in typeItem:
raise KeyError("'base' not provided")
if 'dims' in typeItem:
raise TypeError("'dims' not supported for floating point types")
baseType = getNumpyTypename(typeItem['base'], typeClass='H5T_FLOAT')
dtRet = np.dtype(baseType)
elif typeClass == 'H5T_STRING':
if 'length' not in typeItem:
raise KeyError("'length' not provided")
if 'charSet' not in typeItem:
raise KeyError("'charSet' not provided")
if typeItem['length'] == 'H5T_VARIABLE':
if 'dims' in typeItem:
raise TypeError("'dims' not supported for variable types")
if typeItem['charSet'] == 'H5T_CSET_ASCII':
dtRet = special_dtype(vlen=bytes)
elif typeItem['charSet'] == 'H5T_CSET_UTF8':
dtRet = special_dtype(vlen=unicode)
else:
raise TypeError("unexpected 'charSet' value")
else:
nStrSize = typeItem['length']
if type(nStrSize) != int:
raise TypeError("expecting integer value for 'length'")
type_code = None
if typeItem['charSet'] == 'H5T_CSET_ASCII':
type_code = 'S'
elif typeItem['charSet'] == 'H5T_CSET_UTF8':
raise TypeError("fixed-width unicode strings are not supported")
else:
raise TypeError("unexpected 'charSet' value")
dtRet = np.dtype(type_code + str(nStrSize)) # fixed size string
elif typeClass == 'H5T_VLEN':
if 'dims' in typeItem:
raise TypeError("'dims' not supported for vlen types")
if 'base' not in typeItem:
raise KeyError("'base' not provided")
baseType = createBaseDataType(typeItem['base'])
dtRet = special_dtype(vlen=np.dtype(baseType))
elif typeClass == 'H5T_OPAQUE':
if 'dims' in typeItem:
raise TypeError("'dims' not supported for opaque types")
if 'size' not in typeItem:
raise KeyError("'size' not provided")
nSize = int(typeItem['size'])
if nSize <= 0:
raise TypeError("'size' must be non-negative")
dtRet = np.dtype('V' + str(nSize))
elif typeClass == 'H5T_ARRAY':
if not 'dims' in typeItem:
raise KeyError("'dims' must be provided for array types")
if 'base' not in typeItem:
raise KeyError("'base' not provided")
arrayBaseType = typeItem['base']
if type(arrayBaseType) is dict:
if "class" not in arrayBaseType:
raise KeyError("'class' not provided for array base type")
if arrayBaseType["class"] not in ('H5T_INTEGER', 'H5T_FLOAT', 'H5T_STRING'):
raise TypeError("Array Type base type must be integer, float, or string")
dt_base = createDataType(arrayBaseType)
if type(typeItem['dims']) == int:
dims = (typeItem['dims']) # make into a tuple
elif type(typeItem['dims']) not in (list, tuple):
raise TypeError("expected list or integer for dims")
else:
dims = typeItem['dims']
# create an array type of the base type
dtRet = np.dtype((dt_base, dims))
elif typeClass == 'H5T_REFERENCE':
if 'dims' in typeItem:
raise TypeError("'dims' not supported for reference types")
if 'base' not in typeItem:
raise KeyError("'base' not provided")
if typeItem['base'] == 'H5T_STD_REF_OBJ':
dtRet = special_dtype(ref=Reference)
elif typeItem['base'] == 'H5T_STD_REF_DSETREG':
dtRet = special_dtype(ref=RegionReference)
else:
raise TypeError("Invalid base type for reference type")
elif typeClass == 'H5T_ENUM':
if 'base' not in typeItem:
raise KeyError("Expected 'base' to be provided for enum type")
base_json = typeItem["base"]
if 'class' not in base_json:
raise KeyError("Expected class field in base type")
if base_json['class'] != 'H5T_INTEGER':
raise TypeError("Only integer base types can be used with enum type")
if 'mapping' not in typeItem:
raise KeyError("'mapping' not provided for enum type")
mapping = typeItem["mapping"]
if len(mapping) == 0:
raise KeyError("empty enum map")
dt = createBaseDataType(base_json)
if dt.kind == 'i' and dt.name=='int8' and len(mapping) == 2 and 'TRUE' in mapping and 'FALSE' in mapping:
# convert to numpy boolean type
dtRet = np.dtype("bool")
else:
# not a boolean enum, use h5py special dtype
dtRet = special_dtype(enum=(dt, mapping))
else:
raise TypeError("Invalid type class")
return dtRet
def test_enum_lit(self):
""" (Types) Enum literal is HDF5 integer """
dt = h5t.special_dtype(enum=('i', {'a': 1, 'b': 2}))
htype = h5t.py_create(dt)
self.assertIsInstance(htype, h5t.TypeIntegerID)
def test_regref_log(self):
""" (Types) Region reference logical is HDF5 dset reference """
dt = h5t.special_dtype(ref=h5r.RegionReference)
htype = h5t.py_create(dt, logical=True)
self.assertEqual(htype, h5t.STD_REF_DSETREG)
def test_regref(self):
""" (Types) Region reference literal is Python object """
dt = h5t.special_dtype(ref=h5r.RegionReference)
htype = h5t.py_create(dt)
self.assertEqual(htype, h5t.PYTHON_OBJECT)
def test_objref_log(self):
""" (Types) Object reference logical is HDF5 reference """
dt = h5t.special_dtype(ref=h5r.Reference)
htype = h5t.py_create(dt, logical=True)
self.assertEqual(htype, h5t.STD_REF_OBJ)
def createBaseDataType(typeItem):
dtRet = None
if type(typeItem) in (str, unicode):
# should be one of the predefined types
dtName = getNumpyTypename(typeItem)
dtRet = np.dtype(dtName)
return dtRet # return predefined type
if type(typeItem) != dict:
raise TypeError("Type Error: invalid type")
if 'class' not in typeItem:
raise KeyError("'class' not provided")
typeClass = typeItem['class']
if typeClass == 'H5T_INTEGER':
if 'base' not in typeItem:
raise KeyError("'base' not provided")
if 'dims' in typeItem:
raise TypeError("'dims' not supported for integer types")
baseType = getNumpyTypename(typeItem['base'], typeClass='H5T_INTEGER')
dtRet = np.dtype(baseType)
elif typeClass == 'H5T_FLOAT':
if 'base' not in typeItem:
raise KeyError("'base' not provided")
if 'dims' in typeItem:
raise TypeError("'dims' not supported for floating point types")
baseType = getNumpyTypename(typeItem['base'], typeClass='H5T_FLOAT')
dtRet = np.dtype(baseType)
elif typeClass == 'H5T_STRING':
if 'length' not in typeItem:
raise KeyError("'length' not provided")
if 'charSet' not in typeItem:
raise KeyError("'charSet' not provided")
if typeItem['length'] == 'H5T_VARIABLE':
if 'dims' in typeItem:
raise TypeError("'dims' not supported for variable types")
if typeItem['charSet'] == 'H5T_CSET_ASCII':
dtRet = special_dtype(vlen=bytes)
elif typeItem['charSet'] == 'H5T_CSET_UTF8':
dtRet = special_dtype(vlen=unicode)
else:
raise TypeError("unexpected 'charSet' value")
else:
nStrSize = typeItem['length']
if type(nStrSize) != int:
raise TypeError("expecting integer value for 'length'")
type_code = None
if typeItem['charSet'] == 'H5T_CSET_ASCII':
type_code = 'S'
elif typeItem['charSet'] == 'H5T_CSET_UTF8':
raise TypeError(
"fixed-width unicode strings are not supported")
else:
raise TypeError("unexpected 'charSet' value")
dtRet = np.dtype(type_code + str(nStrSize)) # fixed size string
elif typeClass == 'H5T_VLEN':
if 'dims' in typeItem:
raise TypeError("'dims' not supported for vlen types")
if 'base' not in typeItem:
raise KeyError("'base' not provided")
baseType = createBaseDataType(typeItem['base'])
dtRet = special_dtype(vlen=np.dtype(baseType))
elif typeClass == 'H5T_OPAQUE':
if 'dims' in typeItem:
raise TypeError("'dims' not supported for opaque types")
if 'size' not in typeItem:
raise KeyError("'size' not provided")
nSize = int(typeItem['size'])
if nSize <= 0:
raise TypeError("'size' must be non-negative")
dtRet = np.dtype('V' + str(nSize))
elif typeClass == 'H5T_ARRAY':
if not 'dims' in typeItem:
raise KeyError("'dims' must be provided for array types")
if 'base' not in typeItem:
raise KeyError("'base' not provided")
arrayBaseType = typeItem['base']
if type(arrayBaseType) is dict:
if "class" not in arrayBaseType:
raise KeyError("'class' not provided for array base type")
if arrayBaseType["class"] not in ('H5T_INTEGER', 'H5T_FLOAT',
'H5T_STRING'):
raise TypeError(
"Array Type base type must be integer, float, or string")
dt_base = createDataType(arrayBaseType)
if type(typeItem['dims']) == int:
dims = (typeItem['dims']) # make into a tuple
elif type(typeItem['dims']) not in (list, tuple):
raise TypeError("expected list or integer for dims")
else:
dims = typeItem['dims']
# create an array type of the base type
dtRet = np.dtype((dt_base, dims))
elif typeClass == 'H5T_REFERENCE':
if 'dims' in typeItem:
raise TypeError("'dims' not supported for reference types")
if 'base' not in typeItem:
raise KeyError("'base' not provided")
if typeItem['base'] == 'H5T_STD_REF_OBJ':
dtRet = special_dtype(ref=Reference)
elif typeItem['base'] == 'H5T_STD_REF_DSETREG':
dtRet = special_dtype(ref=RegionReference)
else:
raise TypeError("Invalid base type for reference type")
elif typeClass == 'H5T_ENUM':
if 'base' not in typeItem:
raise KeyError("Expected 'base' to be provided for enum type")
base_json = typeItem["base"]
if 'class' not in base_json:
raise KeyError("Expected class field in base type")
if base_json['class'] != 'H5T_INTEGER':
raise TypeError(
"Only integer base types can be used with enum type")
if 'mapping' not in typeItem:
raise KeyError("'mapping' not provided for enum type")
mapping = typeItem["mapping"]
if len(mapping) == 0:
raise KeyError("empty enum map")
dt = createBaseDataType(base_json)
if dt.kind == 'i' and dt.name == 'int8' and len(
mapping) == 2 and 'TRUE' in mapping and 'FALSE' in mapping:
# convert to numpy boolean type
dtRet = np.dtype("bool")
else:
# not a boolean enum, use h5py special dtype
dtRet = special_dtype(enum=(dt, mapping))
else:
raise TypeError("Invalid type class")
return dtRet
