def test_create(self):
""" Create new soft link by assignment """
g = self.f.create_group('new')
sl = SoftLink('/new')
self.f['alias'] = sl
g2 = self.f['alias']
self.assertEqual(g, g2)
def _set_link(self, path, name, obj):
"""
Set link
name is the symlink path, obj is the obj we want to link to
"""
# pylint: disable=protected-access
if isinstance(obj, BaseGroupWrapper):
self._file[str(HDF5Path(path, name))] = SoftLink(str(obj._path))
else:
raise TypeError("Not a valid definition of {}".format(name))
def test_copy_soft_links(self):
self.f1['bar'] = [1, 2, 3]
foo = self.f1.create_group('foo')
foo['baz'] = SoftLink('/bar')
self.f1.copy(foo, 'qux', expand_soft=True)
self.f2.copy(foo, 'foo', expand_soft=True)
del self.f1['bar']
self.assertIsInstance(self.f1['qux'], Group)
self.assertArrayEqual(self.f1['qux/baz'], np.array([1, 2, 3]))
self.assertIsInstance(self.f2['/foo'], Group)
self.assertArrayEqual(self.f2['foo/baz'], np.array([1, 2, 3]))
def write_link(self, **kwargs):
parent, builder = getargs('parent', 'builder', kwargs)
name = builder.name
target_builder = builder.builder
path = self.__get_path(target_builder)
# source will indicate target_builder's location
if parent.file.filename == target_builder.source:
link_obj = SoftLink(path)
elif target_builder.source is not None:
link_obj = ExternalLink(target_builder.source, path)
else:
msg = 'cannot create external link to %s' % path
raise ValueError(msg)
parent[name] = link_obj
return link_obj
def test_get_link_class(self):
""" Get link classes """
default = object()
sl = SoftLink('/mongoose')
el = ExternalLink('somewhere.hdf5', 'mongoose')
self.f.create_group('hard')
self.f['soft'] = sl
self.f['external'] = el
out_hl = self.f.get('hard', default, getlink=True, getclass=True)
out_sl = self.f.get('soft', default, getlink=True, getclass=True)
out_el = self.f.get('external', default, getlink=True, getclass=True)
self.assertEqual(out_hl, HardLink)
self.assertEqual(out_sl, SoftLink)
self.assertEqual(out_el, ExternalLink)
def test_get_link(self):
""" Get link values """
sl = SoftLink('/mongoose')
el = ExternalLink('somewhere.hdf5', 'mongoose')
self.f.create_group('hard')
self.f['soft'] = sl
self.f['external'] = el
out_hl = self.f.get('hard', getlink=True)
out_sl = self.f.get('soft', getlink=True)
out_el = self.f.get('external', getlink=True)
#TODO: redo with SoftLink/ExternalLink built-in equality
self.assertIsInstance(out_hl, HardLink)
self.assertIsInstance(out_sl, SoftLink)
self.assertEqual(out_sl._path, sl._path)
self.assertIsInstance(out_el, ExternalLink)
self.assertEqual(out_el._path, el._path)
self.assertEqual(out_el._filename, el._filename)
def write_link(self, **kwargs):
parent, builder = getargs('parent', 'builder', kwargs)
if builder.written:
return None
name = builder.name
target_builder = builder.builder
path = self.__get_path(target_builder)
# source will indicate target_builder's location
if parent.file.filename == target_builder.source:
link_obj = SoftLink(path)
elif target_builder.source is not None:
target_filename = os.path.abspath(target_builder.source)
parent_filename = os.path.abspath(parent.file.filename)
relative_path = os.path.relpath(target_filename,
os.path.dirname(parent_filename))
link_obj = ExternalLink(relative_path, path)
else:
msg = 'cannot create external link to %s' % path
raise ValueError(msg)
parent[name] = link_obj
builder.written = True
return link_obj
def test_exc(self):
""" Opening dangling soft link results in KeyError """
self.f['alias'] = SoftLink('new')
with self.assertRaises(KeyError):
self.f['alias']
def test_srepr(self):
""" SoftLink path repr """
sl = SoftLink('/foo')
self.assertIsInstance(repr(sl), str)
def test_spath(self):
""" SoftLink path attribute """
sl = SoftLink('/foo')
self.assertEqual(sl.path, '/foo')
def write_dataset(self, **kwargs):
""" Write a dataset to HDF5
The function uses other dataset-dependent write functions, e.g,
__scalar_fill__, __list_fill__ and __chunked_iter_fill__ to write the data.
"""
parent, builder = getargs('parent', 'builder', kwargs)
name = builder.name
data = builder.data
attributes = builder.attributes
dtype = builder.dtype
dset = None
link = None
if isinstance(data, str):
dset = self.__scalar_fill__(parent, name, data)
elif isinstance(data, DataChunkIterator):
dset = self.__chunked_iter_fill__(parent, name, data)
elif isinstance(data, Dataset):
data_filename = os.path.abspath(data.file.filename)
parent_filename = os.path.abspath(parent.file.filename)
if data_filename != parent_filename:
link = ExternalLink(
os.path.relpath(data_filename,
os.path.dirname(parent_filename)),
data.name)
else:
link = SoftLink(data.name)
parent[name] = link
elif isinstance(data, Builder):
_dtype = self.__dtypes[dtype]
if dtype == 'region':
def _filler():
ref = self.__get_ref(data, builder.region)
dset = parent.create_dataset(name,
data=ref,
shape=None,
dtype=_dtype)
self.set_attributes(dset, attributes)
self.__queue_ref(_filler)
else:
def _filler():
ref = self.__get_ref(data)
dset = parent.create_dataset(name,
data=ref,
shape=None,
dtype=_dtype)
self.set_attributes(dset, attributes)
self.__queue_ref(_filler)
return
elif isinstance(data,
Iterable) and not self.isinstance_inmemory_array(data):
dset = self.__chunked_iter_fill__(
parent, name, DataChunkIterator(data=data, buffer_size=100))
elif hasattr(data, '__len__'):
dset = self.__list_fill__(parent, name, data, dtype_spec=dtype)
else:
dset = self.__scalar_fill__(parent, name, data, dtype=dtype)
if link is None:
self.set_attributes(dset, attributes)
return dset
def write_dataset(self, **kwargs):
""" Write a dataset to HDF5
The function uses other dataset-dependent write functions, e.g,
__scalar_fill__, __list_fill__ and __chunked_iter_fill__ to write the data.
"""
parent, builder = getargs('parent', 'builder', kwargs)
name = builder.name
data = builder.data
options = dict()
if isinstance(data, H5DataIO):
options['compression'] = 'gzip' if data.compress else None
data = data.data
attributes = builder.attributes
options['dtype'] = builder.dtype
dset = None
link = None
if isinstance(options['dtype'], list):
# do some stuff to figure out what data is a reference
refs = list()
for i, dts in enumerate(options['dtype']):
if self.__is_ref(dts):
refs.append(i)
if len(refs) > 0:
_dtype = self.__resolve_dtype__(options['dtype'], data)
dset = parent.require_dataset(name, shape=(len(data),), dtype=_dtype)
@self.__queue_ref
def _filler():
ret = list()
for item in data:
new_item = list(item)
for i in refs:
new_item[i] = self.__get_ref(item[i])
ret.append(tuple(new_item))
dset = parent[name]
dset[:] = ret
self.set_attributes(dset, attributes)
return
else:
dset = self.__list_fill__(parent, name, data, options)
elif self.__is_ref(options['dtype']):
_dtype = self.__dtypes[options['dtype']]
if isinstance(data, RegionBuilder):
dset = parent.require_dataset(name, shape=(), dtype=_dtype)
@self.__queue_ref
def _filler():
ref = self.__get_ref(data.builder, data.region)
dset = parent[name]
dset[()] = ref
self.set_attributes(dset, attributes)
elif isinstance(data, ReferenceBuilder):
dset = parent.require_dataset(name, dtype=_dtype, shape=())
@self.__queue_ref
def _filler():
ref = self.__get_ref(data.builder)
dset = parent[name]
dset[()] = ref
self.set_attributes(dset, attributes)
else:
if options['dtype'] == 'region':
dset = parent.require_dataset(name, dtype=_dtype, shape=(len(data),))
@self.__queue_ref
def _filler():
refs = list()
for item in data:
refs.append(self.__get_ref(item.builder, item.region))
dset = parent[name]
dset[()] = refs
self.set_attributes(dset, attributes)
else:
dset = parent.require_dataset(name, shape=(len(data),), dtype=_dtype)
@self.__queue_ref
def _filler():
refs = list()
for item in data:
refs.append(self.__get_ref(item.builder))
dset = parent[name]
self.set_attributes(dset, attributes)
return
else:
if isinstance(data, str):
dset = self.__scalar_fill__(parent, name, data, options)
elif isinstance(data, DataChunkIterator):
dset = self.__chunked_iter_fill__(parent, name, data, options)
elif isinstance(data, Dataset):
data_filename = os.path.abspath(data.file.filename)
parent_filename = os.path.abspath(parent.file.filename)
if data_filename != parent_filename:
link = ExternalLink(os.path.relpath(data_filename, os.path.dirname(parent_filename)), data.name)
else:
link = SoftLink(data.name)
parent[name] = link
elif isinstance(data, Iterable) and not self.isinstance_inmemory_array(data):
dset = self.__chunked_iter_fill__(parent, name, DataChunkIterator(data=data, buffer_size=100), options)
elif hasattr(data, '__len__'):
dset = self.__list_fill__(parent, name, data, options)
else:
dset = self.__scalar_fill__(parent, name, data, options)
if link is None:
self.set_attributes(dset, attributes)
return dset
def write_dataset(self, **kwargs):
""" Write a dataset to HDF5
The function uses other dataset-dependent write functions, e.g,
__scalar_fill__, __list_fill__ and __chunked_iter_fill__ to write the data.
"""
parent, builder, link_data = getargs('parent', 'builder', 'link_data',
kwargs)
if builder.written:
print('%s already written to %s' %
(self.__get_path(builder), builder.source))
return None
name = builder.name
data = builder.data
options = dict() # dict with additional
if isinstance(data, H5DataIO):
options['io_settings'] = data.io_settings
link_data = data.link_data
data = data.data
else:
options['io_settings'] = {}
attributes = builder.attributes
options['dtype'] = builder.dtype
dset = None
link = None
# The user provided an existing h5py dataset as input and asked to create a link to the dataset
if isinstance(data, Dataset):
# Create a Soft/External link to the dataset
if link_data:
data_filename = os.path.abspath(data.file.filename)
parent_filename = os.path.abspath(parent.file.filename)
if data_filename != parent_filename:
link = ExternalLink(
os.path.relpath(data_filename,
os.path.dirname(parent_filename)),
data.name)
else:
link = SoftLink(data.name)
parent[name] = link
# Copy the dataset
else:
parent.copy(source=data,
dest=parent,
name=name,
expand_soft=False,
expand_external=False,
expand_refs=False,
without_attrs=True)
dset = parent[name]
# Write a compound dataset, i.e, a dataset with compound data type
elif isinstance(options['dtype'], list):
# do some stuff to figure out what data is a reference
refs = list()
for i, dts in enumerate(options['dtype']):
if self.__is_ref(dts):
refs.append(i)
# If one ore more of the parts of the compound data type are references then we need to deal with those
if len(refs) > 0:
_dtype = self.__resolve_dtype__(options['dtype'], data)
dset = parent.require_dataset(name,
shape=(len(data), ),
dtype=_dtype,
**options['io_settings'])
builder.written = True
@self.__queue_ref
def _filler():
ret = list()
for item in data:
new_item = list(item)
for i in refs:
new_item[i] = self.__get_ref(item[i])
ret.append(tuple(new_item))
dset = parent[name]
dset[:] = ret
self.set_attributes(dset, attributes)
return
# If the compound data type contains only regular data (i.e., no references) then we can write it as usual
else:
dset = self.__list_fill__(parent, name, data, options)
# Write a dataset containing references, i.e., a region or object reference.
# NOTE: we can ignore options['io_settings'] for scalar data
elif self.__is_ref(options['dtype']):
_dtype = self.__dtypes[options['dtype']]
# Write a scalar data region reference dataset
if isinstance(data, RegionBuilder):
dset = parent.require_dataset(name, shape=(), dtype=_dtype)
builder.written = True
@self.__queue_ref
def _filler():
ref = self.__get_ref(data.builder, data.region)
dset = parent[name]
dset[()] = ref
self.set_attributes(dset, attributes)
# Write a scalar object reference dataset
elif isinstance(data, ReferenceBuilder):
dset = parent.require_dataset(name, dtype=_dtype, shape=())
builder.written = True
@self.__queue_ref
def _filler():
ref = self.__get_ref(data.builder)
dset = parent[name]
dset[()] = ref
self.set_attributes(dset, attributes)
# Write an array dataset of references
else:
# Write a array of region references
if options['dtype'] == 'region':
dset = parent.require_dataset(name,
dtype=_dtype,
shape=(len(data), ),
**options['io_settings'])
builder.written = True
@self.__queue_ref
def _filler():
refs = list()
for item in data:
refs.append(
self.__get_ref(item.builder, item.region))
dset = parent[name]
dset[()] = refs
self.set_attributes(dset, attributes)
# Write array of object references
else:
dset = parent.require_dataset(name,
shape=(len(data), ),
dtype=_dtype,
**options['io_settings'])
builder.written = True
@self.__queue_ref
def _filler():
refs = list()
for item in data:
refs.append(self.__get_ref(item.builder))
dset = parent[name]
self.set_attributes(dset, attributes)
return
# write a "regular" dataset
else:
# Write a scalar dataset containing a single string
if isinstance(data, (text_type, binary_type)):
dset = self.__scalar_fill__(parent, name, data, options)
# Iterative write of a data chunk iterator
elif isinstance(data, AbstractDataChunkIterator):
dset = self.__chunked_iter_fill__(parent, name, data, options)
# Write a regular in memory array (e.g., numpy array, list etc.)
elif hasattr(data, '__len__'):
dset = self.__list_fill__(parent, name, data, options)
# Write a regular scalar dataset
else:
dset = self.__scalar_fill__(parent, name, data, options)
# Create the attributes on the dataset only if we are the primary and not just a Soft/External link
if link is None:
self.set_attributes(dset, attributes)
# Validate the attributes on the linked dataset
elif len(attributes) > 0:
pass
builder.written = True
return
def test_h5py_soft_link(self, empty_registry):
registries = RegistryContainer(empty_registry)
soft_link = SoftLink("/example")
assert soft_link == registries.dump(soft_link)
def test_h5py_soft_link(self, h5py_file_with_group):
soft_link = SoftLink("example")
registries = new_registry_list()
registries.to_file(h5py_file_with_group, "alias", soft_link)
assert h5py_file_with_group.get("alias",
getlink=True).path == "example"
