def test_issue_212(self):
""" Issue 212
Fails with:
AttributeError: 'SharedConfig' object has no attribute 'lapl'
"""
def closer(x):
def w():
try:
if x:
x.close()
except IOError:
pass
return w
orig_name = self.mktemp()
new_name = self.mktemp()
f = File(orig_name, 'w')
self.addCleanup(closer(f))
f.create_group('a')
f.close()
g = File(new_name, 'w')
self.addCleanup(closer(g))
g['link'] = ExternalLink(orig_name, '/') # note root group
g.close()
h = File(new_name, 'r')
self.addCleanup(closer(h))
self.assertIsInstance(h['link']['a'], Group)
def test_create(self):
""" Creating external links """
self.f['ext'] = ExternalLink(self.ename, '/external')
grp = self.f['ext']
self.ef = grp.file
self.assertNotEqual(self.ef, self.f)
self.assertEqual(grp.name, '/external')
def test_h5py_external_link(self, h5py_file_with_group):
external = ExternalLink("external.hdf5", "example")
registries = new_registry_list()
registries.to_file(h5py_file_with_group, "alias", external)
assert h5py_file_with_group.get("alias",
getlink=True).path == "example"
assert h5py_file_with_group.get(
"alias", getlink=True).filename == "external.hdf5"
def test_unicode_encode(self):
"""
Check that external links encode unicode filenames properly
Testing issue #732
"""
ext_filename = os.path.join(mkdtemp(), u"α.hdf5")
with File(ext_filename, "w") as ext_file:
ext_file.create_group('external')
self.f['ext'] = ExternalLink(ext_filename, '/external')
def test_close_file(self):
""" Files opened by accessing external links can be closed
Issue 189.
"""
self.f['ext'] = ExternalLink(self.ename, '/')
grp = self.f['ext']
f2 = grp.file
f2.close()
self.assertFalse(f2)
def test_unicode_decode(self):
"""
Check that external links decode unicode filenames properly
Testing issue #732
"""
ext_filename = os.path.join(mkdtemp(), u"α.hdf5")
with File(ext_filename, "w") as ext_file:
ext_file.create_group('external')
ext_file["external"].attrs["ext_attr"] = "test"
self.f['ext'] = ExternalLink(ext_filename, '/external')
self.assertEqual(self.f["ext"].attrs["ext_attr"], "test")
def test_unicode_hdf5_path(self):
"""
Check that external links handle unicode hdf5 paths properly
Testing issue #333
"""
ext_filename = os.path.join(mkdtemp(), "external.hdf5")
with File(ext_filename, "w") as ext_file:
ext_file.create_group('α')
ext_file["α"].attrs["ext_attr"] = "test"
self.f['ext'] = ExternalLink(ext_filename, '/α')
self.assertEqual(self.f["ext"].attrs["ext_attr"], "test")
def test_copy_external_links(self):
filename = self.f1.filename
self.f1['foo'] = [1, 2, 3]
self.f2['bar'] = ExternalLink(filename, 'foo')
self.f1.close()
self.f1 = None
self.assertArrayEqual(self.f2['bar'], np.array([1, 2, 3]))
self.f2.copy('bar', 'baz', expand_external=True)
os.unlink(filename)
self.assertArrayEqual(self.f2['baz'], np.array([1, 2, 3]))
def write_link(self, **kwargs):
parent, builder = getargs('parent', 'builder', kwargs)
name = builder.name
target_builder = builder.builder
path = self.__get_path(target_builder)
# source will indicate target_builder's location
if parent.file.filename == target_builder.source:
link_obj = SoftLink(path)
elif target_builder.source is not None:
link_obj = ExternalLink(target_builder.source, path)
else:
msg = 'cannot create external link to %s' % path
raise ValueError(msg)
parent[name] = link_obj
return link_obj
def test_get_link_class(self):
""" Get link classes """
default = object()
sl = SoftLink('/mongoose')
el = ExternalLink('somewhere.hdf5', 'mongoose')
self.f.create_group('hard')
self.f['soft'] = sl
self.f['external'] = el
out_hl = self.f.get('hard', default, getlink=True, getclass=True)
out_sl = self.f.get('soft', default, getlink=True, getclass=True)
out_el = self.f.get('external', default, getlink=True, getclass=True)
self.assertEqual(out_hl, HardLink)
self.assertEqual(out_sl, SoftLink)
self.assertEqual(out_el, ExternalLink)
def test_get_link(self):
""" Get link values """
sl = SoftLink('/mongoose')
el = ExternalLink('somewhere.hdf5', 'mongoose')
self.f.create_group('hard')
self.f['soft'] = sl
self.f['external'] = el
out_hl = self.f.get('hard', getlink=True)
out_sl = self.f.get('soft', getlink=True)
out_el = self.f.get('external', getlink=True)
#TODO: redo with SoftLink/ExternalLink built-in equality
self.assertIsInstance(out_hl, HardLink)
self.assertIsInstance(out_sl, SoftLink)
self.assertEqual(out_sl._path, sl._path)
self.assertIsInstance(out_el, ExternalLink)
self.assertEqual(out_el._path, el._path)
self.assertEqual(out_el._filename, el._filename)
def write_link(self, **kwargs):
parent, builder = getargs('parent', 'builder', kwargs)
if builder.written:
return None
name = builder.name
target_builder = builder.builder
path = self.__get_path(target_builder)
# source will indicate target_builder's location
if parent.file.filename == target_builder.source:
link_obj = SoftLink(path)
elif target_builder.source is not None:
target_filename = os.path.abspath(target_builder.source)
parent_filename = os.path.abspath(parent.file.filename)
relative_path = os.path.relpath(target_filename,
os.path.dirname(parent_filename))
link_obj = ExternalLink(relative_path, path)
else:
msg = 'cannot create external link to %s' % path
raise ValueError(msg)
parent[name] = link_obj
builder.written = True
return link_obj
def test_erepr(self):
""" External link repr """
el = ExternalLink('foo.hdf5', '/foo')
self.assertIsInstance(repr(el), str)
def write_dataset(self, **kwargs):
""" Write a dataset to HDF5
The function uses other dataset-dependent write functions, e.g,
__scalar_fill__, __list_fill__ and __chunked_iter_fill__ to write the data.
"""
parent, builder, link_data = getargs('parent', 'builder', 'link_data',
kwargs)
if builder.written:
print('%s already written to %s' %
(self.__get_path(builder), builder.source))
return None
name = builder.name
data = builder.data
options = dict() # dict with additional
if isinstance(data, H5DataIO):
options['io_settings'] = data.io_settings
link_data = data.link_data
data = data.data
else:
options['io_settings'] = {}
attributes = builder.attributes
options['dtype'] = builder.dtype
dset = None
link = None
# The user provided an existing h5py dataset as input and asked to create a link to the dataset
if isinstance(data, Dataset):
# Create a Soft/External link to the dataset
if link_data:
data_filename = os.path.abspath(data.file.filename)
parent_filename = os.path.abspath(parent.file.filename)
if data_filename != parent_filename:
link = ExternalLink(
os.path.relpath(data_filename,
os.path.dirname(parent_filename)),
data.name)
else:
link = SoftLink(data.name)
parent[name] = link
# Copy the dataset
else:
parent.copy(source=data,
dest=parent,
name=name,
expand_soft=False,
expand_external=False,
expand_refs=False,
without_attrs=True)
dset = parent[name]
# Write a compound dataset, i.e, a dataset with compound data type
elif isinstance(options['dtype'], list):
# do some stuff to figure out what data is a reference
refs = list()
for i, dts in enumerate(options['dtype']):
if self.__is_ref(dts):
refs.append(i)
# If one ore more of the parts of the compound data type are references then we need to deal with those
if len(refs) > 0:
_dtype = self.__resolve_dtype__(options['dtype'], data)
dset = parent.require_dataset(name,
shape=(len(data), ),
dtype=_dtype,
**options['io_settings'])
builder.written = True
@self.__queue_ref
def _filler():
ret = list()
for item in data:
new_item = list(item)
for i in refs:
new_item[i] = self.__get_ref(item[i])
ret.append(tuple(new_item))
dset = parent[name]
dset[:] = ret
self.set_attributes(dset, attributes)
return
# If the compound data type contains only regular data (i.e., no references) then we can write it as usual
else:
dset = self.__list_fill__(parent, name, data, options)
# Write a dataset containing references, i.e., a region or object reference.
# NOTE: we can ignore options['io_settings'] for scalar data
elif self.__is_ref(options['dtype']):
_dtype = self.__dtypes[options['dtype']]
# Write a scalar data region reference dataset
if isinstance(data, RegionBuilder):
dset = parent.require_dataset(name, shape=(), dtype=_dtype)
builder.written = True
@self.__queue_ref
def _filler():
ref = self.__get_ref(data.builder, data.region)
dset = parent[name]
dset[()] = ref
self.set_attributes(dset, attributes)
# Write a scalar object reference dataset
elif isinstance(data, ReferenceBuilder):
dset = parent.require_dataset(name, dtype=_dtype, shape=())
builder.written = True
@self.__queue_ref
def _filler():
ref = self.__get_ref(data.builder)
dset = parent[name]
dset[()] = ref
self.set_attributes(dset, attributes)
# Write an array dataset of references
else:
# Write a array of region references
if options['dtype'] == 'region':
dset = parent.require_dataset(name,
dtype=_dtype,
shape=(len(data), ),
**options['io_settings'])
builder.written = True
@self.__queue_ref
def _filler():
refs = list()
for item in data:
refs.append(
self.__get_ref(item.builder, item.region))
dset = parent[name]
dset[()] = refs
self.set_attributes(dset, attributes)
# Write array of object references
else:
dset = parent.require_dataset(name,
shape=(len(data), ),
dtype=_dtype,
**options['io_settings'])
builder.written = True
@self.__queue_ref
def _filler():
refs = list()
for item in data:
refs.append(self.__get_ref(item.builder))
dset = parent[name]
self.set_attributes(dset, attributes)
return
# write a "regular" dataset
else:
# Write a scalar dataset containing a single string
if isinstance(data, (text_type, binary_type)):
dset = self.__scalar_fill__(parent, name, data, options)
# Iterative write of a data chunk iterator
elif isinstance(data, AbstractDataChunkIterator):
dset = self.__chunked_iter_fill__(parent, name, data, options)
# Write a regular in memory array (e.g., numpy array, list etc.)
elif hasattr(data, '__len__'):
dset = self.__list_fill__(parent, name, data, options)
# Write a regular scalar dataset
else:
dset = self.__scalar_fill__(parent, name, data, options)
# Create the attributes on the dataset only if we are the primary and not just a Soft/External link
if link is None:
self.set_attributes(dset, attributes)
# Validate the attributes on the linked dataset
elif len(attributes) > 0:
pass
builder.written = True
return
def test_h5py_external_link(self, empty_registry):
registries = RegistryContainer(empty_registry)
external_link = ExternalLink("example.hdf5", "/example")
assert external_link == registries.dump(external_link)
def write_dataset(self, **kwargs):
""" Write a dataset to HDF5
The function uses other dataset-dependent write functions, e.g,
__scalar_fill__, __list_fill__ and __chunked_iter_fill__ to write the data.
"""
parent, builder = getargs('parent', 'builder', kwargs)
name = builder.name
data = builder.data
options = dict()
if isinstance(data, H5DataIO):
options['compression'] = 'gzip' if data.compress else None
data = data.data
attributes = builder.attributes
options['dtype'] = builder.dtype
dset = None
link = None
if isinstance(options['dtype'], list):
# do some stuff to figure out what data is a reference
refs = list()
for i, dts in enumerate(options['dtype']):
if self.__is_ref(dts):
refs.append(i)
if len(refs) > 0:
_dtype = self.__resolve_dtype__(options['dtype'], data)
dset = parent.require_dataset(name, shape=(len(data),), dtype=_dtype)
@self.__queue_ref
def _filler():
ret = list()
for item in data:
new_item = list(item)
for i in refs:
new_item[i] = self.__get_ref(item[i])
ret.append(tuple(new_item))
dset = parent[name]
dset[:] = ret
self.set_attributes(dset, attributes)
return
else:
dset = self.__list_fill__(parent, name, data, options)
elif self.__is_ref(options['dtype']):
_dtype = self.__dtypes[options['dtype']]
if isinstance(data, RegionBuilder):
dset = parent.require_dataset(name, shape=(), dtype=_dtype)
@self.__queue_ref
def _filler():
ref = self.__get_ref(data.builder, data.region)
dset = parent[name]
dset[()] = ref
self.set_attributes(dset, attributes)
elif isinstance(data, ReferenceBuilder):
dset = parent.require_dataset(name, dtype=_dtype, shape=())
@self.__queue_ref
def _filler():
ref = self.__get_ref(data.builder)
dset = parent[name]
dset[()] = ref
self.set_attributes(dset, attributes)
else:
if options['dtype'] == 'region':
dset = parent.require_dataset(name, dtype=_dtype, shape=(len(data),))
@self.__queue_ref
def _filler():
refs = list()
for item in data:
refs.append(self.__get_ref(item.builder, item.region))
dset = parent[name]
dset[()] = refs
self.set_attributes(dset, attributes)
else:
dset = parent.require_dataset(name, shape=(len(data),), dtype=_dtype)
@self.__queue_ref
def _filler():
refs = list()
for item in data:
refs.append(self.__get_ref(item.builder))
dset = parent[name]
self.set_attributes(dset, attributes)
return
else:
if isinstance(data, str):
dset = self.__scalar_fill__(parent, name, data, options)
elif isinstance(data, DataChunkIterator):
dset = self.__chunked_iter_fill__(parent, name, data, options)
elif isinstance(data, Dataset):
data_filename = os.path.abspath(data.file.filename)
parent_filename = os.path.abspath(parent.file.filename)
if data_filename != parent_filename:
link = ExternalLink(os.path.relpath(data_filename, os.path.dirname(parent_filename)), data.name)
else:
link = SoftLink(data.name)
parent[name] = link
elif isinstance(data, Iterable) and not self.isinstance_inmemory_array(data):
dset = self.__chunked_iter_fill__(parent, name, DataChunkIterator(data=data, buffer_size=100), options)
elif hasattr(data, '__len__'):
dset = self.__list_fill__(parent, name, data, options)
else:
dset = self.__scalar_fill__(parent, name, data, options)
if link is None:
self.set_attributes(dset, attributes)
return dset
def write_dataset(self, **kwargs):
""" Write a dataset to HDF5
The function uses other dataset-dependent write functions, e.g,
__scalar_fill__, __list_fill__ and __chunked_iter_fill__ to write the data.
"""
parent, builder = getargs('parent', 'builder', kwargs)
name = builder.name
data = builder.data
attributes = builder.attributes
dtype = builder.dtype
dset = None
link = None
if isinstance(data, str):
dset = self.__scalar_fill__(parent, name, data)
elif isinstance(data, DataChunkIterator):
dset = self.__chunked_iter_fill__(parent, name, data)
elif isinstance(data, Dataset):
data_filename = os.path.abspath(data.file.filename)
parent_filename = os.path.abspath(parent.file.filename)
if data_filename != parent_filename:
link = ExternalLink(
os.path.relpath(data_filename,
os.path.dirname(parent_filename)),
data.name)
else:
link = SoftLink(data.name)
parent[name] = link
elif isinstance(data, Builder):
_dtype = self.__dtypes[dtype]
if dtype == 'region':
def _filler():
ref = self.__get_ref(data, builder.region)
dset = parent.create_dataset(name,
data=ref,
shape=None,
dtype=_dtype)
self.set_attributes(dset, attributes)
self.__queue_ref(_filler)
else:
def _filler():
ref = self.__get_ref(data)
dset = parent.create_dataset(name,
data=ref,
shape=None,
dtype=_dtype)
self.set_attributes(dset, attributes)
self.__queue_ref(_filler)
return
elif isinstance(data,
Iterable) and not self.isinstance_inmemory_array(data):
dset = self.__chunked_iter_fill__(
parent, name, DataChunkIterator(data=data, buffer_size=100))
elif hasattr(data, '__len__'):
dset = self.__list_fill__(parent, name, data, dtype_spec=dtype)
else:
dset = self.__scalar_fill__(parent, name, data, dtype=dtype)
if link is None:
self.set_attributes(dset, attributes)
return dset
def test_epath(self):
""" External link paths attributes """
el = ExternalLink('foo.hdf5', '/foo')
self.assertEqual(el.filename, 'foo.hdf5')
self.assertEqual(el.path, '/foo')
def test_exc_missingfile(self):
""" KeyError raised when attempting to open missing file """
self.f['ext'] = ExternalLink('mongoose.hdf5', '/foo')
with self.assertRaises(KeyError):
self.f['ext']
def test_exc(self):
""" KeyError raised when attempting to open broken link """
self.f['ext'] = ExternalLink(self.ename, '/missing')
with self.assertRaises(KeyError):
self.f['ext']
