def get_hdf5_with_external_recursive_links():
ID = "external_recursive_links"
if ID in _file_cache:
return _file_cache[ID][0].name
tmp1 = tempfile.NamedTemporaryFile(prefix=ID + "_", suffix=".h5", delete=True)
tmp1.file.close()
h5_1 = h5py.File(tmp1.name, "w")
tmp2 = tempfile.NamedTemporaryFile(prefix=ID + "_", suffix=".h5", delete=True)
tmp2.file.close()
h5_2 = h5py.File(tmp2.name, "w")
g = h5_1.create_group("group")
g.create_dataset("dataset", data=numpy.int64(10))
h5_1["soft_link_to_group"] = h5py.SoftLink("/group")
h5_1["external_link_to_link"] = h5py.ExternalLink(tmp2.name, "/soft_link_to_group")
h5_1["external_link_to_recursive_link"] = h5py.ExternalLink(tmp2.name, "/external_link_to_recursive_link")
h5_1.close()
g = h5_2.create_group("group")
g.create_dataset("dataset", data=numpy.int64(10))
h5_2["soft_link_to_group"] = h5py.SoftLink("/group")
h5_2["external_link_to_link"] = h5py.ExternalLink(tmp1.name, "/soft_link_to_group")
h5_2["external_link_to_recursive_link"] = h5py.ExternalLink(tmp1.name, "/external_link_to_recursive_link")
h5_2.close()
_file_cache[ID] = (tmp1, tmp2)
return tmp1.name
def example_h5_filename(tmpdir, scope='module'):
ext_filename = (tmpdir / "example-data-external.h5")
filename = (tmpdir / "example-data.h5")
with h5py.File(ext_filename, 'w') as f:
d = f.create_dataset("external_ds",
data=np.array([1, 2, 3, 4], dtype='d'))
g = f.create_group("external_group")
with h5py.File(filename, 'w') as f:
g = f.create_group("group")
g.attrs['unit'] = "Flerbians"
g.attrs['count'] = 123
g2 = g.create_group("subgroup")
g3 = g2.create_group("subsubgroup")
d1 = g.create_dataset("scalar", data=1.23)
d1.attrs['cats'] = np.array(["Kali", "Bustopher Jones"],
dtype=h5py.special_dtype(vlen=str))
d2 = g.create_dataset("vector", data=np.array([1, 2, 3], dtype='i'))
f['link'] = d2
f['softlink'] = h5py.SoftLink('/group/scalar')
f['subsubgroup_hardlink'] = g3
f['extlink'] = h5py.ExternalLink(ext_filename, "external_ds")
f['extgroup'] = h5py.ExternalLink(ext_filename, "external_group")
yield filename
def get_hdf5_with_all_links():
ID = "alllinks"
if ID in _file_cache:
return _file_cache[ID].name
tmp = tempfile.NamedTemporaryFile(prefix=ID + "_", suffix=".h5", delete=True)
tmp.file.close()
h5 = h5py.File(tmp.name, "w")
g = h5.create_group("group")
g.create_dataset("dataset", data=numpy.int64(10))
h5.create_dataset("dataset", data=numpy.int64(10))
h5["hard_link_to_group"] = h5["/group"]
h5["hard_link_to_dataset"] = h5["/dataset"]
h5["soft_link_to_group"] = h5py.SoftLink("/group")
h5["soft_link_to_dataset"] = h5py.SoftLink("/dataset")
h5["soft_link_to_nothing"] = h5py.SoftLink("/foo/bar/2000")
alltypes_filename = get_hdf5_with_all_types()
h5["external_link_to_group"] = h5py.ExternalLink(alltypes_filename, "/arrays")
h5["external_link_to_dataset"] = h5py.ExternalLink(alltypes_filename, "/arrays/cube")
h5["external_link_to_nothing"] = h5py.ExternalLink(alltypes_filename, "/foo/bar/2000")
h5["external_link_to_missing_file"] = h5py.ExternalLink("missing_file.h5", "/")
h5.close()
_file_cache[ID] = tmp
return tmp.name
def create_master(params):
# Get path to create the master file
master_file = os.path.join(params["OutputDir"], params["MasterFileName"])
master_file = ".".join([master_file, "hdf5"])
# Get the list of interpolated files
interp_prefix = os.path.join(params["OutputDir"], params["InterpFileName"])
interp_files = sorted(glob(interp_prefix + "*"))
# Open master file
with h5py.File(master_file, mode="w") as mf:
group = mf.create_group("dm_maps")
group.attrs["desc"] = "This is the master file for the interpolated redshfits"
# Link each interpolated file to master file
for i, fn in enumerate(interp_files):
with h5py.File(fn, mode="r") as file_i:
try:
redshift = file_i[params["Header"]].attrs["Redshift"]
except:
redshift = file_i[params["Header"]].attrs["redshift"]
# Create sub_group
group_name = f"slice_{i:03d}"
sub_group = group.create_group(group_name)
# Save the Header and DM dataset to the subgroup
sub_group[params["Header"]] = h5py.ExternalLink(fn, params["Header"])
sub_group[params["Dataset"]] = h5py.ExternalLink(fn, params["Dataset"])
def write_to_file(f, data):
datasets_group = f.create_group('datasets_group')
datasets_group.attrs['string_attr'] = 'my string attribute'
datasets_group.attrs['int_attr'] = 123
datasets_group.attrs['float_attr'] = 123.456
float_group = datasets_group.create_group('float')
float_group.create_dataset('float32', data=data, dtype='f4')
float_group.create_dataset('float64', data=data, dtype='f8', fillvalue=6)
int_group = datasets_group.create_group('int')
int_group.create_dataset('int8', data=data, dtype='i1')
int_group.create_dataset('int16', data=data, dtype='i2')
int_group.create_dataset('int32', data=data, dtype='i4')
links_group = f.create_group('links_group')
links_group['hard_link_to_int8'] = int_group['int8']
links_group['soft_link_to_int8'] = h5py.SoftLink('/datasets_group/int/int8')
links_group['broken_soft_link'] = h5py.SoftLink('/datasets_group/int/missing_dataset')
links_group['soft_link_to_group'] = h5py.SoftLink('/datasets_group/int')
# Define the external link path relative to this file, to ease testing
links_group['external_link'] = h5py.ExternalLink('test_file_ext.hdf5', '/external_dataset')
links_group['external_link_to_missing_file'] = h5py.ExternalLink('missing_file.hdf5', '/external_dataset')
multiDimensionDatasets = f.create_group('nD_Datasets');
multiDimensionDatasets.create_dataset('3D_float32', data=data_3d, dtype='f4')
multiDimensionDatasets.create_dataset('3D_int32', data=data_3d, dtype='i4')
f.flush()
f.close()
def link_or_copy(group, name, link, copy, absolute_paths=False):
'''
Link or copy a dataset or group
Parameters
----------
group : h5py.Group
The group to create the link, dataset, or group in
name : str
The name of the link, dataset, or group in the new file
link : h5py.ExternalLink
A link to the group or dataset to include
copy : bool
Whether to copy or link to the dataset
absolute_paths : bool
If copy=False, then if absolute_paths is True, absolute filenames
are used in the link, otherwise the path relative to the file is used.
'''
if copy:
f = h5py.File(link.filename, 'r')
f.copy(link.path, group, name=name)
f.close()
else:
if absolute_paths:
group[name] = h5py.ExternalLink(os.path.abspath(link.filename), link.path)
else:
group[name] = h5py.ExternalLink(os.path.relpath(link.filename, os.path.dirname(group.file.filename)), link.path)
try:
group[name]
except KeyError: # indicates linking failed (h5py < 2.1.0)
logger.warn("Linking failed, copying instead (indicates an outdated version of h5py)")
del group[name]
f = h5py.File(link.filename, 'r')
f.copy(link.path, group, name=name)
f.close()
def create_links(h5):
print("- Creating links...")
main_group = h5.create_group("links")
dataset = main_group.create_dataset("dataset", data=numpy.int64(10))
group = main_group.create_group("group")
group["something_inside"] = numpy.int64(20)
main_group["hard_link_to_group"] = main_group["group"]
main_group["hard_link_to_dataset"] = main_group["dataset"]
main_group.create_group("hard_recursive_link")
main_group.create_group("hard_recursive_link2")
main_group["hard_recursive_link/link"] = main_group["hard_recursive_link2"]
main_group["hard_recursive_link2/link"] = main_group["hard_recursive_link"]
main_group["soft_link_to_group"] = h5py.SoftLink(group.name)
main_group["soft_link_to_dataset"] = h5py.SoftLink(dataset.name)
main_group["soft_link_to_nothing"] = h5py.SoftLink("/foo/bar/2000")
main_group["soft_link_to_group_link"] = h5py.SoftLink(main_group.name + "/soft_link_to_group")
main_group["soft_link_to_dataset_link"] = h5py.SoftLink(main_group.name + "/soft_link_to_dataset")
main_group["soft_link_to_itself"] = h5py.SoftLink(main_group.name + "/soft_link_to_itself")
# External links to self file
main_group["external_link_to_group"] = h5py.ExternalLink(h5.file.filename, group.name)
main_group["external_link_to_dataset"] = h5py.ExternalLink(h5.file.filename, dataset.name)
main_group["external_link_to_nothing"] = h5py.ExternalLink(h5.file.filename, "/foo/bar/2000")
main_group["external_link_to_missing_file"] = h5py.ExternalLink(h5.file.filename + "_unknown", "/")
main_group["external_link_to_group_link"] = h5py.ExternalLink(h5.file.filename, main_group.name + "/soft_link_to_group")
main_group["external_link_to_dataset_link"] = h5py.ExternalLink(h5.file.filename, main_group.name + "/soft_link_to_dataset")
main_group["external_link_to_itself"] = h5py.ExternalLink(h5.file.filename, main_group.name + "/external_link_to_itself")
main_group["external_link_to_recursive_link2"] = h5py.ExternalLink(h5.file.filename, main_group.name + "/external_link_to_recursive_link3")
main_group["external_link_to_recursive_link3"] = h5py.ExternalLink(h5.file.filename, main_group.name + "/external_link_to_recursive_link2")
main_group["external_link_to_soft_recursive"] = h5py.ExternalLink(h5.file.filename, main_group.name + "/soft_link_to_itself")
def prepare_h5source(ds_dir,
volume_name,
label_file=("groundtruth_seg.h5", "main"),
raw_file=("im_uint8.h5", "main"),
add_dummy_mask=True):
h5_filepath = ".{}.h5".format(volume_name)
if add_dummy_mask:
with h5py.File(os.path.join(ds_dir, volume_name, label_file[0]),
"r") as f_labels:
mask_shape = f_labels[label_file[1]].shape
with h5py.File(h5_filepath, "w") as h5:
h5["volumes/raw"] = h5py.ExternalLink(
os.path.join(ds_dir, volume_name, raw_file[0]), raw_file[1])
h5["volumes/labels/neuron_ids"] = h5py.ExternalLink(
os.path.join(ds_dir, volume_name, label_file[0]), label_file[1])
datasets = {
VolumeTypes.RAW: 'volumes/raw',
VolumeTypes.GT_LABELS: 'volumes/labels/neuron_ids'
}
if add_dummy_mask:
h5.create_dataset(
name="volumes/labels/mask",
dtype="uint8",
shape=mask_shape,
fillvalue=1,
)
datasets[VolumeTypes.GT_MASK] = 'volumes/labels/mask'
h5source = gunpowder.Hdf5Source(
h5_filepath,
datasets=datasets,
#resolutions=(8, 8, 8),
)
return h5source
def createResource(cls, directory):
filename = os.path.join(directory, "base.h5")
externalFilename = os.path.join(directory, "base__external.h5")
externalh5 = h5py.File(externalFilename, mode="w")
externalh5["target/dataset"] = 50
externalh5["target/link"] = h5py.SoftLink("/target/dataset")
externalh5.close()
h5 = h5py.File(filename, mode="w")
h5["group/dataset"] = 50
h5["link/soft_link"] = h5py.SoftLink("/group/dataset")
h5["link/soft_link_to_group"] = h5py.SoftLink("/group")
h5["link/soft_link_to_link"] = h5py.SoftLink("/link/soft_link")
h5["link/soft_link_to_file"] = h5py.SoftLink("/")
h5["link/external_link"] = h5py.ExternalLink(externalFilename,
"/target/dataset")
h5["link/external_link_to_link"] = h5py.ExternalLink(
externalFilename, "/target/link")
h5["broken_link/external_broken_file"] = h5py.ExternalLink(
externalFilename + "_not_exists", "/target/link")
h5["broken_link/external_broken_link"] = h5py.ExternalLink(
externalFilename, "/target/not_exists")
h5["broken_link/soft_broken_link"] = h5py.SoftLink("/group/not_exists")
h5["broken_link/soft_link_to_broken_link"] = h5py.SoftLink(
"/group/not_exists")
h5.close()
return filename
def link_hdf5_files(post_hdf5_link_list):
logger = logging.getLogger()
PLATE_PREFIX = '/sample/0/plate/'
WELL_PREFIX = PLATE_PREFIX + '%s/experiment/'
POSITION_PREFIX = WELL_PREFIX + '%s/position/'
def get_plate_and_postion(hf_file):
plate = hf_file[PLATE_PREFIX].keys()[0]
well = hf_file[WELL_PREFIX % plate].keys()[0]
position = hf_file[POSITION_PREFIX % (plate, well)].keys()[0]
return plate, well, position
all_pos_hdf5_filename = join(split(post_hdf5_link_list[0])[0],
'_all_positions.ch5')
if exists(all_pos_hdf5_filename):
f = h5py.File(all_pos_hdf5_filename, 'a')
### This is dangerous, several processes open the file for writing...
logger.info(("_all_positons.hdf file found, "
"trying to reuse it by overwrite old external links..."))
if 'definition' in f:
del f['definition']
f['definition'] = h5py.ExternalLink(post_hdf5_link_list[0],
'/definition')
for fname in post_hdf5_link_list:
fh = h5py.File(fname, 'r')
fplate, fwell, fpos = get_plate_and_postion(fh)
fh.close()
msg = "Linking into _all_positons.hdf:" + \
((POSITION_PREFIX + '%s') % (fplate, fwell, fpos))
logger.info(msg)
if (POSITION_PREFIX + '%s') % (fplate, fwell, fpos) in f:
del f[(POSITION_PREFIX + '%s') % (fplate, fwell, fpos)]
f[(POSITION_PREFIX + '%s') % (fplate, fwell, fpos)] = \
h5py.ExternalLink(fname, (POSITION_PREFIX + '%s')
% (fplate, fwell, fpos))
f.close()
else:
f = h5py.File(all_pos_hdf5_filename, 'w')
logger.info("_all_positons.hdf file created...")
f['definition'] = h5py.ExternalLink(post_hdf5_link_list[0],'/definition')
for fname in post_hdf5_link_list:
fh = h5py.File(fname, 'r')
fplate, fwell, fpos = get_plate_and_postion(fh)
fh.close()
msg = "Linking into _all_positons.hdf:" + \
((POSITION_PREFIX + '%s') % (fplate, fwell, fpos))
logger.info(msg)
f[(POSITION_PREFIX + '%s') % (fplate, fwell, fpos)] = \
h5py.ExternalLink(fname, (POSITION_PREFIX + '%s')
% (fplate, fwell, fpos))
f.close()
def saveH5(self):
""" """
"""
Private method to save the object to a file. Creates links to h5 files that all contain only one pattern.
:param output_path: The file where to save the object's data.
:type output_path: string, default b
"""
# Path where individual h5 files are located.
path_to_files = self.output_path
# Setup new file.
with h5py.File(self.output_path + ".h5", "w") as h5_outfile:
# Files to read from.
individual_files = [
os.path.join(path_to_files, f)
for f in os.listdir(path_to_files)
]
individual_files.sort()
# Keep track of global parameters being linked.
global_parameters = False
# Loop over all individual files and link in the top level groups.
for ind_file in individual_files:
# Open file.
with h5py.File(ind_file, 'r') as h5_infile:
# Links must be relative.
relative_link_target = os.path.relpath(
path=ind_file,
start=os.path.dirname(os.path.dirname(ind_file)))
# Link global parameters.
if not global_parameters:
global_parameters = True
h5_outfile["params"] = h5py.ExternalLink(
relative_link_target, "params")
h5_outfile["info"] = h5py.ExternalLink(
relative_link_target, "info")
h5_outfile["misc"] = h5py.ExternalLink(
relative_link_target, "misc")
h5_outfile["version"] = h5py.ExternalLink(
relative_link_target, "version")
for key in h5_infile['data']:
# Link in the data.
ds_path = "data/%s" % (key)
h5_outfile[ds_path] = h5py.ExternalLink(
relative_link_target, ds_path)
# Reset output path.
self.output_path = self.output_path + ".h5"
def main(input_filenames, output_filename):
with h5py.File(output_filename) as output_file:
for i, input_filename in enumerate(input_filenames):
with h5py.File(input_filename, "r") as input_file:
dark_field = input_file["postprocessing/dark_field"]
output_file["dark_field_{:02}".format(i)] = h5py.ExternalLink(
input_filename, "postprocessing/dark_field")
ratio = input_file["postprocessing/ratio"]
output_file["ratio_{:02}".format(i)] = h5py.ExternalLink(
input_filename, "postprocessing/ratio")
def createResource(cls, directory):
filename = os.path.join(directory, "base.h5")
extH5FileName = os.path.join(directory, "base__external.h5")
extDatFileName = os.path.join(directory, "base__external.dat")
externalh5 = h5py.File(extH5FileName, mode="w")
externalh5["target/dataset"] = 50
externalh5["target/link"] = h5py.SoftLink("/target/dataset")
externalh5["/ext/vds0"] = [0, 1]
externalh5["/ext/vds1"] = [2, 3]
externalh5.close()
numpy.array([0, 1, 10, 10, 2, 3]).tofile(extDatFileName)
h5 = h5py.File(filename, mode="w")
h5["group/dataset"] = 50
h5["link/soft_link"] = h5py.SoftLink("/group/dataset")
h5["link/soft_link_to_group"] = h5py.SoftLink("/group")
h5["link/soft_link_to_link"] = h5py.SoftLink("/link/soft_link")
h5["link/soft_link_to_file"] = h5py.SoftLink("/")
h5["group/soft_link_relative"] = h5py.SoftLink("dataset")
h5["link/external_link"] = h5py.ExternalLink(extH5FileName,
"/target/dataset")
h5["link/external_link_to_link"] = h5py.ExternalLink(
extH5FileName, "/target/link")
h5["broken_link/external_broken_file"] = h5py.ExternalLink(
extH5FileName + "_not_exists", "/target/link")
h5["broken_link/external_broken_link"] = h5py.ExternalLink(
extH5FileName, "/target/not_exists")
h5["broken_link/soft_broken_link"] = h5py.SoftLink("/group/not_exists")
h5["broken_link/soft_link_to_broken_link"] = h5py.SoftLink(
"/group/not_exists")
layout = h5py.VirtualLayout((2, 2), dtype=int)
layout[0] = h5py.VirtualSource("base__external.h5",
name="/ext/vds0",
shape=(2, ),
dtype=int)
layout[1] = h5py.VirtualSource("base__external.h5",
name="/ext/vds1",
shape=(2, ),
dtype=int)
h5.create_group("/ext")
h5["/ext"].create_virtual_dataset("virtual", layout)
external = [("base__external.dat", 0, 2 * 8),
("base__external.dat", 4 * 8, 2 * 8)]
h5["/ext"].create_dataset("raw",
shape=(2, 2),
dtype=int,
external=external)
h5.close()
return filename
def test_external_links(self):
# create a file for use a link target
if config.get("use_h5py"):
# for some reason this test is failing in Travis
return
linked_filename = self.getFileName("linked_file")
abs_filepath = os.path.abspath(linked_filename)
if config.get("use_h5py"):
rel_filepath = os.path.relpath(linked_filename)
else:
rel_filepath = "linked_file.h5"
f = h5py.File(linked_filename, 'w')
is_hsds = False
if isinstance(f.id.id, str) and f.id.id.startswith("g-"):
is_hsds = True
g1 = f.create_group("g1")
dset = g1.create_dataset('ds', (5,7), dtype='f4')
dset_id = dset.id.id
f.close()
filename = self.getFileName("external_links")
print("filename:", filename)
f = h5py.File(filename, 'w')
f["missing_link"] = h5py.ExternalLink(abs_filepath, "somepath")
f["abspath_link"] = h5py.ExternalLink(abs_filepath, "/g1/ds")
f["relpath_link"] = h5py.ExternalLink(rel_filepath, "/g1/ds")
try:
linked_obj = f["missing_link"]
self.assertTrue(False)
except KeyError:
pass # expected
try:
linked_obj = f["abspath_link"]
self.assertTrue(linked_obj.name, "/g1/ds")
self.assertEqual(linked_obj.shape, (5, 7))
# The following no longer works for h5py 2.8
# self.assertEqual(linked_obj.id.id, dset_id)
except KeyError:
if config.get("use_h5py") or is_hsds:
# absolute paths aren't working yet for h5serv
self.assertTrue(False)
linked_obj = f["relpath_link"]
self.assertTrue(linked_obj.name, "/g1/ds")
self.assertEqual(linked_obj.shape, (5, 7))
if not config.get("use_h5py"):
self.assertEqual(linked_obj.id.id, dset_id)
f.close()
def add_link_to_coord(self, coord, subfile):
path = coord.get_path()
#self.require_group(path)
try:
self[path] = h5py.ExternalLink(subfile, path)
except RuntimeError as e:
print("Link to path '%s' already exists" % path)
#self.require_group(CH5Const.DEFINITION)
try:
self[CH5Const.DEFINITION] = h5py.ExternalLink(
subfile, CH5Const.DEFINITION)
except RuntimeError as e:
# link to definition is already there
print("Link to path '%s' already exists" % path)
def link(target, parent, name):
""" create link
:param target: file name
:type target: :obj:`str`
:param parent: parent object
:type parent: :class:`FTObject`
:param name: link name
:type name: :obj:`str`
:returns: link object
:rtype: :class:`H5PYLink`
"""
localfname = H5PYLink.getfilename(parent)
if ":/" in target:
filename, path = target.split(":/")
if os.path.abspath(filename) != os.path.abspath(localfname):
parent.h5object[name] = h5py.ExternalLink(filename, path)
else:
parent.h5object[name] = h5py.SoftLink(path)
else:
parent.h5object[name] = h5py.SoftLink(target)
return H5PYLink(parent.h5object.get(name, getlink=True),
parent).setname(name)
def write_master_file(
fname,
data_files,
pixel_mask=None,
raw_master_file=None,
i0=None,
compression=hdf5plugin.Bitshuffle(nelems=0, lz4=True),
):
print(color.info(f"Writing: {fname}"))
f = h5py.File(fname, "w")
nxentry, nxdata = create_entry_and_data(f)
if raw_master_file is not None:
#add attrs
nxdata.attrs["exptime"] = raw_master_file['Exptime'].item()
nxdata.attrs["period"] = raw_master_file['Period'].item()
# Link written data sets:
for i, fname in enumerate(data_files, start=1):
f[f"entry/data/data_{i:06d}"] = h5py.ExternalLink(
fname, "entry/data/data")
#Group to hold instrument specific data
grp = nxentry.create_group("instrument/data")
# Pixel mask recognized by Albula
if pixel_mask is not None:
inst = nxentry.create_group("instrument/detector/detectorSpecific")
inst.create_dataset("pixel_mask",
data=pixel_mask.astype(np.uint8),
**compression)
if i0 is not None:
grp.create_dataset("i0", data=i0, **compression)
f.close()
def create_external_link(fname, dataset_path, out_fname, new_dataset_path):
"""
Creates an external link of `fname:dataset_path` into
`out_fname:new_dataset_path`.
:param fname:
The filename of the file containing the dataset that is to
be linked to.
`type:str`
:param dataset_path:
A `str` containg the path to the dataset contained within `fname`.
:param out_fname:
A `str` for the output filename that will contain the link to
`fname:dataset_path`.
:param new_dataset_path:
A `str` containing the dataset path within `out_fname` that will
link to `fname:dataset_path`.
"""
with h5py.File(out_fname) as fid:
fid[new_dataset_path] = h5py.ExternalLink(fname, dataset_path)
return
def write(self, data):
"""
Write out the averaged TOD to a Level2 continuum file with an external link to the original level 1 data
"""
if not os.path.exists(self.output_dir):
os.makedirs(self.output_dir)
# We will store these in a separate file and link them to the level2s
fname = data.filename.split('/')[-1]
if os.path.exists(self.outfile):
os.remove(self.outfile)
output = h5py.File(self.outfile, 'a')
# Set permissions and group
os.chmod(self.outfile, 0o664)
shutil.chown(self.outfile, group='comap')
# Store datasets in root
dnames = ['tod', 'weights']
dsets = [self.all_tod, self.all_weights]
for (dname, dset) in zip(dnames, dsets):
if dname in output:
del output[dname]
output.create_dataset(dname, data=dset)
output.attrs['version'] = __level3_version__
output.close()
if self.level3 in data.keys():
del data[self.level3]
data[self.level3] = h5py.ExternalLink(self.outfile, '/')
def _link_datafile_to_nexus_file(self, data):
filename = self.exp.meta_data.get('nxs_filename')
with h5py.File(filename, 'a') as nxs_file:
# entry path in nexus file
name = data.get_name()
group_name = self.exp.meta_data.get(['group_name', name])
link_type = self.exp.meta_data.get(['link_type', name])
nxs_entry = '/entry/' + link_type
if link_type == 'final_result':
nxs_entry += '_' + data.get_name()
else:
nxs_entry += "/" + group_name
nxs_entry = nxs_file[nxs_entry]
nxs_entry.attrs['signal'] = 'data'
data_entry = nxs_entry.name + '/data'
# output file path
h5file = data.backing_file.filename
# entry path in output file path
m_data = self.exp.meta_data.get
if not (link_type == 'intermediate'
and m_data('inter_path') != m_data('out_path')):
h5file = h5file.split(m_data('out_folder') + '/')[-1]
nxs_file[data_entry] = \
h5py.ExternalLink(h5file, group_name + '/data')
def buildLink(gdst, linkname, gsrc):
fn = sanitize_posixpath(basename(gsrc.file.filename)) if gsrc else ''
gdst.attrs[linkname + '_PATH'] = gsrc.name if gsrc else ''
gdst.attrs[linkname + '_FILE'] = fn
if gsrc:
gdst[linkname] = h5py.SoftLink(gsrc.name) if (
gsrc.file == gdst.file) else h5py.ExternalLink(fn, gsrc.name)
def test_file(self):
""" (Links) File attribute works correctly on external links """
from h5py.h5 import _global_ids as gi
from h5py import h5i
self.f['ext'] = h5py.ExternalLink(self.ename, '/external')
g = self.f['ext']
self.assertNotEqual(g.file, self.f)
def eigerfile(request):
N_chans = 4
N_points = 27
N_bin = 4096
N_roi = 32
seq_id = np.random.randint(10)
pre_fix = gettempprefix()
out_name = f'{pre_fix}_{seq_id}_master.h5'
with h5py.File("sample_data_000001.h5", "w") as dfile:
data = np.random.rand(N_points, N_chans, N_bin) * 1000
dfile["data_000001"] = data
with h5py.File(out_name, "w") as fout:
for e in EigerHandler.EIGER_MD_LAYOUT.values():
fout[e] = np.random.randint(1, 10)
fout["entry/data/data_000001"] = h5py.ExternalLink(
"sample_data_000001.h5", "data_000001")
def finalize():
os.remove(out_name)
os.remove("sample_data_000001.h5")
request.addfinalizer(finalize)
kwargs = {'images_per_file': 1, 'frame_per_point': 1}
return (out_name, kwargs), (N_points, N_chans, N_bin, N_roi)
def prepare_field_file(self):
df = self.get_data_file()
if 'field_dtype' in df.keys():
# we don't need to do anything, raw binary files are used
return None
last_iteration = df['iteration'].value
cppf = df['parameters/checkpoints_per_file'].value
niter_out = df['parameters/niter_out'].value
with h5py.File(os.path.join(self.work_dir, self.simname + '_fields.h5'), 'a') as ff:
ff.require_group('vorticity')
ff.require_group('vorticity/complex')
checkpoint = 0
while True:
cpf_name = os.path.join(
self.work_dir,
self.simname + '_checkpoint_{0}.h5'.format(checkpoint))
if os.path.exists(cpf_name):
cpf = h5py.File(cpf_name, 'r')
for iter_name in cpf['vorticity/complex'].keys():
if iter_name not in ff['vorticity/complex'].keys():
ff['vorticity/complex/' + iter_name] = h5py.ExternalLink(
cpf_name,
'vorticity/complex/' + iter_name)
checkpoint += 1
else:
break
return None
def _get_ext_link_rec(file_path: Path, hdf_path: str, depth_to_go: int,
func_to_fulfill: Callable[[Path, str], bool]) -> set:
"""
recursive function to return external links of hdf files in variable depth
:param file_path: path of the hdf file
:param hdf_path: hdf_path of the hdf object inside the hdf file
:param depth_to_go: goal depth to go
:param func_to_fulfill: the function that has to be fulfilled in order to be gathered
:return: set of external hdf links
"""
if depth_to_go == 0:
return {h5py.ExternalLink(file_path, hdf_path)} if func_to_fulfill(
file_path, hdf_path) else set()
if depth_to_go > 0:
with h5py.File(file_path, "r") as file:
children_set = set()
for key in file[hdf_path].keys():
children_set.update(
set(
_get_ext_link_rec(file_path=file_path,
hdf_path=hdf_path_combine(
hdf_path, key),
depth_to_go=depth_to_go - 1,
func_to_fulfill=func_to_fulfill)))
return children_set
raise ValueError("depth_to_go should be a non negative integer")
def add_history(wf_file_name, history_file_name):
"""
Add history from pearent file to propagated file
:param wf_file_name: output file
:param history_file_name: peraent file
"""
with h5py.File(wf_file_name) as wf_h5:
with h5py.File(history_file_name) as history_h5:
if "history" in wf_h5:
del wf_h5["history"]
wf_h5.create_group("/history/parent/")
wf_h5.create_group("/history/parent/detail")
for k in history_h5:
if k == "history":
try:
history_h5.copy("/history/parent",
wf_h5["history"]["parent"])
except KeyError:
pass
elif not k == "data":
history_h5.copy(k, wf_h5["history"]["parent"]["detail"])
else:
wf_h5["history"]["parent"]["detail"][
"data"] = h5py.ExternalLink(history_file_name, "/data")
def linkfile(self, data):
fname = data.filename.split('/')[-1]
lvl2 = data[self.level2]
if self.prefix in lvl2:
del lvl2[self.prefix]
lvl2[self.prefix] = h5py.ExternalLink(
'{}/{}_{}'.format(self.output_dir, self.prefix, fname), '/')
def nxDataAddSignals(data, signals, append=True):
"""
Add signals to NXdata instance
:param h5py.Group data:
:param list(2-tuple) signals: name(str), value(None,h5py.Dataset,numpy.ndarray,dict), attrs(dict)
:param bool append:
"""
raiseIsNotNxClass(data, u'NXdata')
if append:
newsignals = nxDataGetSignals(data)
else:
newsignals = []
for name, value, attrs in signals:
if value is None:
pass # is or will be created elsewhere
elif isinstance(value, h5py.Dataset):
if value.file.filename != data.file.filename:
data[name] = h5py.ExternalLink(value.file.filename, value.name)
elif value.parent != data:
data[name] = h5py.SoftLink(value.name)
elif isinstance(value, dict):
data.create_dataset(name, **value)
else:
data[name] = value
if attrs:
data[name].attrs.update(attrs)
newsignals.append(name)
if newsignals:
nxDataSetSignals(data, newsignals)
def test_mode_external(self):
""" Mode property works for files opened via external links
Issue 190.
"""
fname1 = self.mktemp()
fname2 = self.mktemp()
f1 = File(fname1, 'w')
f1.close()
f2 = File(fname2, 'w')
try:
f2['External'] = h5py.ExternalLink(fname1, '/')
f3 = f2['External'].file
self.assertEqual(f3.mode, 'r+')
finally:
f2.close()
f3.close()
f2 = File(fname2, 'r')
try:
f3 = f2['External'].file
self.assertEqual(f3.mode, 'r')
finally:
f2.close()
f3.close()
def postprocess_hdf5_history(
ret: List[OptimizerResult],
storage_file: str,
history_options: HistoryOptions,
) -> None:
"""Create single history file pointing to files of multiple starts.
Create links in `storage_file` to the history of each start contained in
`ret`, the results of the optimization.
Parameters
----------
ret:
The result iterable returned by the optimization.
storage_file:
The filename of the hdf5 file in which the histories
are to be gathered.
history_options:
History options used in the optimization.
"""
# create hdf5 file that gathers the others within history group
with h5py.File(storage_file, mode='w') as f:
# create file and group
get_or_create_group(f, "history")
# append links to each single result file
for result in ret:
id = result['id']
f[f'history/{id}'] = h5py.ExternalLink(
result['history'].file, f'history/{id}'
)
# reset storage file (undo preprocessing changes)
history_options.storage_file = storage_file
