def get_hdf5_with_external_recursive_links(): ID = "external_recursive_links" if ID in _file_cache: return _file_cache[ID][0].name tmp1 = tempfile.NamedTemporaryFile(prefix=ID + "_", suffix=".h5", delete=True) tmp1.file.close() h5_1 = h5py.File(tmp1.name, "w") tmp2 = tempfile.NamedTemporaryFile(prefix=ID + "_", suffix=".h5", delete=True) tmp2.file.close() h5_2 = h5py.File(tmp2.name, "w") g = h5_1.create_group("group") g.create_dataset("dataset", data=numpy.int64(10)) h5_1["soft_link_to_group"] = h5py.SoftLink("/group") h5_1["external_link_to_link"] = h5py.ExternalLink(tmp2.name, "/soft_link_to_group") h5_1["external_link_to_recursive_link"] = h5py.ExternalLink(tmp2.name, "/external_link_to_recursive_link") h5_1.close() g = h5_2.create_group("group") g.create_dataset("dataset", data=numpy.int64(10)) h5_2["soft_link_to_group"] = h5py.SoftLink("/group") h5_2["external_link_to_link"] = h5py.ExternalLink(tmp1.name, "/soft_link_to_group") h5_2["external_link_to_recursive_link"] = h5py.ExternalLink(tmp1.name, "/external_link_to_recursive_link") h5_2.close() _file_cache[ID] = (tmp1, tmp2) return tmp1.name
def example_h5_filename(tmpdir, scope='module'): ext_filename = (tmpdir / "example-data-external.h5") filename = (tmpdir / "example-data.h5") with h5py.File(ext_filename, 'w') as f: d = f.create_dataset("external_ds", data=np.array([1, 2, 3, 4], dtype='d')) g = f.create_group("external_group") with h5py.File(filename, 'w') as f: g = f.create_group("group") g.attrs['unit'] = "Flerbians" g.attrs['count'] = 123 g2 = g.create_group("subgroup") g3 = g2.create_group("subsubgroup") d1 = g.create_dataset("scalar", data=1.23) d1.attrs['cats'] = np.array(["Kali", "Bustopher Jones"], dtype=h5py.special_dtype(vlen=str)) d2 = g.create_dataset("vector", data=np.array([1, 2, 3], dtype='i')) f['link'] = d2 f['softlink'] = h5py.SoftLink('/group/scalar') f['subsubgroup_hardlink'] = g3 f['extlink'] = h5py.ExternalLink(ext_filename, "external_ds") f['extgroup'] = h5py.ExternalLink(ext_filename, "external_group") yield filename
def get_hdf5_with_all_links(): ID = "alllinks" if ID in _file_cache: return _file_cache[ID].name tmp = tempfile.NamedTemporaryFile(prefix=ID + "_", suffix=".h5", delete=True) tmp.file.close() h5 = h5py.File(tmp.name, "w") g = h5.create_group("group") g.create_dataset("dataset", data=numpy.int64(10)) h5.create_dataset("dataset", data=numpy.int64(10)) h5["hard_link_to_group"] = h5["/group"] h5["hard_link_to_dataset"] = h5["/dataset"] h5["soft_link_to_group"] = h5py.SoftLink("/group") h5["soft_link_to_dataset"] = h5py.SoftLink("/dataset") h5["soft_link_to_nothing"] = h5py.SoftLink("/foo/bar/2000") alltypes_filename = get_hdf5_with_all_types() h5["external_link_to_group"] = h5py.ExternalLink(alltypes_filename, "/arrays") h5["external_link_to_dataset"] = h5py.ExternalLink(alltypes_filename, "/arrays/cube") h5["external_link_to_nothing"] = h5py.ExternalLink(alltypes_filename, "/foo/bar/2000") h5["external_link_to_missing_file"] = h5py.ExternalLink("missing_file.h5", "/") h5.close() _file_cache[ID] = tmp return tmp.name
def create_master(params): # Get path to create the master file master_file = os.path.join(params["OutputDir"], params["MasterFileName"]) master_file = ".".join([master_file, "hdf5"]) # Get the list of interpolated files interp_prefix = os.path.join(params["OutputDir"], params["InterpFileName"]) interp_files = sorted(glob(interp_prefix + "*")) # Open master file with h5py.File(master_file, mode="w") as mf: group = mf.create_group("dm_maps") group.attrs["desc"] = "This is the master file for the interpolated redshfits" # Link each interpolated file to master file for i, fn in enumerate(interp_files): with h5py.File(fn, mode="r") as file_i: try: redshift = file_i[params["Header"]].attrs["Redshift"] except: redshift = file_i[params["Header"]].attrs["redshift"] # Create sub_group group_name = f"slice_{i:03d}" sub_group = group.create_group(group_name) # Save the Header and DM dataset to the subgroup sub_group[params["Header"]] = h5py.ExternalLink(fn, params["Header"]) sub_group[params["Dataset"]] = h5py.ExternalLink(fn, params["Dataset"])
def write_to_file(f, data): datasets_group = f.create_group('datasets_group') datasets_group.attrs['string_attr'] = 'my string attribute' datasets_group.attrs['int_attr'] = 123 datasets_group.attrs['float_attr'] = 123.456 float_group = datasets_group.create_group('float') float_group.create_dataset('float32', data=data, dtype='f4') float_group.create_dataset('float64', data=data, dtype='f8', fillvalue=6) int_group = datasets_group.create_group('int') int_group.create_dataset('int8', data=data, dtype='i1') int_group.create_dataset('int16', data=data, dtype='i2') int_group.create_dataset('int32', data=data, dtype='i4') links_group = f.create_group('links_group') links_group['hard_link_to_int8'] = int_group['int8'] links_group['soft_link_to_int8'] = h5py.SoftLink('/datasets_group/int/int8') links_group['broken_soft_link'] = h5py.SoftLink('/datasets_group/int/missing_dataset') links_group['soft_link_to_group'] = h5py.SoftLink('/datasets_group/int') # Define the external link path relative to this file, to ease testing links_group['external_link'] = h5py.ExternalLink('test_file_ext.hdf5', '/external_dataset') links_group['external_link_to_missing_file'] = h5py.ExternalLink('missing_file.hdf5', '/external_dataset') multiDimensionDatasets = f.create_group('nD_Datasets'); multiDimensionDatasets.create_dataset('3D_float32', data=data_3d, dtype='f4') multiDimensionDatasets.create_dataset('3D_int32', data=data_3d, dtype='i4') f.flush() f.close()
def link_or_copy(group, name, link, copy, absolute_paths=False): ''' Link or copy a dataset or group Parameters ---------- group : h5py.Group The group to create the link, dataset, or group in name : str The name of the link, dataset, or group in the new file link : h5py.ExternalLink A link to the group or dataset to include copy : bool Whether to copy or link to the dataset absolute_paths : bool If copy=False, then if absolute_paths is True, absolute filenames are used in the link, otherwise the path relative to the file is used. ''' if copy: f = h5py.File(link.filename, 'r') f.copy(link.path, group, name=name) f.close() else: if absolute_paths: group[name] = h5py.ExternalLink(os.path.abspath(link.filename), link.path) else: group[name] = h5py.ExternalLink(os.path.relpath(link.filename, os.path.dirname(group.file.filename)), link.path) try: group[name] except KeyError: # indicates linking failed (h5py < 2.1.0) logger.warn("Linking failed, copying instead (indicates an outdated version of h5py)") del group[name] f = h5py.File(link.filename, 'r') f.copy(link.path, group, name=name) f.close()
def create_links(h5): print("- Creating links...") main_group = h5.create_group("links") dataset = main_group.create_dataset("dataset", data=numpy.int64(10)) group = main_group.create_group("group") group["something_inside"] = numpy.int64(20) main_group["hard_link_to_group"] = main_group["group"] main_group["hard_link_to_dataset"] = main_group["dataset"] main_group.create_group("hard_recursive_link") main_group.create_group("hard_recursive_link2") main_group["hard_recursive_link/link"] = main_group["hard_recursive_link2"] main_group["hard_recursive_link2/link"] = main_group["hard_recursive_link"] main_group["soft_link_to_group"] = h5py.SoftLink(group.name) main_group["soft_link_to_dataset"] = h5py.SoftLink(dataset.name) main_group["soft_link_to_nothing"] = h5py.SoftLink("/foo/bar/2000") main_group["soft_link_to_group_link"] = h5py.SoftLink(main_group.name + "/soft_link_to_group") main_group["soft_link_to_dataset_link"] = h5py.SoftLink(main_group.name + "/soft_link_to_dataset") main_group["soft_link_to_itself"] = h5py.SoftLink(main_group.name + "/soft_link_to_itself") # External links to self file main_group["external_link_to_group"] = h5py.ExternalLink(h5.file.filename, group.name) main_group["external_link_to_dataset"] = h5py.ExternalLink(h5.file.filename, dataset.name) main_group["external_link_to_nothing"] = h5py.ExternalLink(h5.file.filename, "/foo/bar/2000") main_group["external_link_to_missing_file"] = h5py.ExternalLink(h5.file.filename + "_unknown", "/") main_group["external_link_to_group_link"] = h5py.ExternalLink(h5.file.filename, main_group.name + "/soft_link_to_group") main_group["external_link_to_dataset_link"] = h5py.ExternalLink(h5.file.filename, main_group.name + "/soft_link_to_dataset") main_group["external_link_to_itself"] = h5py.ExternalLink(h5.file.filename, main_group.name + "/external_link_to_itself") main_group["external_link_to_recursive_link2"] = h5py.ExternalLink(h5.file.filename, main_group.name + "/external_link_to_recursive_link3") main_group["external_link_to_recursive_link3"] = h5py.ExternalLink(h5.file.filename, main_group.name + "/external_link_to_recursive_link2") main_group["external_link_to_soft_recursive"] = h5py.ExternalLink(h5.file.filename, main_group.name + "/soft_link_to_itself")
def prepare_h5source(ds_dir, volume_name, label_file=("groundtruth_seg.h5", "main"), raw_file=("im_uint8.h5", "main"), add_dummy_mask=True): h5_filepath = ".{}.h5".format(volume_name) if add_dummy_mask: with h5py.File(os.path.join(ds_dir, volume_name, label_file[0]), "r") as f_labels: mask_shape = f_labels[label_file[1]].shape with h5py.File(h5_filepath, "w") as h5: h5["volumes/raw"] = h5py.ExternalLink( os.path.join(ds_dir, volume_name, raw_file[0]), raw_file[1]) h5["volumes/labels/neuron_ids"] = h5py.ExternalLink( os.path.join(ds_dir, volume_name, label_file[0]), label_file[1]) datasets = { VolumeTypes.RAW: 'volumes/raw', VolumeTypes.GT_LABELS: 'volumes/labels/neuron_ids' } if add_dummy_mask: h5.create_dataset( name="volumes/labels/mask", dtype="uint8", shape=mask_shape, fillvalue=1, ) datasets[VolumeTypes.GT_MASK] = 'volumes/labels/mask' h5source = gunpowder.Hdf5Source( h5_filepath, datasets=datasets, #resolutions=(8, 8, 8), ) return h5source
def createResource(cls, directory): filename = os.path.join(directory, "base.h5") externalFilename = os.path.join(directory, "base__external.h5") externalh5 = h5py.File(externalFilename, mode="w") externalh5["target/dataset"] = 50 externalh5["target/link"] = h5py.SoftLink("/target/dataset") externalh5.close() h5 = h5py.File(filename, mode="w") h5["group/dataset"] = 50 h5["link/soft_link"] = h5py.SoftLink("/group/dataset") h5["link/soft_link_to_group"] = h5py.SoftLink("/group") h5["link/soft_link_to_link"] = h5py.SoftLink("/link/soft_link") h5["link/soft_link_to_file"] = h5py.SoftLink("/") h5["link/external_link"] = h5py.ExternalLink(externalFilename, "/target/dataset") h5["link/external_link_to_link"] = h5py.ExternalLink( externalFilename, "/target/link") h5["broken_link/external_broken_file"] = h5py.ExternalLink( externalFilename + "_not_exists", "/target/link") h5["broken_link/external_broken_link"] = h5py.ExternalLink( externalFilename, "/target/not_exists") h5["broken_link/soft_broken_link"] = h5py.SoftLink("/group/not_exists") h5["broken_link/soft_link_to_broken_link"] = h5py.SoftLink( "/group/not_exists") h5.close() return filename
def link_hdf5_files(post_hdf5_link_list): logger = logging.getLogger() PLATE_PREFIX = '/sample/0/plate/' WELL_PREFIX = PLATE_PREFIX + '%s/experiment/' POSITION_PREFIX = WELL_PREFIX + '%s/position/' def get_plate_and_postion(hf_file): plate = hf_file[PLATE_PREFIX].keys()[0] well = hf_file[WELL_PREFIX % plate].keys()[0] position = hf_file[POSITION_PREFIX % (plate, well)].keys()[0] return plate, well, position all_pos_hdf5_filename = join(split(post_hdf5_link_list[0])[0], '_all_positions.ch5') if exists(all_pos_hdf5_filename): f = h5py.File(all_pos_hdf5_filename, 'a') ### This is dangerous, several processes open the file for writing... logger.info(("_all_positons.hdf file found, " "trying to reuse it by overwrite old external links...")) if 'definition' in f: del f['definition'] f['definition'] = h5py.ExternalLink(post_hdf5_link_list[0], '/definition') for fname in post_hdf5_link_list: fh = h5py.File(fname, 'r') fplate, fwell, fpos = get_plate_and_postion(fh) fh.close() msg = "Linking into _all_positons.hdf:" + \ ((POSITION_PREFIX + '%s') % (fplate, fwell, fpos)) logger.info(msg) if (POSITION_PREFIX + '%s') % (fplate, fwell, fpos) in f: del f[(POSITION_PREFIX + '%s') % (fplate, fwell, fpos)] f[(POSITION_PREFIX + '%s') % (fplate, fwell, fpos)] = \ h5py.ExternalLink(fname, (POSITION_PREFIX + '%s') % (fplate, fwell, fpos)) f.close() else: f = h5py.File(all_pos_hdf5_filename, 'w') logger.info("_all_positons.hdf file created...") f['definition'] = h5py.ExternalLink(post_hdf5_link_list[0],'/definition') for fname in post_hdf5_link_list: fh = h5py.File(fname, 'r') fplate, fwell, fpos = get_plate_and_postion(fh) fh.close() msg = "Linking into _all_positons.hdf:" + \ ((POSITION_PREFIX + '%s') % (fplate, fwell, fpos)) logger.info(msg) f[(POSITION_PREFIX + '%s') % (fplate, fwell, fpos)] = \ h5py.ExternalLink(fname, (POSITION_PREFIX + '%s') % (fplate, fwell, fpos)) f.close()
def saveH5(self): """ """ """ Private method to save the object to a file. Creates links to h5 files that all contain only one pattern. :param output_path: The file where to save the object's data. :type output_path: string, default b """ # Path where individual h5 files are located. path_to_files = self.output_path # Setup new file. with h5py.File(self.output_path + ".h5", "w") as h5_outfile: # Files to read from. individual_files = [ os.path.join(path_to_files, f) for f in os.listdir(path_to_files) ] individual_files.sort() # Keep track of global parameters being linked. global_parameters = False # Loop over all individual files and link in the top level groups. for ind_file in individual_files: # Open file. with h5py.File(ind_file, 'r') as h5_infile: # Links must be relative. relative_link_target = os.path.relpath( path=ind_file, start=os.path.dirname(os.path.dirname(ind_file))) # Link global parameters. if not global_parameters: global_parameters = True h5_outfile["params"] = h5py.ExternalLink( relative_link_target, "params") h5_outfile["info"] = h5py.ExternalLink( relative_link_target, "info") h5_outfile["misc"] = h5py.ExternalLink( relative_link_target, "misc") h5_outfile["version"] = h5py.ExternalLink( relative_link_target, "version") for key in h5_infile['data']: # Link in the data. ds_path = "data/%s" % (key) h5_outfile[ds_path] = h5py.ExternalLink( relative_link_target, ds_path) # Reset output path. self.output_path = self.output_path + ".h5"
def main(input_filenames, output_filename): with h5py.File(output_filename) as output_file: for i, input_filename in enumerate(input_filenames): with h5py.File(input_filename, "r") as input_file: dark_field = input_file["postprocessing/dark_field"] output_file["dark_field_{:02}".format(i)] = h5py.ExternalLink( input_filename, "postprocessing/dark_field") ratio = input_file["postprocessing/ratio"] output_file["ratio_{:02}".format(i)] = h5py.ExternalLink( input_filename, "postprocessing/ratio")
def createResource(cls, directory): filename = os.path.join(directory, "base.h5") extH5FileName = os.path.join(directory, "base__external.h5") extDatFileName = os.path.join(directory, "base__external.dat") externalh5 = h5py.File(extH5FileName, mode="w") externalh5["target/dataset"] = 50 externalh5["target/link"] = h5py.SoftLink("/target/dataset") externalh5["/ext/vds0"] = [0, 1] externalh5["/ext/vds1"] = [2, 3] externalh5.close() numpy.array([0, 1, 10, 10, 2, 3]).tofile(extDatFileName) h5 = h5py.File(filename, mode="w") h5["group/dataset"] = 50 h5["link/soft_link"] = h5py.SoftLink("/group/dataset") h5["link/soft_link_to_group"] = h5py.SoftLink("/group") h5["link/soft_link_to_link"] = h5py.SoftLink("/link/soft_link") h5["link/soft_link_to_file"] = h5py.SoftLink("/") h5["group/soft_link_relative"] = h5py.SoftLink("dataset") h5["link/external_link"] = h5py.ExternalLink(extH5FileName, "/target/dataset") h5["link/external_link_to_link"] = h5py.ExternalLink( extH5FileName, "/target/link") h5["broken_link/external_broken_file"] = h5py.ExternalLink( extH5FileName + "_not_exists", "/target/link") h5["broken_link/external_broken_link"] = h5py.ExternalLink( extH5FileName, "/target/not_exists") h5["broken_link/soft_broken_link"] = h5py.SoftLink("/group/not_exists") h5["broken_link/soft_link_to_broken_link"] = h5py.SoftLink( "/group/not_exists") layout = h5py.VirtualLayout((2, 2), dtype=int) layout[0] = h5py.VirtualSource("base__external.h5", name="/ext/vds0", shape=(2, ), dtype=int) layout[1] = h5py.VirtualSource("base__external.h5", name="/ext/vds1", shape=(2, ), dtype=int) h5.create_group("/ext") h5["/ext"].create_virtual_dataset("virtual", layout) external = [("base__external.dat", 0, 2 * 8), ("base__external.dat", 4 * 8, 2 * 8)] h5["/ext"].create_dataset("raw", shape=(2, 2), dtype=int, external=external) h5.close() return filename
def test_external_links(self): # create a file for use a link target if config.get("use_h5py"): # for some reason this test is failing in Travis return linked_filename = self.getFileName("linked_file") abs_filepath = os.path.abspath(linked_filename) if config.get("use_h5py"): rel_filepath = os.path.relpath(linked_filename) else: rel_filepath = "linked_file.h5" f = h5py.File(linked_filename, 'w') is_hsds = False if isinstance(f.id.id, str) and f.id.id.startswith("g-"): is_hsds = True g1 = f.create_group("g1") dset = g1.create_dataset('ds', (5,7), dtype='f4') dset_id = dset.id.id f.close() filename = self.getFileName("external_links") print("filename:", filename) f = h5py.File(filename, 'w') f["missing_link"] = h5py.ExternalLink(abs_filepath, "somepath") f["abspath_link"] = h5py.ExternalLink(abs_filepath, "/g1/ds") f["relpath_link"] = h5py.ExternalLink(rel_filepath, "/g1/ds") try: linked_obj = f["missing_link"] self.assertTrue(False) except KeyError: pass # expected try: linked_obj = f["abspath_link"] self.assertTrue(linked_obj.name, "/g1/ds") self.assertEqual(linked_obj.shape, (5, 7)) # The following no longer works for h5py 2.8 # self.assertEqual(linked_obj.id.id, dset_id) except KeyError: if config.get("use_h5py") or is_hsds: # absolute paths aren't working yet for h5serv self.assertTrue(False) linked_obj = f["relpath_link"] self.assertTrue(linked_obj.name, "/g1/ds") self.assertEqual(linked_obj.shape, (5, 7)) if not config.get("use_h5py"): self.assertEqual(linked_obj.id.id, dset_id) f.close()
def add_link_to_coord(self, coord, subfile): path = coord.get_path() #self.require_group(path) try: self[path] = h5py.ExternalLink(subfile, path) except RuntimeError as e: print("Link to path '%s' already exists" % path) #self.require_group(CH5Const.DEFINITION) try: self[CH5Const.DEFINITION] = h5py.ExternalLink( subfile, CH5Const.DEFINITION) except RuntimeError as e: # link to definition is already there print("Link to path '%s' already exists" % path)
def link(target, parent, name): """ create link :param target: file name :type target: :obj:`str` :param parent: parent object :type parent: :class:`FTObject` :param name: link name :type name: :obj:`str` :returns: link object :rtype: :class:`H5PYLink` """ localfname = H5PYLink.getfilename(parent) if ":/" in target: filename, path = target.split(":/") if os.path.abspath(filename) != os.path.abspath(localfname): parent.h5object[name] = h5py.ExternalLink(filename, path) else: parent.h5object[name] = h5py.SoftLink(path) else: parent.h5object[name] = h5py.SoftLink(target) return H5PYLink(parent.h5object.get(name, getlink=True), parent).setname(name)
def write_master_file( fname, data_files, pixel_mask=None, raw_master_file=None, i0=None, compression=hdf5plugin.Bitshuffle(nelems=0, lz4=True), ): print(color.info(f"Writing: {fname}")) f = h5py.File(fname, "w") nxentry, nxdata = create_entry_and_data(f) if raw_master_file is not None: #add attrs nxdata.attrs["exptime"] = raw_master_file['Exptime'].item() nxdata.attrs["period"] = raw_master_file['Period'].item() # Link written data sets: for i, fname in enumerate(data_files, start=1): f[f"entry/data/data_{i:06d}"] = h5py.ExternalLink( fname, "entry/data/data") #Group to hold instrument specific data grp = nxentry.create_group("instrument/data") # Pixel mask recognized by Albula if pixel_mask is not None: inst = nxentry.create_group("instrument/detector/detectorSpecific") inst.create_dataset("pixel_mask", data=pixel_mask.astype(np.uint8), **compression) if i0 is not None: grp.create_dataset("i0", data=i0, **compression) f.close()
def create_external_link(fname, dataset_path, out_fname, new_dataset_path): """ Creates an external link of `fname:dataset_path` into `out_fname:new_dataset_path`. :param fname: The filename of the file containing the dataset that is to be linked to. `type:str` :param dataset_path: A `str` containg the path to the dataset contained within `fname`. :param out_fname: A `str` for the output filename that will contain the link to `fname:dataset_path`. :param new_dataset_path: A `str` containing the dataset path within `out_fname` that will link to `fname:dataset_path`. """ with h5py.File(out_fname) as fid: fid[new_dataset_path] = h5py.ExternalLink(fname, dataset_path) return
def write(self, data): """ Write out the averaged TOD to a Level2 continuum file with an external link to the original level 1 data """ if not os.path.exists(self.output_dir): os.makedirs(self.output_dir) # We will store these in a separate file and link them to the level2s fname = data.filename.split('/')[-1] if os.path.exists(self.outfile): os.remove(self.outfile) output = h5py.File(self.outfile, 'a') # Set permissions and group os.chmod(self.outfile, 0o664) shutil.chown(self.outfile, group='comap') # Store datasets in root dnames = ['tod', 'weights'] dsets = [self.all_tod, self.all_weights] for (dname, dset) in zip(dnames, dsets): if dname in output: del output[dname] output.create_dataset(dname, data=dset) output.attrs['version'] = __level3_version__ output.close() if self.level3 in data.keys(): del data[self.level3] data[self.level3] = h5py.ExternalLink(self.outfile, '/')
def _link_datafile_to_nexus_file(self, data): filename = self.exp.meta_data.get('nxs_filename') with h5py.File(filename, 'a') as nxs_file: # entry path in nexus file name = data.get_name() group_name = self.exp.meta_data.get(['group_name', name]) link_type = self.exp.meta_data.get(['link_type', name]) nxs_entry = '/entry/' + link_type if link_type == 'final_result': nxs_entry += '_' + data.get_name() else: nxs_entry += "/" + group_name nxs_entry = nxs_file[nxs_entry] nxs_entry.attrs['signal'] = 'data' data_entry = nxs_entry.name + '/data' # output file path h5file = data.backing_file.filename # entry path in output file path m_data = self.exp.meta_data.get if not (link_type == 'intermediate' and m_data('inter_path') != m_data('out_path')): h5file = h5file.split(m_data('out_folder') + '/')[-1] nxs_file[data_entry] = \ h5py.ExternalLink(h5file, group_name + '/data')
def buildLink(gdst, linkname, gsrc): fn = sanitize_posixpath(basename(gsrc.file.filename)) if gsrc else '' gdst.attrs[linkname + '_PATH'] = gsrc.name if gsrc else '' gdst.attrs[linkname + '_FILE'] = fn if gsrc: gdst[linkname] = h5py.SoftLink(gsrc.name) if ( gsrc.file == gdst.file) else h5py.ExternalLink(fn, gsrc.name)
def test_file(self): """ (Links) File attribute works correctly on external links """ from h5py.h5 import _global_ids as gi from h5py import h5i self.f['ext'] = h5py.ExternalLink(self.ename, '/external') g = self.f['ext'] self.assertNotEqual(g.file, self.f)
def eigerfile(request): N_chans = 4 N_points = 27 N_bin = 4096 N_roi = 32 seq_id = np.random.randint(10) pre_fix = gettempprefix() out_name = f'{pre_fix}_{seq_id}_master.h5' with h5py.File("sample_data_000001.h5", "w") as dfile: data = np.random.rand(N_points, N_chans, N_bin) * 1000 dfile["data_000001"] = data with h5py.File(out_name, "w") as fout: for e in EigerHandler.EIGER_MD_LAYOUT.values(): fout[e] = np.random.randint(1, 10) fout["entry/data/data_000001"] = h5py.ExternalLink( "sample_data_000001.h5", "data_000001") def finalize(): os.remove(out_name) os.remove("sample_data_000001.h5") request.addfinalizer(finalize) kwargs = {'images_per_file': 1, 'frame_per_point': 1} return (out_name, kwargs), (N_points, N_chans, N_bin, N_roi)
def prepare_field_file(self): df = self.get_data_file() if 'field_dtype' in df.keys(): # we don't need to do anything, raw binary files are used return None last_iteration = df['iteration'].value cppf = df['parameters/checkpoints_per_file'].value niter_out = df['parameters/niter_out'].value with h5py.File(os.path.join(self.work_dir, self.simname + '_fields.h5'), 'a') as ff: ff.require_group('vorticity') ff.require_group('vorticity/complex') checkpoint = 0 while True: cpf_name = os.path.join( self.work_dir, self.simname + '_checkpoint_{0}.h5'.format(checkpoint)) if os.path.exists(cpf_name): cpf = h5py.File(cpf_name, 'r') for iter_name in cpf['vorticity/complex'].keys(): if iter_name not in ff['vorticity/complex'].keys(): ff['vorticity/complex/' + iter_name] = h5py.ExternalLink( cpf_name, 'vorticity/complex/' + iter_name) checkpoint += 1 else: break return None
def _get_ext_link_rec(file_path: Path, hdf_path: str, depth_to_go: int, func_to_fulfill: Callable[[Path, str], bool]) -> set: """ recursive function to return external links of hdf files in variable depth :param file_path: path of the hdf file :param hdf_path: hdf_path of the hdf object inside the hdf file :param depth_to_go: goal depth to go :param func_to_fulfill: the function that has to be fulfilled in order to be gathered :return: set of external hdf links """ if depth_to_go == 0: return {h5py.ExternalLink(file_path, hdf_path)} if func_to_fulfill( file_path, hdf_path) else set() if depth_to_go > 0: with h5py.File(file_path, "r") as file: children_set = set() for key in file[hdf_path].keys(): children_set.update( set( _get_ext_link_rec(file_path=file_path, hdf_path=hdf_path_combine( hdf_path, key), depth_to_go=depth_to_go - 1, func_to_fulfill=func_to_fulfill))) return children_set raise ValueError("depth_to_go should be a non negative integer")
def add_history(wf_file_name, history_file_name): """ Add history from pearent file to propagated file :param wf_file_name: output file :param history_file_name: peraent file """ with h5py.File(wf_file_name) as wf_h5: with h5py.File(history_file_name) as history_h5: if "history" in wf_h5: del wf_h5["history"] wf_h5.create_group("/history/parent/") wf_h5.create_group("/history/parent/detail") for k in history_h5: if k == "history": try: history_h5.copy("/history/parent", wf_h5["history"]["parent"]) except KeyError: pass elif not k == "data": history_h5.copy(k, wf_h5["history"]["parent"]["detail"]) else: wf_h5["history"]["parent"]["detail"][ "data"] = h5py.ExternalLink(history_file_name, "/data")
def linkfile(self, data): fname = data.filename.split('/')[-1] lvl2 = data[self.level2] if self.prefix in lvl2: del lvl2[self.prefix] lvl2[self.prefix] = h5py.ExternalLink( '{}/{}_{}'.format(self.output_dir, self.prefix, fname), '/')
def nxDataAddSignals(data, signals, append=True): """ Add signals to NXdata instance :param h5py.Group data: :param list(2-tuple) signals: name(str), value(None,h5py.Dataset,numpy.ndarray,dict), attrs(dict) :param bool append: """ raiseIsNotNxClass(data, u'NXdata') if append: newsignals = nxDataGetSignals(data) else: newsignals = [] for name, value, attrs in signals: if value is None: pass # is or will be created elsewhere elif isinstance(value, h5py.Dataset): if value.file.filename != data.file.filename: data[name] = h5py.ExternalLink(value.file.filename, value.name) elif value.parent != data: data[name] = h5py.SoftLink(value.name) elif isinstance(value, dict): data.create_dataset(name, **value) else: data[name] = value if attrs: data[name].attrs.update(attrs) newsignals.append(name) if newsignals: nxDataSetSignals(data, newsignals)
def test_mode_external(self): """ Mode property works for files opened via external links Issue 190. """ fname1 = self.mktemp() fname2 = self.mktemp() f1 = File(fname1, 'w') f1.close() f2 = File(fname2, 'w') try: f2['External'] = h5py.ExternalLink(fname1, '/') f3 = f2['External'].file self.assertEqual(f3.mode, 'r+') finally: f2.close() f3.close() f2 = File(fname2, 'r') try: f3 = f2['External'].file self.assertEqual(f3.mode, 'r') finally: f2.close() f3.close()
def postprocess_hdf5_history( ret: List[OptimizerResult], storage_file: str, history_options: HistoryOptions, ) -> None: """Create single history file pointing to files of multiple starts. Create links in `storage_file` to the history of each start contained in `ret`, the results of the optimization. Parameters ---------- ret: The result iterable returned by the optimization. storage_file: The filename of the hdf5 file in which the histories are to be gathered. history_options: History options used in the optimization. """ # create hdf5 file that gathers the others within history group with h5py.File(storage_file, mode='w') as f: # create file and group get_or_create_group(f, "history") # append links to each single result file for result in ret: id = result['id'] f[f'history/{id}'] = h5py.ExternalLink( result['history'].file, f'history/{id}' ) # reset storage file (undo preprocessing changes) history_options.storage_file = storage_file