def default_lapl(): """ Default link access property list """ lapl = h5p.create(h5p.LINK_ACCESS) fapl = h5p.create(h5p.FILE_ACCESS) fapl.set_fclose_degree(h5f.CLOSE_STRONG) lapl.set_elink_fapl(fapl) return lapl
def test_obj_track_times(self): """ tests if the object track times set/get """ # test for groups gcid = h5p.create(h5p.GROUP_CREATE) gcid.set_obj_track_times(False) self.assertEqual(False, gcid.get_obj_track_times()) gcid.set_obj_track_times(True) self.assertEqual(True, gcid.get_obj_track_times()) # test for datasets dcid = h5p.create(h5p.DATASET_CREATE) dcid.set_obj_track_times(False) self.assertEqual(False, dcid.get_obj_track_times()) dcid.set_obj_track_times(True) self.assertEqual(True, dcid.get_obj_track_times()) # test for generic objects ocid = h5p.create(h5p.OBJECT_CREATE) ocid.set_obj_track_times(False) self.assertEqual(False, ocid.get_obj_track_times()) ocid.set_obj_track_times(True) self.assertEqual(True, ocid.get_obj_track_times())
def test_obj_track_times(self): """ tests if the object track times set/get """ # test for groups gcid = h5p.create(h5p.GROUP_CREATE) gcid.set_obj_track_times(False) self.assertEqual(False,gcid.get_obj_track_times()) gcid.set_obj_track_times(True) self.assertEqual(True,gcid.get_obj_track_times()) # test for datasets dcid = h5p.create(h5p.DATASET_CREATE) dcid.set_obj_track_times(False) self.assertEqual(False,dcid.get_obj_track_times()) dcid.set_obj_track_times(True) self.assertEqual(True,dcid.get_obj_track_times()) # test for generic objects ocid = h5p.create(h5p.OBJECT_CREATE) ocid.set_obj_track_times(False) self.assertEqual(False,ocid.get_obj_track_times()) ocid.set_obj_track_times(True) self.assertEqual(True,ocid.get_obj_track_times())
def make_fapl(driver,libver,**kwds): """ Set up a file access property list """ plist = h5p.create(h5p.FILE_ACCESS) plist.set_fclose_degree(h5f.CLOSE_STRONG) if libver is not None: if libver in libver_dict: low = libver_dict[libver] high = h5f.LIBVER_LATEST else: low, high = (libver_dict[x] for x in libver) plist.set_libver_bounds(low, high) if driver is None or (driver=='windows' and sys.platform=='win32'): return plist if(driver=='sec2'): plist.set_fapl_sec2(**kwds) elif(driver=='stdio'): plist.set_fapl_stdio(**kwds) elif(driver=='core'): plist.set_fapl_core(**kwds) elif(driver=='family'): plist.set_fapl_family(memb_fapl=plist.copy(), **kwds) else: raise ValueError('Unknown driver type "%s"' % driver) return plist
def create_entry(obj, name, timestamp, **attributes): """Create a new ARF entry under obj, setting required attributes. An entry is an abstract collection of data which all refer to the same time frame. Data can include physiological recordings, sound recordings, and derived data such as spike times and labels. See add_data() for information on how data are stored. name -- the name of the new entry. any valid python string. timestamp -- timestamp of entry (datetime object, or seconds since January 1, 1970). Can be an integer, a float, or a tuple of integers (seconds, microsceconds) Additional keyword arguments are set as attributes on created entry. Returns: newly created entry object """ # create group using low-level interface to store creation order from h5py import h5p, h5g from h5py._hl import group try: gcpl = h5p.create(h5p.GROUP_CREATE) gcpl.set_link_creation_order(h5p.CRT_ORDER_TRACKED | h5p.CRT_ORDER_INDEXED) except AttributeError: grp = obj.create_group(name) else: grp = group.Group(h5g.create(obj.id, name, lcpl=None, gcpl=gcpl)) set_uuid(grp, attributes.pop("uuid", None)) set_attributes(grp, timestamp=convert_timestamp(timestamp), **attributes) return grp
def make_fapl(driver, libver, **kwds): """ Set up a file access property list """ plist = h5p.create(h5p.FILE_ACCESS) plist.set_fclose_degree(h5f.CLOSE_STRONG) if libver is not None: if libver in libver_dict: low = libver_dict[libver] high = h5f.LIBVER_LATEST else: low, high = (libver_dict[x] for x in libver) plist.set_libver_bounds(low, high) if driver is None or (driver == "windows" and sys.platform == "win32"): return plist if driver == "sec2": plist.set_fapl_sec2(**kwds) elif driver == "stdio": plist.set_fapl_stdio(**kwds) elif driver == "core": plist.set_fapl_core(**kwds) elif driver == "family": plist.set_fapl_family(memb_fapl=plist.copy(), **kwds) elif driver == "mpio": kwds.setdefault("info", mpi4py.MPI.Info()) plist.set_fapl_mpio(**kwds) else: raise ValueError('Unknown driver type "%s"' % driver) return plist
def ensureHdf(file, mode): if isinstance(file, str) or file is None: driver, driverKwds = None, {} if file is None: file = tempfile.mktemp(suffix=".hdf") driver, driverKwds = "core", {"backing_store": False} dirname = os.path.dirname(os.path.abspath(file)) if not os.path.exists(dirname): os.makedirs(dirname) return h5py.File(file, mode, libver="earliest", driver=driver, **driverKwds) elif isinstance(file, bytes): fapl = h5p.create(h5p.FILE_ACCESS) fapl.set_fapl_core() fapl.set_file_image(file) fid = h5f.open(tempfile.mktemp(suffix=".hdf").encode(), h5f.ACC_RDONLY, fapl=fapl) return h5py.File(fid) else: return file
def open_file(name, mode=None, driver=None, libver=None, userblock_size=None, **kwargs): """Open an ARF file, creating as necessary. Use this instead of h5py.File to ensure that root-level attributes and group creation property lists are set correctly. """ import os from h5py import h5p from h5py._hl import files exists = os.path.exists(name) try: fcpl = h5p.create(h5p.FILE_CREATE) fcpl.set_link_creation_order(h5p.CRT_ORDER_TRACKED | h5p.CRT_ORDER_INDEXED) except AttributeError: # older version of h5py fp = files.File(name, mode=mode, driver=driver, libver=libver, **kwargs) else: fapl = files.make_fapl(driver, libver, **kwargs) fp = files.File(files.make_fid(name, mode, userblock_size, fapl, fcpl)) if not exists and fp.mode == 'r+': set_attributes(fp, arf_library='python', arf_library_version=__version__, arf_version=spec_version) return fp
def create_entry(group, name, timestamp, **attributes): """Create a new ARF entry under group, setting required attributes. An entry is an abstract collection of data which all refer to the same time frame. Data can include physiological recordings, sound recordings, and derived data such as spike times and labels. See add_data() for information on how data are stored. name -- the name of the new entry. any valid python string. timestamp -- timestamp of entry (datetime object, or seconds since January 1, 1970). Can be an integer, a float, or a tuple of integers (seconds, microsceconds) Additional keyword arguments are set as attributes on created entry. Returns: newly created entry object """ # create group using low-level interface to store creation order from h5py import h5p, h5g, _hl try: gcpl = h5p.create(h5p.GROUP_CREATE) gcpl.set_link_creation_order( h5p.CRT_ORDER_TRACKED | h5p.CRT_ORDER_INDEXED) except AttributeError: grp = group.create_group(name) else: name, lcpl = group._e(name, lcpl=True) grp = _hl.group.Group(h5g.create(group.id, name, lcpl=lcpl, gcpl=gcpl)) set_uuid(grp, attributes.pop("uuid", None)) set_attributes(grp, timestamp=convert_timestamp(timestamp), **attributes) return grp
def make_fapl(driver, libver, **kwds): """ Set up a file access property list """ plist = h5p.create(h5p.FILE_ACCESS) if libver is not None: if libver in libver_dict: low = libver_dict[libver] high = h5f.LIBVER_LATEST else: low, high = (libver_dict[x] for x in libver) plist.set_libver_bounds(low, high) if driver is None or (driver == 'windows' and sys.platform == 'win32'): return plist if (driver == 'sec2'): plist.set_fapl_sec2(**kwds) elif (driver == 'stdio'): plist.set_fapl_stdio(**kwds) elif (driver == 'core'): plist.set_fapl_core(**kwds) elif (driver == 'family'): plist.set_fapl_family(memb_fapl=plist.copy(), **kwds) elif (driver == 'mpio'): kwds.setdefault('info', mpi4py.MPI.Info()) plist.set_fapl_mpio(**kwds) else: raise ValueError('Unknown driver type "%s"' % driver) return plist
def test_set_alignment(self): '''test get/set chunk cache ''' falist = h5p.create(h5p.FILE_ACCESS) threshold = 10 * 1024 # threshold of 10kiB alignment = 1024 * 1024 # threshold of 1kiB falist.set_alignment(threshold, alignment) self.assertEqual((threshold, alignment), falist.get_alignment())
def test_set_alignment(self): """test get/set chunk cache """ falist = h5p.create(h5p.FILE_ACCESS) threshold = 10 * 1024 # threshold of 10kiB alignment = 1024 * 1024 # threshold of 1kiB falist.set_alignment(threshold, alignment) self.assertEqual((threshold, alignment), falist.get_alignment())
def test_virtual_prefix(self): '''test get/set virtual prefix ''' dalist = h5p.create(h5p.DATASET_ACCESS) self.assertEqual(dalist.get_virtual_prefix().decode(), '') virtual_prefix = "path/to/virtual/dataset" dalist.set_virtual_prefix(virtual_prefix.encode('utf-8')) self.assertEqual(dalist.get_virtual_prefix().decode(), virtual_prefix)
def test_link_creation_tracking(self): """ tests the link creation order set/get """ gcid = h5p.create(h5p.GROUP_CREATE) gcid.set_link_creation_order(0) self.assertEqual(0, gcid.get_link_creation_order()) flags = h5p.CRT_ORDER_TRACKED|h5p.CRT_ORDER_INDEXED gcid.set_link_creation_order(flags) self.assertEqual(flags, gcid.get_link_creation_order()) # test for file creation fcpl = h5p.create(h5p.FILE_CREATE) fcpl.set_link_creation_order(flags) self.assertEqual(flags, fcpl.get_link_creation_order())
def test_link_creation_tracking(self): """ tests the link creation order set/get """ gcid = h5p.create(h5p.GROUP_CREATE) gcid.set_link_creation_order(0) self.assertEqual(0, gcid.get_link_creation_order()) flags = h5p.CRT_ORDER_TRACKED | h5p.CRT_ORDER_INDEXED gcid.set_link_creation_order(flags) self.assertEqual(flags, gcid.get_link_creation_order()) # test for file creation fcpl = h5p.create(h5p.FILE_CREATE) fcpl.set_link_creation_order(flags) self.assertEqual(flags, fcpl.get_link_creation_order())
def make_fid(name, mode, userblock_size, fapl, fcpl=None): """ Get a new FileID by opening or creating a file. Also validates mode argument.""" if userblock_size is not None: if mode in ("r", "r+"): raise ValueError("User block may only be specified " "when creating a file") try: userblock_size = int(userblock_size) except (TypeError, ValueError): raise ValueError("User block size must be an integer") if fcpl is None: fcpl = h5p.create(h5p.FILE_CREATE) fcpl.set_userblock(userblock_size) if mode == "r": fid = h5f.open(name, h5f.ACC_RDONLY, fapl=fapl) elif mode == "r+": fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl) elif mode == "w-": fid = h5f.create(name, h5f.ACC_EXCL, fapl=fapl, fcpl=fcpl) elif mode == "w": fid = h5f.create(name, h5f.ACC_TRUNC, fapl=fapl, fcpl=fcpl) elif mode == "a": # Open in append mode (read/write). # If that fails, create a new file only if it won't clobber an # existing one (ACC_EXCL) try: fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl) except IOError: fid = h5f.create(name, h5f.ACC_EXCL, fapl=fapl, fcpl=fcpl) elif mode is None: # Try to open in append mode (read/write). # If that fails, try readonly, and finally create a new file only # if it won't clobber an existing file (ACC_EXCL). try: fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl) except IOError: try: fid = h5f.open(name, h5f.ACC_RDONLY, fapl=fapl) except IOError: fid = h5f.create(name, h5f.ACC_EXCL, fapl=fapl, fcpl=fcpl) else: raise ValueError("Invalid mode; must be one of r, r+, w, w-, a") try: if userblock_size is not None: existing_fcpl = fid.get_create_plist() if existing_fcpl.get_userblock() != userblock_size: raise ValueError( "Requested userblock size (%d) does not match that of existing file (%d)" % (userblock_size, existing_fcpl.get_userblock()) ) except: fid.close() raise return fid
def make_fid(name, mode, userblock_size, fapl, fcpl=None): """ Get a new FileID by opening or creating a file. Also validates mode argument.""" if userblock_size is not None: if mode in ('r', 'r+'): raise ValueError("User block may only be specified " "when creating a file") try: userblock_size = int(userblock_size) except (TypeError, ValueError): raise ValueError("User block size must be an integer") if fcpl is None: fcpl = h5p.create(h5p.FILE_CREATE) fcpl.set_userblock(userblock_size) if mode == 'r': fid = h5f.open(name, h5f.ACC_RDONLY, fapl=fapl) elif mode == 'r+': fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl) elif mode == 'w-': fid = h5f.create(name, h5f.ACC_EXCL, fapl=fapl, fcpl=fcpl) elif mode == 'w': fid = h5f.create(name, h5f.ACC_TRUNC, fapl=fapl, fcpl=fcpl) elif mode == 'a': # Open in append mode (read/write). # If that fails, create a new file only if it won't clobber an # existing one (ACC_EXCL) try: fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl) except IOError: fid = h5f.create(name, h5f.ACC_EXCL, fapl=fapl, fcpl=fcpl) elif mode is None: # Try to open in append mode (read/write). # If that fails, try readonly, and finally create a new file only # if it won't clobber an existing file (ACC_EXCL). try: fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl) except IOError: try: fid = h5f.open(name, h5f.ACC_RDONLY, fapl=fapl) except IOError: fid = h5f.create(name, h5f.ACC_EXCL, fapl=fapl, fcpl=fcpl) else: raise ValueError("Invalid mode; must be one of r, r+, w, w-, a") try: if userblock_size is not None: existing_fcpl = fid.get_create_plist() if existing_fcpl.get_userblock() != userblock_size: raise ValueError( "Requested userblock size (%d) does not match that of existing file (%d)" % (userblock_size, existing_fcpl.get_userblock())) except: fid.close() raise return fid
def test_chunk_cache(self): '''test get/set chunk cache ''' dalist = h5p.create(h5p.DATASET_ACCESS) nslots = 10000 # 40kb hash table nbytes = 1000000 # 1MB cache size w0 = .5 # even blend of eviction strategy dalist.set_chunk_cache(nslots, nbytes, w0) self.assertEqual((nslots, nbytes, w0), dalist.get_chunk_cache())
def test_set_file_locking(self): '''test get/set file locking''' falist = h5p.create(h5p.FILE_ACCESS) use_file_locking = False ignore_when_disabled = False falist.set_file_locking(use_file_locking, ignore_when_disabled) self.assertEqual((use_file_locking, ignore_when_disabled), falist.get_file_locking())
def test_chunk_cache(self): """test get/set chunk cache """ dalist = h5p.create(h5p.DATASET_ACCESS) nslots = 10000 # 40kb hash table nbytes = 1000000 # 1MB cache size w0 = 0.5 # even blend of eviction strategy dalist.set_chunk_cache(nslots, nbytes, w0) self.assertEqual((nslots, nbytes, w0), dalist.get_chunk_cache())
def test_efile_prefix(self): '''test get/set efile prefix ''' dalist = h5p.create(h5p.DATASET_ACCESS) self.assertEqual(dalist.get_efile_prefix().decode(), '') efile_prefix = "path/to/external/dataset" dalist.set_efile_prefix(efile_prefix.encode('utf-8')) self.assertEqual(dalist.get_efile_prefix().decode(), efile_prefix) efile_prefix = "${ORIGIN}" dalist.set_efile_prefix(efile_prefix.encode('utf-8')) self.assertEqual(dalist.get_efile_prefix().decode(), efile_prefix)
def test_file(tmp_path): image = decompress(a2b_base64(compressed_image)) # FAPL: File access property list fapl = h5p.create(h5py.h5p.FILE_ACCESS) fapl.set_fapl_core() fapl.set_file_image(image) file = tmp_path / "disk.h5" fid = h5f.open(bytes(file), h5py.h5f.ACC_RDONLY, fapl=fapl) with h5py.File(fid) as f: assert (f["test"][:] == [1, 2, 3]).all() assert f["test"].dtype == "int64"
def test_load_from_image(self): from binascii import a2b_base64 from zlib import decompress compressed_image = 'eJzr9HBx4+WS4mIAAQ4OBhYGAQZk8B8KKjhQ+TD5BCjNCKU7oPQKJpg4I1hOAiouCDUfXV1IkKsrSPV/NACzx4AFQnMwjIKRCDxcHQNAdASUD0ulJ5hQ1ZWkFpeAaFh69KDQXkYGNohZjDA+JCUzMkIEmKHqELQAWKkAByytOoBJViAPJM7ExATWyAE0B8RgZkyAJmlYDoEAIahukJoNU6+HMTA0UOgT6oBgP38XUI6G5UMFZrzKR8EoGAUjGMDKYVgxDSsuAHcfMK8=' image = decompress(a2b_base64(compressed_image)) fapl = h5p.create(h5py.h5p.FILE_ACCESS) fapl.set_fapl_core() fapl.set_file_image(image) fid = h5f.open(self.mktemp().encode(), h5py.h5f.ACC_RDONLY, fapl=fapl) f = h5py.File(fid) self.assertTrue('test' in f)
def open_file(name, mode=None, driver=None, libver=None, userblock_size=None, **kwargs): """Open an ARF file, creating as necessary. Use this instead of h5py.File to ensure that root-level attributes and group creation property lists are set correctly. """ import sys import os from h5py import h5p from h5py._hl import files try: # If the byte string doesn't match the default # encoding, just pass it on as-is. Note Unicode # objects can always be encoded. name = name.encode(sys.getfilesystemencoding()) except (UnicodeError, LookupError): pass exists = os.path.exists(name) try: fcpl = h5p.create(h5p.FILE_CREATE) fcpl.set_link_creation_order(h5p.CRT_ORDER_TRACKED | h5p.CRT_ORDER_INDEXED) except AttributeError: # older version of h5py fp = files.File(name, mode=mode, driver=driver, libver=libver, **kwargs) else: fapl = files.make_fapl(driver, libver, **kwargs) fp = files.File(files.make_fid(name, mode, userblock_size, fapl, fcpl)) if not exists and fp.mode == 'r+': set_attributes(fp, arf_library='python', arf_library_version=__version__, arf_version=spec_version) return fp
def test_attr_phase_change(self): """ test the attribute phase change """ cid = h5p.create(h5p.OBJECT_CREATE) # test default value ret = cid.get_attr_phase_change() self.assertEqual((8, 6), ret) # max_compact must < 65536 (64kb) with self.assertRaises(ValueError): cid.set_attr_phase_change(65536, 6) # Using dense attributes storage to avoid 64kb size limitation # for a single attribute in compact attribute storage. cid.set_attr_phase_change(0, 0) self.assertEqual((0, 0), cid.get_attr_phase_change())
def _write_dset_low(dset, data, arr_slice, collective=False): memory_space = h5s.create_simple(data.shape) file_space = dset.id.get_space() s = (arr_slice[0].start,arr_slice[1].start,arr_slice[2].start) e = (arr_slice[0].stop,arr_slice[1].stop,arr_slice[2].stop) count = tuple([ee - ss for ss,ee in zip(s,e)]) file_space.select_hyperslab(s, count) if collective: dxpl = h5p.create(h5p.DATASET_XFER) dxpl.set_dxpl_mpio(h5fd.MPIO_COLLECTIVE) else: dxpl = None dset.id.write(memory_space, file_space, np.ascontiguousarray(data),dxpl=dxpl)
def make_fid(name, mode, userblock_size, fapl): """ Get a new FileID by opening or creating a file. Also validates mode argument.""" fcpl = None if userblock_size is not None: if mode in ('r', 'r+'): raise ValueError( "User block may only be specified when creating a file") try: userblock_size = int(userblock_size) except (TypeError, ValueError): raise ValueError("User block size must be an integer") fcpl = h5p.create(h5p.FILE_CREATE) fcpl.set_userblock(userblock_size) if mode == 'r': fid = h5f.open(name, h5f.ACC_RDONLY, fapl=fapl) elif mode == 'r+': fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl) elif mode == 'w-': fid = h5f.create(name, h5f.ACC_EXCL, fapl=fapl, fcpl=fcpl) elif mode == 'w': fid = h5f.create(name, h5f.ACC_TRUNC, fapl=fapl, fcpl=fcpl) elif mode == 'a' or mode is None: try: fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl) try: existing_fcpl = fid.get_create_plist() if userblock_size is not None and existing_fcpl.get_userblock( ) != userblock_size: raise ValueError( "Requested userblock size (%d) does not match that of existing file (%d)" % (userblock_size, existing_fcpl.get_userblock())) except: fid.close() raise except IOError: fid = h5f.create(name, h5f.ACC_EXCL, fapl=fapl, fcpl=fcpl) else: raise ValueError("Invalid mode; must be one of r, r+, w, w-, a") return fid
def make_fid(name, mode, userblock_size, fapl): """ Get a new FileID by opening or creating a file. Also validates mode argument.""" fcpl = None if userblock_size is not None: if mode in ('r', 'r+'): raise ValueError("User block may only be specified " "when creating a file") try: userblock_size = int(userblock_size) except (TypeError, ValueError): raise ValueError("User block size must be an integer") fcpl = h5p.create(h5p.FILE_CREATE) fcpl.set_userblock(userblock_size) if mode == 'r': fid = h5f.open(name, h5f.ACC_RDONLY, fapl=fapl) elif mode == 'r+': fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl) elif mode == 'w-': fid = h5f.create(name, h5f.ACC_EXCL, fapl=fapl, fcpl=fcpl) elif mode == 'w': fid = h5f.create(name, h5f.ACC_TRUNC, fapl=fapl, fcpl=fcpl) elif mode == 'a' or mode is None: try: fid = h5f.open(name, h5f.ACC_RDWR, fapl=fapl) try: existing_fcpl = fid.get_create_plist() if (userblock_size is not None and existing_fcpl.get_userblock() != userblock_size): raise ValueError("Requested userblock size (%d) does not match that of existing file (%d)" % (userblock_size, existing_fcpl.get_userblock())) except: fid.close() raise except IOError: fid = h5f.create(name, h5f.ACC_EXCL, fapl=fapl, fcpl=fcpl) else: raise ValueError("Invalid mode; must be one of r, r+, w, w-, a") return fid
def main(): sec2j.init() fapl = h5p.create(h5p.FILE_ACCESS) print fapl print 'fapl.id:', fapl.id sec2j.set_fapl(fapl.id) fname = "test.h5" if os.path.exists(fname): print 'Opening the file...' fid = h5f.open(fname, fapl=fapl) else: fid = h5f.create(fname, flags=h5f.ACC_TRUNC, fapl=fapl) print 'fid ready:', fid.id #return f = h5py.File(fid) sec2j.tx_start(fid.id) g = f.require_group('bbic/volume/0') for i in range(10000): g.attrs.create('a%d' % i, 640) f.flush() os._exit(-1)
def open_file(name, mode=None, driver=None, libver=None, userblock_size=None, **kwargs): """Open an ARF file, creating as necessary. Use this instead of h5py.File to ensure that root-level attributes and group creation property lists are set correctly. """ import sys import os from h5py import h5p from h5py._hl import files try: # If the byte string doesn't match the default # encoding, just pass it on as-is. Note Unicode # objects can always be encoded. name = name.encode(sys.getfilesystemencoding()) except (UnicodeError, LookupError): pass exists = os.path.exists(name) try: fcpl = h5p.create(h5p.FILE_CREATE) fcpl.set_link_creation_order( h5p.CRT_ORDER_TRACKED | h5p.CRT_ORDER_INDEXED) except AttributeError: # older version of h5py fp = files.File(name, mode=mode, driver=driver, libver=libver, **kwargs) else: fapl = files.make_fapl(driver, libver, **kwargs) fp = files.File(files.make_fid(name, mode, userblock_size, fapl, fcpl)) if not exists and fp.mode == 'r+': set_attributes(fp, arf_library='python', arf_library_version=__version__, arf_version=spec_version) return fp
def copy(self, source, dest, name=None, shallow=False, expand_soft=False, expand_external=False, expand_refs=False, without_attrs=False): """Copy an object or group. The source can be a path, Group, Dataset, or Datatype object. The destination can be either a path or a Group object. The source and destinations need not be in the same file. If the source is a Group object, all objects contained in that group will be copied recursively. When the destination is a Group object, by default the target will be created in that group with its current name (basename of obj.name). You can override that by setting "name" to a string. There are various options which all default to "False": - shallow: copy only immediate members of a group. - expand_soft: expand soft links into new objects. - expand_external: expand external links into new objects. - expand_refs: copy objects that are pointed to by references. - without_attrs: copy object without copying attributes. Example: >>> f = File('myfile.hdf5') >>> f.listnames() ['MyGroup'] >>> f.copy('MyGroup', 'MyCopy') >>> f.listnames() ['MyGroup', 'MyCopy'] """ if isinstance(source, HLObject): source_path = '.' else: # Interpret source as a path relative to this group source_path = source source = self if isinstance(dest, Group): if name is not None: dest_path = name else: # copy source into dest group: dest_name/source_name dest_path = pp.basename(h5i.get_name(source[source_path].id)) elif isinstance(dest, HLObject): raise TypeError("Destination must be path or Group object") else: # Interpret destination as a path relative to this group dest_path = dest dest = self flags = 0 if shallow: flags |= h5o.COPY_SHALLOW_HIERARCHY_FLAG if expand_soft: flags |= h5o.COPY_EXPAND_SOFT_LINK_FLAG if expand_external: flags |= h5o.COPY_EXPAND_EXT_LINK_FLAG if expand_refs: flags |= h5o.COPY_EXPAND_REFERENCE_FLAG if without_attrs: flags |= h5o.COPY_WITHOUT_ATTR_FLAG if flags: copypl = h5p.create(h5p.OBJECT_COPY) copypl.set_copy_object(flags) else: copypl = None h5o.copy(source.id, self._e(source_path), dest.id, self._e(dest_path), copypl, base.dlcpl)
def generate_dcpl(shape, dtype, chunks, compression, compression_opts, shuffle, fletcher32, maxshape, scaleoffset): """ Generate a dataset creation property list. Undocumented and subject to change without warning. """ if shape == (): if any((chunks, compression, compression_opts, shuffle, fletcher32, scaleoffset is not None)): raise TypeError("Scalar datasets don't support chunk/filter options") if maxshape and maxshape != (): raise TypeError("Scalar datasets cannot be extended") return h5p.create(h5p.DATASET_CREATE) def rq_tuple(tpl, name): """ Check if chunks/maxshape match dataset rank """ if tpl in (None, True): return try: tpl = tuple(tpl) except TypeError: raise TypeError('"%s" argument must be None or a sequence object' % name) if len(tpl) != len(shape): raise ValueError('"%s" must have same rank as dataset shape' % name) rq_tuple(chunks, 'chunks') rq_tuple(maxshape, 'maxshape') if compression is not None: if compression not in encode: raise ValueError('Compression filter "%s" is unavailable' % compression) if compression == 'gzip': if compression_opts is None: gzip_level = DEFAULT_GZIP elif compression_opts in range(10): gzip_level = compression_opts else: raise ValueError("GZIP setting must be an integer from 0-9, not %r" % compression_opts) elif compression == 'lzf': if compression_opts is not None: raise ValueError("LZF compression filter accepts no options") elif compression == 'szip': if compression_opts is None: compression_opts = DEFAULT_SZIP err = "SZIP options must be a 2-tuple ('ec'|'nn', even integer 0-32)" try: szmethod, szpix = compression_opts except TypeError: raise TypeError(err) if szmethod not in ('ec', 'nn'): raise ValueError(err) if not (0<szpix<=32 and szpix%2 == 0): raise ValueError(err) elif compression_opts is not None: # Can't specify just compression_opts by itself. raise TypeError("Compression method must be specified") if scaleoffset is not None: # scaleoffset must be an integer when it is not None or False, # except for integral data, for which scaleoffset == True is # permissible (will use SO_INT_MINBITS_DEFAULT) if scaleoffset < 0: raise ValueError('scale factor must be >= 0') if dtype.kind == 'f': if scaleoffset is True: raise ValueError('integer scaleoffset must be provided for ' 'floating point types') elif dtype.kind in ('u', 'i'): if scaleoffset is True: scaleoffset = h5z.SO_INT_MINBITS_DEFAULT else: raise TypeError('scale/offset filter only supported for integer ' 'and floating-point types') # Scale/offset following fletcher32 in the filter chain will (almost?) # always triggera a read error, as most scale/offset settings are # lossy. Since fletcher32 must come first (see comment below) we # simply prohibit the combination of fletcher32 and scale/offset. if fletcher32: raise ValueError('fletcher32 cannot be used with potentially lossy' ' scale/offset filter') # End argument validation if (chunks is True) or \ (chunks is None and any((shuffle, fletcher32, compression, maxshape, scaleoffset is not None))): chunks = guess_chunk(shape, maxshape, dtype.itemsize) if maxshape is True: maxshape = (None,)*len(shape) plist = h5p.create(h5p.DATASET_CREATE) if chunks is not None: plist.set_chunk(chunks) plist.set_fill_time(h5d.FILL_TIME_ALLOC) # prevent resize glitch # MUST be first, to prevent 1.6/1.8 compatibility glitch if fletcher32: plist.set_fletcher32() # scale-offset must come before shuffle and compression if scaleoffset is not None: if dtype.kind in ('u', 'i'): plist.set_scaleoffset(h5z.SO_INT, scaleoffset) else: # dtype.kind == 'f' plist.set_scaleoffset(h5z.SO_FLOAT_DSCALE, scaleoffset) if shuffle: plist.set_shuffle() if compression == 'gzip': plist.set_deflate(gzip_level) elif compression == 'lzf': plist.set_filter(h5z.FILTER_LZF, h5z.FLAG_OPTIONAL) elif compression == 'szip': opts = {'ec': h5z.SZIP_EC_OPTION_MASK, 'nn': h5z.SZIP_NN_OPTION_MASK} plist.set_szip(opts[szmethod], szpix) return plist
def generate_dcpl(shape, dtype, chunks, compression, compression_opts, shuffle, fletcher32, maxshape): """ Generate a dataset creation property list. Undocumented and subject to change without warning. """ # Scalar datasets don't support any fancy features # However, the currently defined behavior is simply to ignore these options if shape == (): #if any((chunks, compression, compression_opts, shuffle, fletcher32)): # raise TypeError("Scalar datasets don't support chunk/filter options") if maxshape and maxshape != (): raise TypeError("Scalar datasets cannot be extended") return h5p.create(h5p.DATASET_CREATE) def rq_tuple(tpl, name): """ Check if chunks/maxshape match dataset rank """ if tpl in (None, True): return try: tpl = tuple(tpl) except TypeError: raise TypeError('"%s" argument must be None or a sequence object' % name) if len(tpl) != len(shape): raise ValueError('"%s" must have same rank as dataset shape' % name) rq_tuple(chunks, 'chunks') rq_tuple(maxshape, 'maxshape') if compression is not None: if compression not in encode: raise ValueError('Compression filter "%s" is unavailable' % compression) if compression == 'gzip': if compression_opts is None: gzip_level = DEFAULT_GZIP elif compression_opts in range(10): gzip_level = compression_opts else: raise ValueError( "GZIP setting must be an integer from 0-9, not %r" % compression_opts) elif compression == 'lzf': if compression_opts is not None: raise ValueError("LZF compression filter accepts no options") elif compression == 'szip': if compression_opts is None: compression_opts = DEFAULT_SZIP err = "SZIP options must be a 2-tuple ('ec'|'nn', even integer 0-32)" try: szmethod, szpix = compression_opts except TypeError: raise TypeError(err) if szmethod not in ('ec', 'nn'): raise ValueError(err) if not (0 < szpix <= 32 and szpix % 2 == 0): raise ValueError(err) elif compression_opts is not None: # Can't specify just compression_opts by itself. raise TypeError("Compression method must be specified") # End argument validation if (chunks is True) or \ (chunks is None and any((shuffle, fletcher32, compression, maxshape))): chunks = guess_chunk(shape, dtype.itemsize) if maxshape is True: maxshape = (None, ) * len(shape) plist = h5p.create(h5p.DATASET_CREATE) if chunks is not None: plist.set_chunk(chunks) plist.set_fill_time(h5d.FILL_TIME_ALLOC) # prevent resize glitch # MUST be first, to prevent 1.6/1.8 compatibility glitch if fletcher32: plist.set_fletcher32() if shuffle: plist.set_shuffle() if compression == 'gzip': plist.set_deflate(gzip_level) elif compression == 'lzf': plist.set_filter(h5z.FILTER_LZF, h5z.FLAG_OPTIONAL) elif compression == 'szip': opts = {'ec': h5z.SZIP_EC_OPTION_MASK, 'nn': h5z.SZIP_NN_OPTION_MASK} plist.set_szip(opts[szmethod], szpix) return plist
def setup_fid(self, name): self.fname = res.get_data_copy(name) plist = h5p.create(h5p.FILE_ACCESS) plist.set_fclose_degree(h5f.CLOSE_STRONG) self.fid = h5f.open(self.fname, h5f.ACC_RDWR, fapl=plist)
def test_mdc_config(self): '''test get/set mdc config ''' falist = h5p.create(h5p.FILE_ACCESS) config = falist.get_mdc_config() falist.set_mdc_config(config)
def _make_new_dset(shape=None, dtype=None, data=None, chunks=None, compression=None, shuffle=None, fletcher32=None, maxshape=None, compression_opts=None, fillvalue=None, scaleoffset=None, track_times=None, external=None, track_order=None, dcpl=None): """ Return a new low-level dataset identifier """ # Convert data to a C-contiguous ndarray if data is not None and not isinstance(data, Empty): # normalize strings -> np.dtype objects if dtype is not None: _dtype = np.dtype(dtype) else: _dtype = None # if we are going to a f2 datatype, pre-convert in python # to workaround a possible h5py bug in the conversion. is_small_float = (_dtype is not None and _dtype.kind == 'f' and _dtype.itemsize == 2) data = np.asarray( data, order="C", dtype=(_dtype if is_small_float else guess_dtype(data))) # Validate shape if shape is None: if data is None: if dtype is None: raise TypeError( "One of data, shape or dtype must be specified") data = Empty(dtype) shape = data.shape else: shape = (shape, ) if isinstance(shape, int) else tuple(shape) if data is not None and (np.product(shape, dtype=np.ulonglong) != np.product(data.shape, dtype=np.ulonglong)): raise ValueError("Shape tuple is incompatible with data") if isinstance(maxshape, int): maxshape = (maxshape, ) # Validate chunk shape if isinstance(chunks, int) and not isinstance(chunks, bool): chunks = (chunks, ) # The original make_new_dset errors here if the shape is less than the # chunk size, but we avoid doing that as we cannot change the chunk size # for a dataset for any version once it is created. See #34. if isinstance(dtype, Datatype): # Named types are used as-is tid = dtype.id dtype = tid.dtype # Following code needs this else: # Validate dtype if dtype is None and data is None: dtype = np.dtype("=f4") elif dtype is None and data is not None: dtype = data.dtype else: dtype = np.dtype(dtype) tid = h5t.py_create(dtype, logical=1) # Legacy if any((compression, shuffle, fletcher32, maxshape, scaleoffset)) and chunks is False: raise ValueError("Chunked format required for given storage options") # Legacy if compression is True: if compression_opts is None: compression_opts = 4 compression = 'gzip' # Legacy if compression in _LEGACY_GZIP_COMPRESSION_VALS: if compression_opts is not None: raise TypeError("Conflict in compression options") compression_opts = compression compression = 'gzip' dcpl = filters.fill_dcpl(dcpl or h5p.create(h5p.DATASET_CREATE), shape, dtype, chunks, compression, compression_opts, shuffle, fletcher32, maxshape, scaleoffset, external) if fillvalue is not None: fillvalue = np.array(fillvalue) dcpl.set_fill_value(fillvalue) if track_times in (True, False): dcpl.set_obj_track_times(track_times) elif track_times is not None: raise TypeError("track_times must be either True or False") if track_order == True: dcpl.set_attr_creation_order(h5p.CRT_ORDER_TRACKED | h5p.CRT_ORDER_INDEXED) elif track_order == False: dcpl.set_attr_creation_order(0) elif track_order is not None: raise TypeError("track_order must be either True or False") if maxshape is not None: maxshape = tuple(m if m is not None else h5s.UNLIMITED for m in maxshape) if isinstance(data, Empty): raise NotImplementedError("Empty datasets") return data
from mpi4py import MPI #Get MPI info comm = MPI.COMM_WORLD NPROCS_LOCAL = int(os.environ['OMPI_COMM_WORLD_LOCAL_SIZE']) #Get number of processes NPROCS = comm.size #Get rank rank = comm.rank #H5PY for storage import h5py from h5py import h5s from h5py import h5p, h5fd dxpl = h5p.create(h5p.DATASET_XFER) dxpl.set_dxpl_mpio(h5fd.MPIO_COLLECTIVE) def task(rk, l): b = rk * l return (b, b + l) class Bunch(object): def __init__(self, adict): self.__dict__.update(adict) def factors(n): return sorted(set(reduce(list.__add__, ([i, n // i] for i in range(
def generate_dcpl(shape, dtype, chunks, compression, compression_opts, shuffle, fletcher32, maxshape): """ Generate a dataset creation property list. Undocumented and subject to change without warning. """ if shape == (): if any((chunks, compression, compression_opts, shuffle, fletcher32)): raise TypeError("Scalar datasets don't support chunk/filter options") if maxshape and maxshape != (): raise TypeError("Scalar datasets cannot be extended") return h5p.create(h5p.DATASET_CREATE) def rq_tuple(tpl, name): """ Check if chunks/maxshape match dataset rank """ if tpl in (None, True): return try: tpl = tuple(tpl) except TypeError: raise TypeError('"%s" argument must be None or a sequence object' % name) if len(tpl) != len(shape): raise ValueError('"%s" must have same rank as dataset shape' % name) rq_tuple(chunks, 'chunks') rq_tuple(maxshape, 'maxshape') if compression is not None: if compression not in encode: raise ValueError('Compression filter "%s" is unavailable' % compression) if compression == 'gzip': if compression_opts is None: gzip_level = DEFAULT_GZIP elif compression_opts in range(10): gzip_level = compression_opts else: raise ValueError("GZIP setting must be an integer from 0-9, not %r" % compression_opts) elif compression == 'lzf': if compression_opts is not None: raise ValueError("LZF compression filter accepts no options") elif compression == 'szip': if compression_opts is None: compression_opts = DEFAULT_SZIP err = "SZIP options must be a 2-tuple ('ec'|'nn', even integer 0-32)" try: szmethod, szpix = compression_opts except TypeError: raise TypeError(err) if szmethod not in ('ec', 'nn'): raise ValueError(err) if not (0<szpix<=32 and szpix%2 == 0): raise ValueError(err) elif compression_opts is not None: # Can't specify just compression_opts by itself. raise TypeError("Compression method must be specified") # End argument validation if (chunks is True) or \ (chunks is None and any((shuffle, fletcher32, compression, maxshape))): chunks = guess_chunk(shape, maxshape, dtype.itemsize) if maxshape is True: maxshape = (None,)*len(shape) plist = h5p.create(h5p.DATASET_CREATE) if chunks is not None: plist.set_chunk(chunks) plist.set_fill_time(h5d.FILL_TIME_ALLOC) # prevent resize glitch # MUST be first, to prevent 1.6/1.8 compatibility glitch if fletcher32: plist.set_fletcher32() if shuffle: plist.set_shuffle() if compression == 'gzip': plist.set_deflate(gzip_level) elif compression == 'lzf': plist.set_filter(h5z.FILTER_LZF, h5z.FLAG_OPTIONAL) elif compression == 'szip': opts = {'ec': h5z.SZIP_EC_OPTION_MASK, 'nn': h5z.SZIP_NN_OPTION_MASK} plist.set_szip(opts[szmethod], szpix) return plist
def test_libver_v110(self): """ Test libver bounds set/get for H5F_LIBVER_V110""" plist = h5p.create(h5p.FILE_ACCESS) plist.set_libver_bounds(h5f.LIBVER_V18, h5f.LIBVER_V110) self.assertEqual((h5f.LIBVER_V18, h5f.LIBVER_V110), plist.get_libver_bounds())
def test_libver(self): """ Test libver bounds set/get """ plist = h5p.create(h5p.FILE_ACCESS) plist.set_libver_bounds(h5f.LIBVER_EARLIEST, h5f.LIBVER_LATEST) self.assertEqual((h5f.LIBVER_EARLIEST, h5f.LIBVER_LATEST), plist.get_libver_bounds())
def default_lcpl(): """ Default link creation property list """ lcpl = h5p.create(h5p.LINK_CREATE) lcpl.set_create_intermediate_group(True) return lcpl
def generate_dcpl(shape, dtype, chunks, compression, compression_opts, shuffle, fletcher32, maxshape, scaleoffset): """ Generate a dataset creation property list. Undocumented and subject to change without warning. """ if shape == (): if any((chunks, compression, compression_opts, shuffle, fletcher32, scaleoffset is not None)): raise TypeError( "Scalar datasets don't support chunk/filter options") if maxshape and maxshape != (): raise TypeError("Scalar datasets cannot be extended") return h5p.create(h5p.DATASET_CREATE) def rq_tuple(tpl, name): """ Check if chunks/maxshape match dataset rank """ if tpl in (None, True): return try: tpl = tuple(tpl) except TypeError: raise TypeError('"%s" argument must be None or a sequence object' % name) if len(tpl) != len(shape): raise ValueError('"%s" must have same rank as dataset shape' % name) rq_tuple(chunks, 'chunks') rq_tuple(maxshape, 'maxshape') if compression is not None: if compression not in encode: raise ValueError('Compression filter "%s" is unavailable' % compression) if compression == 'gzip': if compression_opts is None: gzip_level = DEFAULT_GZIP elif compression_opts in range(10): gzip_level = compression_opts else: raise ValueError( "GZIP setting must be an integer from 0-9, not %r" % compression_opts) elif compression == 'lzf': if compression_opts is not None: raise ValueError("LZF compression filter accepts no options") elif compression == 'szip': if compression_opts is None: compression_opts = DEFAULT_SZIP err = "SZIP options must be a 2-tuple ('ec'|'nn', even integer 0-32)" try: szmethod, szpix = compression_opts except TypeError: raise TypeError(err) if szmethod not in ('ec', 'nn'): raise ValueError(err) if not (0 < szpix <= 32 and szpix % 2 == 0): raise ValueError(err) elif compression_opts is not None: # Can't specify just compression_opts by itself. raise TypeError("Compression method must be specified") if scaleoffset is not None: # scaleoffset must be an integer when it is not None or False, # except for integral data, for which scaleoffset == True is # permissible (will use SO_INT_MINBITS_DEFAULT) if scaleoffset < 0: raise ValueError('scale factor must be >= 0') if dtype.kind == 'f': if scaleoffset is True: raise ValueError('integer scaleoffset must be provided for ' 'floating point types') elif dtype.kind in ('u', 'i'): if scaleoffset is True: scaleoffset = h5z.SO_INT_MINBITS_DEFAULT else: raise TypeError('scale/offset filter only supported for integer ' 'and floating-point types') # Scale/offset following fletcher32 in the filter chain will (almost?) # always triggera a read error, as most scale/offset settings are # lossy. Since fletcher32 must come first (see comment below) we # simply prohibit the combination of fletcher32 and scale/offset. if fletcher32: raise ValueError('fletcher32 cannot be used with potentially lossy' ' scale/offset filter') # End argument validation if (chunks is True) or \ (chunks is None and any((shuffle, fletcher32, compression, maxshape, scaleoffset is not None))): chunks = guess_chunk(shape, maxshape, dtype.itemsize) if maxshape is True: maxshape = (None, ) * len(shape) plist = h5p.create(h5p.DATASET_CREATE) if chunks is not None: plist.set_chunk(chunks) plist.set_fill_time(h5d.FILL_TIME_ALLOC) # prevent resize glitch # MUST be first, to prevent 1.6/1.8 compatibility glitch if fletcher32: plist.set_fletcher32() # scale-offset must come before shuffle and compression if scaleoffset is not None: if dtype.kind in ('u', 'i'): plist.set_scaleoffset(h5z.SO_INT, scaleoffset) else: # dtype.kind == 'f' plist.set_scaleoffset(h5z.SO_FLOAT_DSCALE, scaleoffset) if shuffle: plist.set_shuffle() if compression == 'gzip': plist.set_deflate(gzip_level) elif compression == 'lzf': plist.set_filter(h5z.FILTER_LZF, h5z.FLAG_OPTIONAL) elif compression == 'szip': opts = {'ec': h5z.SZIP_EC_OPTION_MASK, 'nn': h5z.SZIP_NN_OPTION_MASK} plist.set_szip(opts[szmethod], szpix) return plist