def __init__(self, filename, write=False):
if isinstance(filename, six.string_types):
nommap = s3.is_s3_path(filename) or gcs.is_gs_path(filename)
super(Hdf5MemoryMapped, self).__init__(filename,
write=write,
nommap=nommap)
else:
super(Hdf5MemoryMapped, self).__init__(filename.name,
write=write,
nommap=True)
if hasattr(filename, 'read'):
fp = filename # support file handle for testing
self.file_map[self.filename] = fp
else:
mode = 'rb+' if write else 'rb'
if s3.is_s3_path(filename):
fp = s3.open(self.filename)
self.file_map[self.filename] = fp
elif gcs.is_gs_path(filename):
fp = gcs.open(filename)
self.file_map[self.filename] = fp
else:
if self.nommap:
fp = open(self.filename, mode)
self.file_map[self.filename] = fp
else:
# this is the only path that will have regular mmapping
fp = self.filename
self.h5file = h5py.File(fp, "r+" if write else "r")
self.h5table_root_name = None
self._version = 1
self._load()
def __init__(self, path, write=False):
if isinstance(path, six.string_types):
nommap = s3.is_s3_path(path) or gcs.is_gs_path(path)
super(Hdf5MemoryMapped, self).__init__(path, write=write, nommap=nommap)
else:
super(Hdf5MemoryMapped, self).__init__(path.name, write=write, nommap=True)
self._all_mmapped = True
self._open(path)
self.units = {}
self.h5table_root_name = None
self._version = 1
self._load()
if not write: # in write mode, call freeze yourself, so the hashes are computed
self._freeze()
if self._all_mmapped:
self.h5file.close()
def _open(self, path):
if hasattr(path, 'read'):
fp = path # support file handle for testing
self.file_map[self.path] = fp
else:
mode = 'rb+' if self.write else 'rb'
if s3.is_s3_path(path):
fp = s3.open(self.path)
self.file_map[self.path] = fp
elif gcs.is_gs_path(path):
fp = gcs.open(self.path)
self.file_map[self.path] = fp
else:
if self.nommap:
fp = open(self.path, mode)
self.file_map[self.path] = fp
else:
# this is the only path that will have regular mmapping
fp = self.path
self.h5file = h5py.File(fp, "r+" if self.write else "r")