def _raiseValueError(filename):
with CompressedFile(filename, create=True) as uncompressed:
with open(uncompressed, 'w+') as f:
text = ' This is a failed content, ' \
'it should not appear in the compressed file\n'
f.write(text)
raise ValueError
def __init__(self,
file_path_or_obj,
cache_param_list=False,
create=False,
read_only=False):
'''
Opens an HDF file (or accepts and already open h5py.File object) - will
create if does not exist if create=True!
:param cache_param_list: Names of parameters to cache where accessed. A value of True will result in all parameters to be cached.
:type cache_param_list: [str] or bool
:param file_path_or_obj: Can be either the path to an HDF file or an already opened HDF file object.
:type file_path_or_obj: str or h5py.File
:param create: ill allow creation of file if it does not exist.
:type create: bool
'''
if isinstance(file_path_or_obj, h5py.File):
hdf_exists = True
self.hdf = file_path_or_obj
if self.hdf.mode != 'r+':
raise ValueError("hdf_file requires mode 'r+'.")
self.file_path = os.path.abspath(self.hdf.filename)
else:
hdf_exists = os.path.isfile(file_path_or_obj)
if not create and not hdf_exists:
raise IOError('File not found: %s' % file_path_or_obj)
self.file_path = os.path.abspath(file_path_or_obj)
if read_only:
self.compressor = ReadOnlyCompressedFile(self.file_path)
mode = 'r'
else:
self.compressor = CompressedFile(self.file_path)
mode = 'a'
uncompressed_path = self.compressor.load()
self.hdf = h5py.File(uncompressed_path, mode=mode)
self.hdfaccess_version = self.hdf.attrs.get('hdfaccess_version', 1)
if hdf_exists:
# default version is 1
assert self.hdfaccess_version == HDFACCESS_VERSION
else:
# just created this file, add the current version
self.hdf.attrs['hdfaccess_version'] = HDFACCESS_VERSION
if 'series' not in self.hdf.keys():
# The 'series' group is required for storing parameters.
self.hdf.create_group('series')
# cache keys as accessing __iter__ on hdf groups is v.slow
self._cache = defaultdict(SortedSet)
# cache parameters that are used often
self._params_cache = {}
# this is the list of parameters to cache
if cache_param_list is True:
cache_param_list = self.keys()
elif cache_param_list is False:
cache_param_list = []
self.cache_param_list = cache_param_list
def test_fromfile(self):
'''
Create the compressed file from an existing one and compare the
contents of the compressed version.
'''
for compression_format in COMPRESSION_FORMATS.keys():
filename = self.filenames[compression_format]
uncompressed_filename = self.uncompressed_filenames[compression_format]
# Create the compressed version first
cf = CompressedFile(filename, uncompressed_filename)
cf.compress()
with open(uncompressed_filename) as f:
expected = f.readlines()
# next uncompress the file and compare the contents
with CompressedFile(filename) as uncompressed:
with open(uncompressed) as f:
self.assertListEqual(f.readlines(), expected)
def test_append(self):
'''
Normal mode: any changes in the uncompressed file will be saved on
exit from the context manager.
'''
for filename in self.filenames:
# Create contents of the files
self.generateContent(filename)
# Now let's try to write to it
with CompressedFile(filename) as uncompressed:
with open(uncompressed, 'a') as f:
text = ' 2. This is the second line of content\n'
f.write(text)
# Now let's check the content of the file
expected = [
' 1. This is the first line of content\n',
' 2. This is the second line of content\n',
]
with CompressedFile(filename) as uncompressed:
with open(uncompressed) as f:
self.assertListEqual(f.readlines(), expected)
def generateContent(self, filename):
# Create a new compressed file
with CompressedFile(filename, create=True) as uncompressed:
with open(uncompressed, 'w+') as f:
text = ' 1. This is the first line of content\n'
f.write(text)
