def _raiseValueError(filename): with CompressedFile(filename, create=True) as uncompressed: with open(uncompressed, 'w+') as f: text = ' This is a failed content, ' \ 'it should not appear in the compressed file\n' f.write(text) raise ValueError
def __init__(self, file_path_or_obj, cache_param_list=False, create=False, read_only=False): ''' Opens an HDF file (or accepts and already open h5py.File object) - will create if does not exist if create=True! :param cache_param_list: Names of parameters to cache where accessed. A value of True will result in all parameters to be cached. :type cache_param_list: [str] or bool :param file_path_or_obj: Can be either the path to an HDF file or an already opened HDF file object. :type file_path_or_obj: str or h5py.File :param create: ill allow creation of file if it does not exist. :type create: bool ''' if isinstance(file_path_or_obj, h5py.File): hdf_exists = True self.hdf = file_path_or_obj if self.hdf.mode != 'r+': raise ValueError("hdf_file requires mode 'r+'.") self.file_path = os.path.abspath(self.hdf.filename) else: hdf_exists = os.path.isfile(file_path_or_obj) if not create and not hdf_exists: raise IOError('File not found: %s' % file_path_or_obj) self.file_path = os.path.abspath(file_path_or_obj) if read_only: self.compressor = ReadOnlyCompressedFile(self.file_path) mode = 'r' else: self.compressor = CompressedFile(self.file_path) mode = 'a' uncompressed_path = self.compressor.load() self.hdf = h5py.File(uncompressed_path, mode=mode) self.hdfaccess_version = self.hdf.attrs.get('hdfaccess_version', 1) if hdf_exists: # default version is 1 assert self.hdfaccess_version == HDFACCESS_VERSION else: # just created this file, add the current version self.hdf.attrs['hdfaccess_version'] = HDFACCESS_VERSION if 'series' not in self.hdf.keys(): # The 'series' group is required for storing parameters. self.hdf.create_group('series') # cache keys as accessing __iter__ on hdf groups is v.slow self._cache = defaultdict(SortedSet) # cache parameters that are used often self._params_cache = {} # this is the list of parameters to cache if cache_param_list is True: cache_param_list = self.keys() elif cache_param_list is False: cache_param_list = [] self.cache_param_list = cache_param_list
def test_fromfile(self): ''' Create the compressed file from an existing one and compare the contents of the compressed version. ''' for compression_format in COMPRESSION_FORMATS.keys(): filename = self.filenames[compression_format] uncompressed_filename = self.uncompressed_filenames[compression_format] # Create the compressed version first cf = CompressedFile(filename, uncompressed_filename) cf.compress() with open(uncompressed_filename) as f: expected = f.readlines() # next uncompress the file and compare the contents with CompressedFile(filename) as uncompressed: with open(uncompressed) as f: self.assertListEqual(f.readlines(), expected)
def test_append(self): ''' Normal mode: any changes in the uncompressed file will be saved on exit from the context manager. ''' for filename in self.filenames: # Create contents of the files self.generateContent(filename) # Now let's try to write to it with CompressedFile(filename) as uncompressed: with open(uncompressed, 'a') as f: text = ' 2. This is the second line of content\n' f.write(text) # Now let's check the content of the file expected = [ ' 1. This is the first line of content\n', ' 2. This is the second line of content\n', ] with CompressedFile(filename) as uncompressed: with open(uncompressed) as f: self.assertListEqual(f.readlines(), expected)
def generateContent(self, filename): # Create a new compressed file with CompressedFile(filename, create=True) as uncompressed: with open(uncompressed, 'w+') as f: text = ' 1. This is the first line of content\n' f.write(text)