def test_remerging(self):
prev_old_length = self.test_merge_with_linked_derived_parameter(
disable_logging=False)
name = self.traj1
self.bfilename = make_temp_dir(
os.path.join('experiments', 'tests', 'HDF5',
'backup_test%s.hdf5' % self.trajname1))
self.traj1.f_load(load_data=2)
self.traj1.f_backup(backup_filename=self.bfilename)
self.traj3 = load_trajectory(index=-1,
filename=self.bfilename,
load_all=2)
old_length = len(self.traj1)
self.traj1.f_merge(self.traj3, backup=False, remove_duplicates=False)
self.assertTrue(len(self.traj1) > old_length)
self.traj1.f_load(load_data=2)
for run in self.traj1.f_get_run_names():
self.traj1.v_crun = run
idx = self.traj1.v_idx
param = self.traj1['test.crun.gg']
if idx < prev_old_length or old_length <= idx < prev_old_length + old_length:
self.assertTrue(param == 42, '%s != 42' % str(param))
else:
self.assertTrue(param == 44, '%s != 44' % str(param))
param = self.traj1['test.hh.crun']
if idx < prev_old_length or old_length <= idx < prev_old_length + old_length:
self.assertTrue(param == 111, '%s != 111' % str(param))
else:
self.assertTrue(param == 53, '%s != 53' % str(param))
self.assertTrue(len(self.traj1) > old_length)
for irun in range(len(self.traj1.f_get_run_names())):
self.assertTrue(
self.traj1.res.runs['r_%d' % irun].paraBL == self.traj1.paramB)
self.assertTrue(self.traj1.res['r_%d' % irun] == self.traj1.paramB)
self.env1.f_disable_logging()
self.env2.f_disable_logging()
def test_full_store(self):
filename = make_temp_dir('full_store.hdf5')
with Environment(filename=filename,
log_config=get_log_config()) as env:
traj = env.v_trajectory
traj.par.x = Parameter('x', 3, 'jj')
traj.f_explore({'x': [1,2,3]})
env.f_run(add_one_particular_item, True)
traj = load_trajectory(index=-1, filename=filename)
self.assertTrue('hi' in traj)
def test_full_store(self):
filename = make_temp_dir('full_store.hdf5')
with Environment(filename=filename,
log_config=get_log_config()) as env:
traj = env.v_trajectory
traj.par.x = Parameter('x', 3, 'jj')
traj.f_explore({'x': [1,2,3]})
env.f_run(add_one_particular_item, True)
traj = load_trajectory(index=-1, filename=filename)
self.assertTrue('hi' in traj)
def test_remerging(self):
prev_old_length = self.test_merge_with_linked_derived_parameter(disable_logging=False)
name = self.traj1
self.bfilename = make_temp_dir(os.path.join('experiments',
'tests',
'HDF5',
'backup_test%s.hdf5' % self.trajname1))
self.traj1.f_load(load_data=2)
self.traj1.f_backup(backup_filename=self.bfilename)
self.traj3 = load_trajectory(index=-1, filename=self.bfilename, load_all=2)
old_length = len(self.traj1)
self.traj1.f_merge(self.traj3, backup=False, remove_duplicates=False)
self.assertTrue(len(self.traj1) > old_length)
self.traj1.f_load(load_data=2)
for run in self.traj1.f_get_run_names():
self.traj1.v_crun = run
idx = self.traj1.v_idx
param = self.traj1['test.crun.gg']
if idx < prev_old_length or old_length <= idx < prev_old_length + old_length:
self.assertTrue(param == 42, '%s != 42' % str(param))
else:
self.assertTrue(param == 44, '%s != 44' % str(param))
param = self.traj1['test.hh.crun']
if idx < prev_old_length or old_length <= idx < prev_old_length + old_length:
self.assertTrue(param == 111, '%s != 111' % str(param))
else:
self.assertTrue(param == 53, '%s != 53' % str(param))
self.assertTrue(len(self.traj1)>old_length)
for irun in range(len(self.traj1.f_get_run_names())):
self.assertTrue(self.traj1.res.runs['r_%d' % irun].paraBL == self.traj1.paramB)
self.assertTrue(self.traj1.res['r_%d' % irun] == self.traj1.paramB)
self.env1.f_disable_logging()
self.env2.f_disable_logging()
def merge_all_in_folder(folder, ext='.hdf5',
dynamic_imports=None,
storage_service=None,
force=False,
ignore_data=(),
move_data=False,
delete_other_files=False,
keep_info=True,
keep_other_trajectory_info=True,
merge_config=True,
backup=True):
"""Merges all files in a given folder.
IMPORTANT: Does not check if there are more than 1 trajectory in a file. Always
uses the last trajectory in file and ignores the other ones.
Trajectories are merged according to the alphabetical order of the files,
i.e. the resulting merged trajectory is found in the first file
(according to lexicographic ordering).
:param folder: folder (not recursive) where to look for files
:param ext: only files with the given extension are used
:param dynamic_imports: Dynamic imports for loading
:param storage_service: storage service to use, leave `None` to use the default one
:param force: If loading should be forced.
:param delete_other_files: Deletes files of merged trajectories
All other parameters as in `f_merge_many` of the trajectory.
:return: The merged traj
"""
in_dir = os.listdir(folder)
all_files = []
# Find all files with matching extension
for file in in_dir:
full_file = os.path.join(folder, file)
if os.path.isfile(full_file):
_, extension = os.path.splitext(full_file)
if extension == ext:
all_files.append(full_file)
all_files = sorted(all_files)
# Open all trajectories
trajs = []
for full_file in all_files:
traj = load_trajectory(index=-1,
storage_service=storage_service,
filename=full_file,
load_data=0,
force=force,
dynamic_imports=dynamic_imports)
trajs.append(traj)
# Merge all trajectories
first_traj = trajs.pop(0)
first_traj.f_merge_many(trajs,
ignore_data=ignore_data,
move_data=move_data,
delete_other_trajectory=False,
keep_info=keep_info,
keep_other_trajectory_info=keep_other_trajectory_info,
merge_config=merge_config,
backup=backup)
if delete_other_files:
# Delete all but the first file
for file in all_files[1:]:
os.remove(file)
return first_traj
def compact_hdf5_file(filename, name=None, index=None, keep_backup=True):
"""Can compress an HDF5 to reduce file size.
The properties on how to compress the new file are taken from a given
trajectory in the file.
Simply calls ``ptrepack`` from the command line.
(Se also https://pytables.github.io/usersguide/utilities.html#ptrepackdescr)
Currently only supported under Linux, no guarantee for Windows usage.
:param filename:
Name of the file to compact
:param name:
The name of the trajectory from which the compression properties are taken
:param index:
Instead of a name you could also specify an index, i.e -1 for the last trajectory
in the file.
:param keep_backup:
If a back up version of the original file should be kept.
The backup file is named as the original but `_backup` is appended to the end.
:return:
The return/error code of ptrepack
"""
if name is None and index is None:
index = -1
tmp_traj = load_trajectory(name, index, as_new=False, load_all=pypetconstants.LOAD_NOTHING,
force=True, filename=filename)
service = tmp_traj.v_storage_service
complevel = service.complevel
complib = service.complib
shuffle = service.shuffle
fletcher32 = service.fletcher32
name_wo_ext, ext = os.path.splitext(filename)
tmp_filename = name_wo_ext + '_tmp' + ext
abs_filename = os.path.abspath(filename)
abs_tmp_filename = os.path.abspath(tmp_filename)
command = ['ptrepack', '-v',
'--complib', complib,
'--complevel', str(complevel),
'--shuffle', str(int(shuffle)),
'--fletcher32', str(int(fletcher32)),
abs_filename, abs_tmp_filename]
str_command = ' '.join(command)
print('Executing command `%s`' % str_command)
retcode = subprocess.call(command)
if retcode != 0:
print('#### ERROR: Compacting `%s` failed with errorcode %s! ####' %
(filename, str(retcode)))
else:
print('#### Compacting successful ####')
print('Renaming files')
if keep_backup:
backup_file_name = name_wo_ext + '_backup' + ext
os.rename(filename, backup_file_name)
else:
os.remove(filename)
os.rename(tmp_filename, filename)
print('### Compacting and Renaming finished ####')
return retcode
def compact_hdf5_file(filename, name=None, index=None, keep_backup=True):
"""Can compress an HDF5 to reduce file size.
The properties on how to compress the new file are taken from a given
trajectory in the file.
Simply calls ``ptrepack`` from the command line.
(Se also https://pytables.github.io/usersguide/utilities.html#ptrepackdescr)
Currently only supported under Linux, no guarantee for Windows usage.
:param filename:
Name of the file to compact
:param name:
The name of the trajectory from which the compression properties are taken
:param index:
Instead of a name you could also specify an index, i.e -1 for the last trajectory
in the file.
:param keep_backup:
If a back up version of the original file should be kept.
The backup file is named as the original but `_backup` is appended to the end.
:return:
The return/error code of ptrepack
"""
if name is None and index is None:
index = -1
tmp_traj = load_trajectory(name,
index,
as_new=False,
load_all=pypetconstants.LOAD_NOTHING,
force=True,
filename=filename)
service = tmp_traj.v_storage_service
complevel = service.complevel
complib = service.complib
shuffle = service.shuffle
fletcher32 = service.fletcher32
name_wo_ext, ext = os.path.splitext(filename)
tmp_filename = name_wo_ext + '_tmp' + ext
abs_filename = os.path.abspath(filename)
abs_tmp_filename = os.path.abspath(tmp_filename)
command = [
'ptrepack', '-v', '--complib', complib, '--complevel',
str(complevel), '--shuffle',
str(int(shuffle)), '--fletcher32',
str(int(fletcher32)), abs_filename, abs_tmp_filename
]
str_command = ' '.join(command)
print('Executing command `%s`' % str_command)
retcode = subprocess.call(command)
if retcode != 0:
print('#### ERROR: Compacting `%s` failed with errorcode %s! ####' %
(filename, str(retcode)))
else:
print('#### Compacting successful ####')
print('Renaming files')
if keep_backup:
backup_file_name = name_wo_ext + '_backup' + ext
os.rename(filename, backup_file_name)
else:
os.remove(filename)
os.rename(tmp_filename, filename)
print('### Compacting and Renaming finished ####')
return retcode
