def get_pset_files(dir_paramsets):
"""Return a list of parameter set files"""
def pset_filter(fname):
return fname[-3:] == ".py" and fname != "__init__.py"
return listdir_filter(dir_paramsets, pset_filter)
def collect_h5_to_db(dirpath, dbpath, output=True):
"""Collects all HDF5 of a directory and put them into a big HDF5 file.
Parameters
----------
dirpath : str
Directory where the HDF5 files are
dbpath : str
Filename of the resulting HDF5 file
output : bool, optional
True to output progress. Default is True.
"""
h5files = listdir_filter(dirpath, lambda fname: fname[-3:] == '.h5')
try:
h5files.remove(dbpath) # In case db is in the same directory
except ValueError:
pass # Nothing to do if the db path is not in the same directory
n_files = len(h5files)
old_simus_id = get_all_simus_id(dbpath)
# Open the db to put the new simus in
with tables.openFile(dbpath, 'a') as db:
for num_file, h5file in enumerate(h5files):
# Browse all simulation files in dirpath
with tables.openFile(h5file, 'r') as f:
file_time = f.root._v_attrs['time']
file_uuid = f.root._v_attrs['uuid']
# Add the simulation if not in DB
if (file_time, file_uuid) not in old_simus_id:
simu_id = file_time + '__' + file_uuid
simu_id = simu_id.replace(':', '_')
simu_id = simu_id.replace('.', '_')
simu_id = simu_id.replace('-', '_')
simu_name = 'simu' + simu_id
newgroup = db.createGroup(db.root, simu_name)
f.copyNode(f.root, newgroup, recursive=True)
f.copyNodeAttrs(f.root, newgroup)
# Output progress
if output:
print 'File '+str(num_file + 1)+'/'+str(n_files)+' done.'
