def pickle_merge(folder):
"""Merges pickle files and save it to `all.pkl`."""
file_list = _get_file_list(folder)
if len(file_list) == 0:
print('There is nothing to merge because no file' +
' has been found in %s.' % folder)
return
ha = folder_hash(folder, ['all.pkl', '.folder_hash'])
subfolders = [d for d in listdir(folder) if isdir(join(folder, d))]
with temp_folder() as tf:
for sf in tqdm(subfolders, desc='Copying files'):
make_sure_path_exists(join(tf, sf))
cp(join(folder, sf), join(tf, sf))
file_list = _get_file_list(tf)
out = _merge(file_list)
with BeginEnd('Storing pickles'):
pickle(out, join(folder, 'all.pkl'))
_save_cache(folder, ha)
return out
def store_job(job, fpath):
pickle(job, fpath)
def store_jobs(jobs, fpath):
print('Storing jobs...')
pickle({t.jobid: t for t in jobs}, fpath)
print(' %d jobs stored ' % len(jobs))
def store_task_results(task_results, fpath):
print('Storing task results')
pickle({tr.task_id: tr for tr in task_results}, fpath)
print(' %d task results stored ' % len(task_results))
def store_task_args(task_args, fpath):
if os.path.exists(fpath):
return
pickle(task_args, fpath)
def store_tasks(tasks, fpath):
if os.path.exists(fpath):
return
pickle({t.task_id: t for t in tasks}, fpath)
def store(self):
fp = join(config.stdoe_folder(), self.runid, 'cluster_run.pkl')
pickle(self, fp)
def _test_it(folder):
fp = join(folder, "a.pkl")
pickle(A(10), fp)
a = unpickle(fp)
assert a.value == 10