def prepare(rkeys, version, force=False, cores=1, hashing=None): pool = Pool(cores) m = Manager() queue = m.Queue() jobs = [rkey + (version, force, hashing, queue) for rkey in rkeys] bar = Bar("[1/3]", max=len(jobs), suffix="%(percent).1f%% / %(elapsed_td)s / ETA %(eta_td)s") bar.start() res = pool.map_async(prepare2, jobs, chunksize=1) todo = len(jobs) while todo: queue.get() todo -= 1 bar.next() bar.finish() pool.close() pool.join()
def make(rkeys, out=None, hashing=None): dat = [] bar = Bar("[2/3]", max=len(rkeys), suffix="%(percent).1f%% / %(elapsed_td)s / ETA %(eta_td)s") bar.start() for (bid, pid, problem, limit) in rkeys: f_dat = expres.results.path(bid, pid, problem, limit, ext="in" if hashing else "pre") if out: tmp = open(f_dat).read().strip() if tmp: out.write(tmp) out.write("\n") else: dat.extend(open(f_dat).read().strip().split("\n")) bar.next() bar.finish() return dat if not out else None