def reduce(self, params):
from os import listdir
files = [f for f in listdir(".")
if f.startswith(params.prefix) and f.endswith(".root")]
def get_period(s): return s.split("-")[1].split("_")[-1]
by_subperiod = {}
for f in files:
by_subperiod.setdefault(get_period(f), []).append(f)
from pprint import pprint
pprint(by_subperiod)
from multiprocessing import Pool, cpu_count
pool = Pool(cpu_count())
pool.map(mp_merge, [("period%s.root" % p, files) for p, files in by_subperiod.iteritems()])
by_period = {}
for p, files in sorted(by_subperiod.iteritems()):
by_period.setdefault(p[0], []).append("period%s.root" % p)
pprint(by_period)
pool.map(mp_merge, [("period%s.root" % p, files) for p, files in by_period.iteritems()])
from hmerge import merge_files
merge_files("all.root", ["period%s.root" % p for p in by_period])
print "Done."
def mp_merge(args):
from hmerge import merge_files
merge_files(*args)
