def setRepositoryRevisions(self): ###could be inherited from artusWrapper!
# expand possible environment variables in paths
if isinstance(self._args.repo_scan_base_dirs, basestring):
self._args.repo_scan_base_dirs = [self._args.repo_scan_base_dirs]
self._args.repo_scan_base_dirs = [os.path.expandvars(repoScanBaseDir) for repoScanBaseDir in self._args.repo_scan_base_dirs]
# construct possible scan paths
subDirWildcards = ["*/" * level for level in range(self._args.repo_scan_depth+1)]
scanDirWildcards = [os.path.join(repoScanBaseDir, subDirWildcard) for repoScanBaseDir in self._args.repo_scan_base_dirs for subDirWildcard in subDirWildcards]
# globbing and filter for directories
scanDirs = tools.flattenList([glob.glob(scanDirWildcard) for scanDirWildcard in scanDirWildcards])
scanDirs = [scanDir for scanDir in scanDirs if os.path.isdir(scanDir)]
# key: directory to check type of repository
# value: command to extract the revision
repoVersionCommands = {
".git" : "git rev-parse HEAD",
".svn" : "svn info"# | grep Revision | awk '{print $2}'"
}
# loop over dirs and revision control systems and write revisions to the config dict
for repoDir, currentRevisionCommand in repoVersionCommands.items():
repoScanDirs = [os.path.join(scanDir, repoDir) for scanDir in scanDirs]
repoScanDirs = [glob.glob(os.path.join(scanDir, repoDir)) for scanDir in scanDirs]
repoScanDirs = tools.flattenList([glob.glob(os.path.join(scanDir, repoDir)) for scanDir in scanDirs])
repoScanDirs = [os.path.abspath(os.path.join(repoScanDir, "..")) for repoScanDir in repoScanDirs]
for repoScanDir in repoScanDirs:
popenCout, popenCerr = subprocess.Popen(currentRevisionCommand.split(), stdout=subprocess.PIPE, cwd=repoScanDir).communicate()
self._config[repoScanDir] = popenCout.replace("\n", "")
def setRepositoryRevisions(self):
# expand possible environment variables in paths
if isinstance(self._args.repo_scan_base_dirs, basestring):
self._args.repo_scan_base_dirs = [self._args.repo_scan_base_dirs]
self._args.repo_scan_base_dirs = [os.path.expandvars(repoScanBaseDir) for repoScanBaseDir in self._args.repo_scan_base_dirs]
# construct possible scan paths
subDirWildcards = ["*/" * level for level in range(self._args.repo_scan_depth+1)]
scanDirWildcards = [os.path.join(repoScanBaseDir, subDirWildcard) for repoScanBaseDir in self._args.repo_scan_base_dirs for subDirWildcard in subDirWildcards]
# globbing and filter for directories
scanDirs = tools.flattenList([glob.glob(scanDirWildcard) for scanDirWildcard in scanDirWildcards])
scanDirs = [scanDir for scanDir in scanDirs if os.path.isdir(scanDir)]
# key: directory to check type of repository
# value: command to extract the revision
repoVersionCommands = {
".git" : "git rev-parse HEAD",
".svn" : "svn info"# | grep Revision | awk '{print $2}'"
}
# loop over dirs and revision control systems and write revisions to the config dict
for repoDir, currentRevisionCommand in repoVersionCommands.items():
repoScanDirs = [os.path.join(scanDir, repoDir) for scanDir in scanDirs]
repoScanDirs = [glob.glob(os.path.join(scanDir, repoDir)) for scanDir in scanDirs]
repoScanDirs = tools.flattenList([glob.glob(os.path.join(scanDir, repoDir)) for scanDir in scanDirs])
repoScanDirs = [os.path.abspath(os.path.join(repoScanDir, "..")) for repoScanDir in repoScanDirs]
for repoScanDir in repoScanDirs:
popenCout, popenCerr = subprocess.Popen(currentRevisionCommand.split(), stdout=subprocess.PIPE, cwd=repoScanDir).communicate()
self._config[repoScanDir] = popenCout.replace("\n", "")
def prepare_args(self, parser, plotData):
super(InputRoot, self).prepare_args(parser, plotData)
self.prepare_list_args(plotData, [
"nicks", "x_expressions", "y_expressions", "z_expressions",
"x_bins", "y_bins", "z_bins", "scale_factors", "files",
"directories", "folders", "weights", "friend_files",
"friend_folders", "friend_aliases", "tree_draw_options",
"proxy_prefixes"
],
help="InputRoot options")
inputbase.InputBase.prepare_nicks(plotData)
for key in ["folders"]:
plotData.plotdict[key] = [
element.split() if element else [""]
for element in plotData.plotdict[key]
]
for key in ["friend_files", "friend_folders"]:
plotData.plotdict[key] = [
element.split() if element else element
for element in plotData.plotdict[key]
]
if plotData.plotdict["redo_cache"] is None:
plotData.plotdict["redo_cache"] = not any(
tools.flattenList([[
input_file.startswith("/") or ("://" in input_file)
for input_file in files
] for files in plotData.plotdict["files"]]))
if plotData.plotdict["read_config"]:
self.read_input_json_dicts(plotData)
def crossings_graph(x_crossings, y_crossing, x_lims=None):
x_values = tools.flattenList(zip(*([x_crossings]*3)))
y_values = [y_crossing, 0.0, y_crossing]*len(x_crossings)
if x_lims:
x_values = [x_lims[0]]+x_values+[x_lims[1]]
y_values = [y_crossing]+y_values+[y_crossing]
graph = ROOT.TGraph(len(x_values), array.array("d", x_values), array.array("d", y_values))
graph.SetName("graph_" + hashlib.md5("_".join([str(x_crossings), str(y_crossing)])).hexdigest())
return graph
def crossings_graph(x_crossings, y_crossing, x_lims=None):
x_values = tools.flattenList(zip(*([x_crossings] * 3)))
y_values = [y_crossing, 0.0, y_crossing] * len(x_crossings)
if x_lims:
x_values = [x_lims[0]] + x_values + [x_lims[1]]
y_values = [y_crossing] + y_values + [y_crossing]
graph = ROOT.TGraph(len(x_values), array.array("d", x_values),
array.array("d", y_values))
graph.SetName("graph_" + hashlib.md5("_".join(
[str(x_crossings), str(y_crossing)])).hexdigest())
return graph
def prepare_args(self, parser, plotData):
super(InputRoot, self).prepare_args(parser, plotData)
self.prepare_list_args(plotData, ["nicks", "x_expressions", "y_expressions", "z_expressions", "x_bins", "y_bins", "z_bins", "scale_factors", "files", "directories", "folders", "weights", "friend_files", "friend_folders", "friend_aliases", "tree_draw_options", "proxy_prefixes"], help="InputRoot options")
inputbase.InputBase.prepare_nicks(plotData)
for key in ["folders"]:
plotData.plotdict[key] = [element.split() if element else [""] for element in plotData.plotdict[key]]
for key in ["friend_files", "friend_folders"]:
plotData.plotdict[key] = [element.split() if element else element for element in plotData.plotdict[key]]
if plotData.plotdict["redo_cache"] is None:
plotData.plotdict["redo_cache"] = not any(tools.flattenList([[input_file.startswith("/") or ("://" in input_file) for input_file in files] for files in plotData.plotdict["files"]]))
if plotData.plotdict["read_config"]:
self.read_input_json_dicts(plotData)
def merge_graphs(graphs, allow_reversed=False, allow_shuffle=False):
"""
Merge graphs
"""
if len(graphs) == 0:
return None
points = []
for graph in graphs:
x_values = graph.GetX()
y_values = graph.GetY()
points.append([[x_values[index], y_values[index]] for index in xrange(graph.GetN())])
if allow_shuffle:
points = geometry.order_lists_for_smallest_distances(points, allow_reversed=allow_reversed)
merged_x_values, merged_y_values = zip(*tools.flattenList(points))
merged_x_values = array.array("d", merged_x_values)
merged_y_values = array.array("d", merged_y_values)
graph = ROOT.TGraph(len(merged_x_values), merged_x_values, merged_y_values)
return graph
def merge_graphs(graphs, allow_reversed=False, allow_shuffle=False):
"""
Merge graphs
"""
if len(graphs) == 0:
return None
points = []
for graph in graphs:
x_values = graph.GetX()
y_values = graph.GetY()
points.append([[x_values[index], y_values[index]] for index in xrange(graph.GetN())])
if allow_shuffle:
points = geometry.order_lists_for_smallest_distances(points, allow_reversed=allow_reversed)
merged_x_values, merged_y_values = zip(*tools.flattenList(points))
merged_x_values = array.array("d", merged_x_values)
merged_y_values = array.array("d", merged_y_values)
graph = ROOT.TGraph(len(merged_x_values), merged_x_values, merged_y_values)
return graph
os.path.join(channel_category, quark_type, unc_type)
for unc_type in [
"nominal", "alpha_s_up", "alpha_s_down", "pdf_up",
"pdf_down", "qcd_scale_up", "qcd_scale_down"
]
]
config["output_dir"] = os.path.join(args.output_dir, channel,
category, quark_type)
config["filename"] = "energy_distribution"
if not args.www is None:
config["www"] = os.path.join(args.www, channel, category,
quark_type)
plot_configs.append(config)
if log.isEnabledFor(logging.DEBUG):
import pprint
pprint.pprint(plot_configs)
plot_results = higgsplot.HiggsPlotter(list_of_config_dicts=plot_configs,
list_of_args_strings=[args.args],
n_processes=args.n_processes,
n_plots=args.n_plots)
root_filenames = tools.flattenList(plot_results.output_filenames)
merged_output = os.path.join(args.output_dir, "energy_distributions.root")
tools.hadd(target_file=merged_output,
source_files=root_filenames,
hadd_args="-f")
log.info("Merged outputs in " + merged_output)
# plot best fit values of parameter pol from physics model
plot_configs = []
for template in [
"$CMSSW_BASE/src/HiggsAnalysis/KITHiggsToTauTau/data/plots/configs/combine/best_fit_pol_over_channel.json",
"$CMSSW_BASE/src/HiggsAnalysis/KITHiggsToTauTau/data/plots/configs/combine/best_fit_pol_over_channel_tot_stat_unc.json",
"$CMSSW_BASE/src/HiggsAnalysis/KITHiggsToTauTau/data/plots/configs/combine/best_fit_weinberg_angle_over_channel.json",
"$CMSSW_BASE/src/HiggsAnalysis/KITHiggsToTauTau/data/plots/configs/combine/best_fit_weinberg_angle_over_channel_tot_stat_unc.json"
]:
config = jsonTools.JsonDict(os.path.expandvars(template))
config["directories"] = [
" ".join(
set([
os.path.dirname(root_file) for root_file in sorted(
tools.flattenList(values_tree_files.values()))
]))
]
config["x_ticks"] = sorted(values_tree_files.keys())
inv_annotation_replacements = {
value: key
for key, value in annotation_replacements.iteritems()
}
config["x_tick_labels"] = [
inv_annotation_replacements.get(int(value), value)
for value in sorted(values_tree_files.keys())
]
#config["x_tick_labels"] = ["#scale[1.5]{" + ("" if label == "combined" else "channel_") + label + "}" for label in config["x_tick_labels"]]
config["x_tick_labels"] = [
"" + ("" if label == "combined" else "channel_") + label + ""
for label in config["x_tick_labels"]
split_stat_syst_uncs=True,
additional_freeze_nuisances=["r"])
annotation_replacements = {
channel: index
for (index, channel) in enumerate(["combined"] + args.channel)
}
annotation_replacements.update({
binid: index + 1
for (index, binid) in enumerate(
sorted(
list(
set([
datacards.configs.category2binid(
category, channel=category[:2]) for
category in tools.flattenList(args.categories)
]))))
})
if not asimov_polarisation is None:
annotation_replacements.update({
"pol{:04}".format(int(asimov_polarisation * 1000)):
asimov_polarisation
})
values_tree_files = {}
if ("channel" in args.combinations) or ("category"
in args.combinations):
datacards.annotate_trees(
datacards_workspaces, "higgsCombine*.*.mH*.root", ([[
os.path.join(
os.path.dirname(
template.replace("${CHANNEL}", "(.*)").replace(
def main():
parser = argparse.ArgumentParser(
description=
"Collect matching trees from input files into one output tree",
parents=[logger.loggingParser])
parser.add_argument(
"-i",
"--input-dirs",
help=
"Input directories = crab project directories containing the subdirectories with crab tasks",
nargs="+")
parser.add_argument(
"-o",
"--output-dir",
default=None,
help=
"Local output directory. [Default: subdir \"results\" in first input directory]"
)
parser.add_argument(
"-d",
"--dcache-target",
default=None,
help=
"Directory on dCache (srm) where the files should be copied to. [Default: %(default)s]"
)
parser.add_argument(
"--input-trees",
nargs="+",
default=["svfitCache"],
help="Paths of input SVfit cache trees. [Default: %(default)s]")
parser.add_argument(
"--output-tree",
default="svfitCache",
help="Name of output SVfit cache tree. [Default: %(default)s]")
parser.add_argument(
"--previous-cache",
default="",
help=
"Path to a previous cache which will be merged. [Default: %(default)s]"
)
parser.add_argument(
"--dcache",
type=bool,
default=False,
help="Read&Write from and to desy dcache[Default: %(default)s]")
parser.add_argument(
"--no-run",
default=False,
action="store_true",
help="Do not run but only print dict [Default: %(default)s]")
parser.add_argument(
"-n",
"--n-processes",
type=int,
default=1,
help="Number of (parallel) processes. [Default: %(default)s]")
args = parser.parse_args()
logger.initLogger(args)
if args.output_dir is None:
args.output_dir = os.path.join(args.input_dirs[0], "results")
# get paths to crab outputs
#max_n_jobs = 8000
#max_n_retrieve = 500
get_crab_outputs_args = []
for input_dir in args.input_dirs:
#for jobid_start in xrange(1, max_n_jobs, max_n_retrieve):
# jobid_end = jobid_start + max_n_retrieve - 1
# get_crab_outputs_args.append([input_dir, "{jobid_start}-{jobid_end}".format(jobid_start=jobid_start, jobid_end=jobid_end)])
get_crab_outputs_args.append([input_dir, "1-10"])
tar_files = tools.parallelize(_get_crab_outputs,
get_crab_outputs_args,
max(args.n_processes, 2),
description="crab getoutput --dump")
tar_files = tools.flattenList(tar_files)
# download and un-tar
download_untar_args = [[tar_file, args.output_dir]
for tar_file in tar_files]
tools.parallelize(_download_untar,
download_untar_args,
args.n_processes,
description="download and un-tar crab outputs")
root_files = glob.glob(os.path.join(args.output_dir, "*.root"))
# TODO: maybe add more root files from -i arguments, that did not need to be un-tared
root_files_per_sample_nick = {}
for root_file in root_files:
basename = os.path.basename(root_file)
sample_nick = basename[:basename.index("_job_")]
root_files_per_sample_nick.setdefault(sample_nick,
[]).append(root_file)
merged_output_dir = os.path.join(args.output_dir, "merged")
if not os.path.exists(merged_output_dir):
os.makedirs(merged_output_dir)
merge_outputs_args = [[
os.path.join(merged_output_dir, sample_nick + ".root"), tmp_root_files,
"-f"
] for sample_nick, tmp_root_files in
root_files_per_sample_nick.iteritems()]
tools.parallelize(_merge_outputs,
merge_outputs_args,
args.n_processes,
description="merging")
if args.dcache_target:
dcache_copy_commands = [
"gfal-copy -v -f -r " + merged_output_dir + " " +
args.dcache_target
]
tools.parallelize(_call_command,
dcache_copy_commands,
args.n_processes,
description="copying to dCache")
rm_commands = ["rm " + root_file for root_file in root_files]
if args.dcache_target:
rm_commands.extend([
"rm " + os.path.join(merged_output_dir, sample_nick + ".root")
for sample_nick in root_files_per_sample_nick.keys()
])
tools.parallelize(_call_command,
rm_commands,
args.n_processes,
description="deleting temporary files")
log.info("\nJSON configuration for Artus:\n")
config_output_dir = args.dcache_target if args.dcache_target else merged_output_dir
for src, dst in filename_replacements.iteritems():
config_output_dir = config_output_dir.replace(src, dst)
for sample_nick in sorted(root_files_per_sample_nick.keys()):
log.info("\"" + sample_nick + "\" : \"" +
os.path.join(config_output_dir, sample_nick + ".root") +
"\",")
split_stat_syst_uncs=True,
additional_freeze_nuisances=["r"])
annotation_replacements = {
channel: index
for (index, channel) in enumerate(["combined"] + args.channel)
}
annotation_replacements.update({
binid: index + 1
for (index, binid) in enumerate(
sorted(
list(
set([
datacards.configs.category2binid(category,
channel=category[:2])
for category in tools.flattenList(args.categories)
]))))
})
values_tree_files = {}
if ("channel" in args.combinations) or ("category" in args.combinations):
datacards.annotate_trees(
datacards_workspaces, "higgsCombine*.*.mH*.root", ([[
os.path.join(
os.path.dirname(
template.replace("${CHANNEL}", "(.*)").replace(
"${MASS}", "\d*")), ".*.root")
for template in datacard_filename_templates
if "channel" in template
][0]] if "channel" in args.combinations else []) + ([[
os.path.join(
os.path.dirname(
config["labels"][index] = os.path.join(
"%s_%s" %
(channel_renamings.get(channel, channel),
category_renamings.get(category, category)),
label_renamings.get(label, label))
config = uncertainty(config, name).get_config(shift)
if "PrintInfos" in config.get("analysis_modules", []):
config.get("analysis_modules", []).remove("PrintInfos")
harry_configs.append(config)
harry_args.append("-d %s --formats png pdf %s" %
(args["input_dir"], args["args"]))
higgs_plotter = higgsplot.HiggsPlotter(list_of_config_dicts=harry_configs,
list_of_args_strings=harry_args,
n_processes=args["n_processes"],
n_plots=args["n_plots"])
root_outputs = list(
set([
output
for output in tools.flattenList(higgs_plotter.output_filenames)
if output.endswith(".root")
]))
command = "hadd -f %s %s" % (args["root_output"], " ".join(root_outputs))
log.info(command)
logger.subprocessCall(shlex.split(command))
log.info("Merged ROOT output is saved to \"%s\"." % args["root_output"])
for quantity in args["quantities"]:
config["x_expressions"] = quantity
config["x_bins"] = [channel+"_"+quantity]
config["x_label"] = channel+"_"+quantity
if not category is None:
config["output_dir"] = os.path.join(config.setdefault("output_dir", "plots"), category)
for index, label in enumerate(config.setdefault("labels", [])):
config["labels"][index] = os.path.join("%s_%s" % (channel_renamings.get(channel, channel),
category_renamings.get(category, category)),
label_renamings.get(label, label))
config = uncertainty(config, name).get_config(shift)
if "PrintInfos" in config.get("analysis_modules", []):
config.get("analysis_modules", []).remove("PrintInfos")
harry_configs.append(config)
harry_args.append("-d %s --formats png pdf %s" % (args["input_dir"], args["args"]))
higgs_plotter = higgsplot.HiggsPlotter(list_of_config_dicts=harry_configs, list_of_args_strings=harry_args, n_processes=args["n_processes"], n_plots=args["n_plots"])
root_outputs = list(set([output for output in tools.flattenList(higgs_plotter.output_filenames) if output.endswith(".root")]))
command = "hadd -f %s %s" % (args["root_output"], " ".join(root_outputs))
log.info(command)
logger.subprocessCall(shlex.split(command))
log.info("Merged ROOT output is saved to \"%s\"." % args["root_output"])
print("###################### annotations ######################")
if "totstatuncs" in args.steps: # (scaled_lumi is None) and (asimov_polarisation is None):
datacards.combine(
datacards_cbs,
datacards_workspaces,
None,
args.n_processes,
"-M MultiDimFit --algo singles -P pol --redefineSignalPOIs pol "+datacards.stable_options+" -n",
split_stat_syst_uncs=True,
additional_freeze_nuisances=["r"]
)
print("###################### annotations ######################")
annotation_replacements = {channel : index for (index, channel) in enumerate(["combined"] + args.channel)}
annotation_replacements.update({binid : index+1 for (index, binid) in enumerate(sorted(list(set([datacards.configs.category2binid(category, channel=category[:2]) for category in tools.flattenList(args.categories)]))))})
if not asimov_polarisation is None:
annotation_replacements.update({"pol{:04}".format(int(asimov_polarisation*1000)) : asimov_polarisation})
values_tree_files = {}
if ("channel" in args.combinations) or ("category" in args.combinations):
datacards.annotate_trees(
datacards_workspaces,
"higgsCombine*.*.mH*.root",
([[os.path.join(os.path.dirname(template.replace("${CHANNEL}", "(.*)").replace("${MASS}", "\d*")), ".*.root") for template in datacard_filename_templates if "channel" in template][0]] if "channel" in args.combinations else [])+
([[os.path.join(os.path.dirname(template.replace("${BINID}", "(\d*)").replace("${MASS}", "\d*")), ".*.root") for template in datacard_filename_templates if "category" in template][0]] if "category" in args.combinations else [])+
([os.path.join("/(pol-?\d*)", ".*.root")] if not asimov_polarisation is None else []),
annotation_replacements,
args.n_processes,
values_tree_files,
"-t limit -b" + (" channel" if "channel" in args.combinations else "") + (" category" if "category" in args.combinations else "") + (" polarisation" if not asimov_polarisation is None else "")
)
"-M MultiDimFit --algo grid --points 200 -P pol --redefineSignalPOIs pol "+datacards.stable_options+" -n Scan "
)
if "totstatuncs" in args.steps: # (scaled_lumi is None) and (asimov_polarisation is None):
datacards.combine(
datacards_cbs,
datacards_workspaces,
None,
args.n_processes,
"-M MultiDimFit --algo singles -P pol --redefineSignalPOIs pol "+datacards.stable_options+" -n ",
split_stat_syst_uncs=True,
additional_freeze_nuisances=["r"]
)
annotation_replacements = {channel : index for (index, channel) in enumerate(["combined"] + args.channel)}
annotation_replacements.update({binid : index+1 for (index, binid) in enumerate(sorted(list(set([datacards.configs.category2binid(category, channel=category[:2]) for category in tools.flattenList(args.categories)]))))})
if not asimov_polarisation is None:
annotation_replacements.update({"pol{:04}".format(int(asimov_polarisation*1000)) : asimov_polarisation})
values_tree_files = {}
if ("channel" in args.combinations) or ("category" in args.combinations):
datacards.annotate_trees(
datacards_workspaces,
"higgsCombine*.*.mH*.root",
([[os.path.join(os.path.dirname(template.replace("${CHANNEL}", "(.*)").replace("${MASS}", "\d*")), ".*.root") for template in datacard_filename_templates if "channel" in template][0]] if "channel" in args.combinations else [])+
([[os.path.join(os.path.dirname(template.replace("${BINID}", "(\d*)").replace("${MASS}", "\d*")), ".*.root") for template in datacard_filename_templates if "category" in template][0]] if "category" in args.combinations else [])+
([os.path.join("/(pol-?\d*)", ".*.root")] if not asimov_polarisation is None else []),
annotation_replacements,
args.n_processes,
values_tree_files,
"-t limit -b" + (" channel" if "channel" in args.combinations else "") + (" category" if "category" in args.combinations else "") + (" polarisation" if not asimov_polarisation is None else "")
)
def main():
parser = argparse.ArgumentParser(description="Collect matching trees from input files into one output tree",
parents=[logger.loggingParser])
parser.add_argument("-i", "--input-dirs", help="Input directories = crab project directories containing the subdirectories with crab tasks", nargs="+")
parser.add_argument("-o", "--output-dir", default=None,
help="Local output directory. [Default: subdir \"results\" in first input directory]")
parser.add_argument("-d", "--dcache-target", default=None,
help="Directory on dCache (srm) where the files should be copied to. [Default: %(default)s]")
parser.add_argument("--input-trees", nargs="+", default=["svfitCache"],
help="Paths of input SVfit cache trees. [Default: %(default)s]")
parser.add_argument("--output-tree", default="svfitCache",
help="Name of output SVfit cache tree. [Default: %(default)s]")
parser.add_argument("--previous-cache", default="",
help="Path to a previous cache which will be merged. [Default: %(default)s]")
parser.add_argument("--dcache", type=bool, default=False,
help="Read&Write from and to desy dcache[Default: %(default)s]")
parser.add_argument("--no-run", default=False, action="store_true",
help="Do not run but only print dict [Default: %(default)s]")
parser.add_argument("-n", "--n-processes", type=int, default=1,
help="Number of (parallel) processes. [Default: %(default)s]")
args = parser.parse_args()
logger.initLogger(args)
if args.output_dir is None:
args.output_dir = os.path.join(args.input_dirs[0], "results")
# get paths to crab outputs
#max_n_jobs = 8000
#max_n_retrieve = 500
get_crab_outputs_args = []
for input_dir in args.input_dirs:
#for jobid_start in xrange(1, max_n_jobs, max_n_retrieve):
# jobid_end = jobid_start + max_n_retrieve - 1
# get_crab_outputs_args.append([input_dir, "{jobid_start}-{jobid_end}".format(jobid_start=jobid_start, jobid_end=jobid_end)])
get_crab_outputs_args.append([input_dir, "1-10"])
tar_files = tools.parallelize(_get_crab_outputs, get_crab_outputs_args, max(args.n_processes, 2), description="crab getoutput --dump")
tar_files = tools.flattenList(tar_files)
# download and un-tar
download_untar_args = [[tar_file, args.output_dir] for tar_file in tar_files]
tools.parallelize(_download_untar, download_untar_args, args.n_processes, description="download and un-tar crab outputs")
root_files = glob.glob(os.path.join(args.output_dir, "*.root"))
# TODO: maybe add more root files from -i arguments, that did not need to be un-tared
root_files_per_sample_nick = {}
for root_file in root_files:
basename = os.path.basename(root_file)
sample_nick = basename[:basename.index("_job_")]
root_files_per_sample_nick.setdefault(sample_nick, []).append(root_file)
merged_output_dir = os.path.join(args.output_dir, "merged")
if not os.path.exists(merged_output_dir):
os.makedirs(merged_output_dir)
merge_outputs_args = [[os.path.join(merged_output_dir, sample_nick+".root"), tmp_root_files, "-f"] for sample_nick, tmp_root_files in root_files_per_sample_nick.iteritems()]
tools.parallelize(_merge_outputs, merge_outputs_args, args.n_processes, description="merging")
if args.dcache_target:
dcache_copy_commands = ["gfal-copy -v -f -r "+merged_output_dir+" "+args.dcache_target]
tools.parallelize(_call_command, dcache_copy_commands, args.n_processes, description="copying to dCache")
rm_commands = ["rm "+root_file for root_file in root_files]
if args.dcache_target:
rm_commands.extend(["rm "+os.path.join(merged_output_dir, sample_nick+".root") for sample_nick in root_files_per_sample_nick.keys()])
tools.parallelize(_call_command, rm_commands, args.n_processes, description="deleting temporary files")
log.info("\nJSON configuration for Artus:\n")
config_output_dir = args.dcache_target if args.dcache_target else merged_output_dir
for src, dst in filename_replacements.iteritems():
config_output_dir = config_output_dir.replace(src, dst)
for sample_nick in sorted(root_files_per_sample_nick.keys()):
log.info("\""+sample_nick+"\" : \""+os.path.join(config_output_dir, sample_nick+".root")+"\",")
