def main():
parser = argparse.ArgumentParser(
description="Merge Artus outputs per nick name.",
parents=[logger.loggingParser])
parser.add_argument(
"project_dir",
help=
"Artus Project directory containing the files \"output/*/*.root\" to merge"
)
parser.add_argument("-b",
"--batch",
default="host",
help="Select backend. [Default: %(default)s]")
args = parser.parse_args()
logger.initLogger(args)
gc_work_directory = os.path.join(args.project_dir, "workdir_merge")
merged_directory = os.path.join(args.project_dir, "merged")
#remove directories that will be overwritten
directories_to_remove = []
for directory in [gc_work_directory, merged_directory]:
if os.path.exists(directory):
directories_to_remove.append(directory)
logger.subprocessCall(
("rm -rf " + (" ".join(directories_to_remove))).split())
gc_config_base_filename = os.path.expandvars(
"$CMSSW_BASE/src/Artus/Configuration/data/grid-control_merge_artus_outputs_base.conf"
)
gc_config_base = ""
with open(gc_config_base_filename) as gc_config_base_file:
gc_config_base = gc_config_base_file.read().rstrip()
backend = open(
os.path.expandvars(
"$CMSSW_BASE/src/Artus/Configuration/data/grid-control_backend_" +
args.batch + ".conf"), 'r').read()
gc_config = string.Template(gc_config_base).safe_substitute(
BACKEND=backend,
PROJECT_DIR=args.project_dir,
CMSSW_BASE=os.path.expandvars("$CMSSW_BASE"),
SCRAM_ARCH=os.path.expandvars("$SCRAM_ARCH"),
)
gc_config_filename = os.path.join(args.project_dir,
"grid-control_merge_artus_outputs.conf")
if os.path.exists(gc_config_filename):
os.remove(gc_config_filename)
with open(gc_config_filename, "w") as gc_config_file:
gc_config_file.write(gc_config)
command = "go.py " + gc_config_filename
log.info(command)
logger.subprocessCall(command.split())
log.info("Output is written to directory \"%s\"" % merged_directory)
def main():
global opt_all
parser = argparse.ArgumentParser(description="ROOT file compare tool.",
parents=[logger.loggingParser])
parser.add_argument("files", nargs=2,
help="Two ROOT files to compare.")
parser.add_argument("-a", "--all-differences", default=False, action="store_true",
help="Check for all differences instead of stopping after the first difference. [Default: %(default)s]")
args = parser.parse_args()
logger.initLogger(args)
inputFile1 = ROOT.TFile(args.files[0])
inputFile2 = ROOT.TFile(args.files[1])
opt_all = args.all_differences
files_equal = compareTree(inputFile1, inputFile2)
inputFile1.Close()
inputFile2.Close()
if files_equal:
log.info("The files \"" + args.files[0] + "\" and \"" + args.files[1] + "\" are identical.")
exit(0)
else:
log.warning("The files \"" + args.files[0] + "\" and \"" + args.files[1] + "\" are NOT identical.")
if relative_differences_per_leaf != {}:
for key in relative_differences_per_leaf.keys():
log.warning("Average difference for {0}: {1:.2f}%".format(key, sum(relative_differences_per_leaf[key]) / len(relative_differences_per_leaf[key]) * 100.))
exit(1)
def main():
parser = argparse.ArgumentParser(description="Merge Artus outputs per nick name.", parents=[logger.loggingParser])
parser.add_argument("project_dir", help="Artus Project directory containing the files \"output/*/*.root\" to merge")
args = parser.parse_args()
logger.initLogger(args)
output_dirs = glob.glob(os.path.join(args.project_dir, "output/*"))
nick_names = [nick for nick in [output_dir[output_dir.rfind("/")+1:] for output_dir in output_dirs] if not ".tar.gz" in nick]
gc_config_base_filename = os.path.expandvars("$CMSSW_BASE/src/Artus/Configuration/data/gc_merge_artus_outputs_base.conf")
gc_config_base = ""
with open(gc_config_base_filename) as gc_config_base_file:
gc_config_base = gc_config_base_file.read().rstrip()
gc_config = string.Template(gc_config_base).safe_substitute(
PROJECT_DIR=args.project_dir,
SAMPLE_NICKS=" ".join(nick_names),
CMSSW_BASE=os.path.expandvars("$CMSSW_BASE")
)
gc_config_filename = os.path.join(args.project_dir, "gc_merge_artus_outputs.conf")
if os.path.exists(gc_config_filename):
os.remove(gc_config_filename)
logger.subprocessCall(("rm -rf " + os.path.join(args.project_dir, "workdir_merge")).split())
with open(gc_config_filename, "w") as gc_config_file:
gc_config_file.write(gc_config)
command = "go.py -Gcm 1 " + gc_config_filename
log.info(command)
logger.subprocessCall(command.split())
log.info("Output is written to directory \"%s\"" % os.path.join(args.project_dir, "merged"))
def main():
parser = argparse.ArgumentParser(description="Wrapper for hadd avoiding too many bash arguments.", parents=[logger.loggingParser])
parser.add_argument("-t", "--target-file", required=True,
help="Target file.")
parser.add_argument("source_files", nargs="+",
help="Source file. Can be either separate files or lists of files separated by whitespaces in one or more arguments.")
parser.add_argument("-a", "--args", default="",
help="Options for hadd. [Default: %(default)s]")
parser.add_argument("-n", "--max-files", default=500, type=int,
help="Maximum number of source files use per hadd call. [Default: %(default)s]")
args = parser.parse_args()
logger.initLogger(args)
source_files = []
for arg in args.source_files:
for item in shlex.split(arg.replace("\"", "")):
matching_files = glob.glob(item)
if len(matching_files) > 0:
source_files.extend(matching_files)
else:
source_files.append(item)
sys.exit(hadd(target_file=args.target_file,
source_files=source_files,
hadd_args=args.args,
max_files=args.max_files))
def main():
parser = argparse.ArgumentParser(description="Collect outputs of failed GC jobs.", parents=[logger.loggingParser])
parser.add_argument("INPUT_DIR",
help="Project directory of Artus GC run")
parser.add_argument("-o", "--output-dir",
help="Directory where to put symlinks to failed job outputs. [Default: INPUT_DIR/failed]")
args = parser.parse_args()
logger.initLogger(args)
input_dir = args.INPUT_DIR
if args.output_dir == None:
args.output_dir = os.path.join(input_dir, "failed")
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
job_info_file_names = glob.glob(os.path.join(input_dir, "workdir/output/job_*/job.info"))
job_infos = {}
for job_info_file_name in job_info_file_names:
with open(job_info_file_name) as job_info_file:
job_info_content = re.search("JOBID=(?P<job_id>\d*).*EXITCODE=(?P<exit_code>\d*)",
job_info_file.read().replace("\n", "")).groupdict()
job_infos[int(job_info_content["job_id"])] = int(job_info_content["exit_code"])
job_infos = {job_id : exit_code for job_id, exit_code in job_infos.iteritems() if exit_code != 0}
for job_id, exit_code in job_infos.iteritems():
output_dir = os.path.join(args.output_dir, "job_id_%d_exit_code_%d" % (job_id, exit_code))
if not os.path.exists(output_dir):
os.makedirs(output_dir)
os.system("ln -vs %s %s" % (os.path.join(input_dir, "workdir/output/job_%d" % job_id),
os.path.join(output_dir, "gc_job_%d" % job_id)))
os.system("ln -vs %s %s" % (os.path.join(input_dir, "output/*/*_%d_*" % job_id), output_dir))
def __init__(self, executable=None, userArgParsers=None):
self._config = jsonTools.JsonDict()
self._executable = executable
self._parser = None
#Load default argument parser
self._initArgumentParser(userArgParsers)
#Parse command line arguments and return dict
self._args = self._parser.parse_args()
logger.initLogger(self._args)
# expand the environment variables only at the batch node
if self._args.batch:
self._args.envvar_expansion = False
date_now = datetime.now().strftime("%Y-%m-%d_%H-%M")
# write repository revisions to the config
if not self._args.disable_repo_versions:
self.setRepositoryRevisions()
self._config["Date"] = date_now
#Expand Config
self.expandConfig()
self.projectPath = None
if self._args.batch:
self.projectPath = os.path.join(os.path.expandvars(self._args.work), date_now+"_"+self._args.project_name)
def __init__(self, executable=None, userArgParsers=None):
self._config = jsonTools.JsonDict()
self._executable = executable
self._parser = None
#Load default argument parser
self._initArgumentParser(userArgParsers)
#Parse command line arguments and return dict
self._args = self._parser.parse_args()
logger.initLogger(self._args)
# expand the environment variables only at the batch node
if self._args.batch:
self._args.envvar_expansion = False
# write repository revisions to the config
if not self._args.disable_repo_versions:
self.setRepositoryRevisions()
self._config["Date"] = self._args.date
#Expand Config
self.expandConfig()
self.projectPath = None
self.localProjectPath = None
self.remote_se = False
if self._args.batch:
self.projectPath = os.path.join(os.path.expandvars(self._args.work), self._args.date+"_"+self._args.project_name)
self.localProjectPath = self.projectPath
if self.projectPath.startswith("srm://"):
self.remote_se = True
self.localProjectPath = os.path.join(os.path.expandvars(self._parser.get_default("work")), self._args.date+"_"+self._args.project_name)
def main():
parser = argparse.ArgumentParser(description="Print out the sum of entries in the Event trees of all inputs sorted by nick names.",
parents=[logger.loggingParser])
parser.add_argument("files", nargs="+", help="Input files.")
parser.add_argument("-n", "--nick", default="kappa_(?P<nick>.*)_\d+.root",
help="Regular expression to extract nickname from file names. Use 'NONE' to disable nick name matching. [Default: %(default)s]")
args = parser.parse_args()
logger.initLogger(args)
n_entries_per_nick = {}
for file_name in args.files:
# retrieve number of entries
root_file = ROOT.TFile(file_name, "READ")
events = root_file.Get("Events")
n_entries = events.GetEntries()
root_file.Close()
# nickname matching and sum of entries
no_regex_match = True if args.nick=='NONE' else False
nick = ( re.match(args.nick, os.path.basename(file_name)).groupdict().values()[0] if not no_regex_match else '*')
n_entries_per_nick[nick] = n_entries_per_nick.get(nick, 0.0) + n_entries
# print results
log.info("{\n\t\"NumberGeneratedEvents\" : {\n\t\t\"nick\" : {")
for index, (nick, n_entries) in enumerate(n_entries_per_nick.items()):
log.info("\t\t\t\"" + nick + "\" : " + str(int(n_entries)) + ("," if index < len(n_entries_per_nick)-1 else ""))
log.info("\t\t}\n\t}\n}")
def main():
ROOT.gSystem.Load(os.path.expandvars("$CMSSW_BASE/src/Kappa/lib/libKappa"))
parser = argparse.ArgumentParser(
description="Print out the available jet taggers in a kappa skim.",
parents=[logger.loggingParser])
parser.add_argument(
"file",
help="Kappa skim output file containing the tagged jet meta data")
args = parser.parse_args()
logger.initLogger(args)
root_file = ROOT.TFile(args.file, "READ")
lumis = root_file.Get("Lumis")
lumis.GetEntry(0)
log.info("\nNames of available jet taggers:")
tagNames = lumis.jetMetadata.tagNames
for tagName in tagNames:
log.info("\t" + tagName)
log.info("\nNames of available jet IDs:")
idNames = lumis.jetMetadata.idNames
for idName in idNames:
log.info("\t" + idName)
def main():
parser = argparse.ArgumentParser(description="Collect outputs of running GC jobs.", parents=[logger.loggingParser])
parser.add_argument("INPUT_DIR",
help="Project directory of Artus GC run")
parser.add_argument("-o", "--output-dir",
help="Directory where to put symlinks to failed job outputs. [Default: INPUT_DIR/failed]")
args = parser.parse_args()
logger.initLogger(args)
input_dir = args.INPUT_DIR
if args.output_dir == None:
args.output_dir = os.path.join(input_dir, "running")
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
job_info_file_names = glob.glob(os.path.join(input_dir, "workdir/jobs/job_*.txt"))
for job_info_file_name in job_info_file_names:
job_id = re.search(os.path.join(input_dir, "workdir/jobs/job_(?P<job_id>\d*).txt"), job_info_file_name).groupdict()["job_id"]
if not os.path.exists(os.path.join(input_dir, "workdir/output/job_"+job_id+"/job.info")):
output_dir = os.path.join(args.output_dir, "job_id_"+job_id)
if not os.path.exists(output_dir):
os.makedirs(output_dir)
os.system("ln -vs %s %s" % (os.path.join(input_dir, "workdir/jobs/job_"+job_id+".txt"), output_dir))
os.system("ln -vs %s %s" % (os.path.join(input_dir, "workdir/jobs/job_"+job_id+".var"), output_dir))
def main():
ROOT.gROOT.SetBatch(True)
ROOT.gSystem.Load(os.path.expandvars("$CMSSW_BASE/src/Kappa/lib/libKappa"))
parser = argparse.ArgumentParser(
description="Searcg for Kappa files containing certain events.",
parents=[logger.loggingParser])
parser.add_argument("files",
nargs="+",
help="Kappa skim output files to check")
parser.add_argument(
"-c",
"--config",
help="JSON config file containing the event selection.")
args = parser.parse_args()
logger.initLogger(args)
config = jsonTools.JsonDict(args.config)
run_whitelist = config.get("RunWhitelist", [])
lumi_whitelist = config.get("LumiWhitelist", [])
event_whitelist = config.get("EventWhitelist", [])
run_blacklist = config.get("RunBlacklist", [])
lumi_blacklist = config.get("LumiBlacklist", [])
event_blacklist = config.get("EventBlacklist", [])
selections = []
if len(run_whitelist) > 0:
selections.append("||".join(
["(KEventMetadata.nRun==%d)" % run for run in run_whitelist]))
if len(lumi_whitelist) > 0:
selections.append("||".join(
["(KEventMetadata.nLumi==%d)" % lumi for lumi in lumi_whitelist]))
if len(event_whitelist) > 0:
selections.append("||".join([
"(KEventMetadata.nEvent==%d)" % event for event in event_whitelist
]))
if len(run_blacklist) > 0:
selections.append("&&".join(
["(KEventMetadata.nRun!=%d)" % run for run in run_blacklist]))
if len(lumi_blacklist) > 0:
selections.append("&&".join(
["(KEventMetadata.nLumi!=%d)" % lumi for lumi in lumi_blacklist]))
if len(event_blacklist) > 0:
selections.append("&&".join([
"(KEventMetadata.nEvent!=%d)" % event for event in event_blacklist
]))
selection = "&&".join(["(%s)" % cut for cut in selections])
for input_file in args.files:
tree = ROOT.TChain("Events")
tree.AddFile(input_file)
entries = tree.Draw("1", selection, "goff")
log.info("%4d entries: %s" % (entries, input_file))
def main():
parser = argparse.ArgumentParser(description="Tools simplifying dCache usage.", parents=[logger.loggingParser])
parser.add_argument("-c", "--command", required=True, help="Command, e.g. lcg-cp, lcg-del or ddcp.")
parser.add_argument("-a", "--args", help="Arguments (can be left empty).")
parser.add_argument("-s", "--src", required=True, help="Source.")
parser.add_argument("--src-prefix", default="", help="Source prefix. \"gridka\" and \"desy\" are replaced by their dCache locations. \"\" means local path.")
#parser.add_argument("-d", "--dst", help="Destination (can be left empty).")
#parser.add_argument("--dst-prefix", default="", help="Destination prefix. \"gridka\" and \"desy\" are replaced by their dCache locations. \"\" means local path.")
args = parser.parse_args()
logger.initLogger(args)
prefix_replacements = {
"gridka" : "srm://dgridsrm-fzk.gridka.de:8443/srm/managerv2?SFN=",
"desy" : "srm://dcache-se-cms.desy.de:8443/srm/managerv2?SFN=",
}
for replacement_from, replacement_to in prefix_replacements.items():
if args.src_prefix == replacement_from:
args.src_prefix = replacement_to
#if args.dst_prefix == replacement_from:
# args.dst_prefix = replacement_to
src_files = list_of_files(args.src_prefix, args.src)
for src_file in src_files:
logger.subprocessCall((args.command + " " + (args.args if args.args else "") + " " + args.src_prefix + src_file).split())
def main():
parser = argparse.ArgumentParser(description="Compare different repository versions configured in Artus configs. The script has to be executed in the directory of the repository.", parents=[logger.loggingParser])
parser.add_argument("files", help="Two configurations. The configs should be ordered by the expected repository version. Can be either Artus output root files or JSON text files", nargs=2)
args = parser.parse_args()
logger.initLogger(args)
config1 = jsonTools.JsonDict(args.files[0])
config2 = jsonTools.JsonDict(args.files[1])
dirname = os.path.basename(os.getcwd())
repo_key1 = sorted([key for key in config1.keys() if key.startswith("/") and key.endswith(dirname)], key=lambda item: len(item))[-1]
repo_key2 = sorted([key for key in config2.keys() if key.startswith("/") and key.endswith(dirname)], key=lambda item: len(item))[-1]
repo_version1 = config1[repo_key1]
repo_version2 = config2[repo_key2]
diff_string = "%s...%s" % (repo_version1, repo_version2)
command = "git diff %s..%s" % (repo_version1, repo_version2)
if log.isEnabledFor(logging.DEBUG):
log.info("")
logger.subprocessCall(shlex.split(command))
log.info("\n"+command)
popen_cout, popen_cerr = subprocess.Popen("git config remote.origin.url".split(), stdout=subprocess.PIPE).communicate()
remote_url = popen_cout.replace("\n", "")
github_link = os.path.join(remote_url, "compare", diff_string)
log.info(github_link)
def main():
ROOT.gROOT.SetBatch(True)
parser = argparse.ArgumentParser(description="Merge Artus outputs per nick name.", parents=[logger.loggingParser])
parser.add_argument("project_dir", help="Artus Project directory containing the files \"output/*/*.root\" to merge")
args = parser.parse_args()
logger.initLogger(args)
output_dirs = glob.glob(os.path.join(args.project_dir, "output/*"))
nick_names = [nick for nick in [output_dir[output_dir.rfind("/")+1:] for output_dir in output_dirs] if not ".tar.gz" in nick]
outputs_per_nick = {nick : glob.glob(os.path.join(args.project_dir, "output", nick, "*.root")) for nick in nick_names}
outputs_per_nick = {nick : files for nick, files in outputs_per_nick.iteritems() if len(files) > 0}
for nick_name, output_files in pi.ProgressIterator(outputs_per_nick.iteritems(),
length=len(outputs_per_nick),
description="Merging Artus outputs"):
merged_dir = os.path.join(args.project_dir, "merged", nick_name)
if not os.path.exists(merged_dir):
os.makedirs(merged_dir)
command = "hadd -f %s %s" % (os.path.join(merged_dir, nick_name+".root"), " ".join(output_files))
log.info(command)
logger.subprocessCall(command.split())
log.info("\n")
def main():
parser = argparse.ArgumentParser(description="Print qstat summary table.", parents=[logger.loggingParser])
parser.add_argument("-s", "--sort-state", default="r",
help="State to be used for sorting. [Default: %(default)s]")
parser.add_argument("--norm-total-jobs", default=False, action="store_true",
help="Norm progress bar to total number of jobs in batch system. [Default: Norm to user with most job in system.]")
parser.add_argument("-S", "--simple-mode", default=False, action="store_true",
help="Simple mode. Only display all and running jobs.")
parser.add_argument("-c", "--continuous-mode", default=0, type=int,
help="Continuous mode. Argument is refresh interval. Default is non-continuous mode.")
args = parser.parse_args()
logger.initLogger(args)
users = []
while True:
try:
users = print_qstat_table(args, users)
if args.continuous_mode == 0:
break
time.sleep(args.continuous_mode)
except:
sys.stdout.write("\033[?1049l") # switch back to default buffer
sys.stdout.flush()
sys.exit(0)
def main():
parser = argparse.ArgumentParser(description="Read text files and convert dCache paths.", parents=[logger.loggingParser])
parser.add_argument("input_files", nargs="+",
help="Input text files.")
parser.add_argument("-m", "--mode", default="xrd2local",
choices=["dcap2local", "dcap2srm", "dcap2xrd",
"local2xrd", "local2dcap", "local2srm",
"srm2dcap", "srm2local", "srm2xrd",
"xrd2dcap", "xrd2local", "xrd2srm"],
help="Conversion mode. [Default: %(default)s]")
parser.add_argument("-i", "--in-place", default=False, action="store_true",
help="Convert paths in place modifying the input files. [Default: Only print converted files]")
args = parser.parse_args()
logger.initLogger(args)
for input_filename in args.input_files:
with open(input_filename, "rw" if args.in_place else "r") as input_file:
content = input_file.read()
content = getattr(dcachetools, args.mode)(content)
if args.in_place:
input_file.write(content)
else:
log.info(content)
def get_binning_with_equal_entries():
"""Get binning with roughly equal number of entries per bin
Example usage: get_binning_with_equal_entries.py artus_output.root folder/ntuple quantityname --n-bins 10
"""
parser = argparse.ArgumentParser(description="Print binning of histograms", parents=[logger.loggingParser])
parser.add_argument("root_file", help="Input ROOT file")
parser.add_argument("folder", help="Folder to TTree")
parser.add_argument("quantity", help="Quantity for which binning should be determined")
parser.add_argument("--n-bins", type=int, help="Number of bins. [Default: %(default)s]", default=25)
parser.add_argument("--selection", type=str, help="Optional selection", default="1")
args = parser.parse_args()
logger.initLogger(args)
with TFileContextManager(args.root_file, "READ") as root_file:
tree = root_file.Get(args.folder)
events_list = sorted([e[0] for e in eventselectionoverlap.EventSelectionOverlap.get_events_list_from_tree(tree, args.quantity, args.selection)])
assert(len(events_list) > args.n_bins)
# slice events list
bin_border_list = []
for i_bin in range(args.n_bins):
bin_border_list.append(events_list[int(round(i_bin/float(args.n_bins)*len(events_list)))])
bin_border_list.append(events_list[-1])
log.info("Bin borders for {} bins:".format(args.n_bins))
log.info(" ".join([str(i) for i in bin_border_list]))
def main():
parser = argparse.ArgumentParser(
description="Download remote file with timeout.",
parents=[logger.loggingParser])
parser.add_argument("remote", help="Remote source file.")
parser.add_argument("local", help="Local destimation file.")
parser.add_argument(
"--offset",
type=int,
default=30,
help="Wait for at most n seconds (default: %(default)s)")
parser.add_argument(
"--bandwidth",
type=int,
default=100,
help="Require at least n kB/s bandwidth (default: %(default)s)")
args = parser.parse_args()
logger.initLogger(args)
result = tools.download_remote_file(remote=args.remote,
local=args.local,
offset=args.offset,
bandwidth=args.bandwidth)
if not result:
sys.exit(1)
def main():
parser = argparse.ArgumentParser(
description="Profile C++ executable with arguments.",
parents=[logger.loggingParser])
parser.add_argument("command", nargs="+", help="Command to be profiled.")
parser.add_argument("-p",
"--profiler",
default="igprof",
choices=["igprof", "valgrind"],
help="Profiling tool. [Default: %(default)s]")
runningOptionsGroup.add_argument(
"--profile-options",
default="pp",
help=
"Additional options for profiling. Choose memory (mp) or performance (pp). [Default: %(default)s]"
)
parser.add_argument("-o",
"--output-dir",
default=None,
help="Output directory. [Default: tmp. directory]")
args = parser.parse_args()
logger.initLogger(args)
profile_cpp.profile_cpp(command=args.command,
profiler=args.profiler,
profiler_opt=args.profile_options,
output_dir=args.output_dir)
def main():
parser = argparse.ArgumentParser(
description="Run multiple commands in parallel.",
parents=[logger.loggingParser])
parser.add_argument(
"commands",
help=
"Commands to be executed on a batch system. They can also be piped into this program.",
nargs="*",
default=[])
parser.add_argument(
"-n",
"--n-processes",
type=int,
default=1,
help="Number of (parallel) processes. [Default: %(default)s]")
args = parser.parse_args()
logger.initLogger(args)
# prepare commands
if (len(args.commands) == 0) and (not sys.stdin.isatty()):
args.commands.extend(sys.stdin.read().strip().split("\n"))
tools.parallelize(run_command,
args.commands,
n_processes=args.n_processes,
description=os.path.basename(sys.argv[0]))
def main():
parser = argparse.ArgumentParser(description="Merge Artus outputs per nick name.", parents=[logger.loggingParser])
parser.add_argument("project_dir", help="Artus Project directory containing the files \"output/*/*.root\" to merge")
parser.add_argument("-n", "--n-processes", type=int, default=1,
help="Number of (parallel) processes. [Default: %(default)s]")
parser.add_argument("--output-dir", help="Directory to store merged files. Default: Same as project_dir.")
args = parser.parse_args()
logger.initLogger(args)
output_dirs = glob.glob(os.path.join(args.project_dir, "output/*"))
nick_names = [nick for nick in [output_dir[output_dir.rfind("/")+1:] for output_dir in output_dirs] if not ".tar.gz" in nick]
outputs_per_nick = {nick : glob.glob(os.path.join(args.project_dir, "output", nick, "*.root")) for nick in nick_names}
outputs_per_nick = {nick : files for nick, files in outputs_per_nick.iteritems() if len(files) > 0}
commands = []
for nick_name, output_files in pi.ProgressIterator(outputs_per_nick.iteritems(),
length=len(outputs_per_nick),
description="Merging Artus outputs"):
merged_dir = os.path.join(args.project_dir if(args.output_dir == None) else args.output_dir, "merged", nick_name)
if not os.path.exists(merged_dir):
os.makedirs(merged_dir)
commands.append("hadd -f %s %s" % (os.path.join(merged_dir, nick_name+".root"), " ".join(output_files)))
tools.parallelize(_call_command, commands, n_processes=args.n_processes)
def main():
parser = argparse.ArgumentParser(
description="Merge Artus outputs per nick name.",
parents=[logger.loggingParser])
parser.add_argument(
"project_dir",
help=
"Artus Project directory containing the files \"output/*/*.root\" to merge"
)
parser.add_argument(
"-n",
"--n-processes",
type=int,
default=1,
help="Number of (parallel) processes. [Default: %(default)s]")
parser.add_argument(
"--output-dir",
help="Directory to store merged files. Default: Same as project_dir.")
args = parser.parse_args()
logger.initLogger(args)
output_dirs = glob.glob(os.path.join(args.project_dir, "output/*"))
nick_names = [
nick for nick in
[output_dir[output_dir.rfind("/") + 1:] for output_dir in output_dirs]
if not ".tar.gz" in nick
]
outputs_per_nick = {
nick:
glob.glob(os.path.join(args.project_dir, "output", nick, "*.root"))
for nick in nick_names
}
# drop potentially existing SvfitCaches from the filelist
for nick, files in outputs_per_nick.iteritems():
outputs_per_nick[nick] = [
file for file in files if ("SvfitCache" not in file)
]
outputs_per_nick = {
nick: files
for nick, files in outputs_per_nick.iteritems() if len(files) > 0
}
commands = []
for nick_name, output_files in pi.ProgressIterator(
outputs_per_nick.iteritems(),
length=len(outputs_per_nick),
description="Merging Artus outputs"):
merged_dir = os.path.join(
args.project_dir if (args.output_dir == None) else args.output_dir,
"merged", nick_name)
if not os.path.exists(merged_dir):
os.makedirs(merged_dir)
commands.append("hadd.py -a \" -f\" -t %s \"%s\"" % (os.path.join(
merged_dir, nick_name + ".root"), " ".join(output_files)))
tools.parallelize(_call_command, commands, n_processes=args.n_processes)
def main():
parser = argparse.ArgumentParser(description="Sort trees.", parents=[logger.loggingParser])
parser.add_argument("inputs", nargs="+", help="Input files containing the tree to sort.")
parser.add_argument("-t", "--tree", required=True, help="Path to the tree object in the file.")
parser.add_argument(
"-b",
"--branches",
nargs="+",
default=["run", "lumi", "event"],
help="Branch names to be considered for the sorting.",
)
parser.add_argument("-o", "--output", default="output.root", help="Output ROOT file.")
args = parser.parse_args()
logger.initLogger(args)
args.branches = args.branches[:4]
# https://root.cern.ch/root/roottalk/roottalk01/3646.html
log.info("Opening input from")
input_tree = ROOT.TChain()
for input_file in args.inputs:
path = os.path.join(input_file, args.tree)
log.info("\t" + path)
input_tree.Add(path)
input_tree.SetCacheSize(128 * 1024 * 1024)
n_entries = input_tree.GetEntries()
values = [[] for index in xrange(len(args.branches))]
n_entries_per_iteration = 10000000 # larger buffers make problems
for iteration in progressiterator.ProgressIterator(
range(int(math.ceil(n_entries / float(n_entries_per_iteration)))),
description="Retrieving branch values for sorting",
):
cut = "(Entry$>=({i}*{n}))*(Entry$<(({i}+1)*{n}))".format(i=iteration, n=n_entries_per_iteration)
input_tree.Draw(":".join(args.branches), cut, "goff")
buffers = [input_tree.GetV1(), input_tree.GetV2(), input_tree.GetV3(), input_tree.GetV4()][: len(args.branches)]
for index, input_buffer in enumerate(buffers):
values[index].extend(
list(numpy.ndarray(input_tree.GetSelectedRows(), dtype=numpy.double, buffer=input_buffer))
)
log.info("Sorting of the tree entry indices...")
values = zip(*([range(n_entries)] + values))
values.sort(key=lambda value: value[1:])
sorted_entries = list(zip(*values)[0])
log.info("Creating output " + args.output + "...")
with tfilecontextmanager.TFileContextManager(args.output, "RECREATE") as output_file:
output_tree = input_tree.CloneTree(0)
for entry in progressiterator.ProgressIterator(sorted_entries, description="Copying tree entries"):
input_tree.GetEntry(entry)
output_tree.Fill()
output_file.Write()
log.info("Save sorted tree in " + os.path.join(args.output, args.tree) + ".")
def main():
parser = argparse.ArgumentParser(description="Get JEC parameter files using CMSSW tools.", parents=[logger.loggingParser])
parser.add_argument("--gt-data", default="FT_53_V21_AN4::All",
help="Global tag for data. [Default: %(default)s]")
parser.add_argument("--gt-mc", default="START53_V23::All",
help="Global tag for MC. [Default: %(default)s]")
parser.add_argument("--pf-jets", default="AK5PF",
help="Name of PF jets collection. [Default: %(default)s]")
parser.add_argument("--calo-jets", default="",
help="Name of Calo jets collection. [Default: %(default)s]")
args = parser.parse_args()
logger.initLogger(args)
args = vars(args)
python_config_template = string.Template("""
import FWCore.ParameterSet.Config as cms
process = cms.Process("jectxt")
process.load("Configuration.StandardSequences.Services_cff")
process.load("Configuration.StandardSequences.FrontierConditions_GlobalTag_cff")
# define your favorite global tag
process.GlobalTag.globaltag = "$global_tag"
process.maxEvents = cms.untracked.PSet(input = cms.untracked.int32(1))
process.source = cms.Source("EmptySource")
process.readAK5PF = cms.EDAnalyzer("JetCorrectorDBReader",
payloadName = cms.untracked.string("$pf_jets"),
# this is used ONLY for the name of the printed txt files. You can use any name that you like,
# but it is recommended to use the GT name that you retrieved the files from.
globalTag = cms.untracked.string("$global_tag_name"),
printScreen = cms.untracked.bool(False),
createTextFile = cms.untracked.bool(True)
)
process.readAK5Calo = process.readAK5PF.clone(payloadName = "$calo_jets")
process.p = cms.Path(process.readAK5PF * process.readAK5Calo)
""")
for mode in ["data", "mc"]:
python_config = python_config_template.substitute(global_tag=args["gt_"+mode],
global_tag_name=args["gt_"+mode].replace(":", "_"),
pf_jets=args["pf_jets"],
calo_jets=args["calo_jets"])
if args["calo_jets"] == "":
python_config = python_config.replace("\nprocess.readAK5Calo = process.readAK5PF.clone(payloadName = \"\")", "")
python_config = python_config.replace("process.p = cms.Path(process.readAK5PF * process.readAK5Calo)",
"process.p = cms.Path(process.readAK5PF)")
python_config_file_name = None
with tempfile.NamedTemporaryFile(prefix=mode, suffix=".py", delete=False) as python_config_file:
python_config_file_name = python_config_file.name
python_config_file.write(python_config)
if python_config_file_name != None:
command = ["cmsRun", python_config_file_name]
log.info("Execute \"%s\"." % " ".join(command))
exit_code = logger.subprocessCall(command)
log.info("Exit code: " + str(exit_code))
os.remove(python_config_file_name)
def parse_known_args(self, args=None, namespace=None):
known_args, unknown_args = super(HarryParser, self).parse_known_args(args=args, namespace=namespace)
logger.initLogger(known_args)
# Add help after first parsing
self.add_help = True
self.add_argument('-h', '--help', action='help', default=argparse.SUPPRESS, help='show this help message and exit')
return known_args, unknown_args
def main():
parser = argparse.ArgumentParser(description="Checkout packages needed for Artus/KappaAnalysis.", parents=[logger.loggingParser])
parser.add_argument("-n", "--n-trials", default=2, type=int,
help="Maximum number of trials for checkout commands. [Default: %(default)s]")
args = parser.parse_args()
logger.initLogger(args)
checkoutpackages.checkout_packages(max_n_trials=args.n_trials)
def main():
parser = argparse.ArgumentParser(
description=
"Switch to the Kappa commit that has been used for a certain skim.",
parents=[logger.loggingParser])
parser.add_argument(
"file",
help="Kappa skim output file containing the UserInfo in the Lumis tree"
)
parser.add_argument(
"-a",
"--args",
default="",
help="Arguments for git checkout. [Default: %(default)s]")
args = parser.parse_args()
logger.initLogger(args)
root_file = ROOT.TFile(args.file, "READ")
lumis = root_file.Get("Lumis")
lumis.GetEntry(0)
user_infos = lumis.GetUserInfo()
assert user_infos.GetEntries() >= 2
keys = user_infos[0]
values = user_infos[1]
user_infos_dict = {
str(k.GetString()): str(v.GetString())
for k, v in zip(keys, values)
}
cwd = os.getcwd()
if not cwd.endswith("Kappa") or cwd.endswith("Kappa/"):
log.fatal("The script needs to be executed in $CMSSW_BASE/src/Kappa!")
sys.exit(1)
kappa_repo = [
k for k in user_infos_dict.keys()
if k.endswith("Kappa") or k.endswith("Kappa/")
]
if len(kappa_repo) == 0:
log.fatal("Kappa revision not found in file \"%s\"!" % args.file)
sys.exit(1)
else:
kappa_repo = kappa_repo[0]
#sorted([tools.longest_common_substring(key, cwd) for key in user_infos_dict.keys()], key=lambda item: len(item))[-1]
kappa_revision = user_infos_dict[kappa_repo]
command = "git checkout %s %s" % (args.args, kappa_revision)
log.info(command)
if not raw_input("Proceed? [Y|n] ").lower().strip().startswith("n"):
logger.subprocessCall(shlex.split(command))
def main():
parser = argparse.ArgumentParser(description="Tools for JSON files.", parents=[logger.loggingParser])
parser.add_argument("json", nargs="+", help="JSON configs.")
args = parser.parse_args()
logger.initLogger(args)
log.info(jsonTools.JsonDict(args.json).doIncludes().doComments())
def main():
parser = argparse.ArgumentParser(description="Delete filelists and their files. This script also deletes the filelists from the git repositories, if you execute the script inside the repository. Commit and push the changes if wanted afterwards.",
parents=[logger.loggingParser])
parser.add_argument("filelists", nargs="+", help="Filelists.")
args = parser.parse_args()
logger.initLogger(args)
for filelist in args.filelists:
delete_filelist(filelist)
def main():
parser = argparse.ArgumentParser(description="Print stats summary table.",
parents=[logger.loggingParser])
batch_system_group = parser.add_mutually_exclusive_group()
batch_system_group.add_argument("--qstat",
dest="qstat",
default=True,
action="store_true")
batch_system_group.add_argument("--condor",
dest="qstat",
default=True,
action="store_false")
parser.add_argument(
"-s",
"--sort-state",
default="r",
help="State to be used for sorting. [Default: %(default)s]")
parser.add_argument(
"--norm-total-jobs",
default=False,
action="store_true",
help=
"Norm progress bar to total number of jobs in batch system. [Default: Norm to user with most job in system.]"
)
parser.add_argument("-S",
"--simple-mode",
default=False,
action="store_true",
help="Simple mode. Only display all and running jobs.")
parser.add_argument(
"-c",
"--continuous-mode",
default=0,
type=int,
help=
"Continuous mode. Argument is refresh interval. Default is non-continuous mode."
)
args = parser.parse_args()
logger.initLogger(args)
users = []
while True:
try:
users = print_stats_table(args, users)
if args.continuous_mode == 0:
break
time.sleep(args.continuous_mode)
except:
sys.stdout.write("\033[?1049l") # switch back to default buffer
sys.stdout.flush()
sys.exit(0)
def main():
parser = argparse.ArgumentParser(description="Print out the generator weight.",
parents=[logger.loggingParser])
parser.add_argument("files", nargs="+", help="Input files.")
parser.add_argument("-n", "--nick", default="kappa_(?P<nick>.*)_\d+.root",
help="Regular expression to extract nickname from file names. Use 'NONE' to disable nick name matching. [Default: %(default)s]")
args = parser.parse_args()
logger.initLogger(args)
sumweight_per_nick = {}
n_entries_per_nick = {}
for (fileindex, file_name) in enumerate(args.files):
# retrieve weights and total number of events
root_file = ROOT.TFile.Open(file_name, "READ")
lumiTree = root_file.Get("Lumis")
n_entries = lumiTree.GetEntries()
# nickname matching and sum of weights
no_regex_match = True if args.nick=='NONE' else False
nick = ( re.match(args.nick, os.path.basename(file_name)).groupdict().values()[0] if not no_regex_match else '*')
for entry in xrange(n_entries):
lumiTree.GetEntry(entry)
n_entries_per_nick[nick] = n_entries_per_nick.get(nick, 0.0) + lumiTree.filterMetadata.nEventsTotal
sumweight_per_nick[nick] = sumweight_per_nick.get(nick, 0.0) + (lumiTree.filterMetadata.nEventsTotal - 2 * lumiTree.filterMetadata.nNegEventsTotal)
#counter
if (fileindex % 100 == 0):
print "...processing file %d (nickname: %s)" % (fileindex, nick)
print "\tweighted entries = ", sumweight_per_nick[nick]
print "\tentries = ", n_entries_per_nick[nick]
#close rootfile
root_file.Close()
# print results and save to dataset
if not no_regex_match:
dataset = os.path.expandvars("$ARTUSPATH/../Kappa/Skimming/data/datasets.json")
dictionary = load_database(dataset)
for index, (nick, sumweight) in enumerate(sumweight_per_nick.items()):
if not no_regex_match:
sample_name = get_sample_by_nick(nick)
dictionary[nick]["n_events_generated"] = str(int(n_entries_per_nick[nick]))
dictionary[nick]["generatorWeight"] = sumweight/n_entries_per_nick[nick]
log.info("\n\n\"" + sample_name + "\"" + ": {")
log.info("\tn_events_generated: " + str(int(n_entries_per_nick[nick])))
log.info("\tgeneratorWeight: " + str(sumweight/n_entries_per_nick[nick]))
log.info("}")
if not no_regex_match: save_database(dictionary, dataset)
def parse_known_args(self, args=None, namespace=None, **kwargs):
new_logger = not kwargs.pop("from_script", False)
known_args, unknown_args = super(HarryParser, self).parse_known_args(args=args, namespace=namespace, **kwargs)
if new_logger:
logger.initLogger(known_args)
# Add help after first parsing
self.add_help = True
self.add_argument('-h', '--help', action='help', default=argparse.SUPPRESS, help='show this help message and exit')
return known_args, unknown_args
def main():
parser = argparse.ArgumentParser(description="Batch submission of multiple commands.", parents=[logger.loggingParser])
parser.add_argument("commands", help="Commands to be executed on a batch system. They can also be piped into this program.", nargs="*", default=[])
parser.add_argument("-b", "--batch", default="rwthcondor",
help="Run with grid-control and select backend. [Default: %(default)s]")
args = parser.parse_args()
logger.initLogger(args)
batchsubmission.batch_submission(commands=args.commands, batch=args.batch)
def main():
parser = argparse.ArgumentParser(description="Print names of all pipelines from Artus JSON config", parents=[logger.loggingParser])
parser.add_argument("configs", nargs="+",
help="Artus JSON configs. Can be either Artus output root files or JSON text files. Only first is considered.")
args = parser.parse_args()
logger.initLogger(args)
config = jsonTools.JsonDict(args.configs[0])
log.info("\n".join(sorted(config.get("Pipelines", {}).keys())))
def main():
parser = argparse.ArgumentParser(description="Execute bash command.",
parents=[logger.loggingParser])
parser.add_argument("command", nargs="+",
help="Bash command to be executed.")
args = parser.parse_args()
logger.initLogger(args)
logger.subprocessCall(os.path.expandvars(" ".join(args.command)), shell=True)
def main():
parser = argparse.ArgumentParser(
description=
"Delete filelists and their files. This script also deletes the filelists from the git repositories, if you execute the script inside the repository. Commit and push the changes if wanted afterwards.",
parents=[logger.loggingParser])
parser.add_argument("filelists", nargs="+", help="Filelists.")
args = parser.parse_args()
logger.initLogger(args)
for filelist in args.filelists:
delete_filelist(filelist)
def main():
ROOT.gROOT.SetBatch(True)
parser = argparse.ArgumentParser(description="Search for Kappa files containing certain events.",
parents=[logger.loggingParser])
parser.add_argument("files", nargs="+", help="Kappa skim output files to check")
parser.add_argument("-c", "--config", help="JSON config file containing the event selection.")
args = parser.parse_args()
logger.initLogger(args)
config = jsonTools.JsonDict(args.config)
run_whitelist = config.get("RunWhitelist", [])
lumi_whitelist = config.get("LumiWhitelist", [])
event_whitelist = config.get("EventWhitelist", [])
run_blacklist = config.get("RunBlacklist", [])
lumi_blacklist = config.get("LumiBlacklist", [])
event_blacklist = config.get("EventBlacklist", [])
selections = []
if len(run_whitelist) > 0:
selections.append("||".join(["(eventInfo.nRun==%d)" % run for run in run_whitelist]))
if len(lumi_whitelist) > 0:
selections.append("||".join(["(eventInfo.nLumi==%d)" % lumi for lumi in lumi_whitelist]))
if len(event_whitelist) > 0:
selections.append("||".join(["(eventInfo.nEvent==%d)" % event for event in event_whitelist]))
if len(run_blacklist) > 0:
selections.append("&&".join(["(eventInfo.nRun!=%d)" % run for run in run_blacklist]))
if len(lumi_blacklist) > 0:
selections.append("&&".join(["(eventInfo.nLumi!=%d)" % lumi for lumi in lumi_blacklist]))
if len(event_blacklist) > 0:
selections.append("&&".join(["(eventInfo.nEvent!=%d)" % event for event in event_blacklist]))
selection = "&&".join(["(%s)" % cut for cut in selections])
results = []
for input_file in args.files:
tree = ROOT.TChain("Events")
tree.AddFile(input_file)
entries = tree.Draw("1", selection, "goff")
if entries > 0:
results.append((entries, input_file))
log.info("%4d entries: %s" % (entries, input_file))
log.info("\nFound files:")
for entries, input_file in results:
log.info("%4d entries: %s" % (entries, input_file))
def main():
parser = argparse.ArgumentParser(description="Create filelists for Artus inputs from pnfs mount on the NAF.", parents=[logger.loggingParser])
parser.add_argument("-s", "--skimming-dir", required=True,
help="Skimming output directory as specified in se (output) path in GC.")
parser.add_argument("-d", "--date", required=True,
help="Date string.")
parser.add_argument("-o", "--output-dir", default="$CMSSW_BASE/src/HiggsAnalysis/KITHiggsToTauTau/data/Samples/",
help="Output directory for the filelists. [Default: %(default)s]")
parser.add_argument("-r", "--create-recent-symlinks", default=True, action="store_true",
help="Create symlinks to filelists as recent ones. [Default: %(default)s]")
parser.add_argument("-c", "--crab", default=False, action="store_true",
help="Skimming output directory was created by Crab (it has a different structure). [Default: %(default)s]")
args = parser.parse_args()
logger.initLogger(args)
if not args.skimming_dir.startswith("/pnfs"):
log.critical("You need to specify a /pnfs path at the NAF")
sys.exit(1)
args.output_dir = os.path.expandvars(args.output_dir)
skimming_dirs = glob.glob(os.path.join(args.skimming_dir, "*_*TeV*"))
if args.crab:
skimming_dirs = glob.glob(os.path.join(args.skimming_dir, "*/crab_*"))
for skimming_dir in skimming_dirs:
nick = os.path.basename(skimming_dir)
files = sorted(glob.glob(os.path.join(skimming_dir, "*.root")))
if args.crab:
nick = nick.strip("crab_")
files = sorted(glob.glob(os.path.join(skimming_dir, "*/*/*.root")))
if (len(files) == 0):
log.critical("Input file list empty. If the skims have been produced with Crab, switch on the --crab option")
sys.exit(1)
filelists = os.path.join(args.output_dir, "%s_sample_%s_%s.txt" % ("%s", nick, "%s"))
dcache_settings = {
#"NAF" : ["", None],
"DCAP" : ["dcap://dcache-cms-dcap.desy.de/", None],
#"XROOTD" : ["root://cms-xrd-global.cern.ch/", "/pnfs/desy.de/cms/tier2"],
}
for name, settings in dcache_settings.items():
create_filelist([(settings[0] + (root_file.replace(settings[1], "") if settings[1] else root_file)) for root_file in files],
filelists % (name, args.date),
filelists % (name, "recent") if args.create_recent_symlinks else None)
def main():
parser = argparse.ArgumentParser(description="Print out the available tau discriminators in a kappa skim.", parents=[logger.loggingParser])
parser.add_argument("file", help="Kappa skim output file containing the Tau Meta data")
args = parser.parse_args()
logger.initLogger(args)
lumis = ROOT.TChain("Lumis")
lumis.Add(args.file)
lumis.GetEntry(0)
hltNames = lumis.lumiInfo.hltNames
for hltName in hltNames:
log.info(hltName)
def main():
parser = argparse.ArgumentParser(
description="Collect outputs of failed GC jobs.",
parents=[logger.loggingParser])
parser.add_argument("INPUT_DIR", help="Project directory of Artus GC run")
parser.add_argument(
"-o",
"--output-dir",
help=
"Directory where to put symlinks to failed job outputs. [Default: INPUT_DIR/failed]"
)
args = parser.parse_args()
logger.initLogger(args)
input_dir = args.INPUT_DIR
if args.output_dir == None:
args.output_dir = os.path.join(input_dir, "failed")
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
job_info_file_names = glob.glob(
os.path.join(input_dir, "workdir/output/job_*/job.info"))
job_infos = {}
for job_info_file_name in job_info_file_names:
with open(job_info_file_name) as job_info_file:
job_info_content = re.search(
"JOBID=(?P<job_id>\d*).*EXITCODE=(?P<exit_code>\d*)",
job_info_file.read().replace("\n", "")).groupdict()
job_infos[int(job_info_content["job_id"])] = int(
job_info_content["exit_code"])
job_infos = {
job_id: exit_code
for job_id, exit_code in job_infos.iteritems() if exit_code != 0
}
for job_id, exit_code in job_infos.iteritems():
output_dir = os.path.join(
args.output_dir, "job_id_%d_exit_code_%d" % (job_id, exit_code))
if not os.path.exists(output_dir):
os.makedirs(output_dir)
os.system("ln -vs %s %s" %
(os.path.join(input_dir, "workdir/output/job_%d" % job_id),
os.path.join(output_dir, "gc_job_%d" % job_id)))
os.system(
"ln -vs %s %s" %
(os.path.join(input_dir, "output/*/*_%d_*" % job_id), output_dir))
def main():
parser = argparse.ArgumentParser(description="Execute bash command.",
parents=[logger.loggingParser])
parser.add_argument("command",
nargs="+",
help="Bash command to be executed.")
args = parser.parse_args()
logger.initLogger(args)
logger.subprocessCall(os.path.expandvars(" ".join(args.command)),
shell=True)
def __init__(self, executable=None, userArgParsers=None):
self._config = jsonTools.JsonDict()
self._executable = executable
self._parser = None
#Load default argument parser
self._initArgumentParser(userArgParsers)
#Parse command line arguments and return dict
self._args = self._parser.parse_args()
logger.initLogger(self._args)
# expand the environment variables only at the batch node
if self._args.batch:
self._args.envvar_expansion = False
date_now = datetime.now().strftime("%Y-%m-%d_%H-%M")
# write repository revisions to the config
if not self._args.disable_repo_versions:
self.setRepositoryRevisions()
self._config["Date"] = date_now
# write username to the config
try:
self._config["User"] = os.environ["USER"]
except:
import random
import string
self._config["User"] = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for _ in range(10))
#Expand Config
self.expandConfig()
self.projectPath = None
self.localProjectPath = None
self.remote_se = False
if self._args.batch:
self.projectPath = os.path.join(
os.path.expandvars(self._args.work),
date_now + "_" + self._args.project_name)
self.localProjectPath = self.projectPath
if self.projectPath.startswith("srm://"):
self.remote_se = True
self.localProjectPath = os.path.join(
os.path.expandvars(self._parser.get_default("work")),
date_now + "_" + self._args.project_name)
def main():
parser = argparse.ArgumentParser(description="Print out the available LHE weights in a kappa skim.", parents=[logger.loggingParser])
parser.add_argument("file", help="Kappa skim output file")
args = parser.parse_args()
logger.initLogger(args)
lumis = ROOT.TChain("Lumis")
lumis.Add(args.file)
lumis.GetEntry(0)
log.info("\nNames of available LHE weights:")
lheWeightNames = lumis.genEventInfoMetadata.lheWeightNames
for lheWeightName in lheWeightNames:
log.info("\t" + lheWeightName + (" -> "+mapping[lheWeightName] if lheWeightName in mapping else ""))
def main():
ROOT.gSystem.Load(os.path.expandvars("$CMSSW_BASE/src/Kappa/lib/libKappa"))
parser = argparse.ArgumentParser(
description="Print out the user info in the kappa skim.",
parents=[logger.loggingParser])
parser.add_argument(
"file",
help="Kappa skim output file containing the UserInfo in the Lumis tree"
)
parser.add_argument("-k",
"--keys",
nargs="+",
help="Keys to print. [Default: print available keys]")
args = parser.parse_args()
logger.initLogger(args)
root_file = ROOT.TFile(args.file, "READ")
lumis = root_file.Get("Lumis")
lumis.GetEntry(0)
user_infos = lumis.GetUserInfo()
assert user_infos.GetEntries() >= 2
keys = user_infos[0]
values = user_infos[1]
user_infos_dict = {
str(k.GetString()): str(v.GetString())
for k, v in zip(keys, values)
}
if args.keys is None:
log.info("Available keys:")
for key in user_infos_dict.keys():
log.info("\t%s" % key)
else:
if len(args.keys) == 1:
if args.keys[0] in user_infos_dict:
log.info(user_infos_dict[args.keys[0]])
else:
log.error("Key \"%s\" not available!" % args.keys[0])
else:
for key in args.keys:
if key in user_infos_dict:
log.info("%25s : %s" % (key, user_infos_dict[key]))
else:
log.error("Key \"%s\" not available!" % key)
def parse_known_args(self, args=None, namespace=None, **kwargs):
new_logger = not kwargs.pop("from_script", False)
known_args, unknown_args = super(HarryParser, self).parse_known_args(
args=args, namespace=namespace, **kwargs)
if new_logger:
logger.initLogger(known_args)
# Add help after first parsing
self.add_help = True
self.add_argument('-h',
'--help',
action='help',
default=argparse.SUPPRESS,
help='show this help message and exit')
return known_args, unknown_args
def __init__(self, executable=None, userArgParsers=None):
self._config = jsonTools.JsonDict()
self._executable = "HiggsToTauTauAnalysis"
self._parser = None
#Load default argument parser
self._initArgumentParser(userArgParsers)
#Parse command line arguments and return dict
self._args = self._parser.parse_args()
logger.initLogger(self._args)
self._date_now = datetime.now().strftime("%Y-%m-%d_%H-%M")
self.tmp_directory_remote_files = None
self._gridControlInputFiles = {}
def main():
parser = argparse.ArgumentParser(
description="Read text files and convert dCache paths.",
parents=[logger.loggingParser])
parser.add_argument("input_files", nargs="+", help="Input text files.")
parser.add_argument("-m",
"--modes",
default=["xrd2local"],
nargs="+",
choices=[
"dcap2local", "dcap2srm", "dcap2xrd", "local2xrd",
"local2dcap", "local2srm", "srm2dcap", "srm2local",
"srm2xrd", "xrd2dcap", "xrd2local", "xrd2srm",
"xrd2xrd"
],
help="Conversion modes. [Default: %(default)s]")
parser.add_argument(
"-i",
"--in-place",
default=False,
action="store_true",
help=
"Convert paths in place modifying the input files. [Default: Only print converted files]"
)
args = parser.parse_args()
logger.initLogger(args)
for input_filename in args.input_files:
if "://" in input_filename or input_filename.startswith("/pnfs"):
result = input_filename
for mode in args.modes:
result = getattr(dcachetools, mode)(result)
log.info(result)
else:
with open(input_filename,
"rw" if args.in_place else "r") as input_file:
content = input_file.read()
for mode in args.modes:
content = getattr(dcachetools, mode)(content)
if args.in_place:
input_file.write(content)
else:
log.info(content)
def main():
parser = argparse.ArgumentParser(
description="Checkout packages needed for Artus/KappaAnalysis.",
parents=[logger.loggingParser])
parser.add_argument(
"-n",
"--n-trials",
default=2,
type=int,
help=
"Maximum number of trials for checkout commands. [Default: %(default)s]"
)
args = parser.parse_args()
logger.initLogger(args)
checkoutpackages.checkout_packages(max_n_trials=args.n_trials)
def main():
parser = argparse.ArgumentParser(
description=
"Compare different repository versions configured in Artus configs. The script has to be executed in the directory of the repository.",
parents=[logger.loggingParser])
parser.add_argument(
"files",
help=
"Two configurations. The configs should be ordered by the expected repository version. Can be either Artus output root files or JSON text files",
nargs=2)
args = parser.parse_args()
logger.initLogger(args)
config1 = jsonTools.JsonDict(args.files[0])
config2 = jsonTools.JsonDict(args.files[1])
dirname = os.path.basename(os.getcwd())
repo_key1 = sorted([
key for key in config1.keys()
if key.startswith("/") and key.endswith(dirname)
],
key=lambda item: len(item))[-1]
repo_key2 = sorted([
key for key in config2.keys()
if key.startswith("/") and key.endswith(dirname)
],
key=lambda item: len(item))[-1]
repo_version1 = config1[repo_key1]
repo_version2 = config2[repo_key2]
diff_string = "%s...%s" % (repo_version1, repo_version2)
command = "git diff %s..%s" % (repo_version1, repo_version2)
if log.isEnabledFor(logging.DEBUG):
log.info("")
logger.subprocessCall(shlex.split(command))
log.info("\n" + command)
popen_cout, popen_cerr = subprocess.Popen(
"git config remote.origin.url".split(),
stdout=subprocess.PIPE).communicate()
remote_url = popen_cout.replace("\n", "")
github_link = os.path.join(remote_url, "compare", diff_string)
log.info(github_link)
def main():
parser = argparse.ArgumentParser(
description="Print names of all pipelines from Artus JSON config",
parents=[logger.loggingParser])
parser.add_argument(
"configs",
nargs="+",
help=
"Artus JSON configs. Can be either Artus output root files or JSON text files. Only first is considered."
)
args = parser.parse_args()
logger.initLogger(args)
config = jsonTools.JsonDict(args.configs[0])
log.info("\n".join(sorted(config.get("Pipelines", {}).keys())))
def main():
parser = argparse.ArgumentParser(
description=
"Print out the available tau discriminators in a kappa skim.",
parents=[logger.loggingParser])
parser.add_argument(
"file", help="Kappa skim output file containing the Tau Meta data")
args = parser.parse_args()
logger.initLogger(args)
root_file = ROOT.TFile(args.file, "READ")
lumis = root_file.Get("Lumis")
lumis.GetEntry(0)
hltNames = lumis.lumiInfo.hltNames
for hltName in hltNames:
log.info(hltName)
def __init__(self, **kwargs):
logger.initLogger() # get a basic logger without arguments
kwargs["add_help"] = False
kwargs["conflict_handler"] = "resolve"
kwargs["fromfile_prefix_chars"] = "@"
kwargs["formatter_class"] = argparse.RawDescriptionHelpFormatter
kwargs["epilog"] = (
"Parameter replication: Several list parameters are internally matched by\n"
" their indizes. If they are of different length, short parameter lists\n"
" are replicated element wise to meet the required length. For example,\n"
" setting the inputs i1, i2, i3 and weights w1, w2 creates the matching\n"
" i1->w1, i2->w2 and i3->w1.\n"
)
kwargs.setdefault("parents", []).append(logger.loggingParser)
super(HarryParser, self).__init__(**kwargs)
self.add_argument("-h", "--help", default=False, action="store_true",
help="Show this help message and exit.")
self.add_argument("--no-logo", default=True, action="store_true",
help="Don't show the HarryPlotter logo at startup.")
self.add_argument("--comment", default="",
help="Comment for the output JSON file. This argument is filled with the program call in multiplot scripts. [Default: %(default)s]")
self.module_options = self.add_argument_group('Modules')
self.module_options.add_argument("--modules-search-paths", default=[], nargs="+",
help="Additional paths to be searched for modules.")
self.module_options.add_argument("--input-modules", default="InputRoot", nargs="+",
help="Input Modules. [Default: %(default)s]")
self.module_options.add_argument("--analysis-modules", default=[], nargs="+",
help="Analysis Modules. [Default: %(default)s]")
self.module_options.add_argument("--plot-modules", default="PlotMpl", nargs="+",
help="Plot Modules. [Default: %(default)s]")
self.module_options.add_argument("--list-available-modules", default=False, action="store_true",
help="List all available modules.")
self.json_options = self.add_argument_group('JSON Configs')
self.json_options.add_argument("-j", "--json-defaults", nargs="+",
help="JSON config file(s) containing default settings (for a certain plot).")
self.json_options.add_argument("--export-json", default="default", const="update", nargs="?",
help="Export arguments to specified JSON config file. Specify \"update\" or no argument to overwrite the input JSON config file. [Default: <plot output filename>.json]")
# Register new keyword 'bool' for parser
self.register('type','bool',self._str2bool)
def main():
parser = argparse.ArgumentParser(
description="Print out the available electron MVA IDs in a kappa skim.",
parents=[logger.loggingParser])
parser.add_argument(
"file",
help="Kappa skim output file containing the electron meta data")
args = parser.parse_args()
logger.initLogger(args)
lumis = ROOT.TChain("Lumis")
lumis.Add(args.file)
lumis.GetEntry(0)
log.info("\nNames of available electron MVA IDs:")
idNames = lumis.electronMetadata.idNames
for idName in idNames:
log.info("\t" + idName)
