def _print_module_list(self, module_list):
"""Print a list of modules (name and docstring)"""
for module in module_list:
log.info("\t" +
tools.get_colored_string("{}".format(module), "green"))
if inspect.getdoc(self.available_processors[module]):
log.info(
tools.get_indented_text(
"\t\t",
inspect.getdoc(self.available_processors[module])))
def print_comments_from_json_files(json_dir, comment_key):
"""print comments from all json files in a directory"""
#TODO generalize this. works with any string, not only comments
files = [os.path.join(json_dir, f) for f in os.listdir(json_dir) if f.endswith(".json")]
log.info(tools.get_colored_string("json files: ({})".format(json_dir), 'cyan'))
for f in files:
d = JsonDict(f).doIncludes().doComments()
string = "\t" + tools.get_colored_string(os.path.basename(f), "yellow")
if comment_key in d:
string += tools.get_indented_text("\n\t\t", d[comment_key])
log.info(string)
def print_comments_from_json_files(json_dir, comment_key):
"""print comments from all json files in a directory"""
#TODO generalize this. works with any string, not only comments
files = [
os.path.join(json_dir, f) for f in os.listdir(json_dir)
if f.endswith(".json")
]
log.info(
tools.get_colored_string("json files: ({})".format(json_dir), 'cyan'))
for f in files:
d = JsonDict(f).doIncludes().doComments()
string = "\t" + tools.get_colored_string(os.path.basename(f), "yellow")
if comment_key in d:
string += tools.get_indented_text("\n\t\t", d[comment_key])
log.info(string)
def print_plotting_functions(plotting_path):
""" print the comments / docstrings of the plotting configs"""
log.info(
tools.get_colored_string(
"plotting scripts: ({})".format(plotting_path), 'cyan'))
# get docstrings
module_list = get_module_list(plotting_path)
for module in module_list:
log.info(tools.get_colored_string(module.__name__ + ".py", "yellow"))
functions = inspect.getmembers(module, inspect.isfunction)
if len(functions) > 0:
prefix = "\t\t"
for func in functions:
if func[0].startswith("_"):
continue
log.info("\t" + tools.get_colored_string(func[0], "green"))
if inspect.getdoc(func[1]) != None: # has docstring
log.info(
tools.get_indented_text(prefix,
inspect.getdoc(func[1])))
def multi_plots(self, list_of_config_dicts, list_of_args_strings, n_processes=1, n_fast_plots=None):
config_dicts = list_of_config_dicts if isinstance(list_of_config_dicts, collections.Iterable) and not isinstance(list_of_config_dicts, basestring) else [list_of_config_dicts]
args_strings = list_of_args_strings if isinstance(list_of_args_strings, collections.Iterable) and not isinstance(list_of_args_strings, basestring) else [list_of_args_strings]
# fill up missing list elements by repeating previous ones
n_plots = max(len(config_dicts), len(args_strings))
if (len(config_dicts) > 1) and (len(config_dicts) < n_plots):
log.warning("Too few config dicts specified!")
if (len(args_strings) > 1) and (len(args_strings) < n_plots):
log.warning("Too few argument lists specified!")
config_dicts = (config_dicts*n_plots)[:n_plots]
args_strings = (args_strings*n_plots)[:n_plots]
harry_args = []
for config_dict, args_string in zip(config_dicts, args_strings):
if config_dict is None:
harry_args.append(None)
else:
config_dict["comment"] = " ".join(sys.argv)
if "json_defaults" in config_dict:
json_defaults_dict = jsonTools.JsonDict(config_dict["json_defaults"]).doIncludes().doComments()
config_dict.pop("json_defaults")
json_defaults_dict.update(config_dict)
config_dict = json_defaults_dict
harry_args.append("--json-defaults \"%s\"" % jsonTools.JsonDict(config_dict).toString(indent=None).replace("\"", "'"))
if not args_string is None:
if harry_args[-1] is None:
harry_args[-1] = args_string
else:
harry_args[-1] += (" "+args_string)
if config_dict is None:
harry_args[-1] += (" --comment " + (" ".join(sys.argv)))
if not n_fast_plots is None:
harry_args = harry_args[:n_fast_plots]
# multi processing of multiple plots
output_filenames = []
failed_plots = []
if len(harry_args) > 1 and n_processes > 1:
log.info("Creating {:d} plots in {:d} processes".format(len(harry_args), min(n_processes, len(harry_args))))
results = tools.parallelize(pool_plot, zip([self]*len(harry_args), harry_args), n_processes)
tmp_output_filenames, tmp_failed_plots, tmp_error_messages = zip(*([result for result in results if not result is None and result != (None,)]))
output_filenames = [output_filename for output_filename in tmp_output_filenames if not output_filename is None]
failed_plots = [(failed_plot, error_message) for failed_plot, error_message in zip(tmp_failed_plots, tmp_error_messages) if not failed_plot is None]
# single processing of multiple plots
elif len(harry_args) > 1:
log.info("Creating {:d} plots".format(len(harry_args)))
for harry_args in harry_args:
try:
output_filenames.append(self.plot(harry_args))
except SystemExit as e:
failed_plots.append((harry_args, None))
except Exception as e:
log.info(str(e))
failed_plots.append((harry_args, None))
# single plot
elif len(harry_args) > 0:
output_filenames.append(self.plot(harry_args[0]))
if len(failed_plots) > 0:
log.error("%d failed plots:" % len(failed_plots))
for failed_plot in failed_plots:
log.info("\n"+tools.get_colored_string("Failed plot:", color='red'))
log.info("\t%s" % failed_plot[0])
if failed_plot[1] is not None:
try:
log.info(tools.get_indented_text(" ", tools.get_colored_string("Traceback for this plot:", color='red')+"\n" + failed_plot[1]))
except:
log.info("\t Traceback for this plot: \n" + failed_plot[1])
return output_filenames
def _print_module_list(self, module_list):
"""Print a list of modules (name and docstring)"""
for module in module_list:
log.info("\t"+tools.get_colored_string("{}".format(module), "green"))
if inspect.getdoc(self.available_processors[module]):
log.info(tools.get_indented_text("\t\t", inspect.getdoc(self.available_processors[module])))
def multi_plots(self,
list_of_config_dicts,
list_of_args_strings,
n_processes=1,
n_fast_plots=None,
batch=None):
config_dicts = list_of_config_dicts if isinstance(
list_of_config_dicts, collections.Iterable) and not isinstance(
list_of_config_dicts, basestring) else [list_of_config_dicts]
args_strings = list_of_args_strings if isinstance(
list_of_args_strings, collections.Iterable) and not isinstance(
list_of_args_strings, basestring) else [list_of_args_strings]
# fill up missing list elements by repeating previous ones
n_plots = max(len(config_dicts), len(args_strings))
if (len(config_dicts) > 1) and (len(config_dicts) < n_plots):
log.warning("Too few config dicts specified!")
if (len(args_strings) > 1) and (len(args_strings) < n_plots):
log.warning("Too few argument lists specified!")
config_dicts = (config_dicts * n_plots)[:n_plots]
args_strings = (args_strings * n_plots)[:n_plots]
if n_processes > 1:
for i in range(len(args_strings)):
args_strings[i] += (" --hide-progressbar ")
self.harry_args = []
for config_dict, args_string in zip(config_dicts, args_strings):
if config_dict is None:
self.harry_args.append(None)
else:
config_dict["comment"] = " ".join(sys.argv)
if not batch is None:
config_dict["dry_run"] = True
if "json_defaults" in config_dict:
json_defaults_dict = jsonTools.JsonDict(
config_dict["json_defaults"]).doIncludes().doComments(
)
config_dict.pop("json_defaults")
json_defaults_dict.update(config_dict)
config_dict = json_defaults_dict
self.harry_args.append(
"--json-defaults \"%s\"" %
jsonTools.JsonDict(config_dict).toString(
indent=None).replace("\"", "'"))
if not args_string is None:
if self.harry_args[-1] is None:
self.harry_args[-1] = args_string
else:
self.harry_args[-1] += (" " + args_string)
if config_dict is None:
self.harry_args[-1] += (" --comment " +
(" ".join(sys.argv)))
if not batch is None:
self.harry_args[-1] += " --dry-run"
if not n_fast_plots is None:
self.harry_args = self.harry_args[:n_fast_plots]
n_plots = len(self.harry_args)
self.harry_cores = [None] * n_plots
# multi processing of multiple plots
output_filenames = []
failed_plots = []
if (n_plots > 1) and (n_processes > 1):
log.info("Creating {:d} plots in {:d} processes".format(
n_plots, min(n_processes, n_plots)))
results = tools.parallelize(pool_plot,
zip([self] * n_plots, range(n_plots)),
n_processes,
description="Plotting")
tmp_output_filenames, tmp_failed_plots, tmp_error_messages = zip(
*([
result for result in results
if not result is None and result != (None, )
]))
output_filenames = [
output_filename for output_filename in tmp_output_filenames
if not output_filename is None
]
failed_plots = [(failed_plot, error_message)
for failed_plot, error_message in zip(
tmp_failed_plots, tmp_error_messages)
if not failed_plot is None]
# single processing of multiple plots
elif n_plots > 1:
log.info("Creating {:d} plots".format(n_plots))
for plot_index in xrange(n_plots):
try:
output_filenames.append(self.plot(plot_index))
except SystemExit as e:
failed_plots.append((self.harry_args[plot_index], None))
except Exception as e:
log.info(str(e))
failed_plots.append((self.harry_args[plot_index], None))
# single plot
elif n_plots > 0:
output_filenames.append(self.plot(0))
# batch submission
if (not (batch is None)) and (len(failed_plots) < n_plots):
workdir = tempfile.mkdtemp(
prefix="harry_work_" +
datetime.datetime.now().strftime("%Y-%m-%d_%H-%M") + "_")
main_config = ""
with open(
os.path.expandvars(
"$CMSSW_BASE/src/Artus/HarryPlotter/data/grid-control_base_config.conf"
), "r") as main_config_file:
main_config = main_config_file.read()
backend_config = ""
with open(
os.path.expandvars(
"$CMSSW_BASE/src/Artus/Configuration/data/grid-control_backend_"
+ batch + ".conf"), "r") as backend_config_file:
backend_config = backend_config_file.read()
final_config = string.Template(main_config).safe_substitute(
cmsswbase=os.path.expandvars("$CMSSW_BASE"),
hpworkbase=os.path.expandvars("$HP_WORK_BASE"),
cwd=os.getcwd(),
jsonconfigs="\n\t" +
("\n\t".join([item[0] for item in output_filenames])),
executable=self.standalone_executable,
workdir=workdir,
backend=backend_config)
final_config_filename = workdir + ".conf"
with open(final_config_filename, "w") as final_config_file:
final_config_file.write(final_config)
command = "go.py " + final_config_filename
log.info(command)
logger.subprocessCall(shlex.split(command))
if len(failed_plots) > 0:
log.error("%d failed plots:" % len(failed_plots))
for failed_plot in failed_plots:
log.info("\n" +
tools.get_colored_string("Failed plot:", color='red'))
log.info("\t%s" % failed_plot[0])
if failed_plot[1] is not None:
try:
log.info(
tools.get_indented_text(
" ",
tools.get_colored_string(
"Traceback for this plot:", color='red') +
"\n" + failed_plot[1]))
except:
log.info("\t Traceback for this plot: \n" +
failed_plot[1])
return output_filenames
def multi_plots(self, list_of_config_dicts, list_of_args_strings, n_processes=1, n_fast_plots=None):
config_dicts = list_of_config_dicts if isinstance(list_of_config_dicts, collections.Iterable) and not isinstance(list_of_config_dicts, basestring) else [list_of_config_dicts]
args_strings = list_of_args_strings if isinstance(list_of_args_strings, collections.Iterable) and not isinstance(list_of_args_strings, basestring) else [list_of_args_strings]
# fill up missing list elements by repeating previous ones
n_plots = max(len(config_dicts), len(args_strings))
if (len(config_dicts) > 1) and (len(config_dicts) < n_plots):
log.warning("Too few config dicts specified!")
if (len(args_strings) > 1) and (len(args_strings) < n_plots):
log.warning("Too few argument lists specified!")
config_dicts = (config_dicts*n_plots)[:n_plots]
args_strings = (args_strings*n_plots)[:n_plots]
harry_args = []
for config_dict, args_string in zip(config_dicts, args_strings):
if config_dict is None:
harry_args.append(None)
else:
config_dict["comment"] = " ".join(sys.argv)
if "json_defaults" in config_dict:
json_defaults_dict = jsonTools.JsonDict(config_dict["json_defaults"]).doIncludes().doComments()
config_dict.pop("json_defaults")
json_defaults_dict.update(config_dict)
config_dict = json_defaults_dict
harry_args.append("--json-defaults \"%s\"" % jsonTools.JsonDict(config_dict).toString(indent=None).replace("\"", "'"))
if not args_string is None:
if harry_args[-1] is None:
harry_args[-1] = args_string
else:
harry_args[-1] += (" "+args_string)
if config_dict is None:
harry_args[-1] += (" --comment " + (" ".join(sys.argv)))
if not n_fast_plots is None:
harry_args = harry_args[:n_fast_plots]
# multi processing of multiple plots
output_filenames = []
failed_plots = []
if len(harry_args) > 1 and n_processes > 1:
log.info("Creating {:d} plots in {:d} processes".format(len(harry_args), n_processes))
pool = Pool(processes=n_processes)
results = pool.map_async(pool_plot, zip([self]*len(harry_args), harry_args))
# 9999999 is needed for KeyboardInterrupt to work: http://stackoverflow.com/questions/1408356/keyboard-interrupts-with-pythons-multiprocessing-pool
tmp_output_filenames, tmp_failed_plots, tmp_error_messages = zip(*([result for result in results.get(9999999) if not result is None and result != (None,)]))
output_filenames = [output_filename for output_filename in tmp_output_filenames if not output_filename is None]
failed_plots = [(failed_plot, error_message) for failed_plot, error_message in zip(tmp_failed_plots, tmp_error_messages) if not failed_plot is None]
# single processing of multiple plots
elif len(harry_args) > 1:
log.info("Creating {:d} plots".format(len(harry_args)))
for harry_args in harry_args:
try:
output_filenames.append(self.plot(harry_args))
except SystemExit as e:
failed_plots.append((harry_args, None))
except Exception as e:
log.info(str(e))
failed_plots.append((harry_args, None))
# single plot
elif len(harry_args) > 0:
output_filenames.append(self.plot(harry_args[0]))
if len(failed_plots) > 0:
log.error("%d failed plots:" % len(failed_plots))
for failed_plot in failed_plots:
log.info("\n"+tools.get_colored_string("Failed plot:", color='red'))
log.info("\t%s" % failed_plot[0])
if failed_plot[1] is not None:
log.info(tools.get_indented_text(" ", tools.get_colored_string("Traceback for this plot:", color='red')+"\n" + failed_plot[1]))
return output_filenames
def multi_plots(self, list_of_config_dicts, list_of_args_strings, n_processes=1, n_fast_plots=None, batch=None):
config_dicts = list_of_config_dicts if isinstance(list_of_config_dicts, collections.Iterable) and not isinstance(list_of_config_dicts, basestring) else [list_of_config_dicts]
args_strings = list_of_args_strings if isinstance(list_of_args_strings, collections.Iterable) and not isinstance(list_of_args_strings, basestring) else [list_of_args_strings]
# fill up missing list elements by repeating previous ones
n_plots = max(len(config_dicts), len(args_strings))
if (len(config_dicts) > 1) and (len(config_dicts) < n_plots):
log.warning("Too few config dicts specified!")
if (len(args_strings) > 1) and (len(args_strings) < n_plots):
log.warning("Too few argument lists specified!")
config_dicts = (config_dicts*n_plots)[:n_plots]
args_strings = (args_strings*n_plots)[:n_plots]
if n_processes>1:
for i in range(len(args_strings)):
args_strings[i] += (" --hide-progressbar ")
self.harry_args = []
for config_dict, args_string in zip(config_dicts, args_strings):
if config_dict is None:
self.harry_args.append(None)
else:
config_dict["comment"] = " ".join(sys.argv)
if not batch is None:
config_dict["dry_run"] = True
if "json_defaults" in config_dict:
json_defaults_dict = jsonTools.JsonDict(config_dict["json_defaults"]).doIncludes().doComments()
config_dict.pop("json_defaults")
json_defaults_dict.update(config_dict)
config_dict = json_defaults_dict
self.harry_args.append("--json-defaults \"%s\"" % jsonTools.JsonDict(config_dict).toString(indent=None).replace("\"", "'"))
if not args_string is None:
if self.harry_args[-1] is None:
self.harry_args[-1] = args_string
else:
self.harry_args[-1] += (" "+args_string)
if config_dict is None:
self.harry_args[-1] += (" --comment " + (" ".join(sys.argv)))
if not batch is None:
self.harry_args[-1] += " --dry-run"
if not n_fast_plots is None:
self.harry_args = self.harry_args[:n_fast_plots]
n_plots = len(self.harry_args)
self.harry_cores = [None]*n_plots
# multi processing of multiple plots
output_filenames = []
failed_plots = []
if (n_plots > 1) and (n_processes > 1):
log.info("Creating {:d} plots in {:d} processes".format(n_plots, min(n_processes, n_plots)))
results = tools.parallelize(pool_plot, zip([self]*n_plots, range(n_plots)), n_processes, description="Plotting")
tmp_output_filenames, tmp_failed_plots, tmp_error_messages = zip(*([result for result in results if not result is None and result != (None,)]))
output_filenames = [output_filename for output_filename in tmp_output_filenames if not output_filename is None]
failed_plots = [(failed_plot, error_message) for failed_plot, error_message in zip(tmp_failed_plots, tmp_error_messages) if not failed_plot is None]
# single processing of multiple plots
elif n_plots > 1:
log.info("Creating {:d} plots".format(n_plots))
for plot_index in xrange(n_plots):
try:
output_filenames.append(self.plot(plot_index))
except SystemExit as e:
failed_plots.append((self.harry_args[plot_index], None))
except Exception as e:
log.info(str(e))
failed_plots.append((self.harry_args[plot_index], None))
# single plot
elif n_plots > 0:
output_filenames.append(self.plot(0))
# batch submission
if (not (batch is None)) and (len(failed_plots) < n_plots):
try:
os.makedirs(os.path.expandvars("$HP_WORK_BASE"))
except OSError:
if not os.path.isdir(os.path.expandvars("$HP_WORK_BASE")):
raise
workdir = tempfile.mkdtemp(prefix="harry_work_"+datetime.datetime.now().strftime("%Y-%m-%d_%H-%M")+"_", dir=os.path.expandvars("$HP_WORK_BASE"))
main_config = ""
with open(os.path.expandvars("$CMSSW_BASE/src/Artus/HarryPlotter/data/grid-control_base_config.conf"), "r") as main_config_file:
main_config = main_config_file.read()
backend_config = ""
with open(os.path.expandvars("$CMSSW_BASE/src/Artus/Configuration/data/grid-control_backend_" + batch + ".conf"), "r") as backend_config_file:
backend_config = backend_config_file.read()
final_config = string.Template(main_config).safe_substitute(
cmsswbase=os.path.expandvars("$CMSSW_BASE"),
hpworkbase=os.path.expandvars("$HP_WORK_BASE"),
cwd=os.getcwd(),
jsonconfigs="\n\t"+("\n\t".join([item[0] for item in output_filenames])),
executable=self.standalone_executable,
workdir=workdir,
backend=backend_config
)
final_config_filename = workdir+".conf"
with open(final_config_filename, "w") as final_config_file:
final_config_file.write(final_config)
command = "go.py " + final_config_filename
log.info(command)
logger.subprocessCall(shlex.split(command))
if len(failed_plots) > 0:
log.error("%d failed plots:" % len(failed_plots))
for failed_plot in failed_plots:
log.info("\n"+tools.get_colored_string("Failed plot:", color='red'))
log.info("\t%s" % failed_plot[0])
if failed_plot[1] is not None:
try:
log.info(tools.get_indented_text(" ", tools.get_colored_string("Traceback for this plot:", color='red')+"\n" + failed_plot[1]))
except:
log.info("\t Traceback for this plot: \n" + failed_plot[1])
return output_filenames
def multi_plots(self, list_of_config_dicts, list_of_args_strings, n_processes=1, n_fast_plots=None):
config_dicts = list_of_config_dicts if isinstance(list_of_config_dicts, collections.Iterable) and not isinstance(list_of_config_dicts, basestring) else [list_of_config_dicts]
args_strings = list_of_args_strings if isinstance(list_of_args_strings, collections.Iterable) and not isinstance(list_of_args_strings, basestring) else [list_of_args_strings]
# fill up missing list elements by repeating previous ones
n_plots = max(len(config_dicts), len(args_strings))
if (len(config_dicts) > 1) and (len(config_dicts) < n_plots):
log.warning("Too few config dicts specified!")
if (len(args_strings) > 1) and (len(args_strings) < n_plots):
log.warning("Too few argument lists specified!")
config_dicts = (config_dicts*n_plots)[:n_plots]
args_strings = (args_strings*n_plots)[:n_plots]
if n_processes>1:
for i in range(len(args_strings)):
args_strings[i] += (" --hide-progressbar ")
self.harry_args = []
for config_dict, args_string in zip(config_dicts, args_strings):
if config_dict is None:
self.harry_args.append(None)
else:
config_dict["comment"] = " ".join(sys.argv)
if "json_defaults" in config_dict:
json_defaults_dict = jsonTools.JsonDict(config_dict["json_defaults"]).doIncludes().doComments()
config_dict.pop("json_defaults")
json_defaults_dict.update(config_dict)
config_dict = json_defaults_dict
self.harry_args.append("--json-defaults \"%s\"" % jsonTools.JsonDict(config_dict).toString(indent=None).replace("\"", "'"))
if not args_string is None:
if self.harry_args[-1] is None:
self.harry_args[-1] = args_string
else:
self.harry_args[-1] += (" "+args_string)
if config_dict is None:
self.harry_args[-1] += (" --comment " + (" ".join(sys.argv)))
if not n_fast_plots is None:
self.harry_args = self.harry_args[:n_fast_plots]
n_plots = len(self.harry_args)
self.harry_cores = [None]*n_plots
# multi processing of multiple plots
output_filenames = []
failed_plots = []
if (n_plots > 1) and (n_processes > 1):
log.info("Creating {:d} plots in {:d} processes".format(n_plots, min(n_processes, n_plots)))
results = tools.parallelize(pool_plot, zip([self]*n_plots, range(n_plots)), n_processes, description="Plotting")
tmp_output_filenames, tmp_failed_plots, tmp_error_messages = zip(*([result for result in results if not result is None and result != (None,)]))
output_filenames = [output_filename for output_filename in tmp_output_filenames if not output_filename is None]
failed_plots = [(failed_plot, error_message) for failed_plot, error_message in zip(tmp_failed_plots, tmp_error_messages) if not failed_plot is None]
# single processing of multiple plots
elif n_plots > 1:
log.info("Creating {:d} plots".format(n_plots))
for plot_index in xrange(n_plots):
try:
output_filenames.append(self.plot(plot_index))
except SystemExit as e:
failed_plots.append((self.harry_args[plot_index], None))
except Exception as e:
log.info(str(e))
failed_plots.append((self.harry_args[plot_index], None))
# single plot
elif n_plots > 0:
output_filenames.append(self.plot(0))
if len(failed_plots) > 0:
log.error("%d failed plots:" % len(failed_plots))
for failed_plot in failed_plots:
log.info("\n"+tools.get_colored_string("Failed plot:", color='red'))
log.info("\t%s" % failed_plot[0])
if failed_plot[1] is not None:
try:
log.info(tools.get_indented_text(" ", tools.get_colored_string("Traceback for this plot:", color='red')+"\n" + failed_plot[1]))
except:
log.info("\t Traceback for this plot: \n" + failed_plot[1])
return output_filenames
