def _print_module_list(self, module_list): """Print a list of modules (name and docstring)""" for module in module_list: log.info("\t" + tools.get_colored_string("{}".format(module), "green")) if inspect.getdoc(self.available_processors[module]): log.info( tools.get_indented_text( "\t\t", inspect.getdoc(self.available_processors[module])))
def print_comments_from_json_files(json_dir, comment_key): """print comments from all json files in a directory""" #TODO generalize this. works with any string, not only comments files = [os.path.join(json_dir, f) for f in os.listdir(json_dir) if f.endswith(".json")] log.info(tools.get_colored_string("json files: ({})".format(json_dir), 'cyan')) for f in files: d = JsonDict(f).doIncludes().doComments() string = "\t" + tools.get_colored_string(os.path.basename(f), "yellow") if comment_key in d: string += tools.get_indented_text("\n\t\t", d[comment_key]) log.info(string)
def print_comments_from_json_files(json_dir, comment_key): """print comments from all json files in a directory""" #TODO generalize this. works with any string, not only comments files = [ os.path.join(json_dir, f) for f in os.listdir(json_dir) if f.endswith(".json") ] log.info( tools.get_colored_string("json files: ({})".format(json_dir), 'cyan')) for f in files: d = JsonDict(f).doIncludes().doComments() string = "\t" + tools.get_colored_string(os.path.basename(f), "yellow") if comment_key in d: string += tools.get_indented_text("\n\t\t", d[comment_key]) log.info(string)
def print_plotting_functions(plotting_path): """ print the comments / docstrings of the plotting configs""" log.info( tools.get_colored_string( "plotting scripts: ({})".format(plotting_path), 'cyan')) # get docstrings module_list = get_module_list(plotting_path) for module in module_list: log.info(tools.get_colored_string(module.__name__ + ".py", "yellow")) functions = inspect.getmembers(module, inspect.isfunction) if len(functions) > 0: prefix = "\t\t" for func in functions: if func[0].startswith("_"): continue log.info("\t" + tools.get_colored_string(func[0], "green")) if inspect.getdoc(func[1]) != None: # has docstring log.info( tools.get_indented_text(prefix, inspect.getdoc(func[1])))
def multi_plots(self, list_of_config_dicts, list_of_args_strings, n_processes=1, n_fast_plots=None): config_dicts = list_of_config_dicts if isinstance(list_of_config_dicts, collections.Iterable) and not isinstance(list_of_config_dicts, basestring) else [list_of_config_dicts] args_strings = list_of_args_strings if isinstance(list_of_args_strings, collections.Iterable) and not isinstance(list_of_args_strings, basestring) else [list_of_args_strings] # fill up missing list elements by repeating previous ones n_plots = max(len(config_dicts), len(args_strings)) if (len(config_dicts) > 1) and (len(config_dicts) < n_plots): log.warning("Too few config dicts specified!") if (len(args_strings) > 1) and (len(args_strings) < n_plots): log.warning("Too few argument lists specified!") config_dicts = (config_dicts*n_plots)[:n_plots] args_strings = (args_strings*n_plots)[:n_plots] harry_args = [] for config_dict, args_string in zip(config_dicts, args_strings): if config_dict is None: harry_args.append(None) else: config_dict["comment"] = " ".join(sys.argv) if "json_defaults" in config_dict: json_defaults_dict = jsonTools.JsonDict(config_dict["json_defaults"]).doIncludes().doComments() config_dict.pop("json_defaults") json_defaults_dict.update(config_dict) config_dict = json_defaults_dict harry_args.append("--json-defaults \"%s\"" % jsonTools.JsonDict(config_dict).toString(indent=None).replace("\"", "'")) if not args_string is None: if harry_args[-1] is None: harry_args[-1] = args_string else: harry_args[-1] += (" "+args_string) if config_dict is None: harry_args[-1] += (" --comment " + (" ".join(sys.argv))) if not n_fast_plots is None: harry_args = harry_args[:n_fast_plots] # multi processing of multiple plots output_filenames = [] failed_plots = [] if len(harry_args) > 1 and n_processes > 1: log.info("Creating {:d} plots in {:d} processes".format(len(harry_args), min(n_processes, len(harry_args)))) results = tools.parallelize(pool_plot, zip([self]*len(harry_args), harry_args), n_processes) tmp_output_filenames, tmp_failed_plots, tmp_error_messages = zip(*([result for result in results if not result is None and result != (None,)])) output_filenames = [output_filename for output_filename in tmp_output_filenames if not output_filename is None] failed_plots = [(failed_plot, error_message) for failed_plot, error_message in zip(tmp_failed_plots, tmp_error_messages) if not failed_plot is None] # single processing of multiple plots elif len(harry_args) > 1: log.info("Creating {:d} plots".format(len(harry_args))) for harry_args in harry_args: try: output_filenames.append(self.plot(harry_args)) except SystemExit as e: failed_plots.append((harry_args, None)) except Exception as e: log.info(str(e)) failed_plots.append((harry_args, None)) # single plot elif len(harry_args) > 0: output_filenames.append(self.plot(harry_args[0])) if len(failed_plots) > 0: log.error("%d failed plots:" % len(failed_plots)) for failed_plot in failed_plots: log.info("\n"+tools.get_colored_string("Failed plot:", color='red')) log.info("\t%s" % failed_plot[0]) if failed_plot[1] is not None: try: log.info(tools.get_indented_text(" ", tools.get_colored_string("Traceback for this plot:", color='red')+"\n" + failed_plot[1])) except: log.info("\t Traceback for this plot: \n" + failed_plot[1]) return output_filenames
def _print_module_list(self, module_list): """Print a list of modules (name and docstring)""" for module in module_list: log.info("\t"+tools.get_colored_string("{}".format(module), "green")) if inspect.getdoc(self.available_processors[module]): log.info(tools.get_indented_text("\t\t", inspect.getdoc(self.available_processors[module])))
def multi_plots(self, list_of_config_dicts, list_of_args_strings, n_processes=1, n_fast_plots=None, batch=None): config_dicts = list_of_config_dicts if isinstance( list_of_config_dicts, collections.Iterable) and not isinstance( list_of_config_dicts, basestring) else [list_of_config_dicts] args_strings = list_of_args_strings if isinstance( list_of_args_strings, collections.Iterable) and not isinstance( list_of_args_strings, basestring) else [list_of_args_strings] # fill up missing list elements by repeating previous ones n_plots = max(len(config_dicts), len(args_strings)) if (len(config_dicts) > 1) and (len(config_dicts) < n_plots): log.warning("Too few config dicts specified!") if (len(args_strings) > 1) and (len(args_strings) < n_plots): log.warning("Too few argument lists specified!") config_dicts = (config_dicts * n_plots)[:n_plots] args_strings = (args_strings * n_plots)[:n_plots] if n_processes > 1: for i in range(len(args_strings)): args_strings[i] += (" --hide-progressbar ") self.harry_args = [] for config_dict, args_string in zip(config_dicts, args_strings): if config_dict is None: self.harry_args.append(None) else: config_dict["comment"] = " ".join(sys.argv) if not batch is None: config_dict["dry_run"] = True if "json_defaults" in config_dict: json_defaults_dict = jsonTools.JsonDict( config_dict["json_defaults"]).doIncludes().doComments( ) config_dict.pop("json_defaults") json_defaults_dict.update(config_dict) config_dict = json_defaults_dict self.harry_args.append( "--json-defaults \"%s\"" % jsonTools.JsonDict(config_dict).toString( indent=None).replace("\"", "'")) if not args_string is None: if self.harry_args[-1] is None: self.harry_args[-1] = args_string else: self.harry_args[-1] += (" " + args_string) if config_dict is None: self.harry_args[-1] += (" --comment " + (" ".join(sys.argv))) if not batch is None: self.harry_args[-1] += " --dry-run" if not n_fast_plots is None: self.harry_args = self.harry_args[:n_fast_plots] n_plots = len(self.harry_args) self.harry_cores = [None] * n_plots # multi processing of multiple plots output_filenames = [] failed_plots = [] if (n_plots > 1) and (n_processes > 1): log.info("Creating {:d} plots in {:d} processes".format( n_plots, min(n_processes, n_plots))) results = tools.parallelize(pool_plot, zip([self] * n_plots, range(n_plots)), n_processes, description="Plotting") tmp_output_filenames, tmp_failed_plots, tmp_error_messages = zip( *([ result for result in results if not result is None and result != (None, ) ])) output_filenames = [ output_filename for output_filename in tmp_output_filenames if not output_filename is None ] failed_plots = [(failed_plot, error_message) for failed_plot, error_message in zip( tmp_failed_plots, tmp_error_messages) if not failed_plot is None] # single processing of multiple plots elif n_plots > 1: log.info("Creating {:d} plots".format(n_plots)) for plot_index in xrange(n_plots): try: output_filenames.append(self.plot(plot_index)) except SystemExit as e: failed_plots.append((self.harry_args[plot_index], None)) except Exception as e: log.info(str(e)) failed_plots.append((self.harry_args[plot_index], None)) # single plot elif n_plots > 0: output_filenames.append(self.plot(0)) # batch submission if (not (batch is None)) and (len(failed_plots) < n_plots): workdir = tempfile.mkdtemp( prefix="harry_work_" + datetime.datetime.now().strftime("%Y-%m-%d_%H-%M") + "_") main_config = "" with open( os.path.expandvars( "$CMSSW_BASE/src/Artus/HarryPlotter/data/grid-control_base_config.conf" ), "r") as main_config_file: main_config = main_config_file.read() backend_config = "" with open( os.path.expandvars( "$CMSSW_BASE/src/Artus/Configuration/data/grid-control_backend_" + batch + ".conf"), "r") as backend_config_file: backend_config = backend_config_file.read() final_config = string.Template(main_config).safe_substitute( cmsswbase=os.path.expandvars("$CMSSW_BASE"), hpworkbase=os.path.expandvars("$HP_WORK_BASE"), cwd=os.getcwd(), jsonconfigs="\n\t" + ("\n\t".join([item[0] for item in output_filenames])), executable=self.standalone_executable, workdir=workdir, backend=backend_config) final_config_filename = workdir + ".conf" with open(final_config_filename, "w") as final_config_file: final_config_file.write(final_config) command = "go.py " + final_config_filename log.info(command) logger.subprocessCall(shlex.split(command)) if len(failed_plots) > 0: log.error("%d failed plots:" % len(failed_plots)) for failed_plot in failed_plots: log.info("\n" + tools.get_colored_string("Failed plot:", color='red')) log.info("\t%s" % failed_plot[0]) if failed_plot[1] is not None: try: log.info( tools.get_indented_text( " ", tools.get_colored_string( "Traceback for this plot:", color='red') + "\n" + failed_plot[1])) except: log.info("\t Traceback for this plot: \n" + failed_plot[1]) return output_filenames
def multi_plots(self, list_of_config_dicts, list_of_args_strings, n_processes=1, n_fast_plots=None): config_dicts = list_of_config_dicts if isinstance(list_of_config_dicts, collections.Iterable) and not isinstance(list_of_config_dicts, basestring) else [list_of_config_dicts] args_strings = list_of_args_strings if isinstance(list_of_args_strings, collections.Iterable) and not isinstance(list_of_args_strings, basestring) else [list_of_args_strings] # fill up missing list elements by repeating previous ones n_plots = max(len(config_dicts), len(args_strings)) if (len(config_dicts) > 1) and (len(config_dicts) < n_plots): log.warning("Too few config dicts specified!") if (len(args_strings) > 1) and (len(args_strings) < n_plots): log.warning("Too few argument lists specified!") config_dicts = (config_dicts*n_plots)[:n_plots] args_strings = (args_strings*n_plots)[:n_plots] harry_args = [] for config_dict, args_string in zip(config_dicts, args_strings): if config_dict is None: harry_args.append(None) else: config_dict["comment"] = " ".join(sys.argv) if "json_defaults" in config_dict: json_defaults_dict = jsonTools.JsonDict(config_dict["json_defaults"]).doIncludes().doComments() config_dict.pop("json_defaults") json_defaults_dict.update(config_dict) config_dict = json_defaults_dict harry_args.append("--json-defaults \"%s\"" % jsonTools.JsonDict(config_dict).toString(indent=None).replace("\"", "'")) if not args_string is None: if harry_args[-1] is None: harry_args[-1] = args_string else: harry_args[-1] += (" "+args_string) if config_dict is None: harry_args[-1] += (" --comment " + (" ".join(sys.argv))) if not n_fast_plots is None: harry_args = harry_args[:n_fast_plots] # multi processing of multiple plots output_filenames = [] failed_plots = [] if len(harry_args) > 1 and n_processes > 1: log.info("Creating {:d} plots in {:d} processes".format(len(harry_args), n_processes)) pool = Pool(processes=n_processes) results = pool.map_async(pool_plot, zip([self]*len(harry_args), harry_args)) # 9999999 is needed for KeyboardInterrupt to work: http://stackoverflow.com/questions/1408356/keyboard-interrupts-with-pythons-multiprocessing-pool tmp_output_filenames, tmp_failed_plots, tmp_error_messages = zip(*([result for result in results.get(9999999) if not result is None and result != (None,)])) output_filenames = [output_filename for output_filename in tmp_output_filenames if not output_filename is None] failed_plots = [(failed_plot, error_message) for failed_plot, error_message in zip(tmp_failed_plots, tmp_error_messages) if not failed_plot is None] # single processing of multiple plots elif len(harry_args) > 1: log.info("Creating {:d} plots".format(len(harry_args))) for harry_args in harry_args: try: output_filenames.append(self.plot(harry_args)) except SystemExit as e: failed_plots.append((harry_args, None)) except Exception as e: log.info(str(e)) failed_plots.append((harry_args, None)) # single plot elif len(harry_args) > 0: output_filenames.append(self.plot(harry_args[0])) if len(failed_plots) > 0: log.error("%d failed plots:" % len(failed_plots)) for failed_plot in failed_plots: log.info("\n"+tools.get_colored_string("Failed plot:", color='red')) log.info("\t%s" % failed_plot[0]) if failed_plot[1] is not None: log.info(tools.get_indented_text(" ", tools.get_colored_string("Traceback for this plot:", color='red')+"\n" + failed_plot[1])) return output_filenames
def multi_plots(self, list_of_config_dicts, list_of_args_strings, n_processes=1, n_fast_plots=None, batch=None): config_dicts = list_of_config_dicts if isinstance(list_of_config_dicts, collections.Iterable) and not isinstance(list_of_config_dicts, basestring) else [list_of_config_dicts] args_strings = list_of_args_strings if isinstance(list_of_args_strings, collections.Iterable) and not isinstance(list_of_args_strings, basestring) else [list_of_args_strings] # fill up missing list elements by repeating previous ones n_plots = max(len(config_dicts), len(args_strings)) if (len(config_dicts) > 1) and (len(config_dicts) < n_plots): log.warning("Too few config dicts specified!") if (len(args_strings) > 1) and (len(args_strings) < n_plots): log.warning("Too few argument lists specified!") config_dicts = (config_dicts*n_plots)[:n_plots] args_strings = (args_strings*n_plots)[:n_plots] if n_processes>1: for i in range(len(args_strings)): args_strings[i] += (" --hide-progressbar ") self.harry_args = [] for config_dict, args_string in zip(config_dicts, args_strings): if config_dict is None: self.harry_args.append(None) else: config_dict["comment"] = " ".join(sys.argv) if not batch is None: config_dict["dry_run"] = True if "json_defaults" in config_dict: json_defaults_dict = jsonTools.JsonDict(config_dict["json_defaults"]).doIncludes().doComments() config_dict.pop("json_defaults") json_defaults_dict.update(config_dict) config_dict = json_defaults_dict self.harry_args.append("--json-defaults \"%s\"" % jsonTools.JsonDict(config_dict).toString(indent=None).replace("\"", "'")) if not args_string is None: if self.harry_args[-1] is None: self.harry_args[-1] = args_string else: self.harry_args[-1] += (" "+args_string) if config_dict is None: self.harry_args[-1] += (" --comment " + (" ".join(sys.argv))) if not batch is None: self.harry_args[-1] += " --dry-run" if not n_fast_plots is None: self.harry_args = self.harry_args[:n_fast_plots] n_plots = len(self.harry_args) self.harry_cores = [None]*n_plots # multi processing of multiple plots output_filenames = [] failed_plots = [] if (n_plots > 1) and (n_processes > 1): log.info("Creating {:d} plots in {:d} processes".format(n_plots, min(n_processes, n_plots))) results = tools.parallelize(pool_plot, zip([self]*n_plots, range(n_plots)), n_processes, description="Plotting") tmp_output_filenames, tmp_failed_plots, tmp_error_messages = zip(*([result for result in results if not result is None and result != (None,)])) output_filenames = [output_filename for output_filename in tmp_output_filenames if not output_filename is None] failed_plots = [(failed_plot, error_message) for failed_plot, error_message in zip(tmp_failed_plots, tmp_error_messages) if not failed_plot is None] # single processing of multiple plots elif n_plots > 1: log.info("Creating {:d} plots".format(n_plots)) for plot_index in xrange(n_plots): try: output_filenames.append(self.plot(plot_index)) except SystemExit as e: failed_plots.append((self.harry_args[plot_index], None)) except Exception as e: log.info(str(e)) failed_plots.append((self.harry_args[plot_index], None)) # single plot elif n_plots > 0: output_filenames.append(self.plot(0)) # batch submission if (not (batch is None)) and (len(failed_plots) < n_plots): try: os.makedirs(os.path.expandvars("$HP_WORK_BASE")) except OSError: if not os.path.isdir(os.path.expandvars("$HP_WORK_BASE")): raise workdir = tempfile.mkdtemp(prefix="harry_work_"+datetime.datetime.now().strftime("%Y-%m-%d_%H-%M")+"_", dir=os.path.expandvars("$HP_WORK_BASE")) main_config = "" with open(os.path.expandvars("$CMSSW_BASE/src/Artus/HarryPlotter/data/grid-control_base_config.conf"), "r") as main_config_file: main_config = main_config_file.read() backend_config = "" with open(os.path.expandvars("$CMSSW_BASE/src/Artus/Configuration/data/grid-control_backend_" + batch + ".conf"), "r") as backend_config_file: backend_config = backend_config_file.read() final_config = string.Template(main_config).safe_substitute( cmsswbase=os.path.expandvars("$CMSSW_BASE"), hpworkbase=os.path.expandvars("$HP_WORK_BASE"), cwd=os.getcwd(), jsonconfigs="\n\t"+("\n\t".join([item[0] for item in output_filenames])), executable=self.standalone_executable, workdir=workdir, backend=backend_config ) final_config_filename = workdir+".conf" with open(final_config_filename, "w") as final_config_file: final_config_file.write(final_config) command = "go.py " + final_config_filename log.info(command) logger.subprocessCall(shlex.split(command)) if len(failed_plots) > 0: log.error("%d failed plots:" % len(failed_plots)) for failed_plot in failed_plots: log.info("\n"+tools.get_colored_string("Failed plot:", color='red')) log.info("\t%s" % failed_plot[0]) if failed_plot[1] is not None: try: log.info(tools.get_indented_text(" ", tools.get_colored_string("Traceback for this plot:", color='red')+"\n" + failed_plot[1])) except: log.info("\t Traceback for this plot: \n" + failed_plot[1]) return output_filenames
def multi_plots(self, list_of_config_dicts, list_of_args_strings, n_processes=1, n_fast_plots=None): config_dicts = list_of_config_dicts if isinstance(list_of_config_dicts, collections.Iterable) and not isinstance(list_of_config_dicts, basestring) else [list_of_config_dicts] args_strings = list_of_args_strings if isinstance(list_of_args_strings, collections.Iterable) and not isinstance(list_of_args_strings, basestring) else [list_of_args_strings] # fill up missing list elements by repeating previous ones n_plots = max(len(config_dicts), len(args_strings)) if (len(config_dicts) > 1) and (len(config_dicts) < n_plots): log.warning("Too few config dicts specified!") if (len(args_strings) > 1) and (len(args_strings) < n_plots): log.warning("Too few argument lists specified!") config_dicts = (config_dicts*n_plots)[:n_plots] args_strings = (args_strings*n_plots)[:n_plots] if n_processes>1: for i in range(len(args_strings)): args_strings[i] += (" --hide-progressbar ") self.harry_args = [] for config_dict, args_string in zip(config_dicts, args_strings): if config_dict is None: self.harry_args.append(None) else: config_dict["comment"] = " ".join(sys.argv) if "json_defaults" in config_dict: json_defaults_dict = jsonTools.JsonDict(config_dict["json_defaults"]).doIncludes().doComments() config_dict.pop("json_defaults") json_defaults_dict.update(config_dict) config_dict = json_defaults_dict self.harry_args.append("--json-defaults \"%s\"" % jsonTools.JsonDict(config_dict).toString(indent=None).replace("\"", "'")) if not args_string is None: if self.harry_args[-1] is None: self.harry_args[-1] = args_string else: self.harry_args[-1] += (" "+args_string) if config_dict is None: self.harry_args[-1] += (" --comment " + (" ".join(sys.argv))) if not n_fast_plots is None: self.harry_args = self.harry_args[:n_fast_plots] n_plots = len(self.harry_args) self.harry_cores = [None]*n_plots # multi processing of multiple plots output_filenames = [] failed_plots = [] if (n_plots > 1) and (n_processes > 1): log.info("Creating {:d} plots in {:d} processes".format(n_plots, min(n_processes, n_plots))) results = tools.parallelize(pool_plot, zip([self]*n_plots, range(n_plots)), n_processes, description="Plotting") tmp_output_filenames, tmp_failed_plots, tmp_error_messages = zip(*([result for result in results if not result is None and result != (None,)])) output_filenames = [output_filename for output_filename in tmp_output_filenames if not output_filename is None] failed_plots = [(failed_plot, error_message) for failed_plot, error_message in zip(tmp_failed_plots, tmp_error_messages) if not failed_plot is None] # single processing of multiple plots elif n_plots > 1: log.info("Creating {:d} plots".format(n_plots)) for plot_index in xrange(n_plots): try: output_filenames.append(self.plot(plot_index)) except SystemExit as e: failed_plots.append((self.harry_args[plot_index], None)) except Exception as e: log.info(str(e)) failed_plots.append((self.harry_args[plot_index], None)) # single plot elif n_plots > 0: output_filenames.append(self.plot(0)) if len(failed_plots) > 0: log.error("%d failed plots:" % len(failed_plots)) for failed_plot in failed_plots: log.info("\n"+tools.get_colored_string("Failed plot:", color='red')) log.info("\t%s" % failed_plot[0]) if failed_plot[1] is not None: try: log.info(tools.get_indented_text(" ", tools.get_colored_string("Traceback for this plot:", color='red')+"\n" + failed_plot[1])) except: log.info("\t Traceback for this plot: \n" + failed_plot[1]) return output_filenames