def __init__(self, additional_modules_dirs=None, args_from_script=None, parser=None): super(HarryCore, self).__init__() # List of absolute paths to all module directories self._modules_dirs = [] # Dict of all available processors self.available_processors = {} # List of active processors self.processors = [] # First time parsing of cmd arguments if parser == None: parser = harryparser.HarryParser() self.parser = parser self._args_from_script = shlex.split(args_from_script) if args_from_script else None args, unknown_args = self.parser.parse_known_args(self._args_from_script) self.args = vars(args) # Default directories to be searched for plugins default_modules_dirs = ["input_modules/", "analysis_modules/", "plot_modules/"] # Modules search dir from command line arguments if self.args['modules_search_paths']: default_modules_dirs += self.args['modules_search_paths'] # Modules search dir from MODULES_SEARCH_PATH shell variable if tools.get_environment_variable('MODULES_SEARCH_PATH', fail_if_not_existing=False) is not None: default_modules_dirs += tools.get_environment_variable('MODULES_SEARCH_PATH').split(':') # Passed additonal modules dirs if additional_modules_dirs: default_modules_dirs += additional_modules_dirs for directory in default_modules_dirs: self.register_modules_dir(directory)
def save(self): self.plot.finish() if not self.plot is None: for filename in self.plotdict["output_filenames"]: self.plot.save(filename) log.info("Created plot \"%s\"." % filename) if not (self.plotdict["live"]==None): # if 'userpc', get the username and name of desktop machine if self.plotdict["userpc"]: userpc = ("%s@%s" % (tools.get_environment_variable("USER"), tools.get_environment_variable("HARRY_USERPC"))).replace("\n", "") for output_filename in self.plotdict["output_filenames"]: if self.plotdict["userpc"]: PlotData.show_plot_userpc(output_filename, self.plotdict["live"], tools.get_environment_variable('USER'), userpc) else: PlotData.show_plot(output_filename, self.plotdict["live"]) # web plotting # TODO: make this more configurable if users want to user other webspaces etc. if self.plotdict["www"] != None: self.webplotting( www = self.plotdict["www"], output_dir = self.plotdict["output_dir"], output_filenames = self.plotdict["output_filenames"], www_text= self.plotdict["www_text"], www_title = self.plotdict["www_title"], additional_output_files = self.plotdict["additional_output_files"] if "additional_output_files" in self.plotdict else False, save_legend = self.plotdict.get("save_legend", False), export_json = self.plotdict["export_json"] ) return self.plotdict["output_filenames"]
def save(self): self.plot.finish() if not self.plot is None: for filename in self.plotdict["output_filenames"]: self.plot.save(filename) log.info("Created plot \"%s\"." % filename) if not (self.plotdict["live"] == None): # if 'userpc', get the username and name of desktop machine if self.plotdict["userpc"]: userpc = ("%s@%s" % (tools.get_environment_variable("USER"), tools.get_environment_variable("HARRY_USERPC")) ).replace("\n", "") for output_filename in self.plotdict["output_filenames"]: if self.plotdict["userpc"]: PlotData.show_plot_userpc( output_filename, self.plotdict["live"], tools.get_environment_variable('USER'), userpc) else: PlotData.show_plot(output_filename, self.plotdict["live"]) # web plotting # TODO: make this more configurable if users want to user other webspaces etc. if self.plotdict["www"] is not None: try: remote_subdir = self.plotdict["output_dir"].split( 'websync/')[1] except: logger.warning( "Remote subdir may not reflect local gallery structure" ) remote_subdir = None self.webplotting( www=self.plotdict["www"], output_dir=self.plotdict["output_dir"], output_filenames=self.plotdict["output_filenames"], www_text=self.plotdict["www_text"], www_title=self.plotdict["www_title"], www_nodate=self.plotdict["www_nodate"], additional_output_files=self. plotdict["additional_output_files"] if "additional_output_files" in self.plotdict else False, save_legend=self.plotdict.get("save_legend", False), export_json=self.plotdict["export_json"], www_no_overwrite=self.plotdict["www_no_overwrite"] or self.plotdict["no_overwrite"], remote_subdir=remote_subdir, ) return self.plotdict["output_filenames"]
def save(self): self.plot.finish() if not self.plot is None: for filename in self.plotdict["output_filenames"]: self.plot.save(filename) log.info("Created plot \"%s\"." % filename) if not (self.plotdict["live"]==None): # if 'userpc', get the username and name of desktop machine if self.plotdict["userpc"]: userpc = ("%s@%s" % (tools.get_environment_variable("USER"), tools.get_environment_variable("HARRY_USERPC"))).replace("\n", "") for output_filename in self.plotdict["output_filenames"]: if self.plotdict["userpc"]: PlotData.show_plot_userpc(output_filename, self.plotdict["live"], tools.get_environment_variable('USER'), userpc) else: PlotData.show_plot(output_filename, self.plotdict["live"]) # web plotting # TODO: make this more configurable if users want to user other webspaces etc. if self.plotdict["www"] is not None: try: remote_subdir = self.plotdict["output_dir"].split('websync/')[1] except: logger.warning("Remote subdir may not reflect local gallery structure") remote_subdir = None self.webplotting( www=self.plotdict["www"], output_dir=self.plotdict["output_dir"], output_filenames=self.plotdict["output_filenames"], www_text=self.plotdict["www_text"], www_title=self.plotdict["www_title"], www_nodate=self.plotdict["www_nodate"], additional_output_files=self.plotdict["additional_output_files"] if "additional_output_files" in self.plotdict else False, save_legend=self.plotdict.get("save_legend", False), export_json=self.plotdict["export_json"], www_no_overwrite=self.plotdict["www_no_overwrite"] or self.plotdict["no_overwrite"], remote_subdir=remote_subdir, ) return self.plotdict["output_filenames"]
def parse_known_args(self, args=None, namespace=None): known_args, unknown_args = super( HarryParserZJet, self).parse_known_args(args=args, namespace=namespace) if known_args.list_functions: for plot_config_path in tools.get_environment_variable( "PLOTCONFIGS").split(":"): toolsZJet.print_plotting_functions(plot_config_path) sys.exit(0) return known_args, unknown_args
def main(): """This script calls the harry core or python plot functions.""" if len(sys.argv) == 1: sys.argv.append("-h") # the try-except is needed so -h doesnt exit at the parsing here try: parser = argparse.ArgumentParser( description="Merlin, a plotting tool derived\ from HarryPlotter. Please have a look at the documentation for Excalibur\ (https://github.com/KIT-CMS/Excalibur/blob/master/README.md)\ and for HarryPlotter (https://github.com/KIT-CMS/Artus/blob/master/HarryPlotter/README.md)." ) parser.add_argument( '--python', nargs='+', default=[None], help= "execute python function(s). Available functions can be listed with --list-functions" ) known_args, unknown_args = parser.parse_known_args() # call python config function if known_args.python != [None]: logger.initLogger() for function in known_args.python: toolsZJet.call_python_function( function, tools.get_environment_variable("PLOTCONFIGS"), unknown_args) else: harryZJet.HarryPlotterZJet( list_of_args_strings=" ".join(unknown_args)) except SystemExit, e: if '-h' in sys.argv or '--help' in sys.argv: harryZJet.HarryPlotterZJet()
def webplotting(www, output_dir, output_filenames=False, www_text = False, www_title="plots_archive", additional_output_files=False, save_legend=False, export_json = False, no_publish=False): # set some needed variables user = tools.get_environment_variable("HARRY_REMOTE_USER") html_content = "" overview_filename = "index.html" date = datetime.date.today().strftime('%Y_%m_%d') remote_subdir = os.path.expandvars(os.path.join(date, (www if type(www)==str else ""))) url = os.path.expandvars(os.path.join("$HARRY_URL", remote_subdir, overview_filename)) plots_for_gallery = [p for p in sorted(os.listdir(output_dir)) if (os.path.isfile(os.path.join(output_dir, p)) and all([not p.endswith("."+ext) for ext in ["json", "html", "root"]]))] # get the html templates files_to_copy = [] for galleryplot in plots_for_gallery: files_to_copy.append(os.path.join(output_dir, galleryplot)) html_texts = {} for var in ['overview', 'description', 'plot']: with open(os.path.expandvars("$ARTUSPATH/HarryPlotter/data/template_webplotting_{}.html".format(var))) as htmlfile: html_texts[var] = string.Template(htmlfile.read()) html_texts['description'] = html_texts['description'].substitute(url=url) if www_text: html_texts['description'] = www_text # loop over plots, make gallery (one entry for multiple formats) for plot in sorted(list(set([os.path.splitext(plot)[0] for plot in plots_for_gallery]))): formats = [os.path.splitext(p)[1] for p in plots_for_gallery if (plot == os.path.splitext(p)[0])] # use png for preview, if it exists image = plot + ('.png' if (plot +'.png' in plots_for_gallery) else formats[0]) # links for the different formats links = "" for fileformat in formats: links +=' <a href="{}">{}</a>'.format(plot+fileformat, fileformat[1:]) html_content += html_texts['plot'].substitute( title=plot, image=image, links=links, json=plot+".json" ) # put the html parts together and write with open(os.path.join(output_dir, overview_filename), "w") as overview_file: overview_file.write(html_texts['overview'].substitute( html_content=html_content, title=www_title, text=html_texts['description'] )) if no_publish: return 0 # find out which files to copy if(output_filenames != False): files_to_copy = ( output_filenames ) files_to_copy += [os.path.join(output_dir, overview_filename)] if additional_output_files != False: files_to_copy += additional_output_files if export_json != False: files_to_copy += [export_json] if save_legend: files_to_copy += [os.path.join(output_dir, save_legend + _format) for _format in formats] # create remote dir, copy files mkdir_command = os.path.expandvars("$WEB_PLOTTING_MKDIR_COMMAND").format(subdir=remote_subdir) copy_command = os.path.expandvars("$WEB_PLOTTING_COPY_COMMAND").format(source=" ".join(files_to_copy), subdir=remote_subdir) log.info("Copying plots to webspace...") log.debug("\nIssueing mkdir command: " + mkdir_command) logger.subprocessCall(mkdir_command.split()) log.debug("\nIssueing rsync command: " + copy_command) logger.subprocessCall(copy_command.split()) log.info("Copied {0}; see {1}".format(" ".join([f.split("/")[-1] for f in files_to_copy]), url))
def save(self): self.plot.finish() if not self.plot is None: for filename in self.plotdict["output_filenames"]: self.plot.save(filename) log.info('Created plot "%s".' % filename) if not (self.plotdict["live"] == None): # if 'userpc', get the username and name of desktop machine if self.plotdict["userpc"]: userpc = ( "%s@%s" % (tools.get_environment_variable("USER"), tools.get_environment_variable("HARRY_USERPC")) ).replace("\n", "") for output_filename in self.plotdict["output_filenames"]: if self.plotdict["userpc"]: extrafunctions.show_plot_userpc( output_filename, self.plotdict["live"], tools.get_environment_variable("USER"), userpc ) else: extrafunctions.show_plot(output_filename, self.plotdict["live"]) # web plotting # TODO: make this more configurable if users want to user other webspaces etc. if self.plotdict["www"] != None: # set some needed variables user = tools.get_environment_variable("HARRY_REMOTE_USER") overview_filename = "overview.html" date = datetime.date.today().strftime("%Y_%m_%d") remote_dir = tools.get_environment_variable("HARRY_REMOTE_DIR") + "/%s/%s/" % ( date, (self.plotdict["www"] if type(self.plotdict["www"]) == str else ""), ) remote_path = tools.get_environment_variable("HARRY_REMOTE_PATH") + "/%s" % remote_dir url = tools.get_environment_variable("HARRY_URL") + "/%s/%s" % (remote_dir, overview_filename) plots_for_gallery = [ p for p in sorted(os.listdir(self.plotdict["output_dir"])) if ((".png" in p) or (".pdf" in p)) ] plots_to_copy = [os.path.basename(filename), overview_filename] if self.plotdict.get("save_legend", False): for f in self.plotdict["formats"]: plots_to_copy.append(".".join([self.plotdict["save_legend"], f])) html_content = "" log.info("Copying plots to webspace...") # loop over plots, make gallery for plot in [p for p in plots_for_gallery if ((".png" in p) or (".pdf" in p))]: # try to link to pdf file, if it exists href = plot.replace(".png", ".pdf") if href not in plots_for_gallery: href = plot title = plot.split("/")[-1][:-4].replace("_", " ") html_content += htmlTemplatePlot % (title, href, title, plot) with open(self.plotdict["output_dir"] + "/" + overview_filename, "w") as f: f.write(htmlTemplate % (url, html_content)) # create remote dir, copy plots and overview file create_dir_command = ["ssh", tools.get_environment_variable("HARRY_SSHPC"), "mkdir -p", remote_path] log.debug("\nIssueing mkdir command: " + " ".join(create_dir_command)) subprocess.call(create_dir_command) rsync_command = ( ["rsync", "-u"] + [os.path.join(self.plotdict["output_dir"], p) for p in plots_to_copy] + ["%s:%s" % (tools.get_environment_variable("HARRY_SSHPC"), remote_path)] ) log.debug("\nIssueing rsync command: " + " ".join(rsync_command) + "\n") subprocess.call(rsync_command) log.info("Copied {0}; see {1}".format(filename.split("/")[-1], url)) return self.plotdict["output_filenames"]
def save(self): self.plot.finish() if not self.plot is None: for filename in self.plotdict["output_filenames"]: self.plot.save(filename) log.info('Created plot "%s".' % filename) if not (self.plotdict["live"] == None): # if 'userpc', get the username and name of desktop machine if self.plotdict["userpc"]: userpc = ( "%s@%s" % (tools.get_environment_variable("USER"), tools.get_environment_variable("HARRY_USERPC")) ).replace("\n", "") for output_filename in self.plotdict["output_filenames"]: if self.plotdict["userpc"]: extrafunctions.show_plot_userpc( output_filename, self.plotdict["live"], tools.get_environment_variable("USER"), userpc ) else: extrafunctions.show_plot(output_filename, self.plotdict["live"]) # web plotting # TODO: make this more configurable if users want to user other webspaces etc. if self.plotdict["www"] != None: # set some needed variables user = tools.get_environment_variable("HARRY_REMOTE_USER") overview_filename = "overview.html" date = datetime.date.today().strftime("%Y_%m_%d") remote_dir = os.path.expandvars( os.path.join( "$HARRY_REMOTE_DIR", date, (self.plotdict["www"] if type(self.plotdict["www"]) == str else "") ) ) remote_path = os.path.expandvars(os.path.join("$HARRY_REMOTE_PATH", remote_dir)) url = os.path.expandvars(os.path.join("$HARRY_URL", remote_dir, overview_filename)) plots_for_gallery = [ p for p in sorted(os.listdir(self.plotdict["output_dir"])) if ( os.path.isfile(os.path.join(self.plotdict["output_dir"], p)) and all([not p.endswith("." + ext) for ext in ["json", "html", "root"]]) ) ] html_content = "" log.info("Copying plots to webspace...") # loop over plots, make gallery html_template = None with open( os.path.expandvars("$ARTUSPATH/HarryPlotter/data/template_webplotting_overview.html") ) as html_template_file: html_template = string.Template(html_template_file.read()) html_template_plot = None with open( os.path.expandvars("$ARTUSPATH/HarryPlotter/data/template_webplotting_plot.html") ) as html_template_plot_file: html_template_plot = string.Template(html_template_plot_file.read()) for plot in [p for p in plots_for_gallery]: # try to link to pdf file, if it exists href = plot.replace(".png", ".pdf") if href not in plots_for_gallery: href = plot filename = os.path.splitext(plot)[0] html_content += html_template_plot.substitute( title=filename, href=href, plot=plot, json=filename + ".json" ) with open(os.path.join(self.plotdict["output_dir"], overview_filename), "w") as overview_file: overview_file.write(html_template.substitute(url=url, html_content=html_content)) # create remote dir, copy plots and overview file sshpc = tools.get_environment_variable("HARRY_SSHPC") create_dir_command = ["ssh", user + "@" + sshpc, "mkdir -p", remote_path] log.debug("\nIssueing mkdir command: " + " ".join(create_dir_command)) logger.subprocessCall(create_dir_command) rsync_command = ( ["rsync", "-u"] + glob.glob(os.path.join(self.plotdict["output_dir"], "*.*")) + ["%s@%s:%s" % (user, sshpc, remote_path)] ) log.debug("\nIssueing rsync command: " + " ".join(rsync_command) + "\n") logger.subprocessCall(rsync_command) log.info("Copied {0}; see {1}".format(filename.split("/")[-1], url)) return self.plotdict["output_filenames"]