def __init__(self, additional_modules_dirs=None, args_from_script=None, parser=None):
super(HarryCore, self).__init__()
# List of absolute paths to all module directories
self._modules_dirs = []
# Dict of all available processors
self.available_processors = {}
# List of active processors
self.processors = []
# First time parsing of cmd arguments
if parser == None:
parser = harryparser.HarryParser()
self.parser = parser
self._args_from_script = shlex.split(args_from_script) if args_from_script else None
args, unknown_args = self.parser.parse_known_args(self._args_from_script)
self.args = vars(args)
# Default directories to be searched for plugins
default_modules_dirs = ["input_modules/", "analysis_modules/", "plot_modules/"]
# Modules search dir from command line arguments
if self.args['modules_search_paths']:
default_modules_dirs += self.args['modules_search_paths']
# Modules search dir from MODULES_SEARCH_PATH shell variable
if tools.get_environment_variable('MODULES_SEARCH_PATH', fail_if_not_existing=False) is not None:
default_modules_dirs += tools.get_environment_variable('MODULES_SEARCH_PATH').split(':')
# Passed additonal modules dirs
if additional_modules_dirs:
default_modules_dirs += additional_modules_dirs
for directory in default_modules_dirs:
self.register_modules_dir(directory)
def save(self):
self.plot.finish()
if not self.plot is None:
for filename in self.plotdict["output_filenames"]:
self.plot.save(filename)
log.info("Created plot \"%s\"." % filename)
if not (self.plotdict["live"]==None):
# if 'userpc', get the username and name of desktop machine
if self.plotdict["userpc"]:
userpc = ("%s@%s" % (tools.get_environment_variable("USER"), tools.get_environment_variable("HARRY_USERPC"))).replace("\n", "")
for output_filename in self.plotdict["output_filenames"]:
if self.plotdict["userpc"]:
PlotData.show_plot_userpc(output_filename, self.plotdict["live"], tools.get_environment_variable('USER'), userpc)
else:
PlotData.show_plot(output_filename, self.plotdict["live"])
# web plotting
# TODO: make this more configurable if users want to user other webspaces etc.
if self.plotdict["www"] != None:
self.webplotting(
www = self.plotdict["www"],
output_dir = self.plotdict["output_dir"],
output_filenames = self.plotdict["output_filenames"],
www_text= self.plotdict["www_text"],
www_title = self.plotdict["www_title"],
additional_output_files = self.plotdict["additional_output_files"] if "additional_output_files" in self.plotdict else False,
save_legend = self.plotdict.get("save_legend", False),
export_json = self.plotdict["export_json"]
)
return self.plotdict["output_filenames"]
def __init__(self, additional_modules_dirs=None, args_from_script=None, parser=None):
super(HarryCore, self).__init__()
# List of absolute paths to all module directories
self._modules_dirs = []
# Dict of all available processors
self.available_processors = {}
# List of active processors
self.processors = []
# First time parsing of cmd arguments
if parser == None:
parser = harryparser.HarryParser()
self.parser = parser
self._args_from_script = shlex.split(args_from_script) if args_from_script else None
args, unknown_args = self.parser.parse_known_args(self._args_from_script)
self.args = vars(args)
# Default directories to be searched for plugins
default_modules_dirs = ["input_modules/", "analysis_modules/", "plot_modules/"]
# Modules search dir from command line arguments
if self.args['modules_search_paths']:
default_modules_dirs += self.args['modules_search_paths']
# Modules search dir from MODULES_SEARCH_PATH shell variable
if tools.get_environment_variable('MODULES_SEARCH_PATH', fail_if_not_existing=False) is not None:
default_modules_dirs += tools.get_environment_variable('MODULES_SEARCH_PATH').split(':')
# Passed additonal modules dirs
if additional_modules_dirs:
default_modules_dirs += additional_modules_dirs
for directory in default_modules_dirs:
self.register_modules_dir(directory)
def save(self):
self.plot.finish()
if not self.plot is None:
for filename in self.plotdict["output_filenames"]:
self.plot.save(filename)
log.info("Created plot \"%s\"." % filename)
if not (self.plotdict["live"] == None):
# if 'userpc', get the username and name of desktop machine
if self.plotdict["userpc"]:
userpc = ("%s@%s" %
(tools.get_environment_variable("USER"),
tools.get_environment_variable("HARRY_USERPC"))
).replace("\n", "")
for output_filename in self.plotdict["output_filenames"]:
if self.plotdict["userpc"]:
PlotData.show_plot_userpc(
output_filename, self.plotdict["live"],
tools.get_environment_variable('USER'), userpc)
else:
PlotData.show_plot(output_filename,
self.plotdict["live"])
# web plotting
# TODO: make this more configurable if users want to user other webspaces etc.
if self.plotdict["www"] is not None:
try:
remote_subdir = self.plotdict["output_dir"].split(
'websync/')[1]
except:
logger.warning(
"Remote subdir may not reflect local gallery structure"
)
remote_subdir = None
self.webplotting(
www=self.plotdict["www"],
output_dir=self.plotdict["output_dir"],
output_filenames=self.plotdict["output_filenames"],
www_text=self.plotdict["www_text"],
www_title=self.plotdict["www_title"],
www_nodate=self.plotdict["www_nodate"],
additional_output_files=self.
plotdict["additional_output_files"]
if "additional_output_files" in self.plotdict else False,
save_legend=self.plotdict.get("save_legend", False),
export_json=self.plotdict["export_json"],
www_no_overwrite=self.plotdict["www_no_overwrite"]
or self.plotdict["no_overwrite"],
remote_subdir=remote_subdir,
)
return self.plotdict["output_filenames"]
def save(self):
self.plot.finish()
if not self.plot is None:
for filename in self.plotdict["output_filenames"]:
self.plot.save(filename)
log.info("Created plot \"%s\"." % filename)
if not (self.plotdict["live"]==None):
# if 'userpc', get the username and name of desktop machine
if self.plotdict["userpc"]:
userpc = ("%s@%s" % (tools.get_environment_variable("USER"), tools.get_environment_variable("HARRY_USERPC"))).replace("\n", "")
for output_filename in self.plotdict["output_filenames"]:
if self.plotdict["userpc"]:
PlotData.show_plot_userpc(output_filename, self.plotdict["live"], tools.get_environment_variable('USER'), userpc)
else:
PlotData.show_plot(output_filename, self.plotdict["live"])
# web plotting
# TODO: make this more configurable if users want to user other webspaces etc.
if self.plotdict["www"] is not None:
try:
remote_subdir = self.plotdict["output_dir"].split('websync/')[1]
except:
logger.warning("Remote subdir may not reflect local gallery structure")
remote_subdir = None
self.webplotting(
www=self.plotdict["www"],
output_dir=self.plotdict["output_dir"],
output_filenames=self.plotdict["output_filenames"],
www_text=self.plotdict["www_text"],
www_title=self.plotdict["www_title"],
www_nodate=self.plotdict["www_nodate"],
additional_output_files=self.plotdict["additional_output_files"] if "additional_output_files" in self.plotdict else False,
save_legend=self.plotdict.get("save_legend", False),
export_json=self.plotdict["export_json"],
www_no_overwrite=self.plotdict["www_no_overwrite"] or self.plotdict["no_overwrite"],
remote_subdir=remote_subdir,
)
return self.plotdict["output_filenames"]
def parse_known_args(self, args=None, namespace=None):
known_args, unknown_args = super(
HarryParserZJet, self).parse_known_args(args=args,
namespace=namespace)
if known_args.list_functions:
for plot_config_path in tools.get_environment_variable(
"PLOTCONFIGS").split(":"):
toolsZJet.print_plotting_functions(plot_config_path)
sys.exit(0)
return known_args, unknown_args
def main():
"""This script calls the harry core or python plot functions."""
if len(sys.argv) == 1:
sys.argv.append("-h")
# the try-except is needed so -h doesnt exit at the parsing here
try:
parser = argparse.ArgumentParser(
description="Merlin, a plotting tool derived\
from HarryPlotter. Please have a look at the documentation for Excalibur\
(https://github.com/KIT-CMS/Excalibur/blob/master/README.md)\
and for HarryPlotter (https://github.com/KIT-CMS/Artus/blob/master/HarryPlotter/README.md)."
)
parser.add_argument(
'--python',
nargs='+',
default=[None],
help=
"execute python function(s). Available functions can be listed with --list-functions"
)
known_args, unknown_args = parser.parse_known_args()
# call python config function
if known_args.python != [None]:
logger.initLogger()
for function in known_args.python:
toolsZJet.call_python_function(
function, tools.get_environment_variable("PLOTCONFIGS"),
unknown_args)
else:
harryZJet.HarryPlotterZJet(
list_of_args_strings=" ".join(unknown_args))
except SystemExit, e:
if '-h' in sys.argv or '--help' in sys.argv:
harryZJet.HarryPlotterZJet()
def webplotting(www, output_dir, output_filenames=False, www_text = False, www_title="plots_archive", additional_output_files=False, save_legend=False, export_json = False, no_publish=False):
# set some needed variables
user = tools.get_environment_variable("HARRY_REMOTE_USER")
html_content = ""
overview_filename = "index.html"
date = datetime.date.today().strftime('%Y_%m_%d')
remote_subdir = os.path.expandvars(os.path.join(date, (www if type(www)==str else "")))
url = os.path.expandvars(os.path.join("$HARRY_URL", remote_subdir, overview_filename))
plots_for_gallery = [p for p in sorted(os.listdir(output_dir)) if (os.path.isfile(os.path.join(output_dir, p)) and all([not p.endswith("."+ext) for ext in ["json", "html", "root"]]))]
# get the html templates
files_to_copy = []
for galleryplot in plots_for_gallery:
files_to_copy.append(os.path.join(output_dir, galleryplot))
html_texts = {}
for var in ['overview', 'description', 'plot']:
with open(os.path.expandvars("$ARTUSPATH/HarryPlotter/data/template_webplotting_{}.html".format(var))) as htmlfile:
html_texts[var] = string.Template(htmlfile.read())
html_texts['description'] = html_texts['description'].substitute(url=url)
if www_text:
html_texts['description'] = www_text
# loop over plots, make gallery (one entry for multiple formats)
for plot in sorted(list(set([os.path.splitext(plot)[0] for plot in plots_for_gallery]))):
formats = [os.path.splitext(p)[1] for p in plots_for_gallery if (plot == os.path.splitext(p)[0])]
# use png for preview, if it exists
image = plot + ('.png' if (plot +'.png' in plots_for_gallery) else formats[0])
# links for the different formats
links = ""
for fileformat in formats:
links +=' <a href="{}">{}</a>'.format(plot+fileformat, fileformat[1:])
html_content += html_texts['plot'].substitute(
title=plot,
image=image,
links=links,
json=plot+".json"
)
# put the html parts together and write
with open(os.path.join(output_dir, overview_filename), "w") as overview_file:
overview_file.write(html_texts['overview'].substitute(
html_content=html_content,
title=www_title,
text=html_texts['description']
))
if no_publish:
return 0
# find out which files to copy
if(output_filenames != False):
files_to_copy = ( output_filenames )
files_to_copy += [os.path.join(output_dir, overview_filename)]
if additional_output_files != False:
files_to_copy += additional_output_files
if export_json != False:
files_to_copy += [export_json]
if save_legend:
files_to_copy += [os.path.join(output_dir, save_legend + _format) for _format in formats]
# create remote dir, copy files
mkdir_command = os.path.expandvars("$WEB_PLOTTING_MKDIR_COMMAND").format(subdir=remote_subdir)
copy_command = os.path.expandvars("$WEB_PLOTTING_COPY_COMMAND").format(source=" ".join(files_to_copy), subdir=remote_subdir)
log.info("Copying plots to webspace...")
log.debug("\nIssueing mkdir command: " + mkdir_command)
logger.subprocessCall(mkdir_command.split())
log.debug("\nIssueing rsync command: " + copy_command)
logger.subprocessCall(copy_command.split())
log.info("Copied {0}; see {1}".format(" ".join([f.split("/")[-1] for f in files_to_copy]), url))
def save(self):
self.plot.finish()
if not self.plot is None:
for filename in self.plotdict["output_filenames"]:
self.plot.save(filename)
log.info('Created plot "%s".' % filename)
if not (self.plotdict["live"] == None):
# if 'userpc', get the username and name of desktop machine
if self.plotdict["userpc"]:
userpc = (
"%s@%s"
% (tools.get_environment_variable("USER"), tools.get_environment_variable("HARRY_USERPC"))
).replace("\n", "")
for output_filename in self.plotdict["output_filenames"]:
if self.plotdict["userpc"]:
extrafunctions.show_plot_userpc(
output_filename, self.plotdict["live"], tools.get_environment_variable("USER"), userpc
)
else:
extrafunctions.show_plot(output_filename, self.plotdict["live"])
# web plotting
# TODO: make this more configurable if users want to user other webspaces etc.
if self.plotdict["www"] != None:
# set some needed variables
user = tools.get_environment_variable("HARRY_REMOTE_USER")
overview_filename = "overview.html"
date = datetime.date.today().strftime("%Y_%m_%d")
remote_dir = tools.get_environment_variable("HARRY_REMOTE_DIR") + "/%s/%s/" % (
date,
(self.plotdict["www"] if type(self.plotdict["www"]) == str else ""),
)
remote_path = tools.get_environment_variable("HARRY_REMOTE_PATH") + "/%s" % remote_dir
url = tools.get_environment_variable("HARRY_URL") + "/%s/%s" % (remote_dir, overview_filename)
plots_for_gallery = [
p for p in sorted(os.listdir(self.plotdict["output_dir"])) if ((".png" in p) or (".pdf" in p))
]
plots_to_copy = [os.path.basename(filename), overview_filename]
if self.plotdict.get("save_legend", False):
for f in self.plotdict["formats"]:
plots_to_copy.append(".".join([self.plotdict["save_legend"], f]))
html_content = ""
log.info("Copying plots to webspace...")
# loop over plots, make gallery
for plot in [p for p in plots_for_gallery if ((".png" in p) or (".pdf" in p))]:
# try to link to pdf file, if it exists
href = plot.replace(".png", ".pdf")
if href not in plots_for_gallery:
href = plot
title = plot.split("/")[-1][:-4].replace("_", " ")
html_content += htmlTemplatePlot % (title, href, title, plot)
with open(self.plotdict["output_dir"] + "/" + overview_filename, "w") as f:
f.write(htmlTemplate % (url, html_content))
# create remote dir, copy plots and overview file
create_dir_command = ["ssh", tools.get_environment_variable("HARRY_SSHPC"), "mkdir -p", remote_path]
log.debug("\nIssueing mkdir command: " + " ".join(create_dir_command))
subprocess.call(create_dir_command)
rsync_command = (
["rsync", "-u"]
+ [os.path.join(self.plotdict["output_dir"], p) for p in plots_to_copy]
+ ["%s:%s" % (tools.get_environment_variable("HARRY_SSHPC"), remote_path)]
)
log.debug("\nIssueing rsync command: " + " ".join(rsync_command) + "\n")
subprocess.call(rsync_command)
log.info("Copied {0}; see {1}".format(filename.split("/")[-1], url))
return self.plotdict["output_filenames"]
def save(self):
self.plot.finish()
if not self.plot is None:
for filename in self.plotdict["output_filenames"]:
self.plot.save(filename)
log.info('Created plot "%s".' % filename)
if not (self.plotdict["live"] == None):
# if 'userpc', get the username and name of desktop machine
if self.plotdict["userpc"]:
userpc = (
"%s@%s"
% (tools.get_environment_variable("USER"), tools.get_environment_variable("HARRY_USERPC"))
).replace("\n", "")
for output_filename in self.plotdict["output_filenames"]:
if self.plotdict["userpc"]:
extrafunctions.show_plot_userpc(
output_filename, self.plotdict["live"], tools.get_environment_variable("USER"), userpc
)
else:
extrafunctions.show_plot(output_filename, self.plotdict["live"])
# web plotting
# TODO: make this more configurable if users want to user other webspaces etc.
if self.plotdict["www"] != None:
# set some needed variables
user = tools.get_environment_variable("HARRY_REMOTE_USER")
overview_filename = "overview.html"
date = datetime.date.today().strftime("%Y_%m_%d")
remote_dir = os.path.expandvars(
os.path.join(
"$HARRY_REMOTE_DIR", date, (self.plotdict["www"] if type(self.plotdict["www"]) == str else "")
)
)
remote_path = os.path.expandvars(os.path.join("$HARRY_REMOTE_PATH", remote_dir))
url = os.path.expandvars(os.path.join("$HARRY_URL", remote_dir, overview_filename))
plots_for_gallery = [
p
for p in sorted(os.listdir(self.plotdict["output_dir"]))
if (
os.path.isfile(os.path.join(self.plotdict["output_dir"], p))
and all([not p.endswith("." + ext) for ext in ["json", "html", "root"]])
)
]
html_content = ""
log.info("Copying plots to webspace...")
# loop over plots, make gallery
html_template = None
with open(
os.path.expandvars("$ARTUSPATH/HarryPlotter/data/template_webplotting_overview.html")
) as html_template_file:
html_template = string.Template(html_template_file.read())
html_template_plot = None
with open(
os.path.expandvars("$ARTUSPATH/HarryPlotter/data/template_webplotting_plot.html")
) as html_template_plot_file:
html_template_plot = string.Template(html_template_plot_file.read())
for plot in [p for p in plots_for_gallery]:
# try to link to pdf file, if it exists
href = plot.replace(".png", ".pdf")
if href not in plots_for_gallery:
href = plot
filename = os.path.splitext(plot)[0]
html_content += html_template_plot.substitute(
title=filename, href=href, plot=plot, json=filename + ".json"
)
with open(os.path.join(self.plotdict["output_dir"], overview_filename), "w") as overview_file:
overview_file.write(html_template.substitute(url=url, html_content=html_content))
# create remote dir, copy plots and overview file
sshpc = tools.get_environment_variable("HARRY_SSHPC")
create_dir_command = ["ssh", user + "@" + sshpc, "mkdir -p", remote_path]
log.debug("\nIssueing mkdir command: " + " ".join(create_dir_command))
logger.subprocessCall(create_dir_command)
rsync_command = (
["rsync", "-u"]
+ glob.glob(os.path.join(self.plotdict["output_dir"], "*.*"))
+ ["%s@%s:%s" % (user, sshpc, remote_path)]
)
log.debug("\nIssueing rsync command: " + " ".join(rsync_command) + "\n")
logger.subprocessCall(rsync_command)
log.info("Copied {0}; see {1}".format(filename.split("/")[-1], url))
return self.plotdict["output_filenames"]