def multi_plot():
"""Initializes and runs the core."""
parser = argparse.ArgumentParser(description='Proces multiple plot configs.')
parser.add_argument("-l", "--load-configs", default=[], nargs='+',
help="Process multiple configs.")
parser.add_argument("-j", "--jobs", default=6, type=int, help="Number of jobs.")
parser.add_argument("-i", "--job-ids", type=int, nargs='+', default=None, help="Job id to process.")
parser.add_argument("--no-mp", default=False, action='store_true', help="Do not use multiproccessing, but a simple loop.")
parser.add_argument("--log-level", default='info', help="Set the log level.")
args = vars(parser.parse_args())
# empty sys args
sys.argv = sys.argv[:1]
# Process pool
configs = []
for item in args.pop('load_configs', []):
if item.endswith('.json'):
configs.append(read_config(item))
elif item.endswith('.py'):
conf_module = imp.load_source('config', item)
config = conf_module.get_config()
if isinstance(config, (list, tuple)):
configs += config
else:
configs.append(config)
else:
raise ValueError('The file type of {0} is not supported.'.format(item))
if args['no_mp']:
for i,config in enumerate(configs):
if args['job_ids'] is None or i in args['job_ids']:
plot(config, log_level=args['log_level'])
else:
print 'Initializing {0} worker processes'.format(args['jobs'])
pool = Pool(processes=args['jobs'], initializer=init_worker)
try:
for i, config in enumerate(configs):
a = pool.apply_async(run_worker, (config, args['log_level'], i))
a.get()
pool.close()
pool.join()
except KeyboardInterrupt:
pool.terminate()
pool.join()
def build_config(self, input_config=None, log_level='info'):
""" Parse arguments. To set log level, load additional module parsers etc. the sys.args[1:] are parsed
multiple times.
:return: dict of parsed args
"""
base_parser = SettingParser(add_help=False)
base_parser_group = base_parser.add_argument_group(title='Base Parser', description='')
base_parser_group.add_argument("--log-level", default=log_level, help="Set the log level.")
base_parser_group.add_argument("--list-modules", action='store_true', help="List all available modules.")
base_parser_group.add_argument("-l", "--load-config", default=[], nargs='+',
help="Load json configs, with decreasing precedence, or a python scripts from file.")
# Parse only log level to set it as early as possible
args = vars(base_parser.parse_known_args()[0])
# Set log level
log_level = getattr(logging, args['log_level'].upper(), None)
if not isinstance(log_level, int):
raise ValueError('Invalid log level: %s' % log_level)
logging.basicConfig(format='%(message)s', level=log_level)
# Discover all available modules
self._all_modules = discover_modules()
# If module list requested, print it and exit program
if args['list_modules']:
for label, module in self._all_modules.iteritems():
print label
sys.exit(0)
# Load configs/python scripts from files if requested
file_config = ConfigDict()
for item in args.pop('load_config', []):
if item.endswith('.json'):
merge(file_config, read_config(item))
elif item.endswith('.py'):
log.info('Running python file {}'.format(item))
runpy.run_path(item)
else:
raise ValueError('The file type of {0} is not supported.'.format(item))
base_parser_group.add_argument("--input-modules", nargs='+', default=['RootModule'], help="Input modules .")
base_parser_group.add_argument("--ana-modules", nargs='+', default=[], help="Analysis modules.")
base_parser_group.add_argument("--output-modules", nargs='+', default=['PlotModule'], help="Output modules.")
# Parse also the modules to add their parsers
args = vars(base_parser.parse_known_args()[0])
module_parsers = [self._all_modules[name]().parser for name in
args['input_modules'] + args['ana_modules'] + args['output_modules']]
# Additional arguments for complete parser
base_parser_group.add_argument("-p", "--print-config", default=False, action="store_true",
help="Print out the JSON config before running Artus.")
base_parser_group.add_argument("--store-json", type='bool', default=True,
help="Save the config as json file.")
base_parser_group.add_argument("--merge-args", nargs='+', default=[],
help=("If json file configs and command line configs are provided the settings are "
"merged for the provided args. Works only for list arguments."))
base_parser_group.add_argument('--output-path', default='plot.png', help='Path to output file.')
base_parser_group.add_argument('--output-prefix', default='plots/', help='Prefix to output paths.')
# Final parser consists of baseparser + active module parsers
parser = SettingParser(parents=[base_parser] + module_parsers,
description='''Plotting tool to read, manipulate and plot root objects.''')
# Triggers before actual parameters are parsed.
callbacks.trigger('before_parsing', config=file_config)
# Final parsing of parameters
args = vars(parser.parse_args())
# If a config was loaded, merge it with the args
if file_config:
provided_args = args.pop('provided_args')
merge_args = args.pop('merge_args')
merge(file_config, args, precedence_keys=provided_args, merge_keys=merge_args)
config = file_config
if input_config:
merge(input_config, config, precedence_keys=provided_args, merge_keys=merge_args)
config = input_config
else:
config = args
# ConfigDict is just a simple wrapper around a dict with some helper functions.
return ConfigDict(config)