Exemplo n.º 1
0
Arquivo: plots.py Projeto: claria/plot
def multi_plot():
    """Initializes and runs the core."""
    parser = argparse.ArgumentParser(description='Proces multiple plot configs.')
    parser.add_argument("-l", "--load-configs", default=[], nargs='+', 
                        help="Process multiple configs.")
    parser.add_argument("-j", "--jobs", default=6, type=int, help="Number of jobs.")
    parser.add_argument("-i", "--job-ids", type=int, nargs='+', default=None, help="Job id to process.")
    parser.add_argument("--no-mp", default=False, action='store_true', help="Do not use multiproccessing, but a simple loop.")
    parser.add_argument("--log-level", default='info', help="Set the log level.")
    args = vars(parser.parse_args()) 
    # empty sys args
    sys.argv = sys.argv[:1]

    # Process pool

    configs = []

    for item in args.pop('load_configs', []):
            if item.endswith('.json'):
                configs.append(read_config(item))
            elif item.endswith('.py'):
                conf_module = imp.load_source('config', item)
                config = conf_module.get_config()
                if isinstance(config, (list, tuple)):
                    configs += config
                else:
                    configs.append(config)
            else:
                raise ValueError('The file type of {0} is not supported.'.format(item))

    if args['no_mp']:
        for i,config in enumerate(configs):
            if args['job_ids'] is None or i in args['job_ids']:
                plot(config, log_level=args['log_level'])
    else:
        print 'Initializing {0} worker processes'.format(args['jobs'])
        pool = Pool(processes=args['jobs'], initializer=init_worker)

        try:
            for i, config in enumerate(configs):
                a = pool.apply_async(run_worker, (config, args['log_level'], i))
                a.get()
            pool.close()
            pool.join()
        except KeyboardInterrupt:
            pool.terminate()
            pool.join()
Exemplo n.º 2
0
Arquivo: core.py Projeto: claria/plot
    def build_config(self, input_config=None, log_level='info'):
        """ Parse arguments. To set log level, load additional module parsers etc. the sys.args[1:] are parsed
            multiple times.
        :return: dict of parsed args
        """
        base_parser = SettingParser(add_help=False)
        base_parser_group = base_parser.add_argument_group(title='Base Parser', description='')
        base_parser_group.add_argument("--log-level", default=log_level, help="Set the log level.")
        base_parser_group.add_argument("--list-modules", action='store_true', help="List all available modules.")
        base_parser_group.add_argument("-l", "--load-config", default=[], nargs='+',
                                       help="Load json configs, with decreasing precedence,  or a python scripts from file.")

        # Parse only log level to set it as early as possible
        args = vars(base_parser.parse_known_args()[0])
        # Set log level
        log_level = getattr(logging, args['log_level'].upper(), None)
        if not isinstance(log_level, int):
            raise ValueError('Invalid log level: %s' % log_level)
        logging.basicConfig(format='%(message)s', level=log_level)

        # Discover all available modules
        self._all_modules = discover_modules()

        # If module list requested, print it and exit program
        if args['list_modules']:
            for label, module in self._all_modules.iteritems():
                print label
            sys.exit(0)

        # Load configs/python scripts from files if requested
        file_config = ConfigDict()
        for item in args.pop('load_config', []):
            if item.endswith('.json'):
                merge(file_config, read_config(item))
            elif item.endswith('.py'):
                log.info('Running python file {}'.format(item))
                runpy.run_path(item)
            else:
                raise ValueError('The file type of {0} is not supported.'.format(item))

        base_parser_group.add_argument("--input-modules", nargs='+', default=['RootModule'], help="Input modules .")
        base_parser_group.add_argument("--ana-modules", nargs='+', default=[], help="Analysis modules.")
        base_parser_group.add_argument("--output-modules", nargs='+', default=['PlotModule'], help="Output modules.")
        # Parse also the modules to add their parsers
        args = vars(base_parser.parse_known_args()[0])

        module_parsers = [self._all_modules[name]().parser for name in
                          args['input_modules'] + args['ana_modules'] + args['output_modules']]

        # Additional arguments for complete parser
        base_parser_group.add_argument("-p", "--print-config", default=False, action="store_true",
                                       help="Print out the JSON config before running Artus.")
        base_parser_group.add_argument("--store-json", type='bool', default=True,
                                       help="Save the config as json file.")
        base_parser_group.add_argument("--merge-args", nargs='+', default=[],
                                       help=("If json file configs and command line configs are provided the settings are "
                                       "merged for the provided args. Works only for list arguments."))
        base_parser_group.add_argument('--output-path', default='plot.png', help='Path to output file.')
        base_parser_group.add_argument('--output-prefix', default='plots/', help='Prefix to output paths.')


        # Final parser consists of baseparser + active module parsers
        parser = SettingParser(parents=[base_parser] + module_parsers,
                               description='''Plotting tool to read, manipulate and plot root objects.''')

        # Triggers before actual parameters are parsed.
        callbacks.trigger('before_parsing', config=file_config)

        # Final parsing of parameters
        args = vars(parser.parse_args())

        # If a config was loaded, merge it with the args
        if file_config:
            provided_args = args.pop('provided_args')
            merge_args = args.pop('merge_args')
            merge(file_config, args, precedence_keys=provided_args, merge_keys=merge_args)
            config = file_config

            if input_config:
                merge(input_config, config, precedence_keys=provided_args, merge_keys=merge_args)
                config = input_config
        else:
            config = args

        # ConfigDict is just a simple wrapper around a dict with some helper functions.
        return ConfigDict(config)