def __call__(self) -> None: # If only 1 pipeline stage is passed, then the list of stages to run is # parsed as a non-iterable integer, which can cause the generator to # fail to be created. So make it iterable in that case as well. if not isinstance(self.args.pipeline, Iterable): self.args.pipeline = [self.args.pipeline] if 5 not in self.args.pipeline: controller = ControllerGeneratorParser()(self.args) sgp = pm.module_load_tiered( project=self.args.project, path='generators.scenario_generator_parser') scenario = sgp.ScenarioGeneratorParser().to_scenario_name( self.args) self.logger.info("Controller=%s, Scenario=%s", controller, scenario) cmdopts = rdg.from_cmdline(self.args) pipeline = Pipeline(self.args, controller, cmdopts) else: pipeline = Pipeline(self.args, None, {}) try: pipeline.run() except KeyboardInterrupt: self.logger.info("Exiting on user cancel") sys.exit()
def factory(cli_arg: str, main_config: types.YAMLDict, cmdopts: types.Cmdopts) -> PopulationVariableDensity: """ Factory to create :class:`PopulationVariableDensity` derived classes from the command line definition. """ attr = vd.Parser()(cli_arg) sgp = pm.module_load_tiered(project=cmdopts['project'], path='generators.scenario_generator_parser') kw = sgp.ScenarioGeneratorParser().to_dict(cmdopts['scenario']) extent = ArenaExtent(Vector3D(kw['arena_x'], kw['arena_y'], kw['arena_z'])) densities = [ x for x in np.linspace( attr['density_min'], attr['density_max'], num=attr['cardinality']) ] def __init__(self) -> None: PopulationVariableDensity.__init__(self, cli_arg, main_config, cmdopts['batch_input_root'], densities, extent) return type( cli_arg, # type: ignore (PopulationVariableDensity, ), {"__init__": __init__})
def __init__(self, **kwargs) -> None: cmdopts = kwargs['cmdopts'] self.logger = logging.getLogger(__name__) module = pm.module_load_tiered(project=cmdopts['project'], path='generators.scenario_generators') generator_name = module.gen_generator_name(spec.scenario_name) self.scenario_generator = getattr(module, generator_name)( controller=controller, spec=spec, **kwargs)
def _create_exp_run(self, run_exp_def: xml.XMLLuigi, cmds_generator: bindings.IExpShellCmdsGenerator, run_num: int, seeds: tp.List[int]) -> None: run_output_dir = "{0}_{1}_output".format(self.main_input_name, run_num) # If the project defined per-run configuration, apply # it. Otherwise, just apply the configuration in the SIERRA core. per_run = pm.module_load_tiered(project=self.cmdopts['project'], path='generators.exp_generators') run_output_root = os.path.join(self.exp_output_root, run_output_dir) stem_path = self._get_launch_file_stempath(run_num) per_run.ExpRunDefUniqueGenerator(run_num, run_output_root, stem_path, self.random_seeds[run_num], self.cmdopts).generate(run_exp_def) # Write out the experimental run launch file run_exp_def.write(stem_path) # Perform any necessary programmatic (i.e., stuff you can do in python # and don't need a shell for) per-run configuration. configurer = platform.ExpConfigurer(self.cmdopts) configurer.for_exp_run(self.exp_input_root, run_output_root) if configurer.cmdfile_paradigm() == 'per-exp': # Update GNU Parallel commands file with the command for the # configured experimental run. fpath = f"{self.commands_fpath}{config.kGNUParallel['cmdfile_ext']}" with open(fpath, 'a') as cmds_file: self._update_cmds_file(cmds_file, cmds_generator, 'per-exp', run_num, self._get_launch_file_stempath(run_num), 'slave') elif configurer.cmdfile_paradigm() == 'per-run': # Write new GNU Parallel commands file with the commends for the # experimental run. ext = config.kGNUParallel['cmdfile_ext'] master_fpath = f"{self.commands_fpath}_run{run_num}_master{ext}" slave_fpath = f"{self.commands_fpath}_run{run_num}_slave{ext}" self.logger.trace("Updating slave cmdfile %s", slave_fpath) with open(slave_fpath, 'w') as cmds_file: self._update_cmds_file(cmds_file, cmds_generator, 'per-run', run_num, self._get_launch_file_stempath(run_num), 'slave') self.logger.trace("Updating master cmdfile %s", master_fpath) with open(master_fpath, 'w') as cmds_file: self._update_cmds_file(cmds_file, cmds_generator, 'per-run', run_num, self._get_launch_file_stempath(run_num), 'master')
def __init__(self, criteria: bc.IConcreteBatchCriteria, exp_num: int, cmdopts: types.Cmdopts) -> None: self.exp_num = exp_num self.exp_input_root = os.path.join( cmdopts['batch_input_root'], criteria.gen_exp_dirnames(cmdopts)[exp_num]) self.exp_def_fpath = os.path.join(self.exp_input_root, sierra.core.config.kPickleLeaf) self.logger = logging.getLogger(__name__) self.criteria = criteria from_bivar_bc1 = False from_bivar_bc2 = False from_univar_bc = False if criteria.is_bivar(): bivar = tp.cast(bc.BivarBatchCriteria, criteria) from_bivar_bc1 = hasattr(bivar.criteria1, 'exp_scenario_name') from_bivar_bc2 = hasattr(bivar.criteria2, 'exp_scenario_name') else: from_univar_bc = hasattr(criteria, 'exp_scenario_name') # Need to get per-experiment arena dimensions from batch criteria, as # they might be different for each experiment if from_univar_bc: self.arena_dim = criteria.arena_dims()[exp_num] self.scenario_name = criteria.exp_scenario_name(exp_num) self.logger.debug( "Read scenario dimensions '%s' from univariate batch criteria", self.arena_dim) elif from_bivar_bc1 or from_bivar_bc2: self.arena_dim = criteria.arena_dims()[exp_num] self.logger.debug( "Read scenario dimensions '%s' bivariate batch criteria", self.arena_dim) self.scenario_name = criteria.exp_scenario_name(exp_num) else: # Default case: scenario dimensions read from cmdline sgp = pm.module_load_tiered( project=cmdopts['project'], path='generators.scenario_generator_parser') kw = sgp.ScenarioGeneratorParser().to_dict(cmdopts['scenario']) self.arena_dim = ArenaExtent( Vector3D(kw['arena_x'], kw['arena_y'], kw['arena_z'])) self.logger.debug("Read scenario dimensions %s from cmdline spec", self.arena_dim) self.scenario_name = cmdopts['scenario']
def __call__(self, main_config: types.YAMLDict, controller_config: types.YAMLDict, LN_config: types.YAMLDict, HM_config: types.YAMLDict, criteria: bc.IConcreteBatchCriteria) -> None: """ Parameters: main_config: Parsed dictionary of main YAML configuration controller_config: Parsed dictionary of controller YAML configuration. LN_config: Parsed dictionary of intra-experiment linegraph configuration. HM_config: Parsed dictionary of intra-experiment heatmap configuration. criteria: The :term:`Batch Criteria` used for the batch experiment. """ exp_to_gen = sierra.core.utils.exp_range_calc( self.cmdopts, self.cmdopts['batch_output_root'], criteria) for exp in exp_to_gen: exp = os.path.split(exp)[1] cmdopts = copy.deepcopy(self.cmdopts) cmdopts["exp_input_root"] = os.path.join( self.cmdopts['batch_input_root'], exp) cmdopts["exp_output_root"] = os.path.join( self.cmdopts['batch_output_root'], exp) cmdopts["exp_graph_root"] = os.path.join( self.cmdopts['batch_graph_root'], exp) cmdopts["exp_model_root"] = os.path.join( cmdopts['batch_model_root'], exp) cmdopts["exp_stat_root"] = os.path.join(cmdopts["batch_stat_root"], exp) if os.path.isdir(cmdopts["exp_stat_root"]): generator = pm.module_load_tiered( project=self.cmdopts['project'], path='pipeline.stage4.intra_exp_graph_generator') generator.IntraExpGraphGenerator(main_config, controller_config, LN_config, HM_config, cmdopts)(criteria) else: self.logger.warning( "Skipping experiment '%s': %s does not exist", exp, cmdopts['exp_stat_root'])
def _gather_worker(gatherq: mp.Queue, processq: mp.Queue, main_config: dict, project: str, storage_medium: str) -> None: while True: # Wait for 3 seconds after the queue is empty before bailing try: batch_output_root, exp = gatherq.get(True, 3) module = pm.module_load_tiered( project=project, path='pipeline.stage3.run_collator') module.ExperimentalRunCSVGatherer(main_config, batch_output_root, exp, storage_medium, processq)() gatherq.task_done() except queue.Empty: break
def factory(cli_arg: str, main_config: types.YAMLDict, cmdopts: types.Cmdopts) -> PopulationConstantDensity: """ Factory to create :class:`PopulationConstantDensity` derived classes from the command line definition. """ attr = cd.Parser()(cli_arg) sgp = pm.module_load_tiered(project=cmdopts['project'], path='generators.scenario_generator_parser') kw = sgp.ScenarioGeneratorParser().to_dict(cmdopts['scenario']) is_2x1 = kw['arena_x'] == 2 * kw['arena_y'] is_1x1 = kw['arena_x'] == kw['arena_y'] if is_2x1: r = range(kw['arena_x'], kw['arena_x'] + attr['cardinality'] * attr['arena_size_inc'], attr['arena_size_inc']) dims = [ sierra.core.utils.ArenaExtent( Vector3D(x, int(x / 2), kw['arena_z'])) for x in r ] elif is_1x1: r = range(kw['arena_x'], kw['arena_x'] + attr['cardinality'] * attr['arena_size_inc'], attr['arena_size_inc']) dims = [ sierra.core.utils.ArenaExtent(Vector3D(x, x, kw['arena_z'])) for x in r ] else: raise NotImplementedError( "Unsupported arena X,Y scaling '{0}': Must be [2x1,1x1]") def __init__(self) -> None: PopulationConstantDensity.__init__(self, cli_arg, main_config, cmdopts['batch_input_root'], attr["target_density"], dims, kw['scenario_tag']) return type( cli_arg, # type: ignore (PopulationConstantDensity, ), {"__init__": __init__})
def _run_inter_graph_generation( self, criteria: bc.IConcreteBatchCriteria) -> None: """ Generate inter-experiment graphs (duh). """ targets = self._calc_inter_LN_targets() self.logger.info("Generating inter-experiment graphs...") start = time.time() generator = pm.module_load_tiered( project=self.cmdopts['project'], path='pipeline.stage4.inter_exp_graph_generator') generator.InterExpGraphGenerator(self.main_config, self.cmdopts, targets)(criteria) elapsed = int(time.time() - start) sec = datetime.timedelta(seconds=elapsed) self.logger.info("Inter-experiment graph generation complete: %s", str(sec))
def __init__(self, main_config: types.YAMLDict, cmdopts: types.Cmdopts) -> None: self.cmdopts = cmdopts self.main_config = main_config self.controller_config = yaml.load( open( os.path.join(self.cmdopts['project_config_root'], sierra.core.config.kYAML['controllers'])), yaml.FullLoader) self.logger = logging.getLogger(__name__) # Load YAML config loader = pm.module_load_tiered( project=self.cmdopts['project'], path='pipeline.stage4.yaml_config_loader') config = loader.YAMLConfigLoader()(self.cmdopts) self.intra_LN_config = config['intra_LN'] self.intra_HM_config = config['intra_HM'] self.inter_LN_config = config['inter_LN'] if self.cmdopts['models_enable']: self._load_models()
def factory(cli_arg: str, main_config: types.YAMLDict, cmdopts: types.Cmdopts) -> PopulationSize: """ Factory to create :class:`PopulationSize` derived classes from the command line definition. """ parser = population_size.Parser() attr = parser(cli_arg) max_sizes = parser.to_sizes(attr) if cmdopts['robot_positions']: positions = [ Vector3D.from_str(s, astype=float) for s in cmdopts['robot_positions'] ] else: # Get the dimensions of the effective arena from the scenario so we can # place robots randomly within it. sgp = pm.module_load_tiered( project=cmdopts['project'], path='generators.scenario_generator_parser') kw = sgp.ScenarioGeneratorParser().to_dict(cmdopts['scenario']) xs = random.choices(range(0, kw['arena_x']), k=max_sizes[-1]) ys = random.choices(range(0, kw['arena_y']), k=max_sizes[-1]) zs = random.choices(range(0, kw['arena_z']), k=max_sizes[-1]) positions = [Vector3D(x, y, z) for x, y, z in zip(xs, ys, zs)] def __init__(self) -> None: PopulationSize.__init__(self, cli_arg, main_config, cmdopts['batch_input_root'], cmdopts['robot'], max_sizes, positions) return type( cli_arg, # type: ignore (PopulationSize, ), {"__init__": __init__})
def __init__(self, args: argparse.Namespace, controller: str, rdg_opts: types.Cmdopts) -> None: self.args = args self.logger = logging.getLogger(__name__) assert all(stage in [1, 2, 3, 4, 5] for stage in args.pipeline), \ f"Invalid pipeline stage in {args.pipeline}: Only 1-5 valid" self.cmdopts = { # general 'sierra_root': self.args.sierra_root, 'scenario': self.args.scenario, 'template_input_file': self.args.template_input_file, 'project': self.args.project, 'exec_env': args.exec_env, 'platform_vc': self.args.platform_vc, "n_runs": args.n_runs, 'project_imagizing': self.args.project_imagizing, 'exp_overwrite': self.args.exp_overwrite, 'exp_range': self.args.exp_range, 'dist_stats': self.args.dist_stats, 'skip_collate': self.args.skip_collate, 'platform': self.args.platform, # stage 1 'no_preserve_seeds': self.args.no_preserve_seeds, # stage 2 'nodefile': self.args.nodefile, # stage 3 'skip_verify_results': self.args.skip_verify_results, 'render_cmd_opts': self.args.render_cmd_opts, 'processing_mem_limit': self.args.processing_mem_limit, 'serial_processing': self.args.serial_processing, 'storage_medium': self.args.storage_medium, # stage 4 'exp_graphs': self.args.exp_graphs, 'project_no_yaml_LN': self.args.project_no_yaml_LN, 'project_no_yaml_HM': self.args.project_no_yaml_HM, 'project_rendering': self.args.project_rendering, 'plot_log_xscale': self.args.plot_log_xscale, 'plot_enumerated_xscale': self.args.plot_enumerated_xscale, 'plot_log_yscale': self.args.plot_log_yscale, 'plot_regression_lines': self.args.plot_regression_lines, 'plot_primary_axis': self.args.plot_primary_axis, 'plot_large_text': self.args.plot_large_text, 'models_enable': self.args.models_enable, # stage 5 'controllers_list': self.args.controllers_list, 'controllers_legend': self.args.controllers_legend, 'scenarios_list': self.args.scenarios_list, 'scenarios_legend': self.args.scenarios_legend, 'scenario_comparison': self.args.scenario_comparison, 'controller_comparison': self.args.controller_comparison, 'comparison_type': self.args.comparison_type, 'transpose_graphs': self.args.transpose_graphs, } # Load additional cmdline options from platform self.logger.debug("Updating cmdopts with extensions from '%s'", self.cmdopts['platform']) module = pm.module_load_tiered("cmdline", platform=self.cmdopts['platform']) module.PlatformCmdline.cmdopts_update(self.args, self.cmdopts) if rdg_opts is not None: self.cmdopts.update(rdg_opts) # Load additional cmdline options from project. This is mandatory, # because all projects have to define --controller and --scenario # at a minimum. self.logger.debug("Updating cmdopts with extensions from '%s'", self.cmdopts['project']) path = "{0}.cmdline".format(self.cmdopts['project']) module = pm.module_load(path) module.Cmdline.cmdopts_update(self.args, self.cmdopts) self.cmdopts['plugin_root'] = os.path.join('sierra', 'plugins') project = pm.pipeline.get_plugin(self.cmdopts['project']) path = os.path.join(project['parent_dir'], self.cmdopts['project']) self.cmdopts['project_root'] = path self.cmdopts['project_config_root'] = os.path.join(path, 'config') self.cmdopts['project_model_root'] = os.path.join(path, 'models') self._load_config() if 5 not in self.args.pipeline: self.batch_criteria = bc.factory(self.main_config, self.cmdopts, self.args) self.controller = controller
def _gen_csv(self, cmdopts: types.Cmdopts, batch_leaf: str, src_stem: str, dest_stem: str) -> None: """ Helper function for generating a set of .csv files for use in inter-scenario graph generation. Generates: - ``.csv`` file containing results for each scenario the controller is being compared across, 1 per line. - ``.stddev`` file containing stddev for the generated ``.csv`` file, 1 per line. - ``.model`` file containing model predictions for controller behavior during each scenario, 1 per line (not generated if models were not run the performance measures we are generating graphs for). - ``.legend`` file containing legend values for models to plot (not generated if models were not run for the performance measures we are generating graphs for). """ csv_ipath = os.path.join(cmdopts['batch_output_root'], cmdopts['batch_stat_collate_root'], src_stem + ".csv") stddev_ipath = os.path.join(cmdopts['batch_output_root'], cmdopts['batch_stat_collate_root'], src_stem + ".stddev") model_ipath_stem = os.path.join(cmdopts['batch_model_root'], src_stem) model_opath_stem = os.path.join(self.sc_model_root, dest_stem + "-" + self.controller) opath_stem = os.path.join(self.sc_csv_root, dest_stem + "-" + self.controller) # Some experiments might not generate the necessary performance measure # .csvs for graph generation, which is OK. if not utils.path_exists(csv_ipath): self.logger.warning("%s missing for controller %s", csv_ipath, self.controller) return # Collect performance measure results. Append to existing dataframe if # it exists, otherwise start a new one. data_df = self._accum_df(csv_ipath, opath_stem + '.csv', src_stem) storage.DataFrameWriter('storage.csv')(data_df, opath_stem + '.csv', index=False) # Collect performance results stddev. Append to existing dataframe if it # exists, otherwise start a new one. stddev_df = self._accum_df(stddev_ipath, opath_stem + '.stddev', src_stem) if stddev_df is not None: storage.DataFrameWriter('storage.csv')(stddev_df, opath_stem + '.stddev', index=False) # Collect performance results models and legends. Append to existing # dataframes if they exist, otherwise start new ones. model_df = self._accum_df(model_ipath_stem + '.model', model_opath_stem + '.model', src_stem) if model_df is not None: storage.DataFrameWriter('storage.csv')(model_df, model_opath_stem + '.model', index=False) with open(model_opath_stem + '.legend', 'a') as f: _, scenario, _ = rdg.parse_batch_leaf(batch_leaf) sgp = pm.module_load_tiered( project=cmdopts['project'], path='generators.scenario_generator_parser') kw = sgp.ScenarioGeneratorParser().to_dict(scenario) f.write("{0} Prediction\n".format(kw['scenario_tag']))