def __call__(self, main_config: types.YAMLDict,
criteria: bc.IConcreteBatchCriteria) -> None:
exp_to_run = utils.exp_range_calc(self.cmdopts,
self.cmdopts['batch_output_root'],
criteria)
exp_dirnames = criteria.gen_exp_dirnames(self.cmdopts)
for i, exp in enumerate(exp_to_run):
exp = os.path.split(exp)[1]
exp_index = exp_dirnames.index(exp)
cmdopts = copy.deepcopy(self.cmdopts)
cmdopts["exp0_output_root"] = os.path.join(
self.cmdopts["batch_output_root"], exp_dirnames[0])
cmdopts["exp0_stat_root"] = os.path.join(
self.cmdopts["batch_stat_root"], exp_dirnames[0])
cmdopts["exp_input_root"] = os.path.join(
self.cmdopts['batch_input_root'], exp)
cmdopts["exp_output_root"] = os.path.join(
self.cmdopts['batch_output_root'], exp)
cmdopts["exp_graph_root"] = os.path.join(
self.cmdopts['batch_graph_root'], exp)
cmdopts["exp_stat_root"] = os.path.join(
self.cmdopts["batch_stat_root"], exp)
cmdopts["exp_model_root"] = os.path.join(
cmdopts['batch_model_root'], exp)
utils.dir_create_checked(cmdopts['exp_model_root'], exist_ok=True)
for model in self.models:
if not model.run_for_exp(criteria, cmdopts, exp_index):
self.logger.debug(
"Skip running intra-experiment model from '%s' for exp%s",
str(model), exp_index)
continue
# Run the model
self.logger.debug("Run intra-experiment model '%s' for exp%s",
str(model), exp_index)
dfs = model.run(criteria, exp_index, cmdopts)
for df, csv_stem in zip(dfs, model.target_csv_stems()):
path_stem = os.path.join(cmdopts['exp_model_root'],
csv_stem)
# Write model legend file so the generated graph can find it
with open(path_stem + '.legend', 'w') as f:
for i, search in enumerate(dfs):
if search.values.all() == df.values.all():
legend = model.legend_names()[i]
f.write(legend)
break
# Write model .csv file
storage.DataFrameWriter('storage.csv')(df,
path_stem +
'.model',
index=False)
def __call__(self, criteria: bc.IConcreteBatchCriteria, target: dict,
stat_collate_root: str) -> None:
self.logger.info(
"Stage4: Collating bivariate files from batch in %s for graph '%s'...",
self.cmdopts['batch_output_root'], target['src_stem'])
self.logger.trace(json.dumps(target, indent=4))
exp_dirs = utils.exp_range_calc(self.cmdopts,
self.cmdopts['batch_output_root'],
criteria)
xlabels, ylabels = utils.bivar_exp_labels_calc(exp_dirs)
if self.cmdopts['dist_stats'] in ['conf95', 'all']:
exts = [
config.kStatsExtensions['mean'],
config.kStatsExtensions['stddev']
]
elif self.cmdopts['dist_stats'] in ['bw', 'all']:
exts = [
config.kStatsExtensions['min'], config.kStatsExtensions['max'],
config.kStatsExtensions['mean'],
config.kStatsExtensions['whislo'],
config.kStatsExtensions['whishi'],
config.kStatsExtensions['cilo'],
config.kStatsExtensions['cihi'],
config.kStatsExtensions['median']
]
stats = [
BivarGraphCollationInfo(df_ext=ext,
xlabels=xlabels,
ylabels=ylabels) for ext in exts
]
for i, diri in enumerate(exp_dirs):
# We get full paths back from the exp dirs calculation, and we need to work with path
# leaves
diri = os.path.split(diri)[1]
self._collate_exp(target, diri, stats)
for stat in stats:
if stat.all_srcs_exist:
storage.DataFrameWriter('storage.csv')(
stat.df,
os.path.join(stat_collate_root,
target['dest_stem'] + stat.df_ext),
index=False)
elif stat.some_srcs_exist:
self.logger.warning(
"Not all experiments in '%s' produced '%s%s'",
self.cmdopts['batch_output_root'], target['src_stem'],
stat.df_ext)
def __call__(self) -> None:
"""
Runs experiments in the batch according to configuration.
"""
self.logger.info("Platform='%s' exec_env='%s'",
self.cmdopts['platform'], self.cmdopts['exec_env'])
module = pm.pipeline.get_plugin_module(self.cmdopts['platform'])
module.pre_exp_diagnostics(self.cmdopts, self.logger)
exp_all = [
os.path.join(self.batch_exp_root, d)
for d in self.criteria.gen_exp_dirnames(self.cmdopts)
]
exp_to_run = utils.exp_range_calc(self.cmdopts, self.batch_exp_root,
self.criteria)
# Verify environment is OK before running anything
platform.ExecEnvChecker(self.cmdopts)()
# Calculate path for to file for logging execution times
now = datetime.datetime.now()
exec_times_fpath = os.path.join(self.batch_stat_exec_root,
now.strftime("%Y-%m-%e-%H:%M"))
# Start a new process for the experiment shell so pre-run commands have
# an effect (if they set environment variables, etc.).
shell = ExpShell()
# Run the experiment!
for exp in exp_to_run:
exp_num = exp_all.index(exp)
# Run cmds for platform-specific things to setup the experiment
# (e.g., start daemons) if needed.
generator = platform.ExpShellCmdsGenerator(self.cmdopts, exp_num)
for spec in generator.pre_exp_cmds():
shell.run_from_spec(spec)
runner = ExpRunner(self.cmdopts, exec_times_fpath, generator,
shell)
runner(exp, exp_num)
# Run cmds to cleanup platform-specific things now that the experiment
# is done (if needed).
for spec in generator.post_exp_cmds():
shell.run_from_spec(spec)
def graph_yticklabels(self,
cmdopts: types.Cmdopts,
exp_dirs: tp.Optional[tp.List[str]] = None) -> tp.List[str]:
dirs = []
all_dirs = utils.exp_range_calc(cmdopts,
cmdopts['batch_output_root'],
self)
for c2 in self.criteria2.gen_exp_dirnames(cmdopts):
for y in all_dirs:
leaf = os.path.split(y)[1]
if c2 in leaf.split('+')[1]:
dirs.append(leaf)
break
return self.criteria2.graph_xticklabels(cmdopts, dirs)