def find_best(logdir, metric, mode): """Find the best experiment checkpoint as measured by a metric.""" import logging from ray.tune.analysis import Analysis logging.getLogger("ray.tune").setLevel("ERROR") analysis = Analysis(logdir) best_logdir = analysis.get_best_logdir(metric, mode=mode) last_checkpoint_path = get_last_checkpoint_path(best_logdir) click.echo(last_checkpoint_path)
def main(): ray.init(logging_level=logging.WARNING) best = Analysis( "results/HparamSearch-Dim10", default_metric="true_value", default_mode="max", ).get_best_config() config = { "wandb_dir": os.getcwd(), "wandb_tags": "unstable controllable".split(), "seed": tune.grid_search(list(range(10))), "env_dim": tune.grid_search(list(range(2, 11))), "estimator": tune.grid_search("dpg maac".split()), "K": 8, "B": best["B"], "optimizer": "SGD", "learning_rate": best["learning_rate"], "clip_grad_norm": 100, } tune.run( SuboptimalityGap, config=config, num_samples=1, stop=dict(time_total_s=300), local_dir="./results", ) ray.shutdown()
def list_trials(experiment_path, sort=None, output=None, filter_op=None, info_keys=None, limit=None, desc=False): """Lists trials in the directory subtree starting at the given path. Args: experiment_path (str): Directory where trials are located. Like Experiment.local_dir/Experiment.name/experiment*.json. sort (list): Keys to sort by. output (str): Name of file where output is saved. filter_op (str): Filter operation in the format "<column> <operator> <value>". info_keys (list): Keys that are displayed. limit (int): Number of rows to display. desc (bool): Sort ascending vs. descending. """ _check_tabulate() try: checkpoints_df = Analysis(experiment_path).dataframe() # last result except TuneError as e: raise click.ClickException("No trial data found!") from e def key_filter(k): return k in DEFAULT_CLI_KEYS or k.startswith(CONFIG_PREFIX) col_keys = [k for k in checkpoints_df.columns if key_filter(k)] if info_keys: for k in info_keys: if k not in checkpoints_df.columns: raise click.ClickException("Provided key invalid: {}. " "Available keys: {}.".format( k, checkpoints_df.columns)) col_keys = [k for k in checkpoints_df.columns if k in info_keys] if not col_keys: raise click.ClickException("No columns to output.") checkpoints_df = checkpoints_df[col_keys] if "last_update_time" in checkpoints_df: with pd.option_context("mode.use_inf_as_null", True): datetime_series = checkpoints_df["last_update_time"].dropna() datetime_series = datetime_series.apply( lambda t: datetime.fromtimestamp(t).strftime(TIMESTAMP_FORMAT)) checkpoints_df["last_update_time"] = datetime_series if "logdir" in checkpoints_df: # logdir often too long to view in table, so drop experiment_path checkpoints_df["logdir"] = checkpoints_df["logdir"].str.replace( experiment_path, "") if filter_op: col, op, val = filter_op.split(" ") col_type = checkpoints_df[col].dtype if is_numeric_dtype(col_type): val = float(val) elif is_string_dtype(col_type): val = str(val) # TODO(Andrew): add support for datetime and boolean else: raise click.ClickException("Unsupported dtype for {}: {}".format( val, col_type)) op = OPERATORS[op] filtered_index = op(checkpoints_df[col], val) checkpoints_df = checkpoints_df[filtered_index] if sort: for key in sort: if key not in checkpoints_df: raise click.ClickException("{} not in: {}".format( key, list(checkpoints_df))) ascending = not desc checkpoints_df = checkpoints_df.sort_values( by=sort, ascending=ascending) if limit: checkpoints_df = checkpoints_df[:limit] print_format_output(checkpoints_df) if output: file_extension = os.path.splitext(output)[1].lower() if file_extension in (".p", ".pkl", ".pickle"): checkpoints_df.to_pickle(output) elif file_extension == ".csv": checkpoints_df.to_csv(output, index=False) else: raise click.ClickException( "Unsupported filetype: {}".format(output)) click.secho("Output saved at {}".format(output), fg="green")
# but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # Apache License Version 2.0 for more details. # ============================================================================ import sys import pandas as pd from absl import flags from ray.tune.analysis import Analysis FLAGS = flags.FLAGS flags.DEFINE_string("logdir", "", "") FLAGS(sys.argv) analysis = Analysis(experiment_dir=FLAGS.logdir) all_configs = analysis.get_all_configs() rows = {} for path, df in analysis.trial_dataframes.items(): df = df.filter(regex="val|test|iteration").assign( metric=lambda x: x.filter(regex="val/(?:roc_auc|f1)").sum(axis=1)) idx = df["metric"].idxmax() rows[path] = df.iloc[idx].to_dict() rows[path]["seed"] = all_configs[path]["seed"] df = pd.DataFrame.from_dict(rows, orient="index") df = df.filter(regex="test/(?:roc_auc|f1|recall)").rename( mapper=lambda x: x.split("/")[-1], axis=1).rename(columns={ "roc_auc": "AUC", "f1": "F1", "recall": "Recall"
def summarize_run(path: str, cp_dir: str = 'None', infer_cp_dir: bool = True, train_metric: str = DEFAULT_TRAIN_METRIC, eval_metric: str = DEFAULT_VALID_METRIC, overwrite: bool = False, return_analysis: bool = False) -> None: print(f'\nLoading experiment from: \n{path}\n') #if not os.path.isfile(path): # raise ValueError(f'Path does not exist or is directory:\n{path}') #if path[-5:] != '.json': # raise ValueError(f'Not a .json file:\n{path}') summary_dir = os.path.join(path, 'summary') if os.path.isdir(summary_dir): if not overwrite: raise ValueError( f'Target directory `{summary_dir}` exists, use `--overwrite` to replace.' ) shutil.rmtree(summary_dir) os.makedirs(summary_dir) path_split = path.split(BASE_PATH)[1].split('/') experiment = path_split[0] name = path_split[1] mode = path_split[2] if infer_cp_dir: cp_dir = os.path.join(DEFAULT_TARGET_BASE_DIR, experiment, name, mode, 'summary') print(f'\nInfering cp_dir:\n {cp_dir}\n') if cp_dir != 'None': if cp_dir[-1] == '/': cp_dir = cp_dir[:-1] cp_dir_base = os.path.dirname(cp_dir) if os.path.isdir(cp_dir): shutil.rmtree(cp_dir) os.makedirs(cp_dir_base, exist_ok=True) exp = Analysis(path) if return_analysis: return exp configs = exp.dataframe() configs['rundir'] = [ os.path.join(l, 'progress.csv') for l in configs['logdir'] ] runs = [] for i, f in enumerate(configs['rundir']): df = pd.read_csv(f) df['uid'] = i runs.append(df) runs = pd.concat(runs) best_run_dir = exp.get_best_logdir(eval_metric, mode='min') best_run_file = os.path.join(best_run_dir, 'progress.csv') best_run = df = pd.read_csv(best_run_file) print(f'Best run ID: {best_run_dir}') for f in ['json', 'pkl']: in_file = os.path.join(best_run_dir, f'params.{f}') out_file = os.path.join(summary_dir, f'best_params.{f}') copyfile(in_file, out_file) # Plot runs. plot_all(runs, eval_metric, os.path.join(summary_dir, 'all_runs.png')) plot_single(best_run, eval_metric, os.path.join(summary_dir, 'best_run.png')) if cp_dir != 'None': shutil.copytree(summary_dir, cp_dir)