def list_experiments(project_path, sort=None, output=None, filter_op=None, info_keys=None, limit=None, desc=False): """Lists experiments in the directory subtree. Args: project_path (str): Directory where experiments are located. Corresponds to Experiment.local_dir. sort (list): Keys to sort by. output (str): Name of file where output is saved. filter_op (str): Filter operation in the format "<column> <operator> <value>". info_keys (list): Keys that are displayed. limit (int): Number of rows to display. desc (bool): Sort ascending vs. descending. """ _check_tabulate() base, experiment_folders, _ = next(os.walk(project_path)) experiment_data_collection = [] for experiment_dir in experiment_folders: analysis_obj, checkpoints_df = None, None try: analysis_obj = ExperimentAnalysis( os.path.join(project_path, experiment_dir)) checkpoints_df = analysis_obj.dataframe() except TuneError: logger.debug("No experiment state found in %s", experiment_dir) continue # Format time-based values. stats = analysis_obj.stats() time_values = { "start_time": stats.get("_start_time"), "last_updated": stats.get("timestamp"), } formatted_time_values = { key: datetime.fromtimestamp(val).strftime(TIMESTAMP_FORMAT) if val else None for key, val in time_values.items() } experiment_data = { "name": experiment_dir, "total_trials": checkpoints_df.shape[0], "running_trials": (checkpoints_df["status"] == Trial.RUNNING).sum(), "terminated_trials": (checkpoints_df["status"] == Trial.TERMINATED).sum(), "error_trials": (checkpoints_df["status"] == Trial.ERROR).sum(), } experiment_data.update(formatted_time_values) experiment_data_collection.append(experiment_data) if not experiment_data_collection: print("No experiments found!") sys.exit(0) info_df = pd.DataFrame(experiment_data_collection) if not info_keys: info_keys = DEFAULT_PROJECT_INFO_KEYS col_keys = [k for k in list(info_keys) if k in info_df] if not col_keys: print("None of keys {} in experiment data!".format(info_keys)) sys.exit(0) info_df = info_df[col_keys] if filter_op: col, op, val = filter_op.split(" ") col_type = info_df[col].dtype if is_numeric_dtype(col_type): val = float(val) elif is_string_dtype(col_type): val = str(val) # TODO(Andrew): add support for datetime and boolean else: raise ValueError("Unsupported dtype for {}: {}".format( val, col_type)) op = OPERATORS[op] filtered_index = op(info_df[col], val) info_df = info_df[filtered_index] if sort: if sort not in info_df: raise KeyError("{} not in: {}".format(sort, list(info_df))) ascending = not desc info_df = info_df.sort_values(by=sort, ascending=ascending) if limit: info_df = info_df[:limit] print_format_output(info_df) if output: file_extension = os.path.splitext(output)[1].lower() if file_extension in (".p", ".pkl", ".pickle"): info_df.to_pickle(output) elif file_extension == ".csv": info_df.to_csv(output, index=False) else: raise ValueError("Unsupported filetype: {}".format(output)) print("Output saved at:", output)
class ExperimentAnalysisSuite(unittest.TestCase): def setUp(self): ray.init(local_mode=True) self.test_dir = tempfile.mkdtemp() self.test_name = "analysis_exp" self.num_samples = 10 self.metric = "episode_reward_mean" self.test_path = os.path.join(self.test_dir, self.test_name) self.run_test_exp() self.ea = ExperimentAnalysis(self.test_path) def tearDown(self): shutil.rmtree(self.test_dir, ignore_errors=True) ray.shutdown() def run_test_exp(self): ahb = AsyncHyperBandScheduler(time_attr="training_iteration", reward_attr=self.metric, grace_period=5, max_t=100) run(MyTrainableClass, name=self.test_name, scheduler=ahb, local_dir=self.test_dir, **{ "stop": { "training_iteration": 1 }, "num_samples": 10, "config": { "width": sample_from(lambda spec: 10 + int(90 * random.random())), "height": sample_from(lambda spec: int(100 * random.random())), }, }) def testDataframe(self): df = self.ea.dataframe() self.assertTrue(isinstance(df, pd.DataFrame)) self.assertEquals(df.shape[0], self.num_samples) def testTrialDataframe(self): cs = self.ea._checkpoints idx = random.randint(0, len(cs) - 1) trial_df = self.ea.trial_dataframe( cs[idx]["trial_id"]) # random trial df self.assertTrue(isinstance(trial_df, pd.DataFrame)) self.assertEqual(trial_df.shape[0], 1) def testBestTrainable(self): best_trainable = self.ea.get_best_trainable(self.metric, MyTrainableClass) self.assertTrue(isinstance(best_trainable, MyTrainableClass)) def testBestConfig(self): best_config = self.ea.get_best_config(self.metric) self.assertTrue(isinstance(best_config, dict)) self.assertTrue("width" in best_config) self.assertTrue("height" in best_config) def testBestTrial(self): best_trial = self.ea._get_best_trial(self.metric) self.assertTrue(isinstance(best_trial, dict)) self.assertTrue("local_dir" in best_trial) self.assertEqual(best_trial["local_dir"], os.path.expanduser(self.test_path)) self.assertTrue("config" in best_trial) self.assertTrue("width" in best_trial["config"]) self.assertTrue("height" in best_trial["config"]) self.assertTrue("last_result" in best_trial) self.assertTrue(self.metric in best_trial["last_result"]) def testCheckpoints(self): checkpoints = self.ea._checkpoints self.assertTrue(isinstance(checkpoints, list)) self.assertTrue(isinstance(checkpoints[0], dict)) self.assertEqual(len(checkpoints), self.num_samples) def testStats(self): stats = self.ea.stats() self.assertTrue(isinstance(stats, dict)) self.assertTrue("start_time" in stats) self.assertTrue("timestamp" in stats) def testRunnerData(self): runner_data = self.ea.runner_data() self.assertTrue(isinstance(runner_data, dict)) self.assertTrue("_metadata_checkpoint_dir" in runner_data) self.assertEqual(runner_data["_metadata_checkpoint_dir"], os.path.expanduser(self.test_path))