def _get_latest_checkpoint(self, experiment_checkpoint_path: str) -> List[str]: if os.path.isdir(experiment_checkpoint_path): # Case 1: Dir specified, find latest checkpoint. latest_checkpoint = find_newest_experiment_checkpoint( experiment_checkpoint_path) if not latest_checkpoint: latest_checkpoint = [] for fname in os.listdir(experiment_checkpoint_path): fname = os.path.join(experiment_checkpoint_path, fname) latest_checkpoint_subdir = find_newest_experiment_checkpoint( fname) if latest_checkpoint_subdir: latest_checkpoint.append(latest_checkpoint_subdir) if not latest_checkpoint: raise ValueError( f"The directory `{experiment_checkpoint_path}` does not " f"contain a Ray Tune experiment checkpoint.") elif not os.path.isfile(experiment_checkpoint_path): # Case 2: File specified, but does not exist. raise ValueError( f"The file `{experiment_checkpoint_path}` does not " f"exist and cannot be loaded for experiment analysis.") else: # Case 3: File specified, use as latest checkpoint. latest_checkpoint = experiment_checkpoint_path if not isinstance(latest_checkpoint, list): latest_checkpoint = [latest_checkpoint] return latest_checkpoint
def _get_latest_checkpoint(self, experiment_checkpoint_path: Path) -> List[str]: # Case 1: Dir specified, find latest checkpoint. if experiment_checkpoint_path.is_dir(): latest_checkpoint = find_newest_experiment_checkpoint( str(experiment_checkpoint_path)) # If no checkpoint in this folder the sub-directory is searched. # In this case also multiple experiment folders could exist in # the same root. In this case the length of `latest_checkpoint` # will be greater than 1. if not latest_checkpoint: latest_checkpoint = [] for fname in experiment_checkpoint_path.iterdir(): fname = experiment_checkpoint_path.joinpath(fname) latest_checkpoint_subdir = find_newest_experiment_checkpoint( str(fname)) if latest_checkpoint_subdir: latest_checkpoint.append(latest_checkpoint_subdir) if not latest_checkpoint: # This avoid nested experiment directories of the form # `experiment_name1/experiment_name2/experiment_state.json`. experiment_checkpoint_path = str(experiment_checkpoint_path) raise ValueError( f"The directory `{experiment_checkpoint_path}` does not " "contain a Ray Tune experiment checkpoint.") elif not experiment_checkpoint_path.is_file(): # Case 2: File specified, but does not exist. experiment_checkpoint_path = str(experiment_checkpoint_path) raise ValueError( f"The file `{experiment_checkpoint_path}` does not " f"exist and cannot be loaded for experiment analysis.") else: # Case 3: File specified, use as latest checkpoint. latest_checkpoint = str(experiment_checkpoint_path) if not isinstance(latest_checkpoint, list): latest_checkpoint = [latest_checkpoint] return latest_checkpoint
def load_experiment_checkpoint_from_state_file( experiment_dir: str) -> ExperimentStateCheckpoint: newest_ckpt_path = find_newest_experiment_checkpoint(experiment_dir) with open(newest_ckpt_path, "r") as f: runner_state = json.load(f, cls=TuneFunctionDecoder) trials = [] for trial_cp_str in runner_state["checkpoints"]: parsed = json.loads(trial_cp_str, cls=TuneFunctionDecoder) trial = TrialStub(**parsed) trials.append(trial) runner_data = runner_state["runner_data"] return ExperimentStateCheckpoint(runner_data, trials)