def discover_and_download_checkpoint_from_sibling_runs(self) -> None: """ Downloads checkpoints from sibling runs in a hyperdrive run. This is used to gather results from all splits in a hyperdrive run. """ self.run_recovery = RunRecovery.download_checkpoints_from_run( self.model_config, self.run_context) # Check paths are good, just in case for path in self.run_recovery.checkpoints_roots: if not path.is_dir(): raise NotADirectoryError( f"Does not exist or is not a directory: {path}")
def create_ensemble_model(self) -> None: """ Call MLRunner again after training cross-validation models, to create an ensemble model from them. """ # Import only here in case of dependency issues in reduced environment from InnerEye.ML.utils.run_recovery import RunRecovery with logging_section("Downloading checkpoints from sibling runs"): run_recovery = RunRecovery.download_checkpoints_from_run( self.azure_config, self.model_config, PARENT_RUN_CONTEXT, output_subdir_name=OTHER_RUNS_SUBDIR_NAME) # Check paths are good, just in case for path in run_recovery.checkpoints_roots: if not path.is_dir(): raise NotADirectoryError( f"Does not exist or is not a directory: {path}") # Adjust parameters self.azure_config.hyperdrive = False self.model_config.number_of_cross_validation_splits = 0 self.model_config.is_train = False best_epoch = self.create_ml_runner().run_inference_and_register_model( run_recovery, model_proc=ModelProcessing.ENSEMBLE_CREATION) crossval_dir = self.plot_cross_validation_and_upload_results() Runner.generate_report(self.model_config, best_epoch, ModelProcessing.ENSEMBLE_CREATION) # CrossValResults should have been uploaded to the parent run, so we don't need it here. remove_file_or_directory(crossval_dir) # We can also remove OTHER_RUNS under the root, as it is no longer useful and only contains copies of files # available elsewhere. However, first we need to upload relevant parts of OTHER_RUNS/ENSEMBLE. other_runs_dir = self.model_config.outputs_folder / OTHER_RUNS_SUBDIR_NAME other_runs_ensemble_dir = other_runs_dir / ENSEMBLE_SPLIT_NAME if PARENT_RUN_CONTEXT is not None: if other_runs_ensemble_dir.exists(): # Only keep baseline Wilcoxon results and scatterplots and reports for subdir in other_runs_ensemble_dir.glob("*"): if subdir.name not in [ BASELINE_WILCOXON_RESULTS_FILE, SCATTERPLOTS_SUBDIR_NAME, REPORT_HTML, REPORT_IPYNB ]: remove_file_or_directory(subdir) PARENT_RUN_CONTEXT.upload_folder( name=BASELINE_COMPARISONS_FOLDER, path=str(other_runs_ensemble_dir)) else: logging.warning( f"Directory not found for upload: {other_runs_ensemble_dir}" ) remove_file_or_directory(other_runs_dir)