def test_model_inference_config() -> None: # check if normal path works normal_path = "/".join(list(map(str, range(1, 91)))) assert len(normal_path) == 260 ModelInferenceConfig(model_name="Test", checkpoint_paths=[normal_path]) # check if long path fails ie: > 260 long_path = normal_path + "/" assert len(long_path) == 261 with pytest.raises(ValueError): ModelInferenceConfig(model_name="Test", checkpoint_paths=[long_path])
def test_model_inference_config() -> None: # check if normal path works normal_path = "/".join(list(map(str, range(1, 91)))) assert len(normal_path) == 260 ModelInferenceConfig(model_name="Test", checkpoint_paths=[normal_path], structure_names=["organ1", "tumour2"], colours=[(255, 0, 0), (255, 0, 0)], fill_holes=[True, False]) # check if long path fails ie: > 260 long_path = normal_path + "/" assert len(long_path) == 261 with pytest.raises(ValueError): ModelInferenceConfig(model_name="Test", checkpoint_paths=[long_path], structure_names=["organ1", "tumour2"], colours=[(255, 0, 0), (255, 0, 0)], fill_holes=[True, False])
def copy_child_paths_to_folder(self, model_folder: Path, checkpoint_paths: List[Path]) -> None: """ Gets the files that are required to register a model for inference. The necessary files are copied from the current folder structure into the given temporary folder. The folder will contain all source code in the InnerEye folder, possibly additional source code from the extra_code_directory, and all checkpoints in a newly created "checkpoints" folder inside the model. :param model_folder: The folder into which all files should be copied. :param checkpoint_paths: A list with absolute paths to checkpoint files. They are expected to be inside of the model's checkpoint folder. """ def copy_folder(source_folder: Path, destination_folder: str = "") -> None: logging.info(f"Copying folder for registration: {source_folder}") destination_folder = destination_folder or source_folder.name shutil.copytree(str(source_folder), str(model_folder / destination_folder), ignore=shutil.ignore_patterns('*.pyc')) def copy_file(source: Path, destination_file: str) -> None: logging.info( f"Copying file for registration: {source} to {destination_file}" ) destination = model_folder / destination_file if destination.is_file(): # This could happen if there is score.py inside of the InnerEye package and also inside the calling # project. The latter will have precedence logging.warning( f"Overwriting existing {source.name} with {source}") destination.parent.mkdir(parents=True, exist_ok=True) shutil.copy(str(source), str(destination)) relative_checkpoint_paths = [] for checkpoint in checkpoint_paths: if checkpoint.is_absolute(): try: # Checkpoints live in a folder structure in the checkpoint folder. There can be multiple of # them, with identical names, coming from an ensemble run. Hence, preserve their folder structure. checkpoint_relative = checkpoint.relative_to( self.model_config.checkpoint_folder) except ValueError: raise ValueError( f"Checkpoint file {checkpoint} was expected to be in a subfolder of " f"{self.model_config.checkpoint_folder}") # Checkpoints go into a newly created folder "checkpoints" inside of the model folder relative_checkpoint_paths.append( str(Path(CHECKPOINT_FOLDER) / checkpoint_relative)) else: raise ValueError( f"Expected an absolute path to a checkpoint file, but got: {checkpoint}" ) model_folder.mkdir(parents=True, exist_ok=True) model_inference_config = ModelInferenceConfig( model_name=self.model_config.model_name, structure_names=self.model_config.ground_truth_ids_display_names, colours=self.model_config.colours, fill_holes=self.model_config.fill_holes, model_configs_namespace=self.model_config.__class__.__module__, checkpoint_paths=relative_checkpoint_paths) # Inference configuration must live in the root folder of the registered model full_path_to_config = model_folder / fixed_paths.MODEL_INFERENCE_JSON_FILE_NAME full_path_to_config.write_text(model_inference_config.to_json(), encoding='utf-8') # type: ignore # Merge the conda files into one merged environment file at the root of the model merged_conda_file = model_folder / fixed_paths.ENVIRONMENT_YAML_FILE_NAME merge_conda_files(get_all_environment_files(self.project_root), result_file=merged_conda_file) # InnerEye package: This can be either in Python's package folder, or a plain folder. In both cases, # we can identify it by going up the folder structure off a known file (repository_root does exactly that) repository_root = fixed_paths.repository_root_directory() copy_folder(repository_root / INNEREYE_PACKAGE_NAME) # Extra code directory is expected to be relative to the project root folder. if self.azure_config.extra_code_directory: extra_code_folder = self.project_root / self.azure_config.extra_code_directory if extra_code_folder.is_dir(): copy_folder(extra_code_folder) else: logging.warning( f"The `extra_code_directory` is set to '{self.azure_config.extra_code_directory}', " "but this folder does not exist in the project root folder." ) # All files at project root should be copied as-is. Those should be essential things like score.py that # are needed for inference to run. First try to find them at repository root (but they might not be there # if InnerEye is used as a package), then at project root. files_to_copy = list(repository_root.glob("*.py")) if repository_root != self.project_root: files_to_copy.extend(self.project_root.glob("*.py")) for f in files_to_copy: copy_file(f, destination_file=f.name) for (checkpoint_source, checkpoint_destination) in zip(checkpoint_paths, relative_checkpoint_paths): if checkpoint_source.is_file(): copy_file(checkpoint_source, destination_file=str(checkpoint_destination)) else: raise ValueError( f"Checkpoint file {checkpoint_source} does not exist")
def read_model_inference_config( path_to_model_inference_config: str) -> ModelInferenceConfig: with open(path_to_model_inference_config, 'r', encoding='utf-8') as file: model_inference_config = ModelInferenceConfig.from_json( file.read()) # type: ignore return model_inference_config
def register_segmentation_model(self, best_epoch: int, best_epoch_dice: float, checkpoint_paths: List[Path], model_proc: ModelProcessing, run: Optional[Run] = None, workspace: Optional[Workspace] = None, tags: Optional[Dict[str, str]] = None) -> \ Tuple[Optional[Model], Optional[Path], Any]: """ Registers a new model in the workspace's model registry to be deployed further, and creates a model zip for portal deployment (if required). This model, is the model checkpoint with the highest test accuracy. :param best_epoch: The training epoch that resulted in the highest validation score. :param best_epoch_dice: Dice metric for the best epoch :param checkpoint_paths: Checkpoint paths to use to upload model checkpoints to AML. :param model_proc: whether it's a single or ensemble model. :param run: If provided then the run's workspace and tags will be used to register the model. :param workspace: If provided, then this workspace will be used to register the model instead of the workspace associated with the provided run. :param tags: If provided, then these will be used instead of the tags found in the provided run. :returns AML model object, the path to the specially-deployed model if any, and a further object relating to model deployment; if model_deployment_hook is None, the last two are also None. However if a model cannot be registered because the run is an _OfflineRun, or the model_config is not for a segmentation model, None is returned instead of a model. """ if not isinstance(self.model_config, SegmentationModelBase): logging.warning("Non-segmentation models cannot be registered") return None, None, None if (run is None) == (workspace is None): raise ValueError( "Either a run or a workspace must be provided but not both") elif run: if not hasattr(run, 'experiment'): logging.warning( "Not registering a model, because the run has no associated experiment" ) return None, None, None workspace = run.experiment.workspace tags = run.get_tags() relative_checkpoint_paths = [ x.relative_to(self.project_root) if x.is_absolute() else x for x in checkpoint_paths ] model_inference_config = ModelInferenceConfig( model_name=self.model_config.model_name, structure_names=self.model_config.ground_truth_ids_display_names, colours=self.model_config.colours, fill_holes=self.model_config.fill_holes, model_configs_namespace=self.model_config.__class__.__module__, checkpoint_paths=list(map(str, relative_checkpoint_paths))) full_path_to_config = self.project_root / fixed_paths.MODEL_INFERENCE_JSON_FILE_NAME full_path_to_config.write_text(model_inference_config.to_json(), encoding='utf-8') # type: ignore relative_child_paths = self.get_child_paths(checkpoint_paths) # Add experiment and run ID to tags if run is not None: tags = self.tags_with_run_information(run, tags) model = Model.register( workspace=workspace, model_path=str(self.project_root), child_paths=relative_child_paths, model_name=self.model_config.model_name, tags=tags, description="Best epoch: {}, Accuracy : {}".format( best_epoch, best_epoch_dice)) logging.info( f"Registered {model_proc.value} model: {model.name}, with Id: {model.id}" ) # update the run's tags with the registered model information if not self.model_config.is_offline_run: update_run_tags(run, {MODEL_ID_KEY_NAME: model.id}) # create a version of the model for deployment if the hook is provided if self.model_deployment_hook is not None: assert isinstance(self.model_config, SegmentationModelBase) deployment_model_path, deployment_model_spec = self.model_deployment_hook( self.model_config, self.azure_config, model, model_proc) return model, deployment_model_path, deployment_model_spec return model, None, None