def test_task_environment(self): """ <b>Description:</b> Check the TaskEnvironment can correctly return the value <b>Input data:</b> Dummy data <b>Expected results:</b> Test passes if incoming data is processed correctly <b>Steps</b> 1. Using an already created dummy environment. 2. Checking class fields """ env = environment() __dummy_config = dummy_config() assert env == TaskEnvironment( model=None, model_template=env.model_template, hyper_parameters=env.get_hyper_parameters(), label_schema=env.label_schema, ) assert isinstance(env, TaskEnvironment) assert env != "Fail params" assert env.get_labels() == [] for i in ["header", "description", "visible_in_ui"]: assert (getattr( env.get_model_configuration().configurable_parameters, i) == __dummy_config[i]) assert env.get_model_configuration().configurable_parameters.id == ID() for param in __dummy_config: getattr(env.get_hyper_parameters(), param) == __dummy_config[param] assert env.get_hyper_parameters().id == ID() assert "model=None" in repr(env) assert "label_schema=LabelSchemaEntity(label_groups=[LabelGroup(id=" in repr( env) assert "name=from_label_list" in repr(env) assert "group_type=LabelGroupType.EXCLUSIVE" in repr(env) assert ( "labels=[LabelEntity(123456789, name=car, hotkey=ctrl+0, domain=DETECTION" in repr(env)) assert ( "LabelEntity(987654321, name=person, hotkey=ctrl+0, domain=DETECTION" in repr(env)) assert ( "CONFIGURABLE_PARAMETERS(header='Configuration for an object detection task -- TEST ONLY'" in repr(env)) assert ( "description='Configuration for an object detection task -- TEST ONLY'" in repr(env)) assert "visible_in_ui=True" in repr(env) assert "id=ID()" in repr(env)
def environment(): """ Return TaskEnvironment """ car = LabelEntity(id=ID(123456789), name="car", domain=Domain.DETECTION, is_empty=True) person = LabelEntity(id=ID(987654321), name="person", domain=Domain.DETECTION, is_empty=True) labels_list = [car, person] dummy_template = __get_path_to_file("./dummy_template.yaml") model_template = parse_model_template(dummy_template) hyper_parameters = model_template.hyper_parameters.data params = ote_config_helper.create(hyper_parameters) labels_schema = LabelSchemaEntity.from_labels(labels_list) environment = TaskEnvironment( model=None, hyper_parameters=params, label_schema=labels_schema, model_template=model_template, ) return environment
def __init__(self, task_environment: TaskEnvironment): logger.info("Loading OTEClassificationTask.") self._scratch_space = tempfile.mkdtemp(prefix="ote-cls-scratch-") logger.info(f"Scratch space created at {self._scratch_space}") self._task_environment = task_environment if len(task_environment.get_labels(False)) == 1: self._labels = task_environment.get_labels(include_empty=True) else: self._labels = task_environment.get_labels(include_empty=False) self._empty_label = get_empty_label(task_environment.label_schema) self._multilabel = len(task_environment.label_schema.get_groups(False)) > 1 and \ len(task_environment.label_schema.get_groups(False)) == \ len(task_environment.get_labels(include_empty=False)) self._hierarchical = False if not self._multilabel and len( task_environment.label_schema.get_groups(False)) > 1: self._labels = get_leaf_labels(task_environment.label_schema) self._hierarchical = True template_file_path = task_environment.model_template.model_template_path self._base_dir = os.path.abspath(os.path.dirname(template_file_path)) self._cfg = get_default_config() self._patch_config(self._base_dir) if self._multilabel: assert self._cfg.model.type == 'multilabel', task_environment.model_template.model_template_path + \ ' model template does not support multiclass classification' else: assert self._cfg.model.type == 'classification', task_environment.model_template.model_template_path + \ ' model template does not support multilabel classification' self.device = torch.device( "cuda:0") if torch.cuda.device_count() else torch.device("cpu") self._model = self._load_model(task_environment.model, device=self.device) self.stop_callback = StopCallback() self.metrics_monitor = DefaultMetricsMonitor() # Set default model attributes. self._optimization_methods = [] self._precision = [ModelPrecision.FP32] self._optimization_type = ModelOptimizationType.MO
def main(): """ Main function that is used for model exporting. """ args = parse_args() # Load template.yaml file. template = find_and_parse_model_template(args.template) # Get class for Task. task_class = get_impl_class(template.entrypoints.base) # Get hyper parameters schema. hyper_parameters = create(template.hyper_parameters.data) assert hyper_parameters environment = TaskEnvironment( model=None, hyper_parameters=hyper_parameters, label_schema=read_label_schema( os.path.join(os.path.dirname(args.load_weights), "label_schema.json")), model_template=template, ) model_adapters = { "weights.pth": ModelAdapter(read_binary(args.load_weights)) } model = ModelEntity( configuration=environment.get_model_configuration(), model_adapters=model_adapters, train_dataset=None, ) environment.model = model task = task_class(task_environment=environment) exported_model = ModelEntity(None, environment.get_model_configuration(), model_status=ModelStatus.NOT_READY) task.export(ExportType.OPENVINO, exported_model) os.makedirs(args.save_model_to, exist_ok=True) save_model_data(exported_model, args.save_model_to)
def main(): """ Main function that is used for model evaluation. """ # Parses input arguments. args = parse_args() # Reads model template file. template = find_and_parse_model_template(args.template) # Get hyper parameters schema. hyper_parameters = template.hyper_parameters.data assert hyper_parameters # Get classes for Task, ConfigurableParameters and Dataset. if not args.load_weights.endswith(".bin") and not args.load_weights.endswith( ".xml" ): raise RuntimeError("Only OpenVINO-exported models are supported.") task_class = get_impl_class(template.entrypoints.openvino) environment = TaskEnvironment( model=None, hyper_parameters=create(hyper_parameters), label_schema=read_label_schema( os.path.join(os.path.dirname(args.load_weights), "label_schema.json") ), model_template=template, ) environment.model = read_model( environment.get_model_configuration(), args.load_weights, None ) task = task_class(task_environment=environment) deployed_model = ModelEntity(None, environment.get_model_configuration()) os.makedirs(args.save_model_to, exist_ok=True) task.deploy(deployed_model) with open(os.path.join(args.save_model_to, "openvino.zip"), "wb") as write_file: write_file.write(deployed_model.exportable_code)
def _create_environment_and_task(params, labels_schema, model_template): environment = TaskEnvironment( model=None, hyper_parameters=params, label_schema=labels_schema, model_template=model_template, ) logger.info("Create base Task") task_impl_path = model_template.entrypoints.base task_cls = get_impl_class(task_impl_path) task = task_cls(task_environment=environment) return environment, task
def __init__(self, task_environment: TaskEnvironment): self.task_environment = task_environment self.model_name = task_environment.model_template.name self.labels = task_environment.get_labels() # Hyperparameters. self.project_path: str = tempfile.mkdtemp(prefix="ote-anomalib") self.config = self.get_config() self.model = self.load_model(ote_model=task_environment.model) self.trainer: Trainer
def init_environment(params, model_template, number_of_images=10): resolution = (224, 224) colors = [(0, 255, 0), (0, 0, 255)] cls_names = ['b', 'g'] texts = ['Blue', 'Green'] env_labels = [ LabelEntity(name=name, domain=Domain.CLASSIFICATION, is_empty=False, id=ID(i)) for i, name in enumerate(cls_names) ] items = [] for _ in range(0, number_of_images): for j, lbl in enumerate(env_labels): class_img = np.zeros((*resolution, 3), dtype=np.uint8) class_img[:] = colors[j] class_img = cv.putText(class_img, texts[j], (50, 50), cv.FONT_HERSHEY_SIMPLEX, .8 + j * .2, colors[j - 1], 2, cv.LINE_AA) image = Image(data=class_img) labels = [ScoredLabel(label=lbl, probability=1.0)] shapes = [Annotation(Rectangle.generate_full_box(), labels)] annotation_scene = AnnotationSceneEntity( kind=AnnotationSceneKind.ANNOTATION, annotations=shapes) items.append( DatasetItemEntity(media=image, annotation_scene=annotation_scene)) rng = random.Random() rng.seed(100) rng.shuffle(items) for i, _ in enumerate(items): subset_region = i / number_of_images if subset_region >= 0.9: subset = Subset.TESTING elif subset_region >= 0.6: subset = Subset.VALIDATION else: subset = Subset.TRAINING items[i].subset = subset dataset = DatasetEntity(items) labels_schema = generate_label_schema(dataset.get_labels(), multilabel=False) environment = TaskEnvironment(model=None, hyper_parameters=params, label_schema=labels_schema, model_template=model_template) return environment, dataset
def main(): """ Main function that is used for model demonstration. """ # Dynamically create an argument parser based on override parameters. args, template, hyper_parameters = parse_args() # Get new values from user's input. updated_hyper_parameters = gen_params_dict_from_args(args) # Override overridden parameters by user's values. override_parameters(updated_hyper_parameters, hyper_parameters) hyper_parameters = create(hyper_parameters) # Get classes for Task, ConfigurableParameters and Dataset. if args.load_weights.endswith(".bin") or args.load_weights.endswith( ".xml"): task_class = get_impl_class(template.entrypoints.openvino) else: task_class = get_impl_class(template.entrypoints.base) environment = TaskEnvironment( model=None, hyper_parameters=hyper_parameters, label_schema=read_label_schema( os.path.join(os.path.dirname(args.load_weights), "label_schema.json")), model_template=template, ) environment.model = read_model(environment.get_model_configuration(), args.load_weights, None) task = task_class(task_environment=environment) capture = open_images_capture(args.input, args.loop) elapsed_times = deque(maxlen=10) frame_index = 0 while True: frame = capture.read() if frame is None: break predictions, elapsed_time = get_predictions(task, frame) elapsed_times.append(elapsed_time) elapsed_time = np.mean(elapsed_times) frame = draw_predictions(template.task_type, predictions, frame, args.fit_to_size) if args.display_perf: put_text_on_rect_bg( frame, f"time: {elapsed_time:.4f} sec.", (0, frame.shape[0] - 30), color=(255, 255, 255), ) if args.delay >= 0: cv2.imshow("frame", frame) if cv2.waitKey(args.delay) == ESC_BUTTON: break else: print(f"{frame_index=}, {elapsed_time=}, {len(predictions)=}")
def main(): """ Main function that is used for model evaluation. """ # Dynamically create an argument parser based on override parameters. args, template, hyper_parameters = parse_args() # Get new values from user's input. updated_hyper_parameters = gen_params_dict_from_args(args) # Override overridden parameters by user's values. override_parameters(updated_hyper_parameters, hyper_parameters) hyper_parameters = create(hyper_parameters) # Get classes for Task, ConfigurableParameters and Dataset. if args.load_weights.endswith(".bin") or args.load_weights.endswith(".xml"): task_class = get_impl_class(template.entrypoints.openvino) else: task_class = get_impl_class(template.entrypoints.base) dataset_class = get_dataset_class(template.task_type) dataset = dataset_class( test_subset={"ann_file": args.test_ann_files, "data_root": args.test_data_roots} ) dataset_label_schema = generate_label_schema(dataset, template.task_type) check_label_schemas( read_label_schema( os.path.join(os.path.dirname(args.load_weights), "label_schema.json") ), dataset_label_schema, ) environment = TaskEnvironment( model=None, hyper_parameters=hyper_parameters, label_schema=dataset_label_schema, model_template=template, ) model = read_model(environment.get_model_configuration(), args.load_weights, None) environment.model = model task = task_class(task_environment=environment) validation_dataset = dataset.get_subset(Subset.TESTING) predicted_validation_dataset = task.infer( validation_dataset.with_empty_annotations(), InferenceParameters(is_evaluation=True), ) resultset = ResultSetEntity( model=model, ground_truth_dataset=validation_dataset, prediction_dataset=predicted_validation_dataset, ) task.evaluate(resultset) assert resultset.performance is not None print(resultset.performance) if args.save_performance: with open(args.save_performance, "w", encoding="UTF-8") as write_file: json.dump( {resultset.performance.score.name: resultset.performance.score.value}, write_file, )
def main(): """ Main function that is used for model training. """ # Dynamically create an argument parser based on override parameters. args, template, hyper_parameters = parse_args() # Get new values from user's input. updated_hyper_parameters = gen_params_dict_from_args(args) # Override overridden parameters by user's values. override_parameters(updated_hyper_parameters, hyper_parameters) hyper_parameters = create(hyper_parameters) # Get classes for Task, ConfigurableParameters and Dataset. task_class = get_impl_class(template.entrypoints.base) dataset_class = get_dataset_class(template.task_type) # Create instances of Task, ConfigurableParameters and Dataset. dataset = dataset_class( train_subset={ "ann_file": args.train_ann_files, "data_root": args.train_data_roots, }, val_subset={"ann_file": args.val_ann_files, "data_root": args.val_data_roots}, ) environment = TaskEnvironment( model=None, hyper_parameters=hyper_parameters, label_schema=generate_label_schema(dataset, template.task_type), model_template=template, ) if args.load_weights: environment.model = ModelEntity( train_dataset=dataset, configuration=environment.get_model_configuration(), model_adapters={ "weights.pth": ModelAdapter(read_binary(args.load_weights)) }, ) if args.enable_hpo: run_hpo(args, environment, dataset, template.task_type) task = task_class(task_environment=environment) output_model = ModelEntity( dataset, environment.get_model_configuration(), model_status=ModelStatus.NOT_READY, ) task.train(dataset, output_model, train_parameters=TrainParameters()) save_model_data(output_model, args.save_model_to) validation_dataset = dataset.get_subset(Subset.VALIDATION) predicted_validation_dataset = task.infer( validation_dataset.with_empty_annotations(), InferenceParameters(is_evaluation=True), ) resultset = ResultSetEntity( model=output_model, ground_truth_dataset=validation_dataset, prediction_dataset=predicted_validation_dataset, ) task.evaluate(resultset) assert resultset.performance is not None print(resultset.performance)
def test_model_entity_sets_values(self): """ <b>Description:</b> Check that ModelEntity correctly returns the set values <b>Expected results:</b> Test passes if ModelEntity correctly returns the set values <b>Steps</b> 1. Check set values in the ModelEntity """ def __get_path_to_file(filename: str): """ Return the path to the file named 'filename', which lives in the tests/entities directory """ return str(Path(__file__).parent / Path(filename)) car = LabelEntity(name="car", domain=Domain.DETECTION) labels_list = [car] dummy_template = __get_path_to_file("./dummy_template.yaml") model_template = parse_model_template(dummy_template) hyper_parameters = model_template.hyper_parameters.data params = ote_config_helper.create(hyper_parameters) labels_schema = LabelSchemaEntity.from_labels(labels_list) environment = TaskEnvironment( model=None, hyper_parameters=params, label_schema=labels_schema, model_template=model_template, ) item = self.generate_random_image() dataset = DatasetEntity(items=[item]) score_metric = ScoreMetric(name="Model accuracy", value=0.5) model_entity = ModelEntity(train_dataset=self.dataset(), configuration=self.configuration()) set_params = { "configuration": environment.get_model_configuration(), "train_dataset": dataset, "id": ID(1234567890), "creation_date": self.creation_date, "previous_trained_revision": 5, "previous_revision": 2, "version": 2, "tags": ["tree", "person"], "model_status": ModelStatus.TRAINED_NO_STATS, "model_format": ModelFormat.BASE_FRAMEWORK, "performance": Performance(score_metric), "training_duration": 5.8, "precision": [ModelPrecision.INT8], "latency": 328, "fps_throughput": 20, "target_device": TargetDevice.GPU, "target_device_type": "notebook", "optimization_methods": [OptimizationMethod.QUANTIZATION], "optimization_type": ModelOptimizationType.MO, "optimization_objectives": { "param": "Test param" }, "performance_improvement": {"speed", 0.5}, "model_size_reduction": 1.0, } for key, value in set_params.items(): setattr(model_entity, key, value) assert getattr(model_entity, key) == value assert model_entity.is_optimized() is True
def run_hpo_trainer( hp_config, model, hyper_parameters, model_template, dataset_paths, task_type, ): """Run each training of each trial with given hyper parameters""" if isinstance(hyper_parameters, dict): current_params = {} for val in hyper_parameters["parameters"]: current_params[val] = hyper_parameters[val] hyper_parameters = create(model_template.hyper_parameters.data) HpoManager.set_hyperparameter(hyper_parameters, current_params) if dataset_paths is None: raise ValueError("Dataset is not defined.") impl_class = get_dataset_class(task_type) dataset = impl_class( train_subset={ "ann_file": dataset_paths.get("train_ann_file", None), "data_root": dataset_paths.get("train_data_root", None), }, val_subset={ "ann_file": dataset_paths.get("val_ann_file", None), "data_root": dataset_paths.get("val_data_root", None), }, ) train_env = TaskEnvironment( model=model, hyper_parameters=hyper_parameters, label_schema=generate_label_schema(dataset, task_type), model_template=model_template, ) hyper_parameters = train_env.get_hyper_parameters() # set epoch if task_type == TaskType.CLASSIFICATION: (hyper_parameters.learning_parameters.max_num_epochs ) = hp_config["iterations"] elif task_type in (TaskType.DETECTION, TaskType.SEGMENTATION): hyper_parameters.learning_parameters.num_iters = hp_config[ "iterations"] # set hyper-parameters and print them HpoManager.set_hyperparameter(hyper_parameters, hp_config["params"]) print(f"hyper parameter of current trial : {hp_config['params']}") train_env.set_hyper_parameters(hyper_parameters) train_env.model_template.hpo = { "hp_config": hp_config, "metric": hp_config["metric"], } impl_class = get_impl_class(train_env.model_template.entrypoints.base) task = impl_class(task_environment=train_env) dataset = HpoDataset(dataset, hp_config) if train_env.model: train_env.model.train_dataset = dataset train_env.model.confiugration.configurable_parameters = hyper_parameters output_model = ModelEntity( dataset, train_env.get_model_configuration(), model_status=ModelStatus.NOT_READY, ) # make callback to report score to hpopt every epoch train_param = TrainParameters( False, HpoCallback(hp_config, hp_config["metric"], task), None) train_param.train_on_empty_model = None task.train(dataset=dataset, output_model=output_model, train_parameters=train_param) hpopt.finalize_trial(hp_config)