def __init__(
self,
hparams: OTEClassificationParameters,
label_schema: LabelSchemaEntity,
model_file: Union[str, bytes],
weight_file: Union[str, bytes, None] = None,
device: str = "CPU",
num_requests: int = 1,
):
"""
Inferencer implementation for OTEDetection using OpenVINO backend.
:param model: Path to model to load, `.xml`, `.bin` or `.onnx` file.
:param hparams: Hyper parameters that the model should use.
:param num_requests: Maximum number of requests that the inferencer can make.
Good value is the number of available cores. Defaults to 1.
:param device: Device to run inference on, such as CPU, GPU or MYRIAD. Defaults to "CPU".
"""
multilabel = len(label_schema.get_groups(False)) > 1 and \
len(label_schema.get_groups(False)) == len(label_schema.get_labels(include_empty=False))
self.label_schema = label_schema
model_adapter = OpenvinoAdapter(create_core(),
model_file,
weight_file,
device=device,
max_num_requests=num_requests)
self.configuration = {'multilabel': multilabel}
self.model = Model.create_model("ote_classification",
model_adapter,
self.configuration,
preload=True)
self.converter = ClassificationToAnnotationConverter(self.label_schema)
def get_empty_label(label_schema: LabelSchemaEntity) -> Optional[LabelEntity]:
"""
Get first empty label from label_schema
"""
empty_candidates = list(
set(label_schema.get_labels(include_empty=True))
- set(label_schema.get_labels(include_empty=False))
)
if empty_candidates:
return empty_candidates[0]
return None
def get_leaf_labels(label_schema: LabelSchemaEntity) -> List[LabelEntity]:
"""
Get leafs from label tree
"""
leaf_labels = []
all_labels = label_schema.get_labels(False)
for lbl in all_labels:
if not label_schema.get_children(lbl):
leaf_labels.append(lbl)
return leaf_labels
def generate_label_schema(not_empty_labels, multilabel=False):
assert len(not_empty_labels) > 1
label_schema = LabelSchemaEntity()
if multilabel:
emptylabel = LabelEntity(name="Empty label",
is_empty=True,
domain=Domain.CLASSIFICATION)
empty_group = LabelGroup(name="empty",
labels=[emptylabel],
group_type=LabelGroupType.EMPTY_LABEL)
single_groups = []
for label in not_empty_labels:
single_groups.append(
LabelGroup(name=label.name,
labels=[label],
group_type=LabelGroupType.EXCLUSIVE))
label_schema.add_group(single_groups[-1])
label_schema.add_group(empty_group, exclusive_with=single_groups)
else:
main_group = LabelGroup(name="labels",
labels=not_empty_labels,
group_type=LabelGroupType.EXCLUSIVE)
label_schema.add_group(main_group)
return label_schema
def test_optimization_interface(self):
"""
<b>Description:</b>
Check IOptimizationTask class object initialization
<b>Input data:</b>
IOptimizationTask object
<b>Expected results:</b>
Test passes if IOptimizationTask object optimize method raises NotImplementedError exception
"""
dataset = DatasetEntity()
configuration = ModelConfiguration(
configurable_parameters=ConfigurableParameters(
header="Test Header"),
label_schema=LabelSchemaEntity(),
)
model_entity = ModelEntity(configuration=configuration,
train_dataset=dataset)
optimization_parameters = OptimizationParameters()
with pytest.raises(NotImplementedError):
IOptimizationTask().optimize(
optimization_type=OptimizationType.POT,
dataset=dataset,
output_model=model_entity,
optimization_parameters=optimization_parameters,
)
def test_training_interface(self):
"""
<b>Description:</b>
Check ITrainingTask class object initialization
<b>Input data:</b>
ITrainingTask object
<b>Expected results:</b>
Test passes if ITrainingTask object methods raise NotImplementedError exception
"""
i_training_task = ITrainingTask()
dataset = DatasetEntity()
configuration = ModelConfiguration(
configurable_parameters=ConfigurableParameters(
header="Test Header"),
label_schema=LabelSchemaEntity(),
)
model_entity = ModelEntity(configuration=configuration,
train_dataset=dataset)
train_parameters = TrainParameters()
with pytest.raises(NotImplementedError):
i_training_task.save_model(model_entity)
with pytest.raises(NotImplementedError):
i_training_task.train(
dataset=dataset,
output_model=model_entity,
train_parameters=train_parameters,
)
with pytest.raises(NotImplementedError):
i_training_task.cancel_training()
def environment():
"""
Return TaskEnvironment
"""
car = LabelEntity(id=ID(123456789),
name="car",
domain=Domain.DETECTION,
is_empty=True)
person = LabelEntity(id=ID(987654321),
name="person",
domain=Domain.DETECTION,
is_empty=True)
labels_list = [car, person]
dummy_template = __get_path_to_file("./dummy_template.yaml")
model_template = parse_model_template(dummy_template)
hyper_parameters = model_template.hyper_parameters.data
params = ote_config_helper.create(hyper_parameters)
labels_schema = LabelSchemaEntity.from_labels(labels_list)
environment = TaskEnvironment(
model=None,
hyper_parameters=params,
label_schema=labels_schema,
model_template=model_template,
)
return environment
def test_evaluate_interface(self):
"""
<b>Description:</b>
Check IEvaluationTask class object initialization
<b>Input data:</b>
IEvaluationTask object
<b>Expected results:</b>
Test passes if IEvaluationTask object evaluate method raises NotImplementedError exception
"""
dataset = DatasetEntity()
configuration = ModelConfiguration(
configurable_parameters=ConfigurableParameters(
header="Test Header"),
label_schema=LabelSchemaEntity(),
)
model_entity = ModelEntity(configuration=configuration,
train_dataset=dataset)
with pytest.raises(NotImplementedError):
IEvaluationTask().evaluate(
ResultSetEntity(
model=model_entity,
ground_truth_dataset=dataset,
prediction_dataset=dataset,
))
def get_ancestors_by_prediction(
label_schema: LabelSchemaEntity, prediction: ScoredLabel
) -> List[ScoredLabel]:
"""
Get all the ancestors for a given label node
"""
ancestor_labels = label_schema.get_ancestors(prediction.get_label())
return [ScoredLabel(al, prediction.probability) for al in ancestor_labels]
def __init__(self, label_schema: LabelSchemaEntity):
labels = label_schema.get_labels(include_empty=False)
self.normal_label = [
label for label in labels if label.name == "Normal"
][0]
self.anomalous_label = [
label for label in labels if label.name == "Anomalous"
][0]
def forward(
instance: LabelSchemaEntity,
) -> dict:
"""Serializes to dict."""
label_groups = [
LabelGroupMapper().forward(group)
for group in instance.get_groups(include_empty=True)
]
output_dict = {
"label_tree": LabelGraphMapper().forward(instance.label_tree),
"exclusivity_graph": LabelGraphMapper().forward(instance.exclusivity_graph),
"label_groups": label_groups,
}
output_dict["all_labels"] = {
IDMapper().forward(label.id): LabelMapper().forward(label)
for label in instance.get_labels(True)
}
return output_dict
def metadata_item_with_model() -> MetadataItemEntity:
data = TensorEntity(
name="appended_metadata_with_model",
numpy=np.random.randint(low=0, high=255, size=(10, 15, 3)),
)
configuration = ModelConfiguration(
configurable_parameters=ConfigurableParameters(
header="Test Header"),
label_schema=LabelSchemaEntity(),
)
model = ModelEntity(configuration=configuration,
train_dataset=DatasetEntity())
metadata_item_with_model = MetadataItemEntity(data=data, model=model)
return metadata_item_with_model
def __init__(self, label_schema: LabelSchemaEntity):
if len(label_schema.get_labels(False)) == 1:
self.labels = label_schema.get_labels(include_empty=True)
else:
self.labels = label_schema.get_labels(include_empty=False)
self.empty_label = get_empty_label(label_schema)
multilabel = len(label_schema.get_groups(False)) > 1 and len(
label_schema.get_groups(False)) == len(
label_schema.get_labels(include_empty=False))
self.hierarchical = False
if not multilabel and len(label_schema.get_groups(False)) > 1:
self.labels = get_leaf_labels(label_schema)
self.hierarchical = True
self.label_schema = label_schema
def test_model_configuration(self):
"""
<b>Description:</b>
Check that ModelConfiguration correctly returns the configuration
<b>Input data:</b>
ConfigurableParameters, LabelSchemaEntity
<b>Expected results:</b>
Test passes if ModelConfiguration correctly returns the configuration
<b>Steps</b>
1. Check configuration params in the ModelConfiguration
"""
parameters = ConfigurableParameters(header="Test header")
label_schema = LabelSchemaEntity()
model_configuration = ModelConfiguration(
configurable_parameters=parameters, label_schema=label_schema)
assert model_configuration.configurable_parameters == parameters
assert model_configuration.label_schema == label_schema
def generate_label_schema(dataset, task_type):
"""
Generates label schema depending on task type.
"""
if task_type == TaskType.CLASSIFICATION and dataset.is_multilabel():
not_empty_labels = dataset.get_labels()
assert len(not_empty_labels) > 1
label_schema = LabelSchemaEntity()
empty_label = LabelEntity(
name="Empty label", is_empty=True, domain=Domain.CLASSIFICATION
)
empty_group = LabelGroup(
name="empty", labels=[empty_label], group_type=LabelGroupType.EMPTY_LABEL
)
single_groups = []
for label in not_empty_labels:
single_groups.append(
LabelGroup(
name=label.name, labels=[label], group_type=LabelGroupType.EXCLUSIVE
)
)
label_schema.add_group(single_groups[-1])
label_schema.add_group(empty_group, exclusive_with=single_groups)
return label_schema
if task_type == TaskType.ANOMALY_CLASSIFICATION:
return LabelSchemaEntity.from_labels(
[
LabelEntity(
name="Normal", domain=Domain.ANOMALY_CLASSIFICATION, id=ID(0)
),
LabelEntity(
name="Anomalous", domain=Domain.ANOMALY_CLASSIFICATION, id=ID(1)
),
]
)
return LabelSchemaEntity.from_labels(dataset.get_labels())
def backward(instance: dict) -> LabelSchemaEntity:
"""Deserializes from dict."""
all_labels = {
IDMapper().backward(id): LabelMapper().backward(label)
for id, label in instance["all_labels"].items()
}
exclusivity_graph = LabelGraphMapper().backward(
instance["exclusivity_graph"], all_labels
)
label_tree = LabelGraphMapper().backward(instance["label_tree"], all_labels)
label_groups = [
LabelGroupMapper().backward(label_group, all_labels)
for label_group in instance["label_groups"]
]
output = LabelSchemaEntity(
exclusivity_graph=cast(LabelGraph, exclusivity_graph),
label_tree=cast(LabelTree, label_tree),
label_groups=label_groups,
)
return output
def test_export_interface(self):
"""
<b>Description:</b>
Check IExportTask class object initialization
<b>Input data:</b>
IExportTask object
<b>Expected results:</b>
Test passes if IExportTask object export method raises NotImplementedError exception
"""
dataset = DatasetEntity()
configuration = ModelConfiguration(
configurable_parameters=ConfigurableParameters(
header="Test Header"),
label_schema=LabelSchemaEntity(),
)
model_entity = ModelEntity(configuration=configuration,
train_dataset=dataset)
with pytest.raises(NotImplementedError):
IExportTask().export(export_type=ExportType.OPENVINO,
output_model=model_entity)
def add_hierarchy(
self, label_schema: LabelSchemaEntity
) -> Tuple[LabelEntity, LabelEntity, LabelEntity]:
"""Adds children to flowering, no_plant and vegetative"""
label_schema.add_group(
LabelGroup(
"plant_state",
[self.flowering, self.no_plant, self.vegetative],
LabelGroupType.EXCLUSIVE,
))
flower_partial_visible = self.new_label_by_name(
"flower_partial_visible")
flower_fully_visible = self.new_label_by_name("flower_fully_visible")
label_schema.add_group(
LabelGroup(
"flowering_state",
[flower_fully_visible, flower_partial_visible],
LabelGroupType.EXCLUSIVE,
))
label_schema.add_child(self.flowering, flower_partial_visible)
label_schema.add_child(self.flowering, flower_fully_visible)
assert self.flowering == label_schema.get_parent(
flower_partial_visible)
assert label_schema.get_parent(self.no_plant) is None
few_leaves = self.new_label_by_name("few_leaves")
label_schema.add_group(
LabelGroup("leaf_state", [few_leaves], LabelGroupType.EXCLUSIVE))
label_schema.add_child(self.vegetative, few_leaves)
return few_leaves, flower_fully_visible, flower_partial_visible
def test_flat_label_schema_serialization(self):
"""
This test serializes flat LabelSchema and checks serialized representation.
Then it compares deserialized LabelSchema with original one.
"""
cur_date = now()
names = ["cat", "dog", "mouse"]
colors = [
Color(
randint(0, 255), # nosec
randint(0, 255), # nosec
randint(0, 255), # nosec
randint(0, 255), # nosec
) # nosec # noqa
for _ in range(3)
]
labels = [
LabelEntity(
name=name,
domain=Domain.CLASSIFICATION,
creation_date=cur_date,
id=ID(i),
color=colors[i],
)
for i, name in enumerate(names)
]
label_shema = LabelSchemaEntity.from_labels(labels)
serialized = LabelSchemaMapper.forward(label_shema)
assert serialized == {
"label_tree": {"type": "tree", "directed": True, "nodes": [], "edges": []},
"exclusivity_graph": {
"type": "graph",
"directed": False,
"nodes": [],
"edges": [],
},
"label_groups": [
{
"_id": label_shema.get_groups()[0].id,
"name": "from_label_list",
"label_ids": ["0", "1", "2"],
"relation_type": "EXCLUSIVE",
}
],
"all_labels": {
"0": {
"_id": "0",
"name": "cat",
"color": ColorMapper.forward(colors[0]),
"hotkey": "",
"domain": "CLASSIFICATION",
"creation_date": DatetimeMapper.forward(cur_date),
"is_empty": False,
},
"1": {
"_id": "1",
"name": "dog",
"color": ColorMapper.forward(colors[1]),
"hotkey": "",
"domain": "CLASSIFICATION",
"creation_date": DatetimeMapper.forward(cur_date),
"is_empty": False,
},
"2": {
"_id": "2",
"name": "mouse",
"color": ColorMapper.forward(colors[2]),
"hotkey": "",
"domain": "CLASSIFICATION",
"creation_date": DatetimeMapper.forward(cur_date),
"is_empty": False,
},
},
}
deserialized = LabelSchemaMapper.backward(serialized)
assert label_shema == deserialized
def __init__(self, label_schema: LabelSchemaEntity):
labels = label_schema.get_labels(include_empty=False)
self.label_map = dict(enumerate(labels, 1))
def other_configuration(self):
parameters = ConfigurableParameters(header="Other test header")
label_schema = LabelSchemaEntity()
return ModelConfiguration(configurable_parameters=parameters,
label_schema=label_schema)
def test_model_entity_sets_values(self):
"""
<b>Description:</b>
Check that ModelEntity correctly returns the set values
<b>Expected results:</b>
Test passes if ModelEntity correctly returns the set values
<b>Steps</b>
1. Check set values in the ModelEntity
"""
def __get_path_to_file(filename: str):
"""
Return the path to the file named 'filename', which lives in the tests/entities directory
"""
return str(Path(__file__).parent / Path(filename))
car = LabelEntity(name="car", domain=Domain.DETECTION)
labels_list = [car]
dummy_template = __get_path_to_file("./dummy_template.yaml")
model_template = parse_model_template(dummy_template)
hyper_parameters = model_template.hyper_parameters.data
params = ote_config_helper.create(hyper_parameters)
labels_schema = LabelSchemaEntity.from_labels(labels_list)
environment = TaskEnvironment(
model=None,
hyper_parameters=params,
label_schema=labels_schema,
model_template=model_template,
)
item = self.generate_random_image()
dataset = DatasetEntity(items=[item])
score_metric = ScoreMetric(name="Model accuracy", value=0.5)
model_entity = ModelEntity(train_dataset=self.dataset(),
configuration=self.configuration())
set_params = {
"configuration": environment.get_model_configuration(),
"train_dataset": dataset,
"id": ID(1234567890),
"creation_date": self.creation_date,
"previous_trained_revision": 5,
"previous_revision": 2,
"version": 2,
"tags": ["tree", "person"],
"model_status": ModelStatus.TRAINED_NO_STATS,
"model_format": ModelFormat.BASE_FRAMEWORK,
"performance": Performance(score_metric),
"training_duration": 5.8,
"precision": [ModelPrecision.INT8],
"latency": 328,
"fps_throughput": 20,
"target_device": TargetDevice.GPU,
"target_device_type": "notebook",
"optimization_methods": [OptimizationMethod.QUANTIZATION],
"optimization_type": ModelOptimizationType.MO,
"optimization_objectives": {
"param": "Test param"
},
"performance_improvement": {"speed", 0.5},
"model_size_reduction": 1.0,
}
for key, value in set_params.items():
setattr(model_entity, key, value)
assert getattr(model_entity, key) == value
assert model_entity.is_optimized() is True
def __init__(self, labels: LabelSchemaEntity):
self.labels = labels.get_labels(include_empty=False)