class TestPackagerCodeInsightImporter:
def test_get_type(self):
assert PackagerCodeInsightImporter.get_type(
) == InsightType.packager_code
def test_get_required_prediction_types(self):
assert PackagerCodeInsightImporter.get_required_prediction_types() == {
PredictionType.packager_code
}
def test_is_conflicting_insight(self):
assert PackagerCodeInsightImporter.is_conflicting_insight(
ProductInsight(value="tag1"), ProductInsight(value="tag1"))
assert not PackagerCodeInsightImporter.is_conflicting_insight(
ProductInsight(value="tag1"), ProductInsight(value="tag2"))
@pytest.mark.parametrize(
"product,emb_code,expected",
[
(
Product({"emb_codes_tags": ["FR 40.261.001 CE"]}),
"fr 40261001 ce",
False,
),
(
Product({"emb_codes_tags": ["FR 40.261.001 CE"]}),
"fr 50262601 ce",
True,
),
],
)
def test_is_prediction_valid(self, product, emb_code, expected):
assert (PackagerCodeInsightImporter.is_prediction_valid(
product, emb_code) is expected)
def test_generate_candidates(self):
prediction = Prediction(type=PredictionType.packager_code,
value="fr 40.261.001 ce")
selected = list(
PackagerCodeInsightImporter.generate_candidates(
Product({"emb_codes_tags": ["FR 50.200.000 CE"]}),
[prediction],
))
assert len(selected) == 1
insight = selected[0]
assert isinstance(insight, ProductInsight)
assert insight.value == prediction.value
assert insight.type == InsightType.packager_code
def updated_product_update_insights(barcode: str):
product_dict = get_product(barcode)
if product_dict is None:
logger.warn("Updated product does not exist: {}".format(barcode))
return
category_added = updated_product_add_category_insight(
barcode, product_dict)
if category_added:
logger.info("Product {} updated".format(barcode))
product = Product(product_dict)
validators: Dict[str, InsightValidator] = {}
for insight in (ProductInsight.select().where(
ProductInsight.annotation.is_null(),
ProductInsight.barcode == barcode).iterator()):
if insight.type not in validators:
validators[insight.type] = InsightValidatorFactory.create(
insight.type, None)
validator = validators[insight.type]
insight_deleted = delete_invalid_insight(insight,
validator=validator,
product=product)
if insight_deleted:
logger.info("Insight {} deleted (type: {})".format(
insight.id, insight.type))
def test_generate_candidates(self):
prediction = Prediction(type=PredictionType.packager_code,
value="fr 40.261.001 ce")
selected = list(
PackagerCodeInsightImporter.generate_candidates(
Product({"emb_codes_tags": ["FR 50.200.000 CE"]}),
[prediction],
))
assert len(selected) == 1
insight = selected[0]
assert isinstance(insight, ProductInsight)
assert insight.value == prediction.value
assert insight.type == InsightType.packager_code
def update_insights(barcode: str, server_domain: str):
# Sleep 10s to let the OFF update request that triggered the webhook call
# to finish
time.sleep(10)
product_dict = get_product(barcode)
if product_dict is None:
logger.warn("Updated product does not exist: {}".format(barcode))
return
updated = updated_product_predict_insights(barcode, product_dict, server_domain)
if updated:
logger.info("Product {} updated".format(barcode))
update_ingredients(barcode, product_dict, server_domain)
product = Product(product_dict)
validators: Dict[str, Optional[InsightValidator]] = {}
for insight in (
ProductInsight.select()
.where(
ProductInsight.annotation.is_null(),
ProductInsight.barcode == barcode,
ProductInsight.server_domain == server_domain,
)
.iterator()
):
if insight.type not in validators:
validators[insight.type] = InsightValidatorFactory.create(
insight.type, None
)
validator = validators[insight.type]
if validator is not None:
result = validate_insight(insight, validator=validator, product=product)
if result == InsightValidationResult.deleted:
logger.info(
"Insight {} deleted (type: {})".format(insight.id, insight.type)
)
elif result == InsightValidationResult.updated:
logger.info(
"Insight {} converted to latent (type: {})".format(
insight.id, insight.type
)
)
def _fake_store(monkeypatch, barcode):
monkeypatch.setattr(
robotoff.insights.importer,
"get_product_store",
lambda: {
barcode: Product(
{
"code": barcode, # needed to validate brand/label
# needed to validate image
"images": {
"2": {"rev": 1, "uploaded_t": datetime.utcnow().timestamp()}
},
}
)
},
)
def test_generate_insights_automatic_processing(self, mocker):
class FakeImporter(InsightImporter):
@classmethod
def generate_candidates(cls, product, predictions):
yield from (ProductInsight(**prediction.to_dict())
for prediction in predictions)
@classmethod
def get_insight_update(cls, candidates, references):
return candidates, references
mocker.patch(
"robotoff.insights.importer.get_existing_insight",
return_value=[],
)
prediction = Prediction(
type=PredictionType.category,
barcode=DEFAULT_BARCODE,
data={},
automatic_processing=True,
)
generated = list(
FakeImporter.generate_insights(
[prediction],
DEFAULT_SERVER_DOMAIN,
automatic=True,
product_store=FakeProductStore(
data={DEFAULT_BARCODE: Product({"code": DEFAULT_BARCODE})
}),
))
assert len(generated) == 1
to_create, to_delete = generated[0]
assert not to_delete
assert len(to_create) == 1
created_insight = to_create[0]
assert isinstance(created_insight.process_after, datetime.datetime)
def get_product(quantity: Optional[str] = None):
return Product({"code": DEFAULT_BARCODE, "quantity": quantity})
class TestCategoryImporter:
def test_get_type(self):
assert CategoryImporter.get_type() == InsightType.category
def test_get_required_prediction_types(self):
assert CategoryImporter.get_required_prediction_types() == {
PredictionType.category
}
@pytest.mark.parametrize(
"category,to_check_categories,expected",
[
("en:salmons", {"en:smoked-salmons"}, True),
("en:smoked-salmons", {"en:salmons"}, False),
("en:snacks", {"en:dairies"}, False),
("en:dairies", {"en:snacks"}, False),
],
)
def test_is_parent_category(self, category, to_check_categories, expected):
assert (CategoryImporter.is_parent_category(
category, to_check_categories) is expected)
@pytest.mark.parametrize(
"predictions,product,expected_value_tags",
[
(
[
Prediction(PredictionType.category, value_tag="en:meats"),
],
Product({
"code": DEFAULT_BARCODE,
"categories_tags": ["en:meats"]
}),
[],
),
(
[
Prediction(PredictionType.category,
value_tag="en:non-existing-tag"),
],
Product({"code": DEFAULT_BARCODE}),
[],
),
(
[
Prediction(PredictionType.category, value_tag="en:meats"),
],
Product({"code": DEFAULT_BARCODE}),
["en:meats"],
),
(
[
Prediction(PredictionType.category, value_tag="en:meats"),
Prediction(PredictionType.category, value_tag="en:pork"),
],
Product({"code": DEFAULT_BARCODE}),
["en:pork"],
),
(
[
Prediction(PredictionType.category,
value_tag="en:miso-soup"),
Prediction(PredictionType.category, value_tag="en:meats"),
Prediction(PredictionType.category, value_tag="en:soups"),
Prediction(PredictionType.category,
value_tag="en:apricots"),
],
Product({
"code": DEFAULT_BARCODE,
"categories_tags": ["en:apricots"]
}),
["en:miso-soup", "en:meats"],
),
],
)
def test_generate_candidates(self, predictions, product,
expected_value_tags):
candidates = list(
CategoryImporter.generate_candidates(product, predictions))
assert all(isinstance(c, ProductInsight) for c in candidates)
assert len(candidates) == len(expected_value_tags)
for candidate, expected_value_tag in zip(candidates,
expected_value_tags):
assert candidate.value_tag == expected_value_tag
class TestLabelInsightImporter:
def test_get_type(self):
assert LabelInsightImporter.get_type() == InsightType.label
def test_get_required_prediction_types(self):
assert LabelInsightImporter.get_required_prediction_types() == {
PredictionType.label
}
@pytest.mark.parametrize(
"label,to_check_labels,expected",
[
("en:organic", {"en:eu-organic"}, True),
("en:eu-organic", {"en:organic"}, False),
("en:organic", {"en:fsc"}, False),
("en:fsc", {"en:organic"}, False),
],
)
def test_is_parent_label(self, label, to_check_labels, expected):
assert LabelInsightImporter.is_parent_label(
label, to_check_labels) is expected
@pytest.mark.parametrize(
"predictions,product,expected",
[
(
[
Prediction(PredictionType.label, value_tag="en:organic"),
],
Product({
"code": DEFAULT_BARCODE,
"labels_tags": ["en:organic"]
}),
[],
),
(
[
Prediction(PredictionType.label,
value_tag="en:non-existing-tag"),
],
Product({"code": DEFAULT_BARCODE}),
[],
),
(
[
Prediction(PredictionType.label, value_tag="en:organic"),
],
Product({"code": DEFAULT_BARCODE}),
[("en:organic", True)],
),
(
[
Prediction(PredictionType.label, value_tag="en:organic"),
Prediction(PredictionType.label, value_tag="en:ecoveg"),
],
Product({"code": DEFAULT_BARCODE}),
[("en:ecoveg", False)],
),
(
[
Prediction(PredictionType.label, value_tag="en:vegan"),
Prediction(PredictionType.label, value_tag="en:ecoveg"),
Prediction(PredictionType.label,
value_tag="en:non-existing-tag"),
Prediction(PredictionType.label,
value_tag="en:max-havelaar"),
Prediction(PredictionType.label, value_tag="en:organic"),
],
Product({
"code": DEFAULT_BARCODE,
"labels_tags": ["en:vegan"]
}),
[("en:ecoveg", False), ("en:max-havelaar", True)],
),
],
)
def test_generate_candidates(self, predictions, product, expected):
candidates = list(
LabelInsightImporter.generate_candidates(product, predictions))
assert all(isinstance(c, ProductInsight) for c in candidates)
assert len(candidates) == len(expected)
for candidate, (value_tag,
automatic_processing) in zip(candidates, expected):
assert candidate.value_tag == value_tag
assert candidate.automatic_processing is automatic_processing
def test_generate_insights_creation_and_deletion(self, mocker):
"""Test `get_insight_update` method in the following case:
- product exists
- an insight of the same type already exists for this product
- the insight update triggers the deletion of the old insight and
the creation of a new one
"""
class FakeImporter(InsightImporter):
@classmethod
def generate_candidates(cls, product, predictions):
yield from (ProductInsight(**prediction.to_dict())
for prediction in predictions)
@classmethod
def get_insight_update(cls, candidates, references):
return candidates, references
reference = ProductInsight(barcode=DEFAULT_BARCODE,
type=InsightType.category,
value_tag="tag1")
get_existing_insight_mock = mocker.patch(
"robotoff.insights.importer.get_existing_insight",
return_value=[reference],
)
prediction = Prediction(
type=PredictionType.category,
barcode=DEFAULT_BARCODE,
value_tag="tag2",
data={"k": "v"},
automatic_processing=True,
source_image="/images/products/322/982/001/9192/8.jpg",
)
generated = list(
FakeImporter.generate_insights(
[prediction],
DEFAULT_SERVER_DOMAIN,
automatic=False,
product_store=FakeProductStore(
data={
DEFAULT_BARCODE:
Product({
"code": DEFAULT_BARCODE,
"images": {
"8": {
"uploaded_t": DEFAULT_UPLOADED_T
}
},
})
}),
))
assert len(generated) == 1
to_create, to_delete = generated[0]
assert len(to_create) == 1
created_insight = to_create[0]
assert isinstance(created_insight, ProductInsight)
assert created_insight.automatic_processing is False
assert isinstance(created_insight.timestamp, datetime.datetime)
assert created_insight.type == "category"
assert created_insight.value_tag == "tag2"
assert created_insight.data == {"k": "v"}
assert created_insight.barcode == DEFAULT_BARCODE
assert created_insight.server_domain == DEFAULT_SERVER_DOMAIN
assert created_insight.server_type == "off"
assert created_insight.process_after is None
uuid.UUID(created_insight.id)
assert to_delete == [reference]
get_existing_insight_mock.assert_called_once()
def fake_product_store(self):
return {barcode1: Product({"categories_tags": ["en:Fish"]})}