def _process_product_insights(
self,
product: Optional[Product],
barcode: str,
insights: List[Insight],
automatic: bool,
server_domain: str,
) -> Iterator[Insight]:
for insight in self.process_product_insights(product, barcode,
insights, server_domain):
source_image: Optional[str] = insight.source_image
if (product and source_image
and not is_valid_image(product.images, source_image)):
logger.info("Invalid image for product {}: {}".format(
barcode, source_image))
continue
if not product and self.product_store.is_real_time():
# if product store is in real time, the product does not exist (deleted)
logger.info("Insight of deleted product {}".format(barcode))
continue
if not automatic:
insight.automatic_processing = False
elif insight.automatic_processing is None:
insight.automatic_processing = not self.need_validation(
insight)
yield insight
def generate_fiber_quality_facet():
product_store: DBProductStore = get_product_store()
collection = product_store.collection
added = 0
seen_set: Set[str] = set()
for insight in (ProductInsight.select(
ProductInsight.barcode, ProductInsight.source_image).where(
ProductInsight.type == InsightType.nutrient_mention.name,
ProductInsight.data["mentions"].contains("fiber"),
ProductInsight.source_image.is_null(False),
).iterator()):
barcode = insight.barcode
if barcode in seen_set:
continue
product = product_store.get_product(
barcode, ["nutriments", "data_quality_tags", "images"])
if product is None:
continue
nutriments = product.get("nutriments", {})
data_quality_tags = product.get("data_quality_tags", {})
images = product.get("images", {})
if (not is_valid_image(images, insight.source_image)
or "fiber" in nutriments or "fiber_prepared" in nutriments):
continue
facets = []
if FIBER_QUALITY_FACET_NAME not in data_quality_tags:
facets.append(FIBER_QUALITY_FACET_NAME)
if (FIBER_NUTRITION_QUALITY_FACET_NAME not in data_quality_tags
and is_nutrition_image(images, insight.source_image)):
facets.append(FIBER_NUTRITION_QUALITY_FACET_NAME)
if not facets:
continue
logger.info("Adding facets to {}: {}".format(barcode, facets))
seen_set.add(barcode)
added += 1
collection.update_one(
{"code": barcode},
{
"$push": {
"data_quality_tags": {
"$each": facets
},
"data_quality_warnings_tags": {
"$each": facets
},
}
},
)
logger.info("Fiber quality facets added on {} products".format(added))
def has_invalid_image(self,
insight: ProductInsight,
product: Optional[Product] = None) -> bool:
if (product and insight.source_image
and not is_valid_image(product.images, insight.source_image)):
return True
return False
def is_valid_product_prediction(prediction: Prediction,
product: Optional[Product] = None) -> bool:
"""Return True if the Prediction is valid and can be imported,
i.e:
- if the source image (if any) is valid
- if the product was not deleted
:param prediction: The Prediction to check
:param product: The Product fetched from DBProductStore
:return: Whether the Prediction is valid
"""
if not product:
# the product does not exist (deleted)
logger.info(f"Prediction of deleted product {prediction.barcode}")
return False
if prediction.source_image and not is_valid_image(product.images,
prediction.source_image):
logger.info(
f"Invalid image for product {product.barcode}: {prediction.source_image}"
)
return False
return True
def test_is_valid_image(
images: JSONType,
image_path: str,
output: bool,
):
assert is_valid_image(images, image_path) is output