def save_nearest_neighbors(logos: List[LogoAnnotation]) -> int:
logo_ids_params = ",".join((str(logo.id) for logo in logos))
r = http_session.get(
settings.BaseURLProvider().robotoff().get() +
"/api/v1/ann/batch?logo_ids=" + logo_ids_params,
timeout=30,
)
r.raise_for_status()
response = r.json()
results = {int(key): value for key, value in response["results"].items()}
logo_id_to_logo = {logo.id: logo for logo in logos}
missing_logo_ids = set(logo_id_to_logo.keys()).difference(
set(results.keys()))
if missing_logo_ids:
logger.warning("Missing logo IDs in response: %s", missing_logo_ids)
saved = 0
for logo_id, logo_results in results.items():
if logo_id in logo_id_to_logo:
logo = logo_id_to_logo[logo_id]
distances = [n["distance"] for n in logo_results]
logo_ids = [n["logo_id"] for n in logo_results]
logo.nearest_neighbors = {
"distances": distances,
"logo_ids": logo_ids,
}
logo.save()
saved += 1
return saved
def add_logos_to_ann(image: ImageModel, logos: List[LogoAnnotation]) -> int:
if not logos:
return 0
image_url = settings.OFF_IMAGE_BASE_URL + image.source_image
data = {
"image_url":
image_url,
"logos": [{
"bounding_box": logo.bounding_box,
"id": logo.id
} for logo in logos],
}
r = http_session.post(
settings.BaseURLProvider().robotoff().get() + "/api/v1/ann/add",
json=data,
timeout=30,
)
if not r.ok:
logger.warning(
f"error while adding image to ANN ({r.status_code}): %s", r.text)
return 0
return r.json()["added"]
def print_generic_insight(insight: JSONType) -> None:
for key, value in insight.items():
click.echo("{}: {}".format(key, str(value)))
click.echo("url: {}/product/{}".format(settings.BaseURLProvider().get(),
insight["barcode"]))
if "source" in insight:
click.echo("image: {}{}".format(settings.OFF_IMAGE_BASE_URL,
insight["source"]))
click.echo("")
def generate_metrics_from_path(
country_tag: str,
path: str,
target_datetime: datetime.datetime,
count: Optional[int] = None,
facet: Optional[str] = None,
) -> List:
inserts: List = []
url = settings.BaseURLProvider().country(country_tag + "-en").get() + path
if facet is None:
facet = get_facet_name(url)
try:
r = requests.get(url, timeout=60)
except requests.exceptions.Timeout:
logger.error("OFF request timeout (60s): {}".format(url))
return inserts
if not r.ok:
logger.error("Error during OFF request: {}".format(r.status_code))
return inserts
try:
data = r.json()
except json.JSONDecodeError as e:
logger.error("Error during OFF request JSON decoding:\n{}".format(e))
return inserts
for tag in data["tags"]:
name = tag["name"]
products = tag["products"]
fields = {"products": products}
if "percent" in tag:
fields["percent"] = float(tag["percent"])
elif count is not None:
fields["percent"] = products * 100 / count
tag_id = tag["id"]
inserts.append({
"measurement": "facets",
"tags": {
"tag_name": name,
"tag_id": tag_id,
"country": country_tag,
"facet": facet,
},
"time": target_datetime.isoformat(),
"fields": fields,
})
return inserts
def print_ingredient_spellcheck_insight(insight: JSONType) -> None:
for key in ("id", "type", "barcode", "countries"):
value = insight.get(key)
click.echo("{}: {}".format(key, str(value)))
click.echo("url: {}/product/{}".format(settings.BaseURLProvider().get(),
insight["barcode"]))
original_snippet = insight["original_snippet"]
corrected_snippet = insight["corrected_snippet"]
click.echo(generate_colored_diff(original_snippet, corrected_snippet))
click.echo("")
def get_stored_logo_ids() -> Set[int]:
r = http_session.get(settings.BaseURLProvider().robotoff().get() +
"/api/v1/ann/stored",
timeout=30)
if not r.ok:
logger.warning(
f"error while fetching stored logo IDs ({r.status_code}): %s",
r.text)
return set()
return set(r.json()["stored"])
def move_to(barcode: str, to: ServerType, timeout: Optional[int] = 10) -> bool:
if get_product(barcode, server=to) is not None:
return False
url = "{}/cgi/product_jqm.pl".format(settings.BaseURLProvider().get())
params = {
"type": "edit",
"code": barcode,
"new_code": str(to),
**settings.off_credentials(),
}
r = http_session.get(url, params=params, timeout=timeout)
data = r.json()
return data["status"] == 1
def send_logo_notification(self, logo: LogoAnnotation,
probs: Dict[LogoLabelType, float]):
crop_url = logo.get_crop_image_url()
prob_text = "\n".join(
(f"{label[0]} - {label[1]}: {prob:.2g}" for label, prob in sorted(
probs.items(), key=operator.itemgetter(1), reverse=True)))
barcode = logo.image_prediction.image.barcode
base_off_url = settings.BaseURLProvider().get()
text = (
f"Prediction for <{crop_url}|image> "
f"(<https://hunger.openfoodfacts.org/logos?logo_id={logo.id}|annotate logo>, "
f"<{base_off_url}/product/{barcode}|product>):\n{prob_text}")
self._post_message(_slack_message_block(text),
self.ROBOTOFF_ALERT_CHANNEL)
def test_add_category_insight_no_insights(mocker):
mocker.patch(
"robotoff.workers.tasks.product_updated.predict_category_from_product_es",
return_value=None,
)
mocker.patch(
"robotoff.workers.tasks.product_updated.CategoryClassifier.predict",
return_value=[],
)
import_insights_mock = mocker.patch(
"robotoff.workers.tasks.product_updated.import_insights")
imported = add_category_insight("123", {"code": "123"},
settings.BaseURLProvider().get())
assert not import_insights_mock.called
assert not imported
def test_add_category_insight_with_ml_insights(mocker):
expected_prediction = Prediction(
barcode="123",
type=PredictionType.category,
value_tag="en:chicken",
data={
"lang": "xx",
"model": "neural",
"confidence": 0.9
},
automatic_processing=True,
)
mocker.patch(
"robotoff.workers.tasks.product_updated.predict_category_from_product_es",
return_value=None,
)
mocker.patch(
"robotoff.workers.tasks.product_updated.CategoryClassifier.predict",
return_value=[expected_prediction],
)
import_insights_mock = mocker.patch(
"robotoff.workers.tasks.product_updated.import_insights",
return_value=1,
)
server_domain = settings.BaseURLProvider().get()
imported = add_category_insight("123", {"code": "123"}, server_domain)
import_insights_mock.assert_called_once_with(
[
Prediction(
barcode="123",
type=PredictionType.category,
value_tag="en:chicken",
data={
"lang": "xx",
"model": "neural",
"confidence": 0.9
},
automatic_processing=True,
)
],
server_domain,
automatic=True,
)
assert imported
import json
from difflib import SequenceMatcher
from typing import Dict, Optional
import click
from robotoff import settings
from robotoff.utils import http_session
from robotoff.utils.types import JSONType
LOCAL = False
if LOCAL:
BASE_URL = "http://localhost:5500/api/v1"
else:
BASE_URL = settings.BaseURLProvider().robotoff().get() + "/api/v1"
RANDOM_INSIGHT_URL = BASE_URL + "/insights/random"
ANNOTATE_INSIGHT_URL = BASE_URL + "/insights/annotate"
class NoInsightException(Exception):
pass
def run(insight_type: Optional[str], country: Optional[str]):
while True:
try:
run_loop(insight_type, country)
except NoInsightException:
click.echo("No insight left")
"Unable to extract username from session cookie: %s",
self.session_cookie,
)
return None
class ServerType(enum.Enum):
off = 1
obf = 2
opff = 3
opf = 4
API_URLS: Dict[ServerType, str] = {
ServerType.off: settings.BaseURLProvider().get(),
ServerType.obf: "https://world.openbeautyfacts.org",
ServerType.opf: "https://world.openproductfacts.org",
ServerType.opff: "https://world.openpetfoodfacts.org",
}
BARCODE_PATH_REGEX = re.compile(r"^(...)(...)(...)(.*)$")
def get_source_from_url(ocr_url: str) -> str:
url_path = urlparse(ocr_url).path
if url_path.startswith("/images/products"):
url_path = url_path[len("/images/products") :]
def get_product_count(country_tag: str) -> int:
r = requests.get(settings.BaseURLProvider().country(country_tag).get() +
"/3.json?fields=null").json()
return int(r["count"])
from typing import Dict, List, Optional
from robotoff import settings
from robotoff.insights import InsightType
from robotoff.models import ProductInsight
from robotoff.mongo import MONGO_CLIENT_CACHE
from robotoff.off import generate_image_url, get_product
from robotoff.taxonomy import Taxonomy, TaxonomyType, get_taxonomy
from robotoff.utils import get_logger
from robotoff.utils.i18n import TranslationStore
from robotoff.utils.types import JSONType
logger = get_logger(__name__)
LABEL_IMG_BASE_URL = "https://{}/images/lang".format(
settings.BaseURLProvider().static().get())
LABEL_IMAGES = {
"en:eu-organic": LABEL_IMG_BASE_URL + "/en/labels/eu-organic.135x90.svg",
"en:ab-agriculture-biologique":
LABEL_IMG_BASE_URL + "/fr/labels/ab-agriculture-biologique.74x90.svg",
"en:european-vegetarian-union":
LABEL_IMG_BASE_URL + "/en/labels/european-vegetarian-union.90x90.svg",
"en:pgi": LABEL_IMG_BASE_URL + "/en/labels/pgi.90x90.png",
}
class Question(metaclass=abc.ABCMeta):
@abc.abstractmethod
def serialize(self) -> JSONType:
pass
def crop_image_url(source_image, bounding_box) -> str:
base_url = settings.OFF_IMAGE_BASE_URL + source_image
y_min, x_min, y_max, x_max = bounding_box
base_robotoff_url = settings.BaseURLProvider().robotoff().get()
return f"{base_robotoff_url}/api/v1/images/crop?image_url={base_url}&y_min={y_min}&x_min={x_min}&y_max={y_max}&x_max={x_max}"