def save_nearest_neighbors(logos: List[LogoAnnotation]) -> int: logo_ids_params = ",".join((str(logo.id) for logo in logos)) r = http_session.get( settings.BaseURLProvider().robotoff().get() + "/api/v1/ann/batch?logo_ids=" + logo_ids_params, timeout=30, ) r.raise_for_status() response = r.json() results = {int(key): value for key, value in response["results"].items()} logo_id_to_logo = {logo.id: logo for logo in logos} missing_logo_ids = set(logo_id_to_logo.keys()).difference( set(results.keys())) if missing_logo_ids: logger.warning("Missing logo IDs in response: %s", missing_logo_ids) saved = 0 for logo_id, logo_results in results.items(): if logo_id in logo_id_to_logo: logo = logo_id_to_logo[logo_id] distances = [n["distance"] for n in logo_results] logo_ids = [n["logo_id"] for n in logo_results] logo.nearest_neighbors = { "distances": distances, "logo_ids": logo_ids, } logo.save() saved += 1 return saved
def add_logos_to_ann(image: ImageModel, logos: List[LogoAnnotation]) -> int: if not logos: return 0 image_url = settings.OFF_IMAGE_BASE_URL + image.source_image data = { "image_url": image_url, "logos": [{ "bounding_box": logo.bounding_box, "id": logo.id } for logo in logos], } r = http_session.post( settings.BaseURLProvider().robotoff().get() + "/api/v1/ann/add", json=data, timeout=30, ) if not r.ok: logger.warning( f"error while adding image to ANN ({r.status_code}): %s", r.text) return 0 return r.json()["added"]
def print_generic_insight(insight: JSONType) -> None: for key, value in insight.items(): click.echo("{}: {}".format(key, str(value))) click.echo("url: {}/product/{}".format(settings.BaseURLProvider().get(), insight["barcode"])) if "source" in insight: click.echo("image: {}{}".format(settings.OFF_IMAGE_BASE_URL, insight["source"])) click.echo("")
def generate_metrics_from_path( country_tag: str, path: str, target_datetime: datetime.datetime, count: Optional[int] = None, facet: Optional[str] = None, ) -> List: inserts: List = [] url = settings.BaseURLProvider().country(country_tag + "-en").get() + path if facet is None: facet = get_facet_name(url) try: r = requests.get(url, timeout=60) except requests.exceptions.Timeout: logger.error("OFF request timeout (60s): {}".format(url)) return inserts if not r.ok: logger.error("Error during OFF request: {}".format(r.status_code)) return inserts try: data = r.json() except json.JSONDecodeError as e: logger.error("Error during OFF request JSON decoding:\n{}".format(e)) return inserts for tag in data["tags"]: name = tag["name"] products = tag["products"] fields = {"products": products} if "percent" in tag: fields["percent"] = float(tag["percent"]) elif count is not None: fields["percent"] = products * 100 / count tag_id = tag["id"] inserts.append({ "measurement": "facets", "tags": { "tag_name": name, "tag_id": tag_id, "country": country_tag, "facet": facet, }, "time": target_datetime.isoformat(), "fields": fields, }) return inserts
def print_ingredient_spellcheck_insight(insight: JSONType) -> None: for key in ("id", "type", "barcode", "countries"): value = insight.get(key) click.echo("{}: {}".format(key, str(value))) click.echo("url: {}/product/{}".format(settings.BaseURLProvider().get(), insight["barcode"])) original_snippet = insight["original_snippet"] corrected_snippet = insight["corrected_snippet"] click.echo(generate_colored_diff(original_snippet, corrected_snippet)) click.echo("")
def get_stored_logo_ids() -> Set[int]: r = http_session.get(settings.BaseURLProvider().robotoff().get() + "/api/v1/ann/stored", timeout=30) if not r.ok: logger.warning( f"error while fetching stored logo IDs ({r.status_code}): %s", r.text) return set() return set(r.json()["stored"])
def move_to(barcode: str, to: ServerType, timeout: Optional[int] = 10) -> bool: if get_product(barcode, server=to) is not None: return False url = "{}/cgi/product_jqm.pl".format(settings.BaseURLProvider().get()) params = { "type": "edit", "code": barcode, "new_code": str(to), **settings.off_credentials(), } r = http_session.get(url, params=params, timeout=timeout) data = r.json() return data["status"] == 1
def send_logo_notification(self, logo: LogoAnnotation, probs: Dict[LogoLabelType, float]): crop_url = logo.get_crop_image_url() prob_text = "\n".join( (f"{label[0]} - {label[1]}: {prob:.2g}" for label, prob in sorted( probs.items(), key=operator.itemgetter(1), reverse=True))) barcode = logo.image_prediction.image.barcode base_off_url = settings.BaseURLProvider().get() text = ( f"Prediction for <{crop_url}|image> " f"(<https://hunger.openfoodfacts.org/logos?logo_id={logo.id}|annotate logo>, " f"<{base_off_url}/product/{barcode}|product>):\n{prob_text}") self._post_message(_slack_message_block(text), self.ROBOTOFF_ALERT_CHANNEL)
def test_add_category_insight_no_insights(mocker): mocker.patch( "robotoff.workers.tasks.product_updated.predict_category_from_product_es", return_value=None, ) mocker.patch( "robotoff.workers.tasks.product_updated.CategoryClassifier.predict", return_value=[], ) import_insights_mock = mocker.patch( "robotoff.workers.tasks.product_updated.import_insights") imported = add_category_insight("123", {"code": "123"}, settings.BaseURLProvider().get()) assert not import_insights_mock.called assert not imported
def test_add_category_insight_with_ml_insights(mocker): expected_prediction = Prediction( barcode="123", type=PredictionType.category, value_tag="en:chicken", data={ "lang": "xx", "model": "neural", "confidence": 0.9 }, automatic_processing=True, ) mocker.patch( "robotoff.workers.tasks.product_updated.predict_category_from_product_es", return_value=None, ) mocker.patch( "robotoff.workers.tasks.product_updated.CategoryClassifier.predict", return_value=[expected_prediction], ) import_insights_mock = mocker.patch( "robotoff.workers.tasks.product_updated.import_insights", return_value=1, ) server_domain = settings.BaseURLProvider().get() imported = add_category_insight("123", {"code": "123"}, server_domain) import_insights_mock.assert_called_once_with( [ Prediction( barcode="123", type=PredictionType.category, value_tag="en:chicken", data={ "lang": "xx", "model": "neural", "confidence": 0.9 }, automatic_processing=True, ) ], server_domain, automatic=True, ) assert imported
import json from difflib import SequenceMatcher from typing import Dict, Optional import click from robotoff import settings from robotoff.utils import http_session from robotoff.utils.types import JSONType LOCAL = False if LOCAL: BASE_URL = "http://localhost:5500/api/v1" else: BASE_URL = settings.BaseURLProvider().robotoff().get() + "/api/v1" RANDOM_INSIGHT_URL = BASE_URL + "/insights/random" ANNOTATE_INSIGHT_URL = BASE_URL + "/insights/annotate" class NoInsightException(Exception): pass def run(insight_type: Optional[str], country: Optional[str]): while True: try: run_loop(insight_type, country) except NoInsightException: click.echo("No insight left")
"Unable to extract username from session cookie: %s", self.session_cookie, ) return None class ServerType(enum.Enum): off = 1 obf = 2 opff = 3 opf = 4 API_URLS: Dict[ServerType, str] = { ServerType.off: settings.BaseURLProvider().get(), ServerType.obf: "https://world.openbeautyfacts.org", ServerType.opf: "https://world.openproductfacts.org", ServerType.opff: "https://world.openpetfoodfacts.org", } BARCODE_PATH_REGEX = re.compile(r"^(...)(...)(...)(.*)$") def get_source_from_url(ocr_url: str) -> str: url_path = urlparse(ocr_url).path if url_path.startswith("/images/products"): url_path = url_path[len("/images/products") :]
def get_product_count(country_tag: str) -> int: r = requests.get(settings.BaseURLProvider().country(country_tag).get() + "/3.json?fields=null").json() return int(r["count"])
from typing import Dict, List, Optional from robotoff import settings from robotoff.insights import InsightType from robotoff.models import ProductInsight from robotoff.mongo import MONGO_CLIENT_CACHE from robotoff.off import generate_image_url, get_product from robotoff.taxonomy import Taxonomy, TaxonomyType, get_taxonomy from robotoff.utils import get_logger from robotoff.utils.i18n import TranslationStore from robotoff.utils.types import JSONType logger = get_logger(__name__) LABEL_IMG_BASE_URL = "https://{}/images/lang".format( settings.BaseURLProvider().static().get()) LABEL_IMAGES = { "en:eu-organic": LABEL_IMG_BASE_URL + "/en/labels/eu-organic.135x90.svg", "en:ab-agriculture-biologique": LABEL_IMG_BASE_URL + "/fr/labels/ab-agriculture-biologique.74x90.svg", "en:european-vegetarian-union": LABEL_IMG_BASE_URL + "/en/labels/european-vegetarian-union.90x90.svg", "en:pgi": LABEL_IMG_BASE_URL + "/en/labels/pgi.90x90.png", } class Question(metaclass=abc.ABCMeta): @abc.abstractmethod def serialize(self) -> JSONType: pass
def crop_image_url(source_image, bounding_box) -> str: base_url = settings.OFF_IMAGE_BASE_URL + source_image y_min, x_min, y_max, x_max = bounding_box base_robotoff_url = settings.BaseURLProvider().robotoff().get() return f"{base_robotoff_url}/api/v1/images/crop?image_url={base_url}&y_min={y_min}&x_min={x_min}&y_max={y_max}&x_max={x_max}"