def export_logo_annotation( output: pathlib.Path, server_domain: Optional[str] = None, annotated: Optional[bool] = None, ): from robotoff.models import db, LogoAnnotation, ImageModel, ImagePrediction from robotoff.utils import dump_jsonl with db: where_clauses = [] if server_domain is not None: where_clauses.append(ImageModel.server_domain == server_domain) if annotated is not None: where_clauses.append( LogoAnnotation.annotation_value.is_null(not annotated)) query = LogoAnnotation.select().join(ImagePrediction).join( ImageModel) if where_clauses: query = query.where(*where_clauses) logo_iter = query.iterator() dict_iter = (l.to_dict() for l in logo_iter) dump_jsonl(output, dict_iter)
def search(self, req: falcon.Request, resp: falcon.Response): count: int = req.get_param_as_int( "count", min_value=1, max_value=2000, default=25 ) type_: Optional[str] = req.get_param("type") barcode: Optional[str] = req.get_param("barcode") value: Optional[str] = req.get_param("value") min_confidence: Optional[float] = req.get_param_as_float("min_confidence") random: bool = req.get_param_as_bool("random", default=False) server_domain: Optional[str] = req.get_param("server_domain") annotated: bool = req.get_param_as_bool("annotated", default=False) where_clauses = [LogoAnnotation.annotation_value.is_null(not annotated)] join_image_prediction = False join_image_model = False if server_domain: where_clauses.append(ImageModel.server_domain == server_domain) join_image_model = True if min_confidence is not None: where_clauses.append(ImagePrediction.max_confidence >= min_confidence) join_image_prediction = True if barcode is not None: where_clauses.append(ImageModel.barcode == barcode) join_image_model = True if type_ is not None: where_clauses.append(LogoAnnotation.annotation_type == type_) if value is not None: value_tag = get_tag(value) where_clauses.append(LogoAnnotation.annotation_value_tag == value_tag) query = LogoAnnotation.select() join_image_prediction = join_image_prediction or join_image_model if join_image_prediction: query = query.join(ImagePrediction) if join_image_model: query = query.join(ImageModel) if where_clauses: query = query.where(*where_clauses) query_count = query.count() if random: query = query.order_by(peewee.fn.Random()) query = query.limit(count) items = [item.to_dict() for item in query.iterator()] for item in items: image_prediction = item.pop("image_prediction") item["image"] = image_prediction["image"] resp.media = {"logos": items, "count": query_count}
def fetch_logos(self, logo_ids: List[str], resp: falcon.Response): logos = [] for logo in (LogoAnnotation.select().join(ImagePrediction).join( ImageModel).where(LogoAnnotation.id.in_(logo_ids)).iterator()): logo_dict = logo.to_dict() image_prediction = logo_dict.pop("image_prediction") logo_dict["image"] = image_prediction["image"] logos.append(logo_dict) resp.media = {"logos": logos, "count": len(logos)}
def get_logo_annotations() -> Dict[int, LogoLabelType]: annotations: Dict[int, LogoLabelType] = {} for logo in (LogoAnnotation.select( LogoAnnotation.id, LogoAnnotation.annotation_type, LogoAnnotation.annotation_value, LogoAnnotation.taxonomy_value, ).where(LogoAnnotation.annotation_type.is_null(False)).iterator()): if logo.annotation_value is None: annotations[logo.id] = (logo.annotation_type, None) elif logo.taxonomy_value is not None: annotations[logo.id] = (logo.annotation_type, logo.taxonomy_value) return annotations
def add_logo_to_ann(sleep_time: float): from itertools import groupby import time import requests import tqdm from robotoff.logos import add_logos_to_ann, get_stored_logo_ids from robotoff.models import db, ImageModel, ImagePrediction, LogoAnnotation from robotoff.utils import get_logger logger = get_logger() seen = get_stored_logo_ids() with db: logos_iter = tqdm.tqdm(LogoAnnotation.select().join( ImagePrediction).join(ImageModel).where( LogoAnnotation.nearest_neighbors.is_null()).order_by( ImageModel.id).iterator()) for _, logo_batch in groupby( logos_iter, lambda x: x.image_prediction.image.id): logos = list(logo_batch) if all(l.id in seen for l in logos): continue image = logos[0].image_prediction.image logger.info(f"Adding logos of image {image.id}") try: added = add_logos_to_ann(image, logos) except requests.exceptions.ReadTimeout: logger.warn("Request timed-out during logo addition") continue logger.info(f"Added: {added}") if sleep_time: time.sleep(sleep_time)
import json from robotoff import settings from robotoff.models import LogoAnnotation, db annotations = {} with db: for logo_annotation in (LogoAnnotation.select( LogoAnnotation.id, LogoAnnotation.taxonomy_value).where( LogoAnnotation.taxonomy_value.is_null(False)).iterator()): annotations[logo_annotation.id] = logo_annotation.taxonomy_value with (settings.DATASET_DIR / "annotations.jsonl").open("w") as f: json.dump(annotations, f)