def export_logo_annotation(
output: pathlib.Path,
server_domain: Optional[str] = None,
annotated: Optional[bool] = None,
):
from robotoff.models import db, LogoAnnotation, ImageModel, ImagePrediction
from robotoff.utils import dump_jsonl
with db:
where_clauses = []
if server_domain is not None:
where_clauses.append(ImageModel.server_domain == server_domain)
if annotated is not None:
where_clauses.append(
LogoAnnotation.annotation_value.is_null(not annotated))
query = LogoAnnotation.select().join(ImagePrediction).join(
ImageModel)
if where_clauses:
query = query.where(*where_clauses)
logo_iter = query.iterator()
dict_iter = (l.to_dict() for l in logo_iter)
dump_jsonl(output, dict_iter)
def search(self, req: falcon.Request, resp: falcon.Response):
count: int = req.get_param_as_int(
"count", min_value=1, max_value=2000, default=25
)
type_: Optional[str] = req.get_param("type")
barcode: Optional[str] = req.get_param("barcode")
value: Optional[str] = req.get_param("value")
min_confidence: Optional[float] = req.get_param_as_float("min_confidence")
random: bool = req.get_param_as_bool("random", default=False)
server_domain: Optional[str] = req.get_param("server_domain")
annotated: bool = req.get_param_as_bool("annotated", default=False)
where_clauses = [LogoAnnotation.annotation_value.is_null(not annotated)]
join_image_prediction = False
join_image_model = False
if server_domain:
where_clauses.append(ImageModel.server_domain == server_domain)
join_image_model = True
if min_confidence is not None:
where_clauses.append(ImagePrediction.max_confidence >= min_confidence)
join_image_prediction = True
if barcode is not None:
where_clauses.append(ImageModel.barcode == barcode)
join_image_model = True
if type_ is not None:
where_clauses.append(LogoAnnotation.annotation_type == type_)
if value is not None:
value_tag = get_tag(value)
where_clauses.append(LogoAnnotation.annotation_value_tag == value_tag)
query = LogoAnnotation.select()
join_image_prediction = join_image_prediction or join_image_model
if join_image_prediction:
query = query.join(ImagePrediction)
if join_image_model:
query = query.join(ImageModel)
if where_clauses:
query = query.where(*where_clauses)
query_count = query.count()
if random:
query = query.order_by(peewee.fn.Random())
query = query.limit(count)
items = [item.to_dict() for item in query.iterator()]
for item in items:
image_prediction = item.pop("image_prediction")
item["image"] = image_prediction["image"]
resp.media = {"logos": items, "count": query_count}
def fetch_logos(self, logo_ids: List[str], resp: falcon.Response):
logos = []
for logo in (LogoAnnotation.select().join(ImagePrediction).join(
ImageModel).where(LogoAnnotation.id.in_(logo_ids)).iterator()):
logo_dict = logo.to_dict()
image_prediction = logo_dict.pop("image_prediction")
logo_dict["image"] = image_prediction["image"]
logos.append(logo_dict)
resp.media = {"logos": logos, "count": len(logos)}
def get_logo_annotations() -> Dict[int, LogoLabelType]:
annotations: Dict[int, LogoLabelType] = {}
for logo in (LogoAnnotation.select(
LogoAnnotation.id,
LogoAnnotation.annotation_type,
LogoAnnotation.annotation_value,
LogoAnnotation.taxonomy_value,
).where(LogoAnnotation.annotation_type.is_null(False)).iterator()):
if logo.annotation_value is None:
annotations[logo.id] = (logo.annotation_type, None)
elif logo.taxonomy_value is not None:
annotations[logo.id] = (logo.annotation_type, logo.taxonomy_value)
return annotations
def add_logo_to_ann(sleep_time: float):
from itertools import groupby
import time
import requests
import tqdm
from robotoff.logos import add_logos_to_ann, get_stored_logo_ids
from robotoff.models import db, ImageModel, ImagePrediction, LogoAnnotation
from robotoff.utils import get_logger
logger = get_logger()
seen = get_stored_logo_ids()
with db:
logos_iter = tqdm.tqdm(LogoAnnotation.select().join(
ImagePrediction).join(ImageModel).where(
LogoAnnotation.nearest_neighbors.is_null()).order_by(
ImageModel.id).iterator())
for _, logo_batch in groupby(
logos_iter, lambda x: x.image_prediction.image.id):
logos = list(logo_batch)
if all(l.id in seen for l in logos):
continue
image = logos[0].image_prediction.image
logger.info(f"Adding logos of image {image.id}")
try:
added = add_logos_to_ann(image, logos)
except requests.exceptions.ReadTimeout:
logger.warn("Request timed-out during logo addition")
continue
logger.info(f"Added: {added}")
if sleep_time:
time.sleep(sleep_time)
import json
from robotoff import settings
from robotoff.models import LogoAnnotation, db
annotations = {}
with db:
for logo_annotation in (LogoAnnotation.select(
LogoAnnotation.id, LogoAnnotation.taxonomy_value).where(
LogoAnnotation.taxonomy_value.is_null(False)).iterator()):
annotations[logo_annotation.id] = logo_annotation.taxonomy_value
with (settings.DATASET_DIR / "annotations.jsonl").open("w") as f:
json.dump(annotations, f)