def test_serialize_offer_group():
offer = offers_factories.OfferFactory(extraData={})
serialized = appsearch.AppSearchBackend().serialize_offer(offer)
assert serialized["group"] == str(offer.id)
offer.extraData["visa"] = "56070"
serialized = appsearch.AppSearchBackend().serialize_offer(offer)
assert serialized["group"] == "56070"
offer.extraData["isbn"] = "123456789"
serialized = appsearch.AppSearchBackend().serialize_offer(offer)
assert serialized["group"] == "123456789"
def test_serialize_venue():
venue = offers_factories.VenueFactory(
venueTypeCode="VISUAL_ARTS",
contact__email="*****@*****.**",
contact__website=None,
contact__phone_number=None,
contact__social_medias={
"facebook": None,
"instagram": None,
"snapchat": None,
"twitter": "https://twitter.com/my.venue",
},
)
serialized = appsearch.AppSearchBackend().serialize_venue(venue)
assert serialized == {
"id": venue.id,
"name": venue.name,
"offerer_name": venue.managingOfferer.name,
"venue_type": venue.venueTypeCode.name,
"position": f"{venue.latitude},{venue.longitude}",
"description": venue.description,
"email": "*****@*****.**",
"twitter": "https://twitter.com/my.venue",
"audio_disability": 0,
"mental_disability": 0,
"motor_disability": 0,
"visual_disability": 0,
}
def test_serialize_offer_dates_and_times():
offer = offers_factories.OfferFactory(
subcategoryId=subcategories.SEANCE_CINE.id)
dt = datetime.datetime(2032, 1, 1, 12, 15)
offers_factories.EventStockFactory(offer=offer, beginningDatetime=dt)
serialized = appsearch.AppSearchBackend().serialize_offer(offer)
assert serialized["date_created"] == offer.dateCreated
assert serialized["dates"] == [dt]
assert serialized["times"] == [12 * 60 * 60 + 15 * 60]
def test_do_no_return_booleans():
# If the backend's `serialize()` method returned boolean values,
# they would be left as booleans when JSON-encoded, and the App
# Search API would reject them (because it does not support this
# data type).
offer = offers_factories.OfferFactory()
serialized = appsearch.AppSearchBackend().serialize_offer(offer)
for key, value in serialized.items():
assert not isinstance(value, bool), f"Key {key}should not be a boolean"
def test_serialize_disability_related_fields():
venue = offers_factories.VenueFactory(
audioDisabilityCompliant=True,
mentalDisabilityCompliant=False,
motorDisabilityCompliant=None,
visualDisabilityCompliant=True,
)
serialized = appsearch.AppSearchBackend().serialize_venue(venue)
assert serialized["audio_disability"] == 1
assert serialized["mental_disability"] == 0
assert serialized["visual_disability"] == 1
assert "motor_disability" not in serialized
def test_check_number_of_sql_queries():
offer = offers_factories.OfferFactory()
# FIXME (dbaty, 2021-07-05): we should put these `joinedload` in a
# function, and call that function from `_reindex_offer_ids()`
# where we fetch offers.
offer = (offers_models.Offer.query.options(
joinedload(offers_models.Offer.venue).joinedload(
offerers_models.Venue.managingOfferer)).options(
joinedload(offers_models.Offer.criteria)).options(
joinedload(offers_models.Offer.mediations)).options(
joinedload(offers_models.Offer.product)).options(
joinedload(offers_models.Offer.stocks)).one())
# Make sure that the JOINs above are enough to avoid any extra SQL
# query below where serializing an offer.
with assert_num_queries(0):
appsearch.AppSearchBackend().serialize_offer(offer)
def index_venues():
# FIXME (antoinewg, 2021-09-15): late import to avoid import loop of models.
import pcapi.core.offerers.models as offerers_models
permanent_venues = []
for venue in offerers_models.Venue.query.all():
if venue.isPermanent:
print(f"Found {venue} to add to index ({venue.name[:20]}...)")
permanent_venues.append(venue)
if not permanent_venues:
print("ERR: Could not find any permanent venues to index")
return
backend = appsearch.AppSearchBackend()
backend.index_venues(permanent_venues)
print(f"Successfully created or updated {len(permanent_venues)} venues")
def index_offers():
# FIXME (dbaty, 2021-07-01): late import to avoid import loop of models.
import pcapi.core.offers.models as offers_models
bookable_offers = []
for offer in offers_models.Offer.query.all():
if offer.isBookable:
print(f"Found {offer} to add to index ({offer.name[:20]}...)")
bookable_offers.append(offer)
if not bookable_offers:
print("ERR: Could not find any bookable offers to index")
return
backend = appsearch.AppSearchBackend()
backend.index_offers(bookable_offers)
print(f"Successfully created or updated {len(bookable_offers)} offers")
def test_serialize_offer():
offer = offers_factories.OfferFactory(
name="Titre formidable",
description="Un LIVRE qu'il est bien pour le lire",
extraData={
"author": "Author",
"isbn": "2221001648",
"performer": "Performer",
"speaker": "Speaker",
"stageDirector": "Stage Director",
},
rankingWeight=2,
subcategoryId=subcategories.LIVRE_PAPIER.id,
venue__id=127,
venue__name="La Moyenne Librairie SA",
venue__publicName="La Moyenne Librairie",
venue__managingOfferer__name="Les Librairies Associées",
)
stock = offers_factories.StockFactory(offer=offer, price=10)
serialized = appsearch.AppSearchBackend().serialize_offer(offer)
assert serialized == {
"artist": "Author Performer Speaker Stage Director",
"category": "LIVRE",
"date_created": offer.dateCreated,
"description": "livre bien lire",
"group": "2221001648",
"is_digital": 0,
"is_duo": 0,
"is_educational": 0,
"is_event": 0,
"is_thing": 1,
"name": "Titre formidable",
"id": offer.id,
"label": "Livre",
"prices": [1000],
"ranking_weight": 2,
"search_group_name": subcategories.SearchGroups.LIVRE.name,
"stocks_date_created": [stock.dateCreated],
"subcategory_id": subcategories.LIVRE_PAPIER.id,
"offerer_name": "Les Librairies Associées",
"venue_id": 127,
"venue_department_code": "75",
"venue_name": "La Moyenne Librairie SA",
"venue_position": "48.87004,2.37850",
"venue_public_name": "La Moyenne Librairie",
}
def test_serialize_offer_artist_empty():
offer = offers_factories.OfferFactory(extraData={})
serialized = appsearch.AppSearchBackend().serialize_offer(offer)
assert "artist" not in serialized
def test_serialize_offer_artist_strip():
offer = offers_factories.OfferFactory(extraData={"author": "Author"})
serialized = appsearch.AppSearchBackend().serialize_offer(offer)
assert serialized["artist"] == "Author"
def setup_educational_offers_engines():
backend = appsearch.AppSearchBackend()
setup_engine(backend.offers_engine,
engine_names=[appsearch.EDUCATIONAL_OFFER_ENGINE_NAME])
def test_serialize_offer_thumb_url():
offer = offers_factories.OfferFactory(product__thumbCount=1)
serialized = appsearch.AppSearchBackend().serialize_offer(offer)
assert serialized[
"thumb_url"] == f"/storage/thumbs/products/{humanize(offer.productId)}"
def test_serialize_offer_tags():
tag = offers_factories.OfferCriterionFactory(criterion__name="formidable")
serialized = appsearch.AppSearchBackend().serialize_offer(tag.offer)
assert serialized["tags"] == ["formidable"]
def get_backend():
return appsearch.AppSearchBackend()
def setup_venues_engine():
backend = appsearch.AppSearchBackend()
setup_engine(backend.venues_engine,
engine_names=[appsearch.VENUES_ENGINE_NAME])
def full_index_offers(backend, start, end):
"""Reindex all bookable offers.
The script iterates over all active offers. For each offer, it
indexes it on the selected backend Algolia or App Search) if the
offer is bookable. Otherwise, the offer is NOT unindexed, as we
suppose that the offer has already been unindexed through the
normally-run code. That way, this script skips a lot of
unnecessary unindexation requests.
This script processes batches of 1.000 offers (note that HTTP
requests to App Search cannot have include more than 100 offers)
and reports back every 10.000 offers.
Errors are logged and are not blocking. You MUST check the logs
for batches that failed and MUST re-run all these batches.
Obviously, the script takes a lot of time. You should run multiple
instances of it at the same time. 4 concurrent instances looks
like a good compromise.
Usage:
$ flask full_index_offers.py algolia 10_000_000 20_000_000
Using "_" as thousands separator is supported (and encouraged for
clarity).
"""
if start > end:
raise ValueError('"start" must be less than "end"')
backend = {"algolia": algolia.AlgoliaBackend(), "appsearch": appsearch.AppSearchBackend()}[backend]
queue = []
def enqueue_or_index(q, offer, force_index=False):
if offer:
q.append(offer)
if force_index or len(q) > BATCH_SIZE:
try:
backend.index_offers(q)
except Exception as exc: # pylint: disable=broad-except
logger.exception(
"Full offer reindexation: error while reindexing from %d to %d: %s", q[0].id, q[-1].id, exc
)
q.clear()
to_report = 0
elapsed_per_batch = []
while start <= end:
start_time = time.perf_counter()
offers = (
offers_models.Offer.query.options(
joinedload(offers_models.Offer.venue).joinedload(offerers_models.Venue.managingOfferer)
)
.options(joinedload(offers_models.Offer.criteria))
.options(joinedload(offers_models.Offer.mediations))
.options(joinedload(offers_models.Offer.product))
.options(joinedload(offers_models.Offer.stocks))
.filter(offers_models.Offer.isActive.is_(True), offers_models.Offer.id.between(start, start + BATCH_SIZE))
.order_by(offers_models.Offer.id)
)
for offer in offers:
if offer.is_eligible_for_search:
enqueue_or_index(queue, offer)
elapsed_per_batch.append(int(time.perf_counter() - start_time))
start = start + BATCH_SIZE
eta = _get_eta(end, start, elapsed_per_batch)
to_report += BATCH_SIZE
if to_report >= REPORT_EVERY:
to_report = 0
print(f" => OK: {start} | eta = {eta}")
enqueue_or_index(queue, offer=None, force_index=True)
print("Done")
def setup_offers_engines():
backend = appsearch.AppSearchBackend()
setup_engine(backend.offers_engine,
engine_names=appsearch.OFFERS_ENGINE_NAMES,
meta_name=appsearch.OFFERS_META_ENGINE_NAME)
