def wrapper(*args, **kwargs):
job_description = f"{func.__name__} {args}"
logger.info(build_job_log_message(job=job_description, status=JobStatus.STARTED))
result = func(*args, **kwargs)
logger.info(build_job_log_message(job=job_description, status=JobStatus.ENDED))
return result
def log_redis_connection_status():
try:
conn.ping()
logger.info("Worker: redis connection OK")
except redis.exceptions.ConnectionError as e:
logger.critical("Worker: redis connection KO: %s", e)
def save_reimbursable_thing_offer(venue: VenueSQLEntity):
paid_reimbursable_offer = create_offer_with_thing_product(
venue, thing_name="Roman cool", thing_type=ThingType.LIVRE_EDITION)
reimbursable_stock = create_stock(offer=paid_reimbursable_offer, price=30)
repository.save(reimbursable_stock)
logger.info("created 1 reimbursable thing offer with 1 paid stock of 30 €")
return reimbursable_stock
def create_industrial_criteria() -> dict:
logger.info("create_industrial_criteria")
criteria_by_name = {}
criterion1 = Criterion()
criterion1.name = "Bonne offre d’appel"
criterion1.description = "Offre déjà beaucoup réservée par les autres jeunes"
criteria_by_name[criterion1.name] = criterion1
criterion2 = Criterion()
criterion2.name = "Mauvaise accroche"
criterion2.description = "Offre ne possédant pas une accroche de qualité suffisante"
criteria_by_name[criterion2.name] = criterion2
criterion3 = Criterion()
criterion3.name = "Offre de médiation spécifique"
criterion3.description = "Offre possédant une médiation orientée pour les jeunes de 18 ans"
criteria_by_name[criterion3.name] = criterion3
repository.save(*criteria_by_name.values())
logger.info("created %d criteria", len(criteria_by_name))
return criteria_by_name
def get_zip_file_from_ftp(zip_file_name: str, folder_name: str) -> ZipFile:
data_file = BytesIO()
data_file.name = zip_file_name
file_path = "RETR " + folder_name + "/" + zip_file_name
logger.info("Downloading file %s", file_path)
connect_to_titelive_ftp().retrbinary(file_path, data_file.write)
return ZipFile(data_file, "r")
def create_industrial_thing_stocks(thing_offers_by_name):
logger.info("create_industrial_thing_stocks")
thing_stocks_by_name = {}
short_names_to_increase_price = []
thing_offer_items = list(thing_offers_by_name.items())
thing_offer_items_with_stocks = remove_every(thing_offer_items, THING_OFFERS_WITH_STOCK_REMOVE_MODULO)
for thing_offer_item_with_stocks in thing_offer_items_with_stocks:
(thing_offer_with_stocks_name, thing_offer_with_stocks) = thing_offer_item_with_stocks
quantity = 20
short_name = get_occurrence_short_name(thing_offer_with_stocks_name)
price = get_price_by_short_name(short_name)
price_counter = short_names_to_increase_price.count(short_name)
if price_counter > 2:
price = price + price_counter
short_names_to_increase_price.append(short_name)
name = thing_offer_with_stocks_name + " / " + str(quantity) + " / " + str(price)
thing_stocks_by_name[name] = create_stock_from_offer(thing_offer_with_stocks, quantity=quantity, price=price)
repository.save(*thing_stocks_by_name.values())
logger.info("created %d thing_stocks", len(thing_stocks_by_name))
def send_payments_details(payments: List[Payment],
recipients: List[str]) -> None:
if not recipients:
raise Exception(
"[BATCH][PAYMENTS] Missing PASS_CULTURE_PAYMENTS_DETAILS_RECIPIENTS in environment variables"
)
if all(
map(lambda x: x.currentStatus.status == TransactionStatus.ERROR,
payments)):
logger.warning(
"[BATCH][PAYMENTS] Not sending payments details as all payments have an ERROR status"
)
else:
details = create_all_payments_details(payments)
csv = generate_payment_details_csv(details)
logger.info("[BATCH][PAYMENTS] Sending %s details of %s payments",
len(details), len(payments))
logger.info("[BATCH][PAYMENTS] Recipients of email : %s", recipients)
try:
send_payment_details_email(csv, recipients)
except MailServiceException as exception:
logger.exception(
"[BATCH][PAYMENTS] Error while sending payment details email to MailJet: %s",
exception)
def create_industrial_pro_users(offerers_by_name: Dict) -> Dict:
logger.info("create_industrial_pro_users")
users_by_name = {}
offerers = list(offerers_by_name.values())
for index, departement_code in enumerate(departement_codes):
for pro_count in range(PROS_COUNT):
email = "pctest.pro{}.{}@example.com".format(
departement_code, pro_count)
user = create_user(
reset_password_token_validity_limit=datetime.utcnow() +
timedelta(hours=24),
is_beneficiary=False,
date_of_birth=None,
departement_code=str(departement_code),
email=email,
first_name="PC Test Pro",
is_admin=False,
last_name="{} {}".format(departement_code, pro_count),
postal_code="{}100".format(departement_code),
public_name="PC Test Pro {} {}".format(departement_code,
pro_count),
)
users_by_name["pro{} {}".format(departement_code,
pro_count)] = user
UserOffererFactory(offerer=offerers[index], user=user)
repository.save(*users_by_name.values())
logger.info("created %d users", len(users_by_name))
return users_by_name
def create_industrial_bookings(offers_by_name, users_by_name):
logger.info("create_industrial_bookings")
bookings_by_name = {}
list_of_users_with_no_more_money = []
token = 100000
active_offers_with_stocks = {
booking_key: offer
for booking_key, offer in offers_by_name.items()
if offer.venue.managingOfferer.isValidated is True
and len(offer.stocks) > 0
}
for (user_name, user) in users_by_name.items():
if (user.firstName != "PC Test Jeune" or "has-signed-up" in user_name
or "has-filled-cultural-survey" in user_name):
continue
if "has-booked-some" in user.email:
_create_has_booked_some_bookings(bookings_by_name,
active_offers_with_stocks, user,
user_name)
else:
token = _create_bookings_for_other_beneficiaries(
bookings_by_name, list_of_users_with_no_more_money,
active_offers_with_stocks, token, user, user_name)
repository.save(*bookings_by_name.values())
logger.info("created %d bookings", len(bookings_by_name))
def cancel_banned_bookings() -> None:
logger.info("[CANCEL BANNED BOOKINGS] START")
bookings_to_update = get_bookings_banned_and_sent()
bookings_in_error = []
updated_bookings = []
for booking in bookings_to_update:
booking.isCancelled = True
booking.isUsed = False
booking.dateUsed = None
try:
repository.save(booking)
updated_bookings.append(booking.id)
except ApiErrors as error:
logger.exception("%s for booking %s", error.errors, booking.id)
bookings_in_error.append(booking.id)
logger.info("%i BOOKINGS UPDATED", len(updated_bookings))
logger.info("LIST OF UPDATED BOOKINGS")
logger.info(updated_bookings)
if len(bookings_in_error) > 0:
logger.error("LIST OF BOOKINGS IN ERROR")
logger.error(bookings_in_error)
logger.info("[CANCEL BANNED BOOKINGS] END")
def create_industrial_venue_types() -> List[VenueType]:
logger.info("create_industrial_venue_types")
labels = [
"Arts visuels, arts plastiques et galeries",
"Centre culturel",
"Cours et pratique artistiques",
"Culture scientifique",
"Festival",
"Jeux / Jeux vidéos",
"Librairie",
"Bibliothèque ou médiathèque",
"Musée",
"Musique - Disquaire",
"Musique - Magasin d’instruments",
"Musique - Salle de concerts",
"Offre numérique",
"Patrimoine et tourisme",
"Cinéma - Salle de projections",
"Spectacle vivant",
"Autre",
]
venue_types = [create_venue_type(label=label) for label in labels]
repository.save(*venue_types)
logger.info("created %i venue types", len(venue_types))
return venue_types
def create_industrial_admin_users():
logger.info("create_industrial_admin_users")
users_by_name = {}
for departement_code in departement_codes:
for admin_count in range(ADMINS_COUNT):
email = "pctest.admin{}.{}@btmx.fr".format(departement_code,
admin_count)
users_by_name["admin{} {}".format(
departement_code, admin_count)] = create_user(
reset_password_token_validity_limit=datetime.utcnow() +
timedelta(hours=24),
is_beneficiary=False,
date_of_birth=None,
departement_code=str(departement_code),
email=email,
first_name="PC Test Admin",
is_admin=True,
last_name="{} {}".format(departement_code, admin_count),
postal_code="{}100".format(departement_code),
public_name="PC Test Admin {} {}".format(
departement_code, admin_count),
)
repository.save(*users_by_name.values())
logger.info("created %d users", len(users_by_name))
return users_by_name
def create_industrial_payments():
logger.info("create_industrial_payments")
pending_payments, not_processable_payments = generate_new_payments()
logger.info("created %d payments",
len(pending_payments + not_processable_payments))
def create_industrial_user_offerers(users_by_name, offerers_by_name):
logger.info("create_industrial_user_offerers")
user_offerers_by_name = {}
# special validation
user = users_by_name["pro93 real-validation"]
offerer = offerers_by_name["414819409 lat:48.8 lon:1.48"]
user_offerers_by_name["pro93 real-validation / 414819409 lat:48.8 lon:1.48"] = create_user_offerer(
offerer=offerer, user=user
)
# loop on users
for (user_name, user) in users_by_name.items():
for (offerer_name, offerer) in offerers_by_name.items():
if (
PostalCode(offerer.postalCode).get_departement_code() != user.departementCode
or "real-validation" in user_name
):
continue
user_offerers_by_name["{} / {}".format(user_name, offerer_name)] = create_user_offerer(
offerer=offerer, user=user
)
repository.save(*user_offerers_by_name.values())
logger.info("created %d user_offerers", len(user_offerers_by_name))
return user_offerers_by_name
def _run_indexing(client: Redis, venue_provider: Dict) -> None:
venue_provider_id = venue_provider["id"]
provider_id = venue_provider["providerId"]
venue_id = venue_provider["venueId"]
run_algolia_venue_provider_command = (
f"python src/pcapi/scripts/pc.py process_venue_provider_offers_for_algolia "
f"--provider-id {provider_id} "
f"--venue-provider-id {venue_provider_id} "
f"--venue-id {venue_id}")
try:
container_id = run_process_in_one_off_container(
run_algolia_venue_provider_command)
add_venue_provider_currently_in_sync(
client=client,
container_id=container_id,
venue_provider_id=venue_provider_id)
logger.info(
"[ALGOLIA][Worker] Indexing offers from VenueProvider %s in container %s",
venue_provider_id, container_id)
except ScalingoApiException as error:
logger.exception(
"[ALGOLIA][Worker] Error indexing offers from VenueProvider %s with errors: %s",
venue_provider_id,
error,
)
def create_industrial_event_stocks(event_occurrences_by_name):
logger.info("create_industrial_event_stocks")
event_stocks_by_name = {}
short_names_to_increase_price = []
event_occurrence_items = list(event_occurrences_by_name.items())
event_occurrence_items_with_stocks = remove_every(
event_occurrence_items, EVENT_OCCURRENCES_WITH_STOCKS_REMOVE_MODULO
)
for event_occurrence_item_with_stocks in event_occurrence_items_with_stocks:
(event_occurrence_with_stocks_name, event_occurrence_with_stocks) = event_occurrence_item_with_stocks
available = 10
short_name = get_occurrence_short_name(event_occurrence_with_stocks_name)
price = get_price_by_short_name(short_name)
price_counter = short_names_to_increase_price.count(short_name)
if price_counter > 2:
price = price + price_counter
short_names_to_increase_price.append(short_name)
if event_occurrence_with_stocks["offer"].product.offerType["value"] == str(EventType.ACTIVATION):
price = 0
name = event_occurrence_with_stocks_name + " / " + str(available) + " / " + str(price)
event_stocks_by_name[name] = create_stock_from_event_occurrence(
event_occurrence_with_stocks, price=price, quantity=available
)
repository.save(*event_stocks_by_name.values())
logger.info("created %d event_stocks", len(event_stocks_by_name))
def create_beneficiary_user() -> User:
import_status = ImportStatus.CREATED
beneficiary_user = create_user(email=f"{str(import_status)}@email.com")
repository.save(beneficiary_user)
logger.info("created 1 beneficiary user")
return beneficiary_user
def create_industrial_webapp_users():
logger.info("create_industrial_webapp_users")
users_by_name = {}
validation_prefix, validation_suffix = "AZERTY", 123
validation_suffix += 1
for departement_code in departement_codeS:
for tag in WEBAPP_TAGS:
short_tag = "".join([chunk[0].upper() for chunk in tag.split("-")])
if tag == "has-signed-up":
reset_password_token = "{}{}".format(validation_prefix,
validation_suffix)
validation_suffix += 1
else:
reset_password_token = None
cultural_survey_id = None
needs_to_fill_cultural_survey = False
has_seen_tutorials = True
if tag == "has-filled-cultural-survey":
has_seen_tutorials = False
if tag == "has-signed-up":
cultural_survey_id = uuid.uuid4()
needs_to_fill_cultural_survey = True
has_seen_tutorials = False
email = "pctest.jeune{}.{}@btmx.fr".format(departement_code, tag)
users_by_name["jeune{} {}".format(
departement_code, tag
)] = create_user(
cultural_survey_id=cultural_survey_id,
departement_code=str(departement_code),
email=email,
first_name="PC Test Jeune",
date_of_birth=datetime(2003, 1, 1),
has_seen_tutorials=has_seen_tutorials,
last_name="{} {}".format(departement_code, short_tag),
needs_to_fill_cultural_survey=needs_to_fill_cultural_survey,
postal_code="{}100".format(departement_code),
public_name="PC Test Jeune {} {}".format(
departement_code, short_tag),
reset_password_token=reset_password_token,
reset_password_token_validity_limit=datetime.utcnow() +
timedelta(hours=24),
)
repository.save(*users_by_name.values())
logger.info("created %d users", len(users_by_name))
return users_by_name
def create_industrial_algolia_indexed_objects() -> None:
if settings.ALGOLIA_TRIGGER_INDEXATION:
logger.info("create_industrial_algolia_objects")
offer_ids = Offer.query.with_entities(Offer.id).all()
clear_index()
delete_all_indexed_offers(client=app.redis_client)
process_eligible_offers(client=app.redis_client,
offer_ids=offer_ids,
from_provider_update=False)
def save_sandbox(name, with_clean=True):
if with_clean:
logger.info("Cleaning database")
clean_all_database()
logger.info("All databases cleaned")
script_name = "sandbox_" + name
sandbox_module = getattr(scripts, script_name)
sandbox_module.save_sandbox()
def _modify_allocine_price_rule_for_venue(
allocine_venue_provider: AllocineVenueProvider,
new_price: Decimal) -> None:
venue_provider_price_rule = AllocineVenueProviderPriceRule.query.filter_by(
allocineVenueProviderId=allocine_venue_provider.id).one_or_none()
if venue_provider_price_rule is not None:
venue_provider_price_rule.price = new_price
repository.save(venue_provider_price_rule)
logger.info("Venue priceRule updated")
def _delete_user_offerers_from_rows(csv_rows: Iterable) -> None:
user_offerers_successful = []
user_offerers_in_error = []
csv_rows_iterable = iter(csv_rows)
headers = next(csv_rows_iterable)
for row in csv_rows_iterable:
if _is_blank_row(row):
continue
row = dict(zip(headers, row))
user_id = row[USER_ID_COLUMN_HEADER]
offerer_id = row[OFFERER_ID_COLUMN_HEADER]
user_offerer = find_one_or_none_by_user_id_and_offerer_id(
int(user_id), int(offerer_id))
if user_offerer is None:
continue
user_offerer_id = user_offerer.id
logger.info(
"[DELETE USER_OFFERERS FROM FILE] Suppression du rattachement pour le user d'id %s et l'offerer "
"d'id %s est lancée",
user_id,
offerer_id,
)
try:
repository.delete(user_offerer)
logger.info(
"[DELETE USER_OFFERERS FROM FILE] Suppression du rattachement pour le user d'id %s et l'offerer "
"d'id %s réussie",
user_id,
offerer_id,
)
user_offerers_successful.append(user_offerer_id)
except ApiErrors as error:
logger.exception(
"[DELETE USER_OFFERERS FROM FILE] %s pour le rattachement avec le user d'id %s et l'offerer d'id %s",
error.errors,
user_id,
offerer_id,
)
user_offerers_in_error.append(user_offerer_id)
logger.info("[DELETE USER_OFFERERS FROM FILE] %i RATTACHEMENT SUPPRIMES",
len(user_offerers_successful))
logger.info(
"[DELETE USER_OFFERERS FROM FILE] LISTE DES RATTACHEMENT SUPPRIMES")
logger.info(user_offerers_successful)
if len(user_offerers_in_error) > 0:
logger.error(
"[DELETE USER_OFFERERS FROM FILE] LISTE DES RATTACHEMENTS EN ERREUR"
)
logger.error(user_offerers_in_error)
def is_accessible(self):
authorized = current_user.is_authenticated and current_user.isAdmin
if authorized:
logger.info("[ADMIN] Accès autorisé à l'interface d'administation pour %s", current_user)
else:
logger.warning("[ADMIN] Tentative d'accès non autorisé à l'interface d'administation par %s", current_user)
return authorized
def log_database_connection_status() -> None:
with app.app_context():
if check_database_connection():
logger.info("Worker: database connection OK")
else:
logger.critical("Worker: database connection KO")
db.session.remove()
db.session.close()
db.engine.dispose()
def create_or_update_users(rows: Iterable[dict]) -> List[User]:
# The purpose of this function is to recreate test users on
# staging after the staging database is reset. It's not meant to
# be used anywhere else, and certainly not on production.
if settings.IS_PROD:
raise ValueError(
"This function is not supposed to be run on production")
users = []
for row in rows:
user = find_user_by_email(row["Mail"])
birthdate = datetime.strptime(row["Date de naissance"], "%Y-%m-%d")
if user:
user.dateOfBirth = birthdate
user.setPassword(settings.STAGING_TEST_USER_PASSWORD)
else:
user = users_api.create_account(
email=row["Mail"],
password=settings.STAGING_TEST_USER_PASSWORD,
birthdate=birthdate,
is_email_validated=True,
send_activation_mail=False,
)
deposit = payments_api.create_deposit(user, "import_users (csv)")
repository.save(deposit)
user.isBeneficiary = True
user.lastName = row["Nom"]
user.firstName = row["Prénom"]
user.publicName = f"{user.firstName} {user.lastName}"
user.phoneNumber = row["Téléphone"]
user.departementCode = row["Département"]
user.postalCode = row["Code postal"]
repository.save(user)
users.append(user)
logger.info("Created or updated user=%s from CSV import", user.id)
admin = find_user_by_email("*****@*****.**")
if not admin:
admin = users_api.create_account(
email="*****@*****.**",
password=settings.STAGING_TEST_USER_PASSWORD,
birthdate=datetime(1946, 12, 24),
is_email_validated=True,
send_activation_mail=False,
)
admin.setPassword(settings.STAGING_TEST_USER_PASSWORD)
admin.isAdmin = True
admin.isBeneficiary = False
admin.firstName = "Jeanne"
admin.lastName = "Admin"
admin.publicName = f"{user.firstName} {user.lastName}"
repository.save(admin)
logger.info("Created or updated admin user=%s", admin.id)
return users
def save_paid_online_book_offer(venue: VenueSQLEntity):
paid_reimbursable_offer = create_offer_with_thing_product(
venue,
thing_name="Roman cool",
thing_type=ThingType.LIVRE_EDITION,
url="https://mycoolbook.fr")
reimbursable_stock = create_stock(offer=paid_reimbursable_offer, price=20)
repository.save(reimbursable_stock)
logger.info("created 1 online book offer with 1 paid stock of 20 €")
return reimbursable_stock
def wrapper(*args, **kwargs):
start_time = time.time()
logger.info(build_cron_log_message(name=func.__name__, status=CronStatus.STARTED))
result = func(*args, **kwargs)
end_time = time.time()
duration = end_time - start_time
logger.info(build_cron_log_message(name=func.__name__, status=CronStatus.ENDED, duration=duration))
return result
def batch_indexing_offers_in_algolia_by_offer(client: Redis) -> None:
offer_ids = get_offer_ids(client=client)
if len(offer_ids) > 0:
logger.info("[ALGOLIA] processing %i offers...", len(offer_ids))
process_eligible_offers(client=client,
offer_ids=offer_ids,
from_provider_update=False)
delete_offer_ids(client=client)
logger.info("[ALGOLIA] %i offers processed!", len(offer_ids))
def get_movies_showtimes(
api_key: str, theater_id: str, get_movies_showtimes_from_api: Callable = get_movies_showtimes_from_allocine
) -> iter:
api_response = get_movies_showtimes_from_api(api_key, theater_id)
movies_showtimes = api_response["movieShowtimeList"]["edges"]
movies_number = api_response["movieShowtimeList"]["totalCount"]
filtered_movies_showtimes = _exclude_movie_showtimes_with_special_event_type(movies_showtimes)
logger.info("[ALLOCINE] Total : %s movies", movies_number)
return iter(filtered_movies_showtimes)
def iterate_rows_for_user_offerers(csv_rows: List[List[str]]) -> List:
user_offerers = []
for row in csv_rows:
if _is_header_or_blank_line(row):
continue
user_offerer = create_activated_user_offerer(row)
if user_offerer:
user_offerers.append(user_offerer)
logger.info("Enregistrement de %s comptes pro", len(user_offerers))
return user_offerers
