def test_should_not_delete_beneficiary_when_import_already_exists(
self, app):
# Given
two_days_ago = datetime.utcnow() - timedelta(days=2)
beneficiary = create_user()
with freeze_time(two_days_ago):
save_beneficiary_import_with_status(
ImportStatus.CREATED,
123,
source=BeneficiaryImportSources.demarches_simplifiees,
source_id=14562,
user=beneficiary,
)
# When
save_beneficiary_import_with_status(
ImportStatus.REJECTED,
123,
source=BeneficiaryImportSources.demarches_simplifiees,
source_id=14562,
user=None,
)
# Then
beneficiary_imports = BeneficiaryImport.query.filter_by(
applicationId=123).first()
assert beneficiary_imports.beneficiary == beneficiary
def test_a_status_is_set_on_an_existing_import(self, app):
# given
two_days_ago = datetime.utcnow() - timedelta(days=2)
with freeze_time(two_days_ago):
save_beneficiary_import_with_status(
ImportStatus.DUPLICATE,
123,
source=BeneficiaryImportSources.demarches_simplifiees,
source_id=14562,
user=None,
)
beneficiary = create_user()
# when
save_beneficiary_import_with_status(
ImportStatus.CREATED,
123,
source=BeneficiaryImportSources.demarches_simplifiees,
source_id=14562,
user=beneficiary,
)
# then
beneficiary_imports = BeneficiaryImport.query.filter_by(
applicationId=123).all()
assert len(beneficiary_imports) == 1
assert beneficiary_imports[0].currentStatus == ImportStatus.CREATED
assert beneficiary_imports[0].beneficiary == beneficiary
def _process_error(error_messages: List[str], application_id: int,
procedure_id: int) -> None:
error = f"Le dossier {application_id} contient des erreurs et a été ignoré - Procedure {procedure_id}"
logger.error("[BATCH][REMOTE IMPORT BENEFICIARIES] %s", error)
error_messages.append(error)
save_beneficiary_import_with_status(
ImportStatus.ERROR,
application_id,
source=BeneficiaryImportSources.demarches_simplifiees,
source_id=procedure_id,
detail=error,
)
def _process_rejection(information: Dict, procedure_id: int) -> None:
save_beneficiary_import_with_status(
ImportStatus.REJECTED,
information["application_id"],
source=BeneficiaryImportSources.demarches_simplifiees,
source_id=procedure_id,
detail="Compte existant avec cet email",
)
logger.warning(
"[BATCH][REMOTE IMPORT BENEFICIARIES] Rejected application %s because of already existing email - Procedure %s",
information["application_id"],
procedure_id,
)
def _process_duplication(
duplicate_users: list[users_models.User], information: fraud_models.DMSContent, procedure_id: int
) -> None:
number_of_beneficiaries = len(duplicate_users)
duplicate_ids = ", ".join([str(u.id) for u in duplicate_users])
message = f"{number_of_beneficiaries} utilisateur(s) en doublon {duplicate_ids} pour le dossier {information.application_id} - Procedure {procedure_id}"
logger.warning("[BATCH][REMOTE IMPORT BENEFICIARIES] Duplicate beneficiaries found : %s", message)
save_beneficiary_import_with_status(
ImportStatus.DUPLICATE,
information.application_id,
source=BeneficiaryImportSources.demarches_simplifiees,
source_id=procedure_id,
detail=f"Utilisateur en doublon : {duplicate_ids}",
)
def test_a_status_is_set_on_a_new_import(self, app):
# when
save_beneficiary_import_with_status(
ImportStatus.DUPLICATE,
123,
source=BeneficiaryImportSources.demarches_simplifiees,
source_id=14562,
user=None,
)
# then
beneficiary_import = BeneficiaryImport.query.filter_by(
applicationId=123).first()
assert beneficiary_import.currentStatus == ImportStatus.DUPLICATE
def test_a_beneficiary_import_is_saved_with_all_fields(self, app):
# when
save_beneficiary_import_with_status(
ImportStatus.DUPLICATE,
123,
source=BeneficiaryImportSources.demarches_simplifiees,
source_id=145236,
user=None,
)
# then
beneficiary_import = BeneficiaryImport.query.filter_by(
applicationId=123).first()
assert beneficiary_import.applicationId == 123
assert beneficiary_import.sourceId == 145236
assert beneficiary_import.source == "demarches_simplifiees"
def process_parsing_exception(exception: Exception, procedure_id: int, application_id: int) -> None:
logger.info(
"[BATCH][REMOTE IMPORT BENEFICIARIES] Application %s in procedure %s had errors and was ignored: %s",
application_id,
procedure_id,
exception,
exc_info=True,
)
error = f"Le dossier {application_id} contient des erreurs et a été ignoré - Procedure {procedure_id}"
save_beneficiary_import_with_status(
ImportStatus.ERROR,
application_id,
source=BeneficiaryImportSources.demarches_simplifiees,
source_id=procedure_id,
detail=error,
)
def _process_duplication(duplicate_users: List[User],
error_messages: List[str], information: Dict,
procedure_id: int) -> None:
number_of_beneficiaries = len(duplicate_users)
duplicate_ids = ", ".join([str(u.id) for u in duplicate_users])
message = f"{number_of_beneficiaries} utilisateur(s) en doublon {duplicate_ids} pour le dossier {information['application_id']}"
logger.warning(
"[BATCH][REMOTE IMPORT BENEFICIARIES] Duplicate beneficiaries found : %s - Procedure %s",
message, procedure_id)
error_messages.append(message)
save_beneficiary_import_with_status(
ImportStatus.DUPLICATE,
information["application_id"],
source=BeneficiaryImportSources.demarches_simplifiees,
source_id=procedure_id,
detail=f"Utilisateur en doublon : {duplicate_ids}",
)
def _process_rejection(
information: fraud_models.DMSContent, procedure_id: int, reason: str, user: users_models.User = None
) -> None:
save_beneficiary_import_with_status(
ImportStatus.REJECTED,
information.application_id,
source=BeneficiaryImportSources.demarches_simplifiees,
source_id=procedure_id,
detail=reason,
user=user,
)
logger.warning(
"[BATCH][REMOTE IMPORT BENEFICIARIES] Rejected application %s because of '%s' - Procedure %s",
information.application_id,
reason,
procedure_id,
)
def _process_creation(
error_messages: List[str],
information: Dict,
new_beneficiaries: List[User],
procedure_id: int,
user: Optional[User] = None,
) -> None:
new_beneficiary = create_beneficiary_from_application(information,
user=user)
try:
repository.save(new_beneficiary)
except ApiErrors as api_errors:
logger.warning(
"[BATCH][REMOTE IMPORT BENEFICIARIES] Could not save application %s, because of error: %s - Procedure %s",
information["application_id"],
api_errors,
procedure_id,
)
error_messages.append(str(api_errors))
else:
logger.info(
"[BATCH][REMOTE IMPORT BENEFICIARIES] Successfully created user for application %s - Procedure %s",
information["application_id"],
procedure_id,
)
save_beneficiary_import_with_status(
ImportStatus.CREATED,
information["application_id"],
source=BeneficiaryImportSources.demarches_simplifiees,
source_id=procedure_id,
user=new_beneficiary,
)
new_beneficiaries.append(new_beneficiary)
try:
if user is None:
send_activation_email(new_beneficiary)
else:
send_accepted_as_beneficiary_email(new_beneficiary)
except MailServiceException as mail_service_exception:
logger.exception(
"Email send_activation_email failure for application %s - Procedure %s : %s",
information["application_id"],
procedure_id,
mail_service_exception,
)
def process_parsing_error(exception: DMSParsingError, procedure_id: int, application_id: int) -> None:
logger.info(
"[BATCH][REMOTE IMPORT BENEFICIARIES] Invalid values (%r) detected in Application %s in procedure %s",
exception.errors,
application_id,
procedure_id,
)
user = find_user_by_email(exception.user_email)
user_emails.send_dms_wrong_values_emails(
exception.user_email, exception.errors.get("postal_code"), exception.errors.get("id_piece_number")
)
if user:
subscription_messages.on_dms_application_parsing_errors(user, list(exception.errors.keys()))
errors = ",".join([f"'{key}' ({value})" for key, value in sorted(exception.errors.items())])
error_detail = f"Erreur dans les données soumises dans le dossier DMS : {errors}"
# keep a compatibility with BeneficiaryImport table
save_beneficiary_import_with_status(
ImportStatus.ERROR,
application_id,
source=BeneficiaryImportSources.demarches_simplifiees,
source_id=procedure_id,
detail=error_detail,
user=user,
)
def process_beneficiary_application(
information: fraud_models.DMSContent,
procedure_id: int,
preexisting_account: Optional[users_models.User] = None,
) -> None:
"""
Create/update a user account and complete the import process.
Note that a 'user' is not always a beneficiary.
"""
user = create_beneficiary_from_application(information, user=preexisting_account)
user.hasCompletedIdCheck = True
try:
repository.save(user)
except ApiErrors as api_errors:
logger.warning(
"[BATCH][REMOTE IMPORT BENEFICIARIES] Could not save application %s, because of error: %s - Procedure %s",
information.application_id,
api_errors,
procedure_id,
)
return
logger.info(
"[BATCH][REMOTE IMPORT BENEFICIARIES] Successfully created user for application %s - Procedure %s",
information.application_id,
procedure_id,
)
beneficiary_import = save_beneficiary_import_with_status(
ImportStatus.CREATED,
information.application_id,
source=BeneficiaryImportSources.demarches_simplifiees,
source_id=procedure_id,
user=user,
)
if not users_api.steps_to_become_beneficiary(user):
deposit_source = beneficiary_import.get_detailed_source()
subscription_api.activate_beneficiary(user, deposit_source)
else:
users_external.update_external_user(user)
def run(
process_applications_updated_after: datetime,
get_all_applications_ids: Callable[
..., List[int]] = get_closed_application_ids_for_demarche_simplifiee,
get_applications_ids_to_retry: Callable[
..., List[int]] = find_applications_ids_to_retry,
get_details: Callable[..., Dict] = get_application_details,
already_imported: Callable[..., bool] = is_already_imported,
already_existing_user: Callable[..., User] = find_user_by_email,
) -> None:
procedure_id = settings.DMS_NEW_ENROLLMENT_PROCEDURE_ID
logger.info(
"[BATCH][REMOTE IMPORT BENEFICIARIES] Start import from Démarches Simplifiées for "
"procedure = %s - Procedure %s",
procedure_id,
procedure_id,
)
error_messages: List[str] = []
new_beneficiaries: List[User] = []
applications_ids = get_all_applications_ids(
procedure_id, settings.DMS_TOKEN, process_applications_updated_after)
retry_ids = get_applications_ids_to_retry()
logger.info(
"[BATCH][REMOTE IMPORT BENEFICIARIES] %i new applications to process - Procedure %s",
len(applications_ids),
procedure_id,
)
logger.info(
"[BATCH][REMOTE IMPORT BENEFICIARIES] %i previous applications to retry - Procedure %s",
len(retry_ids),
procedure_id,
)
for application_id in retry_ids + applications_ids:
details = get_details(application_id, procedure_id, settings.DMS_TOKEN)
try:
information = parse_beneficiary_information(details)
except Exception as exc: # pylint: disable=broad-except
logger.info(
"[BATCH][REMOTE IMPORT BENEFICIARIES] Application %s in procedure %s had errors and was ignored: %s",
application_id,
procedure_id,
exc,
exc_info=True,
)
error = f"Le dossier {application_id} contient des erreurs et a été ignoré - Procedure {procedure_id}"
error_messages.append(error)
save_beneficiary_import_with_status(
ImportStatus.ERROR,
application_id,
source=BeneficiaryImportSources.demarches_simplifiees,
source_id=procedure_id,
detail=error,
)
continue
if already_existing_user(information["email"]):
_process_rejection(information, procedure_id=procedure_id)
continue
if not already_imported(information["application_id"]):
process_beneficiary_application(
information=information,
error_messages=error_messages,
new_beneficiaries=new_beneficiaries,
retry_ids=retry_ids,
procedure_id=procedure_id,
)
logger.info(
"[BATCH][REMOTE IMPORT BENEFICIARIES] End import from Démarches Simplifiées - Procedure %s",
procedure_id)
def execute(
self,
application_id: int,
run_fraud_detection: bool = True,
ignore_id_piece_number_field: bool = False,
fraud_detection_ko: bool = False,
) -> None:
try:
jouve_content = jouve_backend.get_application_content(
application_id,
ignore_id_piece_number_field=ignore_id_piece_number_field)
beneficiary_pre_subscription = jouve_backend.get_subscription_from_content(
jouve_content)
except jouve_backend.ApiJouveException as api_jouve_exception:
logger.error(
api_jouve_exception.message,
extra={
"route": api_jouve_exception.route,
"statusCode": api_jouve_exception.status_code,
"applicationId": application_id,
},
)
return
except jouve_backend.JouveContentValidationError as exc:
logger.error(
"Validation error when parsing Jouve content: %s",
exc.message,
extra={
"application_id": application_id,
"validation_errors": exc.errors
},
)
return
preexisting_account = find_user_by_email(
beneficiary_pre_subscription.email)
if not preexisting_account:
save_beneficiary_import_with_status(
ImportStatus.ERROR,
application_id,
source=BeneficiaryImportSources.demarches_simplifiees,
source_id=jouve_backend.DEFAULT_JOUVE_SOURCE_ID,
detail=
f"Aucun utilisateur trouvé pour l'email {beneficiary_pre_subscription.email}",
)
return
try:
on_jouve_result(preexisting_account, jouve_content)
except Exception as exc: # pylint: disable=broad-except
logger.exception("Error on jouve result: %s", exc)
try:
validate(
beneficiary_pre_subscription,
preexisting_account=preexisting_account,
ignore_id_piece_number_field=ignore_id_piece_number_field,
)
if fraud_detection_ko:
raise FraudDetected(
"Forced by 'fraud_detection_ko' script argument")
if run_fraud_detection:
validate_fraud(beneficiary_pre_subscription)
except SuspiciousFraudDetected:
send_fraud_suspicion_email(beneficiary_pre_subscription)
subscription_messages.create_message_jouve_manual_review(
preexisting_account, application_id=application_id)
except FraudDetected as cant_register_beneficiary_exception:
# detail column cannot contain more than 255 characters
detail = f"Fraud controls triggered: {cant_register_beneficiary_exception}"[:
255]
self.beneficiary_repository.reject(
beneficiary_pre_subscription,
detail=detail,
user=preexisting_account,
)
except BeneficiaryIsADuplicate as exception:
exception_reason = str(exception)
logger.info(
"User is a duplicate : cannot register user from application",
extra={
"applicationId": application_id,
"reason": exception_reason,
},
)
subscription_messages.on_duplicate_user(preexisting_account)
self.beneficiary_repository.reject(beneficiary_pre_subscription,
detail=exception_reason,
user=preexisting_account)
old_user_emails.send_rejection_email_to_beneficiary_pre_subscription(
beneficiary_pre_subscription=beneficiary_pre_subscription,
beneficiary_is_eligible=True)
except SubscriptionJourneyOnHold as exc:
logger.warning("User subscription is on hold",
extra={
"applicationId": application_id,
"reason": str(exc)
})
except CantRegisterBeneficiary as cant_register_beneficiary_exception:
exception_reason = str(cant_register_beneficiary_exception)
logger.warning(
"Couldn't register user from application",
extra={
"applicationId": application_id,
"reason": exception_reason,
},
)
self.beneficiary_repository.reject(beneficiary_pre_subscription,
detail=exception_reason,
user=preexisting_account)
old_user_emails.send_rejection_email_to_beneficiary_pre_subscription(
beneficiary_pre_subscription=beneficiary_pre_subscription,
beneficiary_is_eligible=False)
else:
user = self.beneficiary_repository.save(
beneficiary_pre_subscription, user=preexisting_account)
logger.info("User registered from application",
extra={
"applicationId": application_id,
"userId": user.id
})
def process_application(
procedure_id: int, application_id: int, application_details: dict, retry_ids: list[int], parsing_function
) -> None:
try:
information = parsing_function(application_details, procedure_id)
except DMSParsingError as exc:
process_parsing_error(exc, procedure_id, application_id)
return
except Exception as exc: # pylint: disable=broad-except
process_parsing_exception(exc, procedure_id, application_id)
return
user = find_user_by_email(information.email)
if not user:
save_beneficiary_import_with_status(
ImportStatus.ERROR,
application_id,
source=BeneficiaryImportSources.demarches_simplifiees,
source_id=procedure_id,
detail=f"Aucun utilisateur trouvé pour l'email {information.email}",
)
return
try:
fraud_api.on_dms_fraud_check(user, information)
except Exception as exc: # pylint: disable=broad-except
logger.exception("Error on dms fraud check result: %s", exc)
# TODO: Handle switch from underage_beneficiary to beneficiary
if user.is_beneficiary is True:
_process_rejection(information, procedure_id=procedure_id, reason="Compte existant avec cet email")
return
if information.id_piece_number:
_duplicated_user = users_models.User.query.filter(
users_models.User.idPieceNumber == information.id_piece_number
).first()
if _duplicated_user:
subscription_messages.on_duplicate_user(user)
_process_rejection(
information,
procedure_id=procedure_id,
reason=f"Nr de piece déjà utilisé par {_duplicated_user.id}",
user=user,
)
return
if not is_already_imported(information.application_id):
duplicate_users = beneficiary_by_civility_query(
first_name=information.first_name,
last_name=information.last_name,
date_of_birth=information.birth_date,
exclude_email=information.email,
).all()
if duplicate_users and information.application_id not in retry_ids:
_process_duplication(duplicate_users, information, procedure_id)
subscription_messages.on_duplicate_user(user)
else:
process_beneficiary_application(
information=information,
procedure_id=procedure_id,
preexisting_account=user,
)
