def deliver_sms(self, notification_id): try: current_app.logger.info( "Start sending SMS for notification id: {}".format( notification_id)) notification = notifications_dao.get_notification_by_id( notification_id) if not notification: raise NoResultFound() send_to_providers.send_sms_to_provider(notification) except Exception as e: if isinstance(e, SmsClientResponseException): current_app.logger.warning( "SMS notification delivery for id: {} failed".format( notification_id)) else: current_app.logger.exception( "SMS notification delivery for id: {} failed".format( notification_id)) try: if self.request.retries == 0: self.retry(queue=QueueNames.RETRY, countdown=0) else: self.retry(queue=QueueNames.RETRY) except self.MaxRetriesExceededError: message = "RETRY FAILED: Max retries reached. The task send_sms_to_provider failed for notification {}. " \ "Notification has been updated to technical-failure".format(notification_id) update_notification_status_by_id(notification_id, NOTIFICATION_TECHNICAL_FAILURE) raise NotificationTechnicalFailureException(message)
def create_letters_pdf(self, notification_id): try: notification = get_notification_by_id(notification_id, _raise=True) pdf_data, billable_units = get_letters_pdf( notification.template, contact_block=notification.reply_to_text, filename=notification.service.letter_branding and notification.service.letter_branding.filename, values=notification.personalisation) upload_letter_pdf(notification, pdf_data) if notification.key_type != KEY_TYPE_TEST: notification.billable_units = billable_units dao_update_notification(notification) current_app.logger.info( 'Letter notification reference {reference}: billable units set to {billable_units}' .format(reference=str(notification.reference), billable_units=billable_units)) except (RequestException, BotoClientError): try: current_app.logger.exception( "Letters PDF notification creation for id: {} failed".format( notification_id)) self.retry(queue=QueueNames.RETRY) except MaxRetriesExceededError: current_app.logger.error( "RETRY FAILED: task create_letters_pdf failed for notification {}" .format(notification_id), ) update_notification_status_by_id(notification_id, 'technical-failure')
def deliver_email(self, notification_id): try: current_app.logger.info( "Start sending email for notification id: {}".format( notification_id)) notification = notifications_dao.get_notification_by_id( notification_id) if not notification: raise NoResultFound() send_to_providers.send_email_to_provider(notification) except InvalidEmailError as e: current_app.logger.exception(e) update_notification_status_by_id(notification_id, 'technical-failure') except Exception as e: try: current_app.logger.exception( "RETRY: Email notification {} failed".format(notification_id)) self.retry(queue=QueueNames.RETRY) except self.MaxRetriesExceededError: message = "RETRY FAILED: Max retries reached. " \ "The task send_email_to_provider failed for notification {}. " \ "Notification has been updated to technical-failure".format(notification_id) update_notification_status_by_id(notification_id, NOTIFICATION_TECHNICAL_FAILURE) raise NotificationTechnicalFailureException(message)
def sanitise_letter(self, filename): try: reference = get_reference_from_filename(filename) notification = dao_get_notification_by_reference(reference) current_app.logger.info('Notification ID {} Virus scan passed: {}'.format(notification.id, filename)) if notification.status != NOTIFICATION_PENDING_VIRUS_CHECK: current_app.logger.info('Sanitise letter called for notification {} which is in {} state'.format( notification.id, notification.status)) return notify_celery.send_task( name=TaskNames.SANITISE_LETTER, kwargs={ 'notification_id': str(notification.id), 'filename': filename, 'allow_international_letters': notification.service.has_permission( INTERNATIONAL_LETTERS ), }, queue=QueueNames.SANITISE_LETTERS, ) except Exception: try: current_app.logger.exception( "RETRY: calling sanitise_letter task for notification {} failed".format(notification.id) ) self.retry(queue=QueueNames.RETRY) except self.MaxRetriesExceededError: message = "RETRY FAILED: Max retries reached. " \ "The task sanitise_letter failed for notification {}. " \ "Notification has been updated to technical-failure".format(notification.id) update_notification_status_by_id(notification.id, NOTIFICATION_TECHNICAL_FAILURE) raise NotificationTechnicalFailureException(message)
def deliver_email(self, notification_id): try: current_app.logger.info( "Start sending email for notification id: {}".format( notification_id)) notification = notifications_dao.get_notification_by_id( notification_id) if not notification: raise NoResultFound() send_to_providers.send_email_to_provider(notification) except EmailClientNonRetryableException as e: current_app.logger.exception( f"Email notification {notification_id} failed: {e}") update_notification_status_by_id(notification_id, 'technical-failure') except Exception as e: try: if isinstance(e, AwsSesClientThrottlingSendRateException): current_app.logger.warning( f"RETRY: Email notification {notification_id} was rate limited by SES" ) else: current_app.logger.exception( f"RETRY: Email notification {notification_id} failed") self.retry(queue=QueueNames.RETRY) except self.MaxRetriesExceededError: message = "RETRY FAILED: Max retries reached. " \ "The task send_email_to_provider failed for notification {}. " \ "Notification has been updated to technical-failure".format(notification_id) update_notification_status_by_id(notification_id, NOTIFICATION_TECHNICAL_FAILURE) raise NotificationTechnicalFailureException(message)
def test_should_by_able_to_update_status_by_id_from_pending_to_delivered(sample_template, sample_job): data = _notification_json(sample_template, job_id=sample_job.id, status='sending') notification = Notification(**data) dao_create_notification(notification) assert Notification.query.get(notification.id).status == 'sending' assert update_notification_status_by_id(notification_id=notification.id, status='pending') assert Notification.query.get(notification.id).status == 'pending' assert update_notification_status_by_id(notification.id, 'delivered') assert Notification.query.get(notification.id).status == 'delivered'
def test_should_by_able_to_update_status_by_id_from_pending_to_temporary_failure(sample_template, sample_job): data = _notification_json(sample_template, job_id=sample_job.id, status='sending') notification = Notification(**data) dao_create_notification(notification) assert Notification.query.get(notification.id).status == 'sending' assert update_notification_status_by_id(notification_id=notification.id, status='pending') assert Notification.query.get(notification.id).status == 'pending' assert update_notification_status_by_id( notification.id, status='permanent-failure') assert Notification.query.get(notification.id).status == 'temporary-failure'
def process_pinpoint_results(self, response): if not is_feature_enabled(FeatureFlag.PINPOINT_RECEIPTS_ENABLED): current_app.logger.info( 'Pinpoint receipts toggle is disabled, skipping callback task') return True try: pinpoint_message = json.loads(base64.b64decode(response['Message'])) reference = pinpoint_message['attributes']['message_id'] event_type = pinpoint_message.get('event_type') record_status = pinpoint_message['attributes']['record_status'] current_app.logger.info( f'received callback from Pinpoint with event_type of {event_type} and record_status of {record_status}' f'with reference {reference}') notification_status = get_notification_status(event_type, record_status, reference) notification, should_retry, should_exit = attempt_to_get_notification( reference, notification_status, pinpoint_message['event_timestamp']) if should_retry: self.retry(queue=QueueNames.RETRY) if should_exit: return update_notification_status_by_id(notification.id, notification_status) current_app.logger.info( f"Pinpoint callback return status of {notification_status} for notification: {notification.id}" ) statsd_client.incr(f"callback.pinpoint.{notification_status}") if notification.sent_at: statsd_client.timing_with_dates('callback.pinpoint.elapsed-time', datetime.datetime.utcnow(), notification.sent_at) check_and_queue_callback_task(notification) return True except Retry: raise except Exception as e: current_app.logger.exception( f"Error processing Pinpoint results: {type(e)}") self.retry(queue=QueueNames.RETRY)
def _move_invalid_letter_and_update_status(notification, filename, scan_pdf_object): try: move_scan_to_invalid_pdf_bucket(filename) scan_pdf_object.delete() update_letter_pdf_status( reference=notification.reference, status=NOTIFICATION_VALIDATION_FAILED, billable_units=0) except BotoClientError: current_app.logger.exception( "Error when moving letter with id {} to invalid PDF bucket".format(notification.id) ) update_notification_status_by_id(notification.id, NOTIFICATION_TECHNICAL_FAILURE)
def get_pdf_for_templated_letter(self, notification_id): try: notification = get_notification_by_id(notification_id, _raise=True) letter_filename = get_letter_pdf_filename( reference=notification.reference, crown=notification.service.crown, sending_date=notification.created_at, dont_use_sending_date=notification.key_type == KEY_TYPE_TEST, postage=notification.postage) letter_data = { 'letter_contact_block': notification.reply_to_text, 'template': { "subject": notification.template.subject, "content": notification.template.content, "template_type": notification.template.template_type }, 'values': notification.personalisation, 'logo_filename': notification.service.letter_branding and notification.service.letter_branding.filename, 'letter_filename': letter_filename, "notification_id": str(notification_id), 'key_type': notification.key_type } encrypted_data = encryption.encrypt(letter_data) notify_celery.send_task(name=TaskNames.CREATE_PDF_FOR_TEMPLATED_LETTER, args=(encrypted_data, ), queue=QueueNames.SANITISE_LETTERS) except Exception: try: current_app.logger.exception( f"RETRY: calling create-letter-pdf task for notification {notification_id} failed" ) self.retry(queue=QueueNames.RETRY) except self.MaxRetriesExceededError: message = f"RETRY FAILED: Max retries reached. " \ f"The task create-letter-pdf failed for notification id {notification_id}. " \ f"Notification has been updated to technical-failure" update_notification_status_by_id(notification_id, NOTIFICATION_TECHNICAL_FAILURE) raise NotificationTechnicalFailureException(message)
def _process_for_status(notification_status, client_name, provider_reference): notification = notifications_dao.update_notification_status_by_id( provider_reference, notification_status) if not notification: current_app.logger.warning( "{} callback failed: notification {} either not found or already updated " "from sending. Status {}".format(client_name, provider_reference, notification_status)) return statsd_client.incr('callback.{}.{}'.format(client_name, notification_status)) if not notification.sent_by: set_notification_sent_by(notification, client_name) if notification.sent_at: statsd_client.timing_with_dates( 'callback.{}.elapsed-time'.format(client_name), datetime.utcnow(), notification.sent_at) check_for_callback_and_send_delivery_status_to_service(notification) success = "{} callback succeeded. reference {} updated".format( client_name, provider_reference) return success
def test_should_not_update_status_by_id_if_not_sending_and_does_not_update_job(notify_db, notify_db_session): job = sample_job(notify_db, notify_db_session) notification = sample_notification(notify_db, notify_db_session, status='delivered', job=job) assert Notification.query.get(notification.id).status == 'delivered' assert not update_notification_status_by_id(notification.id, 'failed') assert Notification.query.get(notification.id).status == 'delivered' assert job == Job.query.get(notification.job_id)
def cancel_notification_for_service(service_id, notification_id): notification = notifications_dao.get_notification_by_id( notification_id, service_id) if not notification: raise InvalidRequest('Notification not found', status_code=404) elif notification.notification_type != LETTER_TYPE: raise InvalidRequest( 'Notification cannot be cancelled - only letters can be cancelled', status_code=400) elif not letter_can_be_cancelled(notification.status, notification.created_at): print_day = letter_print_day(notification.created_at) raise InvalidRequest( "It’s too late to cancel this letter. Printing started {} at 5.30pm" .format(print_day), status_code=400) updated_notification = notifications_dao.update_notification_status_by_id( notification_id, NOTIFICATION_CANCELLED, ) return jsonify( notification_with_template_schema.dump(updated_notification).data), 200
def deliver_sms(self, notification_id): try: notification = notifications_dao.get_notification_by_id(notification_id) if not notification: raise NoResultFound() send_to_providers.send_sms_to_provider(notification) except Exception as e: try: current_app.logger.exception( "RETRY: SMS notification {} failed".format(notification_id) ) self.retry(queue="retry", countdown=retry_iteration_to_delay(self.request.retries)) except self.MaxRetriesExceededError: current_app.logger.exception( "RETRY FAILED: task send_sms_to_provider failed for notification {}".format(notification_id), ) update_notification_status_by_id(notification_id, 'technical-failure')
def process_sms_client_response(status, reference, client_name): success = None errors = None # validate reference if reference == 'send-sms-code': success = "{} callback succeeded: send-sms-code".format(client_name) return success, errors try: uuid.UUID(reference, version=4) except ValueError: message = "{} callback with invalid reference {}".format(client_name, reference) return success, message try: response_parser = sms_response_mapper[client_name] except KeyError: return success, 'unknown sms client: {}'.format(client_name) # validate status try: response_dict = response_parser(status) current_app.logger.info('{} callback return status of {} for reference: {}'.format(client_name, status, reference)) except KeyError: msg = "{} callback failed: status {} not found.".format(client_name, status) return success, msg notification_status = response_dict['notification_status'] notification_status_message = response_dict['message'] notification_success = response_dict['success'] # record stats notification = notifications_dao.update_notification_status_by_id(reference, notification_status) if not notification: status_error = "{} callback failed: notification {} either not found or already updated " \ "from sending. Status {}".format(client_name, reference, notification_status_message) return success, status_error if not notification_success: current_app.logger.info( "{} delivery failed: notification {} has error found. Status {}".format(client_name, reference, notification_status_message)) statsd_client.incr('callback.{}.{}'.format(client_name.lower(), notification_status)) if notification.sent_at: statsd_client.timing_with_dates( 'callback.{}.elapsed-time'.format(client_name.lower()), datetime.utcnow(), notification.sent_at ) success = "{} callback succeeded. reference {} updated".format(client_name, reference) return success, errors
def _move_invalid_letter_and_update_status( *, notification, filename, scan_pdf_object, message=None, invalid_pages=None, page_count=None ): try: move_scan_to_invalid_pdf_bucket( source_filename=filename, message=message, invalid_pages=invalid_pages, page_count=page_count ) scan_pdf_object.delete() update_letter_pdf_status( reference=notification.reference, status=NOTIFICATION_VALIDATION_FAILED, billable_units=0) except BotoClientError: current_app.logger.exception( "Error when moving letter with id {} to invalid PDF bucket".format(notification.id) ) update_notification_status_by_id(notification.id, NOTIFICATION_TECHNICAL_FAILURE) raise NotificationTechnicalFailureException
def _process_for_status(notification_status, client_name, provider_reference, detailed_status_code=None): # record stats if client_name == 'Twilio': notification = notifications_dao.update_notification_status_by_reference( reference=provider_reference, status=notification_status) else: notification = notifications_dao.update_notification_status_by_id( notification_id=provider_reference, status=notification_status, sent_by=client_name.lower(), detailed_status_code=detailed_status_code) if not notification: return statsd_client.incr('callback.{}.{}'.format(client_name.lower(), notification_status)) if notification.sent_at: statsd_client.timing_with_dates( 'callback.{}.elapsed-time'.format(client_name.lower()), datetime.utcnow(), notification.sent_at) if notification.billable_units == 0: service = notification.service template_model = dao_get_template_by_id(notification.template_id, notification.template_version) template = SMSMessageTemplate( template_model.__dict__, values=notification.personalisation, prefix=service.name, show_prefix=service.prefix_sms, ) notification.billable_units = template.fragment_count notifications_dao.dao_update_notification(notification) if notification_status != NOTIFICATION_PENDING: service_callback_api = get_service_delivery_status_callback_api_for_service( service_id=notification.service_id) # queue callback task only if the service_callback_api exists if service_callback_api: encrypted_notification = create_delivery_status_callback_data( notification, service_callback_api) send_delivery_status_to_service.apply_async( [str(notification.id), encrypted_notification], queue=QueueNames.CALLBACKS)
def lookup_va_profile_id(self, notification_id): current_app.logger.info( f"Retrieving VA Profile ID from MPI for notification {notification_id}" ) notification = notifications_dao.get_notification_by_id(notification_id) try: va_profile_id = mpi_client.get_va_profile_id(notification) notification.recipient_identifiers.set( RecipientIdentifier(notification_id=notification.id, id_type=IdentifierType.VA_PROFILE_ID.value, id_value=va_profile_id)) notifications_dao.dao_update_notification(notification) current_app.logger.info( f"Successfully updated notification {notification_id} with VA PROFILE ID {va_profile_id}" ) except MpiRetryableException as e: current_app.logger.warning( f"Received {str(e)} for notification {notification_id}.") try: self.retry(queue=QueueNames.RETRY) except self.MaxRetriesExceededError: message = "RETRY FAILED: Max retries reached. " \ f"The task lookup_va_profile_id failed for notification {notification_id}. " \ "Notification has been updated to technical-failure" notifications_dao.update_notification_status_by_id( notification_id, NOTIFICATION_TECHNICAL_FAILURE, status_reason=e.failure_reason) raise NotificationTechnicalFailureException(message) from e except (BeneficiaryDeceasedException, IdentifierNotFound, MultipleActiveVaProfileIdsException) as e: message = f"{e.__class__.__name__} - {str(e)}: " \ f"Can't proceed after querying MPI for VA Profile ID for {notification_id}. " \ "Stopping execution of following tasks. Notification has been updated to permanent-failure." current_app.logger.warning(message) self.request.chain = None notifications_dao.update_notification_status_by_id( notification_id, NOTIFICATION_PERMANENT_FAILURE, status_reason=e.failure_reason) except Exception as e: message = f"Failed to retrieve VA Profile ID from MPI for notification: {notification_id} " \ "Notification has been updated to technical-failure" current_app.logger.exception(message) status_reason = e.failure_reason if hasattr( e, 'failure_reason') else 'Unknown error from MPI' notifications_dao.update_notification_status_by_id( notification_id, NOTIFICATION_TECHNICAL_FAILURE, status_reason=status_reason) raise NotificationTechnicalFailureException(message) from e
def test_should_by_able_to_update_status_by_id(sample_template, sample_job, mmg_provider): with freeze_time('2000-01-01 12:00:00'): data = _notification_json(sample_template, job_id=sample_job.id, status='sending') notification = Notification(**data) dao_create_notification(notification) assert Notification.query.get(notification.id).status == 'sending' with freeze_time('2000-01-02 12:00:00'): updated = update_notification_status_by_id(notification.id, 'delivered') assert updated.status == 'delivered' assert updated.updated_at == datetime(2000, 1, 2, 12, 0, 0) assert Notification.query.get(notification.id).status == 'delivered' assert notification.updated_at == datetime(2000, 1, 2, 12, 0, 0)
def lookup_contact_info(self, notification_id): current_app.logger.info( f"Looking up contact information for notification_id:{notification_id}." ) notification = get_notification_by_id(notification_id) va_profile_id = notification.recipient_identifiers[ IdentifierType.VA_PROFILE_ID.value].id_value try: if EMAIL_TYPE == notification.notification_type: recipient = va_profile_client.get_email(va_profile_id) elif SMS_TYPE == notification.notification_type: recipient = va_profile_client.get_telephone(va_profile_id) else: raise NotImplementedError( f"The task lookup_contact_info failed for notification {notification_id}. " f"{notification.notification_type} is not supported") except VAProfileRetryableException as e: current_app.logger.exception(e) try: self.retry(queue=QueueNames.RETRY) except self.MaxRetriesExceededError: message = "RETRY FAILED: Max retries reached. " \ f"The task lookup_contact_info failed for notification {notification_id}. " \ "Notification has been updated to technical-failure" update_notification_status_by_id(notification_id, NOTIFICATION_TECHNICAL_FAILURE) raise NotificationTechnicalFailureException(message) from e except NoContactInfoException as e: message = f"{e.__class__.__name__} - {str(e)}: " \ f"Can't proceed after querying VA Profile for contact information for {notification_id}. " \ "Stopping execution of following tasks. Notification has been updated to permanent-failure." current_app.logger.warning(message) self.request.chain = None update_notification_status_by_id(notification_id, NOTIFICATION_PERMANENT_FAILURE) except VAProfileNonRetryableException as e: current_app.logger.exception(e) message = f"The task lookup_contact_info failed for notification {notification_id}. " \ "Notification has been updated to technical-failure" update_notification_status_by_id(notification_id, NOTIFICATION_TECHNICAL_FAILURE) raise NotificationTechnicalFailureException(message) from e else: notification.to = recipient dao_update_notification(notification)
def deliver_email(self, notification_id): try: current_app.logger.info( "Start sending email for notification id: {}".format( notification_id)) notification = notifications_dao.get_notification_by_id( notification_id) if not notification: raise NoResultFound() send_to_providers.send_email_to_provider(notification) current_app.logger.info( f"Successfully sent email for notification id: {notification_id}") except InvalidEmailError as e: current_app.logger.exception( f"Email notification {notification_id} failed: {str(e)}") update_notification_status_by_id(notification_id, NOTIFICATION_TECHNICAL_FAILURE) raise NotificationTechnicalFailureException(str(e)) except MalwarePendingException: current_app.logger.info( f"RETRY number {self.request.retries}: Email notification {notification_id} is pending malware scans" ) self.retry(queue=QueueNames.RETRY, countdown=60) except InvalidProviderException as e: current_app.logger.exception( f"Invalid provider for {notification_id}: {str(e)}") update_notification_status_by_id(notification_id, NOTIFICATION_TECHNICAL_FAILURE) raise NotificationTechnicalFailureException(str(e)) except Exception as e: try: if isinstance(e, AwsSesClientThrottlingSendRateException): current_app.logger.warning( f"RETRY number {self.request.retries}: Email notification {notification_id} was rate limited by SES" ) else: current_app.logger.exception( f"RETRY number {self.request.retries}: Email notification {notification_id} failed" ) self.retry(queue=QueueNames.RETRY) except self.MaxRetriesExceededError: message = "RETRY FAILED: Max retries reached. " \ "The task send_email_to_provider failed for notification {}. " \ "Notification has been updated to technical-failure".format(notification_id) update_notification_status_by_id(notification_id, NOTIFICATION_TECHNICAL_FAILURE) raise NotificationTechnicalFailureException(message)
def _process_for_status(notification_status, client_name, provider_reference): # record stats notification = notifications_dao.update_notification_status_by_id( provider_reference, notification_status) if not notification: current_app.logger.warning( "{} callback failed: notification {} either not found or already updated " "from sending. Status {}".format(client_name, provider_reference, notification_status)) return statsd_client.incr('callback.{}.{}'.format(client_name.lower(), notification_status)) if not notification.sent_by: set_notification_sent_by(notification, client_name.lower()) if notification.sent_at: statsd_client.timing_with_dates( 'callback.{}.elapsed-time'.format(client_name.lower()), datetime.utcnow(), notification.sent_at) # queue callback task only if the service_callback_api exists service_callback_api = get_service_delivery_status_callback_api_for_service( service_id=notification.service_id) if service_callback_api: encrypted_notification = create_encrypted_callback_data( notification, service_callback_api) send_delivery_status_to_service.apply_async( [str(notification.id), encrypted_notification], queue=QueueNames.CALLBACKS) success = "{} callback succeeded. reference {} updated".format( client_name, provider_reference) return success
def deliver_sms(self, notification_id): try: current_app.logger.info( "Start sending SMS for notification id: {}".format( notification_id)) notification = notifications_dao.get_notification_by_id( notification_id) if not notification: raise NoResultFound() send_to_providers.send_sms_to_provider(notification) current_app.logger.info( f"Successfully sent sms for notification id: {notification_id}") except InvalidProviderException as e: current_app.logger.exception(e) update_notification_status_by_id(notification_id, NOTIFICATION_TECHNICAL_FAILURE) raise NotificationTechnicalFailureException(str(e)) except NonRetryableException: current_app.logger.exception( f'SMS notification delivery for id: {notification_id} failed. Not retrying.' ) update_notification_status_by_id(notification_id, NOTIFICATION_PERMANENT_FAILURE) notification = notifications_dao.get_notification_by_id( notification_id) check_and_queue_callback_task(notification) except Exception: try: current_app.logger.exception( "SMS notification delivery for id: {} failed".format( notification_id)) if self.request.retries == 0: self.retry(queue=QueueNames.RETRY, countdown=0) else: self.retry(queue=QueueNames.RETRY) except self.MaxRetriesExceededError: message = "RETRY FAILED: Max retries reached. The task send_sms_to_provider failed for notification {}. " \ "Notification has been updated to technical-failure".format(notification_id) update_notification_status_by_id(notification_id, NOTIFICATION_TECHNICAL_FAILURE) raise NotificationTechnicalFailureException(message)
def test_should_return_zero_count_if_no_notification_with_id(): assert not update_notification_status_by_id(str(uuid.uuid4()), 'delivered')
def process_sanitised_letter(self, sanitise_data): letter_details = encryption.decrypt(sanitise_data) filename = letter_details['filename'] notification_id = letter_details['notification_id'] current_app.logger.info('Processing sanitised letter with id {}'.format(notification_id)) notification = get_notification_by_id(notification_id, _raise=True) if notification.status != NOTIFICATION_PENDING_VIRUS_CHECK: current_app.logger.info( 'process-sanitised-letter task called for notification {} which is in {} state'.format( notification.id, notification.status) ) return try: original_pdf_object = s3.get_s3_object(current_app.config['LETTERS_SCAN_BUCKET_NAME'], filename) if letter_details['validation_status'] == 'failed': current_app.logger.info('Processing invalid precompiled pdf with id {} (file {})'.format( notification_id, filename)) _move_invalid_letter_and_update_status( notification=notification, filename=filename, scan_pdf_object=original_pdf_object, message=letter_details['message'], invalid_pages=letter_details['invalid_pages'], page_count=letter_details['page_count'], ) return current_app.logger.info('Processing valid precompiled pdf with id {} (file {})'.format( notification_id, filename)) billable_units = get_billable_units_for_letter_page_count(letter_details['page_count']) is_test_key = notification.key_type == KEY_TYPE_TEST # Updating the notification needs to happen before the file is moved. This is so that if updating the # notification fails, the task can retry because the file is in the same place. update_letter_pdf_status( reference=notification.reference, status=NOTIFICATION_DELIVERED if is_test_key else NOTIFICATION_CREATED, billable_units=billable_units, recipient_address=letter_details['address'] ) # The original filename could be wrong because we didn't know the postage. # Now we know if the letter is international, we can check what the filename should be. upload_file_name = get_letter_pdf_filename( reference=notification.reference, crown=notification.service.crown, created_at=notification.created_at, ignore_folder=True, postage=notification.postage ) move_sanitised_letter_to_test_or_live_pdf_bucket( filename, is_test_key, notification.created_at, upload_file_name, ) # We've moved the sanitised PDF from the sanitise bucket, but still need to delete the original file: original_pdf_object.delete() except BotoClientError: # Boto exceptions are likely to be caused by the file(s) being in the wrong place, so retrying won't help - # we'll need to manually investigate current_app.logger.exception( f"Boto error when processing sanitised letter for notification {notification.id} (file {filename})" ) update_notification_status_by_id(notification.id, NOTIFICATION_TECHNICAL_FAILURE) raise NotificationTechnicalFailureException except Exception: try: current_app.logger.exception( "RETRY: calling process_sanitised_letter task for notification {} failed".format(notification.id) ) self.retry(queue=QueueNames.RETRY) except self.MaxRetriesExceededError: message = "RETRY FAILED: Max retries reached. " \ "The task process_sanitised_letter failed for notification {}. " \ "Notification has been updated to technical-failure".format(notification.id) update_notification_status_by_id(notification.id, NOTIFICATION_TECHNICAL_FAILURE) raise NotificationTechnicalFailureException(message)
def test_should_update_status_by_id_if_created(notify_db, notify_db_session): notification = sample_notification(notify_db, notify_db_session, status='created') assert Notification.query.get(notification.id).status == 'created' updated = update_notification_status_by_id(notification.id, 'failed') assert Notification.query.get(notification.id).status == 'failed' assert updated.status == 'failed'
def process_virus_scan_passed(self, filename): reference = get_reference_from_filename(filename) notification = dao_get_notification_by_reference(reference) current_app.logger.info("notification id {} Virus scan passed: {}".format( notification.id, filename)) is_test_key = notification.key_type == KEY_TYPE_TEST scan_pdf_object = s3.get_s3_object( current_app.config["LETTERS_SCAN_BUCKET_NAME"], filename) old_pdf = scan_pdf_object.get()["Body"].read() try: billable_units = get_page_count(old_pdf) except PdfReadError: current_app.logger.exception( msg="Invalid PDF received for notification_id: {}".format( notification.id)) _move_invalid_letter_and_update_status(notification, filename, scan_pdf_object) return sanitise_response = _sanitise_precompiled_pdf(self, notification, old_pdf) if not sanitise_response: new_pdf = None else: sanitise_response = sanitise_response.json() try: new_pdf = base64.b64decode(sanitise_response["file"].encode()) except JSONDecodeError: new_pdf = sanitise_response.content redaction_failed_message = sanitise_response.get( "redaction_failed_message") if redaction_failed_message and not is_test_key: current_app.logger.info("{} for notification id {} ({})".format( redaction_failed_message, notification.id, filename)) copy_redaction_failed_pdf(filename) # TODO: Remove this once CYSP update their template to not cross over the margins if notification.service_id == UUID( "fe44178f-3b45-4625-9f85-2264a36dd9ec"): # CYSP # Check your state pension submit letters with good addresses and notify tags, so just use their supplied pdf new_pdf = old_pdf if not new_pdf: current_app.logger.info( "Invalid precompiled pdf received {} ({})".format( notification.id, filename)) _move_invalid_letter_and_update_status(notification, filename, scan_pdf_object) return else: current_app.logger.info( "Validation was successful for precompiled pdf {} ({})".format( notification.id, filename)) current_app.logger.info( "notification id {} ({}) sanitised and ready to send".format( notification.id, filename)) try: _upload_pdf_to_test_or_live_pdf_bucket(new_pdf, filename, is_test_letter=is_test_key) update_letter_pdf_status( reference=reference, status=NOTIFICATION_DELIVERED if is_test_key else NOTIFICATION_CREATED, billable_units=billable_units, ) scan_pdf_object.delete() except BotoClientError: current_app.logger.exception( "Error uploading letter to live pdf bucket for notification: {}". format(notification.id)) update_notification_status_by_id(notification.id, NOTIFICATION_TECHNICAL_FAILURE)
def process_virus_scan_passed(self, filename): reference = get_reference_from_filename(filename) notification = dao_get_notification_by_reference(reference) current_app.logger.info('notification id {} Virus scan passed: {}'.format( notification.id, filename)) is_test_key = notification.key_type == KEY_TYPE_TEST scan_pdf_object = s3.get_s3_object( current_app.config['LETTERS_SCAN_BUCKET_NAME'], filename) old_pdf = scan_pdf_object.get()['Body'].read() sanitise_response, result = _sanitise_precompiled_pdf( self, notification, old_pdf) new_pdf = None if result == 'validation_passed': new_pdf = base64.b64decode(sanitise_response["file"].encode()) redaction_failed_message = sanitise_response.get( "redaction_failed_message") if redaction_failed_message and not is_test_key: current_app.logger.info('{} for notification id {} ({})'.format( redaction_failed_message, notification.id, filename)) copy_redaction_failed_pdf(filename) billable_units = get_billable_units_for_letter_page_count( sanitise_response.get("page_count")) # TODO: Remove this once CYSP update their template to not cross over the margins if notification.service_id == UUID( 'fe44178f-3b45-4625-9f85-2264a36dd9ec'): # CYSP # Check your state pension submit letters with good addresses and notify tags, so just use their supplied pdf new_pdf = old_pdf if result == 'validation_failed' and not new_pdf: current_app.logger.info( 'Invalid precompiled pdf received {} ({})'.format( notification.id, filename)) _move_invalid_letter_and_update_status( notification=notification, filename=filename, scan_pdf_object=scan_pdf_object, message=sanitise_response["message"], invalid_pages=sanitise_response.get("invalid_pages"), page_count=sanitise_response.get("page_count")) return current_app.logger.info( 'notification id {} ({}) sanitised and ready to send'.format( notification.id, filename)) try: _upload_pdf_to_test_or_live_pdf_bucket( new_pdf, filename, is_test_letter=is_test_key, created_at=notification.created_at) update_letter_pdf_status(reference=reference, status=NOTIFICATION_DELIVERED if is_test_key else NOTIFICATION_CREATED, billable_units=billable_units) scan_pdf_object.delete() except BotoClientError: current_app.logger.exception( "Error uploading letter to live pdf bucket for notification: {}". format(notification.id)) update_notification_status_by_id(notification.id, NOTIFICATION_TECHNICAL_FAILURE)