def process_virus_scan_passed(self, filename):
reference = get_reference_from_filename(filename)
notification = dao_get_notification_by_reference(reference)
current_app.logger.info('notification id {} Virus scan passed: {}'.format(
notification.id, filename))
is_test_key = notification.key_type == KEY_TYPE_TEST
scan_pdf_object = s3.get_s3_object(
current_app.config['LETTERS_SCAN_BUCKET_NAME'], filename)
old_pdf = scan_pdf_object.get()['Body'].read()
sanitise_response, result = _sanitise_precompiled_pdf(
self, notification, old_pdf)
new_pdf = None
if result == 'validation_passed':
new_pdf = base64.b64decode(sanitise_response["file"].encode())
redaction_failed_message = sanitise_response.get(
"redaction_failed_message")
if redaction_failed_message and not is_test_key:
current_app.logger.info('{} for notification id {} ({})'.format(
redaction_failed_message, notification.id, filename))
copy_redaction_failed_pdf(filename)
billable_units = get_billable_units_for_letter_page_count(
sanitise_response.get("page_count"))
# TODO: Remove this once CYSP update their template to not cross over the margins
if notification.service_id == UUID(
'fe44178f-3b45-4625-9f85-2264a36dd9ec'): # CYSP
# Check your state pension submit letters with good addresses and notify tags, so just use their supplied pdf
new_pdf = old_pdf
if result == 'validation_failed' and not new_pdf:
current_app.logger.info(
'Invalid precompiled pdf received {} ({})'.format(
notification.id, filename))
_move_invalid_letter_and_update_status(
notification=notification,
filename=filename,
scan_pdf_object=scan_pdf_object,
message=sanitise_response["message"],
invalid_pages=sanitise_response.get("invalid_pages"),
page_count=sanitise_response.get("page_count"))
return
current_app.logger.info(
'notification id {} ({}) sanitised and ready to send'.format(
notification.id, filename))
try:
_upload_pdf_to_test_or_live_pdf_bucket(
new_pdf,
filename,
is_test_letter=is_test_key,
created_at=notification.created_at)
update_letter_pdf_status(reference=reference,
status=NOTIFICATION_DELIVERED
if is_test_key else NOTIFICATION_CREATED,
billable_units=billable_units)
scan_pdf_object.delete()
except BotoClientError:
current_app.logger.exception(
"Error uploading letter to live pdf bucket for notification: {}".
format(notification.id))
update_notification_status_by_id(notification.id,
NOTIFICATION_TECHNICAL_FAILURE)
def process_sanitised_letter(self, sanitise_data):
letter_details = encryption.decrypt(sanitise_data)
filename = letter_details['filename']
notification_id = letter_details['notification_id']
current_app.logger.info('Processing sanitised letter with id {}'.format(notification_id))
notification = get_notification_by_id(notification_id, _raise=True)
if notification.status != NOTIFICATION_PENDING_VIRUS_CHECK:
current_app.logger.info(
'process-sanitised-letter task called for notification {} which is in {} state'.format(
notification.id, notification.status)
)
return
try:
original_pdf_object = s3.get_s3_object(current_app.config['LETTERS_SCAN_BUCKET_NAME'], filename)
if letter_details['validation_status'] == 'failed':
current_app.logger.info('Processing invalid precompiled pdf with id {} (file {})'.format(
notification_id, filename))
_move_invalid_letter_and_update_status(
notification=notification,
filename=filename,
scan_pdf_object=original_pdf_object,
message=letter_details['message'],
invalid_pages=letter_details['invalid_pages'],
page_count=letter_details['page_count'],
)
return
current_app.logger.info('Processing valid precompiled pdf with id {} (file {})'.format(
notification_id, filename))
billable_units = get_billable_units_for_letter_page_count(letter_details['page_count'])
is_test_key = notification.key_type == KEY_TYPE_TEST
# Updating the notification needs to happen before the file is moved. This is so that if updating the
# notification fails, the task can retry because the file is in the same place.
update_letter_pdf_status(
reference=notification.reference,
status=NOTIFICATION_DELIVERED if is_test_key else NOTIFICATION_CREATED,
billable_units=billable_units,
recipient_address=letter_details['address']
)
# The original filename could be wrong because we didn't know the postage.
# Now we know if the letter is international, we can check what the filename should be.
upload_file_name = get_letter_pdf_filename(
reference=notification.reference,
crown=notification.service.crown,
created_at=notification.created_at,
ignore_folder=True,
postage=notification.postage
)
move_sanitised_letter_to_test_or_live_pdf_bucket(
filename,
is_test_key,
notification.created_at,
upload_file_name,
)
# We've moved the sanitised PDF from the sanitise bucket, but still need to delete the original file:
original_pdf_object.delete()
except BotoClientError:
# Boto exceptions are likely to be caused by the file(s) being in the wrong place, so retrying won't help -
# we'll need to manually investigate
current_app.logger.exception(
f"Boto error when processing sanitised letter for notification {notification.id} (file {filename})"
)
update_notification_status_by_id(notification.id, NOTIFICATION_TECHNICAL_FAILURE)
raise NotificationTechnicalFailureException
except Exception:
try:
current_app.logger.exception(
"RETRY: calling process_sanitised_letter task for notification {} failed".format(notification.id)
)
self.retry(queue=QueueNames.RETRY)
except self.MaxRetriesExceededError:
message = "RETRY FAILED: Max retries reached. " \
"The task process_sanitised_letter failed for notification {}. " \
"Notification has been updated to technical-failure".format(notification.id)
update_notification_status_by_id(notification.id, NOTIFICATION_TECHNICAL_FAILURE)
raise NotificationTechnicalFailureException(message)
def process_virus_scan_passed(self, filename):
reference = get_reference_from_filename(filename)
notification = dao_get_notification_by_reference(reference)
current_app.logger.info("notification id {} Virus scan passed: {}".format(
notification.id, filename))
is_test_key = notification.key_type == KEY_TYPE_TEST
scan_pdf_object = s3.get_s3_object(
current_app.config["LETTERS_SCAN_BUCKET_NAME"], filename)
old_pdf = scan_pdf_object.get()["Body"].read()
try:
billable_units = get_page_count(old_pdf)
except PdfReadError:
current_app.logger.exception(
msg="Invalid PDF received for notification_id: {}".format(
notification.id))
_move_invalid_letter_and_update_status(notification, filename,
scan_pdf_object)
return
sanitise_response = _sanitise_precompiled_pdf(self, notification, old_pdf)
if not sanitise_response:
new_pdf = None
else:
sanitise_response = sanitise_response.json()
try:
new_pdf = base64.b64decode(sanitise_response["file"].encode())
except JSONDecodeError:
new_pdf = sanitise_response.content
redaction_failed_message = sanitise_response.get(
"redaction_failed_message")
if redaction_failed_message and not is_test_key:
current_app.logger.info("{} for notification id {} ({})".format(
redaction_failed_message, notification.id, filename))
copy_redaction_failed_pdf(filename)
# TODO: Remove this once CYSP update their template to not cross over the margins
if notification.service_id == UUID(
"fe44178f-3b45-4625-9f85-2264a36dd9ec"): # CYSP
# Check your state pension submit letters with good addresses and notify tags, so just use their supplied pdf
new_pdf = old_pdf
if not new_pdf:
current_app.logger.info(
"Invalid precompiled pdf received {} ({})".format(
notification.id, filename))
_move_invalid_letter_and_update_status(notification, filename,
scan_pdf_object)
return
else:
current_app.logger.info(
"Validation was successful for precompiled pdf {} ({})".format(
notification.id, filename))
current_app.logger.info(
"notification id {} ({}) sanitised and ready to send".format(
notification.id, filename))
try:
_upload_pdf_to_test_or_live_pdf_bucket(new_pdf,
filename,
is_test_letter=is_test_key)
update_letter_pdf_status(
reference=reference,
status=NOTIFICATION_DELIVERED
if is_test_key else NOTIFICATION_CREATED,
billable_units=billable_units,
)
scan_pdf_object.delete()
except BotoClientError:
current_app.logger.exception(
"Error uploading letter to live pdf bucket for notification: {}".
format(notification.id))
update_notification_status_by_id(notification.id,
NOTIFICATION_TECHNICAL_FAILURE)