def sanitise_letter(self, filename):
try:
reference = get_reference_from_filename(filename)
notification = dao_get_notification_by_reference(reference)
current_app.logger.info('Notification ID {} Virus scan passed: {}'.format(notification.id, filename))
if notification.status != NOTIFICATION_PENDING_VIRUS_CHECK:
current_app.logger.info('Sanitise letter called for notification {} which is in {} state'.format(
notification.id, notification.status))
return
notify_celery.send_task(
name=TaskNames.SANITISE_LETTER,
kwargs={
'notification_id': str(notification.id),
'filename': filename,
'allow_international_letters': notification.service.has_permission(
INTERNATIONAL_LETTERS
),
},
queue=QueueNames.SANITISE_LETTERS,
)
except Exception:
try:
current_app.logger.exception(
"RETRY: calling sanitise_letter task for notification {} failed".format(notification.id)
)
self.retry(queue=QueueNames.RETRY)
except self.MaxRetriesExceededError:
message = "RETRY FAILED: Max retries reached. " \
"The task sanitise_letter failed for notification {}. " \
"Notification has been updated to technical-failure".format(notification.id)
update_notification_status_by_id(notification.id, NOTIFICATION_TECHNICAL_FAILURE)
raise NotificationTechnicalFailureException(message)
def letter_in_created_state(filename):
# filename looks like '2018-01-13/NOTIFY.ABCDEF1234567890.D.2.C.C.20180113120000.PDF'
subfolder = filename.split('/')[0]
ref = get_reference_from_filename(filename)
notifications = dao_get_notifications_by_references([ref])
if notifications:
if notifications[0].status == NOTIFICATION_CREATED:
return True
current_app.logger.info(
'Collating letters for {} but notification with reference {} already in {}'
.format(subfolder, ref, notifications[0].status))
return False
def process_virus_scan_passed(filename):
reference = get_reference_from_filename(filename)
notification = dao_get_notification_by_reference(reference)
current_app.logger.info('notification id {} Virus scan passed: {}'.format(
notification.id, filename))
is_test_key = notification.key_type == KEY_TYPE_TEST
move_scanned_pdf_to_test_or_live_pdf_bucket(filename,
is_test_letter=is_test_key)
update_letter_pdf_status(
reference,
NOTIFICATION_DELIVERED if is_test_key else NOTIFICATION_CREATED)
def process_virus_scan_error(filename):
move_failed_pdf(filename, ScanErrorType.ERROR)
reference = get_reference_from_filename(filename)
notification = dao_get_notification_by_reference(reference)
updated_count = update_letter_pdf_status(reference,
NOTIFICATION_TECHNICAL_FAILURE)
if updated_count != 1:
raise Exception(
"There should only be one letter notification for each reference. Found {} notifications"
.format(updated_count))
raise VirusScanError('notification id {} Virus scan error: {}'.format(
notification.id, filename))
def process_virus_scan_failed(filename):
move_failed_pdf(filename, ScanErrorType.FAILURE)
reference = get_reference_from_filename(filename)
notification = dao_get_notification_by_reference(reference)
updated_count = update_letter_pdf_status(reference, NOTIFICATION_VIRUS_SCAN_FAILED, billable_units=0)
if updated_count != 1:
raise Exception(
"There should only be one letter notification for each reference. Found {} notifications".format(
updated_count
)
)
error = VirusScanError('notification id {} Virus scan failed: {}'.format(notification.id, filename))
current_app.logger.exception(error)
raise error
def process_virus_scan_passed(self, filename):
reference = get_reference_from_filename(filename)
notification = dao_get_notification_by_reference(reference)
current_app.logger.info('notification id {} Virus scan passed: {}'.format(
notification.id, filename))
is_test_key = notification.key_type == KEY_TYPE_TEST
scan_pdf_object = s3.get_s3_object(
current_app.config['LETTERS_SCAN_BUCKET_NAME'], filename)
old_pdf = scan_pdf_object.get()['Body'].read()
sanitise_response, result = _sanitise_precompiled_pdf(
self, notification, old_pdf)
new_pdf = None
if result == 'validation_passed':
new_pdf = base64.b64decode(sanitise_response["file"].encode())
redaction_failed_message = sanitise_response.get(
"redaction_failed_message")
if redaction_failed_message and not is_test_key:
current_app.logger.info('{} for notification id {} ({})'.format(
redaction_failed_message, notification.id, filename))
copy_redaction_failed_pdf(filename)
billable_units = get_billable_units_for_letter_page_count(
sanitise_response.get("page_count"))
# TODO: Remove this once CYSP update their template to not cross over the margins
if notification.service_id == UUID(
'fe44178f-3b45-4625-9f85-2264a36dd9ec'): # CYSP
# Check your state pension submit letters with good addresses and notify tags, so just use their supplied pdf
new_pdf = old_pdf
if result == 'validation_failed' and not new_pdf:
current_app.logger.info(
'Invalid precompiled pdf received {} ({})'.format(
notification.id, filename))
_move_invalid_letter_and_update_status(
notification=notification,
filename=filename,
scan_pdf_object=scan_pdf_object,
message=sanitise_response["message"],
invalid_pages=sanitise_response.get("invalid_pages"),
page_count=sanitise_response.get("page_count"))
return
current_app.logger.info(
'notification id {} ({}) sanitised and ready to send'.format(
notification.id, filename))
try:
_upload_pdf_to_test_or_live_pdf_bucket(
new_pdf,
filename,
is_test_letter=is_test_key,
created_at=notification.created_at)
update_letter_pdf_status(reference=reference,
status=NOTIFICATION_DELIVERED
if is_test_key else NOTIFICATION_CREATED,
billable_units=billable_units)
scan_pdf_object.delete()
except BotoClientError:
current_app.logger.exception(
"Error uploading letter to live pdf bucket for notification: {}".
format(notification.id))
update_notification_status_by_id(notification.id,
NOTIFICATION_TECHNICAL_FAILURE)
def process_virus_scan_passed(self, filename):
reference = get_reference_from_filename(filename)
notification = dao_get_notification_by_reference(reference)
current_app.logger.info("notification id {} Virus scan passed: {}".format(
notification.id, filename))
is_test_key = notification.key_type == KEY_TYPE_TEST
scan_pdf_object = s3.get_s3_object(
current_app.config["LETTERS_SCAN_BUCKET_NAME"], filename)
old_pdf = scan_pdf_object.get()["Body"].read()
try:
billable_units = get_page_count(old_pdf)
except PdfReadError:
current_app.logger.exception(
msg="Invalid PDF received for notification_id: {}".format(
notification.id))
_move_invalid_letter_and_update_status(notification, filename,
scan_pdf_object)
return
sanitise_response = _sanitise_precompiled_pdf(self, notification, old_pdf)
if not sanitise_response:
new_pdf = None
else:
sanitise_response = sanitise_response.json()
try:
new_pdf = base64.b64decode(sanitise_response["file"].encode())
except JSONDecodeError:
new_pdf = sanitise_response.content
redaction_failed_message = sanitise_response.get(
"redaction_failed_message")
if redaction_failed_message and not is_test_key:
current_app.logger.info("{} for notification id {} ({})".format(
redaction_failed_message, notification.id, filename))
copy_redaction_failed_pdf(filename)
# TODO: Remove this once CYSP update their template to not cross over the margins
if notification.service_id == UUID(
"fe44178f-3b45-4625-9f85-2264a36dd9ec"): # CYSP
# Check your state pension submit letters with good addresses and notify tags, so just use their supplied pdf
new_pdf = old_pdf
if not new_pdf:
current_app.logger.info(
"Invalid precompiled pdf received {} ({})".format(
notification.id, filename))
_move_invalid_letter_and_update_status(notification, filename,
scan_pdf_object)
return
else:
current_app.logger.info(
"Validation was successful for precompiled pdf {} ({})".format(
notification.id, filename))
current_app.logger.info(
"notification id {} ({}) sanitised and ready to send".format(
notification.id, filename))
try:
_upload_pdf_to_test_or_live_pdf_bucket(new_pdf,
filename,
is_test_letter=is_test_key)
update_letter_pdf_status(
reference=reference,
status=NOTIFICATION_DELIVERED
if is_test_key else NOTIFICATION_CREATED,
billable_units=billable_units,
)
scan_pdf_object.delete()
except BotoClientError:
current_app.logger.exception(
"Error uploading letter to live pdf bucket for notification: {}".
format(notification.id))
update_notification_status_by_id(notification.id,
NOTIFICATION_TECHNICAL_FAILURE)
