def test_get_letter_pdf_filename_returns_correct_filename_for_test_letters(
notify_api, mocker):
filename = get_letter_pdf_filename(reference="foo",
crown="C",
is_scan_letter=True)
assert filename == "NOTIFY.FOO.D.2.C.C.20171204172900.PDF"
def _delete_letters_from_s3(notification_type, service_id, date_to_delete_from,
query_limit):
bucket_name = current_app.config['LETTERS_PDF_BUCKET_NAME']
letters_to_delete_from_s3 = db.session.query(Notification).filter(
Notification.notification_type == notification_type,
Notification.created_at < date_to_delete_from,
Notification.service_id == service_id,
# although letters in non completed statuses do have PDFs in s3, they do not exist in the
# production-letters-pdf bucket as they never made it that far so we do not try and delete
# them from it
Notification.status.in_(NOTIFICATION_STATUS_TYPES_COMPLETED)).limit(
query_limit).all()
for letter in letters_to_delete_from_s3:
prefix = get_letter_pdf_filename(
reference=letter.reference,
crown=letter.service.crown,
created_at=letter.created_at,
ignore_folder=letter.key_type == KEY_TYPE_TEST,
postage=letter.postage)
s3_objects = get_s3_bucket_objects(bucket_name=bucket_name,
subfolder=prefix)
for s3_object in s3_objects:
try:
remove_s3_object(bucket_name, s3_object['Key'])
except ClientError:
current_app.logger.exception(
"Could not delete S3 object with filename: {}".format(
s3_object['Key']))
def get_key_and_size_of_letters_to_be_sent_to_print(print_run_deadline,
postage):
letters_awaiting_sending = dao_get_letters_to_be_printed(
print_run_deadline, postage)
letter_pdfs = []
for letter in letters_awaiting_sending:
try:
letter_file_name = get_letter_pdf_filename(
reference=letter.reference,
crown=letter.service.crown,
sending_date=letter.created_at,
postage=letter.postage)
letter_head = s3.head_s3_object(
current_app.config['LETTERS_PDF_BUCKET_NAME'],
letter_file_name)
letter_pdfs.append({
"Key": letter_file_name,
"Size": letter_head['ContentLength']
})
except BotoClientError as e:
current_app.logger.exception(
f"Error getting letter from bucket for notification: {letter.id} with reference: {letter.reference}",
e)
return letter_pdfs
def _delete_letters_from_s3(notification_type, service_id, date_to_delete_from,
query_limit):
letters_to_delete_from_s3 = db.session.query(Notification).filter(
Notification.notification_type == notification_type,
Notification.created_at < date_to_delete_from,
Notification.service_id == service_id).limit(query_limit).all()
for letter in letters_to_delete_from_s3:
bucket_name = current_app.config['LETTERS_PDF_BUCKET_NAME']
# I don't think we need this anymore, we should update the query to get letters sent 7 days ago
if letter.sent_at:
prefix = get_letter_pdf_filename(
reference=letter.reference,
crown=letter.service.crown,
created_at=letter.created_at,
ignore_folder=letter.key_type == KEY_TYPE_TEST,
postage=letter.postage)
s3_objects = get_s3_bucket_objects(bucket_name=bucket_name,
subfolder=prefix)
for s3_object in s3_objects:
try:
remove_s3_object(bucket_name, s3_object['Key'])
except ClientError:
current_app.logger.exception(
"Could not delete S3 object with filename: {}".format(
s3_object['Key']))
def test_get_letter_pdf_filename_returns_correct_filename_for_test_letters(
notify_api, mocker):
sending_date = datetime(2017, 12, 4, 17, 29)
filename = get_letter_pdf_filename(reference='foo', crown='C',
sending_date=sending_date, dont_use_sending_date=True)
assert filename == 'NOTIFY.FOO.D.2.C.C.20171204172900.PDF'
def test_upload_letter_pdf_to_correct_bucket(sample_letter_notification,
mocker, is_precompiled_letter,
bucket_config_name):
if is_precompiled_letter:
sample_letter_notification.template.hidden = True
sample_letter_notification.template.name = PRECOMPILED_TEMPLATE_NAME
mock_s3 = mocker.patch("app.letters.utils.s3upload")
filename = get_letter_pdf_filename(
reference=sample_letter_notification.reference,
crown=sample_letter_notification.service.crown,
is_scan_letter=is_precompiled_letter,
)
upload_letter_pdf(sample_letter_notification,
b"\x00\x01",
precompiled=is_precompiled_letter)
mock_s3.assert_called_once_with(
bucket_name=current_app.config[bucket_config_name],
file_location=filename,
filedata=b"\x00\x01",
region=current_app.config["AWS_REGION"],
)
def test_get_letter_pdf_filename_returns_correct_postage_for_filename(
notify_api, postage, expected_postage):
filename = get_letter_pdf_filename(reference="foo",
crown=True,
postage=postage)
assert filename == "2017-12-04/NOTIFY.FOO.D.{}.C.C.20171204172900.PDF".format(
expected_postage)
def test_get_letter_pdf_filename_returns_tomorrows_filename(
notify_api, mocker):
created_at = datetime(2017, 12, 4, 17, 31)
filename = get_letter_pdf_filename(reference='foo',
crown=True,
created_at=created_at)
assert filename == '2017-12-05/NOTIFY.FOO.D.2.C.C.20171204173100.PDF'
def test_get_letter_pdf_filename_returns_correct_filename(
notify_api, mocker, crown_flag, expected_crown_text):
created_at = datetime(2017, 12, 4, 17, 29)
filename = get_letter_pdf_filename(reference='foo',
crown=crown_flag,
created_at=created_at)
assert filename == '2017-12-04/NOTIFY.FOO.D.2.C.{}.20171204172900.PDF'.format(
expected_crown_text)
def test_get_letter_pdf_filename_returns_correct_filename_for_test_letters(
notify_api, mocker):
created_at = datetime(2017, 12, 4, 17, 29)
filename = get_letter_pdf_filename(reference='foo',
crown='C',
created_at=created_at,
ignore_folder=True)
assert filename == 'NOTIFY.FOO.D.2.C.C.20171204172900.PDF'
def test_get_letter_pdf_filename_returns_correct_postage_for_filename(
notify_api, postage, expected_postage):
created_at = datetime(2017, 12, 4, 17, 29)
filename = get_letter_pdf_filename(reference='foo',
crown=True,
created_at=created_at,
postage=postage)
assert filename == '2017-12-04/NOTIFY.FOO.D.{}.C.C.20171204172900.PDF'.format(
expected_postage)
def send_pdf_letter_notification(service_id, post_data):
service = dao_fetch_service_by_id(service_id)
check_service_has_permission(LETTER_TYPE, service.permissions)
check_service_has_permission(UPLOAD_LETTERS, service.permissions)
check_service_over_daily_message_limit(KEY_TYPE_NORMAL, service)
validate_created_by(service, post_data["created_by"])
template = get_precompiled_letter_template(service.id)
file_location = "service-{}/{}.pdf".format(service.id, post_data["file_id"])
try:
letter = utils_s3download(current_app.config["TRANSIENT_UPLOADED_LETTERS"], file_location)
except S3ObjectNotFound as e:
current_app.logger.exception(
"Letter {}.pdf not in transient {} bucket".format(
post_data["file_id"], current_app.config["TRANSIENT_UPLOADED_LETTERS"]
)
)
raise e
# Getting the page count won't raise an error since admin has already checked the PDF is valid
billable_units = get_page_count(letter.read())
personalisation = {"address_line_1": post_data["filename"]}
# TODO: stop hard-coding postage as 'second' once we get postage from the admin
notification = persist_notification(
notification_id=post_data["file_id"],
template_id=template.id,
template_version=template.version,
template_postage=template.postage,
recipient=post_data["filename"],
service=service,
personalisation=personalisation,
notification_type=LETTER_TYPE,
api_key_id=None,
key_type=KEY_TYPE_NORMAL,
reference=create_one_off_reference(LETTER_TYPE),
client_reference=post_data["filename"],
created_by_id=post_data["created_by"],
billable_units=billable_units,
postage="second",
)
upload_filename = get_letter_pdf_filename(
notification.reference,
notification.service.crown,
is_scan_letter=False,
postage=notification.postage,
)
move_uploaded_pdf_to_letters_bucket(file_location, upload_filename)
return {"id": str(notification.id)}
def get_pdf_for_templated_letter(self, notification_id):
try:
notification = get_notification_by_id(notification_id, _raise=True)
letter_filename = get_letter_pdf_filename(
reference=notification.reference,
crown=notification.service.crown,
sending_date=notification.created_at,
dont_use_sending_date=notification.key_type == KEY_TYPE_TEST,
postage=notification.postage)
letter_data = {
'letter_contact_block':
notification.reply_to_text,
'template': {
"subject": notification.template.subject,
"content": notification.template.content,
"template_type": notification.template.template_type
},
'values':
notification.personalisation,
'logo_filename':
notification.service.letter_branding
and notification.service.letter_branding.filename,
'letter_filename':
letter_filename,
"notification_id":
str(notification_id),
'key_type':
notification.key_type
}
encrypted_data = encryption.encrypt(letter_data)
notify_celery.send_task(name=TaskNames.CREATE_PDF_FOR_TEMPLATED_LETTER,
args=(encrypted_data, ),
queue=QueueNames.SANITISE_LETTERS)
except Exception:
try:
current_app.logger.exception(
f"RETRY: calling create-letter-pdf task for notification {notification_id} failed"
)
self.retry(queue=QueueNames.RETRY)
except self.MaxRetriesExceededError:
message = f"RETRY FAILED: Max retries reached. " \
f"The task create-letter-pdf failed for notification id {notification_id}. " \
f"Notification has been updated to technical-failure"
update_notification_status_by_id(notification_id,
NOTIFICATION_TECHNICAL_FAILURE)
raise NotificationTechnicalFailureException(message)
def test_create_letters_pdf_calls_s3upload(mocker, sample_letter_notification):
mocker.patch('app.celery.letters_pdf_tasks.get_letters_pdf',
return_value=(b'\x00\x01', '1'))
mock_s3 = mocker.patch('app.letters.utils.s3upload')
create_letters_pdf(sample_letter_notification.id)
filename = get_letter_pdf_filename(
reference=sample_letter_notification.reference,
crown=sample_letter_notification.service.crown)
mock_s3.assert_called_with(
bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'],
file_location=filename,
filedata=b'\x00\x01',
region=current_app.config['AWS_REGION'])
def test_upload_letter_pdf_uses_postage_from_notification(
sample_letter_template, mocker, postage, expected_postage):
letter_notification = create_notification(template=sample_letter_template,
postage=postage)
mock_s3 = mocker.patch('app.letters.utils.s3upload')
filename = get_letter_pdf_filename(reference=letter_notification.reference,
crown=letter_notification.service.crown,
is_scan_letter=False,
postage=letter_notification.postage)
upload_letter_pdf(letter_notification, b'\x00\x01', precompiled=False)
mock_s3.assert_called_once_with(
bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'],
file_location=filename,
filedata=b'\x00\x01',
region=current_app.config['AWS_REGION'])
def test_upload_letter_pdf_to_correct_bucket(
sample_letter_notification, mocker, is_precompiled_letter, bucket_config_name
):
if is_precompiled_letter:
sample_letter_notification.template.hidden = True
sample_letter_notification.template.name = PRECOMPILED_TEMPLATE_NAME
mock_s3 = mocker.patch('app.letters.utils.s3upload')
filename = get_letter_pdf_filename(
reference=sample_letter_notification.reference,
crown=sample_letter_notification.service.crown,
sending_date=sample_letter_notification.created_at,
dont_use_sending_date=is_precompiled_letter
)
upload_letter_pdf(sample_letter_notification, b'\x00\x01', precompiled=is_precompiled_letter)
mock_s3.assert_called_once_with(
bucket_name=current_app.config[bucket_config_name],
file_location=filename,
filedata=b'\x00\x01',
region=current_app.config['AWS_REGION']
)
def send_pdf_letter_notification(service_id, post_data):
service = dao_fetch_service_by_id(service_id)
check_service_has_permission(LETTER_TYPE,
[p.permission for p in service.permissions])
check_service_over_daily_message_limit(KEY_TYPE_NORMAL, service)
validate_created_by(service, post_data['created_by'])
validate_and_format_recipient(
send_to=post_data['recipient_address'],
key_type=KEY_TYPE_NORMAL,
service=service,
notification_type=LETTER_TYPE,
allow_guest_list_recipients=False,
)
template = get_precompiled_letter_template(service.id)
file_location = 'service-{}/{}.pdf'.format(service.id,
post_data['file_id'])
try:
letter = utils_s3download(
current_app.config['TRANSIENT_UPLOADED_LETTERS'], file_location)
except S3ObjectNotFound as e:
current_app.logger.exception(
'Letter {}.pdf not in transient {} bucket'.format(
post_data['file_id'],
current_app.config['TRANSIENT_UPLOADED_LETTERS']))
raise e
# Getting the page count won't raise an error since admin has already checked the PDF is valid
page_count = get_page_count(letter.read())
billable_units = get_billable_units_for_letter_page_count(page_count)
personalisation = {'address_line_1': post_data['filename']}
notification = persist_notification(
notification_id=post_data['file_id'],
template_id=template.id,
template_version=template.version,
recipient=urllib.parse.unquote(post_data['recipient_address']),
service=service,
personalisation=personalisation,
notification_type=LETTER_TYPE,
api_key_id=None,
key_type=KEY_TYPE_NORMAL,
reference=create_one_off_reference(LETTER_TYPE),
client_reference=post_data['filename'],
created_by_id=post_data['created_by'],
billable_units=billable_units,
postage=post_data['postage'] or template.postage,
)
upload_filename = get_letter_pdf_filename(
reference=notification.reference,
crown=notification.service.crown,
created_at=notification.created_at,
ignore_folder=False,
postage=notification.postage)
move_uploaded_pdf_to_letters_bucket(file_location, upload_filename)
return {'id': str(notification.id)}
def test_get_letter_pdf_filename_returns_correct_filename(
notify_api, mocker, crown_flag, expected_crown_text):
filename = get_letter_pdf_filename(reference="foo", crown=crown_flag)
assert filename == "2017-12-04/NOTIFY.FOO.D.2.C.{}.20171204172900.PDF".format(
expected_crown_text)
def process_sanitised_letter(self, sanitise_data):
letter_details = encryption.decrypt(sanitise_data)
filename = letter_details['filename']
notification_id = letter_details['notification_id']
current_app.logger.info('Processing sanitised letter with id {}'.format(notification_id))
notification = get_notification_by_id(notification_id, _raise=True)
if notification.status != NOTIFICATION_PENDING_VIRUS_CHECK:
current_app.logger.info(
'process-sanitised-letter task called for notification {} which is in {} state'.format(
notification.id, notification.status)
)
return
try:
original_pdf_object = s3.get_s3_object(current_app.config['LETTERS_SCAN_BUCKET_NAME'], filename)
if letter_details['validation_status'] == 'failed':
current_app.logger.info('Processing invalid precompiled pdf with id {} (file {})'.format(
notification_id, filename))
_move_invalid_letter_and_update_status(
notification=notification,
filename=filename,
scan_pdf_object=original_pdf_object,
message=letter_details['message'],
invalid_pages=letter_details['invalid_pages'],
page_count=letter_details['page_count'],
)
return
current_app.logger.info('Processing valid precompiled pdf with id {} (file {})'.format(
notification_id, filename))
billable_units = get_billable_units_for_letter_page_count(letter_details['page_count'])
is_test_key = notification.key_type == KEY_TYPE_TEST
# Updating the notification needs to happen before the file is moved. This is so that if updating the
# notification fails, the task can retry because the file is in the same place.
update_letter_pdf_status(
reference=notification.reference,
status=NOTIFICATION_DELIVERED if is_test_key else NOTIFICATION_CREATED,
billable_units=billable_units,
recipient_address=letter_details['address']
)
# The original filename could be wrong because we didn't know the postage.
# Now we know if the letter is international, we can check what the filename should be.
upload_file_name = get_letter_pdf_filename(
reference=notification.reference,
crown=notification.service.crown,
created_at=notification.created_at,
ignore_folder=True,
postage=notification.postage
)
move_sanitised_letter_to_test_or_live_pdf_bucket(
filename,
is_test_key,
notification.created_at,
upload_file_name,
)
# We've moved the sanitised PDF from the sanitise bucket, but still need to delete the original file:
original_pdf_object.delete()
except BotoClientError:
# Boto exceptions are likely to be caused by the file(s) being in the wrong place, so retrying won't help -
# we'll need to manually investigate
current_app.logger.exception(
f"Boto error when processing sanitised letter for notification {notification.id} (file {filename})"
)
update_notification_status_by_id(notification.id, NOTIFICATION_TECHNICAL_FAILURE)
raise NotificationTechnicalFailureException
except Exception:
try:
current_app.logger.exception(
"RETRY: calling process_sanitised_letter task for notification {} failed".format(notification.id)
)
self.retry(queue=QueueNames.RETRY)
except self.MaxRetriesExceededError:
message = "RETRY FAILED: Max retries reached. " \
"The task process_sanitised_letter failed for notification {}. " \
"Notification has been updated to technical-failure".format(notification.id)
update_notification_status_by_id(notification.id, NOTIFICATION_TECHNICAL_FAILURE)
raise NotificationTechnicalFailureException(message)
def test_get_letter_pdf_filename_returns_tomorrows_filename(
notify_api, mocker):
filename = get_letter_pdf_filename(reference="foo", crown=True)
assert filename == "2017-12-05/NOTIFY.FOO.D.2.C.C.20171204173100.PDF"
