def test_process_letter_task_check_virus_scan_passed_when_sanitise_fails(
sample_letter_notification, mocker, key_type, aws_credentials):
filename = 'NOTIFY.{}'.format(sample_letter_notification.reference)
source_bucket_name = current_app.config['LETTERS_SCAN_BUCKET_NAME']
target_bucket_name = current_app.config['INVALID_PDF_BUCKET_NAME']
conn = boto3.resource('s3', region_name='us-east-1')
conn.create_bucket(Bucket=source_bucket_name)
conn.create_bucket(Bucket=target_bucket_name)
s3 = boto3.client('s3', region_name='us-east-1')
s3.put_object(Bucket=source_bucket_name, Key=filename, Body=b'pdf_content')
sample_letter_notification.status = NOTIFICATION_PENDING_VIRUS_CHECK
sample_letter_notification.key_type = key_type
mock_move_s3 = mocker.patch('app.letters.utils._move_s3_object')
mock_sanitise = mocker.patch(
'app.celery.letters_pdf_tasks._sanitise_precompiled_pdf',
return_value=None)
mock_get_page_count = mocker.patch(
'app.celery.letters_pdf_tasks.get_page_count', return_value=2)
process_virus_scan_passed(filename)
assert sample_letter_notification.status == NOTIFICATION_VALIDATION_FAILED
assert sample_letter_notification.billable_units == 0
mock_sanitise.assert_called_once_with(ANY, sample_letter_notification,
b'pdf_content')
mock_move_s3.assert_called_once_with(source_bucket_name, filename,
target_bucket_name, filename)
mock_get_page_count.assert_called_once_with(b'pdf_content')
def test_process_letter_task_check_virus_scan_passed_when_file_cannot_be_opened(
sample_letter_notification, mocker, key_type):
filename = "NOTIFY.{}".format(sample_letter_notification.reference)
source_bucket_name = current_app.config["LETTERS_SCAN_BUCKET_NAME"]
target_bucket_name = current_app.config["INVALID_PDF_BUCKET_NAME"]
conn = boto3.resource("s3", region_name="ca-central-1")
conn.create_bucket(Bucket=source_bucket_name)
conn.create_bucket(Bucket=target_bucket_name)
s3 = boto3.client("s3", region_name="ca-central-1")
s3.put_object(Bucket=source_bucket_name, Key=filename, Body=b"pdf_content")
sample_letter_notification.status = NOTIFICATION_PENDING_VIRUS_CHECK
sample_letter_notification.key_type = key_type
mock_move_s3 = mocker.patch("app.letters.utils._move_s3_object")
mock_get_page_count = mocker.patch(
"app.celery.letters_pdf_tasks.get_page_count",
side_effect=PdfReadError)
mock_sanitise = mocker.patch(
"app.celery.letters_pdf_tasks._sanitise_precompiled_pdf")
process_virus_scan_passed(filename)
mock_sanitise.assert_not_called()
mock_get_page_count.assert_called_once_with(b"pdf_content")
mock_move_s3.assert_called_once_with(source_bucket_name, filename,
target_bucket_name, filename)
assert sample_letter_notification.status == NOTIFICATION_VALIDATION_FAILED
assert sample_letter_notification.billable_units == 0
def test_process_letter_task_check_virus_scan_passed(
sample_letter_template,
mocker,
key_type,
noti_status,
bucket_config_name,
destination_folder,
):
letter_notification = create_notification(
template=sample_letter_template,
billable_units=0,
status="pending-virus-check",
key_type=key_type,
reference="{} letter".format(key_type),
)
filename = "NOTIFY.{}".format(letter_notification.reference)
source_bucket_name = current_app.config["LETTERS_SCAN_BUCKET_NAME"]
target_bucket_name = current_app.config[bucket_config_name]
conn = boto3.resource("s3", region_name="ca-central-1")
conn.create_bucket(Bucket=source_bucket_name)
conn.create_bucket(Bucket=target_bucket_name)
s3 = boto3.client("s3", region_name="ca-central-1")
s3.put_object(Bucket=source_bucket_name, Key=filename, Body=b"old_pdf")
mock_get_page_count = mocker.patch(
"app.celery.letters_pdf_tasks.get_page_count", return_value=1)
mock_s3upload = mocker.patch("app.celery.letters_pdf_tasks.s3upload")
endpoint = "http://localhost:9999/precompiled/sanitise"
with requests_mock.mock() as rmock:
rmock.request(
"POST",
endpoint,
json={
"file": base64.b64encode(b"new_pdf").decode("utf-8"),
"validation_passed": True,
"errors": {
"content_outside_of_printable_area": [],
"document_not_a4_size_portrait_orientation": [],
},
},
status_code=200,
)
process_virus_scan_passed(filename)
assert letter_notification.status == noti_status
assert letter_notification.billable_units == 1
assert rmock.called
assert rmock.request_history[0].url == endpoint
mock_s3upload.assert_called_once_with(
bucket_name=target_bucket_name,
filedata=b"new_pdf",
file_location=destination_folder + filename,
region="ca-central-1",
)
mock_get_page_count.assert_called_once_with(b"old_pdf")
def test_process_letter_task_check_virus_scan_passed(
sample_letter_template, mocker, key_type, noti_status,
bucket_config_name, destination_folder, aws_credentials):
letter_notification = create_notification(
template=sample_letter_template,
billable_units=0,
status='pending-virus-check',
key_type=key_type,
reference='{} letter'.format(key_type))
filename = 'NOTIFY.{}'.format(letter_notification.reference)
source_bucket_name = current_app.config['LETTERS_SCAN_BUCKET_NAME']
target_bucket_name = current_app.config[bucket_config_name]
target_aws_region = current_app.config['AWS_REGION']
conn = boto3.resource('s3', region_name=target_aws_region)
conn.create_bucket(
Bucket=source_bucket_name,
CreateBucketConfiguration={'LocationConstraint': target_aws_region})
conn.create_bucket(
Bucket=target_bucket_name,
CreateBucketConfiguration={'LocationConstraint': target_aws_region})
s3 = boto3.client('s3', region_name=target_aws_region)
s3.put_object(Bucket=source_bucket_name, Key=filename, Body=b'old_pdf')
mock_get_page_count = mocker.patch(
'app.celery.letters_pdf_tasks.get_page_count', return_value=1)
mock_s3upload = mocker.patch('app.celery.letters_pdf_tasks.s3upload')
endpoint = 'http://localhost:9999/precompiled/sanitise'
with requests_mock.mock() as rmock:
rmock.request("POST",
endpoint,
json={
"file": base64.b64encode(b"new_pdf").decode("utf-8"),
"validation_passed": True,
"errors": {
"content_outside_of_printable_area": [],
"document_not_a4_size_portrait_orientation": [],
}
},
status_code=200)
process_virus_scan_passed(filename)
assert letter_notification.status == noti_status
assert letter_notification.billable_units == 1
assert rmock.called
assert rmock.request_history[0].url == endpoint
mock_s3upload.assert_called_once_with(
bucket_name=target_bucket_name,
filedata=b'new_pdf',
file_location=destination_folder + filename,
region=target_aws_region,
)
mock_get_page_count.assert_called_once_with(b'old_pdf')
def test_process_virus_scan_passed_logs_error_and_sets_tech_failure_if_s3_error_uploading_to_live_bucket(
mocker,
sample_letter_notification,
):
mock_logger = mocker.patch("app.celery.tasks.current_app.logger.exception")
sample_letter_notification.status = NOTIFICATION_PENDING_VIRUS_CHECK
filename = "NOTIFY.{}".format(sample_letter_notification.reference)
source_bucket_name = current_app.config["LETTERS_SCAN_BUCKET_NAME"]
conn = boto3.resource("s3", region_name="ca-central-1")
conn.create_bucket(Bucket=source_bucket_name)
s3 = boto3.client("s3", region_name="ca-central-1")
s3.put_object(Bucket=source_bucket_name, Key=filename, Body=b"pdf_content")
mocker.patch("app.celery.letters_pdf_tasks.get_page_count", return_value=1)
error_response = {
"Error": {
"Code": "InvalidParameterValue",
"Message": "some error message from amazon",
"Type": "Sender",
}
}
mocker.patch(
"app.celery.letters_pdf_tasks._upload_pdf_to_test_or_live_pdf_bucket",
side_effect=ClientError(error_response, "operation_name"),
)
endpoint = "http://localhost:9999/precompiled/sanitise"
with requests_mock.mock() as rmock:
rmock.request(
"POST",
endpoint,
json={
"file": base64.b64encode(b"new_pdf").decode("utf-8"),
"validation_passed": True,
"errors": {
"content_outside_of_printable_area": [],
"document_not_a4_size_portrait_orientation": [],
},
},
status_code=200,
)
process_virus_scan_passed(filename)
assert sample_letter_notification.status == NOTIFICATION_TECHNICAL_FAILURE
mock_logger.assert_called_once_with(
"Error uploading letter to live pdf bucket for notification: {}".
format(sample_letter_notification.id))
def test_process_letter_task_check_virus_scan_passed_when_redaction_fails(
sample_letter_notification,
mocker,
key_type,
notification_status,
bucket_config_name,
):
filename = "NOTIFY.{}".format(sample_letter_notification.reference)
bucket_name = current_app.config["LETTERS_SCAN_BUCKET_NAME"]
target_bucket_name = current_app.config[bucket_config_name]
conn = boto3.resource("s3", region_name="eu-west-1")
conn.create_bucket(Bucket=bucket_name)
conn.create_bucket(Bucket=target_bucket_name)
s3 = boto3.client("s3", region_name="eu-west-1")
s3.put_object(Bucket=bucket_name, Key=filename, Body=b"pdf_content")
sample_letter_notification.status = NOTIFICATION_PENDING_VIRUS_CHECK
sample_letter_notification.key_type = key_type
mock_copy_s3 = mocker.patch("app.letters.utils._copy_s3_object")
mocker.patch("app.celery.letters_pdf_tasks.get_page_count", return_value=2)
endpoint = "http://localhost:9999/precompiled/sanitise"
with requests_mock.mock() as rmock:
rmock.request(
"POST",
endpoint,
json={
"file": base64.b64encode(b"new_pdf").decode("utf-8"),
"validation_passed": True,
"redaction_failed_message":
"No matches for address block during redaction procedure",
"errors": {
"content_outside_of_printable_area": [],
"document_not_a4_size_portrait_orientation": [],
},
},
status_code=200,
)
process_virus_scan_passed(filename)
assert sample_letter_notification.billable_units == 2
assert sample_letter_notification.status == notification_status
if key_type == KEY_TYPE_NORMAL:
mock_copy_s3.assert_called_once_with(bucket_name, filename,
bucket_name,
"REDACTION_FAILURE/" + filename)
else:
mock_copy_s3.assert_not_called()
def test_process_letter_task_check_virus_scan_passed(
sample_letter_notification, mocker, key_type, is_test_letter,
noti_status):
filename = 'NOTIFY.{}'.format(sample_letter_notification.reference)
sample_letter_notification.status = 'pending-virus-check'
sample_letter_notification.key_type = key_type
mock_move_pdf = mocker.patch(
'app.celery.letters_pdf_tasks.move_scanned_pdf_to_test_or_live_pdf_bucket'
)
process_virus_scan_passed(filename)
mock_move_pdf.assert_called_once_with(filename,
is_test_letter=is_test_letter)
assert sample_letter_notification.status == noti_status
def test_process_virus_scan_passed_logs_error_and_sets_tech_failure_if_s3_error_uploading_to_live_bucket(
mocker,
sample_letter_notification,
):
mock_logger = mocker.patch('app.celery.tasks.current_app.logger.exception')
sample_letter_notification.status = NOTIFICATION_PENDING_VIRUS_CHECK
filename = 'NOTIFY.{}'.format(sample_letter_notification.reference)
source_bucket_name = current_app.config['LETTERS_SCAN_BUCKET_NAME']
conn = boto3.resource('s3', region_name='eu-west-1')
conn.create_bucket(Bucket=source_bucket_name)
s3 = boto3.client('s3', region_name='eu-west-1')
s3.put_object(Bucket=source_bucket_name, Key=filename, Body=b'pdf_content')
error_response = {
'Error': {
'Code': 'InvalidParameterValue',
'Message': 'some error message from amazon',
'Type': 'Sender'
}
}
mocker.patch('app.celery.letters_pdf_tasks._upload_pdf_to_test_or_live_pdf_bucket',
side_effect=ClientError(error_response, 'operation_name'))
endpoint = 'http://localhost:9999/precompiled/sanitise'
with requests_mock.mock() as rmock:
rmock.request(
"POST",
endpoint,
json={
"file": base64.b64encode(b"new_pdf").decode("utf-8"),
"validation_passed": True,
"message": "",
"invalid_pages": [],
"page_count": 1
},
status_code=200
)
process_virus_scan_passed(filename)
assert sample_letter_notification.status == NOTIFICATION_TECHNICAL_FAILURE
mock_logger.assert_called_once_with(
'Error uploading letter to live pdf bucket for notification: {}'.format(sample_letter_notification.id)
)
def test_process_letter_task_check_virus_scan_passed_when_sanitise_fails(
sample_letter_notification, mocker, key_type
):
filename = 'NOTIFY.{}'.format(sample_letter_notification.reference)
source_bucket_name = current_app.config['LETTERS_SCAN_BUCKET_NAME']
target_bucket_name = current_app.config['INVALID_PDF_BUCKET_NAME']
conn = boto3.resource('s3', region_name='eu-west-1')
conn.create_bucket(Bucket=source_bucket_name)
conn.create_bucket(Bucket=target_bucket_name)
s3 = boto3.client('s3', region_name='eu-west-1')
s3.put_object(Bucket=source_bucket_name, Key=filename, Body=b'pdf_content')
sample_letter_notification.status = NOTIFICATION_PENDING_VIRUS_CHECK
sample_letter_notification.key_type = key_type
mock_move_s3 = mocker.patch('app.letters.utils._move_s3_object')
sanitise_response = {
"file": base64.b64encode(b"nyan").decode("utf-8"),
"validation_passed": False,
"message": "content-outside-printable-area",
"invalid_pages": [1, 2],
"page_count": 2
}
mock_sanitise = mocker.patch(
'app.celery.letters_pdf_tasks._sanitise_precompiled_pdf', return_value=(sanitise_response, "validation_failed")
)
process_virus_scan_passed(filename)
assert sample_letter_notification.status == NOTIFICATION_VALIDATION_FAILED
assert sample_letter_notification.billable_units == 0
mock_sanitise.assert_called_once_with(
ANY,
sample_letter_notification,
b'pdf_content'
)
mock_move_s3.assert_called_once_with(
source_bucket=source_bucket_name, source_filename=filename,
target_bucket=target_bucket_name, target_filename=filename, metadata={
"message": "content-outside-printable-area",
"invalid_pages": "[1, 2]",
"page_count": "2"
}
)
def test_process_letter_task_check_virus_scan_passed(
sample_letter_template, mocker, key_type, noti_status, bucket_config_name, destination_folder
):
letter_notification = create_notification(template=sample_letter_template, billable_units=0,
status='pending-virus-check', key_type=key_type,
reference='{} letter'.format(key_type))
filename = 'NOTIFY.{}'.format(letter_notification.reference)
source_bucket_name = current_app.config['LETTERS_SCAN_BUCKET_NAME']
target_bucket_name = current_app.config[bucket_config_name]
conn = boto3.resource('s3', region_name='eu-west-1')
conn.create_bucket(Bucket=source_bucket_name)
conn.create_bucket(Bucket=target_bucket_name)
s3 = boto3.client('s3', region_name='eu-west-1')
s3.put_object(Bucket=source_bucket_name, Key=filename, Body=b'old_pdf')
mock_s3upload = mocker.patch('app.celery.letters_pdf_tasks.s3upload')
endpoint = 'http://localhost:9999/precompiled/sanitise'
with requests_mock.mock() as rmock:
rmock.request(
"POST",
endpoint,
json={
"file": base64.b64encode(b"new_pdf").decode("utf-8"),
"validation_passed": True,
"message": "",
"invalid_pages": [],
"page_count": 1
},
status_code=200
)
process_virus_scan_passed(filename)
assert letter_notification.status == noti_status
assert letter_notification.billable_units == 1
assert rmock.called
assert rmock.request_history[0].url == endpoint
mock_s3upload.assert_called_once_with(
bucket_name=target_bucket_name,
filedata=b'new_pdf',
file_location=destination_folder + filename,
region='eu-west-1',
)
def test_process_letter_task_check_virus_scan_passed_when_redaction_fails(
sample_letter_notification, mocker, key_type, notification_status, bucket_config_name
):
filename = 'NOTIFY.{}'.format(sample_letter_notification.reference)
bucket_name = current_app.config['LETTERS_SCAN_BUCKET_NAME']
target_bucket_name = current_app.config[bucket_config_name]
conn = boto3.resource('s3', region_name='eu-west-1')
conn.create_bucket(Bucket=bucket_name)
conn.create_bucket(Bucket=target_bucket_name)
s3 = boto3.client('s3', region_name='eu-west-1')
s3.put_object(Bucket=bucket_name, Key=filename, Body=b'pdf_content')
sample_letter_notification.status = NOTIFICATION_PENDING_VIRUS_CHECK
sample_letter_notification.key_type = key_type
mock_copy_s3 = mocker.patch('app.letters.utils._copy_s3_object')
endpoint = 'http://localhost:9999/precompiled/sanitise'
with requests_mock.mock() as rmock:
rmock.request(
"POST",
endpoint,
json={
"file": base64.b64encode(b"new_pdf").decode("utf-8"),
"validation_passed": True,
"redaction_failed_message": "No matches for address block during redaction procedure",
"message": "",
"invalid_pages": "",
"page_count": 3
},
status_code=200
)
process_virus_scan_passed(filename)
assert sample_letter_notification.billable_units == 2
assert sample_letter_notification.status == notification_status
if key_type == KEY_TYPE_NORMAL:
mock_copy_s3.assert_called_once_with(
bucket_name, filename,
bucket_name, 'REDACTION_FAILURE/' + filename
)
else:
mock_copy_s3.assert_not_called()
def test_process_letter_task_check_virus_scan_passed_when_file_cannot_be_opened(
sample_letter_notification, mocker, key_type
):
filename = 'NOTIFY.{}'.format(sample_letter_notification.reference)
source_bucket_name = current_app.config['LETTERS_SCAN_BUCKET_NAME']
target_bucket_name = current_app.config['INVALID_PDF_BUCKET_NAME']
conn = boto3.resource('s3', region_name='eu-west-1')
conn.create_bucket(Bucket=source_bucket_name)
conn.create_bucket(Bucket=target_bucket_name)
s3 = boto3.client('s3', region_name='eu-west-1')
s3.put_object(Bucket=source_bucket_name, Key=filename, Body=b'pdf_content')
sample_letter_notification.status = NOTIFICATION_PENDING_VIRUS_CHECK
sample_letter_notification.key_type = key_type
mock_move_s3 = mocker.patch('app.letters.utils._move_s3_object')
endpoint = 'http://localhost:9999/precompiled/sanitise'
with requests_mock.mock() as rmock:
rmock.request(
"POST",
endpoint,
json={
"page_count": None,
"recipient_address": None,
"message": 'unable-to-read-the-file',
"invalid_pages": None,
"file": None
},
status_code=400
)
process_virus_scan_passed(filename)
mock_move_s3.assert_called_once_with(
source_bucket=source_bucket_name, source_filename=filename,
target_bucket=target_bucket_name, target_filename=filename, metadata={'message': 'unable-to-read-the-file'}
)
assert sample_letter_notification.status == NOTIFICATION_VALIDATION_FAILED
assert sample_letter_notification.billable_units == 0
