def test_should_get_references_out_of_s3_filenames(mocks): filenames = ['2017-01-01/TEST1.PDF'] zip_and_send_letter_pdfs(filenames, 'foo.zip') mocks.get_notification_references_from_s3_filenames.assert_called_once_with( filenames)
def test_zip_and_send_should_retry_if_s3_client_error(mocks): mocks.get_zip_of_letter_pdfs_from_s3.side_effect = ClientError({}, 'operation') filenames = ['2017-01-01/TEST1.PDF'] with pytest.raises(Retry): zip_and_send_letter_pdfs(filenames, 'foo.zip') mocks.zip_and_send_retry.assert_called_once_with(queue='process-ftp-tasks')
def test_zip_and_send_should_retry_if_celery_hits_soft_time_limit(mocks): mocks.file_exists_on_s3.side_effect = SoftTimeLimitExceeded mocks.zip_and_send_retry.side_effect = Retry filenames = ['2017-01-01/TEST1.PDF'] with pytest.raises(Retry): zip_and_send_letter_pdfs(filenames, 'foo.zip') mocks.zip_and_send_retry.assert_called_once_with(queue='process-ftp-tasks')
def test_zip_and_send_should_retry_if_cant_check_zips_sent(mocks): mocks.file_exists_on_s3.side_effect = ClientError({}, 'operation') mocks.zip_and_send_retry.side_effect = Retry filenames = ['2017-01-01/TEST1.PDF'] with pytest.raises(Retry): zip_and_send_letter_pdfs(filenames, 'foo.zip') mocks.zip_and_send_retry.assert_called_once_with(queue='process-ftp-tasks')
def test_zip_and_send_should_update_notifications_to_success(mocks): filenames = ['2017-01-01/TEST1.PDF'] zip_and_send_letter_pdfs(filenames, 'foo.zip') assert not mocks.file_exists_with_correct_size.called mocks.send_task.assert_called_once_with( name='update-letter-notifications-to-sent', args=(['1', '2', '3'], ), queue='notify-internal-tasks')
def test_zip_and_send_should_set_to_error_if_cant_check_zips_sent(mocks): mocks.file_exists_on_s3.side_effect = ClientError({}, 'operation') filenames = ['2017-01-01/TEST1.PDF'] zip_and_send_letter_pdfs(filenames, 'foo.zip') mocks.send_task.assert_called_once_with( name='update-letter-notifications-to-error', args=(['1', '2', '3'], ), queue='notify-internal-tasks')
def test_should_upload_record_of_zipfile_contents_to_s3(notify_ftp, mocks): filenames = ['2017-01-01/TEST1.PDF'] zip_and_send_letter_pdfs(filenames, 'foo.zip') assert mocks.upload_to_s3.call_args_list == [ call(bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'], file_location='2017-01-01/zips_sent/foo.zip.TXT', filedata=b'["2017-01-01/TEST1.PDF"]', region='eu-west-1') ]
def test_zip_and_send_should_update_notifications_to_success_if_send_zip_fails_but_files_uploaded( mocks): mocks.send_zip.side_effect = FtpException filenames = ['2017-01-01/TEST1.PDF'] zip_and_send_letter_pdfs(filenames, 'foo.zip') mocks.file_exists_with_correct_size.assert_called_once_with('foo.zip', 2) mocks.send_task.assert_called_once_with( name='update-letter-notifications-to-sent', args=(['1', '2', '3'], ), queue='notify-internal-tasks')
def test_zip_and_send_should_retry_if_send_zip_fails_and_files_did_not_upload( mocks): mocks.send_zip.side_effect = FtpException mocks.file_exists_with_correct_size.side_effect = FtpException filenames = ['2017-01-01/TEST1.PDF'] with pytest.raises(Retry): zip_and_send_letter_pdfs(filenames, 'foo.zip') mocks.file_exists_with_correct_size.assert_called_once_with('foo.zip', 2) mocks.zip_and_send_retry.assert_called_once_with(queue='process-ftp-tasks') assert not mocks.send_task.called
def test_zip_and_send_should_update_notification_if_max_retries_when_s3_client_error( mocks): mocks.get_zip_of_letter_pdfs_from_s3.side_effect = ClientError({}, 'operation') mocks.zip_and_send_retry.side_effect = MaxRetriesExceededError filenames = ['2017-01-01/TEST1.PDF'] zip_and_send_letter_pdfs(filenames, 'foo.zip') mocks.send_task.assert_called_once_with( name='update-letter-notifications-to-error', args=(['1', '2', '3'], ), queue='notify-internal-tasks')
def test_zip_and_send_should_set_to_error_after_max_retries_if_celery_hits_soft_time_limit( mocks): mocks.file_exists_on_s3.side_effect = SoftTimeLimitExceeded mocks.zip_and_send_retry.side_effect = MaxRetriesExceededError filenames = ['2017-01-01/TEST1.PDF'] zip_and_send_letter_pdfs(filenames, 'foo.zip') mocks.send_task.assert_called_once_with( name='update-letter-notifications-to-error', args=(['1', '2', '3'], ), queue='notify-internal-tasks')
def test_zip_and_send_should_update_notifications_to_success_in_1k_batches( mocker, mocks): mocker.patch( 'app.celery.tasks.get_notification_references_from_s3_filenames', return_value=list(range(10000))) filenames = ['2017-01-01/TEST1.PDF'] zip_and_send_letter_pdfs(filenames, 'foo.zip') assert mocks.send_task.call_count == 10 assert mocks.send_task.mock_calls[0][2]['args'] == (list(range(1000)), ) assert mocks.send_task.mock_calls[1][2]['args'] == (list(range(1000, 2000)), ) assert mocks.send_task.mock_calls[2][2]['args'] == (list(range(2000, 3000)), )
def test_zip_and_send_should_set_to_error_after_max_retries_after_files_did_not_upload( mocks): mocks.send_zip.side_effect = FtpException mocks.file_exists_with_correct_size.side_effect = FtpException mocks.zip_and_send_retry.side_effect = MaxRetriesExceededError filenames = ['2017-01-01/TEST1.PDF'] zip_and_send_letter_pdfs(filenames, 'foo.zip') mocks.file_exists_with_correct_size.assert_called_once_with('foo.zip', 2) mocks.send_task.assert_called_once_with( name='update-letter-notifications-to-error', args=(['1', '2', '3'], ), queue='notify-internal-tasks')
def test_zip_and_send_should_skip_if_record_already_in_zips_sent( mocks, caplog): mocks.file_exists_on_s3.return_value = True filenames = ['2017-01-01/TEST1.PDF'] zip_and_send_letter_pdfs(filenames, 'foo.zip') mocks.file_exists_on_s3.assert_called_once_with( current_app.config['LETTERS_PDF_BUCKET_NAME'], '2017-01-01/zips_sent/foo.zip.TXT', ) # no update notification tasks should be triggered assert mocks.send_task.call_count == 0 assert any( rec.message == '2017-01-01/zips_sent/foo.zip.TXT already exists in S3, skipping DVLA upload' for rec in caplog.records)
def test_should_send_zip_file(mocks): filenames = ['2017-01-01/TEST1.PDF'] zip_and_send_letter_pdfs(filenames, 'foo.zip') mocks.send_zip.assert_called_once_with(b'\x00\x01', 'foo.zip')
def test_should_get_zip_of_letter_pdfs_from_s3(mocks): filenames = ['2017-01-01/TEST1.PDF', '2017-01-01/TEST2.PDF'] zip_and_send_letter_pdfs(filenames, 'foo.zip') mocks.get_zip_of_letter_pdfs_from_s3.assert_called_once_with(filenames)