def test_get_s3_file_makes_correct_call(notify_api, mocker):
get_s3_mock = mocker.patch('app.aws.s3.get_s3_object')
get_s3_file('foo-bucket', 'bar-file.txt')
get_s3_mock.assert_called_with(
'foo-bucket',
'bar-file.txt'
)
def update_letter_notifications_statuses(self, filename):
bucket_location = '{}-ftp'.format(current_app.config['NOTIFY_EMAIL_DOMAIN'])
response_file_content = s3.get_s3_file(bucket_location, filename)
sorted_letter_counts = defaultdict(int)
try:
notification_updates = process_updates_from_file(response_file_content)
except TypeError:
raise DVLAException('DVLA response file: {} has an invalid format'.format(filename))
else:
temporary_failures = []
for update in notification_updates:
check_billable_units(update)
update_letter_notification(filename, temporary_failures, update)
sorted_letter_counts[update.cost_threshold] += 1
try:
if sorted_letter_counts.keys() - {'Unsorted', 'Sorted'}:
unknown_status = sorted_letter_counts.keys() - {'Unsorted', 'Sorted'}
message = 'DVLA response file: {} contains unknown Sorted status {}'.format(
filename, unknown_status
)
raise DVLAException(message)
billing_date = get_billing_date_in_aet_from_filename(filename)
persist_daily_sorted_letter_counts(day=billing_date,
file_name=filename,
sorted_letter_counts=sorted_letter_counts)
finally:
if temporary_failures:
# This will alert Notify that DVLA was unable to deliver the letters, we need to investigate
message = "DVLA response file: {filename} has failed letters with notification.reference {failures}" \
.format(filename=filename, failures=temporary_failures)
raise DVLAException(message)
def letter_raise_alert_if_no_ack_file_for_zip():
# get a list of zip files since yesterday
zip_file_set = set()
for key in s3.get_list_of_files_by_suffix(
bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'],
subfolder=datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent',
suffix='.TXT'):
subname = key.split('/')[-1] # strip subfolder in name
zip_file_set.add(subname.upper().rstrip('.TXT'))
# get acknowledgement file
ack_file_set = set()
yesterday = datetime.now(tz=pytz.utc) - timedelta(
days=1) # AWS datetime format
for key in s3.get_list_of_files_by_suffix(
bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'],
subfolder='root/dispatch',
suffix='.ACK.txt',
last_modified=yesterday):
ack_file_set.add(key)
today_str = datetime.utcnow().strftime('%Y%m%d')
ack_content_set = set()
for key in ack_file_set:
if today_str in key:
content = s3.get_s3_file(
current_app.config['DVLA_RESPONSE_BUCKET_NAME'], key)
for zip_file in content.split('\n'): # each line
s = zip_file.split('|')
ack_content_set.add(s[0].upper())
deskpro_message = "Letter ack file does not contains all zip files sent. " \
"Missing ack for zip files: {}, " \
"pdf bucket: {}, subfolder: {}, " \
"ack bucket: {}".format(str(sorted(zip_file_set - ack_content_set)),
current_app.config['LETTERS_PDF_BUCKET_NAME'],
datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent',
current_app.config['DVLA_RESPONSE_BUCKET_NAME'])
# strip empty element before comparison
ack_content_set.discard('')
zip_file_set.discard('')
if len(zip_file_set - ack_content_set) > 0:
if current_app.config['NOTIFY_ENVIRONMENT'] in [
'live', 'production', 'test'
]:
deskpro_client.create_ticket(subject="Letter acknowledge error",
message=deskpro_message,
ticket_type='alert')
current_app.logger.error(deskpro_message)
if len(ack_content_set - zip_file_set) > 0:
current_app.logger.info(
"letter ack contains zip that is not for today: {}".format(
ack_content_set - zip_file_set))
def get_letter_details_from_zips_sent_file(file_paths):
"""Get notification details from letters listed in zips_sent file(s)
This takes one or more file paths for the zips_sent files in S3 as its parameters, for example:
get-letter-details-from-zips-sent-file '2019-04-01/zips_sent/filename_1' '2019-04-01/zips_sent/filename_2'
"""
rows_from_file = []
for path in file_paths:
file_contents = s3.get_s3_file(
bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'],
file_location=path
)
rows_from_file.extend(json.loads(file_contents))
notification_references = tuple(row[18:34] for row in rows_from_file)
sql = """
SELECT id, service_id, reference, job_id, created_at
FROM notifications
WHERE reference IN :notification_references
ORDER BY service_id, job_id"""
result = db.session.execute(sql, {'notification_references': notification_references}).fetchall()
with open('zips_sent_details.csv', 'w') as csvfile:
csv_writer = csv.writer(csvfile)
csv_writer.writerow(['notification_id', 'service_id', 'reference', 'job_id', 'created_at'])
for row in result:
csv_writer.writerow(row)
def parse_dvla_file(filename):
bucket_location = '{}-ftp'.format(current_app.config['NOTIFY_EMAIL_DOMAIN'])
response_file_content = s3.get_s3_file(bucket_location, filename)
try:
return process_updates_from_file(response_file_content)
except TypeError:
raise DVLAException('DVLA response file: {} has an invalid format'.format(filename))
def get_letter_details_from_zips_sent_file(file_paths):
"""Get notification details from letters listed in zips_sent file(s)
This takes one or more file paths for the zips_sent files in S3 as its parameters, for example:
get-letter-details-from-zips-sent-file '2019-04-01/zips_sent/filename_1' '2019-04-01/zips_sent/filename_2'
"""
rows_from_file = []
for path in file_paths:
file_contents = s3.get_s3_file(
bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'],
file_location=path)
rows_from_file.extend(json.loads(file_contents))
notification_references = tuple(row[18:34] for row in rows_from_file)
get_letters_data_from_references(notification_references)
def test_get_s3_file_makes_correct_call(notify_api, mocker):
get_s3_mock = mocker.patch("app.aws.s3.get_s3_object")
get_s3_file("foo-bucket", "bar-file.txt")
get_s3_mock.assert_called_with("foo-bucket", "bar-file.txt")
