def letter_raise_alert_if_no_ack_file_for_zip():
# get a list of zip files since yesterday
zip_file_set = set()
for key in s3.get_list_of_files_by_suffix(
bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'],
subfolder=datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent',
suffix='.TXT'):
subname = key.split('/')[-1] # strip subfolder in name
zip_file_set.add(subname.upper().rstrip('.TXT'))
# get acknowledgement file
ack_file_set = set()
yesterday = datetime.now(tz=pytz.utc) - timedelta(
days=1) # AWS datetime format
for key in s3.get_list_of_files_by_suffix(
bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'],
subfolder='root/dispatch',
suffix='.ACK.txt',
last_modified=yesterday):
ack_file_set.add(key)
today_str = datetime.utcnow().strftime('%Y%m%d')
ack_content_set = set()
for key in ack_file_set:
if today_str in key:
content = s3.get_s3_file(
current_app.config['DVLA_RESPONSE_BUCKET_NAME'], key)
for zip_file in content.split('\n'): # each line
s = zip_file.split('|')
ack_content_set.add(s[0].upper())
deskpro_message = "Letter ack file does not contains all zip files sent. " \
"Missing ack for zip files: {}, " \
"pdf bucket: {}, subfolder: {}, " \
"ack bucket: {}".format(str(sorted(zip_file_set - ack_content_set)),
current_app.config['LETTERS_PDF_BUCKET_NAME'],
datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent',
current_app.config['DVLA_RESPONSE_BUCKET_NAME'])
# strip empty element before comparison
ack_content_set.discard('')
zip_file_set.discard('')
if len(zip_file_set - ack_content_set) > 0:
if current_app.config['NOTIFY_ENVIRONMENT'] in [
'live', 'production', 'test'
]:
deskpro_client.create_ticket(subject="Letter acknowledge error",
message=deskpro_message,
ticket_type='alert')
current_app.logger.error(deskpro_message)
if len(ack_content_set - zip_file_set) > 0:
current_app.logger.info(
"letter ack contains zip that is not for today: {}".format(
ack_content_set - zip_file_set))
def test_get_list_of_files_by_suffix(notify_api, mocker, suffix_str,
days_before, returned_no):
paginator_mock = mocker.patch("app.aws.s3.client")
multiple_pages_s3_object = [
{
"Contents": [
single_s3_object_stub("bar/foo.ACK.txt",
datetime_in_past(1, 0)),
]
},
{
"Contents": [
single_s3_object_stub("bar/foo1.rs.txt",
datetime_in_past(2, 0)),
]
},
]
paginator_mock.return_value.get_paginator.return_value.paginate.return_value = multiple_pages_s3_object
if days_before:
key = get_list_of_files_by_suffix(
"foo-bucket",
subfolder="bar",
suffix=suffix_str,
last_modified=datetime.now(tz=pytz.utc) -
timedelta(days=days_before),
)
else:
key = get_list_of_files_by_suffix("foo-bucket",
subfolder="bar",
suffix=suffix_str)
assert sum(1 for x in key) == returned_no
for k in key:
assert k == "bar/foo.ACK.txt"
def letter_raise_alert_if_no_ack_file_for_zip():
# get a list of zip files since yesterday
zip_file_set = set()
today_str = datetime.utcnow().strftime("%Y-%m-%d")
yesterday = datetime.now(tz=pytz.utc) - timedelta(
days=1) # AWS datetime format
for key in s3.get_list_of_files_by_suffix(
bucket_name=current_app.config["LETTERS_PDF_BUCKET_NAME"],
subfolder=today_str + "/zips_sent",
suffix=".TXT",
):
subname = key.split("/")[-1] # strip subfolder in name
zip_file_set.add(subname.upper().replace(".ZIP.TXT", ""))
# get acknowledgement file
ack_file_set = set()
for key in s3.get_list_of_files_by_suffix(
bucket_name=current_app.config["DVLA_RESPONSE_BUCKET_NAME"],
subfolder="root/dispatch",
suffix=".ACK.txt",
last_modified=yesterday,
):
ack_file_set.add(
key.lstrip("root/dispatch").upper().replace(".ACK.TXT", ""))
message = ("Letter ack file does not contain all zip files sent. "
"Missing ack for zip files: {}, "
"pdf bucket: {}, subfolder: {}, "
"ack bucket: {}").format(
str(sorted(zip_file_set - ack_file_set)),
current_app.config["LETTERS_PDF_BUCKET_NAME"],
datetime.utcnow().strftime("%Y-%m-%d") + "/zips_sent",
current_app.config["DVLA_RESPONSE_BUCKET_NAME"],
)
# strip empty element before comparison
ack_file_set.discard("")
zip_file_set.discard("")
if len(zip_file_set - ack_file_set) > 0:
if current_app.config["NOTIFY_ENVIRONMENT"] in [
"live", "production", "test"
]:
zendesk_client.create_ticket(
subject="Letter acknowledge error",
message=message,
ticket_type=zendesk_client.TYPE_INCIDENT,
)
current_app.logger.error(message)
if len(ack_file_set - zip_file_set) > 0:
current_app.logger.info(
"letter ack contains zip that is not for today: {}".format(
ack_file_set - zip_file_set))
def replay_daily_sorted_count_files(file_extension):
bucket_location = '{}-ftp'.format(current_app.config['NOTIFY_EMAIL_DOMAIN'])
for filename in s3.get_list_of_files_by_suffix(bucket_name=bucket_location,
subfolder='root/dispatch',
suffix=file_extension or '.rs.txt'):
print("Create task to record daily sorted counts for file: ", filename)
record_daily_sorted_counts.apply_async([filename], queue=QueueNames.NOTIFY)
def test_get_list_of_files_by_suffix_empty_contents_return_with_no_error(notify_api, mocker):
paginator_mock = mocker.patch('app.aws.s3.client')
multiple_pages_s3_object = [
{
"other_content": [
'some_values',
]
}
]
paginator_mock.return_value.get_paginator.return_value.paginate.return_value = multiple_pages_s3_object
key = get_list_of_files_by_suffix('foo-bucket', subfolder='bar', suffix='.pdf')
assert sum(1 for x in key) == 0
