def test_get_s3_file_makes_correct_call(notify_api, mocker): get_s3_mock = mocker.patch('app.aws.s3.get_s3_object') get_s3_file('foo-bucket', 'bar-file.txt') get_s3_mock.assert_called_with( 'foo-bucket', 'bar-file.txt' )
def update_letter_notifications_statuses(self, filename): bucket_location = '{}-ftp'.format(current_app.config['NOTIFY_EMAIL_DOMAIN']) response_file_content = s3.get_s3_file(bucket_location, filename) sorted_letter_counts = defaultdict(int) try: notification_updates = process_updates_from_file(response_file_content) except TypeError: raise DVLAException('DVLA response file: {} has an invalid format'.format(filename)) else: temporary_failures = [] for update in notification_updates: check_billable_units(update) update_letter_notification(filename, temporary_failures, update) sorted_letter_counts[update.cost_threshold] += 1 try: if sorted_letter_counts.keys() - {'Unsorted', 'Sorted'}: unknown_status = sorted_letter_counts.keys() - {'Unsorted', 'Sorted'} message = 'DVLA response file: {} contains unknown Sorted status {}'.format( filename, unknown_status ) raise DVLAException(message) billing_date = get_billing_date_in_aet_from_filename(filename) persist_daily_sorted_letter_counts(day=billing_date, file_name=filename, sorted_letter_counts=sorted_letter_counts) finally: if temporary_failures: # This will alert Notify that DVLA was unable to deliver the letters, we need to investigate message = "DVLA response file: {filename} has failed letters with notification.reference {failures}" \ .format(filename=filename, failures=temporary_failures) raise DVLAException(message)
def letter_raise_alert_if_no_ack_file_for_zip(): # get a list of zip files since yesterday zip_file_set = set() for key in s3.get_list_of_files_by_suffix( bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'], subfolder=datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent', suffix='.TXT'): subname = key.split('/')[-1] # strip subfolder in name zip_file_set.add(subname.upper().rstrip('.TXT')) # get acknowledgement file ack_file_set = set() yesterday = datetime.now(tz=pytz.utc) - timedelta( days=1) # AWS datetime format for key in s3.get_list_of_files_by_suffix( bucket_name=current_app.config['DVLA_RESPONSE_BUCKET_NAME'], subfolder='root/dispatch', suffix='.ACK.txt', last_modified=yesterday): ack_file_set.add(key) today_str = datetime.utcnow().strftime('%Y%m%d') ack_content_set = set() for key in ack_file_set: if today_str in key: content = s3.get_s3_file( current_app.config['DVLA_RESPONSE_BUCKET_NAME'], key) for zip_file in content.split('\n'): # each line s = zip_file.split('|') ack_content_set.add(s[0].upper()) deskpro_message = "Letter ack file does not contains all zip files sent. " \ "Missing ack for zip files: {}, " \ "pdf bucket: {}, subfolder: {}, " \ "ack bucket: {}".format(str(sorted(zip_file_set - ack_content_set)), current_app.config['LETTERS_PDF_BUCKET_NAME'], datetime.utcnow().strftime('%Y-%m-%d') + '/zips_sent', current_app.config['DVLA_RESPONSE_BUCKET_NAME']) # strip empty element before comparison ack_content_set.discard('') zip_file_set.discard('') if len(zip_file_set - ack_content_set) > 0: if current_app.config['NOTIFY_ENVIRONMENT'] in [ 'live', 'production', 'test' ]: deskpro_client.create_ticket(subject="Letter acknowledge error", message=deskpro_message, ticket_type='alert') current_app.logger.error(deskpro_message) if len(ack_content_set - zip_file_set) > 0: current_app.logger.info( "letter ack contains zip that is not for today: {}".format( ack_content_set - zip_file_set))
def get_letter_details_from_zips_sent_file(file_paths): """Get notification details from letters listed in zips_sent file(s) This takes one or more file paths for the zips_sent files in S3 as its parameters, for example: get-letter-details-from-zips-sent-file '2019-04-01/zips_sent/filename_1' '2019-04-01/zips_sent/filename_2' """ rows_from_file = [] for path in file_paths: file_contents = s3.get_s3_file( bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'], file_location=path ) rows_from_file.extend(json.loads(file_contents)) notification_references = tuple(row[18:34] for row in rows_from_file) sql = """ SELECT id, service_id, reference, job_id, created_at FROM notifications WHERE reference IN :notification_references ORDER BY service_id, job_id""" result = db.session.execute(sql, {'notification_references': notification_references}).fetchall() with open('zips_sent_details.csv', 'w') as csvfile: csv_writer = csv.writer(csvfile) csv_writer.writerow(['notification_id', 'service_id', 'reference', 'job_id', 'created_at']) for row in result: csv_writer.writerow(row)
def parse_dvla_file(filename): bucket_location = '{}-ftp'.format(current_app.config['NOTIFY_EMAIL_DOMAIN']) response_file_content = s3.get_s3_file(bucket_location, filename) try: return process_updates_from_file(response_file_content) except TypeError: raise DVLAException('DVLA response file: {} has an invalid format'.format(filename))
def get_letter_details_from_zips_sent_file(file_paths): """Get notification details from letters listed in zips_sent file(s) This takes one or more file paths for the zips_sent files in S3 as its parameters, for example: get-letter-details-from-zips-sent-file '2019-04-01/zips_sent/filename_1' '2019-04-01/zips_sent/filename_2' """ rows_from_file = [] for path in file_paths: file_contents = s3.get_s3_file( bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'], file_location=path) rows_from_file.extend(json.loads(file_contents)) notification_references = tuple(row[18:34] for row in rows_from_file) get_letters_data_from_references(notification_references)
def test_get_s3_file_makes_correct_call(notify_api, mocker): get_s3_mock = mocker.patch("app.aws.s3.get_s3_object") get_s3_file("foo-bucket", "bar-file.txt") get_s3_mock.assert_called_with("foo-bucket", "bar-file.txt")