def upload_letter_to_s3(data,
*,
file_location,
status,
page_count,
filename,
message=None,
invalid_pages=None,
recipient=None):
# Use of urllib.parse.quote encodes metadata into ascii, which is required by s3.
# Making sure data for displaying to users is decoded is taken care of by LetterMetadata
metadata = {
'status': status,
'page_count': str(page_count),
'filename': urllib.parse.quote(filename),
}
if message:
metadata['message'] = message
if invalid_pages:
metadata['invalid_pages'] = json.dumps(invalid_pages)
if recipient:
metadata['recipient'] = urllib.parse.quote(recipient)
utils_s3upload(
filedata=data,
region=current_app.config['AWS_REGION'],
bucket_name=current_app.config['TRANSIENT_UPLOADED_LETTERS'],
file_location=file_location,
metadata=metadata,
)
def upload_letter_to_s3(data,
*,
file_location,
status,
page_count,
filename,
message=None,
invalid_pages=None,
recipient=None):
metadata = {
'status': status,
'page_count': str(page_count),
'filename': filename,
}
if message:
metadata['message'] = message
if invalid_pages:
metadata['invalid_pages'] = json.dumps(invalid_pages)
if recipient:
metadata['recipient'] = format_recipient(recipient)
utils_s3upload(
filedata=data,
region=current_app.config['AWS_REGION'],
bucket_name=current_app.config['TRANSIENT_UPLOADED_LETTERS'],
file_location=file_location,
metadata=metadata,
)
def upload_letter_png_logo(filename, filedata, region):
bucket_name = current_app.config['LOGO_UPLOAD_BUCKET_NAME']
utils_s3upload(filedata=filedata,
region=region,
bucket_name=bucket_name,
file_location=filename,
content_type='image/png')
def s3upload(service_id, filedata, region):
upload_id = str(uuid.uuid4())
upload_file_name = FILE_LOCATION_STRUCTURE.format(service_id, upload_id)
utils_s3upload(filedata=filedata['data'],
region=region,
bucket_name=current_app.config['CSV_UPLOAD_BUCKET_NAME'],
file_location=upload_file_name)
return upload_id
def s3upload(service_id, filedata, region):
upload_id = str(uuid.uuid4())
bucket_name, file_location = get_csv_location(service_id, upload_id)
utils_s3upload(
filedata=filedata['data'],
region=region,
bucket_name=bucket_name,
file_location=file_location,
)
return upload_id
def upload_job_to_s3(service_id, file_data):
upload_id = str(uuid.uuid4())
bucket, location = get_job_location(service_id, upload_id)
utils_s3upload(
filedata=file_data,
region=current_app.config["AWS_REGION"],
bucket_name=bucket,
file_location=location,
)
return upload_id
def s3upload(service_id, filedata, region):
client = clients.csv
upload_id = str(uuid.uuid4())
upload_file_name = FILE_LOCATION_STRUCTURE.format(service_id, upload_id)
utils_s3upload(filedata=filedata['data'],
region=region,
bucket_name=client.bucket_name,
file_location=upload_file_name,
session=client.session)
return upload_id
def upload_letter_temp_logo(filename, filedata, region, user_id):
upload_filename = LETTER_TEMP_LOGO_LOCATION.format(user_id=user_id, unique_id=str(uuid.uuid4()), filename=filename)
bucket_name = current_app.config["LOGO_UPLOAD_BUCKET_NAME"]
utils_s3upload(
filedata=filedata,
region=region,
bucket_name=bucket_name,
file_location=upload_filename,
content_type="image/svg+xml",
)
return upload_filename
def upload_email_logo(filename, filedata, region, user_id):
upload_file_name = EMAIL_LOGO_LOCATION_STRUCTURE.format(
temp=TEMP_TAG.format(user_id=user_id),
unique_id=str(uuid.uuid4()),
filename=filename)
bucket_name = current_app.config['LOGO_UPLOAD_BUCKET_NAME']
utils_s3upload(filedata=filedata,
region=region,
bucket_name=bucket_name,
file_location=upload_file_name,
content_type='image/png')
return upload_file_name
def zip_and_send_letter_pdfs(filenames_to_zip, upload_filename):
folder_date = filenames_to_zip[0].split('/')[0]
zips_sent_filename = '{}/zips_sent/{}.TXT'.format(folder_date,
upload_filename)
current_app.logger.info(
"Starting to zip {file_count} letter PDFs in memory from {folder} into dvla file {upload_filename}"
.format( # noqa
file_count=len(filenames_to_zip),
folder=folder_date,
upload_filename=upload_filename))
try:
if file_exists_on_s3(current_app.config['LETTERS_PDF_BUCKET_NAME'],
zips_sent_filename):
current_app.logger.warning(
'{} already exists in S3, skipping DVLA upload'.format(
zips_sent_filename))
return
zip_data = get_zip_of_letter_pdfs_from_s3(filenames_to_zip)
ftp_client.send_zip(zip_data, upload_filename)
# upload a record to s3 of each zip file we send to DVLA - this is just a list of letter filenames so we can
# match up their references with DVLA
utils_s3upload(
filedata=json.dumps(filenames_to_zip).encode(),
region=current_app.config['AWS_REGION'],
bucket_name=current_app.config['LETTERS_PDF_BUCKET_NAME'],
file_location=zips_sent_filename)
except ClientError:
current_app.logger.exception(
'FTP app failed to download PDF from S3 bucket {}'.format(
folder_date))
task_name = "update-letter-notifications-to-error"
except FtpException:
try:
# check if file exists with the right size.
# It has happened that an IOError occurs but the files are present on the remote server.
ftp_client.file_exists_with_correct_size(upload_filename,
len(zip_data))
task_name = "update-letter-notifications-to-sent"
except FtpException:
current_app.logger.exception('FTP app failed to send api messages')
task_name = "update-letter-notifications-to-error"
else:
task_name = "update-letter-notifications-to-sent"
refs = get_notification_references_from_s3_filenames(filenames_to_zip)
update_notifications(task_name, refs)
def upload_logo(filename, filedata, region, user_id):
client = clients.logo
upload_file_name = LOGO_LOCATION_STRUCTURE.format(
temp=TEMP_TAG.format(user_id=user_id),
unique_id=str(uuid.uuid4()),
filename=filename)
utils_s3upload(filedata=filedata,
region=region,
bucket_name=client.bucket_name,
file_location=upload_file_name,
content_type='image/png',
acl='public-read',
session=client.session)
return upload_file_name
