def upload():
"""Upload file to S3."""
if "file" not in request.files:
return jsonify(success=False, message="File key is missing."), 400
file_data = request.files["file"]
filename = file_data.filename
mime_type = file_data.mimetype
_, extension = splitext(filename)
if extension not in current_app.config["EDITOR_UPLOAD_ALLOWED_EXTENSIONS"]:
return (
jsonify(success=False,
message=f"File extension '{extension}' is not supported."),
400,
)
key = hash_data(file_data.read())
bucket = current_app.config.get("S3_EDITOR_BUCKET")
file_data.seek(0)
current_s3_instance.upload_file(file_data, key, filename, mime_type,
current_app.config["S3_FILE_ACL"], bucket)
file_url = current_s3_instance.get_s3_url(key, bucket)
return jsonify({"path": file_url}), 200
def test_upload_file(inspire_app, s3):
create_s3_bucket(KEY)
filename = "file.txt"
mimetype = "text/*"
acl = "public-read"
expected_content = "This is a demo file\n"
record_fixture_path = pkg_resources.resource_filename(
__name__, os.path.join("fixtures", "file.txt")
)
with open(record_fixture_path, "rb") as data:
current_s3_instance.upload_file(data, KEY, filename, mimetype, acl)
result = current_s3_instance.client.head_object(
Bucket=current_s3_instance.get_bucket_for_file_key(KEY), Key=KEY
)
assert result["ContentDisposition"] == f'inline; filename="{filename}"'
assert result["ContentType"] == mimetype
content = (
current_s3_instance.resource.Object(
current_s3_instance.get_bucket_for_file_key(KEY), KEY
)
.get()["Body"]
.read()
.decode("utf-8")
)
assert content == expected_content
def write_sitemap_page_content(page, page_content):
bucket = current_app.config["S3_SITEMAP_BUCKET"]
file_data = page_content.encode("utf8")
file_data = BytesIO(file_data)
filename = get_sitemap_page_filename(page)
current_s3_instance.upload_file(
file_data,
filename,
filename,
SITEMAP_MIME_TYPE,
current_app.config["S3_FILE_ACL"],
bucket,
)
def generate_bibliography():
content_length = request.content_length
if content_length is not None and content_length > 10 * 1024 * 1024:
raise FileTooBigError
requested_format = request.args.get("format")
bytes_file = request.files["file"]
if not allowed_file(bytes_file.filename):
raise FileFormatNotSupportedError
text_file = TextIOWrapper(bytes_file, errors="ignore")
reference_names = get_references(text_file)
references, errors = find_references(reference_names, requested_format)
if not references:
raise NoReferencesFoundError
file_data = "\n\n".join(references).encode("utf-8")
key = hash_data(file_data)
file_data = BytesIO(file_data)
filename = get_filename(bytes_file.filename, requested_format)
mime_type = get_mimetype(requested_format)
bucket = current_app.config.get("S3_BIBLIOGRAPHY_GENERATOR_BUCKET")
current_s3_instance.upload_file(file_data, key, filename, mime_type,
current_app.config["S3_FILE_ACL"], bucket)
download_url = current_s3_instance.get_s3_url(key, bucket)
formatted_errors = []
for error in errors:
if error["type"] == "ambiguous":
formatted_errors.append({
"message":
f"Ambiguous reference to {error['ref']} on line {error['line']}"
})
else:
formatted_errors.append({
"message":
f"Reference to {error['ref']} not found on line {error['line']}"
})
response = {
"data": {
"download_url": download_url
},
"errors": formatted_errors
}
return jsonify(response)
def add_file(
app_context,
control_number,
uuid,
url,
original_url=None,
key=None,
filename=None,
*args,
**kwargs,
):
"""Adds files to s3.
Args:
app_context: Original app context should be passed here if running in separate thread
"""
with app_context.app.app_context():
if current_s3_instance.is_s3_url(
url) and not current_app.config.get(
"UPDATE_S3_FILES_METADATA", False):
result = {}
if key not in url:
filename = filename or key
key = url.split("/")[-1]
result.update({"key": key, "filename": filename})
LOGGER.info(
"File already on S3 - Skipping",
url=url,
key=key,
recid=control_number,
uuid=uuid,
thread=threading.get_ident(),
)
return result
file_data = download_file_from_url(url)
new_key = hash_data(file_data)
mimetype = magic.from_buffer(file_data, mime=True)
file_data = BytesIO(file_data)
filename = filename or key
acl = current_app.config["S3_FILE_ACL"]
if current_s3_instance.file_exists(new_key):
LOGGER.info(
"Replacing file metadata",
key=new_key,
recid=control_number,
uuid=uuid,
thread=threading.get_ident(),
)
current_s3_instance.replace_file_metadata(
new_key, filename, mimetype, acl)
else:
LOGGER.info(
"Uploading file to s3",
key=new_key,
recid=control_number,
uuid=uuid,
thread=threading.get_ident(),
)
current_s3_instance.upload_file(file_data, new_key, filename,
mimetype, acl)
result = {
"key": new_key,
"filename": filename,
"url": current_s3_instance.get_file_url(new_key),
}
if (url.startswith("http")
and not current_s3_instance.is_s3_url(url)
and not original_url):
result["original_url"] = url
return result
def add_file(
app_context,
url,
original_url=None,
key=None,
filename=None,
*args,
**kwargs,
):
"""Adds files to s3.
Args:
app_context: Original app context should be passed here if running in separate thread
"""
with app_context.app.app_context():
is_s3_or_public_url = current_s3_instance.is_s3_url_with_bucket_prefix(
url) or current_s3_instance.is_public_url(url)
if is_s3_or_public_url and not current_app.config.get(
"UPDATE_S3_FILES_METADATA", False):
result = {}
if key not in url:
filename = filename or key
key = url.split("/")[-1]
result.update({"key": key, "filename": filename})
if current_s3_instance.is_s3_url(url):
url = current_s3_instance.get_public_url(key)
result.update({"url": url})
LOGGER.info(
"File already on S3 - Skipping",
url=url,
key=key,
thread=threading.get_ident(),
)
return result
file_data = download_file_from_url(url)
new_key = hash_data(file_data)
mimetype = magic.from_buffer(file_data, mime=True)
file_data = BytesIO(file_data)
filename = filename or key
if not filename:
filename = new_key
if mimetype in current_app.config.get(
"FILES_RESTRICTED_MIMETYPES"):
LOGGER.error(
"Unsupported file type - Aborting",
key=key,
mimetype=mimetype,
thread=threading.get_ident(),
)
raise UnsupportedFileError(mimetype)
acl = current_app.config["S3_FILE_ACL"]
if current_s3_instance.file_exists(new_key):
LOGGER.info(
"Replacing file metadata",
key=new_key,
thread=threading.get_ident(),
)
current_s3_instance.replace_file_metadata(
new_key, filename, mimetype, acl)
else:
LOGGER.info(
"Uploading file to s3",
key=new_key,
thread=threading.get_ident(),
)
current_s3_instance.upload_file(file_data, new_key, filename,
mimetype, acl)
result = {
"key": new_key,
"filename": filename,
"url": current_s3_instance.get_public_url(new_key),
}
if (url.startswith("http")
and not current_s3_instance.is_s3_url(url)
and not current_s3_instance.is_public_url(url)
and not original_url):
result["original_url"] = url
return result