def test_file_exists_when_file_is_there(inspire_app, s3):
expected_result = True
create_s3_bucket(KEY)
create_s3_file(current_s3_instance.get_bucket_for_file_key(KEY), KEY,
"this is my data")
result = current_s3_instance.file_exists(KEY)
assert result == expected_result
def add_file(
app_context,
control_number,
uuid,
url,
original_url=None,
key=None,
filename=None,
*args,
**kwargs,
):
"""Adds files to s3.
Args:
app_context: Original app context should be passed here if running in separate thread
"""
with app_context.app.app_context():
if current_s3_instance.is_s3_url(
url) and not current_app.config.get(
"UPDATE_S3_FILES_METADATA", False):
result = {}
if key not in url:
filename = filename or key
key = url.split("/")[-1]
result.update({"key": key, "filename": filename})
LOGGER.info(
"File already on S3 - Skipping",
url=url,
key=key,
recid=control_number,
uuid=uuid,
thread=threading.get_ident(),
)
return result
file_data = download_file_from_url(url)
new_key = hash_data(file_data)
mimetype = magic.from_buffer(file_data, mime=True)
file_data = BytesIO(file_data)
filename = filename or key
acl = current_app.config["S3_FILE_ACL"]
if current_s3_instance.file_exists(new_key):
LOGGER.info(
"Replacing file metadata",
key=new_key,
recid=control_number,
uuid=uuid,
thread=threading.get_ident(),
)
current_s3_instance.replace_file_metadata(
new_key, filename, mimetype, acl)
else:
LOGGER.info(
"Uploading file to s3",
key=new_key,
recid=control_number,
uuid=uuid,
thread=threading.get_ident(),
)
current_s3_instance.upload_file(file_data, new_key, filename,
mimetype, acl)
result = {
"key": new_key,
"filename": filename,
"url": current_s3_instance.get_file_url(new_key),
}
if (url.startswith("http")
and not current_s3_instance.is_s3_url(url)
and not original_url):
result["original_url"] = url
return result
def add_file(
app_context,
url,
original_url=None,
key=None,
filename=None,
*args,
**kwargs,
):
"""Adds files to s3.
Args:
app_context: Original app context should be passed here if running in separate thread
"""
with app_context.app.app_context():
is_s3_or_public_url = current_s3_instance.is_s3_url_with_bucket_prefix(
url) or current_s3_instance.is_public_url(url)
if is_s3_or_public_url and not current_app.config.get(
"UPDATE_S3_FILES_METADATA", False):
result = {}
if key not in url:
filename = filename or key
key = url.split("/")[-1]
result.update({"key": key, "filename": filename})
if current_s3_instance.is_s3_url(url):
url = current_s3_instance.get_public_url(key)
result.update({"url": url})
LOGGER.info(
"File already on S3 - Skipping",
url=url,
key=key,
thread=threading.get_ident(),
)
return result
file_data = download_file_from_url(url)
new_key = hash_data(file_data)
mimetype = magic.from_buffer(file_data, mime=True)
file_data = BytesIO(file_data)
filename = filename or key
if not filename:
filename = new_key
if mimetype in current_app.config.get(
"FILES_RESTRICTED_MIMETYPES"):
LOGGER.error(
"Unsupported file type - Aborting",
key=key,
mimetype=mimetype,
thread=threading.get_ident(),
)
raise UnsupportedFileError(mimetype)
acl = current_app.config["S3_FILE_ACL"]
if current_s3_instance.file_exists(new_key):
LOGGER.info(
"Replacing file metadata",
key=new_key,
thread=threading.get_ident(),
)
current_s3_instance.replace_file_metadata(
new_key, filename, mimetype, acl)
else:
LOGGER.info(
"Uploading file to s3",
key=new_key,
thread=threading.get_ident(),
)
current_s3_instance.upload_file(file_data, new_key, filename,
mimetype, acl)
result = {
"key": new_key,
"filename": filename,
"url": current_s3_instance.get_public_url(new_key),
}
if (url.startswith("http")
and not current_s3_instance.is_s3_url(url)
and not current_s3_instance.is_public_url(url)
and not original_url):
result["original_url"] = url
return result
def test_file_exists_when_error_occurs(mock_client_head_object, inspire_app,
s3):
with pytest.raises(ClientError):
current_s3_instance.file_exists(KEY)
def test_file_exists_when_file_is_missing(inspire_app, s3):
expected_result = False
create_s3_bucket(KEY)
result = current_s3_instance.file_exists(KEY)
assert result == expected_result