def test_download_file_from_url_fails(inspire_app):
url = "https://inspirehep.net/record/1759380/files/channelxi3.png"
with requests_mock.Mocker() as mocker:
mocker.get(
"https://inspirehep.net/record/1759380/files/channelxi3.png",
status_code=404,
)
with pytest.raises(DownloadFileError):
download_file_from_url(url)
def test_download_file_from_url_with_full_url(inspire_app):
url = "https://inspirehep.net/record/1759380/files/channelxi3.png"
expected_content = b"This is the file data"
with requests_mock.Mocker() as mocker:
mocker.get(
"https://inspirehep.net/record/1759380/files/channelxi3.png",
status_code=200,
content=expected_content,
)
result = download_file_from_url(url)
assert result == expected_content
def test_download_file_from_url_with_relative_url(inspire_app):
url = "/record/1759380/files/channelxi3.png"
expected_content = b"This is the file data"
with requests_mock.Mocker() as mocker:
mocker.get(
"http://localhost:5000/record/1759380/files/channelxi3.png",
status_code=200,
content=expected_content,
)
result = download_file_from_url(url)
assert result == expected_content
def add_file(
app_context,
control_number,
uuid,
url,
original_url=None,
key=None,
filename=None,
*args,
**kwargs,
):
"""Adds files to s3.
Args:
app_context: Original app context should be passed here if running in separate thread
"""
with app_context.app.app_context():
if current_s3_instance.is_s3_url(
url) and not current_app.config.get(
"UPDATE_S3_FILES_METADATA", False):
result = {}
if key not in url:
filename = filename or key
key = url.split("/")[-1]
result.update({"key": key, "filename": filename})
LOGGER.info(
"File already on S3 - Skipping",
url=url,
key=key,
recid=control_number,
uuid=uuid,
thread=threading.get_ident(),
)
return result
file_data = download_file_from_url(url)
new_key = hash_data(file_data)
mimetype = magic.from_buffer(file_data, mime=True)
file_data = BytesIO(file_data)
filename = filename or key
acl = current_app.config["S3_FILE_ACL"]
if current_s3_instance.file_exists(new_key):
LOGGER.info(
"Replacing file metadata",
key=new_key,
recid=control_number,
uuid=uuid,
thread=threading.get_ident(),
)
current_s3_instance.replace_file_metadata(
new_key, filename, mimetype, acl)
else:
LOGGER.info(
"Uploading file to s3",
key=new_key,
recid=control_number,
uuid=uuid,
thread=threading.get_ident(),
)
current_s3_instance.upload_file(file_data, new_key, filename,
mimetype, acl)
result = {
"key": new_key,
"filename": filename,
"url": current_s3_instance.get_file_url(new_key),
}
if (url.startswith("http")
and not current_s3_instance.is_s3_url(url)
and not original_url):
result["original_url"] = url
return result
def add_file(
app_context,
url,
original_url=None,
key=None,
filename=None,
*args,
**kwargs,
):
"""Adds files to s3.
Args:
app_context: Original app context should be passed here if running in separate thread
"""
with app_context.app.app_context():
is_s3_or_public_url = current_s3_instance.is_s3_url_with_bucket_prefix(
url) or current_s3_instance.is_public_url(url)
if is_s3_or_public_url and not current_app.config.get(
"UPDATE_S3_FILES_METADATA", False):
result = {}
if key not in url:
filename = filename or key
key = url.split("/")[-1]
result.update({"key": key, "filename": filename})
if current_s3_instance.is_s3_url(url):
url = current_s3_instance.get_public_url(key)
result.update({"url": url})
LOGGER.info(
"File already on S3 - Skipping",
url=url,
key=key,
thread=threading.get_ident(),
)
return result
file_data = download_file_from_url(url)
new_key = hash_data(file_data)
mimetype = magic.from_buffer(file_data, mime=True)
file_data = BytesIO(file_data)
filename = filename or key
if not filename:
filename = new_key
if mimetype in current_app.config.get(
"FILES_RESTRICTED_MIMETYPES"):
LOGGER.error(
"Unsupported file type - Aborting",
key=key,
mimetype=mimetype,
thread=threading.get_ident(),
)
raise UnsupportedFileError(mimetype)
acl = current_app.config["S3_FILE_ACL"]
if current_s3_instance.file_exists(new_key):
LOGGER.info(
"Replacing file metadata",
key=new_key,
thread=threading.get_ident(),
)
current_s3_instance.replace_file_metadata(
new_key, filename, mimetype, acl)
else:
LOGGER.info(
"Uploading file to s3",
key=new_key,
thread=threading.get_ident(),
)
current_s3_instance.upload_file(file_data, new_key, filename,
mimetype, acl)
result = {
"key": new_key,
"filename": filename,
"url": current_s3_instance.get_public_url(new_key),
}
if (url.startswith("http")
and not current_s3_instance.is_s3_url(url)
and not current_s3_instance.is_public_url(url)
and not original_url):
result["original_url"] = url
return result