def validate_upload(database):
"""Validate the upload request.
:returns: `None` if the validation succeeds. Otherwise error
response if validation failed.
"""
response = None
# Update used_quota also at the start of the function
# since multiple users might by using the same project
database.user(request.authorization.username).update_used_quota(
current_app.config.get("UPLOAD_PATH"))
# Check that Content-Length header is provided
if request.content_length is None:
response = utils.make_response(400, "Missing Content-Length header")
# Check that Content-Type is supported if the header is provided
content_type = request.content_type
if content_type and content_type not in SUPPORTED_TYPES:
response = utils.make_response(
415, "Unsupported Content-Type: %s" % content_type)
# Check user quota
if request.content_length > current_app.config.get("MAX_CONTENT_LENGTH"):
response = utils.make_response(413, "Max single file size exceeded")
elif _request_exceeds_quota(database):
response = utils.make_response(413, "Quota exceeded")
return response
def upload_archive():
"""Upload and extract the archive at <UPLOAD_PATH>/project.
:returns: HTTP Response
"""
database = db.Database()
response = up.validate_upload(database)
if response:
return response
upload_dir = request.args.get("dir", default=None)
file_path, file_name = utils.get_tmp_upload_path()
# Create directory if it does not exist
if not os.path.exists(file_path):
os.makedirs(file_path)
file_path = safe_join(file_path, file_name)
try:
response = up.save_archive(database, file_path, upload_dir)
except (MemberOverwriteError, up.OverwriteError) as error:
return utils.make_response(409, str(error))
except MemberTypeError as error:
return utils.make_response(415, str(error))
except MemberNameError as error:
return utils.make_response(400, str(error))
except up.QuotaError as error:
return utils.make_response(413, str(error))
return response
def get_path(fpath):
"""Get filepath, name and checksum.
:returns: HTTP Response
"""
username = request.authorization.username
database = db.Database()
project = database.user(username).get_project()
root_upload_path = current_app.config.get("UPLOAD_PATH")
fpath, fname = utils.get_upload_path(project, fpath, root_upload_path)
fpath = os.path.join(fpath, fname)
if os.path.isfile(fpath):
file_path = utils.get_return_path(project, fpath, root_upload_path)
response = jsonify({
"file_path":
file_path,
"metax_identifier":
database.files.get_identifier(fpath),
"md5":
database.checksums.get_checksum(os.path.abspath(fpath)),
"timestamp":
md.iso8601_timestamp(fpath)
})
elif os.path.isdir(fpath):
dir_tree = _get_dir_tree(project, fpath, root_upload_path)
response = jsonify(dict(file_path=dir_tree))
else:
return utils.make_response(404, "File not found")
response.status_code = 200
return response
def upload_file(fpath):
"""Save the uploaded file at <UPLOAD_PATH>/project/fpath.
:returns: HTTP Response
"""
username = request.authorization.username
database = db.Database()
project = database.user(username).get_project()
response = up.validate_upload(database)
if response:
return response
file_path, file_name = utils.get_upload_path(project, fpath)
# Create directory if it does not exist
if not os.path.exists(file_path):
os.makedirs(file_path)
file_path = os.path.join(file_path, file_name)
try:
response = up.save_file(database, project, file_path)
except (up.OverwriteError) as error:
return utils.make_response(409, str(error))
database.user(request.authorization.username).update_used_quota(
current_app.config.get("UPLOAD_PATH"))
return response
def delete_files():
"""Delete all files of a user.
:returns: HTTP Response
"""
username = request.authorization.username
project = db.Database().user(username).get_project()
root_upload_path = current_app.config.get("UPLOAD_PATH")
fpath = safe_join(root_upload_path, secure_filename(project))
if not os.path.exists(fpath):
return utils.make_response(404, "No files found")
task_id = enqueue_background_job(
task_func="upload_rest_api.jobs.files.delete_files",
queue_name=FILES_QUEUE,
username=username,
job_kwargs={
"fpath": fpath,
"username": username
})
polling_url = utils.get_polling_url(TASK_STATUS_API_V1.name, task_id)
response = jsonify({
"file_path": "/",
"message": "Deleting files and metadata",
"polling_url": polling_url,
"status": "pending"
})
location = url_for(TASK_STATUS_API_V1.name + ".task_status",
task_id=task_id)
response.headers[b'Location'] = location
response.status_code = 202
return response
def save_archive(database, fpath, upload_dir):
"""Uploads the archive on disk at fpath by reading
the upload stream in 1MB chunks. Extracts the archive file
and checks that no symlinks are created.
:param database: Database object
:param fpath: Path where to save the file
:param upload_dir: Directory to which the archive is extracted
:returns: HTTP Response
"""
username = request.authorization.username
project = database.user(username).get_project()
dir_path = utils.get_project_path(project)
if upload_dir:
dir_path = safe_join(dir_path, upload_dir)
if os.path.isdir(dir_path):
raise OverwriteError("Directory '%s' already exists" % upload_dir)
os.makedirs(dir_path)
_save_stream(fpath)
# If zip or tar file was uploaded, extract all files
if zipfile.is_zipfile(fpath) or tarfile.is_tarfile(fpath):
# Check the uncompressed size
quota, used_quota, extracted_size = _check_extraction_size(
database, fpath, username)
if quota - used_quota - extracted_size < 0:
# Remove the archive and raise an exception
os.remove(fpath)
raise QuotaError("Quota exceeded")
database.user(username).set_used_quota(used_quota + extracted_size)
task_id = enqueue_background_job(
task_func="upload_rest_api.jobs.upload.extract_task",
queue_name=UPLOAD_QUEUE,
username=username,
job_kwargs={
"fpath": fpath,
"dir_path": dir_path
})
polling_url = utils.get_polling_url(TASK_STATUS_API_V1.name, task_id)
response = jsonify({
"file_path": "/",
"message": "Uploading archive",
"polling_url": polling_url,
"status": "pending"
})
location = url_for(TASK_STATUS_API_V1.name + ".task_status",
task_id=task_id)
response.headers[b'Location'] = location
response.status_code = 202
else:
os.remove(fpath)
response = utils.make_response(
400, "Uploaded file is not a supported archive")
return response
def get_files():
"""Get all files of the user.
:return: HTTP Response
"""
username = request.authorization.username
project = db.Database().user(username).get_project()
root_upload_path = current_app.config.get("UPLOAD_PATH")
fpath = safe_join(root_upload_path, secure_filename(project))
if not os.path.exists(fpath):
return utils.make_response(404, "No files found")
response = jsonify(_get_dir_tree(project, fpath, root_upload_path))
response.status_code = 200
return response
def http_error_generic(error):
"""Generic HTTP error handler."""
current_app.logger.error(error, exc_info=True)
code = error.code
message = "Page not found" if code == 404 else six.text_type(error)
return utils.make_response(code, message)
def http_error_500(error):
"""Error handler for status code 500."""
current_app.logger.error(error, exc_info=True)
return utils.make_response(500, "Internal server error")
def delete_path(fpath):
"""Delete fpath under user's project.
If fpath resolves to a directory, the whole directory is recursively
removed.
:returns: HTTP Response
"""
root_upload_path = current_app.config.get("UPLOAD_PATH")
username = request.authorization.username
database = db.Database()
project = database.user(username).get_project()
fpath, fname = utils.get_upload_path(project, fpath)
fpath = os.path.join(fpath, fname)
if os.path.isfile(fpath):
# Remove metadata from Metax
try:
response = md.MetaxClient().delete_file_metadata(
project, fpath, root_upload_path)
except md.MetaxClientError as exception:
response = str(exception)
# Remove checksum from mongo
database.checksums.delete_one(os.path.abspath(fpath))
os.remove(fpath)
elif os.path.isdir(fpath):
# Remove all file metadata of files under dir fpath from Metax
task_id = enqueue_background_job(
task_func="upload_rest_api.jobs.files.delete_files",
queue_name=FILES_QUEUE,
username=username,
job_kwargs={
"fpath": fpath,
"username": username
})
polling_url = utils.get_polling_url(TASK_STATUS_API_V1.name, task_id)
response = jsonify({
"file_path":
fpath[len(os.path.join(root_upload_path, project)):],
"message":
"Deleting files and metadata",
"polling_url":
polling_url,
"status":
"pending"
})
location = url_for(TASK_STATUS_API_V1.name + ".task_status",
task_id=task_id)
response.headers[b'Location'] = location
response.status_code = 202
return response
else:
return utils.make_response(404, "File not found")
database.user(username).update_used_quota(root_upload_path)
response = jsonify({
"file_path":
utils.get_return_path(project, fpath, root_upload_path),
"message":
"deleted",
"metax":
response
})
response.status_code = 200
return response