def setUp(self):
super(UploadUtilsTests, self).setUp()
self.request_id = 'FOIL-TEST-UUT'
self.filename = "iamafile.txt"
self.quarantine_basepath = os.path.join(
current_app.config['UPLOAD_QUARANTINE_DIRECTORY'],
self.request_id
)
self.quarantine_path = os.path.join(
self.quarantine_basepath,
self.filename
)
self.upload_basepath = os.path.join(
current_app.config['UPLOAD_DIRECTORY'],
self.request_id,
)
self.upload_path = os.path.join(
self.upload_basepath,
self.filename
)
self.update_path = os.path.join(
self.upload_basepath,
UPDATED_FILE_DIRNAME,
self.filename
)
os.mkdir(self.quarantine_basepath)
os.mkdir(self.upload_basepath)
open(self.quarantine_path, 'w').close()
self.key = get_upload_key(self.request_id, self.filename)
self.key_update = get_upload_key(
self.request_id, self.filename, for_update=True)
def setUp(self):
super(UploadViewsTests, self).setUp()
self.filename = "$ome fi/le.txt"
self.filename_secure = secure_filename(self.filename)
self.filename_encoded = b64encode(
self.filename_secure.encode()).decode().strip('=')
self.request_id = 'FOIL-TEST-UVT'
self.quarantine_basepath = os.path.join(
current_app.config['UPLOAD_QUARANTINE_DIRECTORY'],
self.request_id
)
self.quarantine_path = os.path.join(
self.quarantine_basepath,
self.filename_secure
)
self.upload_basepath = os.path.join(
current_app.config['UPLOAD_DIRECTORY'],
self.request_id
)
self.upload_path = os.path.join(
self.upload_basepath,
self.filename_secure
)
self.key = get_upload_key(self.request_id, self.filename_secure)
self.key_update = get_upload_key(
self.request_id, self.filename_secure, for_update=True)
def status():
"""
Check the status of an upload.
Request Parameters:
- request_id
- filename
- for_update (bool, optional)
:returns: {
"status": upload status
}
"""
try:
status = redis.get(
get_upload_key(request.args['request_id'],
secure_filename(request.args['filename']),
eval_request_bool(request.args.get('for_update'))))
if status is not None:
response = {"status": status.decode("utf-8")}
else:
response = {"error": "Upload status not found."}
status_code = 200
except KeyError:
response = {}
status_code = 422
return jsonify(response), status_code
def status():
"""
Check the status of an upload.
Request Parameters:
- request_id
- filename
- for_update (bool, optional)
:returns: {
"status": upload status
}
"""
try:
status = redis.get(
get_upload_key(
request.args['request_id'],
secure_filename(request.args['filename']),
eval_request_bool(request.args.get('for_update'))
)
)
if status is not None:
response = {"status": status.decode("utf-8")}
else:
response = {"error": "Upload status not found."}
status_code = 200
except KeyError:
sentry.captureException()
response = {}
status_code = 422
return jsonify(response), status_code
def _move_validated_upload(request_id, tmp_path):
"""
Move an approved upload to the upload directory.
:param request_id: the id of the request associated with the upload
:param tmp_path: the temporary file path to the upload
generated by app.request.utils._quarantine_upload_no_id()
"""
dst_dir = os.path.join(current_app.config['UPLOAD_DIRECTORY'], request_id)
if not fu.exists(dst_dir):
fu.mkdir(dst_dir)
valid_name = os.path.basename(tmp_path).split('.',
1)[1] # remove 'tmp' prefix
valid_path = os.path.join(dst_dir, valid_name)
# store file metadata in redis
redis_set_file_metadata(request_id, tmp_path)
fu.move(tmp_path, valid_path)
upload_redis.set(get_upload_key(request_id, valid_name),
upload_status.READY)
return valid_path
def _move_validated_upload(request_id, tmp_path):
"""
Move an approved upload to the upload directory.
:param request_id: the id of the request associated with the upload
:param tmp_path: the temporary file path to the upload
generated by app.request.utils._quarantine_upload_no_id()
"""
dst_dir = os.path.join(
current_app.config['UPLOAD_DIRECTORY'],
request_id)
if not fu.exists(dst_dir):
fu.mkdir(dst_dir)
valid_name = os.path.basename(tmp_path).split('.', 1)[1] # remove 'tmp' prefix
valid_path = os.path.join(dst_dir, valid_name)
# store file metadata in redis
redis_set_file_metadata(request_id, tmp_path)
fu.move(tmp_path, valid_path)
upload_redis.set(
get_upload_key(request_id, valid_name),
upload_status.READY)
return valid_path
def post(request_id):
"""
Create a new upload.
Handles chunked files through the Content-Range header.
For filesize validation and more upload logic, see:
/static/js/upload/fileupload.js
Optional request body parameters:
- update (bool)
save the uploaded file to the 'updated' directory
(this indicates the file is meant to replace
a previously uploaded file)
- response_id (int)
the id of a response associated with the file
this upload is replacing
- REQUIRED if 'update' is 'true'
- ignored if 'update' is 'false'
:returns: {
"name": file name,
"size": file size
}
"""
files = request.files
file_ = files[next(files.keys())]
filename = secure_filename(file_.filename)
is_update = eval_request_bool(request.form.get('update'))
agency_ein = Requests.query.filter_by(id=request_id).one().agency.ein
if is_allowed(user=current_user, request_id=request_id, permission=permission.ADD_FILE) or \
is_allowed(user=current_user, request_id=request_id, permission=permission.EDIT_FILE):
response_id = request.form.get('response_id') if is_update else None
if upload_exists(request_id, filename, response_id):
response = {
"files": [{
"name":
filename,
"error":
"A file with this name has already "
"been uploaded for this request."
# TODO: "link": <link-to-existing-file> ? would be nice
}]
}
else:
upload_path = os.path.join(
current_app.config['UPLOAD_QUARANTINE_DIRECTORY'], request_id)
if not os.path.exists(upload_path):
os.mkdir(upload_path)
filepath = os.path.join(upload_path, filename)
key = get_upload_key(request_id, filename, is_update)
try:
if CONTENT_RANGE_HEADER in request.headers:
start, size = parse_content_range(
request.headers[CONTENT_RANGE_HEADER])
# Only validate mime type on first chunk
valid_file_type = True
file_type = None
if start == 0:
valid_file_type, file_type = is_valid_file_type(file_)
if current_user.is_agency_active(agency_ein):
valid_file_type = True
if os.path.exists(filepath):
# remove existing file (upload 'restarted' for same file)
os.remove(filepath)
if valid_file_type:
redis.set(key, upload_status.PROCESSING)
with open(filepath, 'ab') as fp:
fp.seek(start)
fp.write(file_.stream.read())
# scan if last chunk written
if os.path.getsize(filepath) == size:
scan_and_complete_upload.delay(
request_id, filepath, is_update, response_id)
else:
valid_file_type, file_type = is_valid_file_type(file_)
if current_user.is_agency_active(agency_ein):
valid_file_type = True
if valid_file_type:
redis.set(key, upload_status.PROCESSING)
file_.save(filepath)
scan_and_complete_upload.delay(request_id, filepath,
is_update, response_id)
if not valid_file_type:
response = {
"files": [{
"name":
filename,
"error":
"The file type '{}' is not allowed.".format(
file_type)
}]
}
else:
response = {
"files": [{
"name": filename,
"original_name": file_.filename,
"size": os.path.getsize(filepath),
}]
}
except Exception as e:
redis.set(key, upload_status.ERROR)
current_app.logger.exception(
"Upload for file '{}' failed: {}".format(filename, e))
response = {
"files": [{
"name":
filename,
"error":
"There was a problem uploading this file."
}]
}
return jsonify(response), 200
def delete(r_id_type, r_id, filecode):
"""
Removes an uploaded file.
:param r_id_type: "response" or "request"
:param r_id: the Response or Request identifier
:param filecode: the encoded name of the uploaded file
(base64 without padding)
Optional request body parameters:
- quarantined_only (bool)
only delete the file if it is quarantined
(beware: takes precedence over 'updated_only')
- updated_only (bool)
only delete the file if it is in the 'updated' directory
:returns:
On success:
{ "deleted": filename }
On failure:
{ "error": error message }
"""
filename = secure_filename(b64decode_lenient(filecode))
if r_id_type not in ["request", "response"]:
response = {"error": "Invalid ID type."}
else:
try:
if r_id_type == "response":
response = Responses.query.filter_by(id=r_id, deleted=False)
r_id = response.request_id
path = ''
quarantined_only = eval_request_bool(
request.form.get('quarantined_only'))
has_add_edit = (is_allowed(user=current_user,
request_id=r_id,
permission=permission.ADD_FILE)
or is_allowed(user=current_user,
request_id=r_id,
permission=permission.EDIT_FILE))
if quarantined_only and has_add_edit:
path = os.path.join(
current_app.config['UPLOAD_QUARANTINE_DIRECTORY'], r_id)
elif eval_request_bool(request.form.get('updated_only')) and \
is_allowed(user=current_user, request_id=r_id, permission=permission.EDIT_FILE):
path = os.path.join(current_app.config['UPLOAD_DIRECTORY'],
r_id, UPDATED_FILE_DIRNAME)
else:
path_for_status = {
upload_status.PROCESSING:
current_app.config['UPLOAD_QUARANTINE_DIRECTORY'],
upload_status.SCANNING:
current_app.config['UPLOAD_QUARANTINE_DIRECTORY'],
upload_status.READY:
current_app.config['UPLOAD_DIRECTORY']
}
status = redis.get(get_upload_key(r_id, filename))
if status is not None:
dest_path = path_for_status[status.decode("utf-8")]
if (dest_path ==
current_app.config['UPLOAD_QUARANTINE_DIRECTORY']
and has_add_edit
) or (dest_path
== current_app.config['UPLOAD_DIRECTORY']
and is_allowed(user=current_user,
request_id=r_id,
permission=permission.ADD_FILE)):
path = os.path.join(dest_path, r_id)
filepath = os.path.join(path, filename)
found = False
if path != '':
if quarantined_only:
if os.path.exists(filepath):
os.remove(filepath)
found = True
else:
if fu.exists(filepath):
fu.remove(filepath)
found = True
if found:
response = {"deleted": filename}
else:
response = {"error": "Upload not found."}
except Exception as e:
current_app.logger.exception(
"Error on DELETE /upload/: {}".format(e))
response = {"error": "Failed to delete '{}'".format(filename)}
return jsonify(response), 200
def post(request_id):
"""
Create a new upload.
Handles chunked files through the Content-Range header.
For filesize validation and more upload logic, see:
/static/js/upload/fileupload.js
Optional request body parameters:
- update (bool)
save the uploaded file to the 'updated' directory
(this indicates the file is meant to replace
a previously uploaded file)
- response_id (int)
the id of a response associated with the file
this upload is replacing
- REQUIRED if 'update' is 'true'
- ignored if 'update' is 'false'
:returns: {
"name": file name,
"size": file size
}
"""
files = request.files
file_ = files[next(files.keys())]
filename = secure_filename(file_.filename)
is_update = eval_request_bool(request.form.get('update'))
agency_ein = Requests.query.filter_by(id=request_id).one().agency.ein
if is_allowed(user=current_user, request_id=request_id, permission=permission.ADD_FILE) or \
is_allowed(user=current_user, request_id=request_id, permission=permission.EDIT_FILE):
response_id = request.form.get('response_id') if is_update else None
if upload_exists(request_id, filename, response_id):
response = {
"files": [{
"name": filename,
"error": "A file with this name has already "
"been uploaded for this request."
# TODO: "link": <link-to-existing-file> ? would be nice
}]
}
else:
upload_path = os.path.join(
current_app.config['UPLOAD_QUARANTINE_DIRECTORY'],
request_id)
if not os.path.exists(upload_path):
os.mkdir(upload_path)
filepath = os.path.join(upload_path, filename)
key = get_upload_key(request_id, filename, is_update)
try:
if CONTENT_RANGE_HEADER in request.headers:
start, size = parse_content_range(
request.headers[CONTENT_RANGE_HEADER])
# Only validate mime type on first chunk
valid_file_type = True
file_type = None
if start == 0:
valid_file_type, file_type = is_valid_file_type(file_)
if current_user.is_agency_active(agency_ein):
valid_file_type = True
if os.path.exists(filepath):
# remove existing file (upload 'restarted' for same file)
os.remove(filepath)
if valid_file_type:
redis.set(key, upload_status.PROCESSING)
with open(filepath, 'ab') as fp:
fp.seek(start)
fp.write(file_.stream.read())
# scan if last chunk written
if os.path.getsize(filepath) == size:
scan_and_complete_upload.delay(request_id, filepath, is_update, response_id)
else:
valid_file_type, file_type = is_valid_file_type(file_)
if current_user.is_agency_active(agency_ein):
valid_file_type = True
if valid_file_type:
redis.set(key, upload_status.PROCESSING)
file_.save(filepath)
scan_and_complete_upload.delay(request_id, filepath, is_update, response_id)
if not valid_file_type:
response = {
"files": [{
"name": filename,
"error": "The file type '{}' is not allowed.".format(
file_type)
}]
}
else:
response = {
"files": [{
"name": filename,
"original_name": file_.filename,
"size": os.path.getsize(filepath),
}]
}
except Exception as e:
sentry.captureException()
redis.set(key, upload_status.ERROR)
current_app.logger.exception("Upload for file '{}' failed: {}".format(filename, e))
response = {
"files": [{
"name": filename,
"error": "There was a problem uploading this file."
}]
}
return jsonify(response), 200
def delete(r_id_type, r_id, filecode):
"""
Removes an uploaded file.
:param r_id_type: "response" or "request"
:param r_id: the Response or Request identifier
:param filecode: the encoded name of the uploaded file
(base64 without padding)
Optional request body parameters:
- quarantined_only (bool)
only delete the file if it is quarantined
(beware: takes precedence over 'updated_only')
- updated_only (bool)
only delete the file if it is in the 'updated' directory
:returns:
On success:
{ "deleted": filename }
On failure:
{ "error": error message }
"""
filename = secure_filename(b64decode_lenient(filecode))
if r_id_type not in ["request", "response"]:
response = {"error": "Invalid ID type."}
else:
try:
if r_id_type == "response":
response = Responses.query.filter_by(id=r_id, deleted=False)
r_id = response.request_id
path = ''
quarantined_only = eval_request_bool(request.form.get('quarantined_only'))
has_add_edit = (is_allowed(user=current_user, request_id=r_id, permission=permission.ADD_FILE) or
is_allowed(user=current_user, request_id=r_id, permission=permission.EDIT_FILE))
if quarantined_only and has_add_edit:
path = os.path.join(
current_app.config['UPLOAD_QUARANTINE_DIRECTORY'],
r_id
)
elif eval_request_bool(request.form.get('updated_only')) and \
is_allowed(user=current_user, request_id=r_id, permission=permission.EDIT_FILE):
path = os.path.join(
current_app.config['UPLOAD_DIRECTORY'],
r_id,
UPDATED_FILE_DIRNAME
)
else:
path_for_status = {
upload_status.PROCESSING: current_app.config['UPLOAD_QUARANTINE_DIRECTORY'],
upload_status.SCANNING: current_app.config['UPLOAD_QUARANTINE_DIRECTORY'],
upload_status.READY: current_app.config['UPLOAD_DIRECTORY']
}
status = redis.get(get_upload_key(r_id, filename))
if status is not None:
dest_path = path_for_status[status.decode("utf-8")]
if (dest_path == current_app.config['UPLOAD_QUARANTINE_DIRECTORY'] and has_add_edit) or (
dest_path == current_app.config['UPLOAD_DIRECTORY'] and
is_allowed(user=current_user, request_id=r_id, permission=permission.ADD_FILE)
):
path = os.path.join(
dest_path,
r_id
)
filepath = os.path.join(path, filename)
found = False
if path != '':
if quarantined_only:
if os.path.exists(filepath):
os.remove(filepath)
found = True
else:
if fu.exists(filepath):
fu.remove(filepath)
found = True
if found:
response = {"deleted": filename}
else:
response = {"error": "Upload not found."}
except Exception as e:
sentry.captureException()
current_app.logger.exception("Error on DELETE /upload/: {}".format(e))
response = {"error": "Failed to delete '{}'".format(filename)}
return jsonify(response), 200
