def test_delete_request_id(self):
endpoint = '/upload/request/{}/{}'.format(
self.request_id,
self.filename_encoded
)
expected_response = {
"deleted": self.filename_secure
}
# test for data/quarantine/
os.mkdir(self.quarantine_basepath)
open(self.quarantine_path, 'w').close()
redis.set(self.key, upload_status.SCANNING)
response = self.client.delete(endpoint)
self.assertEqual(
json.loads(response.data.decode()),
expected_response
)
self.assertFalse(os.path.exists(self.quarantine_path))
# test for data/
os.mkdir(self.upload_basepath)
open(self.upload_path, 'w').close()
redis.set(self.key, upload_status.READY)
response = self.client.delete(endpoint)
self.assertEqual(
json.loads(response.data.decode()),
expected_response
)
self.assertFalse(os.path.exists(self.upload_path))
def redis_set_file_metadata(request_or_response_id, filepath, is_update=False):
"""
Stores a file's size, mime type, and hash.
"""
redis.set(
_get_file_metadata_key(request_or_response_id, filepath, is_update),
':'.join((str(os.path.getsize(filepath)), os_get_mime_type(filepath),
os_get_hash(filepath))))
def scan_and_complete_upload(request_id, filepath, is_update=False, response_id=None):
"""
Scans an uploaded file (see scan_file) and moves
it to the data directory if it is clean. If is_update is set,
the file will also be placed under the 'updated' directory.
Updates redis accordingly.
:param request_id: id of request associated with the upload
:param filepath: path to uploaded and quarantined file
:param is_update: will the file replace an existing one?
:param response_id: id of response associated with the upload
"""
if is_update:
assert response_id is not None
else:
assert response_id is None
filename = os.path.basename(filepath)
key = get_upload_key(request_id, filename, is_update)
redis.set(key, upload_status.SCANNING)
try:
scan_file(filepath)
except VirusDetectedException:
sentry.captureException()
redis.delete(key)
else:
# complete upload
dst_dir = os.path.join(
current_app.config['UPLOAD_DIRECTORY'],
request_id
)
if is_update:
dst_dir = os.path.join(
dst_dir,
UPDATED_FILE_DIRNAME
)
# store file metadata in redis
redis_set_file_metadata(response_id or request_id, filepath, is_update)
if not fu.exists(dst_dir):
try:
fu.makedirs(dst_dir)
except OSError as e:
sentry.captureException()
# in the time between the call to fu.exists
# and fu.makedirs, the directory was created
current_app.logger.error("OS Error: {}".format(e.args))
fu.move(
filepath,
os.path.join(dst_dir, filename)
)
redis.set(key, upload_status.READY)
def redis_set_file_metadata(request_or_response_id, filepath, is_update=False):
"""
Stores a file's size, mime type, and hash.
"""
redis.set(
_get_file_metadata_key(request_or_response_id, filepath, is_update),
':'.join((
str(os.path.getsize(filepath)),
os_get_mime_type(filepath),
os_get_hash(filepath)
))
)
def test_delete_missing_file(self):
redis.set(self.key, upload_status.READY)
response = self.client.delete(
'/upload/request/{}/{}'.format(
self.request_id,
self.filename_encoded
)
)
self.assertEqual(
json.loads(response.data.decode()),
{
"error": "Upload not found."
}
)
def test_status(self):
redis.set(self.key, upload_status.PROCESSING)
response = self.client.get(
'/upload/status',
query_string={
"request_id": self.request_id,
"filename": self.filename
}
)
self.assertEqual(
json.loads(response.data.decode()),
{
"status": upload_status.PROCESSING
}
)
def _move_validated_upload(request_id, tmp_path):
"""
Move an approved upload to the upload directory.
:param request_id: the id of the request associated with the upload
:param tmp_path: the temporary file path to the upload
generated by app.request.utils._quarantine_upload_no_id()
"""
dst_dir = os.path.join(current_app.config['UPLOAD_DIRECTORY'], request_id)
if not fu.exists(dst_dir):
fu.mkdir(dst_dir)
valid_name = os.path.basename(tmp_path).split('.',
1)[1] # remove 'tmp' prefix
valid_path = os.path.join(dst_dir, valid_name)
# store file metadata in redis
redis_set_file_metadata(request_id, tmp_path)
fu.move(tmp_path, valid_path)
upload_redis.set(get_upload_key(request_id, valid_name),
upload_status.READY)
return valid_path
def _move_validated_upload(request_id, tmp_path):
"""
Move an approved upload to the upload directory.
:param request_id: the id of the request associated with the upload
:param tmp_path: the temporary file path to the upload
generated by app.request.utils._quarantine_upload_no_id()
"""
dst_dir = os.path.join(
current_app.config['UPLOAD_DIRECTORY'],
request_id)
if not fu.exists(dst_dir):
fu.mkdir(dst_dir)
valid_name = os.path.basename(tmp_path).split('.', 1)[1] # remove 'tmp' prefix
valid_path = os.path.join(dst_dir, valid_name)
# store file metadata in redis
redis_set_file_metadata(request_id, tmp_path)
fu.move(tmp_path, valid_path)
upload_redis.set(
get_upload_key(request_id, valid_name),
upload_status.READY)
return valid_path
def post(request_id):
"""
Create a new upload.
Handles chunked files through the Content-Range header.
For filesize validation and more upload logic, see:
/static/js/upload/fileupload.js
Optional request body parameters:
- update (bool)
save the uploaded file to the 'updated' directory
(this indicates the file is meant to replace
a previously uploaded file)
- response_id (int)
the id of a response associated with the file
this upload is replacing
- REQUIRED if 'update' is 'true'
- ignored if 'update' is 'false'
:returns: {
"name": file name,
"size": file size
}
"""
files = request.files
file_ = files[next(files.keys())]
filename = secure_filename(file_.filename)
is_update = eval_request_bool(request.form.get('update'))
agency_ein = Requests.query.filter_by(id=request_id).one().agency.ein
if is_allowed(user=current_user, request_id=request_id, permission=permission.ADD_FILE) or \
is_allowed(user=current_user, request_id=request_id, permission=permission.EDIT_FILE):
response_id = request.form.get('response_id') if is_update else None
if upload_exists(request_id, filename, response_id):
response = {
"files": [{
"name":
filename,
"error":
"A file with this name has already "
"been uploaded for this request."
# TODO: "link": <link-to-existing-file> ? would be nice
}]
}
else:
upload_path = os.path.join(
current_app.config['UPLOAD_QUARANTINE_DIRECTORY'], request_id)
if not os.path.exists(upload_path):
os.mkdir(upload_path)
filepath = os.path.join(upload_path, filename)
key = get_upload_key(request_id, filename, is_update)
try:
if CONTENT_RANGE_HEADER in request.headers:
start, size = parse_content_range(
request.headers[CONTENT_RANGE_HEADER])
# Only validate mime type on first chunk
valid_file_type = True
file_type = None
if start == 0:
valid_file_type, file_type = is_valid_file_type(file_)
if current_user.is_agency_active(agency_ein):
valid_file_type = True
if os.path.exists(filepath):
# remove existing file (upload 'restarted' for same file)
os.remove(filepath)
if valid_file_type:
redis.set(key, upload_status.PROCESSING)
with open(filepath, 'ab') as fp:
fp.seek(start)
fp.write(file_.stream.read())
# scan if last chunk written
if os.path.getsize(filepath) == size:
scan_and_complete_upload.delay(
request_id, filepath, is_update, response_id)
else:
valid_file_type, file_type = is_valid_file_type(file_)
if current_user.is_agency_active(agency_ein):
valid_file_type = True
if valid_file_type:
redis.set(key, upload_status.PROCESSING)
file_.save(filepath)
scan_and_complete_upload.delay(request_id, filepath,
is_update, response_id)
if not valid_file_type:
response = {
"files": [{
"name":
filename,
"error":
"The file type '{}' is not allowed.".format(
file_type)
}]
}
else:
response = {
"files": [{
"name": filename,
"original_name": file_.filename,
"size": os.path.getsize(filepath),
}]
}
except Exception as e:
redis.set(key, upload_status.ERROR)
current_app.logger.exception(
"Upload for file '{}' failed: {}".format(filename, e))
response = {
"files": [{
"name":
filename,
"error":
"There was a problem uploading this file."
}]
}
return jsonify(response), 200
def post(request_id):
"""
Create a new upload.
Handles chunked files through the Content-Range header.
For filesize validation and more upload logic, see:
/static/js/upload/fileupload.js
Optional request body parameters:
- update (bool)
save the uploaded file to the 'updated' directory
(this indicates the file is meant to replace
a previously uploaded file)
- response_id (int)
the id of a response associated with the file
this upload is replacing
- REQUIRED if 'update' is 'true'
- ignored if 'update' is 'false'
:returns: {
"name": file name,
"size": file size
}
"""
files = request.files
file_ = files[next(files.keys())]
filename = secure_filename(file_.filename)
is_update = eval_request_bool(request.form.get('update'))
agency_ein = Requests.query.filter_by(id=request_id).one().agency.ein
if is_allowed(user=current_user, request_id=request_id, permission=permission.ADD_FILE) or \
is_allowed(user=current_user, request_id=request_id, permission=permission.EDIT_FILE):
response_id = request.form.get('response_id') if is_update else None
if upload_exists(request_id, filename, response_id):
response = {
"files": [{
"name": filename,
"error": "A file with this name has already "
"been uploaded for this request."
# TODO: "link": <link-to-existing-file> ? would be nice
}]
}
else:
upload_path = os.path.join(
current_app.config['UPLOAD_QUARANTINE_DIRECTORY'],
request_id)
if not os.path.exists(upload_path):
os.mkdir(upload_path)
filepath = os.path.join(upload_path, filename)
key = get_upload_key(request_id, filename, is_update)
try:
if CONTENT_RANGE_HEADER in request.headers:
start, size = parse_content_range(
request.headers[CONTENT_RANGE_HEADER])
# Only validate mime type on first chunk
valid_file_type = True
file_type = None
if start == 0:
valid_file_type, file_type = is_valid_file_type(file_)
if current_user.is_agency_active(agency_ein):
valid_file_type = True
if os.path.exists(filepath):
# remove existing file (upload 'restarted' for same file)
os.remove(filepath)
if valid_file_type:
redis.set(key, upload_status.PROCESSING)
with open(filepath, 'ab') as fp:
fp.seek(start)
fp.write(file_.stream.read())
# scan if last chunk written
if os.path.getsize(filepath) == size:
scan_and_complete_upload.delay(request_id, filepath, is_update, response_id)
else:
valid_file_type, file_type = is_valid_file_type(file_)
if current_user.is_agency_active(agency_ein):
valid_file_type = True
if valid_file_type:
redis.set(key, upload_status.PROCESSING)
file_.save(filepath)
scan_and_complete_upload.delay(request_id, filepath, is_update, response_id)
if not valid_file_type:
response = {
"files": [{
"name": filename,
"error": "The file type '{}' is not allowed.".format(
file_type)
}]
}
else:
response = {
"files": [{
"name": filename,
"original_name": file_.filename,
"size": os.path.getsize(filepath),
}]
}
except Exception as e:
sentry.captureException()
redis.set(key, upload_status.ERROR)
current_app.logger.exception("Upload for file '{}' failed: {}".format(filename, e))
response = {
"files": [{
"name": filename,
"error": "There was a problem uploading this file."
}]
}
return jsonify(response), 200
