def add_file(self,
filepath=None,
mime_type='text/plain',
contents=None,
title=None):
if filepath is None:
filename = str(uuid.uuid4())
filepath = os.path.join(current_app.config['UPLOAD_DIRECTORY'],
self.request.id, filename)
else:
filename = os.path.basename(filepath)
self.filepaths.append(filepath)
# create an empty file if the specified path does not exist
if not fu.exists(filepath):
fu.makedirs(os.path.dirname(filepath), exist_ok=True)
with open(filepath, 'w') as fp:
fp.write(contents or ''.join(
random.choice(ascii_letters)
for _ in range(random.randrange(100, 500))))
response = Files(self.request.id, PRIVATE, title or filename,
filename, mime_type, fu.getsize(filepath),
fu.get_hash(filepath))
# TODO: add Events FILE_ADDED
create_object(response)
return response
def add_file(
self,
title=None,
filepath=None,
name=None, # will be ignored if filepath supplied
privacy=response_privacy.PRIVATE,
user=None):
if filepath is None:
filename = name or fake.file_name(extension='txt')
filepath = os.path.join(current_app.config["UPLOAD_DIRECTORY"],
self.request.id, filename)
else:
filename = os.path.basename(filepath)
if not fu.exists(filepath):
fu.makedirs(os.path.dirname(filepath), exist_ok=True)
with open(filepath, "w") as fp:
fp.write(fake.file_content())
self.__files.append(filepath)
response = Files(
self.request.id,
privacy=privacy,
title=title or fake.title(),
name=filename,
mime_type=fu.get_mime_type(filepath),
size=fu.getsize(filepath),
hash_=fu.get_hash(filepath),
)
create_object(response)
self.__create_event(event_type.FILE_ADDED, response, user)
return response
def __del__(self):
"""
Remove any *non-database* artifacts created for this request.
"""
if self.__clean:
for path in self.__files:
if fu.exists(path):
fu.remove(path)
def __del__(self):
"""
Clean up time!
- remove any files created by this factory
- ...
"""
if self.clean:
for path in self.filepaths:
if fu.exists(path):
fu.remove(path)
def scan_and_complete_upload(request_id, filepath, is_update=False, response_id=None):
"""
Scans an uploaded file (see scan_file) and moves
it to the data directory if it is clean. If is_update is set,
the file will also be placed under the 'updated' directory.
Updates redis accordingly.
:param request_id: id of request associated with the upload
:param filepath: path to uploaded and quarantined file
:param is_update: will the file replace an existing one?
:param response_id: id of response associated with the upload
"""
if is_update:
assert response_id is not None
else:
assert response_id is None
filename = os.path.basename(filepath)
key = get_upload_key(request_id, filename, is_update)
redis.set(key, upload_status.SCANNING)
try:
scan_file(filepath)
except VirusDetectedException:
sentry.captureException()
redis.delete(key)
else:
# complete upload
dst_dir = os.path.join(
current_app.config['UPLOAD_DIRECTORY'],
request_id
)
if is_update:
dst_dir = os.path.join(
dst_dir,
UPDATED_FILE_DIRNAME
)
# store file metadata in redis
redis_set_file_metadata(response_id or request_id, filepath, is_update)
if not fu.exists(dst_dir):
try:
fu.makedirs(dst_dir)
except OSError as e:
sentry.captureException()
# in the time between the call to fu.exists
# and fu.makedirs, the directory was created
current_app.logger.error("OS Error: {}".format(e.args))
fu.move(
filepath,
os.path.join(dst_dir, filename)
)
redis.set(key, upload_status.READY)
def _move_validated_upload(request_id, tmp_path):
"""
Move an approved upload to the upload directory.
:param request_id: the id of the request associated with the upload
:param tmp_path: the temporary file path to the upload
generated by app.request.utils._quarantine_upload_no_id()
"""
dst_dir = os.path.join(current_app.config['UPLOAD_DIRECTORY'], request_id)
if not fu.exists(dst_dir):
fu.mkdir(dst_dir)
valid_name = os.path.basename(tmp_path).split('.',
1)[1] # remove 'tmp' prefix
valid_path = os.path.join(dst_dir, valid_name)
# store file metadata in redis
redis_set_file_metadata(request_id, tmp_path)
fu.move(tmp_path, valid_path)
upload_redis.set(get_upload_key(request_id, valid_name),
upload_status.READY)
return valid_path
def _move_validated_upload(request_id, tmp_path):
"""
Move an approved upload to the upload directory.
:param request_id: the id of the request associated with the upload
:param tmp_path: the temporary file path to the upload
generated by app.request.utils._quarantine_upload_no_id()
"""
dst_dir = os.path.join(
current_app.config['UPLOAD_DIRECTORY'],
request_id)
if not fu.exists(dst_dir):
fu.mkdir(dst_dir)
valid_name = os.path.basename(tmp_path).split('.', 1)[1] # remove 'tmp' prefix
valid_path = os.path.join(dst_dir, valid_name)
# store file metadata in redis
redis_set_file_metadata(request_id, tmp_path)
fu.move(tmp_path, valid_path)
upload_redis.set(
get_upload_key(request_id, valid_name),
upload_status.READY)
return valid_path
def get_response_content(response_id):
"""
Currently only supports File Responses.
Request Parameters:
- token: (optional) ephemeral access token
:return: response file contents or
redirect to login if user not authenticated and no token provided or
400 error if response/file not found
"""
response_ = Responses.query.filter_by(id=response_id, deleted=False).one()
if response_ is not None and response_.type == FILE:
upload_path = os.path.join(
current_app.config["UPLOAD_DIRECTORY"],
response_.request_id
)
filepath_parts = (
upload_path,
response_.name
)
filepath = os.path.join(*filepath_parts)
serving_path = os.path.join(
current_app.config['UPLOAD_SERVING_DIRECTORY'],
response_.request_id,
response_.name
)
token = flask_request.args.get('token')
if fu.exists(filepath):
if response_.is_public:
# then we just serve the file, anyone can view it
@after_this_request
def remove(resp):
os.remove(serving_path)
return resp
return fu.send_file(*filepath_parts, as_attachment=True)
else:
# check presence of token in url
if token is not None:
resptok = ResponseTokens.query.filter_by(
token=token, response_id=response_id).first()
if resptok is not None:
if response_.privacy != PRIVATE:
@after_this_request
def remove(resp):
os.remove(serving_path)
return resp
return fu.send_file(*filepath_parts, as_attachment=True)
else:
delete_object(resptok)
# if token not included, nonexistent, or is expired, but user is logged in
if current_user.is_authenticated:
# user is agency or is public and response is not private
if (((current_user.is_public and response_.privacy != PRIVATE)
or current_user.is_agency)
# user is associated with request
and UserRequests.query.filter_by(
request_id=response_.request_id,
user_guid=current_user.guid
).first() is not None):
@after_this_request
def remove(resp):
os.remove(serving_path)
return resp
return fu.send_file(*filepath_parts, as_attachment=True)
# user does not have permission to view file
return abort(403)
else:
# redirect to login
return redirect(login_url(
login_manager.login_view,
next_url=url_for('request.view', request_id=response_.request_id)
))
return abort(404) # file does not exist
def get_response_content(response_id):
"""
Currently only supports File Responses.
Request Parameters:
- token: (optional) ephemeral access token
:return: response file contents or
redirect to login if user not authenticated and no token provided or
400 error if response/file not found
"""
response_ = Responses.query.filter_by(id=response_id, deleted=False).one()
# if ((current_user.is_authenticated
# and current_user not in response_.request.agency_users
# and response_.request.requester != current_user)
# or flask_request.args.get('token') is None):
# return abort(403)
if response_ is not None and response_.type == FILE:
upload_path = os.path.join(
current_app.config["UPLOAD_DIRECTORY"],
response_.request_id
)
filepath_parts = (
upload_path,
response_.name
)
filepath = os.path.join(*filepath_parts)
serving_path = os.path.join(
current_app.config['UPLOAD_SERVING_DIRECTORY'],
response_.request_id,
response_.name
)
token = flask_request.args.get('token')
if fu.exists(filepath):
if token is not None:
resptok = ResponseTokens.query.filter_by(
token=token, response_id=response_id).first()
if resptok is not None:
if (datetime.utcnow() < resptok.expiration_date
and response_.privacy != PRIVATE):
@after_this_request
def remove(resp):
os.remove(serving_path)
return resp
return fu.send_file(*filepath_parts, as_attachment=True)
else:
delete_object(resptok)
else:
if current_user.is_authenticated:
# user is agency or is public and response is not private
if (((current_user.is_public and response_.privacy != PRIVATE)
or current_user.is_agency)
# user is associated with request
and UserRequests.query.filter_by(
request_id=response_.request_id,
user_guid=current_user.guid,
auth_user_type=current_user.auth_user_type
).first() is not None):
@after_this_request
def remove(resp):
os.remove(serving_path)
return resp
return fu.send_file(*filepath_parts, as_attachment=True)
return abort(403)
else:
# response is release and public # TODO: Responses.is_release_public property
if (response_.privacy == RELEASE_AND_PUBLIC
and response_.release_date is not None
and datetime.utcnow() > response_.release_date):
@after_this_request
def remove(resp):
os.remove(serving_path)
return resp
return fu.send_file(*filepath_parts, as_attachment=True)
else:
return redirect(url_for(
'auth.login',
return_to_url=url_for('request.view', request_id=response_.request_id)
))
# TODO: restore after saml/oauth implementation
# return redirect(url_for(
# 'auth.index',
# sso2=True,
# return_to=flask_request.base_url))
return abort(404)
def delete(r_id_type, r_id, filecode):
"""
Removes an uploaded file.
:param r_id_type: "response" or "request"
:param r_id: the Response or Request identifier
:param filecode: the encoded name of the uploaded file
(base64 without padding)
Optional request body parameters:
- quarantined_only (bool)
only delete the file if it is quarantined
(beware: takes precedence over 'updated_only')
- updated_only (bool)
only delete the file if it is in the 'updated' directory
:returns:
On success:
{ "deleted": filename }
On failure:
{ "error": error message }
"""
filename = secure_filename(b64decode_lenient(filecode))
if r_id_type not in ["request", "response"]:
response = {"error": "Invalid ID type."}
else:
try:
if r_id_type == "response":
response = Responses.query.filter_by(id=r_id, deleted=False)
r_id = response.request_id
path = ''
quarantined_only = eval_request_bool(
request.form.get('quarantined_only'))
has_add_edit = (is_allowed(user=current_user,
request_id=r_id,
permission=permission.ADD_FILE)
or is_allowed(user=current_user,
request_id=r_id,
permission=permission.EDIT_FILE))
if quarantined_only and has_add_edit:
path = os.path.join(
current_app.config['UPLOAD_QUARANTINE_DIRECTORY'], r_id)
elif eval_request_bool(request.form.get('updated_only')) and \
is_allowed(user=current_user, request_id=r_id, permission=permission.EDIT_FILE):
path = os.path.join(current_app.config['UPLOAD_DIRECTORY'],
r_id, UPDATED_FILE_DIRNAME)
else:
path_for_status = {
upload_status.PROCESSING:
current_app.config['UPLOAD_QUARANTINE_DIRECTORY'],
upload_status.SCANNING:
current_app.config['UPLOAD_QUARANTINE_DIRECTORY'],
upload_status.READY:
current_app.config['UPLOAD_DIRECTORY']
}
status = redis.get(get_upload_key(r_id, filename))
if status is not None:
dest_path = path_for_status[status.decode("utf-8")]
if (dest_path ==
current_app.config['UPLOAD_QUARANTINE_DIRECTORY']
and has_add_edit
) or (dest_path
== current_app.config['UPLOAD_DIRECTORY']
and is_allowed(user=current_user,
request_id=r_id,
permission=permission.ADD_FILE)):
path = os.path.join(dest_path, r_id)
filepath = os.path.join(path, filename)
found = False
if path != '':
if quarantined_only:
if os.path.exists(filepath):
os.remove(filepath)
found = True
else:
if fu.exists(filepath):
fu.remove(filepath)
found = True
if found:
response = {"deleted": filename}
else:
response = {"error": "Upload not found."}
except Exception as e:
current_app.logger.exception(
"Error on DELETE /upload/: {}".format(e))
response = {"error": "Failed to delete '{}'".format(filename)}
return jsonify(response), 200
def get_response_content(response_id):
"""
Currently only supports File Responses.
Request Parameters:
- token: (optional) ephemeral access token
:return: response file contents or
redirect to login if user not authenticated and no token provided or
400 error if response/file not found
"""
response_ = Responses.query.filter_by(id=response_id, deleted=False).one()
if response_ is not None and response_.type == FILE:
upload_path = os.path.join(
current_app.config["UPLOAD_DIRECTORY"],
response_.request_id
)
filepath_parts = (
upload_path,
response_.name
)
filepath = os.path.join(*filepath_parts)
serving_path = os.path.join(
current_app.config['UPLOAD_SERVING_DIRECTORY'],
response_.request_id,
response_.name
)
token = flask_request.args.get('token')
if fu.exists(filepath):
if response_.is_public:
# then we just serve the file, anyone can view it
@after_this_request
def remove(resp):
os.remove(serving_path)
return resp
return fu.send_file(*filepath_parts, as_attachment=True)
else:
# check presence of token in url
if token is not None:
resptok = ResponseTokens.query.filter_by(
token=token, response_id=response_id).first()
if resptok is not None:
if response_.privacy != PRIVATE:
@after_this_request
def remove(resp):
os.remove(serving_path)
return resp
return fu.send_file(*filepath_parts, as_attachment=True)
else:
delete_object(resptok)
# if token not included, nonexistent, or is expired, but user is logged in
if current_user.is_authenticated:
# user is agency or is public and response is not private
if (((current_user.is_public and response_.privacy != PRIVATE)
or current_user.is_agency)
# user is associated with request
and UserRequests.query.filter_by(
request_id=response_.request_id,
user_guid=current_user.guid
).first() is not None):
@after_this_request
def remove(resp):
os.remove(serving_path)
return resp
return fu.send_file(*filepath_parts, as_attachment=True)
# user does not have permission to view file
return abort(403)
else:
# redirect to login
return redirect(login_url(
login_manager.login_view,
next_url=url_for('request.view', request_id=response_.request_id)
))
return abort(404) # file does not exist
def delete(r_id_type, r_id, filecode):
"""
Removes an uploaded file.
:param r_id_type: "response" or "request"
:param r_id: the Response or Request identifier
:param filecode: the encoded name of the uploaded file
(base64 without padding)
Optional request body parameters:
- quarantined_only (bool)
only delete the file if it is quarantined
(beware: takes precedence over 'updated_only')
- updated_only (bool)
only delete the file if it is in the 'updated' directory
:returns:
On success:
{ "deleted": filename }
On failure:
{ "error": error message }
"""
filename = secure_filename(b64decode_lenient(filecode))
if r_id_type not in ["request", "response"]:
response = {"error": "Invalid ID type."}
else:
try:
if r_id_type == "response":
response = Responses.query.filter_by(id=r_id, deleted=False)
r_id = response.request_id
path = ''
quarantined_only = eval_request_bool(request.form.get('quarantined_only'))
has_add_edit = (is_allowed(user=current_user, request_id=r_id, permission=permission.ADD_FILE) or
is_allowed(user=current_user, request_id=r_id, permission=permission.EDIT_FILE))
if quarantined_only and has_add_edit:
path = os.path.join(
current_app.config['UPLOAD_QUARANTINE_DIRECTORY'],
r_id
)
elif eval_request_bool(request.form.get('updated_only')) and \
is_allowed(user=current_user, request_id=r_id, permission=permission.EDIT_FILE):
path = os.path.join(
current_app.config['UPLOAD_DIRECTORY'],
r_id,
UPDATED_FILE_DIRNAME
)
else:
path_for_status = {
upload_status.PROCESSING: current_app.config['UPLOAD_QUARANTINE_DIRECTORY'],
upload_status.SCANNING: current_app.config['UPLOAD_QUARANTINE_DIRECTORY'],
upload_status.READY: current_app.config['UPLOAD_DIRECTORY']
}
status = redis.get(get_upload_key(r_id, filename))
if status is not None:
dest_path = path_for_status[status.decode("utf-8")]
if (dest_path == current_app.config['UPLOAD_QUARANTINE_DIRECTORY'] and has_add_edit) or (
dest_path == current_app.config['UPLOAD_DIRECTORY'] and
is_allowed(user=current_user, request_id=r_id, permission=permission.ADD_FILE)
):
path = os.path.join(
dest_path,
r_id
)
filepath = os.path.join(path, filename)
found = False
if path != '':
if quarantined_only:
if os.path.exists(filepath):
os.remove(filepath)
found = True
else:
if fu.exists(filepath):
fu.remove(filepath)
found = True
if found:
response = {"deleted": filename}
else:
response = {"error": "Upload not found."}
except Exception as e:
sentry.captureException()
current_app.logger.exception("Error on DELETE /upload/: {}".format(e))
response = {"error": "Failed to delete '{}'".format(filename)}
return jsonify(response), 200
