def add_file(self, filepath=None, mime_type='text/plain', contents=None, title=None): if filepath is None: filename = str(uuid.uuid4()) filepath = os.path.join(current_app.config['UPLOAD_DIRECTORY'], self.request.id, filename) else: filename = os.path.basename(filepath) self.filepaths.append(filepath) # create an empty file if the specified path does not exist if not fu.exists(filepath): fu.makedirs(os.path.dirname(filepath), exist_ok=True) with open(filepath, 'w') as fp: fp.write(contents or ''.join( random.choice(ascii_letters) for _ in range(random.randrange(100, 500)))) response = Files(self.request.id, PRIVATE, title or filename, filename, mime_type, fu.getsize(filepath), fu.get_hash(filepath)) # TODO: add Events FILE_ADDED create_object(response) return response
def add_file( self, title=None, filepath=None, name=None, # will be ignored if filepath supplied privacy=response_privacy.PRIVATE, user=None): if filepath is None: filename = name or fake.file_name(extension='txt') filepath = os.path.join(current_app.config["UPLOAD_DIRECTORY"], self.request.id, filename) else: filename = os.path.basename(filepath) if not fu.exists(filepath): fu.makedirs(os.path.dirname(filepath), exist_ok=True) with open(filepath, "w") as fp: fp.write(fake.file_content()) self.__files.append(filepath) response = Files( self.request.id, privacy=privacy, title=title or fake.title(), name=filename, mime_type=fu.get_mime_type(filepath), size=fu.getsize(filepath), hash_=fu.get_hash(filepath), ) create_object(response) self.__create_event(event_type.FILE_ADDED, response, user) return response
def __del__(self): """ Remove any *non-database* artifacts created for this request. """ if self.__clean: for path in self.__files: if fu.exists(path): fu.remove(path)
def __del__(self): """ Clean up time! - remove any files created by this factory - ... """ if self.clean: for path in self.filepaths: if fu.exists(path): fu.remove(path)
def scan_and_complete_upload(request_id, filepath, is_update=False, response_id=None): """ Scans an uploaded file (see scan_file) and moves it to the data directory if it is clean. If is_update is set, the file will also be placed under the 'updated' directory. Updates redis accordingly. :param request_id: id of request associated with the upload :param filepath: path to uploaded and quarantined file :param is_update: will the file replace an existing one? :param response_id: id of response associated with the upload """ if is_update: assert response_id is not None else: assert response_id is None filename = os.path.basename(filepath) key = get_upload_key(request_id, filename, is_update) redis.set(key, upload_status.SCANNING) try: scan_file(filepath) except VirusDetectedException: sentry.captureException() redis.delete(key) else: # complete upload dst_dir = os.path.join( current_app.config['UPLOAD_DIRECTORY'], request_id ) if is_update: dst_dir = os.path.join( dst_dir, UPDATED_FILE_DIRNAME ) # store file metadata in redis redis_set_file_metadata(response_id or request_id, filepath, is_update) if not fu.exists(dst_dir): try: fu.makedirs(dst_dir) except OSError as e: sentry.captureException() # in the time between the call to fu.exists # and fu.makedirs, the directory was created current_app.logger.error("OS Error: {}".format(e.args)) fu.move( filepath, os.path.join(dst_dir, filename) ) redis.set(key, upload_status.READY)
def _move_validated_upload(request_id, tmp_path): """ Move an approved upload to the upload directory. :param request_id: the id of the request associated with the upload :param tmp_path: the temporary file path to the upload generated by app.request.utils._quarantine_upload_no_id() """ dst_dir = os.path.join(current_app.config['UPLOAD_DIRECTORY'], request_id) if not fu.exists(dst_dir): fu.mkdir(dst_dir) valid_name = os.path.basename(tmp_path).split('.', 1)[1] # remove 'tmp' prefix valid_path = os.path.join(dst_dir, valid_name) # store file metadata in redis redis_set_file_metadata(request_id, tmp_path) fu.move(tmp_path, valid_path) upload_redis.set(get_upload_key(request_id, valid_name), upload_status.READY) return valid_path
def _move_validated_upload(request_id, tmp_path): """ Move an approved upload to the upload directory. :param request_id: the id of the request associated with the upload :param tmp_path: the temporary file path to the upload generated by app.request.utils._quarantine_upload_no_id() """ dst_dir = os.path.join( current_app.config['UPLOAD_DIRECTORY'], request_id) if not fu.exists(dst_dir): fu.mkdir(dst_dir) valid_name = os.path.basename(tmp_path).split('.', 1)[1] # remove 'tmp' prefix valid_path = os.path.join(dst_dir, valid_name) # store file metadata in redis redis_set_file_metadata(request_id, tmp_path) fu.move(tmp_path, valid_path) upload_redis.set( get_upload_key(request_id, valid_name), upload_status.READY) return valid_path
def get_response_content(response_id): """ Currently only supports File Responses. Request Parameters: - token: (optional) ephemeral access token :return: response file contents or redirect to login if user not authenticated and no token provided or 400 error if response/file not found """ response_ = Responses.query.filter_by(id=response_id, deleted=False).one() if response_ is not None and response_.type == FILE: upload_path = os.path.join( current_app.config["UPLOAD_DIRECTORY"], response_.request_id ) filepath_parts = ( upload_path, response_.name ) filepath = os.path.join(*filepath_parts) serving_path = os.path.join( current_app.config['UPLOAD_SERVING_DIRECTORY'], response_.request_id, response_.name ) token = flask_request.args.get('token') if fu.exists(filepath): if response_.is_public: # then we just serve the file, anyone can view it @after_this_request def remove(resp): os.remove(serving_path) return resp return fu.send_file(*filepath_parts, as_attachment=True) else: # check presence of token in url if token is not None: resptok = ResponseTokens.query.filter_by( token=token, response_id=response_id).first() if resptok is not None: if response_.privacy != PRIVATE: @after_this_request def remove(resp): os.remove(serving_path) return resp return fu.send_file(*filepath_parts, as_attachment=True) else: delete_object(resptok) # if token not included, nonexistent, or is expired, but user is logged in if current_user.is_authenticated: # user is agency or is public and response is not private if (((current_user.is_public and response_.privacy != PRIVATE) or current_user.is_agency) # user is associated with request and UserRequests.query.filter_by( request_id=response_.request_id, user_guid=current_user.guid ).first() is not None): @after_this_request def remove(resp): os.remove(serving_path) return resp return fu.send_file(*filepath_parts, as_attachment=True) # user does not have permission to view file return abort(403) else: # redirect to login return redirect(login_url( login_manager.login_view, next_url=url_for('request.view', request_id=response_.request_id) )) return abort(404) # file does not exist
def get_response_content(response_id): """ Currently only supports File Responses. Request Parameters: - token: (optional) ephemeral access token :return: response file contents or redirect to login if user not authenticated and no token provided or 400 error if response/file not found """ response_ = Responses.query.filter_by(id=response_id, deleted=False).one() # if ((current_user.is_authenticated # and current_user not in response_.request.agency_users # and response_.request.requester != current_user) # or flask_request.args.get('token') is None): # return abort(403) if response_ is not None and response_.type == FILE: upload_path = os.path.join( current_app.config["UPLOAD_DIRECTORY"], response_.request_id ) filepath_parts = ( upload_path, response_.name ) filepath = os.path.join(*filepath_parts) serving_path = os.path.join( current_app.config['UPLOAD_SERVING_DIRECTORY'], response_.request_id, response_.name ) token = flask_request.args.get('token') if fu.exists(filepath): if token is not None: resptok = ResponseTokens.query.filter_by( token=token, response_id=response_id).first() if resptok is not None: if (datetime.utcnow() < resptok.expiration_date and response_.privacy != PRIVATE): @after_this_request def remove(resp): os.remove(serving_path) return resp return fu.send_file(*filepath_parts, as_attachment=True) else: delete_object(resptok) else: if current_user.is_authenticated: # user is agency or is public and response is not private if (((current_user.is_public and response_.privacy != PRIVATE) or current_user.is_agency) # user is associated with request and UserRequests.query.filter_by( request_id=response_.request_id, user_guid=current_user.guid, auth_user_type=current_user.auth_user_type ).first() is not None): @after_this_request def remove(resp): os.remove(serving_path) return resp return fu.send_file(*filepath_parts, as_attachment=True) return abort(403) else: # response is release and public # TODO: Responses.is_release_public property if (response_.privacy == RELEASE_AND_PUBLIC and response_.release_date is not None and datetime.utcnow() > response_.release_date): @after_this_request def remove(resp): os.remove(serving_path) return resp return fu.send_file(*filepath_parts, as_attachment=True) else: return redirect(url_for( 'auth.login', return_to_url=url_for('request.view', request_id=response_.request_id) )) # TODO: restore after saml/oauth implementation # return redirect(url_for( # 'auth.index', # sso2=True, # return_to=flask_request.base_url)) return abort(404)
def delete(r_id_type, r_id, filecode): """ Removes an uploaded file. :param r_id_type: "response" or "request" :param r_id: the Response or Request identifier :param filecode: the encoded name of the uploaded file (base64 without padding) Optional request body parameters: - quarantined_only (bool) only delete the file if it is quarantined (beware: takes precedence over 'updated_only') - updated_only (bool) only delete the file if it is in the 'updated' directory :returns: On success: { "deleted": filename } On failure: { "error": error message } """ filename = secure_filename(b64decode_lenient(filecode)) if r_id_type not in ["request", "response"]: response = {"error": "Invalid ID type."} else: try: if r_id_type == "response": response = Responses.query.filter_by(id=r_id, deleted=False) r_id = response.request_id path = '' quarantined_only = eval_request_bool( request.form.get('quarantined_only')) has_add_edit = (is_allowed(user=current_user, request_id=r_id, permission=permission.ADD_FILE) or is_allowed(user=current_user, request_id=r_id, permission=permission.EDIT_FILE)) if quarantined_only and has_add_edit: path = os.path.join( current_app.config['UPLOAD_QUARANTINE_DIRECTORY'], r_id) elif eval_request_bool(request.form.get('updated_only')) and \ is_allowed(user=current_user, request_id=r_id, permission=permission.EDIT_FILE): path = os.path.join(current_app.config['UPLOAD_DIRECTORY'], r_id, UPDATED_FILE_DIRNAME) else: path_for_status = { upload_status.PROCESSING: current_app.config['UPLOAD_QUARANTINE_DIRECTORY'], upload_status.SCANNING: current_app.config['UPLOAD_QUARANTINE_DIRECTORY'], upload_status.READY: current_app.config['UPLOAD_DIRECTORY'] } status = redis.get(get_upload_key(r_id, filename)) if status is not None: dest_path = path_for_status[status.decode("utf-8")] if (dest_path == current_app.config['UPLOAD_QUARANTINE_DIRECTORY'] and has_add_edit ) or (dest_path == current_app.config['UPLOAD_DIRECTORY'] and is_allowed(user=current_user, request_id=r_id, permission=permission.ADD_FILE)): path = os.path.join(dest_path, r_id) filepath = os.path.join(path, filename) found = False if path != '': if quarantined_only: if os.path.exists(filepath): os.remove(filepath) found = True else: if fu.exists(filepath): fu.remove(filepath) found = True if found: response = {"deleted": filename} else: response = {"error": "Upload not found."} except Exception as e: current_app.logger.exception( "Error on DELETE /upload/: {}".format(e)) response = {"error": "Failed to delete '{}'".format(filename)} return jsonify(response), 200
def delete(r_id_type, r_id, filecode): """ Removes an uploaded file. :param r_id_type: "response" or "request" :param r_id: the Response or Request identifier :param filecode: the encoded name of the uploaded file (base64 without padding) Optional request body parameters: - quarantined_only (bool) only delete the file if it is quarantined (beware: takes precedence over 'updated_only') - updated_only (bool) only delete the file if it is in the 'updated' directory :returns: On success: { "deleted": filename } On failure: { "error": error message } """ filename = secure_filename(b64decode_lenient(filecode)) if r_id_type not in ["request", "response"]: response = {"error": "Invalid ID type."} else: try: if r_id_type == "response": response = Responses.query.filter_by(id=r_id, deleted=False) r_id = response.request_id path = '' quarantined_only = eval_request_bool(request.form.get('quarantined_only')) has_add_edit = (is_allowed(user=current_user, request_id=r_id, permission=permission.ADD_FILE) or is_allowed(user=current_user, request_id=r_id, permission=permission.EDIT_FILE)) if quarantined_only and has_add_edit: path = os.path.join( current_app.config['UPLOAD_QUARANTINE_DIRECTORY'], r_id ) elif eval_request_bool(request.form.get('updated_only')) and \ is_allowed(user=current_user, request_id=r_id, permission=permission.EDIT_FILE): path = os.path.join( current_app.config['UPLOAD_DIRECTORY'], r_id, UPDATED_FILE_DIRNAME ) else: path_for_status = { upload_status.PROCESSING: current_app.config['UPLOAD_QUARANTINE_DIRECTORY'], upload_status.SCANNING: current_app.config['UPLOAD_QUARANTINE_DIRECTORY'], upload_status.READY: current_app.config['UPLOAD_DIRECTORY'] } status = redis.get(get_upload_key(r_id, filename)) if status is not None: dest_path = path_for_status[status.decode("utf-8")] if (dest_path == current_app.config['UPLOAD_QUARANTINE_DIRECTORY'] and has_add_edit) or ( dest_path == current_app.config['UPLOAD_DIRECTORY'] and is_allowed(user=current_user, request_id=r_id, permission=permission.ADD_FILE) ): path = os.path.join( dest_path, r_id ) filepath = os.path.join(path, filename) found = False if path != '': if quarantined_only: if os.path.exists(filepath): os.remove(filepath) found = True else: if fu.exists(filepath): fu.remove(filepath) found = True if found: response = {"deleted": filename} else: response = {"error": "Upload not found."} except Exception as e: sentry.captureException() current_app.logger.exception("Error on DELETE /upload/: {}".format(e)) response = {"error": "Failed to delete '{}'".format(filename)} return jsonify(response), 200