def test_delete_request_id(self): endpoint = '/upload/request/{}/{}'.format( self.request_id, self.filename_encoded ) expected_response = { "deleted": self.filename_secure } # test for data/quarantine/ os.mkdir(self.quarantine_basepath) open(self.quarantine_path, 'w').close() redis.set(self.key, upload_status.SCANNING) response = self.client.delete(endpoint) self.assertEqual( json.loads(response.data.decode()), expected_response ) self.assertFalse(os.path.exists(self.quarantine_path)) # test for data/ os.mkdir(self.upload_basepath) open(self.upload_path, 'w').close() redis.set(self.key, upload_status.READY) response = self.client.delete(endpoint) self.assertEqual( json.loads(response.data.decode()), expected_response ) self.assertFalse(os.path.exists(self.upload_path))
def redis_set_file_metadata(request_or_response_id, filepath, is_update=False): """ Stores a file's size, mime type, and hash. """ redis.set( _get_file_metadata_key(request_or_response_id, filepath, is_update), ':'.join((str(os.path.getsize(filepath)), os_get_mime_type(filepath), os_get_hash(filepath))))
def scan_and_complete_upload(request_id, filepath, is_update=False, response_id=None): """ Scans an uploaded file (see scan_file) and moves it to the data directory if it is clean. If is_update is set, the file will also be placed under the 'updated' directory. Updates redis accordingly. :param request_id: id of request associated with the upload :param filepath: path to uploaded and quarantined file :param is_update: will the file replace an existing one? :param response_id: id of response associated with the upload """ if is_update: assert response_id is not None else: assert response_id is None filename = os.path.basename(filepath) key = get_upload_key(request_id, filename, is_update) redis.set(key, upload_status.SCANNING) try: scan_file(filepath) except VirusDetectedException: sentry.captureException() redis.delete(key) else: # complete upload dst_dir = os.path.join( current_app.config['UPLOAD_DIRECTORY'], request_id ) if is_update: dst_dir = os.path.join( dst_dir, UPDATED_FILE_DIRNAME ) # store file metadata in redis redis_set_file_metadata(response_id or request_id, filepath, is_update) if not fu.exists(dst_dir): try: fu.makedirs(dst_dir) except OSError as e: sentry.captureException() # in the time between the call to fu.exists # and fu.makedirs, the directory was created current_app.logger.error("OS Error: {}".format(e.args)) fu.move( filepath, os.path.join(dst_dir, filename) ) redis.set(key, upload_status.READY)
def redis_set_file_metadata(request_or_response_id, filepath, is_update=False): """ Stores a file's size, mime type, and hash. """ redis.set( _get_file_metadata_key(request_or_response_id, filepath, is_update), ':'.join(( str(os.path.getsize(filepath)), os_get_mime_type(filepath), os_get_hash(filepath) )) )
def test_delete_missing_file(self): redis.set(self.key, upload_status.READY) response = self.client.delete( '/upload/request/{}/{}'.format( self.request_id, self.filename_encoded ) ) self.assertEqual( json.loads(response.data.decode()), { "error": "Upload not found." } )
def test_status(self): redis.set(self.key, upload_status.PROCESSING) response = self.client.get( '/upload/status', query_string={ "request_id": self.request_id, "filename": self.filename } ) self.assertEqual( json.loads(response.data.decode()), { "status": upload_status.PROCESSING } )
def _move_validated_upload(request_id, tmp_path): """ Move an approved upload to the upload directory. :param request_id: the id of the request associated with the upload :param tmp_path: the temporary file path to the upload generated by app.request.utils._quarantine_upload_no_id() """ dst_dir = os.path.join(current_app.config['UPLOAD_DIRECTORY'], request_id) if not fu.exists(dst_dir): fu.mkdir(dst_dir) valid_name = os.path.basename(tmp_path).split('.', 1)[1] # remove 'tmp' prefix valid_path = os.path.join(dst_dir, valid_name) # store file metadata in redis redis_set_file_metadata(request_id, tmp_path) fu.move(tmp_path, valid_path) upload_redis.set(get_upload_key(request_id, valid_name), upload_status.READY) return valid_path
def _move_validated_upload(request_id, tmp_path): """ Move an approved upload to the upload directory. :param request_id: the id of the request associated with the upload :param tmp_path: the temporary file path to the upload generated by app.request.utils._quarantine_upload_no_id() """ dst_dir = os.path.join( current_app.config['UPLOAD_DIRECTORY'], request_id) if not fu.exists(dst_dir): fu.mkdir(dst_dir) valid_name = os.path.basename(tmp_path).split('.', 1)[1] # remove 'tmp' prefix valid_path = os.path.join(dst_dir, valid_name) # store file metadata in redis redis_set_file_metadata(request_id, tmp_path) fu.move(tmp_path, valid_path) upload_redis.set( get_upload_key(request_id, valid_name), upload_status.READY) return valid_path
def post(request_id): """ Create a new upload. Handles chunked files through the Content-Range header. For filesize validation and more upload logic, see: /static/js/upload/fileupload.js Optional request body parameters: - update (bool) save the uploaded file to the 'updated' directory (this indicates the file is meant to replace a previously uploaded file) - response_id (int) the id of a response associated with the file this upload is replacing - REQUIRED if 'update' is 'true' - ignored if 'update' is 'false' :returns: { "name": file name, "size": file size } """ files = request.files file_ = files[next(files.keys())] filename = secure_filename(file_.filename) is_update = eval_request_bool(request.form.get('update')) agency_ein = Requests.query.filter_by(id=request_id).one().agency.ein if is_allowed(user=current_user, request_id=request_id, permission=permission.ADD_FILE) or \ is_allowed(user=current_user, request_id=request_id, permission=permission.EDIT_FILE): response_id = request.form.get('response_id') if is_update else None if upload_exists(request_id, filename, response_id): response = { "files": [{ "name": filename, "error": "A file with this name has already " "been uploaded for this request." # TODO: "link": <link-to-existing-file> ? would be nice }] } else: upload_path = os.path.join( current_app.config['UPLOAD_QUARANTINE_DIRECTORY'], request_id) if not os.path.exists(upload_path): os.mkdir(upload_path) filepath = os.path.join(upload_path, filename) key = get_upload_key(request_id, filename, is_update) try: if CONTENT_RANGE_HEADER in request.headers: start, size = parse_content_range( request.headers[CONTENT_RANGE_HEADER]) # Only validate mime type on first chunk valid_file_type = True file_type = None if start == 0: valid_file_type, file_type = is_valid_file_type(file_) if current_user.is_agency_active(agency_ein): valid_file_type = True if os.path.exists(filepath): # remove existing file (upload 'restarted' for same file) os.remove(filepath) if valid_file_type: redis.set(key, upload_status.PROCESSING) with open(filepath, 'ab') as fp: fp.seek(start) fp.write(file_.stream.read()) # scan if last chunk written if os.path.getsize(filepath) == size: scan_and_complete_upload.delay( request_id, filepath, is_update, response_id) else: valid_file_type, file_type = is_valid_file_type(file_) if current_user.is_agency_active(agency_ein): valid_file_type = True if valid_file_type: redis.set(key, upload_status.PROCESSING) file_.save(filepath) scan_and_complete_upload.delay(request_id, filepath, is_update, response_id) if not valid_file_type: response = { "files": [{ "name": filename, "error": "The file type '{}' is not allowed.".format( file_type) }] } else: response = { "files": [{ "name": filename, "original_name": file_.filename, "size": os.path.getsize(filepath), }] } except Exception as e: redis.set(key, upload_status.ERROR) current_app.logger.exception( "Upload for file '{}' failed: {}".format(filename, e)) response = { "files": [{ "name": filename, "error": "There was a problem uploading this file." }] } return jsonify(response), 200
def post(request_id): """ Create a new upload. Handles chunked files through the Content-Range header. For filesize validation and more upload logic, see: /static/js/upload/fileupload.js Optional request body parameters: - update (bool) save the uploaded file to the 'updated' directory (this indicates the file is meant to replace a previously uploaded file) - response_id (int) the id of a response associated with the file this upload is replacing - REQUIRED if 'update' is 'true' - ignored if 'update' is 'false' :returns: { "name": file name, "size": file size } """ files = request.files file_ = files[next(files.keys())] filename = secure_filename(file_.filename) is_update = eval_request_bool(request.form.get('update')) agency_ein = Requests.query.filter_by(id=request_id).one().agency.ein if is_allowed(user=current_user, request_id=request_id, permission=permission.ADD_FILE) or \ is_allowed(user=current_user, request_id=request_id, permission=permission.EDIT_FILE): response_id = request.form.get('response_id') if is_update else None if upload_exists(request_id, filename, response_id): response = { "files": [{ "name": filename, "error": "A file with this name has already " "been uploaded for this request." # TODO: "link": <link-to-existing-file> ? would be nice }] } else: upload_path = os.path.join( current_app.config['UPLOAD_QUARANTINE_DIRECTORY'], request_id) if not os.path.exists(upload_path): os.mkdir(upload_path) filepath = os.path.join(upload_path, filename) key = get_upload_key(request_id, filename, is_update) try: if CONTENT_RANGE_HEADER in request.headers: start, size = parse_content_range( request.headers[CONTENT_RANGE_HEADER]) # Only validate mime type on first chunk valid_file_type = True file_type = None if start == 0: valid_file_type, file_type = is_valid_file_type(file_) if current_user.is_agency_active(agency_ein): valid_file_type = True if os.path.exists(filepath): # remove existing file (upload 'restarted' for same file) os.remove(filepath) if valid_file_type: redis.set(key, upload_status.PROCESSING) with open(filepath, 'ab') as fp: fp.seek(start) fp.write(file_.stream.read()) # scan if last chunk written if os.path.getsize(filepath) == size: scan_and_complete_upload.delay(request_id, filepath, is_update, response_id) else: valid_file_type, file_type = is_valid_file_type(file_) if current_user.is_agency_active(agency_ein): valid_file_type = True if valid_file_type: redis.set(key, upload_status.PROCESSING) file_.save(filepath) scan_and_complete_upload.delay(request_id, filepath, is_update, response_id) if not valid_file_type: response = { "files": [{ "name": filename, "error": "The file type '{}' is not allowed.".format( file_type) }] } else: response = { "files": [{ "name": filename, "original_name": file_.filename, "size": os.path.getsize(filepath), }] } except Exception as e: sentry.captureException() redis.set(key, upload_status.ERROR) current_app.logger.exception("Upload for file '{}' failed: {}".format(filename, e)) response = { "files": [{ "name": filename, "error": "There was a problem uploading this file." }] } return jsonify(response), 200