class StaticLargeObject(object): """ StaticLargeObject Middleware See above for a full description. The proxy logs created for any subrequests made will have swift.source set to "SLO". :param app: The next WSGI filter or app in the paste.deploy chain. :param conf: The configuration dict for the middleware. """ def __init__(self, app, conf): self.conf = conf self.app = app self.logger = get_logger(conf, log_route='slo') self.max_manifest_segments = int(self.conf.get('max_manifest_segments', 1000)) self.max_manifest_size = int(self.conf.get('max_manifest_size', 1024 * 1024 * 2)) self.min_segment_size = int(self.conf.get('min_segment_size', 1024 * 1024)) self.bulk_deleter = Bulk( app, {'max_deletes_per_request': self.max_manifest_segments}) def handle_multipart_put(self, req): """ Will handle the PUT of a SLO manifest. Heads every object in manifest to check if is valid and if so will save a manifest generated from the user input. :params req: a swob.Request with an obj in path :raises: HttpException on errors """ try: vrs, account, container, obj = req.split_path(1, 4, True) except ValueError: return self.app if req.content_length > self.max_manifest_size: raise HTTPRequestEntityTooLarge( "Manifest File > %d bytes" % self.max_manifest_size) if req.headers.get('X-Copy-From'): raise HTTPMethodNotAllowed( 'Multipart Manifest PUTs cannot be Copy requests') if req.content_length is None and \ req.headers.get('transfer-encoding', '').lower() != 'chunked': raise HTTPLengthRequired(request=req) parsed_data = parse_input(req.body_file.read(self.max_manifest_size)) problem_segments = [] if len(parsed_data) > self.max_manifest_segments: raise HTTPRequestEntityTooLarge( 'Number segments must be <= %d' % self.max_manifest_segments) total_size = 0 out_content_type = req.accept.best_match(ACCEPTABLE_FORMATS) if not out_content_type: out_content_type = 'text/plain' data_for_storage = [] for index, seg_dict in enumerate(parsed_data): obj_path = '/'.join( ['', vrs, account, seg_dict['path'].lstrip('/')]) try: seg_size = int(seg_dict['size_bytes']) except (ValueError, TypeError): raise HTTPBadRequest('Invalid Manifest File') if seg_size < self.min_segment_size and \ (index == 0 or index < len(parsed_data) - 1): raise HTTPBadRequest( 'Each segment, except the last, must be larger than ' '%d bytes.' % self.min_segment_size) new_env = req.environ.copy() if isinstance(obj_path, unicode): obj_path = obj_path.encode('utf-8') new_env['PATH_INFO'] = obj_path new_env['REQUEST_METHOD'] = 'HEAD' new_env['swift.source'] = 'SLO' del(new_env['wsgi.input']) del(new_env['QUERY_STRING']) new_env['CONTENT_LENGTH'] = 0 new_env['HTTP_USER_AGENT'] = \ '%s MultipartPUT' % req.environ.get('HTTP_USER_AGENT') head_seg_resp = \ Request.blank(obj_path, new_env).get_response(self.app) if head_seg_resp.status_int // 100 == 2: total_size += seg_size if seg_size != head_seg_resp.content_length: problem_segments.append([quote(obj_path), 'Size Mismatch']) if seg_dict['etag'] != head_seg_resp.etag: problem_segments.append([quote(obj_path), 'Etag Mismatch']) if head_seg_resp.last_modified: last_modified = head_seg_resp.last_modified else: # shouldn't happen last_modified = datetime.now() last_modified_formatted = \ last_modified.strftime('%Y-%m-%dT%H:%M:%S.%f') data_for_storage.append( {'name': '/' + seg_dict['path'].lstrip('/'), 'bytes': seg_size, 'hash': seg_dict['etag'], 'content_type': head_seg_resp.content_type, 'last_modified': last_modified_formatted}) else: problem_segments.append([quote(obj_path), head_seg_resp.status]) if problem_segments: resp_body = get_response_body( out_content_type, {}, problem_segments) raise HTTPBadRequest(resp_body, content_type=out_content_type) env = req.environ if not env.get('CONTENT_TYPE'): guessed_type, _junk = mimetypes.guess_type(req.path_info) env['CONTENT_TYPE'] = guessed_type or 'application/octet-stream' env['swift.content_type_overriden'] = True env['CONTENT_TYPE'] += ";swift_bytes=%d" % total_size env['HTTP_X_STATIC_LARGE_OBJECT'] = 'True' json_data = json.dumps(data_for_storage) env['CONTENT_LENGTH'] = str(len(json_data)) env['wsgi.input'] = StringIO(json_data) return self.app def handle_multipart_delete(self, req): """ Will delete all the segments in the SLO manifest and then, if successful, will delete the manifest file. :params req: a swob.Request with an obj in path :raises HTTPServerError: on invalid manifest :returns: swob.Response on failure, otherwise self.app """ new_env = req.environ.copy() new_env['REQUEST_METHOD'] = 'GET' del(new_env['wsgi.input']) new_env['QUERY_STRING'] = 'multipart-manifest=get' new_env['CONTENT_LENGTH'] = 0 new_env['HTTP_USER_AGENT'] = \ '%s MultipartDELETE' % req.environ.get('HTTP_USER_AGENT') new_env['swift.source'] = 'SLO' get_man_resp = \ Request.blank('', new_env).get_response(self.app) if get_man_resp.status_int // 100 == 2: if not config_true_value( get_man_resp.headers.get('X-Static-Large-Object')): raise HTTPBadRequest('Not an SLO manifest') try: manifest = json.loads(get_man_resp.body) except ValueError: raise HTTPServerError('Invalid manifest file') delete_resp = self.bulk_deleter.handle_delete( req, objs_to_delete=[o['name'].encode('utf-8') for o in manifest], user_agent='MultipartDELETE', swift_source='SLO') if delete_resp.status_int // 100 == 2: # delete the manifest file itself return self.app else: return delete_resp return get_man_resp @wsgify def __call__(self, req): """ WSGI entry point """ try: vrs, account, container, obj = req.split_path(1, 4, True) except ValueError: return self.app if obj: if req.method == 'PUT' and \ req.params.get('multipart-manifest') == 'put': return self.handle_multipart_put(req) if req.method == 'DELETE' and \ req.params.get('multipart-manifest') == 'delete': return self.handle_multipart_delete(req) if 'X-Static-Large-Object' in req.headers: raise HTTPBadRequest( request=req, body='X-Static-Large-Object is a reserved header. ' 'To create a static large object add query param ' 'multipart-manifest=put.') return self.app
class StaticLargeObject(object): def __init__(self, app, conf): self.conf = conf self.app = app self.logger = get_logger(conf, log_route='slo') self.max_manifest_segments = int(self.conf.get('max_manifest_segments', 1000)) self.max_manifest_size = int(self.conf.get('max_manifest_size', 1024 * 1024 * 2)) self.min_segment_size = int(self.conf.get('min_segment_size', 1024 * 1024)) self.bulk_deleter = Bulk( app, {'max_deletes_per_request': self.max_manifest_segments}) def handle_multipart_put(self, req): try: vrs, account, container, obj = split_path(req.path,1, 4, True) except ValueError: return self.app if req.content_length > self.max_manifest_size: raise HTTPRequestEntityTooLarge( "Manifest File > %d bytes" % self.max_manifest_size) if req.headers.get('X-Copy-From') or req.headers.get('Destination'): raise HTTPMethodNotAllowed( 'Multipart Manifest PUTs cannot be Copy requests') if req.content_length is None and \ req.headers.get('transfer-encoding', '').lower() != 'chunked': raise HTTPLengthRequired(request=req) parsed_data = parse_input(req.environ['wsgi.input'].read(self.max_manifest_size)) problem_segments = [] if len(parsed_data) > self.max_manifest_segments: raise HTTPRequestEntityTooLarge( 'Number segments must be <= %d' % self.max_manifest_segments) total_size = 0 out_content_type = 'application/json' if not out_content_type: out_content_type = 'text/plain' data_for_storage = [] for index, seg_dict in enumerate(parsed_data): obj_path = '/'.join( ['', vrs, account, seg_dict['path'].lstrip('/')]) try: seg_size = int(seg_dict['size_bytes']) except (ValueError, TypeError): raise HTTPBadRequest('Invalid Manifest File') new_env = req.environ.copy() if isinstance(obj_path, unicode): obj_path = obj_path.encode('utf-8') new_env['PATH_INFO'] = obj_path new_env['REQUEST_METHOD'] = 'HEAD' new_env['swift.source'] = 'SLO' del(new_env['wsgi.input']) del(new_env['QUERY_STRING']) new_env['CONTENT_LENGTH'] = 0 new_env['HTTP_USER_AGENT'] = \ '%s MultipartPUT' % req.environ.get('HTTP_USER_AGENT') head_seg_resp = \ Request.blank(obj_path, new_env).get_response(self.app) if head_seg_resp.status_int // 100 == 2: total_size += seg_size if seg_size != head_seg_resp.content_length: problem_segments.append([quote(obj_path), 'Size Mismatch']) if seg_dict['etag'] != head_seg_resp.etag: problem_segments.append([quote(obj_path), 'Etag Mismatch']) data_for_storage.append( {'name': '/' + seg_dict['path'].lstrip('/'), 'bytes': seg_size, 'hash': seg_dict['etag']}) else: problem_segments.append([quote(obj_path), head_seg_resp.status]) if problem_segments: resp_body = get_response_body( out_content_type, {}, problem_segments) raise jresponse('-1','badrequest',req,400,param=resp_body) env = req.environ env['swift.content_type_overriden'] = True env['HTTP_X_STATIC_LARGE_OBJECT'] = 'True' json_data = json.dumps(data_for_storage) env['CONTENT_LENGTH'] = str(len(json_data)) env['wsgi.input'] = StringIO(json_data) return self.app def handle_multipart_delete(self, req): new_env = req.environ.copy() new_env['REQUEST_METHOD'] = 'GET' del(new_env['wsgi.input']) new_env['QUERY_STRING'] = 'multipart-manifest=get' new_env['CONTENT_LENGTH'] = 0 new_env['HTTP_USER_AGENT'] = \ '%s MultipartDELETE' % req.environ.get('HTTP_USER_AGENT') new_env['swift.source'] = 'SLO' get_man_resp = \ Request.blank('', new_env).get_response(self.app) if get_man_resp.status_int // 100 == 2: if not config_true_value( get_man_resp.headers.get('X-Static-Large-Object')): raise HTTPBadRequest('Not an SLO manifest') try: manifest = json.loads(get_man_resp.body) except ValueError: raise HTTPServerError('Invalid manifest file') delete_resp = self.bulk_deleter.handle_delete( req, objs_to_delete=[o['name'].encode('utf-8') for o in manifest], user_agent='MultipartDELETE', swift_source='SLO') if delete_resp.status_int // 100 == 2: # delete the manifest file itself return self.app else: return delete_resp return get_man_resp @wsgify def __call__(self, req): """ WSGI entry point """ try: vrs, account, container, obj = split_path(req.path,1, 4, True) except ValueError: return self.app if obj: if req.method == 'PUT' and \ req.GET.get('multipart-manifest') == 'put': return self.handle_multipart_put(req) if req.method == 'DELETE' and \ req.GET.get('multipart-manifest') == 'delete': return self.handle_multipart_delete(req) if 'X-Static-Large-Object' in req.headers: raise HTTPBadRequest( request=req, body='X-Static-Large-Object is a reserved header. ' 'To create a static large object add query param ' 'multipart-manifest=put.') return self.app