def validate_metadata(metadata): """ Validates that metadata falls within acceptable limits. :param metadata: to be validated :raises HTTPBadRequest: if MAX_META_COUNT or MAX_META_OVERALL_SIZE is exceeded, or if metadata contains non-UTF-8 data """ meta_count = 0 meta_size = 0 for key, (value, timestamp) in metadata.items(): if key and not isinstance(key, six.text_type): if not check_utf8(key): raise HTTPBadRequest('Metadata must be valid UTF-8') # Promote to a natural string for the checks below if six.PY3: key = key.decode('utf8') if value and not isinstance(value, six.text_type): if not check_utf8(value): raise HTTPBadRequest('Metadata must be valid UTF-8') key = key.lower() if value and key.startswith(('x-account-meta-', 'x-container-meta-')): prefix = 'x-account-meta-' if key.startswith('x-container-meta-'): prefix = 'x-container-meta-' key = key[len(prefix):] meta_count = meta_count + 1 meta_size = meta_size + len(key) + len(value) if meta_count > MAX_META_COUNT: raise HTTPBadRequest('Too many metadata items; max %d' % MAX_META_COUNT) if meta_size > MAX_META_OVERALL_SIZE: raise HTTPBadRequest('Total metadata too large; max %d' % MAX_META_OVERALL_SIZE)
def validate_metadata(metadata): """ Validates that metadata falls within acceptable limits. :param metadata: to be validated :raises: HTTPBadRequest if MAX_META_COUNT or MAX_META_OVERALL_SIZE is exceeded, or if metadata contains non-UTF-8 data """ meta_count = 0 meta_size = 0 for key, (value, timestamp) in metadata.items(): key = key.lower() if value != '' and (key.startswith('x-account-meta') or key.startswith('x-container-meta')): prefix = 'x-account-meta-' if key.startswith('x-container-meta-'): prefix = 'x-container-meta-' key = key[len(prefix):] meta_count = meta_count + 1 meta_size = meta_size + len(key) + len(value) bad_key = key and not check_utf8(key) bad_value = value and not check_utf8(value) if bad_key or bad_value: raise HTTPBadRequest('Metadata must be valid UTF-8') if meta_count > MAX_META_COUNT: raise HTTPBadRequest('Too many metadata items; max %d' % MAX_META_COUNT) if meta_size > MAX_META_OVERALL_SIZE: raise HTTPBadRequest('Total metadata too large; max %d' % MAX_META_OVERALL_SIZE)
def PUT(self, req): """Handle HTTP PUT request.""" try: drive, part, account, container = split_path(unquote(req.path), 3, 4) if (account and not check_utf8(account)) or \ (container and not check_utf8(container)): raise ValueError('NULL characters not allowed in names') except ValueError, err: return HTTPBadRequest(body=str(err), content_type='text/plain', request=req)
def test_check_utf8(self): unicode_sample = u"\uc77c\uc601" valid_utf8_str = unicode_sample.encode("utf-8") invalid_utf8_str = unicode_sample.encode("utf-8")[::-1] unicode_with_null = u"abc\u0000def" utf8_with_null = unicode_with_null.encode("utf-8") for false_argument in [None, "", invalid_utf8_str, unicode_with_null, utf8_with_null]: self.assertFalse(constraints.check_utf8(false_argument)) for true_argument in ["this is ascii and utf-8, too", unicode_sample, valid_utf8_str]: self.assertTrue(constraints.check_utf8(true_argument))
def test_check_utf8(self): unicode_sample = u'\uc77c\uc601' valid_utf8_str = unicode_sample.encode('utf-8') invalid_utf8_str = unicode_sample.encode('utf-8')[::-1] for false_argument in [None, '', invalid_utf8_str, ]: self.assertFalse(constraints.check_utf8(false_argument)) for true_argument in ['this is ascii and utf-8, too', unicode_sample, valid_utf8_str]: self.assertTrue(constraints.check_utf8(true_argument))
def __call__(self, env, start_response): start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(wsgi_to_str(req.path_info), internal=True): res = HTTPPreconditionFailed(body='Invalid UTF8') else: try: # disallow methods which are not publicly accessible if req.method not in self.allowed_methods: res = HTTPMethodNotAllowed() else: res = getattr(self, req.method)(req) except HTTPException as error_response: res = error_response except (Exception, Timeout): self.logger.exception(_('ERROR __call__ error with %(method)s' ' %(path)s '), {'method': req.method, 'path': req.path}) res = HTTPInternalServerError(body=traceback.format_exc()) if self.log_requests: trans_time = time.time() - start_time additional_info = '' if res.headers.get('x-container-timestamp') is not None: additional_info += 'x-container-timestamp: %s' % \ res.headers['x-container-timestamp'] log_msg = get_log_line(req, res, trans_time, additional_info, self.log_format, self.anonymization_method, self.anonymization_salt) if req.method.upper() == 'REPLICATE': self.logger.debug(log_msg) else: self.logger.info(log_msg) return res(env, start_response)
def handle_request(self, req): try: self.logger.set_statsd_prefix('proxy-server') if req.content_length and req.content_length < 0: return jresponse('-1','Invalid Content-Length',req,400) try: if not check_utf8(req.path_info): return jresponse('-1','Invalid UTF8',req,412) except UnicodeError: return jresponse('-1','Invalid UTF8',req,412) try: controller, path_parts = self.get_controller(req) p = req.path_info if isinstance(p, unicode): p = p.encode('utf-8') except ValueError: return jresponse('-1','not found',req,404) if not controller: return jresponse('-1','Bad URL',req,412) if self.deny_host_headers and \ req.host.split(':')[0] in self.deny_host_headers: return HTTPForbidden(request=req, body='Invalid host header') if not check_path_parts(path_parts): return HTTPForbidden(request=req, body='Invalid path_parts header') self.logger.set_statsd_prefix('proxy-server.' + controller.server_type.lower()) controller = controller(self, **path_parts) if 'swift.trans_id' not in req.environ: # if this wasn't set by an earlier middleware, set it now trans_id = 'tx' + uuid.uuid4().hex req.environ['swift.trans_id'] = trans_id self.logger.txn_id = trans_id req.headers['x-trans-id'] = req.environ['swift.trans_id'] controller.trans_id = req.environ['swift.trans_id'] self.logger.client_ip = get_remote_client(req) try: if req.GET.get('op'): req.method = req.GET.get('op') handler = getattr(controller, req.method) getattr(handler, 'publicly_accessible') except AttributeError: return HTTPMethodNotAllowed(request=req) if path_parts['version']: req.path_info_pop() req.environ['swift.orig_req_method'] = req.method return handler(req) except (Exception, Timeout): self.logger.exception(_('ERROR Unhandled exception in request')) return jresponse('-1','ServerERROR',req,500)
def __call__(self, env, start_response): start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8') else: try: # disallow methods which have not been marked 'public' try: method = getattr(self, req.method) getattr(method, 'publicly_accessible') except AttributeError: res = HTTPMethodNotAllowed() else: res = method(req) except (Exception, Timeout): self.logger.exception(_('ERROR __call__ error with %(method)s' ' %(path)s '), {'method': req.method, 'path': req.path}) res = HTTPInternalServerError(body=traceback.format_exc()) trans_time = '%.4f' % (time.time() - start_time) log_message = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %s' % ( req.remote_addr, time.strftime('%d/%b/%Y:%H:%M:%S +0000', time.gmtime()), req.method, req.path, res.status.split()[0], res.content_length or '-', req.headers.get('x-trans-id', '-'), req.referer or '-', req.user_agent or '-', trans_time) if req.method.upper() == 'REPLICATE': self.logger.debug(log_message) else: self.logger.info(log_message) return res(env, start_response)
def handle_extract(self, req, compress_type): """ :params req: a swob Request :params compress_type: specifying the compression type of the tar. Accepts '', 'gz, or 'bz2' :raises HTTPException: on unhandled errors :returns: a swob response to request """ success_count = 0 failed_files = [] existing_containers = set() out_content_type = req.accept.best_match(ACCEPTABLE_FORMATS) if not out_content_type: return HTTPNotAcceptable(request=req) if req.content_length is None and req.headers.get("transfer-encoding", "").lower() != "chunked": return HTTPBadRequest("Invalid request: no content sent.") try: vrs, account, extract_base = req.split_path(2, 3, True) except ValueError: return HTTPNotFound(request=req) extract_base = extract_base or "" extract_base = extract_base.rstrip("/") try: tar = tarfile.open(mode="r|" + compress_type, fileobj=req.body_file) while True: tar_info = tar.next() if tar_info is None or len(failed_files) >= self.max_failed_extractions: break if tar_info.isfile(): obj_path = tar_info.name if obj_path.startswith("./"): obj_path = obj_path[2:] obj_path = obj_path.lstrip("/") if extract_base: obj_path = extract_base + "/" + obj_path if "/" not in obj_path: continue # ignore base level file destination = "/".join(["", vrs, account, obj_path]) container = obj_path.split("/", 1)[0] if not check_utf8(destination): failed_files.append([quote(destination[:MAX_PATH_LENGTH]), HTTPPreconditionFailed().status]) continue if tar_info.size > MAX_FILE_SIZE: failed_files.append([quote(destination[:MAX_PATH_LENGTH]), HTTPRequestEntityTooLarge().status]) continue if container not in existing_containers: try: self.create_container(req, "/".join(["", vrs, account, container])) existing_containers.add(container) except CreateContainerError, err: if err.status_int == HTTP_UNAUTHORIZED: return HTTPUnauthorized(request=req) failed_files.append([quote(destination[:MAX_PATH_LENGTH]), err.status]) continue except ValueError: failed_files.append([quote(destination[:MAX_PATH_LENGTH]), HTTP_BAD_REQUEST]) continue if len(existing_containers) > self.max_containers: return HTTPBadRequest("More than %d base level containers in tar." % self.max_containers)
def __call__(self, env, start_response): start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL') else: try: # disallow methods which have not been marked 'public' try: if req.method not in self.allowed_methods: raise AttributeError('Not allowed method.') except AttributeError: res = HTTPMethodNotAllowed() else: method = getattr(self, req.method) res = method(req) except HTTPException as error_response: res = error_response except (Exception, Timeout): self.logger.exception(_( 'ERROR __call__ error with %(method)s %(path)s '), {'method': req.method, 'path': req.path}) res = HTTPInternalServerError(body=traceback.format_exc()) if self.log_requests: trans_time = time.time() - start_time log_message = get_log_line(req, res, trans_time, '') if req.method.upper() == 'REPLICATE': self.logger.debug(log_message) else: self.logger.info(log_message) return res(env, start_response)
def get_segments_to_delete_iter(self, req): """ A generator function to be used to delete all the segments and sub-segments referenced in a manifest. :params req: a swob.Request with an SLO manifest in path :raises HTTPPreconditionFailed: on invalid UTF8 in request path :raises HTTPBadRequest: on too many buffered sub segments and on invalid SLO manifest path """ if not check_utf8(req.path_info): raise HTTPPreconditionFailed(request=req, body="Invalid UTF8 or contains NULL") vrs, account, container, obj = req.split_path(4, 4, True) segments = [{"sub_slo": True, "name": ("/%s/%s" % (container, obj)).decode("utf-8")}] while segments: if len(segments) > MAX_BUFFERED_SLO_SEGMENTS: raise HTTPBadRequest("Too many buffered slo segments to delete.") seg_data = segments.pop(0) if seg_data.get("sub_slo"): try: segments.extend(self.get_slo_segments(seg_data["name"], req)) except HTTPException as err: # allow bulk delete response to report errors seg_data["error"] = {"code": err.status_int, "message": err.body} # add manifest back to be deleted after segments seg_data["sub_slo"] = False segments.append(seg_data) else: seg_data["name"] = seg_data["name"].encode("utf-8") yield seg_data
def __call__(self, env, start_response): start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get("x-trans-id", None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body="Invalid UTF8 or contains NULL") else: try: # disallow methods which are not publicly accessible try: if req.method not in self.allowed_methods: raise AttributeError("Not allowed method.") except AttributeError: res = HTTPMethodNotAllowed() else: method = getattr(self, req.method) res = method(req) except HTTPException as error_response: res = error_response except (Exception, Timeout): self.logger.exception( _("ERROR __call__ error with %(method)s" " %(path)s "), {"method": req.method, "path": req.path} ) res = HTTPInternalServerError(body=traceback.format_exc()) if self.log_requests: trans_time = time.time() - start_time additional_info = "" if res.headers.get("x-container-timestamp") is not None: additional_info += "x-container-timestamp: %s" % res.headers["x-container-timestamp"] log_msg = get_log_line(req, res, trans_time, additional_info) if req.method.upper() == "REPLICATE": self.logger.debug(log_msg) else: self.logger.info(log_msg) return res(env, start_response)
def __call__(self, env, start_response): req = Request(env) if 'd' != req.headers.get('X-Ftype'): return self.app(env,start_response) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = jresponse('-1', 'invalid UTF8', req,412) else: try: # disallow methods which have not been marked 'public' try: method = getattr(self, req.method) getattr(method, 'publicly_accessible') except AttributeError: res = jresponse('-1', 'method not allowed', req,405) else: res = method(req) except (Exception, Timeout): self.logger.exception(_('ERROR __call__ error with %(method)s' ' %(path)s '), {'method': req.method, 'path': req.path}) res = jresponse('-1', 'internal server error', req,500) return res(env, start_response)
def __call__(self, env, start_response): start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8') else: try: if hasattr(self, req.method): res = getattr(self, req.method)(req) else: res = HTTPMethodNotAllowed() except (Exception, Timeout): self.logger.exception(_('ERROR __call__ error with %(method)s' ' %(path)s '), {'method': req.method, 'path': req.path}) res = HTTPInternalServerError(body=traceback.format_exc()) trans_time = '%.4f' % (time.time() - start_time) additional_info = '' if res.headers.get('x-container-timestamp') is not None: additional_info += 'x-container-timestamp: %s' % \ res.headers['x-container-timestamp'] log_message = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %s "%s"' % ( req.remote_addr, time.strftime('%d/%b/%Y:%H:%M:%S +0000', time.gmtime()), req.method, req.path, res.status.split()[0], res.content_length or '-', req.headers.get('x-trans-id', '-'), req.referer or '-', req.user_agent or '-', trans_time, additional_info) if req.method.upper() == 'REPLICATE': self.logger.debug(log_message) else: self.logger.info(log_message) return res(env, start_response)
def __call__(self, env, start_response): start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get("x-trans-id", None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body="Invalid UTF8") else: try: if hasattr(self, req.method): res = getattr(self, req.method)(req) else: res = HTTPMethodNotAllowed() except (Exception, Timeout): self.logger.exception( _("ERROR __call__ error with %(method)s" " %(path)s "), {"method": req.method, "path": req.path} ) res = HTTPInternalServerError(body=traceback.format_exc()) trans_time = "%.4f" % (time.time() - start_time) log_message = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %s' % ( req.remote_addr, time.strftime("%d/%b/%Y:%H:%M:%S +0000", time.gmtime()), req.method, req.path, res.status.split()[0], res.content_length or "-", req.headers.get("x-trans-id", "-"), req.referer or "-", req.user_agent or "-", trans_time, ) if req.method.upper() == "REPLICATE": self.logger.debug(log_message) else: self.logger.info(log_message) return res(env, start_response)
def __call__(self, env, start_response): start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = jresponse('-1','Invalid UTF8',req,412) else: try: # disallow methods which have not been marked 'public' try: method = getattr(self, req.method) getattr(method, 'publicly_accessible') except AttributeError: res = jresponse('-1', 'method not allowed', req,405) else: res = method(req) # if req.method == 'PUT': # print 'path: '+req.path + ' status: '+str(res.status_int) + ' msg: '+res.body except (Exception, Timeout): self.logger.exception(_('ERROR __call__ error with %(method)s' ' %(path)s '), {'method': req.method, 'path': req.path}) res = jresponse('-1', 'InternalServerError', req,500) return res(env, start_response)
def __call__(self, env, start_response): """WSGI Application entry point for the Swift Object Server.""" start_time = time.time() req = Request(env) if 'l' != req.headers.get('X-Ftype'): return self.app(env,start_response) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res =jresponse('-1', 'Invalid UTF8', req,412) else: try: # disallow methods which have not been marked 'public' try: method = getattr(self, req.method) getattr(method, 'publicly_accessible') except AttributeError: res = jresponse('-1', 'method not allowed', req,405) else: res = method(req) except (Exception, Timeout): self.logger.exception(_('ERROR __call__ error with %(method)s' ' %(path)s '), {'method': req.method, 'path': req.path}) res = jresponse('-1', 'InternalServerError', req,500) trans_time = time.time() - start_time if req.method in ('PUT', 'DELETE'): slow = self.slow - trans_time if slow > 0: sleep(slow) return res(env, start_response)
def __call__(self, env, start_response): start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL') else: try: # disallow methods which have not been marked 'public' if req.method not in self.allowed_methods: res = HTTPMethodNotAllowed() else: res = getattr(self, req.method)(req) except HTTPException as error_response: res = error_response except (Exception, Timeout): self.logger.exception( _('ERROR __call__ error with %(method)s %(path)s '), { 'method': req.method, 'path': req.path }) res = HTTPInternalServerError(body=traceback.format_exc()) if self.log_requests: trans_time = time.time() - start_time log_message = get_log_line(req, res, trans_time, '') if req.method.upper() == 'REPLICATE': self.logger.debug(log_message) else: self.logger.info(log_message) return res(env, start_response)
def test_check_utf8(self): unicode_sample = u'\uc77c\uc601' valid_utf8_str = unicode_sample.encode('utf-8') invalid_utf8_str = unicode_sample.encode('utf-8')[::-1] unicode_with_null = u'abc\u0000def' utf8_with_null = unicode_with_null.encode('utf-8') for false_argument in [ None, '', invalid_utf8_str, unicode_with_null, utf8_with_null ]: self.assertFalse(constraints.check_utf8(false_argument)) for true_argument in [ 'this is ascii and utf-8, too', unicode_sample, valid_utf8_str ]: self.assertTrue(constraints.check_utf8(true_argument))
def delete_filter(predicate, objs_to_delete): for obj_to_delete in objs_to_delete: obj_name = obj_to_delete['name'] if not obj_name: continue if not predicate(obj_name): continue if obj_to_delete.get('error'): if obj_to_delete['error']['code'] == HTTP_NOT_FOUND: resp_dict['Number Not Found'] += 1 else: failed_files.append([ quote(obj_name), obj_to_delete['error']['message'] ]) continue delete_path = '/'.join( ['', vrs, account, obj_name.lstrip('/')]) if not constraints.check_utf8(delete_path): failed_files.append( [quote(obj_name), HTTPPreconditionFailed().status]) continue yield (obj_name, delete_path)
def __call__(self, env, start_response): start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get('x-cf-trans-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8') else: try: if hasattr(self, req.method): res = getattr(self, req.method)(req) else: res = HTTPMethodNotAllowed() except Exception: self.logger.exception(_('ERROR __call__ error with %(method)s' ' %(path)s '), {'method': req.method, 'path': req.path}) res = HTTPInternalServerError(body=traceback.format_exc()) trans_time = '%.4f' % (time.time() - start_time) additional_info = '' if res.headers.get('x-container-timestamp') is not None: additional_info += 'x-container-timestamp: %s' % \ res.headers['x-container-timestamp'] log_message = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %s "%s"' % ( req.remote_addr, time.strftime('%d/%b/%Y:%H:%M:%S +0000', time.gmtime()), req.method, req.path, res.status.split()[0], res.content_length or '-', req.headers.get('x-cf-trans-id', '-'), req.referer or '-', req.user_agent or '-', trans_time, additional_info) if req.method.upper() == 'REPLICATE': self.logger.debug(log_message) else: self.logger.info(log_message) return res(env, start_response)
def __call__(self, env, start_response): start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL') else: try: # disallow methods which are not publicly accessible if req.method not in self.allowed_methods: res = HTTPMethodNotAllowed() else: res = getattr(self, req.method)(req) except HTTPException as error_response: res = error_response except (Exception, Timeout): self.logger.exception(_('ERROR __call__ error with %(method)s' ' %(path)s '), {'method': req.method, 'path': req.path}) res = HTTPInternalServerError(body=traceback.format_exc()) if self.log_requests: trans_time = time.time() - start_time additional_info = '' if res.headers.get('x-container-timestamp') is not None: additional_info += 'x-container-timestamp: %s' % \ res.headers['x-container-timestamp'] log_msg = get_log_line(req, res, trans_time, additional_info, self.log_format, self.anonymization_method, self.anonymization_salt) if req.method.upper() == 'REPLICATE': self.logger.debug(log_msg) else: self.logger.info(log_msg) return res(env, start_response)
def __call__(self, env, start_response): start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = jresponse('-1', 'invalid utf8', req,412) else: try: # disallow methods which are not publicly accessible try: method = getattr(self, req.method) getattr(method, 'publicly_accessible') except AttributeError: res = jresponse('-1', 'method not allowed', req,405) else: res = method(req) except (Exception, Timeout): self.logger.exception(_('ERROR __call__ error with %(method)s' ' %(path)s '), {'method': req.method, 'path': req.path}) res = jresponse('-1', 'InternalServerError', req,500) additional_info = '' if res.headers.get('x-container-timestamp') is not None: additional_info += 'x-container-timestamp: %s' % \ res.headers['x-container-timestamp'] return res(env, start_response)
def handle_delete(self, req): """ :params req: a swob Request :raises HTTPException: on unhandled errors :returns: a swob Response """ try: vrs, account, _junk = req.split_path(2, 3, True) except ValueError: return HTTPNotFound(request=req) incoming_format = req.headers.get("Content-Type") if incoming_format and not incoming_format.startswith("text/plain"): # For now only accept newline separated object names return HTTPNotAcceptable(request=req) out_content_type = req.accept.best_match(ACCEPTABLE_FORMATS) if not out_content_type: return HTTPNotAcceptable(request=req) objs_to_delete = self.get_objs_to_delete(req) failed_files = [] success_count = not_found_count = 0 failed_file_response_type = HTTPBadRequest for obj_to_delete in objs_to_delete: obj_to_delete = obj_to_delete.strip().lstrip("/") if not obj_to_delete: continue obj_to_delete = unquote(obj_to_delete) delete_path = "/".join(["", vrs, account, obj_to_delete]) if not check_utf8(delete_path): failed_files.append([quote(delete_path), HTTPPreconditionFailed().status]) continue new_env = req.environ.copy() new_env["PATH_INFO"] = delete_path del (new_env["wsgi.input"]) new_env["CONTENT_LENGTH"] = 0 new_env["HTTP_USER_AGENT"] = "%s BulkDelete" % req.environ.get("HTTP_USER_AGENT") delete_obj_req = Request.blank(delete_path, new_env) resp = delete_obj_req.get_response(self.app) if resp.status_int // 100 == 2: success_count += 1 elif resp.status_int == HTTP_NOT_FOUND: not_found_count += 1 elif resp.status_int == HTTP_UNAUTHORIZED: return HTTPUnauthorized(request=req) else: if resp.status_int // 100 == 5: failed_file_response_type = HTTPBadGateway failed_files.append([quote(delete_path), resp.status]) resp_body = self.get_response_body( out_content_type, {"Number Deleted": success_count, "Number Not Found": not_found_count}, failed_files ) if (success_count or not_found_count) and not failed_files: return HTTPOk(resp_body, content_type=out_content_type) if failed_files: return failed_file_response_type(resp_body, content_type=out_content_type) return HTTPBadRequest("Invalid bulk delete.")
def test_check_utf8_lone_surrogates(self): self.assertFalse(constraints.check_utf8(b'\xed\xa0\xbc')) self.assertFalse(constraints.check_utf8(u'\ud83c')) self.assertFalse(constraints.check_utf8(b'\xed\xb9\x88')) self.assertFalse(constraints.check_utf8(u'\ude48')) self.assertFalse(constraints.check_utf8(u'\ud800')) self.assertFalse(constraints.check_utf8(u'\udc00')) self.assertFalse(constraints.check_utf8(u'\udcff')) self.assertFalse(constraints.check_utf8(u'\udfff'))
def __call__(self, env, start_response): """WSGI Application entry point for the Swift Object Server.""" start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get("x-trans-id", None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body="Invalid UTF8 or contains NULL") else: try: # disallow methods which have not been marked 'public' try: method = getattr(self, req.method) getattr(method, "publicly_accessible") replication_method = getattr(method, "replication", False) if self.replication_server is not None and self.replication_server != replication_method: raise AttributeError("Not allowed method.") except AttributeError: res = HTTPMethodNotAllowed() else: res = method(req) except DiskFileCollision: res = HTTPForbidden(request=req) except HTTPException as error_response: res = error_response except (Exception, Timeout): self.logger.exception( _("ERROR __call__ error with %(method)s" " %(path)s "), {"method": req.method, "path": req.path} ) res = HTTPInternalServerError(body=traceback.format_exc()) trans_time = time.time() - start_time if self.log_requests: log_line = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %.4f' % ( req.remote_addr, time.strftime("%d/%b/%Y:%H:%M:%S +0000", time.gmtime()), req.method, req.path, res.status.split()[0], res.content_length or "-", req.referer or "-", req.headers.get("x-trans-id", "-"), req.user_agent or "-", trans_time, ) if req.method in ("REPLICATE", "REPLICATION") or "X-Backend-Replication" in req.headers: self.logger.debug(log_line) else: self.logger.info(log_line) if req.method in ("PUT", "DELETE"): slow = self.slow - trans_time if slow > 0: sleep(slow) return res(env, start_response)
def test_check_utf8_non_canonical(self): self.assertFalse(constraints.check_utf8(b'\xed\xa0\xbc\xed\xbc\xb8')) self.assertTrue(constraints.check_utf8(u'\U0001f338')) self.assertTrue(constraints.check_utf8(b'\xf0\x9f\x8c\xb8')) self.assertTrue(constraints.check_utf8(u'\U0001f338'.encode('utf8'))) self.assertFalse(constraints.check_utf8(b'\xed\xa0\xbd\xed\xb9\x88')) self.assertTrue(constraints.check_utf8(u'\U0001f648'))
def __call__(self, env, start_response): """WSGI Application entry point for the Swift Object Server.""" start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL') else: try: # disallow methods which have not been marked 'public' try: method = getattr(self, req.method) getattr(method, 'publicly_accessible') replication_method = getattr(method, 'replication', False) if (self.replication_server is not None and self.replication_server != replication_method): raise AttributeError('Not allowed method.') except AttributeError: res = HTTPMethodNotAllowed() else: res = method(req) except DiskFileCollision: res = HTTPForbidden(request=req) except HTTPException as error_response: res = error_response except (Exception, Timeout): self.logger.exception(_( 'ERROR __call__ error with %(method)s' ' %(path)s '), {'method': req.method, 'path': req.path}) res = HTTPInternalServerError(body=traceback.format_exc()) trans_time = time.time() - start_time if self.log_requests: log_line = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %.4f' % ( req.remote_addr, time.strftime('%d/%b/%Y:%H:%M:%S +0000', time.gmtime()), req.method, req.path, res.status.split()[0], res.content_length or '-', req.referer or '-', req.headers.get('x-trans-id', '-'), req.user_agent or '-', trans_time) if req.method in ('REPLICATE', 'REPLICATION') or \ 'X-Backend-Replication' in req.headers: self.logger.debug(log_line) else: self.logger.info(log_line) if req.method in ('PUT', 'DELETE'): slow = self.slow - trans_time if slow > 0: sleep(slow) return res(env, start_response)
def split_path(path, minsegs=1, maxsegs=None, rest_with_last=False): """ This is a copy/paste of swift.common.utils.split_path. I will probably be adding the unquote into swift's version soon and this will prevent dependencies on that. Validate and split the given HTTP request path. **Examples**:: ['a'] = split_path('/a') ['a', None] = split_path('/a', 1, 2) ['a', 'c'] = split_path('/a/c', 1, 2) ['a', 'c', 'o/r'] = split_path('/a/c/o/r', 1, 3, True) :param path: HTTP Request path to be split :param minsegs: Minimum number of segments to be extracted :param maxsegs: Maximum number of segments to be extracted :param rest_with_last: If True, trailing data will be returned as part of last segment. If False, and there is trailing data, raises ValueError. :returns: list of segments with a length of maxsegs (non-existant segments will return as None) :raises: ValueError if given an invalid path :raises: InvalidUtf8 if path contains invalid UTF-8 """ path = unquote(path) if not check_utf8(path): raise InvalidUtf8('Invalid UTF8') if not maxsegs: maxsegs = minsegs if minsegs > maxsegs: raise ValueError('minsegs > maxsegs: %d > %d' % (minsegs, maxsegs)) if rest_with_last: segs = path.split('/', maxsegs) minsegs += 1 maxsegs += 1 count = len(segs) if segs[0] or count < minsegs or count > maxsegs or \ '' in segs[1:minsegs]: raise ValueError('Invalid path: %s' % quote(path)) else: minsegs += 1 maxsegs += 1 segs = path.split('/', maxsegs) count = len(segs) if segs[0] or count < minsegs or count > maxsegs + 1 or \ '' in segs[1:minsegs] or (count == maxsegs + 1 and segs[maxsegs]): raise ValueError('Invalid path: %s' % quote(path)) segs = segs[1:maxsegs] segs.extend([None] * (maxsegs - 1 - len(segs))) return segs
def test_check_utf8(self): unicode_sample = u'\uc77c\uc601' unicode_with_reserved = u'abc%sdef' % utils.RESERVED_STR # Some false-y values self.assertFalse(constraints.check_utf8(None)) self.assertFalse(constraints.check_utf8('')) self.assertFalse(constraints.check_utf8(b'')) self.assertFalse(constraints.check_utf8(u'')) # invalid utf8 bytes self.assertFalse( constraints.check_utf8(unicode_sample.encode('utf-8')[::-1])) # unicode with null self.assertFalse(constraints.check_utf8(unicode_with_reserved)) # utf8 bytes with null self.assertFalse( constraints.check_utf8(unicode_with_reserved.encode('utf8'))) self.assertTrue(constraints.check_utf8('this is ascii and utf-8, too')) self.assertTrue(constraints.check_utf8(unicode_sample)) self.assertTrue(constraints.check_utf8(unicode_sample.encode('utf8')))
def test_check_utf8(self): unicode_sample = u'\uc77c\uc601' unicode_with_null = u'abc\u0000def' # Some false-y values self.assertFalse(constraints.check_utf8(None)) self.assertFalse(constraints.check_utf8('')) self.assertFalse(constraints.check_utf8(b'')) self.assertFalse(constraints.check_utf8(u'')) # invalid utf8 bytes self.assertFalse( constraints.check_utf8(unicode_sample.encode('utf-8')[::-1])) # unicode with null self.assertFalse(constraints.check_utf8(unicode_with_null)) # utf8 bytes with null self.assertFalse( constraints.check_utf8(unicode_with_null.encode('utf8'))) self.assertTrue(constraints.check_utf8('this is ascii and utf-8, too')) self.assertTrue(constraints.check_utf8(unicode_sample)) self.assertTrue(constraints.check_utf8(unicode_sample.encode('utf8')))
def test_check_utf8(self): unicode_sample = u'\uc77c\uc601' unicode_with_null = u'abc\u0000def' # Some false-y values self.assertFalse(constraints.check_utf8(None)) self.assertFalse(constraints.check_utf8('')) self.assertFalse(constraints.check_utf8(b'')) self.assertFalse(constraints.check_utf8(u'')) # invalid utf8 bytes self.assertFalse(constraints.check_utf8( unicode_sample.encode('utf-8')[::-1])) # unicode with null self.assertFalse(constraints.check_utf8(unicode_with_null)) # utf8 bytes with null self.assertFalse(constraints.check_utf8( unicode_with_null.encode('utf8'))) self.assertTrue(constraints.check_utf8('this is ascii and utf-8, too')) self.assertTrue(constraints.check_utf8(unicode_sample)) self.assertTrue(constraints.check_utf8(unicode_sample.encode('utf8')))
def _parse_uri(self): if not check_utf8(self.environ['PATH_INFO']): raise InvalidURI(self.path) if self.bucket_in_host: obj = self.environ['PATH_INFO'][1:] or None return self.bucket_in_host, obj bucket, obj = self.split_path(0, 2, True) if bucket and not validate_bucket_name(bucket): # Ignore GET service case raise InvalidBucketName(bucket) return (bucket, obj)
def __call__(self, env, start_response): """ Boilerplate code for how the server's code gets called upon receiving a request. Taken directly from other servers. """ # start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL') else: try: # disallow methods which have not been marked 'public' try: method = getattr(self, req.method) getattr(method, 'publicly_accessible') replication_method = getattr(method, 'replication', False) if (self.replication_server is not None and self.replication_server != replication_method): raise AttributeError('Not allowed method.') except AttributeError: res = HTTPMethodNotAllowed() else: res = method(req) except HTTPException as error_response: res = error_response except (Exception, Timeout): self.logger.exception( _('ERROR __call__ error with %(method)s %(path)s '), { 'method': req.method, 'path': req.path }) res = HTTPInternalServerError(body=traceback.format_exc()) # trans_time = '%.4f' % (time.time() - start_time) # if self.log_requests: # log_message = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %s' % ( # req.remote_addr, # time.strftime('%d/%b/%Y:%H:%M:%S +0000', # time.gmtime()), # req.method, req.path, # res.status.split()[0], res.content_length or '-', # req.headers.get('x-trans-id', '-'), # req.referer or '-', req.user_agent or '-', # trans_time) # if req.method.upper() == 'REPLICATE': # self.logger.debug(log_message) # else: # self.logger.info(log_message) return res(env, start_response)
def check_object_creation(req, object_name): """ Check to ensure that everything is alright about an object to be created. :param req: HTTP request object :param object_name: name of object to be created :raises HTTPRequestEntityTooLarge: the object is too large :raises HTTPLengthRequered: missing content-length header and not a chunked request :raises HTTPBadRequest: missing or bad content-type header, or bad metadata """ if req.content_length and req.content_length > MAX_FILE_SIZE: return HTTPRequestEntityTooLarge(body='Your request is too large.', request=req, content_type='text/plain') if req.content_length is None and \ req.headers.get('transfer-encoding') != 'chunked': return HTTPLengthRequired(request=req) if 'X-Copy-From' in req.headers and req.content_length: return HTTPBadRequest(body='Copy requests require a zero byte body', request=req, content_type='text/plain') for obj in object_name.split('/'): if not validate_obj_name(obj): return HTTPBadRequest(body='Invalid object name %s' % (obj), request=req, content_type='text/plain') if 'Content-Type' not in req.headers: return HTTPBadRequest(request=req, content_type='text/plain', body='No content type') if not check_utf8(req.headers['Content-Type']): return HTTPBadRequest(request=req, body='Invalid Content-Type', content_type='text/plain') if 'x-object-manifest' in req.headers: value = req.headers['x-object-manifest'] container = prefix = None try: container, prefix = value.split('/', 1) except ValueError: pass if not container or not prefix or '?' in value or '&' in value or \ prefix[0] == '/': return HTTPBadRequest( request=req, body='X-Object-Manifest must in the format container/prefix') return check_metadata(req, 'object')
def __call__(self, env, start_response): """WSGI Application entry point for the Swift Object Server.""" start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL') else: try: # disallow methods which have not been marked 'public' try: method = getattr(self, req.method) getattr(method, 'publicly_accessible') replication_method = getattr(method, 'replication', False) if (self.replication_server is not None and self.replication_server != replication_method): raise AttributeError('Not allowed method.') except AttributeError: res = HTTPMethodNotAllowed() else: res = method(req) except DiskFileCollision: res = HTTPForbidden(request=req) except (Exception, Timeout): self.logger.exception(_( 'ERROR __call__ error with %(method)s' ' %(path)s '), {'method': req.method, 'path': req.path}) res = HTTPInternalServerError(body=traceback.format_exc()) trans_time = time.time() - start_time if self.log_requests: log_line = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %.4f' % ( req.remote_addr, time.strftime('%d/%b/%Y:%H:%M:%S +0000', time.gmtime()), req.method, req.path, res.status.split()[0], res.content_length or '-', req.referer or '-', req.headers.get('x-trans-id', '-'), req.user_agent or '-', trans_time) if req.method == 'REPLICATE': self.logger.debug(log_line) else: self.logger.info(log_line) if req.method in ('PUT', 'DELETE'): slow = self.slow - trans_time if slow > 0: sleep(slow) return res(env, start_response)
def __call__(self, env, start_response): start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get("x-trans-id", None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body="Invalid UTF8 or contains NULL") else: try: # disallow methods which are not publicly accessible try: method = getattr(self, req.method) getattr(method, "publicly_accessible") replication_method = getattr(method, "replication", False) if self.replication_server is not None and self.replication_server != replication_method: raise AttributeError("Not allowed method.") except AttributeError: res = HTTPMethodNotAllowed() else: res = method(req) except HTTPException as error_response: res = error_response except (Exception, Timeout): self.logger.exception( _("ERROR __call__ error with %(method)s" " %(path)s "), {"method": req.method, "path": req.path} ) res = HTTPInternalServerError(body=traceback.format_exc()) trans_time = "%.4f" % (time.time() - start_time) additional_info = "" if res.headers.get("x-container-timestamp") is not None: additional_info += "x-container-timestamp: %s" % res.headers["x-container-timestamp"] log_message = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %s "%s"' % ( req.remote_addr, time.strftime("%d/%b/%Y:%H:%M:%S +0000", time.gmtime()), req.method, req.path, res.status.split()[0], res.content_length or "-", req.headers.get("x-trans-id", "-"), req.referer or "-", req.user_agent or "-", trans_time, additional_info, ) if req.method.upper() == "REPLICATE": self.logger.debug(log_message) else: self.logger.info(log_message) return res(env, start_response)
def __call__(self, env, start_response): """ Boilerplate code for how the server's code gets called upon receiving a request. Taken directly from other servers. """ # start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL') else: try: # disallow methods which have not been marked 'public' try: method = getattr(self, req.method) getattr(method, 'publicly_accessible') replication_method = getattr(method, 'replication', False) if (self.replication_server is not None and self.replication_server != replication_method): raise AttributeError('Not allowed method.') except AttributeError: res = HTTPMethodNotAllowed() else: res = method(req) except HTTPException as error_response: res = error_response except (Exception, Timeout): self.logger.exception(_( 'ERROR __call__ error with %(method)s %(path)s '), {'method': req.method, 'path': req.path}) res = HTTPInternalServerError(body=traceback.format_exc()) # trans_time = '%.4f' % (time.time() - start_time) # if self.log_requests: # log_message = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %s' % ( # req.remote_addr, # time.strftime('%d/%b/%Y:%H:%M:%S +0000', # time.gmtime()), # req.method, req.path, # res.status.split()[0], res.content_length or '-', # req.headers.get('x-trans-id', '-'), # req.referer or '-', req.user_agent or '-', # trans_time) # if req.method.upper() == 'REPLICATE': # self.logger.debug(log_message) # else: # self.logger.info(log_message) return res(env, start_response)
def __call__(self, env, start_response): print "account server __call__" start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL') else: try: # disallow methods which are not publicly accessible try: method = getattr(self, req.method) getattr(method, 'publicly_accessible') replication_method = getattr(method, 'replication', False) if (self.replication_server is not None and self.replication_server != replication_method): raise AttributeError('Not allowed method.') except AttributeError: res = HTTPMethodNotAllowed() else: res = method(req) except HTTPException as error_response: res = error_response except (Exception, Timeout): self.logger.exception(_('ERROR __call__ error with %(method)s' ' %(path)s '), {'method': req.method, 'path': req.path}) res = HTTPInternalServerError(body=traceback.format_exc()) trans_time = '%.4f' % (time.time() - start_time) additional_info = '' if res.headers.get('x-container-timestamp') is not None: additional_info += 'x-container-timestamp: %s' % \ res.headers['x-container-timestamp'] log_message = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %s "%s"' % ( req.remote_addr, time.strftime('%d/%b/%Y:%H:%M:%S +0000', time.gmtime()), req.method, req.path, res.status.split()[0], res.content_length or '-', req.headers.get('x-trans-id', '-'), req.referer or '-', req.user_agent or '-', trans_time, additional_info) if req.method.upper() == 'REPLICATE': self.logger.debug(log_message) else: self.logger.info(log_message) return res(env, start_response)
def __call__(self, env, start_response): print "account server __call__" start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL') else: try: # disallow methods which are not publicly accessible try: method = getattr(self, req.method) getattr(method, 'publicly_accessible') replication_method = getattr(method, 'replication', False) if (self.replication_server is not None and self.replication_server != replication_method): raise AttributeError('Not allowed method.') except AttributeError: res = HTTPMethodNotAllowed() else: res = method(req) except HTTPException as error_response: res = error_response except (Exception, Timeout): self.logger.exception( _('ERROR __call__ error with %(method)s' ' %(path)s '), { 'method': req.method, 'path': req.path }) res = HTTPInternalServerError(body=traceback.format_exc()) trans_time = '%.4f' % (time.time() - start_time) additional_info = '' if res.headers.get('x-container-timestamp') is not None: additional_info += 'x-container-timestamp: %s' % \ res.headers['x-container-timestamp'] log_message = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %s "%s"' % ( req.remote_addr, time.strftime('%d/%b/%Y:%H:%M:%S +0000', time.gmtime()), req.method, req.path, res.status.split()[0], res.content_length or '-', req.headers.get('x-trans-id', '-'), req.referer or '-', req.user_agent or '-', trans_time, additional_info) if req.method.upper() == 'REPLICATE': self.logger.debug(log_message) else: self.logger.info(log_message) return res(env, start_response)
def __call__(self, env, start_response): """WSGI Application entry point for the Swift Object Server.""" start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL') else: try: # disallow methods which have not been marked 'public' try: method = getattr(self, req.method) getattr(method, 'publicly_accessible') replication_method = getattr(method, 'replication', False) if (self.replication_server is not None and self.replication_server != replication_method): raise AttributeError('Not allowed method.') except AttributeError: res = HTTPMethodNotAllowed() else: res = method(req) except DiskFileCollision: res = HTTPForbidden(request=req) except HTTPException as error_response: res = error_response except (Exception, Timeout): self.logger.exception(_( 'ERROR __call__ error with %(method)s' ' %(path)s '), {'method': req.method, 'path': req.path}) res = HTTPInternalServerError(body=traceback.format_exc()) trans_time = time.time() - start_time if self.log_requests: log_line = get_log_line(req, res, trans_time, '') if req.method in ('REPLICATE', 'REPLICATION') or \ 'X-Backend-Replication' in req.headers: self.logger.debug(log_line) else: self.logger.info(log_line) if req.method in ('PUT', 'DELETE'): slow = self.slow - trans_time if slow > 0: sleep(slow) my_debug('returning from __call__ or objserver', res) my_debug('returnign from __call__ of objserver', start_response) return res(env, start_response)
def get_segments_to_delete_iter(self, req): """ A generator function to be used to delete all the segments and sub-segments referenced in a manifest. :params req: a swob.Request with an SLO manifest in path :raises HTTPPreconditionFailed: on invalid UTF8 in request path :raises HTTPBadRequest: on too many buffered sub segments and on invalid SLO manifest path """ if not check_utf8(req.path_info): raise HTTPPreconditionFailed(request=req, body='Invalid UTF8 or contains NULL') try: vrs, account, container, obj = req.split_path(4, 4, True) except ValueError: raise HTTPBadRequest('Invalid SLO manifiest path') segments = [{ 'sub_slo': True, 'name': ('/%s/%s' % (container, obj)).decode('utf-8') }] while segments: if len(segments) > MAX_BUFFERED_SLO_SEGMENTS: raise HTTPBadRequest( 'Too many buffered slo segments to delete.') seg_data = segments.pop(0) if seg_data.get('sub_slo'): try: segments.extend( self.get_slo_segments(seg_data['name'], req)) except HTTPException as err: # allow bulk delete response to report errors seg_data['error'] = { 'code': err.status_int, 'message': err.body } # add manifest back to be deleted after segments seg_data['sub_slo'] = False segments.append(seg_data) else: seg_data['name'] = seg_data['name'].encode('utf-8') yield seg_data
def handle_multipart_delete(self, req): """ Will delete all the segments in the SLO manifest and then, if successful, will delete the manifest file. :params req: a swob.Request with an obj in path :raises HTTPServerError: on invalid manifest :returns: swob.Response whose app_iter set to Bulk.handle_delete_iter """ if not check_utf8(req.path_info): raise HTTPPreconditionFailed(request=req, body='Invalid UTF8 or contains NULL') try: vrs, account, container, obj = req.split_path(4, 4, True) except ValueError: raise HTTPBadRequest('Not an SLO manifest') new_env = req.environ.copy() new_env['REQUEST_METHOD'] = 'GET' del (new_env['wsgi.input']) new_env['QUERY_STRING'] = 'multipart-manifest=get' new_env['CONTENT_LENGTH'] = 0 new_env['HTTP_USER_AGENT'] = \ '%s MultipartDELETE' % req.environ.get('HTTP_USER_AGENT') new_env['swift.source'] = 'SLO' get_man_resp = \ Request.blank('', new_env).get_response(self.app) if get_man_resp.status_int // 100 == 2: if not config_true_value( get_man_resp.headers.get('X-Static-Large-Object')): raise HTTPBadRequest('Not an SLO manifest') try: manifest = json.loads(get_man_resp.body) # append the manifest file for deletion at the end manifest.append( {'name': '/'.join(['', container, obj]).decode('utf-8')}) except ValueError: raise HTTPServerError('Invalid manifest file') resp = HTTPOk(request=req) resp.app_iter = self.bulk_deleter.handle_delete_iter( req, objs_to_delete=[o['name'].encode('utf-8') for o in manifest], user_agent='MultipartDELETE', swift_source='SLO') return resp return get_man_resp
def __call__(self, env, start_response): start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL') else: try: # disallow methods which are not publicly accessible try: method = getattr(self, req.method) getattr(method, 'publicly_accessible') replication_method = getattr(method, 'replication', False) if (self.replication_server is not None and self.replication_server != replication_method): raise AttributeError('Not allowed method.') except AttributeError: res = HTTPMethodNotAllowed() else: res = method(req) except HTTPException as error_response: res = error_response except (Exception, Timeout): self.logger.exception( _('ERROR __call__ error with %(method)s' ' %(path)s '), { 'method': req.method, 'path': req.path }) res = HTTPInternalServerError(body=traceback.format_exc()) if self.log_requests: trans_time = time.time() - start_time additional_info = '' if res.headers.get('x-container-timestamp') is not None: additional_info += 'x-container-timestamp: %s' % \ res.headers['x-container-timestamp'] log_msg = get_log_line(req, res, trans_time, additional_info) if req.method.upper() == 'REPLICATE': self.logger.debug(log_msg) else: self.logger.info(log_msg) return res(env, start_response)
def handle_multipart_delete(self, req): """ Will delete all the segments in the SLO manifest and then, if successful, will delete the manifest file. :params req: a swob.Request with an obj in path :raises HTTPServerError: on invalid manifest :returns: swob.Response whose app_iter set to Bulk.handle_delete_iter """ if not check_utf8(req.path_info): raise HTTPPreconditionFailed( request=req, body='Invalid UTF8 or contains NULL') resp = HTTPOk(request=req) out_content_type = req.accept.best_match(ACCEPTABLE_FORMATS) if out_content_type: resp.content_type = out_content_type resp.app_iter = self.bulk_deleter.handle_delete_iter( req, objs_to_delete=self.get_segments_to_delete_iter(req), user_agent='MultipartDELETE', swift_source='SLO', out_content_type=out_content_type) return resp
def __call__(self, env, start_response): """WSGI Application entry point for the Swift Object Server.""" start_time = time.time() self.plugin(env) req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8') else: try: if hasattr(self, req.method): res = getattr(self, req.method)(req) else: res = HTTPMethodNotAllowed() except (Exception, Timeout): self.logger.exception( _('ERROR __call__ error with %(method)s' ' %(path)s '), { 'method': req.method, 'path': req.path }) res = HTTPInternalServerError(body=traceback.format_exc()) trans_time = time.time() - start_time if self.log_requests: log_line = '%s - - [%s] "%s %s" %s %s "%s" "%s" "%s" %.4f' % ( req.remote_addr, time.strftime('%d/%b/%Y:%H:%M:%S +0000', time.gmtime()), req.method, req.path, res.status.split()[0], res.content_length or '-', req.referer or '-', req.headers.get( 'x-trans-id', '-'), req.user_agent or '-', trans_time) if req.method == 'REPLICATE': self.logger.debug(log_line) else: self.logger.info(log_line) if req.method in ('PUT', 'DELETE'): slow = self.slow - trans_time if slow > 0: sleep(slow) return res(env, start_response)
def handle_extract(self, req, compress_type): """ :params req: a swob Request :params compress_type: specifying the compression type of the tar. Accepts '', 'gz, or 'bz2' :raises HTTPException: on unhandled errors :returns: a swob response to request """ success_count = 0 failed_files = [] existing_containers = set() out_content_type = req.accept.best_match(ACCEPTABLE_FORMATS) if not out_content_type: return HTTPNotAcceptable(request=req) if req.content_length is None and \ req.headers.get('transfer-encoding', '').lower() != 'chunked': return HTTPLengthRequired(request=req) try: vrs, account, extract_base = req.split_path(2, 3, True) except ValueError: return HTTPNotFound(request=req) extract_base = extract_base or '' extract_base = extract_base.rstrip('/') try: tar = tarfile.open(mode='r|' + compress_type, fileobj=req.body_file) while True: tar_info = tar.next() if tar_info is None or \ len(failed_files) >= self.max_failed_extractions: break if tar_info.isfile(): obj_path = tar_info.name if obj_path.startswith('./'): obj_path = obj_path[2:] obj_path = obj_path.lstrip('/') if extract_base: obj_path = extract_base + '/' + obj_path if '/' not in obj_path: continue # ignore base level file destination = '/'.join(['', vrs, account, obj_path]) container = obj_path.split('/', 1)[0] if not check_utf8(destination): failed_files.append([ quote(destination[:MAX_PATH_LENGTH]), HTTPPreconditionFailed().status ]) continue if tar_info.size > MAX_FILE_SIZE: failed_files.append([ quote(destination[:MAX_PATH_LENGTH]), HTTPRequestEntityTooLarge().status ]) continue if container not in existing_containers: try: self.create_container( req, '/'.join(['', vrs, account, container])) existing_containers.add(container) except CreateContainerError, err: if err.status_int == HTTP_UNAUTHORIZED: return HTTPUnauthorized(request=req) failed_files.append([ quote(destination[:MAX_PATH_LENGTH]), err.status ]) continue except ValueError: failed_files.append([ quote(destination[:MAX_PATH_LENGTH]), HTTP_BAD_REQUEST ]) continue if len(existing_containers) > self.max_containers: return HTTPBadRequest( 'More than %d base level containers in tar.' % self.max_containers)
def handle_extract_iter(self, req, compress_type, out_content_type='text/plain'): """ A generator that can be assigned to a swob Response's app_iter which, when iterated over, will extract and PUT the objects pulled from the request body. Will occasionally yield whitespace while request is being processed. When the request is completed will yield a response body that can be parsed to determine success. See above documentation for details. :params req: a swob Request :params compress_type: specifying the compression type of the tar. Accepts '', 'gz', or 'bz2' """ resp_dict = { 'Response Status': HTTPCreated().status, 'Response Body': '', 'Number Files Created': 0 } failed_files = [] last_yield = time() if out_content_type and out_content_type.endswith('/xml'): to_yield = b'<?xml version="1.0" encoding="UTF-8"?>\n' else: to_yield = b' ' separator = b'' containers_accessed = set() req.environ['eventlet.minimum_write_chunk_size'] = 0 try: if not out_content_type: raise HTTPNotAcceptable(request=req) if req.content_length is None and \ req.headers.get('transfer-encoding', '').lower() != 'chunked': raise HTTPLengthRequired(request=req) try: vrs, account, extract_base = req.split_path(2, 3, True) except ValueError: raise HTTPNotFound(request=req) extract_base = extract_base or '' extract_base = extract_base.rstrip('/') tar = tarfile.open(mode='r|' + compress_type, fileobj=req.body_file) failed_response_type = HTTPBadRequest containers_created = 0 while True: if last_yield + self.yield_frequency < time(): last_yield = time() yield to_yield to_yield, separator = b' ', b'\r\n\r\n' tar_info = tar.next() if tar_info is None or \ len(failed_files) >= self.max_failed_extractions: break if tar_info.isfile(): obj_path = tar_info.name if not six.PY2: obj_path = obj_path.encode('utf-8', 'surrogateescape') obj_path = bytes_to_wsgi(obj_path) if obj_path.startswith('./'): obj_path = obj_path[2:] obj_path = obj_path.lstrip('/') if extract_base: obj_path = extract_base + '/' + obj_path if '/' not in obj_path: continue # ignore base level file destination = '/'.join(['', vrs, account, obj_path]) container = obj_path.split('/', 1)[0] if not constraints.check_utf8(wsgi_to_str(destination)): failed_files.append([ wsgi_quote(obj_path[:self.max_path_length]), HTTPPreconditionFailed().status ]) continue if tar_info.size > constraints.MAX_FILE_SIZE: failed_files.append([ wsgi_quote(obj_path[:self.max_path_length]), HTTPRequestEntityTooLarge().status ]) continue container_failure = None if container not in containers_accessed: cont_path = '/'.join(['', vrs, account, container]) try: if self.create_container(req, cont_path): containers_created += 1 if containers_created > self.max_containers: raise HTTPBadRequest( 'More than %d containers to create ' 'from tar.' % self.max_containers) except CreateContainerError as err: # the object PUT to this container still may # succeed if acls are set container_failure = [ wsgi_quote(cont_path[:self.max_path_length]), err.status ] if err.status_int == HTTP_UNAUTHORIZED: raise HTTPUnauthorized(request=req) except ValueError: failed_files.append([ wsgi_quote(obj_path[:self.max_path_length]), HTTPBadRequest().status ]) continue tar_file = tar.extractfile(tar_info) create_headers = { 'Content-Length': tar_info.size, 'X-Auth-Token': req.headers.get('X-Auth-Token'), } # Copy some whitelisted headers to the subrequest for k, v in req.headers.items(): if ((k.lower() in ('x-delete-at', 'x-delete-after')) or is_user_meta('object', k)): create_headers[k] = v create_obj_req = make_subrequest( req.environ, method='PUT', path=wsgi_quote(destination), headers=create_headers, agent='%(orig)s BulkExpand', swift_source='EA') create_obj_req.environ['wsgi.input'] = tar_file for pax_key, pax_value in tar_info.pax_headers.items(): header_name = pax_key_to_swift_header(pax_key) if header_name: # Both pax_key and pax_value are unicode # strings; the key is already UTF-8 encoded, but # we still have to encode the value. create_obj_req.headers[header_name] = \ pax_value.encode("utf-8") resp = create_obj_req.get_response(self.app) containers_accessed.add(container) if resp.is_success: resp_dict['Number Files Created'] += 1 else: if container_failure: failed_files.append(container_failure) if resp.status_int == HTTP_UNAUTHORIZED: failed_files.append([ wsgi_quote(obj_path[:self.max_path_length]), HTTPUnauthorized().status ]) raise HTTPUnauthorized(request=req) if resp.status_int // 100 == 5: failed_response_type = HTTPBadGateway failed_files.append([ wsgi_quote(obj_path[:self.max_path_length]), resp.status ]) if failed_files: resp_dict['Response Status'] = failed_response_type().status elif not resp_dict['Number Files Created']: resp_dict['Response Status'] = HTTPBadRequest().status resp_dict['Response Body'] = 'Invalid Tar File: No Valid Files' except HTTPException as err: resp_dict['Response Status'] = err.status resp_dict['Response Body'] = err.body.decode('utf-8') except (tarfile.TarError, zlib.error) as tar_error: resp_dict['Response Status'] = HTTPBadRequest().status resp_dict['Response Body'] = 'Invalid Tar File: %s' % tar_error except Exception: self.logger.exception('Error in extract archive.') resp_dict['Response Status'] = HTTPServerError().status yield separator + get_response_body(out_content_type, resp_dict, failed_files, 'extract')
def handle_delete(self, req, objs_to_delete=None, user_agent='BulkDelete', swift_source='BD'): """ :params req: a swob Request :raises HTTPException: on unhandled errors :returns: a swob Response """ try: vrs, account, _junk = req.split_path(2, 3, True) except ValueError: return HTTPNotFound(request=req) incoming_format = req.headers.get('Content-Type') if incoming_format and not incoming_format.startswith('text/plain'): # For now only accept newline separated object names return HTTPNotAcceptable(request=req) out_content_type = req.accept.best_match(ACCEPTABLE_FORMATS) if not out_content_type: return HTTPNotAcceptable(request=req) if objs_to_delete is None: objs_to_delete = self.get_objs_to_delete(req) failed_files = [] success_count = not_found_count = 0 failed_file_response_type = HTTPBadRequest for obj_to_delete in objs_to_delete: obj_to_delete = obj_to_delete.strip().lstrip('/') if not obj_to_delete: continue delete_path = '/'.join(['', vrs, account, obj_to_delete]) if not check_utf8(delete_path): failed_files.append( [quote(delete_path), HTTPPreconditionFailed().status]) continue new_env = req.environ.copy() new_env['PATH_INFO'] = delete_path del (new_env['wsgi.input']) new_env['CONTENT_LENGTH'] = 0 new_env['HTTP_USER_AGENT'] = \ '%s %s' % (req.environ.get('HTTP_USER_AGENT'), user_agent) new_env['swift.source'] = swift_source delete_obj_req = Request.blank(delete_path, new_env) resp = delete_obj_req.get_response(self.app) if resp.status_int // 100 == 2: success_count += 1 elif resp.status_int == HTTP_NOT_FOUND: not_found_count += 1 elif resp.status_int == HTTP_UNAUTHORIZED: return HTTPUnauthorized(request=req) else: if resp.status_int // 100 == 5: failed_file_response_type = HTTPBadGateway failed_files.append([quote(delete_path), resp.status]) resp_body = get_response_body(out_content_type, { 'Number Deleted': success_count, 'Number Not Found': not_found_count }, failed_files) if (success_count or not_found_count) and not failed_files: return HTTPOk(resp_body, content_type=out_content_type) if failed_files: return failed_file_response_type(resp_body, content_type=out_content_type) return HTTPBadRequest('Invalid bulk delete.')
def handle_request(self, req): """ Entry point for proxy server. Should return a WSGI-style callable (such as swob.Response). :param req: swob.Request object """ try: self.logger.set_statsd_prefix('proxy-server') # content_length存在且小于0,返回错误。 if req.content_length and req.content_length < 0: self.logger.increment('errors') return HTTPBadRequest(request=req, body='Invalid Content-Length') try: if not check_utf8(req.path_info): self.logger.increment('errors') return HTTPPreconditionFailed( request=req, body='Invalid UTF8 or contains NULL') except UnicodeError: self.logger.increment('errors') return HTTPPreconditionFailed( request=req, body='Invalid UTF8 or contains NULL') try: # 根据 path获取相应的 controller, 如: # 1. AccountController. 2. ContainerController # 3. ECObjectController 4. ReplicatedObjectController # 5. InfoController # path_parts is a dict : (version=version, # account_name=account, # container_name=container, # object_name=obj) controller, path_parts = self.get_controller(req) except APIVersionError: self.logger.increment('errors') return HTTPBadRequest(request=req) except ValueError: self.logger.increment('errors') return HTTPNotFound(request=req) if not controller: self.logger.increment('errors') return HTTPPreconditionFailed(request=req, body='Bad URL') if self.deny_host_headers and \ req.host.split(':')[0] in self.deny_host_headers: return HTTPForbidden(request=req, body='Invalid host header') self.logger.set_statsd_prefix('proxy-server.' + controller.server_type.lower()) # 实例化 controller controller = controller(self, **path_parts) if 'swift.trans_id' not in req.environ: # if this wasn't set by an earlier middleware, set it now trans_id_suffix = self.trans_id_suffix trans_id_extra = req.headers.get('x-trans-id-extra') if trans_id_extra: trans_id_suffix += '-' + trans_id_extra[:32] trans_id = generate_trans_id(trans_id_suffix) req.environ['swift.trans_id'] = trans_id self.logger.txn_id = trans_id req.headers['x-trans-id'] = req.environ['swift.trans_id'] controller.trans_id = req.environ['swift.trans_id'] self.logger.client_ip = get_remote_client(req) if req.method not in controller.allowed_methods: return HTTPMethodNotAllowed( request=req, headers={'Allow': ', '.join(controller.allowed_methods)}) # getattr(object, name) 返回一个对象的属性值 # 根据 req.method 的值返回controller的 HEAD() GET() PUT() 等函数 handler = getattr(controller, req.method) old_authorize = None # swift.authrize 是 swift_auth 提供的句柄 if 'swift.authorize' in req.environ: # We call authorize before the handler, always. If authorized, # we remove the swift.authorize hook so isn't ever called # again. If not authorized, we return the denial unless the # controller's method indicates it'd like to gather more # information and try again later. resp = req.environ['swift.authorize'](req) if not resp: # No resp means authorized, no delayed recheck required. old_authorize = req.environ['swift.authorize'] else: # Response indicates denial, but we might delay the denial # and recheck later. If not delayed, return the error now. if not getattr(handler, 'delay_denial', None): return resp # Save off original request method (GET, POST, etc.) in case it # gets mutated during handling. This way logging can display the # method the client actually sent. req.environ.setdefault('swift.orig_req_method', req.method) try: if old_authorize: req.environ.pop('swift.authorize', None) return handler(req) # call HEAD(),GET(),PUT(),etc... finally: if old_authorize: req.environ['swift.authorize'] = old_authorize except HTTPException as error_response: return error_response except (Exception, Timeout): self.logger.exception(_('ERROR Unhandled exception in request')) return HTTPServerError(request=req)
def test_check_utf8_non_canonical(self): self.assertFalse(constraints.check_utf8('\xed\xa0\xbc\xed\xbc\xb8')) self.assertFalse(constraints.check_utf8('\xed\xa0\xbd\xed\xb9\x88'))
def handle_request(self, req): """ Entry point for proxy server. Should return a WSGI-style callable (such as swob.Response). :param req: swob.Request object """ try: self.logger.set_statsd_prefix('proxy-server') if req.content_length and req.content_length < 0: self.logger.increment('errors') return HTTPBadRequest(request=req, body='Invalid Content-Length') try: if not check_utf8(req.path_info): self.logger.increment('errors') return HTTPPreconditionFailed( request=req, body='Invalid UTF8 or contains NULL') except UnicodeError: self.logger.increment('errors') return HTTPPreconditionFailed( request=req, body='Invalid UTF8 or contains NULL') try: controller, path_parts = self.get_controller(req) p = req.path_info if isinstance(p, six.text_type): p = p.encode('utf-8') except APIVersionError: self.logger.increment('errors') return HTTPBadRequest(request=req) except ValueError: self.logger.increment('errors') return HTTPNotFound(request=req) if not controller: self.logger.increment('errors') return HTTPPreconditionFailed(request=req, body='Bad URL') if self.deny_host_headers and \ req.host.split(':')[0] in self.deny_host_headers: return HTTPForbidden(request=req, body='Invalid host header') self.logger.set_statsd_prefix('proxy-server.' + controller.server_type.lower()) controller = controller(self, **path_parts) if 'swift.trans_id' not in req.environ: # if this wasn't set by an earlier middleware, set it now trans_id_suffix = self.trans_id_suffix trans_id_extra = req.headers.get('x-trans-id-extra') if trans_id_extra: trans_id_suffix += '-' + trans_id_extra[:32] trans_id = generate_trans_id(trans_id_suffix) req.environ['swift.trans_id'] = trans_id self.logger.txn_id = trans_id req.headers['x-trans-id'] = req.environ['swift.trans_id'] controller.trans_id = req.environ['swift.trans_id'] self.logger.client_ip = get_remote_client(req) try: handler = getattr(controller, req.method) getattr(handler, 'publicly_accessible') except AttributeError: allowed_methods = getattr(controller, 'allowed_methods', set()) return HTTPMethodNotAllowed( request=req, headers={'Allow': ', '.join(allowed_methods)}) old_authorize = None if 'swift.authorize' in req.environ: # We call authorize before the handler, always. If authorized, # we remove the swift.authorize hook so isn't ever called # again. If not authorized, we return the denial unless the # controller's method indicates it'd like to gather more # information and try again later. resp = req.environ['swift.authorize'](req) if not resp and not req.headers.get('X-Copy-From-Account') \ and not req.headers.get('Destination-Account'): # No resp means authorized, no delayed recheck required. old_authorize = req.environ['swift.authorize'] else: # Response indicates denial, but we might delay the denial # and recheck later. If not delayed, return the error now. if not getattr(handler, 'delay_denial', None): return resp # Save off original request method (GET, POST, etc.) in case it # gets mutated during handling. This way logging can display the # method the client actually sent. req.environ['swift.orig_req_method'] = req.method try: if old_authorize: req.environ.pop('swift.authorize', None) return handler(req) finally: if old_authorize: req.environ['swift.authorize'] = old_authorize except HTTPException as error_response: return error_response except (Exception, Timeout): self.logger.exception(_('ERROR Unhandled exception in request')) return HTTPServerError(request=req)
def test_check_utf8_lone_surrogates(self): self.assertFalse(constraints.check_utf8('\xed\xa0\xbc')) self.assertFalse(constraints.check_utf8('\xed\xb9\x88'))
def __call__(self, env, start_response): """WSGI Application entry point for the Swift Object Server.""" start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = HTTPPreconditionFailed(body='Invalid UTF8 or contains NULL') else: try: # disallow methods which have not been marked 'public' try: if req.method not in self.allowed_methods: raise AttributeError('Not allowed method.') except AttributeError: res = HTTPMethodNotAllowed() else: method = getattr(self, req.method) res = method(req) except DiskFileCollision: res = HTTPForbidden(request=req) except HTTPException as error_response: res = error_response except (Exception, Timeout): self.logger.exception( _('ERROR __call__ error with %(method)s' ' %(path)s '), { 'method': req.method, 'path': req.path }) res = HTTPInternalServerError(body=traceback.format_exc()) trans_time = time.time() - start_time if self.log_requests: log_line = get_log_line(req, res, trans_time, '') if req.method in ('REPLICATE', 'REPLICATION') or \ 'X-Backend-Replication' in req.headers: self.logger.debug(log_line) else: self.logger.info(log_line) if req.method in ('PUT', 'DELETE'): slow = self.slow - trans_time if slow > 0: sleep(slow) # To be able to zero-copy send the object, we need a few things. # First, we have to be responding successfully to a GET, or else we're # not sending the object. Second, we have to be able to extract the # socket file descriptor from the WSGI input object. Third, the # diskfile has to support zero-copy send. # # There's a good chance that this could work for 206 responses too, # but the common case is sending the whole object, so we'll start # there. if req.method == 'GET' and res.status_int == 200 and \ isinstance(env['wsgi.input'], wsgi.Input): app_iter = getattr(res, 'app_iter', None) checker = getattr(app_iter, 'can_zero_copy_send', None) if checker and checker(): # For any kind of zero-copy thing like sendfile or splice, we # need the file descriptor. Eventlet doesn't provide a clean # way of getting that, so we resort to this. wsock = env['wsgi.input'].get_socket() wsockfd = wsock.fileno() # Don't call zero_copy_send() until after we force the HTTP # headers out of Eventlet and into the socket. def zero_copy_iter(): # If possible, set TCP_CORK so that headers don't # immediately go on the wire, but instead, wait for some # response body to make the TCP frames as large as # possible (and hence as few packets as possible). # # On non-Linux systems, we might consider TCP_NODELAY, but # since the only known zero-copy-capable diskfile uses # Linux-specific syscalls, we'll defer that work until # someone needs it. if hasattr(socket, 'TCP_CORK'): wsock.setsockopt(socket.IPPROTO_TCP, socket.TCP_CORK, 1) yield EventletPlungerString() try: app_iter.zero_copy_send(wsockfd) except Exception: self.logger.exception("zero_copy_send() blew up") raise yield '' # Get headers ready to go out res(env, start_response) return zero_copy_iter() else: return res(env, start_response) else: return res(env, start_response)