def __call__(self, env, start_response): start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = jresponse('-1', 'invalid utf8', req,412) else: try: # disallow methods which are not publicly accessible try: method = getattr(self, req.method) getattr(method, 'publicly_accessible') except AttributeError: res = jresponse('-1', 'method not allowed', req,405) else: res = method(req) except (Exception, Timeout): self.logger.exception(_('ERROR __call__ error with %(method)s' ' %(path)s '), {'method': req.method, 'path': req.path}) res = jresponse('-1', 'InternalServerError', req,500) additional_info = '' if res.headers.get('x-container-timestamp') is not None: additional_info += 'x-container-timestamp: %s' % \ res.headers['x-container-timestamp'] return res(env, start_response)
def check_object_creation(req, object_name): """ Check to ensure that everything is alright about an object to be created. :param req: HTTP request object :param object_name: name of object to be created :raises HTTPRequestEntityTooLarge: the object is too large :raises HTTPLengthRequered: missing content-length header and not a chunked request :raises HTTPBadRequest: missing or bad content-type header, or bad metadata """ if req.content_length and req.content_length > MAX_FILE_SIZE: respbody='Your request is too large.' return jresponse('-1', respbody, req,413) if req.content_length is None and \ req.headers.get('transfer-encoding') != 'chunked': return jresponse('-1', 'length required', req,411) if len(object_name) > MAX_OBJECT_NAME_LENGTH: respbody='Object name length of %d longer than %d' % (len(object_name), MAX_OBJECT_NAME_LENGTH) return jresponse('-1', respbody, req,400) return None
def __call__(self, request): if request.method not in ("PUT","COPY"): return self.app try: split_path(request.path,2, 4, rest_with_last=True) except ValueError: return self.app new_quota = request.headers.get('X-Account-Meta-Quota-Bytes') if new_quota: if not new_quota.isdigit(): return jresponse('-1', 'bad request', request, 400) return self.app account_info = get_account_info(request.environ, self.app) new_size = int(account_info['bytes']) + (request.content_length or 0) quota = int(account_info['meta'].get('quota-bytes', -1)) if 0 <= quota < new_size: respbody='Your request is too large.' return jresponse('-1', respbody, request,413) return self.app
def MOVE(self,req): if 'ftype' not in req.GET: return jresponse('-1', 'param error', req,404) (container_partition, containers,object_versions) = self.container_info(self.account_name, self.container_name, account_autocreate=self.app.account_autocreate) if not containers: return jresponse('-1', 'not found', req,404) direr_partition, direr_nodes = self.app.direr_ring.get_nodes(self.account_name, self.container_name, self.direr_name) headers = [] for container in containers: nheaders = {'X-Timestamp': normalize_timestamp(time.time()), 'x-trans-id': self.trans_id, 'X-Container-Host': '%(ip)s:%(port)s' % container, 'X-Container-Partition': container_partition, 'X-Container-Device': container['device'], 'x-move-dst':req.headers['Destination'], 'x-ftype':req.GET['ftype'], 'Connection': 'close'} if object_versions: nheaders['x-versions-location'] = object_versions self.transfer_headers(req.headers, nheaders) headers.append(nheaders) resp = self.make_requests(self.account_name,req, self.app.direr_ring, direr_partition, 'MOVE', req.path_info, headers) return resp
def __call__(self, env, start_response): """WSGI Application entry point for the Swift Object Server.""" start_time = time.time() req = Request(env) if 'l' != req.headers.get('X-Ftype'): return self.app(env,start_response) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res =jresponse('-1', 'Invalid UTF8', req,412) else: try: # disallow methods which have not been marked 'public' try: method = getattr(self, req.method) getattr(method, 'publicly_accessible') except AttributeError: res = jresponse('-1', 'method not allowed', req,405) else: res = method(req) except (Exception, Timeout): self.logger.exception(_('ERROR __call__ error with %(method)s' ' %(path)s '), {'method': req.method, 'path': req.path}) res = jresponse('-1', 'InternalServerError', req,500) trans_time = time.time() - start_time if req.method in ('PUT', 'DELETE'): slow = self.slow - trans_time if slow > 0: sleep(slow) return res(env, start_response)
def PUT(self, req): """HTTP PUT request handler.""" if req.body: return jresponse('-1','param error',req,400) if len(self.container_name) > MAX_CONTAINER_NAME_LENGTH: respbody = 'Container name length of %d longer than %d' % \ (len(self.container_name), MAX_CONTAINER_NAME_LENGTH) return jresponse('-1', respbody, req,400) account_partition, accounts = \ self.account_info(self.account_name, autocreate=self.app.account_autocreate) if not accounts: return jresponse('-1', 'not found', req,404) container_partition, containers = self.app.container_ring.get_nodes(self.account_name, self.container_name) headers = [] for account in accounts: nheaders = {'X-Timestamp': normalize_timestamp(time.time()), 'x-trans-id': self.trans_id, 'X-Account-Host': '%(ip)s:%(port)s' % account, 'X-Account-Partition': account_partition, 'X-Account-Device': self.account_name, 'Connection': 'close'} self.transfer_headers(req.headers, nheaders) headers.append(nheaders) resp = self.make_requests(self.account_name,req, self.app.container_ring, container_partition, 'PUT', req.path_info, headers) return resp
def __call__(self, env, start_response): req = Request(env) if 'd' != req.headers.get('X-Ftype'): return self.app(env,start_response) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = jresponse('-1', 'invalid UTF8', req,412) else: try: # disallow methods which have not been marked 'public' try: method = getattr(self, req.method) getattr(method, 'publicly_accessible') except AttributeError: res = jresponse('-1', 'method not allowed', req,405) else: res = method(req) except (Exception, Timeout): self.logger.exception(_('ERROR __call__ error with %(method)s' ' %(path)s '), {'method': req.method, 'path': req.path}) res = jresponse('-1', 'internal server error', req,500) return res(env, start_response)
def __call__(self, env, start_response): start_time = time.time() req = Request(env) self.logger.txn_id = req.headers.get('x-trans-id', None) if not check_utf8(req.path_info): res = jresponse('-1','Invalid UTF8',req,412) else: try: # disallow methods which have not been marked 'public' try: method = getattr(self, req.method) getattr(method, 'publicly_accessible') except AttributeError: res = jresponse('-1', 'method not allowed', req,405) else: res = method(req) # if req.method == 'PUT': # print 'path: '+req.path + ' status: '+str(res.status_int) + ' msg: '+res.body except (Exception, Timeout): self.logger.exception(_('ERROR __call__ error with %(method)s' ' %(path)s '), {'method': req.method, 'path': req.path}) res = jresponse('-1', 'InternalServerError', req,500) return res(env, start_response)
def PUT(self, req): """Handle HTTP PUT request.""" if req.body: return jresponse('-1', 'param error', req,400) start_time = time.time() try: drive, part, account, container, obj = split_path( unquote(req.path), 4, 5, True) validate_device_partition(drive, part) except ValueError, err: return jresponse('-1', 'bad request', req,400)
def MOVE(self,req): if 'ftype' not in req.GET: return jresponse('-1', 'param error', req,404) (container_partition, containers,object_versions) = self.container_info(self.account_name, self.container_name, account_autocreate=self.app.account_autocreate) if not containers: return jresponse('-1', 'not found', req,404) object_partition, object_nodes = self.app.object_ring.get_nodes(self.account_name, self.container_name, self.object_name) headers = [] req.GET['ftype'] = 'f' for container in containers: nheaders = {'X-Timestamp': normalize_timestamp(time.time()), 'x-trans-id': self.trans_id, 'X-Container-Host': '%(ip)s:%(port)s' % container, 'X-Container-Partition': container_partition, 'X-Container-Device': container['device'], 'x-move-dst':req.headers['Destination'], 'x-ftype':req.GET['ftype'], 'x-overwrite':req.GET.get('overwrite','false'), 'Connection': 'close'} self.transfer_headers(req.headers, nheaders) headers.append(nheaders) resp = self.make_requests(self.account_name,req, self.app.object_ring, object_partition, 'MOVE', req.path_info, headers) if False and object_versions: # this is a version manifest and needs to be handled differently lcontainer = object_versions.split('/')[0] # prefix_len = '%03x' % len(self.object_name) lprefix = self.object_name + '/' last_item = None try: for last_item in self._listing_iter(lcontainer, lprefix, req.environ): pass except ListingIterNotFound: # no worries, last_item is None pass except ListingIterNotAuthorized, err: return err.aresp except ListingIterError: return jresponse('-1','ServerERROR',req,500)
def DELETE(self, req): """HTTP DELETE request handler.""" account_partition, accounts = self.account_info(self.account_name,autocreate=False) account = accounts[0] (container_partition, containers,object_versions) = self.container_info(self.account_name, self.container_name) if not containers: return jresponse('-1', 'not found', req,404) partition, nodes = self.app.object_ring.get_nodes(self.account_name, self.container_name, self.object_name) headers = [] for container in containers: nheaders = dict(req.headers.iteritems()) nheaders['X-Timestamp']= normalize_timestamp(time.time()) nheaders['Connection'] = 'close' nheaders['X-Container-Host'] = '%(ip)s:%(port)s' % container nheaders['X-Container-Partition'] = container_partition nheaders['X-Container-Device'] = container['device'] nheaders['X-Account-Host'] = '%(ip)s:%(port)s' % account nheaders['X-Account-Partition'] = account_partition nheaders['X-Account-Device'] = self.account_name headers.append(nheaders) resp = self.make_requests(self.account_name,req, self.app.object_ring, partition, 'DELETE_RECYCLE', req.path_info, headers) if object_versions and req.GET.get('cover') == 'true': # this is a version manifest and needs to be handled differently lcontainer = object_versions.split('/')[0] # prefix_len = '%03x' % len(self.object_name) lprefix = self.object_name + '/' last_item = None try: for last_item in self._listing_iter(lcontainer, lprefix, req.environ): pass except ListingIterNotFound: # no worries, last_item is None pass except ListingIterNotAuthorized, err: return err.aresp except ListingIterError: return jresponse('-1','ServerERROR',req,500)
def handle_register(self, req): rdatas = {'failed_files':[],'success_count':0,'not_found_count':0} req.accept = 'application/json' out_content_type = 'application/json' self.handle_normal(req,rdatas) self.handle_quota(req,rdatas) self.handle_normal_versions(req,rdatas) self.handle_normal_metadata(req,rdatas) self.handle_segments(req,rdatas) self.handle_recycle(req,rdatas) self.handle_recycle_meta(req,rdatas) self.handle_recycle_user(req,rdatas) self.handle_private(req,rdatas) self.handle_private_versions(req,rdatas) self.handle_private_metadata(req,rdatas) self.handle_backup(req,rdatas) self.handle_backup_versions(req,rdatas) self.handle_backup_metadata(req,rdatas) resp_body = get_response_body( out_content_type, {'Number successed': rdatas['success_count'], 'Number failed': rdatas['not_found_count']}, rdatas['failed_files']) if (rdatas['success_count'] or rdatas['not_found_count']) and not rdatas['failed_files']: return jresponse('0','',req,200,param=resp_body) if rdatas['failed_files']: return jresponse('-1',json.dumps(resp_body), req,400) return jresponse('-1', 'Invalid userinit register', req, 400)
def PUT(self, req): (container_partition, containers,_) = self.container_info(self.account_name, self.container_name, account_autocreate=self.app.account_autocreate) if not containers: return jresponse('-1', 'not found', req,404) direr_partition, direr_nodes = self.app.direr_ring.get_nodes(self.account_name, self.container_name, self.direr_name) headers = [] for container in containers: nheaders = {'X-Timestamp': normalize_timestamp(time.time()), 'x-trans-id': self.trans_id, 'X-Container-Host': '%(ip)s:%(port)s' % container, 'X-Container-Partition': container_partition, 'X-Container-Device': container['device'], 'x-ftype':req.GET['ftype'], 'Connection': 'close'} self.transfer_headers(req.headers, nheaders) headers.append(nheaders) resp = self.make_requests(self.account_name,req, self.app.direr_ring, direr_partition, 'PUT', req.path_info, headers) return resp
def best_response(self, req, statuses, reasons, bodies, server_type, etag=None,jsondata=None): """ :param req: webob.Request object :param statuses: list of statuses returned :param reasons: list of reasons for each status :param bodies: bodies of each response :param server_type: type of server the responses came from :param etag: etag :returns: webob.Response object with the correct status, body, etc. set """ resp = Response(request=req) if len(statuses): for hundred in (HTTP_OK, HTTP_MULTIPLE_CHOICES, HTTP_BAD_REQUEST): hstatuses = \ [s for s in statuses if hundred <= s < hundred + 100] if len(hstatuses) > len(statuses) / 2: status = max(hstatuses) status_index = statuses.index(status) resp.status = '%s %s' % (status, reasons[status_index]) resp.body = bodies[status_index] return resp resp.status = '503 Internal Server Error' return jresponse('-1', 'internal server error', req,503)
def DELETE_RECYCLE(self, req): try: device, partition, account, src_container, src_obj = split_path( unquote(req.path), 4, 5, True) validate_device_partition(device, partition) except ValueError, err: return jresponse('-1', 'bad request', req,400)
def META(self, req): """Handler for HTTP GET/HEAD requests.""" partition, nodes = self.app.account_ring.get_nodes(self.account_name) shuffle(nodes) resp = self.META_base(req, _('Account'), partition, nodes, req.path_info.rstrip('/'), len(nodes)) if resp.status_int == HTTP_NOT_FOUND and self.app.account_autocreate: if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH: respbody = 'Account name length of %d longer than %d' % \ (len(self.account_name), MAX_ACCOUNT_NAME_LENGTH) return jresponse('-1', respbody, req,400) headers = {'X-Timestamp': normalize_timestamp(time.time()), 'X-Trans-Id': self.trans_id, 'Connection': 'close'} resp = self.make_requests(self.account_name, Request.blank('/v1/' + self.account_name), self.app.account_ring, partition, 'PUT', '/' + self.account_name, [headers] * len(nodes)) if not is_success(resp.status_int): self.app.logger.warning('Could not autocreate account %r' % self.account_name) return resp resp = self.META_base(req, _('Account'), partition, nodes, req.path_info.rstrip('/'), len(nodes)) return resp
def gluster_check_object_creation(req, object_name): """ Check to ensure that everything is alright about an object to be created. Monkey patches swift.common.constraints.check_object_creation, invoking the original, and then adding an additional check for individual object name components. :param req: HTTP request object :param object_name: name of object to be created :raises HTTPRequestEntityTooLarge: the object is too large :raises HTTPLengthRequered: missing content-length header and not a chunked request :raises HTTPBadRequest: missing or bad content-type header, or bad metadata """ ret = __check_object_creation(req, object_name) if ret is None: for obj in object_name.split('/'): reason = validate_obj_name_component(obj) if reason: bdy = 'Invalid object name "%s", component "%s" %s' \ % (object_name, obj, reason) ret = jresponse('-1',bdy,req,400) return ret
def POST(self, req): """HTTP POST request handler.""" account_partition, accounts = self.app.account_ring.get_nodes(self.account_name) headers = {'X-Timestamp': normalize_timestamp(time.time()), 'X-Trans-Id': self.trans_id, 'Connection': 'close'} self.transfer_headers(req.headers, headers) resp = self.make_requests(self.account_name,req, self.app.account_ring, account_partition, 'POST', req.path_info, [headers] * len(accounts)) if resp.status_int == HTTP_NOT_FOUND and self.app.account_autocreate: if len(self.account_name) > MAX_ACCOUNT_NAME_LENGTH: respbody = 'Account name length of %d longer than %d' % \ (len(self.account_name), MAX_ACCOUNT_NAME_LENGTH) return jresponse('-1', respbody, req,400) resp = self.make_requests(self.account_name, Request.blank('/v1/' + self.account_name), self.app.account_ring, account_partition, 'PUT', '/' + self.account_name, [headers] * len(accounts)) if not is_success(resp.status_int): self.app.logger.warning('Could not autocreate account %r' % self.account_name) return resp return resp
def GET(self, req): try: drive, part, account, container, direr = split_path( unquote(req.path), 4, 5, True) validate_device_partition(drive, part) except ValueError, err: return jresponse('-1', 'bad request', req,400)
def POST(self, req): error_response = check_metadata(req, 'object') if error_response: return error_response container_partition, containers,_ = self.container_info(self.account_name, self.container_name, account_autocreate=self.app.account_autocreate) if not containers: return jresponse('-1', 'not found', req,404) partition, nodes = self.app.object_ring.get_nodes(self.account_name, self.container_name, self.object_name) req.headers['X-Timestamp'] = normalize_timestamp(time.time()) headers = [] for container in containers: nheaders = dict(req.headers.iteritems()) nheaders['Connection'] = 'close' nheaders['X-Container-Host'] = '%(ip)s:%(port)s' % container nheaders['X-Container-Partition'] = container_partition nheaders['X-Container-Device'] = container['device'] headers.append(nheaders) resp = self.make_requests(self.account_name,req, self.app.object_ring, partition, 'POST', req.path_info, headers) return resp
def DELETE_RECYCLE(self, req): try: drive, part, account, src_container, src_direr = split_path( unquote(req.path), 4, 5, True) validate_device_partition(drive, part) except ValueError, err: return jresponse('-1', str(err), req,400)
def MOVE_VERSION(self,req): (container_partition, containers,_) = self.container_info(self.account_name, self.container_name, account_autocreate=self.app.account_autocreate) if not containers: return jresponse('-1', 'not found', req,404) object_partition, object_nodes = self.app.object_ring.get_nodes(self.account_name, self.container_name, self.object_name) headers = [] req.GET['ftype'] = 'f' for container in containers: nheaders = {'X-Timestamp': normalize_timestamp(time.time()), 'x-trans-id': self.trans_id, 'X-Container-Host': '%(ip)s:%(port)s' % container, 'X-Container-Partition': container_partition, 'X-Container-Device': container['device'], 'x-move-dst':req.headers['Destination'], 'x-ftype':req.GET['ftype'], 'x-fhr-dir':'True', 'Connection': 'close'} self.transfer_headers(req.headers, nheaders) headers.append(nheaders) resp = self.make_requests(self.account_name,req, self.app.object_ring, object_partition, 'MOVE', req.path_info, headers) return resp
def verify_request(self,req,env): try: version, account, container, obj = split_path(req.path_info, minsegs=1, maxsegs=4, rest_with_last=True) except ValueError: self.logger.increment('errors') return jresponse('-1','not found',req,404) token = env.get('HTTP_X_AUTH_TOKEN', env.get('HTTP_X_STORAGE_TOKEN')) verify_flag = False if token : user_info = self.get_cache_user_info(env, token) if user_info: tenant = 'AUTH_' + user_info.replace('@','').replace('.','') if account != tenant: self.logger.increment('unauthorized') verify_flag = False verify_flag = True else: self.logger.increment('unauthorized') verify_flag = False else: self.logger.increment('unauthorized') verify_flag = False oauth_data_list = json.dumps({'verify_flag':str(verify_flag).lower()}) return Response(body=oauth_data_list,request=req)
def DELETE(self, req): """Handle HTTP DELETE request.""" try: drive, part, account = split_path(unquote(req.path), 3) validate_device_partition(drive, part) except ValueError, err: return jresponse('-1', 'bad request',req,400)
def PUT(self, request): try: device, partition, account, src_container, src_link = \ split_path(unquote(request.path), 5, 5, True) validate_device_partition(device, partition) except ValueError, err: return jresponse('-1', 'bad request', request,400)
def HEAD(self, req): """Handle HTTP HEAD request.""" try: drive, part, account, container = split_path(unquote(req.path), 3, 4) validate_device_partition(drive, part) except ValueError, err: return jresponse('-1', 'bad request', req,400)
def POST(self, req): """Handle HTTP POST request.""" start_time = time.time() try: drive, part, account, container = split_path(unquote(req.path), 4) validate_device_partition(drive, part) except ValueError, err: return jresponse('-1', 'bad request', req,400)
def DELETE(self, request): """Handle HTTP DELETE requests for the Swift Object Server.""" start_time = time.time() try: device, partition, account, container, obj = \ split_path(unquote(request.path), 5, 5, True) validate_device_partition(device, partition) except ValueError, e: return jresponse('-1', 'bad request', request,400)
def GET(self, request): # request is global , can not be modify # response can be modify start_time = time.time() try: device, partition, account, container, obj = \ split_path(unquote(request.path), 5, 5, True) validate_device_partition(device, partition) except ValueError, err: return jresponse('-1', 'bad request', request,400)
def check_metadata(req, target_type): """ Check metadata sent in the request headers. :param req: request object :param target_type: str: one of: object, container, or account: indicates which type the target storage for the metadata is :raises HTTPBadRequest: bad metadata """ prefix = 'x-%s' % (target_type) meta_count = 0 meta_size = 0 for key, value in req.headers.iteritems(): if not key.lower().startswith(prefix): continue if not key: respbody='Metadata name cannot be empty' return jresponse('-1', respbody, req,400) meta_count += 1 meta_size += len(key) + len(value) if len(key) > MAX_META_NAME_LENGTH: respbody='Metadata name too long; max %d' % MAX_META_NAME_LENGTH return jresponse('-1', respbody, req,400) elif len(value) > MAX_META_VALUE_LENGTH: respbody='Metadata value too long; max %d' % MAX_META_VALUE_LENGTH return jresponse('-1', respbody, req,400) elif meta_count > MAX_META_COUNT: respbody='Too many metadata items; max %d' % MAX_META_COUNT return jresponse('-1', respbody, req,400) elif meta_size > MAX_META_OVERALL_SIZE: respbody='Total metadata too large; max %d' % MAX_META_OVERALL_SIZE return jresponse('-1', respbody, req,400) return None