def _authorize_unconfirmed_identity(self, req, obj, referrers, roles): """" Perform authorization for access that does not require a confirmed identity. :returns: A boolean if authorization is granted or denied. None if a determination could not be made. """ # Allow container sync. if (req.environ.get('swift_sync_key') and req.environ['swift_sync_key'] == req.headers.get('x-container-sync-key', None) and 'x-timestamp' in req.headers and (req.remote_addr in self.allowed_sync_hosts or swift_utils.get_remote_client(req) in self.allowed_sync_hosts)): log_msg = 'allowing proxy %s for container-sync' % req.remote_addr self.logger.debug(log_msg) return True # Check if referrer is allowed. if swift_acl.referrer_allowed(req.referer, referrers): if obj or '.rlistings' in roles: log_msg = 'authorizing %s via referer ACL' % req.referrer self.logger.debug(log_msg) return True return False
def authorize(self, req): env = req.environ env_identity = env.get('keystone.identity', {}) tenant = env_identity.get('tenant') try: version, account, container, obj = split_path(req.path, 1, 4, True) except ValueError: return HTTPNotFound(request=req) if account != '%s_%s' % (self.reseller_prefix, tenant[0]): self.logger.debug('tenant mismatch') return self.denied_response(req) # If user is in the swift operator group then make the owner of it. user_groups = env_identity.get('roles', []) for _group in self.keystone_swift_operator_roles.split(','): _group = _group.strip() if _group in user_groups: self.logger.debug( "User is in group: %s allow him to do whatever it wants" % (_group)) req.environ['swift_owner'] = True return None # If user is of the same name of the tenant then make owner of it. user = env_identity.get('user', '') if self.keystone_tenant_user_admin and user == tenant[1]: self.logger.debug("user: %s == %s tenant and option "\ "keystone_tenant_user_admin is set" % \ (user, tenant)) req.environ['swift_owner'] = True return None # Allow container sync if (req.environ.get('swift_sync_key') and req.environ['swift_sync_key'] == req.headers.get('x-container-sync-key', None) and 'x-timestamp' in req.headers and (req.remote_addr in self.allowed_sync_hosts or get_remote_client(req) in self.allowed_sync_hosts)): self.logger.debug('allowing container-sync') return None # Check if Referrer allow it referrers, groups = parse_acl(getattr(req, 'acl', None)) if referrer_allowed(req.referer, referrers): if obj or '.rlistings' in groups: self.logger.debug('authorizing via ACL') return None return self.denied_response(req) # Check if we have the group in the usergroups and allow it for user_group in user_groups: if user_group in groups: self.logger.debug('user in group which is allowed in" \ " ACL: %s authorizing' % (user_group)) return None # last but not least retun deny return self.denied_response(req)
def authorize(self, req): env = req.environ env_identity = env.get('keystone.identity', {}) tenant = env_identity.get('tenant') try: version, account, container, obj = split_path(req.path, 1, 4, True) except ValueError: return HTTPNotFound(request=req) if account != '%s_%s' % (self.reseller_prefix, tenant[0]): self.logger.debug('tenant mismatch: %s != %s_%s' % \ (account, self.reseller_prefix, tenant[0])) return self.denied_response(req) # If user is in the swift operator group then make the owner of it. user_groups = env_identity.get('roles', []) for _group in self.keystone_swift_operator_roles.split(','): _group = _group.strip() if _group in user_groups: self.logger.debug("User is in group: %s allows" % (_group)) req.environ['swift_owner'] = True return None # If user is of the same name of the tenant then make owner of it. user = env_identity.get('user', '') if self.keystone_tenant_user_admin and user == tenant[1]: self.logger.debug("user: %s == %s tenant and option "\ "keystone_tenant_user_admin is set" % \ (user, tenant)) req.environ['swift_owner'] = True return None # Allow container sync if (req.environ.get('swift_sync_key') and req.environ['swift_sync_key'] == req.headers.get( 'x-container-sync-key', None) and 'x-timestamp' in req.headers and (req.remote_addr in self.allowed_sync_hosts or get_remote_client(req) in self.allowed_sync_hosts)): self.logger.debug('allowing container-sync') return None # Check if Referrer allow it referrers, groups = parse_acl(getattr(req, 'acl', None)) if referrer_allowed(req.referer, referrers): if obj or '.rlistings' in groups: self.logger.debug('authorizing via ACL') return None return self.denied_response(req) # Check if we have the group in the usergroups and allow it for user_group in user_groups: if user_group in groups: self.logger.debug('user in group which is allowed in" \ " ACL: %s authorizing' % (user_group)) return None # last but not least retun deny return self.denied_response(req)
def handle_request(self, req): try: self.logger.set_statsd_prefix('proxy-server') if req.content_length and req.content_length < 0: return jresponse('-1','Invalid Content-Length',req,400) try: if not check_utf8(req.path_info): return jresponse('-1','Invalid UTF8',req,412) except UnicodeError: return jresponse('-1','Invalid UTF8',req,412) try: controller, path_parts = self.get_controller(req) p = req.path_info if isinstance(p, unicode): p = p.encode('utf-8') except ValueError: return jresponse('-1','not found',req,404) if not controller: return jresponse('-1','Bad URL',req,412) if self.deny_host_headers and \ req.host.split(':')[0] in self.deny_host_headers: return HTTPForbidden(request=req, body='Invalid host header') if not check_path_parts(path_parts): return HTTPForbidden(request=req, body='Invalid path_parts header') self.logger.set_statsd_prefix('proxy-server.' + controller.server_type.lower()) controller = controller(self, **path_parts) if 'swift.trans_id' not in req.environ: # if this wasn't set by an earlier middleware, set it now trans_id = 'tx' + uuid.uuid4().hex req.environ['swift.trans_id'] = trans_id self.logger.txn_id = trans_id req.headers['x-trans-id'] = req.environ['swift.trans_id'] controller.trans_id = req.environ['swift.trans_id'] self.logger.client_ip = get_remote_client(req) try: if req.GET.get('op'): req.method = req.GET.get('op') handler = getattr(controller, req.method) getattr(handler, 'publicly_accessible') except AttributeError: return HTTPMethodNotAllowed(request=req) if path_parts['version']: req.path_info_pop() req.environ['swift.orig_req_method'] = req.method return handler(req) except (Exception, Timeout): self.logger.exception(_('ERROR Unhandled exception in request')) return jresponse('-1','ServerERROR',req,500)
def _authorize_unconfirmed_identity(self, req, obj, referrers, roles): """" Perform authorization for access that does not require a confirmed identity. :returns: A boolean if authorization is granted or denied. None if a determination could not be made. """ # Allow container sync. if (req.environ.get('swift_sync_key') and req.environ['swift_sync_key'] == req.headers.get( 'x-container-sync-key', None) and 'x-timestamp' in req.headers and (req.remote_addr in self.allowed_sync_hosts or swift_utils.get_remote_client(req) in self.allowed_sync_hosts)): log_msg = 'allowing proxy %s for container-sync' % req.remote_addr self.logger.debug(log_msg) return True # Check if referrer is allowed. if swift_acl.referrer_allowed(req.referer, referrers): if obj or '.rlistings' in roles: log_msg = 'authorizing %s via referer ACL' % req.referrer self.logger.debug(log_msg) return True return False
def authorize(self, req): env = req.environ identity = env.get('cloudstack.identity', {}) try: version, _account, container, obj = split_path(req.path, minsegs=1, maxsegs=4, rest_with_last=True) except ValueError: return HTTPNotFound(request=req) if not _account or not _account.startswith(self.reseller_prefix): return self.denied_response(req) # Remove the reseller_prefix from the account. if self.reseller_prefix != '': account = _account[len(self.reseller_prefix) + 1:] else: account = _account user_roles = identity.get('roles', []) # If this user is part of this account or is the global admin, give access. if account == identity.get( 'account') or self.cs_roles[1] in user_roles: req.environ['swift_owner'] = True return None # Allow container sync if (req.environ.get('swift_sync_key') and req.environ['swift_sync_key'] == req.headers.get( 'x-container-sync-key', None) and 'x-timestamp' in req.headers and (req.remote_addr in self.allowed_sync_hosts or get_remote_client(req) in self.allowed_sync_hosts)): self.logger.debug('Allowing container-sync') return None # Check if Referrer allow it referrers, groups = parse_acl(getattr(req, 'acl', None)) if referrer_allowed(req.referer, referrers): if obj or '.rlistings' in groups: self.logger.debug('Authorizing via ACL') return None return self.denied_response(req) # Check if we have the group in the user_roles and allow if we do for role in user_roles: if role in groups: self.logger.debug('User has role %s, allowing via ACL' % (role)) return None # This user is not authorized, deny request. return self.denied_response(req)
def log_request(self, req, status_int, bytes_received, bytes_sent, request_time): """ Log a request. :param req: swob.Request object for the request :param status_int: integer code for the response status :param bytes_received: bytes successfully read from the request body :param bytes_sent: bytes yielded to the WSGI server :param request_time: time taken to satisfy the request, in seconds """ if self.req_already_logged(req): return req_path = get_valid_utf8_str(req.path) the_request = quote(unquote(req_path), QUOTE_SAFE) if req.query_string: the_request = the_request + '?' + req.query_string logged_headers = None if self.log_hdrs: logged_headers = '\n'.join('%s: %s' % (k, v) for k, v in req.headers.items()) method = self.method_from_req(req) self.access_logger.info(' '.join( quote(str(x) if x else '-', QUOTE_SAFE) for x in ( get_remote_client(req), req.remote_addr, time.strftime('%d/%b/%Y/%H/%M/%S', time.gmtime()), method, the_request, req.environ.get('SERVER_PROTOCOL'), status_int, req.referer, req.user_agent, self.obscure_sensitive(req.headers.get('x-auth-token')), bytes_received, bytes_sent, req.headers.get('etag', None), req.environ.get('swift.trans_id'), logged_headers, '%.4f' % request_time, req.environ.get('swift.source'), ','.join(req.environ.get('swift.log_info') or '-'), ))) self.mark_req_logged(req) # Log timing and bytes-transfered data to StatsD metric_name = self.statsd_metric_name(req, status_int, method) # Only log data for valid controllers (or SOS) to keep the metric count # down (egregious errors will get logged by the proxy server itself). if metric_name: self.access_logger.timing(metric_name + '.timing', request_time * 1000) self.access_logger.update_stats(metric_name + '.xfer', bytes_received + bytes_sent)
def log_request(self, req, status_int, bytes_received, bytes_sent, request_time): """ Log a request. :param req: swob.Request object for the request :param status_int: integer code for the response status :param bytes_received: bytes successfully read from the request body :param bytes_sent: bytes yielded to the WSGI server :param request_time: time taken to satisfy the request, in seconds """ if self.req_already_logged(req): return req_path = get_valid_utf8_str(req.path) the_request = quote(unquote(req_path), QUOTE_SAFE) if req.query_string: the_request = the_request + '?' + req.query_string logged_headers = None if self.log_hdrs: logged_headers = '\n'.join('%s: %s' % (k, v) for k, v in req.headers.items()) method = self.method_from_req(req) self.access_logger.info(' '.join( quote(str(x) if x else '-', QUOTE_SAFE) for x in ( get_remote_client(req), req.remote_addr, time.strftime('%d/%b/%Y/%H/%M/%S', time.gmtime()), method, the_request, req.environ.get('SERVER_PROTOCOL'), status_int, req.referer, req.user_agent, self.obscure_sensitive(req.headers.get('x-auth-token')), bytes_received, bytes_sent, req.headers.get('etag', None), req.environ.get('swift.trans_id'), logged_headers, '%.4f' % request_time, req.environ.get('swift.source'), ','.join(req.environ.get('swift.log_info') or ''), ))) self.mark_req_logged(req) # Log timing and bytes-transfered data to StatsD metric_name = self.statsd_metric_name(req, status_int, method) # Only log data for valid controllers (or SOS) to keep the metric count # down (egregious errors will get logged by the proxy server itself). if metric_name: self.access_logger.timing(metric_name + '.timing', request_time * 1000) self.access_logger.update_stats(metric_name + '.xfer', bytes_received + bytes_sent)
def __call__(self, env, start_response): req = Request(env) if req.method in self.verb_acl: remote = get_remote_client(req) for block in self.verb_acl[req.method]: if remote.startswith(block): break else: raise HTTPForbidden(request=req, body='Forbidden method for %s' % remote) return self.app(env, start_response)
def authorize(self, req): env = req.environ identity = env.get("cloudstack.identity", {}) try: version, _account, container, obj = split_path(req.path, minsegs=1, maxsegs=4, rest_with_last=True) except ValueError: return HTTPNotFound(request=req) if not _account or not _account.startswith(self.reseller_prefix): return self.denied_response(req) # Remove the reseller_prefix from the account. if self.reseller_prefix != "": account = _account[len(self.reseller_prefix) + 1 :] else: account = _account user_roles = identity.get("roles", []) # If this user is part of this account or is the global admin, give access. if account == identity.get("account") or self.cs_roles[1] in user_roles: req.environ["swift_owner"] = True return None # Allow container sync if ( req.environ.get("swift_sync_key") and req.environ["swift_sync_key"] == req.headers.get("x-container-sync-key", None) and "x-timestamp" in req.headers and (req.remote_addr in self.allowed_sync_hosts or get_remote_client(req) in self.allowed_sync_hosts) ): self.logger.debug("Allowing container-sync") return None # Check if Referrer allow it referrers, groups = parse_acl(getattr(req, "acl", None)) if referrer_allowed(req.referer, referrers): if obj or ".rlistings" in groups: self.logger.debug("Authorizing via ACL") return None return self.denied_response(req) # Check if we have the group in the user_roles and allow if we do for role in user_roles: if role in groups: self.logger.debug("User has role %s, allowing via ACL" % (role)) return None # This user is not authorized, deny request. return self.denied_response(req)
def authorize(self, req): env = req.environ env_identity = env.get('keystone.identity', {}) tenant = env_identity.get('tenant') try: version, account, container, obj = split_path(req.path, 1, 4, True) except ValueError: return HTTPNotFound(request=req) if account != '%s_%s' % (self.reseller_prefix, tenant): self.logger.debug('tenant mismatch: %s != %s_%s' % \ (account, self.reseller_prefix, tenant)) return self.denied_response(req) user_groups = env_identity.get('roles', []) #TODO: setting? if self.keystone_admin_group in user_groups: req.environ['swift_owner'] = True return None if (req.environ.get('swift_sync_key') and req.environ['swift_sync_key'] == req.headers.get('x-container-sync-key', None) and 'x-timestamp' in req.headers and (req.remote_addr in self.allowed_sync_hosts or get_remote_client(req) in self.allowed_sync_hosts)): self.logger.debug('allowing container-sync') return None # Check if Referrer allow it #TODO: check if it works referrers, groups = parse_acl(getattr(req, 'acl', None)) if referrer_allowed(req.referer, referrers): if obj or '.rlistings' in groups: self.logger.debug('authorizing via ACL') return None return self.denied_response(req) # Check if we have the group in the group user and allow it for user_group in user_groups: if user_group in groups: self.logger.debug('user in group: %s authorizing' % \ (user_group)) return None return self.denied_response(req)
def authorize(self, req): """ Returns None if the request is authorized to continue or a standard WSGI response callable if not. """ try: version, account, container, obj = split_path(req.path, 1, 4, True) except ValueError: self.logger.increment('errors') return HTTPNotFound(request=req) if not account or not account.startswith(self.reseller_prefix): return self.denied_response(req) user_groups = (req.remote_user or '').split(',') if '.reseller_admin' in user_groups and \ account != self.reseller_prefix and \ account[len(self.reseller_prefix)] != '.': req.environ['swift_owner'] = True return None if account in user_groups and \ (req.method not in ('DELETE', 'PUT') or container): # If the user is admin for the account and is not trying to do an # account DELETE or PUT... req.environ['swift_owner'] = True return None if (req.environ.get('swift_sync_key') and req.environ['swift_sync_key'] == req.headers.get('x-container-sync-key', None) and 'x-timestamp' in req.headers and (req.remote_addr in self.allowed_sync_hosts or get_remote_client(req) in self.allowed_sync_hosts)): return None if req.method == 'OPTIONS': #allow OPTIONS requests to proceed as normal return None referrers, groups = parse_acl(getattr(req, 'acl', None)) if referrer_allowed(req.referer, referrers): if obj or '.rlistings' in groups: return None return self.denied_response(req) if not req.remote_user: return self.denied_response(req) for user_group in user_groups: if user_group in groups: return None return self.denied_response(req)
def log_request(self, env, status_int, bytes_received, bytes_sent, request_time, client_disconnect): """ Log a request. :param env: WSGI environment :param status_int: integer code for the response status :param bytes_received: bytes successfully read from the request body :param bytes_sent: bytes yielded to the WSGI server :param request_time: time taken to satisfy the request, in seconds """ req = Request(env) if client_disconnect: # log disconnected clients as '499' status code status_int = 499 req_path = get_valid_utf8_str(req.path) the_request = quote(unquote(req_path)) if req.query_string: the_request = the_request + "?" + req.query_string logged_headers = None if self.log_hdrs: logged_headers = "\n".join("%s: %s" % (k, v) for k, v in req.headers.items()) self.access_logger.info( " ".join( quote(str(x) if x else "-") for x in ( get_remote_client(req), req.remote_addr, time.strftime("%d/%b/%Y/%H/%M/%S", time.gmtime()), req.method, the_request, req.environ.get("SERVER_PROTOCOL"), status_int, req.referer, req.user_agent, req.headers.get("x-auth-token"), bytes_received, bytes_sent, req.headers.get("etag", None), req.environ.get("swift.trans_id"), logged_headers, "%.4f" % request_time, req.environ.get("swift.source"), ) ) ) self.access_logger.txn_id = None
def log_request(self, env, status_int, bytes_received, bytes_sent, request_time, client_disconnect): """ Log a request. :param env: WSGI environment :param status_int: integer code for the response status :param bytes_received: bytes successfully read from the request body :param bytes_sent: bytes yielded to the WSGI server :param request_time: time taken to satisfy the request, in seconds """ req = Request(env) if client_disconnect: # log disconnected clients as '499' status code status_int = 499 the_request = quote(unquote(req.path)) if req.query_string: the_request = the_request + '?' + req.query_string logged_headers = None if self.log_hdrs: logged_headers = '\n'.join('%s: %s' % (k, v) for k, v in req.headers.items()) self.access_logger.info(' '.join(quote(str(x) if x else '-') for x in ( get_remote_client(req), req.remote_addr, time.strftime('%d/%b/%Y/%H/%M/%S', time.gmtime()), req.method, the_request, req.environ.get('SERVER_PROTOCOL'), status_int, req.referer, req.user_agent, req.headers.get('x-auth-token'), bytes_received, bytes_sent, req.headers.get('etag', None), req.environ.get('swift.trans_id'), logged_headers, '%.4f' % request_time, req.environ.get('swift.source'), ))) self.access_logger.txn_id = None
def log_request(self, env, status_int, bytes_received, bytes_sent, request_time, client_disconnect): """ Log a request. :param env: WSGI environment :param status_int: integer code for the response status :param bytes_received: bytes successfully read from the request body :param bytes_sent: bytes yielded to the WSGI server :param request_time: time taken to satisfy the request, in seconds """ req = Request(env) if client_disconnect: # log disconnected clients as '499' status code status_int = 499 the_request = quote(unquote(req.path)) if req.query_string: the_request = the_request + '?' + req.query_string logged_headers = None if self.log_hdrs: logged_headers = '\n'.join('%s: %s' % (k, v) for k, v in req.headers.items()) self.access_logger.info(' '.join( quote(str(x) if x else '-') for x in ( get_remote_client(req), req.remote_addr, time.strftime('%d/%b/%Y/%H/%M/%S', time.gmtime()), req.method, the_request, req.environ.get('SERVER_PROTOCOL'), status_int, req.referer, req.user_agent, req.headers.get('x-auth-token'), bytes_received, bytes_sent, req.headers.get('etag', None), req.environ.get('swift.trans_id'), logged_headers, '%.4f' % request_time, req.environ.get('swift.source'), ))) self.access_logger.txn_id = None
def handle_request(self, req): """ Entry point for proxy server. Should return a WSGI-style callable (such as swob.Response). :param req: swob.Request object """ try: #设置日志的前缀为proxy-server self.logger.set_statsd_prefix('proxy-server') #如果请求长度为负数,报错 if req.content_length and req.content_length < 0: self.logger.increment('errors') return HTTPBadRequest(request=req, body='Invalid Content-Length') try: #如果路径信息不是有效的utf-8编码,报错 if not check_utf8(req.path_info): self.logger.increment('errors') return HTTPPreconditionFailed( request=req, body='Invalid UTF8 or contains NULL') except UnicodeError: #解码utf-8失败,报错 self.logger.increment('errors') return HTTPPreconditionFailed( request=req, body='Invalid UTF8 or contains NULL') try: #1、根据请求的路径信息,获取对应的控制器对象,并返回路径字典 controller, path_parts = self.get_controller(req) p = req.path_info if isinstance(p, six.text_type): p = p.encode('utf-8') except APIVersionError: self.logger.increment('errors') return HTTPBadRequest(request=req) except ValueError: self.logger.increment('errors') return HTTPNotFound(request=req) if not controller: self.logger.increment('errors') return HTTPPreconditionFailed(request=req, body='Bad URL') if self.deny_host_headers and \ req.host.split(':')[0] in self.deny_host_headers: return HTTPForbidden(request=req, body='Invalid host header') self.logger.set_statsd_prefix('proxy-server.' + controller.server_type.lower()) #2、生成控制器对象 controller = controller(self, **path_parts) #如果没有在请求的env中设置swift.trans_id,那么现在设置 if 'swift.trans_id' not in req.environ: # if this wasn't set by an earlier middleware, set it now trans_id_suffix = self.trans_id_suffix trans_id_extra = req.headers.get('x-trans-id-extra') if trans_id_extra: trans_id_suffix += '-' + trans_id_extra[:32] trans_id = generate_trans_id(trans_id_suffix) req.environ['swift.trans_id'] = trans_id self.logger.txn_id = trans_id req.headers['x-trans-id'] = req.environ['swift.trans_id'] controller.trans_id = req.environ['swift.trans_id'] self.logger.client_ip = get_remote_client(req) try: #3、根据请求方法,获取对应的函数指针handler handler = getattr(controller, req.method) getattr(handler, 'publicly_accessible') except AttributeError: allowed_methods = getattr(controller, 'allowed_methods', set()) return HTTPMethodNotAllowed( request=req, headers={'Allow': ', '.join(allowed_methods)}) old_authorize = None #4、如果请求的env中有鉴权方法,调用该鉴权方法,进行鉴权 if 'swift.authorize' in req.environ: # We call authorize before the handler, always. If authorized, # we remove the swift.authorize hook so isn't ever called # again. If not authorized, we return the denial unless the # controller's method indicates it'd like to gather more # information and try again later. resp = req.environ['swift.authorize'](req) if not resp and not req.headers.get('X-Copy-From-Account') \ and not req.headers.get('Destination-Account'): # No resp means authorized, no delayed recheck required. old_authorize = req.environ['swift.authorize'] else: # 返回resp代表鉴权失败,但是我们可能延迟后重新检查,如果没有设置延迟检查,则返回失败 # Response indicates denial, but we might delay the denial # and recheck later. If not delayed, return the error now. if not getattr(handler, 'delay_denial', None): return resp # Save off original request method (GET, POST, etc.) in case it # gets mutated during handling. This way logging can display the # method the client actually sent. req.environ['swift.orig_req_method'] = req.method try: #将鉴权方法从请求的env中取出,以免后续再次调用 if old_authorize: req.environ.pop('swift.authorize', None) #5、调用处理请求的方法,处理请求 return handler(req) finally: if old_authorize: req.environ['swift.authorize'] = old_authorize except HTTPException as error_response: return error_response except (Exception, Timeout): self.logger.exception(_('ERROR Unhandled exception in request')) return HTTPServerError(request=req)
def log_request(self, req, status_int, bytes_received, bytes_sent, start_time, end_time, resp_headers=None): """ Log a request. :param req: swob.Request object for the request :param status_int: integer code for the response status :param bytes_received: bytes successfully read from the request body :param bytes_sent: bytes yielded to the WSGI server :param start_time: timestamp request started :param end_time: timestamp request completed :param resp_headers: dict of the response headers """ resp_headers = resp_headers or {} req_path = get_valid_utf8_str(req.path) the_request = quote(unquote(req_path), QUOTE_SAFE) if req.query_string: the_request = the_request + '?' + req.query_string logged_headers = None if self.log_hdrs: if self.log_hdrs_only: logged_headers = '\n'.join('%s: %s' % (k, v) for k, v in req.headers.items() if k in self.log_hdrs_only) else: logged_headers = '\n'.join('%s: %s' % (k, v) for k, v in req.headers.items()) method = self.method_from_req(req) end_gmtime_str = time.strftime('%d/%b/%Y/%H/%M/%S', time.gmtime(end_time)) duration_time_str = "%.4f" % (end_time - start_time) start_time_str = "%.9f" % start_time end_time_str = "%.9f" % end_time policy_index = get_policy_index(req.headers, resp_headers) self.access_logger.info(' '.join( quote(str(x) if x else '-', QUOTE_SAFE) for x in ( get_remote_client(req), req.remote_addr, end_gmtime_str, method, the_request, req.environ.get('SERVER_PROTOCOL'), status_int, req.referer, req.user_agent, self.obscure_sensitive(req.headers.get('x-auth-token')), bytes_received, bytes_sent, req.headers.get('etag', None), req.environ.get('swift.trans_id'), logged_headers, duration_time_str, req.environ.get('swift.source'), ','.join(req.environ.get('swift.log_info') or ''), start_time_str, end_time_str, policy_index ))) # Log timing and bytes-transferred data to StatsD metric_name = self.statsd_metric_name(req, status_int, method) # Only log data for valid controllers (or SOS) to keep the metric count # down (egregious errors will get logged by the proxy server itself). if metric_name: self.access_logger.timing(metric_name + '.timing', (end_time - start_time) * 1000) self.access_logger.update_stats(metric_name + '.xfer', bytes_received + bytes_sent)
def handle_request(self, req): """ Entry point for proxy server. Should return a WSGI-style callable (such as swob.Response). :param req: swob.Request object """ try: self.logger.set_statsd_prefix('proxy-server') # content_length存在且小于0,返回错误。 if req.content_length and req.content_length < 0: self.logger.increment('errors') return HTTPBadRequest(request=req, body='Invalid Content-Length') try: if not check_utf8(req.path_info): self.logger.increment('errors') return HTTPPreconditionFailed( request=req, body='Invalid UTF8 or contains NULL') except UnicodeError: self.logger.increment('errors') return HTTPPreconditionFailed( request=req, body='Invalid UTF8 or contains NULL') try: # 根据 path获取相应的 controller, 如: # 1. AccountController. 2. ContainerController # 3. ECObjectController 4. ReplicatedObjectController # 5. InfoController # path_parts is a dict : (version=version, # account_name=account, # container_name=container, # object_name=obj) controller, path_parts = self.get_controller(req) except APIVersionError: self.logger.increment('errors') return HTTPBadRequest(request=req) except ValueError: self.logger.increment('errors') return HTTPNotFound(request=req) if not controller: self.logger.increment('errors') return HTTPPreconditionFailed(request=req, body='Bad URL') if self.deny_host_headers and \ req.host.split(':')[0] in self.deny_host_headers: return HTTPForbidden(request=req, body='Invalid host header') self.logger.set_statsd_prefix('proxy-server.' + controller.server_type.lower()) # 实例化 controller controller = controller(self, **path_parts) if 'swift.trans_id' not in req.environ: # if this wasn't set by an earlier middleware, set it now trans_id_suffix = self.trans_id_suffix trans_id_extra = req.headers.get('x-trans-id-extra') if trans_id_extra: trans_id_suffix += '-' + trans_id_extra[:32] trans_id = generate_trans_id(trans_id_suffix) req.environ['swift.trans_id'] = trans_id self.logger.txn_id = trans_id req.headers['x-trans-id'] = req.environ['swift.trans_id'] controller.trans_id = req.environ['swift.trans_id'] self.logger.client_ip = get_remote_client(req) if req.method not in controller.allowed_methods: return HTTPMethodNotAllowed( request=req, headers={'Allow': ', '.join(controller.allowed_methods)}) # getattr(object, name) 返回一个对象的属性值 # 根据 req.method 的值返回controller的 HEAD() GET() PUT() 等函数 handler = getattr(controller, req.method) old_authorize = None # swift.authrize 是 swift_auth 提供的句柄 if 'swift.authorize' in req.environ: # We call authorize before the handler, always. If authorized, # we remove the swift.authorize hook so isn't ever called # again. If not authorized, we return the denial unless the # controller's method indicates it'd like to gather more # information and try again later. resp = req.environ['swift.authorize'](req) if not resp: # No resp means authorized, no delayed recheck required. old_authorize = req.environ['swift.authorize'] else: # Response indicates denial, but we might delay the denial # and recheck later. If not delayed, return the error now. if not getattr(handler, 'delay_denial', None): return resp # Save off original request method (GET, POST, etc.) in case it # gets mutated during handling. This way logging can display the # method the client actually sent. req.environ.setdefault('swift.orig_req_method', req.method) try: if old_authorize: req.environ.pop('swift.authorize', None) return handler(req) # call HEAD(),GET(),PUT(),etc... finally: if old_authorize: req.environ['swift.authorize'] = old_authorize except HTTPException as error_response: return error_response except (Exception, Timeout): self.logger.exception(_('ERROR Unhandled exception in request')) return HTTPServerError(request=req)
def log_request(self, req, status_int, bytes_received, bytes_sent, start_time, end_time, resp_headers=None): """ Log a request. :param req: swob.Request object for the request :param status_int: integer code for the response status :param bytes_received: bytes successfully read from the request body :param bytes_sent: bytes yielded to the WSGI server :param start_time: timestamp request started :param end_time: timestamp request completed :param resp_headers: dict of the response headers """ resp_headers = resp_headers or {} logged_headers = None if self.log_hdrs: if self.log_hdrs_only: logged_headers = '\n'.join('%s: %s' % (k, v) for k, v in req.headers.items() if k in self.log_hdrs_only) else: logged_headers = '\n'.join('%s: %s' % (k, v) for k, v in req.headers.items()) method = self.method_from_req(req) end_gmtime_str = time.strftime('%d/%b/%Y/%H/%M/%S', time.gmtime(end_time)) duration_time_str = "%.4f" % (end_time - start_time) start_time_str = "%.9f" % start_time end_time_str = "%.9f" % end_time policy_index = get_policy_index(req.headers, resp_headers) self.access_logger.info(' '.join( quote(str(x) if x else '-', QUOTE_SAFE) for x in (get_remote_client(req), req.remote_addr, end_gmtime_str, method, req.path_qs, req.environ.get('SERVER_PROTOCOL'), status_int, req.referer, req.user_agent, self.obscure_sensitive(req.headers.get('x-auth-token')), bytes_received, bytes_sent, req.headers.get('etag', None), req.environ.get('swift.trans_id'), logged_headers, duration_time_str, req.environ.get('swift.source'), ','.join(req.environ.get('swift.log_info') or ''), start_time_str, end_time_str, policy_index))) # Log timing and bytes-transferred data to StatsD metric_name = self.statsd_metric_name(req, status_int, method) metric_name_policy = self.statsd_metric_name_policy( req, status_int, method, policy_index) # Only log data for valid controllers (or SOS) to keep the metric count # down (egregious errors will get logged by the proxy server itself). if metric_name: self.access_logger.timing(metric_name + '.timing', (end_time - start_time) * 1000) self.access_logger.update_stats(metric_name + '.xfer', bytes_received + bytes_sent) if metric_name_policy: self.access_logger.timing(metric_name_policy + '.timing', (end_time - start_time) * 1000) self.access_logger.update_stats(metric_name_policy + '.xfer', bytes_received + bytes_sent)
def handle_request(self, req): """ Entry point for proxy server. Should return a WSGI-style callable (such as webob.Response). :param req: webob.Request object """ try: self.logger.set_statsd_prefix('proxy-server') if req.content_length and req.content_length < 0: self.logger.increment('errors') return HTTPBadRequest(request=req, body='Invalid Content-Length') try: if not check_utf8(req.path_info): self.logger.increment('errors') return HTTPPreconditionFailed(request=req, body='Invalid UTF8') except UnicodeError: self.logger.increment('errors') return HTTPPreconditionFailed(request=req, body='Invalid UTF8') try: controller, path_parts = self.get_controller(req.path) p = req.path_info if isinstance(p, unicode): p = p.encode('utf-8') except ValueError: self.logger.increment('errors') return HTTPNotFound(request=req) if not controller: self.logger.increment('errors') return HTTPPreconditionFailed(request=req, body='Bad URL') if self.deny_host_headers and \ req.host.split(':')[0] in self.deny_host_headers: return HTTPForbidden(request=req, body='Invalid host header') self.logger.set_statsd_prefix('proxy-server.' + controller.server_type) controller = controller(self, **path_parts) if 'swift.trans_id' not in req.environ: # if this wasn't set by an earlier middleware, set it now trans_id = 'tx' + uuid.uuid4().hex req.environ['swift.trans_id'] = trans_id self.logger.txn_id = trans_id req.headers['x-trans-id'] = req.environ['swift.trans_id'] controller.trans_id = req.environ['swift.trans_id'] self.logger.client_ip = get_remote_client(req) try: handler = getattr(controller, req.method) getattr(handler, 'publicly_accessible') except AttributeError: self.logger.increment('method_not_allowed') return HTTPMethodNotAllowed(request=req) if path_parts['version']: req.path_info_pop() if 'swift.authorize' in req.environ: # We call authorize before the handler, always. If authorized, # we remove the swift.authorize hook so isn't ever called # again. If not authorized, we return the denial unless the # controller's method indicates it'd like to gather more # information and try again later. resp = req.environ['swift.authorize'](req) if not resp: # No resp means authorized, no delayed recheck required. del req.environ['swift.authorize'] else: # Response indicates denial, but we might delay the denial # and recheck later. If not delayed, return the error now. if not getattr(handler, 'delay_denial', None): self.logger.increment('auth_short_circuits') return resp return handler(req) except (Exception, Timeout): self.logger.exception(_('ERROR Unhandled exception in request')) return HTTPServerError(request=req)
def handle_request(self, req): """ Entry point for proxy server. Should return a WSGI-style callable (such as swob.Response). :param req: swob.Request object """ try: self.logger.set_statsd_prefix("proxy-server") if req.content_length and req.content_length < 0: self.logger.increment("errors") return HTTPBadRequest(request=req, body="Invalid Content-Length") try: if not check_utf8(req.path_info): self.logger.increment("errors") return HTTPPreconditionFailed(request=req, body="Invalid UTF8 or contains NULL") except UnicodeError: self.logger.increment("errors") return HTTPPreconditionFailed(request=req, body="Invalid UTF8 or contains NULL") try: controller, path_parts = self.get_controller(req) p = req.path_info if isinstance(p, six.text_type): p = p.encode("utf-8") except APIVersionError: self.logger.increment("errors") return HTTPBadRequest(request=req) except ValueError: self.logger.increment("errors") return HTTPNotFound(request=req) if not controller: self.logger.increment("errors") return HTTPPreconditionFailed(request=req, body="Bad URL") if self.deny_host_headers and req.host.split(":")[0] in self.deny_host_headers: return HTTPForbidden(request=req, body="Invalid host header") self.logger.set_statsd_prefix("proxy-server." + controller.server_type.lower()) controller = controller(self, **path_parts) if "swift.trans_id" not in req.environ: # if this wasn't set by an earlier middleware, set it now trans_id_suffix = self.trans_id_suffix trans_id_extra = req.headers.get("x-trans-id-extra") if trans_id_extra: trans_id_suffix += "-" + trans_id_extra[:32] trans_id = generate_trans_id(trans_id_suffix) req.environ["swift.trans_id"] = trans_id self.logger.txn_id = trans_id req.headers["x-trans-id"] = req.environ["swift.trans_id"] controller.trans_id = req.environ["swift.trans_id"] self.logger.client_ip = get_remote_client(req) try: handler = getattr(controller, req.method) getattr(handler, "publicly_accessible") except AttributeError: allowed_methods = getattr(controller, "allowed_methods", set()) return HTTPMethodNotAllowed(request=req, headers={"Allow": ", ".join(allowed_methods)}) old_authorize = None if "swift.authorize" in req.environ: # We call authorize before the handler, always. If authorized, # we remove the swift.authorize hook so isn't ever called # again. If not authorized, we return the denial unless the # controller's method indicates it'd like to gather more # information and try again later. resp = req.environ["swift.authorize"](req) if ( not resp and not req.headers.get("X-Copy-From-Account") and not req.headers.get("Destination-Account") ): # No resp means authorized, no delayed recheck required. old_authorize = req.environ["swift.authorize"] else: # Response indicates denial, but we might delay the denial # and recheck later. If not delayed, return the error now. if not getattr(handler, "delay_denial", None): return resp # Save off original request method (GET, POST, etc.) in case it # gets mutated during handling. This way logging can display the # method the client actually sent. req.environ["swift.orig_req_method"] = req.method try: if old_authorize: req.environ.pop("swift.authorize", None) return handler(req) finally: if old_authorize: req.environ["swift.authorize"] = old_authorize except HTTPException as error_response: return error_response except (Exception, Timeout): self.logger.exception(_("ERROR Unhandled exception in request")) return HTTPServerError(request=req)
def authorize(self, req): env = req.environ identity = env.get('mauth.identity', {}) try: version, _account, container, obj = split_path(req.path, minsegs=1, maxsegs=4, rest_with_last=True) except ValueError: return HTTPNotFound(request=req) if not _account or not _account.startswith(self.reseller_prefix): return self.denied_response(req) user_roles = identity.get('roles', []) # If this user is part of this account, give access. if _account == identity.get('account_part'): req.environ['swift_owner'] = True return None # Allow container sync if (req.environ.get('swift_sync_key') and req.environ['swift_sync_key'] == req.headers.get('x-container-sync-key', None) and 'x-timestamp' in req.headers and (req.remote_addr in self.allowed_sync_hosts or get_remote_client(req) in self.allowed_sync_hosts)): self.logger.debug('Allowing container-sync') return None # Check if Referrer allow it referrers, groups = parse_acl(getattr(req, 'acl', None)) if referrer_allowed(req.referer, referrers): if obj or '.rlistings' in groups: self.logger.debug('Authorizing via ACL') return None return self.denied_response(req) # Check if we have the group in the user_roles and allow if we do for role in user_roles: if role in groups: self.logger.debug('User has role %s, allowing via ACL' % (role)) return None # This user is not authorized, deny request. return self.denied_response(req)
def log_request(self, env, status_int, bytes_received, bytes_sent, request_time, client_disconnect): """ Log a request. :param env: WSGI environment :param status_int: integer code for the response status :param bytes_received: bytes successfully read from the request body :param bytes_sent: bytes yielded to the WSGI server :param request_time: time taken to satisfy the request, in seconds """ if env.get("swift.proxy_access_log_made"): return req = Request(env) if client_disconnect: # log disconnected clients as '499' status code status_int = 499 req_path = get_valid_utf8_str(req.path) the_request = quote(unquote(req_path)) if req.query_string: the_request = the_request + "?" + req.query_string logged_headers = None if self.log_hdrs: logged_headers = "\n".join("%s: %s" % (k, v) for k, v in req.headers.items()) method = req.environ.get("swift.orig_req_method", req.method) self.access_logger.info( " ".join( quote(str(x) if x else "-") for x in ( get_remote_client(req), req.remote_addr, time.strftime("%d/%b/%Y/%H/%M/%S", time.gmtime()), method, the_request, req.environ.get("SERVER_PROTOCOL"), status_int, req.referer, req.user_agent, req.headers.get("x-auth-token"), bytes_received, bytes_sent, req.headers.get("etag", None), req.environ.get("swift.trans_id"), logged_headers, "%.4f" % request_time, req.environ.get("swift.source"), ) ) ) env["swift.proxy_access_log_made"] = True # Log timing and bytes-transfered data to StatsD if req.path.startswith("/v1/"): try: stat_type = [None, "account", "container", "object"][req.path.strip("/").count("/")] except IndexError: stat_type = "object" else: stat_type = env.get("swift.source") # Only log data for valid controllers (or SOS) to keep the metric count # down (egregious errors will get logged by the proxy server itself). if stat_type: stat_method = method if method in self.valid_methods else "BAD_METHOD" metric_name = ".".join((stat_type, stat_method, str(status_int))) self.access_logger.timing(metric_name + ".timing", request_time * 1000) self.access_logger.update_stats(metric_name + ".xfer", bytes_received + bytes_sent)
def authorize(self, req): env = req.environ env_identity = env.get('keystone.identity', {}) tenant = env_identity.get('tenant') try: part = swift_utils.split_path(req.path, 1, 4, True) version, account, container, obj = part except ValueError: return webob.exc.HTTPNotFound(request=req) user_roles = env_identity.get('roles', []) # Give unconditional access to a user with the reseller_admin # role. if self.reseller_admin_role in user_roles: msg = 'User %s has reseller admin authorizing' self.logger.debug(msg % tenant[0]) req.environ['swift_owner'] = True return # Check if a user tries to access an account that does not match their # token if not self._reseller_check(account, tenant[0]): log_msg = 'tenant mismatch: %s != %s' % (account, tenant[0]) self.logger.debug(log_msg) return self.denied_response(req) # Check the roles the user is belonging to. If the user is # part of the role defined in the config variable # operator_roles (like admin) then it will be # promoted as an admin of the account/tenant. for role in self.operator_roles.split(','): role = role.strip() if role in user_roles: log_msg = 'allow user with role %s as account admin' % (role) self.logger.debug(log_msg) req.environ['swift_owner'] = True return # If user is of the same name of the tenant then make owner of it. user = env_identity.get('user', '') if self.is_admin and user == tenant[1]: req.environ['swift_owner'] = True return # Allow container sync. if (req.environ.get('swift_sync_key') and req.environ['swift_sync_key'] == req.headers.get('x-container-sync-key', None) and 'x-timestamp' in req.headers and (req.remote_addr in self.allowed_sync_hosts or swift_utils.get_remote_client(req) in self.allowed_sync_hosts)): log_msg = 'allowing proxy %s for container-sync' % req.remote_addr self.logger.debug(log_msg) return # Check if referrer is allowed. referrers, roles = swift_acl.parse_acl(getattr(req, 'acl', None)) if swift_acl.referrer_allowed(req.referer, referrers): #TODO(chmou): convert .rlistings to Keystone type role. if obj or '.rlistings' in roles: log_msg = 'authorizing %s via referer ACL' % req.referrer self.logger.debug(log_msg) return return self.denied_response(req) # Allow ACL at individual user level (tenant:user format) if '%s:%s' % (tenant[0], user) in roles: log_msg = 'user %s:%s allowed in ACL authorizing' self.logger.debug(log_msg % (tenant[0], user)) return # Check if we have the role in the userroles and allow it for user_role in user_roles: if user_role in roles: log_msg = 'user %s:%s allowed in ACL: %s authorizing' self.logger.debug(log_msg % (tenant[0], user, user_role)) return return self.denied_response(req)
def log_request(self, req, status_int, bytes_received, bytes_sent, start_time, end_time, resp_headers=None, ttfb=0, wire_status_int=None): """ Log a request. :param req: swob.Request object for the request :param status_int: integer code for the response status :param bytes_received: bytes successfully read from the request body :param bytes_sent: bytes yielded to the WSGI server :param start_time: timestamp request started :param end_time: timestamp request completed :param resp_headers: dict of the response headers :param wire_status_int: the on the wire status int """ resp_headers = resp_headers or {} logged_headers = None if self.log_hdrs: if self.log_hdrs_only: logged_headers = '\n'.join('%s: %s' % (k, v) for k, v in req.headers.items() if k in self.log_hdrs_only) else: logged_headers = '\n'.join('%s: %s' % (k, v) for k, v in req.headers.items()) method = self.method_from_req(req) duration_time_str = "%.4f" % (end_time - start_time) policy_index = get_policy_index(req.headers, resp_headers) acc, cont, obj = None, None, None swift_path = req.environ.get('swift.backend_path', req.path) if swift_path.startswith('/v1/'): _, acc, cont, obj = split_path(swift_path, 1, 4, True) replacements = { # Time information 'end_time': StrFormatTime(end_time), 'start_time': StrFormatTime(start_time), # Information worth to anonymize 'client_ip': StrAnonymizer(get_remote_client(req), self.anonymization_method, self.anonymization_salt), 'remote_addr': StrAnonymizer(req.remote_addr, self.anonymization_method, self.anonymization_salt), 'path': StrAnonymizer(req.path_qs, self.anonymization_method, self.anonymization_salt), 'referer': StrAnonymizer(req.referer, self.anonymization_method, self.anonymization_salt), 'user_agent': StrAnonymizer(req.user_agent, self.anonymization_method, self.anonymization_salt), 'headers': StrAnonymizer(logged_headers, self.anonymization_method, self.anonymization_salt), 'client_etag': StrAnonymizer(req.headers.get('etag'), self.anonymization_method, self.anonymization_salt), 'account': StrAnonymizer(acc, self.anonymization_method, self.anonymization_salt), 'container': StrAnonymizer(cont, self.anonymization_method, self.anonymization_salt), 'object': StrAnonymizer(obj, self.anonymization_method, self.anonymization_salt), # Others information 'method': method, 'protocol': req.environ.get('SERVER_PROTOCOL'), 'status_int': status_int, 'auth_token': self.obscure_sensitive(req.headers.get('x-auth-token')), 'bytes_recvd': bytes_received, 'bytes_sent': bytes_sent, 'transaction_id': req.environ.get('swift.trans_id'), 'request_time': duration_time_str, 'source': req.environ.get('swift.source'), 'log_info': ','.join(req.environ.get('swift.log_info', '')), 'policy_index': policy_index, 'ttfb': ttfb, 'pid': self.pid, 'wire_status_int': wire_status_int or status_int, } self.access_logger.info( self.log_formatter.format(self.log_msg_template, **replacements)) # Log timing and bytes-transferred data to StatsD metric_name = self.statsd_metric_name(req, status_int, method) metric_name_policy = self.statsd_metric_name_policy( req, status_int, method, policy_index) # Only log data for valid controllers (or SOS) to keep the metric count # down (egregious errors will get logged by the proxy server itself). if metric_name: self.access_logger.timing(metric_name + '.timing', (end_time - start_time) * 1000) self.access_logger.update_stats(metric_name + '.xfer', bytes_received + bytes_sent) if metric_name_policy: self.access_logger.timing(metric_name_policy + '.timing', (end_time - start_time) * 1000) self.access_logger.update_stats(metric_name_policy + '.xfer', bytes_received + bytes_sent)
def authorize(self, req): env = req.environ identity = env.get('cloudstack.identity', {}) try: version, _account, container, obj = split_path(req.path, minsegs=1, maxsegs=4, rest_with_last=True) except ValueError: return HTTPNotFound(request=req) if not _account or not _account.startswith(self.reseller_prefix): return self.denied_response(req) # Remove the reseller_prefix from the account. if self.reseller_prefix != '': account = _account[len(self.reseller_prefix)+1:] else: account = _account user_roles = identity.get('roles', []) # If this user is part of this account or is the global admin, give access. if account == identity.get('account') or self.cs_roles[1] in user_roles: req.environ['swift_owner'] = True self.logger.debug("User %s is global admin or owner, authorizing" % identity.get('username')) return None # Allow container sync if (req.environ.get('swift_sync_key') and req.environ['swift_sync_key'] == req.headers.get('x-container-sync-key', None) and 'x-timestamp' in req.headers and (req.remote_addr in self.allowed_sync_hosts or get_remote_client(req) in self.allowed_sync_hosts)): self.logger.debug('Allowing container-sync') return None if req.method == 'OPTIONS': #allow OPTIONS requests to proceed as normal self.logger.debug("Allow OPTIONS request.") return None # Check if Referrer allow it referrers, groups = parse_acl(getattr(req, 'acl', None)) if referrer_allowed(req.referer, referrers): if obj or '.rlistings' in groups: self.logger.debug('Authorizing via ACL') return None return self.denied_response(req) # Check if we have the group in the user_roles and allow if we do for role in user_roles: if role in groups: self.logger.debug('User has role %s, allowing via ACL' % (role)) return None # This user is not authorized, deny request. return self.denied_response(req)
def handle_request(self, req): """ Entry point for proxy server. Should return a WSGI-style callable (such as swob.Response). :param req: swob.Request object """ try: self.logger.set_statsd_prefix('proxy-server') if req.content_length and req.content_length < 0: self.logger.increment('errors') return HTTPBadRequest(request=req, body='Invalid Content-Length') try: if not check_utf8(req.path_info): self.logger.increment('errors') return HTTPPreconditionFailed( request=req, body='Invalid UTF8 or contains NULL') except UnicodeError: self.logger.increment('errors') return HTTPPreconditionFailed( request=req, body='Invalid UTF8 or contains NULL') try: controller, path_parts = self.get_controller(req) p = req.path_info if isinstance(p, six.text_type): p = p.encode('utf-8') except APIVersionError: self.logger.increment('errors') return HTTPBadRequest(request=req) except ValueError: self.logger.increment('errors') return HTTPNotFound(request=req) if not controller: self.logger.increment('errors') return HTTPPreconditionFailed(request=req, body='Bad URL') if self.deny_host_headers and \ req.host.split(':')[0] in self.deny_host_headers: return HTTPForbidden(request=req, body='Invalid host header') self.logger.set_statsd_prefix('proxy-server.' + controller.server_type.lower()) controller = controller(self, **path_parts) if 'swift.trans_id' not in req.environ: # if this wasn't set by an earlier middleware, set it now trans_id_suffix = self.trans_id_suffix trans_id_extra = req.headers.get('x-trans-id-extra') if trans_id_extra: trans_id_suffix += '-' + trans_id_extra[:32] trans_id = generate_trans_id(trans_id_suffix) req.environ['swift.trans_id'] = trans_id self.logger.txn_id = trans_id req.headers['x-trans-id'] = req.environ['swift.trans_id'] controller.trans_id = req.environ['swift.trans_id'] self.logger.client_ip = get_remote_client(req) try: handler = getattr(controller, req.method) getattr(handler, 'publicly_accessible') except AttributeError: allowed_methods = getattr(controller, 'allowed_methods', set()) return HTTPMethodNotAllowed( request=req, headers={'Allow': ', '.join(allowed_methods)}) old_authorize = None if 'swift.authorize' in req.environ: # We call authorize before the handler, always. If authorized, # we remove the swift.authorize hook so isn't ever called # again. If not authorized, we return the denial unless the # controller's method indicates it'd like to gather more # information and try again later. resp = req.environ['swift.authorize'](req) if not resp and not req.headers.get('X-Copy-From-Account') \ and not req.headers.get('Destination-Account'): # No resp means authorized, no delayed recheck required. old_authorize = req.environ['swift.authorize'] else: # Response indicates denial, but we might delay the denial # and recheck later. If not delayed, return the error now. if not getattr(handler, 'delay_denial', None): return resp # Save off original request method (GET, POST, etc.) in case it # gets mutated during handling. This way logging can display the # method the client actually sent. req.environ['swift.orig_req_method'] = req.method try: if old_authorize: req.environ.pop('swift.authorize', None) return handler(req) finally: if old_authorize: req.environ['swift.authorize'] = old_authorize except HTTPException as error_response: return error_response except (Exception, Timeout): self.logger.exception(_('ERROR Unhandled exception in request')) return HTTPServerError(request=req)
def handle_request(self, req): """ Entry point for proxy server. Should return a WSGI-style callable (such as swob.Response). :param req: swob.Request object """ try: self.logger.set_statsd_prefix('proxy-server') if req.content_length and req.content_length < 0: self.logger.increment('errors') return HTTPBadRequest(request=req, body='Invalid Content-Length') try: if not check_utf8(req.path_info): self.logger.increment('errors') return HTTPPreconditionFailed( request=req, body='Invalid UTF8 or contains NULL') except UnicodeError: self.logger.increment('errors') return HTTPPreconditionFailed( request=req, body='Invalid UTF8 or contains NULL') try: # 这里确定controller的类型,根据req的内容分别返回: # account controller,container controller 或者 object controller。 controller, path_parts = self.get_controller(req) p = req.path_info if isinstance(p, unicode): p = p.encode('utf-8') except APIVersionError: self.logger.increment('errors') return HTTPBadRequest(request=req) except ValueError: self.logger.increment('errors') return HTTPNotFound(request=req) if not controller: self.logger.increment('errors') return HTTPPreconditionFailed(request=req, body='Bad URL') if self.deny_host_headers and \ req.host.split(':')[0] in self.deny_host_headers: return HTTPForbidden(request=req, body='Invalid host header') self.logger.set_statsd_prefix('proxy-server.' + controller.server_type.lower()) controller = controller(self, **path_parts) if 'swift.trans_id' not in req.environ: # if this wasn't set by an earlier middleware, set it now trans_id_suffix = self.trans_id_suffix trans_id_extra = req.headers.get('x-trans-id-extra') if trans_id_extra: trans_id_suffix += '-' + trans_id_extra[:32] trans_id = generate_trans_id(trans_id_suffix) req.environ['swift.trans_id'] = trans_id self.logger.txn_id = trans_id req.headers['x-trans-id'] = req.environ['swift.trans_id'] controller.trans_id = req.environ['swift.trans_id'] self.logger.client_ip = get_remote_client(req) try: # 这里的handler是相应controller类中对应于req中方法的方法。 # 这些方法包括:GET,HEAD,POST,PUT,DELETE,COPY handler = getattr(controller, req.method) getattr(handler, 'publicly_accessible') except AttributeError: allowed_methods = getattr(controller, 'allowed_methods', set()) return HTTPMethodNotAllowed( request=req, headers={'Allow': ', '.join(allowed_methods)}) if 'swift.authorize' in req.environ: # We call authorize before the handler, always. If authorized, # we remove the swift.authorize hook so isn't ever called # again. If not authorized, we return the denial unless the # controller's method indicates it'd like to gather more # information and try again later. resp = req.environ['swift.authorize'](req) if not resp and not req.headers.get('X-Copy-From-Account') \ and not req.headers.get('Destination-Account'): # No resp means authorized, no delayed recheck required. del req.environ['swift.authorize'] else: # Response indicates denial, but we might delay the denial # and recheck later. If not delayed, return the error now. if not getattr(handler, 'delay_denial', None): return resp # Save off original request method (GET, POST, etc.) in case it # gets mutated during handling. This way logging can display the # method the client actually sent. req.environ['swift.orig_req_method'] = req.method # 这里返回的是相应于req的controller中具体的处理这个类型的req的方法。 # 经过层层调用之后,handler(req)其实是一个Response类的实例。 # 调用过程为(以get object为例): # swift.proxy.controllers.ObjectControllerRouter 中找到对应的controller # 在 BaseObjectController 中找到 GET 方法,GET 方法使用 GETorHEAD 方法 # GETorHEAD 方法跳到 ReplicatedObjectController 中的 _get_or_head_response 方法 # _get_or_head_response 使用 swift.proxy.controller.Controller(base.py)中的 GETorHEAD_base 方法 # GETorHEAD_base 中调用 GetOrHeadHandler 类中的 get_working_response 方法 # get_working_response 方法构建一个 Response 类的实例,并返回。 # 这个 Response 类的实例就是这里被返回的值。 return handler(req) except HTTPException as error_response: return error_response except (Exception, Timeout): self.logger.exception(_('ERROR Unhandled exception in request')) return HTTPServerError(request=req)
def handle_request(self, req): """ Entry point for proxy server. Should return a WSGI-style callable (such as swob.Response). :param req: swob.Request object """ try: self.logger.set_statsd_prefix('proxy-server') if req.content_length and req.content_length < 0: self.logger.increment('errors') return HTTPBadRequest(request=req, body='Invalid Content-Length') try: if not check_utf8(req.path_info): self.logger.increment('errors') return HTTPPreconditionFailed( request=req, body='Invalid UTF8 or contains NULL') except UnicodeError: self.logger.increment('errors') return HTTPPreconditionFailed( request=req, body='Invalid UTF8 or contains NULL') try: controller, path_parts = self.get_controller(req.path) p = req.path_info if isinstance(p, unicode): p = p.encode('utf-8') except ValueError: self.logger.increment('errors') return HTTPNotFound(request=req) if not controller: self.logger.increment('errors') return HTTPPreconditionFailed(request=req, body='Bad URL') if self.deny_host_headers and \ req.host.split(':')[0] in self.deny_host_headers: return HTTPForbidden(request=req, body='Invalid host header') self.logger.set_statsd_prefix('proxy-server.' + controller.server_type.lower()) controller = controller(self, **path_parts) if 'swift.trans_id' not in req.environ: # if this wasn't set by an earlier middleware, set it now trans_id = generate_trans_id(self.trans_id_suffix) req.environ['swift.trans_id'] = trans_id self.logger.txn_id = trans_id req.headers['x-trans-id'] = req.environ['swift.trans_id'] controller.trans_id = req.environ['swift.trans_id'] self.logger.client_ip = get_remote_client(req) try: handler = getattr(controller, req.method) getattr(handler, 'publicly_accessible') except AttributeError: allowed_methods = getattr(controller, 'allowed_methods', set()) return HTTPMethodNotAllowed( request=req, headers={'Allow': ', '.join(allowed_methods)}) if 'swift.authorize' in req.environ: # We call authorize before the handler, always. If authorized, # we remove the swift.authorize hook so isn't ever called # again. If not authorized, we return the denial unless the # controller's method indicates it'd like to gather more # information and try again later. resp = req.environ['swift.authorize'](req) if not resp: # No resp means authorized, no delayed recheck required. del req.environ['swift.authorize'] else: # Response indicates denial, but we might delay the denial # and recheck later. If not delayed, return the error now. if not getattr(handler, 'delay_denial', None): return resp # Save off original request method (GET, POST, etc.) in case it # gets mutated during handling. This way logging can display the # method the client actually sent. req.environ['swift.orig_req_method'] = req.method return handler(req) except HTTPException as error_response: return error_response except (Exception, Timeout): self.logger.exception(_('ERROR Unhandled exception in request')) return HTTPServerError(request=req)
def log_request(self, req, status_int, bytes_received, bytes_sent, start_time, end_time, resp_headers=None): """ Log a request. :param req: swob.Request object for the request :param status_int: integer code for the response status :param bytes_received: bytes successfully read from the request body :param bytes_sent: bytes yielded to the WSGI server :param start_time: timestamp request started :param end_time: timestamp request completed :param resp_headers: dict of the response headers """ resp_headers = resp_headers or {} logged_headers = None if self.log_hdrs: if self.log_hdrs_only: logged_headers = '\n'.join('%s: %s' % (k, v) for k, v in req.headers.items() if k in self.log_hdrs_only) else: logged_headers = '\n'.join('%s: %s' % (k, v) for k, v in req.headers.items()) method = self.method_from_req(req) duration_time_str = "%.4f" % (end_time - start_time) policy_index = get_policy_index(req.headers, resp_headers) acc, cont, obj = None, None, None if req.path.startswith('/v1/'): _, acc, cont, obj = split_path(req.path, 1, 4, True) replacements = { # Time information 'end_time': StrFormatTime(end_time), 'start_time': StrFormatTime(start_time), # Information worth to anonymize 'client_ip': StrAnonymizer(get_remote_client(req), self.anonymization_method, self.anonymization_salt), 'remote_addr': StrAnonymizer(req.remote_addr, self.anonymization_method, self.anonymization_salt), 'path': StrAnonymizer(req.path_qs, self.anonymization_method, self.anonymization_salt), 'referer': StrAnonymizer(req.referer, self.anonymization_method, self.anonymization_salt), 'user_agent': StrAnonymizer(req.user_agent, self.anonymization_method, self.anonymization_salt), 'headers': StrAnonymizer(logged_headers, self.anonymization_method, self.anonymization_salt), 'client_etag': StrAnonymizer(req.headers.get('etag'), self.anonymization_method, self.anonymization_salt), 'account': StrAnonymizer(acc, self.anonymization_method, self.anonymization_salt), 'container': StrAnonymizer(cont, self.anonymization_method, self.anonymization_salt), 'object': StrAnonymizer(obj, self.anonymization_method, self.anonymization_salt), # Others information 'method': method, 'protocol': req.environ.get('SERVER_PROTOCOL'), 'status_int': status_int, 'auth_token': self.obscure_sensitive( req.headers.get('x-auth-token')), 'bytes_recvd': bytes_received, 'bytes_sent': bytes_sent, 'transaction_id': req.environ.get('swift.trans_id'), 'request_time': duration_time_str, 'source': req.environ.get('swift.source'), 'log_info': ','.join(req.environ.get('swift.log_info', '')), 'policy_index': policy_index, } self.access_logger.info( self.log_formatter.format(self.log_msg_template, **replacements)) # Log timing and bytes-transferred data to StatsD metric_name = self.statsd_metric_name(req, status_int, method) metric_name_policy = self.statsd_metric_name_policy(req, status_int, method, policy_index) # Only log data for valid controllers (or SOS) to keep the metric count # down (egregious errors will get logged by the proxy server itself). if metric_name: self.access_logger.timing(metric_name + '.timing', (end_time - start_time) * 1000) self.access_logger.update_stats(metric_name + '.xfer', bytes_received + bytes_sent) if metric_name_policy: self.access_logger.timing(metric_name_policy + '.timing', (end_time - start_time) * 1000) self.access_logger.update_stats(metric_name_policy + '.xfer', bytes_received + bytes_sent)
def log_request(self, env, status_int, bytes_received, bytes_sent, request_time, client_disconnect): """ Log a request. :param env: WSGI environment :param status_int: integer code for the response status :param bytes_received: bytes successfully read from the request body :param bytes_sent: bytes yielded to the WSGI server :param request_time: time taken to satisfy the request, in seconds """ req = Request(env) if client_disconnect: # log disconnected clients as '499' status code status_int = 499 req_path = get_valid_utf8_str(req.path) the_request = quote(unquote(req_path)) if req.query_string: the_request = the_request + '?' + req.query_string logged_headers = None if self.log_hdrs: logged_headers = '\n'.join('%s: %s' % (k, v) for k, v in req.headers.items()) method = req.environ.get('swift.orig_req_method', req.method) self.access_logger.info(' '.join( quote(str(x) if x else '-') for x in ( get_remote_client(req), req.remote_addr, time.strftime('%d/%b/%Y/%H/%M/%S', time.gmtime()), method, the_request, req.environ.get('SERVER_PROTOCOL'), status_int, req.referer, req.user_agent, req.headers.get('x-auth-token'), bytes_received, bytes_sent, req.headers.get('etag', None), req.environ.get('swift.trans_id'), logged_headers, '%.4f' % request_time, req.environ.get('swift.source'), ))) # Log timing and bytes-transfered data to StatsD if req.path.startswith('/v1/'): try: stat_type = [None, 'account', 'container', 'object'][req.path.strip('/').count('/')] except IndexError: stat_type = 'object' else: stat_type = env.get('swift.source') # Only log data for valid controllers (or SOS) to keep the metric count # down (egregious errors will get logged by the proxy server itself). if stat_type: stat_method = method if method in self.valid_methods \ else 'BAD_METHOD' metric_name = '.'.join((stat_type, stat_method, str(status_int))) self.access_logger.timing(metric_name + '.timing', request_time * 1000) self.access_logger.update_stats(metric_name + '.xfer', bytes_received + bytes_sent)
def __call__(self, env, start_response): req = Request(env) try: version, account, container, obj = req.split_path(1, 4, True) except ValueError: return self.app(env, start_response) if account is None: return self.app(env, start_response) if env.get('swift.authorize_override', False): return self.app(env, start_response) # First, restrict modification of auth meta-data to only users with # the admin role (or roles that have been specially enabled in # the swift config). if req.method == "POST": # following code to get roles is borrowed from keystoneauth roles = set() if (env.get('HTTP_X_IDENTITY_STATUS') == 'Confirmed' or env.get('HTTP_X_SERVICE_IDENTITY_STATUS') in (None, 'Confirmed')): roles = set(list_from_csv(env.get('HTTP_X_ROLES', ''))) if not roles.intersection(self.allowed_meta_write_roles): for k, v in req.headers.iteritems(): if k.startswith('X-Container-Meta-Allowed-Iprange') \ or k.startswith('X-Account-Meta-Allowed-Iprange'): return Response(status=403, body=deny_meta_change, request=req)(env, start_response) # Grab the metadata for the account and container if container is not None and container != "": try: container_info = \ get_container_info(req.environ, self.app, swift_source='IPRangeACLMiddleware') except ValueError: # if we can't get container info, then we deny the request return Response(status=403, body="Invalid container (%s)" % container, request=req)(env, start_response) else: container_info = None try: acc_info = get_account_info(req.environ, self.app, swift_source='IPRangeACLMiddleware') except ValueError: # if we can't get account info, then we deny the request return Response(status=403, body="Invalid account (%s)" % account, request=req)(env, start_response) remote_ip = get_remote_client(req) allowed = set() default = "denied" # Read any account-level ACLs meta = acc_info['meta'] for k, v in meta.iteritems(): if k.startswith("allowed-iprange") and len(v) > 0: allowed.add(v) # This key is used to set the default access policy in # cases where no ACLs are present in the meta-data. if k == "ipacl-default": default = v # If the request is for a container or object, check for any # container-level ACLs if container_info is not None: meta = container_info['meta'] for k, v in meta.iteritems(): # Each allowed range must have a unique meta-data key, but # the key must begin with 'allowed-iprange-' if k.startswith('allowed-iprange-') and len(v) > 0: allowed.add(v) # This key is used to set the default access policy in # cases where no ACLs are present in the meta-data. # NOTE: Container-level default behaviour will override # account-level defaults. if k == "ipacl-default": default = v # XXX Could probably condense this into one tree, but not sure # whether Pytricia is OK with mixing IPv4 and IPv6 prefixes. self.pyt = pytricia.PyTricia(32) self.pyt6 = pytricia.PyTricia(128) # If there are no IP range ACLs in the meta-data and the # default policy is "allowed", then we can grant access. if len(allowed) == 0 and default == "allowed": return self.app(env, start_response) else: # Build the patricia tree of allowed IP prefixes for pref in allowed: if ':' in pref: try: addrcheck = ipaddress.IPv6Network(unicode(pref), False) except ipaddress.AddressValueError: self.logger.debug( "iprange_acl -- skipping invalid IP prefix: %(pref)s", {'pref': pref}) continue self.pyt6[pref] = "allowed" else: try: addrcheck = ipaddress.IPv4Network(unicode(pref), False) except ipaddress.AddressValueError: self.logger.debug( "iprange_acl -- skipping invalid IP prefix: %(pref)s", {'pref': pref}) continue self.pyt[pref] = "allowed" # Always allow our own IP, otherwise we could lock ourselves out from # the container! if ':' in self.local_ip: self.pyt6[self.local_ip] = "allowed" else: self.pyt[self.local_ip] = "allowed" # Add our default allowed IP ranges to the patricia tree for default_range in self.default_ranges: if ':' in default_range: try: addrcheck = ipaddress.IPv6Network(unicode(default_range), \ False) except ipaddress.AddressValueError: self.logger.debug("Invalid always_allow prefix for IPv6: %s" \ % (default_range)) else: self.pyt6[default_range] = "allowed" else: try: addrcheck = ipaddress.IPv4Network(unicode(default_range), \ False) except ipaddress.AddressValueError: self.logger.debug("Invalid always_allow prefix for IPv4: %s" \ % (default_range)) else: self.pyt[default_range] = "allowed" # Look up the address of the client in the patricia tree if ':' in remote_ip: status = self.pyt6.get(remote_ip) else: status = self.pyt.get(remote_ip) if status == "allowed": return self.app(env, start_response) return Response(status=403, body=self.deny_message, request=req)(env, start_response)
def authorize(self, req): ret = super(ZodiacAuth, self).authorize(req) if ret is None and req.method == "GET": # passed swauth rules, now check zodiac rules # split the path # this should be safe since it was already checked against an error # in the super call version, account, container, obj = split_path(req.path, 1, 4, True) # grab our acl to use acl = self.zodiac_acl # get the current zodiac sign for this access request access_sign = zodiac_sign_datetime(datetime.datetime.now()) # get the client ip client_addr = get_remote_client(req) if container: # there is a container so let's try to get the timestamp container_nodes = self.app.container_ring.get_nodes(account, container) if container_nodes: # direct head requests return a timestamp whereas calls # to the proxy do not. this might not be the best thing # to do. open to suggestions. try: container_meta = direct_head_container( container_nodes[1][0], container_nodes[0], account, container ) except ClientException: return ret container_date = datetime.datetime.fromtimestamp(float(container_meta["x-timestamp"])) container_sign = zodiac_sign_datetime(container_date) # ensure the container sign has access rules if container_sign in acl and access_sign in acl[container_sign]: if client_addr not in acl[container_sign][access_sign]: ret = self.denied_response(req) else: # sign missing from acl rules or access sign not present ret = self.denied_response(req) if ret is None and obj: # we passed the container permissions and there is an # object. # get the object's store sign and check permissions obj_nodes = self.app.container_ring.get_nodes(account, container, obj) if obj_nodes: try: obj_meta = direct_head_object( obj_nodes[1][0], container_nodes[0], account, container, obj ) except ClientException: return ret obj_date = datetime.datetime.fromtimestamp(float(obj_meta["x-timestamp"])) obj_sign = zodiac_sign_datetime(obj_date) # ensure the object sign has access rules if obj_sign in acl and access_sign in acl[obj_sign]: if client_addr not in acl[obj_sign][access_sign]: ret = self.denied_response(req) else: # object sign missing from acl rules or # access sign not present ret = self.denied_response(req) return ret