def pass_file(self, req, path, content_type=None): """ pass a file to client """ resp = Response() if content_type: resp.content_type = content_type else: (ctype, enc) = guess_type(basename(path)) resp.content_type = ctype resp.charset = None try: with open(join(self.path, path)) as f: resp.app_iter = iter(f.read()) return resp except IOError: return HTTPNotFound(request=req)
def pass_file(self, req, path, content_type=None): """ pass a file to client """ resp = Response() if content_type: resp.content_type = content_type else: (ctype, enc) = guess_type(basename(path)) resp.content_type = ctype resp.charset = None try: with open(join(self.path, path)) as f: resp.app_iter = iter(f.read()) return resp except IOError: return HTTPNotFound(request=req)
def best_response(self, req, statuses, reasons, bodies, server_type, etag=None): """ Given a list of responses from several servers, choose the best to return to the API. :param req: swob.Request object :param statuses: list of statuses returned :param reasons: list of reasons for each status :param bodies: bodies of each response :param server_type: type of server the responses came from :param etag: etag :returns: swob.Response object with the correct status, body, etc. set """ resp = Response(request=req) if len(statuses): for hundred in (HTTP_OK, HTTP_MULTIPLE_CHOICES, HTTP_BAD_REQUEST): hstatuses = \ [s for s in statuses if hundred <= s < hundred + 100] if len(hstatuses) > len(statuses) / 2: status = max(hstatuses) status_index = statuses.index(status) resp.status = '%s %s' % (status, reasons[status_index]) resp.body = bodies[status_index] resp.content_type = 'text/html' if etag: resp.headers['etag'] = etag.strip('"') return resp self.app.logger.error(_('%(type)s returning 503 for %(statuses)s'), {'type': server_type, 'statuses': statuses}) resp.status = '503 Internal Server Error' return resp
def GET(self, req): """ Load ring data from .gz files. """ self._reload() ring_info = self._to_dict() gz_file_to_ring_info = json.dumps(ring_info, sort_keys=True) ring_info_json_md5 = self._ringinfo_md5(gz_file_to_ring_info) resp = Response(request=req) resp.etag = ring_info_json_md5 resp.content_length = len(gz_file_to_ring_info) resp.body = gz_file_to_ring_info resp.status_init = 200 resp.content_type = 'application/json; charset=UTF-8' return resp
def get_working_response(self, req): source, node = self._get_source_and_node() res = None if source: res = Response(request=req) if req.method == "GET" and source.status in (HTTP_OK, HTTP_PARTIAL_CONTENT): res.app_iter = self._make_app_iter(node, source) # See NOTE: swift_conn at top of file about this. res.swift_conn = source.swift_conn res.status = source.status update_headers(res, source.getheaders()) if not res.environ: res.environ = {} res.environ["swift_x_timestamp"] = source.getheader("x-timestamp") res.accept_ranges = "bytes" res.content_length = source.getheader("Content-Length") if source.getheader("Content-Type"): res.charset = None res.content_type = source.getheader("Content-Type") return res
def get_working_response(self, req): source, node = self._get_source_and_node() res = None if source: res = Response(request=req) if req.method == 'GET' and \ source.status in (HTTP_OK, HTTP_PARTIAL_CONTENT): res.app_iter = self._make_app_iter(node, source) # See NOTE: swift_conn at top of file about this. res.swift_conn = source.swift_conn res.status = source.status update_headers(res, source.getheaders()) if not res.environ: res.environ = {} res.environ['swift_x_timestamp'] = \ source.getheader('x-timestamp') res.accept_ranges = 'bytes' res.content_length = source.getheader('Content-Length') if source.getheader('Content-Type'): res.charset = None res.content_type = source.getheader('Content-Type') return res
def best_response(self, req, statuses, reasons, bodies, server_type, etag=None): """ Given a list of responses from several servers, choose the best to return to the API. :param req: swob.Request object :param statuses: list of statuses returned :param reasons: list of reasons for each status :param bodies: bodies of each response :param server_type: type of server the responses came from :param etag: etag :returns: swob.Response object with the correct status, body, etc. set """ resp = Response(request=req) if len(statuses): for hundred in (HTTP_OK, HTTP_MULTIPLE_CHOICES, HTTP_BAD_REQUEST): hstatuses = \ [s for s in statuses if hundred <= s < hundred + 100] if len(hstatuses) > len(statuses) / 2: status = max(hstatuses) status_index = statuses.index(status) resp.status = '%s %s' % (status, reasons[status_index]) resp.body = bodies[status_index] resp.content_type = 'text/html' if etag: resp.headers['etag'] = etag.strip('"') return resp self.app.logger.error(_('%(type)s returning 503 for %(statuses)s'), { 'type': server_type, 'statuses': statuses }) resp.status = '503 Internal Server Error' return resp
def GETorHEAD_base(self, req, server_type, ring, partition, path): """ Base handler for HTTP GET or HEAD requests. :param req: swob.Request object :param server_type: server type :param ring: the ring to obtain nodes from :param partition: partition :param path: path for the request :returns: swob.Response object """ statuses = [] reasons = [] bodies = [] source_headers = [] sources = [] newest = config_true_value(req.headers.get('x-newest', 'f')) headers = self.generate_request_headers(req, additional=req.headers) for node in self.iter_nodes(ring, partition): start_node_timing = time.time() try: with ConnectionTimeout(self.app.conn_timeout): conn = http_connect( node['ip'], node['port'], node['device'], partition, req.method, path, headers=headers, query_string=req.query_string) self.app.set_node_timing(node, time.time() - start_node_timing) with Timeout(self.app.node_timeout): possible_source = conn.getresponse() # See NOTE: swift_conn at top of file about this. possible_source.swift_conn = conn except (Exception, Timeout): self.exception_occurred( node, server_type, _('Trying to %(method)s %(path)s') % {'method': req.method, 'path': req.path}) continue if self.is_good_source(possible_source): # 404 if we know we don't have a synced copy if not float(possible_source.getheader('X-PUT-Timestamp', 1)): statuses.append(HTTP_NOT_FOUND) reasons.append('') bodies.append('') source_headers.append('') self.close_swift_conn(possible_source) else: statuses.append(possible_source.status) reasons.append(possible_source.reason) bodies.append('') source_headers.append('') sources.append((possible_source, node)) if not newest: # one good source is enough break else: statuses.append(possible_source.status) reasons.append(possible_source.reason) bodies.append(possible_source.read()) source_headers.append(possible_source.getheaders()) if possible_source.status == HTTP_INSUFFICIENT_STORAGE: self.error_limit(node, _('ERROR Insufficient Storage')) elif is_server_error(possible_source.status): self.error_occurred(node, _('ERROR %(status)d %(body)s ' 'From %(type)s Server') % {'status': possible_source.status, 'body': bodies[-1][:1024], 'type': server_type}) res = None if sources: sources.sort(key=lambda s: source_key(s[0])) source, node = sources.pop() for src, _junk in sources: self.close_swift_conn(src) res = Response(request=req) if req.method == 'GET' and \ source.status in (HTTP_OK, HTTP_PARTIAL_CONTENT): res.app_iter = self._make_app_iter(node, source) # See NOTE: swift_conn at top of file about this. res.swift_conn = source.swift_conn res.status = source.status update_headers(res, source.getheaders()) if not res.environ: res.environ = {} res.environ['swift_x_timestamp'] = \ source.getheader('x-timestamp') res.accept_ranges = 'bytes' res.content_length = source.getheader('Content-Length') if source.getheader('Content-Type'): res.charset = None res.content_type = source.getheader('Content-Type') if not res: res = self.best_response(req, statuses, reasons, bodies, '%s %s' % (server_type, req.method), headers=source_headers) try: (account, container) = split_path(req.path_info, 1, 2) _set_info_cache(self.app, req.environ, account, container, res) except ValueError: pass try: (account, container, obj) = split_path(req.path_info, 3, 3, True) _set_object_info_cache(self.app, req.environ, account, container, obj, res) except ValueError: pass return res
def _GET_using_cache(self, req): # It may be possible to fulfil the request from cache: we only reach # here if request record_type is 'shard' or 'auto', so if the container # state is 'sharded' then look for cached shard ranges. However, if # X-Newest is true then we always fetch from the backend servers. get_newest = config_true_value(req.headers.get('x-newest', False)) if get_newest: self.app.logger.debug( 'Skipping shard cache lookup (x-newest) for %s', req.path_qs) info = None else: info = _get_info_from_caches(self.app, req.environ, self.account_name, self.container_name) if (info and is_success(info['status']) and info.get('sharding_state') == 'sharded'): # container is sharded so we may have the shard ranges cached headers = headers_from_container_info(info) if headers: # only use cached values if all required headers available infocache = req.environ.setdefault('swift.infocache', {}) memcache = cache_from_env(req.environ, True) cache_key = get_cache_key(self.account_name, self.container_name, shard='listing') cached_ranges = infocache.get(cache_key) if cached_ranges is None and memcache: cached_ranges = memcache.get(cache_key) if cached_ranges is not None: infocache[cache_key] = tuple(cached_ranges) # shard ranges can be returned from cache self.app.logger.debug('Found %d shards in cache for %s', len(cached_ranges), req.path_qs) headers.update({ 'x-backend-record-type': 'shard', 'x-backend-cached-results': 'true' }) shard_range_body = self._filter_resp_shard_ranges( req, cached_ranges) # mimic GetOrHeadHandler.get_working_response... # note: server sets charset with content_type but proxy # GETorHEAD_base does not, so don't set it here either resp = Response(request=req, body=shard_range_body) update_headers(resp, headers) resp.last_modified = math.ceil( float(headers['x-put-timestamp'])) resp.environ['swift_x_timestamp'] = headers.get( 'x-timestamp') resp.accept_ranges = 'bytes' resp.content_type = 'application/json' return resp # The request was not fulfilled from cache so send to the backend # server, but instruct the backend server to ignore name constraints in # request params if returning shard ranges so that the response can # potentially be cached. Only do this if the container state is # 'sharded'. We don't attempt to cache shard ranges for a 'sharding' # container as they may include the container itself as a 'gap filler' # for shard ranges that have not yet cleaved; listings from 'gap # filler' shard ranges are likely to become stale as the container # continues to cleave objects to its shards and caching them is # therefore more likely to result in stale or incomplete listings on # subsequent container GETs. req.headers['x-backend-override-shard-name-filter'] = 'sharded' resp = self._GETorHEAD_from_backend(req) sharding_state = resp.headers.get('x-backend-sharding-state', '').lower() resp_record_type = resp.headers.get('x-backend-record-type', '').lower() complete_listing = config_true_value( resp.headers.pop('x-backend-override-shard-name-filter', False)) # given that we sent 'x-backend-override-shard-name-filter=sharded' we # should only receive back 'x-backend-override-shard-name-filter=true' # if the sharding state is 'sharded', but check them both anyway... if (resp_record_type == 'shard' and sharding_state == 'sharded' and complete_listing): # backend returned unfiltered listing state shard ranges so parse # them and replace response body with filtered listing cache_key = get_cache_key(self.account_name, self.container_name, shard='listing') data = self._parse_listing_response(req, resp) backend_shard_ranges = self._parse_shard_ranges(req, data, resp) if backend_shard_ranges is not None: cached_ranges = [dict(sr) for sr in backend_shard_ranges] if resp.headers.get('x-backend-sharding-state') == 'sharded': # cache in infocache even if no shard ranges returned; this # is unexpected but use that result for this request infocache = req.environ.setdefault('swift.infocache', {}) infocache[cache_key] = tuple(cached_ranges) memcache = cache_from_env(req.environ, True) if memcache and cached_ranges: # cache in memcache only if shard ranges as expected self.app.logger.debug('Caching %d shards for %s', len(cached_ranges), req.path_qs) memcache.set( cache_key, cached_ranges, time=self.app.recheck_listing_shard_ranges) # filter returned shard ranges according to request constraints resp.body = self._filter_resp_shard_ranges(req, cached_ranges) return resp
if is_subdir: output_list.append('<subdir name="%s" />' % name) else: item = '<container><name>%s</name><count>%s</count>' \ '<bytes>%s</bytes></container>' % \ (name, object_count, bytes_used) output_list.append(item) output_list.append('</account>') account_list = '\n'.join(output_list) else: if not account_list: self.logger.timing_since('GET.timing', start_time) return HTTPNoContent(request=req, headers=resp_headers) account_list = '\n'.join(r[0] for r in account_list) + '\n' ret = Response(body=account_list, request=req, headers=resp_headers) ret.content_type = out_content_type ret.charset = 'utf-8' self.logger.timing_since('GET.timing', start_time) return ret @public def REPLICATE(self, req): """ Handle HTTP REPLICATE request. Handler for RPC calls for account replication. """ start_time = time.time() try: post_args = split_path(unquote(req.path), 3) drive, partition, hash = post_args validate_device_partition(drive, partition)
def GETorHEAD_base(self, req, server_type, ring, partition, path): """ Base handler for HTTP GET or HEAD requests. :param req: swob.Request object :param server_type: server type :param ring: the ring to obtain nodes from :param partition: partition :param path: path for the request :returns: swob.Response object """ statuses = [] reasons = [] bodies = [] sources = [] newest = config_true_value(req.headers.get('x-newest', 'f')) for node in self.iter_nodes(ring, partition): start_node_timing = time.time() try: with ConnectionTimeout(self.app.conn_timeout): headers = dict(req.headers) headers['Connection'] = 'close' conn = http_connect(node['ip'], node['port'], node['device'], partition, req.method, path, headers=headers, query_string=req.query_string) self.app.set_node_timing(node, time.time() - start_node_timing) with Timeout(self.app.node_timeout): possible_source = conn.getresponse() # See NOTE: swift_conn at top of file about this. possible_source.swift_conn = conn except (Exception, Timeout): self.exception_occurred( node, server_type, _('Trying to %(method)s %(path)s') % { 'method': req.method, 'path': req.path }) continue if self.is_good_source(possible_source): # 404 if we know we don't have a synced copy if not float(possible_source.getheader('X-PUT-Timestamp', 1)): statuses.append(HTTP_NOT_FOUND) reasons.append('') bodies.append('') self.close_swift_conn(possible_source) else: statuses.append(possible_source.status) reasons.append(possible_source.reason) bodies.append('') sources.append((possible_source, node)) if not newest: # one good source is enough break else: statuses.append(possible_source.status) reasons.append(possible_source.reason) bodies.append(possible_source.read()) if possible_source.status == HTTP_INSUFFICIENT_STORAGE: self.error_limit(node, _('ERROR Insufficient Storage')) elif is_server_error(possible_source.status): self.error_occurred( node, _('ERROR %(status)d %(body)s ' 'From %(type)s Server') % { 'status': possible_source.status, 'body': bodies[-1][:1024], 'type': server_type }) if sources: sources.sort(key=lambda s: source_key(s[0])) source, node = sources.pop() for src, _junk in sources: self.close_swift_conn(src) res = Response(request=req, conditional_response=True) if req.method == 'GET' and \ source.status in (HTTP_OK, HTTP_PARTIAL_CONTENT): res.app_iter = self._make_app_iter(node, source) # See NOTE: swift_conn at top of file about this. res.swift_conn = source.swift_conn res.status = source.status update_headers(res, source.getheaders()) if not res.environ: res.environ = {} res.environ['swift_x_timestamp'] = \ source.getheader('x-timestamp') res.accept_ranges = 'bytes' res.content_length = source.getheader('Content-Length') if source.getheader('Content-Type'): res.charset = None res.content_type = source.getheader('Content-Type') return res return self.best_response(req, statuses, reasons, bodies, '%s %s' % (server_type, req.method))
def GETorHEAD_base(self, req, server_type, partition, nodes, path, attempts): """ Base handler for HTTP GET or HEAD requests. :param req: swob.Request object :param server_type: server type :param partition: partition :param nodes: nodes :param path: path for the request :param attempts: number of attempts to try :returns: swob.Response object """ statuses = [] reasons = [] bodies = [] sources = [] newest = config_true_value(req.headers.get('x-newest', 'f')) nodes = iter(nodes) while len(statuses) < attempts: try: node = nodes.next() except StopIteration: break if self.error_limited(node): continue start_node_timing = time.time() try: with ConnectionTimeout(self.app.conn_timeout): headers = dict(req.headers) headers['Connection'] = 'close' conn = http_connect( node['ip'], node['port'], node['device'], partition, req.method, path, headers=headers, query_string=req.query_string) self.app.set_node_timing(node, time.time() - start_node_timing) with Timeout(self.app.node_timeout): possible_source = conn.getresponse() # See NOTE: swift_conn at top of file about this. possible_source.swift_conn = conn except (Exception, Timeout): self.exception_occurred( node, server_type, _('Trying to %(method)s %(path)s') % {'method': req.method, 'path': req.path}) continue if self.is_good_source(possible_source): # 404 if we know we don't have a synced copy if not float(possible_source.getheader('X-PUT-Timestamp', 1)): statuses.append(HTTP_NOT_FOUND) reasons.append('') bodies.append('') self.close_swift_conn(possible_source) else: statuses.append(possible_source.status) reasons.append(possible_source.reason) bodies.append('') sources.append(possible_source) if not newest: # one good source is enough break else: statuses.append(possible_source.status) reasons.append(possible_source.reason) bodies.append(possible_source.read()) if possible_source.status == HTTP_INSUFFICIENT_STORAGE: self.error_limit(node) elif is_server_error(possible_source.status): self.error_occurred(node, _('ERROR %(status)d %(body)s ' 'From %(type)s Server') % {'status': possible_source.status, 'body': bodies[-1][:1024], 'type': server_type}) if sources: sources.sort(key=source_key) source = sources.pop() for src in sources: self.close_swift_conn(src) res = Response(request=req, conditional_response=True) if req.method == 'GET' and \ source.status in (HTTP_OK, HTTP_PARTIAL_CONTENT): res.app_iter = self._make_app_iter(node, source) # See NOTE: swift_conn at top of file about this. res.swift_conn = source.swift_conn res.status = source.status update_headers(res, source.getheaders()) if not res.environ: res.environ = {} res.environ['swift_x_timestamp'] = \ source.getheader('x-timestamp') res.accept_ranges = 'bytes' res.content_length = source.getheader('Content-Length') if source.getheader('Content-Type'): res.charset = None res.content_type = source.getheader('Content-Type') return res return self.best_response(req, statuses, reasons, bodies, '%s %s' % (server_type, req.method))
def GETorHEAD_base(self, req, server_type, partition, nodes, path, attempts): """ Base handler for HTTP GET or HEAD requests. :param req: swob.Request object :param server_type: server type :param partition: partition :param nodes: nodes :param path: path for the request :param attempts: number of attempts to try :returns: swob.Response object """ statuses = [] reasons = [] bodies = [] sources = [] newest = config_true_value(req.headers.get("x-newest", "f")) nodes = iter(nodes) while len(statuses) < attempts: try: node = nodes.next() except StopIteration: break if self.error_limited(node): continue try: with ConnectionTimeout(self.app.conn_timeout): headers = dict(req.headers) headers["Connection"] = "close" conn = http_connect( node["ip"], node["port"], node["device"], partition, req.method, path, headers=headers, query_string=req.query_string, ) with Timeout(self.app.node_timeout): possible_source = conn.getresponse() # See NOTE: swift_conn at top of file about this. possible_source.swift_conn = conn except (Exception, Timeout): self.exception_occurred( node, server_type, _("Trying to %(method)s %(path)s") % {"method": req.method, "path": req.path} ) continue if self.is_good_source(possible_source): # 404 if we know we don't have a synced copy if not float(possible_source.getheader("X-PUT-Timestamp", 1)): statuses.append(HTTP_NOT_FOUND) reasons.append("") bodies.append("") self.close_swift_conn(possible_source) else: statuses.append(possible_source.status) reasons.append(possible_source.reason) bodies.append("") sources.append(possible_source) if not newest: # one good source is enough break else: statuses.append(possible_source.status) reasons.append(possible_source.reason) bodies.append(possible_source.read()) if possible_source.status == HTTP_INSUFFICIENT_STORAGE: self.error_limit(node) elif is_server_error(possible_source.status): self.error_occurred( node, _("ERROR %(status)d %(body)s " "From %(type)s Server") % {"status": possible_source.status, "body": bodies[-1][:1024], "type": server_type}, ) if sources: sources.sort(key=source_key) source = sources.pop() for src in sources: self.close_swift_conn(src) res = Response(request=req, conditional_response=True) if req.method == "GET" and source.status in (HTTP_OK, HTTP_PARTIAL_CONTENT): res.app_iter = self._make_app_iter(node, source) # See NOTE: swift_conn at top of file about this. res.swift_conn = source.swift_conn res.status = source.status update_headers(res, source.getheaders()) if not res.environ: res.environ = {} res.environ["swift_x_timestamp"] = source.getheader("x-timestamp") res.accept_ranges = "bytes" res.content_length = source.getheader("Content-Length") if source.getheader("Content-Type"): res.charset = None res.content_type = source.getheader("Content-Type") return res return self.best_response(req, statuses, reasons, bodies, "%s %s" % (server_type, req.method))