def GET(self, request): """Handle HTTP GET requests for the Swift Object Server.""" device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) keep_cache = self.keep_cache_private or ( 'X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers) try: disk_file = self.get_diskfile(device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: with disk_file.open(): metadata = disk_file.get_metadata() obj_size = int(metadata['Content-Length']) file_x_ts = metadata['X-Timestamp'] file_x_ts_flt = float(file_x_ts) try: if_unmodified_since = request.if_unmodified_since except (OverflowError, ValueError): # catches timestamps before the epoch return HTTPPreconditionFailed(request=request) file_x_ts_utc = datetime.fromtimestamp(file_x_ts_flt, UTC) if if_unmodified_since and file_x_ts_utc > if_unmodified_since: return HTTPPreconditionFailed(request=request) try: if_modified_since = request.if_modified_since except (OverflowError, ValueError): # catches timestamps before the epoch return HTTPPreconditionFailed(request=request) if if_modified_since and file_x_ts_utc <= if_modified_since: return HTTPNotModified(request=request) keep_cache = (self.keep_cache_private or ('X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers)) response = Response( app_iter=disk_file.reader(keep_cache=keep_cache), request=request, conditional_response=True) response.headers['Content-Type'] = metadata.get( 'Content-Type', 'application/octet-stream') for key, value in metadata.iteritems(): if is_user_meta('object', key) or \ key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] response.last_modified = math.ceil(file_x_ts_flt) response.content_length = obj_size try: response.content_encoding = metadata['Content-Encoding'] except KeyError: pass response.headers['X-Timestamp'] = file_x_ts resp = request.get_response(response) except (DiskFileNotExist, DiskFileQuarantined): resp = HTTPNotFound(request=request, conditional_response=True) return resp
def _if_none_match_wrapper(self, req, *args, **kwargs): if req.if_none_match is None: return fnc(self, req, *args, **kwargs) oio_headers = {REQID_HEADER: self.trans_id} try: metadata = self.app.storage.object_get_properties( self.account_name, self.container_name, self.object_name, version=obj_version_from_env(req.environ), headers=oio_headers) except (NoSuchObject, NoSuchContainer): return fnc(self, req, *args, **kwargs) # req.if_none_match will check for '*'. if metadata.get('hash') in req.if_none_match: if req.method in ('HEAD', 'GET'): raise HTTPNotModified(request=req) else: raise HTTPPreconditionFailed(request=req) return fnc(self, req, *args, **kwargs)
def _if_none_match_wrapper(self, req, *args, **kwargs): if req.if_none_match is None: return fnc(self, req, *args, **kwargs) oio_headers = {'X-oio-req-id': self.trans_id} try: metadata = self.app.storage.object_show(self.account_name, self.container_name, self.object_name, version=req.environ.get( 'oio.query', {}).get('version'), headers=oio_headers) except (NoSuchObject, NoSuchContainer): return fnc(self, req, *args, **kwargs) # req.if_none_match will check for '*'. if metadata.get('hash') in req.if_none_match: if req.method in ('HEAD', 'GET'): raise HTTPNotModified(request=req) else: raise HTTPPreconditionFailed(request=req) return fnc(self, req, *args, **kwargs)
def GET(self, request): """Handle HTTP GET requests for the Swift Object Server.""" device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) try: disk_file = self._diskfile(device, partition, account, container, obj, keep_data_fp=True, iter_hook=sleep) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) if disk_file.is_deleted() or disk_file.is_expired(): if request.headers.get('if-match') == '*': return HTTPPreconditionFailed(request=request) else: return HTTPNotFound(request=request) try: file_size = disk_file.get_data_file_size() except (DiskFileError, DiskFileNotExist): disk_file.quarantine() return HTTPNotFound(request=request) if request.headers.get('if-match') not in (None, '*') and \ disk_file.metadata['ETag'] not in request.if_match: disk_file.close() return HTTPPreconditionFailed(request=request) if request.headers.get('if-none-match') is not None: if disk_file.metadata['ETag'] in request.if_none_match: resp = HTTPNotModified(request=request) resp.etag = disk_file.metadata['ETag'] disk_file.close() return resp try: if_unmodified_since = request.if_unmodified_since except (OverflowError, ValueError): # catches timestamps before the epoch return HTTPPreconditionFailed(request=request) if if_unmodified_since and \ datetime.fromtimestamp( float(disk_file.metadata['X-Timestamp']), UTC) > \ if_unmodified_since: disk_file.close() return HTTPPreconditionFailed(request=request) try: if_modified_since = request.if_modified_since except (OverflowError, ValueError): # catches timestamps before the epoch return HTTPPreconditionFailed(request=request) if if_modified_since and \ datetime.fromtimestamp( float(disk_file.metadata['X-Timestamp']), UTC) < \ if_modified_since: disk_file.close() return HTTPNotModified(request=request) response = Response(app_iter=disk_file, request=request, conditional_response=True) response.headers['Content-Type'] = disk_file.metadata.get( 'Content-Type', 'application/octet-stream') for key, value in disk_file.metadata.iteritems(): if key.lower().startswith('x-object-meta-') or \ key.lower() in self.allowed_headers: response.headers[key] = value response.etag = disk_file.metadata['ETag'] response.last_modified = float(disk_file.metadata['X-Timestamp']) response.content_length = file_size if response.content_length < self.keep_cache_size and \ (self.keep_cache_private or ('X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers)): disk_file.keep_cache = True if 'Content-Encoding' in disk_file.metadata: response.content_encoding = disk_file.metadata['Content-Encoding'] response.headers['X-Timestamp'] = disk_file.metadata['X-Timestamp'] return request.get_response(response)
def GET(self, request): """Handle HTTP GET requests for the Swift Object Server.""" device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) keep_cache = self.keep_cache_private or ( 'X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers) try: disk_file = self.get_diskfile( device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: with disk_file.open(): metadata = disk_file.get_metadata() obj_size = int(metadata['Content-Length']) if request.headers.get('if-match') not in (None, '*') and \ metadata['ETag'] not in request.if_match: return HTTPPreconditionFailed(request=request) if request.headers.get('if-none-match') is not None: if metadata['ETag'] in request.if_none_match: resp = HTTPNotModified(request=request) resp.etag = metadata['ETag'] return resp file_x_ts = metadata['X-Timestamp'] file_x_ts_flt = float(file_x_ts) try: if_unmodified_since = request.if_unmodified_since except (OverflowError, ValueError): # catches timestamps before the epoch return HTTPPreconditionFailed(request=request) file_x_ts_utc = datetime.fromtimestamp(file_x_ts_flt, UTC) if if_unmodified_since and file_x_ts_utc > if_unmodified_since: return HTTPPreconditionFailed(request=request) try: if_modified_since = request.if_modified_since except (OverflowError, ValueError): # catches timestamps before the epoch return HTTPPreconditionFailed(request=request) if if_modified_since and file_x_ts_utc < if_modified_since: return HTTPNotModified(request=request) keep_cache = (self.keep_cache_private or ('X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers)) response = Response( app_iter=disk_file.reader(keep_cache=keep_cache), request=request, conditional_response=True) response.headers['Content-Type'] = metadata.get( 'Content-Type', 'application/octet-stream') for key, value in metadata.iteritems(): if key.lower().startswith('x-object-meta-') or \ key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] response.last_modified = file_x_ts_flt response.content_length = obj_size try: response.content_encoding = metadata[ 'Content-Encoding'] except KeyError: pass response.headers['X-Timestamp'] = file_x_ts resp = request.get_response(response) except DiskFileNotExist: if request.headers.get('if-match') == '*': resp = HTTPPreconditionFailed(request=request) else: resp = HTTPNotFound(request=request) except DiskFileQuarantined: resp = HTTPNotFound(request=request) return resp
def GET(self, request): """Handle HTTP GET requests for the Swift Object Server.""" device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) try: disk_file = self._diskfile(device, partition, account, container, obj, iter_hook=sleep) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) disk_file.open() if disk_file.is_deleted() or disk_file.is_expired(): if request.headers.get('if-match') == '*': return HTTPPreconditionFailed(request=request) else: return HTTPNotFound(request=request) try: file_size = disk_file.get_data_file_size() except (DiskFileError, DiskFileNotExist): disk_file.quarantine() return HTTPNotFound(request=request) metadata = disk_file.get_metadata() if request.headers.get('if-match') not in (None, '*') and \ metadata['ETag'] not in request.if_match: disk_file.close() return HTTPPreconditionFailed(request=request) if request.headers.get('if-none-match') is not None: if metadata['ETag'] in request.if_none_match: resp = HTTPNotModified(request=request) resp.etag = metadata['ETag'] disk_file.close() return resp try: if_unmodified_since = request.if_unmodified_since except (OverflowError, ValueError): # catches timestamps before the epoch return HTTPPreconditionFailed(request=request) if if_unmodified_since and \ datetime.fromtimestamp( float(metadata['X-Timestamp']), UTC) > \ if_unmodified_since: disk_file.close() return HTTPPreconditionFailed(request=request) try: if_modified_since = request.if_modified_since except (OverflowError, ValueError): # catches timestamps before the epoch return HTTPPreconditionFailed(request=request) if if_modified_since and \ datetime.fromtimestamp( float(metadata['X-Timestamp']), UTC) < \ if_modified_since: disk_file.close() return HTTPNotModified(request=request) response = Response(app_iter=disk_file, request=request, conditional_response=True) response.headers['Content-Type'] = metadata.get( 'Content-Type', 'application/octet-stream') for key, value in metadata.iteritems(): if key.lower().startswith('x-object-meta-') or \ key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] response.last_modified = float(metadata['X-Timestamp']) response.content_length = file_size if response.content_length < self.keep_cache_size and \ (self.keep_cache_private or ('X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers)): disk_file.keep_cache = True if 'Content-Encoding' in metadata: response.content_encoding = metadata['Content-Encoding'] response.headers['X-Timestamp'] = metadata['X-Timestamp'] return request.get_response(response)
class CdnHandler(OriginBase): def __init__(self, app, conf, logger): OriginBase.__init__(self, app, conf, logger) self.logger = logger self.max_cdn_file_size = int( conf.get('max_cdn_file_size', 10 * 1024**3)) self.allowed_origin_remote_ips = [] remote_ips = conf.get('allowed_origin_remote_ips') if remote_ips: self.allowed_origin_remote_ips = \ [ip.strip() for ip in remote_ips.split(',') if ip.strip()] if not bool(conf.get('incoming_url_regex')): raise InvalidConfiguration('Invalid config for CdnHandler') self.cdn_regexes = [] for key, val in conf['incoming_url_regex'].items(): regex = re.compile(val) self.cdn_regexes.append(regex) def _getCacheHeaders(self, ttl): return { 'Expires': strftime("%a, %d %b %Y %H:%M:%S GMT", gmtime(time() + ttl)), 'Cache-Control': 'max-age=%d, public' % ttl } def _getCdnHeaders(self, req): headers = {'X-Web-Mode': 'True', 'User-Agent': 'SOS Origin'} for header in ['If-Modified-Since', 'If-Match', 'Range', 'If-Range']: if header in req.headers: headers[header] = req.headers[header] return headers def handle_request(self, env, req): if req.method not in ('GET', 'HEAD'): headers = self._getCacheHeaders(CACHE_BAD_URL) return HTTPMethodNotAllowed(request=req, headers=headers) if self.allowed_origin_remote_ips and \ req.remote_addr not in self.allowed_origin_remote_ips: raise OriginRequestNotAllowed( 'SOS Origin: Remote IP %s not allowed' % req.remote_addr) # allow earlier middleware to override hash and obj_name hsh = env.get('swift.cdn_hash') object_name = env.get('swift.cdn_object_name') if hsh is None or object_name is None: for regex in self.cdn_regexes: match_obj = regex.match(req.url) if match_obj: match_dict = match_obj.groupdict() if not hsh: hsh = match_dict.get('hash') if not object_name: object_name = match_dict.get('object_name') break if not hsh: self.logger.debug('Hash %s not found in %s' % (hsh, req.url)) headers = self._getCacheHeaders(CACHE_BAD_URL) return HTTPNotFound(request=req, headers=headers) if hsh.find('-') >= 0: hsh = hsh.split('-', 1)[1] try: cdn_obj_path = self.get_hsh_obj_path(hsh) except ValueError, e: self.logger.debug('get_hsh_obj_path error: %s' % e) headers = self._getCacheHeaders(CACHE_BAD_URL) return HTTPBadRequest(request=req, headers=headers) hash_data = self.get_cdn_data(env, cdn_obj_path) if hash_data and hash_data.cdn_enabled: # this is a cdn enabled container, proxy req to swift if env.get('swift.cdn_authorize'): auth_resp, ttl = env['swift.cdn_authorize']( env, hash_data.account.encode('utf-8')) if auth_resp: return auth_resp(request=req, headers=self._getCacheHeaders(ttl)) swift_path = quote('/v1/%s/%s/' % (hash_data.account.encode('utf-8'), hash_data.container.encode('utf-8'))) if object_name: swift_path += object_name headers = self._getCdnHeaders(req) env['swift.source'] = 'SOS' resp = make_pre_authed_request(env, req.method, swift_path, headers=headers, agent='SwiftOrigin', swift_source='SOS').get_response( self.app) if resp.status_int == 301 and 'Location' in resp.headers: loc_parsed = urlparse(resp.headers['Location']) acc_cont_path = '/v1/%s/%s' % (hash_data.account.encode( 'utf-8'), hash_data.container.encode('utf-8')) if loc_parsed.path.startswith(acc_cont_path): sos_loc = loc_parsed.path[len(acc_cont_path):] resp = SosResponse( headers=self._getCacheHeaders(hash_data.ttl)) resp.headers['Location'] = sos_loc resp.status = 301 return resp else: self.logger.exception( 'Unexpected Location header ' 'returned. %s does not begin with expected ' 'path: %s' % (loc_parsed.geturl(), acc_cont_path)) return HTTPInternalServerError('Unexpected Relocation') if resp.status_int == 304: return HTTPNotModified(request=req, headers=self._getCacheHeaders( hash_data.ttl)) if resp.status_int == 416: return HTTPRequestedRangeNotSatisfiable( request=req, headers=self._getCacheHeaders(CACHE_404)) if resp.status_int // 100 == 2 or resp.status_int == 404: if resp.content_length > self.max_cdn_file_size: return HTTPBadRequest( request=req, headers=self._getCacheHeaders(CACHE_404)) cdn_resp = Response(request=req, app_iter=resp.app_iter) cdn_resp.status = resp.status_int cdn_resp.headers.update(resp.headers) if resp.status_int == 404: cdn_resp.headers.update(self._getCacheHeaders(CACHE_404)) else: cdn_resp.headers.update( self._getCacheHeaders(hash_data.ttl)) return cdn_resp self.logger.error('Unexpected response from ' 'Swift: %s, %s: %s' % (resp.status, swift_path, resp.body[:40])) return HTTPNotFound(request=req, headers=self._getCacheHeaders(CACHE_404))
if request.headers.get('if-match') == '*': return HTTPPreconditionFailed(request=request) else: return HTTPNotFound(request=request) try: file_size = file.get_data_file_size() except (DiskFileError, DiskFileNotExist): file.quarantine() return HTTPNotFound(request=request) if request.headers.get('if-match') not in (None, '*') and \ file.metadata['ETag'] not in request.if_match: file.close() return HTTPPreconditionFailed(request=request) if request.headers.get('if-none-match') is not None: if file.metadata['ETag'] in request.if_none_match: resp = HTTPNotModified(request=request) resp.etag = file.metadata['ETag'] file.close() return resp try: if_unmodified_since = request.if_unmodified_since except (OverflowError, ValueError): # catches timestamps before the epoch return HTTPPreconditionFailed(request=request) if if_unmodified_since and \ datetime.fromtimestamp( float(file.metadata['X-Timestamp']), UTC) > \ if_unmodified_since: file.close() return HTTPPreconditionFailed(request=request) try:
def GET(self, request): """Handle HTTP GET requests for the Swift Object Server.""" device, partition, account, container, obj = split_and_validate_path(request, 5, 5, True) try: disk_file = self._diskfile(device, partition, account, container, obj, keep_data_fp=True, iter_hook=sleep) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) if disk_file.is_deleted() or disk_file.is_expired(): if request.headers.get("if-match") == "*": return HTTPPreconditionFailed(request=request) else: return HTTPNotFound(request=request) try: file_size = disk_file.get_data_file_size() except (DiskFileError, DiskFileNotExist): disk_file.quarantine() return HTTPNotFound(request=request) if request.headers.get("if-match") not in (None, "*") and disk_file.metadata["ETag"] not in request.if_match: disk_file.close() return HTTPPreconditionFailed(request=request) if request.headers.get("if-none-match") is not None: if disk_file.metadata["ETag"] in request.if_none_match: resp = HTTPNotModified(request=request) resp.etag = disk_file.metadata["ETag"] disk_file.close() return resp try: if_unmodified_since = request.if_unmodified_since except (OverflowError, ValueError): # catches timestamps before the epoch return HTTPPreconditionFailed(request=request) if ( if_unmodified_since and datetime.fromtimestamp(float(disk_file.metadata["X-Timestamp"]), UTC) > if_unmodified_since ): disk_file.close() return HTTPPreconditionFailed(request=request) try: if_modified_since = request.if_modified_since except (OverflowError, ValueError): # catches timestamps before the epoch return HTTPPreconditionFailed(request=request) if ( if_modified_since and datetime.fromtimestamp(float(disk_file.metadata["X-Timestamp"]), UTC) < if_modified_since ): disk_file.close() return HTTPNotModified(request=request) response = Response(app_iter=disk_file, request=request, conditional_response=True) response.headers["Content-Type"] = disk_file.metadata.get("Content-Type", "application/octet-stream") for key, value in disk_file.metadata.iteritems(): if key.lower().startswith("x-object-meta-") or key.lower() in self.allowed_headers: response.headers[key] = value response.etag = disk_file.metadata["ETag"] response.last_modified = float(disk_file.metadata["X-Timestamp"]) response.content_length = file_size if response.content_length < self.keep_cache_size and ( self.keep_cache_private or ("X-Auth-Token" not in request.headers and "X-Storage-Token" not in request.headers) ): disk_file.keep_cache = True if "Content-Encoding" in disk_file.metadata: response.content_encoding = disk_file.metadata["Content-Encoding"] response.headers["X-Timestamp"] = disk_file.metadata["X-Timestamp"] return request.get_response(response)