def GET(self, request):
"""Handle HTTP GET requests for the Swift Object Server."""
device, partition, account, container, obj = \
split_and_validate_path(request, 5, 5, True)
keep_cache = self.keep_cache_private or (
'X-Auth-Token' not in request.headers
and 'X-Storage-Token' not in request.headers)
try:
disk_file = self.get_diskfile(device, partition, account,
container, obj)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
try:
with disk_file.open():
metadata = disk_file.get_metadata()
obj_size = int(metadata['Content-Length'])
file_x_ts = metadata['X-Timestamp']
file_x_ts_flt = float(file_x_ts)
try:
if_unmodified_since = request.if_unmodified_since
except (OverflowError, ValueError):
# catches timestamps before the epoch
return HTTPPreconditionFailed(request=request)
file_x_ts_utc = datetime.fromtimestamp(file_x_ts_flt, UTC)
if if_unmodified_since and file_x_ts_utc > if_unmodified_since:
return HTTPPreconditionFailed(request=request)
try:
if_modified_since = request.if_modified_since
except (OverflowError, ValueError):
# catches timestamps before the epoch
return HTTPPreconditionFailed(request=request)
if if_modified_since and file_x_ts_utc <= if_modified_since:
return HTTPNotModified(request=request)
keep_cache = (self.keep_cache_private or
('X-Auth-Token' not in request.headers
and 'X-Storage-Token' not in request.headers))
response = Response(
app_iter=disk_file.reader(keep_cache=keep_cache),
request=request,
conditional_response=True)
response.headers['Content-Type'] = metadata.get(
'Content-Type', 'application/octet-stream')
for key, value in metadata.iteritems():
if is_user_meta('object', key) or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = metadata['ETag']
response.last_modified = math.ceil(file_x_ts_flt)
response.content_length = obj_size
try:
response.content_encoding = metadata['Content-Encoding']
except KeyError:
pass
response.headers['X-Timestamp'] = file_x_ts
resp = request.get_response(response)
except (DiskFileNotExist, DiskFileQuarantined):
resp = HTTPNotFound(request=request, conditional_response=True)
return resp
def _if_none_match_wrapper(self, req, *args, **kwargs):
if req.if_none_match is None:
return fnc(self, req, *args, **kwargs)
oio_headers = {REQID_HEADER: self.trans_id}
try:
metadata = self.app.storage.object_get_properties(
self.account_name,
self.container_name,
self.object_name,
version=obj_version_from_env(req.environ),
headers=oio_headers)
except (NoSuchObject, NoSuchContainer):
return fnc(self, req, *args, **kwargs)
# req.if_none_match will check for '*'.
if metadata.get('hash') in req.if_none_match:
if req.method in ('HEAD', 'GET'):
raise HTTPNotModified(request=req)
else:
raise HTTPPreconditionFailed(request=req)
return fnc(self, req, *args, **kwargs)
def _if_none_match_wrapper(self, req, *args, **kwargs):
if req.if_none_match is None:
return fnc(self, req, *args, **kwargs)
oio_headers = {'X-oio-req-id': self.trans_id}
try:
metadata = self.app.storage.object_show(self.account_name,
self.container_name,
self.object_name,
version=req.environ.get(
'oio.query',
{}).get('version'),
headers=oio_headers)
except (NoSuchObject, NoSuchContainer):
return fnc(self, req, *args, **kwargs)
# req.if_none_match will check for '*'.
if metadata.get('hash') in req.if_none_match:
if req.method in ('HEAD', 'GET'):
raise HTTPNotModified(request=req)
else:
raise HTTPPreconditionFailed(request=req)
return fnc(self, req, *args, **kwargs)
def GET(self, request):
"""Handle HTTP GET requests for the Swift Object Server."""
device, partition, account, container, obj = \
split_and_validate_path(request, 5, 5, True)
try:
disk_file = self._diskfile(device,
partition,
account,
container,
obj,
keep_data_fp=True,
iter_hook=sleep)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
if disk_file.is_deleted() or disk_file.is_expired():
if request.headers.get('if-match') == '*':
return HTTPPreconditionFailed(request=request)
else:
return HTTPNotFound(request=request)
try:
file_size = disk_file.get_data_file_size()
except (DiskFileError, DiskFileNotExist):
disk_file.quarantine()
return HTTPNotFound(request=request)
if request.headers.get('if-match') not in (None, '*') and \
disk_file.metadata['ETag'] not in request.if_match:
disk_file.close()
return HTTPPreconditionFailed(request=request)
if request.headers.get('if-none-match') is not None:
if disk_file.metadata['ETag'] in request.if_none_match:
resp = HTTPNotModified(request=request)
resp.etag = disk_file.metadata['ETag']
disk_file.close()
return resp
try:
if_unmodified_since = request.if_unmodified_since
except (OverflowError, ValueError):
# catches timestamps before the epoch
return HTTPPreconditionFailed(request=request)
if if_unmodified_since and \
datetime.fromtimestamp(
float(disk_file.metadata['X-Timestamp']), UTC) > \
if_unmodified_since:
disk_file.close()
return HTTPPreconditionFailed(request=request)
try:
if_modified_since = request.if_modified_since
except (OverflowError, ValueError):
# catches timestamps before the epoch
return HTTPPreconditionFailed(request=request)
if if_modified_since and \
datetime.fromtimestamp(
float(disk_file.metadata['X-Timestamp']), UTC) < \
if_modified_since:
disk_file.close()
return HTTPNotModified(request=request)
response = Response(app_iter=disk_file,
request=request,
conditional_response=True)
response.headers['Content-Type'] = disk_file.metadata.get(
'Content-Type', 'application/octet-stream')
for key, value in disk_file.metadata.iteritems():
if key.lower().startswith('x-object-meta-') or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = disk_file.metadata['ETag']
response.last_modified = float(disk_file.metadata['X-Timestamp'])
response.content_length = file_size
if response.content_length < self.keep_cache_size and \
(self.keep_cache_private or
('X-Auth-Token' not in request.headers and
'X-Storage-Token' not in request.headers)):
disk_file.keep_cache = True
if 'Content-Encoding' in disk_file.metadata:
response.content_encoding = disk_file.metadata['Content-Encoding']
response.headers['X-Timestamp'] = disk_file.metadata['X-Timestamp']
return request.get_response(response)
def GET(self, request):
"""Handle HTTP GET requests for the Swift Object Server."""
device, partition, account, container, obj = \
split_and_validate_path(request, 5, 5, True)
keep_cache = self.keep_cache_private or (
'X-Auth-Token' not in request.headers and
'X-Storage-Token' not in request.headers)
try:
disk_file = self.get_diskfile(
device, partition, account, container, obj)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
try:
with disk_file.open():
metadata = disk_file.get_metadata()
obj_size = int(metadata['Content-Length'])
if request.headers.get('if-match') not in (None, '*') and \
metadata['ETag'] not in request.if_match:
return HTTPPreconditionFailed(request=request)
if request.headers.get('if-none-match') is not None:
if metadata['ETag'] in request.if_none_match:
resp = HTTPNotModified(request=request)
resp.etag = metadata['ETag']
return resp
file_x_ts = metadata['X-Timestamp']
file_x_ts_flt = float(file_x_ts)
try:
if_unmodified_since = request.if_unmodified_since
except (OverflowError, ValueError):
# catches timestamps before the epoch
return HTTPPreconditionFailed(request=request)
file_x_ts_utc = datetime.fromtimestamp(file_x_ts_flt, UTC)
if if_unmodified_since and file_x_ts_utc > if_unmodified_since:
return HTTPPreconditionFailed(request=request)
try:
if_modified_since = request.if_modified_since
except (OverflowError, ValueError):
# catches timestamps before the epoch
return HTTPPreconditionFailed(request=request)
if if_modified_since and file_x_ts_utc < if_modified_since:
return HTTPNotModified(request=request)
keep_cache = (self.keep_cache_private or
('X-Auth-Token' not in request.headers and
'X-Storage-Token' not in request.headers))
response = Response(
app_iter=disk_file.reader(keep_cache=keep_cache),
request=request, conditional_response=True)
response.headers['Content-Type'] = metadata.get(
'Content-Type', 'application/octet-stream')
for key, value in metadata.iteritems():
if key.lower().startswith('x-object-meta-') or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = metadata['ETag']
response.last_modified = file_x_ts_flt
response.content_length = obj_size
try:
response.content_encoding = metadata[
'Content-Encoding']
except KeyError:
pass
response.headers['X-Timestamp'] = file_x_ts
resp = request.get_response(response)
except DiskFileNotExist:
if request.headers.get('if-match') == '*':
resp = HTTPPreconditionFailed(request=request)
else:
resp = HTTPNotFound(request=request)
except DiskFileQuarantined:
resp = HTTPNotFound(request=request)
return resp
def GET(self, request):
"""Handle HTTP GET requests for the Swift Object Server."""
device, partition, account, container, obj = \
split_and_validate_path(request, 5, 5, True)
try:
disk_file = self._diskfile(device, partition, account, container,
obj, iter_hook=sleep)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
disk_file.open()
if disk_file.is_deleted() or disk_file.is_expired():
if request.headers.get('if-match') == '*':
return HTTPPreconditionFailed(request=request)
else:
return HTTPNotFound(request=request)
try:
file_size = disk_file.get_data_file_size()
except (DiskFileError, DiskFileNotExist):
disk_file.quarantine()
return HTTPNotFound(request=request)
metadata = disk_file.get_metadata()
if request.headers.get('if-match') not in (None, '*') and \
metadata['ETag'] not in request.if_match:
disk_file.close()
return HTTPPreconditionFailed(request=request)
if request.headers.get('if-none-match') is not None:
if metadata['ETag'] in request.if_none_match:
resp = HTTPNotModified(request=request)
resp.etag = metadata['ETag']
disk_file.close()
return resp
try:
if_unmodified_since = request.if_unmodified_since
except (OverflowError, ValueError):
# catches timestamps before the epoch
return HTTPPreconditionFailed(request=request)
if if_unmodified_since and \
datetime.fromtimestamp(
float(metadata['X-Timestamp']), UTC) > \
if_unmodified_since:
disk_file.close()
return HTTPPreconditionFailed(request=request)
try:
if_modified_since = request.if_modified_since
except (OverflowError, ValueError):
# catches timestamps before the epoch
return HTTPPreconditionFailed(request=request)
if if_modified_since and \
datetime.fromtimestamp(
float(metadata['X-Timestamp']), UTC) < \
if_modified_since:
disk_file.close()
return HTTPNotModified(request=request)
response = Response(app_iter=disk_file,
request=request, conditional_response=True)
response.headers['Content-Type'] = metadata.get(
'Content-Type', 'application/octet-stream')
for key, value in metadata.iteritems():
if key.lower().startswith('x-object-meta-') or \
key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = metadata['ETag']
response.last_modified = float(metadata['X-Timestamp'])
response.content_length = file_size
if response.content_length < self.keep_cache_size and \
(self.keep_cache_private or
('X-Auth-Token' not in request.headers and
'X-Storage-Token' not in request.headers)):
disk_file.keep_cache = True
if 'Content-Encoding' in metadata:
response.content_encoding = metadata['Content-Encoding']
response.headers['X-Timestamp'] = metadata['X-Timestamp']
return request.get_response(response)
class CdnHandler(OriginBase):
def __init__(self, app, conf, logger):
OriginBase.__init__(self, app, conf, logger)
self.logger = logger
self.max_cdn_file_size = int(
conf.get('max_cdn_file_size', 10 * 1024**3))
self.allowed_origin_remote_ips = []
remote_ips = conf.get('allowed_origin_remote_ips')
if remote_ips:
self.allowed_origin_remote_ips = \
[ip.strip() for ip in remote_ips.split(',') if ip.strip()]
if not bool(conf.get('incoming_url_regex')):
raise InvalidConfiguration('Invalid config for CdnHandler')
self.cdn_regexes = []
for key, val in conf['incoming_url_regex'].items():
regex = re.compile(val)
self.cdn_regexes.append(regex)
def _getCacheHeaders(self, ttl):
return {
'Expires': strftime("%a, %d %b %Y %H:%M:%S GMT",
gmtime(time() + ttl)),
'Cache-Control': 'max-age=%d, public' % ttl
}
def _getCdnHeaders(self, req):
headers = {'X-Web-Mode': 'True', 'User-Agent': 'SOS Origin'}
for header in ['If-Modified-Since', 'If-Match', 'Range', 'If-Range']:
if header in req.headers:
headers[header] = req.headers[header]
return headers
def handle_request(self, env, req):
if req.method not in ('GET', 'HEAD'):
headers = self._getCacheHeaders(CACHE_BAD_URL)
return HTTPMethodNotAllowed(request=req, headers=headers)
if self.allowed_origin_remote_ips and \
req.remote_addr not in self.allowed_origin_remote_ips:
raise OriginRequestNotAllowed(
'SOS Origin: Remote IP %s not allowed' % req.remote_addr)
# allow earlier middleware to override hash and obj_name
hsh = env.get('swift.cdn_hash')
object_name = env.get('swift.cdn_object_name')
if hsh is None or object_name is None:
for regex in self.cdn_regexes:
match_obj = regex.match(req.url)
if match_obj:
match_dict = match_obj.groupdict()
if not hsh:
hsh = match_dict.get('hash')
if not object_name:
object_name = match_dict.get('object_name')
break
if not hsh:
self.logger.debug('Hash %s not found in %s' % (hsh, req.url))
headers = self._getCacheHeaders(CACHE_BAD_URL)
return HTTPNotFound(request=req, headers=headers)
if hsh.find('-') >= 0:
hsh = hsh.split('-', 1)[1]
try:
cdn_obj_path = self.get_hsh_obj_path(hsh)
except ValueError, e:
self.logger.debug('get_hsh_obj_path error: %s' % e)
headers = self._getCacheHeaders(CACHE_BAD_URL)
return HTTPBadRequest(request=req, headers=headers)
hash_data = self.get_cdn_data(env, cdn_obj_path)
if hash_data and hash_data.cdn_enabled:
# this is a cdn enabled container, proxy req to swift
if env.get('swift.cdn_authorize'):
auth_resp, ttl = env['swift.cdn_authorize'](
env, hash_data.account.encode('utf-8'))
if auth_resp:
return auth_resp(request=req,
headers=self._getCacheHeaders(ttl))
swift_path = quote('/v1/%s/%s/' %
(hash_data.account.encode('utf-8'),
hash_data.container.encode('utf-8')))
if object_name:
swift_path += object_name
headers = self._getCdnHeaders(req)
env['swift.source'] = 'SOS'
resp = make_pre_authed_request(env,
req.method,
swift_path,
headers=headers,
agent='SwiftOrigin',
swift_source='SOS').get_response(
self.app)
if resp.status_int == 301 and 'Location' in resp.headers:
loc_parsed = urlparse(resp.headers['Location'])
acc_cont_path = '/v1/%s/%s' % (hash_data.account.encode(
'utf-8'), hash_data.container.encode('utf-8'))
if loc_parsed.path.startswith(acc_cont_path):
sos_loc = loc_parsed.path[len(acc_cont_path):]
resp = SosResponse(
headers=self._getCacheHeaders(hash_data.ttl))
resp.headers['Location'] = sos_loc
resp.status = 301
return resp
else:
self.logger.exception(
'Unexpected Location header '
'returned. %s does not begin with expected '
'path: %s' % (loc_parsed.geturl(), acc_cont_path))
return HTTPInternalServerError('Unexpected Relocation')
if resp.status_int == 304:
return HTTPNotModified(request=req,
headers=self._getCacheHeaders(
hash_data.ttl))
if resp.status_int == 416:
return HTTPRequestedRangeNotSatisfiable(
request=req, headers=self._getCacheHeaders(CACHE_404))
if resp.status_int // 100 == 2 or resp.status_int == 404:
if resp.content_length > self.max_cdn_file_size:
return HTTPBadRequest(
request=req, headers=self._getCacheHeaders(CACHE_404))
cdn_resp = Response(request=req, app_iter=resp.app_iter)
cdn_resp.status = resp.status_int
cdn_resp.headers.update(resp.headers)
if resp.status_int == 404:
cdn_resp.headers.update(self._getCacheHeaders(CACHE_404))
else:
cdn_resp.headers.update(
self._getCacheHeaders(hash_data.ttl))
return cdn_resp
self.logger.error('Unexpected response from '
'Swift: %s, %s: %s' %
(resp.status, swift_path, resp.body[:40]))
return HTTPNotFound(request=req,
headers=self._getCacheHeaders(CACHE_404))
if request.headers.get('if-match') == '*':
return HTTPPreconditionFailed(request=request)
else:
return HTTPNotFound(request=request)
try:
file_size = file.get_data_file_size()
except (DiskFileError, DiskFileNotExist):
file.quarantine()
return HTTPNotFound(request=request)
if request.headers.get('if-match') not in (None, '*') and \
file.metadata['ETag'] not in request.if_match:
file.close()
return HTTPPreconditionFailed(request=request)
if request.headers.get('if-none-match') is not None:
if file.metadata['ETag'] in request.if_none_match:
resp = HTTPNotModified(request=request)
resp.etag = file.metadata['ETag']
file.close()
return resp
try:
if_unmodified_since = request.if_unmodified_since
except (OverflowError, ValueError):
# catches timestamps before the epoch
return HTTPPreconditionFailed(request=request)
if if_unmodified_since and \
datetime.fromtimestamp(
float(file.metadata['X-Timestamp']), UTC) > \
if_unmodified_since:
file.close()
return HTTPPreconditionFailed(request=request)
try:
def GET(self, request):
"""Handle HTTP GET requests for the Swift Object Server."""
device, partition, account, container, obj = split_and_validate_path(request, 5, 5, True)
try:
disk_file = self._diskfile(device, partition, account, container, obj, keep_data_fp=True, iter_hook=sleep)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
if disk_file.is_deleted() or disk_file.is_expired():
if request.headers.get("if-match") == "*":
return HTTPPreconditionFailed(request=request)
else:
return HTTPNotFound(request=request)
try:
file_size = disk_file.get_data_file_size()
except (DiskFileError, DiskFileNotExist):
disk_file.quarantine()
return HTTPNotFound(request=request)
if request.headers.get("if-match") not in (None, "*") and disk_file.metadata["ETag"] not in request.if_match:
disk_file.close()
return HTTPPreconditionFailed(request=request)
if request.headers.get("if-none-match") is not None:
if disk_file.metadata["ETag"] in request.if_none_match:
resp = HTTPNotModified(request=request)
resp.etag = disk_file.metadata["ETag"]
disk_file.close()
return resp
try:
if_unmodified_since = request.if_unmodified_since
except (OverflowError, ValueError):
# catches timestamps before the epoch
return HTTPPreconditionFailed(request=request)
if (
if_unmodified_since
and datetime.fromtimestamp(float(disk_file.metadata["X-Timestamp"]), UTC) > if_unmodified_since
):
disk_file.close()
return HTTPPreconditionFailed(request=request)
try:
if_modified_since = request.if_modified_since
except (OverflowError, ValueError):
# catches timestamps before the epoch
return HTTPPreconditionFailed(request=request)
if (
if_modified_since
and datetime.fromtimestamp(float(disk_file.metadata["X-Timestamp"]), UTC) < if_modified_since
):
disk_file.close()
return HTTPNotModified(request=request)
response = Response(app_iter=disk_file, request=request, conditional_response=True)
response.headers["Content-Type"] = disk_file.metadata.get("Content-Type", "application/octet-stream")
for key, value in disk_file.metadata.iteritems():
if key.lower().startswith("x-object-meta-") or key.lower() in self.allowed_headers:
response.headers[key] = value
response.etag = disk_file.metadata["ETag"]
response.last_modified = float(disk_file.metadata["X-Timestamp"])
response.content_length = file_size
if response.content_length < self.keep_cache_size and (
self.keep_cache_private
or ("X-Auth-Token" not in request.headers and "X-Storage-Token" not in request.headers)
):
disk_file.keep_cache = True
if "Content-Encoding" in disk_file.metadata:
response.content_encoding = disk_file.metadata["Content-Encoding"]
response.headers["X-Timestamp"] = disk_file.metadata["X-Timestamp"]
return request.get_response(response)
