def handle_request(self, env, req):
"""
Handles the POST /origin/.prep call for preparing the backing store
Swift cluster for use with the origin subsystem. Can only be called by
.origin_admin
:param req: The webob.Request to process.
:returns: webob.Response, 204 on success
"""
if not self.is_origin_admin(req):
return HTTPForbidden(request=req)
try:
vsn, account = split_path(req.path, 2, 2)
except ValueError:
return HTTPBadRequest(request=req)
if account == '.prep':
path = '/v1/%s' % self.origin_account
resp = make_pre_authed_request(req.environ, 'PUT',
path, agent='SwiftOrigin').get_response(self.app)
if resp.status_int // 100 != 2:
raise Exception(
'Could not create the main origin account: %s %s' %
(path, resp.status))
for i in xrange(self.num_hash_cont):
cont_name = '.hash_%d' % i
path = '/v1/%s/%s' % (self.origin_account, cont_name)
resp = make_pre_authed_request(req.environ, 'PUT',
path, agent='SwiftOrigin').get_response(self.app)
if resp.status_int // 100 != 2:
raise Exception('Could not create %s container: %s %s' %
(cont_name, path, resp.status))
return HTTPNoContent(request=req)
return HTTPNotFound(request=req)
def test_pre_auth_req_drops_query(self):
r = wsgi.make_pre_authed_request({"QUERY_STRING": "original"}, "GET", "path")
self.assertEquals(r.query_string, "original")
r = wsgi.make_pre_authed_request({"QUERY_STRING": "original"}, "GET", "path?replacement")
self.assertEquals(r.query_string, "replacement")
r = wsgi.make_pre_authed_request({"QUERY_STRING": "original"}, "GET", "path?")
self.assertEquals(r.query_string, "")
def _set_hash_data(self, env, cdn_obj_path, new_hash_data,
update_listings=True):
"""
Actually sets the data in the .origin account. If not successful on
any of the several updates this has to do, will raise a OriginDbFailure
"""
cdn_obj_data = new_hash_data.get_json_str()
cdn_obj_etag = md5(cdn_obj_data).hexdigest()
# this is always a PUT because a POST needs to update the file
cdn_obj_resp = make_pre_authed_request(
env, 'PUT', cdn_obj_path, body=cdn_obj_data,
headers={'Etag': cdn_obj_etag},
agent='SwiftOrigin', swift_source='SOS').get_response(self.app)
if cdn_obj_resp.status_int // 100 != 2:
raise OriginDbFailure(
'Could not PUT .hash obj in origin '
'db: %s %s' % (cdn_obj_path, cdn_obj_resp.status_int))
memcache_client = utils.cache_from_env(env)
if memcache_client:
memcache_key = self.cdn_data_memcache_key(cdn_obj_path)
memcache_client.delete(memcache_key)
if not update_listings:
return
listing_cont_path = quote('/v1/%s/%s' % (self.origin_account,
new_hash_data.account))
resp = make_pre_authed_request(
env, 'HEAD', listing_cont_path,
agent='SwiftOrigin', swift_source='SOS').get_response(self.app)
if resp.status_int == 404:
# create new container for listings
resp = make_pre_authed_request(
env, 'PUT', listing_cont_path,
agent='SwiftOrigin', swift_source='SOS').get_response(self.app)
if resp.status_int // 100 != 2:
raise OriginDbFailure(
'Could not create listing container '
'in origin db: %s %s' % (listing_cont_path, resp.status))
cdn_list_path = quote('/v1/%s/%s/%s' % (
self.origin_account, new_hash_data.account.encode('utf-8'),
new_hash_data.container.encode('utf-8')))
listing_content_type = new_hash_data.gen_listing_content_type()
cdn_list_resp = make_pre_authed_request(
env, 'PUT', cdn_list_path,
headers={'Content-Type': listing_content_type,
'Content-Length': 0},
agent='SwiftOrigin', swift_source='SOS').get_response(self.app)
if cdn_list_resp.status_int // 100 != 2:
raise OriginDbFailure(
'Could not PUT/POST to cdn listing in '
'origin db: %s %s' % (cdn_list_path, cdn_list_resp.status_int))
def test_pre_auth_req_drops_query(self):
r = wsgi.make_pre_authed_request(
{'QUERY_STRING': 'original'}, 'GET', 'path')
self.assertEquals(r.query_string, 'original')
r = wsgi.make_pre_authed_request(
{'QUERY_STRING': 'original'}, 'GET', 'path?replacement')
self.assertEquals(r.query_string, 'replacement')
r = wsgi.make_pre_authed_request(
{'QUERY_STRING': 'original'}, 'GET', 'path?')
self.assertEquals(r.query_string, '')
def test_pre_auth_req(self):
class FakeReq(object):
@classmethod
def fake_blank(cls, path, environ={}, body="", headers={}):
self.assertEquals(environ["swift.authorize"]("test"), None)
self.assertFalse("HTTP_X_TRANS_ID" in environ)
was_blank = Request.blank
Request.blank = FakeReq.fake_blank
wsgi.make_pre_authed_request({"HTTP_X_TRANS_ID": "1234"}, "PUT", "/", body="tester", headers={})
wsgi.make_pre_authed_request({"HTTP_X_TRANS_ID": "1234"}, "PUT", "/", headers={})
Request.blank = was_blank
def test_pre_auth_req(self):
class FakeReq(object):
@classmethod
def fake_blank(cls, path, environ={}, body='', headers={}):
self.assertEquals(environ['swift.authorize']('test'), None)
self.assertFalse('HTTP_X_TRANS_ID' in environ)
was_blank = Request.blank
Request.blank = FakeReq.fake_blank
wsgi.make_pre_authed_request({'HTTP_X_TRANS_ID': '1234'},
'PUT', '/', body='tester', headers={})
wsgi.make_pre_authed_request({'HTTP_X_TRANS_ID': '1234'},
'PUT', '/', headers={})
Request.blank = was_blank
def test_pre_auth_req_with_env_path_and_script(self):
env = {"PATH_INFO": "/unquoted path with %20", "SCRIPT_NAME": "/script"}
r = wsgi.make_pre_authed_request(env, "GET")
expected_path = quote(env["SCRIPT_NAME"] + env["PATH_INFO"])
self.assertEquals(r.path, expected_path)
env = {"PATH_INFO": "", "SCRIPT_NAME": "/script"}
r = wsgi.make_pre_authed_request(env, "GET")
self.assertEquals(r.path, "/script")
env = {"PATH_INFO": "/path", "SCRIPT_NAME": ""}
r = wsgi.make_pre_authed_request(env, "GET")
self.assertEquals(r.path, "/path")
env = {"PATH_INFO": "", "SCRIPT_NAME": ""}
r = wsgi.make_pre_authed_request(env, "GET")
self.assertEquals(r.path, "")
def origin_db_delete(self, env, req):
""" Handles DELETEs in the Origin database """
if not self.delete_enabled:
return HTTPMethodNotAllowed(request=req)
try:
vsn, account, container = split_path(req.path, 3, 3)
except ValueError:
return HTTPBadRequest(
'Invalid request. '
'URI format: /<api version>/<account>/<container>')
hsh = self.hash_path(account, container)
cdn_obj_path = self.get_hsh_obj_path(hsh)
# Remove memcache entry
memcache_client = utils.cache_from_env(env)
if memcache_client:
memcache_key = self.cdn_data_memcache_key(cdn_obj_path)
memcache_client.delete(memcache_key)
resp = make_pre_authed_request(env,
'DELETE',
cdn_obj_path,
agent='SwiftOrigin',
swift_source='SOS').get_response(
self.app)
# A 404 means it's already deleted, which is okay
if resp.status_int // 100 != 2 and resp.status_int != 404:
raise OriginDbFailure('Could not DELETE .hash obj in origin '
'db: %s %s' %
(cdn_obj_path, resp.status_int))
cdn_list_path = quote('/v1/%s/%s/%s' %
(self.origin_account, account, container))
list_resp = make_pre_authed_request(env,
'DELETE',
cdn_list_path,
agent='SwiftOrigin',
swift_source='SOS').get_response(
self.app)
if list_resp.status_int // 100 != 2 and list_resp.status_int != 404:
raise OriginDbFailure('Could not DELETE listing path in origin '
'db: %s %s' %
(cdn_list_path, list_resp.status_int))
# Return 404 if container didn't exist
if resp.status_int == 404 and list_resp.status_int == 404:
return HTTPNotFound(request=req)
return HTTPNoContent(request=req)
def test_pre_auth_req_with_env_path_and_script(self):
env = {'PATH_INFO': '/unquoted path with %20',
'SCRIPT_NAME': '/script'}
r = wsgi.make_pre_authed_request(env, 'GET')
expected_path = quote(env['SCRIPT_NAME'] + env['PATH_INFO'])
self.assertEquals(r.path, expected_path)
env = {'PATH_INFO': '', 'SCRIPT_NAME': '/script'}
r = wsgi.make_pre_authed_request(env, 'GET')
self.assertEquals(r.path, '/script')
env = {'PATH_INFO': '/path', 'SCRIPT_NAME': ''}
r = wsgi.make_pre_authed_request(env, 'GET')
self.assertEquals(r.path, '/path')
env = {'PATH_INFO': '', 'SCRIPT_NAME': ''}
r = wsgi.make_pre_authed_request(env, 'GET')
self.assertEquals(r.path, '')
def _get_details(self, req, access_key):
"""Get access key details.
:return: (secret_key, account) as tuple or (None, None) if not found.
"""
memcache_client = cache_from_env(req.environ)
if memcache_client:
memcache_key = MEMCACHE_KEY_FORMAT % (self.reseller_prefix,
access_key)
data = memcache_client.get(memcache_key)
if data:
return data[0], data[1]
path = quote(self.akd_container_url + access_key)
resp = make_pre_authed_request(req.environ, 'GET',
path).get_response(self.app)
if resp.status_int // 100 == 2:
data = json.loads(resp.body)
secret_key, account = data['secret_key'], data['account']
if memcache_client:
memcache_client.set(memcache_key, (secret_key, account),
time=self.cache_time)
return secret_key, account
elif resp.status_int // 100 == 4:
return None, None
else:
raise Exception('Could not GET access key details: {} {}'.format(
path, resp.status_int))
def get_listings(marker):
listing_path = quote('/v1/%s/%s' % (self.origin_account, account))
listing_path += '?format=json&marker=' + quote(marker)
# no limit in request because may have to filter on cdn_enabled
resp = make_pre_authed_request(
env, 'GET', listing_path,
agent='SwiftOrigin', swift_source='SOS').get_response(self.app)
resp_headers = {}
listing_formatted = []
if resp.status_int // 100 == 2:
cont_listing = json.loads(resp.body)
for listing_dict in cont_listing:
if limit is None or len(listing_formatted) < limit:
try:
formatted_data = self._parse_container_listing(
account, listing_dict, list_format,
only_cdn_enabled=enabled_only)
if formatted_data:
listing_formatted.append(formatted_data)
except InvalidContentType, e:
self.logger.exception(e)
continue
else:
break
if cont_listing and not listing_formatted:
# there were rows returned but none matched enabled_only-
# requery with new marker
new_marker = cont_listing[-1]['name']
if isinstance(new_marker, unicode):
new_marker = new_marker.encode('utf-8')
return get_listings(new_marker)
def iter_objects_by_prefix(account, container, prefix, swift_client=None,
app=None):
marker = ''
while True:
param = 'format=json&marker=%s' % marker
if marker == '':
param = '%s&prefix=%s' % (param, prefix)
if swift_client:
path = swift_client.make_path(account, container)
resp = swift_client.make_request('GET', '%s?%s' % (path, param),
{}, (2, 4))
elif app:
path = '/v1/%s/%s' % (account, container)
env = make_pre_authed_env({}, method='GET', path=path,
query_string=param)
req = make_pre_authed_request(env)
resp = req.get_response(app)
if not resp.status_int == 200:
break
data = json.loads(resp.body)
if not data:
break
for item in data:
yield item
marker = data[-1]['name'].encode('utf8')
def get_account_info(env, app):
"""
Get the info structure for an account, based on env and app.
This is useful to middlewares.
Note: This call bypasses auth. Success does not imply that the
request has authorization to the account_info.
"""
(version, account, container, _) = \
split_path(env['PATH_INFO'], 2, 4, True)
new_env = env.copy()
obj_path = '/%s/%s' % (version, account)
if isinstance(obj_path, unicode):
obj_path = obj_path.encode('utf-8')
new_env['PATH_INFO'] = obj_path
if new_env.has_key('wsgi.input'):
del(new_env['wsgi.input'])
if new_env.has_key('QUERY_STRING'):
del(new_env['QUERY_STRING'])
new_env['CONTENT_LENGTH'] = 0
resp = make_pre_authed_request(new_env, 'HEAD', '/%s/%s' % (version, account)).get_response(app)
account_info = headers_to_account_info(
resp.headers, resp.status_int)
return account_info
def _put_versioned_obj(self, req, put_path_info, source_resp):
# Create a new Request object to PUT to the versions container, copying
# all headers from the source object apart from x-timestamp.
put_req = make_pre_authed_request(req.environ, path=put_path_info, method="PUT", swift_source="VW")
copy_header_subset(source_resp, put_req, lambda k: k.lower() != "x-timestamp")
put_req.environ["wsgi.input"] = FileLikeIter(source_resp.app_iter)
return put_req.get_response(self.app)
def get_chunks(self, env, chunks, container):
file_compress_content = []
self.app.logger.info(
'StackSync API: get_chunks: chunks: %s container: %s', str(chunks),
str(container))
seg_resp = None
for chunk in chunks:
new_path = "/v1/" + env[
'stacksync_user_account'] + "/" + container + "/" + str(chunk)
seg_req = make_pre_authed_request(env,
method='GET',
path=new_path,
body="",
agent='%(orig)s')
seg_resp = seg_req.get_response(self.app)
if is_valid_status(seg_resp.status_int):
file_compress_content.append(seg_resp.body)
else:
file_compress_content = []
break
if seg_resp:
return file_compress_content, seg_resp.status_int
else:
# No chunks
return file_compress_content, 200
def _listing_pages_iter(self, account_name, lcontainer, lprefix, env):
marker = ''
while True:
lreq = make_pre_authed_request(env,
method='GET',
swift_source='VW',
path='/v1/%s/%s' %
(account_name, lcontainer))
lreq.environ['QUERY_STRING'] = \
'format=json&prefix=%s&reverse=on&marker=%s' % (
quote(lprefix), quote(marker))
lresp = lreq.get_response(self.app)
if not is_success(lresp.status_int):
if lresp.status_int == HTTP_NOT_FOUND:
raise ListingIterNotFound()
elif is_client_error(lresp.status_int):
raise HTTPPreconditionFailed()
else:
raise ListingIterError()
if not lresp.body:
break
sublisting = json.loads(lresp.body)
if not sublisting:
break
marker = sublisting[-1]['name'].encode('utf-8')
yield sublisting
def _get_container_info(self, env):
"""
Retrieves x-container-meta-web-index, x-container-meta-web-error,
x-container-meta-web-listings, and x-container-meta-web-listings-css
from memcache or from the cluster and stores the result in memcache and
in self._index, self._error, self._listings, and self._listings_css.
:param env: The WSGI environment dict.
"""
self._index = self._error = self._listings = self._listings_css = None
memcache_client = cache_from_env(env)
if memcache_client:
memcache_key = "/staticweb/%s/%s/%s" % (self.version, self.account, self.container)
cached_data = memcache_client.get(memcache_key)
if cached_data:
(self._index, self._error, self._listings, self._listings_css) = cached_data
return
resp = make_pre_authed_request(
env, "HEAD", "/%s/%s/%s" % (self.version, self.account, self.container), agent=self.agent
).get_response(self.app)
if is_success(resp.status_int):
self._index = resp.headers.get("x-container-meta-web-index", "").strip()
self._error = resp.headers.get("x-container-meta-web-error", "").strip()
self._listings = resp.headers.get("x-container-meta-web-listings", "").strip()
self._listings_css = resp.headers.get("x-container-meta-web-listings-css", "").strip()
if memcache_client:
memcache_client.set(
memcache_key,
(self._index, self._error, self._listings, self._listings_css),
timeout=self.cache_timeout,
)
def iter_objects(self, env, path, prefix, marker, end, count):
path_with_params = '%s?format=json&prefix=%s' % (path, prefix)
seg = ''
force_break = False
while count > 0:
l = 1000 if count > 1000 else count
count -= 1000
rpath = path_with_params + ('&marker=%s' % marker) + (
'&limit=%d' % l)
req = make_pre_authed_request(env, 'GET', rpath)
req.environ['swift.proxy_access_log_made'] = True
resp = req.get_response(self.app)
segments = json.loads(resp.body)
for seg in segments:
name = seg['name']
record_ts = int(name.split('/')[1])
if record_ts > end:
force_break = True
break
yield name
if force_break:
break
if len(segments) != l:
break
if segments:
marker = seg['name']
else:
break
def origin_db_get(self, env, req):
'''
Handles GETs to the Origin database
The only part of the path this pays attention to is the account.
'''
#TODO: this does not return transfer-encoding: chunked
try:
account = req.path.split('/')[2]
except IndexError:
return HTTPBadRequest('Invalid request. '
'URI format: /<api version>/<account>')
#TODO: make sure to test with unicode container names
marker = get_param(req, 'marker', default='')
list_format = get_param(req, 'format')
enabled_only = get_param(req, 'enabled',
default='false') in TRUE_VALUES
limit = get_param(req, 'limit')
if limit:
try:
limit = int(limit)
except ValueError:
return HTTPBadRequest('Invalid limit, must be an integer')
listing_path = '/v1/%s/%s?format=json&marker=%s' % \
(self.origin_account, account, marker)
# no limit in request because may have to filter on cdn_enabled
resp = make_pre_authed_request(env, 'GET',
listing_path, agent='SwiftOrigin').get_response(self.app)
resp_headers = {}
# {'Transfer-Encoding': 'chunked'}
#TODO is this right? was chunked in old one
if resp.status_int // 100 == 2:
cont_listing = json.loads(resp.body)
# TODO: is it ok to load the whole thing? do i have a choice?
listing_formatted = []
for listing_dict in cont_listing:
if limit is None or len(listing_formatted) < limit:
try:
formatted_data = self._parse_container_listing(
account, listing_dict, list_format,
only_cdn_enabled=enabled_only)
if formatted_data:
listing_formatted.append(formatted_data)
except InvalidContentType, e:
self.logger.exception(e)
continue
else:
break
if list_format == 'xml':
resp_headers['Content-Type'] = 'application/xml'
response_body = ('<?xml version="1.0" encoding="UTF-8"?>\n'
'<account name="%s">\n%s\n</account>') % (account,
'\n'.join(listing_formatted))
elif list_format == 'json':
resp_headers['Content-Type'] = 'application/json'
response_body = json.dumps(listing_formatted)
else:
resp_headers['Content-Type'] = 'text/plain; charset=UTF-8'
response_body = '\n'.join(listing_formatted)
return Response(body=response_body, headers=resp_headers)
def remove_old_chunks(self, env, chunks_diff, container):
error = False
self.app.logger.info('StackSync API: remove old chunks: container: %s',
str(container))
for chunk_name in chunks_diff:
env_aux = env.copy()
new_path = "/v1/" + env[
'stacksync_user_account'] + "/" + container + "/" + str(
chunk_name)
del env_aux['HTTP_STACKSYNC_API']
seg_req = make_pre_authed_request(env_aux,
method='DELETE',
path=new_path,
agent=str(container))
seg_resp = seg_req.get_response(self.app)
if not is_valid_status(seg_resp.status_int):
self.app.logger.error(
'StackSync API: upload_file_chunks: error deleting old chunk %s',
str(chunk_name))
error = True
break
if error:
self.app.logger.error(
'StackSync API: upload_file_chunks: status: %s description: Error uploading chunks to storage backend',
seg_resp.status)
response = create_error_response(
500, "Error uploading chunks to storage backend")
else:
response = HTTPCreated()
return response
def get_container_info(env, app, swift_source=None):
"""
Get the info structure for a container, based on env and app.
This is useful to middlewares.
"""
cache = cache_from_env(env)
if not cache:
return None
(version, account, container, obj) = \
split_path(env['PATH_INFO'], 2, 4, True)
cache_key = get_container_memcache_key(account, container)
# Use a unique environment cache key per container. If you copy this env
# to make a new request, it won't accidentally reuse the old container info
env_key = 'swift.%s' % cache_key
if env_key not in env:
container_info = cache.get(cache_key)
if not container_info:
resp = make_pre_authed_request(
env, 'HEAD', '/%s/%s/%s' % (version, account, container),
swift_source=swift_source,
).get_response(app)
container_info = headers_to_container_info(
resp.headers, resp.status_int)
env[env_key] = container_info
return env[env_key]
def remove_chunks(self, env, chunks_names, container):
error = False
self.app.logger.info('StackSync API: internal remove uploaded chunks: container: %s', str(container))
for chunk_name in chunks_names:
env_aux = env.copy()
new_path = "/v1/" + env['stacksync_user_account'] + "/" + container + "/" + str(chunk_name)
del env_aux['HTTP_STACKSYNC_API']
seg_req = make_pre_authed_request(env_aux, method='DELETE', path=new_path,
agent=str(container))
seg_resp = seg_req.get_response(self.app)
if not is_valid_status(seg_resp.status_int):
self.app.logger.error('StackSync API: remove_chunks: error deleting uploaded chunks %s', str(chunk_name))
error = True
break
if error:
self.app.logger.error(
'StackSync API: upload_file_chunks: status: %s description: Error uploading chunks to storage backend',
seg_resp.status)
return False
return True
def test_pre_auth_req_with_quoted_path(self):
r = wsgi.make_pre_authed_request({'HTTP_X_TRANS_ID': '1234'},
'PUT',
path=quote('/a space'),
body='tester',
headers={})
self.assertEquals(r.path, quote('/a space'))
def get_account_info(env, app, swift_source=None):
"""
Get the info structure for an account, based on env and app.
This is useful to middlewares.
Note: This call bypasses auth. Success does not imply that the
request has authorization to the account_info.
"""
cache = cache_from_env(env)
if not cache:
return None
(version, account, _junk, _junk) = \
split_path(env['PATH_INFO'], 2, 4, True)
cache_key = get_account_memcache_key(account)
# Use a unique environment cache key per account. If you copy this env
# to make a new request, it won't accidentally reuse the old account info
env_key = 'swift.%s' % cache_key
if env_key not in env:
account_info = cache.get(cache_key)
if not account_info:
resp = make_pre_authed_request(
env,
'HEAD',
'/%s/%s' % (version, account),
swift_source=swift_source,
).get_response(app)
account_info = headers_to_account_info(resp.headers,
resp.status_int)
env[env_key] = account_info
return env[env_key]
def upload_file_chunks(self, env, chunked_file, container):
error = False
self.app.logger.info('StackSync API: upload_file_chunks: container: %s', str(container))
upload_chunks = []
for i in range(len(chunked_file.chunks)):
chunk_name = chunked_file.name_list[i-1]
chunk_content = chunked_file.chunks[i-1]
env_aux = env.copy()
new_path = "/v1/" + env['stacksync_user_account'] + "/" + container + "/" + chunk_name
del env_aux['HTTP_STACKSYNC_API']
seg_req = make_pre_authed_request(env_aux, method='PUT', path=new_path, body=chunk_content,
agent=str(container))
seg_resp = seg_req.get_response(self.app)
if not is_valid_status(seg_resp.status_int):
self.app.logger.error('StackSync API: upload_file_chunks: error uploading chunk %s', chunk_name)
error = True
break
upload_chunks.append(chunk_name)
if error:
self.app.logger.error(
'StackSync API: upload_file_chunks: status: %s description: Error uploading chunks to storage backend',
seg_resp.status)
response = create_error_response(500, "Error uploading chunks to storage backend")
self.remove_chunks(env, upload_chunks, container)
else:
response = HTTPCreated()
return response
def _listing_pages_iter(self, account_name, lcontainer, lprefix, env):
marker = ''
while True:
lreq = make_pre_authed_request(
env, method='GET', swift_source='VW',
path='/v1/%s/%s' % (account_name, lcontainer))
lreq.environ['QUERY_STRING'] = \
'format=json&prefix=%s&marker=%s' % (quote(lprefix),
quote(marker))
lresp = lreq.get_response(self.app)
if not is_success(lresp.status_int):
if lresp.status_int == HTTP_NOT_FOUND:
raise ListingIterNotFound()
elif is_client_error(lresp.status_int):
raise HTTPPreconditionFailed()
else:
raise ListingIterError()
if not lresp.body:
break
sublisting = json.loads(lresp.body)
if not sublisting:
break
marker = sublisting[-1]['name'].encode('utf-8')
yield sublisting
def get_account_info(env, app, swift_source=None):
"""
Get the info structure for an account, based on env and app.
This is useful to middlewares.
Note: This call bypasses auth. Success does not imply that the
request has authorization to the account_info.
"""
cache = cache_from_env(env)
if not cache:
return None
(version, account, _junk, _junk) = \
split_path(env['PATH_INFO'], 2, 4, True)
cache_key = get_account_memcache_key(account)
# Use a unique environment cache key per account. If you copy this env
# to make a new request, it won't accidentally reuse the old account info
env_key = 'swift.%s' % cache_key
if env_key not in env:
account_info = cache.get(cache_key)
if not account_info:
resp = make_pre_authed_request(
env, 'HEAD', '/%s/%s' % (version, account),
swift_source=swift_source,
).get_response(app)
account_info = headers_to_account_info(
resp.headers, resp.status_int)
env[env_key] = account_info
return env[env_key]
def handle_prep(self, req):
"""Prepare the backing store Swiftcluster for use with the auth system.
Required headers:
- `x-s3auth-prep-key`: must be same as key in config
- `x-s3auth-hash-key`: hash key used for hashing admin key
- `x-s3auth-admin-key`: admin key
Note: The call can also be used to change current s3auth-admin key.
"""
prep_key = req.headers.get("x-s3auth-prep-key")
hash_key = req.headers.get("x-s3auth-hash-key")
admin_key = req.headers.get('x-s3auth-admin-key')
if not all((prep_key, hash_key, admin_key)):
return HTTPBadRequest(
body='Headers x-s3auth-prep-key, x-s3auth-hash-key, '
'x-s3auth-admin-key all required',
request=req)
if self.prep_key != prep_key:
return _denied_response(req)
hashed_admin_key = _hash_msg(admin_key, hash_key)
path = quote('/v1/{}'.format(self.auth_account))
resp = make_pre_authed_request(req.environ,
'PUT',
path,
headers={
HKEY_HASH_KEY: hash_key,
HKEY_HASHED_ADMIN_KEY:
hashed_admin_key,
}).get_response(self.app)
if resp.status_int // 100 != 2:
raise Exception('Could not PUT auth account: {} {}'.format(
path, resp.status))
path = quote(self.akd_container_url)
resp = make_pre_authed_request(req.environ, 'PUT',
path).get_response(self.app)
if resp.status_int // 100 != 2:
raise Exception(
'Could not PUT access key details container: {} {}'.format(
path, resp.status))
return HTTPOk(request=req)
def test_pre_auth_req_swift_source(self):
r = wsgi.make_pre_authed_request({'QUERY_STRING': 'original'},
'GET',
'path',
'the body',
swift_source='UT')
self.assertEquals(r.body, 'the body')
self.assertEquals(r.environ['swift.source'], 'UT')
def _listing_pages_iter(self, account_name, lcontainer, lprefix,
env, marker='', end_marker='', reverse=True):
'''Get "pages" worth of objects that start with a prefix.
The optional keyword arguments ``marker``, ``end_marker``, and
``reverse`` are used similar to how they are for containers. We're
either coming:
- directly from ``_listing_iter``, in which case none of the
optional args are specified, or
- from ``_in_proxy_reverse_listing``, in which case ``reverse``
is ``False`` and both ``marker`` and ``end_marker`` are specified
(although they may still be blank).
'''
while True:
lreq = make_pre_authed_request(
env, method='GET', swift_source='VW',
path='/v1/%s/%s' % (account_name, lcontainer))
lreq.environ['QUERY_STRING'] = \
'format=json&prefix=%s&marker=%s' % (
quote(lprefix), quote(marker))
if end_marker:
lreq.environ['QUERY_STRING'] += '&end_marker=%s' % (
quote(end_marker))
if reverse:
lreq.environ['QUERY_STRING'] += '&reverse=on'
lresp = lreq.get_response(self.app)
if not is_success(lresp.status_int):
close_if_possible(lresp.app_iter)
if lresp.status_int == HTTP_NOT_FOUND:
raise ListingIterNotFound()
elif is_client_error(lresp.status_int):
raise HTTPPreconditionFailed()
else:
raise ListingIterError()
if not lresp.body:
break
sublisting = json.loads(lresp.body)
if not sublisting:
break
# When using the ``reverse`` param, check that the listing is
# actually reversed
first_item = sublisting[0]['name'].encode('utf-8')
last_item = sublisting[-1]['name'].encode('utf-8')
page_is_after_marker = marker and first_item > marker
if reverse and (first_item < last_item or page_is_after_marker):
# Apparently there's at least one pre-2.6.0 container server
yield self._in_proxy_reverse_listing(
account_name, lcontainer, lprefix,
env, marker, sublisting)
return
marker = last_item
yield sublisting
def origin_db_delete(self, env, req):
"""
Handles DELETEs in the Origin database.
This is not really a delete- it will remove the object from the
container listing and set cdn_enabled=false and a deleted flag in the
.hash_* obj that says that the obj is deleted. This way the container
won't show up in the listings, HEAD to the object will return 404s but
behind the scenes lookups to the object will be able to determine
the account and container from a container_hash.
"""
try:
vsn, account, container = split_path(req.path, 3, 3)
except ValueError:
return HTTPBadRequest(
'Invalid request. '
'URI format: /<api version>/<account>/<container>')
if self.extra_header_for_deletes and not req.headers.get(
self.extra_header_for_deletes, 'f').lower() in TRUE_VALUES:
# only do delete if header is set (assuming you want the header)
return HTTPMethodNotAllowed(request=req)
hsh = self.hash_path(account, container)
cdn_obj_path = self.get_hsh_obj_path(hsh)
# Remove memcache entry
memcache_client = utils.cache_from_env(env)
if memcache_client:
memcache_key = self.cdn_data_memcache_key(cdn_obj_path)
memcache_client.delete(memcache_key)
ref_hash_data = HashData(account,
container,
self.default_ttl,
False,
False,
deleted=True)
self._set_hash_data(env,
cdn_obj_path,
ref_hash_data,
update_listings=False)
cdn_list_path = quote('/v1/%s/%s/%s' %
(self.origin_account, account, container))
list_resp = make_pre_authed_request(env,
'DELETE',
cdn_list_path,
agent='SwiftOrigin',
swift_source='SOS').get_response(
self.app)
if list_resp.status_int // 100 != 2 and list_resp.status_int != 404:
raise OriginDbFailure('Could not DELETE listing path in origin '
'db: %s %s' %
(cdn_list_path, list_resp.status_int))
# Return 404 if container didn't exist
if list_resp.status_int == 404:
return HTTPNotFound(request=req)
return HTTPNoContent(request=req)
def _listing_pages_iter(self, account_name, lcontainer, lprefix,
req, marker='', end_marker='', reverse=True):
'''Get "pages" worth of objects that start with a prefix.
The optional keyword arguments ``marker``, ``end_marker``, and
``reverse`` are used similar to how they are for containers. We're
either coming:
- directly from ``_listing_iter``, in which case none of the
optional args are specified, or
- from ``_in_proxy_reverse_listing``, in which case ``reverse``
is ``False`` and both ``marker`` and ``end_marker`` are specified
(although they may still be blank).
'''
while True:
lreq = make_pre_authed_request(
req.environ, method='GET', swift_source='VW',
path=wsgi_quote('/v1/%s/%s' % (account_name, lcontainer)))
lreq.environ['QUERY_STRING'] = \
'prefix=%s&marker=%s' % (wsgi_quote(lprefix),
wsgi_quote(marker))
if end_marker:
lreq.environ['QUERY_STRING'] += '&end_marker=%s' % (
wsgi_quote(end_marker))
if reverse:
lreq.environ['QUERY_STRING'] += '&reverse=on'
lresp = lreq.get_response(self.app)
if not is_success(lresp.status_int):
close_if_possible(lresp.app_iter)
if lresp.status_int == HTTP_NOT_FOUND:
raise ListingIterNotFound()
elif is_client_error(lresp.status_int):
raise HTTPPreconditionFailed(request=req)
else:
raise ListingIterError()
if not lresp.body:
break
sublisting = json.loads(lresp.body)
if not sublisting:
break
# When using the ``reverse`` param, check that the listing is
# actually reversed
first_item = bytes_to_wsgi(sublisting[0]['name'].encode('utf-8'))
last_item = bytes_to_wsgi(sublisting[-1]['name'].encode('utf-8'))
page_is_after_marker = marker and first_item > marker
if reverse and (first_item < last_item or page_is_after_marker):
# Apparently there's at least one pre-2.6.0 container server
yield self._in_proxy_reverse_listing(
account_name, lcontainer, lprefix,
req, marker, sublisting)
return
marker = last_item
yield sublisting
def swift_account(self, env, tenant_id):
path = '/v1/%s/%s?format=json&prefix=account/' \
% (self.aggregate_account, tenant_id)
req = make_pre_authed_request(env, 'GET', path)
req.environ['swift.proxy_access_log_made'] = True
resp = req.get_response(self.app)
if resp.status_int == 404:
return None
return json.loads(resp.body)[0]['name'].split('/')[1]
def get_quota(self, env, version, account):
""" Get quota und currently used storage from account """
request = make_pre_authed_request(env, 'HEAD',
'/%s/%s' % (version, account))
response = request.get_response(self.app)
bytes_used = response.headers.get('x-account-bytes-used', 0)
quota = response.headers.get('x-account-meta-quota-bytes', -1)
return (bytes_used, quota)
def origin_db_delete(self, env, req):
""" Handles DELETEs in the Origin database """
if not self.delete_enabled:
return HTTPMethodNotAllowed(request=req)
try:
vsn, account, container = split_path(req.path, 3, 3)
except ValueError:
return HTTPBadRequest(
'Invalid request. '
'URI format: /<api version>/<account>/<container>')
hsh = self.hash_path(account, container)
cdn_obj_path = self.get_hsh_obj_path(hsh)
# Remove memcache entry
memcache_client = utils.cache_from_env(env)
if memcache_client:
memcache_key = self.cdn_data_memcache_key(cdn_obj_path)
memcache_client.delete(memcache_key)
resp = make_pre_authed_request(
env, 'DELETE', cdn_obj_path,
agent='SwiftOrigin', swift_source='SOS').get_response(self.app)
# A 404 means it's already deleted, which is okay
if resp.status_int // 100 != 2 and resp.status_int != 404:
raise OriginDbFailure(
'Could not DELETE .hash obj in origin '
'db: %s %s' % (cdn_obj_path, resp.status_int))
cdn_list_path = quote('/v1/%s/%s/%s' % (self.origin_account,
account, container))
list_resp = make_pre_authed_request(
env, 'DELETE', cdn_list_path,
agent='SwiftOrigin', swift_source='SOS').get_response(self.app)
if list_resp.status_int // 100 != 2 and list_resp.status_int != 404:
raise OriginDbFailure(
'Could not DELETE listing path in origin '
'db: %s %s' % (cdn_list_path, list_resp.status_int))
# Return 404 if container didn't exist
if resp.status_int == 404 and list_resp.status_int == 404:
return HTTPNotFound(request=req)
return HTTPNoContent(request=req)
def _put_versioned_obj(self, req, put_path_info, source_resp):
# Create a new Request object to PUT to the versions container, copying
# all headers from the source object apart from x-timestamp.
put_req = make_pre_authed_request(
req.environ, path=put_path_info, method='PUT',
swift_source='VW')
copy_header_subset(source_resp, put_req,
lambda k: k.lower() != 'x-timestamp')
put_req.environ['wsgi.input'] = FileLikeIter(source_resp.app_iter)
return put_req.get_response(self.app)
def get_account_info(self, env, account):
path = '/v1/%s' % account
req = make_pre_authed_request(env, 'HEAD', path)
req.environ['swift.proxy_access_log_made'] = True
resp = req.get_response(self.app)
if not resp.status_int // 100 == 2:
return (0, 0, 0)
return (int(resp.headers.get('x-account-container-count', 0)),
int(resp.headers.get('x-account-object-count', 0)),
int(resp.headers.get('x-account-bytes-used', 0)))
def __call__(self, request):
try:
(version, account, container, objname) = split_path(
request.path_info, 1, 4, True)
except ValueError:
return self.app(environ, start_response)
if container and not objname:
if request.method in ('DELETE', 'HEAD'):
return self.app
if request.method == 'POST':
# Deny setting if there are any objects in base container
# Otherwise these objects won't be visible
if request.headers.get('X-Container-Meta-Storage-Path'):
container_info = get_container_info(request.environ, self.app)
objects = container_info.get('object_count')
if objects and int(objects) > 0:
return HTTPBadRequest()
# ACL set
groups = (request.remote_user or '').split(',')
account_name = groups[0].split(':')[0]
read_acl = request.environ.get('HTTP_X_CONTAINER_READ', '')
for target_account in read_acl.split(','):
target_account = target_account.split(':')[0]
target_storage_path = self._get_storage_path(request,
target_account)
if not target_storage_path or account_name == target_account:
continue
container_path = "/%s/%s/%s" % (version, account, container)
headers = {'X-Container-Meta-Storage-Path': container_path}
request_path = "%s/%s%s_%s" % (target_storage_path,
self.prefix,
account_name,
container)
req = make_pre_authed_request(request.environ, 'PUT',
request_path, headers=headers)
req.get_response(self.app)
if container:
container_info = get_container_info(request.environ, self.app)
meta = container_info.get('meta', {})
storage_path = meta.get('storage-path')
if storage_path:
if objname:
storage_path += '/' + objname
request.environ['PATH_INFO'] = storage_path
request.environ['RAW_PATH_INFO'] = storage_path
return self.app
def _post_slomd5_header(self, env, SLOmd5):
headers = {}
headers['X-Object-Meta-SLOmd5'] = SLOmd5
post_req = make_pre_authed_request(env, method='POST',
swift_source='SLOmd5',
path=env['PATH_INFO'],
headers=headers)
post_resp = post_req.get_response(self.app)
if not is_success(post_resp.status_int):
self.logger.info('POST with SLOmd5 header failed: ' +
str(post_resp.body))
def handle_request(self, env, req):
"""
Handles the POST /origin/.prep call for preparing the backing store
Swift cluster for use with the origin subsystem. Can only be called by
.origin_admin
:param req: The swob.Request to process.
:returns: swob.Response, 204 on success
"""
if not self.is_origin_admin(req):
return HTTPForbidden(request=req)
try:
vsn, account = split_path(req.path, 2, 2)
except ValueError:
return HTTPBadRequest(request=req)
if account == '.prep':
path = '/v1/%s' % self.origin_account
resp = make_pre_authed_request(req.environ,
'PUT',
path,
agent='SwiftOrigin',
swift_source='SOS').get_response(
self.app)
if resp.status_int // 100 != 2:
raise Exception(
'Could not create the main origin account: %s %s' %
(path, resp.status))
for i in xrange(self.num_hash_cont):
cont_name = '.hash_%d' % i
path = '/v1/%s/%s' % (self.origin_account, cont_name)
resp = make_pre_authed_request(
req.environ,
'PUT',
path,
agent='SwiftOrigin',
swift_source='SOS').get_response(self.app)
if resp.status_int // 100 != 2:
raise Exception('Could not create %s container: %s %s' %
(cont_name, path, resp.status))
return HTTPNoContent(request=req)
return HTTPNotFound(request=req)
def _put_versioned_obj(self, req, put_path_info, source_resp):
# Create a new Request object to PUT to the versions container, copying
# all headers from the source object apart from x-timestamp.
put_req = make_pre_authed_request(
req.environ, path=quote(put_path_info), method='PUT',
swift_source='VW')
copy_header_subset(source_resp, put_req,
lambda k: k.lower() != 'x-timestamp')
put_req.environ['wsgi.input'] = FileLikeIter(source_resp.app_iter)
put_resp = put_req.get_response(self.app)
close_if_possible(source_resp.app_iter)
return put_resp
def list_containers_iter(self, *args, **kwargs):
""" Returns a list of containers the user has access to """
try:
version, account = self.request.split_path(2, 2)
except ValueError:
pass
path = "/%s/%s?format=json" % (version, account)
for key in ('limit', 'marker', 'end_marker', 'prefix', 'delimiter'):
value = self.request.params.get(key)
if value:
path+= '&%s=%s' % (key, value)
if self.memcache_client is None:
self.memcache_client = cache_from_env(self.request.environ)
memcache_key = 'containerlist%s%s' % (path, str(self.groups))
containers = self.memcache_client.get(memcache_key)
if containers is not None:
return containers
req = make_pre_authed_request(self.request.environ, 'GET', path)
resp = req.get_response(self.app)
tmp_containers = json.loads(resp.body)
# No cached result? -> ratelimit request to prevent abuse
memcache_key_sleep = 'containerlist_sleep/%s' % self.account
last_request_time = self.memcache_client.get(memcache_key_sleep)
if last_request_time and len(tmp_containers) > 0:
last_request = time.time() - last_request_time
if last_request < self.min_sleep:
eventlet.sleep(self.min_sleep - last_request)
containers = []
for container in tmp_containers:
tmp_env = copy.copy(self.request.environ)
container_name = container['name'].encode("utf8")
path_info = "/%s/%s/%s" % (version, account, container_name)
tmp_env['PATH_INFO'] = path_info
container_info = get_container_info(tmp_env, self.app)
acl = (container_info.get('read_acl') or '').split(',')
if (list(set(self.groups) & set(acl))):
containers.append((container['name'],
container['count'],
container['bytes'],
0))
self.memcache_client.set(memcache_key, containers, time=60)
self.memcache_client.set(memcache_key_sleep, time.time())
return containers
def _get_quota(self, account, env):
""" Get quota und currently used storage from account """
request = make_pre_authed_request(env, 'HEAD', '/v1/' + account)
response = request.get_response(self.app)
quota = -1
for (key, value) in response.headers.items():
if key == 'x-account-bytes-used':
bytes_used = int(value)
if key == 'x-account-meta-bytes-limit':
quota = int(value)
return (bytes_used, quota)
def _get_container_info(self, env):
"""
Retrieves x-container-meta-web-index, x-container-meta-web-error,
x-container-meta-web-listings, x-container-meta-web-listings-css,
and x-container-meta-web-directory-type from memcache or from the
cluster and stores the result in memcache and in self._index,
self._error, self._listings, self._listings_css and self._dir_type.
:param env: The WSGI environment dict.
"""
self._index = self._error = self._listings = self._listings_css = \
self._dir_type = None
memcache_client = cache_from_env(env)
if memcache_client:
cached_data = memcache_client.get(
get_memcache_key(self.version, self.account, self.container))
if cached_data:
(self._index, self._error, self._listings, self._listings_css,
self._dir_type) = cached_data
return
else:
cached_data = memcache_client.get(
get_compat_memcache_key(
self.version, self.account, self.container))
if cached_data:
(self._index, self._error, self._listings,
self._listings_css) = cached_data
self._dir_type = ''
return
resp = make_pre_authed_request(
env, 'HEAD', '/%s/%s/%s' % (
self.version, self.account, self.container),
agent=self.agent, swift_source='SW').get_response(self.app)
if is_success(resp.status_int):
self._index = \
resp.headers.get('x-container-meta-web-index', '').strip()
self._error = \
resp.headers.get('x-container-meta-web-error', '').strip()
self._listings = \
resp.headers.get('x-container-meta-web-listings', '').strip()
self._listings_css = \
resp.headers.get('x-container-meta-web-listings-css',
'').strip()
self._dir_type = \
resp.headers.get('x-container-meta-web-directory-type',
'').strip()
if memcache_client:
memcache_client.set(
get_memcache_key(
self.version, self.account, self.container),
(self._index, self._error, self._listings,
self._listings_css, self._dir_type),
time=self.cache_timeout)
def handle_delete(self, req, start_response):
"""
Handle request to delete a user's container.
As part of deleting a container, this middleware will also delete
the hidden container holding object versions.
Before a user's container can be deleted, swift must check
if there are still old object versions from that container.
Only after disabling versioning and deleting *all* object versions
can a container be deleted.
"""
container_info = get_container_info(req.environ,
self.app,
swift_source='OV')
versions_cont = unquote(
container_info.get('sysmeta', {}).get('versions-container', ''))
if versions_cont:
account = req.split_path(3, 3, True)[1]
# using a HEAD request here as opposed to get_container_info
# to make sure we get an up-to-date value
versions_req = make_pre_authed_request(
req.environ,
method='HEAD',
swift_source='OV',
path=wsgi_quote('/v1/%s/%s' %
(account, str_to_wsgi(versions_cont))),
headers={'X-Backend-Allow-Reserved-Names': 'true'})
vresp = versions_req.get_response(self.app)
drain_and_close(vresp)
if vresp.is_success and int(
vresp.headers.get('X-Container-Object-Count', 0)) > 0:
raise HTTPConflict(
'Delete all versions before deleting container.',
request=req)
elif not vresp.is_success and vresp.status_int != 404:
raise HTTPInternalServerError(
'Error deleting versioned container')
else:
versions_req.method = 'DELETE'
resp = versions_req.get_response(self.app)
drain_and_close(resp)
if not is_success(resp.status_int) and resp.status_int != 404:
raise HTTPInternalServerError(
'Error deleting versioned container')
app_resp = self._app_call(req.environ)
start_response(self._response_status, self._response_headers,
self._response_exc_info)
return app_resp
def get_cdn_data(self, env, cdn_obj_path):
"""
Retrieves HashData object from memcache or by doing a GET
of the cdn_obj_path which should be what is returned from
get_hsh_obj_path. Will return None if the HashData is "deleted"
:returns: HashData object or None.
"""
memcache_client = utils.cache_from_env(env)
memcache_key = self.cdn_data_memcache_key(cdn_obj_path)
if memcache_client:
cached_cdn_data = memcache_client.get(memcache_key)
if cached_cdn_data == '404':
return None
if cached_cdn_data:
try:
hash_data = HashData.create_from_json(cached_cdn_data)
if hash_data.deleted:
return None
else:
return hash_data
except ValueError:
pass
resp = make_pre_authed_request(env,
'GET',
cdn_obj_path,
agent='SwiftOrigin',
swift_source='SOS').get_response(
self.app)
if resp.status_int // 100 == 2:
try:
if memcache_client:
memcache_client.set(memcache_key,
resp.body,
serialize=False,
time=MEMCACHE_TIMEOUT)
hash_data = HashData.create_from_json(resp.body)
if not hash_data.deleted:
return hash_data
except ValueError:
self.logger.warn('Invalid HashData json: %s' % cdn_obj_path)
if resp.status_int == 404:
if memcache_client:
# only memcache for 30 secs in case adding container to swift
memcache_client.set(memcache_key,
'404',
serialize=False,
time=CACHE_404)
return None
def update_last_modified_meta(self, req, env):
vrs, account, container, obj = req.split_path(1, 4, True)
path = env['PATH_INFO']
if obj:
path = path.split('/%s' % obj)[0]
metakey = 'X-Container-Meta-%s' % self.key_name
headers = {metakey: str(time.time())}
set_meta_req = wsgi.make_pre_authed_request(env,
method='POST',
path=path,
headers=headers,
swift_source='lm')
set_meta_req.get_response(self.app)
def get_data_from_url(url, app, key, logger, env):
if not url or not key:
return None
req = make_pre_authed_request(env,
method='GET',
path='%s/%s' % (url, quote(key)),
swift_source='liteauth',
agent='Liteauth')
resp = req.get_response(app)
if resp.status_int >= 300:
logger.info('get_data response for %s is %s %s'
% (req.path, resp.status, resp.body))
return None
return resp.body.strip()
def _get_source_object(self, req, path_info):
# make a pre_auth request in case the user has write access
# to container, but not READ. This was allowed in previous version
# (i.e., before middleware) so keeping the same behavior here
get_req = make_pre_authed_request(
req.environ, path=path_info,
headers={'X-Newest': 'True'}, method='GET', swift_source='VW')
source_resp = get_req.get_response(self.app)
if source_resp.content_length is None or \
source_resp.content_length > MAX_FILE_SIZE:
return HTTPRequestEntityTooLarge(request=req)
return source_resp
def handle_get_listing(self, req):
"""Retrieve a new-line separated list of all access keys.
Required headers:
- `x-s3auth-admin-key`: admin key
"""
path = quote(self.akd_container_url)
resp = make_pre_authed_request(req.environ, 'GET',
path).get_response(self.app)
if resp.status_int // 100 == 2:
listing = '\n'.join([e['name'] for e in json.loads(resp.body)])
return HTTPOk(request=req, body=listing)
else:
raise Exception('Could not GET access key listing: {} {}'.format(
path, resp.status_int))
def store_data_in_url(url, app, key, data, env):
if not url or not key:
return False
req = make_pre_authed_request(env,
method='PUT',
path='%s/%s' % (url, quote(key)),
body=str(data),
headers={'content-type': 'text/plain'},
swift_source='liteauth',
agent='Liteauth')
req.environ['liteauth.new_service'] = \
env.get('liteauth.new_service', None)
resp = req.get_response(app)
if resp.status_int >= 300:
return False
return True
def _put_versioned_obj(self, req, put_path_info, source_resp):
# Create a new Request object to PUT to the container, copying
# all headers from the source object apart from x-timestamp.
put_req = make_pre_authed_request(
req.environ, path=wsgi_quote(put_path_info), method='PUT',
swift_source='VW')
copy_header_subset(source_resp, put_req,
lambda k: k.lower() != 'x-timestamp')
slo_size = put_req.headers.get('X-Object-Sysmeta-Slo-Size')
if slo_size:
put_req.headers['Content-Type'] += '; swift_bytes=' + slo_size
put_req.environ['swift.content_type_overridden'] = True
put_req.environ['wsgi.input'] = FileLikeIter(source_resp.app_iter)
put_resp = put_req.get_response(self.app)
close_if_possible(source_resp.app_iter)
return put_resp
def __call__(self, env, start_response):
if env['REQUEST_METHOD'] != 'PUT':
return self.app(env, start_response)
version, account, container, obj = split_path(env['PATH_INFO'], 1, 4,
True)
if not obj:
return self.app(env, start_response)
if not obj.lower().endswith('.jpg') and\
not obj.lower().endswith('.jpeg'):
return self.app(env, start_response)
extractor = MetaExtractor(env['wsgi.input'])
env['wsgi.input'] = extractor
try:
resp = self._app_call(env)
except Exception:
resp = HTTPServerError(request=Request(env), body="error")
return resp(env, start_response)
start_response(self._response_status, self._response_headers,
self._response_exc_info)
status = int(self._response_status.split()[0])
if status < 200 or status > 300:
return resp
headers = {}
for tag in extractor.exif_parser.tags:
if tag.tag().startswith('Unknown'):
continue
if len(tag.value()) > MAX_META_VALUE_LENGTH:
continue
header = 'X-Object-Meta-' + tag.tag().replace(' ', '-')
headers[header] = tag.value()
# TODO: merge with the PUT headers
self.logger.info(headers)
post_req = make_pre_authed_request(env,
method='POST',
swift_source='JpegMeta',
path=env['PATH_INFO'],
headers=headers)
post_resp = post_req.get_response(self.app)
if not is_success(post_resp.status_int):
self.logger.info('POST with JPEG headers failed: ' +
str(post_resp.body))
return resp
def handle_obj_versions_delete_push(self, req, versions_cont, api_version,
account_name, container_name,
object_name):
"""
Handle DELETE requests when in history mode.
Copy current version of object to versions_container and write a
delete marker before proceeding with original request.
:param req: original request.
:param versions_cont: container where previous versions of the object
are stored.
:param api_version: api version.
:param account_name: account name.
:param object_name: name of object of original request
"""
self._copy_current(req, versions_cont, api_version, account_name,
object_name)
marker_path = "/%s/%s/%s/%s" % (
api_version, account_name, versions_cont,
self._build_versions_object_name(object_name, time.time()))
marker_headers = {
# Definitive source of truth is Content-Type, and since we add
# a swift_* param, we know users haven't set it themselves.
# This is still open to users POSTing to update the content-type
# but they're just shooting themselves in the foot then.
'content-type': DELETE_MARKER_CONTENT_TYPE,
'content-length': '0',
'x-auth-token': req.headers.get('x-auth-token')
}
marker_req = make_pre_authed_request(req.environ,
path=wsgi_quote(marker_path),
headers=marker_headers,
method='PUT',
swift_source='VW')
marker_req.environ['swift.content_type_overridden'] = True
marker_resp = marker_req.get_response(self.app)
self._check_response_error(req, marker_resp)
drain_and_close(marker_resp)
# successfully copied and created delete marker; safe to delete
return self.app
def _set_details(self, req, access_key, secret_key, account):
"""Set access key details."""
path = quote(self.akd_container_url + access_key)
resp = make_pre_authed_request(
env=req.environ,
method='PUT',
path=path,
body=json.dumps({'secret_key': secret_key, 'account': account})).\
get_response(self.app)
if resp.status_int // 100 == 2:
# Remove old data from cache.
memcache_client = cache_from_env(req.environ)
if memcache_client:
memcache_key = MEMCACHE_KEY_FORMAT % (self.reseller_prefix,
access_key)
memcache_client.delete(memcache_key)
else:
raise Exception('Could not PUT access key details: {} {}'.format(
path, resp.status_int))
def _get_source_object(self, req, path_info):
# make a pre_auth request in case the user has write access
# to container, but not READ. This was allowed in previous version
# (i.e., before middleware) so keeping the same behavior here
get_req = make_pre_authed_request(req.environ,
path=wsgi_quote(path_info) +
'?symlink=get',
headers={'X-Newest': 'True'},
method='GET',
swift_source='VW')
source_resp = get_req.get_response(self.app)
if source_resp.content_length is None or \
source_resp.content_length > MAX_FILE_SIZE:
# Consciously *don't* drain the response before closing;
# any logged 499 is actually rather appropriate here
close_if_possible(source_resp.app_iter)
return HTTPRequestEntityTooLarge(request=req)
return source_resp
def upload_file_chunks(self, env, chunked_file, container):
error = False
self.app.logger.info(
'StackSync API: upload_file_chunks: container: %s', str(container))
upload_chunks = []
for i in range(len(chunked_file.chunks)):
chunk_name = chunked_file.name_list[i - 1]
chunk_content = chunked_file.chunks[i - 1]
env_aux = env.copy()
new_path = "/v1/" + env[
'stacksync_user_account'] + "/" + container + "/" + chunk_name
del env_aux['HTTP_STACKSYNC_API']
seg_req = make_pre_authed_request(env_aux,
method='PUT',
path=new_path,
body=chunk_content,
agent=str(container))
seg_resp = seg_req.get_response(self.app)
if not is_valid_status(seg_resp.status_int):
self.app.logger.error(
'StackSync API: upload_file_chunks: error uploading chunk %s',
chunk_name)
error = True
break
upload_chunks.append(chunk_name)
if error:
self.app.logger.error(
'StackSync API: upload_file_chunks: status: %s description: Error uploading chunks to storage backend',
seg_resp.status)
response = create_error_response(
500, "Error uploading chunks to storage backend")
self.remove_chunks(env, upload_chunks, container)
else:
response = HTTPCreated()
return response
