def __call__(self, request):
try:
(version, account, container,
objname) = split_path(request.path_info, 4, 4, True)
except ValueError:
return self.app
preview_path = '/%s/%s/%s_%s/%s' % (version, account, container,
self.suffix, objname)
if request.method == 'GET' and request.params.has_key('preview'):
request.path_info = preview_path
if request.method == 'PUT':
preview = create_preview(request.body)
if preview:
sub = wsgi.make_subrequest(request.environ,
path=preview_path,
body=preview)
sub.get_response(self.app)
if request.method == 'DELETE':
sub = wsgi.make_subrequest(request.environ, path=preview_path)
sub.get_response(self.app)
return self.app
def __call__(self, request):
try:
(version, account, container, objname) = split_path(request.path_info, 4, 4, True)
except ValueError:
return self.app
preview_path = '/%s/%s/%s_%s/%s' % (version, account, container, self.suffix, objname)
if request.method == 'GET' and request.params.has_key('preview'):
request.path_info = preview_path
if request.method == 'PUT':
if hasattr(request, 'body_file'):
data = ""
while True:
chunk = request.body_file.read()
if not chunk:
break
data += chunk
request.body = data
preview = create_preview(data)
else:
preview = create_preview(request.body)
if preview:
sub = wsgi.make_subrequest(request.environ, path=preview_path, body=preview)
sub.get_response(self.app)
if request.method == 'DELETE':
sub = wsgi.make_subrequest(request.environ, path=preview_path)
sub.get_response(self.app)
return self.app
def create_container(self, req, container_path):
"""
Checks if the container exists and if not try to create it.
:params container_path: an unquoted path to a container to be created
:returns: True if created container, False if container exists
:raises CreateContainerError: when unable to create container
"""
head_cont_req = make_subrequest(
req.environ, method='HEAD', path=wsgi_quote(container_path),
headers={'X-Auth-Token': req.headers.get('X-Auth-Token')},
swift_source='EA')
resp = head_cont_req.get_response(self.app)
if resp.is_success:
return False
if resp.status_int == HTTP_NOT_FOUND:
create_cont_req = make_subrequest(
req.environ, method='PUT', path=wsgi_quote(container_path),
headers={'X-Auth-Token': req.headers.get('X-Auth-Token')},
swift_source='EA')
resp = create_cont_req.get_response(self.app)
if resp.is_success:
return True
raise CreateContainerError(
"Create Container Failed: " + container_path,
resp.status_int, resp.status)
def create_container(self, req, container_path):
"""
Checks if the container exists and if not try to create it.
:params container_path: an unquoted path to a container to be created
:returns: True if created container, False if container exists
:raises CreateContainerError: when unable to create container
"""
head_cont_req = make_subrequest(
req.environ,
method='HEAD',
path=wsgi_quote(container_path),
headers={'X-Auth-Token': req.headers.get('X-Auth-Token')},
swift_source='EA')
resp = head_cont_req.get_response(self.app)
if resp.is_success:
return False
if resp.status_int == HTTP_NOT_FOUND:
create_cont_req = make_subrequest(
req.environ,
method='PUT',
path=wsgi_quote(container_path),
headers={'X-Auth-Token': req.headers.get('X-Auth-Token')},
swift_source='EA')
resp = create_cont_req.get_response(self.app)
if resp.is_success:
return True
raise CreateContainerError(
"Create Container Failed: " + container_path, resp.status_int,
resp.status)
def verify_access(vertigo, path):
"""
Verifies access to the specified object in swift
:param vertigo: swift_vertigo.vertigo_handler.VertigoProxyHandler instance
:param path: swift path of the object to check
:returns: headers of the object whether exists
"""
vertigo.logger.debug('Vertigo - Verify access to %s' % path)
new_env = dict(vertigo.request.environ)
if 'HTTP_TRANSFER_ENCODING' in new_env.keys():
del new_env['HTTP_TRANSFER_ENCODING']
for key in DEFAULT_MD_STRING.keys():
env_key = 'HTTP_X_VERTIGO_' + key.upper()
if env_key in new_env.keys():
del new_env[env_key]
auth_token = vertigo.request.headers.get('X-Auth-Token')
sub_req = make_subrequest(
new_env, 'HEAD', path,
headers={'X-Auth-Token': auth_token},
swift_source='Vertigo')
return sub_req.get_response(vertigo.app)
def create_link(self, link_path, dest_path, heads):
"""
Creates a link to a actual object
:param link_path: swift path of the link
:param dest_path: swift path of the object to link
:param heads: original object headers
"""
self.logger.debug('Creating a link from %s to %s' %
(link_path, dest_path))
new_env = dict(self.request.environ)
if 'HTTP_TRANSFER_ENCODING' in new_env.keys():
del new_env['HTTP_TRANSFER_ENCODING']
if 'HTTP_X_COPY_FROM' in new_env.keys():
del new_env['HTTP_X_COPY_FROM']
auth_token = self.request.headers.get('X-Auth-Token')
link_path = os.path.join('/', self.api_version,
self.account, link_path)
sub_req = make_subrequest(
new_env, 'PUT', link_path,
headers={'X-Auth-Token': auth_token,
'Content-Length': 0,
'Content-Type': 'link',
'Original-Content-Length': heads["Content-Length"],
'X-Object-Sysmeta-Link-To': dest_path},
swift_source='softlink_middleware')
resp = sub_req.get_response(self.app)
return resp
def create_link(vertigo, link_path, dest_path, heads):
"""
Creates a link to a real object
:param vertigo: swift_vertigo.vertigo_handler.VertigoProxyHandler instance
:param link_path: swift path of the link
:param dest_path: swift path of the object to link
:param headers: original object headers
"""
vertigo.logger.debug('Vertigo - Creating link from %s to %s' % (link_path,
dest_path))
new_env = dict(vertigo.request.environ)
if 'HTTP_TRANSFER_ENCODING' in new_env.keys():
del new_env['HTTP_TRANSFER_ENCODING']
if 'HTTP_X_COPY_FROM' in new_env.keys():
del new_env['HTTP_X_COPY_FROM']
auth_token = vertigo.request.headers.get('X-Auth-Token')
link_path = os.path.join('/', vertigo.api_version,
vertigo.account, link_path)
sub_req = make_subrequest(
new_env, 'PUT', link_path,
headers={'X-Auth-Token': auth_token,
'Content-Length': 0,
'Content-Type': 'vertigo/link',
'Original-Content-Length': heads["Content-Length"],
'X-Object-Sysmeta-Vertigo-Link-to': dest_path},
swift_source='Vertigo')
resp = sub_req.get_response(vertigo.app)
return resp
def _get_object_list(self, path):
"""
Gets an object list of a specified path. The path may be '*', that means
it returns all objects inside the container or a pseudo-folder, that means
it only returns the objects inside the pseudo-folder.
:param path: pseudo-folder path (ended with *), or '*'
:return: list of objects
"""
obj_list = list()
dest_path = os.path.join('/', self.api_version, self.account, self.container)
new_env = dict(self.request.environ)
auth_token = self.request.headers.get('X-Auth-Token')
if path == '*':
# All objects inside a container hierarchy
obj_list.append('')
else:
# All objects inside a pseudo-folder hierarchy
obj_split = self.obj.rsplit('/', 1)
pseudo_folder = obj_split[0] + '/'
new_env['QUERY_STRING'] = 'prefix='+pseudo_folder
sub_req = make_subrequest(new_env, 'GET', dest_path,
headers={'X-Auth-Token': auth_token},
swift_source='Vertigo')
response = sub_req.get_response(self.app)
for obj in response.body.split('\n'):
if obj != '':
obj_list.append(obj)
self._augment_object_list(obj_list)
return obj_list
def _fetch_sub_slo_segments(self, req, version, acc, con, obj):
"""
Fetch the submanifest, parse it, and return it.
Raise exception on failures.
"""
sub_req = make_subrequest(
req.environ,
path="/".join(["", version, acc, con, obj]),
method="GET",
headers={"x-auth-token": req.headers.get("x-auth-token")},
agent=("%(orig)s " + "SLO MultipartGET"),
swift_source="SLO",
)
sub_resp = sub_req.get_response(self.slo.app)
if not is_success(sub_resp.status_int):
close_if_possible(sub_resp.app_iter)
raise ListingIterError(
"ERROR: while fetching %s, GET of submanifest %s "
"failed with status %d" % (req.path, sub_req.path, sub_resp.status_int)
)
try:
with closing_if_possible(sub_resp.app_iter):
return json.loads("".join(sub_resp.app_iter))
except ValueError as err:
raise ListingIterError(
"ERROR: while fetching %s, JSON-decoding of submanifest %s "
"failed with %s" % (req.path, sub_req.path, err)
)
def POST(self):
"""
POST handler on Proxy
Deals with storlet ACL updates
"""
# Get the current container's ACL
# We perform a sub request rather than get_container_info
# since get_container_info bypasses authorization, and we
# prefer to be on the safe side.
target = ["", self.api_version, self.account, self.container]
sub_req = make_subrequest(self.request.environ, "HEAD", "/".join(target), agent=self.agent)
sub_resp = sub_req.get_response(self.app)
if sub_resp.status_int != 204:
self.logger.info("Failed to retreive container metadata")
return HTTPUnauthorized(("Unauthorized to get or modify " "the container ACL"))
# Add the requested ACL
read_acl = sub_resp.headers.get("X-Container-Read", None)
if read_acl:
new_read_acl = ",".join([read_acl, self.acl_string])
else:
new_read_acl = self.acl_string
self.request.headers["X-Container-Read"] = new_read_acl
resp = self.request.get_response(self.app)
self.logger.info("Got post response, %s" % resp.status)
return resp
def _augment_object_list(self, obj_list):
"""
Checks the object list and creates those pseudo-folders that are not in
the obj_list, but there are objects within them.
:param obj_list: object list
"""
for obj in obj_list:
if '/' in obj:
obj_split = obj.rsplit('/', 1)
pseudo_folder = obj_split[0] + '/'
if pseudo_folder not in obj_list:
path = os.path.join('/', self.api_version, self.account,
self.container, pseudo_folder)
new_env = dict(self.request.environ)
auth_token = self.request.headers.get('X-Auth-Token')
sub_req = make_subrequest(new_env,
'PUT',
path,
headers={
'X-Auth-Token': auth_token,
'Content-Length': 0
},
swift_source='Vertigo')
response = sub_req.get_response(self.app)
if response.is_success:
obj_list.append(pseudo_folder)
else:
raise ValueError(
"Vertigo - Error creating pseudo-folder")
def do_delete(obj_name, delete_path):
delete_obj_req = make_subrequest(
req.environ, method='DELETE', path=quote(delete_path),
headers={'X-Auth-Token': req.headers.get('X-Auth-Token')},
body='', agent='%(orig)s ' + user_agent,
swift_source=swift_source)
return (delete_obj_req.get_response(self.app), obj_name, 0)
def gather_extra_sources(self):
# (kota_): I know this is a crazy hack to set the resp
# dinamically so that this is a temprorary way to make sure
# the capability, this aboslutely needs cleanup more genelic
if 'X-Storlet-Extra-Resources' in self.request.headers:
try:
resources = list_from_csv(
self.request.headers['X-Storlet-Extra-Resources'])
# resourece should be /container/object
for resource in resources:
# sanity check, if it's invalid path ValueError
# will be raisen
swift_path = ['', self.api_version, self.account]
swift_path.extend(split_path(resource, 2, 2, True))
sub_req = make_subrequest(
self.request.environ,
'GET', '/'.join(swift_path),
agent=self.agent)
sub_resp = sub_req.get_response(self.app)
# TODO(kota_): make this in another green thread
# expicially, in parallel with primary GET
self.extra_sources.append(
self._build_storlet_request(
self.request, sub_resp.headers,
sub_resp.app_iter))
except ValueError:
raise HTTPBadRequest(
'X-Storlet-Extra-Resource must be a csv with'
'/container/object format')
def POST(self):
"""
POST handler on Proxy
Deals with storlet ACL updates
"""
# Get the current container's ACL
# We perform a sub request rather than get_container_info
# since get_container_info bypasses authorization, and we
# prefer to be on the safe side.
sub_req = make_subrequest(self.request.environ,
'HEAD', self.path,
agent=self.agent)
sub_resp = sub_req.get_response(self.app)
if sub_resp.status_int != 204:
self.logger.info("Failed to retrieve container metadata")
return HTTPUnauthorized(('Unauthorized to get or modify '
'the container ACL'))
# Add the requested ACL
read_acl = sub_resp.headers.get("X-Container-Read")
if read_acl:
new_read_acl = ','.join([read_acl, self.acl_string])
else:
new_read_acl = self.acl_string
self.request.headers['X-Container-Read'] = new_read_acl
resp = self.request.get_response(self.app)
return resp
def verify_access_to_storlet(self):
"""
Verify access to the storlet object
:return: storlet parameters
:raises HTTPUnauthorized: If it fails to verify access
"""
sobj = self.request.headers.get("X-Run-Storlet")
spath = "/".join(["", self.api_version, self.account, self.storlet_container, sobj])
self.logger.debug("Verify access to %s" % spath)
new_env = dict(self.request.environ)
if "HTTP_TRANSFER_ENCODING" in new_env.keys():
del new_env["HTTP_TRANSFER_ENCODING"]
for key in CONDITIONAL_KEYS:
env_key = "HTTP_" + key
if env_key in new_env.keys():
del new_env[env_key]
auth_token = self.request.headers.get("X-Auth-Token")
storlet_req = make_subrequest(
new_env, "HEAD", spath, headers={"X-Auth-Token": auth_token}, swift_source=self.agent
)
resp = storlet_req.get_response(self.app)
if not resp.is_success:
raise HTTPUnauthorized("Failed to verify access to the storlet", request=self.request)
params = self._parse_storlet_params(resp.headers)
for key in ["Content-Length", "X-Timestamp"]:
params[key] = resp.headers[key]
return params
def _update_local_cache_from_swift(self):
"""
Updates the local cache of functions.
"""
self.logger.info('Function - Updating local cache from swift')
if self.disaggregated_compute:
new_env = dict(self.req.environ)
swift_path = os.path.join('/', 'v1', self.account,
self.functions_container,
self.function_obj_name)
sub_req = make_subrequest(new_env,
'GET',
swift_path,
swift_source='function_middleware')
resp = sub_req.get_response(self.app)
else:
resp = make_swift_request('GET', self.account,
self.functions_container,
self.function_obj_name)
if resp.status_int != 200:
self.logger.info(
'Function - It is not possible to update the local cache')
raise FileNotFoundError
with open(self.cached_function_obj, 'wb') as fn:
fn.write(resp.body)
self.logger.info('Function - Local cache updated: ' +
self.cached_function_obj)
self.function_metadata = resp.headers
set_object_metadata(self.cached_function_obj, resp.headers)
def get_auth_token(self, env):
if (_WMFRewriteContext.auth_token and
_WMFRewriteContext.auth_token_expiry > monotonic.monotonic()):
self.logger.debug(
'Reusing existing auth token for thumbnail expiry update')
return _WMFRewriteContext.auth_token
headers = {
'X-Auth-User': self.thumbnail_user,
'X-Auth-Key': self.thumbnail_key,
}
sub = make_subrequest(env, path='/auth/v1.0', headers=headers)
resp = sub.get_response(self.app)
if resp.status_int != 200:
self.logger.error(
'Could not acquire auth token while updating thumbnail expiry')
return False
_WMFRewriteContext.auth_token = resp.headers['X-Auth-Token']
expiry = int(resp.headers['X-Auth-Token-Expires'])
_WMFRewriteContext.auth_token_expiry = monotonic.monotonic() + expiry
return _WMFRewriteContext.auth_token
def PUT(self, req):
"""HTTP PUT request handler."""
container_info = self.container_info(
self.account_name, self.container_name, req)
req.acl = container_info['write_acl']
req.environ['swift_sync_key'] = container_info['sync_key']
# is request authorized
if 'swift.authorize' in req.environ:
aresp = req.environ['swift.authorize'](req)
if aresp:
return aresp
old_slo_manifest = None
# If versioning is disabled, we must check if the object exists.
# If it's a SLO, we will have to delete the parts if the current
# operation is a success.
if (self.app.delete_slo_parts and
not container_info['sysmeta'].get('versions-location', None)):
try:
dest_info = get_object_info(req.environ, self.app)
if 'slo-size' in dest_info['sysmeta']:
manifest_env = req.environ.copy()
manifest_env['QUERY_STRING'] = 'multipart-manifest=get'
manifest_req = make_subrequest(manifest_env, 'GET')
manifest_resp = manifest_req.get_response(self.app)
old_slo_manifest = json.loads(manifest_resp.body)
except Exception as exc:
self.app.logger.warn(('Failed to check existence of %s. If '
'overwriting a SLO, old parts may '
'remain. Error was: %s') %
(req.path, exc))
self._update_content_type(req)
self._update_x_timestamp(req)
# check constraints on object name and request headers
error_response = check_object_creation(req, self.object_name) or \
check_content_type(req)
if error_response:
return error_response
if req.headers.get('Oio-Copy-From'):
return self._link_object(req)
data_source = req.environ['wsgi.input']
if req.content_length:
data_source = ExpectedSizeReader(data_source, req.content_length)
headers = self._prepare_headers(req)
with closing_if_possible(data_source):
resp = self._store_object(req, data_source, headers)
if old_slo_manifest and resp.is_success:
self.app.logger.debug(
'Previous object %s was a SLO, deleting parts',
req.path)
self._delete_slo_parts(req, old_slo_manifest)
return resp
def _fetch_sub_slo_segments(self, req, version, acc, con, obj):
"""
Fetch the submanifest, parse it, and return it.
Raise exception on failures.
"""
sub_req = make_subrequest(
req.environ, path='/'.join(['', version, acc, con, obj]),
method='GET',
headers={'x-auth-token': req.headers.get('x-auth-token')},
agent=('%(orig)s ' + 'SLO MultipartGET'), swift_source='SLO')
sub_resp = sub_req.get_response(self.slo.app)
if not is_success(sub_resp.status_int):
close_if_possible(sub_resp.app_iter)
raise ListingIterError(
'ERROR: while fetching %s, GET of submanifest %s '
'failed with status %d' % (req.path, sub_req.path,
sub_resp.status_int))
try:
with closing_if_possible(sub_resp.app_iter):
return json.loads(''.join(sub_resp.app_iter))
except ValueError as err:
raise ListingIterError(
'ERROR: while fetching %s, JSON-decoding of submanifest %s '
'failed with %s' % (req.path, sub_req.path, err))
def _verify_access(self, cont, obj):
"""
Verifies access to the specified object in swift
:param cont: swift container name
:param obj: swift object name
:raise HTTPNotFound: if the object doesn't exists in swift
:return response: Object response
"""
if obj:
path = os.path.join('/', self.api_version, self.account, cont, obj)
else:
path = os.path.join('/', self.api_version, self.account, cont)
self.logger.debug('Verifying access to %s' % path)
new_env = dict(self.req.environ)
if 'HTTP_TRANSFER_ENCODING' in new_env.keys():
del new_env['HTTP_TRANSFER_ENCODING']
auth_token = self.req.headers.get('X-Auth-Token')
sub_req = make_subrequest(new_env,
'HEAD',
path,
headers={'X-Auth-Token': auth_token},
swift_source='function_middleware')
resp = sub_req.get_response(self.app)
if not resp.is_success:
if resp.status_int == 401:
raise HTTPUnauthorized('Unauthorized to access to this '
'resource: ' + path + '\n')
else:
raise HTTPNotFound('There was an error: "' + path +
' doesn\'t exists in Swift.\n')
def _get_container_listing(self, req, version, account, container,
prefix, marker=''):
'''
:param version: whatever
:param account: native
:param container: native
:param prefix: native
:param marker: native
'''
con_req = make_subrequest(
req.environ,
path=wsgi_quote('/'.join([
'', str_to_wsgi(version),
str_to_wsgi(account), str_to_wsgi(container)])),
method='GET',
headers={'x-auth-token': req.headers.get('x-auth-token')},
agent=('%(orig)s ' + 'DLO MultipartGET'), swift_source='DLO')
con_req.query_string = 'prefix=%s' % quote(prefix)
if marker:
con_req.query_string += '&marker=%s' % quote(marker)
con_resp = con_req.get_response(self.dlo.app)
if not is_success(con_resp.status_int):
if req.method == 'HEAD':
con_resp.body = b''
return con_resp, None
with closing_if_possible(con_resp.app_iter):
return None, json.loads(b''.join(con_resp.app_iter))
def _get_container_listing(self, req, version, account, container,
prefix, marker=''):
'''
:param version: whatever
:param account: native
:param container: native
:param prefix: native
:param marker: native
'''
con_req = make_subrequest(
req.environ,
path=wsgi_quote('/'.join([
'', str_to_wsgi(version),
str_to_wsgi(account), str_to_wsgi(container)])),
method='GET',
headers={'x-auth-token': req.headers.get('x-auth-token')},
agent=('%(orig)s ' + 'DLO MultipartGET'), swift_source='DLO')
con_req.query_string = 'prefix=%s' % quote(prefix)
if marker:
con_req.query_string += '&marker=%s' % quote(marker)
con_resp = con_req.get_response(self.dlo.app)
if not is_success(con_resp.status_int):
if req.method == 'HEAD':
con_resp.body = b''
return con_resp, None
with closing_if_possible(con_resp.app_iter):
return None, json.loads(b''.join(con_resp.app_iter))
def create_link(vertigo, link_path, dest_path, heads):
"""
Creates a link to a real object
:param vertigo: swift_vertigo.vertigo_handler.VertigoProxyHandler instance
:param link_path: swift path of the link
:param dest_path: swift path of the object to link
:param headers: original object headers
"""
vertigo.logger.debug('Vertigo - Creating link from %s to %s' % (link_path,
dest_path))
new_env = dict(vertigo.request.environ)
if 'HTTP_TRANSFER_ENCODING' in new_env.keys():
del new_env['HTTP_TRANSFER_ENCODING']
if 'HTTP_X_COPY_FROM' in new_env.keys():
del new_env['HTTP_X_COPY_FROM']
auth_token = vertigo.request.headers.get('X-Auth-Token')
link_path = os.path.join('/', vertigo.api_version,
vertigo.account, link_path)
sub_req = make_subrequest(
new_env, 'PUT', link_path,
headers={'X-Auth-Token': auth_token,
'Content-Length': 0,
'Content-Type': 'vertigo/link',
'Original-Content-Length': heads["Content-Length"],
'X-Object-Sysmeta-Vertigo-Link-to': dest_path},
swift_source='Vertigo')
resp = sub_req.get_response(vertigo.app)
return resp
def verify_access(vertigo, path):
"""
Verifies access to the specified object in swift
:param vertigo: swift_vertigo.vertigo_handler.VertigoProxyHandler instance
:param path: swift path of the object to check
:returns: headers of the object whether exists
"""
vertigo.logger.debug('Vertigo - Verify access to %s' % path)
new_env = dict(vertigo.request.environ)
if 'HTTP_TRANSFER_ENCODING' in new_env.keys():
del new_env['HTTP_TRANSFER_ENCODING']
for key in DEFAULT_MD_STRING.keys():
env_key = 'HTTP_X_VERTIGO_' + key.upper()
if env_key in new_env.keys():
del new_env[env_key]
auth_token = vertigo.request.headers.get('X-Auth-Token')
sub_req = make_subrequest(
new_env, 'HEAD', path,
headers={'X-Auth-Token': auth_token},
swift_source='Vertigo')
return sub_req.get_response(vertigo.app)
def _fetch_sub_slo_segments(self, req, version, acc, con, obj):
"""
Fetch the submanifest, parse it, and return it.
Raise exception on failures.
"""
sub_req = make_subrequest(
req.environ,
path='/'.join(['', version, acc, con, obj]),
method='GET',
headers={'x-auth-token': req.headers.get('x-auth-token')},
agent='%(orig)s SLO MultipartGET',
swift_source='SLO')
sub_resp = sub_req.get_response(self.slo.app)
if not is_success(sub_resp.status_int):
close_if_possible(sub_resp.app_iter)
raise ListingIterError(
'ERROR: while fetching %s, GET of submanifest %s '
'failed with status %d' %
(req.path, sub_req.path, sub_resp.status_int))
try:
with closing_if_possible(sub_resp.app_iter):
return json.loads(''.join(sub_resp.app_iter))
except ValueError as err:
raise ListingIterError(
'ERROR: while fetching %s, JSON-decoding of submanifest %s '
'failed with %s' % (req.path, sub_req.path, err))
def _check_conditions(self, filter_metadata):
"""
This method ckecks the object_tag, object_type and object_size parameters
introduced by the dashborad to run the filter.
"""
if not filter_metadata['object_type'] and \
not filter_metadata['object_tag'] and \
not filter_metadata['object_size']:
return True
metadata = {}
if self.method == 'put':
for key in self.request.headers.keys():
metadata[key.lower()] = self.request.headers.get(key)
else:
sub_req = make_subrequest(self.request.environ,
method='HEAD',
path=self.request.path_info,
headers=self.request.headers,
swift_source='Crystal Filter Middleware')
resp = sub_req.get_response(self.app)
metadata = resp.headers
correct_type = True
correct_size = True
correct_tags = True
try:
if filter_metadata['object_type']:
object_name = filter_metadata['object_name']
filename = self.request.environ['PATH_INFO']
pattern = re.compile(object_name)
if not pattern.search(filename):
correct_type = False
if filter_metadata['object_tag']:
tags = filter_metadata['object_tag'].split(',')
tag_checking = list()
for tag in tags:
key, value = tag.split(':')
meta_key = ('X-Object-Meta-' + key).lower()
sysmeta_key = ('X-Object-Sysmeta-Meta-' + key).lower()
correct_tag = (meta_key in metadata and
metadata[meta_key] == value) or \
(sysmeta_key in metadata and
metadata[sysmeta_key] == value)
tag_checking.append(correct_tag)
correct_tags = all(tag_checking)
if filter_metadata['object_size']:
object_size = filter_metadata['object_size']
op = mappings[object_size[0]]
obj_lenght = int(object_size[1])
correct_size = op(int(metadata['Content-Length']), obj_lenght)
except Exception as e:
self.logger.error(str(e))
return False
return correct_type and correct_size and correct_tags
def _check_conditions(self, filter_metadata):
"""
This method ckecks the object_tag, object_type and object_size parameters
introduced by the dashborad to run the filter.
"""
if not filter_metadata['object_type'] and \
not filter_metadata['object_tag'] and \
not filter_metadata['object_size']:
return True
metadata = {}
if self.method == 'put':
for key in self.request.headers.keys():
metadata[key.lower()] = self.request.headers.get(key)
else:
sub_req = make_subrequest(self.request.environ, method='HEAD',
path=self.request.path_info,
headers=self.request.headers,
swift_source='Crystal Filter Middleware')
resp = sub_req.get_response(self.app)
metadata = resp.headers
correct_type = True
correct_size = True
correct_tags = True
try:
if filter_metadata['object_type']:
object_name = filter_metadata['object_name']
filename = self.request.environ['PATH_INFO']
pattern = re.compile(object_name)
if not pattern.search(filename):
correct_type = False
if filter_metadata['object_tag']:
tags = filter_metadata['object_tag'].split(',')
tag_checking = list()
for tag in tags:
key, value = tag.split(':')
meta_key = ('X-Object-Meta-'+key).lower()
sysmeta_key = ('X-Object-Sysmeta-Meta-'+key).lower()
correct_tag = (meta_key in metadata and
metadata[meta_key] == value) or \
(sysmeta_key in metadata and
metadata[sysmeta_key] == value)
tag_checking.append(correct_tag)
correct_tags = all(tag_checking)
if filter_metadata['object_size']:
object_size = filter_metadata['object_size']
op = mappings[object_size[0]]
obj_lenght = int(object_size[1])
correct_size = op(int(metadata['Content-Length']),
obj_lenght)
except Exception as e:
self.logger.error(str(e))
return False
return correct_type and correct_size and correct_tags
def get_container_metadata(vertigo, container):
new_env = dict(vertigo.request.environ)
auth_token = vertigo.request.headers.get('X-Auth-Token')
sub_req = make_subrequest(new_env, 'HEAD', container,
headers={'X-Auth-Token': auth_token},
swift_source='Vertigo')
response = sub_req.get_response(vertigo.app)
return response.headers
def get_container_metadata(vertigo, container):
new_env = dict(vertigo.request.environ)
auth_token = vertigo.request.headers.get('X-Auth-Token')
sub_req = make_subrequest(new_env, 'HEAD', container,
headers={'X-Auth-Token': auth_token},
swift_source='Vertigo')
response = sub_req.get_response(vertigo.app)
return response.headers
def _list_objects(self,
env,
account,
ct_parts,
header_cb,
prefix='',
recursive=True,
limit=10000):
"""
If `recursive` is set (the default), for each subdirectory marker
encountered, make a listing subrequest, and yield object list.
If `recursive` is False, list objects and directory markers (but
do not recurse).
"""
sub_path = quote_plus(
self.DELIMITER.join(
('', 'v1', account, self.ENCODED_DELIMITER.join(ct_parts))))
LOG.debug("%s: listing objects from '%s'", self.SWIFT_SOURCE, sub_path)
sub_req = make_subrequest(env.copy(),
method='GET',
path=sub_path,
body='',
swift_source=self.SWIFT_SOURCE)
params = sub_req.params
params['delimiter'] = self.DELIMITER
params['limit'] = str(limit) # FIXME: why is it str?
params['prefix'] = prefix
params['format'] = 'json'
sub_req.params = params
resp = sub_req.get_response(self.app)
obj_prefix = ''
if len(ct_parts) > 1:
obj_prefix = self.DELIMITER.join(ct_parts[1:] + ('', ))
if not resp.is_success or resp.content_length == 0:
LOG.warn("Failed to recursively list '%s': %s", obj_prefix,
resp.status)
return
with closing_if_possible(resp.app_iter):
items = json.loads(resp.body)
if header_cb:
header_cb(resp.headers)
subdirs = [x['subdir'][:-1] for x in items if 'subdir' in x]
for obj in items:
if 'name' in obj:
obj['name'] = obj_prefix + obj['name']
yield obj
elif not recursive and 'subdir' in obj:
obj['subdir'] = obj_prefix + obj['subdir']
yield obj
if recursive:
for subdir in subdirs:
for obj in self._list_objects(env, account,
ct_parts + (subdir, ),
header_cb):
yield obj
def handle_slo_get_or_head(self, req, start_response):
"""
Takes a request and a start_response callable and does the normal WSGI
thing with them. Returns an iterator suitable for sending up the WSGI
chain.
:param req: swob.Request object; is a GET or HEAD request aimed at
what may be a static large object manifest (or may not).
:param start_response: WSGI start_response callable
"""
resp_iter = self._app_call(req.environ)
# make sure this response is for a static large object manifest
for header, value in self._response_headers:
if (header.lower() == 'x-static-large-object' and
config_true_value(value)):
break
else:
# Not a static large object manifest. Just pass it through.
start_response(self._response_status,
self._response_headers,
self._response_exc_info)
return resp_iter
# Handle pass-through request for the manifest itself
if req.params.get('multipart-manifest') == 'get':
new_headers = []
for header, value in self._response_headers:
if header.lower() == 'content-type':
new_headers.append(('Content-Type',
'application/json; charset=utf-8'))
else:
new_headers.append((header, value))
self._response_headers = new_headers
start_response(self._response_status,
self._response_headers,
self._response_exc_info)
return resp_iter
if self._need_to_refetch_manifest(req):
req.environ['swift.non_client_disconnect'] = True
close_if_possible(resp_iter)
del req.environ['swift.non_client_disconnect']
get_req = make_subrequest(
req.environ, method='GET',
headers={'x-auth-token': req.headers.get('x-auth-token')},
agent=('%(orig)s ' + 'SLO MultipartGET'), swift_source='SLO')
resp_iter = self._app_call(get_req.environ)
# Any Content-Range from a manifest is almost certainly wrong for the
# full large object.
resp_headers = [(h, v) for h, v in self._response_headers
if not h.lower() == 'content-range']
response = self.get_or_head_response(
req, resp_headers, resp_iter)
return response(req.environ, start_response)
def make_encrypted_subrequest(env, method=None, path=None, body=None,
headers=None, agent='Swift', swift_source=None,
make_env=make_encrypted_env):
"""
Same as :py:func:`make_subrequest` but with encryption env, if available.
"""
return wsgi.make_subrequest(
env, method=method, path=path, body=body, headers=headers, agent=agent,
swift_source=swift_source, make_env=make_env)
def handle_slo_get_or_head(self, req, start_response):
"""
Takes a request and a start_response callable and does the normal WSGI
thing with them. Returns an iterator suitable for sending up the WSGI
chain.
:param req: swob.Request object; is a GET or HEAD request aimed at
what may be a static large object manifest (or may not).
:param start_response: WSGI start_response callable
"""
resp_iter = self._app_call(req.environ)
# make sure this response is for a static large object manifest
for header, value in self._response_headers:
if (header.lower() == 'x-static-large-object'
and config_true_value(value)):
break
else:
# Not a static large object manifest. Just pass it through.
start_response(self._response_status, self._response_headers,
self._response_exc_info)
return resp_iter
# Handle pass-through request for the manifest itself
if req.params.get('multipart-manifest') == 'get':
new_headers = []
for header, value in self._response_headers:
if header.lower() == 'content-type':
new_headers.append(
('Content-Type', 'application/json; charset=utf-8'))
else:
new_headers.append((header, value))
self._response_headers = new_headers
start_response(self._response_status, self._response_headers,
self._response_exc_info)
return resp_iter
if self._need_to_refetch_manifest(req):
req.environ['swift.non_client_disconnect'] = True
close_if_possible(resp_iter)
del req.environ['swift.non_client_disconnect']
get_req = make_subrequest(
req.environ,
method='GET',
headers={'x-auth-token': req.headers.get('x-auth-token')},
agent=('%(orig)s ' + 'SLO MultipartGET'),
swift_source='SLO')
resp_iter = self._app_call(get_req.environ)
# Any Content-Range from a manifest is almost certainly wrong for the
# full large object.
resp_headers = [(h, v) for h, v in self._response_headers
if not h.lower() == 'content-range']
response = self.get_or_head_response(req, resp_headers, resp_iter)
return response(req.environ, start_response)
def _coalesce_requests(self):
start_time = time.time()
pending_req = None
pending_etag = None
pending_size = None
try:
for seg_path, seg_etag, seg_size, first_byte, last_byte \
in self.listing_iter:
first_byte = first_byte or 0
go_to_end = last_byte is None or (
seg_size is not None and last_byte == seg_size - 1)
if time.time() - start_time > self.max_get_time:
raise SegmentError(
'ERROR: While processing manifest %s, '
'max LO GET time of %ds exceeded' %
(self.name, self.max_get_time))
# Make sure that the segment is a plain old object, not some
# flavor of large object, so that we can check its MD5.
path = seg_path + '?multipart-manifest=get'
seg_req = make_subrequest(
self.req.environ, path=path, method='GET',
headers={'x-auth-token': self.req.headers.get(
'x-auth-token')},
agent=('%(orig)s ' + self.ua_suffix),
swift_source=self.swift_source)
if first_byte != 0 or not go_to_end:
seg_req.headers['Range'] = "bytes=%s-%s" % (
first_byte, '' if go_to_end else last_byte)
# We can only coalesce if paths match and we know the segment
# size (so we can check that the ranges will be allowed)
if pending_req and pending_req.path == seg_req.path and \
seg_size is not None:
new_range = '%s,%s' % (
pending_req.headers.get('Range',
'bytes=0-%s' % (seg_size - 1)),
seg_req.headers['Range'].split('bytes=')[1])
if Range(new_range).ranges_for_length(seg_size):
# Good news! We can coalesce the requests
pending_req.headers['Range'] = new_range
continue
# else, Too many ranges, or too much backtracking, or ...
if pending_req:
yield pending_req, pending_etag, pending_size
pending_req = seg_req
pending_etag = seg_etag
pending_size = seg_size
finally:
if time.time() - start_time > self.max_get_time:
raise SegmentError(
'ERROR: While processing manifest %s, '
'max LO GET time of %ds exceeded' %
(self.name, self.max_get_time))
if pending_req:
yield pending_req, pending_etag, pending_size
def do_head(obj_name):
obj_path = '/'.join(['', vrs, account,
get_valid_utf8_str(obj_name).lstrip('/')])
sub_req = make_subrequest(
req.environ, path=obj_path + '?', # kill the query string
method='HEAD',
headers={'x-auth-token': req.headers.get('x-auth-token')},
agent='%(orig)s SLO MultipartPUT', swift_source='SLO')
return obj_name, sub_req.get_response(self)
def do_delete(obj_name, delete_path):
delete_obj_req = make_subrequest(
req.environ,
method='DELETE',
path=wsgi_quote(str_to_wsgi(delete_path)),
headers={'X-Auth-Token': req.headers.get('X-Auth-Token')},
body='',
agent='%(orig)s ' + user_agent,
swift_source=swift_source)
return (delete_obj_req.get_response(self.app), obj_name, 0)
def _delete_slo_parts(self, req, manifest):
"""Delete parts of an obsolete SLO."""
# We cannot use bulk-delete here,
# because we are at the end of the pipeline, after 'bulk'.
for part in manifest:
path = '/'.join(('', 'v1', self.account_name)) + part['name']
try:
del_req = make_subrequest(req.environ, 'DELETE', path=path)
del_req.get_response(self.app)
except Exception as exc:
self.app.logger.warn('Failed to delete SLO part %s: %s',
path, exc)
def _delete_slo_parts(self, req, manifest):
"""Delete parts of an obsolete SLO."""
# We cannot use bulk-delete here,
# because we are at the end of the pipeline, after 'bulk'.
for part in manifest:
path = '/'.join(('', 'v1', self.account_name)) + part['name']
try:
del_req = make_subrequest(req.environ, 'DELETE', path=path)
del_req.get_response(self.app)
except Exception as exc:
self.app.logger.warn('Failed to delete SLO part %s: %s', path,
exc)
def _validate_etag_and_update_sysmeta(self, req, symlink_target_path,
etag):
if req.environ.get('swift.symlink_override'):
req.headers[TGT_ETAG_SYSMETA_SYMLINK_HDR] = etag
req.headers[TGT_BYTES_SYSMETA_SYMLINK_HDR] = \
req.headers[TGT_BYTES_SYMLINK_HDR]
return
# next we'll make sure the E-Tag matches a real object
new_req = make_subrequest(req.environ,
path=wsgi_quote(symlink_target_path),
method='HEAD',
swift_source='SYM')
if req.allow_reserved_names:
new_req.headers['X-Backend-Allow-Reserved-Names'] = 'true'
self._last_target_path = symlink_target_path
resp = self._recursive_get_head(new_req,
target_etag=etag,
follow_softlinks=False)
if self._get_status_int() == HTTP_NOT_FOUND:
raise HTTPConflict(body='X-Symlink-Target does not exist',
request=req,
headers={
'Content-Type': 'text/plain',
'Content-Location': self._last_target_path
})
if not is_success(self._get_status_int()):
drain_and_close(resp)
raise status_map[self._get_status_int()](request=req)
response_headers = HeaderKeyDict(self._response_headers)
# carry forward any etag update params (e.g. "slo_etag"), we'll append
# symlink_target_* params to this header after this method returns
override_header = get_container_update_override_key('etag')
if override_header in response_headers and \
override_header not in req.headers:
sep, params = response_headers[override_header].partition(';')[1:]
req.headers[override_header] = MD5_OF_EMPTY_STRING + sep + params
# It's troublesome that there's so much leakage with SLO
if 'X-Object-Sysmeta-Slo-Etag' in response_headers and \
override_header not in req.headers:
req.headers[override_header] = '%s; slo_etag=%s' % (
MD5_OF_EMPTY_STRING,
response_headers['X-Object-Sysmeta-Slo-Etag'])
req.headers[TGT_BYTES_SYSMETA_SYMLINK_HDR] = (
response_headers.get('x-object-sysmeta-slo-size')
or response_headers['Content-Length'])
req.headers[TGT_ETAG_SYSMETA_SYMLINK_HDR] = etag
if not req.headers.get('Content-Type'):
req.headers['Content-Type'] = response_headers['Content-Type']
def _get_linked_object(self, dest_obj):
"""
Makes a subrequest to the provided container/object
:param dest_obj: container/object
:return: swift.common.swob.Response Instance
"""
dest_path = os.path.join('/', self.api_version, self.account, dest_obj)
new_env = dict(self.request.environ)
sub_req = make_subrequest(new_env, 'GET', dest_path,
headers=self.request.headers,
swift_source='Vertigo')
return sub_req.get_response(self.app)
def do_head(obj_name):
obj_path = '/'.join(
['', vrs, account,
get_valid_utf8_str(obj_name).lstrip('/')])
sub_req = make_subrequest(
req.environ,
path=obj_path + '?', # kill the query string
method='HEAD',
headers={'x-auth-token': req.headers.get('x-auth-token')},
agent='%(orig)s SLO MultipartPUT',
swift_source='SLO')
return obj_name, sub_req.get_response(self)
def __call__(self, request):
try:
(version, account, container, objname) = split_path(request.path_info, 4, 4, True)
except ValueError:
return self.app
preview_path = '/%s/%s/%s_%s/%s' % (version, account, container, self.suffix, objname)
if request.method == 'GET' and request.params.has_key('preview'):
request.path_info = preview_path
if request.method == 'PUT':
preview = create_preview(request.body)
if preview:
sub = wsgi.make_subrequest(request.environ, path=preview_path, body=preview)
sub.get_response(self.app)
if request.method == 'DELETE':
sub = wsgi.make_subrequest(request.environ, path=preview_path)
sub.get_response(self.app)
return self.app
def set_container_metadata(vertigo, metadata):
"""
Sets the swift metadata to the container
:param metadata: metadata dictionary
"""
container = os.path.join('/', vertigo.api_version, vertigo.account, vertigo.container)
new_env = dict(vertigo.request.environ)
auth_token = vertigo.request.headers.get('X-Auth-Token')
metadata.update({'X-Auth-Token': auth_token})
sub_req = make_subrequest(new_env, 'POST', container,
headers=metadata,
swift_source='Vertigo')
sub_req.get_response(vertigo.app)
def get_linked_object(self, dest_obj):
"""
Makes a subrequest to the provided container/object
:param dest_obj: container/object
:return: swift.common.swob.Response Instance
"""
dest_path = os.path.join('/', self.api_version, self.account,
dest_obj)
new_env = dict(self.req.environ)
sub_req = make_subrequest(new_env, 'GET', dest_path,
headers=self.req.headers,
swift_source='softlink_middleware')
return sub_req.get_response(self.app)
def __call__(self, req):
if req.method == 'DELETE':
try:
self._parse_vaco(req)
except:
# No object request
return req.get_response(self.app)
if self.obj and self.container != RECYCLE_BIN_CONTAINER:
new_env = req.environ.copy()
auth_token = req.headers.get('X-Auth-Token')
new_container = os.path.join('/', self.api, self.account, RECYCLE_BIN_CONTAINER)
new_path = os.path.join(RECYCLE_BIN_CONTAINER, self.container, self.obj)
sub_req = make_subrequest(new_env, 'PUT', new_container,
# headers={'X-Auth-Token': auth_token},
swift_source='recyclebin_filter')
resp = sub_req.get_response(self.app)
if 'max_time' in self.parameters:
max_time = self.parameters['max_time']
else:
max_time = DEFAULT_MAX_TIME
sub_req = make_subrequest(new_env, 'COPY', req.path,
headers={'X-Auth-Token': auth_token,
'X-Delete-After': max_time,
'Destination': new_path},
swift_source='recyclebin_filter')
resp = sub_req.get_response(self.app)
if resp.is_success:
return req.get_response(self.app)
else:
return resp
return req.get_response(self.app)
def get_container_metadata(self, container):
"""
Retrieves the swift metadata of the request container
:returns: dictionary with all swift metadata
"""
new_env = dict(self.request.environ)
auth_token = self.request.headers.get('X-Auth-Token')
dest_path = os.path.join('/', self.api_version, self.account, container)
sub_req = make_subrequest(new_env, 'HEAD', dest_path,
headers={'X-Auth-Token': auth_token},
swift_source='micro-controllers_middleware')
response = sub_req.get_response(self.app)
return response.headers
def _get_container_listing(self, req, version, account, container,
prefix, marker=''):
con_req = make_subrequest(
req.environ, path='/'.join(['', version, account, container]),
method='GET',
headers={'x-auth-token': req.headers.get('x-auth-token')},
agent=('%(orig)s ' + 'DLO MultipartGET'), swift_source='DLO')
con_req.query_string = 'format=json&prefix=%s' % quote(prefix)
if marker:
con_req.query_string += '&marker=%s' % quote(marker)
con_resp = con_req.get_response(self.dlo.app)
if not is_success(con_resp.status_int):
return con_resp, None
return None, json.loads(''.join(con_resp.app_iter))
def __call__(self, request):
try:
(version, account, container,
objname) = split_path(request.path_info, 4, 4, True)
except ValueError:
return self.app
preview_path = '/%s/%s/%s_%s/%s' % (version, account, container,
self.suffix, objname)
if request.method == 'GET' and request.params.has_key('preview'):
request.path_info = preview_path
if request.method == 'PUT':
if hasattr(request, 'body_file'):
data = ""
while True:
chunk = request.body_file.read()
if not chunk:
break
data += chunk
request.body = data
preview = create_preview(data)
else:
preview = create_preview(request.body)
if preview:
sub = wsgi.make_subrequest(request.environ,
path=preview_path,
body=preview)
sub.get_response(self.app)
if request.method == 'DELETE':
sub = wsgi.make_subrequest(request.environ, path=preview_path)
sub.get_response(self.app)
return self.app
def post_expiry(self, env):
# Since the request to the swift proxy can be unauthenticated, we need to
# get au auth token for the POST to update the X-Delete- header
auth_token = self.get_auth_token(env)
if not auth_token:
self.logger.error('Could not acquire auth token, cannot update thumbnail expiry')
return False
headers = {
'X-Delete-After': self.thumbnail_expiry,
'X-Auth-Token': auth_token,
}
sub = make_subrequest(env, method='POST', headers=headers)
return sub.get_response(self.app)
def _get_container_listing(self, req, version, account, container,
prefix, marker=''):
con_req = make_subrequest(
req.environ, path='/'.join(['', version, account, container]),
method='GET',
headers={'x-auth-token': req.headers.get('x-auth-token')},
agent=('%(orig)s ' + 'DLO MultipartGET'), swift_source='DLO')
con_req.query_string = 'prefix=%s' % quote(prefix)
if marker:
con_req.query_string += '&marker=%s' % quote(marker)
con_resp = con_req.get_response(self.dlo.app)
if not is_success(con_resp.status_int):
return con_resp, None
with closing_if_possible(con_resp.app_iter):
return None, json.loads(''.join(con_resp.app_iter))
def _list_objects(self, env, account, ct_parts, header_cb,
prefix='', limit=DEFAULT_LIMIT,
marker=None, force_master=False):
"""
returns items
"""
sub_path = quote(self.DELIMITER.join(
('', 'v1', account, self.ENCODED_DELIMITER.join(ct_parts))))
LOG.debug("%s: listing objects from '%s' "
"(limit=%d, prefix=%s, marker=%s)",
self.SWIFT_SOURCE, sub_path, limit, prefix, marker)
sub_req = make_subrequest(env.copy(), method='GET', path=sub_path,
body='',
swift_source=self.SWIFT_SOURCE)
params = sub_req.params
params.pop('delimiter', None) # allow list-multipart-uploads
params['limit'] = str(limit) # FIXME: why is it str?
params['prefix'] = prefix
params['format'] = 'json'
if marker:
params['marker'] = marker
else:
params.pop('marker', None)
if force_master:
sub_req.environ.setdefault('oio.query', {})
sub_req.environ['oio.query']['force_master'] = True
sub_req.params = params
resp = sub_req.get_response(self.app)
obj_prefix = ''
if len(ct_parts) > 1:
obj_prefix = self.DELIMITER.join(ct_parts[1:] + ['', ])
if not resp.is_success or resp.content_length == 0:
LOG.warn("%s: Failed to list %s",
self.SWIFT_SOURCE, sub_path)
return
with closing_if_possible(resp.app_iter):
items = json.loads(resp.body)
if header_cb:
header_cb(resp.headers)
for obj in items:
if 'name' in obj:
obj['name'] = obj_prefix.decode('utf-8') + obj['name']
yield obj
def _get_container_listing(self, req, version, account, container, prefix, marker=""):
con_req = make_subrequest(
req.environ,
path="/".join(["", version, account, container]),
method="GET",
headers={"x-auth-token": req.headers.get("x-auth-token")},
agent=("%(orig)s " + "DLO MultipartGET"),
swift_source="DLO",
)
con_req.query_string = "format=json&prefix=%s" % quote(prefix)
if marker:
con_req.query_string += "&marker=%s" % quote(marker)
con_resp = con_req.get_response(self.dlo.app)
if not is_success(con_resp.status_int):
return con_resp, None
return None, json.loads("".join(con_resp.app_iter))
def build_traversal_req(symlink_target):
"""
:returns: new request for target path if it's symlink otherwise
None
"""
version, account, _junk = split_path(req.path, 2, 3, True)
account = self._response_header_value(
TGT_ACCT_SYSMETA_SYMLINK_HDR) or account
target_path = os.path.join(
'/', version, account,
symlink_target.lstrip('/'))
self._last_target_path = target_path
new_req = make_subrequest(
req.environ, path=target_path, method=req.method,
headers=req.headers, swift_source='SYM')
new_req.headers.pop('X-Backend-Storage-Policy-Index', None)
return new_req
def build_traversal_req(symlink_target):
"""
:returns: new request for target path if it's symlink otherwise
None
"""
version, account, _junk = split_path(req.path, 2, 3, True)
account = self._response_header_value(
TGT_ACCT_SYSMETA_SYMLINK_HDR) or account
target_path = os.path.join(
'/', version, account,
symlink_target.lstrip('/'))
self._last_target_path = target_path
new_req = make_subrequest(
req.environ, path=target_path, method=req.method,
headers=req.headers, swift_source='SYM')
new_req.headers.pop('X-Backend-Storage-Policy-Index', None)
return new_req
def post_expiry(self, env):
# Since the request to the swift proxy can be unauthenticated, we need to
# get an auth token for the POST to update the X-Delete- header
auth_token = self.get_auth_token(env)
if not auth_token:
self.logger.error(
'Could not acquire auth token, cannot update thumbnail expiry')
return False
headers = {
'X-Delete-After': self.thumbnail_expiry,
'X-Auth-Token': auth_token,
}
sub = make_subrequest(env, method='POST', headers=headers)
return sub.get_response(self.app)
def handle_object_put(self, req, start_response, sync_profile,
per_account):
status, headers, app_iter = req.call_application(self.app)
if not status.startswith('404 '):
status, headers, app_iter = req.call_application(self.app)
start_response(status, headers)
return app_iter
provider = create_provider(sync_profile,
max_conns=1,
per_account=per_account)
headers = {}
if sync_profile.get('protocol') == 'swift':
try:
headers = get_container_headers(provider)
except RemoteHTTPError as e:
self.logger.warning(
'Failed to query the remote container (%d): %s' %
(e.resp.status, e.resp.body))
status, headers, app_iter = req.call_application(self.app)
start_response(status, headers)
return app_iter
vers, acct, cont, _ = req.split_path(4, 4, True)
container_path = '/%s' % '/'.join(
[vers, utils.quote(acct),
utils.quote(cont)])
put_container_req = make_subrequest(req.environ,
method='PUT',
path=container_path,
headers=headers,
swift_source='CloudSync Shunt')
put_container_req.environ['swift_owner'] = True
status, headers, body = put_container_req.call_application(self.app)
utils.close_if_possible(body)
if int(status.split()[0]) // 100 != 2:
self.logger.warning('Failed to create container: %s' % status)
status, headers, app_iter = req.call_application(self.app)
start_response(status, headers)
return app_iter
def make_encrypted_subrequest(env,
method=None,
path=None,
body=None,
headers=None,
agent='Swift',
swift_source=None,
make_env=make_encrypted_env):
"""
Same as :py:func:`make_subrequest` but with encryption env, if available.
"""
return wsgi.make_subrequest(env,
method=method,
path=path,
body=body,
headers=headers,
agent=agent,
swift_source=swift_source,
make_env=make_env)
def __call__(self, env, start_response):
path = SUB_PUT_POST_PATH
if env['REQUEST_METHOD'] == 'GET':
path = SUB_GET_PATH
# Make a subrequest that will be logged
hdrs = {'content-type': 'text/plain'}
sub_req = make_subrequest(env, path=path,
method=self.conf['subrequest_type'],
headers=hdrs,
agent='FakeApp',
swift_source='FA')
self.register(self.conf['subrequest_type'],
path, HTTPOk, headers=hdrs)
resp = sub_req.get_response(self.app)
close_if_possible(resp.app_iter)
return self.app(env, start_response)
def __call__(self, env, start_response):
path = SUB_PUT_POST_PATH
if env['REQUEST_METHOD'] == 'GET':
path = SUB_GET_PATH
# Make a subrequest that will be logged
hdrs = {'content-type': 'text/plain'}
sub_req = make_subrequest(env,
path=path,
method=self.conf['subrequest_type'],
headers=hdrs,
agent='FakeApp',
swift_source='FA')
self.register(self.conf['subrequest_type'], path, HTTPOk, headers=hdrs)
resp = sub_req.get_response(self.app)
close_if_possible(resp.app_iter)
return self.app(env, start_response)
def get_auth_token(self, env):
if _WMFRewriteContext.auth_token and _WMFRewriteContext.auth_token_expiry > monotonic.monotonic():
self.logger.debug('Reusing existing auth token for thumbnail expiry update')
return _WMFRewriteContext.auth_token
headers = {
'X-Auth-User': self.thumbnail_user,
'X-Auth-Key': self.thumbnail_key,
}
sub = make_subrequest(env, path='/auth/v1.0', headers=headers)
resp = sub.get_response(self.app)
if resp.status_int != 200:
self.logger.error('Could not acquire auth token while updating thumbnail expiry')
return False
_WMFRewriteContext.auth_token = resp.headers['X-Auth-Token']
_WMFRewriteContext.auth_token_expiry = monotonic.monotonic() + int(resp.headers['X-Auth-Token-Expires'])
return _WMFRewriteContext.auth_token
