def process_object_update(self, update_path, device, policy):
"""
Process the object information to be updated and update.
:param update_path: path to pickled object update file
:param device: path to device
:param policy: storage policy of object update
"""
try:
update = pickle.load(open(update_path, 'rb'))
except Exception:
self.logger.exception(_('ERROR Pickle problem, quarantining %s'),
update_path)
self.logger.increment('quarantines')
target_path = os.path.join(device, 'quarantined', 'objects',
os.path.basename(update_path))
renamer(update_path, target_path, fsync=False)
return
successes = update.get('successes', [])
part, nodes = self.get_container_ring().get_nodes(
update['account'], update['container'])
obj = '/%s/%s/%s' % \
(update['account'], update['container'], update['obj'])
headers_out = HeaderKeyDict(update['headers'])
headers_out['user-agent'] = 'object-updater %s' % os.getpid()
headers_out.setdefault('X-Backend-Storage-Policy-Index',
str(int(policy)))
events = [
spawn(self.object_update, node, part, update['op'], obj,
headers_out) for node in nodes if node['id'] not in successes
]
success = True
new_successes = False
for event in events:
event_success, node_id = event.wait()
if event_success is True:
successes.append(node_id)
new_successes = True
else:
success = False
if success:
self.successes += 1
self.logger.increment('successes')
self.logger.debug('Update sent for %(obj)s %(path)s', {
'obj': obj,
'path': update_path
})
self.logger.increment("unlinks")
os.unlink(update_path)
else:
self.failures += 1
self.logger.increment('failures')
self.logger.debug('Update failed for %(obj)s %(path)s', {
'obj': obj,
'path': update_path
})
if new_successes:
update['successes'] = successes
write_pickle(update, update_path,
os.path.join(device, get_tmp_dir(policy)))
def __call__(self, env, start_response):
method = env['REQUEST_METHOD']
path = env['PATH_INFO']
_, acc, cont, obj = split_path(env['PATH_INFO'], 0, 4,
rest_with_last=True)
if env.get('QUERY_STRING'):
path += '?' + env['QUERY_STRING']
if 'swift.authorize' in env:
resp = env['swift.authorize'](swob.Request(env))
if resp:
return resp(env, start_response)
req_headers = swob.Request(env).headers
self.swift_sources.append(env.get('swift.source'))
self.txn_ids.append(env.get('swift.trans_id'))
try:
resp_class, raw_headers, body = self._find_response(method, path)
headers = HeaderKeyDict(raw_headers)
except KeyError:
if (env.get('QUERY_STRING')
and (method, env['PATH_INFO']) in self._responses):
resp_class, raw_headers, body = self._find_response(
method, env['PATH_INFO'])
headers = HeaderKeyDict(raw_headers)
elif method == 'HEAD' and ('GET', path) in self._responses:
resp_class, raw_headers, body = self._find_response(
'GET', path)
body = None
headers = HeaderKeyDict(raw_headers)
elif method == 'GET' and obj and path in self.uploaded:
resp_class = swob.HTTPOk
headers, body = self.uploaded[path]
else:
raise KeyError("Didn't find %r in allowed responses" % (
(method, path),))
self._calls.append((method, path, req_headers))
# simulate object PUT
if method == 'PUT' and obj:
input = env['wsgi.input'].read()
etag = md5(input).hexdigest()
headers.setdefault('Etag', etag)
headers.setdefault('Content-Length', len(input))
# keep it for subsequent GET requests later
self.uploaded[path] = (deepcopy(headers), input)
if "CONTENT_TYPE" in env:
self.uploaded[path][0]['Content-Type'] = env["CONTENT_TYPE"]
# range requests ought to work, which require conditional_response=True
req = swob.Request(env)
resp = resp_class(req=req, headers=headers, body=body,
conditional_response=req.method in ('GET', 'HEAD'))
wsgi_iter = resp(env, start_response)
self.mark_opened(path)
return LeakTrackingIter(wsgi_iter, self, path)
def process_object_update(self, update_path, device, policy):
"""
Process the object information to be updated and update.
:param update_path: path to pickled object update file
:param device: path to device
:param policy: storage policy of object update
"""
try:
update = pickle.load(open(update_path, 'rb'))
except Exception:
self.logger.exception(
_('ERROR Pickle problem, quarantining %s'), update_path)
self.stats.quarantines += 1
self.logger.increment('quarantines')
target_path = os.path.join(device, 'quarantined', 'objects',
os.path.basename(update_path))
renamer(update_path, target_path, fsync=False)
return
successes = update.get('successes', [])
part, nodes = self.get_container_ring().get_nodes(
update['account'], update['container'])
obj = '/%s/%s/%s' % \
(update['account'], update['container'], update['obj'])
headers_out = HeaderKeyDict(update['headers'])
headers_out['user-agent'] = 'object-updater %s' % os.getpid()
headers_out.setdefault('X-Backend-Storage-Policy-Index',
str(int(policy)))
events = [spawn(self.object_update,
node, part, update['op'], obj, headers_out)
for node in nodes if node['id'] not in successes]
success = True
new_successes = False
for event in events:
event_success, node_id = event.wait()
if event_success is True:
successes.append(node_id)
new_successes = True
else:
success = False
if success:
self.stats.successes += 1
self.logger.increment('successes')
self.logger.debug('Update sent for %(obj)s %(path)s',
{'obj': obj, 'path': update_path})
self.stats.unlinks += 1
self.logger.increment('unlinks')
os.unlink(update_path)
else:
self.stats.failures += 1
self.logger.increment('failures')
self.logger.debug('Update failed for %(obj)s %(path)s',
{'obj': obj, 'path': update_path})
if new_successes:
update['successes'] = successes
write_pickle(update, update_path, os.path.join(
device, get_tmp_dir(policy)))
def __call__(self, env, start_response):
method = env["REQUEST_METHOD"]
if method not in self.ALLOWED_METHODS:
raise HTTPNotImplemented()
path = env["PATH_INFO"]
_, acc, cont, obj = split_path(env["PATH_INFO"], 0, 4, rest_with_last=True)
if env.get("QUERY_STRING"):
path += "?" + env["QUERY_STRING"]
if "swift.authorize" in env:
resp = env["swift.authorize"](swob.Request(env))
if resp:
return resp(env, start_response)
req_headers = swob.Request(env).headers
self.swift_sources.append(env.get("swift.source"))
self.txn_ids.append(env.get("swift.trans_id"))
try:
resp_class, raw_headers, body = self._find_response(method, path)
headers = HeaderKeyDict(raw_headers)
except KeyError:
if env.get("QUERY_STRING") and (method, env["PATH_INFO"]) in self._responses:
resp_class, raw_headers, body = self._find_response(method, env["PATH_INFO"])
headers = HeaderKeyDict(raw_headers)
elif method == "HEAD" and ("GET", path) in self._responses:
resp_class, raw_headers, body = self._find_response("GET", path)
body = None
headers = HeaderKeyDict(raw_headers)
elif method == "GET" and obj and path in self.uploaded:
resp_class = swob.HTTPOk
headers, body = self.uploaded[path]
else:
raise KeyError("Didn't find %r in allowed responses" % ((method, path),))
self._calls.append((method, path, req_headers))
# simulate object PUT
if method == "PUT" and obj:
input = env["wsgi.input"].read()
etag = md5(input).hexdigest()
headers.setdefault("Etag", etag)
headers.setdefault("Content-Length", len(input))
# keep it for subsequent GET requests later
self.uploaded[path] = (deepcopy(headers), input)
if "CONTENT_TYPE" in env:
self.uploaded[path][0]["Content-Type"] = env["CONTENT_TYPE"]
# range requests ought to work, which require conditional_response=True
req = swob.Request(env)
resp = resp_class(req=req, headers=headers, body=body, conditional_response=req.method in ("GET", "HEAD"))
wsgi_iter = resp(env, start_response)
self.mark_opened(path)
return LeakTrackingIter(wsgi_iter, self, path)
def gen_headers(hdrs_in=None, add_ts=True):
"""
Get the headers ready for a request. All requests should have a User-Agent
string, but if one is passed in don't over-write it. Not all requests will
need an X-Timestamp, but if one is passed in do not over-write it.
:param headers: dict or None, base for HTTP headers
:param add_ts: boolean, should be True for any "unsafe" HTTP request
:returns: HeaderKeyDict based on headers and ready for the request
"""
hdrs_out = HeaderKeyDict(hdrs_in) if hdrs_in else HeaderKeyDict()
if add_ts and 'X-Timestamp' not in hdrs_out:
hdrs_out['X-Timestamp'] = Timestamp.now().internal
if 'user-agent' not in hdrs_out:
hdrs_out['User-Agent'] = 'direct-client %s' % os.getpid()
hdrs_out.setdefault('X-Backend-Allow-Reserved-Names', 'true')
return hdrs_out
def test_setdefault(self):
headers = HeaderKeyDict()
# it gets set
headers.setdefault('x-rubber-ducky', 'the one')
self.assertEqual(headers['X-Rubber-Ducky'], 'the one')
# it has the right return value
ret = headers.setdefault('x-boat', 'dinghy')
self.assertEqual(ret, 'dinghy')
ret = headers.setdefault('x-boat', 'yacht')
self.assertEqual(ret, 'dinghy')
# shouldn't crash
headers.setdefault('x-sir-not-appearing-in-this-request', None)
def test_setdefault(self):
headers = HeaderKeyDict()
# it gets set
headers.setdefault('x-rubber-ducky', 'the one')
self.assertEqual(headers['X-Rubber-Ducky'], 'the one')
# it has the right return value
ret = headers.setdefault('x-boat', 'dinghy')
self.assertEqual(ret, 'dinghy')
ret = headers.setdefault('x-boat', 'yacht')
self.assertEqual(ret, 'dinghy')
# shouldn't crash
headers.setdefault('x-sir-not-appearing-in-this-request', None)
def test_setdefault(self):
headers = HeaderKeyDict()
# it gets set
headers.setdefault("x-rubber-ducky", "the one")
self.assertEqual(headers["X-Rubber-Ducky"], "the one")
# it has the right return value
ret = headers.setdefault("x-boat", "dinghy")
self.assertEqual(ret, "dinghy")
ret = headers.setdefault("x-boat", "yacht")
self.assertEqual(ret, "dinghy")
# shouldn't crash
headers.setdefault("x-sir-not-appearing-in-this-request", None)
def __call__(self, env, start_response):
method = env['REQUEST_METHOD']
if method not in self.ALLOWED_METHODS:
raise HTTPNotImplemented()
path = env['PATH_INFO']
_, acc, cont, obj = split_path(env['PATH_INFO'], 0, 4,
rest_with_last=True)
if env.get('QUERY_STRING'):
path += '?' + env['QUERY_STRING']
if 'swift.authorize' in env:
resp = env['swift.authorize'](swob.Request(env))
if resp:
return resp(env, start_response)
req = swob.Request(env)
self.swift_sources.append(env.get('swift.source'))
self.txn_ids.append(env.get('swift.trans_id'))
try:
resp_class, raw_headers, body = self._find_response(method, path)
headers = HeaderKeyDict(raw_headers)
except KeyError:
if (env.get('QUERY_STRING')
and (method, env['PATH_INFO']) in self._responses):
resp_class, raw_headers, body = self._find_response(
method, env['PATH_INFO'])
headers = HeaderKeyDict(raw_headers)
elif method == 'HEAD' and ('GET', path) in self._responses:
resp_class, raw_headers, body = self._find_response(
'GET', path)
body = None
headers = HeaderKeyDict(raw_headers)
elif method == 'GET' and obj and path in self.uploaded:
resp_class = swob.HTTPOk
headers, body = self.uploaded[path]
else:
raise KeyError("Didn't find %r in allowed responses" % (
(method, path),))
# simulate object PUT
if method == 'PUT' and obj:
put_body = ''.join(iter(env['wsgi.input'].read, ''))
if 'swift.callback.update_footers' in env:
footers = HeaderKeyDict()
env['swift.callback.update_footers'](footers)
req.headers.update(footers)
etag = md5(put_body).hexdigest()
headers.setdefault('Etag', etag)
headers.setdefault('Content-Length', len(put_body))
# keep it for subsequent GET requests later
self.uploaded[path] = (dict(req.headers), put_body)
if "CONTENT_TYPE" in env:
self.uploaded[path][0]['Content-Type'] = env["CONTENT_TYPE"]
# simulate object POST
elif method == 'POST' and obj:
metadata, data = self.uploaded.get(path, ({}, None))
# select items to keep from existing...
new_metadata = dict(
(k, v) for k, v in metadata.items()
if (not is_user_meta('object', k) and not
is_object_transient_sysmeta(k)))
# apply from new
new_metadata.update(
dict((k, v) for k, v in req.headers.items()
if (is_user_meta('object', k) or
is_object_transient_sysmeta(k) or
k.lower == 'content-type')))
self.uploaded[path] = new_metadata, data
# note: tests may assume this copy of req_headers is case insensitive
# so we deliberately use a HeaderKeyDict
self._calls.append(
FakeSwiftCall(method, path, HeaderKeyDict(req.headers)))
backend_etag_header = req.headers.get('X-Backend-Etag-Is-At')
conditional_etag = None
if backend_etag_header and backend_etag_header in headers:
# Apply conditional etag overrides
conditional_etag = headers[backend_etag_header]
# range requests ought to work, hence conditional_response=True
if isinstance(body, list):
resp = resp_class(
req=req, headers=headers, app_iter=body,
conditional_response=req.method in ('GET', 'HEAD'),
conditional_etag=conditional_etag)
else:
resp = resp_class(
req=req, headers=headers, body=body,
conditional_response=req.method in ('GET', 'HEAD'),
conditional_etag=conditional_etag)
wsgi_iter = resp(env, start_response)
self.mark_opened(path)
return LeakTrackingIter(wsgi_iter, self, path)
def __call__(self, env, start_response):
method = env['REQUEST_METHOD']
if method not in self.ALLOWED_METHODS:
raise HTTPNotImplemented()
path = env['PATH_INFO']
_, acc, cont, obj = split_path(env['PATH_INFO'],
0,
4,
rest_with_last=True)
if env.get('QUERY_STRING'):
path += '?' + env['QUERY_STRING']
if 'swift.authorize' in env:
resp = env['swift.authorize'](swob.Request(env))
if resp:
return resp(env, start_response)
req = swob.Request(env)
self.swift_sources.append(env.get('swift.source'))
self.txn_ids.append(env.get('swift.trans_id'))
try:
resp_class, raw_headers, body = self._find_response(method, path)
headers = HeaderKeyDict(raw_headers)
except KeyError:
if (env.get('QUERY_STRING')
and (method, env['PATH_INFO']) in self._responses):
resp_class, raw_headers, body = self._find_response(
method, env['PATH_INFO'])
headers = HeaderKeyDict(raw_headers)
elif method == 'HEAD' and ('GET', path) in self._responses:
resp_class, raw_headers, body = self._find_response(
'GET', path)
body = None
headers = HeaderKeyDict(raw_headers)
elif method == 'GET' and obj and path in self.uploaded:
resp_class = swob.HTTPOk
headers, body = self.uploaded[path]
else:
raise KeyError("Didn't find %r in allowed responses" %
((method, path), ))
# simulate object PUT
if method == 'PUT' and obj:
put_body = ''.join(iter(env['wsgi.input'].read, ''))
if 'swift.callback.update_footers' in env:
footers = HeaderKeyDict()
env['swift.callback.update_footers'](footers)
req.headers.update(footers)
etag = md5(put_body).hexdigest()
headers.setdefault('Etag', etag)
headers.setdefault('Content-Length', len(put_body))
# keep it for subsequent GET requests later
self.uploaded[path] = (dict(req.headers), put_body)
if "CONTENT_TYPE" in env:
self.uploaded[path][0]['Content-Type'] = env["CONTENT_TYPE"]
# simulate object POST
elif method == 'POST' and obj:
metadata, data = self.uploaded.get(path, ({}, None))
# select items to keep from existing...
new_metadata = dict((k, v) for k, v in metadata.items()
if (not is_user_meta('object', k)
and not is_object_transient_sysmeta(k)))
# apply from new
new_metadata.update(
dict((k, v) for k, v in req.headers.items() if (
is_user_meta('object', k) or is_object_transient_sysmeta(k)
or k.lower == 'content-type')))
self.uploaded[path] = new_metadata, data
# note: tests may assume this copy of req_headers is case insensitive
# so we deliberately use a HeaderKeyDict
self._calls.append(
FakeSwiftCall(method, path, HeaderKeyDict(req.headers)))
# Apply conditional etag overrides
conditional_etag = resolve_etag_is_at_header(req, headers)
# range requests ought to work, hence conditional_response=True
if isinstance(body, list):
resp = resp_class(req=req,
headers=headers,
app_iter=body,
conditional_response=req.method
in ('GET', 'HEAD'),
conditional_etag=conditional_etag)
else:
resp = resp_class(req=req,
headers=headers,
body=body,
conditional_response=req.method
in ('GET', 'HEAD'),
conditional_etag=conditional_etag)
wsgi_iter = resp(env, start_response)
self.mark_opened(path)
return LeakTrackingIter(wsgi_iter, self.mark_closed, path)
def do_update():
successes = update.get('successes', [])
headers_out = HeaderKeyDict(update['headers'].copy())
headers_out['user-agent'] = 'object-updater %s' % os.getpid()
headers_out.setdefault('X-Backend-Storage-Policy-Index',
str(int(policy)))
headers_out.setdefault('X-Backend-Accept-Redirect', 'true')
headers_out.setdefault('X-Backend-Accept-Quoted-Location', 'true')
container_path = update.get('container_path')
if container_path:
acct, cont = split_path('/' + container_path, minsegs=2)
else:
acct, cont = update['account'], update['container']
part, nodes = self.get_container_ring().get_nodes(acct, cont)
obj = '/%s/%s/%s' % (acct, cont, update['obj'])
events = [
spawn(self.object_update, node, part, update['op'], obj,
headers_out) for node in nodes
if node['id'] not in successes
]
success = True
new_successes = rewrite_pickle = False
redirect = None
redirects = set()
for event in events:
event_success, node_id, redirect = event.wait()
if event_success is True:
successes.append(node_id)
new_successes = True
else:
success = False
if redirect:
redirects.add(redirect)
if success:
self.stats.successes += 1
self.logger.increment('successes')
self.logger.debug('Update sent for %(obj)s %(path)s', {
'obj': obj,
'path': update_path
})
self.stats.unlinks += 1
self.logger.increment('unlinks')
os.unlink(update_path)
try:
# If this was the last async_pending in the directory,
# then this will succeed. Otherwise, it'll fail, and
# that's okay.
os.rmdir(os.path.dirname(update_path))
except OSError:
pass
elif redirects:
# erase any previous successes
update.pop('successes', None)
redirect = max(redirects, key=lambda x: x[-1])[0]
redirect_history = update.setdefault('redirect_history', [])
if redirect in redirect_history:
# force next update to be sent to root, reset history
update['container_path'] = None
update['redirect_history'] = []
else:
update['container_path'] = redirect
redirect_history.append(redirect)
self.stats.redirects += 1
self.logger.increment("redirects")
self.logger.debug(
'Update redirected for %(obj)s %(path)s to %(shard)s', {
'obj': obj,
'path': update_path,
'shard': update['container_path']
})
rewrite_pickle = True
else:
self.stats.failures += 1
self.logger.increment('failures')
self.logger.debug('Update failed for %(obj)s %(path)s', {
'obj': obj,
'path': update_path
})
if new_successes:
update['successes'] = successes
rewrite_pickle = True
return rewrite_pickle, redirect
def __call__(self, env, start_response):
method = env["REQUEST_METHOD"]
if method not in self.ALLOWED_METHODS:
raise HTTPNotImplemented()
path = env["PATH_INFO"]
_, acc, cont, obj = split_path(env["PATH_INFO"], 0, 4, rest_with_last=True)
if env.get("QUERY_STRING"):
path += "?" + env["QUERY_STRING"]
if "swift.authorize" in env:
resp = env["swift.authorize"](swob.Request(env))
if resp:
return resp(env, start_response)
req = swob.Request(env)
self.swift_sources.append(env.get("swift.source"))
self.txn_ids.append(env.get("swift.trans_id"))
try:
resp_class, raw_headers, body = self._find_response(method, path)
headers = HeaderKeyDict(raw_headers)
except KeyError:
if env.get("QUERY_STRING") and (method, env["PATH_INFO"]) in self._responses:
resp_class, raw_headers, body = self._find_response(method, env["PATH_INFO"])
headers = HeaderKeyDict(raw_headers)
elif method == "HEAD" and ("GET", path) in self._responses:
resp_class, raw_headers, body = self._find_response("GET", path)
body = None
headers = HeaderKeyDict(raw_headers)
elif method == "GET" and obj and path in self.uploaded:
resp_class = swob.HTTPOk
headers, body = self.uploaded[path]
else:
raise KeyError("Didn't find %r in allowed responses" % ((method, path),))
# simulate object PUT
if method == "PUT" and obj:
put_body = "".join(iter(env["wsgi.input"].read, ""))
if "swift.callback.update_footers" in env:
footers = HeaderKeyDict()
env["swift.callback.update_footers"](footers)
req.headers.update(footers)
etag = md5(put_body).hexdigest()
headers.setdefault("Etag", etag)
headers.setdefault("Content-Length", len(put_body))
# keep it for subsequent GET requests later
self.uploaded[path] = (dict(req.headers), put_body)
if "CONTENT_TYPE" in env:
self.uploaded[path][0]["Content-Type"] = env["CONTENT_TYPE"]
# simulate object POST
elif method == "POST" and obj:
metadata, data = self.uploaded.get(path, ({}, None))
# select items to keep from existing...
new_metadata = dict(
(k, v)
for k, v in metadata.items()
if (not is_user_meta("object", k) and not is_object_transient_sysmeta(k))
)
# apply from new
new_metadata.update(
dict(
(k, v)
for k, v in req.headers.items()
if (is_user_meta("object", k) or is_object_transient_sysmeta(k) or k.lower == "content-type")
)
)
self.uploaded[path] = new_metadata, data
# note: tests may assume this copy of req_headers is case insensitive
# so we deliberately use a HeaderKeyDict
self._calls.append((method, path, HeaderKeyDict(req.headers)))
# range requests ought to work, hence conditional_response=True
if isinstance(body, list):
resp = resp_class(
req=req, headers=headers, app_iter=body, conditional_response=req.method in ("GET", "HEAD")
)
else:
resp = resp_class(req=req, headers=headers, body=body, conditional_response=req.method in ("GET", "HEAD"))
wsgi_iter = resp(env, start_response)
self.mark_opened(path)
return LeakTrackingIter(wsgi_iter, self, path)
