def process_object_update(self, update_path, device, policy): """ Process the object information to be updated and update. :param update_path: path to pickled object update file :param device: path to device :param policy: storage policy of object update """ try: update = pickle.load(open(update_path, 'rb')) except Exception: self.logger.exception(_('ERROR Pickle problem, quarantining %s'), update_path) self.logger.increment('quarantines') target_path = os.path.join(device, 'quarantined', 'objects', os.path.basename(update_path)) renamer(update_path, target_path, fsync=False) return successes = update.get('successes', []) part, nodes = self.get_container_ring().get_nodes( update['account'], update['container']) obj = '/%s/%s/%s' % \ (update['account'], update['container'], update['obj']) headers_out = HeaderKeyDict(update['headers']) headers_out['user-agent'] = 'object-updater %s' % os.getpid() headers_out.setdefault('X-Backend-Storage-Policy-Index', str(int(policy))) events = [ spawn(self.object_update, node, part, update['op'], obj, headers_out) for node in nodes if node['id'] not in successes ] success = True new_successes = False for event in events: event_success, node_id = event.wait() if event_success is True: successes.append(node_id) new_successes = True else: success = False if success: self.successes += 1 self.logger.increment('successes') self.logger.debug('Update sent for %(obj)s %(path)s', { 'obj': obj, 'path': update_path }) self.logger.increment("unlinks") os.unlink(update_path) else: self.failures += 1 self.logger.increment('failures') self.logger.debug('Update failed for %(obj)s %(path)s', { 'obj': obj, 'path': update_path }) if new_successes: update['successes'] = successes write_pickle(update, update_path, os.path.join(device, get_tmp_dir(policy)))
def __call__(self, env, start_response): method = env['REQUEST_METHOD'] path = env['PATH_INFO'] _, acc, cont, obj = split_path(env['PATH_INFO'], 0, 4, rest_with_last=True) if env.get('QUERY_STRING'): path += '?' + env['QUERY_STRING'] if 'swift.authorize' in env: resp = env['swift.authorize'](swob.Request(env)) if resp: return resp(env, start_response) req_headers = swob.Request(env).headers self.swift_sources.append(env.get('swift.source')) self.txn_ids.append(env.get('swift.trans_id')) try: resp_class, raw_headers, body = self._find_response(method, path) headers = HeaderKeyDict(raw_headers) except KeyError: if (env.get('QUERY_STRING') and (method, env['PATH_INFO']) in self._responses): resp_class, raw_headers, body = self._find_response( method, env['PATH_INFO']) headers = HeaderKeyDict(raw_headers) elif method == 'HEAD' and ('GET', path) in self._responses: resp_class, raw_headers, body = self._find_response( 'GET', path) body = None headers = HeaderKeyDict(raw_headers) elif method == 'GET' and obj and path in self.uploaded: resp_class = swob.HTTPOk headers, body = self.uploaded[path] else: raise KeyError("Didn't find %r in allowed responses" % ( (method, path),)) self._calls.append((method, path, req_headers)) # simulate object PUT if method == 'PUT' and obj: input = env['wsgi.input'].read() etag = md5(input).hexdigest() headers.setdefault('Etag', etag) headers.setdefault('Content-Length', len(input)) # keep it for subsequent GET requests later self.uploaded[path] = (deepcopy(headers), input) if "CONTENT_TYPE" in env: self.uploaded[path][0]['Content-Type'] = env["CONTENT_TYPE"] # range requests ought to work, which require conditional_response=True req = swob.Request(env) resp = resp_class(req=req, headers=headers, body=body, conditional_response=req.method in ('GET', 'HEAD')) wsgi_iter = resp(env, start_response) self.mark_opened(path) return LeakTrackingIter(wsgi_iter, self, path)
def process_object_update(self, update_path, device, policy): """ Process the object information to be updated and update. :param update_path: path to pickled object update file :param device: path to device :param policy: storage policy of object update """ try: update = pickle.load(open(update_path, 'rb')) except Exception: self.logger.exception( _('ERROR Pickle problem, quarantining %s'), update_path) self.stats.quarantines += 1 self.logger.increment('quarantines') target_path = os.path.join(device, 'quarantined', 'objects', os.path.basename(update_path)) renamer(update_path, target_path, fsync=False) return successes = update.get('successes', []) part, nodes = self.get_container_ring().get_nodes( update['account'], update['container']) obj = '/%s/%s/%s' % \ (update['account'], update['container'], update['obj']) headers_out = HeaderKeyDict(update['headers']) headers_out['user-agent'] = 'object-updater %s' % os.getpid() headers_out.setdefault('X-Backend-Storage-Policy-Index', str(int(policy))) events = [spawn(self.object_update, node, part, update['op'], obj, headers_out) for node in nodes if node['id'] not in successes] success = True new_successes = False for event in events: event_success, node_id = event.wait() if event_success is True: successes.append(node_id) new_successes = True else: success = False if success: self.stats.successes += 1 self.logger.increment('successes') self.logger.debug('Update sent for %(obj)s %(path)s', {'obj': obj, 'path': update_path}) self.stats.unlinks += 1 self.logger.increment('unlinks') os.unlink(update_path) else: self.stats.failures += 1 self.logger.increment('failures') self.logger.debug('Update failed for %(obj)s %(path)s', {'obj': obj, 'path': update_path}) if new_successes: update['successes'] = successes write_pickle(update, update_path, os.path.join( device, get_tmp_dir(policy)))
def __call__(self, env, start_response): method = env["REQUEST_METHOD"] if method not in self.ALLOWED_METHODS: raise HTTPNotImplemented() path = env["PATH_INFO"] _, acc, cont, obj = split_path(env["PATH_INFO"], 0, 4, rest_with_last=True) if env.get("QUERY_STRING"): path += "?" + env["QUERY_STRING"] if "swift.authorize" in env: resp = env["swift.authorize"](swob.Request(env)) if resp: return resp(env, start_response) req_headers = swob.Request(env).headers self.swift_sources.append(env.get("swift.source")) self.txn_ids.append(env.get("swift.trans_id")) try: resp_class, raw_headers, body = self._find_response(method, path) headers = HeaderKeyDict(raw_headers) except KeyError: if env.get("QUERY_STRING") and (method, env["PATH_INFO"]) in self._responses: resp_class, raw_headers, body = self._find_response(method, env["PATH_INFO"]) headers = HeaderKeyDict(raw_headers) elif method == "HEAD" and ("GET", path) in self._responses: resp_class, raw_headers, body = self._find_response("GET", path) body = None headers = HeaderKeyDict(raw_headers) elif method == "GET" and obj and path in self.uploaded: resp_class = swob.HTTPOk headers, body = self.uploaded[path] else: raise KeyError("Didn't find %r in allowed responses" % ((method, path),)) self._calls.append((method, path, req_headers)) # simulate object PUT if method == "PUT" and obj: input = env["wsgi.input"].read() etag = md5(input).hexdigest() headers.setdefault("Etag", etag) headers.setdefault("Content-Length", len(input)) # keep it for subsequent GET requests later self.uploaded[path] = (deepcopy(headers), input) if "CONTENT_TYPE" in env: self.uploaded[path][0]["Content-Type"] = env["CONTENT_TYPE"] # range requests ought to work, which require conditional_response=True req = swob.Request(env) resp = resp_class(req=req, headers=headers, body=body, conditional_response=req.method in ("GET", "HEAD")) wsgi_iter = resp(env, start_response) self.mark_opened(path) return LeakTrackingIter(wsgi_iter, self, path)
def gen_headers(hdrs_in=None, add_ts=True): """ Get the headers ready for a request. All requests should have a User-Agent string, but if one is passed in don't over-write it. Not all requests will need an X-Timestamp, but if one is passed in do not over-write it. :param headers: dict or None, base for HTTP headers :param add_ts: boolean, should be True for any "unsafe" HTTP request :returns: HeaderKeyDict based on headers and ready for the request """ hdrs_out = HeaderKeyDict(hdrs_in) if hdrs_in else HeaderKeyDict() if add_ts and 'X-Timestamp' not in hdrs_out: hdrs_out['X-Timestamp'] = Timestamp.now().internal if 'user-agent' not in hdrs_out: hdrs_out['User-Agent'] = 'direct-client %s' % os.getpid() hdrs_out.setdefault('X-Backend-Allow-Reserved-Names', 'true') return hdrs_out
def test_setdefault(self): headers = HeaderKeyDict() # it gets set headers.setdefault('x-rubber-ducky', 'the one') self.assertEqual(headers['X-Rubber-Ducky'], 'the one') # it has the right return value ret = headers.setdefault('x-boat', 'dinghy') self.assertEqual(ret, 'dinghy') ret = headers.setdefault('x-boat', 'yacht') self.assertEqual(ret, 'dinghy') # shouldn't crash headers.setdefault('x-sir-not-appearing-in-this-request', None)
def test_setdefault(self): headers = HeaderKeyDict() # it gets set headers.setdefault('x-rubber-ducky', 'the one') self.assertEqual(headers['X-Rubber-Ducky'], 'the one') # it has the right return value ret = headers.setdefault('x-boat', 'dinghy') self.assertEqual(ret, 'dinghy') ret = headers.setdefault('x-boat', 'yacht') self.assertEqual(ret, 'dinghy') # shouldn't crash headers.setdefault('x-sir-not-appearing-in-this-request', None)
def test_setdefault(self): headers = HeaderKeyDict() # it gets set headers.setdefault("x-rubber-ducky", "the one") self.assertEqual(headers["X-Rubber-Ducky"], "the one") # it has the right return value ret = headers.setdefault("x-boat", "dinghy") self.assertEqual(ret, "dinghy") ret = headers.setdefault("x-boat", "yacht") self.assertEqual(ret, "dinghy") # shouldn't crash headers.setdefault("x-sir-not-appearing-in-this-request", None)
def __call__(self, env, start_response): method = env['REQUEST_METHOD'] if method not in self.ALLOWED_METHODS: raise HTTPNotImplemented() path = env['PATH_INFO'] _, acc, cont, obj = split_path(env['PATH_INFO'], 0, 4, rest_with_last=True) if env.get('QUERY_STRING'): path += '?' + env['QUERY_STRING'] if 'swift.authorize' in env: resp = env['swift.authorize'](swob.Request(env)) if resp: return resp(env, start_response) req = swob.Request(env) self.swift_sources.append(env.get('swift.source')) self.txn_ids.append(env.get('swift.trans_id')) try: resp_class, raw_headers, body = self._find_response(method, path) headers = HeaderKeyDict(raw_headers) except KeyError: if (env.get('QUERY_STRING') and (method, env['PATH_INFO']) in self._responses): resp_class, raw_headers, body = self._find_response( method, env['PATH_INFO']) headers = HeaderKeyDict(raw_headers) elif method == 'HEAD' and ('GET', path) in self._responses: resp_class, raw_headers, body = self._find_response( 'GET', path) body = None headers = HeaderKeyDict(raw_headers) elif method == 'GET' and obj and path in self.uploaded: resp_class = swob.HTTPOk headers, body = self.uploaded[path] else: raise KeyError("Didn't find %r in allowed responses" % ( (method, path),)) # simulate object PUT if method == 'PUT' and obj: put_body = ''.join(iter(env['wsgi.input'].read, '')) if 'swift.callback.update_footers' in env: footers = HeaderKeyDict() env['swift.callback.update_footers'](footers) req.headers.update(footers) etag = md5(put_body).hexdigest() headers.setdefault('Etag', etag) headers.setdefault('Content-Length', len(put_body)) # keep it for subsequent GET requests later self.uploaded[path] = (dict(req.headers), put_body) if "CONTENT_TYPE" in env: self.uploaded[path][0]['Content-Type'] = env["CONTENT_TYPE"] # simulate object POST elif method == 'POST' and obj: metadata, data = self.uploaded.get(path, ({}, None)) # select items to keep from existing... new_metadata = dict( (k, v) for k, v in metadata.items() if (not is_user_meta('object', k) and not is_object_transient_sysmeta(k))) # apply from new new_metadata.update( dict((k, v) for k, v in req.headers.items() if (is_user_meta('object', k) or is_object_transient_sysmeta(k) or k.lower == 'content-type'))) self.uploaded[path] = new_metadata, data # note: tests may assume this copy of req_headers is case insensitive # so we deliberately use a HeaderKeyDict self._calls.append( FakeSwiftCall(method, path, HeaderKeyDict(req.headers))) backend_etag_header = req.headers.get('X-Backend-Etag-Is-At') conditional_etag = None if backend_etag_header and backend_etag_header in headers: # Apply conditional etag overrides conditional_etag = headers[backend_etag_header] # range requests ought to work, hence conditional_response=True if isinstance(body, list): resp = resp_class( req=req, headers=headers, app_iter=body, conditional_response=req.method in ('GET', 'HEAD'), conditional_etag=conditional_etag) else: resp = resp_class( req=req, headers=headers, body=body, conditional_response=req.method in ('GET', 'HEAD'), conditional_etag=conditional_etag) wsgi_iter = resp(env, start_response) self.mark_opened(path) return LeakTrackingIter(wsgi_iter, self, path)
def __call__(self, env, start_response): method = env['REQUEST_METHOD'] if method not in self.ALLOWED_METHODS: raise HTTPNotImplemented() path = env['PATH_INFO'] _, acc, cont, obj = split_path(env['PATH_INFO'], 0, 4, rest_with_last=True) if env.get('QUERY_STRING'): path += '?' + env['QUERY_STRING'] if 'swift.authorize' in env: resp = env['swift.authorize'](swob.Request(env)) if resp: return resp(env, start_response) req = swob.Request(env) self.swift_sources.append(env.get('swift.source')) self.txn_ids.append(env.get('swift.trans_id')) try: resp_class, raw_headers, body = self._find_response(method, path) headers = HeaderKeyDict(raw_headers) except KeyError: if (env.get('QUERY_STRING') and (method, env['PATH_INFO']) in self._responses): resp_class, raw_headers, body = self._find_response( method, env['PATH_INFO']) headers = HeaderKeyDict(raw_headers) elif method == 'HEAD' and ('GET', path) in self._responses: resp_class, raw_headers, body = self._find_response( 'GET', path) body = None headers = HeaderKeyDict(raw_headers) elif method == 'GET' and obj and path in self.uploaded: resp_class = swob.HTTPOk headers, body = self.uploaded[path] else: raise KeyError("Didn't find %r in allowed responses" % ((method, path), )) # simulate object PUT if method == 'PUT' and obj: put_body = ''.join(iter(env['wsgi.input'].read, '')) if 'swift.callback.update_footers' in env: footers = HeaderKeyDict() env['swift.callback.update_footers'](footers) req.headers.update(footers) etag = md5(put_body).hexdigest() headers.setdefault('Etag', etag) headers.setdefault('Content-Length', len(put_body)) # keep it for subsequent GET requests later self.uploaded[path] = (dict(req.headers), put_body) if "CONTENT_TYPE" in env: self.uploaded[path][0]['Content-Type'] = env["CONTENT_TYPE"] # simulate object POST elif method == 'POST' and obj: metadata, data = self.uploaded.get(path, ({}, None)) # select items to keep from existing... new_metadata = dict((k, v) for k, v in metadata.items() if (not is_user_meta('object', k) and not is_object_transient_sysmeta(k))) # apply from new new_metadata.update( dict((k, v) for k, v in req.headers.items() if ( is_user_meta('object', k) or is_object_transient_sysmeta(k) or k.lower == 'content-type'))) self.uploaded[path] = new_metadata, data # note: tests may assume this copy of req_headers is case insensitive # so we deliberately use a HeaderKeyDict self._calls.append( FakeSwiftCall(method, path, HeaderKeyDict(req.headers))) # Apply conditional etag overrides conditional_etag = resolve_etag_is_at_header(req, headers) # range requests ought to work, hence conditional_response=True if isinstance(body, list): resp = resp_class(req=req, headers=headers, app_iter=body, conditional_response=req.method in ('GET', 'HEAD'), conditional_etag=conditional_etag) else: resp = resp_class(req=req, headers=headers, body=body, conditional_response=req.method in ('GET', 'HEAD'), conditional_etag=conditional_etag) wsgi_iter = resp(env, start_response) self.mark_opened(path) return LeakTrackingIter(wsgi_iter, self.mark_closed, path)
def do_update(): successes = update.get('successes', []) headers_out = HeaderKeyDict(update['headers'].copy()) headers_out['user-agent'] = 'object-updater %s' % os.getpid() headers_out.setdefault('X-Backend-Storage-Policy-Index', str(int(policy))) headers_out.setdefault('X-Backend-Accept-Redirect', 'true') headers_out.setdefault('X-Backend-Accept-Quoted-Location', 'true') container_path = update.get('container_path') if container_path: acct, cont = split_path('/' + container_path, minsegs=2) else: acct, cont = update['account'], update['container'] part, nodes = self.get_container_ring().get_nodes(acct, cont) obj = '/%s/%s/%s' % (acct, cont, update['obj']) events = [ spawn(self.object_update, node, part, update['op'], obj, headers_out) for node in nodes if node['id'] not in successes ] success = True new_successes = rewrite_pickle = False redirect = None redirects = set() for event in events: event_success, node_id, redirect = event.wait() if event_success is True: successes.append(node_id) new_successes = True else: success = False if redirect: redirects.add(redirect) if success: self.stats.successes += 1 self.logger.increment('successes') self.logger.debug('Update sent for %(obj)s %(path)s', { 'obj': obj, 'path': update_path }) self.stats.unlinks += 1 self.logger.increment('unlinks') os.unlink(update_path) try: # If this was the last async_pending in the directory, # then this will succeed. Otherwise, it'll fail, and # that's okay. os.rmdir(os.path.dirname(update_path)) except OSError: pass elif redirects: # erase any previous successes update.pop('successes', None) redirect = max(redirects, key=lambda x: x[-1])[0] redirect_history = update.setdefault('redirect_history', []) if redirect in redirect_history: # force next update to be sent to root, reset history update['container_path'] = None update['redirect_history'] = [] else: update['container_path'] = redirect redirect_history.append(redirect) self.stats.redirects += 1 self.logger.increment("redirects") self.logger.debug( 'Update redirected for %(obj)s %(path)s to %(shard)s', { 'obj': obj, 'path': update_path, 'shard': update['container_path'] }) rewrite_pickle = True else: self.stats.failures += 1 self.logger.increment('failures') self.logger.debug('Update failed for %(obj)s %(path)s', { 'obj': obj, 'path': update_path }) if new_successes: update['successes'] = successes rewrite_pickle = True return rewrite_pickle, redirect
def __call__(self, env, start_response): method = env["REQUEST_METHOD"] if method not in self.ALLOWED_METHODS: raise HTTPNotImplemented() path = env["PATH_INFO"] _, acc, cont, obj = split_path(env["PATH_INFO"], 0, 4, rest_with_last=True) if env.get("QUERY_STRING"): path += "?" + env["QUERY_STRING"] if "swift.authorize" in env: resp = env["swift.authorize"](swob.Request(env)) if resp: return resp(env, start_response) req = swob.Request(env) self.swift_sources.append(env.get("swift.source")) self.txn_ids.append(env.get("swift.trans_id")) try: resp_class, raw_headers, body = self._find_response(method, path) headers = HeaderKeyDict(raw_headers) except KeyError: if env.get("QUERY_STRING") and (method, env["PATH_INFO"]) in self._responses: resp_class, raw_headers, body = self._find_response(method, env["PATH_INFO"]) headers = HeaderKeyDict(raw_headers) elif method == "HEAD" and ("GET", path) in self._responses: resp_class, raw_headers, body = self._find_response("GET", path) body = None headers = HeaderKeyDict(raw_headers) elif method == "GET" and obj and path in self.uploaded: resp_class = swob.HTTPOk headers, body = self.uploaded[path] else: raise KeyError("Didn't find %r in allowed responses" % ((method, path),)) # simulate object PUT if method == "PUT" and obj: put_body = "".join(iter(env["wsgi.input"].read, "")) if "swift.callback.update_footers" in env: footers = HeaderKeyDict() env["swift.callback.update_footers"](footers) req.headers.update(footers) etag = md5(put_body).hexdigest() headers.setdefault("Etag", etag) headers.setdefault("Content-Length", len(put_body)) # keep it for subsequent GET requests later self.uploaded[path] = (dict(req.headers), put_body) if "CONTENT_TYPE" in env: self.uploaded[path][0]["Content-Type"] = env["CONTENT_TYPE"] # simulate object POST elif method == "POST" and obj: metadata, data = self.uploaded.get(path, ({}, None)) # select items to keep from existing... new_metadata = dict( (k, v) for k, v in metadata.items() if (not is_user_meta("object", k) and not is_object_transient_sysmeta(k)) ) # apply from new new_metadata.update( dict( (k, v) for k, v in req.headers.items() if (is_user_meta("object", k) or is_object_transient_sysmeta(k) or k.lower == "content-type") ) ) self.uploaded[path] = new_metadata, data # note: tests may assume this copy of req_headers is case insensitive # so we deliberately use a HeaderKeyDict self._calls.append((method, path, HeaderKeyDict(req.headers))) # range requests ought to work, hence conditional_response=True if isinstance(body, list): resp = resp_class( req=req, headers=headers, app_iter=body, conditional_response=req.method in ("GET", "HEAD") ) else: resp = resp_class(req=req, headers=headers, body=body, conditional_response=req.method in ("GET", "HEAD")) wsgi_iter = resp(env, start_response) self.mark_opened(path) return LeakTrackingIter(wsgi_iter, self, path)