def _make_request(self, nodes, part, method, path, headers, query,
logger_thread_locals):
self.app.logger.thread_locals = logger_thread_locals
for node in nodes:
try:
with ConnectionTimeout(self.app.conn_timeout):
conn = http_connect(node['ip'],
node['port'],
node['device'],
part,
method,
path,
headers=headers,
query_string=query)
conn.node = node
with Timeout(self.app.node_timeout):
resp = conn.getresponse()
if not is_informational(resp.status) and \
not is_server_error(resp.status):
return resp.status, resp.reason, resp.read()
elif resp.status == HTTP_INSUFFICIENT_STORAGE:
self.error_limit(node)
except (Exception, Timeout):
self.exception_occurred(
node, self.server_type,
_('Trying to %(method)s %(path)s') % {
'method': method,
'path': path
})
def _make_request(self, nodes, part, method, path, headers, query, logger_thread_locals):
self.app.logger.thread_locals = logger_thread_locals
for node in nodes:
try:
with ConnectionTimeout(self.app.conn_timeout):
conn = http_connect(
node["ip"],
node["port"],
node["device"],
part,
method,
path,
headers=headers,
query_string=query,
)
conn.node = node
with Timeout(self.app.node_timeout):
resp = conn.getresponse()
if not is_informational(resp.status) and not is_server_error(resp.status):
return resp.status, resp.reason, resp.read()
elif resp.status == HTTP_INSUFFICIENT_STORAGE:
self.error_limit(node)
except (Exception, Timeout):
self.exception_occurred(
node, self.server_type, _("Trying to %(method)s %(path)s") % {"method": method, "path": path}
)
def _make_request(self, nodes, part, method, path, headers, query,
logger_thread_locals):
"""
Iterates over the given node iterator, sending an HTTP request to one
node at a time. The first non-informational, non-server-error
response is returned. If no non-informational, non-server-error
response is received from any of the nodes, returns None.
:param nodes: an iterator of the backend server and handoff servers
:param part: the partition number
:param method: the method to send to the backend
:param path: the path to send to the backend
(full path ends up being /<$device>/<$part>/<$path>)
:param headers: a list of dicts, where each dict represents one
backend request that should be made.
:param query: query string to send to the backend.
:param logger_thread_locals: The thread local values to be set on the
self.app.logger to retain transaction
logging information.
:returns: a swob.Response object, or None if no responses were received
"""
self.app.logger.thread_locals = logger_thread_locals
for node in nodes:
try:
start_node_timing = time.time()
with ConnectionTimeout(self.app.conn_timeout):
conn = http_connect(node['ip'],
node['port'],
node['device'],
part,
method,
path,
headers=headers,
query_string=query)
conn.node = node
self.app.set_node_timing(node, time.time() - start_node_timing)
with Timeout(self.app.node_timeout):
resp = conn.getresponse()
if not is_informational(resp.status) and \
not is_server_error(resp.status):
return resp.status, resp.reason, resp.getheaders(), \
resp.read()
elif resp.status == HTTP_INSUFFICIENT_STORAGE:
self.app.error_limit(node,
_('ERROR Insufficient Storage'))
except (Exception, Timeout):
self.app.exception_occurred(
node, self.server_type,
_('Trying to %(method)s %(path)s') % {
'method': method,
'path': path
})
def retry(func, *args, **kwargs):
"""
Helper function to retry a given function a number of times.
:param func: callable to be called
:param retries: number of retries
:param error_log: logger for errors
:param args: arguments to send to func
:param kwargs: keyward arguments to send to func (if retries or
error_log are sent, they will be deleted from kwargs
before sending on to func)
:returns: result of func
:raises ClientException: all retries failed
"""
retries = 5
if 'retries' in kwargs:
retries = kwargs['retries']
del kwargs['retries']
error_log = None
if 'error_log' in kwargs:
error_log = kwargs['error_log']
del kwargs['error_log']
attempts = 0
backoff = 1
while attempts <= retries:
attempts += 1
try:
return attempts, func(*args, **kwargs)
except (socket.error, HTTPException, Timeout) as err:
if error_log:
error_log(err)
if attempts > retries:
raise
except ClientException as err:
if error_log:
error_log(err)
if attempts > retries or not is_server_error(err.http_status) or \
err.http_status == HTTP_INSUFFICIENT_STORAGE:
raise
sleep(backoff)
backoff *= 2
# Shouldn't actually get down here, but just in case.
if args and 'ip' in args[0]:
raise ClientException('Raise too many retries',
http_host=args[0]['ip'],
http_port=args[0]['port'],
http_device=args[0]['device'])
else:
raise ClientException('Raise too many retries')
def _make_request(self, nodes, part, method, path, headers, query,
logger_thread_locals):
"""
Sends an HTTP request to a single node and aggregates the result.
It attempts the primary node, then iterates over the handoff nodes
as needed.
:param nodes: an iterator of the backend server and handoff servers
:param part: the partition number
:param method: the method to send to the backend
:param path: the path to send to the backend
:param headers: a list of dicts, where each dict represents one
backend request that should be made.
:param query: query string to send to the backend.
:param logger_thread_locals: The thread local values to be set on the
self.app.logger to retain transaction
logging information.
:returns: a swob.Response object
"""
self.app.logger.thread_locals = logger_thread_locals
for node in nodes:
try:
start_node_timing = time.time()
with ConnectionTimeout(self.app.conn_timeout):
conn = http_connect(node['ip'],
node['port'],
node['device'],
part,
method,
path,
headers=headers,
query_string=query)
conn.node = node
self.app.set_node_timing(node, time.time() - start_node_timing)
with Timeout(self.app.node_timeout):
resp = conn.getresponse()
if not is_informational(resp.status) and \
not is_server_error(resp.status):
return resp.status, resp.reason, resp.getheaders(), \
resp.read()
elif resp.status == HTTP_INSUFFICIENT_STORAGE:
self.error_limit(node, _('ERROR Insufficient Storage'))
except (Exception, Timeout):
self.exception_occurred(
node, self.server_type,
_('Trying to %(method)s %(path)s') % {
'method': method,
'path': path
})
def retry(func, *args, **kwargs):
"""
Helper function to retry a given function a number of times.
:param func: callable to be called
:param retries: number of retries
:param error_log: logger for errors
:param args: arguments to send to func
:param kwargs: keyward arguments to send to func (if retries or
error_log are sent, they will be deleted from kwargs
before sending on to func)
:returns: result of func
:raises ClientException: all retries failed
"""
retries = 5
if 'retries' in kwargs:
retries = kwargs['retries']
del kwargs['retries']
error_log = None
if 'error_log' in kwargs:
error_log = kwargs['error_log']
del kwargs['error_log']
attempts = 0
backoff = 1
while attempts <= retries:
attempts += 1
try:
return attempts, func(*args, **kwargs)
except (socket.error, HTTPException, Timeout) as err:
if error_log:
error_log(err)
if attempts > retries:
raise
except ClientException as err:
if error_log:
error_log(err)
if attempts > retries or not is_server_error(err.http_status) or \
err.http_status == HTTP_INSUFFICIENT_STORAGE:
raise
sleep(backoff)
backoff *= 2
# Shouldn't actually get down here, but just in case.
if args and 'ip' in args[0]:
raise ClientException('Raise too many retries',
http_host=args[
0]['ip'], http_port=args[0]['port'],
http_device=args[0]['device'])
else:
raise ClientException('Raise too many retries')
def get_controller(self, req):
"""
Get the controller to handle a request.
:param req: the request
:returns: tuple of (controller class, path dictionary)
:raises ValueError: (thrown by split_path) if given invalid path
"""
if req.path == '/info':
d = dict(version=None,
expose_info=self.expose_info,
disallowed_sections=self.disallowed_sections,
admin_key=self.admin_key)
return InfoController, d
version, account, container, obj = split_path(wsgi_to_str(req.path), 1,
4, True)
d = dict(version=version,
account_name=account,
container_name=container,
object_name=obj)
if account and not valid_api_version(version):
raise APIVersionError('Invalid path')
if obj and container and account:
info = get_container_info(req.environ, self)
if is_server_error(info.get('status')):
raise HTTPServiceUnavailable(request=req)
policy_index = req.headers.get('X-Backend-Storage-Policy-Index',
info['storage_policy'])
policy = POLICIES.get_by_index(policy_index)
if not policy:
# This indicates that a new policy has been created,
# with rings, deployed, released (i.e. deprecated =
# False), used by a client to create a container via
# another proxy that was restarted after the policy
# was released, and is now cached - all before this
# worker was HUPed to stop accepting new
# connections. There should never be an "unknown"
# index - but when there is - it's probably operator
# error and hopefully temporary.
raise HTTPServiceUnavailable('Unknown Storage Policy')
return self.obj_controller_router[policy], d
elif container and account:
return ContainerController, d
elif account and not container and not obj:
return AccountController, d
return None, d
def _make_request(self, nodes, part, method, path, headers, query, logger_thread_locals):
"""
Iterates over the given node iterator, sending an HTTP request to one
node at a time. The first non-informational, non-server-error
response is returned. If no non-informational, non-server-error
response is received from any of the nodes, returns None.
:param nodes: an iterator of the backend server and handoff servers
:param part: the partition number
:param method: the method to send to the backend
:param path: the path to send to the backend
:param headers: a list of dicts, where each dict represents one
backend request that should be made.
:param query: query string to send to the backend.
:param logger_thread_locals: The thread local values to be set on the
self.app.logger to retain transaction
logging information.
:returns: a swob.Response object, or None if no responses were received
"""
self.app.logger.thread_locals = logger_thread_locals
for node in nodes:
try:
start_node_timing = time.time()
with ConnectionTimeout(self.app.conn_timeout):
conn = http_connect(
node["ip"],
node["port"],
node["device"],
part,
method,
path,
headers=headers,
query_string=query,
)
conn.node = node
self.app.set_node_timing(node, time.time() - start_node_timing)
with Timeout(self.app.node_timeout):
resp = conn.getresponse()
if not is_informational(resp.status) and not is_server_error(resp.status):
return resp.status, resp.reason, resp.getheaders(), resp.read()
elif resp.status == HTTP_INSUFFICIENT_STORAGE:
self.error_limit(node, _("ERROR Insufficient Storage"))
except (Exception, Timeout):
self.exception_occurred(
node, self.server_type, _("Trying to %(method)s %(path)s") % {"method": method, "path": path}
)
def _connect_put_node(self, nodes, part, path, headers,
logger_thread_locals):
"""
Make a connection for a replicated object.
Connects to the first working node that it finds in node_iter
and sends over the request headers. Returns an HTTPConnection
object to handle the rest of the streaming.
"""
self.app.logger.thread_locals = logger_thread_locals
for node in nodes:
try:
start_time = time.time()
with ConnectionTimeout(self.app.conn_timeout):
conn = http_connect(
node['ip'], node['port'], node['device'], part, 'PUT',
path, headers)
self.app.set_node_timing(node, time.time() - start_time)
with Timeout(self.app.node_timeout):
resp = conn.getexpect()
if resp.status == HTTP_CONTINUE:
conn.resp = None
conn.node = node
return conn
elif is_success(resp.status) or resp.status == HTTP_CONFLICT:
conn.resp = resp
conn.node = node
return conn
elif headers['If-None-Match'] is not None and \
resp.status == HTTP_PRECONDITION_FAILED:
conn.resp = resp
conn.node = node
return conn
elif resp.status == HTTP_INSUFFICIENT_STORAGE:
self.app.error_limit(node, _('ERROR Insufficient Storage'))
elif is_server_error(resp.status):
self.app.error_occurred(
node,
_('ERROR %(status)d Expect: 100-continue '
'From Object Server') % {
'status': resp.status})
except (Exception, Timeout):
self.app.exception_occurred(
node, _('Object'),
_('Expect: 100-continue on %s') % path)
def _connect_put_node(self, nodes, part, path, headers,
logger_thread_locals):
"""
Make a connection for a replicated object.
Connects to the first working node that it finds in node_iter
and sends over the request headers. Returns an HTTPConnection
object to handle the rest of the streaming.
"""
self.app.logger.thread_locals = logger_thread_locals
for node in nodes:
print("===In _connect_put_node===", node)
try:
start_time = time.time()
with ConnectionTimeout(self.app.conn_timeout):
conn = http_connect(node['ip'], node['port'],
node['device'], part, 'PUT', path,
headers)
self.app.set_node_timing(node, time.time() - start_time)
with Timeout(self.app.node_timeout):
resp = conn.getexpect()
if resp.status == HTTP_CONTINUE:
conn.resp = None
conn.node = node
return conn
elif is_success(resp.status) or resp.status == HTTP_CONFLICT:
conn.resp = resp
conn.node = node
return conn
elif headers['If-None-Match'] is not None and \
resp.status == HTTP_PRECONDITION_FAILED:
conn.resp = resp
conn.node = node
return conn
elif resp.status == HTTP_INSUFFICIENT_STORAGE:
self.app.error_limit(node, _('ERROR Insufficient Storage'))
elif is_server_error(resp.status):
self.app.error_occurred(
node,
_('ERROR %(status)d Expect: 100-continue '
'From Object Server') % {'status': resp.status})
except (Exception, Timeout):
self.app.exception_occurred(
node, _('Object'),
_('Expect: 100-continue on %s') % path)
def _make_request(self, nodes, part, method, path, headers, query,
logger_thread_locals):
"""
Sends an HTTP request to a single node and aggregates the result.
It attempts the primary node, then iterates over the handoff nodes
as needed.
:param nodes: an iterator of the backend server and handoff servers
:param part: the partition number
:param method: the method to send to the backend
:param path: the path to send to the backend
:param headers: a list of dicts, where each dict represents one
backend request that should be made.
:param query: query string to send to the backend.
:param logger_thread_locals: The thread local values to be set on the
self.app.logger to retain transaction
logging information.
:returns: a swob.Response object
"""
self.app.logger.thread_locals = logger_thread_locals
for node in nodes:
try:
start_node_timing = time.time()
with ConnectionTimeout(self.app.conn_timeout):
conn = http_connect(node['ip'], node['port'],
node['device'], part, method, path,
headers=headers, query_string=query)
conn.node = node
self.app.set_node_timing(node, time.time() - start_node_timing)
with Timeout(self.app.node_timeout):
resp = conn.getresponse()
if not is_informational(resp.status) and \
not is_server_error(resp.status):
return resp.status, resp.reason, resp.getheaders(), \
resp.read()
elif resp.status == HTTP_INSUFFICIENT_STORAGE:
self.error_limit(node, _('ERROR Insufficient Storage'))
except (Exception, Timeout):
self.exception_occurred(node, self.server_type,
_('Trying to %(method)s %(path)s') %
{'method': method, 'path': path})
def _connect_put_node(self, nodes, part, path, headers,
logger_thread_locals):
"""Method for a file PUT connect"""
self.app.logger.thread_locals = logger_thread_locals
for node in nodes:
try:
start_time = time.time()
with ConnectionTimeout(self.app.conn_timeout):
conn = http_connect(node['ip'], node['port'],
node['device'], part, 'PUT', path,
headers)
self.app.set_node_timing(node, time.time() - start_time)
with Timeout(self.app.node_timeout):
resp = conn.getexpect()
if resp.status == HTTP_CONTINUE:
conn.resp = None
conn.node = node
return conn
elif is_success(resp.status) or resp.status == HTTP_CONFLICT:
conn.resp = resp
conn.node = node
return conn
elif headers['If-None-Match'] is not None and \
resp.status == HTTP_PRECONDITION_FAILED:
conn.resp = resp
conn.node = node
return conn
elif resp.status == HTTP_INSUFFICIENT_STORAGE:
self.app.error_limit(node, _('ERROR Insufficient Storage'))
elif is_server_error(resp.status):
self.app.error_occurred(
node,
_('ERROR %(status)d Expect: 100-continue '
'From Object Server') % {'status': resp.status})
except (Exception, Timeout):
self.app.exception_occurred(
node, _('Object'),
_('Expect: 100-continue on %s') % path)
def retry(func, *args, **kwargs):
"""
Helper function to retry a given function a number of times.
:param func: callable to be called
:param retries: number of retries
:param error_log: logger for errors
:param args: arguments to send to func
:param kwargs: keyward arguments to send to func (if retries or
error_log are sent, they will be deleted from kwargs
before sending on to func)
:returns: restult of func
"""
retries = 5
if 'retries' in kwargs:
retries = kwargs['retries']
del kwargs['retries']
error_log = None
if 'error_log' in kwargs:
error_log = kwargs['error_log']
del kwargs['error_log']
attempts = 0
backoff = 1
while attempts <= retries:
attempts += 1
try:
return attempts, func(*args, **kwargs)
except (socket.error, HTTPException, Timeout), err:
if error_log:
error_log(err)
if attempts > retries:
raise
except ClientException, err:
if error_log:
error_log(err)
if attempts > retries or not is_server_error(err.http_status) or \
err.http_status == HTTP_INSUFFICIENT_STORAGE:
raise
def _connect_put_node(self, nodes, part, path, headers,
logger_thread_locals):
"""Method for a file PUT connect"""
self.app.logger.thread_locals = logger_thread_locals
for node in nodes:
try:
start_time = time.time()
with ConnectionTimeout(self.app.conn_timeout):
conn = http_connect(
node['ip'], node['port'], node['device'], part, 'PUT',
path, headers)
self.app.set_node_timing(node, time.time() - start_time)
with Timeout(self.app.node_timeout):
resp = conn.getexpect()
if resp.status == HTTP_CONTINUE:
conn.resp = None
conn.node = node
return conn
elif is_success(resp.status):
conn.resp = resp
conn.node = node
return conn
elif headers['If-None-Match'] is not None and \
resp.status == HTTP_PRECONDITION_FAILED:
conn.resp = resp
conn.node = node
return conn
elif resp.status == HTTP_INSUFFICIENT_STORAGE:
self.app.error_limit(node, _('ERROR Insufficient Storage'))
elif is_server_error(resp.status):
self.app.error_occurred(
node, _('ERROR %(status)d Expect: 100-continue '
'From Object Server') % {
'status': resp.status})
except (Exception, Timeout):
self.app.exception_occurred(
node, _('Object'),
_('Expect: 100-continue on %s') % path)
def retry(func, *args, **kwargs):
"""
Helper function to retry a given function a number of times.
:param func: callable to be called
:param retries: number of retries
:param error_log: logger for errors
:param args: arguments to send to func
:param kwargs: keyward arguments to send to func (if retries or
error_log are sent, they will be deleted from kwargs
before sending on to func)
:returns: restult of func
"""
retries = 5
if 'retries' in kwargs:
retries = kwargs['retries']
del kwargs['retries']
error_log = None
if 'error_log' in kwargs:
error_log = kwargs['error_log']
del kwargs['error_log']
attempts = 0
backoff = 1
while attempts <= retries:
attempts += 1
try:
return attempts, func(*args, **kwargs)
except (socket.error, HTTPException, Timeout), err:
if error_log:
error_log(err)
if attempts > retries:
raise
except ClientException, err:
if error_log:
error_log(err)
if attempts > retries or not is_server_error(err.http_status) or \
err.http_status == HTTP_INSUFFICIENT_STORAGE:
raise
def _make_request(self, nodes, part, method, path, headers, query,
logger_thread_locals):
self.app.logger.thread_locals = logger_thread_locals
for node in nodes:
try:
start_node_timing = time.time()
with ConnectionTimeout(self.app.conn_timeout):
conn = http_connect(node['ip'], node['port'],
node['device'], part, method, path,
headers=headers, query_string=query)
conn.node = node
self.app.set_node_timing(node, time.time() - start_node_timing)
with Timeout(self.app.node_timeout):
resp = conn.getresponse()
if not is_informational(resp.status) and \
not is_server_error(resp.status):
return resp.status, resp.reason, resp.read()
elif resp.status == HTTP_INSUFFICIENT_STORAGE:
self.error_limit(node, _('ERROR Insufficient Storage'))
except (Exception, Timeout):
self.exception_occurred(node, self.server_type,
_('Trying to %(method)s %(path)s') %
{'method': method, 'path': path})
def GETorHEAD_base(self, req, server_type, partition, nodes, path, attempts):
"""
Base handler for HTTP GET or HEAD requests.
:param req: swob.Request object
:param server_type: server type
:param partition: partition
:param nodes: nodes
:param path: path for the request
:param attempts: number of attempts to try
:returns: swob.Response object
"""
statuses = []
reasons = []
bodies = []
sources = []
newest = config_true_value(req.headers.get("x-newest", "f"))
nodes = iter(nodes)
while len(statuses) < attempts:
try:
node = nodes.next()
except StopIteration:
break
if self.error_limited(node):
continue
try:
with ConnectionTimeout(self.app.conn_timeout):
headers = dict(req.headers)
headers["Connection"] = "close"
conn = http_connect(
node["ip"],
node["port"],
node["device"],
partition,
req.method,
path,
headers=headers,
query_string=req.query_string,
)
with Timeout(self.app.node_timeout):
possible_source = conn.getresponse()
# See NOTE: swift_conn at top of file about this.
possible_source.swift_conn = conn
except (Exception, Timeout):
self.exception_occurred(
node, server_type, _("Trying to %(method)s %(path)s") % {"method": req.method, "path": req.path}
)
continue
if self.is_good_source(possible_source):
# 404 if we know we don't have a synced copy
if not float(possible_source.getheader("X-PUT-Timestamp", 1)):
statuses.append(HTTP_NOT_FOUND)
reasons.append("")
bodies.append("")
self.close_swift_conn(possible_source)
else:
statuses.append(possible_source.status)
reasons.append(possible_source.reason)
bodies.append("")
sources.append(possible_source)
if not newest: # one good source is enough
break
else:
statuses.append(possible_source.status)
reasons.append(possible_source.reason)
bodies.append(possible_source.read())
if possible_source.status == HTTP_INSUFFICIENT_STORAGE:
self.error_limit(node)
elif is_server_error(possible_source.status):
self.error_occurred(
node,
_("ERROR %(status)d %(body)s " "From %(type)s Server")
% {"status": possible_source.status, "body": bodies[-1][:1024], "type": server_type},
)
if sources:
sources.sort(key=source_key)
source = sources.pop()
for src in sources:
self.close_swift_conn(src)
res = Response(request=req, conditional_response=True)
if req.method == "GET" and source.status in (HTTP_OK, HTTP_PARTIAL_CONTENT):
res.app_iter = self._make_app_iter(node, source)
# See NOTE: swift_conn at top of file about this.
res.swift_conn = source.swift_conn
res.status = source.status
update_headers(res, source.getheaders())
if not res.environ:
res.environ = {}
res.environ["swift_x_timestamp"] = source.getheader("x-timestamp")
res.accept_ranges = "bytes"
res.content_length = source.getheader("Content-Length")
if source.getheader("Content-Type"):
res.charset = None
res.content_type = source.getheader("Content-Type")
return res
return self.best_response(req, statuses, reasons, bodies, "%s %s" % (server_type, req.method))
def make_request(
self, method, path, headers, acceptable_statuses, body_file=None,
params=None):
"""Makes a request to Swift with retries.
:param method: HTTP method of request.
:param path: Path of request.
:param headers: Headers to be sent with request.
:param acceptable_statuses: List of acceptable statuses for request.
:param body_file: Body file to be passed along with request,
defaults to None.
:param params: A dict of params to be set in request query string,
defaults to None.
:returns: Response object on success.
:raises UnexpectedResponse: Exception raised when make_request() fails
to get a response with an acceptable status
:raises Exception: Exception is raised when code fails in an
unexpected way.
"""
headers = dict(headers)
headers['user-agent'] = self.user_agent
for attempt in range(self.request_tries):
resp = exc_type = exc_value = exc_traceback = None
req = Request.blank(
path, environ={'REQUEST_METHOD': method}, headers=headers)
if body_file is not None:
if hasattr(body_file, 'seek'):
body_file.seek(0)
req.body_file = body_file
if params:
req.params = params
try:
resp = req.get_response(self.app)
except (Exception, Timeout):
exc_type, exc_value, exc_traceback = exc_info()
else:
if resp.status_int in acceptable_statuses or \
resp.status_int // 100 in acceptable_statuses:
return resp
elif not is_server_error(resp.status_int):
# No sense retrying when we expect the same result
break
# sleep only between tries, not after each one
if attempt < self.request_tries - 1:
if resp:
# always close any resp.app_iter before we discard it
with closing_if_possible(resp.app_iter):
# for non 2XX requests it's safe and useful to drain
# the response body so we log the correct status code
if resp.status_int // 100 != 2:
for iter_body in resp.app_iter:
pass
sleep(2 ** (attempt + 1))
if resp:
msg = 'Unexpected response: %s' % resp.status
if resp.status_int // 100 != 2 and resp.body:
# provide additional context (and drain the response body) for
# non 2XX responses
msg += ' (%s)' % resp.body
raise UnexpectedResponse(msg, resp)
if exc_type:
# To make pep8 tool happy, in place of raise t, v, tb:
six.reraise(exc_type, exc_value, exc_traceback)
def GETorHEAD_base(self, req, server_type, ring, partition, path):
"""
Base handler for HTTP GET or HEAD requests.
:param req: swob.Request object
:param server_type: server type
:param ring: the ring to obtain nodes from
:param partition: partition
:param path: path for the request
:returns: swob.Response object
"""
statuses = []
reasons = []
bodies = []
source_headers = []
sources = []
newest = config_true_value(req.headers.get('x-newest', 'f'))
headers = self.generate_request_headers(req, additional=req.headers)
for node in self.iter_nodes(ring, partition):
start_node_timing = time.time()
try:
with ConnectionTimeout(self.app.conn_timeout):
conn = http_connect(
node['ip'], node['port'], node['device'], partition,
req.method, path, headers=headers,
query_string=req.query_string)
self.app.set_node_timing(node, time.time() - start_node_timing)
with Timeout(self.app.node_timeout):
possible_source = conn.getresponse()
# See NOTE: swift_conn at top of file about this.
possible_source.swift_conn = conn
except (Exception, Timeout):
self.exception_occurred(
node, server_type, _('Trying to %(method)s %(path)s') %
{'method': req.method, 'path': req.path})
continue
if self.is_good_source(possible_source):
# 404 if we know we don't have a synced copy
if not float(possible_source.getheader('X-PUT-Timestamp', 1)):
statuses.append(HTTP_NOT_FOUND)
reasons.append('')
bodies.append('')
source_headers.append('')
self.close_swift_conn(possible_source)
else:
statuses.append(possible_source.status)
reasons.append(possible_source.reason)
bodies.append('')
source_headers.append('')
sources.append((possible_source, node))
if not newest: # one good source is enough
break
else:
statuses.append(possible_source.status)
reasons.append(possible_source.reason)
bodies.append(possible_source.read())
source_headers.append(possible_source.getheaders())
if possible_source.status == HTTP_INSUFFICIENT_STORAGE:
self.error_limit(node, _('ERROR Insufficient Storage'))
elif is_server_error(possible_source.status):
self.error_occurred(node, _('ERROR %(status)d %(body)s '
'From %(type)s Server') %
{'status': possible_source.status,
'body': bodies[-1][:1024],
'type': server_type})
res = None
if sources:
sources.sort(key=lambda s: source_key(s[0]))
source, node = sources.pop()
for src, _junk in sources:
self.close_swift_conn(src)
res = Response(request=req)
if req.method == 'GET' and \
source.status in (HTTP_OK, HTTP_PARTIAL_CONTENT):
res.app_iter = self._make_app_iter(node, source)
# See NOTE: swift_conn at top of file about this.
res.swift_conn = source.swift_conn
res.status = source.status
update_headers(res, source.getheaders())
if not res.environ:
res.environ = {}
res.environ['swift_x_timestamp'] = \
source.getheader('x-timestamp')
res.accept_ranges = 'bytes'
res.content_length = source.getheader('Content-Length')
if source.getheader('Content-Type'):
res.charset = None
res.content_type = source.getheader('Content-Type')
if not res:
res = self.best_response(req, statuses, reasons, bodies,
'%s %s' % (server_type, req.method),
headers=source_headers)
try:
(account, container) = split_path(req.path_info, 1, 2)
_set_info_cache(self.app, req.environ, account, container, res)
except ValueError:
pass
try:
(account, container, obj) = split_path(req.path_info, 3, 3, True)
_set_object_info_cache(self.app, req.environ, account,
container, obj, res)
except ValueError:
pass
return res
def account_info(self, account, autocreate=False):
"""
Get account information, and also verify that the account exists.
:param account: name of the account to get the info for
:returns: tuple of (account partition, account nodes, container_count)
or (None, None, None) if it does not exist
"""
partition, nodes = self.app.account_ring.get_nodes(account)
account_info = {'status': 0,
'container_count': 0,
'total_object_count': None,
'bytes': None,
'meta': {}}
# 0 = no responses, 200 = found, 404 = not found, -1 = mixed responses
if self.app.memcache:
cache_key = get_account_memcache_key(account)
cache_value = self.app.memcache.get(cache_key)
if not isinstance(cache_value, dict):
result_code = cache_value
container_count = 0
else:
result_code = cache_value['status']
try:
container_count = int(cache_value['container_count'])
except ValueError:
container_count = 0
if result_code == HTTP_OK:
return partition, nodes, container_count
elif result_code == HTTP_NOT_FOUND and not autocreate:
return None, None, None
result_code = 0
path = '/%s' % account
headers = {'x-trans-id': self.trans_id, 'Connection': 'close'}
for node in self.iter_nodes(self.app.account_ring, partition):
try:
start_node_timing = time.time()
with ConnectionTimeout(self.app.conn_timeout):
conn = http_connect(node['ip'], node['port'],
node['device'], partition, 'HEAD',
path, headers)
self.app.set_node_timing(node, time.time() - start_node_timing)
with Timeout(self.app.node_timeout):
resp = conn.getresponse()
body = resp.read()
if is_success(resp.status):
result_code = HTTP_OK
account_info.update(
headers_to_account_info(resp.getheaders()))
break
elif resp.status == HTTP_NOT_FOUND:
if result_code == 0:
result_code = HTTP_NOT_FOUND
elif result_code != HTTP_NOT_FOUND:
result_code = -1
elif resp.status == HTTP_INSUFFICIENT_STORAGE:
self.error_limit(node, _('ERROR Insufficient Storage'))
continue
else:
result_code = -1
if is_server_error(resp.status):
self.error_occurred(
node,
_('ERROR %(status)d %(body)s From Account '
'Server') %
{'status': resp.status, 'body': body[:1024]})
except (Exception, Timeout):
self.exception_occurred(node, _('Account'),
_('Trying to get account info for %s')
% path)
if result_code == HTTP_NOT_FOUND and autocreate:
if len(account) > MAX_ACCOUNT_NAME_LENGTH:
return None, None, None
headers = {'X-Timestamp': normalize_timestamp(time.time()),
'X-Trans-Id': self.trans_id,
'Connection': 'close'}
resp = self.make_requests(Request.blank('/v1' + path),
self.app.account_ring, partition, 'PUT',
path, [headers] * len(nodes))
if not is_success(resp.status_int):
self.app.logger.warning('Could not autocreate account %r' %
path)
return None, None, None
result_code = HTTP_OK
if self.app.memcache and result_code in (HTTP_OK, HTTP_NOT_FOUND):
if result_code == HTTP_OK:
cache_timeout = self.app.recheck_account_existence
else:
cache_timeout = self.app.recheck_account_existence * 0.1
account_info.update(status=result_code)
self.app.memcache.set(cache_key,
account_info,
time=cache_timeout)
if result_code == HTTP_OK:
try:
container_count = int(account_info['container_count'])
except ValueError:
container_count = 0
return partition, nodes, container_count
return None, None, None
def container_info(self, account, container, account_autocreate=False):
"""
Get container information and thusly verify container existence.
This will also make a call to account_info to verify that the
account exists.
:param account: account name for the container
:param container: container name to look up
:returns: dict containing at least container partition ('partition'),
container nodes ('containers'), container read
acl ('read_acl'), container write acl ('write_acl'),
and container sync key ('sync_key').
Values are set to None if the container does not exist.
"""
part, nodes = self.app.container_ring.get_nodes(account, container)
path = '/%s/%s' % (account, container)
container_info = {'status': 0, 'read_acl': None,
'write_acl': None, 'sync_key': None,
'count': None, 'bytes': None,
'versions': None, 'partition': None,
'nodes': None}
if self.app.memcache:
cache_key = get_container_memcache_key(account, container)
cache_value = self.app.memcache.get(cache_key)
if isinstance(cache_value, dict):
if 'container_size' in cache_value:
cache_value['count'] = cache_value['container_size']
if is_success(cache_value['status']):
container_info.update(cache_value)
container_info['partition'] = part
container_info['nodes'] = nodes
return container_info
if not self.account_info(account, autocreate=account_autocreate)[1]:
return container_info
headers = {'x-trans-id': self.trans_id, 'Connection': 'close'}
for node in self.iter_nodes(self.app.container_ring, part):
try:
start_node_timing = time.time()
with ConnectionTimeout(self.app.conn_timeout):
conn = http_connect(node['ip'], node['port'],
node['device'], part, 'HEAD',
path, headers)
self.app.set_node_timing(node, time.time() - start_node_timing)
with Timeout(self.app.node_timeout):
resp = conn.getresponse()
body = resp.read()
if is_success(resp.status):
container_info.update(
headers_to_container_info(resp.getheaders()))
break
elif resp.status == HTTP_NOT_FOUND:
container_info['status'] = HTTP_NOT_FOUND
else:
container_info['status'] = -1
if resp.status == HTTP_INSUFFICIENT_STORAGE:
self.error_limit(node, _('ERROR Insufficient Storage'))
elif is_server_error(resp.status):
self.error_occurred(node, _(
'ERROR %(status)d %(body)s From Container '
'Server') %
{'status': resp.status, 'body': body[:1024]})
except (Exception, Timeout):
self.exception_occurred(
node, _('Container'),
_('Trying to get container info for %s') % path)
if self.app.memcache:
if container_info['status'] == HTTP_OK:
self.app.memcache.set(
cache_key, container_info,
time=self.app.recheck_container_existence)
elif container_info['status'] == HTTP_NOT_FOUND:
self.app.memcache.set(
cache_key, container_info,
time=self.app.recheck_container_existence * 0.1)
if container_info['status'] == HTTP_OK:
container_info['partition'] = part
container_info['nodes'] = nodes
return container_info
def _requests_to_bytes_iter(self):
# Take the requests out of self._coalesce_requests, actually make
# the requests, and generate the bytes from the responses.
#
# Yields 2-tuples (segment-name, byte-chunk). The segment name is
# used for logging.
for data_or_req, seg_etag, seg_size in self._coalesce_requests():
if isinstance(data_or_req, bytes): # ugly, awful overloading
yield ('data segment', data_or_req)
continue
seg_req = data_or_req
seg_resp = seg_req.get_response(self.app)
if not is_success(seg_resp.status_int):
# Error body should be short
body = seg_resp.body
if not six.PY2:
body = body.decode('utf8')
msg = 'While processing manifest %s, got %d (%s) ' \
'while retrieving %s' % (
self.name, seg_resp.status_int,
body if len(body) <= 60 else body[:57] + '...',
seg_req.path)
if is_server_error(seg_resp.status_int):
self.logger.error(msg)
raise HTTPServiceUnavailable(request=seg_req,
content_type='text/plain')
raise SegmentError(msg)
elif (
(seg_etag and (seg_resp.etag != seg_etag)) or
(seg_size and
(seg_resp.content_length != seg_size) and not seg_req.range)):
# The content-length check is for security reasons. Seems
# possible that an attacker could upload a >1mb object and
# then replace it with a much smaller object with same
# etag. Then create a big nested SLO that calls that
# object many times which would hammer our obj servers. If
# this is a range request, don't check content-length
# because it won't match.
close_if_possible(seg_resp.app_iter)
raise SegmentError(
'Object segment no longer valid: '
'%(path)s etag: %(r_etag)s != %(s_etag)s or '
'%(r_size)s != %(s_size)s.' % {
'path': seg_req.path,
'r_etag': seg_resp.etag,
'r_size': seg_resp.content_length,
's_etag': seg_etag,
's_size': seg_size
})
else:
self.current_resp = seg_resp
resp_len = 0
seg_hash = None
if seg_resp.etag and not seg_req.headers.get('Range'):
# Only calculate the MD5 if it we can use it to validate
seg_hash = md5(usedforsecurity=False)
document_iters = maybe_multipart_byteranges_to_document_iters(
seg_resp.app_iter, seg_resp.headers['Content-Type'])
for chunk in itertools.chain.from_iterable(document_iters):
if seg_hash:
seg_hash.update(chunk)
resp_len += len(chunk)
yield (seg_req.path, chunk)
close_if_possible(seg_resp.app_iter)
if seg_hash:
if resp_len != seg_resp.content_length:
raise SegmentError(
"Bad response length for %(seg)s as part of %(name)s: "
"headers had %(from_headers)s, but response length "
"was actually %(actual)s" % {
'seg': seg_req.path,
'from_headers': seg_resp.content_length,
'name': self.name,
'actual': resp_len
})
if seg_hash.hexdigest() != seg_resp.etag:
raise SegmentError(
"Bad MD5 checksum for %(seg)s as part of %(name)s: "
"headers had %(etag)s, but object MD5 was actually "
"%(actual)s" % {
'seg': seg_req.path,
'etag': seg_resp.etag,
'name': self.name,
'actual': seg_hash.hexdigest()
})
def _get_source_and_node(self):
self.statuses = []
self.reasons = []
self.bodies = []
self.source_headers = []
sources = []
for node in self.app.iter_nodes(self.ring, self.partition):
if node in self.used_nodes:
continue
start_node_timing = time.time()
try:
with ConnectionTimeout(self.app.conn_timeout):
conn = http_connect(node['ip'],
node['port'],
node['device'],
self.partition,
self.req_method,
self.path,
headers=self.backend_headers,
query_string=self.req_query_string)
self.app.set_node_timing(node, time.time() - start_node_timing)
with Timeout(self.app.node_timeout):
possible_source = conn.getresponse()
# See NOTE: swift_conn at top of file about this.
possible_source.swift_conn = conn
except (Exception, Timeout):
self.app.exception_occurred(
node, self.server_type,
_('Trying to %(method)s %(path)s') % {
'method': self.req_method,
'path': self.req_path
})
continue
if self.is_good_source(possible_source):
# 404 if we know we don't have a synced copy
if not float(possible_source.getheader('X-PUT-Timestamp', 1)):
self.statuses.append(HTTP_NOT_FOUND)
self.reasons.append('')
self.bodies.append('')
self.source_headers.append('')
close_swift_conn(possible_source)
else:
if self.used_source_etag:
src_headers = dict(
(k.lower(), v)
for k, v in possible_source.getheaders())
if src_headers.get('etag', '').strip('"') != \
self.used_source_etag:
self.statuses.append(HTTP_NOT_FOUND)
self.reasons.append('')
self.bodies.append('')
self.source_headers.append('')
continue
self.statuses.append(possible_source.status)
self.reasons.append(possible_source.reason)
self.bodies.append('')
self.source_headers.append('')
sources.append((possible_source, node))
if not self.newest: # one good source is enough
break
else:
self.statuses.append(possible_source.status)
self.reasons.append(possible_source.reason)
self.bodies.append(possible_source.read())
self.source_headers.append(possible_source.getheaders())
if possible_source.status == HTTP_INSUFFICIENT_STORAGE:
self.app.error_limit(node, _('ERROR Insufficient Storage'))
elif is_server_error(possible_source.status):
self.app.error_occurred(
node,
_('ERROR %(status)d %(body)s '
'From %(type)s Server') % {
'status': possible_source.status,
'body': self.bodies[-1][:1024],
'type': self.server_type
})
if sources:
sources.sort(key=lambda s: source_key(s[0]))
source, node = sources.pop()
for src, _junk in sources:
close_swift_conn(src)
self.used_nodes.append(node)
src_headers = dict(
(k.lower(), v) for k, v in possible_source.getheaders())
self.used_source_etag = src_headers.get('etag', '').strip('"')
return source, node
return None, None
def GETorHEAD_base(self, req, server_type, ring, partition, path):
"""
Base handler for HTTP GET or HEAD requests.
:param req: swob.Request object
:param server_type: server type
:param ring: the ring to obtain nodes from
:param partition: partition
:param path: path for the request
:returns: swob.Response object
"""
statuses = []
reasons = []
bodies = []
sources = []
newest = config_true_value(req.headers.get('x-newest', 'f'))
for node in self.iter_nodes(ring, partition):
start_node_timing = time.time()
try:
with ConnectionTimeout(self.app.conn_timeout):
headers = dict(req.headers)
headers['Connection'] = 'close'
conn = http_connect(node['ip'],
node['port'],
node['device'],
partition,
req.method,
path,
headers=headers,
query_string=req.query_string)
self.app.set_node_timing(node, time.time() - start_node_timing)
with Timeout(self.app.node_timeout):
possible_source = conn.getresponse()
# See NOTE: swift_conn at top of file about this.
possible_source.swift_conn = conn
except (Exception, Timeout):
self.exception_occurred(
node, server_type,
_('Trying to %(method)s %(path)s') % {
'method': req.method,
'path': req.path
})
continue
if self.is_good_source(possible_source):
# 404 if we know we don't have a synced copy
if not float(possible_source.getheader('X-PUT-Timestamp', 1)):
statuses.append(HTTP_NOT_FOUND)
reasons.append('')
bodies.append('')
self.close_swift_conn(possible_source)
else:
statuses.append(possible_source.status)
reasons.append(possible_source.reason)
bodies.append('')
sources.append((possible_source, node))
if not newest: # one good source is enough
break
else:
statuses.append(possible_source.status)
reasons.append(possible_source.reason)
bodies.append(possible_source.read())
if possible_source.status == HTTP_INSUFFICIENT_STORAGE:
self.error_limit(node, _('ERROR Insufficient Storage'))
elif is_server_error(possible_source.status):
self.error_occurred(
node,
_('ERROR %(status)d %(body)s '
'From %(type)s Server') % {
'status': possible_source.status,
'body': bodies[-1][:1024],
'type': server_type
})
if sources:
sources.sort(key=lambda s: source_key(s[0]))
source, node = sources.pop()
for src, _junk in sources:
self.close_swift_conn(src)
res = Response(request=req, conditional_response=True)
if req.method == 'GET' and \
source.status in (HTTP_OK, HTTP_PARTIAL_CONTENT):
res.app_iter = self._make_app_iter(node, source)
# See NOTE: swift_conn at top of file about this.
res.swift_conn = source.swift_conn
res.status = source.status
update_headers(res, source.getheaders())
if not res.environ:
res.environ = {}
res.environ['swift_x_timestamp'] = \
source.getheader('x-timestamp')
res.accept_ranges = 'bytes'
res.content_length = source.getheader('Content-Length')
if source.getheader('Content-Type'):
res.charset = None
res.content_type = source.getheader('Content-Type')
return res
return self.best_response(req, statuses, reasons, bodies,
'%s %s' % (server_type, req.method))
def _get_source_and_node(self):
self.statuses = []
self.reasons = []
self.bodies = []
self.source_headers = []
sources = []
node_timeout = self.app.node_timeout
if self.server_type == 'Object' and not self.newest:
node_timeout = self.app.recoverable_node_timeout
for node in self.app.iter_nodes(self.ring, self.partition):
if node in self.used_nodes:
continue
start_node_timing = time.time()
try:
with ConnectionTimeout(self.app.conn_timeout):
conn = http_connect(
node['ip'], node['port'], node['device'],
self.partition, self.req_method, self.path,
headers=self.backend_headers,
query_string=self.req_query_string)
self.app.set_node_timing(node, time.time() - start_node_timing)
with Timeout(node_timeout):
possible_source = conn.getresponse()
# See NOTE: swift_conn at top of file about this.
possible_source.swift_conn = conn
except (Exception, Timeout):
self.app.exception_occurred(
node, self.server_type,
_('Trying to %(method)s %(path)s') %
{'method': self.req_method, 'path': self.req_path})
continue
if self.is_good_source(possible_source):
# 404 if we know we don't have a synced copy
if not float(possible_source.getheader('X-PUT-Timestamp', 1)):
self.statuses.append(HTTP_NOT_FOUND)
self.reasons.append('')
self.bodies.append('')
self.source_headers.append('')
close_swift_conn(possible_source)
else:
if self.used_source_etag:
src_headers = dict(
(k.lower(), v) for k, v in
possible_source.getheaders())
if src_headers.get('etag', '').strip('"') != \
self.used_source_etag:
self.statuses.append(HTTP_NOT_FOUND)
self.reasons.append('')
self.bodies.append('')
self.source_headers.append('')
continue
self.statuses.append(possible_source.status)
self.reasons.append(possible_source.reason)
self.bodies.append('')
self.source_headers.append('')
sources.append((possible_source, node))
if not self.newest: # one good source is enough
break
else:
self.statuses.append(possible_source.status)
self.reasons.append(possible_source.reason)
self.bodies.append(possible_source.read())
self.source_headers.append(possible_source.getheaders())
if possible_source.status == HTTP_INSUFFICIENT_STORAGE:
self.app.error_limit(node, _('ERROR Insufficient Storage'))
elif is_server_error(possible_source.status):
self.app.error_occurred(
node, _('ERROR %(status)d %(body)s '
'From %(type)s Server') %
{'status': possible_source.status,
'body': self.bodies[-1][:1024],
'type': self.server_type})
if sources:
sources.sort(key=lambda s: source_key(s[0]))
source, node = sources.pop()
for src, _junk in sources:
close_swift_conn(src)
self.used_nodes.append(node)
src_headers = dict(
(k.lower(), v) for k, v in
possible_source.getheaders())
self.used_source_etag = src_headers.get('etag', '').strip('"')
return source, node
return None, None
def container_info(self, account, container, account_autocreate=False):
"""
Get container information and thusly verify container existence.
This will also make a call to account_info to verify that the
account exists.
:param account: account name for the container
:param container: container name to look up
:returns: dict containing at least container partition ('partition'),
container nodes ('containers'), container read
acl ('read_acl'), container write acl ('write_acl'),
and container sync key ('sync_key').
Values are set to None if the container does not exist.
"""
part, nodes = self.app.container_ring.get_nodes(account, container)
path = '/%s/%s' % (account, container)
container_info = {
'status': 0,
'read_acl': None,
'write_acl': None,
'sync_key': None,
'count': None,
'bytes': None,
'versions': None,
'partition': None,
'nodes': None
}
if self.app.memcache:
cache_key = get_container_memcache_key(account, container)
cache_value = self.app.memcache.get(cache_key)
if isinstance(cache_value, dict):
if 'container_size' in cache_value:
cache_value['count'] = cache_value['container_size']
if is_success(cache_value['status']):
container_info.update(cache_value)
container_info['partition'] = part
container_info['nodes'] = nodes
return container_info
if not self.account_info(account, autocreate=account_autocreate)[1]:
return container_info
headers = {'x-trans-id': self.trans_id, 'Connection': 'close'}
for node in self.iter_nodes(self.app.container_ring, part):
try:
start_node_timing = time.time()
with ConnectionTimeout(self.app.conn_timeout):
conn = http_connect(node['ip'], node['port'],
node['device'], part, 'HEAD', path,
headers)
self.app.set_node_timing(node, time.time() - start_node_timing)
with Timeout(self.app.node_timeout):
resp = conn.getresponse()
body = resp.read()
if is_success(resp.status):
container_info.update(
headers_to_container_info(resp.getheaders()))
break
elif resp.status == HTTP_NOT_FOUND:
container_info['status'] = HTTP_NOT_FOUND
else:
container_info['status'] = -1
if resp.status == HTTP_INSUFFICIENT_STORAGE:
self.error_limit(node, _('ERROR Insufficient Storage'))
elif is_server_error(resp.status):
self.error_occurred(
node,
_('ERROR %(status)d %(body)s From Container '
'Server') % {
'status': resp.status,
'body': body[:1024]
})
except (Exception, Timeout):
self.exception_occurred(
node, _('Container'),
_('Trying to get container info for %s') % path)
if self.app.memcache:
if container_info['status'] == HTTP_OK:
self.app.memcache.set(
cache_key,
container_info,
time=self.app.recheck_container_existence)
elif container_info['status'] == HTTP_NOT_FOUND:
self.app.memcache.set(
cache_key,
container_info,
time=self.app.recheck_container_existence * 0.1)
if container_info['status'] == HTTP_OK:
container_info['partition'] = part
container_info['nodes'] = nodes
return container_info
def account_info(self, account, autocreate=False):
"""
Get account information, and also verify that the account exists.
:param account: name of the account to get the info for
:returns: tuple of (account partition, account nodes, container_count)
or (None, None, None) if it does not exist
"""
partition, nodes = self.app.account_ring.get_nodes(account)
account_info = {
'status': 0,
'container_count': 0,
'total_object_count': None,
'bytes': None,
'meta': {}
}
# 0 = no responses, 200 = found, 404 = not found, -1 = mixed responses
if self.app.memcache:
cache_key = get_account_memcache_key(account)
cache_value = self.app.memcache.get(cache_key)
if not isinstance(cache_value, dict):
result_code = cache_value
container_count = 0
else:
result_code = cache_value['status']
try:
container_count = int(cache_value['container_count'])
except ValueError:
container_count = 0
if result_code == HTTP_OK:
return partition, nodes, container_count
elif result_code == HTTP_NOT_FOUND and not autocreate:
return None, None, None
result_code = 0
path = '/%s' % account
headers = {'x-trans-id': self.trans_id, 'Connection': 'close'}
for node in self.iter_nodes(self.app.account_ring, partition):
try:
start_node_timing = time.time()
with ConnectionTimeout(self.app.conn_timeout):
conn = http_connect(node['ip'], node['port'],
node['device'], partition, 'HEAD',
path, headers)
self.app.set_node_timing(node, time.time() - start_node_timing)
with Timeout(self.app.node_timeout):
resp = conn.getresponse()
body = resp.read()
if is_success(resp.status):
result_code = HTTP_OK
account_info.update(
headers_to_account_info(resp.getheaders()))
break
elif resp.status == HTTP_NOT_FOUND:
if result_code == 0:
result_code = HTTP_NOT_FOUND
elif result_code != HTTP_NOT_FOUND:
result_code = -1
elif resp.status == HTTP_INSUFFICIENT_STORAGE:
self.error_limit(node, _('ERROR Insufficient Storage'))
continue
else:
result_code = -1
if is_server_error(resp.status):
self.error_occurred(
node,
_('ERROR %(status)d %(body)s From Account '
'Server') % {
'status': resp.status,
'body': body[:1024]
})
except (Exception, Timeout):
self.exception_occurred(
node, _('Account'),
_('Trying to get account info for %s') % path)
if result_code == HTTP_NOT_FOUND and autocreate:
if len(account) > MAX_ACCOUNT_NAME_LENGTH:
return None, None, None
headers = {
'X-Timestamp': normalize_timestamp(time.time()),
'X-Trans-Id': self.trans_id,
'Connection': 'close'
}
resp = self.make_requests(Request.blank('/v1' + path),
self.app.account_ring, partition, 'PUT',
path, [headers] * len(nodes))
if not is_success(resp.status_int):
self.app.logger.warning('Could not autocreate account %r' %
path)
return None, None, None
result_code = HTTP_OK
if self.app.memcache and result_code in (HTTP_OK, HTTP_NOT_FOUND):
if result_code == HTTP_OK:
cache_timeout = self.app.recheck_account_existence
else:
cache_timeout = self.app.recheck_account_existence * 0.1
account_info.update(status=result_code)
self.app.memcache.set(cache_key, account_info, time=cache_timeout)
if result_code == HTTP_OK:
try:
container_count = int(account_info['container_count'])
except ValueError:
container_count = 0
return partition, nodes, container_count
return None, None, None
def account_info(self, account, req=None):
"""
Get account information, and also verify that the account exists.
:param account: name of the account to get the info for
:param req: caller's HTTP request context object (optional)
:param autocreate: whether or not to automatically create the given
account or not (optional, default: False)
:returns: tuple of (account partition, account nodes, container_count)
or (None, None, None) if it does not exist
"""
partition, nodes = self.app.account_ring.get_nodes(account)
account_info = {
'status': 0,
'container_count': 0,
'total_object_count': None,
'bytes': None,
'meta': {}
}
# 0 = no responses, 200 = found, 404 = not found, -1 = mixed responses
if self.app.memcache:
cache_key = get_account_memcache_key(account)
cache_value = self.app.memcache.get(cache_key)
if not isinstance(cache_value, dict):
result_code = cache_value
container_count = 0
else:
result_code = cache_value['status']
try:
container_count = int(cache_value['container_count'])
except ValueError:
container_count = 0
if result_code == HTTP_OK:
return partition, nodes, container_count
elif result_code == HTTP_NOT_FOUND:
return None, None, None
result_code = 0
path = '/%s' % account
headers = self.generate_request_headers(req)
for node in self.iter_nodes(self.app.account_ring, partition):
try:
start_node_timing = time.time()
with ConnectionTimeout(self.app.conn_timeout):
conn = http_connect(node['ip'], node['port'],
node['device'], partition, 'HEAD',
path, headers)
self.app.set_node_timing(node, time.time() - start_node_timing)
with Timeout(self.app.node_timeout):
resp = conn.getresponse()
body = resp.read()
if is_success(resp.status):
result_code = HTTP_OK
account_info.update(
headers_to_account_info(resp.getheaders()))
break
elif resp.status == HTTP_NOT_FOUND:
if result_code == 0:
result_code = HTTP_NOT_FOUND
elif result_code != HTTP_NOT_FOUND:
result_code = -1
elif resp.status == HTTP_INSUFFICIENT_STORAGE:
self.error_limit(node, _('ERROR Insufficient Storage'))
continue
else:
result_code = -1
if is_server_error(resp.status):
self.error_occurred(
node,
_('ERROR %(status)d %(body)s From Account '
'Server') % {
'status': resp.status,
'body': body[:1024]
})
except (Exception, Timeout):
self.exception_occurred(
node, _('Account'),
_('Trying to get account info for %s') % path)
if self.app.memcache and result_code in (HTTP_OK, HTTP_NOT_FOUND):
if result_code == HTTP_OK:
cache_timeout = self.app.recheck_account_existence
else:
cache_timeout = self.app.recheck_account_existence * 0.1
account_info.update(status=result_code)
self.app.memcache.set(cache_key, account_info, time=cache_timeout)
if result_code == HTTP_OK:
try:
container_count = int(account_info['container_count'])
except ValueError:
container_count = 0
return partition, nodes, container_count
return None, None, None
def GETorHEAD_base(self, req, server_type, partition, nodes, path,
attempts):
"""
Base handler for HTTP GET or HEAD requests.
:param req: webob.Request object
:param server_type: server type
:param partition: partition
:param nodes: nodes
:param path: path for the request
:param attempts: number of attempts to try
:returns: webob.Response object
"""
statuses = []
reasons = []
bodies = []
source = None
sources = []
newest = req.headers.get('x-newest', 'f').lower() in TRUE_VALUES
nodes = iter(nodes)
while len(statuses) < attempts:
try:
node = nodes.next()
except StopIteration:
break
if self.error_limited(node):
continue
try:
with ConnectionTimeout(self.app.conn_timeout):
headers = dict(req.headers)
headers['Connection'] = 'close'
conn = http_connect(node['ip'], node['port'],
node['device'], partition, req.method, path,
headers=headers,
query_string=req.query_string)
with Timeout(self.app.node_timeout):
possible_source = conn.getresponse()
# See NOTE: swift_conn at top of file about this.
possible_source.swift_conn = conn
except (Exception, Timeout):
self.exception_occurred(node, server_type,
_('Trying to %(method)s %(path)s') %
{'method': req.method, 'path': req.path})
continue
if possible_source.status == HTTP_INSUFFICIENT_STORAGE:
self.error_limit(node)
continue
if is_success(possible_source.status) or \
is_redirection(possible_source.status):
# 404 if we know we don't have a synced copy
if not float(possible_source.getheader('X-PUT-Timestamp', 1)):
statuses.append(HTTP_NOT_FOUND)
reasons.append('')
bodies.append('')
possible_source.read()
continue
if newest:
if sources:
ts = float(source.getheader('x-put-timestamp') or
source.getheader('x-timestamp') or 0)
pts = float(
possible_source.getheader('x-put-timestamp') or
possible_source.getheader('x-timestamp') or 0)
if pts > ts:
sources.insert(0, possible_source)
else:
sources.append(possible_source)
else:
sources.insert(0, possible_source)
source = sources[0]
statuses.append(source.status)
reasons.append(source.reason)
bodies.append('')
continue
else:
source = possible_source
break
statuses.append(possible_source.status)
reasons.append(possible_source.reason)
bodies.append(possible_source.read())
if is_server_error(possible_source.status):
self.error_occurred(node, _('ERROR %(status)d %(body)s ' \
'From %(type)s Server') %
{'status': possible_source.status,
'body': bodies[-1][:1024], 'type': server_type})
if source:
if req.method == 'GET' and \
source.status in (HTTP_OK, HTTP_PARTIAL_CONTENT):
if newest:
# we need to close all hanging swift_conns
sources.pop(0)
for src in sources:
self.close_swift_conn(src)
res = Response(request=req, conditional_response=True)
res.app_iter = self._make_app_iter(node, source, res)
# See NOTE: swift_conn at top of file about this.
res.swift_conn = source.swift_conn
update_headers(res, source.getheaders())
# Used by container sync feature
if res.environ is None:
res.environ = dict()
res.environ['swift_x_timestamp'] = \
source.getheader('x-timestamp')
update_headers(res, {'accept-ranges': 'bytes'})
res.status = source.status
res.content_length = source.getheader('Content-Length')
if source.getheader('Content-Type'):
res.charset = None
res.content_type = source.getheader('Content-Type')
return res
elif is_success(source.status) or is_redirection(source.status):
res = status_map[source.status](request=req)
update_headers(res, source.getheaders())
# Used by container sync feature
if res.environ is None:
res.environ = dict()
res.environ['swift_x_timestamp'] = \
source.getheader('x-timestamp')
update_headers(res, {'accept-ranges': 'bytes'})
res.content_length = source.getheader('Content-Length')
if source.getheader('Content-Type'):
res.charset = None
res.content_type = source.getheader('Content-Type')
return res
return self.best_response(req, statuses, reasons, bodies,
'%s %s' % (server_type, req.method))
def GETorHEAD_base(self, req, server_type, partition, nodes, path,
attempts):
"""
Base handler for HTTP GET or HEAD requests.
:param req: webob.Request object
:param server_type: server type
:param partition: partition
:param nodes: nodes
:param path: path for the request
:param attempts: number of attempts to try
:returns: webob.Response object
"""
statuses = []
reasons = []
bodies = []
source = None
sources = []
newest = req.headers.get('x-newest', 'f').lower() in TRUE_VALUES
nodes = iter(nodes)
while len(statuses) < attempts:
try:
node = nodes.next()
except StopIteration:
break
if self.error_limited(node):
continue
try:
with ConnectionTimeout(self.app.conn_timeout):
headers = dict(req.headers)
headers['Connection'] = 'close'
conn = http_connect(node['ip'],
node['port'],
node['device'],
partition,
req.method,
path,
headers=headers,
query_string=req.query_string)
with Timeout(self.app.node_timeout):
possible_source = conn.getresponse()
# See NOTE: swift_conn at top of file about this.
possible_source.swift_conn = conn
except (Exception, Timeout):
self.exception_occurred(
node, server_type,
_('Trying to %(method)s %(path)s') % {
'method': req.method,
'path': req.path
})
continue
if possible_source.status == HTTP_INSUFFICIENT_STORAGE:
self.error_limit(node)
continue
if is_success(possible_source.status) or \
is_redirection(possible_source.status):
# 404 if we know we don't have a synced copy
if not float(possible_source.getheader('X-PUT-Timestamp', 1)):
statuses.append(HTTP_NOT_FOUND)
reasons.append('')
bodies.append('')
possible_source.read()
continue
if newest:
if sources:
ts = float(
source.getheader('x-put-timestamp')
or source.getheader('x-timestamp') or 0)
pts = float(
possible_source.getheader('x-put-timestamp')
or possible_source.getheader('x-timestamp') or 0)
if pts > ts:
sources.insert(0, possible_source)
else:
sources.append(possible_source)
else:
sources.insert(0, possible_source)
source = sources[0]
statuses.append(source.status)
reasons.append(source.reason)
bodies.append('')
continue
else:
source = possible_source
break
statuses.append(possible_source.status)
reasons.append(possible_source.reason)
bodies.append(possible_source.read())
if is_server_error(possible_source.status):
self.error_occurred(node, _('ERROR %(status)d %(body)s ' \
'From %(type)s Server') %
{'status': possible_source.status,
'body': bodies[-1][:1024], 'type': server_type})
if source:
if req.method == 'GET' and \
source.status in (HTTP_OK, HTTP_PARTIAL_CONTENT):
if newest:
# we need to close all hanging swift_conns
sources.pop(0)
for src in sources:
self.close_swift_conn(src)
res = Response(request=req, conditional_response=True)
res.app_iter = self._make_app_iter(node, source)
# See NOTE: swift_conn at top of file about this.
res.swift_conn = source.swift_conn
update_headers(res, source.getheaders())
# Used by container sync feature
if res.environ is None:
res.environ = dict()
res.environ['swift_x_timestamp'] = \
source.getheader('x-timestamp')
update_headers(res, {'accept-ranges': 'bytes'})
res.status = source.status
res.content_length = source.getheader('Content-Length')
if source.getheader('Content-Type'):
res.charset = None
res.content_type = source.getheader('Content-Type')
return res
elif is_success(source.status) or is_redirection(source.status):
res = status_map[source.status](request=req)
update_headers(res, source.getheaders())
# Used by container sync feature
if res.environ is None:
res.environ = dict()
res.environ['swift_x_timestamp'] = \
source.getheader('x-timestamp')
update_headers(res, {'accept-ranges': 'bytes'})
res.content_length = source.getheader('Content-Length')
if source.getheader('Content-Type'):
res.charset = None
res.content_type = source.getheader('Content-Type')
return res
return self.best_response(req, statuses, reasons, bodies,
'%s %s' % (server_type, req.method))
def GETorHEAD_base(self, req, server_type, partition, nodes, path,
attempts):
"""
Base handler for HTTP GET or HEAD requests.
:param req: swob.Request object
:param server_type: server type
:param partition: partition
:param nodes: nodes
:param path: path for the request
:param attempts: number of attempts to try
:returns: swob.Response object
"""
statuses = []
reasons = []
bodies = []
sources = []
newest = config_true_value(req.headers.get('x-newest', 'f'))
nodes = iter(nodes)
while len(statuses) < attempts:
try:
node = nodes.next()
except StopIteration:
break
if self.error_limited(node):
continue
start_node_timing = time.time()
try:
with ConnectionTimeout(self.app.conn_timeout):
headers = dict(req.headers)
headers['Connection'] = 'close'
conn = http_connect(
node['ip'], node['port'], node['device'], partition,
req.method, path, headers=headers,
query_string=req.query_string)
self.app.set_node_timing(node, time.time() - start_node_timing)
with Timeout(self.app.node_timeout):
possible_source = conn.getresponse()
# See NOTE: swift_conn at top of file about this.
possible_source.swift_conn = conn
except (Exception, Timeout):
self.exception_occurred(
node, server_type, _('Trying to %(method)s %(path)s') %
{'method': req.method, 'path': req.path})
continue
if self.is_good_source(possible_source):
# 404 if we know we don't have a synced copy
if not float(possible_source.getheader('X-PUT-Timestamp', 1)):
statuses.append(HTTP_NOT_FOUND)
reasons.append('')
bodies.append('')
self.close_swift_conn(possible_source)
else:
statuses.append(possible_source.status)
reasons.append(possible_source.reason)
bodies.append('')
sources.append(possible_source)
if not newest: # one good source is enough
break
else:
statuses.append(possible_source.status)
reasons.append(possible_source.reason)
bodies.append(possible_source.read())
if possible_source.status == HTTP_INSUFFICIENT_STORAGE:
self.error_limit(node)
elif is_server_error(possible_source.status):
self.error_occurred(node, _('ERROR %(status)d %(body)s '
'From %(type)s Server') %
{'status': possible_source.status,
'body': bodies[-1][:1024],
'type': server_type})
if sources:
sources.sort(key=source_key)
source = sources.pop()
for src in sources:
self.close_swift_conn(src)
res = Response(request=req, conditional_response=True)
if req.method == 'GET' and \
source.status in (HTTP_OK, HTTP_PARTIAL_CONTENT):
res.app_iter = self._make_app_iter(node, source)
# See NOTE: swift_conn at top of file about this.
res.swift_conn = source.swift_conn
res.status = source.status
update_headers(res, source.getheaders())
if not res.environ:
res.environ = {}
res.environ['swift_x_timestamp'] = \
source.getheader('x-timestamp')
res.accept_ranges = 'bytes'
res.content_length = source.getheader('Content-Length')
if source.getheader('Content-Type'):
res.charset = None
res.content_type = source.getheader('Content-Type')
return res
return self.best_response(req, statuses, reasons, bodies,
'%s %s' % (server_type, req.method))
def account_info(self, account, req=None):
"""
Get account information, and also verify that the account exists.
:param account: name of the account to get the info for
:param req: caller's HTTP request context object (optional)
:param autocreate: whether or not to automatically create the given
account or not (optional, default: False)
:returns: tuple of (account partition, account nodes, container_count)
or (None, None, None) if it does not exist
"""
partition, nodes = self.app.account_ring.get_nodes(account)
account_info = {'status': 0,
'container_count': 0,
'total_object_count': None,
'bytes': None,
'meta': {}}
# 0 = no responses, 200 = found, 404 = not found, -1 = mixed responses
if self.app.memcache:
cache_key = get_account_memcache_key(account)
cache_value = self.app.memcache.get(cache_key)
if not isinstance(cache_value, dict):
result_code = cache_value
container_count = 0
else:
result_code = cache_value['status']
try:
container_count = int(cache_value['container_count'])
except ValueError:
container_count = 0
if result_code == HTTP_OK:
return partition, nodes, container_count
elif result_code == HTTP_NOT_FOUND:
return None, None, None
result_code = 0
path = '/%s' % account
headers = self.generate_request_headers(req)
for node in self.iter_nodes(self.app.account_ring, partition):
try:
start_node_timing = time.time()
with ConnectionTimeout(self.app.conn_timeout):
conn = http_connect(node['ip'], node['port'],
node['device'], partition, 'HEAD',
path, headers)
self.app.set_node_timing(node, time.time() - start_node_timing)
with Timeout(self.app.node_timeout):
resp = conn.getresponse()
body = resp.read()
if is_success(resp.status):
result_code = HTTP_OK
account_info.update(
headers_to_account_info(resp.getheaders()))
break
elif resp.status == HTTP_NOT_FOUND:
if result_code == 0:
result_code = HTTP_NOT_FOUND
elif result_code != HTTP_NOT_FOUND:
result_code = -1
elif resp.status == HTTP_INSUFFICIENT_STORAGE:
self.error_limit(node, _('ERROR Insufficient Storage'))
continue
else:
result_code = -1
if is_server_error(resp.status):
self.error_occurred(
node,
_('ERROR %(status)d %(body)s From Account '
'Server') %
{'status': resp.status, 'body': body[:1024]})
except (Exception, Timeout):
self.exception_occurred(node, _('Account'),
_('Trying to get account info for %s')
% path)
if self.app.memcache and result_code in (HTTP_OK, HTTP_NOT_FOUND):
if result_code == HTTP_OK:
cache_timeout = self.app.recheck_account_existence
else:
cache_timeout = self.app.recheck_account_existence * 0.1
account_info.update(status=result_code)
self.app.memcache.set(cache_key,
account_info,
time=cache_timeout)
if result_code == HTTP_OK:
try:
container_count = int(account_info['container_count'])
except ValueError:
container_count = 0
return partition, nodes, container_count
return None, None, None
def make_request(self,
method,
path,
headers,
acceptable_statuses,
body_file=None,
params=None):
"""Makes a request to Swift with retries.
:param method: HTTP method of request.
:param path: Path of request.
:param headers: Headers to be sent with request.
:param acceptable_statuses: List of acceptable statuses for request.
:param body_file: Body file to be passed along with request,
defaults to None.
:param params: A dict of params to be set in request query string,
defaults to None.
:returns: Response object on success.
:raises UnexpectedResponse: Exception raised when make_request() fails
to get a response with an acceptable status
:raises Exception: Exception is raised when code fails in an
unexpected way.
"""
headers = dict(headers)
headers['user-agent'] = self.user_agent
for attempt in range(self.request_tries):
resp = exc_type = exc_value = exc_traceback = None
req = Request.blank(path,
environ={'REQUEST_METHOD': method},
headers=headers)
if body_file is not None:
if hasattr(body_file, 'seek'):
body_file.seek(0)
req.body_file = body_file
if params:
req.params = params
try:
resp = req.get_response(self.app)
except (Exception, Timeout):
exc_type, exc_value, exc_traceback = exc_info()
else:
if resp.status_int in acceptable_statuses or \
resp.status_int // 100 in acceptable_statuses:
return resp
elif not is_server_error(resp.status_int):
# No sense retrying when we expect the same result
break
# sleep only between tries, not after each one
if attempt < self.request_tries - 1:
if resp:
# always close any resp.app_iter before we discard it
with closing_if_possible(resp.app_iter):
# for non 2XX requests it's safe and useful to drain
# the response body so we log the correct status code
if resp.status_int // 100 != 2:
for iter_body in resp.app_iter:
pass
sleep(2**(attempt + 1))
if resp:
msg = 'Unexpected response: %s' % resp.status
if resp.status_int // 100 != 2 and resp.body:
# provide additional context (and drain the response body) for
# non 2XX responses
msg += ' (%s)' % resp.body
raise UnexpectedResponse(msg, resp)
if exc_type:
# To make pep8 tool happy, in place of raise t, v, tb:
six.reraise(exc_type, exc_value, exc_traceback)
def _get_source_and_node(self):
self.statuses = []
self.reasons = []
self.bodies = []
self.source_headers = []
sources = []
for node in self.app.iter_nodes(self.ring, self.partition):
if node in self.used_nodes:
continue
start_node_timing = time.time()
try:
with ConnectionTimeout(self.app.conn_timeout):
conn = http_connect(
node["ip"],
node["port"],
node["device"],
self.partition,
self.req_method,
self.path,
headers=self.backend_headers,
query_string=self.req_query_string,
)
self.app.set_node_timing(node, time.time() - start_node_timing)
with Timeout(self.app.node_timeout):
possible_source = conn.getresponse()
# See NOTE: swift_conn at top of file about this.
possible_source.swift_conn = conn
except (Exception, Timeout):
self.app.exception_occurred(
node,
self.server_type,
_("Trying to %(method)s %(path)s") % {"method": self.req_method, "path": self.req_path},
)
continue
if self.is_good_source(possible_source):
# 404 if we know we don't have a synced copy
if not float(possible_source.getheader("X-PUT-Timestamp", 1)):
self.statuses.append(HTTP_NOT_FOUND)
self.reasons.append("")
self.bodies.append("")
self.source_headers.append("")
close_swift_conn(possible_source)
else:
if self.used_source_etag:
src_headers = dict((k.lower(), v) for k, v in possible_source.getheaders())
if src_headers.get("etag", "").strip('"') != self.used_source_etag:
self.statuses.append(HTTP_NOT_FOUND)
self.reasons.append("")
self.bodies.append("")
self.source_headers.append("")
continue
self.statuses.append(possible_source.status)
self.reasons.append(possible_source.reason)
self.bodies.append("")
self.source_headers.append("")
sources.append((possible_source, node))
if not self.newest: # one good source is enough
break
else:
self.statuses.append(possible_source.status)
self.reasons.append(possible_source.reason)
self.bodies.append(possible_source.read())
self.source_headers.append(possible_source.getheaders())
if possible_source.status == HTTP_INSUFFICIENT_STORAGE:
self.app.error_limit(node, _("ERROR Insufficient Storage"))
elif is_server_error(possible_source.status):
self.app.error_occurred(
node,
_("ERROR %(status)d %(body)s " "From %(type)s Server")
% {"status": possible_source.status, "body": self.bodies[-1][:1024], "type": self.server_type},
)
if sources:
sources.sort(key=lambda s: source_key(s[0]))
source, node = sources.pop()
for src, _junk in sources:
close_swift_conn(src)
self.used_nodes.append(node)
src_headers = dict((k.lower(), v) for k, v in possible_source.getheaders())
self.used_source_etag = src_headers.get("etag", "").strip('"')
return source, node
return None, None