def updates(self):
"""
Handles the UPDATES step of an SSYNC request.
Receives a set of PUT and DELETE subrequests that will be
routed to the object server itself for processing. These
contain the information requested by the MISSING_CHECK step.
The PUT and DELETE subrequests are formatted pretty much
exactly like regular HTTP requests, excepting the HTTP
version on the first request line.
The process is generally:
1. Sender sends `:UPDATES: START` and begins sending the
PUT and DELETE subrequests.
2. Receiver gets `:UPDATES: START` and begins routing the
subrequests to the object server.
3. Sender sends `:UPDATES: END`.
4. Receiver gets `:UPDATES: END` and sends `:UPDATES:
START` and `:UPDATES: END` (assuming no errors).
5. Sender gets `:UPDATES: START` and `:UPDATES: END`.
If too many subrequests fail, as configured by
replication_failure_threshold and replication_failure_ratio,
the receiver will hang up the request early so as to not
waste any more time.
At step 4, the receiver will send back an error if there were
any failures (that didn't cause a hangup due to the above
thresholds) so the sender knows the whole was not entirely a
success. This is so the sender knows if it can remove an out
of place partition, for example.
"""
with exceptions.MessageTimeout(self.app.client_timeout,
'updates start'):
line = self.fp.readline(self.app.network_chunk_size)
if line.strip() != ':UPDATES: START':
raise Exception('Looking for :UPDATES: START got %r' % line[:1024])
successes = 0
failures = 0
while True:
with exceptions.MessageTimeout(self.app.client_timeout,
'updates line'):
line = self.fp.readline(self.app.network_chunk_size)
if not line or line.strip() == ':UPDATES: END':
break
# Read first line METHOD PATH of subrequest.
method, path = line.strip().split(' ', 1)
subreq = swob.Request.blank('/%s/%s%s' %
(self.device, self.partition, path),
environ={'REQUEST_METHOD': method})
# Read header lines.
content_length = None
replication_headers = []
while True:
with exceptions.MessageTimeout(self.app.client_timeout):
line = self.fp.readline(self.app.network_chunk_size)
if not line:
raise Exception('Got no headers for %s %s' %
(method, path))
line = line.strip()
if not line:
break
header, value = line.split(':', 1)
header = header.strip().lower()
value = value.strip()
subreq.headers[header] = value
if header != 'etag':
# make sure ssync doesn't cause 'Etag' to be added to
# obj metadata in addition to 'ETag' which object server
# sets (note capitalization)
replication_headers.append(header)
if header == 'content-length':
content_length = int(value)
# Establish subrequest body, if needed.
if method in ('DELETE', 'POST'):
if content_length not in (None, 0):
raise Exception('%s subrequest with content-length %s' %
(method, path))
elif method == 'PUT':
if content_length is None:
raise Exception('No content-length sent for %s %s' %
(method, path))
def subreq_iter():
left = content_length
while left > 0:
with exceptions.MessageTimeout(self.app.client_timeout,
'updates content'):
chunk = self.fp.read(
min(left, self.app.network_chunk_size))
if not chunk:
raise exceptions.ChunkReadError(
'Early termination for %s %s' % (method, path))
left -= len(chunk)
yield chunk
subreq.environ['wsgi.input'] = utils.FileLikeIter(
subreq_iter())
else:
raise Exception('Invalid subrequest method %s' % method)
subreq.headers['X-Backend-Storage-Policy-Index'] = int(self.policy)
subreq.headers['X-Backend-Replication'] = 'True'
if self.node_index is not None:
# primary node should not 409 if it has a non-primary fragment
subreq.headers['X-Backend-Ssync-Frag-Index'] = self.node_index
if replication_headers:
subreq.headers['X-Backend-Replication-Headers'] = \
' '.join(replication_headers)
# Route subrequest and translate response.
resp = subreq.get_response(self.app)
if http.is_success(resp.status_int) or \
resp.status_int == http.HTTP_NOT_FOUND:
successes += 1
else:
self.app.logger.warning(
'ssync subrequest failed with %s: %s %s' %
(resp.status_int, method, subreq.path))
failures += 1
if failures >= self.app.replication_failure_threshold and (
not successes or float(failures) / successes >
self.app.replication_failure_ratio):
raise Exception('Too many %d failures to %d successes' %
(failures, successes))
# The subreq may have failed, but we want to read the rest of the
# body from the remote side so we can continue on with the next
# subreq.
for junk in subreq.environ['wsgi.input']:
pass
if failures:
raise swob.HTTPInternalServerError(
'ERROR: With :UPDATES: %d failures to %d successes' %
(failures, successes))
yield ':UPDATES: START\r\n'
yield ':UPDATES: END\r\n'
def handle_listing(self, req, start_response, sync_profile, cont,
per_account):
limit = int(req.params.get(
'limit', constraints.CONTAINER_LISTING_LIMIT))
marker = req.params.get('marker', '')
prefix = req.params.get('prefix', '')
delimiter = req.params.get('delimiter', '')
path = req.params.get('path', None)
if path:
# We do not support the path parameter in listings
status, headers, app_iter = req.call_application(self.app)
start_response(status, headers)
return app_iter
resp_type = get_listing_content_type(req)
# We always make the request with the json format and convert to the
# client-expected response.
req.params = dict(req.params, format='json')
status, headers, app_iter = req.call_application(self.app)
if not status.startswith('200 '):
# Only splice 200 (since it's JSON, we know there won't be a 204)
start_response(status, headers)
return app_iter
provider = create_provider(sync_profile, max_conns=1,
per_account=per_account)
cloud_status, resp = provider.list_objects(
marker, limit, prefix, delimiter)
if cloud_status != 200:
self.logger.error('Failed to list the remote store: %s' % resp)
resp = []
if not resp:
start_response(status, headers)
return app_iter
internal_resp = json.load(utils.FileLikeIter(app_iter))
spliced_response = []
internal_index = 0
cloud_index = 0
while True:
if len(spliced_response) == limit:
break
if len(resp) == cloud_index and \
len(internal_resp) == internal_index:
break
if internal_index < len(internal_resp):
if 'name' in internal_resp[internal_index]:
internal_name = internal_resp[internal_index]['name']
elif 'subdir' in internal_resp[internal_index]:
internal_name = internal_resp[internal_index]['subdir']
else:
internal_name = None
if cloud_index < len(resp):
if 'name' in resp[cloud_index]:
cloud_name = resp[cloud_index]['name']
else:
cloud_name = resp[cloud_index]['subdir']
else:
cloud_name = None
if cloud_name is not None:
if internal_name is None or \
(internal_name is not None and
cloud_name <= internal_name):
spliced_response.append(resp[cloud_index])
cloud_index += 1
if len(resp) == cloud_index:
# WSGI supplies the request parameters as UTF-8 encoded
# strings. We should do the same when submitting
# subsequent requests.
cloud_status, resp = provider.list_objects(
cloud_name.encode('utf-8'), limit, prefix,
delimiter)
if cloud_status != 200:
self.logger.error(
'Failed to list the remote store: %s' % resp)
resp = []
cloud_index = 0
continue
spliced_response.append(internal_resp[internal_index])
internal_index += 1
res = self._format_listing_response(spliced_response, resp_type, cont)
dict_headers = dict(headers)
dict_headers['Content-Length'] = len(res)
dict_headers['Content-Type'] = resp_type
start_response(status, dict_headers.items())
return res
