def updates(self): """ Handles the UPDATES step of an SSYNC request. Receives a set of PUT and DELETE subrequests that will be routed to the object server itself for processing. These contain the information requested by the MISSING_CHECK step. The PUT and DELETE subrequests are formatted pretty much exactly like regular HTTP requests, excepting the HTTP version on the first request line. The process is generally: 1. Sender sends `:UPDATES: START` and begins sending the PUT and DELETE subrequests. 2. Receiver gets `:UPDATES: START` and begins routing the subrequests to the object server. 3. Sender sends `:UPDATES: END`. 4. Receiver gets `:UPDATES: END` and sends `:UPDATES: START` and `:UPDATES: END` (assuming no errors). 5. Sender gets `:UPDATES: START` and `:UPDATES: END`. If too many subrequests fail, as configured by replication_failure_threshold and replication_failure_ratio, the receiver will hang up the request early so as to not waste any more time. At step 4, the receiver will send back an error if there were any failures (that didn't cause a hangup due to the above thresholds) so the sender knows the whole was not entirely a success. This is so the sender knows if it can remove an out of place partition, for example. """ with exceptions.MessageTimeout(self.app.client_timeout, 'updates start'): line = self.fp.readline(self.app.network_chunk_size) if line.strip() != ':UPDATES: START': raise Exception('Looking for :UPDATES: START got %r' % line[:1024]) successes = 0 failures = 0 while True: with exceptions.MessageTimeout(self.app.client_timeout, 'updates line'): line = self.fp.readline(self.app.network_chunk_size) if not line or line.strip() == ':UPDATES: END': break # Read first line METHOD PATH of subrequest. method, path = line.strip().split(' ', 1) subreq = swob.Request.blank('/%s/%s%s' % (self.device, self.partition, path), environ={'REQUEST_METHOD': method}) # Read header lines. content_length = None replication_headers = [] while True: with exceptions.MessageTimeout(self.app.client_timeout): line = self.fp.readline(self.app.network_chunk_size) if not line: raise Exception('Got no headers for %s %s' % (method, path)) line = line.strip() if not line: break header, value = line.split(':', 1) header = header.strip().lower() value = value.strip() subreq.headers[header] = value if header != 'etag': # make sure ssync doesn't cause 'Etag' to be added to # obj metadata in addition to 'ETag' which object server # sets (note capitalization) replication_headers.append(header) if header == 'content-length': content_length = int(value) # Establish subrequest body, if needed. if method in ('DELETE', 'POST'): if content_length not in (None, 0): raise Exception('%s subrequest with content-length %s' % (method, path)) elif method == 'PUT': if content_length is None: raise Exception('No content-length sent for %s %s' % (method, path)) def subreq_iter(): left = content_length while left > 0: with exceptions.MessageTimeout(self.app.client_timeout, 'updates content'): chunk = self.fp.read( min(left, self.app.network_chunk_size)) if not chunk: raise exceptions.ChunkReadError( 'Early termination for %s %s' % (method, path)) left -= len(chunk) yield chunk subreq.environ['wsgi.input'] = utils.FileLikeIter( subreq_iter()) else: raise Exception('Invalid subrequest method %s' % method) subreq.headers['X-Backend-Storage-Policy-Index'] = int(self.policy) subreq.headers['X-Backend-Replication'] = 'True' if self.node_index is not None: # primary node should not 409 if it has a non-primary fragment subreq.headers['X-Backend-Ssync-Frag-Index'] = self.node_index if replication_headers: subreq.headers['X-Backend-Replication-Headers'] = \ ' '.join(replication_headers) # Route subrequest and translate response. resp = subreq.get_response(self.app) if http.is_success(resp.status_int) or \ resp.status_int == http.HTTP_NOT_FOUND: successes += 1 else: self.app.logger.warning( 'ssync subrequest failed with %s: %s %s' % (resp.status_int, method, subreq.path)) failures += 1 if failures >= self.app.replication_failure_threshold and ( not successes or float(failures) / successes > self.app.replication_failure_ratio): raise Exception('Too many %d failures to %d successes' % (failures, successes)) # The subreq may have failed, but we want to read the rest of the # body from the remote side so we can continue on with the next # subreq. for junk in subreq.environ['wsgi.input']: pass if failures: raise swob.HTTPInternalServerError( 'ERROR: With :UPDATES: %d failures to %d successes' % (failures, successes)) yield ':UPDATES: START\r\n' yield ':UPDATES: END\r\n'
def handle_listing(self, req, start_response, sync_profile, cont, per_account): limit = int(req.params.get( 'limit', constraints.CONTAINER_LISTING_LIMIT)) marker = req.params.get('marker', '') prefix = req.params.get('prefix', '') delimiter = req.params.get('delimiter', '') path = req.params.get('path', None) if path: # We do not support the path parameter in listings status, headers, app_iter = req.call_application(self.app) start_response(status, headers) return app_iter resp_type = get_listing_content_type(req) # We always make the request with the json format and convert to the # client-expected response. req.params = dict(req.params, format='json') status, headers, app_iter = req.call_application(self.app) if not status.startswith('200 '): # Only splice 200 (since it's JSON, we know there won't be a 204) start_response(status, headers) return app_iter provider = create_provider(sync_profile, max_conns=1, per_account=per_account) cloud_status, resp = provider.list_objects( marker, limit, prefix, delimiter) if cloud_status != 200: self.logger.error('Failed to list the remote store: %s' % resp) resp = [] if not resp: start_response(status, headers) return app_iter internal_resp = json.load(utils.FileLikeIter(app_iter)) spliced_response = [] internal_index = 0 cloud_index = 0 while True: if len(spliced_response) == limit: break if len(resp) == cloud_index and \ len(internal_resp) == internal_index: break if internal_index < len(internal_resp): if 'name' in internal_resp[internal_index]: internal_name = internal_resp[internal_index]['name'] elif 'subdir' in internal_resp[internal_index]: internal_name = internal_resp[internal_index]['subdir'] else: internal_name = None if cloud_index < len(resp): if 'name' in resp[cloud_index]: cloud_name = resp[cloud_index]['name'] else: cloud_name = resp[cloud_index]['subdir'] else: cloud_name = None if cloud_name is not None: if internal_name is None or \ (internal_name is not None and cloud_name <= internal_name): spliced_response.append(resp[cloud_index]) cloud_index += 1 if len(resp) == cloud_index: # WSGI supplies the request parameters as UTF-8 encoded # strings. We should do the same when submitting # subsequent requests. cloud_status, resp = provider.list_objects( cloud_name.encode('utf-8'), limit, prefix, delimiter) if cloud_status != 200: self.logger.error( 'Failed to list the remote store: %s' % resp) resp = [] cloud_index = 0 continue spliced_response.append(internal_resp[internal_index]) internal_index += 1 res = self._format_listing_response(spliced_response, resp_type, cont) dict_headers = dict(headers) dict_headers['Content-Length'] = len(res) dict_headers['Content-Type'] = resp_type start_response(status, dict_headers.items()) return res