def updates(self): """ Handles the sender-side of the UPDATES step of a REPLICATION request. Full documentation of this can be found at :py:meth:`.Receiver.updates`. """ # First, send all our subrequests based on the send_list. with exceptions.MessageTimeout(self.daemon.node_timeout, 'updates start'): msg = ':UPDATES: START\r\n' self.connection.send('%x\r\n%s\r\n' % (len(msg), msg)) for object_hash in self.send_list: try: df = self.daemon._diskfile_mgr.get_diskfile_from_hash( self.job['device'], self.job['partition'], object_hash, self.policy_idx) except exceptions.DiskFileNotExist: continue url_path = urllib.quote('/%s/%s/%s' % (df.account, df.container, df.obj)) try: df.open() except exceptions.DiskFileDeleted as err: self.send_delete(url_path, err.timestamp) except exceptions.DiskFileError: pass else: self.send_put(url_path, df) with exceptions.MessageTimeout(self.daemon.node_timeout, 'updates end'): msg = ':UPDATES: END\r\n' self.connection.send('%x\r\n%s\r\n' % (len(msg), msg)) # Now, read their response for any issues. while True: with exceptions.MessageTimeout(self.daemon.http_timeout, 'updates start wait'): line = self.readline() if not line: raise exceptions.ReplicationException('Early disconnect') line = line.strip() if line == ':UPDATES: START': break elif line: raise exceptions.ReplicationException( 'Unexpected response: %r' % line[:1024]) while True: with exceptions.MessageTimeout(self.daemon.http_timeout, 'updates line wait'): line = self.readline() if not line: raise exceptions.ReplicationException('Early disconnect') line = line.strip() if line == ':UPDATES: END': break elif line: raise exceptions.ReplicationException( 'Unexpected response: %r' % line[:1024])
def missing_check(self): """ Handles the sender-side of the MISSING_CHECK step of a SSYNC request. Full documentation of this can be found at :py:meth:`.Receiver.missing_check`. """ # First, send our list. with exceptions.MessageTimeout(self.daemon.node_timeout, 'missing_check start'): msg = ':MISSING_CHECK: START\r\n' self.connection.send('%x\r\n%s\r\n' % (len(msg), msg)) hash_gen = self.df_mgr.yield_hashes( self.job['device'], self.job['partition'], self.job['policy'], self.suffixes, frag_index=self.job.get('frag_index')) if self.remote_check_objs is not None: hash_gen = ifilter( lambda path_objhash_timestamp: path_objhash_timestamp[1] in self.remote_check_objs, hash_gen) for path, object_hash, timestamp in hash_gen: self.available_map[object_hash] = timestamp with exceptions.MessageTimeout(self.daemon.node_timeout, 'missing_check send line'): msg = '%s %s\r\n' % (urllib.quote(object_hash), urllib.quote(timestamp)) self.connection.send('%x\r\n%s\r\n' % (len(msg), msg)) with exceptions.MessageTimeout(self.daemon.node_timeout, 'missing_check end'): msg = ':MISSING_CHECK: END\r\n' self.connection.send('%x\r\n%s\r\n' % (len(msg), msg)) # Now, retrieve the list of what they want. while True: with exceptions.MessageTimeout(self.daemon.http_timeout, 'missing_check start wait'): line = self.readline() if not line: raise exceptions.ReplicationException('Early disconnect') line = line.strip() if line == ':MISSING_CHECK: START': break elif line: raise exceptions.ReplicationException( 'Unexpected response: %r' % line[:1024]) while True: with exceptions.MessageTimeout(self.daemon.http_timeout, 'missing_check line wait'): line = self.readline() if not line: raise exceptions.ReplicationException('Early disconnect') line = line.strip() if line == ':MISSING_CHECK: END': break parts = line.split() if parts: self.send_list.append(parts[0])
def missing_check(self): """ Handles the sender-side of the MISSING_CHECK step of a REPLICATION request. Full documentation of this can be found at :py:meth:`.Receiver.missing_check`. """ # First, send our list. with exceptions.MessageTimeout( self.daemon.node_timeout, 'missing_check start'): msg = ':MISSING_CHECK: START\r\n' self.connection.send('%x\r\n%s\r\n' % (len(msg), msg)) for path, object_hash, timestamp in \ self.daemon._diskfile_mgr.yield_hashes( self.job['device'], self.job['partition'], self.policy_idx, self.suffixes): with exceptions.MessageTimeout( self.daemon.node_timeout, 'missing_check send line'): msg = '%s %s\r\n' % ( urllib.quote(object_hash), urllib.quote(timestamp)) self.connection.send('%x\r\n%s\r\n' % (len(msg), msg)) with exceptions.MessageTimeout( self.daemon.node_timeout, 'missing_check end'): msg = ':MISSING_CHECK: END\r\n' self.connection.send('%x\r\n%s\r\n' % (len(msg), msg)) # Now, retrieve the list of what they want. while True: with exceptions.MessageTimeout( self.daemon.http_timeout, 'missing_check start wait'): line = self.readline() if not line: raise exceptions.ReplicationException('Early disconnect') line = line.strip() if line == ':MISSING_CHECK: START': break elif line: raise exceptions.ReplicationException( 'Unexpected response: %r' % line[:1024]) self.send_list = [] while True: with exceptions.MessageTimeout( self.daemon.http_timeout, 'missing_check line wait'): line = self.readline() if not line: raise exceptions.ReplicationException('Early disconnect') line = line.strip() if line == ':MISSING_CHECK: END': break if line: self.send_list.append(line)
def send_put(self, url_path, df): """ Sends a PUT subrequest for the url_path using the source df (DiskFile) and content_length. """ msg = ['PUT ' + url_path, 'Content-Length: ' + str(df.content_length)] # Sorted to make it easier to test. for key, value in sorted(df.get_datafile_metadata().items()): if key not in ('name', 'Content-Length'): msg.append('%s: %s' % (key, value)) msg = '\r\n'.join(msg) + '\r\n\r\n' with exceptions.MessageTimeout(self.daemon.node_timeout, 'send_put'): self.connection.send('%x\r\n%s\r\n' % (len(msg), msg)) bytes_read = 0 for chunk in df.reader(): bytes_read += len(chunk) with exceptions.MessageTimeout(self.daemon.node_timeout, 'send_put chunk'): self.connection.send('%x\r\n%s\r\n' % (len(chunk), chunk)) if bytes_read != df.content_length: # Since we may now have partial state on the receiver we have to # prevent the receiver finalising what may well be a bad or # partially written diskfile. Unfortunately we have no other option # than to pull the plug on this ssync session. If ssync supported # multiphase PUTs like the proxy uses for EC we could send a bad # etag in a footer of this subrequest, but that is not supported. raise exceptions.ReplicationException( 'Sent data length does not match content-length')
def connect(self): """ Establishes a connection and starts an SSYNC request with the object server. """ with exceptions.MessageTimeout(self.daemon.conn_timeout, 'connect send'): self.connection = bufferedhttp.BufferedHTTPConnection( '%s:%s' % (self.node['replication_ip'], self.node['replication_port'])) self.connection.putrequest( 'SSYNC', '/%s/%s' % (self.node['device'], self.job['partition'])) self.connection.putheader('Transfer-Encoding', 'chunked') self.connection.putheader('X-Backend-Storage-Policy-Index', int(self.job['policy'])) # a sync job must use the node's index for the frag_index of the # rebuilt fragments instead of the frag_index from the job which # will be rebuilding them self.connection.putheader( 'X-Backend-Ssync-Frag-Index', self.node.get('index', self.job.get('frag_index', ''))) # a revert job to a handoff will not have a node index self.connection.putheader('X-Backend-Ssync-Node-Index', self.node.get('index', '')) self.connection.endheaders() with exceptions.MessageTimeout(self.daemon.node_timeout, 'connect receive'): self.response = self.connection.getresponse() if self.response.status != http.HTTP_OK: self.response.read() raise exceptions.ReplicationException( 'Expected status %s; got %s' % (http.HTTP_OK, self.response.status))
def send_subrequest(self, method, url_path, headers, df): msg = ['%s %s' % (method, url_path)] for key, value in sorted(headers.items()): msg.append('%s: %s' % (key, value)) msg = '\r\n'.join(msg) + '\r\n\r\n' with exceptions.MessageTimeout(self.daemon.node_timeout, 'send_%s' % method.lower()): self.connection.send('%x\r\n%s\r\n' % (len(msg), msg)) if df: bytes_read = 0 for chunk in df.reader(): bytes_read += len(chunk) with exceptions.MessageTimeout( self.daemon.node_timeout, 'send_%s chunk' % method.lower()): self.connection.send('%x\r\n%s\r\n' % (len(chunk), chunk)) if bytes_read != df.content_length: # Since we may now have partial state on the receiver we have # to prevent the receiver finalising what may well be a bad or # partially written diskfile. Unfortunately we have no other # option than to pull the plug on this ssync session. If ssync # supported multiphase PUTs like the proxy uses for EC we could # send a bad etag in a footer of this subrequest, but that is # not supported. raise exceptions.ReplicationException( 'Sent data length does not match content-length')
def connect(self): """ Establishes a connection and starts an SSYNC request with the object server. """ connection = response = None with exceptions.MessageTimeout(self.daemon.conn_timeout, 'connect send'): connection = SsyncBufferedHTTPConnection( '%s:%s' % (self.node['replication_ip'], self.node['replication_port'])) connection.putrequest( 'SSYNC', '/%s/%s' % (self.node['device'], self.job['partition'])) connection.putheader('Transfer-Encoding', 'chunked') connection.putheader('X-Backend-Storage-Policy-Index', int(self.job['policy'])) # a sync job must use the node's backend_index for the frag_index # of the rebuilt fragments instead of the frag_index from the job # which will be rebuilding them frag_index = self.node.get('backend_index') if frag_index is not None: connection.putheader('X-Backend-Ssync-Frag-Index', frag_index) # Node-Index header is for backwards compat 2.4.0-2.20.0 connection.putheader('X-Backend-Ssync-Node-Index', frag_index) connection.endheaders() with exceptions.MessageTimeout(self.daemon.node_timeout, 'connect receive'): response = connection.getresponse() if response.status != http.HTTP_OK: err_msg = response.read()[:1024] raise exceptions.ReplicationException( 'Expected status %s; got %s (%s)' % (http.HTTP_OK, response.status, err_msg)) return connection, response
def connect(self): """ Establishes a connection and starts an SSYNC request with the object server. """ with exceptions.MessageTimeout(self.daemon.conn_timeout, 'connect send'): self.connection = bufferedhttp.BufferedHTTPConnection( '%s:%s' % (self.node['replication_ip'], self.node['replication_port'])) self.connection.putrequest( 'SSYNC', '/%s/%s' % (self.node['device'], self.job['partition'])) self.connection.putheader('Transfer-Encoding', 'chunked') self.connection.putheader('X-Backend-Storage-Policy-Index', int(self.job['policy'])) self.connection.putheader('X-Backend-Ssync-Frag-Index', self.node['index']) self.connection.endheaders() with exceptions.MessageTimeout(self.daemon.node_timeout, 'connect receive'): self.response = self.connection.getresponse() if self.response.status != http.HTTP_OK: raise exceptions.ReplicationException( 'Expected status %s; got %s' % (http.HTTP_OK, self.response.status))
def readline(self): """ Reads a line from the REPLICATION response body. httplib has no readline and will block on read(x) until x is read, so we have to do the work ourselves. A bit of this is taken from Python's httplib itself. """ data = self.response_buffer self.response_buffer = '' while '\n' not in data and len(data) < self.daemon.network_chunk_size: if self.response_chunk_left == -1: # EOF-already indicator break if self.response_chunk_left == 0: line = self.response.fp.readline() i = line.find(';') if i >= 0: line = line[:i] # strip chunk-extensions try: self.response_chunk_left = int(line.strip(), 16) except ValueError: # close the connection as protocol synchronisation is # probably lost self.response.close() raise exceptions.ReplicationException('Early disconnect') if self.response_chunk_left == 0: self.response_chunk_left = -1 break chunk = self.response.fp.read( min(self.response_chunk_left, self.daemon.network_chunk_size - len(data))) if not chunk: # close the connection as protocol synchronisation is # probably lost self.response.close() raise exceptions.ReplicationException('Early disconnect') self.response_chunk_left -= len(chunk) if self.response_chunk_left == 0: self.response.fp.read(2) # discard the trailing \r\n data += chunk if '\n' in data: data, self.response_buffer = data.split('\n', 1) data += '\n' return data
def connect(self): """ Establishes a connection and starts an SSYNC request with the object server. """ connection = response = None node_addr = '%s:%s' % (self.node['replication_ip'], self.node['replication_port']) with exceptions.MessageTimeout(self.daemon.conn_timeout, 'connect send'): connection = SsyncBufferedHTTPConnection(node_addr) connection.putrequest( 'SSYNC', '/%s/%s' % (self.node['device'], self.job['partition'])) connection.putheader('Transfer-Encoding', 'chunked') connection.putheader('X-Backend-Storage-Policy-Index', int(self.job['policy'])) # a sync job must use the node's backend_index for the frag_index # of the rebuilt fragments instead of the frag_index from the job # which will be rebuilding them frag_index = self.node.get('backend_index') if frag_index is not None: connection.putheader('X-Backend-Ssync-Frag-Index', frag_index) # Node-Index header is for backwards compat 2.4.0-2.20.0 connection.putheader('X-Backend-Ssync-Node-Index', frag_index) connection.endheaders() with exceptions.MessageTimeout(self.daemon.node_timeout, 'connect receive'): response = connection.getresponse() if response.status != http.HTTP_OK: err_msg = response.read()[:1024] raise exceptions.ReplicationException( 'Expected status %s; got %s (%s)' % (http.HTTP_OK, response.status, err_msg)) if self.include_non_durable and not config_true_value( response.getheader('x-backend-accept-no-commit', False)): # fall back to legacy behaviour if receiver does not understand # X-Backend-Commit self.daemon.logger.warning( 'ssync receiver %s does not accept non-durable fragments' % node_addr) self.include_non_durable = False return connection, response
def connect(self): """ Establishes a connection and starts an SSYNC request with the object server. """ connection = response = None with exceptions.MessageTimeout( self.daemon.conn_timeout, 'connect send'): connection = SsyncBufferedHTTPConnection( '%s:%s' % (self.node['replication_ip'], self.node['replication_port'])) connection.putrequest('SSYNC', '/%s/%s' % ( self.node['device'], self.job['partition'])) connection.putheader('Transfer-Encoding', 'chunked') connection.putheader('X-Backend-Storage-Policy-Index', int(self.job['policy'])) # a sync job must use the node's index for the frag_index of the # rebuilt fragments instead of the frag_index from the job which # will be rebuilding them frag_index = self.node.get('index', self.job.get('frag_index')) if frag_index is None: # replication jobs will not have a frag_index key; # reconstructor jobs with only tombstones will have a # frag_index key explicitly set to the value of None - in both # cases on the wire we write the empty string which # ssync_receiver will translate to None frag_index = '' connection.putheader('X-Backend-Ssync-Frag-Index', frag_index) # a revert job to a handoff will not have a node index connection.putheader('X-Backend-Ssync-Node-Index', self.node.get('index', '')) connection.endheaders() with exceptions.MessageTimeout( self.daemon.node_timeout, 'connect receive'): response = connection.getresponse() if response.status != http.HTTP_OK: err_msg = response.read()[:1024] raise exceptions.ReplicationException( 'Expected status %s; got %s (%s)' % (http.HTTP_OK, response.status, err_msg)) return connection, response
def connect(self): """ Establishes a connection and starts a REPLICATION request with the object server. """ with exceptions.MessageTimeout(self.daemon.conn_timeout, 'connect send'): self.connection = bufferedhttp.BufferedHTTPConnection( '%s:%s' % (self.node['ip'], self.node['port'])) self.connection.putrequest( 'REPLICATION', '/%s/%s' % (self.node['device'], self.job['partition'])) self.connection.putheader('Transfer-Encoding', 'chunked') self.connection.endheaders() with exceptions.MessageTimeout(self.daemon.node_timeout, 'connect receive'): self.response = self.connection.getresponse() if self.response.status != http.HTTP_OK: raise exceptions.ReplicationException( 'Expected status %s; got %s' % (http.HTTP_OK, self.response.status))
def updates(self): """ Handles the sender-side of the UPDATES step of an SSYNC request. Full documentation of this can be found at :py:meth:`.Receiver.updates`. """ # First, send all our subrequests based on the send_map. with exceptions.MessageTimeout(self.daemon.node_timeout, 'updates start'): msg = ':UPDATES: START\r\n' self.connection.send('%x\r\n%s\r\n' % (len(msg), msg)) for object_hash, want in self.send_map.items(): object_hash = urllib.parse.unquote(object_hash) try: df = self.df_mgr.get_diskfile_from_hash( self.job['device'], self.job['partition'], object_hash, self.job['policy'], frag_index=self.job.get('frag_index')) except exceptions.DiskFileNotExist: continue url_path = urllib.parse.quote('/%s/%s/%s' % (df.account, df.container, df.obj)) try: df.open() if want.get('data'): # EC reconstructor may have passed a callback to build an # alternative diskfile - construct it using the metadata # from the data file only. df_alt = self.job.get('sync_diskfile_builder', lambda *args: df)( self.job, self.node, df.get_datafile_metadata()) self.send_put(url_path, df_alt) if want.get('meta') and df.data_timestamp != df.timestamp: self.send_post(url_path, df) except exceptions.DiskFileDeleted as err: if want.get('data'): self.send_delete(url_path, err.timestamp) except exceptions.DiskFileError: # DiskFileErrors are expected while opening the diskfile, # before any data is read and sent. Since there is no partial # state on the receiver it's ok to ignore this diskfile and # continue. The diskfile may however be deleted after a # successful ssync since it remains in the send_map. pass with exceptions.MessageTimeout(self.daemon.node_timeout, 'updates end'): msg = ':UPDATES: END\r\n' self.connection.send('%x\r\n%s\r\n' % (len(msg), msg)) # Now, read their response for any issues. while True: with exceptions.MessageTimeout(self.daemon.http_timeout, 'updates start wait'): line = self.readline() if not line: raise exceptions.ReplicationException('Early disconnect') line = line.strip() if line == ':UPDATES: START': break elif line: raise exceptions.ReplicationException( 'Unexpected response: %r' % line[:1024]) while True: with exceptions.MessageTimeout(self.daemon.http_timeout, 'updates line wait'): line = self.readline() if not line: raise exceptions.ReplicationException('Early disconnect') line = line.strip() if line == ':UPDATES: END': break elif line: raise exceptions.ReplicationException( 'Unexpected response: %r' % line[:1024])
def test_replication_exception(self): self.assertEqual(str(exceptions.ReplicationException()), '') self.assertEqual(str(exceptions.ReplicationException('test')), 'test')
def connect(self): raise exceptions.ReplicationException('test connect')
def missing_check(self, connection, response): """ Handles the sender-side of the MISSING_CHECK step of a SSYNC request. Full documentation of this can be found at :py:meth:`.Receiver.missing_check`. """ available_map = {} send_map = {} # First, send our list. with exceptions.MessageTimeout(self.daemon.node_timeout, 'missing_check start'): msg = b':MISSING_CHECK: START\r\n' connection.send(b'%x\r\n%s\r\n' % (len(msg), msg)) hash_gen = self.df_mgr.yield_hashes( self.job['device'], self.job['partition'], self.job['policy'], self.suffixes, frag_index=self.job.get('frag_index')) if self.remote_check_objs is not None: hash_gen = six.moves.filter( lambda objhash_timestamps: objhash_timestamps[0] in self. remote_check_objs, hash_gen) for object_hash, timestamps in hash_gen: available_map[object_hash] = timestamps with exceptions.MessageTimeout(self.daemon.node_timeout, 'missing_check send line'): msg = b'%s\r\n' % encode_missing(object_hash, **timestamps) connection.send(b'%x\r\n%s\r\n' % (len(msg), msg)) with exceptions.MessageTimeout(self.daemon.node_timeout, 'missing_check end'): msg = b':MISSING_CHECK: END\r\n' connection.send(b'%x\r\n%s\r\n' % (len(msg), msg)) # Now, retrieve the list of what they want. while True: with exceptions.MessageTimeout(self.daemon.http_timeout, 'missing_check start wait'): line = response.readline(size=self.daemon.network_chunk_size) if not line: raise exceptions.ReplicationException('Early disconnect') line = line.strip() if line == b':MISSING_CHECK: START': break elif line: if not six.PY2: try: line = line.decode('ascii') except UnicodeDecodeError: pass raise exceptions.ReplicationException( 'Unexpected response: %r' % line[:1024]) while True: with exceptions.MessageTimeout(self.daemon.http_timeout, 'missing_check line wait'): line = response.readline(size=self.daemon.network_chunk_size) if not line: raise exceptions.ReplicationException('Early disconnect') line = line.strip() if line == b':MISSING_CHECK: END': break parts = line.decode('ascii').split() if parts: send_map[parts[0]] = decode_wanted(parts[1:]) return available_map, send_map