def download(self, acc, container, u_agent, delay=0, request_tries=3): self.logger.info('Prefetching objects with InternalClient with ' + str(delay) + ' seconds of delay.') time.sleep(delay) swift = InternalClient(PROXY_PATH, u_agent, request_tries=request_tries) headers = {} prefetch_list = [] bytes_count = 0 for o in swift.iter_objects(acc, container): if bytes_count + int(o['bytes']) < self.cache_max_size: prefetch_list.append(o['name']) bytes_count += int(o['bytes']) else: break for name in prefetch_list: object_path = '/v1/' + acc + '/' + container + '/' + name oid = hashlib.md5(object_path).hexdigest() status, resp_headers, it = swift.get_object(acc, container, name, headers, ACCEPTABLE_STATUS) object_size = int(resp_headers.get('Content-Length')) object_etag = resp_headers.get('Etag') object_storage_policy_id = '0' # FIXME hardcoded to_evict = self.cache.access_cache("PUT", oid, object_size, object_etag, object_storage_policy_id) for ev_object_id in to_evict: os.remove(os.path.join(self.cache_path, ev_object_id)) self.logger.info('Prefetch Filter - Object ' + name + ' stored in cache with ID: ' + oid) with open(os.path.join(self.cache_path, oid), 'w') as f: for el in it: f.write(el)
def download(oid, acc, container, name, u_agent, token, delay=0, request_tries=5): print 'Prefetching object with InternalClient: ' + oid + ' after ' + str(delay) + ' seconds of delay.' #time.sleep(delay) start_time = dt.now() swift = InternalClient(PROXY_PATH, u_agent, request_tries=request_tries) headers = {} headers['X-Auth-Token'] = token headers['X-No-Prefetch'] = 'True' status, head, it = swift.get_object(acc, container, name, headers, acc_status) data = [el for el in it] end_time = dt.now() diff = end_time - start_time return (oid, data, head, end_time, diff)
class InternalBrainClient(object): def __init__(self, conf_file, account='AUTH_test'): self.swift = InternalClient(conf_file, 'probe-test', 3) self.account = account @translate_client_exception def put_container(self, container_name, headers): return self.swift.create_container(self.account, container_name, headers=headers) @translate_client_exception def post_container(self, container_name, headers): return self.swift.set_container_metadata(self.account, container_name, headers) @translate_client_exception def delete_container(self, container_name): return self.swift.delete_container(self.account, container_name) def parse_qs(self, query_string): if query_string is not None: return {k: v[-1] for k, v in parse_qs(query_string).items()} @translate_client_exception def put_object(self, container_name, object_name, headers, contents, query_string=None): return self.swift.upload_object(StringIO(contents), self.account, container_name, object_name, headers=headers, params=self.parse_qs(query_string)) @translate_client_exception def delete_object(self, container_name, object_name): return self.swift.delete_object( self.account, container_name, object_name) @translate_client_exception def head_object(self, container_name, object_name): return self.swift.get_object_metadata( self.account, container_name, object_name) @translate_client_exception def get_object(self, container_name, object_name, query_string=None): status, headers, resp_iter = self.swift.get_object( self.account, container_name, object_name, params=self.parse_qs(query_string)) return headers, ''.join(resp_iter)
def run(self): self.logger.debug('Prefetching object with InternalClient: ' + self.oid + ' after ' + str(self.delay) + ' seconds of delay.') eventlet.sleep(self.delay) start_time = dt.now() swift = InternalClient(PROXY_PATH, self.user_agent, request_tries=self.request_tries) headers = {} headers['X-Auth-Token'] = self.token headers['X-No-Prefetch'] = 'True' status, head, it = swift.get_object(self.acc, self.container, self.objname, headers, acc_status) data = [el for el in it] end_time = dt.now() diff = end_time - start_time self.log_results(self.oid, data, diff) self.delete_memory()
def download(oid, acc, container, name, u_agent, token, delay=0, request_tries=5): print 'Prefetching object with InternalClient: ' + oid + ' after ' + str( delay.total_seconds()) + ' seconds of delay.' time.sleep(delay.total_seconds()) swift = InternalClient(PROXY_PATH, u_agent, request_tries=request_tries) headers = {} headers['X-Auth-Token'] = token headers['X-No-Prefetch'] = 'True' status, head, it = swift.get_object(acc, container, name, headers, acc_status) print 'Request to Swift - Response Status: ' + str( status) + ' Response headers: ' + str(head) data = [el for el in it] return (oid, data, head)
class ContainerSync(Daemon): """ Daemon to sync syncable containers. This is done by scanning the local devices for container databases and checking for x-container-sync-to and x-container-sync-key metadata values. If they exist, newer rows since the last sync will trigger PUTs or DELETEs to the other container. .. note:: Container sync will sync object POSTs only if the proxy server is set to use "object_post_as_copy = true" which is the default. So-called fast object posts, "object_post_as_copy = false" do not update the container listings and therefore can't be detected for synchronization. The actual syncing is slightly more complicated to make use of the three (or number-of-replicas) main nodes for a container without each trying to do the exact same work but also without missing work if one node happens to be down. Two sync points are kept per container database. All rows between the two sync points trigger updates. Any rows newer than both sync points cause updates depending on the node's position for the container (primary nodes do one third, etc. depending on the replica count of course). After a sync run, the first sync point is set to the newest ROWID known and the second sync point is set to newest ROWID for which all updates have been sent. An example may help. Assume replica count is 3 and perfectly matching ROWIDs starting at 1. First sync run, database has 6 rows: * SyncPoint1 starts as -1. * SyncPoint2 starts as -1. * No rows between points, so no "all updates" rows. * Six rows newer than SyncPoint1, so a third of the rows are sent by node 1, another third by node 2, remaining third by node 3. * SyncPoint1 is set as 6 (the newest ROWID known). * SyncPoint2 is left as -1 since no "all updates" rows were synced. Next sync run, database has 12 rows: * SyncPoint1 starts as 6. * SyncPoint2 starts as -1. * The rows between -1 and 6 all trigger updates (most of which should short-circuit on the remote end as having already been done). * Six more rows newer than SyncPoint1, so a third of the rows are sent by node 1, another third by node 2, remaining third by node 3. * SyncPoint1 is set as 12 (the newest ROWID known). * SyncPoint2 is set as 6 (the newest "all updates" ROWID). In this way, under normal circumstances each node sends its share of updates each run and just sends a batch of older updates to ensure nothing was missed. :param conf: The dict of configuration values from the [container-sync] section of the container-server.conf :param container_ring: If None, the <swift_dir>/container.ring.gz will be loaded. This is overridden by unit tests. """ def __init__(self, conf, container_ring=None, logger=None): #: The dict of configuration values from the [container-sync] section #: of the container-server.conf. self.conf = conf #: Logger to use for container-sync log lines. self.logger = logger or get_logger(conf, log_route='container-sync') #: Path to the local device mount points. self.devices = conf.get('devices', '/srv/node') #: Indicates whether mount points should be verified as actual mount #: points (normally true, false for tests and SAIO). self.mount_check = config_true_value(conf.get('mount_check', 'true')) #: Minimum time between full scans. This is to keep the daemon from #: running wild on near empty systems. self.interval = int(conf.get('interval', 300)) #: Maximum amount of time to spend syncing a container before moving on #: to the next one. If a conatiner sync hasn't finished in this time, #: it'll just be resumed next scan. self.container_time = int(conf.get('container_time', 60)) #: ContainerSyncCluster instance for validating sync-to values. self.realms_conf = ContainerSyncRealms( os.path.join( conf.get('swift_dir', '/etc/swift'), 'container-sync-realms.conf'), self.logger) #: The list of hosts we're allowed to send syncs to. This can be #: overridden by data in self.realms_conf self.allowed_sync_hosts = [ h.strip() for h in conf.get('allowed_sync_hosts', '127.0.0.1').split(',') if h.strip()] self.http_proxies = [ a.strip() for a in conf.get('sync_proxy', '').split(',') if a.strip()] #: ContainerSyncStore instance for iterating over synced containers self.sync_store = ContainerSyncStore(self.devices, self.logger, self.mount_check) #: Number of containers with sync turned on that were successfully #: synced. self.container_syncs = 0 #: Number of successful DELETEs triggered. self.container_deletes = 0 #: Number of successful PUTs triggered. self.container_puts = 0 #: Number of containers whose sync has been turned off, but #: are not yet cleared from the sync store. self.container_skips = 0 #: Number of containers that had a failure of some type. self.container_failures = 0 #: Time of last stats report. self.reported = time() self.swift_dir = conf.get('swift_dir', '/etc/swift') #: swift.common.ring.Ring for locating containers. self.container_ring = container_ring or Ring(self.swift_dir, ring_name='container') bind_ip = conf.get('bind_ip', '0.0.0.0') self._myips = whataremyips(bind_ip) self._myport = int(conf.get('bind_port', 6001)) swift.common.db.DB_PREALLOCATION = \ config_true_value(conf.get('db_preallocation', 'f')) self.conn_timeout = float(conf.get('conn_timeout', 5)) request_tries = int(conf.get('request_tries') or 3) internal_client_conf_path = conf.get('internal_client_conf_path') if not internal_client_conf_path: self.logger.warning( _('Configuration option internal_client_conf_path not ' 'defined. Using default configuration, See ' 'internal-client.conf-sample for options')) internal_client_conf = ConfigString(ic_conf_body) else: internal_client_conf = internal_client_conf_path try: self.swift = InternalClient( internal_client_conf, 'Swift Container Sync', request_tries) except IOError as err: if err.errno != errno.ENOENT: raise raise SystemExit( _('Unable to load internal client from config: %r (%s)') % (internal_client_conf_path, err)) def get_object_ring(self, policy_idx): """ Get the ring object to use based on its policy. :policy_idx: policy index as defined in swift.conf :returns: appropriate ring object """ return POLICIES.get_object_ring(policy_idx, self.swift_dir) def run_forever(self, *args, **kwargs): """ Runs container sync scans until stopped. """ sleep(random() * self.interval) while True: begin = time() for path in self.sync_store.synced_containers_generator(): self.container_sync(path) if time() - self.reported >= 3600: # once an hour self.report() elapsed = time() - begin if elapsed < self.interval: sleep(self.interval - elapsed) def run_once(self, *args, **kwargs): """ Runs a single container sync scan. """ self.logger.info(_('Begin container sync "once" mode')) begin = time() for path in self.sync_store.synced_containers_generator(): self.container_sync(path) if time() - self.reported >= 3600: # once an hour self.report() self.report() elapsed = time() - begin self.logger.info( _('Container sync "once" mode completed: %.02fs'), elapsed) def report(self): """ Writes a report of the stats to the logger and resets the stats for the next report. """ self.logger.info( _('Since %(time)s: %(sync)s synced [%(delete)s deletes, %(put)s ' 'puts], %(skip)s skipped, %(fail)s failed'), {'time': ctime(self.reported), 'sync': self.container_syncs, 'delete': self.container_deletes, 'put': self.container_puts, 'skip': self.container_skips, 'fail': self.container_failures}) self.reported = time() self.container_syncs = 0 self.container_deletes = 0 self.container_puts = 0 self.container_skips = 0 self.container_failures = 0 def container_sync(self, path): """ Checks the given path for a container database, determines if syncing is turned on for that database and, if so, sends any updates to the other container. :param path: the path to a container db """ broker = None try: broker = ContainerBroker(path) # The path we pass to the ContainerBroker is a real path of # a container DB. If we get here, however, it means that this # path is linked from the sync_containers dir. In rare cases # of race or processes failures the link can be stale and # the get_info below will raise a DB doesn't exist exception # In this case we remove the stale link and raise an error # since in most cases the db should be there. try: info = broker.get_info() except DatabaseConnectionError as db_err: if str(db_err).endswith("DB doesn't exist"): self.sync_store.remove_synced_container(broker) raise x, nodes = self.container_ring.get_nodes(info['account'], info['container']) for ordinal, node in enumerate(nodes): if is_local_device(self._myips, self._myport, node['ip'], node['port']): break else: return if not broker.is_deleted(): sync_to = None user_key = None sync_point1 = info['x_container_sync_point1'] sync_point2 = info['x_container_sync_point2'] for key, (value, timestamp) in broker.metadata.items(): if key.lower() == 'x-container-sync-to': sync_to = value elif key.lower() == 'x-container-sync-key': user_key = value if not sync_to or not user_key: self.container_skips += 1 self.logger.increment('skips') return err, sync_to, realm, realm_key = validate_sync_to( sync_to, self.allowed_sync_hosts, self.realms_conf) if err: self.logger.info( _('ERROR %(db_file)s: %(validate_sync_to_err)s'), {'db_file': str(broker), 'validate_sync_to_err': err}) self.container_failures += 1 self.logger.increment('failures') return stop_at = time() + self.container_time next_sync_point = None while time() < stop_at and sync_point2 < sync_point1: rows = broker.get_items_since(sync_point2, 1) if not rows: break row = rows[0] if row['ROWID'] > sync_point1: break key = hash_path(info['account'], info['container'], row['name'], raw_digest=True) # This node will only initially sync out one third of the # objects (if 3 replicas, 1/4 if 4, etc.) and will skip # problematic rows as needed in case of faults. # This section will attempt to sync previously skipped # rows in case the previous attempts by any of the nodes # didn't succeed. if not self.container_sync_row( row, sync_to, user_key, broker, info, realm, realm_key): if not next_sync_point: next_sync_point = sync_point2 sync_point2 = row['ROWID'] broker.set_x_container_sync_points(None, sync_point2) if next_sync_point: broker.set_x_container_sync_points(None, next_sync_point) while time() < stop_at: rows = broker.get_items_since(sync_point1, 1) if not rows: break row = rows[0] key = hash_path(info['account'], info['container'], row['name'], raw_digest=True) # This node will only initially sync out one third of the # objects (if 3 replicas, 1/4 if 4, etc.). It'll come back # around to the section above and attempt to sync # previously skipped rows in case the other nodes didn't # succeed or in case it failed to do so the first time. if unpack_from('>I', key)[0] % \ len(nodes) == ordinal: self.container_sync_row( row, sync_to, user_key, broker, info, realm, realm_key) sync_point1 = row['ROWID'] broker.set_x_container_sync_points(sync_point1, None) self.container_syncs += 1 self.logger.increment('syncs') except (Exception, Timeout): self.container_failures += 1 self.logger.increment('failures') self.logger.exception(_('ERROR Syncing %s'), broker if broker else path) def container_sync_row(self, row, sync_to, user_key, broker, info, realm, realm_key): """ Sends the update the row indicates to the sync_to container. :param row: The updated row in the local database triggering the sync update. :param sync_to: The URL to the remote container. :param user_key: The X-Container-Sync-Key to use when sending requests to the other container. :param broker: The local container database broker. :param info: The get_info result from the local container database broker. :param realm: The realm from self.realms_conf, if there is one. If None, fallback to using the older allowed_sync_hosts way of syncing. :param realm_key: The realm key from self.realms_conf, if there is one. If None, fallback to using the older allowed_sync_hosts way of syncing. :returns: True on success """ try: start_time = time() if row['deleted']: try: headers = {'x-timestamp': row['created_at']} if realm and realm_key: nonce = uuid.uuid4().hex path = urlparse(sync_to).path + '/' + quote( row['name']) sig = self.realms_conf.get_sig( 'DELETE', path, headers['x-timestamp'], nonce, realm_key, user_key) headers['x-container-sync-auth'] = '%s %s %s' % ( realm, nonce, sig) else: headers['x-container-sync-key'] = user_key delete_object(sync_to, name=row['name'], headers=headers, proxy=self.select_http_proxy(), logger=self.logger, timeout=self.conn_timeout) except ClientException as err: if err.http_status != HTTP_NOT_FOUND: raise self.container_deletes += 1 self.logger.increment('deletes') self.logger.timing_since('deletes.timing', start_time) else: part, nodes = \ self.get_object_ring(info['storage_policy_index']). \ get_nodes(info['account'], info['container'], row['name']) shuffle(nodes) exc = None looking_for_timestamp = Timestamp(row['created_at']) timestamp = -1 headers = body = None # look up for the newest one headers_out = {'X-Newest': True, 'X-Backend-Storage-Policy-Index': str(info['storage_policy_index'])} try: source_obj_status, source_obj_info, source_obj_iter = \ self.swift.get_object(info['account'], info['container'], row['name'], headers=headers_out, acceptable_statuses=(2, 4)) except (Exception, UnexpectedResponse, Timeout) as err: source_obj_info = {} source_obj_iter = None exc = err timestamp = Timestamp(source_obj_info.get( 'x-timestamp', 0)) headers = source_obj_info body = source_obj_iter if timestamp < looking_for_timestamp: if exc: raise exc raise Exception( _('Unknown exception trying to GET: ' '%(account)r %(container)r %(object)r'), {'account': info['account'], 'container': info['container'], 'object': row['name']}) for key in ('date', 'last-modified'): if key in headers: del headers[key] if 'etag' in headers: headers['etag'] = headers['etag'].strip('"') if 'content-type' in headers: headers['content-type'] = clean_content_type( headers['content-type']) headers['x-timestamp'] = row['created_at'] if realm and realm_key: nonce = uuid.uuid4().hex path = urlparse(sync_to).path + '/' + quote(row['name']) sig = self.realms_conf.get_sig( 'PUT', path, headers['x-timestamp'], nonce, realm_key, user_key) headers['x-container-sync-auth'] = '%s %s %s' % ( realm, nonce, sig) else: headers['x-container-sync-key'] = user_key put_object(sync_to, name=row['name'], headers=headers, contents=FileLikeIter(body), proxy=self.select_http_proxy(), logger=self.logger, timeout=self.conn_timeout) self.container_puts += 1 self.logger.increment('puts') self.logger.timing_since('puts.timing', start_time) except ClientException as err: if err.http_status == HTTP_UNAUTHORIZED: self.logger.info( _('Unauth %(sync_from)r => %(sync_to)r'), {'sync_from': '%s/%s' % (quote(info['account']), quote(info['container'])), 'sync_to': sync_to}) elif err.http_status == HTTP_NOT_FOUND: self.logger.info( _('Not found %(sync_from)r => %(sync_to)r \ - object %(obj_name)r'), {'sync_from': '%s/%s' % (quote(info['account']), quote(info['container'])), 'sync_to': sync_to, 'obj_name': row['name']}) else: self.logger.exception( _('ERROR Syncing %(db_file)s %(row)s'), {'db_file': str(broker), 'row': row}) self.container_failures += 1 self.logger.increment('failures') return False except (Exception, Timeout) as err: self.logger.exception( _('ERROR Syncing %(db_file)s %(row)s'), {'db_file': str(broker), 'row': row}) self.container_failures += 1 self.logger.increment('failures') return False return True def select_http_proxy(self): return choice(self.http_proxies) if self.http_proxies else None
class ContainerReconciler(Daemon): """ Move objects that are in the wrong storage policy. """ def __init__(self, conf): self.conf = conf # This option defines how long an un-processable misplaced object # marker will be retried before it is abandoned. It is not coupled # with the tombstone reclaim age in the consistency engine. self.reclaim_age = int(conf.get('reclaim_age', 86400 * 7)) self.interval = int(conf.get('interval', 30)) conf_path = conf.get('__file__') or \ '/etc/swift/container-reconciler.conf' self.logger = get_logger(conf, log_route='container-reconciler') request_tries = int(conf.get('request_tries') or 3) self.swift = InternalClient(conf_path, 'Swift Container Reconciler', request_tries) self.stats = defaultdict(int) self.last_stat_time = time.time() def stats_log(self, metric, msg, *args, **kwargs): """ Update stats tracking for metric and emit log message. """ level = kwargs.pop('level', logging.DEBUG) log_message = '%s: ' % metric + msg self.logger.log(level, log_message, *args, **kwargs) self.stats[metric] += 1 def log_stats(self, force=False): """ Dump stats to logger, noop when stats have been already been logged in the last minute. """ now = time.time() should_log = force or (now - self.last_stat_time > 60) if should_log: self.last_stat_time = now self.logger.info('Reconciler Stats: %r', dict(**self.stats)) def pop_queue(self, container, obj, q_ts, q_record): """ Issue a delete object request to the container for the misplaced object queue entry. :param container: the misplaced objects container :param obj: the name of the misplaced object :param q_ts: the timestamp of the misplaced object :param q_record: the timestamp of the queue entry N.B. q_ts will normally be the same time as q_record except when an object was manually re-enqued. """ q_path = '/%s/%s/%s' % (MISPLACED_OBJECTS_ACCOUNT, container, obj) x_timestamp = slightly_later_timestamp(max(q_record, q_ts)) self.stats_log('pop_queue', 'remove %r (%f) from the queue (%s)', q_path, q_ts, x_timestamp) headers = {'X-Timestamp': x_timestamp} direct_delete_container_entry(self.swift.container_ring, MISPLACED_OBJECTS_ACCOUNT, container, obj, headers=headers) def throw_tombstones(self, account, container, obj, timestamp, policy_index, path): """ Issue a delete object request to the given storage_policy. :param account: the account name :param container: the container name :param obj: the object name :param timestamp: the timestamp of the object to delete :param policy_index: the policy index to direct the request :param path: the path to be used for logging """ x_timestamp = slightly_later_timestamp(timestamp) self.stats_log('cleanup_attempt', '%r (%f) from policy_index ' '%s (%s) will be deleted', path, timestamp, policy_index, x_timestamp) headers = { 'X-Timestamp': x_timestamp, 'X-Backend-Storage-Policy-Index': policy_index, } success = False try: self.swift.delete_object(account, container, obj, acceptable_statuses=(2, 404), headers=headers) except UnexpectedResponse as err: self.stats_log( 'cleanup_failed', '%r (%f) was not cleaned up ' 'in storage_policy %s (%s)', path, timestamp, policy_index, err) else: success = True self.stats_log( 'cleanup_success', '%r (%f) was successfully ' 'removed from policy_index %s', path, timestamp, policy_index) return success def _reconcile_object(self, account, container, obj, q_policy_index, q_ts, q_op, path, **kwargs): """ Perform object reconciliation. :param account: the account name of the misplaced object :param container: the container name of the misplaced object :param obj: the object name :param q_policy_index: the policy index of the source indicated by the queue entry. :param q_ts: the timestamp of the misplaced object :param q_op: the operation of the misplaced request :param path: the full path of the misplaced object for logging :returns: True to indicate the request is fully processed successfully, otherwise False. """ container_policy_index = direct_get_container_policy_index( self.swift.container_ring, account, container) if container_policy_index is None: self.stats_log( 'unavailable_container', '%r (%f) unable to ' 'determine the destination policy_index', path, q_ts) return False if container_policy_index == q_policy_index: self.stats_log( 'noop_object', '%r (%f) container policy_index ' '%s matches queue policy index %s', path, q_ts, container_policy_index, q_policy_index) return True # check if object exists in the destination already self.logger.debug( 'checking for %r (%f) in destination ' 'policy_index %s', path, q_ts, container_policy_index) headers = {'X-Backend-Storage-Policy-Index': container_policy_index} dest_obj = self.swift.get_object_metadata(account, container, obj, headers=headers, acceptable_statuses=(2, 4)) dest_ts = Timestamp(dest_obj.get('x-backend-timestamp', 0)) if dest_ts >= q_ts: self.stats_log( 'found_object', '%r (%f) in policy_index %s ' 'is newer than queue (%f)', path, dest_ts, container_policy_index, q_ts) return self.throw_tombstones(account, container, obj, q_ts, q_policy_index, path) # object is misplaced self.stats_log( 'misplaced_object', '%r (%f) in policy_index %s ' 'should be in policy_index %s', path, q_ts, q_policy_index, container_policy_index) # fetch object from the source location self.logger.debug('fetching %r (%f) from storage policy %s', path, q_ts, q_policy_index) headers = {'X-Backend-Storage-Policy-Index': q_policy_index} try: source_obj_status, source_obj_info, source_obj_iter = \ self.swift.get_object(account, container, obj, headers=headers, acceptable_statuses=(2, 4)) except UnexpectedResponse as err: source_obj_status = err.resp.status_int source_obj_info = {} source_obj_iter = None source_ts = Timestamp(source_obj_info.get('x-backend-timestamp', 0)) if source_obj_status == 404 and q_op == 'DELETE': return self.ensure_tombstone_in_right_location( q_policy_index, account, container, obj, q_ts, path, container_policy_index, source_ts) else: return self.ensure_object_in_right_location( q_policy_index, account, container, obj, q_ts, path, container_policy_index, source_ts, source_obj_status, source_obj_info, source_obj_iter) def ensure_object_in_right_location(self, q_policy_index, account, container, obj, q_ts, path, container_policy_index, source_ts, source_obj_status, source_obj_info, source_obj_iter, **kwargs): """ Validate source object will satisfy the misplaced object queue entry and move to destination. :param q_policy_index: the policy_index for the source object :param account: the account name of the misplaced object :param container: the container name of the misplaced object :param obj: the name of the misplaced object :param q_ts: the timestamp of the misplaced object :param path: the full path of the misplaced object for logging :param container_policy_index: the policy_index of the destination :param source_ts: the timestamp of the source object :param source_obj_status: the HTTP status source object request :param source_obj_info: the HTTP headers of the source object request :param source_obj_iter: the body iter of the source object request """ if source_obj_status // 100 != 2 or source_ts < q_ts: if q_ts < time.time() - self.reclaim_age: # it's old and there are no tombstones or anything; give up self.stats_log('lost_source', '%r (%s) was not available in ' 'policy_index %s and has expired', path, q_ts.internal, q_policy_index, level=logging.CRITICAL) return True # the source object is unavailable or older than the queue # entry; a version that will satisfy the queue entry hopefully # exists somewhere in the cluster, so wait and try again self.stats_log('unavailable_source', '%r (%s) in ' 'policy_index %s responded %s (%s)', path, q_ts.internal, q_policy_index, source_obj_status, source_ts.internal, level=logging.WARNING) return False # optimistically move any source with a timestamp >= q_ts ts = max(Timestamp(source_ts), q_ts) # move the object put_timestamp = slightly_later_timestamp(ts, offset=2) self.stats_log( 'copy_attempt', '%r (%f) in policy_index %s will be ' 'moved to policy_index %s (%s)', path, source_ts, q_policy_index, container_policy_index, put_timestamp) headers = source_obj_info.copy() headers['X-Backend-Storage-Policy-Index'] = container_policy_index headers['X-Timestamp'] = put_timestamp try: self.swift.upload_object(FileLikeIter(source_obj_iter), account, container, obj, headers=headers) except UnexpectedResponse as err: self.stats_log('copy_failed', 'upload %r (%f) from ' 'policy_index %s to policy_index %s ' 'returned %s', path, source_ts, q_policy_index, container_policy_index, err, level=logging.WARNING) return False except: # noqa self.stats_log('unhandled_error', 'unable to upload %r (%f) ' 'from policy_index %s to policy_index %s ', path, source_ts, q_policy_index, container_policy_index, level=logging.ERROR, exc_info=True) return False self.stats_log( 'copy_success', '%r (%f) moved from policy_index %s ' 'to policy_index %s (%s)', path, source_ts, q_policy_index, container_policy_index, put_timestamp) return self.throw_tombstones(account, container, obj, q_ts, q_policy_index, path) def ensure_tombstone_in_right_location(self, q_policy_index, account, container, obj, q_ts, path, container_policy_index, source_ts, **kwargs): """ Issue a DELETE request against the destination to match the misplaced DELETE against the source. """ delete_timestamp = slightly_later_timestamp(q_ts, offset=2) self.stats_log( 'delete_attempt', '%r (%f) in policy_index %s ' 'will be deleted from policy_index %s (%s)', path, source_ts, q_policy_index, container_policy_index, delete_timestamp) headers = { 'X-Backend-Storage-Policy-Index': container_policy_index, 'X-Timestamp': delete_timestamp, } try: self.swift.delete_object(account, container, obj, headers=headers) except UnexpectedResponse as err: self.stats_log('delete_failed', 'delete %r (%f) from ' 'policy_index %s (%s) returned %s', path, source_ts, container_policy_index, delete_timestamp, err, level=logging.WARNING) return False except: # noqa self.stats_log('unhandled_error', 'unable to delete %r (%f) ' 'from policy_index %s (%s)', path, source_ts, container_policy_index, delete_timestamp, level=logging.ERROR, exc_info=True) return False self.stats_log('delete_success', '%r (%f) deleted from ' 'policy_index %s (%s)', path, source_ts, container_policy_index, delete_timestamp, level=logging.INFO) return self.throw_tombstones(account, container, obj, q_ts, q_policy_index, path) def reconcile_object(self, info): """ Process a possibly misplaced object write request. Determine correct destination storage policy by checking with primary containers. Check source and destination, copying or deleting into destination and cleaning up the source as needed. This method wraps _reconcile_object for exception handling. :param info: a queue entry dict :returns: True to indicate the request is fully processed successfully, otherwise False. """ self.logger.debug( 'checking placement for %r (%f) ' 'in policy_index %s', info['path'], info['q_ts'], info['q_policy_index']) success = False try: success = self._reconcile_object(**info) except: # noqa self.logger.exception( 'Unhandled Exception trying to ' 'reconcile %r (%f) in policy_index %s', info['path'], info['q_ts'], info['q_policy_index']) if success: metric = 'success' msg = 'was handled successfully' else: metric = 'retry' msg = 'must be retried' msg = '%(path)r (%(q_ts)f) in policy_index %(q_policy_index)s ' + msg self.stats_log(metric, msg, info, level=logging.INFO) self.log_stats() return success def _iter_containers(self): """ Generate a list of containers to process. """ # hit most recent container first instead of waiting on the updaters current_container = get_reconciler_container_name(time.time()) yield current_container container_gen = self.swift.iter_containers(MISPLACED_OBJECTS_ACCOUNT) self.logger.debug('looking for containers in %s', MISPLACED_OBJECTS_ACCOUNT) while True: one_page = None try: one_page = list( itertools.islice(container_gen, constraints.CONTAINER_LISTING_LIMIT)) except UnexpectedResponse as err: self.logger.error( 'Error listing containers in ' 'account %s (%s)', MISPLACED_OBJECTS_ACCOUNT, err) if not one_page: # don't generally expect more than one page break # reversed order since we expect older containers to be empty for c in reversed(one_page): # encoding here is defensive container = c['name'].encode('utf8') if container == current_container: continue # we've already hit this one this pass yield container def _iter_objects(self, container): """ Generate a list of objects to process. :param container: the name of the container to process If the given container is empty and older than reclaim_age this processor will attempt to reap it. """ self.logger.debug('looking for objects in %s', container) found_obj = False try: for raw_obj in self.swift.iter_objects(MISPLACED_OBJECTS_ACCOUNT, container): found_obj = True yield raw_obj except UnexpectedResponse as err: self.logger.error('Error listing objects in container %s (%s)', container, err) if float(container) < time.time() - self.reclaim_age and \ not found_obj: # Try to delete old empty containers so the queue doesn't # grow without bound. It's ok if there's a conflict. self.swift.delete_container(MISPLACED_OBJECTS_ACCOUNT, container, acceptable_statuses=(2, 404, 409, 412)) def reconcile(self): """ Main entry point for processing misplaced objects. Iterate over all queue entries and delegate to reconcile_object. """ self.logger.debug('pulling items from the queue') for container in self._iter_containers(): for raw_obj in self._iter_objects(container): try: obj_info = parse_raw_obj(raw_obj) except Exception: self.stats_log('invalid_record', 'invalid queue record: %r', raw_obj, level=logging.ERROR, exc_info=True) continue finished = self.reconcile_object(obj_info) if finished: self.pop_queue(container, raw_obj['name'], obj_info['q_ts'], obj_info['q_record']) self.log_stats() self.logger.debug('finished container %s', container) def run_once(self, *args, **kwargs): """ Process every entry in the queue. """ try: self.reconcile() except: # noqa self.logger.exception('Unhandled Exception trying to reconcile') self.log_stats(force=True) def run_forever(self, *args, **kwargs): while True: self.run_once(*args, **kwargs) self.stats = defaultdict(int) self.logger.info('sleeping between intervals (%ss)', self.interval) time.sleep(self.interval)
class ContainerReconciler(Daemon): """ Move objects that are in the wrong storage policy. """ def __init__(self, conf): self.conf = conf self.reclaim_age = int(conf.get('reclaim_age', 86400 * 7)) self.interval = int(conf.get('interval', 30)) conf_path = conf.get('__file__') or \ '/etc/swift/container-reconciler.conf' self.logger = get_logger(conf, log_route='container-reconciler') request_tries = int(conf.get('request_tries') or 3) self.swift = InternalClient(conf_path, 'Swift Container Reconciler', request_tries) self.stats = defaultdict(int) self.last_stat_time = time.time() def stats_log(self, metric, msg, *args, **kwargs): """ Update stats tracking for metric and emit log message. """ level = kwargs.pop('level', logging.DEBUG) log_message = '%s: ' % metric + msg self.logger.log(level, log_message, *args, **kwargs) self.stats[metric] += 1 def log_stats(self, force=False): """ Dump stats to logger, noop when stats have been already been logged in the last minute. """ now = time.time() should_log = force or (now - self.last_stat_time > 60) if should_log: self.last_stat_time = now self.logger.info('Reconciler Stats: %r', dict(**self.stats)) def pop_queue(self, container, obj, q_ts, q_record): """ Issue a delete object request to the container for the misplaced object queue entry. :param container: the misplaced objects container :param q_ts: the timestamp of the misplaced object :param q_record: the timestamp of the queue entry N.B. q_ts will normally be the same time as q_record except when an object was manually re-enqued. """ q_path = '/%s/%s/%s' % (MISPLACED_OBJECTS_ACCOUNT, container, obj) x_timestamp = slightly_later_timestamp(max(q_record, q_ts)) self.stats_log('pop_queue', 'remove %r (%f) from the queue (%s)', q_path, q_ts, x_timestamp) headers = {'X-Timestamp': x_timestamp} direct_delete_container_entry( self.swift.container_ring, MISPLACED_OBJECTS_ACCOUNT, container, obj, headers=headers) def throw_tombstones(self, account, container, obj, timestamp, policy_index, path): """ Issue a delete object request to the given storage_policy. :param account: the account name :param container: the container name :param account: the object name :param timestamp: the timestamp of the object to delete :param policy_index: the policy index to direct the request :param path: the path to be used for logging """ x_timestamp = slightly_later_timestamp(timestamp) self.stats_log('cleanup_attempt', '%r (%f) from policy_index ' '%s (%s) will be deleted', path, timestamp, policy_index, x_timestamp) headers = { 'X-Timestamp': x_timestamp, 'X-Backend-Storage-Policy-Index': policy_index, } success = False try: self.swift.delete_object(account, container, obj, acceptable_statuses=(2, 404), headers=headers) except UnexpectedResponse as err: self.stats_log('cleanup_failed', '%r (%f) was not cleaned up ' 'in storage_policy %s (%s)', path, timestamp, policy_index, err) else: success = True self.stats_log('cleanup_success', '%r (%f) was successfully ' 'removed from policy_index %s', path, timestamp, policy_index) return success def _reconcile_object(self, account, container, obj, q_policy_index, q_ts, q_op, path, **kwargs): """ Perform object reconciliation. :param account: the account name of the misplaced object :param container: the container name of the misplaced object :param obj: the object name :param q_policy_index: the policy index of the source indicated by the queue entry. :param q_ts: the timestamp of the misplaced object :param q_op: the operation of the misplaced request :param path: the full path of the misplaced object for logging :returns: True to indicate the request is fully processed successfully, otherwise False. """ container_policy_index = direct_get_container_policy_index( self.swift.container_ring, account, container) if container_policy_index is None: self.stats_log('unavailable_container', '%r (%f) unable to ' 'determine the destination policy_index', path, q_ts) return False if container_policy_index == q_policy_index: self.stats_log('noop_object', '%r (%f) container policy_index ' '%s matches queue policy index %s', path, q_ts, container_policy_index, q_policy_index) return True # check if object exists in the destination already self.logger.debug('checking for %r (%f) in destination ' 'policy_index %s', path, q_ts, container_policy_index) headers = { 'X-Backend-Storage-Policy-Index': container_policy_index} dest_obj = self.swift.get_object_metadata(account, container, obj, headers=headers, acceptable_statuses=(2, 4)) dest_ts = Timestamp(dest_obj.get('x-backend-timestamp', 0)) if dest_ts >= q_ts: self.stats_log('found_object', '%r (%f) in policy_index %s ' 'is newer than queue (%f)', path, dest_ts, container_policy_index, q_ts) return self.throw_tombstones(account, container, obj, q_ts, q_policy_index, path) # object is misplaced self.stats_log('misplaced_object', '%r (%f) in policy_index %s ' 'should be in policy_index %s', path, q_ts, q_policy_index, container_policy_index) # fetch object from the source location self.logger.debug('fetching %r (%f) from storage policy %s', path, q_ts, q_policy_index) headers = { 'X-Backend-Storage-Policy-Index': q_policy_index} try: source_obj_status, source_obj_info, source_obj_iter = \ self.swift.get_object(account, container, obj, headers=headers, acceptable_statuses=(2, 4)) except UnexpectedResponse as err: source_obj_status = err.resp.status_int source_obj_info = {} source_obj_iter = None source_ts = Timestamp(source_obj_info.get('x-backend-timestamp', 0)) if source_obj_status == 404 and q_op == 'DELETE': return self.ensure_tombstone_in_right_location( q_policy_index, account, container, obj, q_ts, path, container_policy_index, source_ts) else: return self.ensure_object_in_right_location( q_policy_index, account, container, obj, q_ts, path, container_policy_index, source_ts, source_obj_status, source_obj_info, source_obj_iter) def ensure_object_in_right_location(self, q_policy_index, account, container, obj, q_ts, path, container_policy_index, source_ts, source_obj_status, source_obj_info, source_obj_iter, **kwargs): """ Validate source object will satisfy the misplaced object queue entry and move to destination. :param q_policy_index: the policy_index for the source object :param account: the account name of the misplaced object :param container: the container name of the misplaced object :param obj: the name of the misplaced object :param q_ts: the timestamp of the misplaced object :param path: the full path of the misplaced object for logging :param container_policy_index: the policy_index of the destination :param source_ts: the timestamp of the source object :param source_obj_status: the HTTP status source object request :param source_obj_info: the HTTP headers of the source object request :param source_obj_iter: the body iter of the source object request """ if source_obj_status // 100 != 2 or source_ts < q_ts: if q_ts < time.time() - self.reclaim_age: # it's old and there are no tombstones or anything; give up self.stats_log('lost_source', '%r (%s) was not available in ' 'policy_index %s and has expired', path, q_ts.internal, q_policy_index, level=logging.CRITICAL) return True # the source object is unavailable or older than the queue # entry; a version that will satisfy the queue entry hopefully # exists somewhere in the cluster, so wait and try again self.stats_log('unavailable_source', '%r (%s) in ' 'policy_index %s responded %s (%s)', path, q_ts.internal, q_policy_index, source_obj_status, source_ts.internal, level=logging.WARNING) return False # optimistically move any source with a timestamp >= q_ts ts = max(Timestamp(source_ts), q_ts) # move the object put_timestamp = slightly_later_timestamp(ts, offset=2) self.stats_log('copy_attempt', '%r (%f) in policy_index %s will be ' 'moved to policy_index %s (%s)', path, source_ts, q_policy_index, container_policy_index, put_timestamp) headers = source_obj_info.copy() headers['X-Backend-Storage-Policy-Index'] = container_policy_index headers['X-Timestamp'] = put_timestamp try: self.swift.upload_object( FileLikeIter(source_obj_iter), account, container, obj, headers=headers) except UnexpectedResponse as err: self.stats_log('copy_failed', 'upload %r (%f) from ' 'policy_index %s to policy_index %s ' 'returned %s', path, source_ts, q_policy_index, container_policy_index, err, level=logging.WARNING) return False except: # noqa self.stats_log('unhandled_error', 'unable to upload %r (%f) ' 'from policy_index %s to policy_index %s ', path, source_ts, q_policy_index, container_policy_index, level=logging.ERROR, exc_info=True) return False self.stats_log('copy_success', '%r (%f) moved from policy_index %s ' 'to policy_index %s (%s)', path, source_ts, q_policy_index, container_policy_index, put_timestamp) return self.throw_tombstones(account, container, obj, q_ts, q_policy_index, path) def ensure_tombstone_in_right_location(self, q_policy_index, account, container, obj, q_ts, path, container_policy_index, source_ts, **kwargs): """ Issue a DELETE request against the destination to match the misplaced DELETE against the source. """ delete_timestamp = slightly_later_timestamp(q_ts, offset=2) self.stats_log('delete_attempt', '%r (%f) in policy_index %s ' 'will be deleted from policy_index %s (%s)', path, source_ts, q_policy_index, container_policy_index, delete_timestamp) headers = { 'X-Backend-Storage-Policy-Index': container_policy_index, 'X-Timestamp': delete_timestamp, } try: self.swift.delete_object(account, container, obj, headers=headers) except UnexpectedResponse as err: self.stats_log('delete_failed', 'delete %r (%f) from ' 'policy_index %s (%s) returned %s', path, source_ts, container_policy_index, delete_timestamp, err, level=logging.WARNING) return False except: # noqa self.stats_log('unhandled_error', 'unable to delete %r (%f) ' 'from policy_index %s (%s)', path, source_ts, container_policy_index, delete_timestamp, level=logging.ERROR, exc_info=True) return False self.stats_log('delete_success', '%r (%f) deleted from ' 'policy_index %s (%s)', path, source_ts, container_policy_index, delete_timestamp, level=logging.INFO) return self.throw_tombstones(account, container, obj, q_ts, q_policy_index, path) def reconcile_object(self, info): """ Process a possibly misplaced object write request. Determine correct destination storage policy by checking with primary containers. Check source and destination, copying or deleting into destination and cleaning up the source as needed. This method wraps _reconcile_object for exception handling. :param info: a queue entry dict :returns: True to indicate the request is fully processed successfully, otherwise False. """ self.logger.debug('checking placement for %r (%f) ' 'in policy_index %s', info['path'], info['q_ts'], info['q_policy_index']) success = False try: success = self._reconcile_object(**info) except: # noqa self.logger.exception('Unhandled Exception trying to ' 'reconcile %r (%f) in policy_index %s', info['path'], info['q_ts'], info['q_policy_index']) if success: metric = 'success' msg = 'was handled successfully' else: metric = 'retry' msg = 'must be retried' msg = '%(path)r (%(q_ts)f) in policy_index %(q_policy_index)s ' + msg self.stats_log(metric, msg, info, level=logging.INFO) self.log_stats() return success def _iter_containers(self): """ Generate a list of containers to process. """ # hit most recent container first instead of waiting on the updaters current_container = get_reconciler_container_name(time.time()) yield current_container container_gen = self.swift.iter_containers(MISPLACED_OBJECTS_ACCOUNT) self.logger.debug('looking for containers in %s', MISPLACED_OBJECTS_ACCOUNT) while True: one_page = None try: one_page = list(itertools.islice( container_gen, constraints.CONTAINER_LISTING_LIMIT)) except UnexpectedResponse as err: self.logger.error('Error listing containers in ' 'account %s (%s)', MISPLACED_OBJECTS_ACCOUNT, err) if not one_page: # don't generally expect more than one page break # reversed order since we expect older containers to be empty for c in reversed(one_page): # encoding here is defensive container = c['name'].encode('utf8') if container == current_container: continue # we've already hit this one this pass yield container def _iter_objects(self, container): """ Generate a list of objects to process. :param container: the name of the container to process If the given container is empty and older than reclaim_age this processor will attempt to reap it. """ self.logger.debug('looking for objects in %s', container) found_obj = False try: for raw_obj in self.swift.iter_objects( MISPLACED_OBJECTS_ACCOUNT, container): found_obj = True yield raw_obj except UnexpectedResponse as err: self.logger.error('Error listing objects in container %s (%s)', container, err) if float(container) < time.time() - self.reclaim_age and \ not found_obj: # Try to delete old empty containers so the queue doesn't # grow without bound. It's ok if there's a conflict. self.swift.delete_container( MISPLACED_OBJECTS_ACCOUNT, container, acceptable_statuses=(2, 404, 409, 412)) def reconcile(self): """ Main entry point for processing misplaced objects. Iterate over all queue entries and delegate to reconcile_object. """ self.logger.debug('pulling items from the queue') for container in self._iter_containers(): for raw_obj in self._iter_objects(container): try: obj_info = parse_raw_obj(raw_obj) except Exception: self.stats_log('invalid_record', 'invalid queue record: %r', raw_obj, level=logging.ERROR, exc_info=True) continue finished = self.reconcile_object(obj_info) if finished: self.pop_queue(container, raw_obj['name'], obj_info['q_ts'], obj_info['q_record']) self.log_stats() self.logger.debug('finished container %s', container) def run_once(self, *args, **kwargs): """ Process every entry in the queue. """ try: self.reconcile() except: self.logger.exception('Unhandled Exception trying to reconcile') self.log_stats(force=True) def run_forever(self, *args, **kwargs): while True: self.run_once(*args, **kwargs) self.stats = defaultdict(int) self.logger.info('sleeping between intervals (%ss)', self.interval) time.sleep(self.interval)
class ContainerSync(Daemon): """ Daemon to sync syncable containers. This is done by scanning the local devices for container databases and checking for x-container-sync-to and x-container-sync-key metadata values. If they exist, newer rows since the last sync will trigger PUTs or DELETEs to the other container. The actual syncing is slightly more complicated to make use of the three (or number-of-replicas) main nodes for a container without each trying to do the exact same work but also without missing work if one node happens to be down. Two sync points are kept per container database. All rows between the two sync points trigger updates. Any rows newer than both sync points cause updates depending on the node's position for the container (primary nodes do one third, etc. depending on the replica count of course). After a sync run, the first sync point is set to the newest ROWID known and the second sync point is set to newest ROWID for which all updates have been sent. An example may help. Assume replica count is 3 and perfectly matching ROWIDs starting at 1. First sync run, database has 6 rows: * SyncPoint1 starts as -1. * SyncPoint2 starts as -1. * No rows between points, so no "all updates" rows. * Six rows newer than SyncPoint1, so a third of the rows are sent by node 1, another third by node 2, remaining third by node 3. * SyncPoint1 is set as 6 (the newest ROWID known). * SyncPoint2 is left as -1 since no "all updates" rows were synced. Next sync run, database has 12 rows: * SyncPoint1 starts as 6. * SyncPoint2 starts as -1. * The rows between -1 and 6 all trigger updates (most of which should short-circuit on the remote end as having already been done). * Six more rows newer than SyncPoint1, so a third of the rows are sent by node 1, another third by node 2, remaining third by node 3. * SyncPoint1 is set as 12 (the newest ROWID known). * SyncPoint2 is set as 6 (the newest "all updates" ROWID). In this way, under normal circumstances each node sends its share of updates each run and just sends a batch of older updates to ensure nothing was missed. :param conf: The dict of configuration values from the [container-sync] section of the container-server.conf :param container_ring: If None, the <swift_dir>/container.ring.gz will be loaded. This is overridden by unit tests. """ def __init__(self, conf, container_ring=None, logger=None): #: The dict of configuration values from the [container-sync] section #: of the container-server.conf. self.conf = conf #: Logger to use for container-sync log lines. self.logger = logger or get_logger(conf, log_route='container-sync') #: Path to the local device mount points. self.devices = conf.get('devices', '/srv/node') #: Indicates whether mount points should be verified as actual mount #: points (normally true, false for tests and SAIO). self.mount_check = config_true_value(conf.get('mount_check', 'true')) #: Minimum time between full scans. This is to keep the daemon from #: running wild on near empty systems. self.interval = int(conf.get('interval', 300)) #: Maximum amount of time to spend syncing a container before moving on #: to the next one. If a container sync hasn't finished in this time, #: it'll just be resumed next scan. self.container_time = int(conf.get('container_time', 60)) #: ContainerSyncCluster instance for validating sync-to values. self.realms_conf = ContainerSyncRealms( os.path.join(conf.get('swift_dir', '/etc/swift'), 'container-sync-realms.conf'), self.logger) #: The list of hosts we're allowed to send syncs to. This can be #: overridden by data in self.realms_conf self.allowed_sync_hosts = [ h.strip() for h in conf.get('allowed_sync_hosts', '127.0.0.1').split(',') if h.strip() ] self.http_proxies = [ a.strip() for a in conf.get('sync_proxy', '').split(',') if a.strip() ] #: ContainerSyncStore instance for iterating over synced containers self.sync_store = ContainerSyncStore(self.devices, self.logger, self.mount_check) #: Number of containers with sync turned on that were successfully #: synced. self.container_syncs = 0 #: Number of successful DELETEs triggered. self.container_deletes = 0 #: Number of successful PUTs triggered. self.container_puts = 0 #: Number of containers whose sync has been turned off, but #: are not yet cleared from the sync store. self.container_skips = 0 #: Number of containers that had a failure of some type. self.container_failures = 0 #: Per container stats. These are collected per container. #: puts - the number of puts that were done for the container #: deletes - the number of deletes that were fot the container #: bytes - the total number of bytes transferred per the container self.container_stats = collections.defaultdict(int) self.container_stats.clear() #: Time of last stats report. self.reported = time() self.swift_dir = conf.get('swift_dir', '/etc/swift') #: swift.common.ring.Ring for locating containers. self.container_ring = container_ring or Ring(self.swift_dir, ring_name='container') bind_ip = conf.get('bind_ip', '0.0.0.0') self._myips = whataremyips(bind_ip) self._myport = int(conf.get('bind_port', 6201)) swift.common.db.DB_PREALLOCATION = \ config_true_value(conf.get('db_preallocation', 'f')) self.conn_timeout = float(conf.get('conn_timeout', 5)) request_tries = int(conf.get('request_tries') or 3) internal_client_conf_path = conf.get('internal_client_conf_path') if not internal_client_conf_path: self.logger.warning( _('Configuration option internal_client_conf_path not ' 'defined. Using default configuration, See ' 'internal-client.conf-sample for options')) internal_client_conf = ConfigString(ic_conf_body) else: internal_client_conf = internal_client_conf_path try: self.swift = InternalClient(internal_client_conf, 'Swift Container Sync', request_tries) except (OSError, IOError) as err: if err.errno != errno.ENOENT and \ not str(err).endswith(' not found'): raise raise SystemExit( _('Unable to load internal client from config: ' '%(conf)r (%(error)s)') % { 'conf': internal_client_conf_path, 'error': err }) def run_forever(self, *args, **kwargs): """ Runs container sync scans until stopped. """ sleep(random() * self.interval) while True: begin = time() for path in self.sync_store.synced_containers_generator(): self.container_stats.clear() self.container_sync(path) if time() - self.reported >= 3600: # once an hour self.report() elapsed = time() - begin if elapsed < self.interval: sleep(self.interval - elapsed) def run_once(self, *args, **kwargs): """ Runs a single container sync scan. """ self.logger.info(_('Begin container sync "once" mode')) begin = time() for path in self.sync_store.synced_containers_generator(): self.container_sync(path) if time() - self.reported >= 3600: # once an hour self.report() self.report() elapsed = time() - begin self.logger.info(_('Container sync "once" mode completed: %.02fs'), elapsed) def report(self): """ Writes a report of the stats to the logger and resets the stats for the next report. """ self.logger.info( _('Since %(time)s: %(sync)s synced [%(delete)s deletes, %(put)s ' 'puts], %(skip)s skipped, %(fail)s failed'), { 'time': ctime(self.reported), 'sync': self.container_syncs, 'delete': self.container_deletes, 'put': self.container_puts, 'skip': self.container_skips, 'fail': self.container_failures }) self.reported = time() self.container_syncs = 0 self.container_deletes = 0 self.container_puts = 0 self.container_skips = 0 self.container_failures = 0 def container_report(self, start, end, sync_point1, sync_point2, info, max_row): self.logger.info( _('Container sync report: %(container)s, ' 'time window start: %(start)s, ' 'time window end: %(end)s, ' 'puts: %(puts)s, ' 'posts: %(posts)s, ' 'deletes: %(deletes)s, ' 'bytes: %(bytes)s, ' 'sync_point1: %(point1)s, ' 'sync_point2: %(point2)s, ' 'total_rows: %(total)s'), { 'container': '%s/%s' % (info['account'], info['container']), 'start': start, 'end': end, 'puts': self.container_stats['puts'], 'posts': 0, 'deletes': self.container_stats['deletes'], 'bytes': self.container_stats['bytes'], 'point1': sync_point1, 'point2': sync_point2, 'total': max_row }) def container_sync(self, path): """ Checks the given path for a container database, determines if syncing is turned on for that database and, if so, sends any updates to the other container. :param path: the path to a container db """ broker = None try: broker = ContainerBroker(path, logger=self.logger) # The path we pass to the ContainerBroker is a real path of # a container DB. If we get here, however, it means that this # path is linked from the sync_containers dir. In rare cases # of race or processes failures the link can be stale and # the get_info below will raise a DB doesn't exist exception # In this case we remove the stale link and raise an error # since in most cases the db should be there. try: info = broker.get_info() except DatabaseConnectionError as db_err: if str(db_err).endswith("DB doesn't exist"): self.sync_store.remove_synced_container(broker) raise x, nodes = self.container_ring.get_nodes(info['account'], info['container']) for ordinal, node in enumerate(nodes): if is_local_device(self._myips, self._myport, node['ip'], node['port']): break else: return if not broker.is_deleted(): sync_to = None user_key = None sync_point1 = info['x_container_sync_point1'] sync_point2 = info['x_container_sync_point2'] for key, (value, timestamp) in broker.metadata.items(): if key.lower() == 'x-container-sync-to': sync_to = value elif key.lower() == 'x-container-sync-key': user_key = value if not sync_to or not user_key: self.container_skips += 1 self.logger.increment('skips') return err, sync_to, realm, realm_key = validate_sync_to( sync_to, self.allowed_sync_hosts, self.realms_conf) if err: self.logger.info( _('ERROR %(db_file)s: %(validate_sync_to_err)s'), { 'db_file': str(broker), 'validate_sync_to_err': err }) self.container_failures += 1 self.logger.increment('failures') return start_at = time() stop_at = start_at + self.container_time next_sync_point = None sync_stage_time = start_at try: while time() < stop_at and sync_point2 < sync_point1: rows = broker.get_items_since(sync_point2, 1) if not rows: break row = rows[0] if row['ROWID'] > sync_point1: break # This node will only initially sync out one third # of the objects (if 3 replicas, 1/4 if 4, etc.) # and will skip problematic rows as needed in case of # faults. # This section will attempt to sync previously skipped # rows in case the previous attempts by any of the # nodes didn't succeed. if not self.container_sync_row(row, sync_to, user_key, broker, info, realm, realm_key): if not next_sync_point: next_sync_point = sync_point2 sync_point2 = row['ROWID'] broker.set_x_container_sync_points(None, sync_point2) if next_sync_point: broker.set_x_container_sync_points( None, next_sync_point) else: next_sync_point = sync_point2 sync_stage_time = time() while sync_stage_time < stop_at: rows = broker.get_items_since(sync_point1, 1) if not rows: break row = rows[0] key = hash_path(info['account'], info['container'], row['name'], raw_digest=True) # This node will only initially sync out one third of # the objects (if 3 replicas, 1/4 if 4, etc.). # It'll come back around to the section above # and attempt to sync previously skipped rows in case # the other nodes didn't succeed or in case it failed # to do so the first time. if unpack_from('>I', key)[0] % \ len(nodes) == ordinal: self.container_sync_row(row, sync_to, user_key, broker, info, realm, realm_key) sync_point1 = row['ROWID'] broker.set_x_container_sync_points(sync_point1, None) sync_stage_time = time() self.container_syncs += 1 self.logger.increment('syncs') finally: self.container_report(start_at, sync_stage_time, sync_point1, next_sync_point, info, broker.get_max_row()) except (Exception, Timeout): self.container_failures += 1 self.logger.increment('failures') self.logger.exception(_('ERROR Syncing %s'), broker if broker else path) def _update_sync_to_headers(self, name, sync_to, user_key, realm, realm_key, method, headers): """ Updates container sync headers :param name: The name of the object :param sync_to: The URL to the remote container. :param user_key: The X-Container-Sync-Key to use when sending requests to the other container. :param realm: The realm from self.realms_conf, if there is one. If None, fallback to using the older allowed_sync_hosts way of syncing. :param realm_key: The realm key from self.realms_conf, if there is one. If None, fallback to using the older allowed_sync_hosts way of syncing. :param method: HTTP method to create sig with :param headers: headers to update with container sync headers """ if realm and realm_key: nonce = uuid.uuid4().hex path = urlparse(sync_to).path + '/' + quote(name) sig = self.realms_conf.get_sig(method, path, headers.get('x-timestamp', 0), nonce, realm_key, user_key) headers['x-container-sync-auth'] = '%s %s %s' % (realm, nonce, sig) else: headers['x-container-sync-key'] = user_key def _object_in_remote_container(self, name, sync_to, user_key, realm, realm_key, timestamp): """ Performs head object on remote to eliminate extra remote put and local get object calls :param name: The name of the object in the updated row in the local database triggering the sync update. :param sync_to: The URL to the remote container. :param user_key: The X-Container-Sync-Key to use when sending requests to the other container. :param realm: The realm from self.realms_conf, if there is one. If None, fallback to using the older allowed_sync_hosts way of syncing. :param realm_key: The realm key from self.realms_conf, if there is one. If None, fallback to using the older allowed_sync_hosts way of syncing. :param timestamp: last modified date of local object :returns: True if object already exists in remote """ headers = {'x-timestamp': timestamp.internal} self._update_sync_to_headers(name, sync_to, user_key, realm, realm_key, 'HEAD', headers) try: metadata, _ = head_object(sync_to, name=name, headers=headers, proxy=self.select_http_proxy(), logger=self.logger, retries=0) remote_ts = Timestamp(metadata.get('x-timestamp', 0)) self.logger.debug("remote obj timestamp %s local obj %s" % (timestamp.internal, remote_ts.internal)) if timestamp <= remote_ts: return True # Object in remote should be updated return False except ClientException as http_err: # Object not in remote if http_err.http_status == 404: return False raise http_err def container_sync_row(self, row, sync_to, user_key, broker, info, realm, realm_key): """ Sends the update the row indicates to the sync_to container. Update can be either delete or put. :param row: The updated row in the local database triggering the sync update. :param sync_to: The URL to the remote container. :param user_key: The X-Container-Sync-Key to use when sending requests to the other container. :param broker: The local container database broker. :param info: The get_info result from the local container database broker. :param realm: The realm from self.realms_conf, if there is one. If None, fallback to using the older allowed_sync_hosts way of syncing. :param realm_key: The realm key from self.realms_conf, if there is one. If None, fallback to using the older allowed_sync_hosts way of syncing. :returns: True on success """ try: start_time = time() # extract last modified time from the created_at value ts_data, ts_ctype, ts_meta = decode_timestamps(row['created_at']) if row['deleted']: # when sync'ing a deleted object, use ts_data - this is the # timestamp of the source tombstone try: headers = {'x-timestamp': ts_data.internal} self._update_sync_to_headers(row['name'], sync_to, user_key, realm, realm_key, 'DELETE', headers) delete_object(sync_to, name=row['name'], headers=headers, proxy=self.select_http_proxy(), logger=self.logger, timeout=self.conn_timeout) except ClientException as err: if err.http_status != HTTP_NOT_FOUND: raise self.container_deletes += 1 self.container_stats['deletes'] += 1 self.logger.increment('deletes') self.logger.timing_since('deletes.timing', start_time) else: # when sync'ing a live object, use ts_meta - this is the time # at which the source object was last modified by a PUT or POST if self._object_in_remote_container(row['name'], sync_to, user_key, realm, realm_key, ts_meta): return True exc = None # look up for the newest one; the symlink=get query-string has # no effect unless symlinks are enabled in the internal client # in which case it ensures that symlink objects retain their # symlink property when sync'd. headers_out = { 'X-Newest': True, 'X-Backend-Storage-Policy-Index': str(info['storage_policy_index']) } try: source_obj_status, headers, body = \ self.swift.get_object(info['account'], info['container'], row['name'], headers=headers_out, acceptable_statuses=(2, 4), params={'symlink': 'get'}) except (Exception, UnexpectedResponse, Timeout) as err: headers = {} body = None exc = err timestamp = Timestamp(headers.get('x-timestamp', 0)) if timestamp < ts_meta: if exc: raise exc raise Exception( _('Unknown exception trying to GET: ' '%(account)r %(container)r %(object)r'), { 'account': info['account'], 'container': info['container'], 'object': row['name'] }) for key in ('date', 'last-modified'): if key in headers: del headers[key] if 'etag' in headers: headers['etag'] = normalize_etag(headers['etag']) if 'content-type' in headers: headers['content-type'] = clean_content_type( headers['content-type']) self._update_sync_to_headers(row['name'], sync_to, user_key, realm, realm_key, 'PUT', headers) put_object(sync_to, name=row['name'], headers=headers, contents=FileLikeIter(body), proxy=self.select_http_proxy(), logger=self.logger, timeout=self.conn_timeout) self.container_puts += 1 self.container_stats['puts'] += 1 self.container_stats['bytes'] += row['size'] self.logger.increment('puts') self.logger.timing_since('puts.timing', start_time) except ClientException as err: if err.http_status == HTTP_UNAUTHORIZED: self.logger.info( _('Unauth %(sync_from)r => %(sync_to)r'), { 'sync_from': '%s/%s' % (quote(info['account']), quote(info['container'])), 'sync_to': sync_to }) elif err.http_status == HTTP_NOT_FOUND: self.logger.info( _('Not found %(sync_from)r => %(sync_to)r \ - object %(obj_name)r'), { 'sync_from': '%s/%s' % (quote(info['account']), quote(info['container'])), 'sync_to': sync_to, 'obj_name': row['name'] }) else: self.logger.exception(_('ERROR Syncing %(db_file)s %(row)s'), { 'db_file': str(broker), 'row': row }) self.container_failures += 1 self.logger.increment('failures') return False except (Exception, Timeout) as err: self.logger.exception(_('ERROR Syncing %(db_file)s %(row)s'), { 'db_file': str(broker), 'row': row }) self.container_failures += 1 self.logger.increment('failures') return False return True def select_http_proxy(self): return choice(self.http_proxies) if self.http_proxies else None
class ContainerSync(Daemon): """ Daemon to sync syncable containers. This is done by scanning the local devices for container databases and checking for x-container-sync-to and x-container-sync-key metadata values. If they exist, newer rows since the last sync will trigger PUTs or DELETEs to the other container. .. note:: Container sync will sync object POSTs only if the proxy server is set to use "object_post_as_copy = true" which is the default. So-called fast object posts, "object_post_as_copy = false" do not update the container listings and therefore can't be detected for synchronization. The actual syncing is slightly more complicated to make use of the three (or number-of-replicas) main nodes for a container without each trying to do the exact same work but also without missing work if one node happens to be down. Two sync points are kept per container database. All rows between the two sync points trigger updates. Any rows newer than both sync points cause updates depending on the node's position for the container (primary nodes do one third, etc. depending on the replica count of course). After a sync run, the first sync point is set to the newest ROWID known and the second sync point is set to newest ROWID for which all updates have been sent. An example may help. Assume replica count is 3 and perfectly matching ROWIDs starting at 1. First sync run, database has 6 rows: * SyncPoint1 starts as -1. * SyncPoint2 starts as -1. * No rows between points, so no "all updates" rows. * Six rows newer than SyncPoint1, so a third of the rows are sent by node 1, another third by node 2, remaining third by node 3. * SyncPoint1 is set as 6 (the newest ROWID known). * SyncPoint2 is left as -1 since no "all updates" rows were synced. Next sync run, database has 12 rows: * SyncPoint1 starts as 6. * SyncPoint2 starts as -1. * The rows between -1 and 6 all trigger updates (most of which should short-circuit on the remote end as having already been done). * Six more rows newer than SyncPoint1, so a third of the rows are sent by node 1, another third by node 2, remaining third by node 3. * SyncPoint1 is set as 12 (the newest ROWID known). * SyncPoint2 is set as 6 (the newest "all updates" ROWID). In this way, under normal circumstances each node sends its share of updates each run and just sends a batch of older updates to ensure nothing was missed. :param conf: The dict of configuration values from the [container-sync] section of the container-server.conf :param container_ring: If None, the <swift_dir>/container.ring.gz will be loaded. This is overridden by unit tests. """ def __init__(self, conf, container_ring=None, logger=None): #: The dict of configuration values from the [container-sync] section #: of the container-server.conf. self.conf = conf #: Logger to use for container-sync log lines. self.logger = logger or get_logger(conf, log_route='container-sync') #: Path to the local device mount points. self.devices = conf.get('devices', '/srv/node') #: Indicates whether mount points should be verified as actual mount #: points (normally true, false for tests and SAIO). self.mount_check = config_true_value(conf.get('mount_check', 'true')) #: Minimum time between full scans. This is to keep the daemon from #: running wild on near empty systems. self.interval = int(conf.get('interval', 300)) #: Maximum amount of time to spend syncing a container before moving on #: to the next one. If a conatiner sync hasn't finished in this time, #: it'll just be resumed next scan. self.container_time = int(conf.get('container_time', 60)) #: ContainerSyncCluster instance for validating sync-to values. self.realms_conf = ContainerSyncRealms( os.path.join(conf.get('swift_dir', '/etc/swift'), 'container-sync-realms.conf'), self.logger) #: The list of hosts we're allowed to send syncs to. This can be #: overridden by data in self.realms_conf self.allowed_sync_hosts = [ h.strip() for h in conf.get('allowed_sync_hosts', '127.0.0.1').split(',') if h.strip() ] self.http_proxies = [ a.strip() for a in conf.get('sync_proxy', '').split(',') if a.strip() ] #: Number of containers with sync turned on that were successfully #: synced. self.container_syncs = 0 #: Number of successful DELETEs triggered. self.container_deletes = 0 #: Number of successful PUTs triggered. self.container_puts = 0 #: Number of containers that didn't have sync turned on. self.container_skips = 0 #: Number of containers that had a failure of some type. self.container_failures = 0 #: Time of last stats report. self.reported = time() self.swift_dir = conf.get('swift_dir', '/etc/swift') #: swift.common.ring.Ring for locating containers. self.container_ring = container_ring or Ring(self.swift_dir, ring_name='container') self._myips = whataremyips() self._myport = int(conf.get('bind_port', 6001)) swift.common.db.DB_PREALLOCATION = \ config_true_value(conf.get('db_preallocation', 'f')) self.conn_timeout = float(conf.get('conn_timeout', 5)) request_tries = int(conf.get('request_tries') or 3) internal_client_conf_path = conf.get('internal_client_conf_path') if not internal_client_conf_path: self.logger.warning( _('Configuration option internal_client_conf_path not ' 'defined. Using default configuration, See ' 'internal-client.conf-sample for options')) internal_client_conf = ConfigString(ic_conf_body) else: internal_client_conf = internal_client_conf_path try: self.swift = InternalClient(internal_client_conf, 'Swift Container Sync', request_tries) except IOError as err: if err.errno != errno.ENOENT: raise raise SystemExit( _('Unable to load internal client from config: %r (%s)') % (internal_client_conf_path, err)) def get_object_ring(self, policy_idx): """ Get the ring object to use based on its policy. :policy_idx: policy index as defined in swift.conf :returns: appropriate ring object """ return POLICIES.get_object_ring(policy_idx, self.swift_dir) def run_forever(self, *args, **kwargs): """ Runs container sync scans until stopped. """ sleep(random() * self.interval) while True: begin = time() all_locs = audit_location_generator(self.devices, DATADIR, '.db', mount_check=self.mount_check, logger=self.logger) for path, device, partition in all_locs: self.container_sync(path) if time() - self.reported >= 3600: # once an hour self.report() elapsed = time() - begin if elapsed < self.interval: sleep(self.interval - elapsed) def run_once(self, *args, **kwargs): """ Runs a single container sync scan. """ self.logger.info(_('Begin container sync "once" mode')) begin = time() all_locs = audit_location_generator(self.devices, DATADIR, '.db', mount_check=self.mount_check, logger=self.logger) for path, device, partition in all_locs: self.container_sync(path) if time() - self.reported >= 3600: # once an hour self.report() self.report() elapsed = time() - begin self.logger.info(_('Container sync "once" mode completed: %.02fs'), elapsed) def report(self): """ Writes a report of the stats to the logger and resets the stats for the next report. """ self.logger.info( _('Since %(time)s: %(sync)s synced [%(delete)s deletes, %(put)s ' 'puts], %(skip)s skipped, %(fail)s failed'), { 'time': ctime(self.reported), 'sync': self.container_syncs, 'delete': self.container_deletes, 'put': self.container_puts, 'skip': self.container_skips, 'fail': self.container_failures }) self.reported = time() self.container_syncs = 0 self.container_deletes = 0 self.container_puts = 0 self.container_skips = 0 self.container_failures = 0 def container_sync(self, path): """ Checks the given path for a container database, determines if syncing is turned on for that database and, if so, sends any updates to the other container. :param path: the path to a container db """ broker = None try: broker = ContainerBroker(path) info = broker.get_info() x, nodes = self.container_ring.get_nodes(info['account'], info['container']) for ordinal, node in enumerate(nodes): if is_local_device(self._myips, self._myport, node['ip'], node['port']): break else: return if not broker.is_deleted(): sync_to = None user_key = None sync_point1 = info['x_container_sync_point1'] sync_point2 = info['x_container_sync_point2'] for key, (value, timestamp) in broker.metadata.iteritems(): if key.lower() == 'x-container-sync-to': sync_to = value elif key.lower() == 'x-container-sync-key': user_key = value if not sync_to or not user_key: self.container_skips += 1 self.logger.increment('skips') return err, sync_to, realm, realm_key = validate_sync_to( sync_to, self.allowed_sync_hosts, self.realms_conf) if err: self.logger.info( _('ERROR %(db_file)s: %(validate_sync_to_err)s'), { 'db_file': str(broker), 'validate_sync_to_err': err }) self.container_failures += 1 self.logger.increment('failures') return stop_at = time() + self.container_time next_sync_point = None while time() < stop_at and sync_point2 < sync_point1: rows = broker.get_items_since(sync_point2, 1) if not rows: break row = rows[0] if row['ROWID'] > sync_point1: break key = hash_path(info['account'], info['container'], row['name'], raw_digest=True) # This node will only initially sync out one third of the # objects (if 3 replicas, 1/4 if 4, etc.) and will skip # problematic rows as needed in case of faults. # This section will attempt to sync previously skipped # rows in case the previous attempts by any of the nodes # didn't succeed. if not self.container_sync_row(row, sync_to, user_key, broker, info, realm, realm_key): if not next_sync_point: next_sync_point = sync_point2 sync_point2 = row['ROWID'] broker.set_x_container_sync_points(None, sync_point2) if next_sync_point: broker.set_x_container_sync_points(None, next_sync_point) while time() < stop_at: rows = broker.get_items_since(sync_point1, 1) if not rows: break row = rows[0] key = hash_path(info['account'], info['container'], row['name'], raw_digest=True) # This node will only initially sync out one third of the # objects (if 3 replicas, 1/4 if 4, etc.). It'll come back # around to the section above and attempt to sync # previously skipped rows in case the other nodes didn't # succeed or in case it failed to do so the first time. if unpack_from('>I', key)[0] % \ len(nodes) == ordinal: self.container_sync_row(row, sync_to, user_key, broker, info, realm, realm_key) sync_point1 = row['ROWID'] broker.set_x_container_sync_points(sync_point1, None) self.container_syncs += 1 self.logger.increment('syncs') except (Exception, Timeout) as err: self.container_failures += 1 self.logger.increment('failures') self.logger.exception(_('ERROR Syncing %s'), broker if broker else path) def container_sync_row(self, row, sync_to, user_key, broker, info, realm, realm_key): """ Sends the update the row indicates to the sync_to container. :param row: The updated row in the local database triggering the sync update. :param sync_to: The URL to the remote container. :param user_key: The X-Container-Sync-Key to use when sending requests to the other container. :param broker: The local container database broker. :param info: The get_info result from the local container database broker. :param realm: The realm from self.realms_conf, if there is one. If None, fallback to using the older allowed_sync_hosts way of syncing. :param realm_key: The realm key from self.realms_conf, if there is one. If None, fallback to using the older allowed_sync_hosts way of syncing. :returns: True on success """ try: start_time = time() if row['deleted']: try: headers = {'x-timestamp': row['created_at']} if realm and realm_key: nonce = uuid.uuid4().hex path = urlparse(sync_to).path + '/' + quote( row['name']) sig = self.realms_conf.get_sig('DELETE', path, headers['x-timestamp'], nonce, realm_key, user_key) headers['x-container-sync-auth'] = '%s %s %s' % ( realm, nonce, sig) else: headers['x-container-sync-key'] = user_key delete_object(sync_to, name=row['name'], headers=headers, proxy=self.select_http_proxy(), logger=self.logger, timeout=self.conn_timeout) except ClientException as err: if err.http_status != HTTP_NOT_FOUND: raise self.container_deletes += 1 self.logger.increment('deletes') self.logger.timing_since('deletes.timing', start_time) else: part, nodes = \ self.get_object_ring(info['storage_policy_index']). \ get_nodes(info['account'], info['container'], row['name']) shuffle(nodes) exc = None looking_for_timestamp = Timestamp(row['created_at']) timestamp = -1 headers = body = None # look up for the newest one headers_out = { 'X-Newest': True, 'X-Backend-Storage-Policy-Index': str(info['storage_policy_index']) } try: source_obj_status, source_obj_info, source_obj_iter = \ self.swift.get_object(info['account'], info['container'], row['name'], headers=headers_out, acceptable_statuses=(2, 4)) except (Exception, UnexpectedResponse, Timeout) as err: source_obj_info = {} source_obj_iter = None exc = err timestamp = Timestamp(source_obj_info.get('x-timestamp', 0)) headers = source_obj_info body = source_obj_iter if timestamp < looking_for_timestamp: if exc: raise exc raise Exception( _('Unknown exception trying to GET: ' '%(account)r %(container)r %(object)r'), { 'account': info['account'], 'container': info['container'], 'object': row['name'] }) for key in ('date', 'last-modified'): if key in headers: del headers[key] if 'etag' in headers: headers['etag'] = headers['etag'].strip('"') if 'content-type' in headers: headers['content-type'] = clean_content_type( headers['content-type']) headers['x-timestamp'] = row['created_at'] if realm and realm_key: nonce = uuid.uuid4().hex path = urlparse(sync_to).path + '/' + quote(row['name']) sig = self.realms_conf.get_sig('PUT', path, headers['x-timestamp'], nonce, realm_key, user_key) headers['x-container-sync-auth'] = '%s %s %s' % ( realm, nonce, sig) else: headers['x-container-sync-key'] = user_key put_object(sync_to, name=row['name'], headers=headers, contents=FileLikeIter(body), proxy=self.select_http_proxy(), logger=self.logger, timeout=self.conn_timeout) self.container_puts += 1 self.logger.increment('puts') self.logger.timing_since('puts.timing', start_time) except ClientException as err: if err.http_status == HTTP_UNAUTHORIZED: self.logger.info( _('Unauth %(sync_from)r => %(sync_to)r'), { 'sync_from': '%s/%s' % (quote(info['account']), quote(info['container'])), 'sync_to': sync_to }) elif err.http_status == HTTP_NOT_FOUND: self.logger.info( _('Not found %(sync_from)r => %(sync_to)r \ - object %(obj_name)r'), { 'sync_from': '%s/%s' % (quote(info['account']), quote(info['container'])), 'sync_to': sync_to, 'obj_name': row['name'] }) else: self.logger.exception(_('ERROR Syncing %(db_file)s %(row)s'), { 'db_file': str(broker), 'row': row }) self.container_failures += 1 self.logger.increment('failures') return False except (Exception, Timeout) as err: self.logger.exception(_('ERROR Syncing %(db_file)s %(row)s'), { 'db_file': str(broker), 'row': row }) self.container_failures += 1 self.logger.increment('failures') return False return True def select_http_proxy(self): return choice(self.http_proxies) if self.http_proxies else None