def create_internal_client(conf, swift_dir): ic_config = conf.get('internal_client_path', os.path.join(swift_dir, 'internal-client.conf')) if not os.path.exists(ic_config): ic_config = ConfigString(INTERNAL_CLIENT_CONFIG) ic_name = conf.get('internal_client_logname', 'ContainerCrawler') return InternalClient(ic_config, ic_name, 3)
def __init__(self, status_dir, settings): self._status_dir = status_dir self._account = settings['account'] self._container = settings['container'] ic_config = ConfigString(self.INTERNAL_CLIENT_CONFIG) self._swift_client = InternalClient(ic_config, 'Metadata sync', 3) self._status_file = os.path.join(self._status_dir, self._account, self._container) self._status_account_dir = os.path.join(self._status_dir, self._account)
def __init__(self, conf, container_ring=None, logger=None): #: The dict of configuration values from the [container-sync] section #: of the container-server.conf. self.conf = conf #: Logger to use for container-sync log lines. self.logger = logger or get_logger(conf, log_route='container-sync') #: Path to the local device mount points. self.devices = conf.get('devices', '/srv/node') #: Indicates whether mount points should be verified as actual mount #: points (normally true, false for tests and SAIO). self.mount_check = config_true_value(conf.get('mount_check', 'true')) #: Minimum time between full scans. This is to keep the daemon from #: running wild on near empty systems. self.interval = int(conf.get('interval', 300)) #: Maximum amount of time to spend syncing a container before moving on #: to the next one. If a container sync hasn't finished in this time, #: it'll just be resumed next scan. self.container_time = int(conf.get('container_time', 60)) #: ContainerSyncCluster instance for validating sync-to values. self.realms_conf = ContainerSyncRealms( os.path.join(conf.get('swift_dir', '/etc/swift'), 'container-sync-realms.conf'), self.logger) #: The list of hosts we're allowed to send syncs to. This can be #: overridden by data in self.realms_conf self.allowed_sync_hosts = [ h.strip() for h in conf.get('allowed_sync_hosts', '127.0.0.1').split(',') if h.strip() ] self.http_proxies = [ a.strip() for a in conf.get('sync_proxy', '').split(',') if a.strip() ] #: ContainerSyncStore instance for iterating over synced containers self.sync_store = ContainerSyncStore(self.devices, self.logger, self.mount_check) #: Number of containers with sync turned on that were successfully #: synced. self.container_syncs = 0 #: Number of successful DELETEs triggered. self.container_deletes = 0 #: Number of successful PUTs triggered. self.container_puts = 0 #: Number of containers whose sync has been turned off, but #: are not yet cleared from the sync store. self.container_skips = 0 #: Number of containers that had a failure of some type. self.container_failures = 0 #: Per container stats. These are collected per container. #: puts - the number of puts that were done for the container #: deletes - the number of deletes that were fot the container #: bytes - the total number of bytes transferred per the container self.container_stats = collections.defaultdict(int) self.container_stats.clear() #: Time of last stats report. self.reported = time() self.swift_dir = conf.get('swift_dir', '/etc/swift') #: swift.common.ring.Ring for locating containers. self.container_ring = container_ring or Ring(self.swift_dir, ring_name='container') bind_ip = conf.get('bind_ip', '0.0.0.0') self._myips = whataremyips(bind_ip) self._myport = int(conf.get('bind_port', 6201)) swift.common.db.DB_PREALLOCATION = \ config_true_value(conf.get('db_preallocation', 'f')) self.conn_timeout = float(conf.get('conn_timeout', 5)) request_tries = int(conf.get('request_tries') or 3) internal_client_conf_path = conf.get('internal_client_conf_path') if not internal_client_conf_path: self.logger.warning( _('Configuration option internal_client_conf_path not ' 'defined. Using default configuration, See ' 'internal-client.conf-sample for options')) internal_client_conf = ConfigString(ic_conf_body) else: internal_client_conf = internal_client_conf_path try: self.swift = InternalClient(internal_client_conf, 'Swift Container Sync', request_tries) except (OSError, IOError) as err: if err.errno != errno.ENOENT and \ not str(err).endswith(' not found'): raise raise SystemExit( _('Unable to load internal client from config: ' '%(conf)r (%(error)s)') % { 'conf': internal_client_conf_path, 'error': err })