def run_once(self, *args, **kwargs): """Run a replication pass once.""" self._zero_stats() dirs = [] ips = whataremyips() if not ips: self.logger.error(_('ERROR Failed to get my own IPs?')) return for node in self.ring.devs: if node and node['ip'] in ips and node['port'] == self.port: if self.mount_check and not os.path.ismount( os.path.join(self.root, node['device'])): self.logger.warn( _('Skipping %(device)s as it is not mounted') % node) continue unlink_older_than( os.path.join(self.root, node['device'], 'tmp'), time.time() - self.reclaim_age) datadir = os.path.join(self.root, node['device'], self.datadir) if os.path.isdir(datadir): dirs.append((datadir, node['id'])) self.logger.info(_('Beginning replication run')) for part, object_file, node_id in self.roundrobin_datadirs(dirs): self.cpool.spawn_n( self._replicate_object, part, object_file, node_id) self.cpool.waitall() self.logger.info(_('Replication run OVER')) self._report_stats()
def collect_jobs(self): """ Returns a sorted list of jobs (dictionaries) that specify the partitions, nodes, etc to be rsynced. """ jobs = [] ips = whataremyips() for local_dev in [dev for dev in self.object_ring.devs if dev and dev['ip'] in ips and dev['port'] == self.port]: dev_path = join(self.devices_dir, local_dev['device']) obj_path = join(dev_path, 'objects') tmp_path = join(dev_path, 'tmp') if self.mount_check and not os.path.ismount(dev_path): self.logger.warn(_('%s is not mounted'), local_dev['device']) continue unlink_older_than(tmp_path, time.time() - self.reclaim_age) if not os.path.exists(obj_path): continue for partition in os.listdir(obj_path): try: nodes = [node for node in self.object_ring.get_part_nodes(int(partition)) if node['id'] != local_dev['id']] jobs.append(dict(path=join(obj_path, partition), nodes=nodes, delete=len(nodes) > self.object_ring.replica_count - 1, partition=partition)) except ValueError: continue random.shuffle(jobs) # Partititons that need to be deleted take priority jobs.sort(key=lambda job: not job['delete']) self.job_count = len(jobs) return jobs
def run_once(self, *args, **kwargs): """Run a replication pass once.""" self._zero_stats() dirs = [] ips = whataremyips() if not ips: self.logger.error(_('ERROR Failed to get my own IPs?')) return for node in self.ring.devs: if node and node['ip'] in ips and node['port'] == self.port: if self.mount_check and not os.path.ismount( os.path.join(self.root, node['device'])): self.logger.warn( _('Skipping %(device)s as it is not mounted') % node) continue unlink_older_than( os.path.join(self.root, node['device'], 'tmp'), time.time() - self.reclaim_age) datadir = os.path.join(self.root, node['device'], self.datadir) if os.path.isdir(datadir): dirs.append((datadir, node['id'])) self.logger.info(_('Beginning replication run')) for part, object_file, node_id in self.roundrobin_datadirs(dirs): self.cpool.spawn_n(self._replicate_object, part, object_file, node_id) self.cpool.waitall() self.logger.info(_('Replication run OVER')) self._report_stats()
def __init__(self, conf, container_ring=None, object_ring=None): #: The dict of configuration values from the [container-sync] section #: of the container-server.conf. self.conf = conf #: Logger to use for container-sync log lines. self.logger = get_logger(conf, log_route='container-sync') #: Path to the local device mount points. self.devices = conf.get('devices', '/srv/node') #: Indicates whether mount points should be verified as actual mount #: points (normally true, false for tests and SAIO). self.mount_check = \ conf.get('mount_check', 'true').lower() in TRUE_VALUES #: Minimum time between full scans. This is to keep the daemon from #: running wild on near empty systems. self.interval = int(conf.get('interval', 300)) #: Maximum amount of time to spend syncing a container before moving on #: to the next one. If a conatiner sync hasn't finished in this time, #: it'll just be resumed next scan. self.container_time = int(conf.get('container_time', 60)) #: The list of hosts we're allowed to send syncs to. self.allowed_sync_hosts = [ h.strip() for h in conf.get('allowed_sync_hosts', '127.0.0.1').split(',') if h.strip() ] self.proxy = conf.get('sync_proxy') #: Number of containers with sync turned on that were successfully #: synced. self.container_syncs = 0 #: Number of successful DELETEs triggered. self.container_deletes = 0 #: Number of successful PUTs triggered. self.container_puts = 0 #: Number of containers that didn't have sync turned on. self.container_skips = 0 #: Number of containers that had a failure of some type. self.container_failures = 0 #: Time of last stats report. self.reported = time() chase_dir = conf.get('chase_dir', '/etc/chase') #: chase.common.ring.Ring for locating containers. self.container_ring = container_ring or \ Ring(os.path.join(chase_dir, 'container.ring.gz')) #: chase.common.ring.Ring for locating objects. self.object_ring = object_ring or \ Ring(os.path.join(chase_dir, 'object.ring.gz')) self._myips = whataremyips() self._myport = int(conf.get('bind_port', 6001))
def __init__(self, conf, container_ring=None, object_ring=None): #: The dict of configuration values from the [container-sync] section #: of the container-server.conf. self.conf = conf #: Logger to use for container-sync log lines. self.logger = get_logger(conf, log_route='container-sync') #: Path to the local device mount points. self.devices = conf.get('devices', '/srv/node') #: Indicates whether mount points should be verified as actual mount #: points (normally true, false for tests and SAIO). self.mount_check = \ conf.get('mount_check', 'true').lower() in TRUE_VALUES #: Minimum time between full scans. This is to keep the daemon from #: running wild on near empty systems. self.interval = int(conf.get('interval', 300)) #: Maximum amount of time to spend syncing a container before moving on #: to the next one. If a conatiner sync hasn't finished in this time, #: it'll just be resumed next scan. self.container_time = int(conf.get('container_time', 60)) #: The list of hosts we're allowed to send syncs to. self.allowed_sync_hosts = [h.strip() for h in conf.get('allowed_sync_hosts', '127.0.0.1').split(',') if h.strip()] self.proxy = conf.get('sync_proxy') #: Number of containers with sync turned on that were successfully #: synced. self.container_syncs = 0 #: Number of successful DELETEs triggered. self.container_deletes = 0 #: Number of successful PUTs triggered. self.container_puts = 0 #: Number of containers that didn't have sync turned on. self.container_skips = 0 #: Number of containers that had a failure of some type. self.container_failures = 0 #: Time of last stats report. self.reported = time() chase_dir = conf.get('chase_dir', '/etc/chase') #: chase.common.ring.Ring for locating containers. self.container_ring = container_ring or \ Ring(os.path.join(chase_dir, 'container.ring.gz')) #: chase.common.ring.Ring for locating objects. self.object_ring = object_ring or \ Ring(os.path.join(chase_dir, 'object.ring.gz')) self._myips = whataremyips() self._myport = int(conf.get('bind_port', 6001))
def __init__(self, conf): self.conf = conf self.logger = get_logger(conf, log_route='account-reaper') self.devices = conf.get('devices', '/srv/node') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') self.interval = int(conf.get('interval', 3600)) chase_dir = conf.get('chase_dir', '/etc/chase') self.account_ring_path = os.path.join(chase_dir, 'account.ring.gz') self.container_ring_path = os.path.join(chase_dir, 'container.ring.gz') self.object_ring_path = os.path.join(chase_dir, 'object.ring.gz') self.account_ring = None self.container_ring = None self.object_ring = None self.node_timeout = int(conf.get('node_timeout', 10)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.myips = whataremyips() self.concurrency = int(conf.get('concurrency', 25)) self.container_concurrency = self.object_concurrency = \ sqrt(self.concurrency) self.container_pool = GreenPool(size=self.container_concurrency)
def __init__(self, conf): self.conf = conf self.logger = get_logger(conf, log_route='account-reaper') self.devices = conf.get('devices', '/srv/node') self.mount_check = conf.get('mount_check', 'true').lower() in \ ('true', 't', '1', 'on', 'yes', 'y') self.interval = int(conf.get('interval', 3600)) chase_dir = conf.get('chase_dir', '/etc/chase') self.account_ring_path = os.path.join(chase_dir, 'account.ring.gz') self.container_ring_path = os.path.join(chase_dir, 'container.ring.gz') self.object_ring_path = os.path.join(chase_dir, 'object.ring.gz') self.account_ring = None self.container_ring = None self.object_ring = None self.node_timeout = int(conf.get('node_timeout', 10)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.myips = whataremyips() self.concurrency = int(conf.get('concurrency', 25)) self.container_concurrency = self.object_concurrency = \ sqrt(self.concurrency) self.container_pool = GreenPool(size=self.container_concurrency)
def collect_jobs(self): """ Returns a sorted list of jobs (dictionaries) that specify the partitions, nodes, etc to be rsynced. """ jobs = [] ips = whataremyips() for local_dev in [ dev for dev in self.object_ring.devs if dev and dev['ip'] in ips and dev['port'] == self.port ]: dev_path = join(self.devices_dir, local_dev['device']) obj_path = join(dev_path, 'objects') tmp_path = join(dev_path, 'tmp') if self.mount_check and not os.path.ismount(dev_path): self.logger.warn(_('%s is not mounted'), local_dev['device']) continue unlink_older_than(tmp_path, time.time() - self.reclaim_age) if not os.path.exists(obj_path): continue for partition in os.listdir(obj_path): try: nodes = [ node for node in self.object_ring.get_part_nodes( int(partition)) if node['id'] != local_dev['id'] ] jobs.append( dict(path=join(obj_path, partition), nodes=nodes, delete=len(nodes) > self.object_ring.replica_count - 1, partition=partition)) except ValueError: continue random.shuffle(jobs) # Partititons that need to be deleted take priority jobs.sort(key=lambda job: not job['delete']) self.job_count = len(jobs) return jobs
def test_whataremyips(self): myips = utils.whataremyips() self.assert_(len(myips) > 1) self.assert_('127.0.0.1' in myips)
def test_whataremyips(self): myips = utils.whataremyips() self.assert_(len(myips) > 1) self.assert_('127.0.0.1' in myips)