def __init__(self, conf, logger=None):
"""
:param conf: configuration object obtained from ConfigParser
:param logger: logging object
"""
self.conf = conf
self.logger = PrefixLoggerAdapter(
logger or get_logger(conf, log_route='object-replicator'), {})
self.devices_dir = conf.get('devices', '/srv/node')
self.mount_check = config_true_value(conf.get('mount_check', 'true'))
self.swift_dir = conf.get('swift_dir', '/etc/swift')
self.bind_ip = conf.get('bind_ip', '0.0.0.0')
self.servers_per_port = int(conf.get('servers_per_port', '0') or 0)
self.port = None if self.servers_per_port else \
int(conf.get('bind_port', 6200))
self.concurrency = int(conf.get('concurrency', 1))
self.replicator_workers = int(conf.get('replicator_workers', 0))
self.stats_interval = int(conf.get('stats_interval', '300'))
self.ring_check_interval = int(conf.get('ring_check_interval', 15))
self.next_check = time.time() + self.ring_check_interval
self.replication_cycle = random.randint(0, 9)
self.partition_times = []
self.interval = int(
conf.get('interval') or conf.get('run_pause') or 30)
self.rsync_timeout = int(
conf.get('rsync_timeout', DEFAULT_RSYNC_TIMEOUT))
self.rsync_io_timeout = conf.get('rsync_io_timeout', '30')
self.rsync_bwlimit = conf.get('rsync_bwlimit', '0')
self.rsync_compress = config_true_value(
conf.get('rsync_compress', 'no'))
self.rsync_module = conf.get('rsync_module', '').rstrip('/')
if not self.rsync_module:
self.rsync_module = '{replication_ip}::object'
self.http_timeout = int(conf.get('http_timeout', 60))
self.recon_cache_path = conf.get('recon_cache_path',
'/var/cache/swift')
self.rcache = os.path.join(self.recon_cache_path, "object.recon")
self._next_rcache_update = time.time() + self.stats_interval
self.conn_timeout = float(conf.get('conn_timeout', 0.5))
self.node_timeout = float(conf.get('node_timeout', 10))
self.sync_method = getattr(self, conf.get('sync_method') or 'rsync')
self.network_chunk_size = int(conf.get('network_chunk_size', 65536))
self.default_headers = {
'Content-Length': '0',
'user-agent': 'object-replicator %s' % os.getpid()
}
self.rsync_error_log_line_length = \
int(conf.get('rsync_error_log_line_length', 0))
self.handoffs_first = config_true_value(
conf.get('handoffs_first', False))
self.handoff_delete = config_auto_int_value(
conf.get('handoff_delete', 'auto'), 0)
if any((self.handoff_delete, self.handoffs_first)):
self.logger.warning('Handoff only mode is not intended for normal '
'operation, please disable handoffs_first and '
'handoff_delete before the next '
'normal rebalance')
self.is_multiprocess_worker = None
self._df_router = DiskFileRouter(conf, self.logger)
self._child_process_reaper_queue = queue.LightQueue()
def __init__(self, watcher_class, watcher_name, conf, logger):
self.watcher_name = watcher_name
self.watcher_in_error = False
self.logger = PrefixLoggerAdapter(logger, {})
self.logger.set_prefix('[audit-watcher %s] ' % watcher_name)
try:
self.watcher = watcher_class(conf, self.logger)
except (Exception, Timeout):
self.logger.exception('Error intializing watcher')
self.watcher_in_error = True
def parallel_process(do_cleanup, conf, logger=None, device_list=None):
logger = logger or logging.getLogger()
# initialise recon dump for collection
# Lets start by always deleting last run's stats
recon_cache = os.path.join(conf['recon_cache_path'], RECON_RELINKER_FILE)
_reset_recon(recon_cache, logger)
device_list = sorted(set(device_list or os.listdir(conf['devices'])))
workers = conf['workers']
if workers == 'auto':
workers = len(device_list)
else:
workers = min(workers, len(device_list))
start = time.time()
logger.info(
'Starting relinker (cleanup=%s) using %d workers: %s' %
(do_cleanup, workers, time.strftime('%X %x %Z', time.gmtime(start))))
if workers == 0 or len(device_list) in (0, 1):
ret = Relinker(conf, logger, device_list, do_cleanup=do_cleanup).run()
logger.info('Finished relinker (cleanup=%s): %s (%s elapsed)' %
(do_cleanup, time.strftime('%X %x %Z', time.gmtime()),
datetime.timedelta(seconds=time.time() - start)))
return ret
children = {}
for worker_devs in distribute_evenly(device_list, workers):
pid = os.fork()
if pid == 0:
dev_logger = PrefixLoggerAdapter(logger, {})
dev_logger.set_prefix('[pid=%s, devs=%s] ' %
(os.getpid(), ','.join(worker_devs)))
os._exit(
Relinker(conf, dev_logger, worker_devs,
do_cleanup=do_cleanup).run())
else:
children[pid] = worker_devs
final_status = EXIT_SUCCESS
final_messages = []
while children:
pid, status = os.wait()
sig = status & 0xff
status = status >> 8
time_delta = time.time() - start
devs = children.pop(pid, ['unknown device'])
worker_desc = '(pid=%s, devs=%s)' % (pid, ','.join(devs))
if sig != 0:
final_status = EXIT_ERROR
final_messages.append(
'Worker %s exited in %.1fs after receiving signal: %s' %
(worker_desc, time_delta, sig))
continue
if status == EXIT_SUCCESS:
continue
if status == EXIT_NO_APPLICABLE_POLICY:
if final_status == EXIT_SUCCESS:
final_status = status
continue
final_status = EXIT_ERROR
if status == EXIT_ERROR:
final_messages.append('Worker %s completed in %.1fs with errors' %
(worker_desc, time_delta))
else:
final_messages.append(
'Worker %s exited in %.1fs with unexpected status %s' %
(worker_desc, time_delta, status))
for msg in final_messages:
logger.warning(msg)
logger.info('Finished relinker (cleanup=%s): %s (%s elapsed)' %
(do_cleanup, time.strftime('%X %x %Z', time.gmtime()),
datetime.timedelta(seconds=time.time() - start)))
return final_status
def parallel_process(do_cleanup, conf, logger=None, device_list=None):
logger = logger or logging.getLogger()
device_list = sorted(set(device_list or os.listdir(conf['devices'])))
workers = conf['workers']
if workers == 'auto':
workers = len(device_list)
else:
workers = min(workers, len(device_list))
if workers == 0 or len(device_list) in (0, 1):
return Relinker(
conf, logger, device_list, do_cleanup=do_cleanup).run()
start = time.time()
children = {}
for worker_devs in distribute_evenly(device_list, workers):
pid = os.fork()
if pid == 0:
dev_logger = PrefixLoggerAdapter(logger, {})
dev_logger.set_prefix('[pid=%s, devs=%s] ' % (
os.getpid(), ','.join(worker_devs)))
os._exit(Relinker(
conf, dev_logger, worker_devs, do_cleanup=do_cleanup).run())
else:
children[pid] = worker_devs
final_status = EXIT_SUCCESS
final_messages = []
while children:
pid, status = os.wait()
sig = status & 0xff
status = status >> 8
time_delta = time.time() - start
devs = children.pop(pid, ['unknown device'])
worker_desc = '(pid=%s, devs=%s)' % (pid, ','.join(devs))
if sig != 0:
final_status = EXIT_ERROR
final_messages.append(
'Worker %s exited in %.1fs after receiving signal: %s'
% (worker_desc, time_delta, sig))
continue
if status == EXIT_SUCCESS:
continue
if status == EXIT_NO_APPLICABLE_POLICY:
if final_status == EXIT_SUCCESS:
final_status = status
continue
final_status = EXIT_ERROR
if status == EXIT_ERROR:
final_messages.append(
'Worker %s completed in %.1fs with errors'
% (worker_desc, time_delta))
else:
final_messages.append(
'Worker %s exited in %.1fs with unexpected status %s'
% (worker_desc, time_delta, status))
for msg in final_messages:
logger.warning(msg)
return final_status