def rebuilder_pass(self, **kwargs):
self.start_time = self.last_report = time.time()
self.log_report('START', force=True)
workers = list()
with ContextPool(self.nworkers + 1) as pool:
# spawn one worker for the retry queue
rqueue = eventlet.Queue(self.nworkers)
pool.spawn(self._read_retry_queue, rqueue, **kwargs)
# spawn workers to rebuild
queue = eventlet.Queue(self.nworkers * 10)
for i in range(self.nworkers):
worker = self._create_worker(**kwargs)
workers.append(worker)
pool.spawn(worker.rebuilder_pass,
i,
queue,
retry_queue=rqueue,
**kwargs)
# fill the queue (with the main thread)
self._fill_queue(queue, **kwargs)
# block until all items are rebuilt
queue.join()
# block until the retry queue is empty
rqueue.join()
self.log_report('DONE', force=True)
return self.total_errors == 0
def __init__(self, conf, tool):
super(_LocalDispatcher, self).__init__(conf, tool)
nb_workers = int_value(self.conf.get('workers'),
self.tool.DEFAULT_WORKERS)
self.max_items_per_second = int_value(
self.conf.get('items_per_second'),
self.tool.DEFAULT_ITEM_PER_SECOND)
self.queue_workers = eventlet.Queue(nb_workers * 2)
self.queue_reply = eventlet.Queue()
self.workers = list()
for _ in range(nb_workers):
worker = self.tool.create_worker(self.queue_workers,
self.queue_reply)
self.workers.append(worker)
def __init__(self, conf, accounts=None, **kwargs):
super(AccountRebuilder, self).__init__(conf, **kwargs)
self._accounts_to_refresh = set()
self._accounts_refreshed = eventlet.Queue()
# input
self.accounts = accounts
self.account_client = AccountClient(self.conf, logger=self.logger)
def __init__(self, conf, tool):
super(_LocalDispatcher, self).__init__(conf, tool)
concurrency = int_value(self.conf.get(
'concurrency'), self.tool.DEFAULT_CONCURRENCY)
self.max_items_per_second = int_value(self.conf.get(
'items_per_second'), self.tool.DEFAULT_ITEM_PER_SECOND)
if self.max_items_per_second > 0:
# Max 5 seconds in advance
queue_size = self.max_items_per_second * 5
else:
queue_size = concurrency * 1024
self.queue_workers = eventlet.Queue(queue_size)
self.queue_reply = eventlet.Queue()
self.workers = list()
for _ in range(concurrency):
worker = self.tool.create_worker(
self.queue_workers, self.queue_reply)
self.workers.append(worker)
def rebuilder_pass(self, **kwargs):
self.start_time = self.last_report = time.time()
self.log_report('START', force=True)
workers = list()
with ContextPool(self.concurrency + 1) as pool:
# spawn one worker for the retry queue
rqueue = eventlet.Queue(self.concurrency)
pool.spawn(self._read_retry_queue, rqueue, **kwargs)
# spawn workers to rebuild
queue = eventlet.Queue(self.concurrency * 10)
for i in range(self.concurrency):
worker = self._create_worker(**kwargs)
workers.append(worker)
pool.spawn(worker.rebuilder_pass,
i,
queue,
retry_queue=rqueue,
**kwargs)
# fill the queue (with the main thread)
try:
self._fill_queue(queue, **kwargs)
except Exception as exc:
if self.running:
self.logger.error("Failed to fill queue: %s", exc)
self.success = False
# block until all items are rebuilt
queue.join()
# block until the retry queue is empty
rqueue.join()
self.log_report('DONE', force=True)
return self.success and self.total_errors == 0
def __init__(self, conf, beanstalkd_addr=None, logger=None):
self.conf = conf
self.logger = logger or get_logger(self.conf)
self.namespace = conf['namespace']
self.success = True
# exit gracefully
self.running = True
signal.signal(signal.SIGINT, self.exit_gracefully)
signal.signal(signal.SIGTERM, self.exit_gracefully)
# counters
self.items_processed = 0
self.total_items_processed = 0
self.errors = 0
self.total_errors = 0
self.total_expected_items = None
# report
self.start_time = 0
self.last_report = 0
self.report_interval = int_value(self.conf.get(
'report_interval'), self.DEFAULT_REPORT_INTERVAL)
# dispatcher
self.dispatcher = None
# input
self.beanstalkd = None
if beanstalkd_addr:
self.beanstalkd = BeanstalkdListener(
beanstalkd_addr,
self.conf.get('beanstalkd_worker_tube')
or self.DEFAULT_BEANSTALKD_WORKER_TUBE,
self.logger)
# retry
self.retryer = None
self.retry_queue = None
if self.beanstalkd:
self.retryer = BeanstalkdSender(
self.beanstalkd.addr, self.beanstalkd.tube, self.logger)
self.retry_queue = eventlet.Queue()
self.retry_delay = int_value(self.conf.get('retry_delay'),
self.DEFAULT_RETRY_DELAY)
def rebuilder_pass(self, **kwargs):
start_time = time.time()
workers = list()
with ContextPool(self.nworkers) as pool:
queue = eventlet.Queue(self.nworkers * 10)
# spawn workers to rebuild
for i in range(self.nworkers):
worker = self._create_worker(**kwargs)
workers.append(worker)
pool.spawn(worker.rebuilder_pass, i, queue)
# fill the queue
self._fill_queue(queue, **kwargs)
# block until all items are rebuilt
queue.join()
passes = 0
errors = 0
total_items_processed = 0
waiting_time = 0
rebuilder_time = 0
info = self._init_info(**kwargs)
for worker in workers:
passes += worker.passes
errors += worker.errors
total_items_processed += worker.total_items_processed
waiting_time += worker.waiting_time
rebuilder_time += worker.rebuilder_time
info = self._compute_info(worker, info, **kwargs)
end_time = time.time()
elapsed = (end_time - start_time) or 0.000001
self.logger.info(
self._get_report(start_time, end_time, passes, errors,
waiting_time, rebuilder_time, elapsed,
total_items_processed, info, **kwargs))
