def processBadAssignedWorkers(self):
assigned_workers = yield GoogleUser.get_assigned_workers()
running_workers = yield Worker.getWorkers(Worker.redis_workers_keys)
running_workers = [
worker.get('name') for worker in running_workers
if not worker.get('name') is None
]
death_workers = yield Worker.getWorkers(
Worker.redis_death_workers_keys)
death_workers = [
worker.get('name') for worker in death_workers
if not worker.get('name') is None
]
registered_workers = set(running_workers + death_workers)
assigned_workers = set(assigned_workers)
bad_workers = assigned_workers.difference(registered_workers)
if bad_workers:
self.log.warning(
'BAD_WORKERS %s are assigned to users. Running %s Death %s',
bad_workers, len(running_workers), len(death_workers))
for worker in bad_workers:
bad_users = yield GoogleUser.get_connected(worker_name=worker)
total_bad_users = len(bad_users)
if total_bad_users > 0:
self.log.info(
'Reconnecting %s users assigned to bad worker %s',
total_bad_users, worker)
last_user_index = total_bad_users - 1
for i in xrange(total_bad_users):
try:
data = bad_users[i]
user = GoogleUser(**data)
user.worker = user.userid
yield user.save()
reactor.callLater(0, self.reloginUser, user, worker,
i == last_user_index)
self.log.info(
'[%s] Reconnecting %s/%s user(s) of worker %s',
user.userid, i + 1, total_bad_users, worker)
except Exception as e:
self.log.err(
e, '[%s] Exception while reconnecting' %
(data['_userid']))
#Remove worker and queue when no users were assigned
if total_bad_users == 0:
yield self.removeWorker(worker)
def reconnectUsers(self):
connected_users = yield GoogleUser.get_connected()
total_users = len(connected_users)
self.log.info('reconnectUsers reconnecting %s users', total_users)
for i in xrange(total_users):
data = connected_users[i]
try:
user = GoogleUser(**data)
worker, user.worker = user.worker, user.userid
yield user.save()
#Enqueing in the next loop iteration of twisted event loop
reactor.callLater(0, self.reloginUser, user, user.worker)
self.log.info('[%s] Reconnecting %s/%s user(s)', user.userid,
i + 1, total_users)
except Exception as e:
self.log.err(
e, '[%s] Exception while reconnecting' % (data['_userid']))
def processDeathWorkers(self):
#avoid process death workers when service is not running
death_workers = yield Worker.getWorkers(
Worker.redis_death_workers_keys) if self.running else []
if death_workers:
self.log.info('DEATH_WORKERS %s',
[worker.get('name') for worker in death_workers])
for worker in death_workers:
name = worker.get('name')
if conf.DIST_QUEUE_LOGIN in worker.get('queues', []):
connected_users = yield GoogleUser.get_connected(name)
total_users = len(connected_users)
self.log.info(
'Reconnecting %s connected user(s) of death worker %s',
total_users, name)
last_user_index = total_users - 1
for i in xrange(total_users):
try:
data = connected_users[i]
user = GoogleUser(**data)
#Update worker as userid to enqueue new jobs in user own queue
user.worker = user.userid
yield user.save()
#Get pending jobs
reactor.callLater(0, self.reloginUser, user, name,
i == last_user_index)
self.log.info(
'[%s] Reconnecting %s/%s user(s) of worker %s',
user.userid, i + 1, total_users, name)
except Exception as e:
self.log.err(
e, '[%s] Exception while reconnecting' %
(data['_userid']))
#Remove worker and queue when no users were assigned
if total_users == 0:
yield self.removeWorker(name)
else:
yield self.removeWorker(name)