def removeWorker(self, name):
self.log.info("Removing worker/queue %s from system", name)
#Remove worker from death workers
worker = Worker(queues=[], name=name)
yield worker.remove(worker.key)
#Remove own queue of worker
queue = Queue(name)
yield queue.empty()
def processBadAssignedWorkers(self):
assigned_workers = yield GoogleUser.get_assigned_workers()
running_workers = yield Worker.getWorkers(Worker.redis_workers_keys)
running_workers = [
worker.get('name') for worker in running_workers
if not worker.get('name') is None
]
death_workers = yield Worker.getWorkers(
Worker.redis_death_workers_keys)
death_workers = [
worker.get('name') for worker in death_workers
if not worker.get('name') is None
]
registered_workers = set(running_workers + death_workers)
assigned_workers = set(assigned_workers)
bad_workers = assigned_workers.difference(registered_workers)
if bad_workers:
self.log.warning(
'BAD_WORKERS %s are assigned to users. Running %s Death %s',
bad_workers, len(running_workers), len(death_workers))
for worker in bad_workers:
bad_users = yield GoogleUser.get_connected(worker_name=worker)
total_bad_users = len(bad_users)
if total_bad_users > 0:
self.log.info(
'Reconnecting %s users assigned to bad worker %s',
total_bad_users, worker)
last_user_index = total_bad_users - 1
for i in xrange(total_bad_users):
try:
data = bad_users[i]
user = GoogleUser(**data)
user.worker = user.userid
yield user.save()
reactor.callLater(0, self.reloginUser, user, worker,
i == last_user_index)
self.log.info(
'[%s] Reconnecting %s/%s user(s) of worker %s',
user.userid, i + 1, total_bad_users, worker)
except Exception as e:
self.log.err(
e, '[%s] Exception while reconnecting' %
(data['_userid']))
#Remove worker and queue when no users were assigned
if total_bad_users == 0:
yield self.removeWorker(worker)
def checkRunningWorkers(self):
workers = yield Worker.getWorkers(Worker.redis_workers_keys)
if workers:
self.log.info('CHECKING_RUNNING_WORKERS %s', len(workers))
for worker in workers:
name = worker.get('name')
key = worker.get('key')
lastTime = worker.get('lastTime')
if key is None or name is None or lastTime is None:
self.log.warning('WORKER_DATA_WRONG %s', worker)
continue
death = worker.get('death')
if death is None:
lastTime = parser.parse(lastTime)
delta = datetime.utcnow() - lastTime
if delta.seconds > conf.SUPERVISOR_WORKER_REFRESH_TIME:
self.log.warning(
'REGISTERING_WORKER_DEATH %s has not been updated since %s second(s)',
name, delta.seconds)
w = Worker([], name=name)
w.log = self.log
yield w.register_death()
def processDeathWorkers(self):
#avoid process death workers when service is not running
death_workers = yield Worker.getWorkers(
Worker.redis_death_workers_keys) if self.running else []
if death_workers:
self.log.info('DEATH_WORKERS %s',
[worker.get('name') for worker in death_workers])
for worker in death_workers:
name = worker.get('name')
if conf.DIST_QUEUE_LOGIN in worker.get('queues', []):
connected_users = yield GoogleUser.get_connected(name)
total_users = len(connected_users)
self.log.info(
'Reconnecting %s connected user(s) of death worker %s',
total_users, name)
last_user_index = total_users - 1
for i in xrange(total_users):
try:
data = connected_users[i]
user = GoogleUser(**data)
#Update worker as userid to enqueue new jobs in user own queue
user.worker = user.userid
yield user.save()
#Get pending jobs
reactor.callLater(0, self.reloginUser, user, name,
i == last_user_index)
self.log.info(
'[%s] Reconnecting %s/%s user(s) of worker %s',
user.userid, i + 1, total_users, name)
except Exception as e:
self.log.err(
e, '[%s] Exception while reconnecting' %
(data['_userid']))
#Remove worker and queue when no users were assigned
if total_users == 0:
yield self.removeWorker(name)
else:
yield self.removeWorker(name)
locale_path="./locale",
login_url="/auth/login",
cookie_secret="32oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=",
base_handler=BaseHandler,
db_handlers=cyclone.util.ObjectDict(
#sqlite=cyclone.sqlite.InlineSQLite(":memory:"),
redis=cyclone.redis.lazyConnectionPool(), ),
more_handlers=[
(r"/websocket", WebSocketHandler),
(r"/xmlrpc", XmlrpcHandler),
])
application = service.Application("Application")
webservice = internet.TCPServer(int(os.getenv('PORT', 8888)),
bottle_app,
interface=os.getenv('LISTTEN', "0.0.0.0"))
webservice.setServiceParent(application)
from katoo.utils.connections import RedisMixin
from katoo.rqtwisted.worker import Worker
RedisMixin.setup()
blocking_seconds = int(os.getenv('DEQUEUE_BLOCKTIME', 1))
workers = int(os.getenv('WORKERS', 1))
for i in xrange(workers):
t = Worker(['default'])
t.setServiceParent(application)
ret = yield API(user.userid, conf.MACHINEID).logout(user.userid)
except XMPPUserNotLogged as e:
my_log.error('Exception launch %s', e)
try:
ret = yield API(user.userid).logout(user.userid)
except XMPPUserNotLogged as e:
my_log.error('Exception launch %s', e)
#Perform a timeout
ret = yield API(user.userid, 'default').login(user)
# reactor.callLater(5, API(user.userid).login, user)
# reactor.callLater(7, API(user.userid).update, user.userid, token="ya29.AHES6ZRDTu4pDWdA_LBrNWF1vnI5NEtdB8V0v6JN46QTaw")
# reactor.callLater(10, API(user.userid).logout, user.userid)
# reactor.callLater(20, API(user.userid).login, user)
#log.startLogging(sys.stdout)
app = KatooApp().app
w = Worker([conf.MACHINEID, conf.DIST_QUEUE_LOGIN],
name=conf.MACHINEID,
loops=conf.REDIS_WORKERS)
w.log = getLoggerAdapter(getLogger('WORKER', level='INFO'), id='WORKER')
w.setServiceParent(app)
KatooApp().service.startService()
reactor.callLater(5, example)
import twisted.python.log
twisted.python.log.startLoggingWithObserver(KatooApp().log.emit)
reactor.callLater(50, reactor.stop)
reactor.run()