def __init__(self, jid, password, host=None, port=5222, logid=None): XMPPClient.__init__(self, jid, password, host=host, port=port) self.log = getLoggerAdapter( log) if logid is None else getLoggerAdapter(log, id=logid) self.factory.addBootstrap(xmlstream.STREAM_ERROR_EVENT, self._onStreamError) self._authFailureTime = None self._lastTimeAuth = None self._connectedTime = None
class XMPPKeepAliveSupervisor(Supervisor, Subject): name = 'XMPP_KEEPALIVE_SUPERVISOR' log = getLoggerAdapter(log, id=name) def __init__(self): Supervisor.__init__(self) Subject.__init__(self) self.lastTime = datetime.utcnow() self.metric = Metric(name='elapsed_time', value=None, unit='msec', source=self.name, scale=1000, sampling=True) def perform_keep_alive(self): self.lastTime = datetime.utcnow() self.notifyObservers() elapsedTime = datetime.utcnow() - self.lastTime usecs = elapsedTime.seconds * 1000000 + elapsedTime.microseconds self.log.info('Finished XMPP_KEEP_ALIVE. Elapsed %s usecs', usecs) self.metric.add(usecs) def startService(self): Supervisor.startService(self) if conf.XMPP_KEEP_ALIVE_TIME > 0: t = LoopingCall(self.perform_keep_alive) self.registerTask(t) t.start(conf.XMPP_KEEP_ALIVE_TIME, now=False)
def new_onRosterSet(f): log = getLoggerAdapter(getLogger('wokkel_onRosterSet', level='INFO'), id='TWISTED') @wraps(f) def wrapper(self, iq): def trapIgnored(failure): failure.trap(RosterPushIgnored) raise error.StanzaError('service-unavailable') request = RosterRequest.fromElement(iq) if (not self.allowAnySender and request.sender and request.sender.userhostJID() != self.parent.jid.userhostJID()): d = defer.fail(RosterPushIgnored()) elif request.item is None: log.info('_onRosterSet iq malformed. %s', iq.toXml()) d = defer.fail(RosterPushIgnored()) elif request.item.remove: d = defer.maybeDeferred(self.removeReceived, request) else: d = defer.maybeDeferred(self.setReceived, request) d.addErrback(trapIgnored) return d return wrapper
def __init__(self, application, request, **kwargs): cyclone.web.RequestHandler.__init__(self, application, request, **kwargs) self.log = getLoggerAdapter(log) self.key = '' self.args = '' self.response = ''
def __init__(self, collectionName, mongourl=None, indexes=None): self.setup(url=mongourl) self.db = self.mongo_db self.collection = collectionName self.pool = self.mongo_conn self.indexes = indexes logger = getLoggerAdapter(log, id="DATA-%s" % (collectionName.upper())) Model.__init__(self, logging=logger)
def constructor(self, key, args_class=None, metric=None): self.log = getLoggerAdapter(log, id=key) self.metric = metric self.key = key self.args = '' if args_class is None else args_class(self).args self.user_agent = CheckUserAgent( self.request.headers.get('User-Agent', '')) if not bool(self.user_agent): raise cyclone.web.HTTPError(403)
def setup(cls, url=None, log=None): if cls.redis_conn is None: if url is None: url = conf.REDIS_URL hostname, port, db, _, password = url_parse(url, 'redis') cls.redis_db = int(db) AuthRedisProtocol.password = password if log is None: log = getLoggerAdapter(getLogger(__name__), id='REDIS_CONNECTIONPOOL') cls.log = AuthRedisProtocol.log = log RedisFactory.protocol = AuthRedisProtocol cls.redis_conn = redis.lazyConnectionPool(host=hostname, port=port, dbid=cls.redis_db, poolsize=conf.REDIS_POOL, reconnect=True)
class HerokuUnidlingSupervisor(Supervisor): name = 'HEROKU_UNIDLING_SUPERVISOR' log = getLoggerAdapter(log, id=name) @defer.inlineCallbacks def avoidHerokuUnidling(self, url): try: self.log.info('AVOIDING HEROKU IDLING: %s', url) yield cyclone.httpclient.fetch(url) except Exception as e: self.log.err(e, 'Exception in avoidHerokuUnidling') def startService(self): Supervisor.startService(self) if not conf.HEROKU_UNIDLING_URL is None: t = LoopingCall(self.avoidHerokuUnidling, conf.HEROKU_UNIDLING_URL) self.registerTask(t) t.start(1800, now=True)
class MetricsSupervisor(Supervisor, Subject): name = 'METRICS_SUPERVISOR' log = getLoggerAdapter(log, id=name) def __init__(self): Supervisor.__init__(self) Subject.__init__(self) def report(self): self.notifyObservers() MetricsHub().report() def startService(self): Supervisor.startService(self) t = LoopingCall(self.report) self.registerTask(t) t.start(conf.METRICS_REPORT_TIME, now=False) def stopService(self): Supervisor.stopService(self)
def __init__(self): handlers = [ (r"/1/google/messages/(.+)", v1.handlers.GoogleMessagesHandler), (r"/1/google/contacts/(.+)", v1.handlers.GoogleContactsHandler), (r"/1/google/(.+)", v1.handlers.GoogleHandler) ] settings = dict(debug=conf.CYCLONE_DEBUG, ) cyclone.web.Application.__init__(self, handlers, **settings) self.log = getLoggerAdapter(log) self.metric = Metric(name='time', value=None, unit=v1.handlers.METRIC_UNIT_TIME, source=v1.handlers.METRIC_SOURCE, sampling=True) self.metric_response_codes = { 1: Metric(name='response_1XX', value=None, unit='requests', source=v1.handlers.METRIC_SOURCE), 2: Metric(name='response_2XX', value=None, unit='requests', source=v1.handlers.METRIC_SOURCE), 3: Metric(name='response_3XX', value=None, unit='requests', source=v1.handlers.METRIC_SOURCE), 4: Metric(name='response_4XX', value=None, unit='requests', source=v1.handlers.METRIC_SOURCE), 5: Metric(name='response_5XX', value=None, unit='requests', source=v1.handlers.METRIC_SOURCE) }
def setup(cls, url=None, log=None): if cls.mongo_conn is None: if url is None: url = conf.MONGO_URL hostname, port, cls.mongo_db, username, password = url_parse( url, 'mongodb') AuthMongoProtocol.database = cls.mongo_db AuthMongoProtocol.username = username AuthMongoProtocol.password = password if log is None: log = getLoggerAdapter(getLogger(__name__), id='MONGO_CONNECTIONPOOL') cls.log = AuthMongoProtocol.log = log txmongo._MongoFactory.protocol = AuthMongoProtocol txmongo._MongoFactory.maxRetries = conf.BACKEND_MAX_RETRIES txmongo._MongoFactory.maxDelay = conf.BACKEND_MAX_DELAY cls.mongo_conn = txmongo.lazyMongoConnectionPool( host=hostname, port=port, reconnect=True, pool_size=conf.MONGO_POOL)
class AsynchronousCall(SynchronousCall): def __init__(self, queue): SynchronousCall.__init__(self, queue) self.sync = conf.DIST_ASYNC_AS_SYNC self.result_ttl = conf.DIST_DEFAULT_TTL if conf.DIST_ASYNC_AS_SYNC else 0 if __name__ == '__main__': from katoo import KatooApp from katoo.api import API from twisted.internet import reactor from katoo.rqtwisted.worker import Worker from katoo.data import GoogleUser from pickle import dumps my_log = getLoggerAdapter(getLogger(__name__)) @defer.inlineCallbacks def example(): job = Job(job_id="123") pickled_rv = dumps({'results': None}) yield job.save() yield job.connection.hset(job.key, 'result', pickled_rv) res = yield job.result my_log.debug('Job result %s', res) user = GoogleUser(_userid="1", _token="asdasdf", _refreshtoken="refreshtoken", _resource="unknownresource", _pushtoken="",
@AsynchronousCall(conf.DIST_QUEUE_PUSH) def sendpush(self, message, token, badgenumber, sound='', **kwargs): self.log.debug( 'SEND_PUSH: %s token: %s, badgenumber: %s, sound: %s kwargs: %s', message, token, badgenumber, sound, kwargs) return self._sendapn(token, message, sound, badgenumber, **kwargs) if __name__ == '__main__': from twisted.internet import reactor, defer, task from katoo import KatooApp from katoo.utils.applog import getLoggerAdapter, getLogger from katoo.rqtwisted import worker import os, translate my_log = getLoggerAdapter(getLogger(__name__, level="DEBUG"), id='MYLOG') #@defer.inlineCallbacks def send(): my_log.debug('Starting send') API('TEST').sendpush( message=translate.TRANSLATORS['en']._('disconnected'), token=os.getenv('PUSHTOKEN', None), badgenumber=0, sound='') my_log.debug('Finished send') @defer.inlineCallbacks def close(): try: my_log.debug('Starting close')
class GlobalSupervisor(Supervisor): name = 'GLOBAL_SUPERVISOR' log = getLoggerAdapter(log, id=name) DISCONNECT_AWAY_METRIC = Metric(name='away_user_disconnected', value=None, unit='events', source='XMPPGOOGLE') def __init__(self): Supervisor.__init__(self) self.lock = defer.DeferredLock() self._checkworkerstasks = [ self.processDeathWorkers, self.processBadAssignedWorkers, self.processOnMigrationUsers, self.checkRunningWorkers ] self._globalmetrics = [RedisMetrics, MongoMetrics] def _attach_global_metrics(self): service = KatooApp().getService(MetricsSupervisor.name) for metric in self._globalmetrics: service.registerObserver(metric()) @defer.inlineCallbacks def checkRunningWorkers(self): workers = yield Worker.getWorkers(Worker.redis_workers_keys) if workers: self.log.info('CHECKING_RUNNING_WORKERS %s', len(workers)) for worker in workers: name = worker.get('name') key = worker.get('key') lastTime = worker.get('lastTime') if key is None or name is None or lastTime is None: self.log.warning('WORKER_DATA_WRONG %s', worker) continue death = worker.get('death') if death is None: lastTime = parser.parse(lastTime) delta = datetime.utcnow() - lastTime if delta.seconds > conf.SUPERVISOR_WORKER_REFRESH_TIME: self.log.warning( 'REGISTERING_WORKER_DEATH %s has not been updated since %s second(s)', name, delta.seconds) w = Worker([], name=name) w.log = self.log yield w.register_death() @defer.inlineCallbacks def processOnMigrationUsers(self): onMigration_users = yield GoogleUser.get_onMigration() total_users = len(onMigration_users) if total_users > 0: self.log.info("ON_MIGRATION_USERS %s", total_users) now = datetime.utcnow() for data in onMigration_users: user = GoogleUser(**data) delta_time = now - user.onMigrationTime if delta_time.seconds < conf.XMPP_KEEP_ALIVE_TIME + 30: if 60 < delta_time.seconds < 70: self.log.warning( '[%s] USER_MIGRATION_STOPPED %s second(s) ago. Waiting to fix the problem with XMPP_KEEP_ALIVE. User state: %s', user.userid, delta_time.seconds, user) continue self.log.warning( '[%s] USER_MIGRATION_STOPPED %s second(s) ago. Performing new relogin. User state: %s', user.userid, delta_time.seconds, user) user.worker = user.userid user.onMigrationTime = '' yield user.save() yield API(user.userid).relogin(user, pending_jobs=[]) @defer.inlineCallbacks def getPendingJobs(self, userid, queue_name): queue = Queue(queue_name) job_ids = yield queue.job_ids jobs = [] index = 0 for job_id in job_ids: try: job = yield Job.fetch(job_id, connection=queue.connection) if job.meta.get('userid') == userid: jobs.append(job_id) except Exception as e: self.log.err( e, '[%s] Exception fetching job %s with index %s while getPendingJobs in queue %s' % (userid, job_id, index, queue_name)) yield queue.remove(job_id) finally: index += 1 defer.returnValue(jobs) @defer.inlineCallbacks def processDeathWorkers(self): #avoid process death workers when service is not running death_workers = yield Worker.getWorkers( Worker.redis_death_workers_keys) if self.running else [] if death_workers: self.log.info('DEATH_WORKERS %s', [worker.get('name') for worker in death_workers]) for worker in death_workers: name = worker.get('name') if conf.DIST_QUEUE_LOGIN in worker.get('queues', []): connected_users = yield GoogleUser.get_connected(name) total_users = len(connected_users) self.log.info( 'Reconnecting %s connected user(s) of death worker %s', total_users, name) last_user_index = total_users - 1 for i in xrange(total_users): try: data = connected_users[i] user = GoogleUser(**data) #Update worker as userid to enqueue new jobs in user own queue user.worker = user.userid yield user.save() #Get pending jobs reactor.callLater(0, self.reloginUser, user, name, i == last_user_index) self.log.info( '[%s] Reconnecting %s/%s user(s) of worker %s', user.userid, i + 1, total_users, name) except Exception as e: self.log.err( e, '[%s] Exception while reconnecting' % (data['_userid'])) #Remove worker and queue when no users were assigned if total_users == 0: yield self.removeWorker(name) else: yield self.removeWorker(name) @defer.inlineCallbacks def processBadAssignedWorkers(self): assigned_workers = yield GoogleUser.get_assigned_workers() running_workers = yield Worker.getWorkers(Worker.redis_workers_keys) running_workers = [ worker.get('name') for worker in running_workers if not worker.get('name') is None ] death_workers = yield Worker.getWorkers( Worker.redis_death_workers_keys) death_workers = [ worker.get('name') for worker in death_workers if not worker.get('name') is None ] registered_workers = set(running_workers + death_workers) assigned_workers = set(assigned_workers) bad_workers = assigned_workers.difference(registered_workers) if bad_workers: self.log.warning( 'BAD_WORKERS %s are assigned to users. Running %s Death %s', bad_workers, len(running_workers), len(death_workers)) for worker in bad_workers: bad_users = yield GoogleUser.get_connected(worker_name=worker) total_bad_users = len(bad_users) if total_bad_users > 0: self.log.info( 'Reconnecting %s users assigned to bad worker %s', total_bad_users, worker) last_user_index = total_bad_users - 1 for i in xrange(total_bad_users): try: data = bad_users[i] user = GoogleUser(**data) user.worker = user.userid yield user.save() reactor.callLater(0, self.reloginUser, user, worker, i == last_user_index) self.log.info( '[%s] Reconnecting %s/%s user(s) of worker %s', user.userid, i + 1, total_bad_users, worker) except Exception as e: self.log.err( e, '[%s] Exception while reconnecting' % (data['_userid'])) #Remove worker and queue when no users were assigned if total_bad_users == 0: yield self.removeWorker(worker) @defer.inlineCallbacks def checkWorkers(self): try: for task in self._checkworkerstasks: if self.running: yield task() else: self.log.info( 'CheckWorkers task %s not launched. Supervisor not running', task) except Exception as e: self.log.err(e, 'Exception in checkWorkers task %s' % (task)) @defer.inlineCallbacks def disconnectAwayUsers(self): away_users = yield GoogleUser.get_away() away_users = [] if not away_users else away_users self.log.info('CHECKING_AWAY_USERS: %s', len(away_users)) for data in away_users: try: user = GoogleUser(**data) API(user.userid, queue=user.worker).disconnect(user.userid) APNSAPI(user.userid).sendpush(message=u'{0} {1}'.format( u'\ue252', translate.TRANSLATORS[user.lang]._('disconnected')), token=user.pushtoken, badgenumber=user.badgenumber, sound='') self.DISCONNECT_AWAY_METRIC.add(1) except Exception as e: self.log.err( e, '[%s] Exception disconnecting user' % (data['_userid'])) @defer.inlineCallbacks def reconnectUsers(self): connected_users = yield GoogleUser.get_connected() total_users = len(connected_users) self.log.info('reconnectUsers reconnecting %s users', total_users) for i in xrange(total_users): data = connected_users[i] try: user = GoogleUser(**data) worker, user.worker = user.worker, user.userid yield user.save() #Enqueing in the next loop iteration of twisted event loop reactor.callLater(0, self.reloginUser, user, user.worker) self.log.info('[%s] Reconnecting %s/%s user(s)', user.userid, i + 1, total_users) except Exception as e: self.log.err( e, '[%s] Exception while reconnecting' % (data['_userid'])) @defer.inlineCallbacks def reloginUser(self, user, last_worker, removeWorker=False): try: pending_jobs = yield self.getPendingJobs(user.userid, last_worker) yield API(user.userid).relogin(user, pending_jobs) except Exception as e: self.log.err(e, '[%s] Exception while reconnecting' % (user.userid)) finally: if removeWorker: yield self.removeWorker(last_worker) @defer.inlineCallbacks def removeWorker(self, name): self.log.info("Removing worker/queue %s from system", name) #Remove worker from death workers worker = Worker(queues=[], name=name) yield worker.remove(worker.key) #Remove own queue of worker queue = Queue(name) yield queue.empty() def startService(self): Supervisor.startService(self) t = LoopingCall(self.disconnectAwayUsers) self.registerTask(t) t.start(conf.TASK_DISCONNECT_SECONDS, now=False) if conf.TASK_RECONNECT_ALL_USERS: reactor.callLater(conf.TWISTED_WARMUP, self.lock.run, self.reconnectUsers) else: reactor.callLater(conf.TWISTED_WARMUP, self.lock.run, self.processDeathWorkers) if conf.REDIS_WORKERS > 0: t = LoopingCall(self.lock.run, self.checkWorkers) self.registerTask(t) t.start(conf.TASK_CHECK_WORKERS, now=False) reactor.callLater(conf.TWISTED_WARMUP, self._attach_global_metrics)
def log(self): if self._log is None: self._log = getLoggerAdapter(log, id=self.key) return self._log
def constructor(self): self._time = datetime.utcnow() self.log = getLoggerAdapter(getLogger(__name__, "INFO"), id='TIMER') self._interval = conf.TIMER_INTERVAL self._maxinterval = self._interval * 3
deferred_list.append(self.user.save()) deferred_list.append(GoogleMessage.updateRemoveTime(self.user.userid, self.user.lastTimeConnected)) deferred_list.append(GoogleRosterItem.remove(self.user.userid)) return defer.DeferredList(deferred_list, consumeErrors=True) def __str__(self): return '<%s object at %s. name: %s>(user: %s)'%(self.__class__.__name__, hex(id(self)), self.name, self.user) if __name__ == '__main__': import os from katoo.data import GoogleUser from wokkel_extensions import XMPPClient from katoo.utils.applog import getLogger, getLoggerAdapter from katoo.apns.api import KatooAPNSService from katoo.supervisor import XMPPKeepAliveSupervisor my_log = getLoggerAdapter(getLogger(__name__, level="INFO"), id='MYLOG') app = KatooApp().app KatooAPNSService().service.setServiceParent(app) KatooApp().start() XMPPKeepAliveSupervisor().setServiceParent(app) import twisted.python.log twisted.python.log.startLoggingWithObserver(KatooApp().log.emit) user = GoogleUser("1", _token=os.getenv('TOKEN'), _refreshtoken=os.getenv('REFRESHTOKEN'), _resource="asdfasdf", _pushtoken=os.getenv('PUSHTOKEN', None), _jid=os.getenv('JID'), _pushsound='cell1.aif', _favoritesound='cell7.aif', _away=True) my_log.info('Instance user: %s', user) xmppclient = XMPPGoogle(user, app) my_log.info('Instance xmppclient: %s', xmppclient) xmppclient.log.info("Instance XMPPGoogle %s. Instance ReauthXMPP %s Instance XMPPClient %s Instance GoogleUser %s", isinstance(xmppclient, XMPPGoogle), isinstance(xmppclient, ReauthXMPPClient), isinstance(xmppclient, XMPPClient), isinstance(xmppclient, GoogleUser)) reactor.run()
@author: pvicente ''' from datetime import timedelta from katoo import conf from katoo.data import GoogleMessage, GoogleRosterItem, GoogleUser from katoo.metrics import Metric from katoo.rqtwisted.queue import Queue, FailedQueue from katoo.utils.applog import getLogger, getLoggerAdapter from katoo.utils.connections import RedisMixin from katoo.utils.patterns import Observer from katoo.utils.time import Timer from twisted.internet import defer log = getLoggerAdapter(getLogger(__name__),id='GLOBAL_METRICS') class RedisMetrics(Observer): SOURCE='REDIS' UNIT='keys' def __init__(self): self._connection = RedisMixin.redis_conn self._keys=Metric(name="keys", value=None, unit=self.UNIT, source=self.SOURCE) failed_queue_name = FailedQueue().name self._items={conf.DIST_QUEUE_LOGIN: Metric(name='queue_%s'%(conf.DIST_QUEUE_LOGIN), value=None, unit=self.UNIT, source=self.SOURCE), conf.DIST_QUEUE_PUSH: Metric(name='queue_%s'%(conf.DIST_QUEUE_PUSH), value=None, unit=self.UNIT, source=self.SOURCE), conf.DIST_QUEUE_RELOGIN: Metric(name='queue_%s'%(conf.DIST_QUEUE_RELOGIN), value=None, unit=self.UNIT, source=self.SOURCE), failed_queue_name: Metric(name='queue_%s'%(failed_queue_name), value=None, unit=self.UNIT, source=self.SOURCE) } @defer.inlineCallbacks
os.environ[ 'ADOPTED_STREAM'] = '' #Avoid to perform Mutlprocess Service in child processes if conf.MULTIPROCESS > 0: m = MultiProcess(__file__, number=conf.MULTIPROCESS) m.setServiceParent(application) metrics_supervisor = MetricsSupervisor() metrics_supervisor.setServiceParent(application) xmpp_keepalive_supervisor = XMPPKeepAliveSupervisor() xmpp_keepalive_supervisor.setServiceParent(application) if conf.REDIS_WORKERS > 0: worker.LOGGING_OK_JOBS = conf.LOGGING_OK_JOBS worker.SLEEP_CALL = sleep worker.MAX_RETRIES = conf.BACKEND_MAX_RETRIES worker.MAX_DELAY_TIME = conf.BACKEND_MAX_DELAY w = worker.Worker( [conf.MACHINEID, conf.DIST_QUEUE_LOGIN, conf.DIST_QUEUE_RELOGIN], name=conf.MACHINEID, loops=conf.REDIS_WORKERS, default_result_ttl=conf.DIST_DEFAULT_TTL, default_warmup=conf.WORKER_WARMUP, default_enqueue_failed_jobs=conf.DIST_ENQUEUE_FAILED_JOBS, default_perform_job_in_thread=conf.DIST_PERFORM_JOB_IN_THREAD, default_thread_pool_size=conf.DIST_THREAD_POOL) w.log = getLoggerAdapter(getLogger('WORKER', level='INFO'), id='WORKER') w.setServiceParent(application)
def __init__(self, key=None, queue=None, synchronous_call=False): self.key = key self.queue_name = None if conf.DIST_DISABLED else queue self.enqueued = False self.sync = synchronous_call self._log = getLoggerAdapter(log, id=self.key)