Example #1
0
class XMPPWorker(object):
    """XMPPWorker class"""

    def __init__(self):
        proctitle("worker")
        log.info("worker started with PID=%s" % os.getpid())

        self.route = prepare_route(route)

        self.qin = Queue("xin", settings.queue_socket)
        self.qout = Queue("xout", settings.queue_socket)

        while True:
            data = self.qin.pop()
            if data:
                try:
                    data = json.loads(data)
                    data_type = data.get("type", "msg")
                    if data_type == "msg":
                        method = self.handle_message
                    elif data_type == "tune":
                        method = self.handle_tune
                    gevent.spawn(method, data)
                except ValueError, err:
                    log.error("%s: %s" % (err.__class__.__name__, err.message))
Example #2
0
    def put_tasks(self):
        dstart = datetime.now()

        qname = 'fqueue:%02d%02d' % (dstart.hour, dstart.minute)
        queue = Queue(qname,
                      settings.feed_queue_socket,
                      blocking=False,
                      channels=False)

        ids = []
        cnt = 0
        while True:
            data = queue.pop()
            if not data:
                break
            self.fqueue.push(data)
            if 'id' in data:
                ids.append(data['id'])
            cnt += 1

        dend = datetime.now()

        td = dend - dstart

        later = settings.feed_queue_update_timeout - td.seconds
        if later < 0:
            later = 0

        if cnt:
            log.info('put_tasks: %d tasks in %d sec for IDs: %s' % \
                     (cnt, td.seconds, str(ids)))
        #gevent.spawn_later(later, self.put_tasks)
        return later
Example #3
0
    def put_tasks(self):
        dstart = datetime.now()

        qname = 'fqueue:%02d%02d' % (dstart.hour, dstart.minute)
        queue = Queue(qname, settings.feed_queue_socket,
                      blocking=False, channels=False)

        ids = []
        cnt = 0
        while True:
            data = queue.pop()
            if not data:
                break
            self.fqueue.push(data)
            if 'id' in data:
                ids.append(data['id'])
            cnt += 1

        dend = datetime.now()

        td = dend - dstart

        later = settings.feed_queue_update_timeout - td.seconds
        if later < 0:
            later = 0

        if cnt:
            log.info('put_tasks: %d tasks in %d sec for IDs: %s' % \
                     (cnt, td.seconds, str(ids)))
        #gevent.spawn_later(later, self.put_tasks)
        return later
Example #4
0
class FeedWorker(object):
    def __init__(self):
        proctitle('feed-worker')
        log.info('FeedWorker started with PID=%d' % os.getpid())

        self.fqueue = Queue('fqueue', settings.feed_queue_socket)

        while True:
            data = self.fqueue.pop()
            if not data or 'id' not in data or not data['id']:
                continue

            try:
                gevent.spawn(self.update_feed, int(data['id']))
            except ValueError:
                continue

    def update_feed(self, id):
        try:
            feed = Feed(id)
        except FeedNotFound:
            log.error('Feed #%s does not exist. Skipped.' % id)
            return

        redis = RedisPool(settings.storage_socket)

        try:
            feed.fetch()

            redis.delete('feed:retries:%s' % feed.id)

            for p in feed.posts():
                if not p.id:
                    if p.tags:
                        p.tags.insert(0, 'news')
                    else:
                        p.tags = ['news']
                    add_post(p)

            log.info('Feed #%s: %s new entries saved' % \
                     (feed.id, len(feed.posts())))

            feed.update_task()

        except FeedFetchError:
            retries = redis.incr('feed:retries:%s' % feed.id)
            log.error('Feed #%s: %s retries failed' % (feed.id, retries))
            if retries > settings.feed_retries:
                redis.delete('feed:retries:%s' % feed.id)
                return
            timeout = settings.feed_retry_timeout * retries
            feed.update_at(datetime.now() + timedelta(seconds=timeout))

        except InvalidFeedType:
            redis.delete('feed:retries:%s' % feed.id)
            feed.update_at(datetime.now() + \
                           timedelta(seconds=settings.feed_max_update_timeout))
Example #5
0
class ImgprocWorker(object):
    proctitle('imgproc-worker')
    def __init__(self):
        self.queue = Queue('imgq', settings.imgproc_socket)

        log.info('imgproc worker started')

        while True:
            data = self.queue.pop()
            if data and isinstance(data, dict):
                fn = data['fn']
                del data['fn']
                try:
                    handlers[fn](**data)
                except Exception, e:
                    log.error(traceback.format_exc())
Example #6
0
class XMPPWorker(object):
    """XMPPWorker class"""
    def __init__(self):
        proctitle('worker')
        log.info('worker started with PID=%s' % os.getpid())

        self.route = prepare_route(route)

        self.qin = Queue('xin', settings.queue_socket)
        self.qout = Queue('xout', settings.queue_socket)

        while True:
            data = self.qin.pop()
            if data:
                try:
                    data = json.loads(data)
                    gevent.spawn(self.handle_message, data)
                except ValueError, err:
                    log.error("%s: %s" % (err.__class__.__name__, err.message))
Example #7
0
class XMPPBot(sleekxmpp.ClientXMPP):
    def __init__(self):
        proctitle('bot')
        log.info('bot started with PID=%d' % os.getpid())

        self._jid = "%s/%s" % (settings.xmpp_jid, settings.xmpp_resource)
        sleekxmpp.ClientXMPP.__init__(self, self._jid, settings.xmpp_password)

        self.register_plugin('xep_0184')
        self.register_plugin('xep_0163')
        self.plugin['xep_0163'].add_interest('http://jabber.org/protocol/tune')
        self.plugin['xep_0060'].map_node_event(
            'http://jabber.org/protocol/tune', 'user_tune')

        self.add_event_handler("session_start", self.session_start)
        self.add_event_handler("message", self.handle_message)
        self.add_event_handler("presence_subscribed", self.handle_subscription)
        self.add_event_handler("user_tune_publish", self.handle_tune)
        self.add_event_handler("got_offline", self.handle_disconnection)

        self.add_event_handler("receipt_received", self.handle_receipt)

        self.xin = Queue('xin', addr=settings.queue_socket)
        self.xout = Queue('xout', addr=settings.queue_socket)

        self.auto_authorize = True
        self.auto_subscribe = True

        spawn(self.listen_queue)

    def session_start(self, event):
        self.send_presence()
        self.get_roster()

    def handle_subscription(self, presence):
        key = 'presence:%s:%s' % (presence['type'], presence['from'].bare)
        data = cache_get(key)
        if data:
            cache_del(key)
            self.send_message(**data)

    def handle_receipt(self, msg):
        if msg['receipt']:
            receipt = msg['receipt']
        elif msg['id']:
            receipt = msg['id']
        if receipt and receipt.startswith('post_'):
            self.xin.push(
                json.dumps({
                    'from': str(msg['from']),
                    'receipt': receipt[5:]
                }))

    def handle_message(self, msg):
        if msg['type'] in ('chat', 'normal'):
            if msg['id'] and msg['id'].startswith('post_'):
                _msg_id = msg['id'].strip()
                self.xin.push(
                    json.dumps({
                        'from': str(msg['from']),
                        'id': _msg_id
                    }))

            try:
                jid, resource = str(msg['to']).split('/', 1)
            except ValueError:
                jid = settings.xmpp_jid
                resource = settings.xmpp_resource
            self.xin.push(
                json.dumps({
                    'from': str(msg['from']),
                    'resource': resource,
                    'body': msg['body'].strip()
                }))

    def handle_tune(self, msg):
        tune = msg['pubsub_event']['items']['item']['payload']
        tune_data = {
            tag_name_without_ns(el): el.text
            for el in tune.getchildren()
        }
        self.xin.push(
            json.dumps({
                'type': 'tune',
                'from': str(msg['from']),
                'tune': tune_data
            }))

    def handle_disconnection(self, presence):
        try:
            jid, resource = str(presence['from']).split('/', 1)
        except ValueError:
            jid = str(presence['from'])
        self.xin.push(json.dumps({'type': 'tune', 'from': jid, 'tune': {}}))

    def listen_queue(self):
        try:
            data = self.xout.pop()
            if data:
                data = json.loads(data)
        except Exception, e:
            log.error('%s %s %s' %
                      (e.__class__.__name__, e.message, type(data), data))
            data = None
        if not data:
            spawn_later(0.05, self.listen_queue)
            return

        try:
            html = None
            if 'html' in data and data['html']:
                html = sleekxmpp.xmlstream.ET.XML(
                    '<div style="margin-top:0">%s</div>' % data['html'])
                #'<html xmlns="http://jabber.org/protocol/xhtml-im">' + \
                #'<body xmlns="http://www.w3.org/1999/xhtml">%s</body></html>'
            if '_resource' in data and data['_resource']:
                mfrom = '%s/%s' % (settings.xmpp_jid, data['_resource'])
            else:
                mfrom = self._jid

            if self.check_subscription(data['to']):
                if '_presence' in data and data['_presence']:
                    pstatus = data['_presence'] \
                                if isinstance(data['_presence'], (str, unicode)) \
                                else "I'm online"
                    self.send_presence(pto=data['to'], pstatus=pstatus)

                mid = data['_msg_id'] if '_msg_id' in data else None

                self.send_message(mfrom=mfrom,
                                  mto=data['to'],
                                  mtype='chat',
                                  mid=mid,
                                  mbody=data['body'],
                                  mhtml=html)
            elif '_authorize' in data and data['_authorize']:
                # TODO: request subscription
                self.sendPresenceSubscription(pto=data['to'])
                cache_store(
                    'presence:subscribed:%s' % JID(data['to']).bare, {
                        'mfrom': mfrom,
                        'mto': data['to'],
                        'mtype': 'chat',
                        'mbody': data['body'],
                        'mhtml': html
                    }, 3600 * 24 * 7)
        finally:
            spawn(self.listen_queue)
Example #8
0
class XMPPSender(object):
    def __init__(self):
        proctitle('sender')
        log.info('sender started with PID=%s' % os.getpid())

        self.xsq = Queue('xsq', settings.queue_socket)
        self.xout = Queue('xout', settings.queue_socket)

        while True:
            msg = self.xsq.pop()
            if msg:
                channel, msg = msg.split(' ', 1)
                data = json.loads(msg)
                if isinstance(data, int):
                    continue
                gevent.spawn(self.handle_message, channel, data)

    def handle_message(self, channel, data):
        tmpl = {}

        #if channel == 'msg' and 'author' in data:
        #redis = RedisPool(settings.redis_socket)
        #avatar = redis.get('avatar32.%s' % data['author'])
        #if not avatar:
        #av_path = os.path.join(settings.avatars_path, '32',
        #'%s.png' % data['author'])
        #if not os.path.exists(av_path):
        #av_path = os.path.join(settings.avatars_path, '32.png')

        #avfd = open(av_path)
        #avatar = 'data:image/png;base64,%s' % b64encode(avfd.read())
        #avfd.close()

        #data['avatar'] = avatar

        if channel == 'confirm':
            if 'type' not in data or data['type'] != 'xmpp' or \
                    not 'address' in data or not data['address'].strip():
                return
            body = xmpp_template('confirm_code', settings.lang, None, **data)
            out = {
                'to': data['address'],
                'body': body['body'],
                '_authorize': True
            }
            self.xout.push(json.dumps(out))
            return

        if channel == 'remember':
            body = xmpp_template('remember', settings.lang, None, **data)
            out = {
                'to': data['address'],
                'body': body['body'],
                '_authorize': True
            }
            self.xout.push(json.dumps(out))
            return

        if not isinstance(data['to'], (list, tuple)):
            data['to'] = [data['to']]

        res = db.fetchall(
            "SELECT * FROM users.profile_im "
            "WHERE id=ANY(%s);", [data['to']])
        profile = {r['id']: dict(r) for r in res}

        for i in data['to']:
            cdata = data.copy()

            user = ImUser.from_data(i, None)
            try:
                jid = user.get_active_account('xmpp')
                if not jid:
                    continue
            except TypeError:
                continue

            if i not in profile:
                profile[i] = user.profile_defaults()
            if profile[i]['off']:
                continue
            lang = user.get_profile('lang')

            cut = None
            if ('type' not in cdata or cdata['type'] != 'feed') and \
                'cut' in data and 'text' in data:
                cut = user.get_profile('im.cut')
                if cut and len(cdata['text']) > cut - 3:
                    cdata['text'] = cdata['text'][:cut] + '...'

            if cut:
                ctmpl = xmpp_template("%s_%s" % (channel, cdata['a']), lang,
                                      'html', **cdata)
            else:
                if not lang in tmpl:
                    tmpl[lang] = xmpp_template("%s_%s" % (channel, cdata['a']),
                                               lang, 'html', **cdata)
                ctmpl = tmpl[lang]

            if profile[i]['xhtml']:
                out = {'to': jid, 'body': ctmpl['body'], 'html': ctmpl['html']}
            else:
                out = {'to': jid, 'body': ctmpl['body']}

            if 'post_id' in cdata and 'comment_id' in cdata:
                out['_msg_id'] = 'post_%s_%s_%s' % (i, cdata['post_id'],
                                                    cdata['comment_id'])
            elif 'post_id' in cdata:
                out['_msg_id'] = 'post_%s_%s' % (i, cdata['post_id'])

            #if channel == 'msg':
            #    if profile[i]['post_resource']:
            #        out['_resource'] = '#%s' % data['id']
            #    elif profile[i]['user_resource']:
            #        out['_resource'] = '@%s' % data['author']

            self.xout.push(json.dumps(out))
Example #9
0
class XMPPSender(object):
    def __init__(self):
        proctitle('sender')
        log.info('sender started with PID=%s' % os.getpid())

        self.xsq = Queue('xsq', settings.queue_socket)
        self.xout = Queue('xout', settings.queue_socket)

        while True:
            msg = self.xsq.pop()
            if msg:
                channel, msg = msg.split(' ', 1)
                data = json.loads(msg)
                if isinstance(data, int):
                    continue
                gevent.spawn(self.handle_message, channel, data)

    def handle_message(self, channel, data):
        tmpl = {}

        #if channel == 'msg' and 'author' in data:
            #redis = RedisPool(settings.redis_socket)
            #avatar = redis.get('avatar32.%s' % data['author'])
            #if not avatar:
                #av_path = os.path.join(settings.avatars_path, '32',
                                       #'%s.png' % data['author'])
                #if not os.path.exists(av_path):
                    #av_path = os.path.join(settings.avatars_path, '32.png')

                #avfd = open(av_path)
                #avatar = 'data:image/png;base64,%s' % b64encode(avfd.read())
                #avfd.close()

            #data['avatar'] = avatar

        if channel == 'confirm':
            if 'type' not in data or data['type'] != 'xmpp' or \
                    not 'address' in data or not data['address'].strip():
                return
            body = xmpp_template('confirm_code', settings.lang, None, **data)
            out = {
                'to': data['address'],
                'body': body['body'],
                '_authorize': True
            }
            self.xout.push(json.dumps(out))
            return

        if channel == 'remember':
            body = xmpp_template('remember', settings.lang, None, **data)
            out = {
                'to': data['address'],
                'body': body['body'],
                '_authorize': True
            }
            self.xout.push(json.dumps(out))
            return

        if not isinstance(data['to'], (list, tuple)):
            data['to'] = [data['to']]

        res = db.fetchall("SELECT * FROM users.profile_im "
                          "WHERE id=ANY(%s);", [data['to']])
        profile = {r['id']:dict(r) for r in res}

        for i in data['to']:
            cdata = data.copy()

            user = ImUser.from_data(i, None)
            try:
                jid = user.get_active_account('xmpp')
                if not jid:
                    continue
            except TypeError:
                continue

            if i not in profile:
                profile[i] = user.profile_defaults()
            if profile[i]['off']:
                continue
            lang = user.get_profile('lang')

            cut = None
            if ('type' not in cdata or cdata['type'] != 'feed') and \
                'cut' in data and 'text' in data:
                cut = user.get_profile('im.cut')
                if cut and len(cdata['text']) > cut-3:
                    cdata['text'] = cdata['text'][:cut] + '...'

            if cut:
                ctmpl = xmpp_template("%s_%s"%(channel, cdata['a']),
                                      lang, 'html', **cdata)
            else:
                if not lang in tmpl:
                    tmpl[lang] = xmpp_template("%s_%s"%(channel, cdata['a']),
                                               lang, 'html', **cdata)
                ctmpl = tmpl[lang]

            if profile[i]['xhtml']:
                out = {'to':jid, 'body':ctmpl['body'],
                                 'html':ctmpl['html']}
            else:
                out = {'to':jid, 'body':ctmpl['body']}

            if 'post_id' in cdata and 'comment_id' in cdata:
                out['_msg_id'] = 'post_%s_%s_%s' % (i, cdata['post_id'], cdata['comment_id'])
            elif 'post_id' in cdata:
                out['_msg_id'] = 'post_%s_%s' % (i, cdata['post_id'])

            #if channel == 'msg':
            #    if profile[i]['post_resource']:
            #        out['_resource'] = '#%s' % data['id']
            #    elif profile[i]['user_resource']:
            #        out['_resource'] = '@%s' % data['author']

            self.xout.push(json.dumps(out))
Example #10
0
class XMPPBot(sleekxmpp.ClientXMPP):
    def __init__(self):
        proctitle('bot')
        log.info('bot started with PID=%d' % os.getpid())

        self._jid = "%s/%s" % (settings.xmpp_jid, settings.xmpp_resource)
        sleekxmpp.ClientXMPP.__init__(self, self._jid, settings.xmpp_password)

        self.register_plugin('xep_0184')
        self.register_plugin('xep_0163')
        self.plugin['xep_0163'].add_interest('http://jabber.org/protocol/tune')
        self.plugin['xep_0060'].map_node_event('http://jabber.org/protocol/tune', 'user_tune')

        self.add_event_handler("session_start", self.session_start)
        self.add_event_handler("message", self.handle_message)
        self.add_event_handler("presence_subscribed", self.handle_subscription)
        self.add_event_handler("user_tune_publish", self.handle_tune)
        self.add_event_handler("got_offline", self.handle_disconnection)

        self.add_event_handler("receipt_received", self.handle_receipt)

        self.xin = Queue('xin', addr=settings.queue_socket)
        self.xout = Queue('xout', addr=settings.queue_socket)

        self.auto_authorize = True
        self.auto_subscribe = True

        spawn(self.listen_queue)

    def session_start(self, event):
        self.send_presence()
        self.get_roster()

    def handle_subscription(self, presence):
        key = 'presence:%s:%s' % (presence['type'], presence['from'].bare)
        data = cache_get(key)
        if data:
            cache_del(key)
            self.send_message(**data)

    def handle_receipt(self, msg):
        if msg['receipt']:
            receipt = msg['receipt']
        elif msg['id']:
            receipt = msg['id']
        if receipt and receipt.startswith('post_'):
            self.xin.push(json.dumps({'from': str(msg['from']),
                                      'receipt': receipt[5:]}))

    def handle_message(self, msg):
        if msg['type'] in ('chat', 'normal'):
            if msg['id'] and msg['id'].startswith('post_'):
                _msg_id = msg['id'].strip()
                self.xin.push(json.dumps({'from': str(msg['from']),
                                          'id': _msg_id}))

            try:
                jid, resource = str(msg['to']).split('/', 1)
            except ValueError:
                jid = settings.xmpp_jid
                resource = settings.xmpp_resource
            self.xin.push(json.dumps({'from': str(msg['from']),
                                      'resource': resource,
                                      'body': msg['body'].strip()}))

    def handle_tune(self, msg):
        tune = msg['pubsub_event']['items']['item']['payload']
        tune_data = { tag_name_without_ns(el):el.text for el in tune.getchildren() }
        self.xin.push(json.dumps({'type': 'tune', 'from': str(msg['from']), 'tune': tune_data}))

    def handle_disconnection(self, presence):
        try:
            jid, resource = str(presence['from']).split('/', 1)
        except ValueError:
            jid = str(presence['from'])
        self.xin.push(json.dumps({'type': 'tune', 'from': jid, 'tune': {}}))

    def listen_queue(self):
        try:
            data = self.xout.pop()
            if data:
                data = json.loads(data)
        except Exception, e:
            log.error('%s %s %s' % (e.__class__.__name__, e.message,
                                    type(data), data))
            data = None
        if not data:
            spawn_later(0.05, self.listen_queue)
            return

        try:
            html = None
            if 'html' in data and data['html']:
                html = sleekxmpp.xmlstream.ET.XML('<div style="margin-top:0">%s</div>' % data['html'])
                    #'<html xmlns="http://jabber.org/protocol/xhtml-im">' + \
                    #'<body xmlns="http://www.w3.org/1999/xhtml">%s</body></html>'
            if '_resource' in data and data['_resource']:
                mfrom = '%s/%s' % (settings.xmpp_jid, data['_resource'])
            else:
                mfrom = self._jid

            if self.check_subscription(data['to']):
                if '_presence' in data and data['_presence']:
                    pstatus = data['_presence'] \
                                if isinstance(data['_presence'], (str, unicode)) \
                                else "I'm online"
                    self.send_presence(pto=data['to'], pstatus=pstatus)

                mid = data['_msg_id'] if '_msg_id' in data else None

                self.send_message(mfrom=mfrom, mto=data['to'], mtype='chat',
                                  mid=mid, mbody=data['body'], mhtml=html)
            elif '_authorize' in data and data['_authorize']:
                # TODO: request subscription
                self.sendPresenceSubscription(pto=data['to'])
                cache_store('presence:subscribed:%s' % JID(data['to']).bare,
                            {'mfrom': mfrom, 'mto': data['to'], 'mtype': 'chat',
                             'mbody': data['body'], 'mhtml': html},
                            3600 * 24 * 7)
        finally:
            spawn(self.listen_queue)