class XMPPWorker(object):
"""XMPPWorker class"""
def __init__(self):
proctitle("worker")
log.info("worker started with PID=%s" % os.getpid())
self.route = prepare_route(route)
self.qin = Queue("xin", settings.queue_socket)
self.qout = Queue("xout", settings.queue_socket)
while True:
data = self.qin.pop()
if data:
try:
data = json.loads(data)
data_type = data.get("type", "msg")
if data_type == "msg":
method = self.handle_message
elif data_type == "tune":
method = self.handle_tune
gevent.spawn(method, data)
except ValueError, err:
log.error("%s: %s" % (err.__class__.__name__, err.message))
def put_tasks(self):
dstart = datetime.now()
qname = 'fqueue:%02d%02d' % (dstart.hour, dstart.minute)
queue = Queue(qname,
settings.feed_queue_socket,
blocking=False,
channels=False)
ids = []
cnt = 0
while True:
data = queue.pop()
if not data:
break
self.fqueue.push(data)
if 'id' in data:
ids.append(data['id'])
cnt += 1
dend = datetime.now()
td = dend - dstart
later = settings.feed_queue_update_timeout - td.seconds
if later < 0:
later = 0
if cnt:
log.info('put_tasks: %d tasks in %d sec for IDs: %s' % \
(cnt, td.seconds, str(ids)))
#gevent.spawn_later(later, self.put_tasks)
return later
def put_tasks(self):
dstart = datetime.now()
qname = 'fqueue:%02d%02d' % (dstart.hour, dstart.minute)
queue = Queue(qname, settings.feed_queue_socket,
blocking=False, channels=False)
ids = []
cnt = 0
while True:
data = queue.pop()
if not data:
break
self.fqueue.push(data)
if 'id' in data:
ids.append(data['id'])
cnt += 1
dend = datetime.now()
td = dend - dstart
later = settings.feed_queue_update_timeout - td.seconds
if later < 0:
later = 0
if cnt:
log.info('put_tasks: %d tasks in %d sec for IDs: %s' % \
(cnt, td.seconds, str(ids)))
#gevent.spawn_later(later, self.put_tasks)
return later
class FeedWorker(object):
def __init__(self):
proctitle('feed-worker')
log.info('FeedWorker started with PID=%d' % os.getpid())
self.fqueue = Queue('fqueue', settings.feed_queue_socket)
while True:
data = self.fqueue.pop()
if not data or 'id' not in data or not data['id']:
continue
try:
gevent.spawn(self.update_feed, int(data['id']))
except ValueError:
continue
def update_feed(self, id):
try:
feed = Feed(id)
except FeedNotFound:
log.error('Feed #%s does not exist. Skipped.' % id)
return
redis = RedisPool(settings.storage_socket)
try:
feed.fetch()
redis.delete('feed:retries:%s' % feed.id)
for p in feed.posts():
if not p.id:
if p.tags:
p.tags.insert(0, 'news')
else:
p.tags = ['news']
add_post(p)
log.info('Feed #%s: %s new entries saved' % \
(feed.id, len(feed.posts())))
feed.update_task()
except FeedFetchError:
retries = redis.incr('feed:retries:%s' % feed.id)
log.error('Feed #%s: %s retries failed' % (feed.id, retries))
if retries > settings.feed_retries:
redis.delete('feed:retries:%s' % feed.id)
return
timeout = settings.feed_retry_timeout * retries
feed.update_at(datetime.now() + timedelta(seconds=timeout))
except InvalidFeedType:
redis.delete('feed:retries:%s' % feed.id)
feed.update_at(datetime.now() + \
timedelta(seconds=settings.feed_max_update_timeout))
class ImgprocWorker(object):
proctitle('imgproc-worker')
def __init__(self):
self.queue = Queue('imgq', settings.imgproc_socket)
log.info('imgproc worker started')
while True:
data = self.queue.pop()
if data and isinstance(data, dict):
fn = data['fn']
del data['fn']
try:
handlers[fn](**data)
except Exception, e:
log.error(traceback.format_exc())
class XMPPWorker(object):
"""XMPPWorker class"""
def __init__(self):
proctitle('worker')
log.info('worker started with PID=%s' % os.getpid())
self.route = prepare_route(route)
self.qin = Queue('xin', settings.queue_socket)
self.qout = Queue('xout', settings.queue_socket)
while True:
data = self.qin.pop()
if data:
try:
data = json.loads(data)
gevent.spawn(self.handle_message, data)
except ValueError, err:
log.error("%s: %s" % (err.__class__.__name__, err.message))
class XMPPBot(sleekxmpp.ClientXMPP):
def __init__(self):
proctitle('bot')
log.info('bot started with PID=%d' % os.getpid())
self._jid = "%s/%s" % (settings.xmpp_jid, settings.xmpp_resource)
sleekxmpp.ClientXMPP.__init__(self, self._jid, settings.xmpp_password)
self.register_plugin('xep_0184')
self.register_plugin('xep_0163')
self.plugin['xep_0163'].add_interest('http://jabber.org/protocol/tune')
self.plugin['xep_0060'].map_node_event(
'http://jabber.org/protocol/tune', 'user_tune')
self.add_event_handler("session_start", self.session_start)
self.add_event_handler("message", self.handle_message)
self.add_event_handler("presence_subscribed", self.handle_subscription)
self.add_event_handler("user_tune_publish", self.handle_tune)
self.add_event_handler("got_offline", self.handle_disconnection)
self.add_event_handler("receipt_received", self.handle_receipt)
self.xin = Queue('xin', addr=settings.queue_socket)
self.xout = Queue('xout', addr=settings.queue_socket)
self.auto_authorize = True
self.auto_subscribe = True
spawn(self.listen_queue)
def session_start(self, event):
self.send_presence()
self.get_roster()
def handle_subscription(self, presence):
key = 'presence:%s:%s' % (presence['type'], presence['from'].bare)
data = cache_get(key)
if data:
cache_del(key)
self.send_message(**data)
def handle_receipt(self, msg):
if msg['receipt']:
receipt = msg['receipt']
elif msg['id']:
receipt = msg['id']
if receipt and receipt.startswith('post_'):
self.xin.push(
json.dumps({
'from': str(msg['from']),
'receipt': receipt[5:]
}))
def handle_message(self, msg):
if msg['type'] in ('chat', 'normal'):
if msg['id'] and msg['id'].startswith('post_'):
_msg_id = msg['id'].strip()
self.xin.push(
json.dumps({
'from': str(msg['from']),
'id': _msg_id
}))
try:
jid, resource = str(msg['to']).split('/', 1)
except ValueError:
jid = settings.xmpp_jid
resource = settings.xmpp_resource
self.xin.push(
json.dumps({
'from': str(msg['from']),
'resource': resource,
'body': msg['body'].strip()
}))
def handle_tune(self, msg):
tune = msg['pubsub_event']['items']['item']['payload']
tune_data = {
tag_name_without_ns(el): el.text
for el in tune.getchildren()
}
self.xin.push(
json.dumps({
'type': 'tune',
'from': str(msg['from']),
'tune': tune_data
}))
def handle_disconnection(self, presence):
try:
jid, resource = str(presence['from']).split('/', 1)
except ValueError:
jid = str(presence['from'])
self.xin.push(json.dumps({'type': 'tune', 'from': jid, 'tune': {}}))
def listen_queue(self):
try:
data = self.xout.pop()
if data:
data = json.loads(data)
except Exception, e:
log.error('%s %s %s' %
(e.__class__.__name__, e.message, type(data), data))
data = None
if not data:
spawn_later(0.05, self.listen_queue)
return
try:
html = None
if 'html' in data and data['html']:
html = sleekxmpp.xmlstream.ET.XML(
'<div style="margin-top:0">%s</div>' % data['html'])
#'<html xmlns="http://jabber.org/protocol/xhtml-im">' + \
#'<body xmlns="http://www.w3.org/1999/xhtml">%s</body></html>'
if '_resource' in data and data['_resource']:
mfrom = '%s/%s' % (settings.xmpp_jid, data['_resource'])
else:
mfrom = self._jid
if self.check_subscription(data['to']):
if '_presence' in data and data['_presence']:
pstatus = data['_presence'] \
if isinstance(data['_presence'], (str, unicode)) \
else "I'm online"
self.send_presence(pto=data['to'], pstatus=pstatus)
mid = data['_msg_id'] if '_msg_id' in data else None
self.send_message(mfrom=mfrom,
mto=data['to'],
mtype='chat',
mid=mid,
mbody=data['body'],
mhtml=html)
elif '_authorize' in data and data['_authorize']:
# TODO: request subscription
self.sendPresenceSubscription(pto=data['to'])
cache_store(
'presence:subscribed:%s' % JID(data['to']).bare, {
'mfrom': mfrom,
'mto': data['to'],
'mtype': 'chat',
'mbody': data['body'],
'mhtml': html
}, 3600 * 24 * 7)
finally:
spawn(self.listen_queue)
class XMPPSender(object):
def __init__(self):
proctitle('sender')
log.info('sender started with PID=%s' % os.getpid())
self.xsq = Queue('xsq', settings.queue_socket)
self.xout = Queue('xout', settings.queue_socket)
while True:
msg = self.xsq.pop()
if msg:
channel, msg = msg.split(' ', 1)
data = json.loads(msg)
if isinstance(data, int):
continue
gevent.spawn(self.handle_message, channel, data)
def handle_message(self, channel, data):
tmpl = {}
#if channel == 'msg' and 'author' in data:
#redis = RedisPool(settings.redis_socket)
#avatar = redis.get('avatar32.%s' % data['author'])
#if not avatar:
#av_path = os.path.join(settings.avatars_path, '32',
#'%s.png' % data['author'])
#if not os.path.exists(av_path):
#av_path = os.path.join(settings.avatars_path, '32.png')
#avfd = open(av_path)
#avatar = 'data:image/png;base64,%s' % b64encode(avfd.read())
#avfd.close()
#data['avatar'] = avatar
if channel == 'confirm':
if 'type' not in data or data['type'] != 'xmpp' or \
not 'address' in data or not data['address'].strip():
return
body = xmpp_template('confirm_code', settings.lang, None, **data)
out = {
'to': data['address'],
'body': body['body'],
'_authorize': True
}
self.xout.push(json.dumps(out))
return
if channel == 'remember':
body = xmpp_template('remember', settings.lang, None, **data)
out = {
'to': data['address'],
'body': body['body'],
'_authorize': True
}
self.xout.push(json.dumps(out))
return
if not isinstance(data['to'], (list, tuple)):
data['to'] = [data['to']]
res = db.fetchall(
"SELECT * FROM users.profile_im "
"WHERE id=ANY(%s);", [data['to']])
profile = {r['id']: dict(r) for r in res}
for i in data['to']:
cdata = data.copy()
user = ImUser.from_data(i, None)
try:
jid = user.get_active_account('xmpp')
if not jid:
continue
except TypeError:
continue
if i not in profile:
profile[i] = user.profile_defaults()
if profile[i]['off']:
continue
lang = user.get_profile('lang')
cut = None
if ('type' not in cdata or cdata['type'] != 'feed') and \
'cut' in data and 'text' in data:
cut = user.get_profile('im.cut')
if cut and len(cdata['text']) > cut - 3:
cdata['text'] = cdata['text'][:cut] + '...'
if cut:
ctmpl = xmpp_template("%s_%s" % (channel, cdata['a']), lang,
'html', **cdata)
else:
if not lang in tmpl:
tmpl[lang] = xmpp_template("%s_%s" % (channel, cdata['a']),
lang, 'html', **cdata)
ctmpl = tmpl[lang]
if profile[i]['xhtml']:
out = {'to': jid, 'body': ctmpl['body'], 'html': ctmpl['html']}
else:
out = {'to': jid, 'body': ctmpl['body']}
if 'post_id' in cdata and 'comment_id' in cdata:
out['_msg_id'] = 'post_%s_%s_%s' % (i, cdata['post_id'],
cdata['comment_id'])
elif 'post_id' in cdata:
out['_msg_id'] = 'post_%s_%s' % (i, cdata['post_id'])
#if channel == 'msg':
# if profile[i]['post_resource']:
# out['_resource'] = '#%s' % data['id']
# elif profile[i]['user_resource']:
# out['_resource'] = '@%s' % data['author']
self.xout.push(json.dumps(out))
class XMPPSender(object):
def __init__(self):
proctitle('sender')
log.info('sender started with PID=%s' % os.getpid())
self.xsq = Queue('xsq', settings.queue_socket)
self.xout = Queue('xout', settings.queue_socket)
while True:
msg = self.xsq.pop()
if msg:
channel, msg = msg.split(' ', 1)
data = json.loads(msg)
if isinstance(data, int):
continue
gevent.spawn(self.handle_message, channel, data)
def handle_message(self, channel, data):
tmpl = {}
#if channel == 'msg' and 'author' in data:
#redis = RedisPool(settings.redis_socket)
#avatar = redis.get('avatar32.%s' % data['author'])
#if not avatar:
#av_path = os.path.join(settings.avatars_path, '32',
#'%s.png' % data['author'])
#if not os.path.exists(av_path):
#av_path = os.path.join(settings.avatars_path, '32.png')
#avfd = open(av_path)
#avatar = 'data:image/png;base64,%s' % b64encode(avfd.read())
#avfd.close()
#data['avatar'] = avatar
if channel == 'confirm':
if 'type' not in data or data['type'] != 'xmpp' or \
not 'address' in data or not data['address'].strip():
return
body = xmpp_template('confirm_code', settings.lang, None, **data)
out = {
'to': data['address'],
'body': body['body'],
'_authorize': True
}
self.xout.push(json.dumps(out))
return
if channel == 'remember':
body = xmpp_template('remember', settings.lang, None, **data)
out = {
'to': data['address'],
'body': body['body'],
'_authorize': True
}
self.xout.push(json.dumps(out))
return
if not isinstance(data['to'], (list, tuple)):
data['to'] = [data['to']]
res = db.fetchall("SELECT * FROM users.profile_im "
"WHERE id=ANY(%s);", [data['to']])
profile = {r['id']:dict(r) for r in res}
for i in data['to']:
cdata = data.copy()
user = ImUser.from_data(i, None)
try:
jid = user.get_active_account('xmpp')
if not jid:
continue
except TypeError:
continue
if i not in profile:
profile[i] = user.profile_defaults()
if profile[i]['off']:
continue
lang = user.get_profile('lang')
cut = None
if ('type' not in cdata or cdata['type'] != 'feed') and \
'cut' in data and 'text' in data:
cut = user.get_profile('im.cut')
if cut and len(cdata['text']) > cut-3:
cdata['text'] = cdata['text'][:cut] + '...'
if cut:
ctmpl = xmpp_template("%s_%s"%(channel, cdata['a']),
lang, 'html', **cdata)
else:
if not lang in tmpl:
tmpl[lang] = xmpp_template("%s_%s"%(channel, cdata['a']),
lang, 'html', **cdata)
ctmpl = tmpl[lang]
if profile[i]['xhtml']:
out = {'to':jid, 'body':ctmpl['body'],
'html':ctmpl['html']}
else:
out = {'to':jid, 'body':ctmpl['body']}
if 'post_id' in cdata and 'comment_id' in cdata:
out['_msg_id'] = 'post_%s_%s_%s' % (i, cdata['post_id'], cdata['comment_id'])
elif 'post_id' in cdata:
out['_msg_id'] = 'post_%s_%s' % (i, cdata['post_id'])
#if channel == 'msg':
# if profile[i]['post_resource']:
# out['_resource'] = '#%s' % data['id']
# elif profile[i]['user_resource']:
# out['_resource'] = '@%s' % data['author']
self.xout.push(json.dumps(out))
class XMPPBot(sleekxmpp.ClientXMPP):
def __init__(self):
proctitle('bot')
log.info('bot started with PID=%d' % os.getpid())
self._jid = "%s/%s" % (settings.xmpp_jid, settings.xmpp_resource)
sleekxmpp.ClientXMPP.__init__(self, self._jid, settings.xmpp_password)
self.register_plugin('xep_0184')
self.register_plugin('xep_0163')
self.plugin['xep_0163'].add_interest('http://jabber.org/protocol/tune')
self.plugin['xep_0060'].map_node_event('http://jabber.org/protocol/tune', 'user_tune')
self.add_event_handler("session_start", self.session_start)
self.add_event_handler("message", self.handle_message)
self.add_event_handler("presence_subscribed", self.handle_subscription)
self.add_event_handler("user_tune_publish", self.handle_tune)
self.add_event_handler("got_offline", self.handle_disconnection)
self.add_event_handler("receipt_received", self.handle_receipt)
self.xin = Queue('xin', addr=settings.queue_socket)
self.xout = Queue('xout', addr=settings.queue_socket)
self.auto_authorize = True
self.auto_subscribe = True
spawn(self.listen_queue)
def session_start(self, event):
self.send_presence()
self.get_roster()
def handle_subscription(self, presence):
key = 'presence:%s:%s' % (presence['type'], presence['from'].bare)
data = cache_get(key)
if data:
cache_del(key)
self.send_message(**data)
def handle_receipt(self, msg):
if msg['receipt']:
receipt = msg['receipt']
elif msg['id']:
receipt = msg['id']
if receipt and receipt.startswith('post_'):
self.xin.push(json.dumps({'from': str(msg['from']),
'receipt': receipt[5:]}))
def handle_message(self, msg):
if msg['type'] in ('chat', 'normal'):
if msg['id'] and msg['id'].startswith('post_'):
_msg_id = msg['id'].strip()
self.xin.push(json.dumps({'from': str(msg['from']),
'id': _msg_id}))
try:
jid, resource = str(msg['to']).split('/', 1)
except ValueError:
jid = settings.xmpp_jid
resource = settings.xmpp_resource
self.xin.push(json.dumps({'from': str(msg['from']),
'resource': resource,
'body': msg['body'].strip()}))
def handle_tune(self, msg):
tune = msg['pubsub_event']['items']['item']['payload']
tune_data = { tag_name_without_ns(el):el.text for el in tune.getchildren() }
self.xin.push(json.dumps({'type': 'tune', 'from': str(msg['from']), 'tune': tune_data}))
def handle_disconnection(self, presence):
try:
jid, resource = str(presence['from']).split('/', 1)
except ValueError:
jid = str(presence['from'])
self.xin.push(json.dumps({'type': 'tune', 'from': jid, 'tune': {}}))
def listen_queue(self):
try:
data = self.xout.pop()
if data:
data = json.loads(data)
except Exception, e:
log.error('%s %s %s' % (e.__class__.__name__, e.message,
type(data), data))
data = None
if not data:
spawn_later(0.05, self.listen_queue)
return
try:
html = None
if 'html' in data and data['html']:
html = sleekxmpp.xmlstream.ET.XML('<div style="margin-top:0">%s</div>' % data['html'])
#'<html xmlns="http://jabber.org/protocol/xhtml-im">' + \
#'<body xmlns="http://www.w3.org/1999/xhtml">%s</body></html>'
if '_resource' in data and data['_resource']:
mfrom = '%s/%s' % (settings.xmpp_jid, data['_resource'])
else:
mfrom = self._jid
if self.check_subscription(data['to']):
if '_presence' in data and data['_presence']:
pstatus = data['_presence'] \
if isinstance(data['_presence'], (str, unicode)) \
else "I'm online"
self.send_presence(pto=data['to'], pstatus=pstatus)
mid = data['_msg_id'] if '_msg_id' in data else None
self.send_message(mfrom=mfrom, mto=data['to'], mtype='chat',
mid=mid, mbody=data['body'], mhtml=html)
elif '_authorize' in data and data['_authorize']:
# TODO: request subscription
self.sendPresenceSubscription(pto=data['to'])
cache_store('presence:subscribed:%s' % JID(data['to']).bare,
{'mfrom': mfrom, 'mto': data['to'], 'mtype': 'chat',
'mbody': data['body'], 'mhtml': html},
3600 * 24 * 7)
finally:
spawn(self.listen_queue)