def make_attach(path, dest, filename, remove=True):
queue = Queue('imgq', settings.imgproc_socket)
queue.push({
'fn': 'attach',
'path': path,
'dest': dest,
'filename': filename,
'remove': remove
})
def update_at(self, dt):
if not self.id:
return
log.info('Feed #%s update_at %s' % (self.id, dt))
qname = 'fqueue:%02d%02d' % (dt.hour, dt.minute)
fqueue = Queue(qname, settings.feed_queue_socket, channels=False)
fqueue.push({'id': self.id})
cache_store('feed:next_update:%s' % self.id, dt.isoformat())
def make_avatar(path, filename, remove=True, old=None):
queue = Queue('imgq', settings.imgproc_socket)
queue.push({
'fn': 'avatar',
'path': path,
'filename': filename,
'remove': remove,
'old': old
})
def update_at(self, dt):
if not self.id:
return
log.info('Feed #%s update_at %s' % (self.id, dt))
qname = 'fqueue:%02d%02d' % (dt.hour, dt.minute)
fqueue = Queue(qname, settings.feed_queue_socket, channels=False)
fqueue.push({'id': self.id})
cache_store('feed:next_update:%s' % self.id, dt.isoformat())
class XMPPSenderQueue(object):
def __init__(self):
proctitle('sender-queue')
log.info('sender queue started with PID=%s' % os.getpid())
self.xsq = Queue('xsq', settings.queue_socket)
pool = RedisPool(settings.pubsub_socket)
pubsub = pool.pubsub()
pubsub.subscribe(['msg', 'sub', 'rec', 'confirm', 'remember'])
for msg in pubsub.listen():
self.xsq.push("%s %s" % (msg['channel'], msg['data']))
class XMPPSenderQueue(object):
def __init__(self):
proctitle('sender-queue')
log.info('sender queue started with PID=%s' % os.getpid())
self.xsq = Queue('xsq', settings.queue_socket)
pool = RedisPool(settings.pubsub_socket)
pubsub = pool.pubsub()
pubsub.subscribe(['msg', 'sub', 'rec', 'confirm', 'remember'])
for msg in pubsub.listen():
self.xsq.push("%s %s" % (msg['channel'], msg['data']))
class FeedManager(object):
def __init__(self):
proctitle('feed-manager')
log.info('FeedManager started with PID=%d' % os.getpid())
self.fqueue = Queue('fqueue',
settings.feed_queue_socket,
channels=False)
#gevent.spawn(self.put_tasks)
gevent.joinall([gevent.spawn(self.run)])
def run(self):
later = 0
while True:
gevent.sleep(later)
later = self.put_tasks()
def put_tasks(self):
dstart = datetime.now()
qname = 'fqueue:%02d%02d' % (dstart.hour, dstart.minute)
queue = Queue(qname,
settings.feed_queue_socket,
blocking=False,
channels=False)
ids = []
cnt = 0
while True:
data = queue.pop()
if not data:
break
self.fqueue.push(data)
if 'id' in data:
ids.append(data['id'])
cnt += 1
dend = datetime.now()
td = dend - dstart
later = settings.feed_queue_update_timeout - td.seconds
if later < 0:
later = 0
if cnt:
log.info('put_tasks: %d tasks in %d sec for IDs: %s' % \
(cnt, td.seconds, str(ids)))
#gevent.spawn_later(later, self.put_tasks)
return later
class FeedManager(object):
def __init__(self):
proctitle('feed-manager')
log.info('FeedManager started with PID=%d' % os.getpid())
self.fqueue = Queue('fqueue', settings.feed_queue_socket,
channels=False)
#gevent.spawn(self.put_tasks)
gevent.joinall([gevent.spawn(self.run)])
def run(self):
later = 0
while True:
gevent.sleep(later)
later = self.put_tasks()
def put_tasks(self):
dstart = datetime.now()
qname = 'fqueue:%02d%02d' % (dstart.hour, dstart.minute)
queue = Queue(qname, settings.feed_queue_socket,
blocking=False, channels=False)
ids = []
cnt = 0
while True:
data = queue.pop()
if not data:
break
self.fqueue.push(data)
if 'id' in data:
ids.append(data['id'])
cnt += 1
dend = datetime.now()
td = dend - dstart
later = settings.feed_queue_update_timeout - td.seconds
if later < 0:
later = 0
if cnt:
log.info('put_tasks: %d tasks in %d sec for IDs: %s' % \
(cnt, td.seconds, str(ids)))
#gevent.spawn_later(later, self.put_tasks)
return later
def make_attach(path, dest, filename, remove=True):
queue = Queue('imgq', settings.imgproc_socket)
queue.push({'fn': 'attach',
'path': path, 'dest': dest, 'filename': filename,
'remove': remove})
class XMPPSender(object):
def __init__(self):
proctitle('sender')
log.info('sender started with PID=%s' % os.getpid())
self.xsq = Queue('xsq', settings.queue_socket)
self.xout = Queue('xout', settings.queue_socket)
while True:
msg = self.xsq.pop()
if msg:
channel, msg = msg.split(' ', 1)
data = json.loads(msg)
if isinstance(data, int):
continue
gevent.spawn(self.handle_message, channel, data)
def handle_message(self, channel, data):
tmpl = {}
#if channel == 'msg' and 'author' in data:
#redis = RedisPool(settings.redis_socket)
#avatar = redis.get('avatar32.%s' % data['author'])
#if not avatar:
#av_path = os.path.join(settings.avatars_path, '32',
#'%s.png' % data['author'])
#if not os.path.exists(av_path):
#av_path = os.path.join(settings.avatars_path, '32.png')
#avfd = open(av_path)
#avatar = 'data:image/png;base64,%s' % b64encode(avfd.read())
#avfd.close()
#data['avatar'] = avatar
if channel == 'confirm':
if 'type' not in data or data['type'] != 'xmpp' or \
not 'address' in data or not data['address'].strip():
return
body = xmpp_template('confirm_code', settings.lang, None, **data)
out = {
'to': data['address'],
'body': body['body'],
'_authorize': True
}
self.xout.push(json.dumps(out))
return
if channel == 'remember':
body = xmpp_template('remember', settings.lang, None, **data)
out = {
'to': data['address'],
'body': body['body'],
'_authorize': True
}
self.xout.push(json.dumps(out))
return
if not isinstance(data['to'], (list, tuple)):
data['to'] = [data['to']]
res = db.fetchall(
"SELECT * FROM users.profile_im "
"WHERE id=ANY(%s);", [data['to']])
profile = {r['id']: dict(r) for r in res}
for i in data['to']:
cdata = data.copy()
user = ImUser.from_data(i, None)
try:
jid = user.get_active_account('xmpp')
if not jid:
continue
except TypeError:
continue
if i not in profile:
profile[i] = user.profile_defaults()
if profile[i]['off']:
continue
lang = user.get_profile('lang')
cut = None
if ('type' not in cdata or cdata['type'] != 'feed') and \
'cut' in data and 'text' in data:
cut = user.get_profile('im.cut')
if cut and len(cdata['text']) > cut - 3:
cdata['text'] = cdata['text'][:cut] + '...'
if cut:
ctmpl = xmpp_template("%s_%s" % (channel, cdata['a']), lang,
'html', **cdata)
else:
if not lang in tmpl:
tmpl[lang] = xmpp_template("%s_%s" % (channel, cdata['a']),
lang, 'html', **cdata)
ctmpl = tmpl[lang]
if profile[i]['xhtml']:
out = {'to': jid, 'body': ctmpl['body'], 'html': ctmpl['html']}
else:
out = {'to': jid, 'body': ctmpl['body']}
if 'post_id' in cdata and 'comment_id' in cdata:
out['_msg_id'] = 'post_%s_%s_%s' % (i, cdata['post_id'],
cdata['comment_id'])
elif 'post_id' in cdata:
out['_msg_id'] = 'post_%s_%s' % (i, cdata['post_id'])
#if channel == 'msg':
# if profile[i]['post_resource']:
# out['_resource'] = '#%s' % data['id']
# elif profile[i]['user_resource']:
# out['_resource'] = '@%s' % data['author']
self.xout.push(json.dumps(out))
class XMPPSender(object):
def __init__(self):
proctitle('sender')
log.info('sender started with PID=%s' % os.getpid())
self.xsq = Queue('xsq', settings.queue_socket)
self.xout = Queue('xout', settings.queue_socket)
while True:
msg = self.xsq.pop()
if msg:
channel, msg = msg.split(' ', 1)
data = json.loads(msg)
if isinstance(data, int):
continue
gevent.spawn(self.handle_message, channel, data)
def handle_message(self, channel, data):
tmpl = {}
#if channel == 'msg' and 'author' in data:
#redis = RedisPool(settings.redis_socket)
#avatar = redis.get('avatar32.%s' % data['author'])
#if not avatar:
#av_path = os.path.join(settings.avatars_path, '32',
#'%s.png' % data['author'])
#if not os.path.exists(av_path):
#av_path = os.path.join(settings.avatars_path, '32.png')
#avfd = open(av_path)
#avatar = 'data:image/png;base64,%s' % b64encode(avfd.read())
#avfd.close()
#data['avatar'] = avatar
if channel == 'confirm':
if 'type' not in data or data['type'] != 'xmpp' or \
not 'address' in data or not data['address'].strip():
return
body = xmpp_template('confirm_code', settings.lang, None, **data)
out = {
'to': data['address'],
'body': body['body'],
'_authorize': True
}
self.xout.push(json.dumps(out))
return
if channel == 'remember':
body = xmpp_template('remember', settings.lang, None, **data)
out = {
'to': data['address'],
'body': body['body'],
'_authorize': True
}
self.xout.push(json.dumps(out))
return
if not isinstance(data['to'], (list, tuple)):
data['to'] = [data['to']]
res = db.fetchall("SELECT * FROM users.profile_im "
"WHERE id=ANY(%s);", [data['to']])
profile = {r['id']:dict(r) for r in res}
for i in data['to']:
cdata = data.copy()
user = ImUser.from_data(i, None)
try:
jid = user.get_active_account('xmpp')
if not jid:
continue
except TypeError:
continue
if i not in profile:
profile[i] = user.profile_defaults()
if profile[i]['off']:
continue
lang = user.get_profile('lang')
cut = None
if ('type' not in cdata or cdata['type'] != 'feed') and \
'cut' in data and 'text' in data:
cut = user.get_profile('im.cut')
if cut and len(cdata['text']) > cut-3:
cdata['text'] = cdata['text'][:cut] + '...'
if cut:
ctmpl = xmpp_template("%s_%s"%(channel, cdata['a']),
lang, 'html', **cdata)
else:
if not lang in tmpl:
tmpl[lang] = xmpp_template("%s_%s"%(channel, cdata['a']),
lang, 'html', **cdata)
ctmpl = tmpl[lang]
if profile[i]['xhtml']:
out = {'to':jid, 'body':ctmpl['body'],
'html':ctmpl['html']}
else:
out = {'to':jid, 'body':ctmpl['body']}
if 'post_id' in cdata and 'comment_id' in cdata:
out['_msg_id'] = 'post_%s_%s_%s' % (i, cdata['post_id'], cdata['comment_id'])
elif 'post_id' in cdata:
out['_msg_id'] = 'post_%s_%s' % (i, cdata['post_id'])
#if channel == 'msg':
# if profile[i]['post_resource']:
# out['_resource'] = '#%s' % data['id']
# elif profile[i]['user_resource']:
# out['_resource'] = '@%s' % data['author']
self.xout.push(json.dumps(out))
class XMPPBot(sleekxmpp.ClientXMPP):
def __init__(self):
proctitle('bot')
log.info('bot started with PID=%d' % os.getpid())
self._jid = "%s/%s" % (settings.xmpp_jid, settings.xmpp_resource)
sleekxmpp.ClientXMPP.__init__(self, self._jid, settings.xmpp_password)
self.register_plugin('xep_0184')
self.register_plugin('xep_0163')
self.plugin['xep_0163'].add_interest('http://jabber.org/protocol/tune')
self.plugin['xep_0060'].map_node_event('http://jabber.org/protocol/tune', 'user_tune')
self.add_event_handler("session_start", self.session_start)
self.add_event_handler("message", self.handle_message)
self.add_event_handler("presence_subscribed", self.handle_subscription)
self.add_event_handler("user_tune_publish", self.handle_tune)
self.add_event_handler("got_offline", self.handle_disconnection)
self.add_event_handler("receipt_received", self.handle_receipt)
self.xin = Queue('xin', addr=settings.queue_socket)
self.xout = Queue('xout', addr=settings.queue_socket)
self.auto_authorize = True
self.auto_subscribe = True
spawn(self.listen_queue)
def session_start(self, event):
self.send_presence()
self.get_roster()
def handle_subscription(self, presence):
key = 'presence:%s:%s' % (presence['type'], presence['from'].bare)
data = cache_get(key)
if data:
cache_del(key)
self.send_message(**data)
def handle_receipt(self, msg):
if msg['receipt']:
receipt = msg['receipt']
elif msg['id']:
receipt = msg['id']
if receipt and receipt.startswith('post_'):
self.xin.push(json.dumps({'from': str(msg['from']),
'receipt': receipt[5:]}))
def handle_message(self, msg):
if msg['type'] in ('chat', 'normal'):
if msg['id'] and msg['id'].startswith('post_'):
_msg_id = msg['id'].strip()
self.xin.push(json.dumps({'from': str(msg['from']),
'id': _msg_id}))
try:
jid, resource = str(msg['to']).split('/', 1)
except ValueError:
jid = settings.xmpp_jid
resource = settings.xmpp_resource
self.xin.push(json.dumps({'from': str(msg['from']),
'resource': resource,
'body': msg['body'].strip()}))
def handle_tune(self, msg):
tune = msg['pubsub_event']['items']['item']['payload']
tune_data = { tag_name_without_ns(el):el.text for el in tune.getchildren() }
self.xin.push(json.dumps({'type': 'tune', 'from': str(msg['from']), 'tune': tune_data}))
def handle_disconnection(self, presence):
try:
jid, resource = str(presence['from']).split('/', 1)
except ValueError:
jid = str(presence['from'])
self.xin.push(json.dumps({'type': 'tune', 'from': jid, 'tune': {}}))
def listen_queue(self):
try:
data = self.xout.pop()
if data:
data = json.loads(data)
except Exception, e:
log.error('%s %s %s' % (e.__class__.__name__, e.message,
type(data), data))
data = None
if not data:
spawn_later(0.05, self.listen_queue)
return
try:
html = None
if 'html' in data and data['html']:
html = sleekxmpp.xmlstream.ET.XML('<div style="margin-top:0">%s</div>' % data['html'])
#'<html xmlns="http://jabber.org/protocol/xhtml-im">' + \
#'<body xmlns="http://www.w3.org/1999/xhtml">%s</body></html>'
if '_resource' in data and data['_resource']:
mfrom = '%s/%s' % (settings.xmpp_jid, data['_resource'])
else:
mfrom = self._jid
if self.check_subscription(data['to']):
if '_presence' in data and data['_presence']:
pstatus = data['_presence'] \
if isinstance(data['_presence'], (str, unicode)) \
else "I'm online"
self.send_presence(pto=data['to'], pstatus=pstatus)
mid = data['_msg_id'] if '_msg_id' in data else None
self.send_message(mfrom=mfrom, mto=data['to'], mtype='chat',
mid=mid, mbody=data['body'], mhtml=html)
elif '_authorize' in data and data['_authorize']:
# TODO: request subscription
self.sendPresenceSubscription(pto=data['to'])
cache_store('presence:subscribed:%s' % JID(data['to']).bare,
{'mfrom': mfrom, 'mto': data['to'], 'mtype': 'chat',
'mbody': data['body'], 'mhtml': html},
3600 * 24 * 7)
finally:
spawn(self.listen_queue)
def make_thumbnail(url):
queue = Queue('imgq', settings.imgproc_socket)
queue.push({'fn': 'thumbnail', 'url': url})
def move_avatar(old, new):
queue = Queue('imgq', settings.imgproc_socket)
queue.push({'fn': 'move_avatar', 'old': old, 'new': new})
def move_avatar(old, new):
queue = Queue('imgq', settings.imgproc_socket)
queue.push({'fn': 'move_avatar', 'old': old, 'new': new})
def make_avatar(path, filename, remove=True, old=None):
queue = Queue('imgq', settings.imgproc_socket)
queue.push({'fn': 'avatar',
'path': path, 'filename': filename, 'remove': remove, 'old': old})
def make_thumbnail(url):
queue = Queue('imgq', settings.imgproc_socket)
queue.push({'fn': 'thumbnail', 'url': url})
def remove_attach(filename):
queue = Queue('imgq', settings.imgproc_socket)
queue.push({'fn': 'remove_attach', 'filename': filename})
class XMPPBot(sleekxmpp.ClientXMPP):
def __init__(self):
proctitle('bot')
log.info('bot started with PID=%d' % os.getpid())
self._jid = "%s/%s" % (settings.xmpp_jid, settings.xmpp_resource)
sleekxmpp.ClientXMPP.__init__(self, self._jid, settings.xmpp_password)
self.register_plugin('xep_0184')
self.register_plugin('xep_0163')
self.plugin['xep_0163'].add_interest('http://jabber.org/protocol/tune')
self.plugin['xep_0060'].map_node_event(
'http://jabber.org/protocol/tune', 'user_tune')
self.add_event_handler("session_start", self.session_start)
self.add_event_handler("message", self.handle_message)
self.add_event_handler("presence_subscribed", self.handle_subscription)
self.add_event_handler("user_tune_publish", self.handle_tune)
self.add_event_handler("got_offline", self.handle_disconnection)
self.add_event_handler("receipt_received", self.handle_receipt)
self.xin = Queue('xin', addr=settings.queue_socket)
self.xout = Queue('xout', addr=settings.queue_socket)
self.auto_authorize = True
self.auto_subscribe = True
spawn(self.listen_queue)
def session_start(self, event):
self.send_presence()
self.get_roster()
def handle_subscription(self, presence):
key = 'presence:%s:%s' % (presence['type'], presence['from'].bare)
data = cache_get(key)
if data:
cache_del(key)
self.send_message(**data)
def handle_receipt(self, msg):
if msg['receipt']:
receipt = msg['receipt']
elif msg['id']:
receipt = msg['id']
if receipt and receipt.startswith('post_'):
self.xin.push(
json.dumps({
'from': str(msg['from']),
'receipt': receipt[5:]
}))
def handle_message(self, msg):
if msg['type'] in ('chat', 'normal'):
if msg['id'] and msg['id'].startswith('post_'):
_msg_id = msg['id'].strip()
self.xin.push(
json.dumps({
'from': str(msg['from']),
'id': _msg_id
}))
try:
jid, resource = str(msg['to']).split('/', 1)
except ValueError:
jid = settings.xmpp_jid
resource = settings.xmpp_resource
self.xin.push(
json.dumps({
'from': str(msg['from']),
'resource': resource,
'body': msg['body'].strip()
}))
def handle_tune(self, msg):
tune = msg['pubsub_event']['items']['item']['payload']
tune_data = {
tag_name_without_ns(el): el.text
for el in tune.getchildren()
}
self.xin.push(
json.dumps({
'type': 'tune',
'from': str(msg['from']),
'tune': tune_data
}))
def handle_disconnection(self, presence):
try:
jid, resource = str(presence['from']).split('/', 1)
except ValueError:
jid = str(presence['from'])
self.xin.push(json.dumps({'type': 'tune', 'from': jid, 'tune': {}}))
def listen_queue(self):
try:
data = self.xout.pop()
if data:
data = json.loads(data)
except Exception, e:
log.error('%s %s %s' %
(e.__class__.__name__, e.message, type(data), data))
data = None
if not data:
spawn_later(0.05, self.listen_queue)
return
try:
html = None
if 'html' in data and data['html']:
html = sleekxmpp.xmlstream.ET.XML(
'<div style="margin-top:0">%s</div>' % data['html'])
#'<html xmlns="http://jabber.org/protocol/xhtml-im">' + \
#'<body xmlns="http://www.w3.org/1999/xhtml">%s</body></html>'
if '_resource' in data and data['_resource']:
mfrom = '%s/%s' % (settings.xmpp_jid, data['_resource'])
else:
mfrom = self._jid
if self.check_subscription(data['to']):
if '_presence' in data and data['_presence']:
pstatus = data['_presence'] \
if isinstance(data['_presence'], (str, unicode)) \
else "I'm online"
self.send_presence(pto=data['to'], pstatus=pstatus)
mid = data['_msg_id'] if '_msg_id' in data else None
self.send_message(mfrom=mfrom,
mto=data['to'],
mtype='chat',
mid=mid,
mbody=data['body'],
mhtml=html)
elif '_authorize' in data and data['_authorize']:
# TODO: request subscription
self.sendPresenceSubscription(pto=data['to'])
cache_store(
'presence:subscribed:%s' % JID(data['to']).bare, {
'mfrom': mfrom,
'mto': data['to'],
'mtype': 'chat',
'mbody': data['body'],
'mhtml': html
}, 3600 * 24 * 7)
finally:
spawn(self.listen_queue)
def remove_attach(filename):
queue = Queue('imgq', settings.imgproc_socket)
queue.push({'fn': 'remove_attach', 'filename': filename})