def comments_count(self):
if self._comments_count is not None:
return self._comments_count
if not self.id:
raise PostNotFound(None)
redis = RedisPool(settings.storage_socket)
cnt = redis.get('cmnt_cnt.%s' % unb26(self.id))
if cnt is not None:
try:
self._comments_count = int(cnt)
except (TypeError, ValueError):
self._comments_count = 0
return self._comments_count
try:
cnt = db.fetchone("SELECT count(comment_id)::int "
"FROM posts.comments "
"WHERE post_id=%s;",
[unb26(self.id)])[0]
except IndexError:
cnt = 0
redis.set('cmnt_cnt.%s' % unb26(self.id), int(cnt))
try:
self._comments_count = int(cnt)
except (TypeError, ValueError):
self._comments_count = 0
return self._comments_count
def __init__(self):
log.info('ws: pubsub init')
pool = RedisPool(settings.pubsub_socket)
pubsub = pool.pubsub()
pubsub.subscribe(['msg', 'msg.self', 'sub', 'rec', 'confirm'])
for msg in pubsub.listen():
self.handle_message(msg)
def __init__(self, _id, addr=None, blocking=True, channels=True):
self._id = _id
self.queue = RedisPool(addr)
self.blocking = blocking
self.channels = channels
log.debug('queue %s connected using %s' % (_id, addr))
def reg_invite_set_login(value):
sess = env.user.session()
if not sess.data():
env.user.session(reg_invite_set_login, key=value)
return 'Please enter your nickname'
if not sess['key']:
env.user.session_destroy()
return 'Fail'
env.user.session_destroy()
if value.startswith('@'):
value = value[1:]
if not validate_nickname(value):
return xmpp_template('reg_invalid')
try:
users.register(login=value, accounts=[('xmpp', env.jid)])
redis = RedisPool(settings.storage_socket)
redis.delete('invite:%s' % sess['key'])
return xmpp_template('reg_ok', login=value)
except UserExists:
return 'User @%s already exists.' % value
except UserError, e:
log.error('%s: %s' % (e.__class__.__name__, e.message))
return e.message
def __init__(self, field=None, value=None):
if isinstance(field, (int, long)):
User.__init__(self, field)
return
if field and value:
User.__init__(self, field, value)
return
sess = Session()
self.id = sess['id']
self.login = sess['login']
self.type = 'user'
self.accounts = []
self.accounts_add = []
self.accounts_del = []
self.profile = {}
self.profile_upd = {}
self.info = {}
self.info_upd = {}
self.password = None
self.redis = RedisPool(settings.storage_socket)
if self.id:
self._get_avatar()
def last_published(self, ts=None):
if not self.id:
return None
redis = RedisPool(settings.storage_socket)
key = "feed:last_published:%s" % self.id
if ts is None:
if not self.id:
if not self._posts:
return None
ts = max([p.created for p in self._posts])
redis.set(key, ts.isoformat())
return ts
ts = redis.get(key)
if ts:
return dateutil.parser.parse(ts)
res = db.fetchone(
"SELECT max(created) FROM posts.posts "
"WHERE author=%s;", [self.id])
if not res or not res[0]:
return None
redis.set(key, res[0].isoformat())
return res[0]
redis.set(key, ts.isoformat())
def user_sessions(user):
redis = RedisPool(settings.session_socket)
try:
sessions = json.loads(redis.get(_sessions_key(user.id)))
#print '>>>>>>> sessions', sessions
return sessions
except (TypeError, ValueError):
return []
def login():
key = sha1(
'%s%s' %
(randint(1000000, 9999999), datetime.now().isoformat())).hexdigest()
redis = RedisPool(settings.storage_socket)
redis.set('login:%s' % key, env.user.id)
redis.expire('login:%s' % key, settings.login_key_expire)
return 'https://%s/login/%s' % (settings.domain, key)
def delete(self):
res = db.fetchone("DELETE FROM posts.comments "
"WHERE post_id=%s AND comment_id=%s "
"RETURNING files;",
[unb26(self.post.id), self.id])
if res and res['files']:
for f in res['files']:
remove_attach(f)
redis = RedisPool(settings.storage_socket)
redis.decr('cmnt_cnt.%s' % unb26(self.post.id))
def cache_store(key, data, expire=None):
log.debug('cache_store %s' % key)
redis = RedisPool(settings.cache_socket)
redis.set(key, json.dumps(data))
if not expire:
try:
expire = settings.cache_expire_max
except AttributeError:
expire = 3600 * 24 # day
redis.expire(key, expire)
def __init__(self):
proctitle('sender-queue')
log.info('sender queue started with PID=%s' % os.getpid())
self.xsq = Queue('xsq', settings.queue_socket)
pool = RedisPool(settings.pubsub_socket)
pubsub = pool.pubsub()
pubsub.subscribe(['msg', 'sub', 'rec', 'confirm', 'remember'])
for msg in pubsub.listen():
self.xsq.push("%s %s" % (msg['channel'], msg['data']))
def gen_invite():
redis = RedisPool(settings.storage_socket)
icnt = redis.decr('icnt:%s' % env.user.id)
if icnt < 0:
return 'Invitation limit exceeded'
key = sha1(
'%s%s' %
(randint(1000000, 9999999), datetime.now().isoformat())).hexdigest()
redis.set('invite:%s' % key, 1)
redis.expire('invite:%s' % key, 3600 * 48)
return key
def __init__(self):
proctitle('support-worker')
log.info('support worker started with PID=%s' % os.getpid())
pool = RedisPool(settings.pubsub_socket)
pubsub = pool.pubsub()
pubsub.subscribe(['msg'])
for msg in pubsub.listen():
try:
data = json.loads(msg['data'])
except TypeError:
continue
if data['a'] in ('post', 'post_edited'):
gevent.spawn(self.handle_post, data)
def request_password(user):
if not user.id:
raise UserNotFound
address = user.get_active_account('xmpp')
if not address:
raise AddressNotFound
code = sha256('%s%s' % (datetime.now(), randint(1000000, 9999999))).hexdigest()
key = 'reset-password:%s' % code
redis = RedisPool(settings.storage_socket)
redis.set(key, user.id)
redis.expire(key, 60 * 60 * 24)
publish('remember', {'type': 'xmpp', 'address': address, 'code': code})
def reset_password(code, password):
redis = RedisPool(settings.storage_socket)
key = 'reset-password:%s' % code
print key
id = redis.get(key)
if not id:
raise UserNotFound
try:
user = User(int(id))
except ValueError:
raise UserNotFound
user.set_password(password)
user.save()
redis.delete(key)
return user
def login_key(key):
if env.user.id:
env.user.logout()
redis = RedisPool(settings.storage_socket)
try:
user_id = int(redis.get('login:%s' % key))
except (TypeError, ValueError):
raise NotFound
redis.delete('login:%s' % key)
env.user = WebUser(user_id)
env.user.authenticate()
return Response(redirect='%s://%s/' % \
(env.request.protocol, settings.domain))
def delete(self):
files = []
for ff in db.fetchall("SELECT files FROM posts.comments "
"WHERE post_id=%s;", [unb26(self.id)]):
if ff and ff['files']:
files.extend(ff['files'])
for ff in db.fetchone("DELETE FROM posts.posts WHERE id=%s "
"RETURNING files;", [unb26(self.id)]):
if ff:
files.extend(ff)
redis = RedisPool(settings.storage_socket)
redis.delete('cmnt_cnt.%s' % unb26(self.id))
for f in files:
remove_attach(f)
def delete(self):
res = db.fetchone("DELETE FROM posts.comments "
"WHERE post_id=%s AND comment_id=%s "
"RETURNING files;",
[unb26(self.post.id), self.id])
if res and res['files']:
for f in res['files']:
remove_attach(f)
redis = RedisPool(settings.storage_socket)
redis.decr('cmnt_cnt.%s' % unb26(self.post.id))
try:
es = elasticsearch.Elasticsearch()
try:
es.delete(index='point-comments', doc_type='post',
id='%s-%s' % (self.post.id, self.id))
except elasticsearch.exceptions.NotFoundError:
pass
except elasticsearch.ConnectionError, e:
log.error('Elasticsearch: %s' % e)
def update_feed(self, id):
try:
feed = Feed(id)
except FeedNotFound:
log.error('Feed #%s does not exist. Skipped.' % id)
return
redis = RedisPool(settings.storage_socket)
try:
feed.fetch()
redis.delete('feed:retries:%s' % feed.id)
for p in feed.posts():
if not p.id:
if p.tags:
p.tags.insert(0, 'news')
else:
p.tags = ['news']
add_post(p)
log.info('Feed #%s: %s new entries saved' % \
(feed.id, len(feed.posts())))
feed.update_task()
except FeedFetchError:
retries = redis.incr('feed:retries:%s' % feed.id)
log.error('Feed #%s: %s retries failed' % (feed.id, retries))
if retries > settings.feed_retries:
redis.delete('feed:retries:%s' % feed.id)
return
timeout = settings.feed_retry_timeout * retries
feed.update_at(datetime.now() + timedelta(seconds=timeout))
except InvalidFeedType:
redis.delete('feed:retries:%s' % feed.id)
feed.update_at(datetime.now() + \
timedelta(seconds=settings.feed_max_update_timeout))
def delete(self):
files = []
for ff in db.fetchall("SELECT files FROM posts.comments "
"WHERE post_id=%s;", [unb26(self.id)]):
if ff and ff['files']:
files.extend(ff['files'])
for ff in db.fetchone("DELETE FROM posts.posts WHERE id=%s "
"RETURNING files;", [unb26(self.id)]):
if ff:
files.extend(ff)
redis = RedisPool(settings.storage_socket)
redis.delete('cmnt_cnt.%s' % unb26(self.id))
for f in files:
remove_attach(f)
es = elasticsearch.Elasticsearch()
try:
es.delete(index='point-posts', doc_type='post', id=self.id)
except elasticsearch.exceptions.NotFoundError:
pass
def reg_invite(key):
redis = RedisPool(settings.storage_socket)
if not redis.get('invite:%s' % key):
return 'Invalid key'
return reg_invite_set_login(key)
def cache_get(key):
log.debug('cache_get %s' % key)
redis = RedisPool(settings.cache_socket)
data = redis.get(key)
if data:
return json.loads(data)
def cache_del(key):
log.debug('cache_del %s' % key)
redis = RedisPool(settings.cache_socket)
redis.delete(key)
def save(self, update=False):
if not self.post.id:
raise PostNotFound
if isinstance(self.author, AnonymousUser):
anon_login = self.author.login
else:
anon_login = None
if not self.created:
self.created = datetime.now()
if isinstance(self.text, str):
self.text = self.text.decode('utf-8', 'ignore')
if update:
res = db.perform("""
UPDATE posts.comments SET (text, updated) = (%s, now())
WHERE post_id = %s AND comment_id = %s;
""", [self.text,
unb26(self.post.id) if isinstance(self.post.id, basestring) else self.post.id,
self.id])
comment_id = self.id
else:
if self.archive and self.id:
comment_id = self.id
res = db.fetchone("INSERT INTO posts.comments "
"(post_id, comment_id, author, created,"
"to_comment_id, anon_login, text, files) "
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s) "
"RETURNING comment_id;",
[unb26(self.post.id), self.id, self.author.id,
self.created,
self.to_comment_id, anon_login, self.text,
self.files])
else:
redis = RedisPool(settings.storage_socket)
while True:
try:
comment_id = redis.incr('cmnt.%s' % self.post.id)
res = db.fetchone("INSERT INTO posts.comments "
"(post_id, comment_id, author, created,"
"to_comment_id, anon_login, text, files) "
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s) "
"RETURNING comment_id;",
[unb26(self.post.id), comment_id,
self.author.id, self.created,
self.to_comment_id,
anon_login, self.text, self.files])
break
except IntegrityError:
pass
if res:
redis.incr('cmnt_cnt.%s' % unb26(self.post.id))
#try:
# es = elasticsearch.Elasticsearch(host=settings.elasticsearch_host, port=settings.elasticsearch_port)
# es.index(index='point-comments',
# id='%s-%s' % (self.post.id, self.id),
# doc_type='post', body={
# 'post_id': self.post.id,
# 'comment_id': self.id,
# 'post_type': self.post.type,
# 'created': self.created,
# 'private': self.post.private,
# 'user_id': self.author.id,
# 'login': self.author.login,
# 'text': self.text,
# })
#except elasticsearch.ConnectionError, e:
# log.error('Elasticsearch: %s' % e)
self.id = comment_id
return comment_id
def comments(self, last=False, all=False, offset=None, limit=None,
cuser=None):
if last:
lim = " LIMIT %d" % limit if limit else ''
offset = 0
order = ' DESC'
elif all:
lim = ''
offset = 0
order = ' ASC'
else:
if not offset:
offset = 0
lim = " LIMIT %d" % limit if limit else ''
order = ' ASC'
if isinstance(cuser, User) and cuser.id:
res = db.fetchall("SELECT c.comment_id, c.to_comment_id,"
"c.author AS user_id, "
"CASE WHEN c.anon_login IS NOT NULL "
"THEN c.anon_login ELSE u1.login END AS login,"
"c.created at time zone %%s AS created, "
"c.text, c.files, "
"c.updated at time zone %%s AS updated, "
"CASE WHEN rc.comment_id IS NOT NULL "
"THEN true ELSE false END AS is_rec, "
"ur.user_id AS recommended, "
"ub.user_id AS bookmarked, "
"ui1.name, ui1.avatar "
"FROM posts.comments c "
"JOIN users.logins u1 ON c.author=u1.id "
"LEFT OUTER JOIN users.info ui1 "
"ON ui1.id=c.author "
"LEFT OUTER JOIN posts.recommendations rc "
"ON rc.post_id=c.post_id "
"AND rc.rcid=c.comment_id "
"LEFT OUTER JOIN posts.recommendations ur "
"ON ur.user_id=%%s "
"AND ur.post_id=c.post_id "
"AND ur.comment_id=c.comment_id "
"LEFT OUTER JOIN posts.bookmarks ub "
"ON ub.user_id=%%s "
"AND ub.post_id=c.post_id "
"AND ub.comment_id=c.comment_id "
"WHERE c.post_id=%%s AND c.comment_id>=%%s "
"ORDER BY c.created%s%s;" % (order, lim),
[self.tz, self.tz, cuser.id, cuser.id, unb26(self.id),
offset])
else:
res = db.fetchall("SELECT c.comment_id, c.to_comment_id,"
"c.author AS user_id, ui1.name,"
"CASE WHEN c.anon_login IS NOT NULL "
"THEN c.anon_login ELSE u1.login END AS login,"
"c.created at time zone %%s AS created, "
"c.text, c.files, "
"c.updated at time zone %%s AS updated, "
"CASE WHEN rc.comment_id IS NOT NULL "
"THEN true ELSE false END AS is_rec, "
"false AS recommended, "
"false AS bookmarked, "
"ui1.avatar "
"FROM posts.comments c "
"JOIN users.logins u1 ON c.author=u1.id "
"LEFT OUTER JOIN users.info ui1 "
"ON ui1.id=c.author "
"LEFT OUTER JOIN posts.recommendations rc "
"ON rc.post_id=c.post_id "
"AND rc.rcid=c.comment_id "
"WHERE c.post_id=%%s AND c.comment_id>=%%s "
"ORDER BY c.created%s%s;" % (order, lim),
[self.tz, self.tz, unb26(self.id), offset])
if last:
res.reverse()
if cuser:
unr = db.fetchall("SELECT comment_id FROM posts.unread_comments "
"WHERE user_id=%s AND post_id=%s;",
[cuser.id, unb26(self.id)])
unread = { r['comment_id']: 1 for r in unr }
else:
unread = {}
comments = []
for c in res:
author = User.from_data(c['user_id'], c['login'],
info={'name': c['name'], 'avatar': c['avatar']})
unr = True if c['comment_id'] in unread else False
comment = Comment.from_data(self, id=c['comment_id'],
to_comment_id=c['to_comment_id'],
author=author,
created=c['created'],
text=c['text'],
recommended=c['recommended'],
bookmarked=c['bookmarked'],
is_rec=c['is_rec'],
files=c['files'],
updated=c['updated'],
unread=unr)
comments.append(comment)
if not limit and not offset:
redis = RedisPool(settings.storage_socket)
redis.set('cmnt_cnt.%s' % unb26(self.id), len(comments))
return comments
def __init__(self, field, value=None):
self.id = None
self.login = None
self.accounts = []
self.accounts_add = []
self.accounts_del = []
self.profile = {}
self.profile_upd = {}
self.info = {}
self.info_upd = {}
self.password = None
self._private = None
self.redis = RedisPool(settings.storage_socket)
if isinstance(field, (int, long)):
self.id = field
self.login = cache_get('login:%s' % field)
if not self.login:
res = db.fetchone(
"SELECT login FROM users.logins WHERE id=%s;", [field])
if not res:
raise UserNotFound
self.login = res[0]
cache_store('login:%s' % field, self.login)
return
if not value:
#raise UserNotFound
# empty user
return
if field == 'login':
r = cache_get('id_login:%s' % value.lower())
if r:
try:
self.id, self.login, self.type = r
except ValueError:
self.id, self.login = r
self.type = 'user'
else:
res = db.fetchone(
"SELECT id, login, type FROM users.logins "
"WHERE lower(login)=%s;", [str(value).lower()])
if not res:
raise UserNotFound(value)
self.id, self.login, self.type = res
cache_store('id_login:%s' % value.lower(),
[res[0], res[1], res[2]])
return
r = cache_get('addr_id_login:%s' % value.lower())
if r:
self.id, self.login = r
else:
res = db.fetchone("SELECT u.id, u.login FROM users.accounts a "
"JOIN users.logins u ON u.id=a.user_id "
"WHERE a.type=%s AND lower(a.address)=%s;",
[field, value.lower()]) #, _cache=3600)
if res:
self.id, self.login = res
cache_store('addr_id_login:%s' % value.lower(),
[res[0], res[1]])
def save_sessions(user, sessions):
redis = RedisPool(settings.session_socket)
redis.set(_sessions_key(user.id), json.dumps(sessions))
def save(self):
if not self.post.id:
raise PostNotFound
if isinstance(self.author, AnonymousUser):
anon_login = self.author.login
else:
anon_login = None
if not self.created:
self.created = datetime.now()
if isinstance(self.text, str):
self.text = self.text.decode('utf-8', 'ignore')
if self.archive and self.id:
comment_id = self.id
res = db.fetchone("INSERT INTO posts.comments "
"(post_id, comment_id, author, created,"
"to_comment_id, anon_login, text, files) "
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s) "
"RETURNING comment_id;",
[unb26(self.post.id), self.id, self.author.id,
self.created,
self.to_comment_id, anon_login, self.text,
self.files])
else:
redis = RedisPool(settings.storage_socket)
while True:
try:
comment_id = redis.incr('cmnt.%s' % self.post.id)
res = db.fetchone("INSERT INTO posts.comments "
"(post_id, comment_id, author, created,"
"to_comment_id, anon_login, text, files) "
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s) "
"RETURNING comment_id;",
[unb26(self.post.id), comment_id,
self.author.id, self.created,
self.to_comment_id,
anon_login, self.text, self.files])
break
except IntegrityError:
pass
if res:
redis.incr('cmnt_cnt.%s' % unb26(self.post.id))
try:
es = elasticsearch.Elasticsearch()
es.index(index='point-comments',
id='%s-%s' % (self.post.id, self.id),
doc_type='post', body={
'post_id': self.post.id,
'comment_id': self.id,
'post_type': self.post.type,
'created': self.created,
'private': self.post.private,
'user_id': self.author.id,
'login': self.author.login,
'text': self.text,
})
except elasticsearch.ConnectionError, e:
log.error('Elasticsearch: %s' % e)
