def update_feed(self, id):
try:
feed = Feed(id)
except FeedNotFound:
log.error('Feed #%s does not exist. Skipped.' % id)
return
redis = RedisPool(settings.storage_socket)
try:
feed.fetch()
redis.delete('feed:retries:%s' % feed.id)
for p in feed.posts():
if not p.id:
if p.tags:
p.tags.insert(0, 'news')
else:
p.tags = ['news']
add_post(p)
log.info('Feed #%s: %s new entries saved' % \
(feed.id, len(feed.posts())))
feed.update_task()
except FeedFetchError:
retries = redis.incr('feed:retries:%s' % feed.id)
log.error('Feed #%s: %s retries failed' % (feed.id, retries))
if retries > settings.feed_retries:
redis.delete('feed:retries:%s' % feed.id)
return
timeout = settings.feed_retry_timeout * retries
feed.update_at(datetime.now() + timedelta(seconds=timeout))
except InvalidFeedType:
redis.delete('feed:retries:%s' % feed.id)
feed.update_at(datetime.now() + \
timedelta(seconds=settings.feed_max_update_timeout))
def save(self, update=False):
if not self.post.id:
raise PostNotFound
if isinstance(self.author, AnonymousUser):
anon_login = self.author.login
else:
anon_login = None
if not self.created:
self.created = datetime.now()
if isinstance(self.text, str):
self.text = self.text.decode('utf-8', 'ignore')
if update:
res = db.perform("""
UPDATE posts.comments SET (text, updated) = (%s, now())
WHERE post_id = %s AND comment_id = %s;
""", [self.text,
unb26(self.post.id) if isinstance(self.post.id, basestring) else self.post.id,
self.id])
comment_id = self.id
else:
if self.archive and self.id:
comment_id = self.id
res = db.fetchone("INSERT INTO posts.comments "
"(post_id, comment_id, author, created,"
"to_comment_id, anon_login, text, files) "
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s) "
"RETURNING comment_id;",
[unb26(self.post.id), self.id, self.author.id,
self.created,
self.to_comment_id, anon_login, self.text,
self.files])
else:
redis = RedisPool(settings.storage_socket)
while True:
try:
comment_id = redis.incr('cmnt.%s' % self.post.id)
res = db.fetchone("INSERT INTO posts.comments "
"(post_id, comment_id, author, created,"
"to_comment_id, anon_login, text, files) "
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s) "
"RETURNING comment_id;",
[unb26(self.post.id), comment_id,
self.author.id, self.created,
self.to_comment_id,
anon_login, self.text, self.files])
break
except IntegrityError:
pass
if res:
redis.incr('cmnt_cnt.%s' % unb26(self.post.id))
#try:
# es = elasticsearch.Elasticsearch(host=settings.elasticsearch_host, port=settings.elasticsearch_port)
# es.index(index='point-comments',
# id='%s-%s' % (self.post.id, self.id),
# doc_type='post', body={
# 'post_id': self.post.id,
# 'comment_id': self.id,
# 'post_type': self.post.type,
# 'created': self.created,
# 'private': self.post.private,
# 'user_id': self.author.id,
# 'login': self.author.login,
# 'text': self.text,
# })
#except elasticsearch.ConnectionError, e:
# log.error('Elasticsearch: %s' % e)
self.id = comment_id
return comment_id
def save(self, update=False):
if not self.post.id:
raise PostNotFound
if isinstance(self.author, AnonymousUser):
anon_login = self.author.login
else:
anon_login = None
if not self.created:
self.created = datetime.now()
if isinstance(self.text, str):
self.text = self.text.decode("utf-8", "ignore")
if update:
res = db.perform(
"""
UPDATE posts.comments SET (text, updated) = (%s, now())
WHERE post_id = %s AND comment_id = %s;
""",
[self.text, unb26(self.post.id) if isinstance(self.post.id, basestring) else self.post.id, self.id],
)
comment_id = self.id
else:
if self.archive and self.id:
comment_id = self.id
res = db.fetchone(
"INSERT INTO posts.comments "
"(post_id, comment_id, author, created,"
"to_comment_id, anon_login, text, files) "
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s) "
"RETURNING comment_id;",
[
unb26(self.post.id),
self.id,
self.author.id,
self.created,
self.to_comment_id,
anon_login,
self.text,
self.files,
],
)
else:
redis = RedisPool(settings.storage_socket)
while True:
try:
comment_id = redis.incr("cmnt.%s" % self.post.id)
res = db.fetchone(
"INSERT INTO posts.comments "
"(post_id, comment_id, author, created,"
"to_comment_id, anon_login, text, files) "
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s) "
"RETURNING comment_id;",
[
unb26(self.post.id),
comment_id,
self.author.id,
self.created,
self.to_comment_id,
anon_login,
self.text,
self.files,
],
)
break
except IntegrityError:
pass
if res:
redis.incr("cmnt_cnt.%s" % unb26(self.post.id))
try:
es = elasticsearch.Elasticsearch()
es.index(
index="point-comments",
id="%s-%s" % (self.post.id, self.id),
doc_type="post",
body={
"post_id": self.post.id,
"comment_id": self.id,
"post_type": self.post.type,
"created": self.created,
"private": self.post.private,
"user_id": self.author.id,
"login": self.author.login,
"text": self.text,
},
)
except elasticsearch.ConnectionError, e:
log.error("Elasticsearch: %s" % e)
def save(self):
if not self.post.id:
raise PostNotFound
if isinstance(self.author, AnonymousUser):
anon_login = self.author.login
else:
anon_login = None
if not self.created:
self.created = datetime.now()
if isinstance(self.text, str):
self.text = self.text.decode('utf-8', 'ignore')
if self.archive and self.id:
comment_id = self.id
res = db.fetchone("INSERT INTO posts.comments "
"(post_id, comment_id, author, created,"
"to_comment_id, anon_login, text, files) "
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s) "
"RETURNING comment_id;",
[unb26(self.post.id), self.id, self.author.id,
self.created,
self.to_comment_id, anon_login, self.text,
self.files])
else:
redis = RedisPool(settings.storage_socket)
while True:
try:
comment_id = redis.incr('cmnt.%s' % self.post.id)
res = db.fetchone("INSERT INTO posts.comments "
"(post_id, comment_id, author, created,"
"to_comment_id, anon_login, text, files) "
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s) "
"RETURNING comment_id;",
[unb26(self.post.id), comment_id,
self.author.id, self.created,
self.to_comment_id,
anon_login, self.text, self.files])
break
except IntegrityError:
pass
if res:
redis.incr('cmnt_cnt.%s' % unb26(self.post.id))
try:
es = elasticsearch.Elasticsearch()
es.index(index='point-comments',
id='%s-%s' % (self.post.id, self.id),
doc_type='post', body={
'post_id': self.post.id,
'comment_id': self.id,
'post_type': self.post.type,
'created': self.created,
'private': self.post.private,
'user_id': self.author.id,
'login': self.author.login,
'text': self.text,
})
except elasticsearch.ConnectionError, e:
log.error('Elasticsearch: %s' % e)
