def generate_site(token):
site = Site.select().where(Site.token == token).get()
rss_title = get_template('rss_title_message').render(site=site.name)
md = markdown.Markdown()
items = []
for row in (Comment.select().join(Site).where(
Site.token == token,
Comment.published).order_by(-Comment.published).limit(10)):
item_link = '%s://%s%s' % (config.get(
config.RSS_PROTO), site.url, row.url)
items.append(
PyRSS2Gen.RSSItem(
title='%s - %s://%s%s' % (config.get(
config.RSS_PROTO), row.author_name, site.url, row.url),
link=item_link,
description=md.convert(row.content),
guid=PyRSS2Gen.Guid('%s/%d' % (item_link, row.id)),
pubDate=row.published,
))
rss = PyRSS2Gen.RSS2(
title=rss_title,
link='%s://%s' % (config.get(config.RSS_PROTO), site.url),
description='Commentaires du site "%s"' % site.name,
lastBuildDate=datetime.now(),
items=items,
)
rss.write_xml(open(config.get(config.RSS_FILE), 'w'), encoding='utf-8')
def query_comments():
comments = []
try:
token = request.args.get('token', '')
url = request.args.get('url', '')
logger.info('retrieve comments for token %s, url %s' % (token, url))
for comment in Comment.select(Comment).join(Site).where(
(Comment.url == url) &
(Comment.published.is_null(False)) &
(Site.token == token)).order_by(+Comment.published):
d = {}
d['author'] = comment.author_name
d['content'] = comment.content
if comment.author_site:
d['site'] = comment.author_site
d['avatar'] = comment.author_gravatar
d['date'] = comment.published.strftime("%Y-%m-%d %H:%M:%S")
logger.debug(d)
comments.append(d)
r = jsonify({'data': comments})
r.status_code = 200
except:
logger.warn('bad request')
r = jsonify({'data': []})
r.status_code = 400
return r
def query_comments():
comments = []
try:
token = request.args.get('token', '')
url = request.args.get('url', '')
logger.info('retrieve comments for url %s' % (url))
for comment in (Comment.select(Comment).join(Site).where(
(Comment.url == url)
& (Comment.published.is_null(False))
& (Site.token == token)).order_by(+Comment.published)):
d = {
'author': comment.author_name,
'content': comment.content,
'avatar': comment.author_gravatar,
'date': comment.published.strftime('%Y-%m-%d %H:%M:%S')
}
if comment.author_site:
d['site'] = comment.author_site
logger.debug(d)
comments.append(d)
r = jsonify({'data': comments})
r.status_code = 200
except:
logger.warn('bad request')
r = jsonify({'data': []})
r.status_code = 400
return r
def submit_new_comment():
for comment in Comment.select().where(Comment.notified.is_null()):
comment_list = (
'author: %s' % comment.author_name,
'site: %s' % comment.author_site,
'date: %s' % comment.created,
'url: %s' % comment.url,
'',
'%s' % comment.content,
'',
)
comment_text = '\n'.join(comment_list)
email_body = get_template('new_comment').render(url=comment.url,
comment=comment_text)
# send email
site = Site.get(Site.id == comment.site)
subject = 'STACOSYS %s: [%d:%s]' % (site.name, comment.id, site.token)
if mailer.send(site.admin_email, subject, email_body):
logger.debug('new comment processed ')
# notify site admin and save notification datetime
comment.notify_site_admin()
else:
logger.warn('rescheduled. send mail failure ' + subject)
def submit_new_comment():
for comment in Comment.select().where(Comment.notified.is_null()):
comment_list = (
"author: %s" % comment.author_name,
"site: %s" % comment.author_site,
"date: %s" % comment.created,
"url: %s" % comment.url,
"",
"%s" % comment.content,
"",
)
comment_text = "\n".join(comment_list)
email_body = get_template("new_comment").render(
url=comment.url, comment=comment_text
)
# send email
site = Site.get(Site.id == comment.site)
subject = "STACOSYS %s: [%d:%s]" % (site.name, comment.id, site.token)
mailer.send(site.admin_email, subject, email_body)
logger.debug("new comment processed ")
# update comment
comment.notified = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
comment.save()
def generate_site(token):
site = Site.select().where(Site.token == token).get()
rss_title = get_template("rss_title_message").render(site=site.name)
md = markdown.Markdown()
items = []
for row in (
Comment.select()
.join(Site)
.where(Site.token == token, Comment.published)
.order_by(-Comment.published)
.limit(10)
):
item_link = "%s://%s%s" % (config.get(config.RSS_PROTO), site.url, row.url)
items.append(
PyRSS2Gen.RSSItem(
title="%s - %s://%s%s"
% (config.get(config.RSS_PROTO), row.author_name, site.url, row.url),
link=item_link,
description=md.convert(row.content),
guid=PyRSS2Gen.Guid("%s/%d" % (item_link, row.id)),
pubDate=row.published,
)
)
rss = PyRSS2Gen.RSS2(
title=rss_title,
link="%s://%s" % (config.get(config.RSS_PROTO), site.url),
description="Commentaires du site '%s'" % site.name,
lastBuildDate=datetime.now(),
items=items,
)
rss.write_xml(open(config.get(config.RSS_FILE), "w"), encoding="utf-8")
def after_insert(self, raw_post: Dict, values_lst: List[SQLValuesToWrite],
records: List[DataRecord]):
for record in records:
statistic_add_comment(record['related_type'], record['related_id'],
record['id'])
post_number = Comment.select().where(
Comment.related_id == record['related_id'],
Comment.id <= record['id']).count()
Comment.update(post_number=post_number).where(
Comment.id == record['id']).execute()
def _reply_comment_email(email: Email):
m = re.search(r'\[(\d+)\:(\w+)\]', email.subject)
if not m:
logger.warn('ignore corrupted email. No token %s' % email.subject)
return
comment_id = int(m.group(1))
token = m.group(2)
# retrieve site and comment rows
try:
comment = Comment.select().where(Comment.id == comment_id).get()
except:
logger.warn('unknown comment %d' % comment_id)
return True
if comment.published:
logger.warn('ignore already published email. token %d' % comment_id)
return
if comment.site.token != token:
logger.warn('ignore corrupted email. Unknown token %d' % comment_id)
return
if not email.plain_text_content:
logger.warn('ignore empty email')
return
# safe logic: no answer or unknown answer is a go for publishing
if email.plain_text_content[:2].upper() in ('NO'):
logger.info('discard comment: %d' % comment_id)
comment.delete_instance()
new_email_body = get_template('drop_comment').render(
original=email.plain_text_content)
if not mailer.send(email.from_addr, 'Re: ' + email.subject,
new_email_body):
logger.warn('minor failure. cannot send rejection mail ' +
email.subject)
else:
# save publishing datetime
comment.publish()
logger.info('commit comment: %d' % comment_id)
# rebuild RSS
rss.generate_site(token)
# send approval confirmation email to admin
new_email_body = get_template('approve_comment').render(
original=email.plain_text_content)
if not mailer.send(email.from_addr, 'Re: ' + email.subject,
new_email_body):
logger.warn('minor failure. cannot send approval email ' +
email.subject)
return True
def after_insert(self, raw_post: Dict, values_lst: List[SQLValuesToWrite], records: List[DataRecord]):
for record in records:
statistic_add_comment(record['related_type'], record['related_id'], record['id'])
post_number = Comment.select().where(Comment.related_id == record['related_id'], Comment.id <= record['id']).count()
Comment.update(post_number=post_number).where(Comment.id == record['id']).execute()
if self.do_mentions:
self.do_mentions(record['user_id'], POST_TYPES.COMMENT, record['id'], {
'comment_info': {
'related_type': record['related_type'],
'related_id': record['related_id'],
}
})
def get_comments_count():
try:
token = request.args.get('token', '')
url = request.args.get('url', '')
count = Comment.select(Comment).join(Site).where(
(Comment.url == url) &
(Comment.published.is_null(False)) &
(Site.token == token)).count()
r = jsonify({'count': count})
r.status_code = 200
except:
r = jsonify({'count': 0})
r.status_code = 200
return r
def get_comments_count():
try:
token = request.args.get('token', '')
url = request.args.get('url', '')
count = (Comment.select(Comment).join(Site).where(
(Comment.url == url)
& (Comment.published.is_null(False))
& (Site.token == token)).count())
r = jsonify({'count': count})
r.status_code = 200
except:
r = jsonify({'count': 0})
r.status_code = 200
return r
def work():
try:
db.execute_sql('DROP TABLE statistic24h;')
db.execute_sql('ALTER TABLE "user" ADD COLUMN "visible" INTEGER NULL DEFAULT NULL;')
db.execute_sql('ALTER TABLE "user" ADD COLUMN "user_id" BYTEA NULL DEFAULT NULL;')
db.execute_sql('ALTER TABLE "user" ADD COLUMN "reset_key" BYTEA NULL DEFAULT NULL;')
db.execute_sql('ALTER TABLE "user" ADD COLUMN "avatar" TEXT NULL DEFAULT NULL;')
db.execute_sql('ALTER TABLE "user" ADD COLUMN "type" INTEGER DEFAULT 0;')
db.execute_sql('ALTER TABLE "user" ADD COLUMN "url" TEXT NULL DEFAULT NULL;')
db.execute_sql('ALTER TABLE "user" ADD COLUMN "location" TEXT NULL DEFAULT NULL;')
db.execute_sql('ALTER TABLE "user" ADD COLUMN "access_time" BIGINT NULL DEFAULT NULL;')
db.execute_sql('ALTER TABLE "user" ADD COLUMN "last_check_in_time" BIGINT NULL DEFAULT NULL;')
db.execute_sql('ALTER TABLE "user" ADD COLUMN "check_in_his" INT DEFAULT 0;')
db.execute_sql('ALTER TABLE "user" ADD COLUMN "exp" INT DEFAULT 0;')
db.execute_sql('ALTER TABLE "user" RENAME "reg_time" TO "time";')
db.execute_sql('ALTER TABLE "board" RENAME "creator_id" TO "user_id";')
db.execute_sql('ALTER TABLE "comment" ADD COLUMN "post_number" INTEGER NULL;')
db.execute_sql('ALTER TABLE "statistic" ADD COLUMN "bookmark_count" INTEGER DEFAULT 0;')
db.execute_sql('ALTER TABLE "statistic" ADD COLUMN "upvote_count" INTEGER DEFAULT 0;')
db.execute_sql('ALTER TABLE "statistic" ADD COLUMN "downvote_count" INTEGER DEFAULT 0;')
db.execute_sql('ALTER TABLE "statistic" ADD COLUMN "thank_count" INTEGER DEFAULT 0;')
db.execute_sql('ALTER TABLE "statistic" ADD COLUMN "vote_weight" BIGINT DEFAULT 0;')
db.execute_sql('ALTER TABLE "statistic" ADD COLUMN "bookmarked_users" BYTEA[] NULL;')
db.execute_sql('ALTER TABLE "statistic" ADD COLUMN "upvoted_users" BYTEA[] NULL;')
db.execute_sql('ALTER TABLE "statistic" ADD COLUMN "downvoted_users" BYTEA[] NULL;')
db.execute_sql('ALTER TABLE "statistic" ADD COLUMN "thanked_users" BYTEA[] NULL;')
except Exception as e:
print(e)
print('failed')
db.rollback()
for i in Comment.select():
post_number = Comment.select().where(Comment.related_id == i.related_id, Comment.id <= i.id).count()
Comment.update(post_number=post_number).where(Comment.id == i.id).execute()
async def after_insert(self, raw_post: Dict, values: SQLValuesToWrite,
record: DataRecord):
post_stats_do_comment(record['related_type'], record['related_id'],
record['id'])
post_number = Comment.select().where(
Comment.related_id == record['related_id'],
Comment.id <= record['id']).count()
Comment.update(post_number=post_number).where(
Comment.id == record['id']).execute()
if self.do_mentions:
# 创建提醒
loc = [record['related_type'], record['related_id']]
# record['related_id']: memoryview
loc_title = POST_TYPES.get_post_title_by_list(loc)[
record['related_id'].tobytes()]
related = [POST_TYPES.COMMENT, record['id']]
self.do_mentions(record['user_id'], loc_title, loc, related)
def update_all(reset=False):
if reset:
try:
es.indices.delete(index=INDEX_NAME)
except elasticsearch.exceptions.NotFoundError:
pass
create_index()
for i in Topic.select(Topic.id):
print('topic', to_hex(i.id))
es_update_topic(i.id)
for i in WikiArticle.select(WikiArticle.id):
print('wiki', to_hex(i.id))
es_update_wiki(i.id)
for i in Comment.select(Comment.id):
print('comment', to_hex(i.id))
es_update_comment(i.id)
async def after_insert(self, values_lst: List[SQLValuesToWrite],
records: List[DataRecord]):
for record in records:
post_stats_do_comment(record['related_type'], record['related_id'],
record['id'])
post_number = Comment.select().where(
Comment.related_id == record['related_id'],
Comment.id <= record['id']).count()
Comment.update(post_number=post_number).where(
Comment.id == record['id']).execute()
if self.do_mentions:
# 创建提醒
loc = [record['related_type'], record['related_id']]
# record['related_id']: memoryview
loc_title = POST_TYPES.get_post_title_by_list(loc)[
get_bytes_from_blob(record['related_id'])]
related = [POST_TYPES.COMMENT, record['id']]
self.do_mentions(record['user_id'], loc_title, loc, related)
if config.SEARCH_ENABLE:
run_in_thread(esdb.es_update_comment, record['id'])
def reply_comment_email(data):
from_email = data["from"]
subject = data["subject"]
message = ""
for part in data["parts"]:
if part["content-type"] == "text/plain":
message = part["content"]
break
m = re.search(r"\[(\d+)\:(\w+)\]", subject)
if not m:
logger.warn("ignore corrupted email. No token %s" % subject)
return
comment_id = int(m.group(1))
token = m.group(2)
# retrieve site and comment rows
try:
comment = Comment.select().where(Comment.id == comment_id).get()
except:
logger.warn("unknown comment %d" % comment_id)
return True
if comment.published:
logger.warn("ignore already published email. token %d" % comment_id)
return
if comment.site.token != token:
logger.warn("ignore corrupted email. Unknown token %d" % comment_id)
return
if not message:
logger.warn("ignore empty email")
return
# safe logic: no answer or unknown answer is a go for publishing
if message[:2].upper() in ("NO", "SP"):
# put a log to help fail2ban
if message[:2].upper() == "SP": # SPAM
if comment.ip:
logger.info(
"SPAM comment from %s: %d" % (comment.ip, comment_id)
)
else:
logger.info("cannot identify SPAM source: %d" % comment_id)
logger.info("discard comment: %d" % comment_id)
comment.delete_instance()
email_body = get_template("drop_comment").render(original=message)
mailer.send(from_email, "Re: " + subject, email_body)
else:
# update Comment row
comment.published = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
comment.ip = None
comment.save()
logger.info("commit comment: %d" % comment_id)
# rebuild RSS
rss.generate_site(token)
# send approval confirmation email to admin
email_body = get_template("approve_comment").render(original=message)
mailer.send(from_email, "Re: " + subject, email_body)
return True