def after_update(self, raw_post: Dict, values: SQLValuesToWrite, old_records: List[DataRecord], records: List[DataRecord]):
for old_record, record in zip(old_records, records):
manage_try_add = lambda column, op: ManageLog.add_by_post_changed(
self, column, op, POST_TYPES.TOPIC, values, old_record, record
)
manage_try_add_with_diff = lambda column, op: ManageLog.add_by_post_changed(
self, column, op, POST_TYPES.TOPIC, values, old_record, record, diff_func=diff
)
title_changed = manage_try_add('title', MOP.POST_TITLE_CHANGE) # 管理日志：标题编辑
content_changed = manage_try_add_with_diff('content', MOP.POST_CONTENT_CHANGE) # 管理日志：正文编辑
if title_changed or content_changed:
post_stats_do_edit(record['id'], record['user_id'])
Topic.update(edit_count=Topic.edit_count + 1).where(Topic.id == record['id']).execute()
manage_try_add('state', MOP.POST_STATE_CHANGE) # 管理日志：状态修改
manage_try_add('visible', MOP.POST_VISIBLE_CHANGE) # 管理日志：改变可见度
manage_try_add('awesome', MOP.TOPIC_AWESOME_CHANGE) # 管理日志：设置精华
manage_try_add('sticky_weight', MOP.TOPIC_STICKY_WEIGHT_CHANGE) # 管理日志：置顶权重
manage_try_add('weight', MOP.TOPIC_WEIGHT_CHANGE) # 管理日志：修改权重
# 管理日志：移动板块
if manage_try_add('board_id', MOP.TOPIC_BOARD_MOVE):
post_stats_topic_move(old_record['board_id'], record['board_id'], record['id'])
def get_summary(topic_path,
summary_size=100,
oracle="accept_reject",
summarizer="sume",
parser=None,
language="english",
rouge_dir="rouge/RELEASE-1.5.5/"):
# relativize the topic path!!!!
if topic_path.startswith("/"):
relative_path = re.search('^(/)(.*)$', topic_path).group(2)
else:
relative_path = topic_path
resolved_topic_path = path.normpath(
path.join(path.expanduser("~"), ".ukpsummarizer",
path.normpath(relative_path)))
topic = Topic(resolved_topic_path)
docs = topic.get_docs()
models = topic.get_models()
if summarizer == "sume":
sw = SumeWrap(language)
summary = sw(docs, summary_size)
return summary
elif summarizer == "custom_weights":
sw = SumeWrap(language)
return "no summary for summarizer type %s" % summarizer
def _msg_data(self, msg):
""" 统一返回格式 - 带上 comment, item
"""
data = Message._msg_data(msg)
if data['link_id']:
topic = Topic.get(data['link_id'])
topic = Topic._topic_data(topic)
data['topic'] = topic
tmp_user = model.user.get_user(user_id=topic['user_id']) # 消息来源用户
if not tmp_user:
return None
data['topic_user'] = {
'id': tmp_user['id'],
'nick': tmp_user['nick'],
'portrait': tmp_user['portrait'],
'type': tmp_user['user_type'],
} if tmp_user else {}
if data['from_uid']:
tmp_user = model.user.get_user(user_id=data['from_uid']) # 消息来源用户
if not tmp_user:
return None
data['from_user'] = {
'id': tmp_user['id'],
'nick': tmp_user['nick'],
'portrait': tmp_user['portrait'],
'type': tmp_user['user_type'],
} if tmp_user else {}
data['class'] = 'message'
return data
def get(self, id):
query = ("SELECT id, name, url FROM topic" "WHERE id=%s")
cursor = self.db.cursor()
cursor.execute(query, (id))
sql_result = cursor.fetchone()
cursor.close()
topic = Topic(sql_result[1], sql_result[2])
topic.id = sql_result[0]
return topic
def after_update(self, raw_post: Dict, values: SQLValuesToWrite,
old_records: List[DataRecord], records: List[DataRecord]):
for old_record, record in zip(old_records, records):
if 'content' in values:
# 管理日志：正文编辑
ManageLog.new(self.current_user, self.current_role,
POST_TYPES.TOPIC, record['id'],
record['user_id'], MOP.TOPIC_CONTENT_CHANGE,
None)
Topic.update(edit_count=Topic.edit_count +
1).where(Topic.id == record['id']).execute()
if 'title' in values:
# 管理日志：标题编辑
ManageLog.new(self.current_user, self.current_role,
POST_TYPES.TOPIC, record['id'],
record['user_id'], MOP.TOPIC_TITLE_CHANGE, None)
# 管理日志：改变状态
ManageLog.add_by_post_changed(self, 'state', MOP.POST_STATE_CHANGE,
POST_TYPES.TOPIC, values, old_record,
record)
# 管理日志：改变可见度
ManageLog.add_by_post_changed(self, 'visible',
MOP.POST_VISIBLE_CHANGE,
POST_TYPES.TOPIC, values, old_record,
record)
# 管理日志：移动板块
if ManageLog.add_by_post_changed(self, 'board_id',
MOP.TOPIC_BOARD_MOVE,
POST_TYPES.TOPIC, values,
old_record, record):
statistic_move_topic(old_record['board_id'],
record['board_id'], record['id'])
# 管理日志：设置精华
ManageLog.add_by_post_changed(self, 'awesome',
MOP.TOPIC_AWESOME_CHANGE,
POST_TYPES.TOPIC, values, old_record,
record)
# 管理日志：置顶权重
ManageLog.add_by_post_changed(self, 'sticky_weight',
MOP.TOPIC_STICKY_WEIGHT_CHANGE,
POST_TYPES.TOPIC, values, old_record,
record)
# 管理日志：修改权重
ManageLog.add_by_post_changed(self, 'weight',
MOP.TOPIC_WEIGHT_CHANGE,
POST_TYPES.TOPIC, values, old_record,
record)
def __init__(self, summary_file):
p, f = path.split(summary_file)
self.topic = Topic(path.normpath(path.join(p, "..")))
self.idx = None
for i, (fn, t) in enumerate(self.topic.get_models()):
if fn.startswith(summary_file):
self.idx = i
print(f, self.idx)
def get_paginated(self, page=1, items=20):
query = ("SELECT id, name, url FROM " "topic LIMIT %s OFFSET %s;")
cursor = self.db.cursor()
cursor.execute(query, (items, items * (page - 1)))
topics = cursor.fetchall()
cursor.close()
if len(topics) == 0:
return False
topic_models = []
for t in topics:
model = Topic(t[1], t[2])
model.id = t[0]
topic_models.append(model)
return topic_models
def check_topic(self, item_id):
if not item_id:
raise HTTPError(404)
item_id = int(item_id)
i = Topic.get(item_id)
if not i:
raise HTTPError(404, 'can not found item with id=%d' % item_id)
def get(self):
page = self.get_argument("p", "1")
count, query = Topic.get_list()
pagination_ret = pagination(count, query, config.TOPIC_PAGE_SIZE, page)
self.render(
"forum/recent.html", nav="index", topics_count=count, pagination=pagination_ret, page_url=self.request.path
)
def lasted_topic(cls, board):
from model.topic import Topic
try:
return Topic.select().where(Topic.board == board)\
.order_by(Topic.time.desc()).get()
except Topic.DoesNotExist:
return
def test_build_formatted_list_of_topics(self):
expectedTopic = Topic("expected", "", "", "")
service.add(expectedTopic.title)
topics = service.all()
assert_that(topics).contains(expectedTopic)
def post(self):
title = self.get_argument("title", "").strip()
board = self.get_argument("board", None)
content = self.get_argument("content", "").strip()
if title and config.TITLE_LENGTH_MIN <= len(title) <= config.TITLE_LENGTH_MAX and Board.exists_by_pk(board):
t = Topic.new(title, self.current_user() or 0, board, content)
self.redirect(url_for("topic", t.id))
else:
# 非标准提交，不用过于客气
self.redirect(url_for("topic_new"))
def post_stats_do_comment(related_type, related_id, comment_id):
# 需要同时更新被评论对象的数字和最后评论id
def func(update, where):
update['last_comment_id'] = comment_id
post_stats_incr(PostStats.comment_count, related_id, 1, cb=func)
# 如果被评论的是文章，需要更新板块数据
if related_type == POST_TYPES.TOPIC:
t = Topic.get_by_pk(related_id)
post_stats_incr(PostStats.comment_count, t.board_id, 1, cb=func)
def get(self, topic_id):
topic = Topic.get_by_pk(topic_id)
if topic:
topic.view_count_inc()
count, user_topics = Topic.get_list_by_user(topic.user)
user_topics = user_topics.limit(10)
follow = JsDict()
if self.current_user():
follow.topic = Follow.exists(OBJECT_TYPES.TOPIC, topic_id, self.current_user())
follow.author = Follow.exists(OBJECT_TYPES.USER, topic.user.id, self.current_user())
self.render(
"forum/topic.html",
nav="index",
page_title=page_title(topic.title, topic.board.title, "社区"),
topic=topic,
user_topics=user_topics,
follow=follow,
)
else:
self.write_error(404)
def get(self, username):
user = User.get_by_username(username)
if user:
page = self.get_argument('p', '1')
count, query = Topic.get_list_by_user(user)
pagination_ret = pagination(count, query, config.TOPIC_PAGE_SIZE, page)
self.render('user/user_page.html', user=user, tab={'user_topic_page': 'active'}, count=count,
avatar_html=avatar_generate(username, user.avatar_color, 167),
pagination=pagination_ret, page_url=self.request.path)
else:
self.write_error(404)
def get(self, topic_id):
topic = Topic.get_by_pk(topic_id)
if not self.topic_check(topic):
return
self.render(
"forum/topic_edit.html",
nav="index",
page_title=page_title("编辑主题 - %s" % topic.title, topic.board.title, "社区"),
topic=topic,
boards=Board.get_list(),
)
def get_topics():
topics = TopicRepository(db)
r = requests.get("https://forum.ubuntuusers.de/")
content = r.content.decode('utf8')
reg = "<tr\\sclass=\"entry.+?<a.{0,100}href=\"(.{0,100}?)\".+?>(.+?)<"
topic_ex = re.compile(reg, re.MULTILINE | re.DOTALL)
matches = topic_ex.finditer(content)
for m in matches:
topic = Topic(m.group(2), m.group(1))
topics.add(topic)
print("Added Topic: {topic}".format(topic=topic.name))
return
async def before_insert(self, raw_post: Dict,
values_lst: List[SQLValuesToWrite]):
values = values_lst[0]
form = TopicNewForm(**raw_post)
if not form.validate():
return self.finish(RETCODE.FAILED, form.errors)
values['user_id'] = self.current_user.id
# 以下通用
if not config.POST_ID_GENERATOR == config.AutoGenerator:
values['id'] = config.POST_ID_GENERATOR().digest()
values['time'] = int(time.time())
values['weight'] = Topic.weight_gen()
def getTopic(topicId):
from model.topic import Topic
c = get_db()
t = text('SELECT topic_name FROM topic WHERE topic_id = :topicId')
results = c.execute(t, topicId = topicId)
for row in results:
name = row['topic_name']
comics = []
tC = text('SELECT comic_id from topic_comic WHERE topic_id = :topicId')
resultsComic = c.execute(tC, topicId=topicId)
for rowCom in resultsComic:
comics.append(getComic(rowCom['comic_id']))
return Topic(topicId, name, comics)
def get_object(cls, related_type, related_id):
from model.user import User
from model.topic import Topic
from model.wiki import WikiItem
if related_type == OBJECT_TYPES.USER:
u = User.get_by_pk(related_id)
if u: return u
elif related_type == OBJECT_TYPES.TOPIC:
t = Topic.get_by_pk(related_id)
if t: return t
elif related_type == OBJECT_TYPES.WIKI:
w = WikiItem.get_by_pk(related_id)
if w: return w
def post(self, topic_id):
topic = Topic.get_by_pk(topic_id)
if not self.topic_check(topic):
return
title = self.get_argument("title", "").strip()
content = self.get_argument("content", "").strip()
if title and config.TITLE_LENGTH_MIN <= len(title) <= config.TITLE_LENGTH_MAX:
topic.edit({"title": title, "content": content}, self.current_user())
self.messages.success("编辑成功")
self.redirect(url_for("topic", topic.id))
else:
# 非标准提交，不用过于客气
self.redirect(url_for("topic_new"))
def get(self, **kwargs):
uid = kwargs['uid']
mode = kwargs['mode']
item_id = int(kwargs['item_id'])
user = hqby.user.get_user_info(uid=uid)
if not user:
raise HTTPError(403, 'can not found this user')
if mode not in ['topic','comment']:
raise HTTPError(400,'type not topic or comment')
if mode == 'topic':
#如果赞的是心得 先从缓存中找赞 没赞过的话就新建赞
key = "0"+uid+str(item_id)
zan = model.like.get_zan(uid,item_id,0,key)
if not zan:
#更新心得里的like_num 新建赞 新建msg
item = Topic.update_topic(item_id, True, False)
item_data = Topic._topic_data(item)
zan = {'user_id':str(uid),'item_id':item_data['topic_id'],'item_type':0}
zan = model.like.create_like(zan)
msg = self.create_msg('TL', user, item_data)
self.write({'status':1,'type':mode,'like_num':item_data['like_num'],'uid':uid,'like_id':zan['id'],'msg':msg})
else:
self.write({'status':0,'type':mode,'msg':'already liked'})
elif mode == 'comment':
#如果赞的是评论 先从缓存中找赞 没赞过的话就新建赞
key = "1"+uid+str(item_id)
zan = model.like.get_zan(uid,item_id,1,key)
if not zan:
item = Comment.update_comment(item_id,True)
zan = {'user_id':str(uid),'item_id':item['id'],'item_type':1}
zan = model.like.create_like(zan)
msg = self.create_msg('CL', user, item)
self.write({'status':1,'type':mode,'like_num':item['like_num'],'uid':uid,'like_id':zan['id'],'msg':msg})
else:
self.write({'status':0,'type':mode,'msg':'already liked'})
return
def get(self, board_id):
board = Board.get_by_pk(board_id)
if board:
page = self.get_argument("p", "1")
count, query = Topic.get_list_by_board(board)
pagination_ret = pagination(count, query, config.TOPIC_PAGE_SIZE, page)
self.render(
"forum/board.html",
nav="index",
page_title=page_title(board.title, "社区"),
board=board,
topics_count=count,
pagination=pagination_ret,
page_url=self.request.path,
)
else:
self.write_error(404)
def save_to_topics(self, rankings, file_name):
"""
this method takes a tuple of rankings: (score, sentence) then saves it to the topic model
"""
topic = Topic(file_name)
foundCore = False
for i in range(len(rankings)):
score = rankings[i][0]
sent = rankings[i][1]
if not foundCore:
if topic.set_idea(sent, score):
foundCore = True
continue
topic.add_supporting_idea(sent, score)
topic.save_to_db()
return topic
class Summary(object):
def __init__(self, summary_file):
p, f = path.split(summary_file)
self.topic = Topic(path.normpath(path.join(p, "..")))
self.idx = None
for i, (fn, t) in enumerate(self.topic.get_models()):
if fn.startswith(summary_file):
self.idx = i
print(f, self.idx)
def get_index(self):
return self.idx
def get_topic(self):
return self.topic
def es_update_topic(id):
post: Topic = Topic.get_by_id(id)
if not post: return
u: User = User.get_by_id(post.user_id)
if not u: return
body = get_post_base_body(post)
body.update({
'user_nickname': u.nickname,
'content': post.content,
'brief': post.content[:100]
})
es.index(
index=INDEX_NAME,
doc_type="doc",
id=to_hex(post.id),
body=body
)
def statistic_add_comment(related_type, related_id, comment_id):
# 关于原子更新
# http://docs.peewee-orm.com/en/latest/peewee/querying.html#atomic-updates
# s: Statistic = cls.get_by_pk(related_id)
Statistic.update(last_comment_id=comment_id, comment_count=Statistic.comment_count + 1)\
.where(Statistic.id == related_id)\
.execute()
Statistic24h.update(comment_count=Statistic24h.comment_count + 1)\
.where(Statistic24h.id == related_id)\
.execute()
if related_type == POST_TYPES.TOPIC:
t = Topic.get_by_pk(related_id)
Statistic.update(last_comment_id=comment_id, comment_count=Statistic.comment_count + 1)\
.where(Statistic.id == t.board_id)\
.execute()
Statistic24h.update(comment_count=Statistic24h.comment_count + 1)\
.where(Statistic24h.id == t.board_id)\
.execute()
def update_all(reset=False):
if reset:
try:
es.indices.delete(index=INDEX_NAME)
except elasticsearch.exceptions.NotFoundError:
pass
create_index()
for i in Topic.select(Topic.id):
print('topic', to_hex(i.id))
es_update_topic(i.id)
for i in WikiArticle.select(WikiArticle.id):
print('wiki', to_hex(i.id))
es_update_wiki(i.id)
for i in Comment.select(Comment.id):
print('comment', to_hex(i.id))
es_update_comment(i.id)
def statistic_add_topic_click(topic_id, board_id=None):
Statistic.update(click_count=Statistic.click_count + 1)\
.where(Statistic.id == topic_id)\
.execute()
Statistic24h.update(click_count=Statistic24h.click_count + 1)\
.where(Statistic24h.id == topic_id)\
.execute()
if not board_id:
t = Topic.get_by_pk(topic_id)
board_id = t.board_id
Statistic.update(click_count=Statistic.click_count + 1)\
.where(Statistic.id == board_id)\
.execute()
Statistic24h.update(click_count=Statistic24h.click_count + 1)\
.where(Statistic24h.id == board_id)\
.execute()
def get_post(cls, related_type, related_id):
from model.user import User
from model.topic import Topic
from model.wiki import WikiItem
if type(related_id) == POST_ID_GENERATOR:
related_id = related_id.to_bin()
if type(related_type) == str:
related_type = int(related_type)
if related_type == POST_TYPES.USER:
u = User.get_by_pk(related_id)
if u: return u
elif related_type == POST_TYPES.TOPIC:
t = Topic.get_by_pk(related_id)
if t: return t
elif related_type == POST_TYPES.WIKI:
w = WikiItem.get_by_pk(related_id)
if w: return w
def topic_count(cls, board, start_time=0):
from model.topic import Topic, TOPIC_STATE
return Topic.select().where(Topic.time>=start_time, Topic.board == board).count()
# is_dataset
d = DataSet(f)
# unroll to get topics
for t in d.get_topics():
for (mf, mt) in t.get_models():
mf = path.normpath(mf)
pref = path.commonprefix([mf, iobasedir])
tn = mf[len(pref) + 1:]
print("shortened:", tn)
queue.append(mf)
# topics.append([t.get_name for t in d.get_topics()])
elif path.exists(path.join(f, "task.json")):
# is topic
t = Topic(f)
for (mf, mt) in t.get_models():
mf = path.normpath(mf)
pref = path.commonprefix([mf, iobasedir])
tn = mf[len(pref) + 1:]
print("shortened:", tn)
queue.append(mf)
elif path.exists(path.join(f, "..", "..", "task.json")) \
and path.exists(f):
# should be model
queue.append(f)
else:
raise BaseException("Invalid file given.", f, " is neither a dataset nor a topic nor a model.")
if args.max_models:
queue = queue[:args.max_models]
def add(self, topic_title):
self.topics.append(Topic(topic_title, "", "", "", ""))
return self.topics
def run(self, topic_path, size=None, max_iteration_count=25):
log = logging.getLogger("GridSearch")
interpretation_types = [
'SimpleNgramFeedbackGraph',
'WordEmbeddingGaussianFeedbackGraph',
'BaselineFeedbackStore',
'WordEmbeddingRandomWalkDiffusionFeedbackGraph',
# 'WordEmbeddingEgoPrFeedbackGraph',
# 'PageRankFeedbackGraph',
]
random.shuffle(interpretation_types)
if topic_path.startswith("/"):
relative_path = re.search('^(/)(.*)$', topic_path).group(2)
else:
relative_path = topic_path
topic = Topic(path.join(self.iobasedir, path.normpath(relative_path)))
embeddings = self.__get_embeddings__(topic.get_language())
run_id = hashlib.sha224(topic_path).hexdigest()
outputdir = path.join(self.scores_dir, run_id)
try:
os.mkdir(outputdir)
except:
pass
concept_embedder = ConceptEmbedder(embeddings)
for itype in interpretation_types:
if itype == 'WordEmbeddingGaussianFeedbackGraph':
mass_reject = [4.0, 1.0, 0.0, -1.0, -4.0]
mass_accept = [4.0, 1.0, 0.0, -1.0, -4.0]
iterations_accept = [16, 128, 1024]
iterations_reject = [2, 4, 8, 16, 64]
cut_off_threshold = [0.998, 0.98, 0.9, 0.6, 0.4]
combinations = list(
itertools.product(mass_reject, mass_accept,
iterations_accept, iterations_reject,
cut_off_threshold))
random.shuffle(combinations)
for (mr, ma, ia, ir, co) in combinations:
log.info(
"WordEmbeddingGaussianFeedbackGraph: %s %s %s %s %s" %
(mr, ma, ia, ir, co))
g = WordEmbeddingGaussianFeedbackGraph(
concept_embedder,
cut_off_threshold=co,
mass_reject=mr,
mass_accept=ma,
iterations_reject=ir,
iterations_accept=ia)
sir = SingleTopicRunner(self.iobasedir,
self.rouge,
scores_dir=outputdir)
sir.run(topic_path,
size,
feedbackstore=g,
summarizer="PROPAGATION",
preload_embeddings=embeddings)
elif itype == 'WordEmbeddingRandomWalkDiffusionFeedbackGraph':
mass_reject = [4.0, 1.0, 0.0, -1.0, -4.0]
mass_accept = [4.0, 1.0, 0.0, -1.0, -4.0]
iterations_accept = [128, 1024, 10000]
iterations_reject = [64, 200, 5000]
cut_off_threshold = [0.998, 0.98, 0.9, 0.6, 0.4]
propagation_abort_threshold = [0.01, 0.1, 0.25, 0.5, 0.75, 0.9]
combinations = list(
itertools.product(mass_reject, mass_accept,
iterations_accept, iterations_reject,
cut_off_threshold,
propagation_abort_threshold))
random.shuffle(combinations)
for (mr, ma, ia, ir, co, pat) in combinations:
log.info(
"WordEmbeddingRandomWalkDiffusionFeedbackGraph: %s %s %s %s %s %s"
% (mr, ma, ia, ir, co, pat))
g = WordEmbeddingRandomWalkDiffusionFeedbackGraph(
concept_embedder,
mass_accept=ma,
mass_reject=mr,
iterations_accept=ia,
iterations_reject=ir,
cut_off_threshold=co,
propagation_abort_threshold=pat)
sir = SingleTopicRunner(self.iobasedir,
self.rouge,
scores_dir=outputdir)
sir.run(topic_path,
size,
feedbackstore=g,
summarizer="PROPAGATION",
preload_embeddings=embeddings)
elif itype == "BaselineFeedbackStore":
log.info("BaselineFeedbackStore")
sir = SingleTopicRunner(self.iobasedir,
self.rouge,
scores_dir=outputdir)
sir.run(topic_path,
size,
summarizer="PROPAGATION",
preload_embeddings=embeddings)
elif itype == "PageRankFeedbackGraph":
log.warning("interpretationtype not implementend. type: %s" %
(itype))
elif itype == "SimpleNgramFeedbackGraph":
window_size = [2, 3, 4, 5]
factor_rejects = [1, 0, 0.05, 0.25, 0.5, 2, 4, 8]
factor_accepts = [1, 0, 0.05, 0.25, 0.5, 2, 4, 8]
stemmer = SnowballStemmer(topic.get_language())
combinations = list(
itertools.product(window_size, factor_rejects,
factor_accepts))
random.shuffle(combinations)
for (ws, fr, fa) in combinations:
log.info(
"SimpleNgramFeedbackGraph: (ws %s, fr %s, fa %s)" %
(ws, fr, fa))
g = SimpleNgramFeedbackGraph(stemmer,
topic.get_language(),
N=ws,
factor_reject=fr,
factor_accept=fa)
sir = SingleTopicRunner(self.iobasedir,
self.rouge,
scores_dir=outputdir)
sir.run(topic_path,
size,
feedbackstore=g,
summarizer="PROPAGATION",
preload_embeddings=embeddings)
else:
log.warning("Got wrong interpretationtype. ignoring type %s" %
(itype))
def get_topics(self):
for file_name in os.listdir(self.root):
topic_location = path.normpath(path.join(self.root, file_name))
if not os.path.isdir(topic_location):
continue
yield Topic(topic_location)
def post(self, **kwargs):
"""
新建一个评论 需要用户登录
"""
token = self._get_token()
uid = token['_id']
user = hqby.user.get_user_info(uid=uid)
if not user:
raise HTTPError(403, 'can not found this user')
try:
params = json_decode(self.request.body)
except ValueError:
params = self.get_form()
comment_type = params.get('comment_type', False)
self.check_comment_type(comment_type)
self.check_topic(params['topic_id'])
params['user_id'] = uid
# 添加一个新的评论
comm = self.create_comment(params,comment_type)
# self.write('haha = %s'%comm)
#return
data = self._comment_data(comm)
data['is_liked'] = 0
topic = Topic.update_topic(data['topic_id'],False,True)
topic_data = Topic._topic_data(topic)
if data['to_user_id']:
to_uid = data['to_user_id']
msg_type = 'CC'
content = user['nick'] + '回复了你的评论 '
else:
to_uid = topic_data['user_id']
msg_type = 'TC'
content = user['nick'] + "评论了你的心得 " + " \"%s\""%topic_data['content']
if uid != to_uid:
msg = Message.set_message(to_uid, msg_type, uid, content, topic_data['topic_id']) #link_id 是topic的id
msg = Message._msg_data(msg)
else:
msg = 'you reply yourself'
#给被回复者发送通知
baidu_apiKey = baidu_push_configs['apiKey']
baidu_secretKey = baidu_push_configs['secretKey']
bind_info = hqby.user.get_bind_info(to_uid)
baidu_uid, baidu_cid = bind_info.get('baidu_uid'), bind_info.get('baidu_cid')
if baidu_uid and baidu_cid and uid != to_uid:
message = {
'title': '读经',
'description': '%s回复了你，快去看看吧' % user['nick'].encode('utf8'),
'open_type': 2,
"aps": {
"alert": '%s回复了你，快去看看吧' % user['nick'].encode('utf8'),
"sound":"",
"badge":0
},
}
message = json.dumps(message)
message_key = "sys"
c = Channel(baidu_apiKey, baidu_secretKey, arr_curlOpts=dict(TIMEOUT=3, CONNECTTIMEOUT=5))
push_type = 1 # 1-单个人, 2-一群人, 3-全部人
optional = dict()
optional[Channel.USER_ID] = baidu_uid
optional[Channel.CHANNEL_ID] = int(baidu_cid)
optional[Channel.MESSAGE_TYPE] = 1 # 0-消息, 1-通知
optional['device_types'] = [3, 4] # 4-ios, 3-安卓, 5-wp设备, 2-pc, 1-浏览器
optional['deploy_status'] = 1 if configs['debug'] else 2 # 1-开发, 2-生产
#job = c.pushMessage(push_type, message, message_key, optional)
job = rq_client.default_queue.enqueue(c.pushMessage, push_type, message, message_key, optional)
#logging.info('log for baidu pusher: %s', str(job))
self.write({'comment':data,'topic':topic_data,'msg':msg})
self.set_status(200)
self.set_header('Content-Type', self._ct('json'))
def post_stats_add_topic_click(topic_id, board_id=None):
if not board_id:
t = Topic.get_by_pk(topic_id)
board_id = t.board_id
post_stats_incr(PostStats.click_count, topic_id)
post_stats_incr(PostStats.click_count, board_id)
def run(self,
topic_path,
size=None,
summarizer="SUME",
summary_idx=None,
parser=None,
oracle="accept",
feedback_log=None,
propagation=False,
max_iteration_count=10,
preload_embeddings=None,
feedbackstore=None,
override_results_files=False,
num_clusters=8):
log = logging.getLogger("SingleTopicRunner")
sf = None # just for the sake of being able to run without simulated feedback...
self.tlog.debug("SingleTopicRunner started")
# relativize the topic path!
if type(topic_path) is Topic:
topic = topic_path
else:
if topic_path.startswith("/"):
relative_path = re.search('^(/)(.*)$', topic_path).group(2)
else:
relative_path = topic_path
topic = Topic(
path.join(self.iobasedir, path.normpath(relative_path)))
language = topic.get_language()
docs = topic.get_docs()
summaries = topic.get_models()
flightrecorder = get_flightrecorder_from_file(feedback_log)
preceding_size = len(
flightrecorder.records
) # the number of iterations that happened due to the provided feedback_log
embeddings = None
"""
if preload_embeddings:
embeddings_path = path.normpath(path.join(self.iobasedir, "embeddings"))
embeddings = load_w2v_embeddings(embeddings_path, language, 'active_learning')
else:
embeddings = preload_embeddings
"""
if summary_idx is not None:
summaries = [summaries[summary_idx]]
if size is None:
use_size = topic.get_summary_size()
else:
use_size = size
clusters_path = path.join(self.iobasedir, 'clustering',
'{}'.format(num_clusters))
#print(clusters_path)
#clusters = get_clusters(clusters_path, topic.docs_dir)
if summarizer == "SUME":
sw = SumeWrap(language)
summary = sw(docs, use_size)
outputfilecontents = {
"summary": summary,
"type": summarizer,
"info_data": []
}
json_content = json.dumps(outputfilecontents)
if self.out is not None:
log.info("writing output to %s" % (self.out))
write_to_file(json_content, self.out)
write_to_file(
json_content,
path.normpath(
path.expanduser(
path.join(self.iobasedir, "tmp", "tmp.json"))))
elif summarizer == "UPPER_BOUND":
ub_summary = load_ub_summary(language,
docs,
summaries,
use_size,
base_dir=self.iobasedir)
summary = '\n'.join(ub_summary)
outputfilecontents = {
"summary": summary,
"type": summarizer,
"info_data": []
}
json_content = json.dumps(outputfilecontents)
if self.out is not None:
log.info("writing output to %s" % (self.out))
write_to_file(json_content, self.out)
write_to_file(
json_content,
path.normpath(
path.expanduser(
path.join(self.iobasedir, "tmp", "tmp.json"))))
elif summarizer == "PROPAGATION":
#UB considering all the summaries
ub_summary = load_ub_summary(language,
docs,
summaries,
use_size,
base_dir=self.iobasedir)
summary = '\n'.join(ub_summary)
ub_scores = self.rouge(summary, summaries, use_size)
log.debug(
"UB scores: R1:%s R2:%s SU4:%s" %
(str(ub_scores[0]), str(ub_scores[1]), str(ub_scores[2])))
ref_summ = random.choice(summaries)
parse_info = []
#parse_info = topic.get_parse_info(summaries.index(ref_summ))
# initialize the Algorithm.
run_config = dict()
run_config['rank_subset'] = True
run_config['relative_k'] = True
run_config['dynamic_k'] = False
for flag in ['adaptive_sampling', 'strategy']:
run_config[flag] = False
r = 0
clusters = None
log.info("recording k_size in summarize %f", self.k)
#TODO: Added summaries instead of one single summary
sf = SimulatedFeedback(
language,
self.rouge,
embeddings=None, #TODO: embeddings
docs=docs,
models=summaries,
summary_length=use_size,
oracle_type=oracle,
ub_score=ub_scores,
ub_summary=ub_summary,
parser_type=parser,
flightrecorder=flightrecorder,
feedbackstore=feedbackstore,
parse_info=parse_info,
run_config=run_config,
k=self.k,
adaptive_window_size=r,
clusters=clusters)
if sf.embeddings is None or sf.embeddings == {}:
embe_var = "none",
else:
if sf.embeddings.embedding_variant is None:
embe_var = "none"
else:
embe_var = sf.embeddings.embedding_variant
if feedbackstore is None:
cfg = {"type": "Unconfigured default"}
else:
cfg = feedbackstore.get_config()
rs = []
for p, t in [ref_summ]:
rs.append({"name": os.path.split(p)[1], "text": t})
run_id_string = "%s-%s-%s-%s-%s-%s-%s-%s" % (
oracle, summarizer, parser, embe_var, topic.get_dataset(),
topic.get_name(), [item["name"]
for item in rs], json.dumps(cfg))
run_id = hashlib.sha224(run_id_string).hexdigest()
filename = path.join(self.scores_storage_path,
"result-%s.json" % (run_id))
if (os.path.exists(filename) and self.out is None
and self.override_results_switch is False):
log.info(
"Skipping run_id '%s' because the result file does already exist. config: %s"
% (run_id, run_id_string))
return
else:
log.info("Doing %s iterations for run_id '%s'\n %s" %
(max_iteration_count, run_id, run_id_string))
write_to_file("", filename)
summary, confirmatory_summary, exploratory_summary = sf.run_full_simulation(
max_iteration_count=max_iteration_count)
recommendations, recom_sentences = sf.get_recommendations()
derived_records = []
# construct table-like array of feedbacks per iteration.
for i, record in enumerate(sf.flight_recorder.records):
for accept in record.accept:
derived_records.append({
"iteration": i,
"concept": accept,
"value": "accept"
})
for reject in record.reject:
derived_records.append({
"iteration": i,
"concept": reject,
"value": "reject"
})
for implicit_reject in record.implicit_reject:
derived_records.append({
"iteration": i,
"concept": implicit_reject,
"value": "implicit_reject"
})
for item in recommendations:
derived_records.append({
"iteration":
-1,
"concept":
item,
"value":
"recommendation",
"weight":
sf.summarizer.weights.get(item, 0.0),
"uncertainity":
sf.svm_uncertainity.get(item, -1.0)
})
result = {
"config_run_id": run_id,
"config_oracle_type": oracle,
"config_summarizer_type": summarizer,
"config_parse_type": str(parser),
#"config_wordembeddings": emb_var,
"config_feedbackstore": sf.feedbackstore.get_config(),
"config_feedback_interpretation": {},
"config_concept_recommendation": {},
"dataset": topic.get_dataset(),
"topic": topic.get_name(),
"models": rs,
"model_rougescores": {
"iteration": -1,
"ROUGE-1 R score": ub_scores[0],
"ROUGE-2 R score": ub_scores[1],
"ROUGE-SU* R score": ub_scores[2],
"accepted": [],
"accept_count": 0,
"rejected": [],
"reject_count": 0,
"summary": ub_summary
},
"result_summary": summary,
"result_rougescores": sf.log_sir_info_data,
"log_feedbacks": derived_records
}
r2 = [{
"iteration": i,
"summary": sf.log_info_data[i]
} for i in range(len(sf.flight_recorder.records))]
log.debug(
"records: %s, infos %s, diff: %s" %
(len(sf.flight_recorder.records), len(sf.log_info_data),
len(sf.flight_recorder.records) - len(sf.log_info_data)))
write_to_file(json.dumps(result), filename)
log.info("Writing results to %s" % (filename))
df = pd.DataFrame(derived_records)
filename = path.join(self.scores_storage_path,
"flightrecorder-%s.csv" % (run_id))
log.info("saving flightrecorder to %s with run_id %s" %
(filename, run_id))
df.to_csv(filename, encoding="UTF-8")
write_to_file(
json.dumps(sf.new_debug_weights_history),
path.join(
self.scores_storage_path,
"weightshistory-%s-%s-%s-%s.json" %
(topic.get_dataset(), topic.get_name(), summarizer,
run_id)))
log.info("Writing weights history to %s" % (filename))
weights_hist = pd.DataFrame(sf.new_debug_weights_history)
filename = path.join(self.scores_storage_path,
"weightshistory-%s.csv" % (run_id))
weights_hist.to_csv(filename, encoding="UTF-8")
log.debug("----------------------------------------------")
log.debug(summary)
log.debug(sf.log_info_data[-1])
log.debug("----------------------------------------------")
if self.pickle_store is not None:
# Pickle dictionary using protocol 0.
print('Pickle in file %s' % self.pickle_store)
self.pickle_write(sf, self.pickle_store, log)
json_content = self.write_summarize_output_json(
sf, confirmatory_summary, derived_records, log,
recom_sentences, result, run_id, summarizer, summary,
self.pickle_store)
# write_to_file(json_content, path.normpath(path.expanduser(path.join(self.iobasedir, "tmp", "tmp.json"))))
else:
raise BaseException("You should tell which summarizer to use")
if sf is not None:
write_details_file([sf.log_info_data],
path.join(self.iobasedir, "tmp", "tmp.csv"))
self.tlog.debug("SingleTopicRunner finished")
def from_dict(old):
args = old
args['student'] = Student.get(id=args['student']['id'])
args['topic'] = Topic.get(id=args['topic']['id'])
return args
