def step1_cal():
"""第一步计算，获取子事件特征词，新文本与特征词匹配分类
"""
print '[%s] ' % ts2datetime(int(time.time(
))), 'event ', eventid, ' %s start step1' % ts2datetime(timestamp)
if initializing:
# 若话题需要做初始聚类，获取话题开始时间之前的文本
results = event.getInitialInfos()
else:
# 若话题已做完初始聚类，获取话题最新一小时的文本
results = event.getInfos(timestamp - 3600, timestamp)
if now_hour == 0:
# 如果不是在做初始化，24时的时候, 一定把当天（大于或等于0时小于24时）产生的簇（非其他簇）下的文本重新做一下匹配, 同时删除这些簇
temp_subeventids = event.getTodayCreatSubeventIds()
temp_infos = event.getTodayCreatSubeventInfos()
event.remove_subevents(temp_subeventids)
results.extend(temp_infos)
print eventid, ' before classify: ', len(results)
# 获取子事件
subevents = event.getSubEvents()
labels_list = []
feature_words_list = []
for subevent in subevents:
subeventid = subevent["_id"]
feature = Feature(subeventid)
# 获取每个子事件最新的特征词
fwords = feature.get_newest()
feature_words_list.append(fwords)
labels_list.append(subeventid)
for r in results:
text = (r['title'] + r['content168']).encode('utf-8')
feature_words_inputs = []
for fwords in feature_words_list:
wcdict = dict()
for w, c in fwords.iteritems():
if isinstance(w, unicode):
w = w.encode('utf-8')
wcdict[w] = c
feature_words_inputs.append(wcdict)
# 单条文本与各子事件的特征词进行匹配，得到每条文本的簇标签
label = subevent_classifier(text, labels_list,
feature_words_inputs)
if label == "other":
label = event.getOtherSubEventID()
news = News(r["_id"], event.id)
news.update_news_subeventid(label)
print '[%s] ' % ts2datetime(int(time.time(
))), 'event ', eventid, ' %s end step1' % ts2datetime(timestamp)
def step1_cal():
"""第一步计算，获取子事件特征词，新文本与特征词匹配分类
"""
print '[%s] ' % ts2datetime(int(time.time())), 'event ', eventid, ' %s start step1' % ts2datetime(timestamp)
if initializing:
# 若话题需要做初始聚类，获取话题开始时间之前的文本
results = event.getInitialInfos()
else:
# 若话题已做完初始聚类，获取话题最新一小时的文本
results = event.getInfos(timestamp - 3600, timestamp)
if now_hour == 0:
# 如果不是在做初始化，24时的时候, 一定把当天（大于或等于0时小于24时）产生的簇（非其他簇）下的文本重新做一下匹配, 同时删除这些簇
temp_subeventids = event.getTodayCreatSubeventIds()
temp_infos = event.getTodayCreatSubeventInfos()
event.remove_subevents(temp_subeventids)
results.extend(temp_infos)
print eventid, ' before classify: ', len(results)
# 获取子事件
subevents = event.getSubEvents()
labels_list = []
feature_words_list = []
for subevent in subevents:
subeventid = subevent["_id"]
feature = Feature(subeventid)
# 获取每个子事件最新的特征词
fwords = feature.get_newest()
feature_words_list.append(fwords)
labels_list.append(subeventid)
for r in results:
text = (r['title'] + r['content168']).encode('utf-8')
feature_words_inputs = []
for fwords in feature_words_list:
wcdict = dict()
for w, c in fwords.iteritems():
if isinstance(w, unicode):
w = w.encode('utf-8')
wcdict[w] = c
feature_words_inputs.append(wcdict)
# 单条文本与各子事件的特征词进行匹配，得到每条文本的簇标签
label = subevent_classifier(text, labels_list, feature_words_inputs)
if label == "other":
label = event.getOtherSubEventID()
news = News(r["_id"], event.id)
news.update_news_subeventid(label)
print '[%s] ' % ts2datetime(int(time.time())), 'event ', eventid, ' %s end step1' % ts2datetime(timestamp)
def test_subevent_classifier():
labels_list = []
feature_words_inputs = []
subevents = event.getSubEvents()
for subevent in subevents:
subeventid = subevent["_id"]
if subeventid != "575612b6-a26f-4df9-a2de-01c85cae56a2":
labels_list.append(subeventid)
feature = Feature(subeventid)
feature_words = feature.get_newest()
new_feature_words = dict()
for k, v in feature_words.iteritems():
new_feature_words[k.encode('utf-8')] = v
feature_words_inputs.append(new_feature_words)
news_id = "http://news.xinhuanet.com/comments/2014-11/03/c_1113084515.htm"
news = News(news_id, event.id)
ns = news.get_news_info()
text = ns['title'].encode('utf-8') + ns['content168'].encode('utf-8')
label = subevent_classifier(text, labels_list, feature_words_inputs)
print label
