def debug(self):
ws_1 = set(self.batch_extract_feature().keys())
ST.remove_emoticon(self._train_xs)
ws_2 = set(self.batch_extract_feature().keys())
linfo('uni_bi_icon_feature_cnt: %s. no_icon: %s' % (len(ws_1), len(ws_2)))
rms = ws_2 - ws_1
for x in rms:
print x
def train_discret_model(self, **config):
linfo('begin train helper discret model: %s' % config)
if not config['emoticon']:
ST.remove_emoticon(self._train_xs)
if not config['parenthesis']:
ST.remove_parenthesis(self._train_xs)
self.txt2bags = {}
self.w2id = self.batch_extract_feature()
linfo('end train helper discret model')
def train_discret_model(self, **config):
linfo('begin train helper discret model: %s' % config)
if not config['emoticon']:
ST.remove_emoticon(self._train_xs)
if not config['parenthesis']:
ST.remove_parenthesis(self._train_xs)
self.txt2bags = {}
self.w2id = self.batch_extract_feature()
linfo('end train helper discret model')
def debug(self):
ws_1 = set(self.batch_extract_feature().keys())
ST.remove_emoticon(self._train_xs)
ws_2 = set(self.batch_extract_feature().keys())
linfo('uni_bi_icon_feature_cnt: %s. no_icon: %s' %
(len(ws_1), len(ws_2)))
rms = ws_2 - ws_1
for x in rms:
print x
def train(self):
self._train_xs, self._train_ys = ST.load_data(self._path)
if not self._emoticon:
ST.remove_emoticon(self._train_xs)
self.gram2gid = self._discretize_gram2gid()
X = self.build_sparse_X(self._train_xs)
self.clf.fit(X, self._train_ys)
self.real_test()
def train(self, icon=True, cross=False):
#word2cnt = BayesClassifier.Word2Cnt()
#txt = '今天天气就是棒[哈哈] [太阳] [飞起来]#'
#return
#self._load_data()
#self._replace_url(fill=True)
self._train_xs, self._train_ys = ST.load_data(self._train_path)
ST.replace_url(self._train_xs, fill=True)
if not icon:
ST.remove_emoticon(self._train_xs)
self._train(cross_validation=cross)
def train(self, icon=True, cross=False):
#word2cnt = BayesClassifier.Word2Cnt()
#txt = '今天天气就是棒[哈哈] [太阳] [飞起来]#'
#return
#self._load_data()
#self._replace_url(fill=True)
self._train_xs, self._train_ys = ST.load_data(self._train_path)
ST.replace_url(self._train_xs, fill=True)
if not icon:
ST.remove_emoticon(self._train_xs)
self._train(cross_validation=cross)
def train(self,cross_validation=False, fold_sz=10, test_path='../../test_data/tri_test_data'):
self._train_xs, self._train_ys = ST.load_data(self._path)
if not self._emoticon:
ST.remove_emoticon(self._train_xs)
self.gram2gid = self._discretize_gram2gid()
if cross_validation:
linfo('begin to cross train')
self._cross_train(fold_sz)
else:
classifier = self._train(self._train_xs, self._train_ys)
self._test_xs, self._test_ys = ST.load_data(test_path)
ST.replace_url(self._test_xs, fill='H')
ST.replace_target(self._test_xs, fill='T')
test_set = [(self._feature_encoding(txt), tag) for txt, tag in zip(self._test_xs, self._test_ys)]
linfo('maxent classifier precision: %.4f' % classify.accuracy(classifier, test_set))