def real_test(self):
self._test_xs, self._test_ys = ST.load_data(self.test_path)
ST.replace_url(self._test_xs, fill='H')
ST.replace_target(self._test_xs, fill='T')
#x_y = [(self.discret_txt(txt), y) for txt, y in zip(self._test_xs, self._test_ys)]
test_mat = self.build_sparse_X(self._test_xs)
self.accuracy(test_mat, self._test_ys)
def format_test(self, emoticon=True, parenthesis=True):
test_path='../test_data/%s_test_data' % self.classifier_type
self._test_xs, self._test_ys = ST.load_data(test_path)
linfo('begin preprocess test data, then sparse')
self._raw_test_xs, self._test_xs = ST.preprocess(self._test_xs)
#ST.replace_url(self._test_xs, fill='H')
#ST.replace_target(self._test_xs, fill='T')
self._test_ys = map(lambda x:self.tag2index[x], self._test_ys)
self.format_sparse(self._test_xs, self._test_ys, '%s/test_data/%s%s_sparse_test_data_%s' % (project_dir, self.flag_prefix, self.classifier_type, 'icon' if emoticon else 'no_icon'))
def train(self,cross_validation=False, fold_sz=10, test_path='../../test_data/tri_test_data'):
self._train_xs, self._train_ys = ST.load_data(self._path)
if not self._emoticon:
ST.remove_emoticon(self._train_xs)
self.gram2gid = self._discretize_gram2gid()
if cross_validation:
linfo('begin to cross train')
self._cross_train(fold_sz)
else:
classifier = self._train(self._train_xs, self._train_ys)
self._test_xs, self._test_ys = ST.load_data(test_path)
ST.replace_url(self._test_xs, fill='H')
ST.replace_target(self._test_xs, fill='T')
test_set = [(self._feature_encoding(txt), tag) for txt, tag in zip(self._test_xs, self._test_ys)]
linfo('maxent classifier precision: %.4f' % classify.accuracy(classifier, test_set))
def train(self):
self._train_xs, self._train_ys = ST.load_data(self._path)
if not self._emoticon:
ST.remove_emoticon(self._train_xs)
self.gram2gid = self._discretize_gram2gid()
X = self.build_sparse_X(self._train_xs)
self.clf.fit(X, self._train_ys)
self.real_test()
def train(self, icon=True, cross=False):
#word2cnt = BayesClassifier.Word2Cnt()
#txt = '今天天气就是棒[哈哈] [太阳] [飞起来]#'
#return
#self._load_data()
#self._replace_url(fill=True)
self._train_xs, self._train_ys = ST.load_data(self._train_path)
ST.replace_url(self._train_xs, fill=True)
if not icon:
ST.remove_emoticon(self._train_xs)
self._train(cross_validation=cross)
def train(self, icon=True, cross=False):
#word2cnt = BayesClassifier.Word2Cnt()
#txt = '今天天气就是棒[哈哈] [太阳] [飞起来]#'
#return
#self._load_data()
#self._replace_url(fill=True)
self._train_xs, self._train_ys = ST.load_data(self._train_path)
ST.replace_url(self._train_xs, fill=True)
if not icon:
ST.remove_emoticon(self._train_xs)
self._train(cross_validation=cross)
def format_test(self, emoticon=True, parenthesis=True):
test_path = '../test_data/%s_test_data' % self.classifier_type
self._test_xs, self._test_ys = ST.load_data(test_path)
linfo('begin preprocess test data, then sparse')
self._raw_test_xs, self._test_xs = ST.preprocess(self._test_xs)
#ST.replace_url(self._test_xs, fill='H')
#ST.replace_target(self._test_xs, fill='T')
self._test_ys = map(lambda x: self.tag2index[x], self._test_ys)
self.format_sparse(
self._test_xs, self._test_ys,
'%s/test_data/%s%s_sparse_test_data_%s' %
(project_dir, self.flag_prefix, self.classifier_type,
'icon' if emoticon else 'no_icon'))
def _all_train(self, total_word2cnt, total_tag2cnt):
if os.path.exists(self._test_path):
test_xs, test_ys = ST.load_data(self._test_path)
#linfo('load manually tagged data count: %s' % len(test_xs))
else:
return
test_t2c = {"P":0,"N":0,"O":0}
for y in test_ys:
if y not in test_t2c:
raise Exception('Key Error in tag2cnt. unknown key: %s' % y)
test_t2c[y] += 1
#print test_t2c
return self._batch_predict(test_xs, test_ys, total_word2cnt, total_tag2cnt, test_t2c)
def _all_train(self, total_word2cnt, total_tag2cnt):
if os.path.exists(self._test_path):
test_xs, test_ys = ST.load_data(self._test_path)
#linfo('load manually tagged data count: %s' % len(test_xs))
else:
return
test_t2c = {"P": 0, "N": 0, "O": 0}
for y in test_ys:
if y not in test_t2c:
raise Exception('Key Error in tag2cnt. unknown key: %s' % y)
test_t2c[y] += 1
#print test_t2c
return self._batch_predict(test_xs, test_ys, total_word2cnt,
total_tag2cnt, test_t2c)
def __init__(self, ct='tri', prefix=''):
if prefix and prefix != 'Dg_':
raise Exception('INVALID PREFIX GIVEN!!!')
self.flag_prefix = prefix
self.train_data_path = '%s/train_data/%s%s_train_data' % (project_dir, prefix, ct)
if ct not in ['bi', 'tri']:
raise Exception('INVALID Classifier Type')
self.classifier_type = ct
self.tag2index = TAG2INDEX
self._train_xs, self._train_ys = ST.load_data(self.train_data_path)
self._train_ys = map(lambda x: self.tag2index[x], self._train_ys)
#self._feature_extract_config = ['unigram', 'bigram']
self._feature_extract_config = feature_config
linfo('feature extract config: %s' % self._feature_extract_config)
linfo('classifier type %s' % ct)
linfo('init %s success' % self)
def __init__(self, ct='tri', prefix=''):
if prefix and prefix != 'Dg_':
raise Exception('INVALID PREFIX GIVEN!!!')
self.flag_prefix = prefix
self.train_data_path = '%s/train_data/%s%s_train_data' % (project_dir,
prefix, ct)
if ct not in ['bi', 'tri']:
raise Exception('INVALID Classifier Type')
self.classifier_type = ct
self.tag2index = TAG2INDEX
self._train_xs, self._train_ys = ST.load_data(self.train_data_path)
self._train_ys = map(lambda x: self.tag2index[x], self._train_ys)
#self._feature_extract_config = ['unigram', 'bigram']
self._feature_extract_config = feature_config
linfo('feature extract config: %s' % self._feature_extract_config)
linfo('classifier type %s' % ct)
linfo('init %s success' % self)