def train(cls, corpus, sim_metric, feature_num=5, sim_model='weighted'):
'''
Extract categories, features, feature weights, from corpus.
Compute the weight for each feature token in each category
The weight is computed as token_count / total_feature_count
'''
print "Training..."
cat_word = {}
for sent, cat in corpus:
cat_word.setdefault(cat,
[]).extend(word_process(word_tokenize(sent)))
features = {cat: Counter(cat_word[cat]) for cat in cat_word}
labels = features.keys()
cat_features = {}
feature_weights = {}
for c, f in features.iteritems():
w_c_pairs = f.most_common(feature_num)
words, counts = zip(*w_c_pairs)
cat_features[c] = words
total_count = float(sum(counts))
word_weights = []
for w, count in w_c_pairs:
word_weights.append((w, count / total_count))
feature_weights[c] = word_weights
return cls(labels, cat_features, feature_weights, sim_metric,
sim_model)
def transform(self, X):
tokenize = lambda x: word_process(word_tokenize(x))
X_tokens = map(tokenize, X)
if self._model == 'onehot':
return map(self.unigram_features, X_tokens)
else:
return map(self.sim_features, X_tokens)
def extract_features(self, corpus, feature_num=10):
cat_word = {}
for sent, cat in corpus:
cat_word.setdefault(cat, []).extend(word_process(word_tokenize(sent)))
features = {cat: Counter(cat_word[cat]) for cat in cat_word}
feature_words = []
for c, f in features.iteritems():
words, counts = zip(*f.most_common(feature_num))
feature_words.extend(list(words))
feature_words = set(feature_words)
return feature_words
def classify_single(self, sent, feature_model='max'):
"""
The input feature words are compared to each category based on category similarity.
Sum the semantic similarity score between features and category.
The category having highest similarity score is the correct category.
:param featuresets: feature sets such as word list
:param method: specify the semantic similarity metric
:param model: similarity combination model 'max', 'sum'. Default is 'max'
:return: the correct category label.
"""
feature_words = list(set(word_process(word_tokenize(sent))))
score = {}
for c in self._categories:
if feature_model == 'max':
score[c] = max([self.category_similarity(w, c) for w in feature_words] + [0.0])
else:
score[c] = sum([self.category_similarity(w, c) for w in feature_words] + [0.0])
return Counter(score).most_common(1)[0][0]
def train(cls, corpus, sim_metric, feature_num=5, sim_model='weighted'):
'''
Extract categories, features, feature weights, from corpus.
Compute the weight for each feature token in each category
The weight is computed as token_count / total_feature_count
'''
print "Training..."
cat_word = {}
for sent, cat in corpus:
cat_word.setdefault(cat, []).extend(word_process(word_tokenize(sent)))
features = {cat: Counter(cat_word[cat]) for cat in cat_word}
labels = features.keys()
cat_features = {}
feature_weights = {}
for c, f in features.iteritems():
w_c_pairs = f.most_common(feature_num)
words, counts = zip(*w_c_pairs)
cat_features[c] = words
total_count = float(sum(counts))
word_weights = []
for w, count in w_c_pairs:
word_weights.append((w, count / total_count))
feature_weights[c] = word_weights
return cls(labels, cat_features, feature_weights, sim_metric, sim_model)
num_features=tfidf_corpus.num_terms)
tfidf_index.num_best = top_N
tfidf_index.save(save_dir + 'tfidf_index/tfidf.index')
if model == 'lsa':
lsa = models.LsiModel(tfidf_corpus,
id2word=dictionary,
num_topics=topic_n)
lsa.save(save_dir + 'lsa.model')
lsa_index = similarities.Similarity(save_dir + 'lsa_index/shard',
lsa[tfidf_corpus],
num_features=topic_n)
lsa_index.num_best = top_N
lsa_index.save(save_dir + 'lsa_index/lsa.index')
return cls(text_process, model, dictionary, tfidf, tfidf_index,
lsa, lsa_index)
return cls(text_process, model, dictionary, tfidf, tfidf_index, None,
None)
try:
data = read_data(DATA_FILE)
except:
prepare_entities()
data = read_data(DATA_FILE)
TextAnalysis.train([d['abstract'] for _, d in enumerate(data)],
lambda x: word_process(word_tokenize(x)))
text_tfidf = TextAnalysis.load(lambda x: word_process(word_tokenize(x)))
text_lsa = TextAnalysis.load(lambda x: word_process(word_tokenize(x)),
model='lsa')
def tokenize(x): return word_process(word_tokenize(x)) X_tokens = map(tokenize, X)