#vectorised_test_documents_tf = vectorizer_tf.transform(test_docs)
term_freq = {}; term_prob = {}; all_terms = {}; totterms = 0.0
def add2dict(k, v, features):
features[k]= features.get(k, 0.0) + v
#Devide features by class
for i in range(num_labels):
classdoc_ids = numpy.nonzero(train_labels[:, i])[0].tolist()
if len(classdoc_ids) == 0:
term_freq[i] = {term_freq[i-1].keys()[0]: 0}
continue
term_freq[i] = get_TF(vectorizer_tf, vectorised_train_documents_tf, classdoc_ids)
map(lambda (k, v): add2dict(k, v, all_terms), term_freq[i].items())
totterms += sum(term_freq[i].values())
all_terms_prob = {k: v/totterms for k, v in all_terms.items()}
#Devide features by class-complements
for i in range(num_labels):
classdoc_ids = numpy.nonzero(train_labels_complement[:, i])[0].tolist()
term_freq[num_labels + i] = get_TF(vectorizer_tf, vectorised_train_documents_tf, classdoc_ids)
#Convert to Probability & Perform Jelinek-Mercer Smoothing
if weight == "tf" or cor_type == "P":
for i in range(num_labels):
term_prob[i] = freqToProbability(term_freq[i], term_freq[num_labels + i], all_terms_prob, lamda)
term_prob[num_labels + i] = freqToProbability(term_freq[num_labels + i], term_freq[i], all_terms_prob, lamda)
vocab_choice = term_prob
#Clear memory for unused variables
#----------------Feature Extraction--------------------------
#Process all documents
# Learn and transform documents [tf]
vectorizer_tf = CountVectorizer(stop_words=stop_words, tokenizer=tokenize)
vectorised_train_documents_tf = vectorizer_tf.fit_transform(train_docs)
#vectorised_test_documents_tf = vectorizer_tf.transform(test_docs)
def add2dict(k, v, all_term):
all_term[k]= all_term.get(k, 0.0) + v
#Devide features by class
term_freq = {}; all_term = {}; tot_freq = 0.0
for i in range(train_labels.shape[1]):
classdoc_ids = numpy.nonzero(train_labels[:, i])[0].tolist()
term_freq[i] = get_TF(vectorizer_tf, vectorised_train_documents_tf, classdoc_ids)
map(lambda (k, v): add2dict(k, v, all_term), term_freq[i].items())
tot_freq += sum(term_freq[i].values())
all_term = {k: v/tot_freq for k, v in all_term.items()}
#Devide features by class-complements
compl_term_freq = {}
for i in range(train_labels.shape[1]):
classdoc_ids = numpy.nonzero(train_labels_complement[:, i])[0].tolist()
compl_term_freq[i] = get_TF(vectorizer_tf, vectorised_train_documents_tf, classdoc_ids)
#Convert to Probability & Perform Jelinek-Mercer Smoothing
term_prob = {}
if weight == "tf" or cor_type == "P":
for i in range(train_labels.shape[1]):
term_prob[i] = freqToProbability(term_freq[i], compl_term_freq[i], all_term, lamda)
