def extractCharacterData(df_comments, tag):
comment_list = extract_global_bag_of_words(df_comments)
train_v, test_v = np.load('train_vect.npy'), np.load('test_vect.npy')
train_list = []
test_list = []
for v in train_v:
train_list.append(comment_list[v])
for v in test_v:
test_list.append(comment_list[v])
print len(train_list)
print len(test_list)
print 'Character Ngrams Binary'
cb_train, cb_test = extract_words(CountVectorizer(analyzer=CharacterAnalyzer(), binary=True, dtype=float), train_list, test_list)
save_sparse_csr(feature_set_path + "binaryCharacterData" + tag + "_train", cb_train)
save_sparse_csr(feature_set_path + "binaryCharacterData" + tag + "_test", cb_test)
print 'Character Ngrams tfidf'
ct_train, ct_test = extract_words(TfidfVectorizer(analyzer=CharacterAnalyzer(), use_idf=True, smooth_idf=True, dtype=float), train_list, test_list)
save_sparse_csr(feature_set_path + "tfidfCharacterData" + tag + "_train", ct_train)
save_sparse_csr(feature_set_path + "tfidfCharacterData" + tag + "_test", ct_test)
print 'Character skipgrams Binary'
sb_train, sb_test = extract_words(CountVectorizer(analyzer=CharacterSkipGramAnalyzer(), binary=True, dtype=float), train_list, test_list)
save_sparse_csr(feature_set_path + "binaryCharacterSkipgramData" + tag + "_train", sb_train)
save_sparse_csr(feature_set_path + "binaryCharacterSkipgramData" + tag + "_test", sb_test)
print 'Character skipgrams TFIDF'
sb_train, sb_test = extract_words(TfidfVectorizer(analyzer=CharacterSkipGramAnalyzer(), use_idf=True, smooth_idf=True, dtype=float), train_list, test_list)
save_sparse_csr(feature_set_path + "tfidfCharacterSkipgramData" + tag + "_train", sb_train)
save_sparse_csr(feature_set_path + "tfidfCharacterSkipgramData" + tag + "_test", sb_test)
def extractLexicalBigramData(articleList, commentList, commentCount):
comment_list = extract_global_bag_of_words(commentList)
train_v, test_v = np.load('train_vect.npy'), np.load('test_vect.npy')
train_list = []
test_list = []
for v in train_v:
train_list.append(comment_list[v])
for v in test_v:
test_list.append(comment_list[v])
print len(train_list)
print len(test_list)
print 'Lexical Ngrams Binary'
cb_train, cb_test = extract_words(
CountVectorizer(analyzer=LexicalBigramUnigramAnalyzer(),
binary=True,
dtype=float), train_list, test_list)
save_sparse_csr(feature_set_path + "binaryLexicalBigramsData_train",
cb_train)
save_sparse_csr(feature_set_path + "binaryLexicalBigramsData_test",
cb_test)
print 'Lexical Ngrams tfidf'
ct_train, ct_test = extract_words(
TfidfVectorizer(analyzer=LexicalBigramUnigramAnalyzer(),
use_idf=True,
smooth_idf=True,
dtype=float), train_list, test_list)
save_sparse_csr(feature_set_path + "tfidfLexicalBigramsData_train",
ct_train)
save_sparse_csr(feature_set_path + "tfidfLexicalBigramsData_test", ct_test)
def extractLexicalBigramData(articleList, commentList, commentCount):
comment_list = extract_global_bag_of_words(commentList)
train_v, test_v = np.load('train_vect.npy'), np.load('test_vect.npy')
train_list = []
test_list = []
for v in train_v:
train_list.append(comment_list[v])
for v in test_v:
test_list.append(comment_list[v])
print len(train_list)
print len(test_list)
print 'Lexical Ngrams Binary'
cb_train, cb_test = extract_words(CountVectorizer(analyzer=LexicalBigramUnigramAnalyzer(), binary=True, dtype=float), train_list, test_list)
save_sparse_csr(feature_set_path + "binaryLexicalBigramsData_train", cb_train)
save_sparse_csr(feature_set_path + "binaryLexicalBigramsData_test", cb_test)
print 'Lexical Ngrams tfidf'
ct_train, ct_test = extract_words(TfidfVectorizer(analyzer=LexicalBigramUnigramAnalyzer(), use_idf=True, smooth_idf=True, dtype=float), train_list, test_list)
save_sparse_csr(feature_set_path + "tfidfLexicalBigramsData_train", ct_train)
save_sparse_csr(feature_set_path + "tfidfLexicalBigramsData_test", ct_test)
def extractWordData(df_comments, tag):
processed_comment_list = extract_global_bag_of_words_processed(df_comments)
print len(processed_comment_list)
train_v, test_v = np.load('train_vect.npy'), np.load('test_vect.npy')
train_list = []
test_list = []
for v in train_v:
train_list.append(processed_comment_list[v])
for v in test_v:
test_list.append(processed_comment_list[v])
# train_list = [' '.join(sent) for sent in train_list]
# test_list = [' '.join(sent) for sent in test_list]
print len(train_list)
print len(test_list)
print 'Unigram Binary'
bwd_train, bwd_test = extract_words(CountVectorizer(analyzer=UnigramAnalyzer(), binary=True, dtype=float), train_list, test_list)
print 'Unigram Frequency'
fwd_train, fwd_test = extract_words(CountVectorizer(analyzer=UnigramAnalyzer(), dtype=float), train_list, test_list)
print 'Unigram TFIDF'
twd_train, twd_test = extract_words(TfidfVectorizer(analyzer=UnigramAnalyzer(), use_idf=True, smooth_idf=True, dtype=float), train_list, test_list)
print 'Bigram Binary'
bbwd_train, bbwd_test = extract_words(CountVectorizer(analyzer=UnigramBigramAnalyzer(), binary=True, dtype=float), train_list, test_list)
print 'Bigram TFIDF'
btwd_train, btwd_test = extract_words(TfidfVectorizer(analyzer=UnigramBigramAnalyzer(), use_idf=True, smooth_idf=True, dtype=float), train_list, test_list)
print 'Trigram Binary'
tbwd_train, tbwd_test = extract_words(CountVectorizer(analyzer=UnigramBigramTrigramAnalyzer(), binary=True, dtype=float), train_list, test_list)
print 'Trigram TFIDF'
ttwd_train, ttwd_test = extract_words(TfidfVectorizer(analyzer=UnigramBigramTrigramAnalyzer(), use_idf=True, smooth_idf=True, dtype=float), train_list, test_list)
print 'Bigram Only Binary'
bowd_train, bowd_test = extract_words(CountVectorizer(analyzer=BigramAnalyzer(), binary=True, dtype=float), train_list, test_list)
print 'Bigram Only TFIDF'
bowd2_train, bowd2_test = extract_words(TfidfVectorizer(analyzer=BigramAnalyzer(), use_idf=True, smooth_idf=True, dtype=float), train_list, test_list)
print 'Trigram Only Binary'
towd_train, towd_test = extract_words(CountVectorizer(analyzer=TrigramAnalyzer(), binary=True, dtype=float), train_list, test_list)
print 'Trigram Only TFIDF'
towd2_train, towd2_test = extract_words(TfidfVectorizer(analyzer=TrigramAnalyzer(), use_idf=True, smooth_idf=True, dtype=float), train_list, test_list)
print(feature_set_path + "binaryWordData_train", bwd_train[123, :])
print(feature_set_path + "freqWordData_train", fwd_train[123, :])
print(feature_set_path + "tfidfWordData_train", twd_train[123, :])
print(feature_set_path + "bigramBinaryWordData_train", bbwd_train[123, :])
print(feature_set_path + "bigramTfidfWordData_train", btwd_train[123, :])
print(feature_set_path + "trigramBinaryWordData_train", tbwd_train[123, :])
print(feature_set_path + "trigramTfidfWordData_train", ttwd_train[123, :])
print(feature_set_path + "bigramOnlyBinaryWordData_train", bowd_train[123, :])
print(feature_set_path + "bigramOnlyTfidfWordData_train", bowd2_train[123, :])
print(feature_set_path + "trigramOnlyBinaryWordData_train", towd_train[123, :])
print(feature_set_path + "trigramOnlyTfidfWordData_train", towd2_train[123, :])
save_sparse_csr(feature_set_path + "binaryWordData" + tag + "_train", bwd_train)
save_sparse_csr(feature_set_path + "freqWordData" + tag + "_train", fwd_train)
save_sparse_csr(feature_set_path + "tfidfWordData" + tag + "_train", twd_train)
save_sparse_csr(feature_set_path + "bigramBinaryWordData" + tag + "_train", bbwd_train)
save_sparse_csr(feature_set_path + "bigramTfidfWordData" + tag + "_train", btwd_train)
save_sparse_csr(feature_set_path + "trigramBinaryWordData" + tag + "_train", tbwd_train)
save_sparse_csr(feature_set_path + "trigramTfidfWordData" + tag + "_train", ttwd_train)
save_sparse_csr(feature_set_path + "bigramOnlyBinaryWordData" + tag + "_train", bowd_train)
save_sparse_csr(feature_set_path + "bigramOnlyTfidfWordData" + tag + "_train", bowd2_train)
save_sparse_csr(feature_set_path + "trigramOnlyBinaryWordData" + tag + "_train", towd_train)
save_sparse_csr(feature_set_path + "trigramOnlyTfidfWordData" + tag + "_train", towd2_train)
save_sparse_csr(feature_set_path + "binaryWordData" + tag + "_test", bwd_test)
save_sparse_csr(feature_set_path + "freqWordData" + tag + "_test", fwd_test)
save_sparse_csr(feature_set_path + "tfidfWordData" + tag + "_test", twd_test)
save_sparse_csr(feature_set_path + "bigramBinaryWordData" + tag + "_test", bbwd_test)
save_sparse_csr(feature_set_path + "bigramTfidfWordData" + tag + "_test", btwd_test)
save_sparse_csr(feature_set_path + "trigramBinaryWordData" + tag + "_test", tbwd_test)
save_sparse_csr(feature_set_path + "trigramTfidfWordData" + tag + "_test", ttwd_test)
save_sparse_csr(feature_set_path + "bigramOnlyBinaryWordData" + tag + "_test", bowd_test)
save_sparse_csr(feature_set_path + "bigramOnlyTfidfWordData" + tag + "_test", bowd2_test)
save_sparse_csr(feature_set_path + "trigramOnlyBinaryWordData" + tag + "_test", towd_test)
save_sparse_csr(feature_set_path + "trigramOnlyTfidfWordData" + tag + "_test", towd2_test)