fid.close() f2.close() # i = i + 1 # print i print len(corpus) vectorizer = CountVectorizer() transformer = TfidfTransformer() tfidf = transformer.fit_transform(vectorizer.fit_transform(corpus)) word = vectorizer.get_feature_names() weight = tfidf.toarray() vectorP = vectorizer.get_params() tfidfP = transformer.get_params() print 'vectorP:', vectorP print 'tfidfP:', tfidfP joblib.dump(vectorizer, "vectorizer" + str(sys.argv[1]) + ".m") joblib.dump(transformer, "tfidf" + str(sys.argv[1]) + ".m") resName = "BaiduTfidf_Result.txt" result = codecs.open(resName, 'w', 'utf-8') for j in range(len(word)): result.write(word[j] + ' ') result.write('\r\n\r\n') for i in range(len(weight)): # print u"-------这里输出第",i,u"类文本的词语tf-idf权重------"
analyzer=my_analyzer, ngram_range=ngram) tf = cv.fit_transform(discussions) print("\nVectorizer Parameters\n", cv, "\n") # LDA For Term Frequency x Doc Matrix n_topics = 9 max_iter = 5 learning_offset = 20. learning_method = 'online' # LDA for TF-IDF x Doc Matrix # First Create Term-Frequency/Inverse Doc Frequency by Review Matrix # This requires constructing Term Freq. x Doc. matrix first tf_idf = TfidfTransformer() print("\nTF-IDF Parameters\n", tf_idf.get_params(),"\n") tf_idf = tf_idf.fit_transform(tf) # Or you can construct the TF/IDF matrix from the data tfidf_vect = TfidfVectorizer(max_df=0.95, min_df=2, max_features=m_features,\ analyzer=my_analyzer, ngram_range=ngram) tf_idf = tfidf_vect.fit_transform(discussions) print("\nTF_IDF Vectorizer Parameters\n", tfidf_vect, "\n") lda = LatentDirichletAllocation(n_components=n_topics, max_iter=max_iter,\ learning_method=learning_method, \ learning_offset=learning_offset, \ random_state=12345) lda.fit_transform(tf_idf) print('{:.<22s}{:>6d}'.format("Number of Reviews", tf.shape[0])) print('{:.<22s}{:>6d}'.format("Number of Terms", tf.shape[1])) print("\nTopics Identified using LDA with TF_IDF")
def generate_vectors(train_url, test_url=None, column='article', trans_type=None, max_n=1, min_df=1, max_df=1.0, max_features=1, sublinear_tf=True, balanced=False, re_weight=0, verbose=False, drop_words=0): """ generate X, y, X_test vectors with csv(with header) url use pandas and CountVectorizer Args: train_url: url to train csv test_url: url to test csv, set to None if not need X_test column: column to use as feature trans_type: specific transformer, {'dc','idf'} max_n: max_n for ngram_range min_df: min_df for CountVectorizer max_df: max_df for CountVectorizer max_features: max_features for CountVectorizer sublinear_tf: sublinear_tf for default TfdcTransformer balanced: balanced for default TfdcTransformer, for idf transformer, it is use_idf re_weight: re_weight for TfdcTransformer verbose: True to show more information drop_words: randomly delete some words from sentences Returns: X, y, X_test """ verbose and print("loading '%s' level data from %s with pandas" % (column, train_url)) train_df = load_to_df(train_url) # vectorizer vec = CountVectorizer(ngram_range=(1, max_n), min_df=min_df, max_df=max_df, max_features=max_features, token_pattern='\w+') s_time = time() verbose and print("finish loading, vectorizing") verbose and print("vectorizer params:", vec.get_params()) sequences = train_df[column] # delete some words randomly for i, row in enumerate(sequences): if drop_words <= 0: break if np.random.ranf() < drop_words: row = np.array(row.split()) sequences.at[i] = ' '.join(row[np.random.ranf(row.shape) > 0.35]) X = vec.fit_transform(sequences) e_time = time() verbose and print("finish vectorizing in %.3f seconds, transforming" % (e_time - s_time)) # transformer if trans_type is None or trans_type == 'idf': trans = TfidfTransformer(sublinear_tf=sublinear_tf, use_idf=balanced) else: trans = TfdcTransformer(sublinear_tf=sublinear_tf, balanced=balanced, re_weight=re_weight) verbose and print("transformer params:", trans.get_params()) y = np.array((train_df["class"]).astype(int)) X = trans.fit_transform(X, y) X_test = None if test_url: verbose and print("transforming test set") test_df = load_to_df(test_url) X_test = vec.transform(test_df[column]) X_test = trans.transform(X_test) s_time = time() verbose and print("finish transforming in %.3f seconds\n" % (s_time - e_time)) return X, y, X_test
def generate_vectors(train_url, test_url=None, column='article', trans_type=None, max_n=1, min_df=1, max_df=1.0, max_features=1, sublinear_tf=True, balanced=False, re_weight=0, verbose=False, drop_words=0, multilabel_out=False, label_col='subjects', only_single=True, shuffle=True, apply_fun=None): """ generate X, y, X_test vectors with csv(with header) url use pandas and CountVectorizer Args: train_url: url to train csv test_url: url to test csv, set to None if not need X_test column: column to use as feature trans_type: specific transformer, {'dc','idf', 'hashing'} max_n: max_n for ngram_range min_df: min_df for CountVectorizer max_df: max_df for CountVectorizer max_features: max_features for CountVectorizer sublinear_tf: sublinear_tf for default TfdcTransformer balanced: balanced for default TfdcTransformer, for idf transformer, it is use_idf re_weight: re_weight for TfdcTransformer verbose: True to show more information drop_words: randomly delete some words from sentences multilabel_out: return y as multilabel format label_col: col name of label only_single: only keep records of single label shuffle: re sample train data apply_fun: callable to be applied on label column Returns: X, y, X_test """ verbose and print("loading '%s' level data from %s with pandas" % (column, train_url)) train_df = pd.read_csv(train_url) if shuffle: train_df = train_df.sample(frac=1) if only_single: train_df = train_df[train_df['subjects'].apply(lambda x: len(x) < 2)] # vectorizer s_time = time() analyzer = 'word' if column == 'word_seg' else 'char' vec = CountVectorizer(analyzer=analyzer, ngram_range=(1, max_n), min_df=min_df, max_df=max_df, max_features=max_features, token_pattern='\w+') verbose and print("finish loading, vectorizing") verbose and print("vectorizer params:", vec.get_params()) sequences = train_df[column] # delete some words randomly for i, row in enumerate(sequences): if drop_words <= 0: break if np.random.ranf() < drop_words: row = np.array(row.split()) sequences.at[i] = ' '.join(row[np.random.ranf(row.shape) > 0.35]) X = sequences if trans_type == 'hashing' else vec.fit_transform(sequences) e_time = time() verbose and print("finish vectorizing in %.3f seconds, transforming" % (e_time - s_time)) # transformer if trans_type is None or trans_type == 'idf': trans = TfidfTransformer(sublinear_tf=sublinear_tf, use_idf=balanced) elif trans_type == 'dc': trans = TfdcTransformer(sublinear_tf=sublinear_tf, balanced=balanced, re_weight=re_weight) else: trans = HashingVectorizer(analyzer=analyzer, ngram_range=(1, max_n), n_features=max_features, token_pattern='\w+', binary=not balanced) verbose and print(trans_type, "transformer params:", trans.get_params()) if multilabel_out: mlb = MultiLabelBinarizer() y = mlb.fit_transform(train_df[label_col].apply(str.split)) verbose and print("multilabel columns:\n", mlb.classes_) else: y = train_df[label_col].apply(apply_fun).values if apply_fun is not None \ else train_df[label_col].values X = trans.fit_transform(X, y) X_test = None if test_url: verbose and print("transforming test set") test_df = pd.read_csv(test_url) X_test = test_df[column] if trans_type == 'hashing' else vec.transform( test_df[column]) X_test = trans.transform(X_test) s_time = time() verbose and print("finish transforming in %.3f seconds\n" % (s_time - e_time)) return X, y, X_test