def random_forest_generator(data_filename, replacer,train_ratio = 0.8):
#initialize variables
data = pd.read_csv(data_filename)
data_labels = data['state'].tolist()
data_texts = data['content']
bow_transformer = CountVectorizer()
#vectorize train data
word_list = preprocess(data_texts, replacer)
vectorized_data = bow_transformer.fit_transform(word_list)
tfidf_transformer = TfidfTransformer().fit(vectorized_data)
worker = Worker(bow = bow_transformer,tfidf = tfidf_transformer)
vectorized_data_list = tfidf_transformer.transform(vectorized_data).toarray()
#print(vectorized_data_list)
all_data = worker.sliceData(vectorized_data_list,data_labels,train_ratio)
train_data = all_data['train_data']
train_labels = all_data['train_labels']
test_data = all_data['test_data']
test_labels = all_data['test_labels']
#build model
forest = RandomForestClassifier(n_estimators=30,n_jobs=-1)
forest = forest.fit(train_data,train_labels)
pred=forest.predict(test_data)
accuracy = worker.calculate_result(test_labels,pred)
#save data and model
joblib.dump(forest,'./var/model/forest_0',compress = 3)
joblib.dump(bow_transformer,'./var/model/bow_0',compress = 3)
joblib.dump(tfidf_transformer,'./var/model/tfidf_0', compress = 3)
return forest
def logistic_regression_generator(data_filename, replacer, train_ratio = 0.8):
data = pd.read_csv(data_filename)
data_labels = data['state'].tolist()
data_texts = data['content']
bow_transformer = CountVectorizer()
#vectorize train data
word_list = preprocess(data_texts, replacer)
vectorized_data = bow_transformer.fit_transform(word_list)
tfidf_transformer = TfidfTransformer().fit(vectorized_data)
worker = Worker(bow = bow_transformer,tfidf = tfidf_transformer)
vectorized_data_list = tfidf_transformer.transform(vectorized_data).toarray()
all_data = worker.sliceData(vectorized_data_list,data_labels,train_ratio)
train_data = all_data['train_data']
train_labels = all_data['train_labels']
test_data = all_data['test_data']
test_labels = all_data['test_labels']
#build model
lr = LogisticRegression(C=30,penalty='l2',random_state=0)
lr.fit(train_data,train_labels)
pred=lr.predict(test_data)
print("generator prediction:")
accuracy = worker.calculate_result(test_labels,pred)
#save data and model
joblib.dump(lr,'./var/model/lr_0',compress = 3)
joblib.dump(bow_transformer,'./var/model/bow_0',compress = 3)
joblib.dump(tfidf_transformer,'./var/model/tfidf_0', compress = 3)
return lr