def set_trained_data(data, NN):
new_data = []
for row in data:
new_data.append(row[1])
G = GloveEmbedding("data/glove.6B.50d.txt", dimensions=50)
word_to_idx, idx_to_word, embedding = G.read_embedding()
S = SentenceToIndices(word_to_idx)
X_Predict_Idx, max_len = S.map_sentence_list(new_data)
i = 0
for s in X_Predict_Idx:
# print(str(i)+ ": ", s)
i = i + 1
#if max_len % 2 != 0:
# max_len = max_len + 1
max_len = 72
print("Max Len", max_len)
P = PadSentences(max_len)
Trim = TrimSentences(max_len)
X_Predict_Final = P.pad_list(X_Predict_Idx)
X_Predict_Final = Trim.trim_list(X_Predict_Final)
X_Predict_Final = np.array(X_Predict_Final)
X_Prediction = NN.predict(X_Predict_Final)
final = np.argmax(X_Prediction, axis=1)
return new_data, final
def get_glove_embedding(self):
g = GloveEmbedding(self.embedding_filename, dimensions=50)
word_to_idx, idx_to_word, embedding = g.read_embedding()
s = SentenceToIndices(word_to_idx)
x_train_indices, max_len = s.map_sentence_list(self.x_all)
if max_len % 2 != 0:
max_len = max_len + 1
p = PadSentences(max_len)
x_train_pad = p.pad_list(x_train_indices)
# TRIM Tweets to remove noisy data
trim_size = max_len
trim = TrimSentences(trim_size)
x_train_pad = trim.trim_list(x_train_pad)
return x_train_pad, max_len, g
def process(self,
json_filename,
h5_filename,
plot=False,
epochs=100,
vect_dimensions=50):
# open the file with tweets
X_all = []
Y_all = []
All = []
#with open(self.labeled_tweets_filename, "r", encoding="ISO-8859-1") as f:
with open(self.labeled_tweets_filename, "r") as f:
i = 0
csv_file = csv.reader(f, delimiter=',')
ones_count = 0
for r in csv_file:
if i != 0:
All.append(r)
i = i + 1
np.random.shuffle(All)
ones_count = 0
two_count = 0
zero_count = 0
for r in All:
tweet = r[0]
label = int(r[1])
if (label == 0):
zero_count += 1
elif (label == 1):
ones_count += 1
else:
two_count += 1
# if (label == 2):
# label = 0
# if (label == 1) and (ones_count <= 4611):
# X_all.append(tweet)
# Y_all.append(label)
# ones_count +=1
# elif (label == 0):
X_all.append(tweet)
Y_all.append(label)
print("len(Y_all): ", len(Y_all))
class_weight_val = class_weight.compute_class_weight(
'balanced', np.unique(Y_all), Y_all)
print("classes: ", np.unique(Y_all))
print("counts for 0, 1, 2: ", zero_count, ones_count, two_count)
print("class weight_val: ", class_weight_val)
class_weight_dictionary = {
0: class_weight_val[0],
1: class_weight_val[1],
2: class_weight_val[2]
}
print("dict: ", class_weight_dictionary)
print("Data Ingested")
# divide the data into training and test
num_data = len(X_all)
limit = math.ceil(num_data * 0.80)
X_train_sentences = X_all
Y_train = Y_all
# Divide after conversions
# divide the data into X_train, Y_train, X_test, Y_test
#X_train_sentences = X_all[0: limit]
#Y_train = Y_all[0: limit]
#X_test_sentences = X_all[limit:]
#Y_test = Y_all[limit:]
#print("Data Divided")
#Get embeeding
#G = Word2VecEmbedding(self.embedding_filename, dimensions=vect_dimensions)
G = GloveEmbedding(self.embedding_filename, dimensions=vect_dimensions)
word_to_idx, idx_to_word, embedding = G.read_embedding()
S = SentenceToIndices(word_to_idx)
X_train_indices, max_len = S.map_sentence_list(X_train_sentences)
print("Train data mappend to indices")
if max_len % 2 != 0:
max_len = max_len + 1
P = PadSentences(max_len)
X_train_pad = P.pad_list(X_train_indices)
print("Train data padded")
# TRIM
trim_size = max_len
#trim_size = 33
Trim = TrimSentences(trim_size)
X_train_pad = Trim.trim_list(X_train_pad)
print("X[0], ", X_train_pad[0])
#convert to numPY arrays
X_train = np.array(X_train_pad)
Y_train = np.array(Y_train)
ones_count = np.count_nonzero(Y_train)
zeros_count = len(Y_train) - ones_count
print("ones count: ", ones_count)
print("zeros count: ", zeros_count)
print("two count: ", two_count)
Y_train_old = Y_train
Y_train = to_categorical(Y_train, num_classes=3)
# Divide the data
X_test_text = X_all[limit:]
X_test = X_train[limit:]
Y_test = Y_train[limit:]
X_train = X_train[0:limit]
Y_train = Y_train[0:limit]
print("data divided on value: ", limit)
print("lengths X_train, Y_train: ", len(X_train), len(Y_train))
print("lengths X_test, Y_test: ", len(X_test), len(Y_test))
print("Train data convert to numpy arrays")
#NN = TweetSentiment2DCNN(trim_size, G)
#NN = TweetSentiment2LSTM2Dense(trim_size, G)
#NN =TweetSentiment2LSTM2Dense3Layer(trim_size, G)
#NN =TweetSentiment2LSTM2Dense4Layer(trim_size, G)
#NN = TweetSentimentCNN(trim_size, G)
#print("Build GRU")
#NN = TweetSentimentGRUSM(max_len, G)
NN = TweetSentiment1D(trim_size, G)
#NN = TweetSentiment1DRev(trim_size, G)
print("model created")
kernel_regularizer = l2(0.001)
#kernel_regularizer = None
NN.build(filters=11,
first_dropout=0,
second_dropout=0.05,
padding='valid',
dense_units=16)
#NN.build(first_layer_units = max_len, second_layer_units = max_len, relu_dense_layer=16, dense_layer_units = 3,
# first_layer_dropout=0, second_layer_dropout=0, third_layer_dropout=0)
print("model built")
NN.summary()
sgd = SGD(lr=0.03, momentum=0.009, decay=0.001, nesterov=True)
rmsprop = RMSprop(decay=0.003)
adam = Adam(lr=0.0003, decay=0.001)
sgd = SGD(lr=0.05)
NN.compile(optimizer=adam,
loss="categorical_crossentropy",
metrics=['accuracy', precision, recall, f1, fprate])
print("model compiled")
print("Begin training")
#callback = TensorBoard(log_dir="/tmp/logs")
#class_weight = {0: 0.67, 1: 0.33}
#class_weight = None
#history = NN.fit(X_train, Y_train, epochs=epochs, batch_size=32, callbacks=[callback], class_weight=class_weight_dictionary)
history = NN.fit(X_train,
Y_train,
epochs=epochs,
batch_size=64,
class_weight=class_weight_dictionary,
validation_split=0.2)
print("Model trained")
print("Predicting")
print("len(X_test): ", X_test)
preds = NN.predict(X_test)
print("len(preds): ", len(preds))
print("type preds: ", type(preds))
print("preds before: ", preds)
preds = np.argmax(preds, axis=1)
print("preds: ", preds)
print("len(preds): ", len(preds))
Y_test = Y_train_old[limit:]
print("Y test: ", Y_test)
c_matrix = confusion_matrix(Y_test, preds)
print("matrix: ", c_matrix)
print("Storing Errors: ")
ErrorAnalysis.store_errors(X_test_text, Y_test, preds, "errorcnn.csv")
print("Errors stored")
print("Confusion matrix: ")
prec_1, recall_1, f1_1, spec_1, t = calculate_cm_metrics(c_matrix, '')
print("C1-> presicion, recall, F1: ", prec_1, recall_1, f1_1)
#
# X_test_indices, max_len = S.map_sentence_list(X_test_sentences)
# print("Test data mapped")
# X_test_pad = P.pad_list(X_test_indices)
# print("Test data padded")
# X_test = np.array(X_test_pad)
# Y_test = np.array(Y_test)
# print("Test data converted to numpy arrays")
# loss, acc = NN.evaluate(X_test, Y_test, callbacks=[callback])
# print("accuracy: ", acc)
T = "I have a bad case of vomit"
X_Predict = [
"my zika is bad", "i love colombia",
"my has been tested for ebola",
"there is a diarrhea outbreak in the city"
]
X_Predict_Idx, max_len2 = S.map_sentence_list(X_Predict)
i = 0
for s in X_Predict_Idx:
print(str(i) + ": ", s)
i = i + 1
print(X_Predict)
X_Predict_Final = P.pad_list(X_Predict_Idx)
X_Predict_Final = Trim.trim_list(X_Predict_Final)
#X_Predict = [X_Predict]
X_Predict_Final = np.array(X_Predict_Final)
print("Predict: ", np.argmax(NN.predict(X_Predict_Final)))
print("Storing model and weights")
NN.save_model(json_filename, h5_filename)
if plot:
print("Ploting")
self.plot(history)
print("Done!")
def process(self, json_filename, h5_filename, plot=False, epochs = 100, vect_dimensions = 100):
np.random.seed(11)
# open the file with tweets
X_all = []
Y_all = []
All = []
Zeros = []
with open(self.labeled_tweets_filename, "r", encoding="ISO-8859-1") as f:
i = 0
csv_file = csv.reader(f, delimiter = ',')
ones_count = 0
Ones = []
for r in csv_file:
if i !=0:
label = int(r[1])
#if label == 0:
# Zeros.append(r)
All.append(r)
# tweet = r[0]
# label = r[1]
# X_all.append(tweet)
# Y_all.append(label)
i = i + 1
print("len(All): ", len(All))
np.random.shuffle(All)
ones_count = 0
for r in All:
tweet = r[0].strip()
label = int(r[1])
if (label == 2):
label = 0
# if (label == 1) and (ones_count <= 4611):
# X_all.append(tweet)
# Y_all.append(label)
# ones_count +=1
# elif (label == 0):
X_all.append(tweet)
Y_all.append(label)
print("Data Ingested")
# divide the data into training and test
num_data = len(X_all)
limit = math.ceil(num_data * 0.60)
X_train_sentences = X_all
Y_train = Y_all
# divide the data into X_train, Y_train, X_test, Y_test
#X_train_sentences = X_all[0: limit]
#Y_train = Y_all[0: limit]
#X_test_sentences = X_all[limit:]
#Y_test = Y_all[limit:]
#print("Data Divided")
#Get embeeding
#G = Word2VecEmbedding(self.embedding_filename, dimensions=vect_dimensions)
G = GloveEmbedding(self.embedding_filename, dimensions=50)
word_to_idx, idx_to_word, embedding = G.read_embedding()
S = SentenceToIndices(word_to_idx)
X_train_indices, max_len = S.map_sentence_list(X_train_sentences)
print("Train data mappend to indices")
if max_len % 2 !=0:
max_len = max_len + 1
P = PadSentences(max_len)
X_train_pad = P.pad_list(X_train_indices)
print("Train data padded")
# TRIM
trim_size = max_len
#trim_size = 45
Trim = TrimSentences(trim_size)
X_train_pad = Trim.trim_list(X_train_pad)
print("X[0], ", X_train_pad[0])
#convert to numPY arrays
X_train_reverse = []
for X in X_train_pad:
t = X[::-1]
X_train_reverse.append(t)
X_train = np.array(X_train_pad)
X_train_reverse = np.array(X_train_reverse)
Y_train = np.array(Y_train)
ones_count = np.count_nonzero(Y_train)
zeros_count = len(Y_train) - ones_count
print("ones count: ", ones_count)
print("zeros count: ", zeros_count)
#Y_train = to_categorical(Y_train, num_classes=3)
print("Train data convert to numpy arrays")
#NN = TweetSentiment2DCNN(trim_size, G)
NN = TweetSentiment2DCNN2Channel(trim_size, G)
#NN = TweetSentimentInception(trim_size, G)
#print("Build GRU")
#NN = TweetSentimentGRUSM(max_len, G)
print("model created")
kernel_regularizer = l2(0.001)
#kernel_regularizer = None
NN.build(filters=11, first_dropout=0, second_dropout=0.1, padding='valid', dense_units=32)
print("model built")
NN.summary()
sgd = SGD(lr=0.03, momentum=0.009, decay=0.001, nesterov=True)
rmsprop = RMSprop(decay=0.003)
adam = Adam(lr=0.1, decay=0.05)
#sgd = SGD(lr=0.05)
NN.compile(optimizer=rmsprop, loss="binary_crossentropy", metrics=['accuracy', precision, recall, f1, fprate])
print("model compiled")
print("Begin training")
callback = TensorBoard(log_dir="/tmp/logs")
#class_weight = {0: 0.67, 1: 0.33}
class_weight = None
history = NN.fit([X_train, X_train_reverse], Y_train, epochs=epochs, batch_size=32, callbacks=[callback], validation_split=0.20, class_weight=class_weight)
print("Model trained")
# X_test_indices, max_len = S.map_sentence_list(X_test_sentences)
# print("Test data mapped")
# X_test_pad = P.pad_list(X_test_indices)
# print("Test data padded")
# X_test = np.array(X_test_pad)
# Y_test = np.array(Y_test)
# print("Test data converted to numpy arrays")
# loss, acc = NN.evaluate(X_test, Y_test, callbacks=[callback])
# print("accuracy: ", acc)
T = "I have a bad case of vomit"
X_Predict = ["my zika is bad", "i love colombia", "my has been tested for ebola", "there is a diarrhea outbreak in the city"]
X_Predict_Idx, max_len2 = S.map_sentence_list(X_Predict)
i =0
for s in X_Predict_Idx:
print(str(i)+ ": ", s)
i = i + 1
print(X_Predict)
X_Predict_Final = P.pad_list(X_Predict_Idx)
X_Predict_Final = Trim.trim_list(X_Predict_Final)
#X_Predict = [X_Predict]
X_Predict_Reverse = []
for r in X_Predict_Final:
t = r[::-1]
X_Predict_Reverse.append(t)
X_Predict_Final = np.array(X_Predict_Final)
X_Predict_Reverse = np.array(X_Predict_Reverse)
Preds = NN.predict([X_Predict_Final, X_Predict_Reverse])
Preds = ((Preds >= 0.5)*1).flatten()
print("Predict: ", Preds)
print("Storing model and weights")
NN.save_model(json_filename, h5_filename)
if plot:
print("Ploting")
self.plot(history)
print("Done!")
def process(self,
json_filename,
h5_filename,
prod_json_file,
prod_h5_filename,
plot=False,
epochs=100,
vect_dimensions=50):
np.random.seed(11)
# open the file with tweets
X_all = []
Y_all = []
All = [] # array with every row in the file
# Load all rows into array All
#with open(self.labeled_tweets_filename, "r", encoding="ISO-8859-1") as f:
with open(self.labeled_tweets_filename, "r") as f:
i = 0
csv_file = csv.reader(f, delimiter=',')
ones_count = 0
for r in csv_file:
All.append(r)
# Mix the rows randoming to prevent order dependencies between runs
np.random.shuffle(All)
X_one_All = []
X_two_All = []
X_three_All = []
X_one_aux_All = []
X_two_aux_All = []
X_three_aux_All = []
X_one_aux_disease = []
X_two_aux_disease = []
X_three_aux_disease = []
X_one_aux_label = []
X_two_aux_label = []
X_three_aux_label = []
Y_t1_t2_relevance = []
Y_t1_t3_relevance = []
for r in All:
# Collect_Tweets
X_one_All.append(r[0].lower().strip())
X_two_All.append(r[3].lower().strip())
X_three_All.append(r[6].lower().strip())
#Collect Aux Info
#X_one_aux_All.append(r[1:3])
#X_two_aux_All.append(r[4:6])
#X_three_aux_All.append(r[7:9])
X_one_aux_disease.append(r[1])
X_one_aux_label.append(r[2])
X_two_aux_disease.append(r[4])
X_two_aux_label.append(r[5])
X_three_aux_disease.append(r[7])
X_three_aux_label.append(r[8])
# Collect Y's
Y_t1_t2_relevance.append(r[9])
Y_t1_t3_relevance.append(r[10])
Y_all.append(r[11])
# Convert the data to a form the NN can understand
num_data = len(All)
#print("All: ", All)
print("All: ")
for r in All:
print(r)
print("All.len: ", len(All))
print("X_one_All: ")
for r in X_one_All:
print(r)
G = Word2VecEmbedding(self.embedding_filename,
dimensions=vect_dimensions)
word_to_idx, idx_to_word, embedding = G.read_embedding()
S = SentenceToIndices(word_to_idx)
X_one_indices, max_len1 = self.map_to_idx(S, X_one_All)
X_two_indices, max_len2 = self.map_to_idx(S, X_two_All)
X_three_indices, max_len3 = self.map_to_idx(S, X_three_All)
#print("X_one_indices: ", X_one_indices)
print("X_one_indices: ")
for r in X_one_indices:
print(r)
print("max_len1 : ", max_len1)
for r in X_two_indices:
print(r)
print("max_len2 : ", max_len2)
for r in X_three_indices:
print(r)
print("max_len3 : ", max_len3)
# get max len of all
max_len = max(max_len1, max_len2)
max_len = max(max_len, max_len3)
if max_len % 2 != 0:
max_len = max_len + 1
# now padd the 3 senteces with 0 to make them all the same size
P = PadSentences(max_len)
X_one_train = P.pad_list(X_one_indices)
X_two_train = P.pad_list(X_two_indices)
X_three_train = P.pad_list(X_three_indices)
# now make the sencentes into np.array
X_one_train = np.array(X_one_train)
X_two_train = np.array(X_two_train)
X_three_train = np.array(X_three_train)
# change to categorical the disease type and the label
X_one_aux_train = self.binarize_aux(X_one_aux_disease, X_one_aux_label)
print('X_one_aux_train.shape: ', X_one_aux_train.shape)
X_two_aux_train = self.binarize_aux(X_two_aux_disease, X_two_aux_label)
X_three_aux_train = self.binarize_aux(X_three_aux_disease,
X_three_aux_label)
# Create the NN
labels_dim = 2
diases_dim = 4
#NN = TweetSimilaryBasic(max_sentence_len=max_len, embedding_builder=G, labels_dim = labels_dim, diases_dim = diases_dim)
#NN = TweetSimilaryBasicBiDirectional(max_sentence_len=max_len, embedding_builder=G, labels_dim = labels_dim, diases_dim = diases_dim)
NN = TweetSimilaryConvInception(max_sentence_len=max_len,
embedding_builder=G,
labels_dim=labels_dim,
diases_dim=diases_dim)
# Build the NN
NN.build()
# Summary
NN.summary()
# Compile the NN
#NN.compile(optimizer='rmsprop', loss=['mean_squared_error','mean_squared_error', 'binary_crossentropy'],
# metrics=['mse', 'mse','acc'], loss_weight=[ 1., 1., 1.0])
NN.compile(optimizer='rmsprop',
loss={
'R1': 'mean_squared_error',
'R2': 'mean_squared_error',
'FINAL': 'binary_crossentropy'
},
metrics={
'R1': 'mse',
'R2': 'mse',
'FINAL': 'acc'
},
loss_weights={
'R1': 0.25,
'R2': 0.25,
'FINAL': 10
})
Y_t1_t2_relevance = np.array(Y_t1_t2_relevance)
Y_t1_t3_relevance = np.array(Y_t1_t3_relevance)
Y_all = np.array(Y_all)
class_weight_val = class_weight.compute_class_weight(
'balanced', np.unique(Y_all), Y_all)
print("type(class_weight_val): ", type(class_weight_val))
print("class_weight_val", class_weight_val)
final_class_weight_val = {
'R1': None,
'R2': None,
'FINAL': class_weight_val
}
print("final_class_weight_val: ", final_class_weight_val)
history = NN.fit(X=[
X_one_train, X_two_train, X_three_train, X_one_aux_train,
X_two_aux_train, X_three_aux_train
],
Y=[Y_t1_t2_relevance, Y_t1_t3_relevance, Y_all],
epochs=epochs,
validation_split=0.20,
class_weight=final_class_weight_val)
# Save the model
NN.save_model_data(json_filename, h5_filename, prod_json_file,
prod_h5_filename)
NN.plot_stats(history)
#print(history)
#print(history.history.keys())
print("Done!")
label = r[1]
X_all.append(tweet)
Y_all.append(label)
i = i + 1
print("Data Ingested")
num_data = len(X_all)
limit = math.ceil(num_data * 0.60)
X_train_sentences = X_all
Y_train = Y_all
G = GloveEmbedding(embedding_filename)
word_to_idx, idx_to_word, embedding = G.read_embedding()
S = SentenceToIndices(word_to_idx)
X_train_indices, max_len = S.map_sentence_list(X_train_sentences)
print("Train data mappend to indices")
P = PadSentences(max_len)
X_train_pad = P.pad_list(X_train_indices)
print("Train data padded")
# convert to numPY arrays
X_train = np.array(X_train_pad)
Y_train = np.array(Y_train)
Y_train = to_categorical(Y_train, num_classes=3)
print("Train data convert to numpy arrays")
model = KerasClassifier(build_fn=create_model(G, max_len))
print("Model created")
# define the grid search parameters
batch_size = [10, 20, 40, 60, 80, 100]
epochs = [10, 50, 100]
param_grid = dict(batch_size=batch_size, epochs=epochs)
grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=1)
print(type(grid))
print("GRID SEARCH WORKING")
def set_prediction(pretain):
data = pretain[:, 0]
#Load Model
model2 = load_model('trained/model1_50d_stoplemma_10e_new_prod.h5',
custom_objects={'tf': tf})
# summarize model.
model2.summary()
# Load data
G = GloveEmbedding("data/glove.twitter.27B.50d.txt", dimensions=50)
word_to_idx, idx_to_word, embedding = G.read_embedding()
S = SentenceToIndices(word_to_idx)
premise = "same busy and just over the flu so feeling great"
premise = "when ebola struck the doctors stepped up to the plate and the rest of us sat and watched them do their stuff to all engineers and environmentalists this is our time to step up and find answers to these consequences of our failure to coexist with nature"
premise = remove_stopwords(premise)
premise = lemmatizer_spacy(premise)
x_premise = remove_stopwords(premise)
x_premise = np.full((len(data)), x_premise)
x_hypothesis = []
for row in data:
#row = row.replace("’", "'")
#row = fix_text_format(row)
row = remove_stopwords(row)
#row = lemmatizer_spacy(row)
#row = remove_stopwords(row)
x_hypothesis.append(row)
x_hypothesis = np.array(x_hypothesis)
X_one_indices, max_len1 = map_to_idx(S, x_premise)
X_two_indices, max_len2 = map_to_idx(S, x_hypothesis)
print("len: ", max_len1, max_len2)
#max_len = max(max_len1, max_len2)
max_len = 44
print("max_len_final: ", max_len)
P = PadSentences(max_len)
Trim = TrimSentences(max_len)
X_one_train = P.pad_list(X_one_indices)
X_two_train = P.pad_list(X_two_indices)
#X_one_train = Trim.trim_list(X_one_indices)
X_two_train = Trim.trim_list(X_two_train)
X_one_train = np.array(X_one_train)
X_two_train = np.array(X_two_train)
X_one_aux_disease = set_disease(x_premise)
X_two_aux_disease = set_disease(x_hypothesis)
new_dis = []
for _ in range(len(data)):
new_dis.append([0, 0, 1, 0, 0, 1])
X_one_aux_train = new_dis
X_two_aux_label = pretain[:, 1]
#X_one_aux_train = binarize_aux(s4, X_one_aux_label)
X_two_aux_train = binarize_aux(X_two_aux_disease, X_two_aux_label)
#new_two = []
#for row in range(len(data)):
# new_two.append([row[0], row[1], row[2], row[3], row[4], row[5], row[6]])
X_two_aux_train = X_two_aux_train.tolist()
for row in X_two_aux_train:
del row[6]
X_one_aux_train = np.array(X_one_aux_train)
X_two_aux_train = np.array(X_two_aux_train)
print("one_aux: ", np.array(X_one_aux_train).shape)
print(X_one_aux_train[:5])
print("two_aux: ", np.array(X_two_aux_train).shape)
print(X_two_aux_train[:5])
model2.load_weights('trained/model1_50d_stoplemma_10e_prod.h5')
#model2.compile(optimizer='rmsprop',loss={'R1': 'mean_squared_error'},metrics={'R1': 'mse'}, loss_weights={'R1': 0.25})
#model2.compile(optimizer='rmsprop')
#model2.load_weights('trained/model1_50d_stoplemma_10e_prod.h5')
X_Prediction = model2.predict(
[X_one_train, X_two_train, X_one_aux_train, X_two_aux_train])
return X_Prediction
def main(model_file, model_weights, labeled_tweets, embedding_filename):
# load json and create model
json_file = open(model_file, 'r')
loaded_model_json = json_file.read()
json_file.close()
loaded_model = model_from_json(loaded_model_json)
# load weights into new model
loaded_model.load_weights(model_weights)
print("Loaded model from disk")
# evaluate loaded model on test data
loaded_model.compile(loss='binary_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
# open the file with tweets
X_all = []
Y_all = []
All = []
with open(labeled_tweets, "r", encoding="ISO-8859-1") as f:
i = 0
csv_file = csv.reader(f, delimiter=',')
ones_count = 0
for r in csv_file:
if i != 0:
label = int(r[1])
if (label == 1) or (label == 2):
if ones_count <= 13000:
All.append(r)
ones_count += 1
else:
All.append(r)
# tweet = r[0]
# label = r[1]
# X_all.append(tweet)
# Y_all.append(label)
i = i + 1
ones_count = 0
for r in All:
tweet = r[0]
label = int(r[1])
if (label == 2):
label = 0
# if (label == 1) and (ones_count <= 4611):
# X_all.append(tweet)
# Y_all.append(label)
# ones_count +=1
# elif (label == 0):
X_all.append(tweet)
Y_all.append(label)
print("Data Ingested")
# divide the data into training and test
num_data = len(X_all)
limit = math.ceil(num_data * 0.60)
X_train_sentences = X_all
Y_train = Y_all
# divide the data into X_train, Y_train, X_test, Y_test
# X_train_sentences = X_all[0: limit]
# Y_train = Y_all[0: limit]
# X_test_sentences = X_all[limit:]
# Y_test = Y_all[limit:]
# print("Data Divided")
# Get embeeding
# G = Word2VecEmbedding(self.embedding_filename, dimensions=vect_dimensions)
G = GloveEmbedding(embedding_filename, dimensions=50)
word_to_idx, idx_to_word, embedding = G.read_embedding()
S = SentenceToIndices(word_to_idx)
X_train_indices, max_len = S.map_sentence_list(X_train_sentences)
print("Train data mappend to indices")
if max_len % 2 != 0:
max_len = max_len + 1
P = PadSentences(max_len)
X_train_pad = P.pad_list(X_train_indices)
print("Train data padded")
# TRIM
trim_size = max_len
Trim = TrimSentences(trim_size)
X_train_pad = Trim.trim_list(X_train_pad)
print("X[0], ", X_train_pad[0])
# convert to numPY arrays
X_train = np.array(X_train_pad)
Y_train = np.array(Y_train)
ones_count = np.count_nonzero(Y_train)
zeros_count = len(Y_train) - ones_count
print("ones count: ", ones_count)
print("zeros count: ", zeros_count)
# Y_train = to_categorical(Y_train, num_classes=3)
print("Train data convert to numpy arrays")
Preds = loaded_model.predict(X_train)
Preds = ((Preds >= 0.5) * 1).flatten()
with open("data/alltweetsanderrors.csv", "w") as f:
csv_writer = csv.writer(f, delimiter=",")
i = 0
err_count = 0
for r in All:
tweet = r[0]
label = int(r[1])
if label == 2:
label = 0
if Preds[i] != label:
err_count += 1
condition = 0
else:
condition = 1
error_pred = []
error_pred.append(tweet)
error_pred.append(label)
error_pred.append(Preds[i])
error_pred.append(condition)
csv_writer.writerow(error_pred)
i += 1
print("All tweets: ", i)
print("Error count: ", err_count)
def process(self,
json_filename,
h5_filename,
plot=False,
epochs=100,
vect_dimensions=100):
np.random.seed(11)
# open the file with tweets
X_all = []
Y_all = []
All = []
with open(self.labeled_tweets_filename, "r",
encoding="ISO-8859-1") as f:
i = 0
csv_file = csv.reader(f, delimiter=',')
for r in csv_file:
if i != 0:
All.append(r)
# tweet = r[0]
# label = r[1]
# X_all.append(tweet)
# Y_all.append(label)
i = i + 1
np.random.shuffle(All)
for r in All:
tweet = r[0]
label = r[1]
if int(label) == 2:
label = '0'
X_all.append(tweet)
Y_all.append(label)
print("Data Ingested")
# divide the data into training and test
num_data = len(X_all)
limit = math.ceil(num_data * 0.60)
X_train_sentences = X_all
Y_train = Y_all
# divide the data into X_train, Y_train, X_test, Y_test
#X_train_sentences = X_all[0: limit]
#Y_train = Y_all[0: limit]
#X_test_sentences = X_all[limit:]
#Y_test = Y_all[limit:]
#print("Data Divided")
#Get embeeding
G = Word2VecEmbedding(self.embedding_filename,
dimensions=vect_dimensions)
#G = GloveEmbedding(self.embedding_filename, dimensions=50)
word_to_idx, idx_to_word, embedding = G.read_embedding()
S = SentenceToIndices(word_to_idx)
X_train_indices, max_len = S.map_sentence_list(X_train_sentences)
print("Train data mappend to indices")
if max_len % 2 != 0:
max_len = max_len + 1
P = PadSentences(max_len)
X_train_pad = P.pad_list(X_train_indices)
print("Train data padded")
# TRIM
trim_size = 40
Trim = TrimSentences(trim_size)
X_train_pad = Trim.trim_list(X_train_pad)
print("X[0], ", X_train_pad[0])
#convert to numPY arrays
X_train = np.array(X_train_pad)
Y_train = np.array(Y_train)
#Y_train = to_categorical(Y_train, num_classes=3)
print("Train data convert to numpy arrays")
#NN = TweetSentiment2DCNN(trim_size, G)
NN = TweetSentiment2DCNNv4(trim_size, G)
#print("Build GRU")
#NN = TweetSentimentGRUSM(max_len, G)
print("model created")
kernel_regularizer = l2(0.001)
#kernel_regularizer = None
NN.build(filters=3,
first_dropout=0.01,
second_dropout=0.01,
padding='valid',
dense_units=16)
print("model built")
NN.summary()
sgd = SGD(lr=0.03, momentum=0.009, decay=0.001, nesterov=True)
rmsprop = RMSprop(decay=0.003)
adam = Adam(lr=0.1, decay=0.05)
sgd = SGD(lr=0.05)
NN.compile(optimizer='adam',
loss="binary_crossentropy",
metrics=['accuracy', f1, precision, recall])
print("model compiled")
print("Begin training")
callback = TensorBoard(log_dir="/tmp/logs")
history = NN.fit(X_train,
Y_train,
epochs=epochs,
batch_size=32,
callbacks=[callback],
validation_split=0.4)
print("Model trained")
# X_test_indices, max_len = S.map_sentence_list(X_test_sentences)
# print("Test data mapped")
# X_test_pad = P.pad_list(X_test_indices)
# print("Test data padded")
# X_test = np.array(X_test_pad)
# Y_test = np.array(Y_test)
# print("Test data converted to numpy arrays")
# loss, acc = NN.evaluate(X_test, Y_test, callbacks=[callback])
# print("accuracy: ", acc)
T = "I have a bad case of vomit"
X_Predict = [
"my zika is bad", "i love colombia",
"my has been tested for ebola",
"there is a diarrhea outbreak in the city"
]
X_Predict_Idx, max_len2 = S.map_sentence_list(X_Predict)
i = 0
for s in X_Predict_Idx:
print(str(i) + ": ", s)
i = i + 1
print(X_Predict)
X_Predict_Final = P.pad_list(X_Predict_Idx)
X_Predict_Final = Trim.trim_list(X_Predict_Final)
#X_Predict = [X_Predict]
X_Predict_Final = np.array(X_Predict_Final)
print("Predict: ", NN.predict(X_Predict_Final))
print("Storing model and weights")
NN.save_model(json_filename, h5_filename)
if plot:
print("Ploting")
self.plot(history)
print("Done!")
def process(self, json_filename, h5_filename, plot=False, epochs=100):
np.random.seed(11)
# open the file with tweets
X_all = []
Y_all = []
with open(self.labeled_tweets_filename, "r",
encoding="ISO-8859-1") as f:
i = 0
csv_file = csv.reader(f, delimiter=',')
for r in csv_file:
if i != 0:
tweet = r[0]
label = r[1]
X_all.append(tweet)
Y_all.append(label)
i = i + 1
print("Data Ingested")
# divide the data into training and test
num_data = len(X_all)
limit = math.ceil(num_data * 0.60)
X_train_sentences = X_all
Y_train = Y_all
# divide the data into X_train, Y_train, X_test, Y_test
#X_train_sentences = X_all[0: limit]
#Y_train = Y_all[0: limit]
#X_test_sentences = X_all[limit:]
#Y_test = Y_all[limit:]
#print("Data Divided")
#Get embeeding
G = GloveEmbedding(self.embedding_filename)
word_to_idx, idx_to_word, embedding = G.read_embedding()
S = SentenceToIndices(word_to_idx)
X_train_indices, max_len = S.map_sentence_list(X_train_sentences)
print("Train data mappend to indices")
P = PadSentences(max_len)
X_train_pad = P.pad_list(X_train_indices)
print("Train data padded")
#convert to numPY arrays
X_train = np.array(X_train_pad)
Y_train = np.array(Y_train)
Y_train = to_categorical(Y_train, num_classes=3)
print("Train data convert to numpy arrays")
NN = TweetSentiment2LSTM2DenseSM(max_len, G)
print("model created")
kernel_regularizer = l2(0.001)
kernel_regularizer = None
NN.build(first_layer_units=max_len,
second_layer_units=max_len,
relu_dense_layer=5,
dense_layer_units=3,
first_layer_dropout=0.3,
second_layer_dropout=0.6,
l2=kernel_regularizer)
print("model built")
NN.summary()
sgd = SGD(lr=0.001, momentum=0.09, decay=0.001, nesterov=True)
rmsprop = RMSprop(decay=0.003)
adam = Adam(lr=0.1, decay=0.05)
NN.compile(optimizer=rmsprop,
loss="categorical_crossentropy",
metrics=['accuracy', precision, recall, f1, fprate])
print("model compiled")
print("Begin training")
callback = TensorBoard(log_dir="/tmp/logs")
w_dict = {0: 0.31, 1: 0.63, 2: 0.06}
history = NN.fit(X_train,
Y_train,
epochs=epochs,
callbacks=[callback],
validation_split=0.2,
class_weight=w_dict)
print("Model trained")
# X_test_indices, max_len = S.map_sentence_list(X_test_sentences)
# print("Test data mapped")
# X_test_pad = P.pad_list(X_test_indices)
# print("Test data padded")
# X_test = np.array(X_test_pad)
# Y_test = np.array(Y_test)
# print("Test data converted to numpy arrays")
# loss, acc = NN.evaluate(X_test, Y_test, callbacks=[callback])
# print("accuracy: ", acc)
T = "I have a bad case of vomit"
X_Predict = [
"my zika is bad", "i love colombia",
"my has been tested for ebola",
"there is a diarrhea outbreak in the city"
]
X_Predict_Idx, max_len2 = S.map_sentence_list(X_Predict)
i = 0
for s in X_Predict_Idx:
print(str(i) + ": ", s)
i = i + 1
print(X_Predict)
X_Predict_Final = P.pad_list(X_Predict_Idx)
#X_Predict = [X_Predict]
X_Predict_Final = np.array(X_Predict_Final)
print("Predict: ", NN.predict(X_Predict_Final))
print("Storing model and weights")
NN.save_model(json_filename, h5_filename)
if plot:
print("Ploting")
self.plot(history)
print("Done!")
def process(self, json_filename, h5_filename):
np.random.seed(11)
# open the file with tweets
X_all = []
Y_all = []
with open(self.labeled_tweets_filename, "r",
encoding="ISO-8859-1") as f:
i = 0
csv_file = csv.reader(f, delimiter=',')
for r in csv_file:
if i != 0:
tweet = r[0]
label = r[1]
X_all.append(tweet)
Y_all.append(label)
i = i + 1
print("Data Ingested")
# divide the data into training and test
num_data = len(X_all)
limit = math.ceil(num_data * 0.60)
# divide the data into X_train, Y_train, X_test, Y_test
X_train_sentences = X_all[0:limit]
Y_train = Y_all[0:limit]
X_test_sentences = X_all[limit:]
Y_test = Y_all[limit:]
print("Data Divided")
#Get embeeding
G = GloveEmbedding(self.embedding_filename)
word_to_idx, idx_to_word, embedding = G.read_embedding()
S = SentenceToIndices(word_to_idx)
X_train_indices, max_len = S.map_sentence_list(X_train_sentences)
print("Train data mappend to indices")
P = PadSentences(max_len)
X_train_pad = P.pad_list(X_train_indices)
print("Train data padded")
# Trim
#trim_size = 40
#Trim = TrimSentences(trim_size)
#X_train_pad = Trim.trim_list(X_train_pad)
#convert to numPY arrays
X_train = np.array(X_train_pad)
Y_train = np.array(Y_train)
print("Train data convert to numpy arrays")
NN = TweetSentiment2LSTM2Dense(max_len, G)
#NN = TweetSentiment2LSTM2Dense(trim_size, G)
print("model created")
NN.build(first_layer_units=128,
dense_layer_units=1,
first_layer_dropout=0,
second_layer_dropout=0)
print("model built")
NN.summary()
sgd = SGD(lr=0.3, momentum=0.001, decay=0.01, nesterov=False)
adam = Adam(lr=0.03)
#NN.compile(loss="binary_crossentropy", metrics=['binary_accuracy'], optimizer=adam)
NN.compile(loss="binary_crossentropy",
metrics=['binary_accuracy'],
optimizer='rmsprop')
print("model compiled")
print("Begin training")
callback = TensorBoard(log_dir="/tmp/logs")
NN.fit(X_train, Y_train, epochs=5, callbacks=[callback])
print("Model trained")
X_test_indices, max_len = S.map_sentence_list(X_test_sentences)
print("Test data mapped")
X_test_pad = P.pad_list(X_test_indices)
print("Test data padded")
X_test = np.array(X_test_pad)
Y_test = np.array(Y_test)
print("Test data converted to numpy arrays")
loss, acc = NN.evaluate(X_test, Y_test)
print("accuracy: ", acc, ", loss: ", loss)
T = "I have a bad case of vomit"
X_Predict = [
"my zika is bad", "i love colombia",
"my has been tested for ebola",
"there is a diarrhea outbreak in the city"
]
X_Predict_Idx, max_len2 = S.map_sentence_list(X_Predict)
i = 0
for s in X_Predict_Idx:
print(str(i) + ": ", s)
i = i + 1
print(X_Predict)
X_Predict_Final = P.pad_list(X_Predict_Idx)
#X_Predict_Final = Trim.trim_list(X_Predict_Final)
#X_Predict = [X_Predict]
X_Predict_Final = np.array(X_Predict_Final)
print("Predict: ", NN.predict(X_Predict_Final))
print("Storing model and weights")
NN.save_model(json_filename, h5_filename)
print("Done!")