return clf1
# Function to predict tags using clf1.
# In[4]:
def predict_tags_clf1(clf1, X):
Y_pred = clf1.predict(X)
return Y_pred
# In[57]:
words_tr, tags_tr = get_all_data_train()
words_dev, tags_dev = get_all_data_dev()
max_f1 = 0.0
best_reg = None
for reg_param in [1.0, 3.0, 5.0, 7.0, 10.0]:
[X_train, Y_train, dict_vectorizer] = clf1_1hot_get_X_Y(words_tr, tags_tr)
clf1 = train_clf1(X_train, Y_train, reg_param)
[X_dev, Y_dev,
dict_vectorizer] = clf1_1hot_get_X_Y(words_dev, tags_dev, dict_vectorizer)
Y_pred_dev = predict_tags_clf1(clf1, X_dev)
P, R, F1 = evaluate_abstract_PRF1(Y_dev, Y_pred_dev)
if (F1 > max_f1):
max_f1 = F1
best_reg = reg_param
print max_f1
print best_reg
# word_dict.update(get_dict_extra_features(sentance[word_ind]))
index += 1
x_t = dict_vectorizer.transform([word_dict])
y_t = clf2.predict(x_t)
Y_pred.extend(y_t)
return Y_pred
# In[ ]:
# In[28]:
words_tr, tags_tr = get_all_data_train()
words_test, tags_test = get_all_data_test()
# In[29]:
n = 3
dict_list, Y_train = clf2_1hot_get_X_Y_dictlist(words_tr, tags_tr, n)
X_train, dict_vectorizer = get_clf2_X_train(dict_list, dict_vectorizer=None)
clf2 = train_clf2(X_train, Y_train, 1.0)
# In[33]:
Y_pred_tr = predict_tags_clf2(clf2, words_tr, n, dict_vectorizer)
tf.flags.DEFINE_boolean("log_device_placement", False,
"Log placement of ops on devices")
FLAGS = tf.flags.FLAGS
FLAGS._parse_flags()
print("\nParameters:")
for attr, value in sorted(FLAGS.__flags.items()):
print("{}={}".format(attr.upper(), value))
print("")
# ## Data Preperation
# In[3]:
# per sentence
word_array, tag_array = get_all_data_train(sentences=True)
X, Y = get_1_hot_sentence_encodings(word_array, tag_array)
# In[4]:
# print Y[0]
# print X[0]
# print word_array[0]
# print tag_array[0]
# In[5]:
# max_document_length = len(X[0])
# vocab_processor = learn.preprocessing.VocabularyProcessor(max_document_length)
# # Randomly shuffle data
# In[3]:
import numpy as np
import tensorflow as tf
session = tf.InteractiveSession()
# njl's Sentiment Analysis example
#https://github.com/nicholaslocascio/tensorflow-nlp-tutorial/blob/master/sentiment-analysis/Sentiment-RNN.ipynb
# In[6]:
from preprocess_data import get_all_data_train
from preprocess_data import x_dict_to_vect
word_array, tag_array = get_all_data_train()
# X,Y = abstracts2features(word_array[1:10],tag_array[1:10],(1,1),False, w2v_size=100)
# X_vect = x_dict_to_vect(X)
# In[5]:
data_placeholder = tf.placeholder(tf.float32, name='data_placeholder')
labels_placeholder = tf.placeholder(tf.float32, name='labels_placeholder')
# In[6]:
feed_dict_train = {
data_placeholder: batch_data,
labels_placeholder: batch_labels,
keep_prob_placeholder: keep_prob_rate
}