def main():
x_train, y_train, x_test, y_test, vocab_size, n_class, word2idx, tag2idx = pos.load_data()
X_train, Y_train = to_train_seq(x_train, y_train)
X_test, Y_test = to_test_seq(x_test, y_test)
print(X_train.shape, Y_train.shape, X_test.shape, Y_test.shape)
params = {'vocab_size': vocab_size, 'n_class': n_class}
tf_estimator = tf.estimator.Estimator(model_fn, params=params)
tf_estimator.train(tf.estimator.inputs.numpy_input_fn(
x = {'inputs':X_train}, y=Y_train,
batch_size = BATCH_SIZE, num_epochs = NUM_EPOCH, shuffle = True))
preds = tf_estimator.predict(tf.estimator.inputs.numpy_input_fn(
x = {'inputs':X_test}, batch_size = BATCH_SIZE, shuffle = False))
preds = np.array(list(preds))
final_acc = (preds == Y_test).mean()
print("final testing accuracy: %.4f" % final_acc)
sample = ['I', 'love', 'you']
idx = np.atleast_2d([word2idx[w] for w in sample] + [0] * (SEQ_LEN - len(sample)))
preds = tf_estimator.predict(tf.estimator.inputs.numpy_input_fn(
x={'inputs':idx}, batch_size=1, shuffle=False))
preds = np.array(list(preds))
idx2tag = {idx : tag for tag, idx in tag2idx.items()}
print(' '.join(sample))
print(' '.join([idx2tag[idx] for idx in preds[0][:len(sample)]]))
SEQ_LEN = 20
BATCH_SIZE = 32
sample = ['I', 'love', 'you']
def to_seq(*args):
data = []
for x in args:
x = x[:(len(x) - len(x) % SEQ_LEN)]
data.append(np.reshape(x, [-1, SEQ_LEN]))
return data
if __name__ == '__main__':
x_train, y_train, x_test, y_test, vocab_size, n_class, word2idx, tag2idx = pos.load_data(
)
X_train, X_test, Y_train, Y_test = to_seq(x_train, x_test, y_train, y_test)
print(X_train.shape, Y_train.shape, X_test.shape, Y_test.shape)
clf = BiRNN_CRF(SEQ_LEN, vocab_size, n_class)
clf.fit(X_train,
Y_train,
val_data=(X_test, Y_test),
keep_prob=0.8,
n_epoch=5,
batch_size=BATCH_SIZE)
y_pred = clf.predict(X_test, batch_size=BATCH_SIZE)
final_acc = (y_pred == Y_test.ravel()).mean()
print("final testing accuracy: %.4f" % final_acc)
idx2tag = {idx: tag for tag, idx in tag2idx.items()}
import pos
import numpy as np
from sklearn.tree import DecisionTreeClassifier
if __name__ == '__main__':
x_train, y_train, x_test, y_test, _, _, _, _ = pos.load_data()
X_train = np.expand_dims(x_train, 1)
X_test = np.expand_dims(x_test, 1)
model = DecisionTreeClassifier()
model.fit(X_train, y_train)
print(model.score(X_train, y_train))
print(model.score(X_test, y_test))
from birnn_crf_clf import BiRNN_CRF
SEQ_LEN = 20
BATCH_SIZE = 32
sample = ['I', 'love', 'you']
def to_seq(*args):
data = []
for x in args:
x = x[: (len(x) - len(x) % SEQ_LEN)]
data.append(np.reshape(x, [-1, SEQ_LEN]))
return data
if __name__ == '__main__':
x_train, y_train, x_test, y_test, vocab_size, n_class, word2idx, tag2idx = pos.load_data()
X_train, X_test, Y_train, Y_test = to_seq(x_train, x_test, y_train, y_test)
print(X_train.shape, Y_train.shape, X_test.shape, Y_test.shape)
clf = BiRNN_CRF(SEQ_LEN, vocab_size, n_class)
clf.fit(X_train, Y_train, val_data=(X_test, Y_test), keep_prob=0.8, n_epoch=5, batch_size=BATCH_SIZE)
y_pred = clf.predict(X_test, batch_size=BATCH_SIZE)
final_acc = (y_pred == Y_test.ravel()).mean()
print("final testing accuracy: %.4f" % final_acc)
idx2tag = {idx : tag for tag, idx in tag2idx.items()}
labels = clf.infer([word2idx[w] for w in sample])
print(' '.join(sample))
print(' '.join([idx2tag[idx] for idx in labels]))
import pos
import numpy as np
from sklearn.tree import DecisionTreeClassifier
if __name__ == '__main__':
x_train, y_train, x_test, y_test, _, _, _, _ = pos.load_data()
X_train = np.expand_dims(x_train, 1)
X_test = np.expand_dims(x_test, 1)
model = DecisionTreeClassifier()
model.fit(X_train, y_train)
print(model.score(X_train, y_train))
print(model.score(X_test, y_test))
