def model_selection(model_name):
if model_name == "cnn":
return TextCNN(sequence_length=train_x.shape[1],
num_classes=train_y.shape[1],
vocab_size=len(vocab_processor.vocabulary_),
embedding_size=FLAGS.embedding_size,
filter_sizes=list(
map(int, FLAGS.filter_sizes.split(","))),
num_filters=FLAGS.num_filters,
l2_reg_lambda=FLAGS.l2_reg_lambda)
elif model_name == "rnn":
return TextRNN(sequence_length=max_document_length,
num_classes=train_y.shape[1],
vocab_size=len(vocab_processor.vocabulary_),
embedding_size=FLAGS.embedding_size,
learning_rate=FLAGS.learning_rate,
batch_size=FLAGS.batch_size,
decay_steps=FLAGS.decay_steps,
decay_rate=FLAGS.decay_rate,
is_training=FLAGS.is_training)
elif model_name == "rcnn":
return TextRCNN(sequence_length=train_x.shape[1],
num_classes=train_y.shape[1],
vocab_size=len(vocab_processor.vocabulary_),
embedding_size=FLAGS.embedding_size,
context_embedding_size=FLAGS.context_embedding_size,
cell_type=FLAGS.cell_type,
hidden_size=FLAGS.hidden_size,
l2_reg_lambda=FLAGS.l2_reg_lambda)
elif model_name == "clstm":
return TextCLSTM(max_len=max_document_length,
num_classes=train_y.shape[1],
vocab_size=len(vocab_processor.vocabulary_),
embedding_size=FLAGS.embedding_size,
filter_sizes=list(
map(int, FLAGS.filter_sizes.split(","))),
num_filters=FLAGS.num_filters,
num_layers=FLAGS.num_layers,
l2_reg_lambda=FLAGS.l2_reg_lambda)
else:
raise NotImplementedError("%s is not implemented" % (model_name))
X_train, Y_train, X_test, Y_true, vocab_processor = utils.get_text_data()
# print("fitting/saving")
# clf = Pipeline([ ("word2vec vectorizer", TfidfEmbeddingVectorizer(w2v)),
# ("logistic regression", linear_model.LogisticRegression())])
with tf.Graph().as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
sess = tf.Session(config=session_conf)
with sess.as_default():
cnn = TextCNN(sequence_length=X_train.shape[1],
num_classes=2,
vocab_size=len(vocab_processor.vocabulary_),
embedding_size=FLAGS.embedding_dim,
filter_sizes=list(map(int,
FLAGS.filter_sizes.split(","))),
num_filters=FLAGS.num_filters)
global_step = tf.Variable(0, name="global_step", trainable=False)
optimizer = tf.train.AdamOptimizer(1e-4)
grads_and_vars = optimizer.compute_gradients(cnn.loss)
train_op = optimizer.apply_gradients(grads_and_vars,
global_step=global_step)
# Output directory for models and summaries
timestamp = str(int(time.time()))
out_dir = os.path.abspath(
os.path.join(os.path.curdir, "runs", timestamp))
print("Writing to {}\n".format(out_dir))
def train(x_train, y_train, vocab_processor, x_dev, y_dev):
with tf.Graph().as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
sess = tf.Session(config=session_conf)
with sess.as_default():
cnn = TextCNN(sequence_length=x_train.shape[1],
num_classes=y_train.shape[1],
vocab_size=len(vocab_processor.vocabulary_),
embedding_size=FLAGS.embedding_dim,
filter_sizes=list(
map(int, FLAGS.filter_sizes.split(","))),
num_filters=FLAGS.num_filters,
l2_reg_lambda=FLAGS.l2_reg_lambda)
# Define Training procedure
global_step = tf.Variable(0, name="global_step", trainable=False)
optimizer = tf.train.AdamOptimizer(1e-3)
grads_and_vars = optimizer.compute_gradients(cnn.loss)
train_op = optimizer.apply_gradients(grads_and_vars,
global_step=global_step)
# Keep track of gradient values and sparsity (optional)
grad_summaries = []
for g, v in grads_and_vars:
if g is not None:
grad_hist_summary = tf.summary.histogram(
"{}/grad/hist".format(v.name), g)
sparsity_summary = tf.summary.scalar(
"{}/grad/sparsity".format(v.name),
tf.nn.zero_fraction(g))
grad_summaries.append(grad_hist_summary)
grad_summaries.append(sparsity_summary)
grad_summaries_merged = tf.summary.merge(grad_summaries)
# Output directory for models and summaries
timestamp = str(int(time.time()))
out_dir = os.path.abspath(
os.path.join(os.path.curdir, "runs", timestamp))
print("Writing to {}\n".format(out_dir))
# Summaries for loss and accuracy
loss_summary = tf.summary.scalar("loss", cnn.loss)
acc_summary = tf.summary.scalar("accuracy", cnn.accuracy)
# Train Summaries
train_summary_op = tf.summary.merge(
[loss_summary, acc_summary, grad_summaries_merged])
train_summary_dir = os.path.join(out_dir, "summaries", "train")
train_summary_writer = tf.summary.FileWriter(
train_summary_dir, sess.graph)
# Dev summaries
dev_summary_op = tf.summary.merge([loss_summary, acc_summary])
dev_summary_dir = os.path.join(out_dir, "summaries", "dev")
dev_summary_writer = tf.summary.FileWriter(dev_summary_dir,
sess.graph)
# Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it
checkpoint_dir = os.path.abspath(
os.path.join(out_dir, "checkpoints"))
checkpoint_prefix = os.path.join(checkpoint_dir, "model")
maxacc_prefix = os.path.join(checkpoint_dir, "maxaccmodel")
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
saver = tf.train.Saver(tf.global_variables(),
max_to_keep=FLAGS.num_checkpoints)
# Write vocabulary
vocab_processor.save(os.path.join(out_dir, "vocab"))
# Initialize all variables
sess.run(tf.global_variables_initializer())
def train_step(x_batch, y_batch):
"""
A single training step
"""
feed_dict = {
cnn.input_x: x_batch,
cnn.input_y: y_batch,
cnn.dropout_keep_prob: FLAGS.dropout_keep_prob
}
_, step, summaries, loss, accuracy = sess.run([
train_op, global_step, train_summary_op, cnn.loss,
cnn.accuracy
], feed_dict)
time_str = datetime.datetime.now().isoformat()
print("{}: step {}, loss {:g}, acc {:g}".format(
time_str, step, loss, accuracy))
train_summary_writer.add_summary(summaries, step)
def dev_step(x_batch, y_batch, writer=None):
"""
Evaluates model on a dev set
"""
feed_dict = {
cnn.input_x: x_batch,
cnn.input_y: y_batch,
cnn.dropout_keep_prob: 1.0
}
step, summaries, loss, accuracy = sess.run(
[global_step, dev_summary_op, cnn.loss, cnn.accuracy],
feed_dict)
time_str = datetime.datetime.now().isoformat()
result = "{}: step {}, loss {:g}, acc {:g}".format(
time_str, step, loss, accuracy)
print(result)
with open(os.path.join(out_dir, "result"), 'a+') as f:
f.write("{}\n".format(result))
if writer:
writer.add_summary(summaries, step)
if tf.MaxAcc < accuracy:
tf.MaxAcc = accuracy
print("Max acc: {:g}".format(tf.MaxAcc))
return True
else:
print("Max acc: {:g}".format(tf.MaxAcc))
return False
# Generate batches
batches = parse_data.batch_iter(list(zip(x_train, y_train)),
FLAGS.batch_size, FLAGS.num_epochs)
# Training loop. For each batch...
for batch in batches:
x_batch, y_batch = zip(*batch)
train_step(x_batch, y_batch)
current_step = tf.train.global_step(sess, global_step)
if current_step % FLAGS.evaluate_every == 0:
print("\nEvaluation:")
save = dev_step(x_dev, y_dev, writer=dev_summary_writer)
print()
if current_step % FLAGS.checkpoint_every == 0:
path = saver.save(sess,
checkpoint_prefix,
global_step=current_step)
print("Saved model checkpoint to {}\n".format(path))
if save:
path = saver.save(sess, maxacc_prefix, None)
print("Saved max acc model checkpoint to {}\n".format(
path))
copy("{}.data-00000-of-00001".format(path))
copy("{}.index".format(path))
copy("{}.meta".format(path))
with tf.device('/gpu:%s' % 3):
with tf.name_scope('%s_%s' % ('tower', 3)):
with tf.Graph().as_default():
session_conf = tf.ConfigProto(
# allow_soft_placement=True,
log_device_placement=False, )
# session_conf.gpu_options.allow_growth = True
sess = tf.Session(config=session_conf) # 初始化 session
with sess.as_default():
cnn = TextCNN(sequence_length=x_train.shape[1],
num_classes=1,
vocab_size=len(vocab_processor.vocabulary_),
embedding_size=100,
filter_sizes=[2, 3, 4, 5],
num_filters=256,
l2_reg_lambda=0.001)
threashold = 0.25 #仅用于人工观察图
# 定义 training 过程
global_step = tf.Variable(0,
name='global_step',
trainable=False)
optimizer = tf.train.AdamOptimizer(0.0005)
grads_and_vars = optimizer.compute_gradients(cnn.loss)
train_op = optimizer.apply_gradients(grads_and_vars,
global_step=global_step)
# 记录 gradient 和 sparsity (画图)
def train():
train_file = 'data/emotion Phrases.csv'
x_raw, y_raw, df, labels, embedding_mat = preprocessing.load_data(
train_file)
parameter_file = './parameters.json'
params = json.loads(open(parameter_file).read())
max_document_length = max([len(x.split(' ')) for x in x_raw])
logging.info(
'The maximum length of all sentences: {}'.format(max_document_length))
vocab_processor = learn.preprocessing.VocabularyProcessor(
max_document_length)
x = np.array(list(vocab_processor.fit_transform(x_raw)))
y = np.array(y_raw)
# print x.shape
x_, x_test, y_, y_test = train_test_split(x,
y,
test_size=0.2,
random_state=42)
shuffle_indices = np.random.permutation(np.arange(len(y_)))
x_shuffled = x_[shuffle_indices]
y_shuffled = y_[shuffle_indices]
x_train, x_dev, y_train, y_dev = train_test_split(x_shuffled,
y_shuffled,
test_size=0.2)
# with open('labels.json', 'w') as outfile:
# json.dump(labels, outfile, indent=4)
logging.info('x_train: {}, x_dev: {}, x_test: {}'.format(
len(x_train), len(x_dev), len(x_test)))
logging.info('y_train: {}, y_dev: {}, y_test: {}'.format(
len(y_train), len(y_dev), len(y_test)))
graph = tf.Graph()
with graph.as_default():
session_conf = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
sess = tf.Session(config=session_conf)
with sess.as_default():
cnn = TextCNN(sequence_length=x_train.shape[1],
num_classes=y_train.shape[1],
vocab_size=9017,
embedding_size=params['embedding_dim'],
filter_sizes=list(
map(int, params['filter_sizes'].split(","))),
num_filters=params['num_filters'],
embedding_mat=embedding_mat,
l2_reg_lambda=params['l2_reg_lambda'])
global_step = tf.Variable(0, name="global_step", trainable=False)
optimizer = tf.train.AdamOptimizer(1e-3)
grads_and_vars = optimizer.compute_gradients(cnn.loss)
train_op = optimizer.apply_gradients(grads_and_vars,
global_step=global_step)
timestamp = str(int(time.time()))
out_dir = os.path.abspath(
os.path.join(os.path.curdir, "trained_model_" + timestamp))
checkpoint_dir = os.path.abspath(
os.path.join(out_dir, "checkpoints"))
checkpoint_prefix = os.path.join(checkpoint_dir, "model")
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
saver = tf.train.Saver(tf.global_variables())
# One training step: train the model with one batch
def train_step(x_batch, y_batch):
feed_dict = {
cnn.input_x: x_batch,
cnn.input_y: y_batch,
cnn.dropout_keep_prob: params['dropout_keep_prob']
}
_, step, loss, acc = sess.run(
[train_op, global_step, cnn.loss, cnn.accuracy], feed_dict)
# One evaluation step: evaluate the model with one batch
def dev_step(x_batch, y_batch):
feed_dict = {
cnn.input_x: x_batch,
cnn.input_y: y_batch,
cnn.dropout_keep_prob: 1.0
}
step, loss, acc, num_correct = sess.run(
[global_step, cnn.loss, cnn.accuracy, cnn.num_correct],
feed_dict)
return num_correct
# Save the word_to_id map since predict.py needs it
vocab_processor.save(os.path.join(out_dir, "vocab.pickle"))
sess.run(tf.global_variables_initializer())
print "Loading Embeddings !"
initW = preprocessing.load_embedding_vectors(
vocab_processor.vocabulary_)
#
#x = np.shape(a)
#print(x)
print(np.shape(initW))
sess.run(cnn.W.assign(initW))
print "Loaded Embeddings !"
# Training starts here
train_batches = preprocessing.generate_batches(
list(zip(x_train, y_train)), params['batch_size'],
params['num_epochs'])
best_accuracy, best_at_step = 0, 0
for train_batch in train_batches:
if len(train_batch) == 0:
continue
x_train_batch, y_train_batch = zip(*train_batch)
train_step(x_train_batch, y_train_batch)
current_step = tf.train.global_step(sess, global_step)
if current_step % params['evaluate_every'] == 0:
dev_batches = preprocessing.generate_batches(
list(zip(x_dev, y_dev)), params['batch_size'], 1)
total_dev_correct = 0
for dev_batch in dev_batches:
if len(dev_batch) == 0:
continue
x_dev_batch, y_dev_batch = zip(*dev_batch)
num_dev_correct = dev_step(x_dev_batch, y_dev_batch)
total_dev_correct += num_dev_correct
dev_accuracy = float(total_dev_correct) / len(y_dev)
logging.critical(
'Accuracy on dev set: {}'.format(dev_accuracy))
if dev_accuracy >= best_accuracy:
best_accuracy, best_at_step = dev_accuracy, current_step
path = saver.save(sess,
checkpoint_prefix,
global_step=current_step)
logging.critical('Saved model {} at step {}'.format(
path, best_at_step))
logging.critical('Best accuracy {} at step {}'.format(
best_accuracy, best_at_step))
test_batches = preprocessing.generate_batches(
list(zip(x_test, y_test)), params['batch_size'], 1)
total_test_correct = 0
for test_batch in test_batches:
if len(test_batch) == 0:
continue
print "Non Zero Length"
x_test_batch, y_test_batch = zip(*test_batch)
num_test_correct = dev_step(x_test_batch, y_test_batch)
total_test_correct += num_test_correct
test_accuracy = float(total_test_correct) / len(y_test)
train_batches = preprocessing.generate_batches(
list(zip(x_train, y_train)), params['batch_size'], 1)
total_train_correct = 0
for train_batch in train_batches:
if len(train_batch) == 0:
continue
print "Non Zero Length"
x_train_batch, y_train_batch = zip(*train_batch)
num_test_correct = dev_step(x_train_batch, y_train_batch)
total_train_correct += num_test_correct
train_accuracy = float(total_train_correct) / len(y_train)
print 'Accuracy on test set is {} based on the best model'.format(
test_accuracy)
print 'Accuracy on train set is {} based on the best model'.format(
train_accuracy)
# logging.critical('Accuracy on test set is {} based on the best model {}'.format(test_accuracy, path))
logging.critical('The training is complete')
# Split train/test set (10%) for training
dev_sample_index = -1 * int(0.1 * float(len(y)))
x_train, x_test = x_shuffled[:dev_sample_index], x_shuffled[dev_sample_index:]
y_train, y_test = y_shuffled[:dev_sample_index], y_shuffled[dev_sample_index:]
print("Vocabulary Size: {:d}".format(len(vocab_processor.vocabulary_)))
with tf.Graph().as_default():
session_conf = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)
sess = tf.Session(config=session_conf)
with sess.as_default():
cnn = TextCNN(x_train.shape[1],
y_train.shape[1],
vocab_size=len(vocab_processor.vocabulary_))
writer = tf.summary.FileWriter('logs')
writer.add_graph(sess.graph)
# Define Training procedure
global_step = tf.Variable(0, name="global_step", trainable=False)
optimizer = tf.train.AdamOptimizer(1e-3)
grads_and_vars = optimizer.compute_gradients(cnn.loss)
train_op = optimizer.apply_gradients(grads_and_vars,
global_step=global_step)
# Output directory for models
timestamp = str(int(time.time()))
out_dir = os.path.abspath(
session_conf = tf.ConfigProto(allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
with tf.Session(config=session_conf) as sess:
seq_length = x_train.shape[1]
num_class = y_train.shape[1]
voc_size = len(voc)
print('initialize cnn filter')
print('sequence length %d, number of class %d, vocab size %d' %
(seq_length, num_class, voc_size))
cnn = TextCNN(seq_length, num_class, voc_size, FLAGS.embedding_dim,
list(map(int, FLAGS.filter_sizes.split(","))),
FLAGS.num_filters, FLAGS.l2_reg_lambda)
global_step = tf.Variable(0, name='global_step', trainable=False)
optimizer = tf.train.AdamOptimizer(1e-3)
grads_and_vars = optimizer.compute_gradients(cnn.loss)
train_op = optimizer.apply_gradients(grads_and_vars,
global_step=global_step)
# Keep track of gradient values and sparsity (optional)
grad_summaries = []
for g, v in grads_and_vars:
if g is not None:
grad_hist_summary = tf.summary.histogram(
"{}/grad/hist".format(v.name), g)
sparsity_summary = tf.summary.scalar(
print "Vocabulary Size: {:d}".format(len(vocb_processor.vocabulary_))
print "Train/Dev: {:d}/{:d}".format(len(x_train), len(x_dev))
graph = tf.Graph()
with graph.as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
sess = tf.Session(config=session_conf)
#sess = tf_debug.LocalCLIDebugWrapperSession(sess)
with sess.as_default():
cnn = TextCNN(sequence_length = x_train.shape[1],
num_classes = y_train.shape[1],
vocb_size = len(vocb_processor.vocabulary_),
embedding_size = FLAGS.embedding_size,
filter_sizes = map(int, FLAGS.filter_sizes.split(",")),
num_filters = FLAGS.num_filters,
l2_reg_lambda = FLAGS.l2_reg_lambda
)
global_step = tf.Variable(0, trainable=False, name="global_step")
optimizer = tf.train.AdamOptimizer(1e-3)
grads_and_vars = optimizer.compute_gradients(cnn.loss)
train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step)
grad_summaries = []
for g, v in grads_and_vars:
if g is not None:
grad_hist_summary = tf.summary.histogram("{}/grad/hist".format(v.name), g)
sparsity_summary = tf.summary.scalar("{}/grad/sparsity".format(v.name), tf.nn.zero_fraction(g))
grad_summaries.append(grad_hist_summary)
grad_summaries.append(sparsity_summary)
def train():
if os.path.exists(VOCAB_FILENAME):
print('load prebuilt vocab file ')
vocab = load_vocab(VOCAB_FILENAME)
else:
print('build vocab from raw text')
data = read_raw_data(TOTAL_FILENAME)
tokens = [t for d in data for t in d[0]]
vocab = build_vocab(tokens)
print('save vocab file')
save_vocab(VOCAB_FILENAME, vocab)
if os.path.exists(TRAIN_DATA_FILENAME):
print('load prebuilt train data ')
input = load_data(TRAIN_DATA_FILENAME)
else:
print('build train data from raw text')
data = read_raw_data(TRAIN_FILENAME)
input = build_input(data, vocab)
print('save train data & vocab file')
save_data(TRAIN_DATA_FILENAME, input)
if os.path.exists(TEST_DATA_FILENAME):
print('load prebuilt test data')
test_input = load_data(TEST_DATA_FILENAME)
else:
print('build test data from raw text')
data = read_raw_data(TEST_FILENAME)
test_input = build_input(data, vocab)
print('save test data ')
save_data(TEST_DATA_FILENAME, test_input)
with tf.Session() as sess:
seq_length = np.shape(input[0][0])[0]
num_class = np.shape(input[0][1])[0]
print('initialize cnn filter')
print('sequence length %d, number of class %d, vocab size %d' % (seq_length, num_class, len(vocab)))
cnn = TextCNN(seq_length, num_class, len(vocab), 128, [3,4,5], 128, 0.0)
global_step = tf.Variable(0, name='global_step', trainable=False)
optimizer = tf.train.AdamOptimizer(1e-3)
grads_and_vars = optimizer.compute_gradients(cnn.loss)
train_op = optimizer.apply_gradients(grads_and_vars, global_step=global_step)
def train_step(x_batch, y_batch):
feed_dict = {
cnn.input : x_batch,
cnn.label : y_batch,
cnn.dropout_keep_prob : 0.5
}
_, step, loss, accuracy = sess.run([train_op, global_step, cnn.loss, cnn.accuracy], feed_dict)
def evaluate(x_batch, y_batch):
feed_dict = {
cnn.input : x_batch,
cnn.label : y_batch,
cnn.dropout_keep_prob : 1.0
}
step, loss, accuracy = sess.run([global_step, cnn.loss, cnn.accuracy], feed_dict)
print("step %d, loss %f, acc %f" % (step, loss, accuracy))
saver = tf.train.Saver()
sess.run(tf.global_variables_initializer())
for i in range(10000):
try:
batch = random.sample(input, 64)
x_batch, y_batch = zip(*batch)
train_step(x_batch, y_batch)
current_step = tf.train.global_step(sess, global_step)
if current_step % 100 == 0:
batch = random.sample(test_input, 64)
x_test, y_test = zip(*batch)
evaluate(x_test, y_test)
if current_step % 1000 == 0:
save_path = saver.save(sess, './textcnn.ckpt')
print('model saved : %s' % save_path)
except:
print ("Unexpected error:", sys.exc_info()[0])
raise
'l2_reg_lambda': 0.1,
'batch_size': 64,
'num_epochs': 40,
'allow_soft_placement': True,
'log_device_placement': False
}
if __name__ == '__main__':
trainx, trainy, testx, testy, vocab = data_helpers.data_process(FLAGS)
conf = tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False,
gpu_options=tf.GPUOptions(allow_growth=True))
with tf.Graph().as_default() as graph, tf.Session(conf) as sess:
model = TextCNN(vocab.embedding, FLAGS)
saver = tf.train.Saver(max_to_keep=20)
sess.run(tf.global_variables_initializer())
stime = time.time()
eval_acc = []
for epoch in range(FLAGS['num_epochs']):
# train
for batch in data_helpers.batch_iter(list(zip(trainx, trainy)),
FLAGS['batch_size'], 1):
x_batch, y_batch = zip(*batch)
step, loss, accuracy = model.train_step(sess, x_batch, y_batch)
if (step + 1) % 200 == 0:
print(
"train epoch: %d, spend-time: %.4f, step: %d, loss: %.4f, acc: %.4f"
% (epoch, time.time() - stime, step, loss, accuracy))