def construct(self, hidden_layer_size):
with self.session.graph.as_default():
with tf.name_scope("inputs"):
self.images = tf.placeholder(
tf.float32, [None, self.WIDTH, self.HEIGHT, 1],
name="images")
self.labels = tf.placeholder(tf.int64, [None], name="labels")
flattened_images = tf_layers.flatten(self.images,
scope="preprocessing")
hidden_layer = tf_layers.fully_connected(
flattened_images,
num_outputs=hidden_layer_size,
activation_fn=tf.nn.relu,
scope="hidden_layer")
output_layer = tf_layers.fully_connected(hidden_layer,
num_outputs=self.LABELS,
activation_fn=None,
scope="output_layer")
self.predictions = tf.argmax(output_layer, 1)
loss = tf_losses.sparse_softmax_cross_entropy(output_layer,
self.labels,
scope="loss")
self.global_step = tf.Variable(0,
dtype=tf.int64,
trainable=False,
name="global_step")
self.training = tf.train.AdamOptimizer().minimize(
loss, global_step=self.global_step)
self.accuracy = tf_metrics.accuracy(self.predictions, self.labels)
# Summaries
self.summaries = {
"training":
tf.merge_summary([
tf.scalar_summary("train/loss", loss),
tf.scalar_summary("train/accuracy", self.accuracy)
])
}
for dataset in ["dev", "test"]:
self.summaries[dataset] = tf.scalar_summary(
dataset + "/accuracy", self.accuracy)
# Initialize variables
self.session.run(tf.initialize_all_variables())
# Finalize graph and log it if requested
self.session.graph.finalize()
if self.summary_writer:
self.summary_writer.add_graph(self.session.graph)
def _softmax_entropy(probabilities, targets, weights=None):
return metric_ops.streaming_mean(losses.sparse_softmax_cross_entropy(
probabilities, math_ops.to_int32(targets)),
weights=weights)
def __init__(self,
rnn_cell,
rnn_cell_dim,
method,
data_train,
logdir,
expname,
threads,
restore_path,
seed=42):
n_words = len(data_train.factors[data_train.FORMS]['words'])
n_tags = len(data_train.factors[data_train.TAGS]['words'])
n_lemmas = len(data_train.factors[data_train.LEMMAS]['words'])
n_mwe = len(data_train.factors[data_train.MWE]['words'])
# Create an empty graph and a session
graph = tf.Graph()
graph.seed = seed
self.session = tf.Session(graph=graph,
config=tf.ConfigProto(
inter_op_parallelism_threads=threads,
intra_op_parallelism_threads=threads))
timestamp = datetime.datetime.now().strftime("%Y-%m-%d_%H%M%S")
self.summary_writer = tf.train.SummaryWriter("{}/{}-{}".format(
logdir, timestamp, expname),
flush_secs=10)
# Construct the graph
with self.session.graph.as_default():
if rnn_cell == "LSTM":
rnn_cell = tf.nn.rnn_cell.LSTMCell(rnn_cell_dim)
elif rnn_cell == "GRU":
rnn_cell = tf.nn.rnn_cell.GRUCell(rnn_cell_dim)
else:
raise ValueError("Unknown rnn_cell {}".format(rnn_cell))
self.global_step = tf.Variable(0,
dtype=tf.int64,
trainable=False,
name="global_step")
self.sentence_lens = tf.placeholder(tf.int32, [None])
self.forms = tf.placeholder(tf.int32, [None, None])
self.tags = tf.placeholder(tf.int32, [None, None])
self.lemmas = tf.placeholder(tf.int32, [None, None])
self.mwe = tf.placeholder(tf.int32, [None, None])
embeddings_forms = tf.get_variable("word_embedding_matrix",
[n_words, rnn_cell_dim],
dtype=tf.float32)
embeddings_lemmas = tf.get_variable("lemma_embedding_matrix",
[n_lemmas, rnn_cell_dim],
dtype=tf.float32)
embeddings_tags = tf.get_variable("tag_embedding_matrix",
[n_tags, rnn_cell_dim],
dtype=tf.float32)
embedding_in_forms = tf.nn.embedding_lookup(
embeddings_forms, self.forms)
embedding_in_lemmas = tf.nn.embedding_lookup(
embeddings_lemmas, self.lemmas)
embedding_in_tags = tf.nn.embedding_lookup(embeddings_tags,
self.tags)
#TODO concatenata tf.cvoncat emb_form emb_lemma emb_tags (zkusit udelat one-hot na tag - nebo zapomenout na to)
embedding_in = tf.concat(
2,
[embedding_in_forms, embedding_in_lemmas, embedding_in_tags
]) #mail Milanovi - je to tak???
self.input_keep_dropout = tf.placeholder_with_default(
1.0, None, name="input_keep")
self.hidden_keep_dropout = tf.placeholder_with_default(
1.0, None, name="hidden_keep")
hidden_layer = tf.nn.dropout(embedding_in, self.input_keep_dropout)
bi_out, bi_out_states = tf.nn.bidirectional_dynamic_rnn(
rnn_cell,
rnn_cell,
embedding_in,
self.sentence_lens,
dtype=tf.float32) # concatenate embedding
layer1 = tf.concat(2, bi_out)
layer = tf_layers.fully_connected(layer1,
n_mwe,
activation_fn=None,
scope="output_layer")
mask = tf.sequence_mask(self.sentence_lens)
masked_mwe = tf.boolean_mask(self.mwe, mask)
self.predictions = tf.cast(tf.argmax(layer, 2, name="predictions"),
dtype=tf.int32)
masked_predictions = tf.boolean_mask(self.predictions, mask)
loss = tf_losses.sparse_softmax_cross_entropy(tf.boolean_mask(
layer, mask),
masked_mwe,
scope="loss")
self.training = tf.train.AdamOptimizer().minimize(
loss, global_step=self.global_step)
self.accuracy = tf_metrics.accuracy(
masked_predictions, masked_mwe
) #vysledni scotre bude jiny !!! cely score na tech MWE
self.dataset_name = tf.placeholder(tf.string, [])
self.summary = tf.merge_summary([
tf.scalar_summary(self.dataset_name + "/loss", loss),
tf.scalar_summary(self.dataset_name + "/accuracy",
self.accuracy)
])
self.saver = tf.train.Saver(max_to_keep=None)
# Initialize variables
self.session.run(tf.initialize_all_variables())
if restore_path is not None:
self.saver.restore(self.session, restore_path)
def main(_):
if not tf.gfile.Exists(FLAGS.data_dir):
print('data direction is not exist!')
return -1
# if tf.gfile.Exists(FLAGS.log_dir):
# tf.gfile.DeleteRecursively(FLAGS.log_dir)
# tf.gfile.MakeDirs(FLAGS.log_dir)
# train_data, train_labels = load_train_data()
# # name = 'cifar10_train'
#
# valid_data, valid_labels = load_valid_data()
# # name = 'cifar10_valid'
# train_data = (train_data - 128) / 128.0
# valid_data = (valid_data - 128) / 128.0
train_example_batch, train_label_batch = input_pipeline(
['data/cifar10_train.tfrecords'], FLAGS.batch_size)
valid_example_batch, valid_label_batch = input_pipeline(
['data/cifar10_valid.tfrecords'], 10000)
with tf.name_scope('input'):
x = tf.placeholder(tf.float32, [None, 32, 32, 3], 'x')
tf.summary.image('show', x, 1)
with tf.name_scope('label'):
y_ = tf.placeholder(tf.int64, [None, 1], 'y')
with tf.variable_scope('net'):
# y, keep_prob = build_net(x)
y, keep_prob = res.build_net(x, 3)
with tf.name_scope('scores'):
loss.sparse_softmax_cross_entropy(y, y_, scope='cross_entropy')
total_loss = tf.contrib.losses.get_total_loss(
add_regularization_losses=True, name='total_loss')
# expp = tf.exp(y)
#
# correct = tf.reduce_sum(tf.multiply(tf.one_hot(y_, 10), y), 1)
#
# total_loss = total_loss + tf.reduce_mean(tf.log(tf.reduce_sum(expp, 1)), 0) - tf.reduce_mean(correct, 0)
tf.summary.scalar('loss', total_loss)
# with tf.name_scope('accuracy'):
# with tf.name_scope('correct_prediction'):
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(tf.reshape(tf.argmax(y, 1), [-1, 1]),
y_)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar('accuracy', accuracy)
# accuracy = tf.metrics.accuracy(labels=y_, predictions=tf.argmax(y, 1), name='accuracy')
# tf.summary.scalar('accuracy', accuracy)
# loss.mean_squared_error(predictions, labels, scope='l2_1')
# loss.mean_squared_error(predictions, labels, scope='l2_2')
# loss_collect = tf.get_collection(tf.GraphKeys.LOSSES)
# print((tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)))
with tf.name_scope('train'):
global_step = tf.Variable(FLAGS.start_step, name="global_step")
# learning_rate = tf.train.exponential_decay(FLAGS.learning_rate,
# global_step, FLAGS.decay_steps, FLAGS.decay_rate, True, "learning_rate")
learning_rate = tf.train.piecewise_constant(global_step,
[32000, 48000],
[0.1, 0.01, 0.001])
train_step = tf.train.MomentumOptimizer(
learning_rate,
momentum=FLAGS.momentum).minimize(total_loss,
global_step=global_step)
tf.summary.scalar('lr', learning_rate)
merged = tf.summary.merge_all()
with tf.Session() as sess:
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
saver = tf.train.Saver(name="saver")
if tf.gfile.Exists(os.path.join(FLAGS.ckpt_dir, 'checkpoint')):
saver.restore(sess, os.path.join(FLAGS.ckpt_dir, 'model.ckpt'))
else:
sess.run(tf.global_variables_initializer())
train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train',
sess.graph)
test_writer = tf.summary.FileWriter(FLAGS.log_dir + '/test',
sess.graph)
train_writer.flush()
test_writer.flush()
def feed_dict(train, kk=FLAGS.dropout):
"""Make a TensorFlow feed_dict: maps data onto Tensor placeholders."""
def get_batch(data, labels):
# id = np.random.randint(low=0, high=labels.shape[0], size=FLAGS.batch_size, dtype=np.int32)
# return data[id, ...], labels[id]
d, l = sess.run([data, labels])
d = d.astype(np.float32)
l = l.astype(np.int64)
d = (d - 128) / 128
return d, l
if train:
tmp, ys = get_batch(train_example_batch, train_label_batch)
xs = tmp
tmp = np.pad(tmp, 4, 'constant')
for ii in range(FLAGS.batch_size):
xx = np.random.randint(0, 9)
yy = np.random.randint(0, 9)
xs[ii,:] = np.fliplr(tmp[ii + 4,xx:xx + 32, yy:yy + 32,4:7]) if np.random.randint(0, 2) == 1 \
else tmp[ii + 4,xx:xx + 32, yy:yy + 32,4:7]
k = kk
else:
xs, ys = get_batch(valid_example_batch, valid_label_batch)
# xs = valid_data
# ys = valid_labels
k = 1.0
return {x: xs, y_: ys, keep_prob: k}
for i in range(FLAGS.start_step, FLAGS.max_steps + 1):
if i % 1000 == 0 and i != 1:
time.sleep(60)
sess.run(train_step, feed_dict=feed_dict(True))
if i % 10 == 0 and i != 0: # Record summaries and test-set accuracy
acc, summary = sess.run([accuracy, merged],
feed_dict=feed_dict(False))
test_writer.add_summary(summary, i)
print(i)
print(acc)
acc, summary = sess.run([accuracy, merged],
feed_dict=feed_dict(True))
print(acc)
coord.request_stop()
coord.join(threads)
train_writer.close()
test_writer.close()
def _softmax_entropy(probabilities, targets, weights=None):
return metric_ops.streaming_mean(losses.sparse_softmax_cross_entropy(
probabilities, math_ops.to_int32(targets)),
weights=weights)
def main(_):
os.environ["CUDA_VISIBLE_DEVICES"] = FLAGS.gpu
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.95
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
# config.log_device_placement = True
if not tf.gfile.Exists(FLAGS.data_dir):
raise RuntimeError('data direction is not exist!')
# if tf.gfile.Exists(FLAGS.log_dir):
# tf.gfile.DeleteRecursively(FLAGS.log_dir)
tf.gfile.MakeDirs(FLAGS.log_dir)
# if not tf.gfile.Exists(FLAGS.ckpt_dir):
tf.gfile.MakeDirs(os.path.join(FLAGS.ckpt_dir, 'best'))
f = open(FLAGS.out_file, 'a')
if not f:
raise RuntimeError('OUTPUT FILE OPEN ERROR!!!!!!')
with tf.device('/cpu:0'):
num_gpus = len(FLAGS.gpu.split(','))
global_step = tf.Variable(FLAGS.start_step,
name='global_step',
trainable=False)
# learning_rate = tf.train.exponential_decay(0.05, global_step, 2000, 0.9, staircase=True)
learning_rate = tf.train.exponential_decay(0.1,
global_step,
1000,
0.95,
staircase=True)
# learning_rate = tf.train.piecewise_constant(global_step, [24000, 48000, 72000, 108000, 144000],
# [0.1, 0.01, 0.001, 0.0001, 0.00001, 0.000001])
tf.summary.scalar('learing rate', learning_rate)
# opt = tf.train.AdamOptimizer(learning_rate)
opt = tf.train.MomentumOptimizer(learning_rate,
momentum=FLAGS.momentum)
# opt = tf.train.GradientDescentOptimizer(learning_rate)
# learning_rate = tf.train.exponential_decay(0.01, global_step, 32000, 0.1)
# opt = tf.train.GradientDescentOptimizer(learning_rate)
tower_grads = []
tower_loss = []
tower_acc = []
tower_acc_v = []
images, labels = input_pipeline(
tf.train.match_filenames_once(
os.path.join(FLAGS.data_dir, 'train', '*.tfrecords')),
FLAGS.batch_size)
batch_queue = tf.contrib.slim.prefetch_queue.prefetch_queue(
[images, labels], capacity=2 * num_gpus)
images_v, labels_v = input_pipeline(
tf.train.match_filenames_once(
os.path.join(FLAGS.data_dir, 'valid', '*.tfrecords')),
128 // num_gpus)
batch_queue_v = tf.contrib.slim.prefetch_queue.prefetch_queue(
[images_v, labels_v], capacity=2 * num_gpus)
for i in range(num_gpus):
with tf.device('/gpu:%d' % i):
with tf.name_scope('tower_%d' % i) as scope:
image_batch, label_batch = batch_queue.dequeue()
logits = build.net(image_batch, is_training, FLAGS)
losses.sparse_softmax_cross_entropy(labels=label_batch,
logits=logits,
scope=scope)
total_loss = losses.get_losses(
scope=scope) + losses.get_regularization_losses(
scope=scope)
total_loss = tf.add_n(total_loss)
grads = opt.compute_gradients(total_loss)
tower_grads.append(grads)
tower_loss.append(losses.get_losses(scope=scope))
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(
tf.reshape(tf.argmax(logits, 1), [-1, 1]),
tf.cast(label_batch, tf.int64))
accuracy = tf.reduce_mean(
tf.cast(correct_prediction, tf.float32))
tower_acc.append(accuracy)
tf.get_variable_scope().reuse_variables()
image_batch_v, label_batch_v = batch_queue_v.dequeue()
logits_v = build.net(image_batch_v, False, FLAGS)
correct_prediction = tf.equal(
tf.reshape(tf.argmax(logits_v, 1), [-1, 1]),
tf.cast(label_batch_v, tf.int64))
accuracy = tf.reduce_mean(
tf.cast(correct_prediction, tf.float32))
tower_acc_v.append(accuracy)
with tf.name_scope('scores'):
with tf.name_scope('accuracy'):
accuracy = tf.reduce_mean(tf.stack(tower_acc, axis=0))
with tf.name_scope('accuracy_v'):
accuracy_v = tf.reduce_mean(tf.stack(tower_acc_v, axis=0))
with tf.name_scope('batch_loss'):
batch_loss = tf.add_n(tower_loss)[0] / num_gpus
tf.summary.scalar('loss', batch_loss)
tf.summary.scalar('accuracy', accuracy)
grads = average_gradients(tower_grads)
variable_averages = tf.train.ExponentialMovingAverage(
0.9999, global_step)
variables_averages_op = variable_averages.apply(
tf.trainable_variables())
with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE):
apply_gradient_op = opt.apply_gradients(grads,
global_step=global_step)
train_op = tf.group(apply_gradient_op, variables_averages_op)
# train_op = apply_gradient_op
# summary_op = tf.summary.merge_all()
# init = tf.global_variables_initializer()
summary_op = tf.summary.merge_all()
saver = tf.train.Saver(name="saver", max_to_keep=10)
saver_best = tf.train.Saver(name='best', max_to_keep=100)
with tf.Session(config=config) as sess:
sess.run(tf.local_variables_initializer())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
if tf.gfile.Exists(os.path.join(FLAGS.ckpt_dir, 'checkpoint')):
saver.restore(sess, tf.train.latest_checkpoint(FLAGS.ckpt_dir))
else:
sess.run(tf.global_variables_initializer())
train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train',
sess.graph)
train_writer.flush()
cache = np.ones(5, dtype=np.float32) / FLAGS.num_classes
cache_v = np.ones(5, dtype=np.float32) / FLAGS.num_classes
d = 1000
best = 0
for i in range(FLAGS.start_step, FLAGS.max_steps + 1):
# feed = feed_dict(True, True)
if i % d == 0: # Record summaries and test-set accuracy
# loss0 = sess.run([total_loss], feed_dict=feed_dict(False, False))
# test_writer.add_summary(summary, i)
# feed[is_training] = FLAGS
acc, loss, summ, lr, acc_v = sess.run(
[
accuracy, batch_loss, summary_op, learning_rate,
accuracy_v
],
feed_dict={is_training: False})
cache[int(i / d) % 5] = acc
cache_v[int(i / d) % 5] = acc_v
train_writer.add_summary(summ, i)
print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
file=f)
print(
'step %d: acc(t)=%f(%f), loss=%f; acc(v)=%f(%f); lr=%e'
% (i, acc, cache.mean(), loss, acc_v, cache_v.mean(),
lr),
file=f)
saver.save(sess,
os.path.join(FLAGS.ckpt_dir, FLAGS.model_name),
global_step=i)
if acc_v > 0.90:
saver_best.save(sess,
os.path.join(FLAGS.ckpt_dir, 'best',
FLAGS.model_name),
global_step=i)
f.flush()
sess.run(train_op, feed_dict={is_training: True})
coord.request_stop()
coord.join(threads)
train_writer.close()
# test_writer.close()
f.close()
def main(_):
if not tf.gfile.Exists(FLAGS.data_dir):
print('data direction is not exist!')
return -1
# if tf.gfile.Exists(FLAGS.log_dir):
# tf.gfile.DeleteRecursively(FLAGS.log_dir)
# tf.gfile.MakeDirs(FLAGS.log_dir)
train_filenames = [('bindata/data_batch_%d.bin' % i) for i in range(1, 6)]
train_example_batch, train_label_batch = input_pipeline(
train_filenames, 128, False)
valid_filenames = ['bindata/test_batch.bin']
valid_example_batch, valid_label_batch = input_pipeline(
valid_filenames, 10000, False)
# train_data, train_labels = load_train_data()
# # name = 'cifar10_train'
#
# valid_data, valid_labels = load_valid_data()
# # name = 'cifar10_valid'
# train_data = (train_data - 128) / 128.0
# valid_data = (valid_data - 128) / 128.0
with tf.name_scope('input'):
x = tf.placeholder(tf.float32, [None, 32, 32, 3], 'x')
tf.summary.image('show', x, 1)
with tf.name_scope('label'):
y_ = tf.placeholder(tf.int64, [None, 1], 'y')
with tf.variable_scope('net'):
# y, keep_prob = build_net(x)
y, keep_prob = res.build_net(x, 3)
with tf.name_scope('scores'):
loss.sparse_softmax_cross_entropy(y, y_, scope='cross_entropy')
total_loss = tf.contrib.losses.get_total_loss(
add_regularization_losses=True, name='total_loss')
# expp = tf.exp(y)
#
# correct = tf.reduce_sum(tf.multiply(tf.one_hot(y_, 10), y), 1)
#
# total_loss = total_loss + tf.reduce_mean(tf.log(tf.reduce_sum(expp, 1)), 0) - tf.reduce_mean(correct, 0)
tf.summary.scalar('loss', total_loss)
# with tf.name_scope('accuracy'):
# with tf.name_scope('correct_prediction'):
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(tf.argmax(y, 1), y_)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar('accuracy', accuracy)
# accuracy = tf.metrics.accuracy(labels=y_, predictions=tf.argmax(y, 1), name='accuracy')
# tf.summary.scalar('accuracy', accuracy)
# loss.mean_squared_error(predictions, labels, scope='l2_1')
# loss.mean_squared_error(predictions, labels, scope='l2_2')
# loss_collect = tf.get_collection(tf.GraphKeys.LOSSES)
# print((tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)))
with tf.name_scope('train'):
global_step = tf.Variable(1, name="global_step")
# learning_rate = tf.train.exponential_decay(FLAGS.learning_rate,
# global_step, FLAGS.decay_steps, FLAGS.decay_rate, True, "learning_rate")
learning_rate = tf.train.piecewise_constant(global_step, [5000, 15000],
[0.1, 0.01, 0.001])
train_step = tf.train.MomentumOptimizer(
learning_rate,
momentum=FLAGS.momentum).minimize(total_loss,
global_step=global_step)
tf.summary.scalar('lr', learning_rate)
merged = tf.summary.merge_all()
with tf.Session() as sess:
saver = tf.train.Saver(name="saver")
if tf.gfile.Exists(os.path.join(FLAGS.ckpt_dir, 'checkpoint')):
saver.restore(sess, os.path.join(FLAGS.ckpt_dir, 'model.ckpt'))
else:
sess.run(tf.global_variables_initializer())
train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train',
sess.graph)
test_writer = tf.summary.FileWriter(FLAGS.log_dir + '/test',
sess.graph)
train_writer.flush()
test_writer.flush()
# def feed_dict(train, kk=FLAGS.dropout):
# """Make a TensorFlow feed_dict: maps data onto Tensor placeholders."""
#
# def get_batch(data, labels):
# id = np.random.randint(low=0, high=labels.shape[0], size=FLAGS.batch_size, dtype=np.int32)
# return data[id, ...], labels[id]
#
# if train:
# xs, ys = get_batch(train_data, train_labels)
# k = kk
# else:
# # xs, ys = get_batch(valid_data, valid_labels)
# xs = valid_data
# ys = valid_labels
# k = 1.0
# return {x: xs, y_: ys, keep_prob: k}
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
for i in range(1, FLAGS.max_steps + 1):
if i % 1000 == 0 and i != 1:
time.sleep(60)
if i % 100 == 0 and i != 0: # Record summaries and test-set accuracy
# start = time.clock()
data_, label_ = sess.run(
[valid_example_batch, valid_label_batch])
data_ = (data_ - 128) / 128.0
acc, summary = sess.run([accuracy, merged],
feed_dict={
x: data_,
y_: label_
})
# end = time.clock()
test_writer.add_summary(summary, i)
# print('Accuracy at step %s: %s; %f seconds' % (i, acc, end - start))
# else: # Record train set summaries, and train
if i % 100 == 99: # Record execution stats
run_options = tf.RunOptions(
trace_level=tf.RunOptions.FULL_TRACE)
run_metadata = tf.RunMetadata()
# feed = feed_dict(True)
data_, label_ = sess.run(
[train_example_batch, train_label_batch])
data_ = (data_ - 128) / 128.0
_, summary = sess.run([train_step, merged],
feed_dict={
x: data_,
y_: label_
},
options=run_options,
run_metadata=run_metadata)
train_writer.add_run_metadata(run_metadata, 'step%03d' % i)
# summary = sess.run(merged, feed_dict=feed)
train_writer.add_summary(summary, i)
print('Adding run metadata for step', i)
saver.save(sess, os.path.join(FLAGS.ckpt_dir, 'model.ckpt'))
else: # Record a summary
data_, label_ = sess.run(
[train_example_batch, train_label_batch])
data_ = (data_ - 128) / 128.0
# print(label_[1])
# plt.imshow(data_[1,:].astype(np.uint8))
# plt.show()
_, summary = sess.run([train_step, merged],
feed_dict={
x: data_,
y_: label_
})
train_writer.add_summary(summary, i)
coord.request_stop()
coord.join(threads)
train_writer.close()
test_writer.close()
def main(_):
os.environ["CUDA_VISIBLE_DEVICES"] = FLAGS.gpu
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.95
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
# config.log_device_placement = True
if not tf.gfile.Exists(FLAGS.data_dir):
raise RuntimeError('data direction is not exist!')
# if tf.gfile.Exists(FLAGS.log_dir):
# tf.gfile.DeleteRecursively(FLAGS.log_dir)
tf.gfile.MakeDirs(FLAGS.log_dir)
# if not tf.gfile.Exists(FLAGS.ckpt_dir):
tf.gfile.MakeDirs(os.path.join(FLAGS.ckpt_dir, 'best'))
f = open(FLAGS.out_file + '.txt',
'a' if FLAGS.start_step is not 0 else 'w')
if not f:
raise RuntimeError('OUTPUT FILE OPEN ERROR!!!!!!')
with tf.device('/cpu:0'):
num_gpus = len(FLAGS.gpu.split(','))
global_step = tf.Variable(FLAGS.start_step,
name='global_step',
trainable=False)
# learning_rate = tf.train.piecewise_constant(global_step,
# [500, 1000, 1500, 2000, 2500, 3000, 3500, 4000, 4500, 5000],
# [0.00001, 0.00005, 0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0])
# step_size = 10000
# learning_rate = tf.train.exponential_decay(1.0, global_step, 2*step_size, 0.5, staircase=True)
# cycle = tf.floor(1 + tf.cast(global_step, tf.float32) / step_size / 2.)
# xx = tf.abs(tf.cast(global_step, tf.float32)/step_size - 2. * tf.cast(cycle, tf.float32) + 1.)
# learning_rate = 1e-4 + (1e-1 - 1e-4) * tf.maximum(0., (1-xx))*learning_rate
# learning_rate = tf.train.piecewise_constant(global_step, [10000, 70000, 120000, 170000, 220000],
# [0.01, 0.1, 0.001, 0.0001, 0.00001, 0.000001])
# learning_rate = tf.constant(0.001)
learning_rate = tf.train.exponential_decay(0.05,
global_step,
30000,
0.1,
staircase=True)
print(
'learning_rate = tf.train.exponential_decay(0.05, global_step, 30000, 0.1, staircase=True)',
file=f)
# opt = tf.train.AdamOptimizer(learning_rate)
opt = tf.train.MomentumOptimizer(learning_rate, momentum=0.9)
# opt = tf.train.GradientDescentOptimizer(learning_rate)
# learning_rate = tf.train.exponential_decay(0.01, global_step, 32000, 0.1)
# opt = tf.train.GradientDescentOptimizer(learning_rate)
print('opt = tf.train.MomentumOptimizer(learning_rate, momentum=0.9)',
file=f)
print('weight decay = %e' % FLAGS.weight_decay, file=f)
f.flush()
tf.summary.scalar('learing rate', learning_rate)
tower_grads = []
tower_loss = []
tower_acc = []
images_t, labels_t = input_pipeline(
tf.train.match_filenames_once(
os.path.join(FLAGS.data_dir, 'train', '*.tfrecords')),
FLAGS.batch_size * num_gpus,
read_threads=len(os.listdir(os.path.join(FLAGS.data_dir,
'train'))))
# batch_queue = tf.contrib.slim.prefetch_queue.prefetch_queue(
# [images, labels], capacity=2 * num_gpus)
images_v, labels_v = input_pipeline(
tf.train.match_filenames_once(
os.path.join(FLAGS.data_dir, 'valid',
'*.tfrecords')), (256 // num_gpus) * num_gpus,
read_threads=len(os.listdir(os.path.join(FLAGS.data_dir,
'valid'))),
if_train=False)
# batch_queue_v = tf.contrib.slim.prefetch_queue.prefetch_queue(
# [images_v, labels_v], capacity=2 * num_gpus)
image_batch0 = tf.placeholder(
tf.float32, [None, FLAGS.patch_size, FLAGS.patch_size, channels],
'imgs')
label_batch0 = tf.placeholder(tf.int32, [None, 1], 'labels')
image_batch = tf.split(image_batch0, num_gpus, 0)
label_batch = tf.split(label_batch0, num_gpus, 0)
for i in range(num_gpus):
with tf.device('/gpu:%d' % i):
with tf.name_scope('tower_%d' % i) as scope:
logits = build.net(image_batch[i], is_training, FLAGS)
losses.sparse_softmax_cross_entropy(labels=label_batch[i],
logits=logits,
scope=scope)
total_loss = losses.get_losses(
scope=scope) + losses.get_regularization_losses(
scope=scope)
total_loss = tf.add_n(total_loss)
grads = opt.compute_gradients(total_loss)
tower_grads.append(grads)
tower_loss.append(losses.get_losses(scope=scope))
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(
tf.reshape(tf.argmax(logits, 1), [-1, 1]),
tf.cast(label_batch[i], tf.int64))
accuracy = tf.reduce_mean(
tf.cast(correct_prediction, tf.float32))
tower_acc.append(accuracy)
tf.get_variable_scope().reuse_variables()
with tf.name_scope('scores'):
with tf.name_scope('accuracy'):
accuracy = tf.reduce_mean(tf.stack(tower_acc, axis=0))
with tf.name_scope('batch_loss'):
batch_loss = tf.add_n(tower_loss)[0] / num_gpus
tf.summary.scalar('loss', batch_loss)
tf.summary.scalar('accuracy', accuracy)
grads = average_gradients(tower_grads)
with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE):
variable_averages = tf.train.ExponentialMovingAverage(
0.9999, global_step)
variables_averages_op = variable_averages.apply(
tf.trainable_variables())
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
apply_gradient_op = opt.apply_gradients(
grads, global_step=global_step)
train_op = tf.group(apply_gradient_op, variables_averages_op)
p_relu_update = tf.get_collection('p_relu')
# train_op = apply_gradient_op
# summary_op = tf.summary.merge_all()
# init = tf.global_variables_initializer()
summary_op = tf.summary.merge_all()
saver = tf.train.Saver(name="saver", max_to_keep=10)
saver_best = tf.train.Saver(name='best', max_to_keep=200)
with tf.Session(config=config) as sess:
sess.run(tf.local_variables_initializer())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
if tf.gfile.Exists(os.path.join(FLAGS.ckpt_dir, 'checkpoint')):
saver.restore(sess, tf.train.latest_checkpoint(FLAGS.ckpt_dir))
else:
sess.run(tf.global_variables_initializer())
if FLAGS.start_step != 0:
sess.run(tf.assign(global_step, FLAGS.start_step))
train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train',
sess.graph)
train_writer.flush()
valid_writer = tf.summary.FileWriter(FLAGS.log_dir + '/valid',
sess.graph)
valid_writer.flush()
cache = np.ones(5, dtype=np.float32) / FLAGS.num_classes
cache_v = np.ones(5, dtype=np.float32) / FLAGS.num_classes
d = 1000
best = 0
for i in range(FLAGS.start_step, FLAGS.max_steps + 1):
def get_batch(set, on_training):
if set == 'train':
img, lb = sess.run([images_t, labels_t])
# x = np.random.randint(0, 64)
# y = np.random.randint(0, 64)
# img = np.roll(np.roll(img, x, 1), y, 2)
elif set == 'valid':
img, lb = sess.run([images_v, labels_v])
else:
raise RuntimeError('Unknown set name')
feed_dict = {}
feed_dict[image_batch0] = img
feed_dict[label_batch0] = lb
feed_dict[is_training] = on_training
return feed_dict
# feed = feed_dict(True, True)
if i % d == 0: # Record summaries and test-set accuracy
# loss0 = sess.run([total_loss], feed_dict=feed_dict(False, False))
# test_writer.add_summary(summary, i)
# feed[is_training] = FLAGS
acc, loss, summ, lr = sess.run(
[accuracy, batch_loss, summary_op, learning_rate],
feed_dict=get_batch('train', False))
acc2 = sess.run(accuracy,
feed_dict=get_batch('train', True))
cache[int(i / d) % 5] = acc
acc_v, loss_v, summ_v = sess.run(
[accuracy, batch_loss, summary_op],
feed_dict=get_batch('valid', False))
acc2_v = sess.run(accuracy,
feed_dict=get_batch('valid', True))
cache_v[int(i / d) % 5] = acc_v
train_writer.add_summary(summ, i)
valid_writer.add_summary(summ_v, i)
print(('step %d, ' % i) +
time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
file=f)
print(
'acc(t)=%f(%f), loss(t)=%f;\nacc(v)=%f(%f), loss(v)=%f; lr=%e'
% (acc, cache.mean(), loss, acc_v, cache_v.mean(),
loss_v, lr),
file=f)
print('%f, %f' % (acc2, acc2_v), file=f)
saver.save(sess,
os.path.join(FLAGS.ckpt_dir, FLAGS.model_name),
global_step=i)
if acc_v > 0.90:
saver_best.save(sess,
os.path.join(FLAGS.ckpt_dir, 'best',
FLAGS.model_name),
global_step=i)
f.flush()
sess.run(train_op, feed_dict=get_batch('train', True))
sess.run(p_relu_update)
coord.request_stop()
coord.join(threads)
train_writer.close()
# test_writer.close()
f.close()
def main(_):
os.environ["CUDA_VISIBLE_DEVICES"] = FLAGS.gpu
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.7
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
if not tf.gfile.Exists(FLAGS.data_dir):
raise RuntimeError('data direction is not exist!')
# if tf.gfile.Exists(FLAGS.log_dir):
# tf.gfile.DeleteRecursively(FLAGS.log_dir)
# tf.gfile.MakeDirs(FLAGS.log_dir)
if not tf.gfile.Exists(FLAGS.ckpt_dir):
tf.gfile.MakeDirs(FLAGS.ckpt_dir)
f = open(FLAGS.out_file, 'w')
if not f:
raise RuntimeError('OUTPUT FILE OPEN ERROR!!!!!!')
with tf.device('/cpu:0'):
global_step = tf.Variable(FLAGS.start_step,
name='global_step',
trainable=False)
learning_rate = tf.train.piecewise_constant(global_step,
[24000, 48000],
[0.1, 0.01, 0.001])
opt = tf.train.MomentumOptimizer(learning_rate,
momentum=FLAGS.momentum)
# learning_rate = tf.train.exponential_decay(0.01, global_step, 32000, 0.1)
# opt = tf.train.GradientDescentOptimizer(learning_rate)
tower_grads = []
num_gpus = len(FLAGS.gpu.split(','))
tower_images = []
tower_labels = []
tower_loss = []
for i in range(num_gpus):
with tf.device('/gpu:%d' % i):
with tf.name_scope('tower_%d' % i) as scope:
# with tf.name_scope('input'):
# images = tf.placeholder(tf.float32, [None, FLAGS.patch_size, FLAGS.patch_size, 3], 'images')
# tf.summary.image('show', images, 1)
#
# with tf.name_scope('label'):
# labels = tf.placeholder(tf.int64, [None, 1], 'y')
images, labels = input_pipeline(
tf.train.match_filenames_once(
os.path.join(FLAGS.data_dir, 'train', '*.tfrecords')),
FLAGS.batch_size)
tower_images.append(images)
tower_labels.append(labels)
logits = build.net(images, FLAGS, is_training,
FLAGS.num_classes)
losses.sparse_softmax_cross_entropy(logits,
labels,
scope=scope)
total_loss = losses.get_losses(
scope=scope) + losses.get_regularization_losses(
scope=scope)
total_loss = tf.add_n(total_loss)
grads = opt.compute_gradients(total_loss)
tower_grads.append(grads)
tower_loss.append(losses.get_losses(scope=scope))
grads = average_gradients(tower_grads)
total_loss = tf.add_n(tower_loss)
# variable_averages = tf.train.ExponentialMovingAverage(
# cifar10.MOVING_AVERAGE_DECAY, global_step)
# variables_averages_op = variable_averages.apply(tf.trainable_variables())
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
apply_gradient_op = opt.apply_gradients(grads, global_step=global_step)
# train_op = tf.group(apply_gradient_op, variables_averages_op)
train_op = apply_gradient_op
# summary_op = tf.summary.merge_all()
# init = tf.global_variables_initializer()
saver = tf.train.Saver(name="saver")
with tf.Session(config=config) as sess:
sess.run(tf.local_variables_initializer())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
if tf.gfile.Exists(os.path.join(FLAGS.ckpt_dir, 'checkpoint')):
saver.restore(sess, os.path.join(FLAGS.ckpt_dir, FLAGS.model_name))
else:
sess.run(tf.global_variables_initializer())
train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train',
sess.graph)
# test_writer = tf.summary.FileWriter(FLAGS.log_dir + '/test', sess.graph)
train_writer.flush()
# test_writer.flush()
# def feed_dict(train, on_training):
# def get_batch(data, labels):
# d, l = sess.run([data, labels])
# d = d.astype(np.float32)
# l = l.astype(np.int64)
# return d, l
# res = {}
# if train:
# for j in range(num_gpus):
# xs, ys = get_batch(train_example_batch, train_label_batch)
# res[tower_images[j]] = xs
# res[tower_labels[j]] = ys
# else:
# for j in range(num_gpus):
# xs, ys = get_batch(valid_example_batch, valid_label_batch)
# res[tower_images[j]] = xs
# res[tower_labels[j]] = ys
# return res
for i in range(FLAGS.start_step, FLAGS.max_steps + 1):
# feed = feed_dict(True, True)
sess.run(train_op, feed_dict={is_training: True})
if i % 10 == 0 and i != 0: # Record summaries and test-set accuracy
# loss0 = sess.run([total_loss], feed_dict=feed_dict(False, False))
# test_writer.add_summary(summary, i)
# feed[is_training] = FLAGS
loss1 = sess.run(total_loss, feed_dict={is_training: False})
# train_writer.add_summary(summary, i)
print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
file=f)
# print('step %d: train_acc=%f, train_loss=%f; test_acc=%f, test_loss=%f' % (i, acc1, loss1, acc0, loss0),
# file=f)
print('step %d: train_loss=%f' % (i, loss1), file=f)
saver.save(sess, os.path.join(FLAGS.ckpt_dir,
FLAGS.model_name))
f.flush()
coord.request_stop()
coord.join(threads)
train_writer.close()
# test_writer.close()
f.close()
def _softmax_entropy(probabilities, targets, weights=None):
return metrics.mean(
losses.sparse_softmax_cross_entropy(probabilities,
math_ops.cast(targets, dtypes.int32)),
weights=weights)
def main(_):
os.environ["CUDA_VISIBLE_DEVICES"] = FLAGS.gpu
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.7
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
if not tf.gfile.Exists(FLAGS.data_dir):
raise RuntimeError('data direction is not exist!')
if tf.gfile.Exists(FLAGS.log_dir):
tf.gfile.DeleteRecursively(FLAGS.log_dir)
tf.gfile.MakeDirs(FLAGS.log_dir)
if not tf.gfile.Exists(FLAGS.ckpt_dir):
tf.gfile.MakeDirs(FLAGS.ckpt_dir)
train_example_batch, train_label_batch = input_pipeline(
tf.train.match_filenames_once(
os.path.join(FLAGS.data_dir, 'train', '*.tfrecords')),
FLAGS.batch_size, FLAGS.patch_size)
valid_example_batch, valid_label_batch = input_pipeline(
tf.train.match_filenames_once(
os.path.join(FLAGS.data_dir, 'valid', '*.tfrecords')),
FLAGS.batch_size, FLAGS.patch_size)
f = open(FLAGS.out_file, 'w')
if not f:
raise RuntimeError('OUTPUT FILE OPEN ERROR!!!!!!')
with tf.name_scope('input'):
x = tf.placeholder(tf.float32,
[None, FLAGS.patch_size, FLAGS.patch_size, 3], 'x')
tf.summary.image('show', x, 1)
with tf.name_scope('label'):
y_ = tf.placeholder(tf.int64, [None, 1], 'y')
is_training = tf.placeholder(tf.bool)
y = build.net(x, FLAGS, is_training, FLAGS)
with tf.name_scope('scores'):
loss.sparse_softmax_cross_entropy(y, y_, scope='cross_entropy')
total_loss = tf.contrib.losses.get_total_loss(
add_regularization_losses=True, name='total_loss')
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(tf.reshape(tf.argmax(y, 1), [-1, 1]),
y_)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar('loss', total_loss)
tf.summary.scalar('accuracy', accuracy)
with tf.name_scope('train'):
global_step = tf.Variable(FLAGS.start_step, name="global_step")
learning_rate = tf.train.piecewise_constant(global_step,
[24000, 48000],
[0.1, 0.01, 0.001])
# learning_rate = tf.train.exponential_decay(0.01, global_step, 32000, 0.1)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_step = tf.train.MomentumOptimizer(
learning_rate,
momentum=FLAGS.momentum).minimize(total_loss,
global_step=global_step)
tf.summary.scalar('lr', learning_rate)
merged = tf.summary.merge_all()
with tf.name_scope("saver"):
saver = tf.train.Saver(name="saver")
with tf.Session(config=config) as sess:
sess.run(tf.local_variables_initializer())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
if tf.gfile.Exists(os.path.join(FLAGS.ckpt_dir, 'checkpoint')):
saver.restore(sess, os.path.join(FLAGS.ckpt_dir, FLAGS.model_name))
else:
sess.run(tf.global_variables_initializer())
train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train',
sess.graph)
test_writer = tf.summary.FileWriter(FLAGS.log_dir + '/test',
sess.graph)
train_writer.flush()
test_writer.flush()
def feed_dict(train, on_training):
def get_batch(data, labels):
d, l = sess.run([data, labels])
d = d.astype(np.float32)
l = l.astype(np.int64)
return d, l
if train:
xs, ys = get_batch(train_example_batch, train_label_batch)
else:
xs, ys = get_batch(valid_example_batch, valid_label_batch)
return {x: xs, y_: ys, is_training: on_training}
for i in range(FLAGS.start_step, FLAGS.max_steps + 1):
feed = feed_dict(True, True)
sess.run(train_step, feed_dict=feed)
if i % 1000 == 0 and i != 0: # Record summaries and test-set accuracy
loss0, acc0, summary = sess.run([total_loss, accuracy, merged],
feed_dict=feed_dict(
False, False))
test_writer.add_summary(summary, i)
# feed[is_training] = FLAGS
loss1, acc1, summary = sess.run([total_loss, accuracy, merged],
feed_dict=feed_dict(
True, False))
train_writer.add_summary(summary, i)
print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
file=f)
print(
'step %d: train_acc=%f, train_loss=%f; test_acc=%f, test_loss=%f'
% (i, acc1, loss1, acc0, loss0),
file=f)
saver.save(sess, os.path.join(FLAGS.ckpt_dir,
FLAGS.model_name))
f.flush()
coord.request_stop()
coord.join(threads)
train_writer.close()
test_writer.close()
f.close()
def main(_):
os.environ["CUDA_VISIBLE_DEVICES"] = FLAGS.gpu
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.7
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
# config.log_device_placement = True
if not tf.gfile.Exists(FLAGS.data_dir):
raise RuntimeError('data direction is not exist!')
# if tf.gfile.Exists(FLAGS.log_dir):
# tf.gfile.DeleteRecursively(FLAGS.log_dir)
# tf.gfile.MakeDirs(FLAGS.log_dir)
if not tf.gfile.Exists(FLAGS.ckpt_dir):
tf.gfile.MakeDirs(FLAGS.ckpt_dir)
f = open(FLAGS.out_file, 'w')
if not f:
raise RuntimeError('OUTPUT FILE OPEN ERROR!!!!!!')
with tf.device('/cpu:0'):
global_step = tf.Variable(FLAGS.start_step,
name='global_step',
trainable=False)
# learning_rate = tf.train.exponential_decay(0.1, global_step, 192000, 0.9, staircase=True)
# tf.summary.scalar('learing rate', learning_rate)
# opt = tf.train.AdamOptimizer(learning_rate)
# opt = tf.train.MomentumOptimizer(learning_rate, momentum=FLAGS.momentum)
# opt = tf.train.GradientDescentOptimizer(learning_rate)
# learning_rate = tf.train.exponential_decay(0.01, global_step, 32000, 0.1)
# opt = tf.train.GradientDescentOptimizer(learning_rate)
# tower_grads = []
num_gpus = len(FLAGS.gpu.split(','))
tower_loss = []
tower_acc = []
images, labels = input_pipeline(
tf.train.match_filenames_once(
os.path.join(FLAGS.data_dir, 'valid', '*.tfrecords')),
int(FLAGS.batch_size / num_gpus))
image_batch = tf.placeholder(
tf.float32, [None, FLAGS.patch_size, FLAGS.patch_size, 3], 'imgs')
label_batch = tf.placeholder(tf.int32, [None, 1], 'labels')
for i in range(num_gpus):
with tf.device('/gpu:%d' % i):
with tf.name_scope('tower_%d' % i) as scope:
# image_batch, label_batch = batch_queue.dequeue()
# image_batch = tf.ones(shape=[128, 64, 64, 3], dtype=tf.float32)
# label_batch = tf.ones(shape=[128, 1], dtype=tf.int32)
logits = build.net(image_batch, False, FLAGS)
losses.sparse_softmax_cross_entropy(labels=label_batch,
logits=logits,
scope=scope)
# total_loss = losses.get_losses(scope=scope) + losses.get_regularization_losses(scope=scope)
# total_loss = tf.add_n(total_loss)
# grads = opt.compute_gradients(total_loss)
# tower_grads.append(grads)
tower_loss.append(losses.get_losses(scope=scope))
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(
tf.reshape(tf.argmax(logits, 1), [-1, 1]),
tf.cast(label_batch, tf.int64))
accuracy = tf.reduce_mean(
tf.cast(correct_prediction, tf.float32))
tower_acc.append(accuracy)
tf.get_variable_scope().reuse_variables()
with tf.name_scope('scores'):
with tf.name_scope('accuracy'):
accuracy = tf.reduce_mean(tf.stack(tower_acc, axis=0))
with tf.name_scope('batch_loss'):
batch_loss = tf.add_n(tower_loss)[0]
tf.summary.scalar('loss', batch_loss)
tf.summary.scalar('accuracy', accuracy)
# grads = average_gradients(tower_grads)
# variable_averages = tf.train.ExponentialMovingAverage(
# cifar10.MOVING_AVERAGE_DECAY, global_step)
# variables_averages_op = variable_averages.apply(tf.trainable_variables())
# with tf.variable_scope(tf.get_variable_scope(), reuse=tf.AUTO_REUSE):
# update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
# with tf.control_dependencies(update_ops):
# apply_gradient_op = opt.apply_gradients(grads, global_step=global_step)
# # train_op = tf.group(apply_gradient_op, variables_averages_op)
# train_op = apply_gradient_op
# summary_op = tf.summary.merge_all()
# init = tf.global_variables_initializer()
summary_op = tf.summary.merge_all()
# variable_averages = tf.train.ExponentialMovingAverage(0.9999)
# variables_to_restore = variable_averages.variables_to_restore()
# saver = tf.train.Saver(variables_to_restore, name='saver')
saver = tf.train.Saver(name="saver")
with tf.Session(config=config) as sess:
sess.run(tf.local_variables_initializer())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
if tf.gfile.Exists(os.path.join(FLAGS.ckpt_dir, 'checkpoint')):
# saver.restore(sess, FLAGS.ckpt_dir+'/model')
saver.restore(
sess,
tf.train.latest_checkpoint(FLAGS.ckpt_dir)
if FLAGS.model_name is None else os.path.join(
FLAGS.ckpt_dir, FLAGS.model_name))
train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/test',
sess.graph)
train_writer.flush()
for i in range(FLAGS.start_step, FLAGS.max_steps + 1):
# feed = feed_dict(True, True)
# if i % 1000 == 0: # Record summaries and test-set accuracy
# loss0 = sess.run([total_loss], feed_dict=feed_dict(False, False))
# test_writer.add_summary(summary, i)
# feed[is_training] = FLAGS
img, lb = sess.run([images, labels])
acc, loss, summ = sess.run([accuracy, batch_loss, summary_op],
feed_dict={
image_batch: img,
label_batch: lb
})
# acc, loss, summ = sess.run([accuracy, batch_loss, summary_op], feed_dict={image_batch: np.ones(shape = [256, 64, 64, 3], dtype=np.float32), label_batch: np.ones(shape=[256, 1], dtype=np.int32)})
train_writer.add_summary(summ, i)
print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()),
file=f)
# print('step %d: train_acc=%f, train_loss=%f; test_acc=%f, test_loss=%f' % (i, acc1, loss1, acc0, loss0),
# file=f)
print('step %d: accuracy=%f, loss=%f' % (i, acc, loss), file=f)
# saver.save(sess, os.path.join(FLAGS.ckpt_dir, FLAGS.model_name))
f.flush()
# sess.run(train_op, feed_dict={is_training: True})
coord.request_stop()
coord.join(threads)
train_writer.close()
# test_writer.close()
f.close()
def _softmax_entropy(probabilities, targets, weights=None):
return metrics.mean(losses.sparse_softmax_cross_entropy(
probabilities, math_ops.cast(targets, dtypes.int32)),
weights=weights)
G, enc_id = generator(inputs, z, is_train=True) ###
D_id_, D_id, D_real_, D_real = discriminator(inputs, is_train=True)
D_G_id_, D_G_id, D_G_real_, D_G_real = discriminator(G,
reuse=True,
is_train=True)
d_id_sum = histogram_summary("d_id", D_id_)
d_real_sum = histogram_summary("d_real", D_real_)
d_G_id_sum = histogram_summary("d_g_id", D_G_id_)
d_G_real_sum = histogram_summary("d_g_real", D_G_real_)
enc_id_sum = histogram_summary("enc_id", enc_id)
G_sum = image_summary("G", G)
#pre-train G_enc
enc_loss = tf.reduce_mean(
losses.sparse_softmax_cross_entropy(labels=y, logits=enc_id))
enc_loss_sum = scalar_summary("enc_loss", enc_loss)
#Loss function set
if MODE == 'wgan-gp':
g_loss_gan = -tf.reduce_mean(D_G_real)
d_loss_gan = tf.reduce_mean(D_G_real) - tf.reduce_mean(D_real)
alpha = tf.random_uniform(
shape=[BATCH_SIZE, 1, 1, 1], #/len(DEVICES)
minval=0.,
maxval=1.)
differences = G - inputs #differences = fake_data - real_data
interpolates = inputs + (alpha * differences)
_, _, _, D_p = discriminator(interpolates, reuse=True, is_train=True)
print(D_p)
