def tower_loss(self, scope, images, labels):
"""Calculate the total loss on a single tower running the CIFAR model.
Args:
scope: unique prefix string identifying the CIFAR tower, e.g. 'tower_0'
images: Images. 4D tensor of shape [batch_size, height, width, 3].
labels: Labels. 1D tensor of shape [batch_size].
Returns:
Tensor of shape [] containing the total loss for a batch of data
"""
# Build inference Graph.
logits = cifar10.inference(images)
# Build the portion of the Graph calculating the losses. Note that we will
# assemble the total_loss using a custom function below.
_ = cifar10.loss(logits, labels)
# Assemble all of the losses for the current tower only.
losses = tf.get_collection('losses', scope)
# Calculate the total loss for the current tower.
total_loss = tf.add_n(losses, name='total_loss')
# Attach a scalar summary to all individual losses and the total loss; do the
# same for the averaged version of the losses.
for l in losses + [total_loss]:
# Remove 'tower_[0-9]/' from the name in case this is a multi-GPU training
# session. This helps the clarity of presentation on tensorboard.
loss_name = re.sub('%s_[0-9]*/' % cifar10.TOWER_NAME, '',
l.op.name)
tf.summary.scalar(loss_name, l)
return total_loss
def evaluate():
"""Eval CIFAR-10 for a number of steps."""
with tf.Graph().as_default() as g:
# Get images and labels for CIFAR-10.
eval_data = FLAGS.eval_data == 'test'
images, labels = cifar10.inputs(eval_data=eval_data)
# Build a Graph that computes the logits predictions from the
# inference model.
logits = cifar10.inference(images)
# Calculate predictions.
top_k_op = tf.nn.in_top_k(logits, labels, 1)
# Restore the moving average version of the learned variables for eval.
variable_averages = tf.train.ExponentialMovingAverage(
cifar10.MOVING_AVERAGE_DECAY)
variables_to_restore = variable_averages.variables_to_restore()
saver = tf.train.Saver(variables_to_restore)
# Build the summary operation based on the TF collection of Summaries.
summary_op = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(FLAGS.eval_dir, g)
while True:
eval_once(saver, summary_writer, top_k_op, summary_op)
if FLAGS.run_once:
break
time.sleep(FLAGS.eval_interval_secs)
def train( name, X_train, Y_train, X_val, Y_val):
sess = tf.Session()
sess.run(tf.global_variables_initializer())
num_training = len(X_train)
batch_size = cifar10.FLAGS.batch_size
cifar10.FLAGS
step = 0
losses = []
accuracies = []
print('-' * 5 + ' Start training ' + '-' * 5)
#Y_train = tf.one_hot(Y_train, 10)
#Y_val = tf.one_hot(Y_val,10)
for epoch in range(num_epoch):
print('train for epoch %d' % epoch)
for i in range(num_training // batch_size):
X_ = X_train[i * batch_size:(i + 1) * batch_size][:]
Y_ = Y_train[i * batch_size:(i + 1) * batch_size]
global_step = tf.contrib.framework.get_or_create_global_step()
# Get images and labels for CIFAR-10.
# Force input pipeline to CPU:0 to avoid operations sometimes ending up on
# GPU and resulting in a slow down.
feed_dict = {images: X_, labels : Y_}
fetches = [self.train_op, self.loss_op, self.accuracy_op]
_, loss, accuracy = sess.run(fetches, feed_dict=feed_dict)
losses.append(loss)
# Build a Graph that computes the logits predictions from the
# inference model.
logits = cifar10.inference(images)
# Calculate loss.
loss = cifar10.loss(logits, labels)
accuracy = eval(logits, labels)
# Build a Graph that trains the model with one batch of examples and
# updates the model parameters.
train_op = cifar10.train(loss, global_step)
losses.append(loss)
accuracies.append(accuracy)
#if step % self.log_step == 0:
# print('iteration (%d): loss = %.3f, accuracy = %.3f' %
# (step, loss, accuracy))
step += 1
# Print validation results
print('validation for epoch %d' % epoch)
#val_accuracy = self.evaluate(sess, X_val, Y_val)
#print('- epoch %d: validation accuracy = %.3f' % (epoch, val_accuracy))
saver = tf.train.Saver()
model_path = saver.save(sess, name+".ckpt")
def buildModel(self):
# Placeholders
self.images = tf.placeholder(tf.float32, [None, 32, 32, 3])
self.labels = tf.placeholder(tf.int64, [None])
# Build a Graph that computes the logits predictions from the
# inference model.
self.logits = cifar10.inference(self.images)
# Calculate loss.
self.loss = cifar10.loss(self.logits, self.labels)
self.accuracy = eval(self.logits, self.labels)
# Build a Graph that trains the model with one batch of examples and
# updates the model parameters.
self.train_op = cifar10.train(self.loss, self.global_step)
def evaluate(eval_data, eval_dir, model_dir, client):
"""Eval CIFAR-10 for a number of steps."""
FLAGS = parser.parse_args()
with tf.Graph().as_default() as g:
# Get images and labels for CIFAR-10.
images, labels = cifar10.inputs(eval_data=eval_data, client=client)
# Build a Graph that computes the logits predictions from the
# inference model.
logits = cifar10.inference(images)
# Calculate predictions.
predict = tf.argmax(logits, 1, output_type=tf.int32)
correct = tf.equal(predict, labels)
accuracy_op = tf.reduce_mean(tf.cast(correct, tf.float32))
top_k_op = tf.nn.in_top_k(logits, labels, 1)
loss = cifar10.loss(logits, labels)
# Restore the moving average version of the learned variables for eval.
variable_averages = tf.train.ExponentialMovingAverage(
cifar10.MOVING_AVERAGE_DECAY)
variables_to_restore = variable_averages.variables_to_restore()
saver = tf.train.Saver(variables_to_restore)
# Build the summary operation based on the TF collection of Summaries.
summary_op = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(eval_dir, g)
while True:
eval_once(saver, summary_writer, top_k_op, accuracy_op, summary_op,
loss, model_dir, eval_dir)
if FLAGS.run_once:
break
time.sleep(FLAGS.eval_interval_secs)
def train(client):
"""Train CIFAR-10 for a number of steps."""
FLAGS = parser.parse_args()
model_dir = modelDir(FLAGS.train_dir, client)
with tf.Graph().as_default():
global_step = tf.contrib.framework.get_or_create_global_step()
# Get images and labels for CIFAR-10.
# Force input pipeline to CPU:0 to avoid operations sometimes ending up on
# GPU and resulting in a slow down.
with tf.device('/cpu:0'):
images, labels = cifar10.distorted_inputs(client)
# Build a Graph that computes the logits predictions from the
# inference model.
logits = cifar10.inference(images)
# Calculate loss.
loss = cifar10.loss(logits, labels)
# Build a Graph that trains the model with one batch of examples and
# updates the model parameters.
train_op = cifar10.train(loss, global_step)
class _LoggerHook(tf.train.SessionRunHook):
"""Logs loss and runtime."""
def begin(self):
self._step = -1
self._start_time = time.time()
def before_run(self, run_context):
self._step += 1
return tf.train.SessionRunArgs(loss) # Asks for loss value.
def after_run(self, run_context, run_values):
if self._step % FLAGS.log_frequency == 0:
current_time = time.time()
duration = current_time - self._start_time
self._start_time = current_time
loss_value = run_values.results
examples_per_sec = FLAGS.log_frequency * FLAGS.batch_size / duration
sec_per_batch = float(duration / FLAGS.log_frequency)
format_str = (
'%s: step %d, loss = %.2f (%.1f examples/sec; %.3f '
'sec/batch)')
print(format_str % (datetime.now(), self._step, loss_value,
examples_per_sec, sec_per_batch))
def end(self, mon_sess):
#print("END!")
weights = [] #np.empty([len(tf.trainable_variables())])
i = 0
for t in tf.trainable_variables():
print(t)
weights.append(t.eval(session=mon_sess))
print(weights)
return weights
with tf.train.MonitoredTrainingSession(
checkpoint_dir=model_dir,
hooks=[
tf.train.StopAtStepHook(last_step=FLAGS.max_steps),
tf.train.NanTensorHook(loss),
_LoggerHook()
],
config=tf.ConfigProto(log_device_placement=FLAGS.
log_device_placement)) as mon_sess:
while not mon_sess.should_stop():
mon_sess.run(train_op)
def train(self):
"""Train CIFAR-10 for a number of steps."""
client = self.index
FLAGS = parser.parse_args()
model_dir=self.modelDir(FLAGS.train_dir,client)
with tf.Graph().as_default():
global_step = tf.contrib.framework.get_or_create_global_step()
# Get images and labels for CIFAR-10.
# Force input pipeline to CPU:0 to avoid operations sometimes ending up on
# GPU and resulting in a slow down.
with tf.device('/cpu:0'):
images, labels = cifar10.distorted_inputs(client)
# Build a Graph that computes the logits predictions from the
# inference model.
logits = cifar10.inference(images)
# Calculate loss.
loss = cifar10.loss(logits, labels)
# Build a Graph that trains the model with one batch of examples and
# updates the model parameters.
train_op = cifar10.train(loss, global_step)
saver = tf.train.Saver()
class _LoggerHook(tf.train.SessionRunHook):
"""Logs loss and runtime."""
def begin(self):
self._step = -1
self._start_time = time.time()
self.loss=[]
def before_run(self, run_context):
self._step += 1
return tf.train.SessionRunArgs(loss) # Asks for loss value.
def after_run(self, run_context, run_values):
if self._step % FLAGS.log_frequency == 0:
current_time = time.time()
duration = current_time - self._start_time
self._start_time = current_time
loss_value = run_values.results
self.loss.append(loss_value)
examples_per_sec = FLAGS.log_frequency * FLAGS.batch_size / duration
sec_per_batch = float(duration / FLAGS.log_frequency)
format_str = ('%s: step %d, loss = %.2f (%.1f examples/sec; %.3f '
'sec/batch)')
print (format_str % (datetime.now(), self._step, loss_value,
examples_per_sec, sec_per_batch))
def end(self, mon_sess):
#print tf.global_variables()
filename = FLAGS.loss_dir + "loss_client"+str(client)+".pkl"
old_loss=[]
if os.path.exists(filename):
with open(filename,'rb') as rfp:
old_loss = pickle.load(rfp)
old_loss.append(self.loss)
with open(filename, "wb") as fp: #Pickling
pickle.dump(old_loss, fp)
cifar10_eval.evaluate(True, "./models/cifar/eval/client"+str(client), FLAGS.train_dir+str(client), client)
for t in tf.trainable_variables():
#print(t)
weights.append(t.eval(session=mon_sess))
with tf.train.MonitoredTrainingSession(
checkpoint_dir=model_dir,
hooks=[tf.train.StopAtStepHook(last_step=FLAGS.max_steps),
tf.train.NanTensorHook(loss),
_LoggerHook()],
config=tf.ConfigProto(
log_device_placement=FLAGS.log_device_placement)) as mon_sess:
#save_path = saver.restore(mon_sess._sess._sess._sess._sess, "/Users/Sara/Dropbox/Class/NN/Project/models/cifar/tfmodel.ckpt")
t = tf.trainable_variables()[0]
#print(t.eval(mon_sess)[0][0][0])
if len(central_weights)>0 :
d = central_weights[0][0][0][0] == t.eval(mon_sess)[0][0][0]
#print("weights", d)
while not mon_sess.should_stop():
mon_sess.run(train_op)
def eval_central_model(client, eval_data, eval_dir, checkpoint_dir):
curr_client = client
"""Eval CIFAR-10 for a number of steps."""
FLAGS = parser.parse_args()
with tf.Graph().as_default() as g:
# Get images and labels for CIFAR-10.
images, labels = process_images()
# Build a Graph that computes the logits predictions from the
# inference model.
logits = cifar10.inference(images)
# Calculate predictions.
top_k_op = tf.nn.in_top_k(logits, labels, 1)
# Restore the moving average version of the learned variables for eval.
variable_averages = tf.train.ExponentialMovingAverage(
cifar10.MOVING_AVERAGE_DECAY)
variables_to_restore = variable_averages.variables_to_restore()
saver = tf.train.Saver(variables_to_restore)
with tf.Session() as sess:
ckpt = tf.train.get_checkpoint_state(checkpoint_dir)
if ckpt and ckpt.model_checkpoint_path:
# Restores from checkpoint
saver.restore(sess, ckpt.model_checkpoint_path)
# Assuming model_checkpoint_path looks something like:
# /my-favorite-path/cifar10_train/model.ckpt-0,
# extract global_step from it.
global_step = ckpt.model_checkpoint_path.split('/')[-1].split(
'-')[-1]
else:
print('No checkpoint file found')
return
coord = tf.train.Coordinator()
try:
threads = []
for qr in tf.get_collection(tf.GraphKeys.QUEUE_RUNNERS):
threads.extend(
qr.create_threads(sess,
coord=coord,
daemon=True,
start=True))
num_iter = int(math.ceil(FLAGS.num_examples /
FLAGS.batch_size))
true_count = 0 # Counts the number of correct predictions.
total_sample_count = num_iter * FLAGS.batch_size
step = 0
while step < num_iter and not coord.should_stop():
predictions = sess.run([top_k_op])
true_count += np.sum(predictions)
step += 1
# Compute precision @ 1.
precision = true_count / total_sample_count
print('%s: precision @ 1 = %.3f' % (datetime.now(), precision))
write(eval_dir, precision, client)
except Exception as e: # pylint: disable=broad-except
coord.request_stop(e)
coord.request_stop()
coord.join(threads, stop_grace_period_secs=10)