def _compute_accuracy(self, inputs, outputs, dataset):
nc = 0
num_left = dataset.get_num_examples()
set_is_training(outputs, False)
loss = 0
while num_left > 0:
X_batch, y_batch = dataset.next_batch(self.batch_size)
X = tf.convert_to_tensor(X_batch, np.float32) #.gpu()
y = tf.convert_to_tensor(y_batch, np.float32) #.gpu()
co.forward({inputs['in']: X})
logits = outputs['out'].val
correct_prediction = tf.equal(tf.argmax(logits, 1),
tf.argmax(y, 1))
num_correct = tf.reduce_sum(tf.cast(correct_prediction, "float"))
loss += tf.reduce_sum(
tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=y))
nc += num_correct
# update the number of examples left.
eff_batch_size = y_batch.shape[0]
num_left -= eff_batch_size
acc = old_div(float(nc), dataset.get_num_examples())
loss = old_div(float(loss), dataset.get_num_examples())
return acc, loss
def evaluate(self, inputs, outputs):
params = M.get_collection()
optimizer = dy.SimpleSGDTrainer(params, self.learning_rate)
num_batches = int(len(self.train_dataset) / self.batch_size)
for epoch in range(self.max_num_training_epochs):
random.shuffle(self.train_dataset)
i = 0
total_loss = 0
while (i < len(self.train_dataset)):
dy.renew_cg()
mbsize = min(self.batch_size, len(self.train_dataset) - i)
minibatch = self.train_dataset[i:i + mbsize]
losses = []
for (label, img) in minibatch:
x = dy.inputVector(img)
co.forward({inputs['in']: x})
logits = outputs['out'].val
loss = dy.pickneglogsoftmax(logits, label)
losses.append(loss)
mbloss = dy.esum(losses) / mbsize
mbloss.backward()
optimizer.update()
total_loss += mbloss.scalar_value()
i += mbsize
val_acc = self.compute_accuracy(inputs, outputs)
if self.log_output_to_terminal and epoch % self.display_step == 0:
print("epoch:", '%d' % (epoch + 1), "loss:",
"{:.9f}".format(total_loss / num_batches),
"validation_accuracy:", "%.5f" % val_acc)
val_acc = self.compute_accuracy(inputs, outputs)
return {'val_acc': val_acc}
def evaluate(self, inputs, outputs):
tf.keras.backend.clear_session()
tf.reset_default_graph()
(x_train, y_train) = self.train_dataset
X = tf.keras.layers.Input(x_train[0].shape)
co.forward({inputs['in']: X})
logits = outputs['out'].val
probs = tf.keras.layers.Softmax()(logits)
model = tf.keras.models.Model(inputs=[inputs['in'].val],
outputs=[probs])
optimizer = tf.keras.optimizers.Adam(lr=self.learning_rate)
model.compile(optimizer=optimizer,
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
model.summary()
history = model.fit(x_train,
y_train,
batch_size=self.batch_size,
epochs=self.max_num_training_epochs,
validation_split=self.val_split)
results = {'val_acc': history.history['val_acc'][-1]}
return results
def _compute_loss(self, inputs, outputs, X, y, loss_metric):
X = tf.constant(X).gpu()
y = tf.constant(y).gpu()
co.forward({inputs['in']: X})
logits = outputs['out'].val
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y))
loss_metric(loss)
return loss
def forward(self, input_name_to_val):
input_to_val = {
self.inputs[name]: val for (name, val) in input_name_to_val.items()
}
co.forward(input_to_val, self._module_seq)
output_name_to_val = {
name: ox.val for (name, ox) in self.outputs.items()
}
return output_name_to_val
def evaluate(self, inputs, outputs):
tf.reset_default_graph()
X_pl = tf.placeholder("float", [None] + self.in_dim)
y_pl = tf.placeholder("float", [None, self.num_classes])
lr_pl = tf.placeholder("float")
co.forward({inputs['in']: X_pl})
logits = outputs['out'].val
train_feed, eval_feed = htf.get_feed_dicts(outputs)
# define loss and optimizer
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=y_pl))
optimizer = tf.train.AdamOptimizer(learning_rate=lr_pl)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
optimizer = optimizer.minimize(loss)
# for computing the accuracy of the model
correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(y_pl, 1))
num_correct = tf.reduce_sum(tf.cast(correct_prediction, "float"))
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
num_batches = int(self.train_dataset.get_num_examples() /
self.batch_size)
for epoch in range(self.num_training_epochs):
avg_loss = 0.
for _ in range(num_batches):
X_batch, y_batch = self.train_dataset.next_batch(
self.batch_size)
train_feed.update({
X_pl: X_batch,
y_pl: y_batch,
lr_pl: self.learning_rate
})
_, c = sess.run([optimizer, loss], feed_dict=train_feed)
avg_loss += c / num_batches
if self.log_output_to_terminal and epoch % self.display_step == 0:
print("epoch:", '%d' % (epoch + 1), "loss:",
"{:.9f}".format(avg_loss))
val_acc = self.compute_accuracy(sess, X_pl, y_pl, num_correct,
self.val_dataset, eval_feed)
print("validation accuracy: %0.4f" % val_acc)
results = {
'validation_accuracy': val_acc,
'num_parameters': htf.get_num_trainable_parameters()
}
return results
def compute_accuracy(self, inputs, outputs):
correct = 0
for (label, img) in self.val_dataset:
dy.renew_cg()
x = dy.inputVector(img)
co.forward({inputs['in']: x})
logits = outputs['out'].val
pred = np.argmax(logits.npvalue())
if (label == pred): correct += 1
return (1.0 * correct / len(self.val_dataset))
def forward(self, input_name_to_val):
"""Forward computation of the module that is represented through the
graph of DeepArchitect modules.
"""
input_to_val = {
ix: input_name_to_val[name]
for (name, ix) in self.inputs.items()
}
co.forward(input_to_val, self._module_seq)
output_name_to_val = {
name: ox.val
for (name, ox) in self.outputs.items()
}
return output_name_to_val
def evaluate(self, inputs, outputs):
keras.backend.clear_session()
X = Input(self.X_train[0].shape)
co.forward({inputs['in']: X})
logits = outputs['out'].val
probs = Activation('softmax')(logits)
model = Model(inputs=[inputs['in'].val], outputs=[probs])
model.compile(optimizer=Adam(lr=self.learning_rate),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
model.summary()
history = model.fit(self.X_train,
self.y_train,
batch_size=self.batch_size,
epochs=self.num_training_epochs,
validation_data=(self.X_val, self.y_val))
results = {'validation_accuracy': history.history['val_accuracy'][-1]}
return results
def forward(self, input_name_to_val):
"""Forward computation of the module that is represented through the
graph of DeepArchitect modules.
"""
if self._module_seq is None:
self._module_seq = co.determine_module_eval_seq(
self.inputs.values())
input_to_val = {
ix: input_name_to_val[name]
for name, ix in iteritems(self.inputs)
}
co.forward(input_to_val, self._module_seq)
output_name_to_val = {
name: ox.val
for name, ox in iteritems(self.outputs)
}
if not self._is_compiled:
modules = get_pytorch_modules(self.outputs)
for i, m in enumerate(modules):
self.add_module(str(i), m)
self._is_compiled = True
return output_name_to_val
def eval(self, inputs, outputs):
tf.reset_default_graph()
X_pl = tf.placeholder("float", [None] + self.in_dim)
y_pl = tf.placeholder("float", [None, self.num_classes])
lr_pl = tf.placeholder("float")
co.forward({inputs['in']: X_pl})
logits = outputs['out'].val
train_feed, eval_feed = htf.get_feed_dicts(outputs)
saver = tf.train.Saver()
# define loss and optimizer
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y_pl))
# chooses the optimizer. (this can be put in a function).
if self.optimizer_type == 'adam':
optimizer = tf.train.AdamOptimizer(learning_rate=lr_pl)
elif self.optimizer_type == 'sgd':
optimizer = tf.train.GradientDescentOptimizer(learning_rate=lr_pl)
elif self.optimizer_type == 'sgd_mom':
optimizer = tf.train.MomentumOptimizer(learning_rate=lr_pl,
momentum=0.99)
else:
raise ValueError("Unknown optimizer.")
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
optimizer = optimizer.minimize(loss)
# for computing the accuracy of the model
correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(y_pl, 1))
num_correct = tf.reduce_sum(tf.cast(correct_prediction, "float"))
init = tf.global_variables_initializer()
# Setting the session to allow growth, so it doesn't allocate all GPU memory.
gpu_ops = tf.GPUOptions(allow_growth=True)
config = tf.ConfigProto(gpu_options=gpu_ops)
seqs = ut.SequenceTracker(abort_if_different_lengths=True)
with tf.Session(config=config) as sess:
sess.run(init)
learning_rate_init = self.learning_rate_init
learning_rate_mult = self.learning_rate_mult
learning_rate_min = self.learning_rate_min
stop_patience = self.stop_patience
learning_rate_patience = self.learning_rate_patience
save_patience = self.save_patience
best_val_acc = -np.inf
best_val_acc_saved = -np.inf
stop_counter = stop_patience
rate_counter = learning_rate_patience
save_counter = save_patience
timer_manager = ut.TimerManager()
timer_manager.create_timer('eval')
# getting the gpu_id based on the environment.
if gpu_utils.is_environment_variable_defined(
'CUDA_VISIBLE_DEVICES'):
s = gpu_utils.get_environment_variable('CUDA_VISIBLE_DEVICES')
s_lst = s.split(',')
if len(s_lst) == 1 and len(s_lst[0]) > 0:
gpu_id = int(s_lst[0])
else:
gpu_id = None
else:
gpus = gpu_utils.get_gpu_information()
if len(gpus) == 1:
gpu_id = 0
else:
gpu_id = None
lr = learning_rate_init
num_batches = int(self.train_dataset.get_num_examples() /
self.batch_size)
for epoch in range(self.max_num_training_epochs):
avg_loss = 0.
for _ in range(num_batches):
X_batch, y_batch = self.train_dataset.next_batch(
self.batch_size)
train_feed.update({X_pl: X_batch, y_pl: y_batch, lr_pl: lr})
_, c = sess.run([optimizer, loss], feed_dict=train_feed)
avg_loss += c / num_batches
# if spent more time than budget, exit.
if (timer_manager.get_time_since_event(
'eval', 'start', 'minutes') >
self.max_eval_time_in_minutes):
break
# early stopping
val_acc = self._compute_accuracy(sess, X_pl, y_pl, num_correct,
self.val_dataset, eval_feed)
# Display logs per epoch step
if self.log_output_to_terminal and epoch % self.display_step == 0:
print("time:", "%7.1f" %
timer_manager.get_time_since_event('eval', 'start'),
"epoch:", '%04d' % (epoch + 1), "loss:",
"{:.9f}".format(avg_loss), "validation_accuracy:",
"%.5f" % val_acc, "learning_rate:", '%.3e' % lr)
d = {
'validation_accuracy':
val_acc,
'training_loss':
avg_loss,
'epoch_number':
epoch + 1,
'learning_rate':
lr,
'time_in_minutes':
timer_manager.get_time_since_event('eval',
'start',
units='minutes'),
}
# adding information about gpu utilization if available.
if gpu_id is not None:
gpus = gpu_utils.get_gpu_information()
d.update({
'gpu_utilization_in_percent':
gpus[gpu_id]['gpu_utilization_in_percent'],
'gpu_memory_utilization_in_gigabytes':
gpus[gpu_id]['gpu_memory_utilization_in_gigabytes']
})
seqs.append(d)
# update the patience counters.
if best_val_acc < val_acc:
best_val_acc = val_acc
# reinitialize all the counters.
stop_counter = stop_patience
rate_counter = learning_rate_patience
save_counter = save_patience
else:
stop_counter -= 1
rate_counter -= 1
if stop_counter == 0:
break
if rate_counter == 0:
lr = max(lr * learning_rate_mult, learning_rate_min)
rate_counter = learning_rate_patience
if best_val_acc_saved < val_acc:
save_counter -= 1
if save_counter == 0:
save_path = saver.save(sess, self.model_path)
print("Model saved in file: %s" % save_path)
save_counter = save_patience
best_val_acc_saved = val_acc
# if spent more time than budget, exit.
if (timer_manager.get_time_since_event('eval', 'start',
'minutes') >
self.max_eval_time_in_minutes):
break
# if the model saved has better performance than the current model,
# load it.
if best_val_acc_saved > val_acc:
saver.restore(sess, self.model_path)
print("Model restored from file: %s" % save_path)
print("Optimization Finished!")
timer_manager.tick_timer('eval')
val_acc = self._compute_accuracy(sess, X_pl, y_pl, num_correct,
self.val_dataset, eval_feed)
t_infer = (
timer_manager.get_time_since_last_tick('eval', 'miliseconds') /
self.val_dataset.get_num_examples())
print("Validation accuracy: %f" % val_acc)
seqs_dict = seqs.get_dict()
results = {
'validation_accuracy': val_acc,
'num_parameters': float(htf.get_num_trainable_parameters()),
'inference_time_per_example_in_miliseconds': t_infer,
'num_training_epochs': seqs_dict['epoch_number'],
'sequences': seqs_dict
}
if 'gpu_utilization_in_percent' in seqs_dict:
results['average_gpu_utilization_in_percent'] = np.mean(
seqs_dict['gpu_utilization_in_percent'])
results[
'average_gpu_memory_utilization_in_gigabytes'] = np.mean(
seqs_dict['gpu_memory_utilization_in_gigabytes'])
if self.test_dataset != None:
test_acc = self._compute_accuracy(sess, X_pl, y_pl, num_correct,
self.test_dataset, eval_feed)
print("Test accuracy: %f" % test_acc)
results['test_accuracy'] = test_acc
results['training_time_in_hours'] = timer_manager.get_time_since_event(
'eval', 'start', units='hours')
return results
def model_fn(features, labels, mode, params):
feature_columns = list(get_feature_columns().values())
images = tf.feature_column.input_layer(
features=features, feature_columns=feature_columns)
images = tf.reshape(images,
shape=(-1, IMAGE_HEIGHT, IMAGE_WIDTH,
IMAGE_DEPTH))
set_recompile(outputs, True)
gc.collect()
htfe.set_is_training(outputs, mode == tf.estimator.ModeKeys.TRAIN)
co.forward({inputs['in']: images})
logits = outputs['out'].val
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
predicted_classes = tf.argmax(logits, 1)
if mode == tf.estimator.ModeKeys.PREDICT:
predictions = {
'class_ids': predicted_classes[:, tf.newaxis],
'probabilities': tf.nn.softmax(logits),
'logits': logits,
}
return tf.estimator.EstimatorSpec(mode,
predictions=predictions)
# define loss and optimizer
train_vars = tf.trainable_variables()
with tf.variable_scope('l2'):
l2_loss = tf.add_n([
tf.nn.l2_loss(v) for v in train_vars if 'kernel' in v.name
]) * self.weight_decay
unreg_loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=labels))
loss = unreg_loss + l2_loss
# Compute evaluation metrics.
accuracy = tf.metrics.accuracy(labels=tf.argmax(labels, 1),
predictions=predicted_classes,
name='acc_op')
metrics = {'accuracy': accuracy}
if mode == tf.estimator.ModeKeys.EVAL:
loss = tf.Print(loss, [
accuracy, l2_loss, unreg_loss, loss,
tf.argmax(labels, 1), predicted_classes
],
summarize=10)
return tf.estimator.EstimatorSpec(mode,
loss=loss,
eval_metric_ops=metrics)
# Create training op.
assert mode == tf.estimator.ModeKeys.TRAIN
step = tf.train.get_or_create_global_step()
learning_rate = self.get_learning_rate(step)
optimizer = tf.train.RMSPropOptimizer(learning_rate,
.9,
momentum=.9,
epsilon=1.0)
loss = tf.Print(loss, [
accuracy, l2_loss, unreg_loss, loss, learning_rate,
tf.argmax(labels, 1), predicted_classes
],
summarize=10)
with tf.control_dependencies(update_ops):
train_op = optimizer.minimize(
loss, global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode,
loss=loss,
train_op=train_op)
def model_fn(features, labels, mode, params):
setRecompile(outputs.values(), True)
gc.collect()
htfe.setTraining(outputs.values(),
mode == tf.estimator.ModeKeys.TRAIN)
step = tf.train.get_or_create_global_step()
if 'In' in inputs:
co.forward({inputs['In']: features})
logits = outputs['Out'].val
else:
co.forward({
inputs['In0']:
features,
inputs['In1']:
tf.math.divide(
tf.cast(step, tf.float32),
float(self.steps_per_epoch *
self.max_num_training_epochs))
})
logits = outputs['Out1'].val
aux_logits = outputs['Out0'].val
predicted_classes = tf.argmax(logits, 1, output_type=tf.int32)
if mode == tf.estimator.ModeKeys.PREDICT:
predictions = {
'class_ids': predicted_classes[:, tf.newaxis],
'probabilities': tf.nn.softmax(logits),
'logits': logits,
}
return tf.estimator.EstimatorSpec(mode, predictions=predictions)
# define loss and optimizer
train_vars = tf.trainable_variables()
with tf.variable_scope('l2'):
l2_loss = tf.add_n([
tf.nn.l2_loss(v) for v in train_vars if 'kernel' in v.name
]) * self.weight_decay
onehot_labels = tf.one_hot(labels, 10)
unreg_loss = tf.losses.softmax_cross_entropy(
onehot_labels=onehot_labels,
logits=logits,
reduction=tf.losses.Reduction.MEAN)
aux_loss = tf.losses.softmax_cross_entropy(
onehot_labels=onehot_labels,
logits=aux_logits,
weights=.5,
reduction=tf.losses.Reduction.MEAN) if 'Out1' in outputs else 0
loss = unreg_loss + l2_loss + aux_loss
if mode == tf.estimator.ModeKeys.EVAL:
return tf.contrib.tpu.TPUEstimatorSpec(
mode,
loss=loss,
eval_metrics=(metric_fn, [labels, predicted_classes]))
# Create training op.
assert mode == tf.estimator.ModeKeys.TRAIN
if self.num_parameters == -1:
self.num_parameters = np.sum([
np.prod(v.get_shape().as_list())
for v in tf.trainable_variables()
])
accuracy = metric_fn(labels, predicted_classes)['accuracy']
tf.identity(accuracy[1], name='train_accuracy')
learning_rate = self.get_learning_rate(step)
metric_dict = {
'batch_loss':
loss,
'learning_rate':
learning_rate,
'batch_accuracy':
tf.reduce_mean(
tf.cast(tf.equal(predicted_classes, labels), tf.float32))
}
host_fn = None
optimizer = self.get_optimizer(learning_rate)
if self.use_tpu:
host_fn = construct_host_fn(metric_dict,
model_dir,
prefix='training/',
max_queue_size=self.steps_per_epoch)
optimizer = tf.contrib.tpu.CrossShardOptimizer(optimizer)
else:
record_summaries(metric_dict, step)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
gvs = optimizer.compute_gradients(loss)
gvs = [(tf.where(tf.is_nan(grad), tf.zeros_like(grad),
grad), val) for grad, val in gvs]
train_op = optimizer.apply_gradients(
gvs, global_step=tf.train.get_or_create_global_step())
return tf.contrib.tpu.TPUEstimatorSpec(mode,
loss=loss,
train_op=train_op,
host_call=host_fn)
def eval(self, inputs, outputs, save_fn=None, state=None):
if state is not None and 'model_dir' in state:
model_dir = state['model_dir']
else:
model_dir = gcu.get_empty_bucket_folder(self.base_dir)
if save_fn:
save_fn({'model_dir': model_dir})
logger.info('Using folder %s for evaluation', model_dir)
latest = tf.train.latest_checkpoint(model_dir)
init_epoch = 0
train_dataset = input_fn('train',
self.data_dir,
batch_size=self.batch_size,
train=True,
num_outputs=len(outputs))
val_dataset = input_fn('validation',
self.data_dir,
batch_size=self.batch_size,
train=False,
num_outputs=len(outputs))
if latest:
model = tf.keras.models.load_model(latest)
init_epoch = int(latest.split('.')[-4])
else:
input_placeholder = tf.keras.Input(
train_dataset.output_shapes[0][1:])
x = input_placeholder
step = tf.train.get_or_create_global_step()
if 'in' in inputs:
co.forward({inputs['in']: x})
logits = outputs['out'].val
logits = tf.keras.layers.Lambda(lambda x: x,
name='final_logits')(logits)
output_tensors = [
logits,
]
loss_weights = [1.0]
losses = [
lambda y_true, y_pred: tf.keras.losses.
sparse_categorical_crossentropy(
y_true, y_pred, from_logits=True),
]
accuracy_metric_name = 'sparse_categorical_accuracy'
else:
co.forward({
inputs['in0']:
x,
inputs['in1']:
float(self.steps_per_epoch * self.max_num_training_epochs)
})
logits = outputs['out1'].val
aux_logits = outputs['out0'].val
logits = tf.keras.layers.Lambda(lambda x: x,
name='final_logits')(logits)
aux_logits = tf.keras.layers.Lambda(
lambda x: x, name='aux_logits')(aux_logits)
output_tensors = [logits, aux_logits]
loss_weights = [1.0, .5]
losses = [
lambda y_true, y_pred: tf.keras.losses.
sparse_categorical_crossentropy(
y_true, y_pred, from_logits=True), lambda y_true,
y_pred: tf.keras.losses.sparse_categorical_crossentropy(
y_true, y_pred, from_logits=True)
]
accuracy_metric_name = 'final_logits_sparse_categorical_accuracy'
metrics = {'final_logits': ['sparse_categorical_accuracy']}
model = tf.keras.Model(inputs=input_placeholder,
outputs=output_tensors)
if self.use_tpu:
my_project_name = subprocess.check_output(
['gcloud', 'config', 'get-value', 'project'])
my_zone = subprocess.check_output(
['gcloud', 'config', 'get-value', 'compute/zone'])
cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(
self.tpu_name.split(','),
zone=my_zone,
project=my_project_name)
strategy = tf.contrib.tpu.TPUDistributionStrategy(
cluster_resolver)
model = tf.contrib.tpu.keras_to_tpu_model(model, strategy)
learning_rate = self.get_learning_rate(step)
optimizer = get_optimizer(self.optimizer_type, learning_rate)
tf.summary.scalar('learning_rate', learning_rate)
tensorboard_callback = tf.keras.callbacks.TensorBoard(
log_dir=model_dir)
checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
filepath=os.path.join(
model_dir, 'weights.{epoch:02d}.{val_loss:.2f}.hdf5'),
period=(self.max_num_training_epochs // 3) + 1)
model.compile(optimizer=optimizer,
loss=losses,
loss_weights=loss_weights,
metrics=metrics)
num_parameters = int(
np.sum([
np.prod(v.get_shape().as_list())
for v in set(model.trainable_weights)
]))
logger.info('Number of parameters for %s: %d', model_dir,
num_parameters)
# print(model.metrics_names)
# return model
timer_manager = ut.TimerManager()
timer_manager.create_timer('eval')
# return model
training_history = model.fit(
(lambda: train_dataset) if self.use_tpu else train_dataset,
epochs=self.max_num_training_epochs,
steps_per_epoch=self.steps_per_epoch,
validation_data=(lambda: val_dataset)
if self.use_tpu else val_dataset,
validation_steps=self.steps_per_val_epoch,
initial_epoch=init_epoch,
callbacks=[tensorboard_callback, checkpoint_callback],
)
test_dataset = input_fn('eval',
self.data_dir,
batch_size=self.batch_size,
train=False,
num_outputs=len(outputs))
test_results = model.evaluate(
(lambda: test_dataset) if self.use_tpu else test_dataset,
steps=5) #self.steps_per_test_epoch)
test_results = {
'test_' + metric_name: test_results[ix]
for (ix, metric_name) in enumerate(model.metrics_names)
}
results = {
'validation_accuracy':
training_history.history['val_' + accuracy_metric_name][-1],
'validation_loss':
training_history.history['val_loss'][-1],
'num_parameters':
num_parameters,
'num_training_epochs':
training_history.epoch,
}
results.update(test_results)
results.update(training_history.history)
results['test_accuracy'] = test_results['test_' + accuracy_metric_name]
results['training_time_in_hours'] = timer_manager.get_time_since_event(
'eval', 'start', units='hours')
return results
