def check_overfitting_valid_data(self, config, dataholder):
if os.path.exists("checkpoints"):
shutil.rmtree("checkpoints")
graph = tf.Graph()
network = DFN(graph, config)
trainer = Trainer(graph, config, network, dataholder)
non_trained_acc = trainer.get_valid_accuracy()
trainer.fit(verbose=False)
trained_acc = trainer.get_valid_accuracy()
condition = non_trained_acc < trained_acc
msg = "Performance on valid data not better after training\n"
msg += " non_trained_acc = {0:.6f}".format(non_trained_acc)
msg += " | trained_acc = {0:.6f}".format(trained_acc)
self.assertTrue(condition, msg=msg)
def optmizers_search(name_tfrecords,
records,
height,
width,
channels,
architecture,
activations,
conv_architecture,
kernel_sizes,
pool_kernel,
batch_size,
epochs,
num_steps,
save_step,
learning_rate,
conv):
"""
Script to run optmizers search,
the result is saved on the file optmizers_results.txt
:param name_tfrecords: name of the used tfrecords
:type name_tfrecords: str
:param records: list of paths to train, test, and valid tfrecords
:type records: list of str
:param height: image height
:type heights: int
:param width: image width
:type width: int
:param channels: image channels
:type channels: int
:param architecture: network architecture
:type architecture: list of int
:param activations: list of different tf functions
:type activations: list of tf.nn.sigmoid, tf.nn.relu, tf.nn.tanh
:param conv_architecture: convolutional architecture
:type conv_architecture: list of int
:param kernel_sizes: filter sizes
:type kernel_sizes: list of int
:param pool_kernel: pooling filter sizes
:type pool_kernel: list of int
:param batch_size: batch size for training
:type batch_size: int
:param epochs: number of epochs
:type epochs: int
:param num_steps: number of iterations for each epoch
:type num_steps: int
:param save_step: when step % save_step == 0, the model
parameters are saved.
:type save_step: int
:param learning_rate: learning rate for the optimizer
:type learning_rate: float
:param conv: param to control if the model will be a CNN
or DFN
:type conv: bool
"""
OT = [tf.train.GradientDescentOptimizer,
tf.train.AdadeltaOptimizer,
tf.train.AdagradOptimizer,
tf.train.AdamOptimizer,
tf.train.FtrlOptimizer,
tf.train.ProximalGradientDescentOptimizer,
tf.train.ProximalAdagradOptimizer,
tf.train.RMSPropOptimizer]
OT_name = ["GradientDescentOptimizer",
"AdadeltaOptimizer",
"AdagradOptimizer",
"AdamOptimizer",
"FtrlOptimizer",
"ProximalGradientDescentOptimizer",
"ProximalAdagradOptimizer",
"RMSPropOptimizer"]
numeric_result = []
results = []
info = []
if conv:
net_name = "CNN"
else:
net_name = "DFN"
header = "\nSearching optimizer for the {} model in the {} data\n".format(net_name, # noqa
name_tfrecords) # noqa
print(header)
for name, opt in zip(OT_name, OT):
config = Config(height=height,
width=width,
channels=channels,
architecture=architecture,
activations=activations,
conv_architecture=conv_architecture,
kernel_sizes=kernel_sizes,
pool_kernel=pool_kernel,
batch_size=batch_size,
epochs=epochs,
num_steps=num_steps,
save_step=save_step,
learning_rate=learning_rate,
optimizer=opt)
data = DataHolder(config,
records=records)
print(name + ":\n")
graph = tf.Graph()
if conv:
network = CNN(graph, config)
else:
network = DFN(graph, config)
trainer = Trainer(graph, config, network, data)
trainer.fit(verbose=True)
valid_acc = trainer.get_valid_accuracy()
numeric_result.append(valid_acc)
name += ': valid_acc = {0:.6f} | '.format(valid_acc)
test_images, test_labels, _ = reconstruct_from_record(data.get_test_tfrecord()) # noqa
test_images = test_images.astype(np.float32) / 255
test_pred = trainer.predict(test_images)
acc_cat = accuracy_per_category(test_pred, test_labels, categories=3)
for i, cat_result in enumerate(acc_cat):
name += int2command[i] + ": = {0:.6f}, ".format(cat_result)
results.append(name)
if os.path.exists("checkpoints"):
shutil.rmtree("checkpoints")
info.append(str(config))
best_result = max(list(zip(numeric_result, OT_name, info)))
result_string = """In an experiment with different optmizers
the best one is {0} with valid accuracy of {1}.
\nThe training uses the following params:
\n{2}\n""".format(best_result[1],
best_result[0],
best_result[2])
file = open("optmizers_results.txt", "w")
file.write(header)
file.write("Results for different optmizers\n")
for result in results:
result += "\n"
file.write(result)
file.write("\n")
file.write(result_string)
file.close()
def architecture_search(name_tfrecords, records, height, width, channels,
conv_architecture, kernel_sizes, pool_kernel,
batch_size, epochs, num_steps, save_step,
learning_rate, optimizer, experiments,
deepest_net_size, conv):
"""
Script to search different architectures for a DFN,
the result is saved on the file architecture_results.txt
:param name_tfrecords: name of the used tfrecords
:type name_tfrecords: str
:param records: list of paths to train, test, and valid tfrecords
:type records: list of str
:param height: image height
:type heights: int
:param width: image width
:type width: int
:param channels: image channels
:type channels: int
:param conv_architecture: convolutional architecture
:type conv_architecture: list of int
:param kernel_sizes: filter sizes
:type kernel_sizes: list of int
:param pool_kernel: pooling filter sizes
:type pool_kernel: list of int
:param batch_size: batch size for training
:type batch_size: int
:param epochs: number of epochs
:type epochs: int
:param num_steps: number of iterations for each epoch
:type num_steps: int
:param save_step: when step % save_step == 0, the model
parameters are saved.
:type save_step: int
:param learning_rate: learning rate for the optimizer
:type learning_rate: float
:param optimizer: a optimizer from tensorflow.
:type optimizer: tf.train.GradientDescentOptimizer,
tf.train.AdadeltaOptimizer,
tf.train.AdagradOptimizer,
tf.train.AdagradDAOptimizer,
tf.train.AdamOptimizer,
tf.train.FtrlOptimizer,
tf.train.ProximalGradientDescentOptimizer,
tf.train.ProximalAdagradOptimizer,
tf.train.RMSPropOptimizer
:param experiments: number of experiments to be made
:type experiments: int
:param deepest_net_size: size of the deepest network
:type deepest_net_size: int
:param conv: param to control if the model will be a CNN
or DFN
:type conv: bool
"""
sizes = np.random.randint(1, deepest_net_size, experiments)
hidden_layers, activations = get_random_architecture_and_activations(
network_sizes=sizes) # noqa
numeric_result = []
results = []
info = []
if conv:
net_name = "CNN"
else:
net_name = "DFN"
header = "\nSearching {} architecture in the {} data\n".format(
net_name, name_tfrecords) # noqa
print(header)
for arch, act in zip(hidden_layers, activations):
config = Config(height=height,
width=width,
channels=channels,
architecture=arch,
activations=act,
conv_architecture=conv_architecture,
kernel_sizes=kernel_sizes,
pool_kernel=pool_kernel,
batch_size=batch_size,
epochs=epochs,
num_steps=num_steps,
save_step=save_step,
learning_rate=learning_rate,
optimizer=optimizer)
data = DataHolder(config, records=records)
name = str(arch)
print(name + ":\n")
graph = tf.Graph()
if conv:
network = CNN(graph, config)
else:
network = DFN(graph, config)
trainer = Trainer(graph, config, network, data)
trainer.fit(verbose=True)
valid_acc = trainer.get_valid_accuracy()
numeric_result.append(valid_acc)
name += ': valid_acc = {0:.6f} | '.format(valid_acc)
valid_images, valid_labels, _ = reconstruct_from_record(
data.get_valid_tfrecord()) # noqa
valid_images = valid_images.astype(np.float32) / 255
valid_pred = trainer.predict(valid_images)
acc_cat = accuracy_per_category(valid_pred, valid_labels, categories=3)
for i, cat_result in enumerate(acc_cat):
name += int2command[i] + ": = {0:.6f}, ".format(cat_result)
results.append(name)
if os.path.exists("checkpoints"):
shutil.rmtree("checkpoints")
info.append(str(config))
best_result = max(list(zip(numeric_result, hidden_layers, info)))
result_string = """In an experiment with {0} architectures
the best one is {1} with valid accuracy of {2}.
\nThe training uses the following params:
\n{3}\n""".format(experiments, best_result[1], best_result[0],
best_result[2])
file = open("architecture_results.txt", "w")
file.write(header)
file.write("Results for different architectures\n")
for result in results:
result += "\n"
file.write(result)
file.write("\n")
file.write(result_string)
file.close()
