def check_overfitting_valid_data(self, config, dataholder):
if os.path.exists("checkpoints"):
shutil.rmtree("checkpoints")
graph = tf.Graph()
network = DFN(graph, config)
trainer = Trainer(graph, config, network, dataholder)
non_trained_acc = trainer.get_valid_accuracy()
trainer.fit(verbose=False)
trained_acc = trainer.get_valid_accuracy()
condition = non_trained_acc < trained_acc
msg = "Performance on valid data not better after training\n"
msg += " non_trained_acc = {0:.6f}".format(non_trained_acc)
msg += " | trained_acc = {0:.6f}".format(trained_acc)
self.assertTrue(condition, msg=msg)
def _train(self, project, classifierPath, modelType, epoch, numCell,
batchSize, maxLen, numClass, embeddingType):
trainer = Trainer(project, classifierPath, modelType, epoch, numCell,
batchSize, maxLen, numClass, embeddingType)
X, Y = None, None
for manual in self.manuals:
x = manual.vectors
if x is None:
continue
y = torch.tensor(manual.id).repeat(len(x))
if X is None:
X = x
Y = y
else:
X = torch.cat((X, x), dim=0)
Y = torch.cat((Y, y), dim=0)
trainer.fit(X, Y)
def check_prediction(self, config, dataholder, num_classes=4):
if os.path.exists("checkpoints"):
shutil.rmtree("checkpoints")
record_path = dataholder.get_test_tfrecord()
images, _, shape = reconstruct_from_record(record_path)
images = images.astype(np.float32) / 255
num_images = images.shape[0]
graph = tf.Graph()
network = DFN(graph, config)
trainer = Trainer(graph, config, network, dataholder)
non_trained_predictions = trainer.predict(images)
trainer.fit(verbose=False)
trained_predictions = trainer.predict(images)
image = images[0].reshape((1, images[0].shape[0]))
single_prediction = trainer.predict(image)
self.assertEqual(non_trained_predictions.shape, (num_images, ))
self.assertEqual(trained_predictions.shape, (num_images, ))
self.assertEqual(np.int32, non_trained_predictions.dtype)
self.assertEqual(np.int32, trained_predictions.dtype)
self.assertEqual(np.int32, single_prediction.dtype)
def train(self,
model,
train_dataloader,
valid_dataloader,
test_dataloader,
num_input=1,
chunk=False,
model_save_name=""):
trainer = Trainer(model=model,
device=self.device,
num_input=num_input,
model_save_name=model_save_name)
trainer.set_dataloader(train=train_dataloader,
valid=valid_dataloader,
test=test_dataloader)
return trainer.fit(chunk=chunk)
def optmizers_search(name_tfrecords,
records,
height,
width,
channels,
architecture,
activations,
conv_architecture,
kernel_sizes,
pool_kernel,
batch_size,
epochs,
num_steps,
save_step,
learning_rate,
conv):
"""
Script to run optmizers search,
the result is saved on the file optmizers_results.txt
:param name_tfrecords: name of the used tfrecords
:type name_tfrecords: str
:param records: list of paths to train, test, and valid tfrecords
:type records: list of str
:param height: image height
:type heights: int
:param width: image width
:type width: int
:param channels: image channels
:type channels: int
:param architecture: network architecture
:type architecture: list of int
:param activations: list of different tf functions
:type activations: list of tf.nn.sigmoid, tf.nn.relu, tf.nn.tanh
:param conv_architecture: convolutional architecture
:type conv_architecture: list of int
:param kernel_sizes: filter sizes
:type kernel_sizes: list of int
:param pool_kernel: pooling filter sizes
:type pool_kernel: list of int
:param batch_size: batch size for training
:type batch_size: int
:param epochs: number of epochs
:type epochs: int
:param num_steps: number of iterations for each epoch
:type num_steps: int
:param save_step: when step % save_step == 0, the model
parameters are saved.
:type save_step: int
:param learning_rate: learning rate for the optimizer
:type learning_rate: float
:param conv: param to control if the model will be a CNN
or DFN
:type conv: bool
"""
OT = [tf.train.GradientDescentOptimizer,
tf.train.AdadeltaOptimizer,
tf.train.AdagradOptimizer,
tf.train.AdamOptimizer,
tf.train.FtrlOptimizer,
tf.train.ProximalGradientDescentOptimizer,
tf.train.ProximalAdagradOptimizer,
tf.train.RMSPropOptimizer]
OT_name = ["GradientDescentOptimizer",
"AdadeltaOptimizer",
"AdagradOptimizer",
"AdamOptimizer",
"FtrlOptimizer",
"ProximalGradientDescentOptimizer",
"ProximalAdagradOptimizer",
"RMSPropOptimizer"]
numeric_result = []
results = []
info = []
if conv:
net_name = "CNN"
else:
net_name = "DFN"
header = "\nSearching optimizer for the {} model in the {} data\n".format(net_name, # noqa
name_tfrecords) # noqa
print(header)
for name, opt in zip(OT_name, OT):
config = Config(height=height,
width=width,
channels=channels,
architecture=architecture,
activations=activations,
conv_architecture=conv_architecture,
kernel_sizes=kernel_sizes,
pool_kernel=pool_kernel,
batch_size=batch_size,
epochs=epochs,
num_steps=num_steps,
save_step=save_step,
learning_rate=learning_rate,
optimizer=opt)
data = DataHolder(config,
records=records)
print(name + ":\n")
graph = tf.Graph()
if conv:
network = CNN(graph, config)
else:
network = DFN(graph, config)
trainer = Trainer(graph, config, network, data)
trainer.fit(verbose=True)
valid_acc = trainer.get_valid_accuracy()
numeric_result.append(valid_acc)
name += ': valid_acc = {0:.6f} | '.format(valid_acc)
test_images, test_labels, _ = reconstruct_from_record(data.get_test_tfrecord()) # noqa
test_images = test_images.astype(np.float32) / 255
test_pred = trainer.predict(test_images)
acc_cat = accuracy_per_category(test_pred, test_labels, categories=3)
for i, cat_result in enumerate(acc_cat):
name += int2command[i] + ": = {0:.6f}, ".format(cat_result)
results.append(name)
if os.path.exists("checkpoints"):
shutil.rmtree("checkpoints")
info.append(str(config))
best_result = max(list(zip(numeric_result, OT_name, info)))
result_string = """In an experiment with different optmizers
the best one is {0} with valid accuracy of {1}.
\nThe training uses the following params:
\n{2}\n""".format(best_result[1],
best_result[0],
best_result[2])
file = open("optmizers_results.txt", "w")
file.write(header)
file.write("Results for different optmizers\n")
for result in results:
result += "\n"
file.write(result)
file.write("\n")
file.write(result_string)
file.close()
model.visualize_results(epoch='pre', pre_training=True)
# model.fit()
shouldTrain = False
downloadModels = False
if shouldTrain:
## Experiments:
# Real
for exp in ['Real']:
model = Trainer(device='cuda:0', batch_size=4, experiment=exp)
model.trainset, model.valset, model.testset = trainset, valset, testset
for optim in ['Adam', 'SGD']:
for scheduler in ['StepLR', 'ReduceLROnPlateau']:
print(f'Running: {exp}, {optim}, {scheduler}')
model.build(optim=optim, scheduler=scheduler)
model.fit()
# Noise
for exp in ['Noise']:
model = Trainer(device='cuda:0', batch_size=4, experiment=exp)
model.trainset, model.valset, model.testset = trainset, valset, testset
for optim in ['Adam', 'SGD']:
for scheduler in ['StepLR', 'ReduceLROnPlateau']:
print(f'Running: {exp}, {optim}, {scheduler}')
model.build(optim=optim, scheduler=scheduler)
model.fit()
# Average
for exp in ['Average']:
model = Trainer(device='cuda:0', batch_size=4, experiment=exp)
model.trainset, model.valset, model.testset = trainset, valset, testset
def train(name_tfrecords,
records,
height,
width,
channels,
architecture,
activations,
conv_architecture,
kernel_sizes,
pool_kernel,
batch_size,
epochs,
num_steps,
save_step,
learning_rate,
optimizer,
verbose,
name,
move,
conv=False):
"""
Trains a model
:param name_tfrecords: name of the used tfrecords
:type name_tfrecords: str
:param records: list of paths to train, test, and valid tfrecords
:type records: list of str
:param height: image height
:type heights: int
:param width: image width
:type width: int
:param channels: image channels
:type channels: int
:param architecture: network architecture
:type architecture: list of int
:param activations: list of different tf functions
:type activations: list of tf.nn.sigmoid, tf.nn.relu, tf.nn.tanh
:param conv_architecture: convolutional architecture
:type conv_architecture: list of int
:param kernel_sizes: filter sizes
:type kernel_sizes: list of int
:param pool_kernel: pooling filter sizes
:type pool_kernel: list of int
:param batch_size: batch size for training
:type batch_size: int
:param epochs: number of epochs
:type epochs: int
:param num_steps: number of iterations for each epoch
:type num_steps: int
:param save_step: when step % save_step == 0, the model
parameters are saved.
:type save_step: int
:param learning_rate: learning rate for the optimizer
:type learning_rate: float
:param optimizer: a optimizer from tensorflow.
:type optimizer: tf.train.GradientDescentOptimizer,
tf.train.AdadeltaOptimizer,
tf.train.AdagradOptimizer,
tf.train.AdagradDAOptimizer,
tf.train.AdamOptimizer,
tf.train.FtrlOptimizer,
tf.train.ProximalGradientDescentOptimizer,
tf.train.ProximalAdagradOptimizer,
tf.train.RMSPropOptimizer
:param verbose: param to control if the trainig will be printed
and if the confusion matrix will be calculated.
:type verbose: bool
:param name: name to save the confusion matrix plot.
:type name: str
:param move: param to control if the checkpoints path
will be moved to the parent folder.
:type move: bool
:param conv: param to control if the model will be a CNN
or DFN
:type conv: bool
"""
if os.path.exists("checkpoints"):
shutil.rmtree("checkpoints")
config = Config(height=height,
width=width,
channels=channels,
architecture=architecture,
activations=activations,
conv_architecture=conv_architecture,
kernel_sizes=kernel_sizes,
pool_kernel=pool_kernel,
batch_size=batch_size,
epochs=epochs,
num_steps=num_steps,
save_step=save_step,
learning_rate=learning_rate,
optimizer=optimizer)
data = DataHolder(config,
records=records)
graph = tf.Graph()
if conv:
network_name = "CNN"
network = CNN(graph, config)
else:
network_name = "DFN"
network = DFN(graph, config)
trainer = Trainer(graph, config, network, data)
print("\nTraining in the {} data using one {}\n".format(name_tfrecords,
network_name))
print("params:\n{}\n".format(str(config)))
trainer.fit(verbose=verbose)
if verbose:
valid_images, valid_labels, _ = reconstruct_from_record(data.get_valid_tfrecord()) # noqa
valid_images = valid_images.astype(np.float32) / 255
valid_pred = trainer.predict(valid_images)
valid_labels = valid_labels.reshape((valid_labels.shape[0],))
plotconfusion(valid_labels,
valid_pred,
name + ".png",
int2command,
classes=["left", "right", "up"])
if move:
dst = os.path.join(parentdir, "checkpoints")
shutil.move("checkpoints", dst)
def _train(self, X, Y, project, modelPath, modelType, epoch, batchSize,
maxLen, numClass):
trainer = Trainer(project, modelPath, modelType, epoch, 300, batchSize,
maxLen, numClass, 'EmbeddingLayer')
trainer.fit(X, Y)
def architecture_search(name_tfrecords, records, height, width, channels,
conv_architecture, kernel_sizes, pool_kernel,
batch_size, epochs, num_steps, save_step,
learning_rate, optimizer, experiments,
deepest_net_size, conv):
"""
Script to search different architectures for a DFN,
the result is saved on the file architecture_results.txt
:param name_tfrecords: name of the used tfrecords
:type name_tfrecords: str
:param records: list of paths to train, test, and valid tfrecords
:type records: list of str
:param height: image height
:type heights: int
:param width: image width
:type width: int
:param channels: image channels
:type channels: int
:param conv_architecture: convolutional architecture
:type conv_architecture: list of int
:param kernel_sizes: filter sizes
:type kernel_sizes: list of int
:param pool_kernel: pooling filter sizes
:type pool_kernel: list of int
:param batch_size: batch size for training
:type batch_size: int
:param epochs: number of epochs
:type epochs: int
:param num_steps: number of iterations for each epoch
:type num_steps: int
:param save_step: when step % save_step == 0, the model
parameters are saved.
:type save_step: int
:param learning_rate: learning rate for the optimizer
:type learning_rate: float
:param optimizer: a optimizer from tensorflow.
:type optimizer: tf.train.GradientDescentOptimizer,
tf.train.AdadeltaOptimizer,
tf.train.AdagradOptimizer,
tf.train.AdagradDAOptimizer,
tf.train.AdamOptimizer,
tf.train.FtrlOptimizer,
tf.train.ProximalGradientDescentOptimizer,
tf.train.ProximalAdagradOptimizer,
tf.train.RMSPropOptimizer
:param experiments: number of experiments to be made
:type experiments: int
:param deepest_net_size: size of the deepest network
:type deepest_net_size: int
:param conv: param to control if the model will be a CNN
or DFN
:type conv: bool
"""
sizes = np.random.randint(1, deepest_net_size, experiments)
hidden_layers, activations = get_random_architecture_and_activations(
network_sizes=sizes) # noqa
numeric_result = []
results = []
info = []
if conv:
net_name = "CNN"
else:
net_name = "DFN"
header = "\nSearching {} architecture in the {} data\n".format(
net_name, name_tfrecords) # noqa
print(header)
for arch, act in zip(hidden_layers, activations):
config = Config(height=height,
width=width,
channels=channels,
architecture=arch,
activations=act,
conv_architecture=conv_architecture,
kernel_sizes=kernel_sizes,
pool_kernel=pool_kernel,
batch_size=batch_size,
epochs=epochs,
num_steps=num_steps,
save_step=save_step,
learning_rate=learning_rate,
optimizer=optimizer)
data = DataHolder(config, records=records)
name = str(arch)
print(name + ":\n")
graph = tf.Graph()
if conv:
network = CNN(graph, config)
else:
network = DFN(graph, config)
trainer = Trainer(graph, config, network, data)
trainer.fit(verbose=True)
valid_acc = trainer.get_valid_accuracy()
numeric_result.append(valid_acc)
name += ': valid_acc = {0:.6f} | '.format(valid_acc)
valid_images, valid_labels, _ = reconstruct_from_record(
data.get_valid_tfrecord()) # noqa
valid_images = valid_images.astype(np.float32) / 255
valid_pred = trainer.predict(valid_images)
acc_cat = accuracy_per_category(valid_pred, valid_labels, categories=3)
for i, cat_result in enumerate(acc_cat):
name += int2command[i] + ": = {0:.6f}, ".format(cat_result)
results.append(name)
if os.path.exists("checkpoints"):
shutil.rmtree("checkpoints")
info.append(str(config))
best_result = max(list(zip(numeric_result, hidden_layers, info)))
result_string = """In an experiment with {0} architectures
the best one is {1} with valid accuracy of {2}.
\nThe training uses the following params:
\n{3}\n""".format(experiments, best_result[1], best_result[0],
best_result[2])
file = open("architecture_results.txt", "w")
file.write(header)
file.write("Results for different architectures\n")
for result in results:
result += "\n"
file.write(result)
file.write("\n")
file.write(result_string)
file.close()
