def trainForAsigment(n_epochs, n_hidden_layers, mode, mnist_data, mnist_test):
optimizer_dict = getOptimizer(mode)
optimizerFunc = optimizer_dict["optimizerFunc"]
has_momentum = optimizer_dict["has_momentum"]
momentum = optimizer_dict["momentum"]
learning_rate = optimizer_dict["learning_rate"]
update_lr = optimizer_dict["update_lr"]
lossFunc = nn.MSELoss()
hf.myprint("Using MSE as loss function")
dataset_full = torch.utils.data.ConcatDataset([mnist_data, mnist_test])
new_train, new_test = torch.utils.data.random_split(
dataset_full, [50000, 20000])
return trainAutoencoder(n_hidden_layers,
new_train,
new_test,
saved_encoder=None,
n_epochs=n_epochs,
optimizerFunc=optimizerFunc,
lossFunc=lossFunc,
learning_rate=learning_rate,
update_lr=update_lr,
has_momentum=has_momentum,
momentum=momentum,
batch_size=1000,
dropout_rate=0.1,
log=True,
log_interval=10)
def train_cnn_for_assigment(n_epochs, mnist_data, mnist_test):
optimizerFunc = torch.optim.Adam
has_momentum = False
momentum = None
learning_rate = 0.001
update_lr = False
lossFunc = nn.MSELoss()
hf.myprint("Using MSE as loss function")
dataset_full = torch.utils.data.ConcatDataset([mnist_data, mnist_test])
new_train, new_test = torch.utils.data.random_split(
dataset_full, [50000, 20000])
return train_cnn(new_train,
new_test,
saved_encoder=None,
n_epochs=n_epochs,
optimizerFunc=optimizerFunc,
lossFunc=lossFunc,
learning_rate=learning_rate,
update_lr=update_lr,
has_momentum=has_momentum,
momentum=momentum,
batch_size=1000,
dropout_rate=0.1,
log=True,
log_interval=10)
def get_learning_rate_update(epoch, learning_rate):
if (epoch % 10 == 0 and epoch != 0):
new_learning_rate = learning_rate / 1.5
hf.myprint(
f"Lowering learning rate from {learning_rate} to {new_learning_rate}"
)
return new_learning_rate
else:
return learning_rate
def evaluateSavedNetwork(filename):
train_loader = torch.utils.data.DataLoader(mnist_data,
batch_size=glval.batch_size,
shuffle=True)
test_loader = torch.utils.data.DataLoader(mnist_test,
batch_size=glval.batch_size)
saved_encoder = torch.load(filename)
test_results = ac.testEncoder(saved_encoder, test_loader, train_loader,
nn.MSELoss(), ac.Flatten())
hf.myprint(
f"Saved Network Train Loss: {test_results[0]} \n\tSaved Network Test Loss: {test_results[1]}"
)
ac.compareEncoder(saved_encoder, mnist_test, (0, 20), save=True)
def train_for_assigment(ac_name, a_encoder, n_epochs, mnist_data, mnist_test):
hf.myprint(f"Starting assigment for {ac_name}...")
optimizerFunc = torch.optim.Adam
learning_rate = 0.001
optimizer = optimizerFunc(a_encoder.parameters(), lr=learning_rate)
batch_size = 250
# lossFunc = nn.MSELoss()
lossFunc = nn.BCELoss()
dataset_full = torch.utils.data.ConcatDataset([mnist_data,mnist_test])
mnist_data, mnist_test = torch.utils.data.random_split(dataset_full, [50000,20000])
train_loader = torch.utils.data.DataLoader(mnist_data, batch_size=batch_size, shuffle=True)
test_loader = torch.utils.data.DataLoader(mnist_test, batch_size=batch_size)
train_losses_av = []
train_error_arr = []
test_error_arr = []
test_results = testEncoder(a_encoder, test_loader, train_loader, lossFunc)
hf.myprint(f"Training {ac_name} with {n_epochs} epochs. Using MSE as loss function and Adam as optimizer with learning rate: {learning_rate}")
hf.myprint(f"Starting Train Loss: {test_results[0]} \n\tStarting Test Loss: {test_results[1]}")
for epoch in range(n_epochs):
train_epoch(epoch, a_encoder, optimizer, lossFunc, train_loader, train_losses_av)
test_results = testEncoder(a_encoder, test_loader, train_loader, lossFunc)
train_error_arr.append(test_results[0])
test_error_arr.append(test_results[1])
hf.myprint(f"Current av Train Loss: {test_results[0]} \n\tCurrent av Test Loss: {test_results[1]}")
return_dict = {
"autoencoder": a_encoder,
"train_losses_av": train_losses_av,
"train_error_arr": train_error_arr,
"test_error_arr": test_error_arr
}
return return_dict
def trainEpoch(epoch, a_encoder, optimizer, lossFunc, train_loader,
train_losses_av):
a_encoder.train()
train_loss_av = 0
for batch_n, (data, _) in enumerate(train_loader):
optimizer.zero_grad()
output = a_encoder(data)
loss = lossFunc(output, data)
train_loss_av += loss.detach().item()
loss.backward()
optimizer.step()
if (batch_n % 5 == 0):
hf.myprint(
'Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
epoch + 1, batch_n * len(data), len(train_loader.dataset),
100. * batch_n / len(train_loader), loss.item()))
train_loss_av /= len(train_loader)
hf.myprint(f"Train loss Average: {train_loss_av}")
train_losses_av.append(train_loss_av)
def train_epoch(epoch, autoenc, optimizer, lossFunc, flatten, train_loader,
log, log_interval, train_losses_drop):
autoenc.train()
train_loss_drop = 0
for batch_n, (data, _) in enumerate(train_loader):
optimizer.zero_grad()
output = autoenc(data)
loss = lossFunc(output, flatten(data))
train_loss_drop += loss.detach().item()
loss.backward()
optimizer.step()
if (log and batch_n % log_interval == 0):
hf.myprint(
'Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
epoch + 1, batch_n * len(data), len(train_loader.dataset),
100. * batch_n / len(train_loader), loss.item()))
train_loss_drop /= len(train_loader)
hf.myprint(f"Train loss Drop: {train_loss_drop}")
train_losses_drop.append(train_loss_drop)
def train_epoch(epoch, autoenc, optimizer, lossFunc, train_loader, train_losses_av):
autoenc.train()
train_loss_av = 0
log_interval = 10
for batch_n, (data, _) in enumerate(train_loader):
data = data.to(global_device)
optimizer.zero_grad()
output = autoenc(data)
loss = get_loss(autoenc, data, lossFunc)
train_loss_av += loss.detach().item()
loss.backward()
optimizer.step()
if (batch_n % log_interval == 0):
hf.myprint('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format(
epoch+1, batch_n * len(data), len(train_loader.dataset),
100. * batch_n / len(train_loader), loss.item()))
train_loss_av /= len(train_loader)
hf.myprint(f"Train loss average: {train_loss_av}")
train_losses_av.append(train_loss_av)
def plotComparison():
#print(type(hf.loadTrainingLog(64, 0)))
hiddenlis = [64, 128, 256, 512]
for m in range(3):
dic_lis = [hf.loadTrainingLog(h, m) for h in hiddenlis]
_, ax = plt.subplots(figsize=(10, 5))
ax.set_xlim(-0.2, 40 + 0.5)
ax.set_ylabel('Error')
ax.set_yscale("log")
ax.set_xlabel('Epoch')
optimFunc = ["SGD", "Adam", "RMSprop"]
plt.title(label=f"Testing Loss with {optimFunc[m]}")
x_values = [x for x in range(40 + 1)]
for i, dic in enumerate(dic_lis):
ax.plot(np.array(x_values),
np.array(dic["test_error_arr"]),
label=f"{hiddenlis[i]} hidden neurons")
ax.legend(loc="upper right")
filename = f'graph_b_m{m}.png'
plt.savefig(filename, dpi=200)
hf.myprint("\tSaved " + filename)
plt.clf()
for h in hiddenlis:
dic_lis = [hf.loadTrainingLog(h, m) for m in range(3)]
_, ax = plt.subplots(figsize=(10, 5))
ax.set_xlim(-0.2, 40 + 0.5)
ax.set_ylabel('Error')
ax.set_yscale("log")
ax.set_xlabel('Epoch')
optimFunc = ["SGD", "Adam", "RMSprop"]
plt.title(label=f"Testing Loss with {h} neurons on the hidden layer")
x_values = [x for x in range(40 + 1)]
for m, dic in enumerate(dic_lis):
ax.plot(np.array(x_values),
np.array(dic["test_error_arr"]),
label=f"{optimFunc[m]}")
ax.legend(loc="upper right")
filename = f'graph_c_h{h}.png'
plt.savefig(filename, dpi=200)
hf.myprint("\tSaved " + filename)
plt.clf()
def getOptimizer(mode):
if mode == 0:
hf.myprint("Using SGD as optimizer")
dic = {
"optimizerFunc": torch.optim.SGD,
"has_momentum": True,
"momentum": 0.5,
"learning_rate": 0.5,
"update_lr": True
}
return dic
elif mode == 1:
hf.myprint("Using Adam as optimizer")
dic = {
"optimizerFunc": torch.optim.Adam,
"has_momentum": False,
"momentum": None,
"learning_rate": 0.001,
"update_lr": True
}
return dic
elif mode == 2:
hf.myprint("Using RMSprop as optimizer")
dic = {
"optimizerFunc": torch.optim.RMSprop,
"has_momentum": True,
"momentum": 0.1,
"learning_rate": 0.001,
"update_lr": False
}
return dic
else:
raise ValueError("Incorrect Mode in getOptimizer()")
def testEncoder(an_encoder, test_loader, train_loader):
hf.myprint("Testing encoder...")
an_encoder.eval()
loss_func1 = nn.MSELoss()
loss_func2 = nn.BCELoss()
test_loss1 = 0
test_loss2 = 0
train_loss1 = 0
train_loss2 = 0
with torch.no_grad():
for data, _ in test_loader:
output = an_encoder(data)
test_loss1 += loss_func1(output, data).item()
test_loss2 += loss_func2(output, data).item()
for data, _ in train_loader:
output = an_encoder(data)
train_loss1 += loss_func1(output, data).item()
train_loss2 += loss_func2(output, data).item()
train_loss1 /= len(train_loader)
train_loss2 /= len(train_loader)
test_loss1 /= len(test_loader)
test_loss2 /= len(test_loader)
return (train_loss1, train_loss2, test_loss1, test_loss2)
def compareEncoder(autoenc,
mnist_test,
number_range,
save=False,
basename='',
tight=False):
flatten = Flatten()
if type(number_range) == type(tuple):
rang = range(number_range[0], number_range[1])
else:
rang = number_range
for i in rang:
_, axes = plt.subplots(figsize=(10, 6), ncols=2, nrows=1)
axes[0].imshow(mnist_test[i][0][0], cmap='gray')
img = autoenc(flatten(
mnist_test[i][0]).unsqueeze(0)).detach().numpy().reshape([28, 28])
axes[1].imshow(
img,
cmap='gray',
)
if save:
filename = basename + f'_comp{i}.png'
if tight:
plt.axis('off') #doesn't work
plt.savefig(filename,
dpi=150,
bbox_inches='tight',
transparent="True",
pad_inches=0)
else:
plt.savefig(filename, dpi=200)
hf.myprint("\tSaved " + filename)
plt.clf()
else:
plt.show()
def train_cnn(n_epochs, mnist_data, mnist_test):
optimizerFunc = torch.optim.Adam
learning_rate = 0.001
lossFunc = nn.BCELoss()
batch_size = 1000
a_encoder = cnn_aut()
hf.myprint(f"Training Convolutionl autoencoder with {n_epochs} epochs")
hf.myprint(
f"Using Binary Cross Entropy as loss function. Adam as optimizer with learning rate: {learning_rate}"
)
dataset_full = torch.utils.data.ConcatDataset([mnist_data, mnist_test])
new_train, new_test = torch.utils.data.random_split(
dataset_full, [50000, 20000])
train_loader = torch.utils.data.DataLoader(mnist_data,
batch_size=batch_size,
shuffle=True)
test_loader = torch.utils.data.DataLoader(mnist_test,
batch_size=batch_size)
optimizer = optimizerFunc(a_encoder.parameters(), lr=learning_rate)
train_losses_av = []
train_error_arr1 = []
train_error_arr2 = []
test_error_arr1 = []
test_error_arr2 = []
test_results = testEncoder(a_encoder, test_loader, train_loader)
train_error_arr1.append(test_results[0])
train_error_arr2.append(test_results[1])
test_error_arr1.append(test_results[2])
test_error_arr2.append(test_results[3])
hf.myprint(
f"Starting Train MSELoss: {test_results[0]} \n\tStarting Test MSELoss: {test_results[2]}"
)
hf.myprint(
f"Starting Train BCELoss: {test_results[1]} \n\tStarting Test BCELoss: {test_results[3]}"
)
for epoch in range(n_epochs):
optimizer = optimizerFunc(a_encoder.parameters(), lr=learning_rate)
trainEpoch(epoch, a_encoder, optimizer, lossFunc, train_loader,
train_losses_av)
test_results = testEncoder(a_encoder, test_loader, train_loader)
train_error_arr1.append(test_results[0])
train_error_arr2.append(test_results[1])
test_error_arr1.append(test_results[2])
test_error_arr2.append(test_results[3])
hf.myprint(
f"Current av Train MSELoss: {test_results[0]} \n\tCurrent av Test MSELoss: {test_results[2]}"
)
hf.myprint(
f"Current av Train BCELoss: {test_results[1]} \n\tCurrent av Test BCELoss: {test_results[3]}"
)
return_dict = {
"autoencoder": a_encoder,
"train_losses_av": train_losses_av,
"train_error_arr1": train_error_arr1,
"train_error_arr2": train_error_arr2,
"test_error_arr1": test_error_arr1,
"test_error_arr2": test_error_arr2
}
return return_dict
def trainAutoencoder(amount_hidden,
mnist_data,
mnist_test,
saved_encoder=None,
n_epochs=glval.n_epochs,
optimizerFunc=glval.optimizerFunc,
lossFunc=glval.lossFunc,
learning_rate=glval.learning_rate,
update_lr=False,
has_momentum=glval.has_momentum,
momentum=glval.momentum,
batch_size=glval.batch_size,
dropout_rate=glval.dropout_rate,
log=True,
log_interval=glval.log_interval):
hf.myprint(
f"Training autoencoder with of {amount_hidden} hidden layers with {n_epochs} epochs"
)
mom = 0 if not has_momentum else momentum
hf.myprint(
f"learning rate: {learning_rate} - dropout rate: {dropout_rate} - momentum: {mom}"
)
hf.myprint(
f"Testing dataset size:{len(mnist_test)} - Training dataset size:{len(mnist_data)}"
)
train_loader = torch.utils.data.DataLoader(mnist_data,
batch_size=batch_size,
shuffle=True)
test_loader = torch.utils.data.DataLoader(mnist_test,
batch_size=batch_size)
if saved_encoder == None:
a_encoder = autoencoder(size_hidden=amount_hidden,
dropout=dropout_rate)
else:
hf.myprint("Loaded saved network")
a_encoder = saved_encoder
if has_momentum:
optimizer = optimizerFunc(a_encoder.parameters(),
lr=learning_rate,
momentum=momentum)
else:
optimizer = optimizerFunc(a_encoder.parameters(), lr=learning_rate)
flatten = Flatten()
train_losses_drop = []
train_error_arr = []
test_error_arr = []
test_results = testEncoder(a_encoder, test_loader, train_loader, lossFunc,
flatten)
train_error_arr.append(test_results[0])
test_error_arr.append(test_results[1])
if log:
hf.myprint(
f"Starting Train Loss: {test_results[0]} \n\tStarting Test Loss: {test_results[1]}"
)
for epoch in range(n_epochs):
learning_rate = get_learning_rate_update(epoch, learning_rate)
if update_lr:
if has_momentum:
optimizer = optimizerFunc(a_encoder.parameters(),
lr=learning_rate,
momentum=momentum)
else:
optimizer = optimizerFunc(a_encoder.parameters(),
lr=learning_rate)
train_epoch(epoch, a_encoder, optimizer, lossFunc, flatten,
train_loader, log, log_interval, train_losses_drop)
test_results = testEncoder(a_encoder, test_loader, train_loader,
lossFunc, flatten)
train_error_arr.append(test_results[0])
test_error_arr.append(test_results[1])
if log:
hf.myprint(
f"Current av Train Loss: {test_results[0]} \n\tCurrent av Test Loss: {test_results[1]}"
)
return_dict = {
"autoencoder": a_encoder,
"train_losses_drop": train_losses_drop,
"train_error_arr": train_error_arr,
"test_error_arr": test_error_arr
}
return return_dict
import torchvision.datasets as datasets
from torch import nn
import autoencoder as ac
import global_var as glval
import warnings
import seaborn as sns
import json
from final_autoencoder import train_cnn
hf.lineprint("Loading datasets and parameters...")
warnings.filterwarnings("ignore", category=UserWarning)
sns.set_style('darkgrid')
sns.set_context('talk')
sns.set(font_scale=0.7)
hf.myprint("Completed")
torch.manual_seed(12345678)
try:
hf.myprint(f"Running on {torch.cuda.get_device_name(0)}")
except Exception:
hf.myprint("No GPU available")
mnist_mean = 0.1307
mnist_standard_deviation = 0.3081
mnist_data = datasets.MNIST('./data',
train=True,
download=True,
transform=torchvision.transforms.Compose([
torchvision.transforms.ToTensor(),
def runConvolutionalAutoencoder(number_of_epochs,
plot_graph=False,
save_network=False,
save_training_log=False,
compare=True,
log=True,
show=True):
hf.lineprint("Processing autoencoder...")
trained_dict = ac.train_cnn_for_assigment(number_of_epochs,
mnist_data=mnist_data,
mnist_test=mnist_test)
basename = f"cnn_epch{number_of_epochs}"
trained_encoder = trained_dict["autoencoder"]
if compare:
ac.compareEncoder(trained_encoder,
mnist_test, (0, 50),
save=True,
basename=basename)
if save_network:
torch.save(trained_encoder, basename + ".savednn")
if save_training_log:
with open(basename + "_traintest.txt", 'w') as f:
writedict = trained_dict
del writedict["autoencoder"]
f.write(json.dumps(writedict))
if plot_graph:
_, ax = plt.subplots(figsize=(10, 5))
ax.set_xlim(-0.2, number_of_epochs + 0.5)
ax.set_ylabel('Error')
ax.set_yscale("log")
ax.set_xlabel('Epoch')
x_values = [x for x in range(number_of_epochs + 1)]
ax.plot(np.array(x_values),
np.array(trained_dict["test_error_arr"]),
label="Test Error")
ax.plot(np.array(x_values),
np.array(trained_dict["train_error_arr"]),
label="Training Error")
ax.legend(loc="upper right")
filename = f'asignmentGraph_a_h{number_hidden}_epch{number_of_epochs}_m{mode}.png'
plt.savefig(filename, dpi=200)
hf.myprint("\tSaved " + filename)
plt.clf()
_, ax = plt.subplots(figsize=(10, 5))
ax.set_xlim(5, number_of_epochs)
ax.set_ylabel('Error')
ax.set_yscale("log")
ax.set_xlabel('Epoch')
x_values = [x for x in range(5, number_of_epochs + 1)]
ax.plot(np.array(x_values),
np.array(trained_dict["test_error_arr"][5:]),
label="Test Error")
ax.plot(np.array(x_values),
np.array(trained_dict["train_error_arr"][5:]),
label="Training Error")
ax.legend(loc="upper right")
filename = f'asignmentGraph_a_h{number_hidden}_epch{number_of_epochs}_m{mode}_zoomed.png'
plt.savefig(filename, dpi=200)
hf.myprint("\tSaved " + filename)
plt.clf()
