def calc_embeddings(config):
"""
"""
save_folder = Path(config.save_folder)
save_folder.mkdir(exist_ok=True, parents=True)
train_loader, test_loader = loaders.loader(batch_size_train=100,
batch_size_test=1000,
shuffle=False)
model1 = VGG_embedding().to(device)
loadVGG(model1, path=Path(config.save_folder) / "vgg.pt")
for trtst in "test", "train":
loader = train_loader if trtst == "train" else test_loader
# Test the model
model1.eval()
embeddings_list = []
labels_list = []
one_hot_pred_list = []
images_list = []
with torch.no_grad(): # turn off gradient calc
for images, labels in tqdm(loader):
images = images.to(device)
labels = labels.to(device)
embeddings = model1.get_embedding(images)
pred = model1.forward_embedding(
embeddings).cpu() # this will output the softmax
embeddings_list.append(
embeddings.cpu()) # torch.Size([100, 512])
labels_list.append(labels) # torch.Size([100])
one_hot_pred_list.append(pred) # torch.Size([100, 27])
images_list.append(images.cpu())
# Calculate space
space_image = (torch.zeros(*images.shape) +
torch.min(images).tolist()).to(device)[0:1]
label = torch.zeros([1]).to(device)
label[0] = 0
embd = model1.get_embedding(space_image)
pred = model1.forward_embedding(embd).cpu()
embeddings_list.append(embd.cpu())
labels_list.append(label)
one_hot_pred_list.append(pred)
images_list.append(space_image.cpu())
lists = [
embeddings_list, labels_list, one_hot_pred_list, images_list
]
for ii, ll in enumerate(lists):
lists[ii] = torch.cat(ll)
# Save new calculated embeddings one hot encoded
# if "normalized" in str(OUTPUT):
# for i in range(0,len(embeddings_list)):
# embeddings_list[i] = torch.nn.functional.normalize(embeddings_list[i], dim=-1)
torch.save((lists), save_folder / f'{trtst}_emb_dataset.pt')
def test_model_load():
""" Initialized model test set accuracy: Less than 5%
Loaded model test set accuracy: usually around 95.3894%"""
num_epochs = 200
learning_rate = .007
train_loader, test_loader = loaders.loader(batch_size_train=100,
batch_size_test=1000)
#model1 = VGG().to(device)
model1 = VGG_embedding().to(device)
# Loss and optimizer
criterion = nn.CrossEntropyLoss()
optimizer1 = torch.optim.Adam(model1.parameters(), lr=learning_rate)
#Check test accuracy before load
model1.eval()
with torch.no_grad(): # turn off gradient calc
correct1 = 0
total1 = 0
for images, labels in test_loader:
images = images.to(device)
labels = labels.to(device)
outputs = model1(images)
_, predicted = torch.max(outputs.data, 1)
total1 += labels.size(0)
correct1 += (predicted == labels).sum().item()
print('Test Accuracy of NN: {} % (improvement)'.format(100 * correct1 /
total1))
loadVGG(model1)
# Test the model
model1.eval()
with torch.no_grad(): # turn off gradient calc
correct1 = 0
total1 = 0
for images, labels in test_loader:
images = images.to(device)
labels = labels.to(device)
outputs = model1(images)
_, predicted = torch.max(outputs.data, 1)
total1 += labels.size(0)
correct1 += (predicted == labels).sum().item()
print('Test Accuracy of NN: {} % (improvement)'.format(100 * correct1 /
total1))
exit()
def main(num_epochs=200,
learning_rate=0.005,
momentum=0.5,
log_interval=500,
*args,
**kwargs):
train_loader, test_loader = loaders.loader(batch_size_train=100,
batch_size_test=1000)
# Train the model
total_step = len(train_loader)
curr_lr1 = learning_rate
model1 = VGG().to(device)
# Loss and optimizer
criterion = nn.CrossEntropyLoss()
optimizer1 = torch.optim.Adam(model1.parameters(), lr=learning_rate)
# Train the model
total_step = len(train_loader)
best_accuracy1 = 0
for epoch in range(num_epochs):
for i, (images, labels) in enumerate(train_loader):
images = images.to(device)
labels = labels.to(device)
# Forward
outputs = model1(images)
loss1 = criterion(outputs, labels)
# Backward and optimize
optimizer1.zero_grad()
loss1.backward()
optimizer1.step()
if i == 499:
print(
"Ordinary Epoch [{}/{}], Step [{}/{}] Loss: {:.4f}".format(
epoch + 1, num_epochs, i + 1, total_step,
loss1.item()))
# Test the model
model1.eval()
with torch.no_grad():
correct1 = 0
total1 = 0
for images, labels in test_loader:
images = images.to(device)
labels = labels.to(device)
outputs = model1(images)
_, predicted = torch.max(outputs.data, 1)
total1 += labels.size(0)
correct1 += (predicted == labels).sum().item()
if best_accuracy1 >= correct1 / total1:
curr_lr1 = learning_rate * np.asscalar(
pow(np.random.rand(1), 3))
update_lr(optimizer1, curr_lr1)
print('Test Accuracy of NN: {} % Best: {} %'.format(
100 * correct1 / total1, 100 * best_accuracy1))
else:
best_accuracy1 = correct1 / total1
net_opt1 = model1
print('Test Accuracy of NN: {} % (improvement)'.format(
100 * correct1 / total1))
model1.train()
def main(config, *args, **kwargs):
# Loads random characters
train_loader, test_loader = loaders.loader(
batch_size_train=config.batch_size_train,
batch_size_test=config.batch_size_test)
# Train the model
total_step = len(train_loader)
curr_lr1 = config.learning_rate
#model1 = VGG().to(device)
model1 = VGG_embedding().to(device)
# Loss and optimizer
criterion = nn.CrossEntropyLoss()
optimizer1 = torch.optim.Adam(model1.parameters(), lr=config.learning_rate)
scheduler = ReduceLROnPlateau(optimizer1,
'min',
patience=config.patience,
factor=config.decay_factor)
# Train the model
total_step = len(train_loader)
best_accuracy1 = 0
best_model = None
space_image = (torch.zeros(28, 28) + -.4242)[None, None, :, :].to(device)
for epoch in range(config.epochs):
model1.train()
for i, (images, labels) in enumerate(
train_loader
): #Modify if lengths need to be returned from collate fn
images = images.to(device)
images = torch.cat([images, space_image], axis=0)
labels = torch.cat([labels, torch.zeros(1).int()])
labels = labels.to(device)
# Forward
outputs = model1(images)
loss1 = criterion(outputs, labels)
# Backward and optimize
optimizer1.zero_grad(
) # clears old gradients from the previus step
loss1.backward(
) # computes the derivative of the loss w,r,t the params using back propogation
optimizer1.step(
) # causes the optimizer to take a step based on the gradients of the params (updates the weights)
#scheduler.step(loss1)
if i == 499:
logger.log(
"Ordinary Epoch [{}/{}], Step [{}/{}] Loss: {:.4f}".format(
epoch + 1, config.epochs, i + 1, total_step,
loss1.item()))
if config.TESTING:
break
# Test the model
model1.eval(
) # turns off dropout layers and batchnorm layers, aka sets model in inference mode
with torch.no_grad(): # turn off gradient calc
correct1 = 0
total1 = 0
for images, labels in test_loader:
images = images.to(device)
labels = labels.to(device)
outputs = model1(images)
_, predicted = torch.max(
outputs.data,
1) # returns vals and indices with dim to reduce = 1(cols)
total1 += labels.size(0)
correct1 += (predicted == labels).sum().item()
if best_accuracy1 >= correct1 / total1:
curr_lr1 = config.learning_rate * np.asscalar(
pow(np.random.rand(1), 3))
update_lr(optimizer1, curr_lr1)
logger.log('Test Accuracy of NN: {} % Best: {} %'.format(
100 * correct1 / total1, 100 * best_accuracy1))
else:
best_accuracy1 = correct1 / total1
net_opt1 = model1
logger.log('Test Accuracy of NN: {} % (improvement)'.format(
100 * correct1 / total1))
# Save best model - Comment out if you intend on using the supercomputer to only write the best model after training
best_model = model1
saveVGG(model1, path=Path(config.save_folder) / "vgg.pt")
def calc_embeddings():
"""
Returns:
"""
num_epochs = 200
learning_rate = .007
train_loader, test_loader = loaders.loader(batch_size_train=100,
batch_size_test=1000)
model1 = VGG_embedding().to(device)
criterion = nn.CrossEntropyLoss()
optimizer1 = torch.optim.Adam(model1.parameters(), lr=learning_rate)
loadVGG(model1)
# Test the model
model1.eval()
data = None
labs = None
preds = None
with torch.no_grad(): # turn off gradient calc
correct1 = 0
total1 = 0
x = list()
y = list()
z = list()
for images, labels in train_loader:
images = images.to(device)
labels = labels.to(device)
# # Calc embeddings for a space ' '
# images = np.full((1, 28, 28), fill_value=-0.4242, dtype=np.float32)
# images = torch.from_numpy(images)
# images = torch.unsqueeze(images, 0)
# labels = torch.tensor([0])
# labels = labels.to(device)
# images = images.to(device)
embeddings = model1.get_embedding(images)
pred = model1.classifier(embeddings)
x.append(embeddings)
y.append(labels)
z.append(pred)
for i in range(len(x)):
if i == 0:
data = x[i]
labs = y[i]
preds = z[i]
else:
data = torch.cat((data, x[i]), 0)
labs = torch.cat((labs, y[i]), 0)
preds = torch.cat((preds, z[i]), )
# Save new calculated embeddings one hot encoded
# save_dataset(data, labs, preds, 'data/train_emb_dataset.pt')
torch.save((data, labs, preds), 'data/train_emb_dataset.pt')
# Calculate Testset embeddings
model1.eval()
data = None
labs = None
preds = None
with torch.no_grad():
correct1 = 0
total1 = 0
x = list()
y = list()
z = list()
for images, labels in test_loader:
images = images.to(device)
labels = labels.to(device)
embeddings = model1.get_embedding(images)
pred = model1.classifier(embeddings)
x.append(embeddings)
y.append(labels)
z.append(pred)
for i in range(len(x)):
if i == 0:
data = x[i]
labs = y[i]
preds = z[i]
else:
data = torch.cat((data, x[i]), 0)
labs = torch.cat((labs, y[i]), 0)
preds = torch.cat((preds, z[i]), )
# Save new calculated embeddings one hot encoded
# save_dataset(data, labs, preds, 'data/test_emb_dataset.pt')
torch.save((data, labs, preds), 'data/test_emb_dataset.pt')
exit()
def main(num_epochs=200,
learning_rate=0.005,
momentum=0.5,
log_interval=500,
*args,
**kwargs):
calc_embeddings()
#test_model_load()
# Loads random characters
train_loader, test_loader = loaders.loader(batch_size_train=100,
batch_size_test=1000)
# Train the model
total_step = len(train_loader)
curr_lr1 = learning_rate
#model1 = VGG().to(device)
model1 = VGG_embedding().to(device)
# Loss and optimizer
criterion = nn.CrossEntropyLoss()
optimizer1 = torch.optim.Adam(model1.parameters(), lr=learning_rate)
# Train the model
total_step = len(train_loader)
best_accuracy1 = 0
best_model = None
### VISUAL PART
# Create model
# Sample letters
# Choose random images of letters
# Restrict language model to predict only A-z
# Pre train the language model
# [Calculate statistics on visual model only OR train it]
for epoch in range(num_epochs):
for i, (images, labels) in enumerate(
train_loader
): #Modify if lengths need to be returned from collate fn
images = images.to(device)
labels = labels.to(device)
# Forward
outputs = model1(images)
loss1 = criterion(outputs, labels)
# Backward and optimize
optimizer1.zero_grad(
) # clears old gradients from the previus step
loss1.backward(
) # computes the derivative of the loss w,r,t the params using back propogation
optimizer1.step(
) # causes the optimizer to take a step based on the gradients of the params (updates the weights)
if i == 499:
print(
"Ordinary Epoch [{}/{}], Step [{}/{}] Loss: {:.4f}".format(
epoch + 1, num_epochs, i + 1, total_step,
loss1.item()))
# Test the model
model1.eval(
) # turns off dropout layers and batchnorm layers, aka sets model in inference mode
with torch.no_grad(): # turn off gradient calc
correct1 = 0
total1 = 0
for images, labels in test_loader:
images = images.to(device)
labels = labels.to(device)
outputs = model1(images)
_, predicted = torch.max(
outputs.data,
1) # returns vals and indices with dim to reduce = 1(cols)
total1 += labels.size(0)
correct1 += (predicted == labels).sum().item()
if best_accuracy1 >= correct1 / total1:
curr_lr1 = learning_rate * np.asscalar(
pow(np.random.rand(1), 3))
update_lr(optimizer1, curr_lr1)
print('Test Accuracy of NN: {} % Best: {} %'.format(
100 * correct1 / total1, 100 * best_accuracy1))
else:
best_accuracy1 = correct1 / total1
net_opt1 = model1
print('Test Accuracy of NN: {} % (improvement)'.format(
100 * correct1 / total1))
# Save best model - Comment out if you intend on using the supercomputer to only write the best model after training
best_model = model1
saveVGG(model1)
model1.train()