def __init__(self, cfg):
super(SRGAN, self).__init__()
# Networks
self.generator = Generator(cfg)
self.discriminator = Discriminator(cfg)
# Optimizers
self.optim_gen = tr.optim.Adam(self.generator.parameters(),
lr=cfg.learning_rate,
betas=(cfg.beta1, 0.999))
self.optim_disc = tr.optim.Adam(self.discriminator.parameters(),
lr=cfg.learning_rate,
betas=(cfg.beta1, 0.999))
# loss models
self.mse_loss = tr.nn.MSELoss()
self.bce_loss = tr.nn.BCELoss(reduction='sum')
self.feature_extractor = FeatureExtractor()
# Get image dataset (cropped)
dataset = get_dataset(cfg.data_dir)
self.dataloader = tr.utils.data.DataLoader(dataset,
batch_size=cfg.batch_size,
shuffle=True)
# For logging results to tensorboard
self.global_step = 0
self.build_writers()
def __init__(self, cfg):
super(SRGAN, self).__init__()
self.dataset = get_dataset(cfg)
self.testset = get_testset(cfg)
# Training stats
self.global_step = 0
# Models
self.generator = Generator(cfg)
self.discriminator = Discriminator(cfg)
# Optimizers
self.generator_optimizer = tf.keras.optimizers.Adam(
cfg.learning_rate, cfg.beta1)
self.discriminator_optimizer = tf.keras.optimizers.Adam(
cfg.learning_rate, cfg.beta1)
# Loss
self.bce_loss = tf.keras.losses.BinaryCrossentropy(from_logits=True)
vgg19 = tf.keras.applications.vgg19.VGG19(
include_top=False,
weights='imagenet',
input_shape=(cfg.crop_size, cfg.crop_size) + (3, ))
self.vgg_features = tf.keras.models.Model(
inputs=vgg19.input, outputs=vgg19.get_layer('block4_conv4').output)
# Build writers for logging
self.build_writers()
def __init__(self):
super(ConvolutionalVariationalAutoencoder, self).__init__()
train_images, test_images = get_dataset()
self.train_dataset = tf.data.Dataset.from_tensor_slices(train_images).shuffle(cfg.train_buf).batch(cfg.batch_size)
self.test_dataset = tf.data.Dataset.from_tensor_slices(test_images).shuffle(cfg.test_buf).batch(cfg.batch_size)
self.optimizer = tf.keras.optimizers.Adam(lr=cfg.learning_rate)
self.global_step = 0
self.model = CVAE(cfg.latent_dim)
self.build_writers()
def main(runsss, overlap, window_size, h_dim, z_dim, batch_size, language):
try:
def train(epoch):
train_loss = 0
tq = tqdm(train_loader)
for batch_idx, (data, _) in enumerate(tq):
data = Variable(data.squeeze().transpose(0, 1))
data = (data - data.min().item()) / (data.max().item() -
data.min().item())
#forward + backward + optimize
optimizer.zero_grad()
kld_loss, nll_loss, _, _ = model(data)
loss = kld_loss + nll_loss
loss.backward()
optimizer.step()
#grad norm clipping, only in pytorch version >= 1.10
nn.utils.clip_grad_norm(model.parameters(), clip)
tq.set_postfix(kld_loss=(kld_loss.item() / batch_size),
nll_loss=(nll_loss.item() / batch_size))
train_loss += loss.item()
return
def test(epoch):
"""uses test data to evaluate
likelihood of the model"""
mean_kld_loss, mean_nll_loss = 0, 0
tq = tqdm(test_loader)
for i, (data, _) in enumerate(tq):
#data = Variable(data)
data = Variable(data.squeeze().transpose(0, 1))
data = (data - data.min().item()) / (data.max().item() -
data.min().item())
kld_loss, nll_loss, _, _ = model(data)
mean_kld_loss += kld_loss.item()
mean_nll_loss += nll_loss.item()
mean_kld_loss /= len(test_loader.dataset)
mean_nll_loss /= len(test_loader.dataset)
print('====> Test set loss: KLD Loss = {:.4f}, NLL Loss = {:.4f} '.
format(mean_kld_loss, mean_nll_loss))
return
def train_classifier(param_string, train_loader_enc, test_loader_enc,
optimizer2, classify, criterion):
num_epochs = 100
best = 0
acc = []
train_acc = []
for epoch in range(num_epochs):
print(f'epoch num: {epoch}\n')
running_loss = 0.0
tq = tqdm(train_loader_enc)
for i, (data, labels) in enumerate(tq):
# zero the parameter gradients
optimizer2.zero_grad()
# forward + backward + optimize
outputs = classify(data)
# print(outputs, labels)
loss = criterion(outputs.float(), labels.long())
loss.backward()
optimizer2.step()
# print statistics
running_loss += loss.item()
tq.set_postfix(running_loss=(running_loss))
acc.append(
test_classifier(train_loader_enc,
test_loader_enc,
classify,
flag=False,
param_string=param_string))
train_acc.append(
test_classifier(train_loader_enc,
test_loader_enc,
classify,
flag=True,
param_string=param_string))
print(acc[-1], best)
if acc[-1] > best:
best = acc[-1]
print(f'best acc of {best}')
f = open('class_acc.txt', 'a+')
for i in range(len(train_acc)):
f.write(param_string)
f.write(f'{i},{train_acc[i]},{acc[i]}\n')
f.close()
return max([*acc, *train_acc])
def test_classifier(train_loader_enc,
test_loader_enc,
classify,
flag=False,
param_string=""):
# evaluate
correct = 0
total = 0
classes = defaultdict(int)
wrong = defaultdict(int)
y_pred = []
y_true = []
with torch.no_grad():
if flag:
tq = tqdm(train_loader_enc)
for data in tq:
dat, labels = data
outputs = classify(dat)
for i in range(len(outputs)):
if outputs[i][0] >= outputs[i][1]:
pred = 0
else:
pred = 1
y_pred.append(pred)
y_true.append(labels[i].item())
if pred == labels[i].item():
correct += 1
classes[pred] += 1
else:
wrong[pred] += 1
total += 1
else:
tq = tqdm(test_loader_enc)
for data in tq:
dat, labels = data
outputs = classify(dat)
for i in range(len(outputs)):
if outputs[i][0] >= outputs[i][1]:
pred = 0
else:
pred = 1
y_pred.append(pred)
y_true.append(labels[i].item())
if pred == labels[i].item():
correct += 1
classes[pred] += 1
else:
wrong[pred] += 1
total += 1
f11 = f1_score(y_true, y_pred, average='binary')
[precision, recall, fbeta_score,
support] = precision_recall_fscore_support(y_true,
y_pred,
average='binary')
paramms = f'correct: {correct}, total: {total}, classes: {classes}, wrong: {wrong}, f1_score: {f11}, precision: {precision}, recall: {recall}, fbeta_score: {fbeta_score}, support: {support}'
print(paramms)
print(f'accuracy: {correct/total}')
if param_string:
f = open('class_acc.txt', 'a+')
f.write(param_string)
f.write(f'y_pred-{y_pred},y_true-{y_true}\n')
f.write(f'{paramms}\n')
f.close()
acc = correct / total
return acc
# transform inputs from test set to encoded vectors, make new training training loaders
def transform_inputs(loader, batch_size=batch_size):
encoded_inputs = []
labels = []
tq = tqdm(loader)
with torch.no_grad():
for batch_idx, (data, label) in enumerate(tq):
data = Variable(data.squeeze().transpose(0, 1))
data = (data - data.min().item()) / (data.max().item() -
data.min().item())
h = model.predict(data)
for i in range(h.shape[1]):
encoded_inputs.append(h[:, i, :].flatten().numpy())
labels.append(label[i].item())
return torch.utils.data.DataLoader(torch.utils.data.TensorDataset(
torch.Tensor(encoded_inputs), torch.Tensor(labels)),
batch_size=batch_size,
shuffle=True)
def run_classifier(epochh, fn):
for b_size in [4, 8, 16, 32]:
train_loader_enc = transform_inputs(train_loader, b_size)
test_loader_enc = transform_inputs(test_loader, b_size)
for intermediate_dim in [5, 10, 20, 40]:
for layers in [True, False]:
f = open(txt_file, 'a+')
f.write(
f'fn="{fn}",runsss={runsss},epochh={epochh},overlap={overlap},window_size={window_size},h_dim={h_dim},z_dim={z_dim},batch_size={batch_size},language={language},b_size={b_size},intermediate_dim={intermediate_dim},layers={layers}\n'
)
f.close()
classify = Classifier(input_dim=h_dim,
intermediate_dim=20,
layers=layers)
print(classify, classify.count_parameters())
criterion = nn.CrossEntropyLoss()
optimizer2 = torch.optim.Adam(classify.parameters(),
lr=0.001)
train_classifier(
f'fn="{fn}",runsss={runsss},epochh={epochh},overlap={overlap},window_size={window_size},h_dim={h_dim},z_dim={z_dim},batch_size={batch_size},language={language},b_size={b_size},intermediate_dim={intermediate_dim},layers={layers}\n',
train_loader_enc, test_loader_enc, optimizer2,
classify, criterion)
accuracy = test_classifier(train_loader_enc,
test_loader_enc,
classify,
flag=False)
print(f'final accuracy ---> {accuracy}')
print('Finished Training')
return
x = get_dataset2(overlap=overlap,
window_size=window_size,
time_steps=20,
language=language,
max_len=(100 if language == "english" else 80))
#hyperparameters
# x_dim = 70
x_dim = x['genuine'].shape[2]
# h_dim = 100
# z_dim = 16
n_layers = 1
n_epochs = 25
clip = 10
learning_rate = 3e-4
batch_size = batch_size
seed = 128
print_every = 10
save_every = 5
#manual seed
torch.manual_seed(seed)
#init model + optimizer + datasets
x_i = []
y_i = []
x_it = []
y_it = []
test_count = defaultdict(int)
tot = 80
for val in ['genuine', 'forged']:
for i in x[val]:
if val == 'genuine' and test_count['genuine'] < tot:
x_it.append(i)
y_it.append([1])
test_count['genuine'] += 1
elif val == 'forged' and test_count['forged'] < tot:
x_it.append(i)
y_it.append([0])
test_count['forged'] += 1
else:
x_i.append(i)
y_i.append([0] if val == 'genuine' else [1])
# print(len(x_i),len(y_i),len(x_it),len(y_it))
x_i, y_i, x_it, y_it = np.array(x_i), np.array(y_i).reshape(
(-1, )), np.array(x_it), np.array(y_it).reshape((-1, ))
if False:
signatures_train, signatures_test, labels_train, labels_test = get_dataset(
)
else:
signatures_train, signatures_test, labels_train, labels_test = x_i, x_it, y_i, y_it
print('input data\n', signatures_train.shape, labels_train.shape,
signatures_test.shape, labels_test.shape)
x_dim = signatures_train.shape[2]
train_loader = torch.utils.data.DataLoader(
torch.utils.data.TensorDataset(torch.Tensor(signatures_train),
torch.Tensor(labels_train)),
batch_size=batch_size,
shuffle=True)
test_loader = torch.utils.data.DataLoader(
torch.utils.data.TensorDataset(torch.Tensor(signatures_test),
torch.Tensor(labels_test)),
batch_size=batch_size,
shuffle=True)
model = VRNN(x_dim, h_dim, z_dim, n_layers)
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
for epoch in range(1, n_epochs + 1):
#training + testing
train(epoch)
test(epoch)
#saving model
if epoch % save_every == 1:
fn = 'saves/vrnn_state_dict_' + f'{runsss},{overlap},{window_size},{h_dim},{z_dim},{batch_size}.{language},{epoch}' + '.pth'
torch.save(model.state_dict(), fn)
print('Saved model to ' + fn)
run_classifier(epoch, fn + '\n')
# freeze model weights
for param in model.parameters():
param.requires_grad = False
except:
print('FAILED RUN')
f = open(txt_file, 'a+')
f.write(
f'FAILED RUN ---> {runsss},{overlap},{window_size},{h_dim},{z_dim},{batch_size}.{language}\n'
)
f.close()
def main(rootpath,
data_dir,
device,
training=True,
model_type="cnn",
sample_name=None,
display=False):
"""
The primary method for controlling the training and testing of data. rootpath is the parent directory of main.py, data_dir
is the subdirectory containing all training data, device is either CPU or GPU. Training is true if we are training a model
and false otherwise. If training is false, sample name is the path to the image that you are converting to LaTeX. If display
is true, print the output of the model for each character it is passed. The model type can be a convolutional neural network (cnn),
k-nearest neighbors (knn), or a decision tree (tree). The knn and tree models have already been trained and will not be trained if
training is true. This will only train the cnn.
"""
dataset = processing.get_dataset(data_dir,
img_size=28,
filename="symbols_rectified")
mean, std = torch.Tensor([.0942]), torch.Tensor(
[.2202]
) # The values that were computed using the processing.normalize() function
dataset.transforms = torchvision.transforms.Compose([
torchvision.transforms.Grayscale(num_output_channels=1),
torchvision.transforms.Resize([224, 224]),
torchvision.transforms.RandomRotation(degrees=5),
torchvision.transforms.ToTensor(),
torchvision.transforms.RandomErasing(p=.2),
torchvision.transforms.Normalize((mean), (std))
])
if training:
train_index, test_index = processing.test_train_split(dataset)
dataloader = processing.get_dataloaders(dataset,
train_index,
test_index,
batch_size=128)
model, val_acc_history = train_cnn(rootpath, dataset, dataloader,
device)
else:
output = []
characters, corners, areas = processing.find_symbols(sample_name,
display=True)
if model_type == "cnn":
num_classes = len(dataset.classes)
model, input_size = cnn.initialize_model(
"simple",
num_classes,
resume_from=os.path.join(rootpath, "weights",
"cnn_weights_epoch_4"))
model.eval()
for character in characters:
img = processing.preprocess(character,
img_size=224,
mean=mean,
std=std,
to_numpy=False,
display=True)
out = model(img.unsqueeze(0))
_, prediction = torch.topk(out, k=2, dim=1)
symbols = []
for pred in prediction[0]:
symbols.append(cnn.label_number_to_name(dataset, pred))
if display:
print(symbols[0])
output.append(symbols)
elif model_type == "knn":
for character in characters:
img = processing.preprocess(character,
img_size=224,
mean=None,
std=None,
to_numpy=True,
display=True)
out = knn.get_nearest_neighbors(
img, k=51) # The two most likely outputs
symbols = [out[0], out[1]]
if display:
print(symbols[0])
output.append(symbols)
elif model_type == "tree":
for character in characters:
img = processing.preprocess(character,
img_size=224,
mean=None,
std=None,
to_numpy=True,
display=True)
out = tree.get_label(img)
symbols = [out, out]
if display:
print(symbols[0])
output.append(symbols)
else:
print("That is not a valid model name")
equation = convert.to_latex(output, corners)
print(equation)
("train" if (option2 == "3") else "validate") + "?\n" + LogColors.ENDC +
"1. Multinomial NB\n"
"2. Bernoulli NB\n"
"3. Gaussian NB\n"
"4. Complement NB\n"
+ LogColors.FAIL + "0. Exit\n" + LogColors.ENDC)
if (int(option3) >= 0) and (int(option3) <= 3):
validOption = True
exit_on_zero(int(option3))
# Training
if int(option2) % 2 == 1:
print(LogColors.OKGREEN + "Training..." + LogColors.ENDC)
ds_training_features, ds_training_labels = get_dataset("ds" + option1 + "/ds" + option1 + "Train.csv")
if option2 == "1":
training.DT_train(ds_training_features, ds_training_labels, option1)
elif option2 == "3":
if option3 == "1":
training.MNB_train(ds_training_features, ds_training_labels, option1)
elif option3 == "2":
training.BNB_train(ds_training_features, ds_training_labels, option1)
elif option3 == "3":
training.GNB_train(ds_training_features, ds_training_labels, option1)
elif option3 == "4":
training.CNB_train(ds_training_features, ds_training_labels, option1)
elif option2 == "5":
training.MLP_NN_train(ds_training_features, ds_training_labels, option1)
def train(self):
'''Train the SRGAN using the just pretrained or past pretrained generator.'''
if os.path.isfile('./pretrained_models/pretrained.pt'):
print("loaded_checkpoint: ./pretrained_models/pretrained.pt")
checkpoint = tr.load('./pretrained_models/pretrained.pt')
self.generator.load_state_dict(checkpoint['generator_state'])
self.optim_gen.load_state_dict(checkpoint['generator_optimizer'])
self.generator.train()
print('Training SRGAN')
ds = tr.FloatTensor(cfg.batch_size, cfg.num_channels,
cfg.cropsize // cfg.factor,
cfg.cropsize // cfg.factor)
# Restart global step for SRGAN tape
self.global_step = 0
real_label = tr.ones((cfg.batch_size, 1))
fake_label = tr.zeros((cfg.batch_size, 1))
for epoch in trange(cfg.epochs):
# Mini-batchs
for i, data in enumerate(self.dataloader):
# Generate data
hr, _ = data
# Prevent error if the dataloader runs out of images
if hr.size(0) < cfg.batch_size:
break
# mini-batch data (ToTensor() normalizes between [0,1])
# randn() instead of rand() for normalized input [-1,1]
nz = tr.rand((hr.size(0), hr.size(1), hr.size(2) // cfg.factor,
hr.size(3) // cfg.factor))
for j in range(cfg.batch_size):
ds[j] = downsample(hr[j])
hr[j] = normalize(hr[j])
nz[j] = normalize(nz[j])
## Begin Training Discriminator
# ----------------------------------------------------------------
self.discriminator.zero_grad()
# Why zero_grad
# https://stackoverflow.com/questions/48001598/why-do-we-need-to-call-zero-grad-in-pytorch
# BCELoss use explained here
# https://pytorch.org/tutorials/beginner/dcgan_faces_tutorial.html
# train real
truth = self.discriminator(hr)
loss_truth = self.bce_loss(truth, real_label)
loss_truth.backward()
# train fake
noise = self.generator(nz)
fake = self.discriminator(noise.detach())
loss_fake = self.bce_loss(fake, fake_label)
loss_fake.backward()
loss_disc = loss_truth + loss_fake
self.optim_disc.step()
# Begin Training Generator
# ----------------------------------------------------------------
self.generator.zero_grad()
# Generate the super resolution image and judge it
sr = self.generator(ds)
generated = self.discriminator(sr)
# Calculate Perceptual Loss
true_features = self.feature_extractor(hr)
generated_features = self.feature_extractor(sr)
content_loss = self.mse_loss(generated_features, true_features)
adversarial_loss = self.bce_loss(generated, real_label)
loss_gen = content_loss + (10e-3 * adversarial_loss)
# Optimize Generator
loss_gen.backward()
self.optim_gen.step()
# Log Losses
self.logger('SRGAN/Content Loss', content_loss)
self.logger('SRGAN/Adversarial Loss', adversarial_loss)
self.logger('SRGAN/Discriminator Loss', loss_disc)
self.logger('SRGAN/Generator Loss', loss_gen)
# Log Images
self.log_state('SRGAN/Original', hr[0])
self.log_state('SRGAN/Downsampled', ds[0])
self.log_state('SRGAN/Generated', sr[0])
# Increment Tape
self.global_step += 1
if epoch % cfg.save_freq == 0:
tr.save(
{
'generator_state': self.generator.state_dict(),
'generator_optimizer': self.optim_gen.state_dict()
}, self.save_path + '.pt')
# Get image dataset (cropped)
dataset = get_dataset(cfg.data_dir)
self.dataloader = tr.utils.data.DataLoader(
dataset, batch_size=cfg.batch_size, shuffle=True)