def px(pae_dict):
saver = tf.train.Saver()
# P(x) Session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
# Restore pretrained model
saver.restore(sess, pxfinestr.split('.meta')[0])
if cp.get('Experiment', 'PREFIX') == 'MNIST':
# Save hidden/output layer results for pipeline training
px_Z_latent = utils.run_OOM(sess,
pae_dict['conv_in'],
XX_full,
pae_dict['conv_z'],
batch_size=batch_size)
else:
px_Z_latent = utils.run_OOM(sess,
pae_dict['sda_in'],
XX_full,
pae_dict['sda_hidden'],
batch_size=batch_size)
# Print clustering ACC
utils.log_accuracy(cp, YY_full, px_Z_latent, 'PX - ACC FULL', SEED)
# Print clustering NMI
utils.log_NMI(cp, YY_full, px_Z_latent, 'PX - NMI FULL', SEED)
# Print clustering CHS score
utils.log_CHS(cp, XX_full, px_Z_latent, 'PX - CHS FULL', SEED)
sess.close()
def evitram(evitramd):
saver = tf.train.Saver()
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
# Restore pretrained model
saver.restore(sess, evitramfinestr.split('.meta')[0])
if cp.get('Experiment', 'PREFIX') == 'MNIST':
# Save hidden/output layer results for pipeline training
px_Z_latent = utils.run_OOM(sess,
evitram_dict['conv_in'],
XX_full,
evitram_dict['conv_z'],
batch_size=batch_size)
# Save latent space
utils.save_OOM(sess,
evitram_dict['conv_in'],
XX_full,
evitram_dict['conv_z'],
path='COND_' + cp.get('Experiment', 'PX_Z_FULL'),
batch_size=batch_size)
# Save reconstruction
utils.save_OOM(sess,
evitram_dict['conv_in'],
XX_full,
evitram_dict['conv_out'],
path='COND_' +
cp.get('Experiment', 'PX_XREC_TRAIN'),
batch_size=batch_size)
else:
px_Z_latent = utils.run_OOM(sess,
evitram_dict['sda_in'],
XX_full,
evitram_dict['sda_hidden'],
batch_size=batch_size)
utils.save_OOM(sess,
evitram_dict['sda_in'],
XX_full,
evitram_dict['sda_hidden'],
path='COND_' + cp.get('Experiment', 'PX_Z_FULL'),
batch_size=batch_size)
# Print clustering ACC
utils.log_accuracy(cp, YY_full, px_Z_latent, 'COND - ACC FULL', SEED)
# Print clustering NMI
utils.log_NMI(cp, YY_full, px_Z_latent, 'COND - NMI FULL', SEED)
# Print clustering CHS score
utils.log_CHS(cp, XX_full, px_Z_latent, 'COND - CHS FULL', SEED)
sess.close()
def train_KT_AT(self):
# summary for current training loop and a running average object for loss
# Use tqdm for progress bar
accuracy_dict = {}
for epoch in tqdm(range(self.num_epochs), total=self.num_epochs):
self.student_model.train()
self.train()
if epoch % self.log_num == 0:
acc = self.test()
accuracy_dict[epoch] = acc
utils.log_accuracy("KD_AT.csv", accuracy_dict)
print(f"\nAccuracy: {acc:05.3f}")
self.save_model()
self.scheduler.step()
utils.plot_accuracy("KD_AT.csv")
def train(self):
best_acc = 0
accuracy_dict = {}
for batch in tqdm(range(self.num_epochs)):
for _ in range(self.ng):
self.generator_optimizer.zero_grad()
# generate guassian noise
z = torch.randn((128, 100)).to(self.device)
# get generator output
psuedo_datapoint = self.generator(z)
# teacher/student outputs: logits, attention1, attention2, attention3
# compute model output, fetch teacher/student output, and compute KD loss
student_logits = self.student_model(psuedo_datapoint)[0]
teacher_logits = self.teacher_model(psuedo_datapoint)[0]
generator_loss = -(utils.KL_Loss(student_logits, teacher_logits))
generator_loss.backward()
# performs updates using calculated gradients
torch.nn.utils.clip_grad_norm_(self.generator.parameters(), 5)
self.generator_optimizer.step()
psuedo_datapoint = psuedo_datapoint.detach()
with torch.no_grad():
teacher_outputs = self.teacher_model(psuedo_datapoint)
for _ in range(self.ns):
self.student_optimizer.zero_grad()
# teacher/student outputs: logits, attention1, attention2, attention3
# compute model output, fetch teacher/student output, and compute KD loss
student_outputs = self.student_model(psuedo_datapoint)
# student_loss = KL_Loss(teacher_logits, teacher_outputs)
student_loss = utils.student_loss_zero_shot(student_outputs, teacher_outputs)
student_loss.backward()
# performs updates using calculated gradients
torch.nn.utils.clip_grad_norm_(self.student_model.parameters(), 5)
self.student_optimizer.step()
if (batch + 1) % self.log_num == 0 or (batch + 1) == self.num_epochs:
acc = self.test()
print(f"\nAccuracy: {acc:05.3f}")
print(f'Student Loss: {student_loss:05.3f}')
utils.writeMetrics({"accuracy": acc}, self.acc_counter)
accuracy_dict[batch] = acc
utils.log_accuracy("zero_shot.csv", accuracy_dict)
self.acc_counter += 1
self.save_model()
if acc > best_acc:
best_acc = acc
utils.writeMetrics({"Student Loss": student_loss, "Generator Loss": generator_loss}, self.counter)
self.counter += 1
self.cosine_annealing_generator.step()
self.cosine_annealing_student.step()
def px(pae_dict):
# Initialize model save string
pxmodelstr = pxfinestr.split('.meta')[0]
# Variable initilization and saving
init = tf.global_variables_initializer()
saver = tf.train.Saver()
if cp.get('Experiment', 'PREFIX') == 'MNIST':
# Tensorboard (comment / uncomment)
######################################################################
from datetime import datetime
now = datetime.utcnow().strftime("%m-%d_%H-%M:%S")
root_logdir = cp.get('Experiment', 'ModelOutputPath')
logdir = "{}/{}{}-{}/".format(
root_logdir,
cp.get('Experiment', 'PREFIX') + '_' +
cp.get('Experiment', 'Enumber') + '_px', sys.argv[2], now)
tf.summary.scalar(name='xrecon loss', tensor=pae_dict['px_mse'])
summary = tf.summary.merge_all()
file_writer = tf.summary.FileWriter(logdir, tf.get_default_graph())
######################################################################
# Start Session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
# Initialize graph variables
init.run()
if cp.get('Experiment', 'PREFIX') == 'MNIST':
train_dict = {
'cp': cp,
'sess': sess,
'data': XX_full,
'sumr': summary,
'savestr': pxmodelstr,
'saver': saver,
'fw': file_writer
}
else:
train_dict = {
'cp': cp,
'sess': sess,
'data': XX_full,
'savestr': pxmodelstr,
'saver': saver,
'ae_ids': ae_ids,
'out_': out_
}
if cp.get('Experiment', 'PREFIX') == 'MNIST':
ConvAE.train(train_dict, pae_dict)
else:
SAE.train(train_dict, pae_dict)
# Get batch size for batch output save
batch_size = cp.getint('Hyperparameters', 'BatchSize')
if cp.get('Experiment', 'PREFIX') == 'MNIST':
# Save hidden/output layer results for pipeline training
px_Z_latent = utils.run_OOM(sess,
pae_dict['conv_in'],
XX_full,
pae_dict['conv_z'],
batch_size=batch_size)
else:
# Save hidden/output layer results for pipeline training
px_Z_latent = utils.run_OOM(sess,
pae_dict['sda_in'],
XX_full,
pae_dict['sda_hidden'],
batch_size=batch_size)
# utils.save_OOM(sess, pae_dict['conv_in'], XX_full,
# pae_dict['conv_out'],
# path=cp.get('Experiment', 'PX_XREC_TRAIN'),
# batch_size=batch_size)
# Print clustering ACC
utils.log_accuracy(cp, YY_full, px_Z_latent, 'PX - ACC FULL', SEED)
# Print clustering NMI
utils.log_NMI(cp, YY_full, px_Z_latent, 'PX - NMI FULL', SEED)
sess.close()
def evitram():
# Restore pretrained model
restorestr = pxfinestr.split('.meta')[0]
# Save model str
evitramstr = evitramfinestr.split('.meta')[0]
# Load pretrained evidence representations for all sources
K = []
for e in sys.argv[3:]:
cp2 = utils.load_config(e)
K.append(cp2.get('Experiment', 'PX_Z_TRAIN'))
sect = 'Experiment'
ev_paths = [cp.get(sect, i) for i in cp.options(sect) if 'evidence' in i]
if cp.get('Experiment', 'PREFIX') == 'MNIST':
evitram_dict = ConvAE.build_EviTRAM(cp, SEED)
else:
# Layerwise autoencoder number
ae_ids = [str(i) for i in xrange(cp.getint('Experiment', 'AENUM'))]
evitram_dict = SAE.build_EviTRAM(cp, ae_ids, SEED)
# Get variables to restore from pretrained model P(x) Encoder
var_list = tf.trainable_variables()
for ev_path_id, ev_path in enumerate(ev_paths):
if cp.get('Experiment', 'PREFIX') == 'MNIST':
# Prepare "restore" variable list
for v in var_list:
if v.name == 'Pre_Q' + str(ev_path_id) + '/kernel:0':
var_list.remove(v)
for v in var_list:
if v.name == 'Pre_Q' + str(ev_path_id) + '/bias:0':
var_list.remove(v)
else:
# Prepare "restore" variable list
for v in var_list:
if v.name == 'Pre_Q' + str(ev_path_id) + '/kernel:0':
var_list.remove(v)
for v in var_list:
if v.name == 'Pre_Q' + str(ev_path_id) + '/bias:0':
var_list.remove(v)
for v in var_list:
if v.name == 'Pre_Comp_Q' + str(ev_path_id) + '/kernel:0':
var_list.remove(v)
for v in var_list:
if v.name == 'Pre_Comp_Q' + str(ev_path_id) + '/bias:0':
var_list.remove(v)
##########################################################
# Tensorboard (comment / uncomment)
##########################################################
from datetime import datetime
now = datetime.utcnow().strftime("%m-%d_%H-%M:%S")
root_logdir = cp.get('Experiment', 'ModelOutputPath')
logdir = "{}/{}{}-{}/".format(
root_logdir,
cp.get('Experiment', 'PREFIX') + '_' +
cp.get('Experiment', 'Enumber') + '_cond', sys.argv[2], now)
tf.summary.scalar(name='cond loss', tensor=evitram_dict['evitram_loss'])
tf.summary.scalar(name='recon loss', tensor=evitram_dict['px_mse'])
summary = tf.summary.merge_all()
file_writer = tf.summary.FileWriter(logdir, tf.get_default_graph())
##########################################################
# Initialize & restore P(x) AE weights
init = tf.global_variables_initializer()
saver = tf.train.Saver(var_list)
saverCOND = tf.train.Saver()
# Task outcomes
EV = [np.load(i) for i in K]
# Start Session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
# Init values
init.run()
# Restore finetuned model
saver.restore(sess, restorestr)
train_dict = {
'cp': cp,
'sess': sess,
'data': XX_full,
'sumr': summary,
'savestr': evitramstr,
'saver': saverCOND,
'fw': file_writer,
'EV': EV,
'ev_paths': ev_paths
}
if cp.get('Experiment', 'PREFIX') == 'MNIST':
ConvAE.evitram_train(train_dict, evitram_dict)
else:
SAE.evitram_train(train_dict, evitram_dict)
# Get batch size for batch output save
batch_size = train_dict['cp'].getint('Hyperparameters', 'BatchSize')
if cp.get('Experiment', 'PREFIX') == 'MNIST':
# Save hidden/output layer results for pipeline training
px_Z_latent = utils.run_OOM(sess,
evitram_dict['conv_in'],
XX_full,
evitram_dict['conv_z'],
batch_size=batch_size)
else:
px_Z_latent = utils.run_OOM(sess,
evitram_dict['sda_in'],
XX_full,
evitram_dict['sda_hidden'],
batch_size=batch_size)
# utils.save_OOM(sess, pae_dict['conv_in'], XX_full,
# pae_dict['conv_out'],
# path=cp.get('Experiment', 'PX_XREC_TRAIN'),
# batch_size=batch_size)
# Print clustering ACC
utils.log_accuracy(cp, YY_full, px_Z_latent, 'COND - ACC FULL', SEED)
# Print clustering NMI
utils.log_NMI(cp, YY_full, px_Z_latent, 'COND - NMI FULL', SEED)
sess.close()
def main_func(device, out_dir_results, random_seed, in_dir_data, batch_size,
workers, num_epochs, num_classes, size_image):
# create an output folder
os.makedirs(OUT_DIR_RESULTS, exist_ok=True)
# fill padded area with ImageNet's mean pixel value converted to range [0, 255]
max_padding = tv.transforms.Lambda(utils.pad_function)
# transform images
transforms_train = tv.transforms.Compose([
max_padding,
tv.transforms.RandomOrder([
tv.transforms.RandomCrop((size_image, size_image)),
tv.transforms.RandomHorizontalFlip()
]),
tv.transforms.ToTensor(),
tv.transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
transforms_eval = tv.transforms.Compose([
max_padding,
tv.transforms.CenterCrop((size_image, size_image)),
tv.transforms.ToTensor(),
tv.transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
# instantiate dataset objects according to the pre-defined splits
ds_train = utils.DatasetBirds(in_dir_data,
transform=transforms_train,
train=True)
ds_val = utils.DatasetBirds(in_dir_data,
transform=transforms_eval,
train=True)
ds_test = utils.DatasetBirds(in_dir_data,
transform=transforms_eval,
train=False)
splits = skms.StratifiedShuffleSplit(n_splits=1,
test_size=0.1,
random_state=random_seed)
idx_train, idx_val = next(
splits.split(np.zeros(len(ds_train)), ds_train.targets))
# set hyper-parameters
params = {'batch_size': batch_size, 'num_workers': workers}
# instantiate data loaders
train_loader = td.DataLoader(dataset=ds_train,
sampler=td.SubsetRandomSampler(idx_train),
**params)
val_loader = td.DataLoader(dataset=ds_val,
sampler=td.SubsetRandomSampler(idx_val),
**params)
test_loader = td.DataLoader(dataset=ds_test, **params)
model_desc = utils.get_model_desc(num_classes=num_classes, pretrained=True)
# instantiate the model
model = tv.models.resnet50(pretrained=True)
model.fc = nn.Linear(model.fc.in_features, num_classes)
model.to(DEVICE)
# instantiate optimizer and scheduler
optimizer = torch.optim.Adam(model.parameters(), lr=1e-4)
scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.95)
# train and validate the model
best_snapshot_path = None
val_acc_avg = list()
best_val_acc = -1.0
for epoch in range(num_epochs):
# train the model
model.train()
train_loss = list()
for batch in train_loader:
x, y = batch
x = x.to(device)
y = y.to(device)
optimizer.zero_grad()
# calculate the loss
y_pred = model(x)
# calculate the loss
loss = F.cross_entropy(y_pred, y)
# backprop & update weights
loss.backward()
optimizer.step()
train_loss.append(loss.item())
# validate the model
model.eval()
val_loss = list()
val_acc = list()
with torch.no_grad():
for batch in val_loader:
x, y = batch
x = x.to(device)
y = y.to(device)
# predict bird species
y_pred = model(x)
# calculate the loss
loss = F.cross_entropy(y_pred, y)
# calculate the accuracy
acc = skm.accuracy_score(
[val.item() for val in y],
[val.item() for val in y_pred.argmax(dim=-1)])
val_loss.append(loss.item())
val_acc.append(acc)
val_acc_avg.append(np.mean(val_acc))
# save the best model snapshot
current_val_acc = val_acc_avg[-1]
if current_val_acc > best_val_acc:
if best_snapshot_path is not None:
os.remove(best_snapshot_path)
best_val_acc = current_val_acc
best_snapshot_path = os.path.join(
out_dir_results,
f'model_{model_desc}_ep={epoch}_acc={best_val_acc}.pt')
torch.save(model.state_dict(), best_snapshot_path)
# adjust the learning rate
scheduler.step()
# print performance metrics
print('Epoch {} |> Train. loss: {:.4f} | Val. loss: {:.4f}'.format(
epoch + 1, np.mean(train_loss), np.mean(val_loss)))
# use the best model snapshot
model.load_state_dict(torch.load(best_snapshot_path, map_location=device))
# test the model
true = list()
pred = list()
with torch.no_grad():
for batch in test_loader:
x, y = batch
x = x.to(device)
y = y.to(device)
y_pred = model(x)
true.extend([val.item() for val in y])
pred.extend([val.item() for val in y_pred.argmax(dim=-1)])
# calculate the accuracy
test_accuracy = skm.accuracy_score(true, pred)
# save the accuracy
path_to_logs = f'{out_dir_results}/logs.csv'
utils.log_accuracy(path_to_logs, model_desc, test_accuracy)
print('Test accuracy: {:.3f}'.format(test_accuracy))
def evitram(evitramd):
saver = tf.train.Saver()
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
# Restore pretrained model
saver.restore(sess, evitramfinestr.split('.meta')[0])
if cp.get('Experiment', 'PREFIX') == 'MNIST' or \
cp.get('Experiment', 'PREFIX') == 'AMNIST':
# Save hidden/output layer results for pipeline training
px_Z_latent = utils.run_OOM(sess, evitram_dict['conv_in'], XX_full,
evitram_dict['conv_z'],
batch_size=batch_size)
elif cp.get('Experiment', 'PREFIX') == 'WEATHER':
# Save hidden/output layer results for pipeline training
px_Z_latent_tr = utils.run_OOM(sess, evitram_dict['conv_in'], XX_full,
evitram_dict['conv_z'],
batch_size=batch_size)
px_Z_latent_te = utils.run_OOM(sess, evitram_dict['conv_in'], XX_test,
evitram_dict['conv_z'],
batch_size=batch_size)
else:
px_Z_latent_tr = utils.run_OOM(sess, evitram_dict['sda_in'], XX_full,
evitram_dict['sda_hidden'],
batch_size=batch_size)
if not(np.array_equal(XX_test, np.zeros(shape=(1,1)))):
px_Z_latent_te = utils.run_OOM(sess, evitram_dict['sda_in'], XX_test,
evitram_dict['sda_hidden'],
batch_size=batch_size)
if 'WEATHER' in cp.get('Experiment', 'PREFIX'):
# Print clustering ACC
utils.log_accuracy_isof(cp, YY_full, px_Z_latent_tr,
'COND - ACC FULL (Train)', SEED)
if not(np.array_equal(XX_test, np.zeros(shape=(1,1)))):
utils.log_accuracy_isof(cp, YY_test, px_Z_latent_te,
'COND - ACC FULL (Test)', SEED)
utils.log_anomalyPRF_isof(cp, YY_full, px_Z_latent_tr,
'COND - PRF FULL (Test)', SEED)
if not(np.array_equal(XX_test, np.zeros(shape=(1,1)))):
utils.log_anomalyPRF_isof(cp, YY_test, px_Z_latent_te,
'COND - PRF FULL (Test)', SEED)
else:
# Print clustering ACC
utils.log_accuracy(cp, YY_full, px_Z_latent,
'PX - ACC FULL', SEED)
# Print clustering NMI
utils.log_NMI(cp, YY_full, px_Z_latent,
'PX - NMI FULL', SEED)
# Print clustering CHS score
utils.log_CHS(cp, XX_full, px_Z_latent,
'PX - CHS FULL', SEED)
sess.close()
def px(pae_dict):
saver = tf.train.Saver()
# P(x) Session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session(config=config) as sess:
# Restore pretrained model
saver.restore(sess, pxfinestr.split('.meta')[0])
if cp.get('Experiment', 'PREFIX') == 'MNIST' or \
cp.get('Experiment', 'PREFIX') == 'AMNIST':
# Save hidden/output layer results for pipeline training
px_Z_latent = utils.run_OOM(sess,
pae_dict['conv_in'],
XX_full,
pae_dict['conv_z'],
batch_size=batch_size)
else:
px_Z_latent = utils.run_OOM(sess,
pae_dict['sda_in'],
XX_full,
pae_dict['sda_hidden'],
batch_size=batch_size)
if not (np.array_equal(XX_test, np.zeros(shape=(1, 1)))):
px_Z_latent_te = utils.run_OOM(sess,
pae_dict['sda_in'],
XX_test,
pae_dict['sda_hidden'],
batch_size=batch_size)
if cp.get('Experiment', 'Enumber') == 'windT':
# Print clustering ACC
utils.log_accuracy_AC(cp, YY_full, px_Z_latent, 'PX - ACC FULL',
SEED)
# utils.log_accuracy(cp, YY_test, px_Z_latent_te,
# 'PX - ACC TEST', SEED)
# Print clustering NMI
utils.log_NMI_AC(cp, YY_full, px_Z_latent, 'PX - NMI FULL', SEED)
# utils.log_NMI(cp, YY_test, px_Z_latent_te,
# 'PX - NMI TEST', SEED)
# Print clustering CHS score
utils.log_anomalyPRF_AC(cp, YY_full, px_Z_latent,
'PX - PRF FULL (FULL)', SEED)
# utils.log_anomalyPRF(cp, YY_test, px_Z_latent_te,
# 'PX - PRF FULL (Test)', SEED)
else:
# Print clustering ACC
utils.log_accuracy(cp, YY_full, px_Z_latent, 'PX - ACC FULL', SEED)
if not (np.array_equal(XX_test, np.zeros(shape=(1, 1)))):
utils.log_accuracy(cp, YY_test, px_Z_latent_te,
'PX - ACC TEST', SEED)
# Print clustering NMI
utils.log_NMI(cp, YY_full, px_Z_latent, 'PX - NMI FULL', SEED)
if not (np.array_equal(XX_test, np.zeros(shape=(1, 1)))):
utils.log_NMI(cp, YY_test, px_Z_latent_te, 'PX - NMI TEST',
SEED)
# Print clustering CHS score
utils.log_anomalyPRF(cp, YY_full, px_Z_latent,
'PX - PRF FULL (FULL)', SEED)
if not (np.array_equal(XX_test, np.zeros(shape=(1, 1)))):
utils.log_anomalyPRF(cp, YY_test, px_Z_latent_te,
'PX - PRF FULL (Test)', SEED)
sess.close()
