def main():
global args
net = UNet(3, 1)
net.load(opt.ckpt_path)
loss = Loss('soft_dice_loss')
torch.cuda.set_device(0)
net = net.cuda()
loss = loss.cuda()
if args.phase == 'train':
# train
dataset = NucleiDetector(opt, phase=args.phase)
train_loader = DataLoader(dataset,
batch_size=opt.batch_size,
shuffle=True,
num_workers=opt.num_workers,
pin_memory=opt.pin_memory)
lr = opt.lr
optimizer = torch.optim.Adam(net.parameters(),
lr=lr,
weight_decay=opt.weight_decay)
previous_loss = None # haven't run
for epoch in range(opt.epoch + 1):
now_loss = train(train_loader, net, loss, epoch, optimizer,
opt.model_save_freq, opt.model_save_path)
if previous_loss is not None and now_loss > previous_loss:
lr *= opt.lr_decay
for param_group in optimizer.param_groups:
param_group['lr'] = lr
save_lr(net.model_name, opt.lr_save_path, lr)
previous_loss = now_loss
elif args.phase == 'val':
# val phase
dataset = NucleiDetector(opt, phase='val')
val_loader = DataLoader(dataset,
batch_size=opt.batch_size,
shuffle=True,
num_workers=opt.num_workers,
pin_memory=opt.pin_memory)
val(val_loader, net, loss)
else:
# test phase
dataset = NucleiDetector(opt, phase='test')
test_loader = DataLoader(dataset,
batch_size=1,
shuffle=True,
num_workers=opt.num_workers,
pin_memory=opt.pin_memory)
test(test_loader, net, opt)
def test(dataset_name,
model_name,
metric_name,
path_history="checkpoints/",
path_best_model=""):
history = ut.load_json(path_history)
transformer = ut.ComposeJoint([[transforms.ToTensor(), None],
[transforms.Normalize(*ut.mean_std), None],
[None, ut.ToLong()]])
test_set = dataset_dict[dataset_name](split="test",
transform_function=transformer)
model = model_dict[model_name](n_classes=test_set.n_classes).cuda()
# path_best_model = "/mnt/home/issam/LCFCNSaves/pascal/State_Dicts/best_model.pth"
model.load_state_dict(torch.load(path_best_model))
model.trained_images = set(history["trained_images"])
testDict = ut.val(model=model,
dataset=test_set,
epoch=history["best_val_epoch"],
metric_name=metric_name)
print(pd.DataFrame([testDict]))
def train(dataset_name, model_name, metric_name, path_history, path_model, path_opt, path_best_model, reset=False):
# SET SEED
np.random.seed(1)
torch.manual_seed(1)
torch.cuda.manual_seed_all(1)
# Train datasets
transformer = ut.ComposeJoint(
[ut.RandomHorizontalFlipJoint(),
[transforms.ToTensor(), None],
[transforms.Normalize(*ut.mean_std), None],
[None, ut.ToLong() ]
])
train_set = dataset_dict[dataset_name](split="train",
transform_function=transformer)
trainloader = torch.utils.data.DataLoader(train_set, batch_size=1,
num_workers=0,
drop_last=False,
sampler=ut.RandomSampler(train_set))
# Val datasets
transformer = ut.ComposeJoint(
[
[transforms.ToTensor(), None],
[transforms.Normalize(*ut.mean_std), None],
[None, ut.ToLong() ]
])
val_set = dataset_dict[dataset_name](split="val",
transform_function=transformer)
test_set = dataset_dict[dataset_name](split="test",
transform_function=transformer)
# Model
model = model_dict[model_name](train_set.n_classes).cuda()
opt = torch.optim.Adam(filter(lambda p: p.requires_grad, model.parameters()),
lr=1e-5, weight_decay=0.0005)
# Train
if os.path.exists(path_history) and not reset:
history = ut.load_json(path_history)
model.load_state_dict(torch.load(path_model))
opt.load_state_dict(torch.load(path_opt))
s_epoch = history["train"][-1]["epoch"]
print("Resuming epoch...{}".format(s_epoch))
else:
history = {"train":[], "val":[], "test":[],
"model_name":model_name,
"dataset_name":dataset_name,
"path_model":path_model,
"path_opt":path_opt,
"path_best_model":path_best_model,
"best_val_epoch":-1, "best_val_mae":np.inf}
s_epoch = 0
print("Starting from scratch...")
for epoch in range(s_epoch + 1, 1000):
train_dict = ut.fit(model, trainloader, opt,
loss_function=losses.lc_loss,
epoch=epoch)
# Update history
history["trained_images"] = list(model.trained_images)
history["train"] += [train_dict]
# Save model, opt and history
torch.save(model.state_dict(), path_model)
torch.save(opt.state_dict(), path_opt)
ut.save_json(path_history, history)
# %%%%%%%%%%% 2. VALIDATION PHASE %%%%%%%%%%%%"
with torch.no_grad():
val_dict = ut.val(model=model, dataset=val_set, epoch=epoch,
metric_name=metric_name)
# Update history
history["val"] += [val_dict]
# Lower is better
if val_dict[metric_name] <= history["best_val_mae"]:
history["best_val_epoch"] = epoch
history["best_val_mae"] = val_dict[metric_name]
torch.save(model.state_dict(), path_best_model)
# Test Model
if not (dataset_name == "penguins" and epoch < 50):
testDict = ut.val(model=model,
dataset=test_set,
epoch=epoch, metric_name=metric_name)
history["test"] += [testDict]
ut.save_json(path_history, history)
nhead=args.nhead,
nlayer=args.nlayer,
norm_mode=args.norm_mode,
norm_scale=args.norm_scale,
residual=args.residual)
net = net.cuda()
optimizer = torch.optim.Adam(net.parameters(), args.lr, weight_decay=args.wd)
criterion = torch.nn.CrossEntropyLoss()
logging.info(net)
# train
best_acc = 0
best_loss = 1e10
for epoch in range(args.epochs):
train_loss, train_acc = train(net, optimizer, criterion, data)
val_loss, val_acc = val(net, criterion, data)
logging.debug('Epoch %d: train loss %.3f train acc: %.3f, val loss: %.3f val acc %.3f.'%
(epoch, train_loss, train_acc, val_loss, val_acc))
# save model
if best_acc < val_acc:
best_acc = val_acc
torch.save(net.state_dict(), OUT_PATH+'checkpoint-best-acc.pkl')
if best_loss > val_loss:
best_loss = val_loss
torch.save(net.state_dict(), OUT_PATH+'checkpoint-best-loss.pkl')
# pick up the best model based on val_acc, then do test
net.load_state_dict(torch.load(OUT_PATH+'checkpoint-best-acc.pkl'))
val_loss, val_acc = val(net, criterion, data)
test_loss, test_acc = test(net, criterion, data)
checkpoint_folder = experiment_folder + '/checkpoint/'
if not os.path.exists(experiment_folder):
os.makedirs(experiment_folder)
if not os.path.exists(checkpoint_folder):
os.makedirs(checkpoint_folder)
for epoch in range(100):
# updating model weights
train_loss, train_time = train(model, optimizer, criterion,
train_loader)
# validation
val_loss, val_time, f1 = val(model, optimizer, criterion, val_loader)
# logging message
log = "Epoch : {} |".format(epoch + 1), \
"LR : {} |".format(get_learning_rate(optimizer)), \
"Train Loss : {:.3f} |".format(train_loss), \
"Val Loss : {:.3f} |".format(val_loss), \
"Val F1 : {:.3f} |".format(f1), \
"Time Train: {:.3f} |".format(train_time), \
"Time Val : {:.3f} |".format(val_time),
print(log)
if not params['debug']:
# save model state
torch.save(model.module.state_dict(),
checkpoint = torch.load('./checkpoint/{}.t7'.format(args.model_name))
net.load_state_dict(checkpoint['net'])
best_F2 = checkpoint['F2']
train_loader = Data.DataLoader(loader('datafile/train.txt'),
batch_size=args.batch_size,
shuffle=True,
num_workers=8,
drop_last=True)
test_loader = Data.DataLoader(loader('datafile/val.txt', test=True),
batch_size=args.batch_size,
num_workers=8)
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(),
lr=args.lr,
momentum=0.9,
weight_decay=5e-4)
for epoch in range(args.num_epoch):
print('Epoch %d' % (epoch))
train(train_loader, net, criterion, optimizer)
acc = val(test_loader, net, criterion)
if acc > best_acc:
print('Saving..')
state = {'net': net.state_dict(), 'acc': acc}
if not os.path.isdir('checkpoint'):
os.mkdir('checkpoint')
torch.save(state, './checkpoint/{}.t7'.format(args.model_name))
best_acc = acc
return student
teacher = getTeacherModel()
student = getStudentModel()
teacher = teacher.cuda()
student = student.cuda()
# Baseline
max_acc_baseline = 0.0
for i in range(epochs):
print("Training epoch {}".format(i))
train(student, teacher, T, train_load, 1.5, 0, 0, lr)
print("Validating epoch {}".format(i))
micro_auprc = val(student, val_load, i)
print("Micro AUPRC: {}".format(micro_auprc))
if (micro_auprc > max_acc_baseline):
torch.save(student.state_dict(), 'weights/dcase_small_baseline')
max_acc_baseline = micro_auprc
student = getStudentModel()
student = student.cuda()
# KD
max_acc_kd = 0.0
for i in range(epochs):
print("Training epoch {}".format(i))
train(student, teacher, T, train_load, 1.5, 1.5, 0, lr)
# for plotting
mvn = torch.distributions.Normal(0, 1)
z_norm = mvn.sample([num_samples, np.ceil(n_in).astype(int)])
val_batch = next(iter(val_loader)).float()
start = time.time()
# for early stopping
i = 0
max_loss = np.inf
epochs_list = []
train_losses = []
val_losses = []
for epoch in range(1, epochs):
epochs_list.append(epoch)
train_loss = train_one_epoch(model, epoch, optimizer, train_loader)
val_loss = val(model, train, val_loader)
train_losses.append(train_loss)
# val_losses.append(val_loss)
val_loss = 100
if val_loss < max_loss:
max_loss = val_loss
i = 0
torch.save(
model,
(path + "model.pt"),
)
else:
i += 1
if i >= 30:
break
print("Patience counter: {}/30".format(i))
import torch
from torch.utils.data import DataLoader
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
import os
from rnvp import RealNVP
from new_model import NewFlow
from utils import train_one_epoch, val, test
from data.hepmass import train
from data.hepmass import val_loader as test_loader
from gif import make_gif
import time
model = torch.load("/Users/edvardhulten/real_nvp_2d/model.pt")
v = val(model, train, test_loader)
test_loss = test(model, train, test_loader)
entropy[rank[args.k:]].mean().item(), i)
i = conf.size(0)
record['s_all_confidence'].update(conf.mean().item(), i)
record['s_all_margin'].update(margin.mean().item(), i)
record['s_all_entropy'].update(entropy.mean().item(), i)
i = t_conf.size(0)
record['t_confidence'].update(t_conf.mean().item(), i)
record['t_margin'].update(t_margin.mean().item(), i)
record['t_entropy'].update(t_entropy.mean().item(), i)
for item in items:
logger.add_scalar('train/{}'.format(item), record[item].avg, epoch + 1)
# val
acc = val(val_loader, args, t_model, s_model, logger, epoch)
if acc > best_acc:
best_acc = acc
state_dict = dict(state_dict=s_model.state_dict(), best_acc=best_acc)
name = osp.join(exp_path, 'ckpt/student_best.pth')
os.makedirs(osp.dirname(name), exist_ok=True)
torch.save(state_dict, name)
scheduler.step()
if args.seed == 0:
counter = counter.cpu().numpy()
np.save(osp.join(exp_path, 'counter.npy'), counter)
def train(args, nci_id):
# fixed seed
torch.manual_seed(0)
# np.random.seed(0)
# cuda test
if torch.cuda.is_available():
is_cuda = True
torch.cuda.manual_seed_all(0)
# is_cuda = False
# load dataset
train_set, test_set = loadNCI.load_nci_data(nci_id, cuda=is_cuda)
# Use Pytorch's DataLoader and th collate function
train_data_loader = DataLoader(train_set,
batch_size=args.batch_size,
shuffle=True,
collate_fn=loadNCI.collate)
test_data_loader = DataLoader(test_set,
batch_size=args.batch_size,
shuffle=True,
collate_fn=loadNCI.collate)
input_dim = train_set[0][0].ndata['feature'].size(1)
output_dim = 2
# create model
model = GINClassifier(args.num_layers, args.num_mlp_layers, input_dim,
args.hidden_dim, output_dim, args.feat_drop,
args.learn_eps, args.graph_pooling_type,
args.neigh_pooling_type, args.final_drop, is_cuda)
if is_cuda:
model.cuda()
loss_func = nn.CrossEntropyLoss() # define loss function
optimizer = optim.Adam(model.parameters(), lr=args.lr)
epoch_losses = []
for epoch in range(args.num_epochs):
model.train()
epoch_loss = 0
train_acc = 0
for iter, (bg, label) in enumerate(
train_data_loader): # bg means batch of graph
prediction = model(bg)
if is_cuda:
label = label.cuda()
loss = loss_func(prediction, label)
optimizer.zero_grad()
# loss.backward(retain_graph=True)
loss.backward()
optimizer.step()
epoch_loss += loss.detach().item()
train_acc += utils.accuracy(prediction, label)
epoch_loss /= (iter + 1)
train_acc /= (iter + 1)
test_acc = utils.val(model, test_data_loader, is_cuda)
print('Task {}: Epoch{}, loss {:.4f}, TrainACC {:.4f}, TestACC {:.4f}'.
format(nci_id, epoch, epoch_loss, train_acc, test_acc))
# print('Task {}: Epoch{}, loss {:.4f}, TrainACC {:.4f}'.format(nci_id, epoch, epoch_loss, train_acc))
epoch_losses.append(epoch_loss)
torch.save(model.state_dict(),
MODEL_PATH + 'Model-NCI-' + str(nci_id) + '.kpl')
test_acc = utils.val(model, test_data_loader, is_cuda)
train_acc = utils.val(model, train_data_loader, is_cuda)
return train_acc, test_acc
