def __init__(self, args, file=None):
if args.verbose:
print("Setting up eval object")
#Initialising some variables
self.device = args.device
self.result_root = args.result_root
self.visualise = args.visualise
self.verbose = args.verbose
if file == None:
file = 'model.pth'
#Load model from file
setup_model(self, file, reload_model=True)
#Setup dataset and dataloaders
test_dataset = Data(args.dataset_path['test'], self.TX_config,
self.TX_input, self.blockage, self.output_nf)
heatmap_dataset = Data(args.dataset_path['heatmap'], self.TX_config,
self.TX_input, self.blockage, self.output_nf)
self.test_data_loader = DataLoader(test_dataset,
batch_size=self.batch_size,
shuffle=True,
num_workers=4)
self.heatmap_loader = DataLoader(heatmap_dataset,
batch_size=self.batch_size,
shuffle=True,
num_workers=4)
#Setting the model to evaluation mode
self.model.eval()
self.result_root = args.result_root
def set_dataset(self):
#Re initialise datasets with different data configurations
self.train_dataset = Data(self.dataset_path['train'], self.TX_config,
self.TX_input, self.blockage, self.output_nf)
self.val_dataset = Data(self.dataset_path['val'], self.TX_config,
self.TX_input, self.blockage, self.output_nf)
#If configuration is changed model size has to be adapted
self.size = config2size(self.TX_config)
def evaluate():
net.eval()
test_loss = 0
targets, outputs = [], []
with torch.no_grad():
for batch_id, (data, target) in enumerate(test_loader):
data, target = data.cuda(), target.cuda()
output = net(data)
batch_loss = criterion(output, target)
targets += [target]
outputs += [output]
test_loss += batch_loss
test_loss /= (batch_id + 1)
test_acc = compute_accuracy(torch.cat(targets), torch.cat(outputs))
targets_temp = torch.cat(targets).cpu().numpy()
outputs_temp = np.argmax(torch.cat(outputs).cpu().numpy(), axis=1)
log_string('Glomerulus Level Classification Confusion Matrix and Accuracy: ')
log_string(str(confusion_matrix(targets_temp, outputs_temp)))
# display
log_string("validation_loss: {0:.4f}, validation_accuracy: {1:.02%}"
.format(test_loss, test_acc))
return outputs, targets, test_loss
def __init__(self, args, id=None, worker_id=0):
#Set the seed for reproducability
if args.verbose:
printMultiLine(worker_id, "Setting up neural network")
#Initialise plenty of variables XS
self.worker_id = worker_id
self.verbose = args.verbose
self.result_root = args.result_root
self.device = args.device
self.batch_size = args.batch_size
self.learning = True
self.visualise = args.visualise
#Initialise model parameters
self.size = args.size
self.model_type = args.model_type
self.nf = args.nf
self.hidden_layers = args.hidden_layers
self.criterion = torch.nn.MSELoss()
self.output_nf = args.output_nf
self.learning_rate = args.learning_rate
#Initialise datasets
self.dataset_path = args.dataset_path
self.TX_config = args.TX_config
self.TX_input = args.TX_input
self.blockage = args.blockage
self.output_nf = args.output_nf
self.train_dataset = Data(self.dataset_path['train'], self.TX_config,
self.TX_input, self.blockage, self.output_nf)
self.val_dataset = Data(self.dataset_path['val'], self.TX_config,
self.TX_input, self.blockage, self.output_nf)
#If normal model is trained (No experiment or PBT training)
if not args.pbt_training and args.experiment == None:
#Setup dataloaders
self.data_loader = DataLoader(self.train_dataset,
batch_size=self.batch_size,
shuffle=True,
num_workers=4)
self.val_data_loader = DataLoader(self.val_dataset,
batch_size=self.batch_size,
shuffle=True,
num_workers=4)
self.step = int(len(self.data_loader) / 10)
#Load checkpoint
setup_model(self, 'checkpoint.pth')
def evaluate(**kwargs):
best_loss = kwargs['best_loss']
best_acc = kwargs['best_acc']
global_step = kwargs['global_step']
net.eval()
test_loss = 0
targets, outputs = [], []
with torch.no_grad():
for batch_id, (data, target) in enumerate(test_loader):
data, target = data.cuda(), target.cuda()
output = net(data)
batch_loss = criterion(output, target)
targets += [target]
outputs += [output]
test_loss += batch_loss
test_loss /= (batch_id + 1)
test_acc = compute_accuracy(torch.cat(targets), torch.cat(outputs))
# check for improvement
loss_str, acc_str = '', ''
if test_loss <= best_loss:
loss_str, best_loss = '(improved)', test_loss
if test_acc >= best_acc:
acc_str, best_acc = '(improved)', test_acc
# display
log_string(
"validation_loss: {0:.4f} {1}, validation_accuracy: {2:.02%}{3}".
format(test_loss, loss_str, test_acc, acc_str))
# write to TensorBoard
info = {'loss': test_loss, 'accuracy': test_acc}
for tag, value in info.items():
test_logger.scalar_summary(tag, value, global_step)
# save checkpoint model
state_dict = net.state_dict()
for key in state_dict.keys():
state_dict[key] = state_dict[key].cpu()
save_path = os.path.join(model_dir, '{}.ckpt'.format(global_step))
torch.save(
{
'global_step': global_step,
'loss': test_loss,
'acc': test_acc,
'save_dir': model_dir,
'state_dict': state_dict
}, save_path)
log_string('Model saved at: {}'.format(save_path))
log_string('--' * 30)
return best_loss, best_acc
def train():
global_step = 0
best_loss = 100
best_acc = 0
for epoch in range(options.epochs):
log_string('**' * 30)
log_string('Training Epoch %03d, Learning Rate %g' %
(epoch + 1, optimizer.param_groups[0]['lr']))
net.train()
train_loss = 0
targets, outputs = [], []
for batch_id, (data, target) in enumerate(train_loader):
global_step += 1
data = data.cuda()
target = target.cuda()
# Forward pass
output = net(data)
batch_loss = criterion(output, target)
targets += [target]
outputs += [output]
train_loss += batch_loss.item()
# Backward and optimize
optimizer.zero_grad()
batch_loss.backward()
optimizer.step()
if (batch_id + 1) % options.disp_freq == 0:
train_loss /= options.disp_freq
train_acc = compute_accuracy(torch.cat(targets),
torch.cat(outputs))
log_string(
"epoch: {0}, step: {1}, train_loss: {2:.4f} train_accuracy: {3:.02%}"
.format(epoch + 1, batch_id + 1, train_loss, train_acc))
info = {'loss': train_loss, 'accuracy': train_acc}
for tag, value in info.items():
train_logger.scalar_summary(tag, value, global_step)
train_loss = 0
targets, outputs = [], []
if (batch_id + 1) % options.val_freq == 0:
log_string('--' * 30)
log_string('Evaluating at step #{}'.format(global_step))
best_loss, best_acc = evaluate(best_loss=best_loss,
best_acc=best_acc,
global_step=global_step)
net.train()
def process_data(args, split):
dataset = Data(args.dataset_path[split], args.TX_config, args.TX_input,
args.blockage, args.output_nf)
data = dataset.get_data()
input_list = []
output_list = []
print("Creating {} data".format(split))
for sample in data:
input = sample[0]
output_list.append(sample[1])
#Get indices of blockage
indices = np.random.choice(np.arange(len(input)),
replace=False,
size=int(args.blockage * len(input)))
for ind in indices:
input[ind] = -1
input_list.append(input)
return np.array(input_list), np.array(output_list)
def evaluate():
net.eval()
test_loss = 0
targets, outputs = [], []
with torch.no_grad():
for batch_id, (data, target) in enumerate(test_loader):
data, target = data.cuda(), target.cuda()
output = net(data)
batch_loss = criterion(output, target)
targets += [target]
outputs += [output]
test_loss += batch_loss
test_loss /= (batch_id + 1)
return outputs, targets, test_loss
def evaluate():
net.eval()
test_loss = 0
targets, outputs = [], []
with torch.no_grad():
for batch_id, (data, target) in enumerate(test_loader):
data, target = data.cuda(), target.cuda()
output = net(data)
batch_loss = criterion(output, target)
targets += [target]
outputs += [output]
test_loss += batch_loss
test_loss /= (batch_id + 1)
test_acc = compute_accuracy(torch.cat(targets), torch.cat(outputs))
# display
log_string(
"validation_loss: {0:.4f}, validation_accuracy: {1:.02%}".format(
test_loss, test_acc))
def mc_evaluate():
net.eval()
# if options.MC:
# net.apply(apply_dropout)
test_loss = 0
targets, outputs, probs = [], [], []
with torch.no_grad():
for batch_id, (data, target) in enumerate(test_loader):
data, target = data.cuda(), target.cuda()
output = net(data)
prob = F.softmax(output)
batch_loss = criterion(output, target)
targets += [target]
outputs += [output]
probs += [prob]
test_loss += batch_loss
test_loss /= (batch_id + 1)
return torch.cat(probs).unsqueeze(0).cpu().numpy(), F.one_hot(torch.cat(targets), options.num_classes).cpu().numpy(), test_loss
if one_channel:
img = img.mean(dim=0)
img = img / 2 + 0.5 # unnormalize
npimg = img.numpy()
if one_channel:
plt.imshow(npimg, cmap="Greys")
else:
plt.imshow(np.transpose(npimg, (1, 2, 0)))
use_cuda = True
device = torch.device("cpu")
if use_cuda:
device = get_default_device()
train_dataset = Data(train=True)
validation_dataset = Data(train=False)
print("done")
# creat train/validation loader
batch_size = 100
train_loader = DataLoader(dataset=train_dataset, batch_size=args.batch_size)
validation_loader = DataLoader(dataset=validation_dataset,
batch_size=args.batch_size)
# Call the model
model = mymodel("resnet18") #models.resnet18(pretrained = True)
model = model.to(device)
model
# trasfer learning(freezed the parameters)
torch.sum(torch.exp(negative_part - max_val), -1)) + 1e-5)
cp_loss = torch.mean(cp_loss)
loss = 0.5 * cluster_loss + 0.5 * cls_loss + cp_loss + mlm_loss * 0.5
loss = loss / float(grad_iter)
if i_batch % 500 == 0:
print('cluster_loss: ', cluster_loss)
print('cls_loss: ', cls_loss)
print('cp_loss: ', cp_loss)
print('mlm_loss: ', mlm_loss)
print('loss: ', loss)
loss.backward()
if i_batch % grad_iter == 0:
optimizer.step()
scheduler.step()
optimizer.zero_grad()
if (i + 1) % config['save_interval'] == 0:
model_dir = './ckpts/ckpt_%d' % i
if not os.path.exists(model_dir):
os.mkdir(model_dir)
bert_encoder.save_pretrained(model_dir)
if __name__ == '__main__':
train_config = json.load(open(sys.argv[1]))
train_data = Data(train_config['train_data'])
tokenizer = BertTokenizer(train_config['vocab'], do_lower_case=False)
bert_encoder = BERT_EM.from_pretrained(train_config['base_model'])
print('Finished loading pre-trained model...')
train(train_data, tokenizer, bert_encoder, train_config)