def setup_class(self):
self.batch = {
'x': torch.rand(
(2, 1, 8, 8)), # batch size 2, 1 input channel, 8x8 pixels
'y': torch.LongTensor(
[0, 1]) # class for those two images (0 and 1 respectively)
}
self.data_loader = DataLoader(DummyDataset(),
batch_size=1,
shuffle=False,
num_workers=4)
self.module = EisenModuleWrapper(Net(),
input_names=['x'],
output_names=['pred'])
self.optimizer = Adam(self.module.parameters(), 0.001)
self.loss = EisenModuleWrapper(module=CrossEntropyLoss(),
input_names=['pred', 'y'],
output_names=['loss'])
self.metric = EisenModuleWrapper(module=CrossEntropyLoss(),
input_names=['pred', 'y'],
output_names=['metric'])
self.testing_workflow = WorkflowTesting(self.module,
self.data_loader,
[self.metric],
gpu=False)
assert isinstance(self.testing_workflow, WorkflowTesting)
def compute_diag_fisher(self):
"""
Arguments: None. Just use global variables (self.model, self.criterion, ...)
Return: Diagonal Fisher matrix.
This function will be used in the function 'update_fisher'
"""
self.fisher_iterator
param_names = []
loglikelihood_grads = {}
for data, label in self.fisher_iterator:
data = Variable(data)
label = Variable(label)
loglikelihood = CrossEntropyLoss()(self.model(data)[self.t], label)
loglikelihood.backward()
for n, p in self.model.named_parameters():
n = n.replace('.', '__')
if p.grad == None:
continue
loglikelihood_grads[n] = (loglikelihood_grads.get(n, 0) +
(p.grad**2)).mean(0)
param_names.append(n)
fisher_diagonals = [g for g in loglikelihood_grads.values()]
dict = {n: f.detach() for n, f in zip(param_names, fisher_diagonals)}
return dict
def __init__(self,
n_inputs,
n_outputs,
n_tasks,
args):
super(Net, self).__init__()
nl, nh = args.n_layers, args.n_hiddens
self.net = MLP([n_inputs] + [nh] * nl + [n_outputs])
self.bce = CrossEntropyLoss()
self.n_outputs = n_outputs
self.opt = optim.SGD(self.parameters(), args.lr)
self.batchSize = int(args.replay_batch_size)
self.memories = args.memories
self.s = float(args.s)
self.gamma = args.gamma
self.steps = int(args.batches_per_example)
# allocate buffer
self.M = []
self.age = 0
# handle gpus if specified
self.cuda = args.cuda
if self.cuda:
self.net = self.net.cuda()
def main(exp, dataset, data_dir, output_dir, num_cls, seed, num_inst, batch,
m_batch, num_updates, num_inner_updates, lr, meta_lr, gpu):
random.seed(seed)
np.random.seed(seed)
setproctitle(exp)
# Print all the args for logging purposes
frame = inspect.currentframe()
args, _, _, values = inspect.getargvalues(frame)
for arg in args:
print(arg, values[arg])
# make output dir
output_dir = '{}/{}'.format(output_dir, exp)
os.makedirs(output_dir, exist_ok=True)
# Set the gpu
print('Setting GPU to', str(gpu))
os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu)
loss_fn = CrossEntropyLoss()
learner = MetaLearner(dataset,
num_cls, num_inst, m_batch, float(meta_lr), batch,
float(lr), num_updates, num_inner_updates, loss_fn)
learner.data_dir = str(data_dir)
learner.output_dir = str(output_dir)
learner.train(exp)
def forward(self,
input_ids,
attention_mask=None,
token_type_ids=None,
position_ids=None,
head_mask=None,
labels=None):
outputs = self.bert(input_ids,
attention_mask=attention_mask,
token_type_ids=token_type_ids,
position_ids=position_ids,
head_mask=head_mask)
pooled_output = outputs[1]
pooled_output = self.dropout(pooled_output + 0.1)
logits = self.classifier(pooled_output)
# add hidden states and attention if they are here
outputs = (logits, ) + outputs[2:]
if labels is not None:
if self.num_labels == 1:
# We are doing regression
loss_fct = BCEWithLogitsLoss()
loss = loss_fct(logits.view(-1), labels.view(-1))
else:
loss_fct = CrossEntropyLoss()
loss = loss_fct(logits.view(-1, self.num_labels),
labels.view(-1))
outputs = (loss, ) + outputs
return outputs # (loss), logits, (hidden_states), (attentions)
def __init__(self, n_inputs, n_outputs, n_tasks, args):
super(Net, self).__init__()
self.args = args
nl, nh = args.n_layers, args.n_hiddens
self.net_o = MLP([n_inputs] + [nh] * nl + [n_outputs])
self.net_n = MLP([n_inputs] + [nh] * nl + [n_outputs])
self.bce = CrossEntropyLoss()
self.mse = nn.MSELoss()
self.l1 = nn.L1Loss()
self.n_outputs = n_outputs
self.opt = optim.SGD(self.net_n.parameters(), args.lr)
self.batchSize = int(args.replay_batch_size)
self.memories = args.memories
# allocate buffer
self.M = []
self.age = 0
# handle gpus if specified
self.cuda = args.cuda
if self.cuda:
self.net_n = self.net_n.cuda()
self.net_o = self.net_o.cuda()
def __init__(self, model, init_weight):
super(RegressionTrain, self).__init__()
self.model = model
self.weights = torch.nn.Parameter(
torch.from_numpy(init_weight).float())
self.ce_loss = CrossEntropyLoss()
def _reformat_predictions(
self, y_true: List[List[int]], y_pred: List[List[int]],
input_ids: List[List[str]]
) -> Tuple[List[List[str]], List[List[str]], List[List[str]]]:
"""
Takes batch of tokens, labels (class indexes) and predictions (class indexes)
and get rid of unwanted tokens, that is, those that have as label the index
to ignore (i.e. padding tokens). It also converts the label and prediction
indexes into their corresponding class name.
Args:
y_true (list of lists `int`): indexes of the real labels
y_pred (list of lists `int`): indexes of the predicted classes
input_ids (list of lists `str`) : tokens
Returns:
Tuple that contains the transformed input arguments
"""
# Map indexes to labels and remove ignored indexes
true_list = [[] for _ in range(len(y_true))]
pred_list = [[] for _ in range(len(y_pred))]
input_list = [[] for _ in range(len(input_ids))]
for i in range(len(y_true)):
for j in range(len(y_true[0])):
if y_true[i][j] != CrossEntropyLoss().ignore_index:
true_list[i].append(self.inds2labels[y_true[i][j]])
pred_list[i].append(self.inds2labels[y_pred[i][j]])
input_list[i].append(input_ids[i][j])
return true_list, pred_list, input_list
def main(exp, dataset, num_cls, num_inst, batch, m_batch, num_updates,
num_inner_updates, lr, meta_lr, gpu):
random.seed(1337)
np.random.seed(1337)
setproctitle(exp)
# Print all the args for logging purposes
frame = inspect.currentframe()
args, _, _, values = inspect.getargvalues(frame)
for arg in args:
print arg, values[arg]
# make output dir
output = '../output/{}'.format(exp)
try:
os.makedirs(output)
except:
pass
# Set the gpu
print 'Setting GPU to', str(gpu)
os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu)
loss_fn = CrossEntropyLoss()
learner = MetaLearner(dataset,
num_cls, num_inst, m_batch, float(meta_lr), batch,
float(lr), num_updates, num_inner_updates, loss_fn)
learner.train(exp)
def __init__(self):
super(RGListener, self).__init__()
# image encoder
self.image_encoder = nn.Sequential(
*list(models.resnet18(pretrained=True).children())[:-2])
# position embedding
self.pos_emb = nn.Parameter(torch.randn(64, 7 * 7))
# caption encoder
self.tokenizer = BertTokenizerFast.from_pretrained("bert-base-uncased")
self.bert = BertModel.from_pretrained("bert-base-uncased")
for param in self.bert.parameters():
param.requires_grad = False
# regressor
self.mlp = nn.Sequential(nn.Linear(768, 256), nn.ReLU(),
nn.Linear(256, 256), nn.ReLU(),
nn.Linear(256, 576))
# trainer
# self.loss_fn = MSELoss()
self.loss_fn = CrossEntropyLoss()
self.optimizer = torch.optim.Adam(self.parameters())
def test_accumulator():
from sklearn.datasets import load_iris
from sklearn.model_selection import train_test_split
from torch.utils.data import TensorDataset, DataLoader
data = load_iris()
X = data['data']
y = data['target']
model = torch.nn.Sequential(nn.Linear(4, 16), nn.Linear(16, 3))
opti = torch.optim.Adam(model.parameters(), lr=1e-2)
criterion = CrossEntropyLoss()
accumulator = GradientAccumulator(accumulation_steps=3)
tx, cvx, ty, cvy = train_test_split(X,
y,
test_size=0.2,
random_state=1)
train_loader = DataLoader(TensorDataset(torch.FloatTensor(tx),
torch.LongTensor(ty)),
batch_size=32)
dev_loader = DataLoader(TensorDataset(torch.FloatTensor(cvx),
torch.LongTensor(cvy)),
batch_size=32)
for i in range(100):
for train_x, train_y in train_loader:
output = model(train_x)
loss = criterion(output, train_y)
with accumulator(loss, opti) as accu:
accu.backward()
accu.step()
for dev_x, dev_y in train_loader:
output = model(dev_x)
true_num = (output.argmax(dim=1) == dev_y).sum()
print("Correct num: {}".format(true_num.item()))
def from_file(file_path,
vocab_size,
pad_token,
embedding_size=300,
n_layers=6):
model = TransformerModel(vocab_size, CrossEntropyLoss(ignore_index=-1),
n_layers, embedding_size, 1000, 6, 1e-5)
model.load_state_dict(torch.load(file_path))
return model
def __init__(self, loss_names, loss_weights, device, num_classes):
"""
:param loss_names: list of loss names,
possible losses=['jaccard', 'nlll', 'crossentropy', 'smooth_jaccard', 'focal', 'dice']
:param loss_weights: list of weight coefficients for each loss from loss_names.
:param device: execution device.
:param num_classes: number of classes in training data.
"""
super(MultiLoss, self).__init__()
assert len(loss_names) == len(loss_weights)
self.device = device
self.losses = dict()
self.loss_weights = dict()
self.num_classes = num_classes
for loss, weight in zip(loss_names, loss_weights):
loss = loss.lower()
if loss == 'jaccard':
if self.num_classes > 1:
self.losses[loss] = JaccardLossMulti()
else:
self.losses[loss] = JaccardLoss()
elif loss == 'nlll':
# if last layer of network is softmax use NLLLoss if isn't use CrossEntropyLoss
self.losses[loss] = NLLLoss()
elif loss == 'crossentropy':
# if last layer of network is softmax use NLLLoss if isn't use CrossEntropyLoss
# or BCEWithLogitsLoss for binary case
if self.num_classes > 1:
self.losses[loss] = CrossEntropyLoss(size_average=True)
else:
self.losses[loss] = BCEWithLogitsLoss(size_average=True)
elif loss == 'smooth_jaccard':
if self.num_classes > 1:
raise ValueError("ERROR: for multiclass case loss is not implemented.")
else:
self.losses[loss] = SmoothJaccardLoss()
elif loss == 'focal':
if self.num_classes > 1:
self.losses[loss] = FocalLossMulti(size_average=False)
else:
self.losses[loss] = FocalLossBinary(size_average=False)
elif loss == 'dice' and self.num_classes < 2:
self.losses[loss] = DiceLoss()
else:
raise ValueError(loss)
self.loss_weights[loss] = weight
def inference(args, model, test_save_path=None):
from datasets.dataset_HuBMAP import HuBMAP_dataset,RandomGenerator
from datasets.dataset_HuBMAP import HuBMAP_dataset, Generator
db_test = HuBMAP_dataset(base_dir=args.root_path, split="test", list_dir=args.list_dir,transform=transforms.Compose(
[Generator(output_size=[args.img_size, args.img_size])]))
testloader = DataLoader(db_test, batch_size=batch_size, shuffle=True, num_workers=1)
logging.info("{} test iterations per epoch".format(len(testloader)))
model.eval()
metric_list = 0.0
### Add validation here
total_test_loss = 0
total_test_dice_loss = 0
batch_num = 0
label_batch_sum = 0
ce_loss = CrossEntropyLoss()
num_classes = args.num_classes
dice_loss = DiceLoss(num_classes)
for i_batch, sampled_batch in enumerate(testloader):
print(" testing progress: {:.2f}".format(batch_num/len(testloader)*100) + "%", end="\r")
model.eval()
image_batch, label_batch = sampled_batch['image'], sampled_batch['label']
image_batch, label_batch = image_batch.cuda(), label_batch.cuda()
#print(label_batch.size())
a = np.sum(label_batch.detach().cpu().numpy())
print(a)
outputs = model(image_batch)
if a>label_batch_sum:
label_batch_sum = a
np.save('test_pred.npy', outputs.detach().cpu().numpy())
np.save('test_img.npy', image_batch.detach().cpu().numpy())
np.save('test_label.npy',label_batch.detach().cpu().numpy())
loss_ce = ce_loss(outputs, label_batch[:].long())
loss_dice = dice_loss(outputs, label_batch, softmax=True)
loss = 0.5 * loss_ce + 0.5 * loss_dice
###
total_test_loss += loss.item()
total_test_dice_loss += loss_dice.item()
###
batch_num = batch_num + 1
avg_test_loss = total_test_loss/batch_num
avg_test_loss_dice = total_test_dice_loss/batch_num
print(avg_test_loss_dice)
writer = SummaryWriter(snapshot_path + '/log')
writer.add_scalar('info/avg_test_loss', avg_test_loss)
writer.add_scalar('info/avg_test_loss_dice', avg_test_loss_dice)
logging.info('test_loss : %f, test_loss_dice: %f' % (avg_test_loss, avg_test_loss_dice))
###
return "Testing Finished!"
def n_layer_nn(optimiser_function, layer_dims=[28*28 + 1, 128, 10], learning_rate=0.1, epochs=100):
layers = len(layer_dims)
assert layers >= 3, "Please give at leaset 3 dimensions"
modules = [Linear(layer_dims[0], layer_dims[1]), Relu()]
for i in range(1, layers - 2):
modules.append(Linear(layer_dims[i], layer_dims[i+1]))
modules.append(Relu())
modules.append(Linear(layer_dims[layers-2], layer_dims[layers-1]))
modules.append(Sigmoid())
print(modules)
model = Sequential(*modules).cuda('cuda:0')
loss_function = CrossEntropyLoss()
optimiser = optimiser_function(model.parameters(), lr=learning_rate)
stopper = EarlyStop(patience=3)
train_losses=[]
val_losses=[]
accuracy=[]
for epoch in range(epochs):
losses=[]
for i,(X, y) in enumerate(get_minibatches(train_loader, device)):
optimiser.zero_grad()
yhat = model.forward(X)
loss = loss_function(yhat, y.argmax(1))
losses.append(loss.item())
loss.backward()
optimiser.step()
train_losses.append(np.mean(losses))
if epoch % 3 == 0:
with torch.no_grad():
losses = []
corrects = 0
for i,(X, y) in enumerate(get_minibatches(val_loader, device)):
y = y.argmax(1)
yhat = model.forward(X)
losses.append(loss_function(yhat, y).item())
ypred = yhat.argmax(1)
corrects += (ypred == y).sum()
val_loss = np.mean(losses)
val_losses.append(val_loss)
acc = corrects.cpu().numpy() / val_size
#print("Accuracy {}".format(acc))
accuracy.append(acc)
if not stopper.continue_still(val_loss):
print("Early stop at epoch {}".format(epoch))
break
return val_losses, accuracy
def calc_losses(y_hats, y, out_dims):
"""
Calculate all losses across all prediction tasks.
Also reformats 'predictions' to be a friendly pytorch tensor for later use.
TODO: this should be a class?
"""
reg_loss = MarginRankingLoss()
clf_loss = CrossEntropyLoss()
if CUDA:
reg_loss = reg_loss.cuda()
clf_loss = clf_loss.cuda()
losses, predictions = [], []
for i, out_dim in enumerate(out_dims):
y_hat = y_hats[i]
y_tru = y[:, i]
# Regression case.
if out_dim == 1:
# Required for margin ranking loss.
y_rank = get_paired_ranks(y_tru)
y1_hat, y2_hat = get_pairs(y_hat)
losses.append(reg_loss(y1_hat, y2_hat, y_rank))
predictions.append(y_hat)
# Classification case.
elif out_dim > 1:
# Cross entropy loss.
losses.append(clf_loss(y_hat, y_tru.long()))
_, preds = torch.max(y_hat.data, 1)
predictions.append(preds.float().unsqueeze(1))
predictions = torch.cat(predictions, dim=1)
return (losses, predictions)
def setup_class(self):
self.batch = {
"x": torch.rand(
(2, 1, 8, 8)), # batch size 2, 1 input channel, 8x8 pixels
"y": torch.LongTensor(
[0, 1]), # class for those two images (0 and 1 respectively)
}
self.data_loader = DataLoader(DummyDataset(),
batch_size=1,
shuffle=False,
num_workers=4)
data_parallel_net = torch.nn.DataParallel(Net())
self.module = EisenModuleWrapper(data_parallel_net,
input_names=["x"],
output_names=["pred"])
self.optimizer = Adam(self.module.parameters(), 0.001)
self.loss = EisenModuleWrapper(module=CrossEntropyLoss(),
input_names=["pred", "y"],
output_names=["loss"])
self.metric = EisenModuleWrapper(
module=CrossEntropyLoss(),
input_names=["pred", "y"],
output_names=["metric"],
)
self.training_workflow = WorkflowTraining(
self.module,
self.data_loader,
[self.loss],
self.optimizer,
[self.metric],
gpu=False,
)
assert isinstance(self.training_workflow, WorkflowTraining)
def train_step(net, train_dataset_loader, epoch):
# set net to train mode
net.train()
learning_rate = BASE_LEARNING_RATE * LEARNING_RATE_DECAY_PER_EPOCH**epoch
optimizer = optim.SGD(net.parameters(),
lr=learning_rate,
momentum=LEARNING_MOMENTUM)
for i, (inputs, labels) in enumerate(train_dataset_loader):
inputs, labels = Variable(inputs), Variable(labels)
optimizer.zero_grad()
output = net.forward(inputs)
loss = CrossEntropyLoss().forward(output, labels)
loss.backward()
optimizer.step()
if i % LOG_INTERVAL == 0:
print('Train Epoch (LR: {}): {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.
format(learning_rate, epoch, i * len(inputs),
len(train_dataset_loader.dataset),
100. * i / len(train_dataset_loader), loss.data[0]))
def main(exp, batch_size, lr, gpu):
setproctitle(exp)
output = '../output/{}'.format(exp)
try:
os.makedirs(output)
except:
pass
os.environ['CUDA_VISIBLE_DEVICES'] = str(gpu)
loss_fn = CrossEntropyLoss()
net = OmniglotNet(10, loss_fn)
# NOTE: load weights from pre-trained model
net.load_state_dict(
torch.load('../output/maml-mnist-10way-5shot/train_iter_4500.pth'))
net.cuda()
opt = Adam(net.parameters(), lr=lr)
train_dataset = MNIST('../data/mnist/mnist_png',
transform=transforms.ToTensor())
test_dataset = MNIST('../data/mnist/mnist_png',
split='test',
transform=transforms.ToTensor())
train_loader = DataLoader(train_dataset,
batch_size=batch_size,
shuffle=True,
num_workers=1,
pin_memory=True)
val_loader = DataLoader(train_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=1,
pin_memory=True)
num_epochs = 10
train_loss = []
train_acc = []
val_loss = []
val_acc = []
for epoch in range(num_epochs):
# train for 1 epoch
t_loss, t_acc, v_loss, v_acc = train(train_loader, val_loader, net,
loss_fn, opt, epoch)
train_loss += t_loss
train_acc += t_acc
val_loss += v_loss
val_acc += v_acc
# eval on the val set
np.save('{}/train_loss.npy'.format(output), np.array(train_loss))
np.save('{}/train_acc.npy'.format(output), np.array(train_acc))
np.save('{}/val_loss.npy'.format(output), np.array(val_loss))
np.save('{}/val_acc.npy'.format(output), np.array(val_acc))
def __init__(self, name, config):
super().__init__(name)
self.loss: nn.Module
self.mode = config.mode
if config.mode == 'regression':
self.loss = MSELoss()
elif config.mode == 'classification':
self.loss = CrossEntropyLoss()
else:
raise ValueError(
f'config.mode must be in [regression, classification] but was {config.mode}'
)
self.head = FinetuneHead(config)
def __init__(self, params) -> None:
super(Solver, self).__init__()
self.params = params
self.device = torch.device(
'cuda') if torch.cuda.is_available() else torch.device('cpu')
self.model = LinearModel(params['feature_type'],
output_size=10).to(self.device)
if params['test'] == True:
self.model.load_state_dict(
torch.load(params['model_path'], map_location=self.device))
else:
self.loss_fn = CrossEntropyLoss()
self.optimizer = torch.optim.Adam(self.model.parameters(),
params['lr'])
self.train_dataset = m_dataset(params['data_path'],
params['feature_type'],
dataType='train')
self.train_dataloader = DataLoader(self.train_dataset,
batch_size=params['batch_size'],
shuffle=True,
num_workers=4,
pin_memory=True)
print('train data load finished!')
print('train data number:{}'.format(len(self.train_dataset)))
self.eval_dataset = m_dataset(params['data_path'],
params['feature_type'],
dataType='evaluate')
self.eval_dataloader = DataLoader(self.eval_dataset,
batch_size=params['batch_size'],
shuffle=True,
num_workers=4,
pin_memory=True)
print('eval data load finished!')
print('eval data number:{}'.format(len(self.eval_dataset)))
self.test_dataset = m_dataset(params['data_path'],
params['feature_type'],
dataType='test')
self.test_dataloader = DataLoader(self.test_dataset,
batch_size=params['batch_size'],
shuffle=True,
num_workers=4,
pin_memory=True)
print('test data load finished!')
print('test data number:{}'.format(len(self.test_dataset)))
self.logger = SummaryWriter(params['logger_dir'])
def __init__(self, type='auto', loss_weight=1.0, softmax_loss_weight=1.0):
"""
:param type: type of loss ('l1', 'l2', 'auto')
:param loss_weight: weight of loss, for 'l1' and 'l2', try with 0.01.
For 'auto', try with 1.0.
Source: https://github.com/Paralysis/ringloss
"""
super().__init__()
self.radius = Parameter(torch.Tensor(1))
self.radius.data.fill_(-1)
self.loss_weight = loss_weight
self.type = type
self.softmax = CrossEntropyLoss()
self.softmax_loss_weight = softmax_loss_weight
def train_batch(self, contents, summaries):
contents_tokenized = [self.tokenizer.encode(cont) for cont in contents]
input_ids, is_masked, labels = self.build_io(contents_tokenized,
summaries)
outputs, = self.model(input_ids)
cross_ent = CrossEntropyLoss(ignore_index=-1)
loss = cross_ent(outputs.view(-1, self.vocab_size), labels.view(-1))
num_masks = torch.sum(is_masked, dim=1).float() + 0.1
with torch.no_grad():
preds = torch.argmax(outputs, dim=2)
accs = torch.sum(preds.eq(labels).long() * is_masked,
dim=1).float() / num_masks
return loss, accs.mean().item()
def __init__(self):
super().__init__()
self.metrics_singleclass = nn.ModuleDict({
'loss':
BCEWithLogitsLoss(),
'acc':
SoftMaxWrapper(Accuracy(), multiclass=False),
'auc':
AUROC(),
})
self.metrics_multiclass = nn.ModuleDict({
'loss':
CrossEntropyLoss(),
'acc':
SoftMaxWrapper(Accuracy(), multiclass=True),
})
def get_lossfn(params, encoder, data):
if params.loss == "mse":
return MSELoss()
elif params.loss == "cosine":
return CosineLoss()
elif params.loss == "ce":
return CrossEntropyLoss(ignore_index=0) # ignore padding symbol
elif params.loss == "fliploss":
if params.baseloss == "cosine":
baseloss = CosineLoss()
elif params.baseloss == "mse":
baseloss = MSELoss()
else:
raise ValueError("Unknown base loss {params.baseloss}.")
bclf = train_binary_classifier(data['Sx'], data['Sy'], encoder, params)
params.latent_binary_classifier = bclf
return FlipLoss(baseloss, bclf, lambda_clfloss=params.lambda_clfloss)
def __init__(self, n_inputs, n_outputs, n_tasks, args):
super(Net, self).__init__()
self.args = args
nl, nh = args.n_layers, args.n_hiddens
config = mf.ModelFactory.get_model(model_type=args.arch,
sizes=[n_inputs] + [nh] * nl +
[n_outputs],
dataset=args.dataset,
args=args)
self.net = Learner.Learner(config, args)
self.opt_wt = optim.SGD(self.parameters(), lr=args.lr)
if self.args.learn_lr:
self.net.define_task_lr_params(alpha_init=args.alpha_init)
self.opt_lr = torch.optim.SGD(list(self.net.alpha_lr.parameters()),
lr=args.opt_lr)
self.loss = CrossEntropyLoss()
self.is_cifar = ((args.dataset == 'cifar100')
or (args.dataset == 'tinyimagenet'))
self.glances = args.glances
self.current_task = 0
self.memories = args.memories
self.batchSize = int(args.replay_batch_size)
# allocate buffer
self.M = []
self.age = 0
# handle gpus if specified
self.cuda = args.cuda
if self.cuda:
self.net = self.net.cuda(args.device_id)
self.n_outputs = n_outputs
if self.is_cifar:
self.nc_per_task = int(n_outputs / n_tasks)
else:
self.nc_per_task = n_outputs
self.args = args
def test_training_accuracy(net, test_dataset_loader, epoch):
# set net to test mode
net.eval()
test_loss = 0
correct = 0
for data, target in test_dataset_loader:
data, target = Variable(data, volatile=True), Variable(target)
output = net.forward(data)
test_loss += CrossEntropyLoss()(output, target).data[0]
pred = output.data.max(1)[
1] # get the index of the max log-probability
correct += pred.eq(target.data).cpu().sum()
test_loss = test_loss
test_loss /= len(
test_dataset_loader) # loss function already averages over batch size
print(
'\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.2f}%)\n'.format(
test_loss, correct, len(test_dataset_loader.dataset),
100. * correct / len(test_dataset_loader.dataset)))
def __init__(self, parameters={}):
# Use dictionary of parameters to class variables of the same name:
self.__dict__.update(NES.P, **parameters)
self.directory = os.getcwd()
self.data_repo = "{0}/{1}".format(self.directory, self.folder)
mkdir(self.data_repo)
np.random.seed(1337)
# Set up multiprocessing:
#cpus = multiprocessing.cpu_count()
#self.pool = Pool(cpus-1)
# Spawn population of networks using self.blueprint as a template:
self.loss_fn = CrossEntropyLoss()
self.population = self.Initialize_Population()
self.tasks = self.Define_Tasks()
# Begin!
self.Evolve()
def Evaluate(individual):
"""
Adapted inner loop of Model Agnostic Meta-Learning to be Baldwinian
a la Fernando et al. 2018.
Source: https://github.com/katerakelly/pytorch-maml/blob/master/src/maml.py
"""
#tasks['t1'] = self.get_task("../data/{}".format(self.dataset), self.num_classes, self.num_inst)
inner_net = InnerLoop(5, CrossEntropyLoss(), 3, 0.01, 100, 10, 3)
for t in range(10):
task = Get_Task("../data/{}".format('omniglot'), 5, 10)
# Outer-loop is completed by NES for G generations
inner_net.copy_weights(individual['network'])
metrics = inner_net.forward(task)
# Want validation accuracy for fitness (tr_loss, tr_acc, val_loss, val_acc):
print(metrics)
def train(args, snapshot_path):
base_lr = args.base_lr
num_classes = args.num_classes
batch_size = args.batch_size
max_iterations = args.max_iterations
model = net_factory(net_type=args.model, in_chns=1, class_num=num_classes)
db_train = BaseDataSets(base_dir=args.root_path,
split="train",
transform=transforms.Compose(
[RandomGenerator(args.patch_size)]),
fold=args.fold,
sup_type=args.sup_type)
db_val = BaseDataSets(base_dir=args.root_path, split="val")
def worker_init_fn(worker_id):
random.seed(args.seed + worker_id)
trainloader = DataLoader(db_train,
batch_size=batch_size,
shuffle=True,
num_workers=8,
pin_memory=True,
worker_init_fn=worker_init_fn)
valloader = DataLoader(db_val, batch_size=1, shuffle=False, num_workers=1)
model.train()
optimizer = optim.SGD(model.parameters(),
lr=base_lr,
momentum=0.9,
weight_decay=0.0001)
ce_loss = CrossEntropyLoss(ignore_index=4)
dice_loss = losses.DiceLoss(num_classes)
gatecrf_loss = ModelLossSemsegGatedCRF()
writer = SummaryWriter(snapshot_path + '/log')
logging.info("{} iterations per epoch".format(len(trainloader)))
iter_num = 0
max_epoch = max_iterations // len(trainloader) + 1
best_performance = 0.0
iterator = tqdm(range(max_epoch), ncols=70)
loss_gatedcrf_kernels_desc = [{"weight": 1, "xy": 6, "rgb": 0.1}]
loss_gatedcrf_radius = 5
for epoch_num in iterator:
for i_batch, sampled_batch in enumerate(trainloader):
volume_batch, label_batch = sampled_batch['image'], sampled_batch[
'label']
volume_batch, label_batch = volume_batch.cuda(), label_batch.cuda()
outputs = model(volume_batch)
outputs_soft = torch.softmax(outputs, dim=1)
loss_ce = ce_loss(outputs, label_batch[:].long())
out_gatedcrf = gatecrf_loss(
outputs_soft,
loss_gatedcrf_kernels_desc,
loss_gatedcrf_radius,
volume_batch,
256,
256,
)["loss"]
loss = loss_ce + 0.1 * out_gatedcrf
optimizer.zero_grad()
loss.backward()
optimizer.step()
lr_ = base_lr * (1.0 - iter_num / max_iterations)**0.9
for param_group in optimizer.param_groups:
param_group['lr'] = lr_
iter_num = iter_num + 1
writer.add_scalar('info/lr', lr_, iter_num)
writer.add_scalar('info/total_loss', loss, iter_num)
writer.add_scalar('info/loss_ce', loss_ce, iter_num)
writer.add_scalar('info/out_gatedcrf', out_gatedcrf, iter_num)
logging.info('iteration %d : loss : %f, loss_ce: %f' %
(iter_num, loss.item(), loss_ce.item()))
if iter_num % 20 == 0:
image = volume_batch[1, 0:1, :, :]
image = (image - image.min()) / (image.max() - image.min())
writer.add_image('train/Image', image, iter_num)
outputs = torch.argmax(torch.softmax(outputs, dim=1),
dim=1,
keepdim=True)
writer.add_image('train/Prediction', outputs[1, ...] * 50,
iter_num)
labs = label_batch[1, ...].unsqueeze(0) * 50
writer.add_image('train/GroundTruth', labs, iter_num)
if iter_num > 0 and iter_num % 200 == 0:
model.eval()
metric_list = 0.0
for i_batch, sampled_batch in enumerate(valloader):
metric_i = test_single_volume(sampled_batch["image"],
sampled_batch["label"],
model,
classes=num_classes)
metric_list += np.array(metric_i)
metric_list = metric_list / len(db_val)
for class_i in range(num_classes - 1):
writer.add_scalar('info/val_{}_dice'.format(class_i + 1),
metric_list[class_i, 0], iter_num)
writer.add_scalar('info/val_{}_hd95'.format(class_i + 1),
metric_list[class_i, 1], iter_num)
performance = np.mean(metric_list, axis=0)[0]
mean_hd95 = np.mean(metric_list, axis=0)[1]
writer.add_scalar('info/val_mean_dice', performance, iter_num)
writer.add_scalar('info/val_mean_hd95', mean_hd95, iter_num)
if performance > best_performance:
best_performance = performance
save_mode_path = os.path.join(
snapshot_path, 'iter_{}_dice_{}.pth'.format(
iter_num, round(best_performance, 4)))
save_best = os.path.join(
snapshot_path, '{}_best_model.pth'.format(args.model))
torch.save(model.state_dict(), save_mode_path)
torch.save(model.state_dict(), save_best)
logging.info('iteration %d : mean_dice : %f mean_hd95 : %f' %
(iter_num, performance, mean_hd95))
model.train()
if iter_num % 3000 == 0:
save_mode_path = os.path.join(snapshot_path,
'iter_' + str(iter_num) + '.pth')
torch.save(model.state_dict(), save_mode_path)
logging.info("save model to {}".format(save_mode_path))
if iter_num >= max_iterations:
break
if iter_num >= max_iterations:
iterator.close()
break
writer.close()
return "Training Finished!"