def test_loss_wrapper_affinity_masking(self):
from neurofire.criteria.loss_transforms import MaskTransitionToIgnoreLabel
from neurofire.criteria.loss_transforms import RemoveSegmentationFromTarget
from neurofire.criteria.loss_wrapper import LossWrapper
from neurofire.transform.affinities import Segmentation2Affinities
offsets = [(1, 0, 0), (0, 1, 0), (0, 0, 1), (9, 0, 0), (0, 9, 0),
(0, 0, 9), (9, 4, 0), (4, 9, 0), (9, 0, 9)]
trafos = Compose(MaskTransitionToIgnoreLabel(offsets, ignore_label=0),
RemoveSegmentationFromTarget())
aff_trafo = Segmentation2Affinities(offsets, retain_segmentation=True)
seg = self.make_segmentation_with_ignore(self.shape)
ignore_mask = self.brute_force_transition_masking(seg, offsets)
target = Variable(torch.Tensor(aff_trafo(seg.astype('float32'))[None]),
requires_grad=False)
tshape = target.size()
pshape = (tshape[0], tshape[1] - 1) + tshape[2:]
prediction = Variable(torch.Tensor(*pshape).uniform_(0, 1),
requires_grad=True)
# apply cross entropy loss
criterion = BCELoss()
# criterion = SorensenDiceLoss()
wrapper = LossWrapper(criterion, trafos)
loss = wrapper.forward(prediction, target)
loss.backward()
grads = prediction.grad.data.numpy().squeeze()
self.assertEqual(grads.shape, ignore_mask.shape)
self.assertTrue((grads[ignore_mask] == 0).all())
self.assertFalse(np.sum(grads[np.logical_not(ignore_mask)]) == 0)
def standard_cross_entropy(log_p_pred,
log_r_pred,
t_pred,
y_true,
r_true,
t_true,
log_r_clip=25.):
r_pred = torch.exp(log_r_pred)
# r_pred = torch.exp(torch.clamp(log_r_pred, -log_r_clip, log_r_clip))
s_hat = 1. / (1. + r_pred)
return BCELoss()(s_hat, y_true)
def fit(self,
points,
features,
actions,
n_epoch=50000,
lr=5e-4,
batch_size=150):
datasets = []
optimizer = optim.Adam(params=self.net.parameters(), lr=lr)
bceLoss = BCELoss()
for p, f, a in zip(points, features, actions):
one_set = TensorDataset(to_tensor(p[:-1]), to_tensor(f[:-1]),
to_tensor(a))
datasets.append(one_set)
train_loader = DataLoader(ConcatDataset(datasets[:-1]),
batch_size=1,
shuffle=True)
test_loader = DataLoader(ConcatDataset(datasets[-1:]),
batch_size=1,
shuffle=False)
print(len(train_loader), len(test_loader))
for e in range(n_epoch):
print(f"Epoch: {e}")
for i, (p, f, a) in enumerate(train_loader):
out = self.net(p, f)
loss = bceLoss(out, a)
loss.backward()
if i % batch_size == 0:
optimizer.step()
optimizer.zero_grad()
sum_loss = 0
with torch.no_grad():
for i, (p, f, a) in enumerate(train_loader):
out = self.net(p, f)
loss = bceLoss(out, a)
sum_loss += loss
sum_loss /= len(train_loader)
print(f"Train Loss: {sum_loss.item()}")
sum_loss = 0
with torch.no_grad():
for i, (p, f, a) in enumerate(test_loader):
out = self.net(p, f)
loss = bceLoss(out, a)
sum_loss += loss
sum_loss /= len(test_loader)
print(f"Test Loss: {sum_loss.item()}")
def __init__(self,
baseloss,
classifier,
lambda_clfloss=0.5,
increase_until=10000,
*args):
super(FlipLoss, self).__init__(*args)
self.baseloss = baseloss
# assumed to return logit for binary classification (sigmoid)
self.classifier = classifier
self.sigmoid = nn.Sigmoid()
self.bce = BCELoss()
self.lambda_clfloss = lambda_clfloss
self.increase_until = increase_until
self.i = 0
for p in self.classifier.parameters():
p.requires_grad = False
self.classifier.eval()
def create_setting_pretraindcae(model=None):
"""
This part is fixed for pretrain DCAE for mnist from paper Deep one-class classification setting.
adam are used in paper.
"""
setting = {}
if cfg.pretrain_solver == 'adam':
optimizer = torch.optim.Adam(params=model.parameters(), lr=cfg.pretrain_lr)
elif cfg.pretrain_solver == 'sgd':
optimizer = torch.optim.SGD(lr=cfg.pretrain_lr, momentum=cfg.pretrain_momentum, nesterov=True, params=model.parameters())
else:
raise ValueError('invalid pretrain solver for using: {}'.format(cfg.pretrain_solver))
setting['optim'] = optimizer
if cfg.ae_loss == 'l2':
#from loss import MSEReconstructionError
#loss = MSEReconstructionError()
print('using MSE')
loss = MSELoss(reduction='none')
if cfg.ae_loss == 'ce':
loss = BCELoss()
setting['criterion'] = loss
return setting
# model ====================================================================
print("Init model...")
# init model with pre-trained weights
model = create_model()
model.load_state_dict(torch.load('../trained_models/baseline_weights/gen_model.pt'))
model.train()
model.cuda()
cudnn.benchmark = True
# loss =====================================================================
print("BCE criterium...")
bce_loss = BCELoss()
# select only decoder parameters, keep vgg16 with pretrained weights
decoder_parameters = []
for i, (a, p) in enumerate(model.named_parameters()):
if i>25:
print(i, a, p.shape)
decoder_parameters.append(p)
# optimizer parameters, added nesterov momentum
optimizer = SGD(decoder_parameters,
lr = args.lr,
momentum=0.9,
weight_decay=0.00001,
nesterov=True)
opt = opts.parse_opt()
if "CUDA_VISIBLE_DEVICES" not in os.environ.keys():
os.environ["CUDA_VISIBLE_DEVICES"] =opt.gpu
#opt.model ='lstm'
#opt.model ='fasttext'
train_iter, test_iter = utils.loadData(opt)
model=models.setup(opt)
if torch.cuda.is_available():
model.cuda()
model.train()
print("# parameters:", sum(param.numel() for param in model.parameters()))
optimizer = optim.Adam(model.parameters(), lr=opt.learning_rate)
optimizer.zero_grad()
loss_fun = BCELoss()
#batch = next(iter(train_iter))
#x=batch.text[0]
#print(utils.evaluation(model,test_iter))
for i in range(opt.max_epoch):
for epoch,batch in enumerate(train_iter):
start= time.time()
predicted = model(batch.text[0])
loss= F.cross_entropy(predicted,batch.label)
loss.backward()
utils.clip_gradient(optimizer, opt.grad_clip)
optimizer.step()
if epoch% 100==0:
if torch.cuda.is_available():
dataset = DiabetesDataset('diabetes.csv')
train_loader = DataLoader(dataset=dataset,
batch_size=32,
shuffle=True,
num_workers=2)
model = Model()
optim = SGD(model.parameters(), lr=0.1)
num_epochs = 5
for epoch in range(num_epochs):
for i, data in enumerate(train_loader, 0):
inputs, labels = data
inputs.to(device)
labels.to(device)
y_pred = model(inputs)
y_pred = y_pred.squeeze(dim=1)
print("Y_pred", y_pred)
print("Y_true", labels)
loss = BCELoss(y_pred, labels)
print(epoch, i, loss.item())
optim.zero_grad()
loss.backward()
optim.step()
# Init optimizers
print('Initializing the optimizers...')
g_adam = Adam(g.parameters(), lr=gen_lr, weight_decay=dis_l2_reg)
d_adam = Adam(d.parameters(), lr=dis_lr, weight_decay=gen_l2_reg)
# Load the dataset
print("Loading dataset...")
transform = Compose([Resize((IMAGE_HEIGHT, IMAGE_WIDTH)), ToTensor()])
char_ds = CharacterDataset(dataset_path, labels_file, transform)
loader = DataLoader(char_ds, batch_size=batch_size, shuffle=True)
# Restore the content of the dataset if the training is not new
if args.add:
next_letter_to_add = args.add
last_char_added_index = character_to_index_mapping[args.add]
for curr_char in character_to_index_mapping.keys():
if curr_char == ' ': # the space is already present
pass
if curr_char == next_letter_to_add:
break
else:
char_ds.add_character_to_training(curr_char)
# Train
print("Initiating GAN...")
gan = CGAN(g, d, BCELoss(), BCELoss(), G_optim=g_adam, D_optim=d_adam,
dataset_loader=loader, dataset=char_ds, device=dev,
writer=writer, current_datetime=current_datetime)
gan.train(add_character_every*len(character_to_index_mapping) + 1000,
next_letter_to_add=next_letter_to_add, use_soft_labels=args.soft_labels)
if (target_numpy[sorted_output_indexes[i]] == 1.0): matches += 1
i += 1
accuracy = matches / match_by
return accuracy
model = Model()
model.setup(80, 80, 6400, 3)
hotspot = HotSpotDataset('train_1.npz', 'train')
#i = 100 ; # id 2277411
for i in range(hotspot.__len__()):
ids, inputs, targets = hotspot.__getitem__(i)
x = Variable(inputs, requires_grad=True).float()
y = Variable(targets, requires_grad=False).float()
criterion = BCELoss()
loss = criterion(model.forward(x), y)
#criterion = BCEWithLogitsLoss(); loss = criterion(model.forward(x), y)
#criterion = MultiMarginLoss(); loss = criterion(model.forward(x), y.long())
#criterion = MultiLabelMarginLoss(); loss = criterion(model.forward(x), y.long())
#criterion = MultiLabelSoftMarginLoss(); loss = criterion(model.forward(x), y)
#loss = binary_cross_entropy(model.forward(x), y);
#loss = binary_cross_entropy_with_logits(x, y);
#x=CustomLoss()(x)
#CustomLoss.apply
# loss = nn.MSELoss()
#loss=CustomLoss.apply()
#loss=model.loss(x, y, 10);
#loss = criterion(model.forward(x), y)
classifier = ClassifierFromElmo(1024, attn = 1)
data = torch.randn(MAX_LEN, 200, 1024).to(device)
mask = torch.ones(MAX_LEN, 200, 2*1024).to(device)
yy = classifier(data, mask)
print(yy.size())
classifier.save('classifiertry')
newc = ClassifierFromElmo(1024, attn = 1, filen = 'classifiertry')
zz = classifier(data, mask)
ff = (yy-zz).sum()
assert ff.abs() < 0.000001
print('works')
assert False
cl = time.clock()
trload, teload = create_embed_loaders()
print('done in {}'.format(time.clock()-cl))
classifier = ClassifierFromElmo(1024).cuda()
for i in range(100):
for j, b in enumerate(trload):
x, y, mask = b['x'], b['y'], b['mask']
mask = mask.squeeze(1)
classifier.zero_grad()
yhat = classifier(x, mask)
loss = BCE(yhat, y)
loss.backward()
print(loss.item())
def __init__(self):
super().__init__()
self.bce = BCELoss(reduction="sum")
self.kl = lambda mu, logvar : torch.sum(-logvar + torch.exp(logvar) + mu ** 2 - 1) / 2
def __init__(self, beta=0.1):
super().__init__()
self.bce = BCELoss(reduction="sum")
self.beta = beta
self.mse = MSELoss(reduction="sum")
def forward(self, x):
x = self.dense1(x)
x = self.dense2(x)
x = F.sigmoid(self.dense3(x))
return x
if __name__ == '__main__':
sparkConf = init_spark_conf().setAppName("testNNClassifer").setMaster(
'local[1]')
sc = init_nncontext(sparkConf)
sqlContext = SQLContext(sc)
df = get_df(sqlContext)
torch_model = SimpleTorchModel()
becloss = BCELoss()
model = TorchNet.from_pytorch(module=torch_model,
input_shape=[1, 2],
lossFunc=becloss.forward,
pred_shape=[1, 1],
label_shape=[1, 1])
classifier = NNEstimator(model, TorchIdentityCriterion(), SeqToTensor([2])) \
.setBatchSize(2) \
.setOptimMethod(Adam()) \
.setLearningRate(0.1) \
.setMaxEpoch(20)
nnClassifierModel = classifier.fit(df)
print("After training: ")
def __init__(self,
name=None,
initial_learning_rate=0.001,
max_epochs=1000,
batch_size=2,
improvement_patience=20,
optimizer_class=Adam,
loss_function=BCELoss(),
best_model_by_fscore=False,
n_epochs_per_checkpoint=10,
checkpoint_export_file=None,
lr_refinement_multiplier=0.2,
number_of_refinement_steps=10,
refinement_patience=2,
best_params_file="default_model.tsd",
):
"""Initialize a training strategy. Includes some validation params.
:param name: Name the training strategy. This is useful for
keeping track of log files: logging will use this name,
e.g. as a TensorBoard comment.
:param initial_learning_rate: Initial learning rate. Passed to
the optimizer
:param max_epochs:
:param batch_size:
:param improvement_patience: How many epochs are we willing to
train without seeing improvement on the validation data before
early-stopping?
:param optimizer_class: A PyTorch optimizer class, like Adam.
(The *class*, not an *instance* of the class.)
:param loss_function: A PyTorch loss, like BCEWithLogitsLoss.
:param best_model_by_fscore: Use the validation aggregated f-score
instead of the loss to keep the best model during training.
:param checkpoint_export_file: Where to dump the checkpoint params?
:param n_epochs_per_checkpoint: Dump checkpoint params export once per
this many epochs.
:param number_of_refinement_steps: How many times should we try to attenuate
the learning rate.
:param lr_refinement_multiplier: Attenuate learning rate by this
multiplicative factor in each refinement step.
:param refinement_patience: How many epochs do we wait for improvement
when in the refining stage.
:param best_params_file: The file to which to save the best model.
"""
self.name = name
# Learning rate
self.initial_learning_rate = initial_learning_rate
# Epochs & batches
self.max_epochs = max_epochs
self.batch_size = batch_size
self.improvement_patience = improvement_patience
# Loss function
self.loss_function = loss_function
# Optimizer
self.optimizer_class = optimizer_class
# Model selection
self.best_model_by_fscore = best_model_by_fscore
# Checkpointing
self.n_epochs_per_checkpoint = n_epochs_per_checkpoint
self.checkpoint_export_file = checkpoint_export_file
# Early-stopping & Refinement
self.lr_refinement_multiplier = lr_refinement_multiplier
self.number_of_refinement_steps = number_of_refinement_steps
self.refinement_patience = refinement_patience
# Persisting the model
self.best_params_file = best_params_file
def train(train_loader, dev_loader, model, label2id):
add_classifier(model=model, labels=label2id)
id2label = {v: k for k, v in label2id.items()}
if torch.cuda.is_available():
model.to('cuda')
max_epochs = model.configuration('MaxEpoch')
lr = model.configuration('LearningRate')
max_patience = model.configuration('Patience')
epoch = 0
early_stop = False
criterion = BCELoss()
optimizer = Adam(lr=lr, params=model.parameters())
while epoch < max_epochs and not early_stop:
model.train()
running_loss = 0.0
steps = 0
bar = tqdm(total=len(train_loader), desc='Training')
for batch in train_loader:
batched_sentences, batched_attn, batched_labels = batch
if torch.cuda.is_available():
batched_sentences = batched_sentences.cuda()
batched_attn = batched_attn.cuda()
batched_labels = batched_labels.cuda()
outputs = model(batched_sentences, attention_mask=batched_attn)
optimizer.zero_grad()
loss = criterion(outputs, batched_labels)
loss.backward()
optimizer.step()
running_loss += loss.item()
steps += 1
bar.update(1)
bar.close()
loss = running_loss / steps
with torch.no_grad():
bar = tqdm(total=len(dev_loader), desc='Eval')
model.eval()
dev_running_loss = 0.0
dev_steps = 0
all_discrete_preds = []
all_labels = []
for batch in dev_loader:
batched_sentences, batched_attn, batched_labels = batch
if torch.cuda.is_available():
batched_sentences = batched_sentences.cuda()
batched_attn = batched_attn.cuda()
batched_labels = batched_labels.cuda()
outputs = model(batched_sentences, attention_mask=batched_attn)
optimizer.zero_grad()
dev_loss = criterion(outputs, batched_labels)
dev_running_loss += dev_loss.item()
dev_steps += 1
discrete_pred = get_discrete_pred(
outputs.detach().cpu().numpy(), id2label=id2label)
discrete_lab = get_discrete_pred(
batched_labels.detach().cpu().numpy(), id2label=id2label)
all_discrete_preds.extend(discrete_pred)
all_labels.extend(discrete_lab)
bar.update(1)
bar.close()
dev_loss = dev_running_loss / dev_steps
avg_pred_number, void_prediction_counter, micro_p, micro_r, micro_f1, macro_p, macro_r, macro_f1 = compute_metrics(
all_discrete_preds, all_labels)
if epoch == 0:
best_macro_f1 = macro_f1
patience = 0
else:
if macro_f1 > best_macro_f1:
best_macro_f1 = macro_f1
patience = 0
else:
patience += 1
if patience >= max_patience:
early_stop = True
print(
'epoch: {}; loss: {:.2f}; val_loss: {:.2f}; macro_f1: {:.2f}; best_macro_f1: {:.2f}, patience: {}'
.format(epoch, loss, dev_loss, macro_f1, best_macro_f1, patience))
if early_stop:
print('early stopped')
epoch += 1
