def __init__(self, **kwargs):
super().__init__()
self.margin = kwargs['margin']
""" Computes triplet loss using all triplets """
self.miner = miners.TripletMarginMiner(margin=1.0,
type_of_triplets="all")
self.loss = losses.TripletMarginLoss(margin=self.margin)
def __init__(self, **kwargs):
super().__init__()
assert kwargs['train_type'] == 'metric_learning', "Triplet Margin Loss can only be used with metric learning."
self.margin = kwargs['margin']
""" Computes triplet loss using all triplets that violate the margin """
self.miner = miners.TripletMarginMiner(margin=self.margin, type_of_triplets="all")
self.loss = losses.TripletMarginLoss(margin=self.margin)
def __init__(self, margin=0.1, **kwargs):
super(TripletLoss, self).__init__()
self.margin = margin
self.miner = miners.TripletMarginMiner(margin,
type_of_triplets='semihard')
self.loss_func = losses.TripletMarginLoss(margin=self.margin)
def train(data, epochs, triplet_alpha, miner, alpha_pow, alpha_mul):
device = torch.device("cpu")
distance = distances.CosineSimilarity()
if miner == 'batch-hard':
mining_func = miners.BatchHardMiner(distance=distance)
elif miner == 'hard':
mining_func = miners.TripletMarginMiner(margin=triplet_alpha,
distance=distance,
type_of_triplets="hard")
elif miner == 'semihard':
mining_func = miners.TripletMarginMiner(margin=triplet_alpha,
distance=distance,
type_of_triplets="semihard")
elif miner == 'all':
mining_func = miners.TripletMarginMiner(margin=triplet_alpha,
distance=distance,
type_of_triplets="all")
loss_func = losses.TripletMarginLoss(margin=triplet_alpha,
distance=distance)
n_templ = 18
p_pow = torch.FloatTensor(n_templ, 1, 1).uniform_(1, 1)
p_pow = torch.tensor(p_pow, device=device, requires_grad=True)
p_mul = torch.FloatTensor(n_templ, 1, 1).uniform_(1, 1)
p_mul = torch.tensor(p_mul, device=device, requires_grad=True)
opt_mul = optim.SGD([p_mul], lr=alpha_mul)
opt_pow = optim.SGD([p_pow], lr=alpha_pow)
__tensors_mul = []
__tensors_pow = []
__loss = []
name = "{}_{}_{}_{}".format(triplet_alpha, miner, alpha_pow, alpha_mul)
os.mkdir('/home/y.kozhevnikov/rust/{}'.format(name))
for epoch in range(epochs):
epoch_losses = []
for word in tqdm(data):
opt_mul.zero_grad()
opt_pow.zero_grad()
golds, vectors = data[word]
golds = torch.tensor(golds)
vectors = vectors.to(device)
vectors = unite(vectors, p_mul, p_pow)
indices_tuple = mining_func(vectors, golds)
loss = loss_func(vectors, golds, indices_tuple)
epoch_losses.append(loss.item())
loss.backward()
opt_mul.step()
opt_pow.step()
print(p_mul.view(n_templ))
print(p_pow.view(n_templ))
__tensors_mul.append(p_mul.clone().detach())
__tensors_pow.append(p_pow.clone().detach())
epoch_loss = torch.mean(torch.tensor(epoch_losses))
__loss.append(epoch_loss)
print(epoch_loss)
for n, t in enumerate(__tensors_mul):
torch.save(t, "/home/y.kozhevnikov/rust/{}/mul_{}.pt".format(name, n))
for n, t in enumerate(__tensors_pow):
torch.save(t, "/home/y.kozhevnikov/rust/{}/pow_{}.pt".format(name, n))
with open("/home/y.kozhevnikov/rust/{}/loss.txt".format(name), 'w') as f:
for l in __loss:
f.write("{}\n".format(l))
return p_pow, p_mul
numerical_input_dim=num_input_dim,
cat_vocab_sizes=cat_vocab_sizes,
cat_embedding_dim=cat_embedding_dim,
embedding_dim=EMBEDDING_DIM,
)
encoder.to(device)
encoder.train()
optimizer = optim.Adam(encoder.parameters(), lr=LR)
distance = distances.CosineSimilarity()
reducer = reducers.ThresholdReducer(low=0) # basically, returns average
loss_func = losses.TripletMarginLoss(margin=0.4,
distance=distance,
reducer=reducer)
mining_func = miners.TripletMarginMiner(margin=0.4,
distance=distance,
type_of_triplets="semihard")
train_losses = train_ml_model(encoder, NUM_EPOCHS, dataloader,
NUM_OF_SUBSEQUENCES, mining_func, loss_func,
optimizer)
fig, axs = plt.subplots(figsize=(12, 6))
plt.plot(train_losses, label='train')
plt.xlabel('iter')
plt.ylabel('loss')
plt.title("final accuracy: {training}")
plt.savefig(f'plots/ML_{arch}_{EMBEDDING_DIM}_{NUM_OBS}_{NUM_EPOCHS}.png')
SCHEDULER_EPOCHS = 2
LR = 0.002
def __init__(self,
train_dl,
val_dl,
unseen_dl,
model,
optimizer,
scheduler,
criterion,
mining_function,
loss,
savePath='./models/',
device='cuda',
BATCH_SIZE=64):
self.device = device
self.train_dl = train_dl
self.val_dl = val_dl
self.unseen_dl = unseen_dl
self.BATCH_SIZE = BATCH_SIZE
self.model = model.to(self.device)
self.optimizer = optimizer
self.scheduler = scheduler
self.criterion = criterion
self.mining_function = mining_function
self.loss = loss
self.distance = distances.LpDistance(normalize_embeddings=True,
p=2,
power=1)
self.reducer = reducers.ThresholdReducer(low=0)
self.regularizer = regularizers.LpRegularizer(p=2)
if self.mining_function == 'triplet':
self.mining_func = miners.TripletMarginMiner(
margin=0.01,
distance=self.distance,
type_of_triplets="semihard")
elif self.mining_function == 'pair':
self.mining_func = miners.PairMarginMiner(pos_margin=0,
neg_margin=0.2)
if self.loss == 'triplet':
self.loss_function = losses.TripletMarginLoss(
margin=0.01, distance=self.distance, reducer=self.reducer)
elif self.loss == 'contrastive':
self.loss_function = losses.ContrastiveLoss(pos_margin=0,
neg_margin=1.5)
elif self.loss == 'panc':
self.loss_function = losses.ProxyAnchorLoss(
9,
128,
margin=0.01,
alpha=5,
reducer=self.reducer,
weight_regularizer=self.regularizer)
elif self.loss == 'pnca':
self.loss_function = losses.ProxyNCALoss(
9,
128,
softmax_scale=1,
reducer=self.reducer,
weight_regularizer=self.regularizer)
elif self.loss == 'normsoftmax':
self.loss_function = losses.NormalizedSoftmaxLoss(
9,
128,
temperature=0.05,
reducer=self.reducer,
weight_regularizer=self.regularizer)
if self.loss in ['normsoftmax', 'panc', 'pnca']:
self.loss_optimizer = optim.SGD(self.loss_function.parameters(),
lr=0.0001,
momentum=0.9)
self.loss_scheduler = lr_scheduler.ReduceLROnPlateau(
self.loss_optimizer,
'min',
patience=3,
threshold=0.0001,
factor=0.1,
verbose=True)
self.savePath = savePath + 'efigi{}_{}_128'.format(
self.mining_function, self.loss)
# Set optimizers (trying out different learnng rates)
#trunk_optimizer = torch.optim.Adam(trunk.parameters(), lr=0.00001, weight_decay=0.0001)
#embedder_optimizer = torch.optim.Adam(embedder.parameters(), lr=0.0001, weight_decay=0.0001)
trunk_optimizer = torch.optim.Adam(trunk.parameters(),
lr=0.0001,
weight_decay=0.0001)
embedder_optimizer = torch.optim.Adam(embedder.parameters(),
lr=0.001,
weight_decay=0.0001)
# Create the loss, miner, sampler, and package them into dictionaries
# Set the loss function
loss = losses.TripletMarginLoss(margin=0.1)
# Set the mining function
miner = miners.TripletMarginMiner(margin=0.1, type_of_triplets="all")
#miner = miners.MultiSimilarityMiner(epsilon=0.1)
# Set the dataloader sampler
# 4 samples each will be returned -> for us m=2 max
sampler = samplers.MPerClassSampler(train_dataset.targets,
m=2,
length_before_new_iter=len(train_dataset))
#sampler = samplers.FixedSetOfTriplets(train_dataset.targets, len(train_dataset))
# Set other training parameters
batch_size = 64
num_epochs = 1
# Package the above stuff into dictionaries.
models = {"trunk": trunk, "embedder": embedder}
def get_miner():
return miners.TripletMarginMiner(margin=0.2, type_of_triplets='all')
def train_app(cfg):
print(cfg.pretty())
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Set trunk model and replace the softmax layer with an identity function
trunk = torchvision.models.__dict__[cfg.model.model_name](pretrained=cfg.model.pretrained)
#resnet18(pretrained=True)
#trunk = models.alexnet(pretrained=True)
#trunk = models.resnet50(pretrained=True)
#trunk = models.resnet152(pretrained=True)
#trunk = models.wide_resnet50_2(pretrained=True)
#trunk = EfficientNet.from_pretrained('efficientnet-b2')
trunk_output_size = trunk.fc.in_features
trunk.fc = Identity()
trunk = torch.nn.DataParallel(trunk.to(device))
embedder = torch.nn.DataParallel(MLP([trunk_output_size, cfg.embedder.size]).to(device))
classifier = torch.nn.DataParallel(MLP([cfg.embedder.size, cfg.embedder.class_out_size])).to(device)
# Set optimizers
if cfg.optimizer.name == "sdg":
trunk_optimizer = torch.optim.SGD(trunk.parameters(), lr=cfg.optimizer.lr, momentum=cfg.optimizer.momentum, weight_decay=cfg.optimizer.weight_decay)
embedder_optimizer = torch.optim.SGD(embedder.parameters(), lr=cfg.optimizer.lr, momentum=cfg.optimizer.momentum, weight_decay=cfg.optimizer.weight_decay)
classifier_optimizer = torch.optim.SGD(classifier.parameters(), lr=cfg.optimizer.lr, momentum=cfg.optimizer.momentum, weight_decay=cfg.optimizer.weight_decay)
elif cfg.optimizer.name == "rmsprop":
trunk_optimizer = torch.optim.RMSprop(trunk.parameters(), lr=cfg.optimizer.lr, momentum=cfg.optimizer.momentum, weight_decay=cfg.optimizer.weight_decay)
embedder_optimizer = torch.optim.RMSprop(embedder.parameters(), lr=cfg.optimizer.lr, momentum=cfg.optimizer.momentum, weight_decay=cfg.optimizer.weight_decay)
classifier_optimizer = torch.optim.RMSprop(classifier.parameters(), lr=cfg.optimizer.lr, momentum=cfg.optimizer.momentum, weight_decay=cfg.optimizer.weight_decay)
# Set the datasets
data_dir = os.environ["DATASET_FOLDER"]+"/"+cfg.dataset.data_dir
print("Data dir: "+data_dir)
train_dataset, val_dataset, val_samples_dataset = get_datasets(data_dir, cfg, mode=cfg.mode.type)
print("Trainset: ",len(train_dataset), "Testset: ",len(val_dataset), "Samplesset: ",len(val_samples_dataset))
# Set the loss function
if cfg.embedder_loss.name == "margin_loss":
loss = losses.MarginLoss(margin=cfg.embedder_loss.margin,nu=cfg.embedder_loss.nu,beta=cfg.embedder_loss.beta)
if cfg.embedder_loss.name == "triplet_margin":
loss = losses.TripletMarginLoss(margin=cfg.embedder_loss.margin)
if cfg.embedder_loss.name == "multi_similarity":
loss = losses.MultiSimilarityLoss(alpha=cfg.embedder_loss.alpha, beta=cfg.embedder_loss.beta, base=cfg.embedder_loss.base)
# Set the classification loss:
classification_loss = torch.nn.CrossEntropyLoss()
# Set the mining function
if cfg.miner.name == "triplet_margin":
#miner = miners.TripletMarginMiner(margin=0.2)
miner = miners.TripletMarginMiner(margin=cfg.miner.margin)
if cfg.miner.name == "multi_similarity":
miner = miners.MultiSimilarityMiner(epsilon=cfg.miner.epsilon)
#miner = miners.MultiSimilarityMiner(epsilon=0.05)
batch_size = cfg.trainer.batch_size
num_epochs = cfg.trainer.num_epochs
iterations_per_epoch = cfg.trainer.iterations_per_epoch
# Set the dataloader sampler
sampler = samplers.MPerClassSampler(train_dataset.targets, m=4, length_before_new_iter=len(train_dataset))
# Package the above stuff into dictionaries.
models = {"trunk": trunk, "embedder": embedder, "classifier": classifier}
optimizers = {"trunk_optimizer": trunk_optimizer, "embedder_optimizer": embedder_optimizer, "classifier_optimizer": classifier_optimizer}
loss_funcs = {"metric_loss": loss, "classifier_loss": classification_loss}
mining_funcs = {"tuple_miner": miner}
# We can specify loss weights if we want to. This is optional
loss_weights = {"metric_loss": cfg.loss.metric_loss, "classifier_loss": cfg.loss.classifier_loss}
schedulers = {
#"metric_loss_scheduler_by_epoch": torch.optim.lr_scheduler.StepLR(classifier_optimizer, cfg.scheduler.step_size, gamma=cfg.scheduler.gamma),
"embedder_scheduler_by_epoch": torch.optim.lr_scheduler.StepLR(embedder_optimizer, cfg.scheduler.step_size, gamma=cfg.scheduler.gamma),
"classifier_scheduler_by_epoch": torch.optim.lr_scheduler.StepLR(classifier_optimizer, cfg.scheduler.step_size, gamma=cfg.scheduler.gamma),
"trunk_scheduler_by_epoch": torch.optim.lr_scheduler.StepLR(embedder_optimizer, cfg.scheduler.step_size, gamma=cfg.scheduler.gamma),
}
experiment_name = "%s_model_%s_cl_%s_ml_%s_miner_%s_mix_ml_%02.2f_mix_cl_%02.2f_resize_%d_emb_size_%d_class_size_%d_opt_%s_lr_%02.2f_m_%02.2f_wd_%02.2f"%(cfg.dataset.name,
cfg.model.model_name,
"cross_entropy",
cfg.embedder_loss.name,
cfg.miner.name,
cfg.loss.metric_loss,
cfg.loss.classifier_loss,
cfg.transform.transform_resize,
cfg.embedder.size,
cfg.embedder.class_out_size,
cfg.optimizer.name,
cfg.optimizer.lr,
cfg.optimizer.momentum,
cfg.optimizer.weight_decay)
record_keeper, _, _ = logging_presets.get_record_keeper("logs/%s"%(experiment_name), "tensorboard/%s"%(experiment_name))
hooks = logging_presets.get_hook_container(record_keeper)
dataset_dict = {"samples": val_samples_dataset, "val": val_dataset}
model_folder = "example_saved_models/%s/"%(experiment_name)
# Create the tester
tester = OneShotTester(
end_of_testing_hook=hooks.end_of_testing_hook,
#size_of_tsne=20
)
#tester.embedding_filename=data_dir+"/embeddings_pretrained_triplet_loss_multi_similarity_miner.pkl"
tester.embedding_filename=data_dir+"/"+experiment_name+".pkl"
end_of_epoch_hook = hooks.end_of_epoch_hook(tester, dataset_dict, model_folder)
trainer = trainers.TrainWithClassifier(models,
optimizers,
batch_size,
loss_funcs,
mining_funcs,
train_dataset,
sampler=sampler,
lr_schedulers=schedulers,
dataloader_num_workers = cfg.trainer.batch_size,
loss_weights=loss_weights,
end_of_iteration_hook=hooks.end_of_iteration_hook,
end_of_epoch_hook=end_of_epoch_hook
)
trainer.train(num_epochs=num_epochs)
tester = OneShotTester()