def train_model(triplet_train_loader, triplet_val_loader, test_fun, margin,
net_output, model_directory):
target_embedding_net = TargetEmbeddingNet(300, net_output)
triple_model = TripletNet(target_embedding_net, target_embedding_net)
if cuda:
triple_model.cuda()
loss_fn = TripletLoss(margin)
lr = 1e-4
optimizer = optim.Adam(triple_model.parameters(), lr=lr)
scheduler = lr_scheduler.StepLR(optimizer, 1000, gamma=0.5, last_epoch=-1)
n_epochs = 20
log_interval = 100
train_losses, val_losses, metrices = fit(triplet_train_loader,
triplet_val_loader,
triple_model,
loss_fn,
optimizer,
scheduler,
n_epochs,
cuda,
log_interval,
callback_test=test_fun,
keep_checkpoint_max=10,
model_dir=model_directory)
return metrices
def __init__(self, gamma, margin):
self.exp_comment = "ripletloss"
self.gamma = gamma
self.embedding_net = nets.Deep4Net_origin(13, 64, 501, batch_norm=True)
self.clf_net = ConvClfNet(self.embedding_net)
self.model = TripletNet(self.clf_net)
self.optimizer = optim.Adam(self.model.parameters())
# self.optimizer = optim.SGD(self.model.parameters(), lr=0.1, momentum=0.9, weight_decay=0.0005)
self.milestones = [20, 40, 60, 80]
self.scheduler = lr_scheduler.MultiStepLR(self.optimizer,
milestones=self.milestones,
gamma=0.1)
self.loss_fn = TripletLoss(margin, gamma)
def __init__(self, gamma, margin):
self.exp_comment = "ripletloss"
self.gamma = gamma
self.embedding_net = nets.Deep4Net_origin(2, 62, 1000)
self.clf_net = ConvClfNet(self.embedding_net)
self.model = TripletNet(self.clf_net)
# self.optimizer = optim.Adam(self.model.parameters())
self.optimizer = optim.AdamW(self.model.parameters(),
lr=0.01,
weight_decay=0.5 * 0.001)
# self.optimizer = optim.SGD(self.model.parameters(), lr=0.001, momentum=0.9, weight_decay=0.0005)
self.milestones = [20, 30, 40, 50]
# self.scheduler = lr_scheduler.MultiStepLR(self.optimizer, milestones=self.milestones, gamma=0.1)
self.scheduler = lr_scheduler.CosineAnnealingLR(self.optimizer,
T_max=100)
self.loss_fn = TripletLoss(margin, gamma)
def binary_output(dataloader):
embedding_net = EmbeddingResnet()
net = TripletNet(embedding_net)
#net = SiameseNet(embedding_net)
#net = ClassificationNet(embedding_net, 100)
net.load_state_dict(torch.load('./model/%s' % args.pretrained))
use_cuda = torch.cuda.is_available()
if use_cuda:
net.cuda()
full_batch_output = torch.cuda.FloatTensor()
full_batch_label = torch.cuda.LongTensor()
net.eval()
for batch_idx, (inputs, targets) in enumerate(dataloader):
if use_cuda:
inputs, targets = inputs.cuda(), targets.cuda()
inputs, targets = Variable(inputs, volatile=True), Variable(targets)
outputs = net.get_embedding(inputs).data
full_batch_output = torch.cat((full_batch_output, outputs.data), 0)
full_batch_label = torch.cat((full_batch_label, targets.data), 0)
return torch.round(full_batch_output), full_batch_label
def load_model(saved_model, grid_type, model_params):
#loading trained model
embedding_net = EmbeddingNet(
in_channels=grid_input_channels_dict[grid_type], **model_params)
model = TripletNet(embedding_net)
cuda = torch.cuda.is_available()
# cuda = False
if cuda:
model.cuda()
map_location = None
else:
map_location = torch.device('cpu')
state_dict = torch.load(saved_model, map_location=map_location)
new_state_dict = OrderedDict()
for k, v in state_dict.items():
name = k.replace('module.', '') # removing ‘moldule.’ from key
new_state_dict[name] = v
model.load_state_dict(new_state_dict)
model.eval()
return model
def main():
import torch
from torch.optim import lr_scheduler
import torch.optim as optim
from torch.autograd import Variable
from trainer import fit
import numpy as np
cuda = torch.cuda.is_available()
# Training settings
parser = argparse.ArgumentParser(description='PyTorch MNIST Example')
parser.add_argument('--batch-size', type=int, default=100, metavar='N',
help='input batch size for training (default: 64)')
parser.add_argument('--test-batch-size', type=int, default=100, metavar='N',
help='input batch size for testing (default: 1000)')
parser.add_argument('--epochs', type=int, default=100, metavar='N',
help='number of epochs to train (default: 10)')
parser.add_argument('--lr', type=float, default=0.001, metavar='LR',
help='learning rate (default: 0.01)')
parser.add_argument('--momentum', type=float, default=0.5, metavar='M',
help='SGD momentum (default: 0.5)')
parser.add_argument('--no-cuda', action='store_true', default=False,
help='disables CUDA training')
parser.add_argument('--seed', type=int, default=1, metavar='S',
help='random seed (default: 1)')
parser.add_argument('--log-interval', type=int, default=10, metavar='N',
help='how many batches to wait before logging training status')
parser.add_argument('--save-model', action='store_true', default=True,
help='For Saving the current Model')
args = parser.parse_args()
use_cuda = not args.no_cuda and torch.cuda.is_available()
torch.manual_seed(args.seed)
np.random.seed(args.seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
device = torch.device("cuda" if use_cuda else "cpu")
kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
from datetime import datetime
import os
loging = False
ismultitask = False
loso = False
if (args.save_model):
model_save_path = 'model/triplet/'
if not os.path.isdir(model_save_path):
os.makedirs(model_save_path)
if loging:
fname = model_save_path + datetime.today().strftime("%m_%d_%H_%M") + ".txt"
f = open(fname, 'w')
x_data, y_data = load_smt()
y_subj = np.zeros([108, 200])
for i in range(108):
y_subj[i, :] = i * 2
y_subj = y_data.reshape(108, 200) + y_subj
y_subj = y_subj.reshape(21600)
# nonbciilli = np.s_[0,1,2,4,5,8,16,17,18,20,21,27,28,29,30,32,35,36,38,42,43,44,51]
valtype = 'sess'
if valtype == 'loso':
pass
elif valtype == 'sess':
from networks import EmbeddingDeep4CNN, TripletNet, FineShallowCNN, EmbeddingDeepCNN, QuintupletNet, EmbeddingShallowCNN
from losses import TripletLoss_dev2
# make model for metric learning
margin = 1
embedding_net = EmbeddingShallowCNN()
# clf_net = nn.Sequential(EmbeddingDeep4CNN(),nn.Linear(1000,2),nn.Dropout(p=1),nn.LogSoftmax(dim=1))
print(embedding_net)
model = TripletNet(embedding_net)
if cuda:
model.cuda()
loss_fn = TripletLoss_dev2(margin).cuda()
n_epochs =1
log_interval = 10
load_model_path = None#'C:\\Users\dk\PycharmProjects\giga_cnn\구모델\\clf_83_8.pt'#'clf_29.pt' #'triplet_mg26.pt'#'clf_triplet2_5.pt' #'triplet_31.pt'
model.fc = nn.Sequential(nn.Linear(1000, 2), nn.Dropout(p=0.5))
if load_model_path is not None:
embedding_net.load_state_dict(torch.load(load_model_path))
# For classification
dataset_train = GigaDataset(x=x_data, y=y_data, valtype=valtype, istrain=True, sess=1)
train_loader = torch.utils.data.DataLoader(dataset_train, batch_size=args.batch_size, shuffle=True,
**kwargs)
dataset_test = GigaDataset(x=x_data, y=y_data, valtype=valtype, istrain=False, sess=2, subj=-1)
test_loader = torch.utils.data.DataLoader(dataset_test, batch_size=args.batch_size, shuffle=False,
**kwargs)
#make model for classification
newmodel = nn.Sequential(model.embedding_net,
nn.Linear(1000, 2),
nn.Dropout(p=0.5),
nn.LogSoftmax(dim=1)
).to(device)
print(newmodel)
newmodel.to(device)
optimizer = optim.SGD(newmodel.parameters(), lr=0.001, momentum=0.9)
# optimizer = optim.Adam(newmodel.parameters())
for epoch in range(0):
train(args, newmodel, device, train_loader, optimizer, epoch)
j_loss, j_score = eval(args, newmodel, device, test_loader)
# For embedding
triplet_dataset_train = TripletGiga(x=x_data, y=y_data,valtype=valtype, istrain=True, sess=1)
triplet_train_loader = torch.utils.data.DataLoader(triplet_dataset_train, batch_size=args.batch_size, shuffle=True, **kwargs)
triplet_dataset_test = TripletGiga(x=x_data, y=y_data,valtype=valtype, istrain=False, sess=2, subj=-1)
triplet_test_loader = torch.utils.data.DataLoader(triplet_dataset_test, batch_size=args.batch_size, shuffle=False, **kwargs)
optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9)
scheduler = lr_scheduler.StepLR(optimizer, 8, gamma=1, last_epoch=-1)
from sklearn.pipeline import Pipeline
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.model_selection import ShuffleSplit, cross_val_score
lda = LinearDiscriminantAnalysis()
Testmodel = nn.Sequential(model.embedding_net,
model.fc,
nn.LogSoftmax(dim=1)).to(device)
# tempEmbeddingNet = nn.Sequential(model.embedding_net,
# nn.Linear(1000,1000),
# nn.Sigmoid())
# model = TripletNet(embedding_net)
print(model)
for temp in range(1, 30): # 10epoch마다 세이브
fit(triplet_train_loader, triplet_test_loader, model, loss_fn, optimizer, scheduler, n_epochs, cuda,
log_interval)
j_loss, j_score = eval(args, Testmodel, device, train_loader)
j_loss, j_score = eval(args, Testmodel, device, test_loader)
torch.save(model.state_dict(), 'clf_' + str(temp) + '.pt')
torch.save(model.state_dict(), 'shallowDG_150epoch_82acc' + str(temp) + '.pt')
#for visualization
dataset_train_subj = GigaDataset(x=x_data, y=y_subj, valtype=valtype, istrain=True, sess=1)
train_loader_subj = torch.utils.data.DataLoader(dataset_train_subj, batch_size=args.batch_size, shuffle=True,
**kwargs)
dataset_test_subj = GigaDataset(x=x_data, y=y_subj, valtype=valtype, istrain=False, sess=2, subj=-1)
test_loader_subj = torch.utils.data.DataLoader(dataset_test_subj, batch_size=args.batch_size, shuffle=False,
**kwargs)
# train_embeddings_tl, train_labels_tl = extract_features(train_loader_subj.dataset, model.embedding_net.convnet,0,100)
# val_embeddings_tl, val_labels_tl = extract_embeddings(test_loader_subj, model.embedding_net, 1000)
train_embeddings_tl, train_labels_tl = extract_embeddings(train_loader_subj, model.embedding_net,1000)
val_embeddings_tl, val_labels_tl = extract_embeddings(test_loader_subj, model.embedding_net,1000)
#
# train_embeddings_tl, train_labels_tl = extract_embeddings(train_loader_subj, model.embedding_net.convnet[0], 1000)
# val_embeddings_tl, val_labels_tl = extract_embeddings(test_loader_subj, model.embedding_net, 1000)
# = train_labels_tl-train_labels_tl%2
# from torchvision import datasets, models, transforms
# temp = model.embedding_net.children()
# newmodel = torch.nn.Sequential(*(list(model.embedding_net.children())[:]))
# for param in model.embedding_net.parameters():
# param.requires_grad = True
#newembedding_net = torch.nn.Sequential(*(list(model.embedding_net.children())[:]))
#
from sklearn.manifold import TSNE
tsne = TSNE(n_components=2,perplexity=30)
#features = np.concatenate([train_embeddings_tl,val_embeddings_tl])
#val_labels_tl = val_labels_tl+2
#labels = np.concatenate([train_labels_tl,val_labels_tl])
train_tsne = tsne.fit_transform(train_embeddings_tl[0:2000])
# plot_embeddings(train_tsne,train_labels_tl[0:1000])
plot_features(train_tsne,train_labels_tl[0:2000])
plot_features3d(train_tsne,train_labels_tl[0:1000]%2)
val_tsne = tsne.fit_transform(val_embeddings_tl)
plot_embeddings(val_tsne, (val_labels_tl-108)-(val_labels_tl-108)%2)
for param in model.embedding_net.parameters():
param.requires_grad = True
#embedding_net2 = EmbeddingDeep4CNN()
newmodel = nn.Sequential(model.embedding_net,
nn.Linear(1000, 2),
nn.Dropout(p=0.5),
nn.LogSoftmax(dim=1),
).to(device)
print(newmodel)
#newmodel.fc_lr = nn.Linear(1000,2)
newmodel.to(device)
optimizer = optim.SGD(newmodel.parameters(), lr=0.001, momentum=0.9)
#optimizer = optim.Adam(newmodel.parameters())
for epoch in range(1, 100):
train(args, newmodel, device, train_loader, optimizer, epoch)
j_loss, j_score = eval(args, newmodel, device, test_loader)
if args.save_model:
torch.save(newmodel.state_dict(),'clf_83_8.pt')
newmodel.load_state_dict(torch.load(load_model_path))
# Visualize feature maps
activation = {}
def get_activation(name):
def hook(model, input, output):
activation[name] = output.detach()
return hook
handle = model.embedding_net.convnet[0].register_forward_hook(get_activation('fc'))
handle.remove()
model.embedding_net.convnet[0]._forward_hooks.clear()
train_loader.dataset
with torch.no_grad():
model.eval()
# num_ftrs = model.embedding_net.fc.out_features
embeddings = np.zeros((len(train_loader.dataset), num_ftrs))
labels = np.zeros(len(train_loader.dataset))
k = 0
for images, target in train_loader:
if cuda:
images = images.cuda()
embeddings[k:k + len(images)] = model.get_embedding(images).data.cpu().numpy()
labels[k:k + len(images)] = target.numpy()
k += len(images)
features = SaveFeatures(model.embedding_net.convnet[0])
temp = features.features.data.cpu().numpy()
del features.features
torch.cuda.empty_cache()
for images, target in train_loader:
if cuda:
images = images.cuda()
output = model.embedding_net(images)
activation = []
def get_activation():
def hook(model, input, output):
activation.append(output)
print(output)
return hook
act = activation['conv1'].squeeze()
fig, axarr = plt.subplots(act.size(0))
for idx in range(act.size(0)):
axarr[idx].imshow(act[idx])
actmap = []
def printnorm(self, input, output):
# input is a tuple of packed inputs
# output is a Tensor. output.data is the Tensor we are interested
print('Inside ' + self.__class__.__name__ + ' forward')
print('')
print('input: ', type(input))
print('input[0]: ', type(input[0]))
print('output: ', type(output))
print('')
print('input size:', input[0].size())
print('output size:', output.data.size())
print('output norm:', output.data.norm())
return output.data
model.embedding_net.convnet[0].register_forward_hook(printnorm)
out = model(input)
fig, axarr = plt.subplots(10)
for idx in range(10):
axarr[idx].imshow(temp[idx,1,:,:])
def find_best_model(top_k, combination_of, num_negs, how_to_choose_k,
log_file):
###### Loading data ##############
ranking_model = RankedTargetModel(
'../../data/idebate/ranking_model.pickle')
train_data = json.load(open('../../data/idebate/train_scored.json', 'r'))
val_data = json.load(open('../../data/idebate/dev_scored.json', 'r'))
train_triplet_samples, train_p_avg, new_target_space = prepare_data(
train_data,
target_space_file=target_space_path,
embedding_method=embedding_method,
triple_sampling=True,
top_k=top_k,
how_to_choose_k=how_to_choose_k,
combination_of=combination_of,
ranking_model=ranking_model,
num_negs=num_negs)
x0, x1, x2 = zip(*train_triplet_samples)
x0 = np.array(x0)
x1 = np.array(x1)
x2 = np.array(x2)
train_triplet_ds = TripletDataset(x0, x1, x2)
val_triplet_samples, val_p_avg, _ = prepare_data(
val_data,
target_space_file=target_space_path,
embedding_method=embedding_method,
triple_sampling=True,
top_k=top_k,
how_to_choose_k=how_to_choose_k,
combination_of=combination_of,
ranking_model=ranking_model,
num_negs=num_negs)
x00, x11, x22 = zip(*val_triplet_samples)
x00 = np.array(x00)
x11 = np.array(x11)
x22 = np.array(x22)
val_triplet_ds = TripletDataset(x00, x11, x22)
triplet_train_loader = torch.utils.data.DataLoader(train_triplet_ds,
batch_size=batch_size,
shuffle=True,
**kwargs)
triplet_test_loader = torch.utils.data.DataLoader(val_triplet_ds,
batch_size=batch_size,
shuffle=False,
**kwargs)
# Prepare the test function
target_space = pickle.load(open(target_space_path, 'rb'))
test_data = json.load(open('../../data/idebate/dev_scored.json', 'r'))
test_data = prepare_test_data(test_data,
embedding_method,
top_k=top_k,
how_to_choose_k=how_to_choose_k,
ranking_model=ranking_model)
test_fun = lambda model: test_model(model, test_data, target_space)
# Build the model
margin = 0.2
target_embedding_net = TargetEmbeddingNet(300)
triple_model = TripletNet(target_embedding_net, target_embedding_net)
if cuda:
triple_model.cuda()
loss_fn = TripletLoss(margin)
lr = 1e-4
optimizer = optim.Adam(triple_model.parameters(), lr=lr)
scheduler = lr_scheduler.StepLR(optimizer, 1000, gamma=0.5, last_epoch=-1)
n_epochs = 50
log_interval = 100
train_losses, val_losses, metrices = fit(triplet_train_loader,
triplet_test_loader,
triple_model,
loss_fn,
optimizer,
scheduler,
n_epochs,
cuda,
log_interval,
callback_test=test_fun)
with open(log_file, 'w') as f:
json.dump(metrices, f)
def main():
import torch
from torch.optim import lr_scheduler
import torch.optim as optim
from torch.autograd import Variable
from trainer import fit
import numpy as np
cuda = torch.cuda.is_available()
# Training settings
parser = argparse.ArgumentParser(
description='cross subject domain adaptation')
parser.add_argument('--batch-size',
type=int,
default=100,
metavar='N',
help='input batch size for training (default: 64)')
parser.add_argument('--test-batch-size',
type=int,
default=100,
metavar='N',
help='input batch size for testing (default: 1000)')
parser.add_argument('--epochs',
type=int,
default=100,
metavar='N',
help='number of epochs to train (default: 10)')
parser.add_argument('--lr',
type=float,
default=0.001,
metavar='LR',
help='learning rate (default: 0.01)')
parser.add_argument('--momentum',
type=float,
default=0.5,
metavar='M',
help='SGD momentum (default: 0.5)')
parser.add_argument('--no-cuda',
action='store_true',
default=False,
help='disables CUDA training')
parser.add_argument(
'--log-interval',
type=int,
default=10,
metavar='N',
help='how many batches to wait before logging training status')
parser.add_argument('--save-model',
action='store_true',
default=True,
help='For Saving the current Model')
# Writer will output to ./runs/ directory by default
fold_idx = 5
gamma = 1
DAsetting = False
args = parser.parse_args()
args.seed = 0
args.use_tensorboard = True
use_cuda = not args.no_cuda and torch.cuda.is_available()
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
torch.cuda.manual_seed(args.seed)
np.random.seed(args.seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
device = torch.device("cuda" if use_cuda else "cpu")
#kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
from datetime import datetime
import os
loging = False
x_data, y_data = load_smt()
#get subject number
y_subj = np.zeros([108, 200])
for i in range(108):
y_subj[i, :] = i * 2
y_subj = y_data.reshape(108, 200) + y_subj
y_subj = y_subj.reshape(21600)
# For classification data
valtype = 'subj'
# #subj - 0-27 train
# train_subj1 = np.r_[0:27]
# train_subj2 = np.r_[0:27]+54
#
# test_subj = np.r_[27:54,54+27:108]
# For Domain adaptation setting
if DAsetting:
train_subj1 = np.r_[27:54]
train_subj2 = np.r_[27:54] + 54
test_subj = np.r_[0:27, 54 + 0:54 + 27]
trial_s = (0, 200)
trial_t = (0, 200)
trial_val = (0, 200)
dataset_train1 = GigaDataset(x=x_data,
y=y_data,
valtype=valtype,
istrain=True,
subj=train_subj1,
trial=trial_s)
dataset_train2 = GigaDataset(x=x_data,
y=y_data,
valtype=valtype,
istrain=True,
subj=train_subj2,
trial=trial_t)
dataset_train = dataset_train1.__add__(dataset_train2)
dataset_test = GigaDataset(x=x_data,
y=y_data,
valtype=valtype,
istrain=False,
subj=test_subj,
trial=trial_val)
triplet_dataset_train1 = TripletGiga(x=x_data,
y=y_data,
valtype=valtype,
istrain=True,
subj=train_subj1,
trial=trial_s)
triplet_dataset_train2 = TripletGiga(x=x_data,
y=y_data,
valtype=valtype,
istrain=True,
subj=train_subj2,
trial=trial_t)
triplet_dataset_train = triplet_dataset_train1.__add__(
triplet_dataset_train2)
triplet_dataset_test = TripletGiga(x=x_data,
y=y_data,
valtype=valtype,
istrain=False,
subj=test_subj,
trial=trial_val)
else: #DG setting
test_subj = np.r_[fold_idx * 9:fold_idx * 9 + 9,
fold_idx + 54:fold_idx + 9 + 54]
train_subj = np.setxor1d(np.r_[0:108], test_subj)
trial_train = (0, 200)
trial_val = (0, 200)
dataset_train = GigaDataset(x=x_data,
y=y_data,
valtype=valtype,
istrain=True,
subj=train_subj,
trial=trial_train)
dataset_test = GigaDataset(x=x_data,
y=y_data,
valtype=valtype,
istrain=False,
subj=test_subj,
trial=trial_val)
triplet_dataset_train = TripletGiga(x=x_data,
y=y_data,
valtype=valtype,
istrain=True,
subj=train_subj,
trial=trial_train)
triplet_dataset_test = TripletGiga(x=x_data,
y=y_data,
valtype=valtype,
istrain=False,
subj=test_subj,
trial=trial_val)
train_loader = torch.utils.data.DataLoader(dataset_train,
batch_size=args.batch_size,
shuffle=True)
test_loader = torch.utils.data.DataLoader(dataset_test,
batch_size=args.batch_size,
shuffle=False)
triplet_train_loader = torch.utils.data.DataLoader(
triplet_dataset_train, batch_size=args.batch_size, shuffle=True)
triplet_test_loader = torch.utils.data.DataLoader(
triplet_dataset_test, batch_size=args.batch_size, shuffle=False)
###################################################################################################################
# make model for metric learning
from networks import Deep4Net, EmbeddingDeep4CNN, EmbeddingDeep4CNN_bn, TripletNet, FineShallowCNN, EmbeddingDeepCNN, QuintupletNet, EmbeddingShallowCNN
from losses import TripletLoss_dev2
margin = 1.0
embedding_net = Deep4Net()
print(embedding_net)
model = TripletNet(embedding_net)
if cuda:
model.cuda()
loss_fn = TripletLoss_dev2(margin, gamma).cuda()
n_epochs = 1
log_interval = 10
# optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9)
# scheduler = lr_scheduler.StepLR(optimizer, 5, gamma=0.5, last_epoch=-1)
optimizer = optim.Adam(model.parameters(), lr=0.01)
scheduler = lr_scheduler.StepLR(optimizer, 10, gamma=1, last_epoch=-1)
#model for validation
evalmodel = nn.Sequential(model.embedding_net, model.fc,
nn.LogSoftmax(dim=1)).to(device)
print('____________DANet____________')
print(model)
comment = 'fold_' + str(fold_idx) + '_g_' + str(gamma)
#save someting
if (args.save_model):
model_save_path = 'model/Deep4Net_(test0604)/' + comment + '/'
if not os.path.isdir(model_save_path):
os.makedirs(model_save_path)
if loging:
fname = model_save_path + datetime.today().strftime(
"%m_%d_%H_%M") + ".txt"
f = open(fname, 'w')
if args.use_tensorboard:
writer = SummaryWriter(comment=comment)
# load_model_path = model_save_path+'danet_0.7_49.pt' #'C:\\Users\dk\PycharmProjects\giga_cnn\구모델\\clf_83_8.pt'#'clf_29.pt' #'triplet_mg26.pt'#'clf_triplet2_5.pt' #'triplet_31.pt'
load_model_path = None
if load_model_path is not None:
model.load_state_dict(torch.load(load_model_path))
for temp in range(1, 50):
fit(triplet_train_loader, triplet_test_loader, model, loss_fn,
optimizer, scheduler, n_epochs, cuda, log_interval)
print(temp)
train_loss, train_score = eval(args, evalmodel, device, train_loader)
eval_loss, eval_score = eval(args, evalmodel, device, test_loader)
if args.use_tensorboard:
writer.add_scalar('Train/Loss', np.mean(train_loss) / 100, temp)
writer.add_scalar('Train/Acc', np.mean(train_score) / 100, temp)
writer.add_scalar('Eval/Loss', np.mean(eval_loss) / 100, temp)
writer.add_scalar('Eval/Acc', np.mean(eval_score) / 100, temp)
writer.close()
torch.save(
model.state_dict(),
model_save_path + 'danet_' + str(gamma) + '_' + str(temp) + '.pt')
def main():
import torch
from torch.optim import lr_scheduler
import torch.optim as optim
from torch.autograd import Variable
from trainer import fit
import numpy as np
cuda = torch.cuda.is_available()
# Training settings
parser = argparse.ArgumentParser(
description='cross subject domain adaptation')
parser.add_argument('--batch-size',
type=int,
default=100,
metavar='N',
help='input batch size for training (default: 64)')
parser.add_argument('--test-batch-size',
type=int,
default=100,
metavar='N',
help='input batch size for testing (default: 1000)')
parser.add_argument('--epochs',
type=int,
default=100,
metavar='N',
help='number of epochs to train (default: 10)')
parser.add_argument('--lr',
type=float,
default=0.001,
metavar='LR',
help='learning rate (default: 0.01)')
parser.add_argument('--momentum',
type=float,
default=0.5,
metavar='M',
help='SGD momentum (default: 0.5)')
parser.add_argument('--no-cuda',
action='store_true',
default=False,
help='disables CUDA training')
parser.add_argument(
'--log-interval',
type=int,
default=10,
metavar='N',
help='how many batches to wait before logging training status')
parser.add_argument('--save-model',
action='store_true',
default=True,
help='For Saving the current Model')
# Writer will output to ./runs/ directory by default
writer = SummaryWriter()
args = parser.parse_args()
args.seed = 0
use_cuda = not args.no_cuda and torch.cuda.is_available()
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
torch.cuda.manual_seed(args.seed)
np.random.seed(args.seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
device = torch.device("cuda" if use_cuda else "cpu")
#kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
from datetime import datetime
import os
loging = False
if (args.save_model):
model_save_path = 'model/DANet(Deep4Net100)_test/'
if not os.path.isdir(model_save_path):
os.makedirs(model_save_path)
if loging:
fname = model_save_path + datetime.today().strftime(
"%m_%d_%H_%M") + ".txt"
f = open(fname, 'w')
x_data, y_data = load_smt()
#get subject number
y_subj = np.zeros([108, 200])
for i in range(108):
y_subj[i, :] = i * 2
y_subj = y_data.reshape(108, 200) + y_subj
y_subj = y_subj.reshape(21600)
# nonbciilli = np.s_[0,1,2,4,5,8,16,17,18,20,21,27,28,29,30,32,35,36,38,42,43,44,51]
valtype = 'sess'
if valtype == 'loso':
pass
elif valtype == 'sess':
from networks import EmbeddingDeep4CNN, TripletNet, FineShallowCNN, EmbeddingDeepCNN, QuintupletNet, EmbeddingShallowCNN
from losses import TripletLoss_dev2
# make model for metric learning
margin = 1.0
gamma = 0.7
embedding_net = EmbeddingDeep4CNN()
print(embedding_net)
model = TripletNet(embedding_net)
#model.fc = nn.Linear(embedding_net.num_hidden,2)
if cuda:
model.cuda()
loss_fn = TripletLoss_dev2(margin, gamma).cuda()
n_epochs = 1
log_interval = 10
#load_model_path = model_save_path+'dgnet1.027.pt' #'C:\\Users\dk\PycharmProjects\giga_cnn\구모델\\clf_83_8.pt'#'clf_29.pt' #'triplet_mg26.pt'#'clf_triplet2_5.pt' #'triplet_31.pt'
load_model_path = None
if load_model_path is not None:
model.load_state_dict(torch.load(load_model_path))
# For classification data
dataset_train = GigaDataset(x=x_data,
y=y_data,
valtype=valtype,
istrain=True,
sess=1)
train_loader = torch.utils.data.DataLoader(
dataset_train,
batch_size=args.batch_size,
shuffle=True,
)
dataset_test = GigaDataset(x=x_data,
y=y_data,
valtype=valtype,
istrain=False,
sess=2,
subj=-1)
test_loader = torch.utils.data.DataLoader(
dataset_test,
batch_size=args.batch_size,
shuffle=False,
)
# For Domain adaptation
triplet_dataset_train = TripletGiga(x=x_data,
y=y_data,
valtype=valtype,
istrain=True,
sess=1)
triplet_train_loader = torch.utils.data.DataLoader(
triplet_dataset_train, batch_size=args.batch_size, shuffle=True)
triplet_dataset_test = TripletGiga(x=x_data,
y=y_data,
valtype=valtype,
istrain=False,
sess=2,
subj=-1)
triplet_test_loader = torch.utils.data.DataLoader(
triplet_dataset_test, batch_size=args.batch_size, shuffle=False)
optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9)
scheduler = lr_scheduler.StepLR(optimizer,
10,
gamma=0.5,
last_epoch=-1)
#model for validation
evalmodel = nn.Sequential(model.embedding_net, model.fc,
nn.LogSoftmax(dim=1)).to(device)
print('____________DANet____________')
print(model)
#
# from torch.utils.tensorboard import SummaryWriter
# writer = SummaryWriter()
# images, labels = next(iter(train_loader))
# import torchvision
# #grid = torchvision.utils.make_grid(images)
# writer.add_images('images',images)
#
# writer.add_embedding(metadata=train_embeddings_tl)
# # writer.add_embedding(metadata = val_embeddings_tl)
# writer.close()
for temp in range(1, 50):
fit(triplet_train_loader, triplet_test_loader, model, loss_fn,
optimizer, scheduler, n_epochs, cuda, log_interval)
train_loss, j_score = eval(args, evalmodel, device, train_loader)
eval_loss, j_score = eval(args, evalmodel, device, test_loader)
writer.add_scalar('Train/Loss', np.mean(train_loss) / 100, temp)
writer.add_scalar('Eval/Loss', np.mean(eval_loss) / 100, temp)
writer.close()
np.mean(train_loss)
torch.save(
model.state_dict(), model_save_path + 'dgnet_' + str(gamma) +
'_' + str(temp) + '.pt')
#for visualization
dataset_train_subj = GigaDataset(x=x_data,
y=y_subj,
valtype='subj',
istrain=True,
sess=1,
subj=np.r_[0:10])
train_loader_subj = torch.utils.data.DataLoader(
dataset_train_subj,
batch_size=args.batch_size,
shuffle=True,
)
dataset_test_subj = GigaDataset(x=x_data,
y=y_subj,
valtype='sess',
istrain=False,
sess=2,
subj=-1)
test_loader_subj = torch.utils.data.DataLoader(
dataset_test_subj,
batch_size=args.batch_size,
shuffle=False,
)
# train_embeddings_tl, train_labels_tl = extract_features(train_loader_subj.dataset, model.embedding_net.convnet,0,100)
# val_embeddings_tl, val_labels_tl = extract_embeddings(test_loader_subj, model.embedding_net, 1000)
train_embeddings_tl, train_labels_tl = extract_embeddings(
train_loader_subj, model.embedding_net,
model.embedding_net.num_hidden)
val_embeddings_tl, val_labels_tl = extract_embeddings(
test_loader_subj, model.embedding_net,
model.embedding_net.num_hidden)
#
# train_embeddings_tl, train_labels_tl = extract_embeddings(train_loader_subj, model.embedding_net.convnet[0], 1000)
# val_embeddings_tl, val_labels_tl = extract_embeddings(test_loader_subj, model.embedding_net, 1000)
# = train_labels_tl-train_labels_tl%2
# from torchvision import datasets, models, transforms
# temp = model.embedding_net.children()
# newmodel = torch.nn.Sequential(*(list(model.embedding_net.children())[:]))
# for param in model.embedding_net.parameters():
# param.requires_grad = True
#newembedding_net = torch.nn.Sequential(*(list(model.embedding_net.children())[:]))
#
from sklearn.manifold import TSNE
tsne = TSNE(n_components=2, perplexity=30)
#features = np.concatenate([train_embeddings_tl,val_embeddings_tl])
#val_labels_tl = val_labels_tl+2
#labels = np.concatenate([train_labels_tl,val_labels_tl])
train_tsne = tsne.fit_transform(train_embeddings_tl)
plot_features(train_tsne, train_labels_tl)
plot_features(train_tsne, train_labels_tl % 2)
for i in range(0, 10):
plot_features(train_tsne[i * 200:(i + 1) * 200],
train_labels_tl[i * 200:(i + 1) * 200])
plot_features3d(train_tsne, train_labels_tl)
val_tsne = tsne.fit_transform(val_embeddings_tl)
plot_features(val_tsne, val_labels_tl - 108)
plot_features(val_tsne, val_labels_tl % 2)
plot_features3d(val_tsne, val_labels_tl % 2)
plot_embeddings(val_tsne,
(val_labels_tl - 108) - (val_labels_tl - 108) % 2)
for param in model.embedding_net.parameters():
param.requires_grad = True
#embedding_net2 = EmbeddingDeep4CNN()
newmodel = nn.Sequential(
model.embedding_net,
nn.Linear(1000, 2),
nn.Dropout(p=0.5),
nn.LogSoftmax(dim=1),
).to(device)
print(newmodel)
#newmodel.fc_lr = nn.Linear(1000,2)
newmodel.to(device)
optimizer = optim.SGD(newmodel.parameters(), lr=0.001, momentum=0.9)
#optimizer = optim.Adam(newmodel.parameters())
for epoch in range(1, 100):
train(args, newmodel, device, train_loader, optimizer, epoch)
j_loss, j_score = eval(args, newmodel, device, test_loader)
if args.save_model:
torch.save(newmodel.state_dict(), 'clf_83_8.pt')
newmodel.load_state_dict(torch.load(load_model_path))
def grid_search_over_k(k_range, how_to_choose_k, test_file_path, output_path,
test_scenario):
target_space_path = '../target_spaces/fasttext_embedded_targets_space.pkl'
target_space = pickle.load(open(target_space_path, 'rb'))
train_data = json.load(open('../../data/idebate/train_scored.json', 'r'))
val_data = json.load(open('../../data/idebate/dev_scored.json', 'r'))
test_data = json.load(open(test_file_path, 'r'))
with open(output_path, 'w') as f:
f.write(
'k, empty_space, model acc @20, model acc @40, model acc @50, model bleu, model meteor, baseline acc @20, baseline acc @40, baseline acc @50, baseline bleu, baseline meteor\n'
)
results = []
for k in k_range:
combination_of = k
train_triplet_ds, train_p_avg, _ = prepare_data(
train_data,
target_space_file=target_space_path,
embedding_method=embedding_method,
triple_sampling=True,
top_k=k,
how_to_choose_k=how_to_choose_k,
combination_of=combination_of,
ranking_model=ranking_model)
val_triplet_ds, val_p_avg, _ = prepare_data(
val_data,
target_space_file=target_space_path,
embedding_method=embedding_method,
triple_sampling=True,
top_k=k,
how_to_choose_k=how_to_choose_k,
combination_of=combination_of,
ranking_model=ranking_model)
prepared_test_data = prepare_test_data(
test_data,
embedding_method,
k,
how_to_choose_k=how_to_choose_k,
ranking_model=ranking_model)
triplet_train_loader = torch.utils.data.DataLoader(
train_triplet_ds,
batch_size=batch_size,
shuffle=True,
**kwargs)
triplet_test_loader = torch.utils.data.DataLoader(
val_triplet_ds, batch_size=batch_size, shuffle=False, **kwargs)
margin = 0.2
target_embedding_net = TargetEmbeddingNet(300)
#avg_embedding_net = TargetEmbeddingNet(300)
triple_model = TripletNet(target_embedding_net,
target_embedding_net)
if cuda:
triple_model.cuda()
loss_fn = TripletLoss(margin)
lr = 1e-4
optimizer = optim.Adam(triple_model.parameters(), lr=lr)
scheduler = lr_scheduler.StepLR(optimizer,
300,
gamma=0.5,
last_epoch=-1)
n_epochs = 20
log_interval = 100
train_losses, val_losses, _ = fit(triplet_train_loader,
triplet_test_loader,
triple_model,
loss_fn,
optimizer,
scheduler,
n_epochs,
cuda,
log_interval,
callback_test=None)
#Test the model on empty space
empty_space_baseline_acc_at_thresholds, empty_space_model_acc_at_thresholds, \
empty_space_baseline_meteor_score, empty_space_baseline_bleu_score, empty_space_model_meteor_score, empty_space_model_bleu_score, _, _ = test_model(triple_model, prepared_test_data, {},
thresholds=[0.2, 0.4, 0.5], test_scenario=test_scenario,combination_of=combination_of)
print('{}, {}, {}, {}'.format(
k, 'yes', empty_space_model_acc_at_thresholds,
empty_space_baseline_acc_at_thresholds))
#Test the model on target space
baseline_acc_at_thresholds, model_acc_at_thresholds,\
baseline_meteor_score, baseline_bleu_score, model_meteor_score, model_bleu_score, _, _ = test_model(triple_model, prepared_test_data, target_space,
thresholds=[0.2, 0.4, 0.5], test_scenario=test_scenario, combination_of=combination_of)
print('{}, {}, {}, {}'.format(k, 'no', model_acc_at_thresholds,
baseline_acc_at_thresholds))
f.write('{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}\n'.format(
k, 'yes', *empty_space_model_acc_at_thresholds,
empty_space_model_bleu_score, empty_space_model_meteor_score,
*empty_space_baseline_acc_at_thresholds,
empty_space_baseline_bleu_score,
empty_space_baseline_meteor_score))
f.write('{}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}, {}\n'.format(
k, 'no', *model_acc_at_thresholds, model_bleu_score,
model_meteor_score, *baseline_acc_at_thresholds,
baseline_bleu_score, baseline_meteor_score))
batch_size=batch_size,
shuffle=False,
**kwargs)
#print(type(triplet_test_loader))
#triplet_m_train_loader = torch.utils.data.DataLoader(triplet_m_train_dataset, batch_size=batch_size, shuffle=True, **kwargs)
#triplet_m_test_loader = torch.utils.data.DataLoader(triplet_m_test_dataset, batch_size=batch_size, shuffle=False, **kwargs)
# Set up the network and training parameters
from networks import EmbeddingNet, TripletNet
from losses import TripletLoss
margin = 1
embedding_net = EmbeddingNet()
model = TripletNet(embedding_net)
if cuda:
model.cuda()
loss_fn = TripletLoss(margin)
lr = 1e-3
optimizer = optim.Adam(model.parameters(), lr=lr, betas=(0.9, 0.999))
scheduler = lr_scheduler.StepLR(optimizer, 10, gamma=0.1, last_epoch=-1)
n_epochs = 1
log_interval = 1
if os.path.isfile('./model/0922_checkpoint'):
checkpoint = torch.load('./model/0922_checkpoint')
model.load_state_dict(checkpoint['model_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
model.train()
torch.save(
triplet_train_loader = torch.utils.data.DataLoader(triplet_train_dataset,
batch_size=batch_size,
shuffle=True,
**kwargs)
triplet_test_loader = torch.utils.data.DataLoader(triplet_test_dataset,
batch_size=batch_size,
shuffle=False,
**kwargs)
# Set up the network and training parameters
from networks import EmbeddingNet, TripletNet
from losses import TripletLoss
margin = 1.
embedding_net = EmbeddingNet()
model = TripletNet(embedding_net)
if cuda:
model.cuda()
loss_fn = TripletLoss(margin)
lr = 1e-3
optimizer = optim.Adam(model.parameters(), lr=lr)
scheduler = lr_scheduler.StepLR(optimizer, 8, gamma=0.1, last_epoch=-1)
n_epochs = 20
log_interval = 100
fit(triplet_train_loader, triplet_test_loader, model, loss_fn, optimizer,
scheduler, n_epochs, cuda, log_interval)
train_embeddings_tl, train_labels_tl = extract_embeddings(train_loader, model)
plot_embeddings(train_embeddings_tl, train_labels_tl)
val_embeddings_tl, val_labels_tl = extract_embeddings(test_loader, model)
def main():
import torch
from torch.optim import lr_scheduler
import torch.optim as optim
from torch.autograd import Variable
from trainer import fit
import numpy as np
cuda = torch.cuda.is_available()
# Training settings
parser = argparse.ArgumentParser(
description='cross subject domain adaptation')
parser.add_argument('--batch-size',
type=int,
default=100,
metavar='N',
help='input batch size for training (default: 64)')
parser.add_argument('--test-batch-size',
type=int,
default=100,
metavar='N',
help='input batch size for testing (default: 1000)')
parser.add_argument('--epochs',
type=int,
default=100,
metavar='N',
help='number of epochs to train (default: 10)')
parser.add_argument('--lr',
type=float,
default=0.001,
metavar='LR',
help='learning rate (default: 0.01)')
parser.add_argument('--momentum',
type=float,
default=0.5,
metavar='M',
help='SGD momentum (default: 0.5)')
parser.add_argument('--no-cuda',
action='store_true',
default=False,
help='disables CUDA training')
parser.add_argument(
'--log-interval',
type=int,
default=10,
metavar='N',
help='how many batches to wait before logging training status')
parser.add_argument('--save-model',
action='store_true',
default=True,
help='For Saving the current Model')
# Writer will output to ./runs/ directory by default
fold_idx = 4
gamma = 1.0
margin = 1.0
DAsetting = False
args = parser.parse_args()
args.seed = 0
args.use_tensorboard = True
args.save_model = True
n_epochs = 100
folder_name = 'exp7_deep100'
comment = 'w/bn fold_' + str(fold_idx) + '_g_' + str(gamma) + '_m_' + str(
margin)
use_cuda = not args.no_cuda and torch.cuda.is_available()
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
torch.cuda.manual_seed(args.seed)
np.random.seed(args.seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = True
device = torch.device("cuda" if use_cuda else "cpu")
#kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
from datetime import datetime
import os
loging = False
x_data, y_data = load_smt()
#get subject number
y_subj = np.zeros([108, 200])
for i in range(108):
y_subj[i, :] = i * 2
y_subj = y_data.reshape(108, 200) + y_subj
y_subj = y_subj.reshape(21600)
#y_subj = np.concatenate([y_data,y_subj],axis=1)
# For classification data
valtype = 'subj'
# if x_data.shape[2] != 60:
# x_data = x_data[:,:,2:,:]
# plt.imshow(x_data[1000,0,:,:])
# #subj - 0-27 train
# train_subj1 = np.r_[0:27]
# train_subj2 = np.r_[0:27]+54
#
# test_subj = np.r_[27:54,54+27:108]
#chidx = np.r_[7:11, 12:15, 17:21, 32:41] #오연조건
# chidx = np.r_[2:56, 60:62]
# x_data = x_data[:,:,chidx,:]
# For Domain adaptation setting
if DAsetting:
train_subj1 = np.r_[27:54]
train_subj2 = np.r_[27:54] + 54
test_subj = np.r_[0:27, 54 + 0:54 + 27]
trial_s = (0, 200)
trial_t = (0, 200)
trial_val = (0, 200)
dataset_train1 = GigaDataset(x=x_data,
y=y_data,
valtype=valtype,
istrain=True,
subj=train_subj1,
trial=trial_s)
dataset_train2 = GigaDataset(x=x_data,
y=y_data,
valtype=valtype,
istrain=True,
subj=train_subj2,
trial=trial_t)
dataset_train = dataset_train1.__add__(dataset_train2)
dataset_test = GigaDataset(x=x_data,
y=y_data,
valtype=valtype,
istrain=False,
subj=test_subj,
trial=trial_val)
triplet_dataset_train1 = TripletGiga(x=x_data,
y=y_subj,
valtype=valtype,
istrain=True,
subj=train_subj1,
trial=trial_s)
triplet_dataset_train2 = TripletGiga(x=x_data,
y=y_subj,
valtype=valtype,
istrain=True,
subj=train_subj2,
trial=trial_t)
triplet_dataset_train = triplet_dataset_train1.__add__(
triplet_dataset_train2)
triplet_dataset_test = TripletGiga(x=x_data,
y=y_data,
valtype=valtype,
istrain=False,
subj=test_subj,
trial=trial_val)
else: #DG setting
test_subj = np.r_[fold_idx * 9:fold_idx * 9 + 9,
fold_idx * 9 + 54:fold_idx * 9 + 9 + 54]
print('test subj:' + str(test_subj))
train_subj = np.setxor1d(np.r_[0:108], test_subj)
trial_train = (0, 200)
trial_val = (0, 200)
dataset_train = GigaDataset(x=x_data,
y=y_data,
valtype=valtype,
istrain=True,
subj=train_subj,
trial=trial_train)
dataset_test = GigaDataset(x=x_data,
y=y_data,
valtype=valtype,
istrain=False,
subj=test_subj,
trial=trial_val)
triplet_dataset_train = TripletGiga2(x=x_data,
y=y_subj,
valtype=valtype,
istrain=True,
subj=train_subj,
trial=trial_train)
triplet_dataset_test = TripletGiga2(x=x_data,
y=y_subj,
valtype=valtype,
istrain=False,
subj=test_subj,
trial=trial_val)
train_loader = torch.utils.data.DataLoader(dataset_train,
batch_size=args.batch_size,
shuffle=True)
test_loader = torch.utils.data.DataLoader(dataset_test,
batch_size=args.batch_size,
shuffle=False)
triplet_train_loader = torch.utils.data.DataLoader(
triplet_dataset_train, batch_size=args.batch_size, shuffle=True)
triplet_test_loader = torch.utils.data.DataLoader(
triplet_dataset_test, batch_size=args.batch_size, shuffle=False)
###################################################################################################################
# make model for metric learning
from networks import basenet, Deep4Net, EmbeddingDeep4CNN, EmbeddingDeep4CNN_bn, TripletNet, FineShallowCNN, EmbeddingDeepCNN, QuintupletNet, EmbeddingShallowCNN
from losses import TripletLoss_dev2, TripLoss
embedding_net = Deep4Net()
print(embedding_net)
model = TripletNet(embedding_net)
#exp3-1 fc레이어 한층더
# model.fc = nn.Sequential(
# nn.Linear(model.num_hidden,128),
# nn.ReLU(),
# nn.Dropout(),
# nn.Linear(128,2)
# )
if cuda:
model.cuda()
loss_fn = TripletLoss_dev2(margin, gamma).cuda()
log_interval = 10
# ##########################################################
# optimizer = optim.Adam(model.parameters(), lr=0.01)
# scheduler = lr_scheduler.StepLR(optimizer, 10, gamma=1, last_epoch=-1)
# exp1 : 62ch 0~5fold까지 셋팅
# optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9)
# scheduler = lr_scheduler.StepLR(optimizer, 5, gamma=0.5, last_epoch=-1)
#exp2 : 운동영역주변 20ch, train성능이 fit하지 않는 현상이 g=0.7,1.0 양족에서 모두 나타나서, 기존의 러닝레이트보다 강하게 줘보고 실험코자함
# optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9)
# scheduler = lr_scheduler.StepLR(optimizer, 5, gamma=1.0, last_epoch=-1)
# #
#exp4, exp5
optimizer = optim.SGD(model.parameters(), lr=0.005 / gamma, momentum=0.9)
scheduler = lr_scheduler.StepLR(optimizer, 5, gamma=0.8,
last_epoch=-1) #너무 빨리 떨구면 언더피팅하는듯
# optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9)
# scheduler = lr_scheduler.StepLR(optimizer, 5, gamma=0.8, last_epoch=-1) #너무 빨리 떨구면 언더피팅하는듯
# exp5
# optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9)
# scheduler = lr_scheduler.StepLR(optimizer, 10, gamma=0.5, last_epoch=-1)
#model for validation
evalmodel = nn.Sequential(model.embedding_net, model.fc,
nn.LogSoftmax(dim=1)).to(device)
print('____________DANet____________')
print(model)
#save someting
if (args.save_model):
model_save_path = 'model/' + folder_name + '/' + comment + '/'
if not os.path.isdir(model_save_path):
os.makedirs(model_save_path)
if loging:
fname = model_save_path + datetime.today().strftime(
"%m_%d_%H_%M") + ".txt"
f = open(fname, 'w')
if args.use_tensorboard:
writer = SummaryWriter(comment=comment)
# load_model_path = 'C:\\Users\dk\PycharmProjects\giga_cnn\model\deep100_negsubj\\fold_0_g_0.7\danet_0.7_49.pt'
#'C:\\Users\dk\PycharmProjects\giga_cnn\구모델\\clf_83_8.pt'#'clf_29.pt' #'triplet_mg26.pt'#'clf_triplet2_5.pt' #'triplet_31.pt'
load_model_path = 'C:\\Users\dk\PycharmProjects\giga_cnn\model\exp6_basenet\\fold_0_g_0.6\danet_0.6_86.pt'
load_model_path = None
if load_model_path is not None:
model.load_state_dict(torch.load(load_model_path))
for epochidx in range(n_epochs):
fit(triplet_train_loader, triplet_test_loader, model, loss_fn,
optimizer, scheduler, epochidx, n_epochs, cuda, log_interval)
print(epochidx)
train_loss, train_score = eval(args, evalmodel, device, train_loader)
eval_loss, eval_score = eval(args, evalmodel, device, test_loader)
if args.use_tensorboard:
writer.add_scalar('Train/Loss',
np.mean(train_loss) / 100, epochidx)
writer.add_scalar('Train/Acc',
np.mean(train_score) / 100, epochidx)
writer.add_scalar('Eval/Loss', np.mean(eval_loss) / 100, epochidx)
writer.add_scalar('Eval/Acc', np.mean(eval_score) / 100, epochidx)
writer.close()
if args.save_model:
torch.save(
model.state_dict(), model_save_path + 'danet_' + str(gamma) +
'_' + str(epochidx) + '.pt')
def main():
import torch
from torch.optim import lr_scheduler
import torch.optim as optim
from torch.autograd import Variable
from trainer import fit
import numpy as np
cuda = torch.cuda.is_available()
# Training settings
parser = argparse.ArgumentParser(description='PyTorch MNIST Example')
parser.add_argument('--batch-size',
type=int,
default=100,
metavar='N',
help='input batch size for training (default: 64)')
parser.add_argument('--test-batch-size',
type=int,
default=100,
metavar='N',
help='input batch size for testing (default: 1000)')
parser.add_argument('--epochs',
type=int,
default=100,
metavar='N',
help='number of epochs to train (default: 10)')
parser.add_argument('--lr',
type=float,
default=0.001,
metavar='LR',
help='learning rate (default: 0.01)')
parser.add_argument('--momentum',
type=float,
default=0.5,
metavar='M',
help='SGD momentum (default: 0.5)')
parser.add_argument('--no-cuda',
action='store_true',
default=False,
help='disables CUDA training')
parser.add_argument('--seed',
type=int,
default=1,
metavar='S',
help='random seed (default: 1)')
parser.add_argument(
'--log-interval',
type=int,
default=10,
metavar='N',
help='how many batches to wait before logging training status')
parser.add_argument('--save-model',
action='store_true',
default=True,
help='For Saving the current Model')
args = parser.parse_args()
use_cuda = not args.no_cuda and torch.cuda.is_available()
torch.manual_seed(args.seed)
np.random.seed(args.seed)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
device = torch.device("cuda" if use_cuda else "cpu")
kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
from datetime import datetime
import os
loging = False
ismultitask = False
loso = False
if (args.save_model):
model_save_path = 'model/triplet/'
if not os.path.isdir(model_save_path):
os.makedirs(model_save_path)
if loging:
fname = model_save_path + datetime.today().strftime(
"%m_%d_%H_%M") + ".txt"
f = open(fname, 'w')
x_data, y_data = load_smt()
y_subj = np.zeros([108, 200])
for i in range(108):
y_subj[i, :] = i * 2
y_subj = y_data.reshape(108, 200) + y_subj
y_subj = y_subj.reshape(21600)
# nonbciilli = np.s_[0,1,2,4,5,8,16,17,18,20,21,27,28,29,30,32,35,36,38,42,43,44,51]
valtype = 'sess'
if valtype == 'loso':
for subj in range(0, 54):
model = Deep4CNN(ismult=ismultitask).to(device)
#model.load_state_dict(torch.load(model_save_path+ "J_" + str(subj) + 'basecnn.pt'))
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum)
optimizer_fine = optim.SGD(model.parameters(),
lr=0.005,
momentum=args.momentum)
dataset_train = GigaDataset(x=x_data,
y=y_data,
valtype=valtype,
istrain=True,
sess=1,
subj=subj)
train_loader = torch.utils.data.DataLoader(
dataset_train,
batch_size=args.batch_size,
shuffle=True,
**kwargs)
dataset_test = GigaDataset(x=x_data,
y=y_data,
valtype=valtype,
istrain=False,
sess=2,
subj=subj)
test_loader = torch.utils.data.DataLoader(
dataset_test,
batch_size=args.batch_size,
shuffle=False,
**kwargs)
# dataset_fine = GigaDataset_LOSO(x=x_data, y=y_data, fine=True, istrain=True, sess=2, subj=subj)
# fine_loader = torch.utils.data.DataLoader(dataset_fine, batch_size=args.batch_size, shuffle=True, **kwargs)
for epoch in range(1, args.epochs + 1):
train(args, model, device, train_loader, optimizer, epoch)
print("joint-train")
#LOSO joint training
j_loss, j_score = eval(args, model, device, test_loader)
if epoch > 30:
if (args.save_model):
torch.save(
model.state_dict(), model_save_path + "model_" +
str(subj) + "_" + str(epoch) + '.pt')
# #fine tuning
# for epoch in range(1, 10):
# train_mt(args, model, device, fine_loader, optimizer_fine, epoch)
#
# print("fine-tuning")
# f_loss, f_score = eval(args, model, device, test_loader)
if (args.save_model):
torch.save(model.state_dict(),
model_save_path + "F_" + str(subj) + 'basecnn.pt')
if loging:
f = open(fname, 'a')
f.write(
str(subj) + " " + "jl : " + str(j_loss) + " " +
str(j_score) + '\n')
f.close()
elif valtype == 'sess':
from networks import EmbeddingDeep4CNN, TripletNet, FineShallowCNN, EmbeddingDeepCNN, QuintupletNet
from losses import TripletLoss_dev2
# make model for metric learning
margin = 1
embedding_net = EmbeddingDeep4CNN()
# clf_net = nn.Sequential(EmbeddingDeep4CNN(),nn.Linear(1000,2),nn.Dropout(p=1),nn.LogSoftmax(dim=1))
print(embedding_net)
model = TripletNet(embedding_net)
if cuda:
model.cuda()
loss_fn = TripletLoss_dev2(margin).cuda()
n_epochs = 1
log_interval = 10
load_model_path = None #'triplet_mg26.pt'#'clf_triplet2_5.pt' #'triplet_31.pt'
model.fc = nn.Sequential(nn.Linear(1000, 2), nn.Dropout(p=0.5))
if load_model_path is not None:
model.load_state_dict(torch.load(load_model_path))
# For classification
dataset_train = GigaDataset(x=x_data,
y=y_data,
valtype=valtype,
istrain=True,
sess=1)
train_loader = torch.utils.data.DataLoader(dataset_train,
batch_size=args.batch_size,
shuffle=True,
**kwargs)
dataset_test = GigaDataset(x=x_data,
y=y_data,
valtype=valtype,
istrain=False,
sess=2,
subj=-1)
test_loader = torch.utils.data.DataLoader(dataset_test,
batch_size=args.batch_size,
shuffle=False,
**kwargs)
#make model for classification
newmodel = nn.Sequential(model.embedding_net, nn.Linear(1000, 2),
nn.LogSoftmax(dim=1)).to(device)
print(newmodel)
newmodel.to(device)
optimizer = optim.SGD(newmodel.parameters(), lr=0.01, momentum=0.9)
# optimizer = optim.Adam(newmodel.parameters())
for epoch in range(0):
train(args, newmodel, device, train_loader, optimizer, epoch)
j_loss, j_score = eval(args, newmodel, device, test_loader)
# For embedding
triplet_dataset_train = TripletGiga(x=x_data,
y=y_data,
valtype=valtype,
istrain=True,
sess=1)
triplet_train_loader = torch.utils.data.DataLoader(
triplet_dataset_train,
batch_size=args.batch_size,
shuffle=True,
**kwargs)
triplet_dataset_test = TripletGiga(x=x_data,
y=y_data,
valtype=valtype,
istrain=False,
sess=2,
subj=-1)
triplet_test_loader = torch.utils.data.DataLoader(
triplet_dataset_test,
batch_size=args.batch_size,
shuffle=False,
**kwargs)
optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.9)
scheduler = lr_scheduler.StepLR(optimizer, 8, gamma=1, last_epoch=-1)
from sklearn.pipeline import Pipeline
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.model_selection import ShuffleSplit, cross_val_score
lda = LinearDiscriminantAnalysis()
Testmodel = nn.Sequential(model.embedding_net, model.fc,
nn.LogSoftmax(dim=1)).to(device)
print(Testmodel)
for temp in range(1, 30): #10epoch마다 세이브
fit(triplet_train_loader, triplet_test_loader, model, loss_fn,
optimizer, scheduler, n_epochs, cuda, log_interval)
j_loss, j_score = eval(args, Testmodel, device, test_loader)
torch.save(model.state_dict(), 'clf_' + str(temp) + '.pt')
Testmodel = nn.Sequential(model.embedding_net, model.fc,
nn.LogSoftmax(dim=1)).to(device)
print(Testmodel)
j_loss, j_score = eval(args, Testmodel, device, test_loader)
dataset_train_subj = GigaDataset(x=x_data,
y=y_subj,
valtype=valtype,
istrain=True,
sess=1)
train_loader_subj = torch.utils.data.DataLoader(
dataset_train_subj,
batch_size=args.batch_size,
shuffle=True,
**kwargs)
dataset_test_subj = GigaDataset(x=x_data,
y=y_subj,
valtype=valtype,
istrain=False,
sess=2,
subj=-1)
test_loader_subj = torch.utils.data.DataLoader(
dataset_test_subj,
batch_size=args.batch_size,
shuffle=False,
**kwargs)
train_embeddings_tl, train_labels_tl = extract_embeddings(
train_loader_subj, model.embedding_net, 1000)
val_embeddings_tl, val_labels_tl = extract_embeddings(
test_loader_subj, model.embedding_net, 1000)
train_labels_tl_subj = train_labels_tl - train_labels_tl % 2
# from torchvision import datasets, models, transforms
# temp = model.embedding_net.children()
# newmodel = torch.nn.Sequential(*(list(model.embedding_net.children())[:]))
# for param in model.embedding_net.parameters():
# param.requires_grad = True
#newembedding_net = torch.nn.Sequential(*(list(model.embedding_net.children())[:]))
#
from sklearn.manifold import TSNE
tsne = TSNE(n_components=2, perplexity=30)
#features = np.concatenate([train_embeddings_tl,val_embeddings_tl])
#val_labels_tl = val_labels_tl+2
#labels = np.concatenate([train_labels_tl,val_labels_tl])
train_tsne = tsne.fit_transform(train_embeddings_tl)
plot_embeddings(train_tsne, train_labels_tl % 2)
val_tsne = tsne.fit_transform(val_embeddings_tl)
plot_embeddings(val_tsne, val_labels_tl % 2)
for param in model.embedding_net.parameters():
param.requires_grad = True
#embedding_net2 = EmbeddingDeep4CNN()
newmodel = nn.Sequential(
model.embedding_net,
nn.Linear(1000, 2),
nn.Dropout(p=0.5),
nn.LogSoftmax(dim=1),
).to(device)
print(newmodel)
#newmodel.fc_lr = nn.Linear(1000,2)
newmodel.to(device)
optimizer = optim.SGD(newmodel.parameters(), lr=0.01, momentum=0.9)
#optimizer = optim.Adam(newmodel.parameters())
for epoch in range(1, 100):
train(args, newmodel, device, train_loader, optimizer, epoch)
j_loss, j_score = eval(args, newmodel, device, test_loader)
if args.save_model:
torch.save(newmodel.state_dict(), 'clf_83_8.pt')
newmodel.load_state_dict(torch.load('clf_83_8.pt'))
])
training_data = TripletVeriDataset(
root_dir=opt.train_images, xml_path=opt.train_annotation_path, transform=train_transform)
validation_data = TripletVeriDataset(
root_dir=opt.test_images, xml_path=opt.test_annotation_path, transform=test_transform)
train_loader = torch.utils.data.DataLoader(training_data,
batch_size=opt.batch_size, shuffle=True, num_workers=opt.num_workers)
val_loader = torch.utils.data.DataLoader(validation_data,
batch_size=opt.batch_size, shuffle=True, num_workers=opt.num_workers)
embedding_net = Net()
# embedding_net = MobileNetv2()
model = TripletNet(embedding_net).to(device)
loss_fn = nn.TripletMarginLoss(margin=0.5)
# loss_fn = TripletLoss(0.5)
# optimizer = optim.Adadelta(
# model.parameters(), lr=opt.learning_rate, weight_decay=5e-4)
optimizer = optim.SGD(model.parameters(), lr=1e-3, momentum=0.9)
# scheduler = StepLR(optimizer, step_size=1, gamma=0.1)
scheduler = ReduceLROnPlateau(
optimizer, 'min', patience=5)
if opt.resume_path:
print('loading checkpoint {}'.format(opt.resume_path))
checkpoint = torch.load(opt.resume_path)
opt.start_epoch = checkpoint['epoch']
trans = transforms.ToPILImage()
im = trans(im)
im.show()
if __name__ == "__main__":
from utils import make_directory
from torchvision.datasets import MNIST, CIFAR10
from networks import CIFAREmbeddingNet
from networks import SiameseNet, TripletNet
from datasets import Cars3D, CarsEPFL
data_path = make_directory("../datasets/")
net = CIFAREmbeddingNet()
model = TripletNet(net)
sampling_method = "triplet"
# data_transforms = transforms.Compose([transforms.ToTensor()])
# train_data = MNIST(root=data_path, train=True, transform=data_transforms)
train_data = CarsEPFL(root=data_path, mode="train")
query_data = CarsEPFL(root=data_path, mode="query")
gallery_data = CarsEPFL(root=data_path, mode="gallery")
train_dataset = BaseData(train_data, sampling_method=sampling_method)
query_dataset = BaseData(query_data, sampling_method=sampling_method)
gallery_dataset = BaseData(gallery_data, sampling_method=sampling_method)
print(train_dataset.n_groundtruths)
print(query_dataset.n_groundtruths)
print(gallery_dataset.n_groundtruths)
query_loader = build_dataloader(
query_set,
cfg.data.imgs_per_gpu,
cfg.data.workers_per_gpu,
len(cfg.gpus.train),
dist=False)
"""
print('dataloader built')
# Build model and load checkpoint
# model = build_retriever(cfg.model)
backbone_model = Vgg16L2(num_dim=128)
margin = 1.
model = TripletNet(backbone_model)
model.cuda()
cuda = True
print('model built')
loss_fn = TripletLoss(margin)
lr = 1e-3
optimizer = optim.Adam(model.parameters(), lr=lr)
scheduler = lr_scheduler.StepLR(optimizer, 8, gamma=0.1, last_epoch=-1)
n_epochs = 20
log_interval = 500
print('start training')
fit(train_loader, query_loader, model, loss_fn, optimizer, scheduler, n_epochs,
cuda, log_interval)
print('done training')
def run_net(params, transforms):
if params["fixed_seed"] is not None:
torch.backends.cudnn.deterministic = True
random.seed(1)
torch.manual_seed(1)
torch.cuda.manual_seed(1)
np.random.seed(1)
if params["gpu_id"] is not None:
device = torch.device("cuda:{}".format(params["gpu_id"]))
else:
device = torch.device('cpu')
if params["dset"] == "CIFAR10":
concatenated = torch.utils.data.ConcatDataset([
CIFAR10(root='./data', train=True, download=True, transform=None),
CIFAR10(root='./data', train=False, download=True, transform=None)
])
elif params["dset"] == "MNIST":
concatenated = torch.utils.data.ConcatDataset([
MNIST(root='./data', train=True, download=True, transform=None),
MNIST(root='./data', train=False, download=True, transform=None)
])
elif params["dset"] == "FASHIONMNIST":
concatenated = torch.utils.data.ConcatDataset([
FashionMNIST(root='./data',
train=True,
download=True,
transform=None),
FashionMNIST(root='./data',
train=False,
download=True,
transform=None)
])
triplet_test = RandomTripletMiningDataset(concatenated,
train=False,
trans=transforms,
device=None,
params=params)
# Initialize model
if params["dset"] == "CIFAR10":
embedding_net = EmbeddingNet(
in_channels=3,
adjusting_constant=5) # Change this to VGG for CIFAR10
elif params["dset"] == "MNIST" or params["dset"] == "FASHIONMNIST":
embedding_net = EmbeddingNet()
model = TripletNet(embedding_net).to(device)
# Sets the datetime string to use as an identifier for the future
if params["run_id"] is None:
params["run_id"] = '_'.join(
(str(datetime.datetime.now()).split('.')[0].split()))
params["embedding_net_path"] = "arches/embedding_net_{}".format(
params["run_id"])
# Train our model
if params["do_learn"]:
# Initialize loss functions
train_loss = TripletLoss(margin=params["margin"])
test_loader = torch.utils.data.DataLoader(
triplet_test, batch_size=params["batch_size"], shuffle=True
) # Test data is the same as train data but test data is not preshuffled
writer = SummaryWriter(comment="triplet_{0}_{1}_{2}".format(
params["dset"], params["num_pairs"], params["rtm_index"]))
optimizer = optim.Adam(model.parameters(),
lr=params["starting_lr"],
weight_decay=params["weight_decay"])
test(model,
device,
test_loader,
writer,
record_histograms=False,
params=params)
for epoch in range(params["num_epochs"]):
params["curr_epoch"] = epoch
torch.save(model.embedding_net.state_dict(),
params["embedding_net_path"])
triplet_train = RandomTripletMiningDataset(concatenated,
train=True,
trans=transforms,
device=device,
params=params)
train_loader = torch.utils.data.DataLoader(
triplet_train, batch_size=params["batch_size"], shuffle=True)
sample_loader = torch.utils.data.DataLoader(
triplet_train,
batch_size=len(triplet_train) // 100,
shuffle=True
) # Sample our data to get a reference point after every so often
sample_data, sample_targets = next(iter(sample_loader))
if params["show_plots"]:
show_datasets(sample_data)
similar_pair_accuracy = np.round(
len(np.where(sample_targets[0] == sample_targets[1])[0]) /
len(sample_targets[0]), 3)
different_pair_accuracy = np.round(
len(np.where(sample_targets[0] != sample_targets[2])[0]) /
len(sample_targets[0]), 3)
print("Similar pair accuracy:", similar_pair_accuracy)
print("Different pair accuracy:", different_pair_accuracy)
params["different_random_pair_accuracy"] = different_pair_accuracy
params["similar_random_pair_accuracy"] = similar_pair_accuracy
train(model,
device,
train_loader,
train_loss,
epoch,
optimizer,
sample_data,
params=params)
embeddings, targets, indices = test(model,
device,
test_loader,
writer,
epoch=epoch,
params=params)
write_to_tensorboard(
params, writer,
epoch) # Writes to tensorboard at the end of the epoch
writer.export_scalars_to_json(".all_scalars.json")
writer.close()
curr_date = datetime.datetime.now()
date_str = curr_date.strftime("./models/%m_%d_%Y_%H_%M_model_cifar")
print("Saving the full model to: %s" % str(date_str))
torch.save(model, date_str)
print("Model was saved successfully")
triplet_train_loader = torch.utils.data.DataLoader(triplet_train_dataset,
batch_size=batch_size,
shuffle=True,
**kwargs)
triplet_test_loader = torch.utils.data.DataLoader(triplet_test_dataset,
batch_size=batch_size,
shuffle=False,
**kwargs)
# Set up the network and training parameters
from networks import EmbeddingNet, TripletNet, HardNet
from losses import TripletLoss
margin = 1.
embedding_net = HardNet()
model = TripletNet(embedding_net)
if cuda:
model.cuda()
loss_fn = TripletLoss(margin)
lr = 1e-3
optimizer = optim.Adam(model.parameters(), lr=lr)
scheduler = lr_scheduler.StepLR(optimizer, 8, gamma=0.1, last_epoch=-1)
n_epochs = 100
log_interval = 10
fit(triplet_train_loader,
triplet_test_loader,
model,
loss_fn,
optimizer,
scheduler,
X = X.values
X_train, X_validation = train_test_split(X, test_size=test_size)
triplet_train_dataset = TripletDataset(X_train)
triplet_test_dataset = TripletDataset(X_validation)
triplet_train_loader = torch.utils.data.DataLoader(triplet_train_dataset,
**params)
triplet_test_loader = torch.utils.data.DataLoader(triplet_test_dataset,
**params)
embedding_net = EmbeddingNetMLP(input_dim, embedding_dim)
#model = TripletNet(embedding_net)
model = TripletNet(embedding_net)
if cuda:
model.cuda()
# loss_fn = TripletLossMLP()
loss_fn = TripletLossMLP()
lr = 1e-3
optimizer = optim.Adam(model.parameters(), lr=lr)
scheduler = lr_scheduler.StepLR(optimizer, 8, gamma=0.1, last_epoch=-1)
log_interval = 100
patience = 4
fit_triplet(triplet_train_loader, triplet_test_loader, model, loss_fn,
optimizer, scheduler, patience, n_epoch, cuda, log_interval)
torch.save(model, "./data/TripletNetwork.pt")
model = torch.load("./data/TripletNetwork.pt")
#below for unlabeled data
def main(args):
PATH = os.path.dirname(os.path.realpath(__file__))
PATH_TRAIN = args.path_train
FILE_NAME = f'{args.train_file}_{args.random_state}'
global MODEL_NAME
MODEL_NAME = args.model_name
training_log = PATH + f'/training_log/{args.model_name}_training_{args.log}.log'
with open(training_log, 'a') as f:
message = f'Training log {args.log} of {args.model_name} \n\n'
message += f'Starts at {datetime.datetime.now()}\n'
message += 'Arguments are: \n'
message += f'{str(args)}\n\n'
f.write(message)
f.flush()
cuda = torch.cuda.is_available()
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
message = f'Training on {torch.cuda.device_count()} {torch.cuda.get_device_name()}\n'
with open(training_log, 'a') as f:
f.write(message + '\n')
f.flush()
model_params = args.model_params
# initialize the model
embedding_net = EmbeddingNet(
in_channels=grid_input_channels_dict[args.grid_type], **model_params)
model = TripletNet(embedding_net)
start_epoch = 0
message = f'Initialize the model architecture\n'
# load saved model
if args.continue_training:
if args.saved_model is None:
message = f'Missing saved model name\n'
with open(training_log, 'a') as f:
f.write(message)
f.flush()
raise ValueError(message)
message += f'Read saved model {args.saved_model}\n'
start_epoch = int(re.search(r'Epoch(\d+)', args.saved_model).group(1))
if cuda:
map_location = None
else:
map_location = torch.device('cpu')
state_dict = torch.load(f'{PATH}/saved_model/{args.saved_model}',
map_location=map_location)
new_state_dict = OrderedDict()
for k, v in state_dict.items():
name = k.replace('module.', '') # removing ‘moldule.’ from key
new_state_dict[name] = v
model.load_state_dict(new_state_dict)
def model_initialization_method(method):
if method == 'xavier_normal':
def weights_init(m):
if isinstance(m, nn.Conv3d):
nn.init.xavier_normal_(m.weight.data)
if m.bias is not None:
torch.nn.init.zeros_(m.bias)
return weights_init
if method == 'xavier_uniform':
def weights_init(m):
if isinstance(m, nn.Conv3d):
nn.init.xavier_uniform_(m.weight.data)
if m.bias is not None:
torch.nn.init.zeros_(m.bias)
return weights_init
if args.model_init != 'default':
model.apply(model_initialization_method(args.model_init))
message += f'Model initialization method:{args.model_init}\n'
pytorch_total_params = sum(p.numel() for p in model.parameters()
if p.requires_grad)
# print total number of trainable parameters and model architecture
with open(training_log, 'a') as f:
message += f'Model architecture:\n{str(model)}\n'
message += f'Total number of trainable parameters is {pytorch_total_params}\n'
message += f'Training starts at : {datetime.datetime.now()}\n'
f.write(message)
f.flush()
# multi gpu
if torch.cuda.device_count() > 1:
model = nn.DataParallel(model)
model.to(device)
def dataset_class(data_type):
if data_type == 'molcoord':
return MolCoordDataset, TripletMolCoord
if data_type == 'grid':
return SingleGridDataset, TripletSingleGrid
DS, TripletDS = dataset_class(args.data_type)
if args.data_type == 'molcoord':
t0 = datetime.datetime.now()
df = pd.read_pickle(f'{PATH_TRAIN}{FILE_NAME}.pkl')
train_index, test_index = train_test_index(
df.shape[0], random_state=args.random_state)
# MolCoordDataset
molcoords = df.mol.apply(
lambda x: MolCoord.from_sdf(Chem.MolToMolBlock(x))).tolist()
# coordination rotation randomly
if args.grid_rotation == 1:
np.random.seed(args.random_state)
axis = np.random.rand(df.shape[0], 3) * 2 - 1
theta = np.random.rand(df.shape[0]) * np.pi * 2
for i in range(len(molcoords)):
matrix = torch.Tensor(matrix_from_axis_angel(
axis[i], theta[i]))
molcoords[i].coord_rotation_(matrix)
train_dataset = DS([molcoords[index] for index in train_index],
np.zeros(len(train_index), dtype=int),
grid_type=args.grid_type,
train=True)
test_dataset = DS([molcoords[index] for index in test_index],
np.zeros(len(test_index), dtype=int),
grid_type=args.grid_type,
train=False)
with open(training_log, 'a') as f:
message = f'Preparing dataset costs {datetime.datetime.now() - t0}'
f.write(message)
f.flush()
#release unreferenced memory
gc.collect()
del df
if args.data_type == 'grid':
grid_path = f'{args.path_train}/grids/grid_{args.grid_type}'
if args.grid_rotation == 1:
grid_path += '_rot'
grid_path += f'/{args.train_file}/{args.train_file}_{args.random_state}_grids'
num_data = int(
re.search(r'^.+training_(\d+)$', args.train_file).group(1))
test_size = 0.2
num_testdata = int(num_data * test_size)
num_traindata = num_data - num_testdata
t0 = datetime.datetime.now()
# GridDataset
train_dataset = SingleGridDataset(grid_path,
np.zeros(num_traindata),
train=True)
test_dataset = SingleGridDataset(grid_path,
np.zeros(num_testdata),
train=False)
with open(training_log, 'a') as f:
message = f'Preparing dataset costs {datetime.datetime.now() - t0}'
f.write(message + '\n')
f.flush()
batch_size = args.batch_size * torch.cuda.device_count()
margin = args.margin
loss_fn = WeightedTripletLoss(margin)
lr = args.learning_rate
optimizer = optim.Adam(model.parameters(), lr=lr)
scheduler = lr_scheduler.StepLR(optimizer,
step_size=8,
gamma=0.1,
last_epoch=-1)
n_epochs = args.n_epochs
log_interval = args.log_interval
metrics = []
es = EarlyStopping(patience=args.es_patience, min_delta=args.es_min_delta)
min_val_loss = np.inf
early_stopping_counter = 0
es_indicator = False
ckpt_interval = 3
# save the model after initialization
if start_epoch == 0:
with open(training_log, 'a') as f:
save(model, 'completed', start_epoch, f, args.log)
for epoch in range(0, start_epoch):
scheduler.step()
for epoch in range(start_epoch, n_epochs):
optimizer.step()
scheduler.step()
t_epoch_start = datetime.datetime.now()
message = f'Epoch {epoch + 1} starts at {t_epoch_start}'
with open(training_log, 'a') as f:
f.write(message + '\n')
f.flush()
t0 = datetime.datetime.now()
# clustering embedding vectors to get labels
if torch.cuda.device_count() > 1:
embedding_net = model.module.embedding_net
embedding_net = nn.DataParallel(embedding_net)
else:
embedding_net = model.embedding_net
embedding_net.eval()
embedding = []
for batch_index in divide_batches(list(range(num_traindata)),
batch_size):
with torch.no_grad():
embedding.append(
embedding_net(
train_dataset[batch_index]['grid'].cuda()).cpu())
for batch_index in divide_batches(list(range(num_testdata)),
batch_size):
with torch.no_grad():
embedding.append(
embedding_net(
test_dataset[batch_index]['grid'].cuda()).cpu())
embedding = torch.cat(embedding)
message = f'Epoch {epoch + 1} embedding computation costs {datetime.datetime.now() - t0}'
with open(training_log, 'a') as f:
f.write(message + '\n')
f.flush()
min_counts_labels = 1
cls_counter = 1
random_state = args.random_state
while min_counts_labels < 2:
t0 = datetime.datetime.now()
message = f'Epoch {epoch + 1} clustering {cls_counter} starts at {t0}\n'
with open(training_log, 'a') as f:
f.write(message)
f.flush()
random.seed(random_state)
random_state = int(random.random() * 1e6)
kwargs = {'verbose': 1}
cls = clustering_method(args.clustering_method, args.num_clusters,
random_state, **kwargs)
cls.fit(embedding)
labels = cls.predict(embedding)
train_labels = labels[:num_traindata]
test_labels = labels[num_traindata:]
# unique_labels, counts_labels = np.unique(labels, return_counts=True)
unique_labels_train, counts_labels_train = np.unique(
train_labels, return_counts=True)
unique_labels_test, counts_labels_test = np.unique(
test_labels, return_counts=True)
min_counts_labels = min(min(counts_labels_train),
min(counts_labels_test))
message = f'Epoch {epoch + 1} clustering {cls_counter} ends at {datetime.datetime.now()}\n'
message += f'Epoch {epoch + 1} clustering {cls_counter} costs {datetime.datetime.now() - t0}\n'
message += f'{len(unique_labels_train)} clusters for train in total\n'
message += f'The minimum number of samples in a cluster for train is {min(counts_labels_train)}\n'
message += f'The maximum number of samples in a cluster for train is {max(counts_labels_train)}\n'
message += f'{len(unique_labels_test)} clusters for test in total\n'
message += f'The minimum number of samples in a cluster for test is {min(counts_labels_test)}\n'
message += f'The maximum number of samples in a cluster for test is {max(counts_labels_test)}\n'
with open(training_log, 'a') as f:
f.write(message + '\n')
f.flush()
cls_counter += 1
if cls_counter > 10:
break
if min_counts_labels < 2:
with open(training_log, 'a') as f:
message = f'Cannot get good clustering results. Stop training.\n'
f.write(message + '\n')
f.flush()
break
if args.weighted_loss:
loss_weights_train = dict(
zip(
unique_labels_train, 1 / counts_labels_train *
len(train_labels) / len(unique_labels_train)))
loss_weights_test = dict(
zip(
unique_labels_test, 1 / counts_labels_test *
len(test_labels) / len(unique_labels_test)))
else:
loss_weights_train = dict(
zip(unique_labels_train, np.ones(len(unique_labels_train))))
loss_weights_test = dict(
zip(unique_labels_test, np.ones(len(unique_labels_test))))
train_dataset.labels = train_labels
test_dataset.labels = test_labels
t0 = datetime.datetime.now()
kwargs = {'num_workers': 0, 'pin_memory': True} if cuda else {}
train_loader = torch.utils.data.DataLoader(TripletDS(
train_dataset, grid_rotation=args.grid_rotation),
batch_size=batch_size,
shuffle=True,
**kwargs)
test_loader = torch.utils.data.DataLoader(TripletDS(
test_dataset, grid_rotation=args.grid_rotation),
batch_size=batch_size,
shuffle=False,
**kwargs)
message = f'Epoch {epoch + 1} dataloader preparation costs {datetime.datetime.now() - t0}'
with open(training_log, 'a') as f:
f.write(message + '\n')
f.flush()
# Train stage
train_loss, metrics = train_epoch(
train_loader,
model,
loss_fn,
loss_weights_train,
optimizer,
cuda,
log_interval,
training_log,
metrics,
epoch,
ckpt_interval,
)
message = f'Epoch: {epoch + 1}/{n_epochs}. Train set: Average loss: {train_loss:.4f}'
for metric in metrics:
message += f'\t{metric.name()}: {metric.value()}'
val_loss, metrics = test_epoch(test_loader, model, loss_fn,
loss_weights_test, cuda, metrics, epoch)
val_loss /= len(test_loader)
message += f'\nEpoch: {epoch + 1}/{n_epochs}. Validation set: Average loss: {val_loss:.4f}'
for metric in metrics:
message += f'\t{metric.name()}: {metric.value()}\n'
message += f'\nEpoch {epoch + 1} costs: {str(datetime.datetime.now() - t_epoch_start)}\n'
print(message)
with open(training_log, 'a') as f:
f.write(message + '\n')
f.flush()
# output train loss and validation loss
with open(
re.search(r'(.+)\.log', training_log).group(1) + '_loss.csv',
'a') as f_loss:
message = f'{epoch + 1},{train_loss},{val_loss}\n'
f_loss.write(message)
f_loss.flush()
min_val_loss, early_stopping_counter, es_indicator = es(
val_loss, min_val_loss, early_stopping_counter)
# save models with improvement on test loss
if early_stopping_counter == 0:
with open(training_log, 'a') as f:
save(model, 'completed', epoch + 1, f, args.log)
# save models with no improvement on test loss
else:
with open(training_log, 'a') as f:
save(model, 'completed', epoch + 1, f, args.log)
message = f'min_val_loss: {min_val_loss:.4f}, # of epochs with no improvement: {early_stopping_counter}\n'
print(message)
f.write(message + '\n')
f.flush()
if es_indicator:
message = f'min_val_loss: {min_val_loss:.4f}, # of epochs with no improvement: {early_stopping_counter}\n'
message += f'Early Stopping after epoch {epoch + 1}\n'
print(message)
with open(training_log, 'a') as f:
f.write(message + '\n')
f.flush()
break
else:
message = f'min_val_loss: {min_val_loss:.4f}, # of epochs with no improvement: {early_stopping_counter}\n'
message += 'Training continued.\n'
print(message)
with open(training_log, 'a') as f:
f.write(message + '\n')
f.flush()
train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True, **kwargs)
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=batch_size, shuffle=False, **kwargs)
triplet_train_dataset = Triplet_inshop(train_dataset) # Returns triplets of images
triplet_test_dataset = Triplet_inshop(test_dataset)
triplet_train_loader = torch.utils.data.DataLoader(triplet_train_dataset, batch_size=batch_size, shuffle=True, **kwargs)
triplet_test_loader = torch.utils.data.DataLoader(triplet_test_dataset, batch_size=batch_size, shuffle=False, **kwargs)
# Set up the network and training parameters
from networks import EmbeddingNet, TripletNet
from losses import TripletLoss
embedding_net = EmbeddingNet(embed_dim)
margin = 1.
model = TripletNet(embedding_net)
if cuda:
if torch.cuda.device_count() > 1:
print("Let's use", torch.cuda.device_count(), "GPUs!")
# dim = 0 [30, xxx] -> [10, ...], [10, ...], [10, ...] on 3 GPUs
model = nn.DataParallel(model)
model.cuda()
loss_fn = TripletLoss(margin)
lr = 1e-3
optimizer = optim.Adam(model.parameters(), lr=lr)
scheduler = lr_scheduler.StepLR(optimizer, 8, gamma=0.1, last_epoch=-1)
n_epochs = 20
log_interval = 10
if not resume_from_pth:
print("begin fit")
# Data loader
train_set = build_dataset(cfg.data.train)
query_set = build_dataset(cfg.data.query)
print('datasets loaded')
train_loader = torch.utils.data.DataLoader(train_set,
batch_size=cfg.data.imgs_per_gpu,
shuffle=True)
query_loader = torch.utils.data.DataLoader(query_set,
batch_size=cfg.data.imgs_per_gpu,
shuffle=True)
print('dataloader created')
# Build model
backbone_model = Vgg16L2(num_dim=128)
model = TripletNet(backbone_model)
model.cuda()
print('model built')
margin = 1.
loss_fn = TripletLoss(margin)
lr = 1e-4
optimizer = optim.Adam(model.parameters(), lr=lr)
scheduler = lr_scheduler.StepLR(optimizer, 8, gamma=0.1, last_epoch=-1)
n_epochs = 20
log_interval = 500
print('start training')
fit(train_loader, query_loader, model, loss_fn, optimizer, scheduler, n_epochs,
cuda, log_interval)
print('done training')
def tune_model_params(log_file_path):
how_to_choose_k = False
#target_space_path= '../target_spaces/glove_embedded_targets_space.pkl'
target_space_path = '../target_spaces/fasttext_embedded_targets_space.pkl'
#target_space_path= '../target_spaces/fasttext_embedded_cmv_targets_space.pkl'
#cmv_train_data = json.load(open('../../data/cmv/train.json', 'r'))
train_data = json.load(open('../../data/idebate/train_scored.json', 'r'))
val_data = json.load(open('../../data/idebate/dev_scored.json', 'r'))
with open(log_file_path, 'w') as logfile:
logfile.write(
'top_k, margin, epoch, acc@20, acc@40, acc@50, bleu, meteor\n')
for top_k in [2, 4, 6, 8]:
combination_of = top_k
num_negs = 1
train_triplet_ds, train_p_avg, new_target_space = prepare_data(
train_data,
target_space_file=target_space_path,
embedding_method=embedding_method,
triple_sampling=True,
top_k=top_k,
how_to_choose_k=how_to_choose_k,
combination_of=combination_of,
ranking_model=ranking_model,
num_negs=num_negs)
val_triplet_ds, val_p_avg, _ = prepare_data(
val_data,
target_space_file=target_space_path,
embedding_method=embedding_method,
triple_sampling=True,
top_k=top_k,
how_to_choose_k=how_to_choose_k,
combination_of=combination_of,
ranking_model=ranking_model,
num_negs=num_negs)
triplet_train_loader = torch.utils.data.DataLoader(
train_triplet_ds,
batch_size=batch_size,
shuffle=True,
**kwargs)
triplet_test_loader = torch.utils.data.DataLoader(
val_triplet_ds, batch_size=batch_size, shuffle=False, **kwargs)
# Prepare the test function
target_space = pickle.load(open(target_space_path, 'rb'))
test_data = json.load(
open('../../data/idebate/dev_scored.json', 'r'))
test_data = prepare_test_data(test_data,
embedding_method,
top_k=top_k,
how_to_choose_k=how_to_choose_k,
ranking_model=ranking_model)
test_fun = lambda model: test_model(model,
test_data,
target_space,
test_scenario='optimistic',
combination_of=combination_of)
for margin in [0.1, 0.2, 0.5, 1.0]:
print('Margin: ', margin)
print('K:', top_k)
target_embedding_net = TargetEmbeddingNet(300)
#avg_embedding_net = TargetEmbeddingNet(300)
triple_model = TripletNet(target_embedding_net,
target_embedding_net)
if cuda:
triple_model.cuda()
loss_fn = TripletLoss(margin)
lr = 1e-4
optimizer = optim.Adam(triple_model.parameters(), lr=lr)
scheduler = lr_scheduler.StepLR(optimizer,
300,
gamma=0.5,
last_epoch=-1)
n_epochs = 31
log_interval = 100
train_losses, val_losses, metrices = fit(
triplet_train_loader,
triplet_test_loader,
triple_model,
loss_fn,
optimizer,
scheduler,
n_epochs,
cuda,
log_interval,
callback_test=test_fun)
for m in metrices:
logfile.write('{}, {}, {}, {}, {}, {}, {}, {}\n'.format(
top_k, margin, m['epoch'], *m['model_acc'],
m['model_bleu'], m['model_meteor']))