def main():
torch.manual_seed(args.seed)
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
use_gpu = torch.cuda.is_available()
if args.use_cpu: use_gpu = False
sys.stdout = Logger(osp.join(args.save_dir, 'log' + '.txt'))
if use_gpu:
print("Currently using GPU: {}".format(args.gpu))
cudnn.benchmark = True
torch.cuda.manual_seed_all(args.seed)
else:
print("Currently using CPU")
with open(loader_path, 'rb') as f:
trainloader, testloader = pickle.load(f)
print("Creating model: {}".format(args.model))
model = models.create(name=args.model, num_classes=num_classes, feature_dim=feature_dim)
if use_gpu:
model = nn.DataParallel(model).cuda()
criterion_xent = nn.CrossEntropyLoss()
criterion_cent = CenterLoss(num_classes=num_classes, feat_dim=args.featdim, use_gpu=use_gpu)
optimizer_model = torch.optim.SGD(model.parameters(), lr=args.lr_model, weight_decay=5e-04, momentum=0.9)
optimizer_centloss = torch.optim.SGD(criterion_cent.parameters(), lr=args.lr_cent)
if args.stepsize > 0:
scheduler = lr_scheduler.StepLR(optimizer_model, step_size=args.stepsize, gamma=args.gamma)
start_time = time.time()
total_loss_list = []
train_acc, test_acc = 0, 0
for epoch in range(args.max_epoch):
adjust_learning_rate(optimizer_model, epoch)
print("==> Epoch {}/{}".format(epoch+1, args.max_epoch))
loss_list, train_acc = train(model, criterion_xent, criterion_cent,
optimizer_model, optimizer_centloss,
trainloader, use_gpu, num_classes, epoch)
total_loss_list.append(loss_list)
if args.stepsize > 0: scheduler.step()
if args.eval_freq > 0 and (epoch+1) % args.eval_freq == 0 or (epoch+1) == args.max_epoch:
print("==> Test")
test_acc = test(model, testloader, use_gpu, num_classes, epoch)
total_loss_list = np.array(total_loss_list).ravel()
elapsed = round(time.time() - start_time)
elapsed = str(datetime.timedelta(seconds=elapsed))
print("Finished. Total elapsed time (h:m:s): {}".format(elapsed))
return total_loss_list,train_acc, test_acc
def main():
homepath = os.environ['HOME']
datapath = os.path.join(homepath, 'data')
mx.file.copy_parallel(args.data_url, datapath)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = MobileNetV3().to(device)
centerloss = CenterLoss(num_classes=75, feat_dim=1280, use_gpu=True)
cross_entropy = nn.CrossEntropyLoss()
optimizer_model = torch.optim.SGD(model.parameters(),
lr=args.lr_model,
weight_decay=5e-04,
momentum=0.9)
optimizer_centloss = torch.optim.SGD(centerloss.parameters(),
lr=args.lr_centloss)
train_iterator, test_iterator = dataprocess(
train_label_path=args.train_label_txt,
data_dirtory=datapath,
test_label_path=args.test_label_txt,
batch_size=args.batch_size)
if args.step > 0:
scheduler = lr_scheduler.StepLR(optimizer_model,
step_size=args.step,
gamma=args.gamma)
if not (os.path.isdir(os.path.join(args.homepath, 'model'))):
os.makedirs(os.path.join(args.homepath, 'model'))
tmp_accuracy = 0
for epoch in range(args.num_epoch):
if args.step > 0:
scheduler.step()
train_loss, train_acc = train(model=model,
device=device,
train_iterator=train_iterator,
optimizer_model=optimizer_model,
optimizer_centloss=optimizer_centloss,
criterion1=cross_entropy,
criterion2=centerloss,
weight_centloss=args.weight)
test_loss, test_acc = eval(model=model,
device=device,
test_iterator=test_iterator,
criterion1=cross_entropy,
criterion2=centerloss,
weight_centloss=args.weight_centloss)
print('|Epoch:', epoch + 1, '|Train loss',
train_loss.item(), '|Train acc:', train_acc.item(), '|Test loss',
test_loss.item(), '|Test acc', test_acc.item())
if test_acc > tmp_accuracy:
MODEL_SAVE_PATH = os.path.join(args.homepath, 'model',
'mymodel_{}.pth'.format(epoch))
torch.save(model.save_dict(), MODEL_SAVE_PATH)
tmp_accuracy = test_acc
mox.file.copy(MODEL_SAVE_PATH,
os.path.join(args.train_url, 'model/mymodel.pth'))
def main():
torch.manual_seed(args.seed)
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
use_gpu = torch.cuda.is_available()
if args.use_cpu: use_gpu = False
sys.stdout = Logger(osp.join(args.save_dir, 'log_' + args.dataset + '.txt'))
if use_gpu:
print("Currently using GPU: {}".format(args.gpu))
cudnn.benchmark = True
torch.cuda.manual_seed_all(args.seed)
else:
print("Currently using CPU")
print("Creating dataset: {}".format(args.dataset))
dataset = datasets.create(
name=args.dataset, batch_size=args.batch_size, use_gpu=use_gpu,
num_workers=args.workers,
)
trainloader, testloader = dataset.trainloader, dataset.testloader
print("Creating model: {}".format(args.model))
model = models.create(name=args.model, num_classes=dataset.num_classes)
if use_gpu:
model = nn.DataParallel(model).cuda()
criterion_xent = nn.CrossEntropyLoss()
criterion_cent = CenterLoss(num_classes=dataset.num_classes, feat_dim=2, use_gpu=use_gpu)
optimizer_model = torch.optim.SGD(model.parameters(), lr=args.lr_model, weight_decay=5e-04, momentum=0.9)
optimizer_centloss = torch.optim.SGD(criterion_cent.parameters(), lr=args.lr_cent)
if args.stepsize > 0:
scheduler = lr_scheduler.StepLR(optimizer_model, step_size=args.stepsize, gamma=args.gamma)
start_time = time.time()
for epoch in range(args.max_epoch):
print("==> Epoch {}/{}".format(epoch+1, args.max_epoch))
train(model, criterion_xent, criterion_cent,
optimizer_model, optimizer_centloss,
trainloader, use_gpu, dataset.num_classes, epoch)
if args.stepsize > 0: scheduler.step()
if args.eval_freq > 0 and (epoch+1) % args.eval_freq == 0 or (epoch+1) == args.max_epoch:
print("==> Test")
acc, err = test(model, testloader, use_gpu, dataset.num_classes, epoch)
print("Accuracy (%): {}\t Error rate (%): {}".format(acc, err))
elapsed = round(time.time() - start_time)
elapsed = str(datetime.timedelta(seconds=elapsed))
print("Finished. Total elapsed time (h:m:s): {}".format(elapsed))
def train():
# Init Training
data_loaders = get_data_loader()
coconut_model = CoconutModel(num_of_classes=args.num_of_classes,
feature_size=args.feature_size)
center_loss = CenterLoss(num_classes=args.num_of_classes,
feat_dim=args.feature_size,
use_gpu=torch.cuda.is_available())
coconut_model.to(device)
center_loss.to(device)
params = list(coconut_model.parameters()) + list(
center_loss.parameters()) + list(coconut_model.bert_model.parameters())
optimizer = RAdam(params=params,
lr=args.lr,
betas=(0.0, 0.999),
eps=1e-3,
weight_decay=args.l2_reg)
lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer=optimizer,
milestones=[80, 150],
gamma=0.1)
starting_epoch = 0
if args.resume:
checkpoints = load_model(model=coconut_model,
optimizer=optimizer,
lr_scheduler=lr_scheduler,
center_loss=center_loss)
(starting_epoch, coconut_model, optimizer, lr_scheduler,
center_loss) = checkpoints
for epoch in range(starting_epoch, args.epoch):
train_model(epoch=epoch,
model=coconut_model,
optimizer=optimizer,
loader=data_loaders["train_loader"],
center_loss=center_loss)
lr_scheduler.step()
eval_model(epoch=epoch,
model=coconut_model,
loader=data_loaders["dev_loader"])
save_mode(epoch=epoch,
model=coconut_model,
optimizer=optimizer,
lr_scheduler=lr_scheduler,
center_loss=center_loss)
return
def main():
train_dataset = datasets.MNIST('../data',
download=True,
train=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307, ),
(0.3081, ))
]))
train_loader = DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_workers)
model = LeNetPP(dim_hidden=args.dim_hidden)
if args.cuda:
model = model.cuda()
nll_loss = nn.NLLLoss()
if args.loss == 0:
center_loss = CenterLoss(dim_hidden=args.dim_hidden,
num_classes=10,
lambda_c=args.lambda_c,
use_cuda=args.cuda)
if args.loss == 1:
center_loss = ContrastiveCenterLoss(dim_hidden=args.dim_hidden,
num_classes=10,
lambda_c=args.lambda_c,
use_cuda=args.cuda)
if args.cuda:
nll_loss, center_loss = nll_loss.cuda(), center_loss.cuda()
criterion = [nll_loss, center_loss]
optimizer_nn = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum)
scheduler = lr_scheduler.StepLR(optimizer_nn, step_size=50, gamma=0.2)
optimizer_c = optim.SGD(center_loss.parameters(), lr=args.alpha)
for epoch in range(args.epochs):
scheduler.step()
train(train_loader, model, criterion, [optimizer_nn, optimizer_c],
epoch + 1)
def main():
torch.manual_seed(args.seed)
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
use_gpu = torch.cuda.is_available()
if use_gpu:
print("Currently using GPU: {}".format(args.gpu))
cudnn.benchmark = True
torch.cuda.manual_seed_all(args.seed)
else:
print("Currently using CPU")
transform = transforms.Compose([transforms.ToTensor()])
# Load dataset
dset_train = MultiViewDataSet(args.datadir, transform=transform)
trainloader = DataLoader(dset_train, batch_size=args.batch_size, shuffle=True, num_workers=0)
model = MVCNN(args.num_classes)
model.cuda()
criterion_xent = nn.CrossEntropyLoss()
criterion_cent = CenterLoss(num_classes=args.num_classes, feat_dim=args.feat_dim, use_gpu=use_gpu)
optimizer_model = torch.optim.SGD(model.parameters(), lr=args.lr_model, momentum=0.9)
optimizer_centloss = torch.optim.SGD(criterion_cent.parameters(), lr=args.lr_cent)
if args.stepsize > 0:
scheduler = lr_scheduler.StepLR(optimizer_model, step_size=args.stepsize, gamma=args.gamma)
for epoch in range(args.max_epoch):
#trainloader = iter(train_loader)
print("==> Epoch {}/{}".format(epoch+1, args.max_epoch))
print("++++++++++++++++++++++++++")
train(model, criterion_xent, criterion_cent,
optimizer_model, optimizer_centloss,
trainloader, use_gpu, args.num_classes, epoch)
if epoch % args.save_model_freq == 0:
torch.save(model.state_dict(), args.model_dir+'/'+'3D_model.pth')
if args.stepsize > 0: scheduler.step()
writer.close()
def train_net(net, train_loader, test_loader, lr, device, prefix):
global tensorboard_writer
tensorboard_writer = SummaryWriter(comment = prefix)
# set net on gpu
net.to(device)
# loss and optimizer
criterion = nn.CrossEntropyLoss()
criterion_cent = CenterLoss(num_classes = 10, feat_dim = 128)
criterion_cent.to(device)
optimizer = optim.SGD(net.parameters(), lr = lr, momentum = MOMENTUM, weight_decay = WEIGHT_DECAY)
optimizer_cent = optim.SGD(criterion_cent.parameters(), lr = lr, momentum = MOMENTUM)
scheduler = lr_scheduler.MultiStepLR(optimizer, milestones = MILESTONES, gamma = GAMMA)
scheduler_cent = lr_scheduler.MultiStepLR(optimizer_cent, milestones = MILESTONES, gamma = GAMMA)
# initial test
eval_net(net, test_loader, 0, device)
# epochs
for epoch in range(EPOCHS):
# train
net.train()
scheduler.step()
scheduler_cent.step()
for i, (images, labels) in enumerate(train_loader):
net.zero_grad()
optimizer.zero_grad()
optimizer_cent.zero_grad()
images = images.to(device)
labels = labels.to(device)
features, outputs = net(images)
loss_xent = criterion(outputs, labels)
loss_cent = 0.1 * criterion(features, labels)
loss = loss_xent + loss_cent
loss.backward()
optimizer.step()
optimizer_cent.step()
print(f'epoch {epoch+1:3d}, {i:3d}|{len(train_loader):3d}, loss_xent: {loss_xent.item():2.4f}, loss_cent: {loss_cent.item():2.4f} ', end = '\r')
tensorboard_writer.add_scalars('train_loss', {'train_loss_xent': loss_xent.item(), 'train_loss_cent': loss_cent.item()}, epoch * len(train_loader) + i)
eval_net(net, test_loader, epoch + 1, device)
torch.save(net.state_dict(), f'zoo/{prefix}_params.pth')
def train(args):
# Create model directory
if not os.path.exists(args.model_path):
os.makedirs(args.model_path)
# Save the arguments.
with open(os.path.join(args.model_path, 'args.json'), 'w') as args_file:
json.dump(args.__dict__, args_file)
# Config logging.
log_format = '%(levelname)-8s %(message)s'
log_file_name = 'train_' + args.train_log_file_suffix + '.log'
logfile = os.path.join(args.model_path, log_file_name)
logging.basicConfig(filename=logfile,
level=logging.INFO,
format=log_format)
logging.getLogger().addHandler(logging.StreamHandler())
logging.info(json.dumps(args.__dict__))
# Image preprocessing
transform = transforms.Compose([
transforms.ToTensor(),
transforms.ToPILImage(),
transforms.RandomResizedCrop(args.crop_size,
scale=(1.00, 1.2),
ratio=(0.75, 1.3333333333333333)),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
# Load vocabulary wrapper.
vocab = load_vocab(args.vocab_path)
# Load the category types.
cat2name = json.load(open(args.cat2name))
# Build data loader
logging.info("Building data loader...")
train_sampler = None
val_sampler = None
if os.path.exists(args.train_dataset_weights):
train_weights = json.load(open(args.train_dataset_weights))
train_weights = torch.DoubleTensor(train_weights)
train_sampler = torch.utils.data.sampler.WeightedRandomSampler(
train_weights, len(train_weights))
if os.path.exists(args.val_dataset_weights):
val_weights = json.load(open(args.val_dataset_weights))
val_weights = torch.DoubleTensor(val_weights)
val_sampler = torch.utils.data.sampler.WeightedRandomSampler(
val_weights, len(val_weights))
data_loader = get_loader(args.dataset,
transform,
args.batch_size,
shuffle=False,
num_workers=args.num_workers,
max_examples=args.max_examples,
sampler=train_sampler)
val_data_loader = get_loader(args.val_dataset,
transform,
args.batch_size,
shuffle=False,
num_workers=args.num_workers,
max_examples=args.max_examples,
sampler=val_sampler)
print('Done loading data ............................')
logging.info("Done")
vqg = create_model(args, vocab)
if args.load_model is not None:
vqg.load_state_dict(torch.load(args.load_model))
logging.info("Done")
# Loss criterion.
pad = vocab(vocab.SYM_PAD) # Set loss weight for 'pad' symbol to 0
criterion = nn.CrossEntropyLoss()
criterion2 = nn.MultiMarginLoss().cuda()
l2_criterion = nn.MSELoss()
alpha = None
if (args.bayes):
alpha = vqg.alpha
# Setup GPUs.
if torch.cuda.is_available():
logging.info("Using available GPU...")
vqg.cuda()
criterion.cuda()
l2_criterion.cuda()
if (alpha is not None):
alpha.cuda()
gen_params = vqg.generator_parameters()
info_params = vqg.info_parameters()
learning_rate = args.learning_rate
info_learning_rate = args.info_learning_rate
gen_optimizer = torch.optim.Adam(gen_params, lr=learning_rate)
info_optimizer = torch.optim.Adam(info_params, lr=info_learning_rate)
if (args.step_two):
cycle_params = vqg.cycle_params()
cycle_optimizer = torch.optim.Adam(cycle_params, lr=learning_rate)
if (args.center_loss):
center_loss = CenterLoss(num_classes=args.num_categories,
feat_dim=args.z_size,
use_gpu=True)
optimizer_centloss = torch.optim.SGD(center_loss.parameters(), lr=0.5)
scheduler = ReduceLROnPlateau(optimizer=gen_optimizer,
mode='min',
factor=0.5,
patience=args.patience,
verbose=True,
min_lr=1e-7)
cycle_scheduler = ReduceLROnPlateau(optimizer=gen_optimizer,
mode='min',
factor=0.99,
patience=args.patience,
verbose=True,
min_lr=1e-7)
info_scheduler = ReduceLROnPlateau(optimizer=info_optimizer,
mode='min',
factor=0.5,
patience=args.patience,
verbose=True,
min_lr=1e-7)
# Train the model.
total_steps = len(data_loader)
start_time = time.time()
n_steps = 0
# Optional losses. Initialized here for logging.
recon_category_loss = 0.0
recon_image_loss = 0.0
kl_loss = 0.0
category_cycle_loss = 0.0
regularisation_loss = 0.0
c_loss = 0.0
cycle_loss = 0.0
if (args.step_two):
category_cycle_loss = 0.0
if (args.bayes):
regularisation_loss = 0.0
if (args.center_loss):
loss_center = 0.0
c_loss = 0.0
for epoch in range(args.num_epochs):
for i, (images, questions, answers, categories,
qindices) in enumerate(data_loader):
n_steps += 1
''' remove answers from dataloader later '''
# Set mini-batch dataset.
if torch.cuda.is_available():
images = images.cuda()
questions = questions.cuda()
answers = answers.cuda()
categories = categories.cuda()
qindices = qindices.cuda()
if (args.bayes):
alpha = alpha.cuda()
# Eval now.
if (args.eval_every_n_steps is not None
and n_steps >= args.eval_every_n_steps
and n_steps % args.eval_every_n_steps == 0):
run_eval(vqg, val_data_loader, criterion, l2_criterion, args,
epoch, scheduler, info_scheduler)
compare_outputs(images, questions, answers, categories, vqg,
vocab, logging, cat2name, args)
# Forward.
vqg.train()
gen_optimizer.zero_grad()
info_optimizer.zero_grad()
if (args.step_two):
cycle_optimizer.zero_grad()
if (args.center_loss):
optimizer_centloss.zero_grad()
image_features = vqg.encode_images(images)
category_features = vqg.encode_categories(categories)
# Question generation.
t_mus, t_logvars, ts = vqg.encode_into_t(image_features,
category_features)
if (args.center_loss):
loss_center = 0.0
c_loss = center_loss(ts, categories)
loss_center += args.lambda_centerloss * c_loss
c_loss = c_loss.item()
loss_center.backward(retain_graph=True)
for param in center_loss.parameters():
param.grad.data *= (1. / args.lambda_centerloss)
optimizer_centloss.step()
qlengths_prev = process_lengths(questions)
(outputs, _,
_), pred_ques = vqg.decode_questions(image_features,
ts,
questions=questions,
teacher_forcing_ratio=1.0)
# Reorder the questions based on length.
questions = torch.index_select(questions, 0, qindices)
# Ignoring the start token.
questions = questions[:, 1:]
qlengths = process_lengths(questions)
# Convert the output from MAX_LEN list of (BATCH x VOCAB) ->
# (BATCH x MAX_LEN x VOCAB).
outputs = [o.unsqueeze(1) for o in outputs]
outputs = torch.cat(outputs, dim=1)
outputs = torch.index_select(outputs, 0, qindices)
if (args.step_two):
category_cycle_loss = 0.0
category_cycle = vqg.encode_questions(pred_ques, qlengths)
cycle_loss = criterion(category_cycle, categories)
category_cycle_loss += args.lambda_c_cycle * cycle_loss
cycle_loss = cycle_loss.item()
category_cycle_loss.backward(retain_graph=True)
cycle_optimizer.step()
# Calculate the generation loss.
targets = pack_padded_sequence(questions,
qlengths,
batch_first=True)[0]
outputs = pack_padded_sequence(outputs, qlengths,
batch_first=True)[0]
gen_loss = criterion(outputs, targets)
total_loss = 0.0
total_loss += args.lambda_gen * gen_loss
gen_loss = gen_loss.item()
# Variational loss.
if (args.bayes):
kl_loss = -0.5 * torch.sum(1 + t_logvars + alpha.pow(2).log() -
alpha.pow(2) *
(t_mus.pow(2) + t_logvars.exp()))
regularisation_loss = l2_criterion(alpha.pow(-1),
torch.ones_like(alpha))
total_loss += args.lambda_t * kl_loss + args.lambda_reg * regularisation_loss
kl_loss = kl_loss.item()
regularisation_loss = regularisation_loss.item()
else:
kl_loss = gaussian_KL_loss(t_mus, t_logvars)
total_loss += args.lambda_t * kl_loss
kl_loss = kl_loss.item()
# Generator Backprop.
total_loss.backward(retain_graph=True)
gen_optimizer.step()
# Reconstruction loss.
recon_image_loss = 0.0
recon_category_loss = 0.0
if not args.no_category_space or not args.no_image_recon:
total_info_loss = 0.0
category_targets = category_features.detach()
image_targets = image_features.detach()
recon_image_features, recon_category_features = vqg.reconstruct_inputs(
image_targets, category_targets)
# Category reconstruction loss.
if not args.no_category_space:
recon_c_loss = l2_criterion(
recon_category_features,
category_targets) # changed to criterion2
total_info_loss += args.lambda_c * recon_c_loss
recon_category_loss = recon_c_loss.item()
# Image reconstruction loss.
if not args.no_image_recon:
recon_i_loss = l2_criterion(recon_image_features,
image_targets)
total_info_loss += args.lambda_i * recon_i_loss
recon_image_loss = recon_i_loss.item()
# Info backprop.
total_info_loss.backward()
info_optimizer.step()
# Print log info
if i % args.log_step == 0:
delta_time = time.time() - start_time
start_time = time.time()
logging.info(
'Time: %.4f, Epoch [%d/%d], Step [%d/%d], '
'LR: %f, Center-Loss: %.4f, KL: %.4f, '
'I-recon: %.4f, C-recon: %.4f, C-cycle: %.4f, Regularisation: %.4f'
% (delta_time, epoch, args.num_epochs, i, total_steps,
gen_optimizer.param_groups[0]['lr'], c_loss, kl_loss,
recon_image_loss, recon_category_loss, cycle_loss,
regularisation_loss))
# Save the models
if args.save_step is not None and (i + 1) % args.save_step == 0:
torch.save(
vqg.state_dict(),
os.path.join(args.model_path,
'vqg-tf-%d-%d.pkl' % (epoch + 1, i + 1)))
torch.save(
vqg.state_dict(),
os.path.join(args.model_path, 'vqg-tf-%d.pkl' % (epoch + 1)))
torch.save(
center_loss.state_dict(),
os.path.join(args.model_path,
'closs-tf-%d-%d.pkl' % (epoch + 1, i + 1)))
# Evaluation and learning rate updates.
run_eval(vqg, val_data_loader, criterion, l2_criterion, args, epoch,
scheduler, info_scheduler)
def train(train_sets, test_sets):
# ---------------------- dataloader ---------------------- #
# dataset loaders
train_loaders, train_iters, test_loaders, test_iters = {}, {}, {}, {}
# 加载有label的训练数据
for domain in opt.domains:
train_loaders[domain] = DataLoader(train_sets[domain],
opt.batch_size,
shuffle=True)
train_iters[domain] = iter(train_loaders[domain])
test_loaders[domain] = DataLoader(test_sets[domain],
opt.batch_size,
shuffle=False)
test_iters[domain] = iter(test_loaders[domain])
# ---------------------- model initialization ---------------------- #
F_d = {}
C = None
if opt.model.lower() == 'mlp':
for domain in opt.domains:
F_d[domain] = MlpFeatureExtractor(opt.feature_num,
opt.F_hidden_sizes,
opt.domain_hidden_size,
opt.dropout, opt.F_bn)
C = SentimentClassifier(opt.C_layers, opt.domain_hidden_size,
opt.domain_hidden_size, opt.num_labels,
opt.dropout, opt.C_bn)
# 转移到gpu上
C = C.to(opt.device)
for f_d in F_d.values():
f_d = f_d.to(opt.device)
criterion_cent = CenterLoss(num_classes=2,
feat_dim=opt.domain_hidden_size,
use_gpu=True)
optimizer_centloss = torch.optim.SGD(criterion_cent.parameters(), lr=0.5)
optimizer = optim.Adam(itertools.chain(
*map(list, [C.parameters()] + [f.parameters() for f in F_d.values()])),
lr=opt.learning_rate)
# training
correct, total = defaultdict(int), defaultdict(int)
# D accuracy
d_correct, d_total = 0, 0
best_acc = 0.0
best_acc_dict = {}
margin = 3
margin_lambda = 0.1
# center_loss_weight_cent = 0.1
for epoch in range(opt.max_epoch):
C.train()
for f in F_d.values():
f.train()
# conceptually view 1 epoch as 1 epoch of the first domain
num_iter = len(train_loaders[opt.domains[0]])
# First_stage
for _ in tqdm(range(num_iter)):
for f_d in F_d.values():
f_d.zero_grad()
C.zero_grad()
optimizer_centloss.zero_grad()
for domain in opt.domains:
inputs, targets = utils.endless_get_next_batch(
train_loaders, train_iters, domain)
targets = targets.to(opt.device)
domain_feat = F_d[domain](inputs)
visual_feature, c_outputs = C(domain_feat)
# loss_cent = criterion_cent(visual_feature, targets)
# loss_cent *= center_loss_weight_cent
loss_cent = 0.0
loss_part_1 = functional.nll_loss(c_outputs, targets)
targets = targets.unsqueeze(1)
targets_onehot = torch.FloatTensor(opt.batch_size, 2)
targets_onehot.zero_()
targets_onehot.scatter_(1, targets.cpu(), 1)
targets_onehot = targets_onehot.to(opt.device)
loss_part_2 = margin_lambda * margin_regularization(
inputs, targets_onehot, F_d[domain], C, margin)
# loss_part_2 = 0.0
print("loss_part_1: " + str(loss_part_1))
print("loss_part_2: " + str(loss_part_2))
print("loss_cent: " + str(loss_cent))
l_c = loss_part_1 + loss_part_2 + loss_cent
l_c.backward()
_, pred = torch.max(c_outputs, 1)
total[domain] += targets.size(0)
correct[domain] += (pred == targets).sum().item()
optimizer.step()
# for param in criterion_cent.parameters():
# param.grad.data *= (1. / center_loss_weight_cent)
optimizer_centloss.step()
# end of epoch
log.info('Ending epoch {}'.format(epoch + 1))
if d_total > 0:
log.info('D Training Accuracy: {}%'.format(100.0 * d_correct /
d_total))
log.info('Training accuracy:')
log.info('\t'.join(opt.domains))
log.info('\t'.join(
[str(100.0 * correct[d] / total[d]) for d in opt.domains]))
log.info('Evaluating test sets:')
test_acc = {}
for domain in opt.domains:
test_acc[domain] = evaluate(domain, test_loaders[domain], F_d[domain],
C)
avg_test_acc = sum([test_acc[d] for d in opt.domains]) / len(opt.domains)
log.info(f'Average test accuracy: {avg_test_acc}')
if avg_test_acc > best_acc:
log.info(f'New best Average test accuracy: {avg_test_acc}')
best_acc = avg_test_acc
best_acc_dict = test_acc
for d in opt.domains:
if d in F_d:
torch.save(
F_d[d].state_dict(),
'{}/net_F_d_{}.pth'.format(opt.exp2_model_save_file, d))
torch.save(C.state_dict(),
'{}/netC.pth'.format(opt.exp2_model_save_file))
log.info(
f'Loading model for feature visualization from {opt.exp2_model_save_file}...'
)
for domain in opt.domains:
F_d[domain].load_state_dict(
torch.load(
os.path.join(opt.exp2_model_save_file,
f'net_F_d_{domain}.pth')))
num_iter = len(train_loaders[opt.domains[0]])
visual_features, senti_labels = get_visual_features(
num_iter, test_loaders, test_iters, F_d, C)
# visual_features, senti_labels = get_visual_features(num_iter, train_loaders, train_iters, F_d)
return best_acc, best_acc_dict, visual_features, senti_labels
def train(args):
"""
Function that starts the model training with evaluation at the end of the each epoch
:param args: Command line arguments parsed with the argparse lib
"""
model = SqueezeModel(num_classes=args.num_classes)
if args.model:
model.load_state_dict(torch.load(args.model))
print("Loaded model from:", args.model)
use_gpu = False
if args.gpu > -1:
use_gpu = True
model.cuda(args.gpu)
# dataset
input_shape = (args.num_channels, args.height, args.width)
train_transform, val_transform = data_transformations(input_shape)
train_dataset = ImageFolder(root=os.path.join(args.dataset, 'train'),
transform=train_transform)
train_dataset_loader = DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.num_threads,
pin_memory=True)
validation_dataset = ImageFolder(root=os.path.join(args.dataset, 'validation'),
transform=val_transform)
validation_dataset_loader = DataLoader(validation_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_threads,
pin_memory=True)
# losses
model_criterion = CrossEntropyLoss()
center_criterion = CenterLoss(num_classes=args.num_classes,
feat_dim=model.num_features,
use_gpu=use_gpu)
# optimizers
optim_params = filter(lambda p: p.requires_grad, model.parameters())
if args.optimizer == 'sgd':
model_optimizer = SGD(params=optim_params, lr=args.learning_rate,
momentum=args.momentum, weight_decay=args.weight_decay)
elif args.optimizer == 'adam':
model_optimizer = Adam(optim_params, lr=args.learning_rate,
weight_decay=args.weight_decay)
else:
raise ValueError('Unknown optimizer')
center_optimizer = Adam(center_criterion.parameters(), lr=args.center_learning_rate,
weight_decay=args.weight_decay)
# schedulers
model_lr_scheduler = ReduceLROnPlateau(
model_optimizer, factor=0.25, patience=5, verbose=True)
center_lr_scheduler = ReduceLROnPlateau(
center_optimizer, factor=0.25, patience=5, verbose=True)
for epoch in range(1, args.max_epoch + 1):
_ = train_epoch(train_dataset_loader,
model, model_criterion, center_criterion,
model_optimizer, center_optimizer, use_gpu)
eval_info = evaluate(validation_dataset_loader, model,
model_criterion, center_criterion, use_gpu)
model_lr_scheduler.step(eval_info['model_loss'])
center_lr_scheduler.step(eval_info['center_loss'])
print_eval_info(eval_info, epoch)
if epoch == 1:
best_f1_val = eval_info['f1']
if eval_info['f1'] >= best_f1_val:
model_filename = (args.name + '_epoch_{:02d}'
'-valLoss_{:.5f}'
'-valF1_{:.5f}'.format(epoch,
eval_info['total_loss'],
eval_info['f1']))
model_path = os.path.join(args.save_dir, model_filename)
torch.save(model.state_dict(), model_path)
print('Epoch {}: Saved the new best model to: {}'.format(
epoch, model_path))
best_f1_val = eval_info['f1']
if args.eval_only and args.pretrained_ckpt is None:
print('***************************')
print('Pretrained Model NOT Loaded')
print('Evaluating on Random Model')
print('***************************')
if args.pretrained_ckpt is not None:
model.load_state_dict(torch.load(args.pretrained_ckpt))
best_acc = 0
optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=0.0005)
criterion_cent_f = CenterLoss(num_classes=dataset.num_class,
feat_dim=1024,
use_gpu=True)
optimizer_centloss_f = torch.optim.SGD(criterion_cent_f.parameters(),
lr=0.1)
criterion_cent_r = CenterLoss(num_classes=dataset.num_class,
feat_dim=1024,
use_gpu=True)
optimizer_centloss_r = torch.optim.SGD(criterion_cent_r.parameters(),
lr=0.1)
criterion_cent_all = [criterion_cent_f, criterion_cent_r]
optimizer_centloss_all = [optimizer_centloss_f, optimizer_centloss_r]
for itr in range(args.max_iter):
dataset.t_max = t_max
if itr % 2 == 0 and itr > 000:
dataset.t_max = t_max_ctc
if not args.eval_only:
# Center loss
if FLAG_CENTER:
criterion_centerloss = CenterLoss(num_classes=num_class,
feat_dim=FEAT_DIM,
use_gpu=True)
else:
pass
# Optimizer
if cfg['OPTIM'].lower() == 'adam':
optimizer_softmax = optim.Adam(model.parameters(),
lr=LR_SOFTMAX,
weight_decay=WEIGHT_DECAY)
if FLAG_CENTER:
optimizer_center = optim.Adam(criterion_centerloss.parameters(),
lr=LR_CENTER)
else:
pass
else:
raise NotImplementedError('Optimizer: Adam')
# Decay LR by a factor of 0.1 every 10 epochs
# scheduler = lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.1)
#======= Train & Validation & Saving Checkpoint =======#
train_batch_loss_history = []
train_batch_acc_history = []
best_model_wts = copy.deepcopy(model.state_dict())
best_roc_auc = 0.0
since = time.time()
def _do_train(self):
# STEP 1 -------------------------------------------------------------------------------------
#data-------------------------------------------------------------------------------------
torch.backends.cudnn.benchmark
if self.use_VGG:
feature_extractor = VGGFc(self.device, model_name='vgg19')
else:
feature_extractor = ResNetFc(self.device, model_name='resnet50')
#### source on which perform training of cls and self-sup task
images, labels = get_split_dataset_info(
self.folder_txt_files + self.source + '_train_all.txt',
self.folder_dataset)
ds_source_ss = CustomDataset(images,
labels,
img_transformer=transform_source_ss,
returns=6,
is_train=True,
ss_classes=self.ss_classes,
n_classes=self.n_classes,
only_4_rotations=self.only_4_rotations,
n_classes_target=self.n_classes_target)
source_train_ss = torch.utils.data.DataLoader(
ds_source_ss,
batch_size=self.batch_size,
shuffle=True,
num_workers=self.n_workers,
pin_memory=True,
drop_last=True)
images, labels = get_split_dataset_info(
self.folder_txt_files + self.target + '_test.txt',
self.folder_dataset)
ds_target_train = CustomDataset(images,
labels,
img_transformer=transform_target_train,
returns=2,
is_train=True,
ss_classes=self.ss_classes,
n_classes=self.n_classes,
only_4_rotations=self.only_4_rotations,
n_classes_target=self.n_classes_target)
target_train = torch.utils.data.DataLoader(ds_target_train,
batch_size=self.batch_size,
shuffle=True,
num_workers=self.n_workers,
pin_memory=True,
drop_last=True)
#### target on which compute the scores to select highest batch (integrate to the learning of ss task) and lower batch (integrate to the learning of cls task for the class unknown)
images, labels = get_split_dataset_info(
self.folder_txt_files + self.target + '_test.txt',
self.folder_dataset)
ds_target_test_for_scores = CustomDataset(
images,
labels,
img_transformer=transform_target_test_for_scores,
returns=2,
is_train=False,
ss_classes=self.ss_classes,
n_classes=self.n_classes,
only_4_rotations=self.only_4_rotations,
n_classes_target=self.n_classes_target)
target_test_for_scores = torch.utils.data.DataLoader(
ds_target_test_for_scores,
batch_size=1,
shuffle=False,
num_workers=self.n_workers,
pin_memory=True,
drop_last=False)
#### target for the final evaluation
images, labels = get_split_dataset_info(
self.folder_txt_files + self.target + '_test.txt',
self.folder_dataset)
ds_target_test = CustomDataset(images,
labels,
img_transformer=transform_target_test,
returns=2,
is_train=False,
ss_classes=self.ss_classes,
n_classes=self.n_classes,
only_4_rotations=self.only_4_rotations,
n_classes_target=self.n_classes_target)
target_test = torch.utils.data.DataLoader(ds_target_test,
batch_size=1,
shuffle=False,
num_workers=self.n_workers,
pin_memory=True,
drop_last=False)
# network -----------------------------------------------------------------------------------------------
if self.only_4_rotations:
discriminator_p = Discriminator(n=1,
n_s=self.ss_classes,
vgg=self.use_VGG)
else:
discriminator_p = Discriminator(n=self.n_classes,
n_s=self.ss_classes,
vgg=self.use_VGG)
cls = CLS(feature_extractor.output_num(),
self.n_classes + 1,
bottle_neck_dim=256,
vgg=self.use_VGG)
discriminator_p.to(self.device)
feature_extractor.to(self.device)
cls.to(self.device)
net = nn.Sequential(feature_extractor, cls).to(self.device)
center_loss = CenterLoss(num_classes=self.ss_classes * self.n_classes,
feat_dim=256 * self.n_classes,
use_gpu=True,
device=self.device)
if self.use_VGG:
center_loss_object = CenterLoss(num_classes=self.n_classes,
feat_dim=4096,
use_gpu=True,
device=self.device)
else:
center_loss_object = CenterLoss(num_classes=self.n_classes,
feat_dim=2048,
use_gpu=True,
device=self.device)
# scheduler, optimizer ---------------------------------------------------------
max_iter = int(self.epochs_step1 * len(source_train_ss))
scheduler = lambda step, initial_lr: inverseDecaySheduler(
step, initial_lr, gamma=10, power=0.75, max_iter=max_iter)
params = list(discriminator_p.parameters())
if self.weight_center_loss > 0:
params = params + list(center_loss.parameters())
optimizer_discriminator_p = OptimWithSheduler(
optim.SGD(params,
lr=self.learning_rate,
weight_decay=5e-4,
momentum=0.9,
nesterov=True), scheduler)
if not self.use_VGG:
for name, param in feature_extractor.named_parameters():
words = name.split('.')
if words[1] == 'layer4':
param.requires_grad = True
else:
param.requires_grad = False
params_cls = list(cls.parameters())
optimizer_cls = OptimWithSheduler(
optim.SGD([{
'params': params_cls
}, {
'params':
feature_extractor.parameters(),
'lr':
(self.learning_rate / self.divison_learning_rate_backbone)
}],
lr=self.learning_rate,
weight_decay=5e-4,
momentum=0.9,
nesterov=True), scheduler)
else:
for name, param in feature_extractor.named_parameters():
words = name.split('.')
if words[1] == 'classifier':
param.requires_grad = True
else:
param.requires_grad = False
params_cls = list(cls.parameters())
optimizer_cls = OptimWithSheduler(
optim.SGD([{
'params': params_cls
}, {
'params':
feature_extractor.parameters(),
'lr':
(self.learning_rate / self.divison_learning_rate_backbone)
}],
lr=self.learning_rate,
weight_decay=5e-4,
momentum=0.9,
nesterov=True), scheduler)
log = Logger(self.folder_log + '/step', clear=True)
target_train = cycle(target_train)
k = 0
print('\n')
print(
'Separation known/unknown phase------------------------------------------------------------------------------------------'
)
print('\n')
while k < self.epochs_step1:
print('Epoch: ', k)
for (i,
(im_source, im_source_ss, label_source, label_source_ss,
label_source_ss_center,
label_source_center_object)) in enumerate(source_train_ss):
(im_target, _) = next(target_train)
global loss_object_class
global acc_train
global loss_rotation
global acc_train_rot
global loss_center
im_source = im_source.to(self.device)
im_target = im_target.to(self.device)
im_source_ss = im_source_ss.to(self.device)
label_source = label_source.to(self.device)
label_source_ss = label_source_ss.to(self.device)
label_source_ss_center = label_source_ss_center.to(self.device)
label_source_center_object = label_source_center_object.to(
self.device)
(_, _, _, predict_prob_source) = net.forward(im_source)
(_, _, _, _) = net.forward(im_target)
fs1_ss = feature_extractor.forward(im_source_ss)
fs1_original = feature_extractor.forward(im_source)
_ = feature_extractor.forward(im_target)
double_input = torch.cat((fs1_original, fs1_ss), 1)
fs1_ss = double_input
p0, p0_center = discriminator_p.forward(fs1_ss)
p0 = nn.Softmax(dim=-1)(p0)
# =========================loss function
ce = CrossEntropyLoss(label_source, predict_prob_source)
d1 = CrossEntropyLoss(label_source_ss, p0)
center, _ = center_loss(p0_center, label_source_ss_center)
with OptimizerManager(
[optimizer_cls, optimizer_discriminator_p]):
loss_object_class = self.cls_weight_source * ce
loss_rotation = self.ss_weight_source * d1
loss_center = self.weight_center_loss * center
loss = loss_object_class + loss_rotation + loss_center
loss.backward()
if self.weight_center_loss > 0:
for param in center_loss.parameters():
param.grad.data *= (1. / self.weight_center_loss)
log.step += 1
k += 1
counter = AccuracyCounter()
counter.addOntBatch(variable_to_numpy(predict_prob_source),
variable_to_numpy(label_source))
acc_train = torch.from_numpy(
np.asarray([counter.reportAccuracy()],
dtype=np.float32)).to(self.device)
counter_ss = AccuracyCounter()
counter_ss.addOntBatch(variable_to_numpy(p0),
variable_to_numpy(label_source_ss))
acc_train_rot = torch.from_numpy(
np.asarray([counter_ss.reportAccuracy()],
dtype=np.float32)).to(self.device)
track_scalars(log, [
'loss_object_class', 'acc_train', 'loss_rotation',
'acc_train_rot', 'loss_center'
], globals())
select_low = compute_scores_all_target(
target_test_for_scores, feature_extractor, discriminator_p, net,
self.use_VGG, self.n_classes, self.ss_classes, self.device,
self.source, self.target, self.folder_txt_files,
self.folder_txt_files_saving)
# ========================= Add target samples to cls and discriminator_p classifiers in function of the score
#data---------------------------------------------------------------------------------------------------------------
self.only_4_rotations = True
images, labels = get_split_dataset_info(
self.folder_txt_files_saving + self.source + '_' + self.target +
'_test_high.txt', self.folder_dataset)
ds_target_high = CustomDataset(
images,
labels,
img_transformer=transform_target_ss_step2,
returns=3,
is_train=True,
ss_classes=self.ss_classes,
n_classes=self.n_classes,
only_4_rotations=self.only_4_rotations,
n_classes_target=self.n_classes_target)
target_train_high = torch.utils.data.DataLoader(
ds_target_high,
batch_size=self.batch_size,
shuffle=True,
num_workers=self.n_workers,
pin_memory=True,
drop_last=True)
images, labels = get_split_dataset_info(
self.folder_txt_files + self.target + '_test.txt',
self.folder_dataset)
ds_target = CustomDataset(images,
labels,
img_transformer=transform_target_ss_step2,
returns=3,
is_train=True,
ss_classes=self.ss_classes,
n_classes=self.n_classes,
only_4_rotations=self.only_4_rotations,
n_classes_target=self.n_classes_target)
target_train = torch.utils.data.DataLoader(ds_target,
batch_size=self.batch_size,
shuffle=True,
num_workers=self.n_workers,
pin_memory=True,
drop_last=True)
images, labels = get_split_dataset_info(
self.folder_txt_files_saving + self.source + '_' + self.target +
'_test_low.txt', self.folder_dataset)
ds_target_low = CustomDataset(
images,
labels,
img_transformer=transform_source_ss_step2,
returns=6,
is_train=True,
ss_classes=self.ss_classes,
n_classes=self.n_classes,
only_4_rotations=self.only_4_rotations,
n_classes_target=self.n_classes_target)
target_train_low = torch.utils.data.DataLoader(
ds_target_low,
batch_size=self.batch_size,
shuffle=True,
num_workers=self.n_workers,
pin_memory=True,
drop_last=True)
# network --------------------------------------------------------------------------------------------------------------------------
discriminator_p = Discriminator(n=1,
n_s=self.ss_classes,
vgg=self.use_VGG)
discriminator_p.to(self.device)
if not self.use_weight_net_first_part:
if self.use_VGG:
feature_extractor = VGGFc(self.device, model_name='vgg19')
else:
feature_extractor = ResNetFc(self.device,
model_name='resnet50')
cls = CLS(feature_extractor.output_num(),
self.n_classes + 1,
bottle_neck_dim=256,
vgg=self.use_VGG)
feature_extractor.to(self.device)
cls.to(self.device)
net = nn.Sequential(feature_extractor, cls).to(self.device)
if len(target_train_low) >= len(target_train_high):
length = len(target_train_low)
else:
length = len(target_train_high)
max_iter = int(self.epochs_step2 * length)
scheduler = lambda step, initial_lr: inverseDecaySheduler(
step, initial_lr, gamma=10, power=0.75, max_iter=max_iter)
params = list(discriminator_p.parameters())
optimizer_discriminator_p = OptimWithSheduler(
optim.SGD(params,
lr=self.learning_rate,
weight_decay=5e-4,
momentum=0.9,
nesterov=True), scheduler)
if not self.use_VGG:
for name, param in feature_extractor.named_parameters():
words = name.split('.')
if words[1] == 'layer4':
param.requires_grad = True
else:
param.requires_grad = False
params_cls = list(cls.parameters())
optimizer_cls = OptimWithSheduler(
optim.SGD([{
'params': params_cls
}, {
'params':
feature_extractor.parameters(),
'lr':
(self.learning_rate / self.divison_learning_rate_backbone)
}],
lr=self.learning_rate,
weight_decay=5e-4,
momentum=0.9,
nesterov=True), scheduler)
else:
for name, param in feature_extractor.named_parameters():
words = name.split('.')
if words[1] == 'classifier':
param.requires_grad = True
else:
param.requires_grad = False
params_cls = list(cls.parameters())
optimizer_cls = OptimWithSheduler(
optim.SGD([{
'params': params_cls
}, {
'params':
feature_extractor.parameters(),
'lr':
(self.learning_rate / self.divison_learning_rate_backbone)
}],
lr=self.learning_rate,
weight_decay=5e-4,
momentum=0.9,
nesterov=True), scheduler)
k = 0
print('\n')
print(
'Adaptation phase--------------------------------------------------------------------------------------------------------'
)
print('\n')
ss_weight_target = self.ss_weight_target
weight_class_unknown = 1 / (select_low *
(self.n_classes /
(len(source_train_ss) * self.batch_size)))
while k < self.epochs_step2:
print('Epoch: ', k)
iteration = cycle(target_train)
if len(target_train_low) > len(target_train_high):
num_iterations = len(target_train_low)
num_iterations_smaller = len(target_train_high)
target_train_low_iter = iter(target_train_low)
target_train_high_iter = cycle(target_train_high)
else:
num_iterations = len(target_train_high)
num_iterations_smaller = len(target_train_low)
target_train_low_iter = cycle(target_train_low)
target_train_high_iter = iter(target_train_high)
for i in range(num_iterations):
global entropy_loss
(im_target_entropy, _, _) = next(iteration)
(im_source, im_source_ss, label_source, label_source_ss, _,
_) = next(target_train_low_iter)
(im_target, im_target_ss,
label_target_ss) = next(target_train_high_iter)
im_source = im_source.to(self.device)
im_source_ss = im_source_ss.to(self.device)
label_source = label_source.to(self.device)
label_source_ss = label_source_ss.to(self.device)
im_target = im_target.to(self.device)
im_target_ss = im_target_ss.to(self.device)
label_target_ss = label_target_ss.to(self.device)
im_target_entropy = im_target_entropy.to(self.device)
ft1_ss = feature_extractor.forward(im_target_ss)
ft1_original = feature_extractor.forward(im_target)
double_input_t = torch.cat((ft1_original, ft1_ss), 1)
ft1_ss = double_input_t
(_, _, _, predict_prob_source) = net.forward(im_source)
(_, _, _, _) = net.forward(im_target_entropy)
(_, _, _, predict_prob_target) = net.forward(im_target)
p0_t, _ = discriminator_p.forward(ft1_ss)
p0_t = nn.Softmax(dim=-1)(p0_t)
# =========================loss function
class_weight = np.ones((self.n_classes + 1),
dtype=np.dtype('f'))
class_weight[
self.
n_classes] = weight_class_unknown * self.weight_class_unknown
class_weight = (torch.from_numpy(class_weight)).to(self.device)
ce = CrossEntropyLoss(label_source, predict_prob_source,
class_weight)
entropy = EntropyLoss(predict_prob_target)
d1_t = CrossEntropyLoss(label_target_ss, p0_t)
with OptimizerManager(
[optimizer_cls, optimizer_discriminator_p]):
loss_object_class = self.cls_weight_source * ce
loss_rotation = ss_weight_target * d1_t
entropy_loss = self.entropy_weight * entropy
loss = loss_object_class + loss_rotation + entropy_loss
loss.backward()
log.step += 1
k += 1
counter = AccuracyCounter()
counter.addOntBatch(variable_to_numpy(predict_prob_source),
variable_to_numpy(label_source))
acc_train = torch.from_numpy(
np.asarray([counter.reportAccuracy()],
dtype=np.float32)).to(self.device)
counter_ss = AccuracyCounter()
counter_ss.addOntBatch(variable_to_numpy(p0_t),
variable_to_numpy(label_target_ss))
acc_train_rot = torch.from_numpy(
np.asarray([counter_ss.reportAccuracy()],
dtype=np.float32)).to(self.device)
track_scalars(log, [
'loss_object_class', 'acc_train', 'loss_rotation',
'acc_train_rot', 'entropy_loss'
], globals())
global predict_prob
global label
global predict_index
# =================================evaluation
if k % 10 == 0 or k == (self.epochs_step2):
with TrainingModeManager([feature_extractor, cls],
train=False) as mgr, Accumulator([
'predict_prob', 'predict_index',
'label'
]) as accumulator:
for (i, (im, label)) in enumerate(target_test):
with torch.no_grad():
im = im.to(self.device)
label = label.to(self.device)
(ss, fs, _, predict_prob) = net.forward(im)
predict_prob, label = [
variable_to_numpy(x)
for x in (predict_prob, label)
]
label = np.argmax(label, axis=-1).reshape(-1, 1)
predict_index = np.argmax(predict_prob,
axis=-1).reshape(-1, 1)
accumulator.updateData(globals())
for x in accumulator.keys():
globals()[x] = accumulator[x]
y_true = label.flatten()
y_pred = predict_index.flatten()
m = extended_confusion_matrix(
y_true,
y_pred,
true_labels=range(self.n_classes_target),
pred_labels=range(self.n_classes + 1))
cm = m
cm = cm.astype(np.float) / np.sum(cm, axis=1, keepdims=True)
acc_os_star = sum([cm[i][i] for i in range(self.n_classes)
]) / (self.n_classes)
unkn = sum([
cm[i][self.n_classes]
for i in range(self.n_classes, self.n_classes_target)
]) / (self.n_classes_target - (self.n_classes))
acc_os = (acc_os_star *
(self.n_classes) + unkn) / (self.n_classes + 1)
hos = (2 * acc_os_star * unkn) / (acc_os_star + unkn)
print('os', acc_os)
print('os*', acc_os_star)
print('unkn', unkn)
print('hos', hos)
net.train()
def main():
torch.manual_seed(args.seed)
os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
use_gpu = torch.cuda.is_available()
if args.use_cpu: use_gpu = False
sys.stdout = Logger(osp.join(args.save_dir,
'log_' + args.dataset + '.txt'))
if use_gpu:
print("Currently using GPU: {}".format(args.gpu))
cudnn.benchmark = True
torch.cuda.manual_seed_all(args.seed)
else:
print("Currently using CPU")
#==================================dataset loading============================
print("Creating dataset: {}".format(args.dataset))
dataset = datasets.create(
name=args.dataset,
batch_size=args.batch_size,
use_gpu=use_gpu,
num_workers=args.workers,
)
trainloader, testloader = dataset.trainloader, dataset.testloader
print("Creating model: {}".format(args.model))
model = models.create(name=args.model, num_classes=dataset.num_classes)
if use_gpu:
model = nn.DataParallel(model).cuda()
criterion_xent = nn.CrossEntropyLoss()
criterion_cent = CenterLoss(num_classes=dataset.num_classes,
feat_dim=2,
use_gpu=use_gpu)
optimizer_model = torch.optim.SGD(model.parameters(),
lr=args.lr_model,
weight_decay=5e-04,
momentum=0.9)
optimizer_centloss = torch.optim.SGD(criterion_cent.parameters(),
lr=args.lr_cent)
if args.stepsize > 0:
scheduler = lr_scheduler.StepLR(optimizer_model,
step_size=args.stepsize,
gamma=args.gamma)
start_time = time.time()
xent_plot = []
cent_plot = []
loss_plot = []
for epoch in range(args.max_epoch):
print("==> Epoch {}/{}".format(epoch + 1, args.max_epoch))
xent_losses, cent_losses, losses = train(model, criterion_xent,
criterion_cent,
optimizer_model,
optimizer_centloss,
trainloader, use_gpu,
dataset.num_classes, epoch)
xent_plot.append(xent_losses.avg)
cent_plot.append(cent_losses.avg)
loss_plot.append(losses.avg)
if args.stepsize > 0: scheduler.step()
# if args.eval_freq > 0 and (epoch+1) % args.eval_freq == 0 or (epoch+1) == args.max_epoch:
# print("==> Test")
# acc, err = test(model, testloader, use_gpu, dataset.num_classes, epoch)
# print("Accuracy (%): {}\t Error rate (%): {}".format(acc, err))
if epoch % 100 == 0:
state = {'cnn': model.state_dict()}
torch.save(
state,
'/home/mg/code/GEI+PTSN/train/pytorch-center-loss-master/snapshots_512/snapshot_%d.t7'
% epoch)
print('model save at epoch %d' % epoch)
plot_losses(xent_plot, cent_plot, loss_plot, prefix='losses')
elapsed = round(time.time() - start_time)
elapsed = str(datetime.timedelta(seconds=elapsed))
print("Finished. Total elapsed time (h:m:s): {}".format(elapsed))
model.cuda()
# Center loss addition
if args.centerloss:
classes = 10 # default for CIFAR & SVHN
dim = 10
centerloss = CenterLoss(num_classes=classes,
feat_dim=dim,
use_gpu=args.cuda)
print(
'Center loss component | Classes: {} | Features: {} | GPU: {}'.format(
classes, dim, args.cuda))
print('Setup optimizer')
if args.centerloss:
params = list(model.parameters()) + list(centerloss.parameters())
else:
params = model.parameters()
optimizer = optim.Adam(params, lr=args.lr, weight_decay=args.wd)
decreasing_lr = list(map(int, args.decreasing_lr.split(',')))
def train(epoch):
model.train()
for batch_idx, (data, target) in enumerate(train_loader):
if args.cuda:
data, target = data.cuda(), target.cuda()
data, target = Variable(data), Variable(target)
optimizer.zero_grad()
output = F.log_softmax(model(data))
if args.centerloss:
use_gpu=True).to(device)
criterion = [celoss, centerloss]
max_lr = 0.001
min_lr = 0.00001
one_cycle = 20
num_cycle = 3
max_epochs = int(num_classes * one_cycle)
net_optimizer = torch.optim.SGD(model_ft.parameters(),
max_lr,
momentum=0.9,
weight_decay=1e-4)
cl_optimimzer = torch.optim.SGD(centerloss.parameters(),
max_lr,
momentum=0.9,
weight_decay=1e-4)
optimizer = [net_optimizer, cl_optimimzer]
lr_scheduler = lr_scheduler.StepLR(net_optimizer,
step_size=step_size,
gamma=gamma)
model_ft = train_model(model_ft,
criterion,
optimizer,
lr_scheduler,
num_epochs=num_epochs)
model.classifier[6] = nn.Linear(4096,num_classes)
# Freezing the convolutional layer weights
for param in model.features.parameters():
param.requires_grad = False
# Freezing the classifier layer weights except the last layer
for n in range(6):
for param in model.classifier[n].parameters():
param.requires_grad = False
feature_model = copy.deepcopy(model)
del(feature_model.classifier[6])
print("Training Data Samples: ", len(train_loader))
# Using center loss and defining the parameters needed for the center loss
center_loss = CenterLoss(num_classes, feat_dim=4096, use_gpu=False)
optimizer_centloss = torch.optim.SGD(center_loss.parameters(), lr=0.5)
optimizer = optim.Adam(model.classifier[6].parameters(), lr = 0.1, betas = (0.9,0.999), weight_decay = 0.00005)
scheduler = ReduceLROnPlateau(optimizer, mode='max', factor=0.1, patience=10, verbose=False, threshold=0.003, threshold_mode='rel', cooldown=0, min_lr=0, eps=1e-08)
###### PARAMETERS NEEDED TO BE MONITORED ##########
train_acc = 0
val_acc = 0
train_loss = 0
val_loss = 0
num_epochs = 100
alpha = 0.1
################ TRAINING THE MODEL ##############
for ep in range(num_epochs):
start = time.time()
print("\n")
print("Epoch Started at:", datetime.datetime.now())
def main():
feature_dim = int(args.feature_size / 2)
tiou_thresholds = np.linspace(0.1, 0.7, 7)
train_subset = 'training'
test_subset = 'validation'
threshold_type = 'mean'
prediction_filename = './data/prediction.json'
fps = 25
stride = 16
if args.dataset_name == 'Thumos14':
t_max = 750
t_max_ctc = 2800
train_subset = 'validation'
test_subset = 'test'
num_class = 20
groundtruth_filename = './data/th14_groundtruth.json'
elif args.dataset_name == 'GTEA':
fps = 15
t_max = 100
t_max_ctc = 150
num_class = 7
groundtruth_filename = './data/gtea_groundtruth.json'
elif args.dataset_name == 'BEOID':
fps = 30
t_max = 100
t_max_ctc = 400
num_class = 34
groundtruth_filename = './data/beoid_groundtruth.json'
else:
raise ValueError('wrong dataset')
device = torch.device("cuda")
if args.background:
num_class += 1
dataset = Dataset(args,
groundtruth_filename,
train_subset=train_subset,
test_subset=test_subset,
mode=args.mode,
use_sf=args.use_sf)
os.system('mkdir -p %s' % args.model_dir)
os.system('mkdir -p %s/%s' % (args.log_dir, args.model_name))
now = datetime.now()
dt_string = now.strftime("%Y%m%d%H%M%S")
logger = Logger('%s/%s_%s' % (args.log_dir, args.model_name, dt_string))
model = SFNET(dataset.feature_size, num_class).to(device)
if args.eval_only and args.resume is None:
print('***************************')
print('Pretrained Model NOT Loaded')
print('Evaluating on Random Model')
print('***************************')
if args.resume is not None:
model.load_state_dict(torch.load(args.resume))
best_acc = 0
optimizer = optim.Adam(model.parameters(), lr=args.lr, weight_decay=0.0005)
criterion_cent_f = CenterLoss(num_classes=num_class,
feat_dim=feature_dim,
use_gpu=True)
optimizer_centloss_f = torch.optim.SGD(criterion_cent_f.parameters(),
lr=0.1)
criterion_cent_r = CenterLoss(num_classes=num_class,
feat_dim=feature_dim,
use_gpu=True)
optimizer_centloss_r = torch.optim.SGD(criterion_cent_r.parameters(),
lr=0.1)
criterion_cent_all = [criterion_cent_f, criterion_cent_r]
optimizer_centloss_all = [optimizer_centloss_f, optimizer_centloss_r]
center_f = criterion_cent_f.get_centers()
center_r = criterion_cent_r.get_centers()
centers = [center_f, center_r]
params = {'alpha': args.alpha, 'beta': args.beta, 'gamma': args.gamma}
ce = torch.nn.CrossEntropyLoss().cuda()
counts = dataset.get_frame_counts()
print('total %d annotated frames' % counts)
for itr in range(args.max_iter + 1):
dataset.t_max = t_max
if itr % 2 == 0 and itr > 000:
dataset.t_max = t_max_ctc
if not args.eval_only:
train_SF(itr,
dataset,
args,
model,
optimizer,
criterion_cent_all,
optimizer_centloss_all,
logger,
device,
ce,
params,
mode=args.mode)
if itr % args.eval_steps == 0 and (not itr == 0 or args.eval_only):
print('model_name: %s' % args.model_name)
acc = evaluate(itr,
dataset,
model,
logger,
groundtruth_filename,
prediction_filename,
background=args.background,
fps=fps,
stride=stride,
subset=test_subset,
threshold_type=threshold_type,
frame_type=args.frame_type,
adjust_mean=args.adjust_mean,
act_weight=args.actionness_weight,
tiou_thresholds=tiou_thresholds,
use_anchor=args.use_anchor)
torch.save(model.state_dict(),
'%s/%s.%d.pkl' % (args.model_dir, args.model_name, itr))
if acc >= best_acc and not args.eval_only:
torch.save(
model.state_dict(),
'%s/%s_best.pkl' % (args.model_dir, args.model_name))
best_acc = acc
if args.expand and itr == args.expand_step:
act_expand(args, dataset, model, device, centers=None)
model = SFNET(dataset.feature_size, num_class).to(device)
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
weight_decay=0.0005)
counts = dataset.get_frame_counts()
print('total %d frames' % counts)
if args.eval_only:
print('Done Eval!')
break
def scAdapt(args, data_set):
## prepare data
batch_size = args.batch_size
kwargs = {'num_workers': 0, 'pin_memory': True}
source_name = args.source_name #"TM_baron_mouse_for_baron"
target_name = args.target_name #"baron_human"
domain_to_indices = np.where(data_set['accessions'] == source_name)[0]
train_set = {'features': data_set['features'][domain_to_indices], 'labels': data_set['labels'][domain_to_indices],
'accessions': data_set['accessions'][domain_to_indices]}
domain_to_indices = np.where(data_set['accessions'] == target_name)[0]
test_set = {'features': data_set['features'][domain_to_indices], 'labels': data_set['labels'][domain_to_indices],
'accessions': data_set['accessions'][domain_to_indices]}
print('source labels:', np.unique(train_set['labels']), ' target labels:', np.unique(test_set['labels']))
test_set_eval = {'features': data_set['features'][domain_to_indices], 'labels': data_set['labels'][domain_to_indices],
'accessions': data_set['accessions'][domain_to_indices]}
print(train_set['features'].shape, test_set['features'].shape)
data = torch.utils.data.TensorDataset(
torch.FloatTensor(train_set['features']), torch.LongTensor(matrix_one_hot(train_set['labels'], int(max(train_set['labels'])+1)).long()))
source_loader = torch.utils.data.DataLoader(data, batch_size=batch_size, shuffle=True, drop_last=True, **kwargs)
data = torch.utils.data.TensorDataset(
torch.FloatTensor(test_set['features']), torch.LongTensor(matrix_one_hot(test_set['labels'], int(max(train_set['labels'])+1)).long()))
target_loader = torch.utils.data.DataLoader(data, batch_size=batch_size, shuffle=True, drop_last=True, **kwargs)
target_test_loader = torch.utils.data.DataLoader(data, batch_size=batch_size, shuffle=False, drop_last=False,
**kwargs)
class_num = max(train_set['labels'])+1
class_num_test = max(test_set['labels']) + 1
### re-weighting the classifier
cls_num_list = [np.sum(train_set['labels'] == i) for i in range(class_num)]
#from https://github.com/YyzHarry/imbalanced-semi-self/blob/master/train.py
# # Normalized weights based on inverse number of effective data per class.
#2019 Learning Imbalanced Datasets with Label-Distribution-Aware Margin Loss
#2020 Rethinking the Value of Labels for Improving Class-Imbalanced Learning
beta = 0.9999
effective_num = 1.0 - np.power(beta, cls_num_list)
per_cls_weights = (1.0 - beta) / np.array(effective_num)
per_cls_weights = per_cls_weights / np.sum(per_cls_weights) * len(cls_num_list)
per_cls_weights = torch.FloatTensor(per_cls_weights).cuda()
## set base network
embedding_size = args.embedding_size
base_network = FeatureExtractor(num_inputs=train_set['features'].shape[1], embed_size = embedding_size).cuda()
label_predictor = LabelPredictor(base_network.output_num(), class_num).cuda()
total_model = nn.Sequential(base_network, label_predictor)
center_loss = CenterLoss(num_classes=class_num, feat_dim=embedding_size, use_gpu=True)
optimizer_centloss = torch.optim.SGD([{'params': center_loss.parameters()}], lr=0.5)
print("output size of FeatureExtractor and LabelPredictor: ", base_network.output_num(), class_num)
ad_net = scAdversarialNetwork(base_network.output_num(), 1024).cuda()
## set optimizer
config_optimizer = {"lr_type": "inv", "lr_param": {"lr": 0.001, "gamma": 0.001, "power": 0.75}}
parameter_list = base_network.get_parameters() + ad_net.get_parameters() + label_predictor.get_parameters()
optimizer = optim.SGD(parameter_list, lr=1e-3, weight_decay=5e-4, momentum=0.9, nesterov=True)
schedule_param = config_optimizer["lr_param"]
lr_scheduler = lr_schedule.schedule_dict[config_optimizer["lr_type"]]
## train
len_train_source = len(source_loader)
len_train_target = len(target_loader)
transfer_loss_value = classifier_loss_value = total_loss_value = 0.0
epoch_global = 0.0
hit = False
s_global_centroid = torch.zeros(class_num, embedding_size).cuda()
t_global_centroid = torch.zeros(class_num, embedding_size).cuda()
for epoch in range(args.num_iterations):
if epoch % (2500) == 0 and epoch != 0:
feature_target = base_network(torch.FloatTensor(test_set['features']).cuda())
output_target = label_predictor.forward(feature_target)
softmax_out = nn.Softmax(dim=1)(output_target)
predict_prob_arr, predict_label_arr = torch.max(softmax_out, 1)
if epoch == args.epoch_th:
data = torch.utils.data.TensorDataset(torch.FloatTensor(test_set['features']), predict_label_arr.cpu())
target_loader_align = torch.utils.data.DataLoader(data, batch_size=batch_size, shuffle=True, drop_last=True,
**kwargs)
result_path = args.result_path #"../results/"
model_file = result_path + 'final_model_' + str(epoch) + source_name + target_name+'.ckpt'
torch.save({'base_network': base_network.state_dict(), 'label_predictor': label_predictor.state_dict()}, model_file)
if not os.path.exists(result_path):
os.makedirs(result_path)
with torch.no_grad():
code_arr_s = base_network(Variable(torch.FloatTensor(train_set['features']).cuda()))
code_arr_t = base_network(Variable(torch.FloatTensor(test_set_eval['features']).cuda()))
code_arr = np.concatenate((code_arr_s.cpu().data.numpy(), code_arr_t.cpu().data.numpy()), 0)
digit_label_dict = pd.read_csv(args.dataset_path + 'digit_label_dict.csv')
digit_label_dict = pd.DataFrame(zip(digit_label_dict.iloc[:,0], digit_label_dict.index), columns=['digit','label'])
digit_label_dict = digit_label_dict.to_dict()['label']
# transform digit label to cell type name
y_pred_label = [digit_label_dict[x] if x in digit_label_dict else x for x in predict_label_arr.cpu().data.numpy()]
pred_labels_file = result_path + 'pred_labels_' + source_name + "_" + target_name + "_" + str(epoch) + ".csv"
pd.DataFrame([predict_prob_arr.cpu().data.numpy(), y_pred_label], index=["pred_probability", "pred_label"]).to_csv(pred_labels_file, sep=',')
embedding_file = result_path + 'embeddings_' + source_name + "_" + target_name + "_" + str(epoch)+ ".csv"
pd.DataFrame(code_arr).to_csv(embedding_file, sep=',')
#### only for evaluation
# acc_by_label = np.zeros( class_num_test )
# all_label = test_set['labels']
# for i in range(class_num_test):
# acc_by_label[i] = np.sum(predict_label_arr.cpu().data.numpy()[all_label == i] == i) / np.sum(all_label == i)
# np.set_printoptions(suppress=True)
# print('iter:', epoch, "average acc over all test cell types: ", round(np.nanmean(acc_by_label), 3))
# print("acc of each test cell type: ", acc_by_label)
# div_score, div_score_all, ent_score, sil_score = evaluate_multibatch(code_arr, train_set, test_set_eval, epoch)
#results_file = result_path + source_name + "_" + target_name + "_" + str(epoch)+ "_acc_div_sil.csv"
#evel_res = [np.nanmean(acc_by_label), div_score, div_score_all, ent_score, sil_score]
#pd.DataFrame(evel_res, index = ["acc","div_score","div_score_all","ent_score","sil_score"], columns=["values"]).to_csv(results_file, sep=',')
# pred_labels_file = result_path + source_name + "_" + target_name + "_" + str(epoch) + "_pred_labels.csv"
# pd.DataFrame([predict_label_arr.cpu().data.numpy(), all_label], index=["pred_label", "true_label"]).to_csv(pred_labels_file, sep=',')
## train one iter
base_network.train(True)
ad_net.train(True)
label_predictor.train(True)
optimizer = lr_scheduler(optimizer, epoch, **schedule_param)
optimizer.zero_grad()
optimizer_centloss.zero_grad()
if epoch % len_train_source == 0:
iter_source = iter(source_loader)
epoch_global = epoch_global + 1
if epoch % len_train_target == 0:
if epoch < args.epoch_th:
iter_target = iter(target_loader)
else:
hit = True
iter_target = iter(target_loader_align)
inputs_source, labels_source = iter_source.next()
inputs_target, labels_target = iter_target.next()
inputs_source, inputs_target, labels_source, labels_target = inputs_source.cuda(), inputs_target.cuda(), labels_source.cuda(), labels_target.cuda()
feature_source = base_network(inputs_source)
feature_target = base_network(inputs_target)
features = torch.cat((feature_source, feature_target), dim=0)
output_source = label_predictor.forward(feature_source)
output_target = label_predictor.forward(feature_target)
######## VAT and BNM loss
# LDS should be calculated before the forward for cross entropy
vat_loss = VATLoss(xi=args.xi, eps=args.eps, ip=args.ip)
lds_loss = vat_loss(total_model, inputs_target)
softmax_tgt = nn.Softmax(dim=1)(output_target[:, 0:class_num])
_, s_tgt, _ = torch.svd(softmax_tgt)
BNM_loss = -torch.mean(s_tgt)
########domain alignment loss
if args.method == 'DANN':
domain_prob_discriminator_1_source = ad_net.forward(feature_source)
domain_prob_discriminator_1_target = ad_net.forward(feature_target)
adv_loss = loss_utility.BCELossForMultiClassification(label=torch.ones_like(domain_prob_discriminator_1_source), \
predict_prob=domain_prob_discriminator_1_source) # domain matching
adv_loss += loss_utility.BCELossForMultiClassification(label=torch.ones_like(domain_prob_discriminator_1_target), \
predict_prob=1 - domain_prob_discriminator_1_target)
transfer_loss = adv_loss
elif args.method == 'mmd':
base = 1.0 # sigma for MMD
sigma_list = [1, 2, 4, 8, 16]
sigma_list = [sigma / base for sigma in sigma_list]
transfer_loss = loss_utility.mix_rbf_mmd2(feature_source, feature_target, sigma_list)
######CrossEntropyLoss
classifier_loss = nn.CrossEntropyLoss(weight=per_cls_weights)(output_source, torch.max(labels_source, dim=1)[1])
# classifier_loss = loss_utility.CrossEntropyLoss(labels_source.float(), nn.Softmax(dim=1)(output_source))
######semantic_loss and center loss
cell_th = args.cell_th
epoch_th = args.epoch_th
if epoch < args.epoch_th or hit == False:
semantic_loss = torch.FloatTensor([0.0]).cuda()
center_loss_src = torch.FloatTensor([0.0]).cuda()
sum_dist_loss = torch.FloatTensor([0.0]).cuda()
# center_loss.centers = feature_source[torch.max(labels_source, dim=1)[1] == 0].mean(dim=0, keepdim=True)
pass
elif hit == True:
center_loss_src = center_loss(feature_source,
labels=torch.max(labels_source, dim=1)[1])
s_global_centroid = center_loss.centers
semantic_loss, s_global_centroid, t_global_centroid = loss_utility.semant_use_s_center(class_num,
s_global_centroid,
t_global_centroid,
feature_source,
feature_target,
torch.max(
labels_source,
dim=1)[1],
labels_target, 0.7,
cell_th) #softmax_tgt
if epoch > epoch_th:
lds_loss = torch.FloatTensor([0.0]).cuda()
if epoch <= args.num_iterations:
progress = epoch / args.epoch_th #args.num_iterations
else:
progress = 1
lambd = 2 / (1 + math.exp(-10 * progress)) - 1
total_loss = classifier_loss + lambd*args.DA_coeff * transfer_loss + lambd*args.BNM_coeff*BNM_loss + lambd*args.alpha*lds_loss\
+ args.semantic_coeff *semantic_loss + args.centerloss_coeff*center_loss_src
total_loss.backward()
optimizer.step()
# multiple (1./centerloss_coeff) in order to remove the effect of centerloss_coeff on updating centers
if args.centerloss_coeff > 0 and center_loss_src > 0:
for param in center_loss.parameters():
param.grad.data *= (1. / args.centerloss_coeff)
optimizer_centloss.step() #optimize the center in center loss
'''--------------- Prepare the criterions --------------- '''
criterion_xent = nn.CrossEntropyLoss()
criterion_cent = CenterLoss(num_classes=num_classes,
feat_dim=args.embed_size,
use_gpu=use_gpu)
'''--------------- Prepare the optimizers --------------- '''
optimizer_model = torch.optim.SGD(model.parameters(),
lr=args.lr_model,
weight_decay=5e-04,
momentum=0.9)
#optimizer_centloss = torch.optim.SGD(criterion_cent.parameters(), lr=args.lr_cent)
#optimizer_model = torch.optim.Adam(model.parameters())
optimizer_centloss = torch.optim.SGD(criterion_cent.parameters(),
lr=args.lr_cent)
if args.stepsize > 0:
scheduler = lr_scheduler.StepLR(optimizer_model,
step_size=args.stepsize,
gamma=args.gamma)
#outputs.shape,features.shape,labels.shape
'''--------------- training loop --------------- '''
start_time = time.time()
for epoch in tqdm.tqdm(range(args.max_epoch)):
print("==> Epoch {}/{}".format(epoch + 1, args.max_epoch))
