def main():
config = [(64, 3, 1, 1), (64, 3, 1, 1), (1, 3, 1, 1)]
#config = [(64, 9, 1, 4), (32, 1, 1, 0), (3, 5, 1, 2)]
#config = [(64, 9, 1, 0), (32, 1, 1, 0), (3, 5, 1, 0)]
# config: (output_ch, kernel_size, stride, padding_size)
model = SRCNN(config).to(DEVICE)
loss_function = nn.MSELoss(reduction='mean')
optimizer = optim.Adam(model.parameters(), lr=0.0001)
test_data = generate_data('test')
for epoch in range(EPOCH):
train_data = generate_data('train')
train(model, train_data, loss_function, optimizer, DEVICE)
test(model, test_data, loss_function, epoch, DEVICE)
def main_(model_main):
#def main(CFG):
model = model_main.to(CFG['device'])
# 模型训练、补训、测试 #################################################################
if CFG['train_or_test'] =='train':
#optimizer = optim.Adadelta(model.parameters(), lr=CFG['lr']) # 求解器
optimizer = optim.SGD(model.parameters(), lr=CFG['lr'], momentum=CFG['momentum'],)
scheduler = StepLR(optimizer, step_size=2, gamma=CFG['gamma']) # 学习速率衰减方式
result_trace = np.zeros([1,7])
loss_trace = np.zeros([1,12]) # 逐个记录每个epoch的loss和acc
acc_trace = np.zeros([1,12])
for epoch in range(0,20): # 遍历每个epoch
start_time = datetime.datetime.now() # 训练开始时间
train_loss_epoch_i, train_acc_epoch_i = train(CFG, model, train_loader, optimizer, epoch) #
end_time = datetime.datetime.now(); time_cost = (end_time - start_time).seconds ; print('耗时:',time_cost) # 训练耗时
test_loss_epoch_i, test_acc_epoch_i, f4t_and_label = test(CFG, model, test_loader, is_print=True)
result_epoch_i = [epoch, train_acc_epoch_i[0], train_loss_epoch_i[0], test_acc_epoch_i[0],test_loss_epoch_i[0], scheduler.get_lr()[0],time_cost]
result_trace = np.vstack([result_trace, np.array(result_epoch_i).reshape(1,len(result_epoch_i))])
loss_trace = np.vstack([loss_trace, np.array([train_loss_epoch_i + test_loss_epoch_i]).reshape(1,12)])
acc_trace = np.vstack([acc_trace, np.array([train_acc_epoch_i + test_acc_epoch_i]).reshape(1,12)])
if epoch>0:
scheduler.step()
if CFG['save_model'] :
pt_name = '[STI]_' + model.name+ '_'+CFG['dataset_choose'] +'_epoch'+str(0)+'to'+str(epoch) +'_sample'+str(CFG['NO_train'][0])+'to'+str(CFG['NO_train'][1]) # 文件主命名, STI表示Source Trained In
# plot_curve(result_trace[:,0],[result_trace[:,1],result_trace[:,3]],'结果图/'+pt_name+'_ACC.png', xlabel='Epoch',ylabel='ACC',title='ACC',legend=['Training_Accuracy','Testing_Accuracy'])
pt = {'model':model.state_dict(), 'optimizer':optimizer.state_dict(),'scheduler':scheduler.state_dict(),'CFG':CFG,
'model_name':model.name, 'result_trace':result_trace, 'loss_trace':loss_trace, 'acc_trace':acc_trace }
torch.save(pt, pt_name + '_' + device.type + '_rand' + str(CFG['random_seed']) + '.pt')
def main():
train_loader, test_loader, criterion, model, optimizer, scheduler, \
starting_epoch, logfilename, model_path, device, writer = prologue(args)
for epoch in range(starting_epoch, args.epochs):
before = time.time()
train_loss, train_acc = train(train_loader, model, criterion,
optimizer, epoch, args.noise_sd, device,
writer)
test_loss, test_acc = test(test_loader, model, criterion, epoch,
args.noise_sd, device, writer,
args.print_freq)
after = time.time()
log(
logfilename, "{}\t{:.3}\t{:.3}\t{:.3}\t{:.3}\t{:.3}\t{:.3}".format(
epoch, after - before,
scheduler.get_lr()[0], train_loss, train_acc, test_loss,
test_acc))
# In PyTorch 1.1.0 and later, you should call `optimizer.step()` before `lr_scheduler.step()`.
# See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate
scheduler.step(epoch)
torch.save(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
}, model_path)
def main():
generator = srgan.SRGAN_gen().to(device)
discriminator = srgan.SRGAN_dis().to(device)
params = list(generator.parameters()) + list(discriminator.parameters())
optimizer = optim.Adam(params, lr=1e-4)
trainset = TrainDataset()
train_loader = DataLoader(dataset=trainset,
batch_size=BATCH_SIZE,
shuffle=True)
test_data = Image.open('./SR_dataset/Set5/001_HR.png')
test_data = transforms.ToTensor()(test_data)
test_data = test_data.unsqueeze(0)
test_data = test_data.to(device)
for epoch in range(10000):
train(generator, discriminator, optimizer, train_loader, device, epoch)
if epoch % 1000 == 0:
test(generator, discriminator, test_data, epoch, device)
def main():
train_loader, test_loader, criterion, model, optimizer, scheduler, \
starting_epoch, logfilename, model_path, device, writer = prologue(args)
if args.attack == 'PGD':
print('Attacker is PGD')
attacker = PGD_L2(steps=args.num_steps,
device=device,
max_norm=args.epsilon)
elif args.attack == 'DDN':
print('Attacker is DDN')
attacker = DDN(steps=args.num_steps,
device=device,
max_norm=args.epsilon,
init_norm=args.init_norm_DDN,
gamma=args.gamma_DDN)
else:
raise Exception('Unknown attack')
for epoch in range(starting_epoch, args.epochs):
attacker.max_norm = np.min(
[args.epsilon, (epoch + 1) * args.epsilon / args.warmup])
attacker.init_norm = np.min(
[args.epsilon, (epoch + 1) * args.epsilon / args.warmup])
before = time.time()
train_loss, train_acc = train(train_loader, model, criterion,
optimizer, epoch, args.noise_sd,
attacker, device, writer)
test_loss, test_acc = test(test_loader, model, criterion, epoch,
args.noise_sd, device, writer,
args.print_freq)
after = time.time()
log(
logfilename, "{}\t{:.3}\t{:.3}\t{:.3}\t{:.3}\t{:.3}\t{:.3}".format(
epoch, after - before,
scheduler.get_lr()[0], train_loss, train_acc, test_loss,
test_acc))
# In PyTorch 1.1.0 and later, you should call `optimizer.step()` before `lr_scheduler.step()`.
# See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate
scheduler.step(epoch)
torch.save(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'optimizer': optimizer.state_dict(),
}, model_path)
def predict(self, X):
args = self.args
test_x = X
test_y = np.zeros(X.shape[0])
test_loader = basic_loader(test_x,
test_y,
args.batch_size,
train_shuffle=False)
model = self.model
model.load_state_dict(self.best_state)
test_loss, test_score, output = test(args,
self.model,
self.device,
test_loader,
'test',
return_output=True)
return torch.cat(output, dim=0).numpy()
def main(tiny_images=None,
model="cnn",
augment=False,
use_scattering=False,
batch_size=2048,
mini_batch_size=256,
lr=1,
lr_start=None,
optim="SGD",
momentum=0.9,
noise_multiplier=1,
max_grad_norm=0.1,
epochs=100,
bn_noise_multiplier=None,
max_epsilon=None,
data_size=550000,
delta=1e-6,
logdir=None):
logger = Logger(logdir)
device = get_device()
bs = batch_size
assert bs % mini_batch_size == 0
n_acc_steps = bs // mini_batch_size
train_data, test_data = get_data("cifar10", augment=augment)
train_loader = torch.utils.data.DataLoader(train_data,
batch_size=100,
shuffle=False,
num_workers=4,
pin_memory=True)
test_loader = torch.utils.data.DataLoader(test_data,
batch_size=100,
shuffle=False,
num_workers=4,
pin_memory=True)
if isinstance(tiny_images, torch.utils.data.Dataset):
train_data_aug = tiny_images
else:
print("loading tiny images...")
train_data_aug, _ = get_data("cifar10_500K",
augment=augment,
aux_data_filename=tiny_images)
scattering, K, (h, w) = None, None, (None, None)
pre_scattered = False
if use_scattering:
scattering, K, (h, w) = get_scatter_transform("cifar10_500K")
scattering.to(device)
# if the whole data fits in memory, pre-compute the scattering
if use_scattering and data_size <= 50000:
loader = torch.utils.data.DataLoader(train_data_aug,
batch_size=100,
shuffle=False,
num_workers=4)
train_data_aug = get_scattered_dataset(loader, scattering, device,
data_size)
pre_scattered = True
assert data_size <= len(train_data_aug)
num_sup = min(data_size, 50000)
num_batches = int(np.ceil(50000 / mini_batch_size)) # cifar-10 equivalent
train_batch_sampler = SemiSupervisedSampler(data_size, num_batches,
mini_batch_size)
train_loader_aug = torch.utils.data.DataLoader(
train_data_aug,
batch_sampler=train_batch_sampler,
num_workers=0 if pre_scattered else 4,
pin_memory=not pre_scattered)
rdp_norm = 0
if model == "cnn":
if use_scattering:
save_dir = f"bn_stats/cifar10_500K"
os.makedirs(save_dir, exist_ok=True)
bn_stats, rdp_norm = scatter_normalization(
train_loader,
scattering,
K,
device,
data_size,
num_sup,
noise_multiplier=bn_noise_multiplier,
orders=ORDERS,
save_dir=save_dir)
model = CNNS["cifar10"](K, input_norm="BN", bn_stats=bn_stats)
model = model.to(device)
if not pre_scattered:
model = nn.Sequential(scattering, model)
else:
model = CNNS["cifar10"](in_channels=3, internal_norm=False)
elif model == "linear":
save_dir = f"bn_stats/cifar10_500K"
os.makedirs(save_dir, exist_ok=True)
bn_stats, rdp_norm = scatter_normalization(
train_loader,
scattering,
K,
device,
data_size,
num_sup,
noise_multiplier=bn_noise_multiplier,
orders=ORDERS,
save_dir=save_dir)
model = ScatterLinear(K, (h, w), input_norm="BN", bn_stats=bn_stats)
model = model.to(device)
if not pre_scattered:
model = nn.Sequential(scattering, model)
else:
raise ValueError(f"Unknown model {model}")
model.to(device)
if pre_scattered:
test_loader = get_scattered_loader(test_loader, scattering, device)
print(f"model has {get_num_params(model)} parameters")
if optim == "SGD":
optimizer = torch.optim.SGD(model.parameters(),
lr=lr,
momentum=momentum)
else:
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
privacy_engine = PrivacyEngine(
model,
bs,
data_size,
alphas=ORDERS,
noise_multiplier=noise_multiplier,
max_grad_norm=max_grad_norm,
)
privacy_engine.attach(optimizer)
best_acc = 0
flat_count = 0
for epoch in range(0, epochs):
print(f"\nEpoch: {epoch} ({privacy_engine.steps} steps)")
train_loss, train_acc = train(model,
train_loader_aug,
optimizer,
n_acc_steps=n_acc_steps)
test_loss, test_acc = test(model, test_loader)
if noise_multiplier > 0:
print(f"sample_rate={privacy_engine.sample_rate}, "
f"mul={privacy_engine.noise_multiplier}, "
f"steps={privacy_engine.steps}")
rdp_sgd = get_renyi_divergence(
privacy_engine.sample_rate,
privacy_engine.noise_multiplier) * privacy_engine.steps
epsilon, _ = get_privacy_spent(rdp_norm + rdp_sgd,
target_delta=delta)
epsilon2, _ = get_privacy_spent(rdp_sgd, target_delta=delta)
print(f"ε = {epsilon:.3f} (sgd only: ε = {epsilon2:.3f})")
if max_epsilon is not None and epsilon >= max_epsilon:
return
else:
epsilon = None
logger.log_epoch(epoch, train_loss, train_acc, test_loss, test_acc,
epsilon)
logger.log_scalar("epsilon/train", epsilon, epoch)
logger.log_scalar("cifar10k_loss/train", train_loss, epoch)
logger.log_scalar("cifar10k_acc/train", train_acc, epoch)
if test_acc > best_acc:
best_acc = test_acc
flat_count = 0
else:
flat_count += 1
if flat_count >= 20:
print("plateau...")
return
def main():
args = parse_args()
args.num_gpus = len(get_available_gpus())
eval(args.NET + '_setup')(args)
set_seed(args.seed)
setup(args)
# Create model and optimizer
if args.resume or args.eval or args.benchmark:
last_epoch, best_epoch, best_val_loss, num_params, \
enc_params, dec_params = parse_experiment(args.odir)
i = last_epoch
if args.eval or args.benchmark:
i = best_epoch
args.resume = model_at(args, i)
model, stats = tf_resume(args, i)
else:
check_overwrite(os.path.join(args.odir, 'trainlog.txt'))
model = eval(args.NET + '_create_model')(args)
stats = []
print('Will save to ' + args.odir)
if not os.path.exists(args.odir):
os.makedirs(args.odir)
if not os.path.exists(args.odir + '/models'):
os.makedirs(args.odir + '/models')
with open(os.path.join(args.odir, 'cmdline.txt'), 'w') as f:
f.write(" ".join([
"'" + a + "'" if (len(a) == 0 or a[0] != '-') else a
for a in sys.argv
]))
args.model = model
args.step = eval(args.NET + '_step')
# Training loop
epoch = args.start_epoch
train_data_queue, train_data_processes = data_setup(args,
'train',
args.nworkers,
repeat=True)
if args.eval == 0:
for epoch in range(args.start_epoch, args.epochs):
print('Epoch {}/{} ({}):'.format(epoch + 1, args.epochs,
args.odir))
loss = train(args, epoch, train_data_queue,
train_data_processes)[0]
if (epoch +
1) % args.test_nth_epoch == 0 or epoch + 1 == args.epochs:
loss_val = test('val', args)[0]
print('-> Train Loss: {}, \tVal loss: {}'.format(
loss, loss_val))
stats.append({
'epoch': epoch + 1,
'loss': loss,
'loss_val': loss_val
})
else:
loss_val = 0
print('-> Train loss: {}'.format(loss))
stats.append({'epoch': epoch + 1, 'loss': loss})
if (epoch +
1) % args.save_nth_epoch == 0 or epoch + 1 == args.epochs:
with open(os.path.join(args.odir, 'trainlog.txt'),
'w') as outfile:
json.dump(stats, outfile)
save_model(args, epoch)
if (epoch +
1) % args.test_nth_epoch == 0 and epoch + 1 < args.epochs:
split = 'val'
predictions = samples(split, args, 20)
cache_pred(predictions, split, args)
metrics(split, args, epoch)
if math.isnan(loss): break
if len(stats) > 0:
with open(os.path.join(args.odir, 'trainlog.txt'), 'w') as outfile:
json.dump(stats, outfile)
kill_data_processes(train_data_queue, train_data_processes)
split = 'val'
predictions = samples(split, args, 20)
cache_pred(predictions, split, args)
metrics(split, args, epoch)
if args.benchmark:
benchmark_results('test', args)
def fit(self, X, y):
x_train, x_valid, y_train, y_valid = train_test_split(X,
y,
test_size=0.2)
# from some github repo...
torch.multiprocessing.set_sharing_strategy('file_system')
args = self.args
args.input_dim = X.shape[1]
args.output_dim = 1
args.task = 'regression'
use_cuda = not args.no_cuda and torch.cuda.is_available()
torch.manual_seed(args.seed)
np.random.seed(args.seed)
device = torch.device("cuda" if use_cuda else "cpu")
self.device = device
train_loader = basic_loader(x_train, y_train, args.batch_size)
valid_loader = basic_loader(x_valid,
y_valid,
args.batch_size,
train_shuffle=False)
# train_loader, valid_loader, test_loader = get_data_loaders(args.dataset, args.batch_size,
# sub_task=args.sub_task, dim=args.input_dim)
# if args.dataset in ['sider_split/', 'tox21_split/']:
# args.dataset = args.dataset[:-1] + '-' + str(args.sub_task)
print('batch number: train={}, valid={}'.format(
len(train_loader), len(valid_loader)))
model = Net(input_dim=args.input_dim,
output_dim=args.output_dim,
hidden_dim=args.hidden_dim,
num_layer=args.depth,
num_back_layer=args.back_n,
dense=True,
drop_type=args.drop_type,
net_type=args.net_type,
approx=args.anneal,
device=device).to(device)
self.model = model
if args.optimizer == 'SGD':
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=args.momentum,
nesterov=True)
elif args.optimizer == 'AMSGrad':
optimizer = optim.Adam(model.parameters(),
lr=args.lr,
amsgrad=True)
scheduler = StepLR(optimizer,
step_size=args.lr_step_size,
gamma=args.gamma)
best_score = -1e30
start_epoch = 1 # start from epoch 1 or last checkpoint epoch
if args.anneal == 'approx':
args.net_type = 'approx_' + args.net_type
best_model_name = './checkpoint/{}/{}/best_seed{}_depth{}_ckpt.t7'.format(
args.dataset.strip('/'), args.net_type, args.seed, args.depth)
last_model_name = './checkpoint/{}/{}/last_seed{}_depth{}_ckpt.t7'.format(
args.dataset.strip('/'), args.net_type, args.seed, args.depth)
best_log_file = 'log/' + args.dataset.strip(
'/') + '/{}/depth{}_backn{}_drop{}_p{}_best.log'.format(
args.net_type, args.depth, args.back_n, args.drop_type, args.p)
last_log_file = 'log/' + args.dataset.strip(
'/') + '/{}/depth{}_backn{}_drop{}_p{}_last.log'.format(
args.net_type, args.depth, args.back_n, args.drop_type, args.p)
model_dir = './checkpoint/{}/{}/'.format(args.dataset.strip('/'),
args.net_type)
if not os.path.exists(model_dir):
os.makedirs(model_dir)
log_dir = 'log/' + args.dataset.strip('/') + '/{}/'.format(
args.net_type)
if not os.path.exists(log_dir):
os.makedirs(log_dir)
if not os.path.isdir('checkpoint'):
os.mkdir('checkpoint')
for epoch in range(start_epoch, args.epochs + start_epoch):
scheduler.step(epoch)
alpha = get_alpha(epoch, args.epochs)
train_approximate_loss = train(args, model, device, train_loader,
optimizer, epoch, args.anneal,
alpha)
# used for plotting learning curves
train_loss, train_score = test(args, model, device, train_loader,
'train')
valid_loss, valid_score = test(args, model, device, valid_loader,
'valid')
# test_loss, test_score = test(args, model, device, test_loader, 'test')
print(train_score, valid_score)
# early stopping version
if valid_score > best_score:
self.best_state = model.state_dict()
state = {'model': model.state_dict()}
torch.save(state, best_model_name)
best_score = valid_score
# "convergent" version
state = {'model': model.state_dict()}
torch.save(state, last_model_name)
# print('Training finished. Loading models from validation...')
# for model_name, log_file, setting in zip([best_model_name, last_model_name], [best_log_file, last_log_file],
# ['best', 'last']):
# print('\nLoading the {} model...'.format(setting))
#
# checkpoint = torch.load(model_name)
# model.load_state_dict(checkpoint['model'])
# train_loss, train_score = test(args, model, device, train_loader, 'train')
# valid_loss, valid_score = test(args, model, device, valid_loader, 'valid')
# test_loss, test_score = test(args, model, device, test_loader, 'test ')
return self
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--DATASET_PATH', type=str, default='/home/zhangdong/database/DUTS/')
parser.add_argument('--WEIGHTS_PATH', type=str, default='/home/yangle/DAVIS/result/models/')
parser.add_argument('--EXPERIMENT', type=str, default='/home/yangle/DAVIS/result/TrainNet/')
parser.add_argument('--N_EPOCHS', type=int, default=200)
parser.add_argument('--MAX_PATIENCE', type=int, default=30)
parser.add_argument('--batch_size', type=int, default=32)
parser.add_argument('--seed', type=int, default=0)
parser.add_argument('--N_CLASSES', type=int, default=2)
parser.add_argument('--LEARNING_RATE', type=float, default=1e-4)
parser.add_argument('--LR_DECAY', type=float, default=0.995)
parser.add_argument('--DECAY_LR_EVERY_N_EPOCHS', type=int, default=1)
parser.add_argument('--WEIGHT_DECAY', type=float, default=0.0001)
parser.add_argument('--CUDNN', type=bool, default=True)
args = parser.parse_args()
torch.cuda.manual_seed(args.seed)
cudnn.benchmark = args.CUDNN
normalize = transforms.Normalize(mean=saliency.mean, std=saliency.std)
train_joint_transformer_img = transforms.Compose([joint_transforms.JointResize(224)])
mask_size_list = [14, 28, 56, 112, 224]
train_dset = saliency.Saliency(
args.DATASET_PATH, 'train',train_joint_transformer_img, mask_size_list,
transform=transforms.Compose([transforms.ToTensor(), normalize, ]))
train_loader = torch.utils.data.DataLoader(
train_dset, batch_size=args.batch_size, shuffle=True)
test_joint_transforms_img = transforms.Compose([joint_transforms.JointResize(224)])
val_dset = saliency.Saliency(
args.DATASET_PATH, 'val',test_joint_transforms_img, mask_size_list,
transform=transforms.Compose([transforms.ToTensor(),normalize]))
val_loader = torch.utils.data.DataLoader(
val_dset, batch_size=args.batch_size, shuffle=False)
print("TrainImages: %d" % len(train_loader.dataset.imgs))
print("ValImages: %d" % len(val_loader.dataset.imgs))
# example_inputs, example_targets = next(iter(train_loader))
# print("InputsBatchSize: ", example_inputs.size())
# print("TargetsBatchSize: ", len(example_targets))
# print("\nInput (size, max, min) ---")
# # input
# i = example_inputs[0]
# print(i.size())
# print(i.max())
# print(i.min())
# print("Target (size, max, min) ---")
# # target
# for mask in example_targets:
# print(mask.size())
# print(mask.max())
# print(mask.min())
# initialize ResNet18 from the pre-trained classification model
resnet = torchvision.models.resnet50(pretrained=True)
pre_trained_dict = resnet.state_dict()
model = SegNet.resnet50()
model_dict = model.state_dict()
# 1. filter out unnecessary keys
pre_trained_dict = {k: v for k, v in pre_trained_dict.items() if k in model_dict}
# 2. overwrite entries in the existing state dict
model_dict.update(pre_trained_dict)
# 3. load the new state dict
model.load_state_dict(model_dict)
model = model.cuda()
#model = torch.nn.DataParallel(model).cuda()
print(' + Number of params: {}'.format(
sum([p.data.nelement() for p in model.parameters()])))
# model.apply(utils.weights_init)
optimizer = optim.RMSprop(model.parameters(), lr=args.LEARNING_RATE,
weight_decay=args.WEIGHT_DECAY, eps=1e-12)
criterion = nn.NLLLoss2d().cuda()
exp_dir = args.EXPERIMENT + 'test'
if os.path.exists(exp_dir):
shutil.rmtree(exp_dir)
exp = experiment.Experiment('test', args.EXPERIMENT)
exp.init()
START_EPOCH = exp.epoch
END_EPOCH = START_EPOCH + args.N_EPOCHS
for epoch in range(START_EPOCH, END_EPOCH):
since = time.time()
# ### Train ###
trn_loss, trn_err = utils.train(model, train_loader, optimizer, criterion, epoch)
print('Epoch {:d}: Train - Loss: {:.4f}\tErr: {:.4f}'.format(epoch, trn_loss, trn_err))
time_elapsed = time.time() - since
print('Train Time {:.0f}m {:.0f}s'.format(
time_elapsed // 60, time_elapsed % 60))
### Test ###
val_loss, val_err = utils.test(model, val_loader, criterion, epoch)
print('Val - Loss: {:.4f}, Error: {:.4f}'.format(val_loss, val_err))
time_elapsed = time.time() - since
print('Total Time {:.0f}m {:.0f}s\n'.format(
time_elapsed // 60, time_elapsed % 60))
### Save Metrics ###
exp.save_history('train', trn_loss, trn_err)
exp.save_history('val', val_loss, val_err)
### Checkpoint ###
exp.save_weights(model, trn_loss, val_loss, trn_err, val_err)
exp.save_optimizer(optimizer, val_loss)
## Early Stopping ##
if (epoch - exp.best_val_loss_epoch) > args.MAX_PATIENCE:
print(("Early stopping at epoch %d since no "
+"better loss found since epoch %.3").format(epoch, exp.best_val_loss))
break
# Adjust Lr ###--old method
utils.adjust_learning_rate(args.LEARNING_RATE, args.LR_DECAY, optimizer,
epoch, args.DECAY_LR_EVERY_N_EPOCHS)
exp.epoch += 1
def main(feature_path=None,
batch_size=2048,
mini_batch_size=256,
lr=1,
optim="SGD",
momentum=0.9,
nesterov=False,
noise_multiplier=1,
max_grad_norm=0.1,
max_epsilon=None,
epochs=100,
logdir=None):
logger = Logger(logdir)
device = get_device()
# get pre-computed features
x_train = np.load(f"{feature_path}_train.npy")
x_test = np.load(f"{feature_path}_test.npy")
train_data, test_data = get_data("cifar10", augment=False)
y_train = np.asarray(train_data.targets)
y_test = np.asarray(test_data.targets)
trainset = torch.utils.data.TensorDataset(torch.from_numpy(x_train),
torch.from_numpy(y_train))
testset = torch.utils.data.TensorDataset(torch.from_numpy(x_test),
torch.from_numpy(y_test))
bs = batch_size
assert bs % mini_batch_size == 0
n_acc_steps = bs // mini_batch_size
train_loader = torch.utils.data.DataLoader(trainset,
batch_size=mini_batch_size,
shuffle=True,
num_workers=1,
pin_memory=True,
drop_last=True)
test_loader = torch.utils.data.DataLoader(testset,
batch_size=mini_batch_size,
shuffle=False,
num_workers=1,
pin_memory=True)
n_features = x_train.shape[-1]
try:
mean = np.load(f"{feature_path}_mean.npy")
var = np.load(f"{feature_path}_var.npy")
except FileNotFoundError:
mean = np.zeros(n_features, dtype=np.float32)
var = np.ones(n_features, dtype=np.float32)
bn_stats = (torch.from_numpy(mean).to(device),
torch.from_numpy(var).to(device))
model = nn.Sequential(StandardizeLayer(bn_stats),
nn.Linear(n_features, 10)).to(device)
if optim == "SGD":
optimizer = torch.optim.SGD(model.parameters(),
lr=lr,
momentum=momentum,
nesterov=nesterov)
else:
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
privacy_engine = PrivacyEngine(
model,
sample_rate=bs / len(train_data),
alphas=ORDERS,
noise_multiplier=noise_multiplier,
max_grad_norm=max_grad_norm,
)
privacy_engine.attach(optimizer)
for epoch in range(0, epochs):
print(f"\nEpoch: {epoch}")
train_loss, train_acc = train(model,
train_loader,
optimizer,
n_acc_steps=n_acc_steps)
test_loss, test_acc = test(model, test_loader)
if noise_multiplier > 0:
rdp_sgd = get_renyi_divergence(
privacy_engine.sample_rate,
privacy_engine.noise_multiplier) * privacy_engine.steps
epsilon, _ = get_privacy_spent(rdp_sgd)
print(f"ε = {epsilon:.3f}")
if max_epsilon is not None and epsilon >= max_epsilon:
return
else:
epsilon = None
logger.log_epoch(epoch, train_loss, train_acc, test_loss, test_acc,
epsilon)
def main(args):
# Setting
warnings.simplefilter("ignore", UserWarning)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# Args Parser
hj_method = args.hj_method
kr_method = args.kr_method
batch_size = args.batch_size
beam_size = args.beam_size
hidden_size = args.hidden_size
embed_size = args.embed_size
vocab_size = args.vocab_size
max_len = args.max_len
padding_index = args.pad_id
n_layers = args.n_layers
stop_ix = args.stop_ix
# Load saved model & Word2vec
save_path = 'save_{}_{}_{}_maxlen_{}'.format(vocab_size, hj_method,
kr_method, max_len)
save_list = sorted(glob.glob(f'./save/{save_path}/*.*'))
save_pt = save_list[-1]
print('Will load {} pt file...'.format(save_pt))
word2vec_hj = Word2Vec.load('./w2v/word2vec_hj_{}_{}.model'.format(
vocab_size, hj_method))
# SentencePiece model load
spm_kr = spm.SentencePieceProcessor()
spm_kr.Load("./spm/m_korean_{}.model".format(vocab_size))
# Test data load
with open('./test_dat.pkl', 'rb') as f:
test_dat = pickle.load(f)
test_dataset = CustomDataset(test_dat['test_hanja'],
test_dat['test_korean'])
test_loader = getDataLoader(test_dataset,
pad_index=padding_index,
shuffle=False,
batch_size=batch_size)
# Model load
print('Model loading...')
encoder = Encoder(vocab_size,
embed_size,
hidden_size,
word2vec_hj,
n_layers=n_layers,
padding_index=padding_index)
decoder = Decoder(embed_size,
hidden_size,
vocab_size,
n_layers=n_layers,
padding_index=padding_index)
seq2seq = Seq2Seq(encoder, decoder, beam_size).cuda()
#optimizer = optim.Adam(seq2seq.parameters(), lr=lr, weight_decay=w_decay)
#scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=scheduler_step_size, gamma=lr_decay)
print(seq2seq)
print('Testing...')
start_time = time.time()
results = test(seq2seq,
test_loader,
vocab_size,
load_pt=save_pt,
stop_ix=stop_ix)
print(time.time() - start_time)
print('Done!')
print("Decoding...")
pred_list = list()
for result_text in tqdm(results):
text = torch.Tensor(result_text).squeeze().tolist()
text = [int(x) for x in text]
prediction_sentence = spm_kr.decode_ids(
text).strip() # Decode with strip
pred_list.append(prediction_sentence)
ref_list = list()
for ref_text in tqdm(test_dat['test_korean'][:stop_ix]):
ref_list.append(spm_kr.decode_ids(ref_text).strip())
print('Done!')
with open(f'./save/{save_path}/test_result.pkl', 'wb') as f:
pickle.dump({
'pred': pred_list,
'reference': ref_list,
}, f)
print('Save file; /test_dat.pkl')
# Calculate BLEU Score
print('Calculate BLEU4, METEOR, Rogue-L...')
chencherry = SmoothingFunction()
bleu4 = corpus_bleu(test_dat['reference'],
test_dat['pred'],
smoothing_function=chencherry.method4)
print('BLEU Score is {}'.format(bleu4))
# Calculate METEOR Score
meteor = meteor_score(test_dat['reference'], test_dat['pred'])
print('METEOR Score is {}'.format(meteor))
# Calculate Rouge-L Score
r = Rouge()
total_test_length = len(test_dat['reference'])
precision_all = 0
recall_all = 0
f_score_all = 0
for i in range(total_test_length):
[precision, recall, f_score] = r.rouge_l([test_dat['reference'][i]],
[test_dat['pred'][i]])
precision_all += precision
recall_all += recall
f_score_all += f_score
print('Precision : {}'.foramt(round(precision_all / total_test_length, 4)))
print('Recall : {}'.foramt(round(recall_all / total_test_length, 4)))
print('F Score : {}'.foramt(round(f_score_all / total_test_length, 4)))
def main(dataset,
augment=False,
batch_size=2048,
mini_batch_size=256,
sample_batches=False,
lr=1,
optim="SGD",
momentum=0.9,
nesterov=False,
noise_multiplier=1,
max_grad_norm=0.1,
epochs=100,
input_norm=None,
num_groups=None,
bn_noise_multiplier=None,
max_epsilon=None,
logdir=None):
logger = Logger(logdir)
device = get_device()
train_data, test_data = get_data(dataset, augment=augment)
scattering, K, (h, w) = get_scatter_transform(dataset)
scattering.to(device)
bs = batch_size
assert bs % mini_batch_size == 0
n_acc_steps = bs // mini_batch_size
# Batch accumulation and data augmentation with Poisson sampling isn't implemented
if sample_batches:
assert n_acc_steps == 1
assert not augment
train_loader = torch.utils.data.DataLoader(train_data,
batch_size=mini_batch_size,
shuffle=True,
num_workers=1,
pin_memory=True)
test_loader = torch.utils.data.DataLoader(test_data,
batch_size=mini_batch_size,
shuffle=False,
num_workers=1,
pin_memory=True)
rdp_norm = 0
if input_norm == "BN":
# compute noisy data statistics or load from disk if pre-computed
save_dir = f"bn_stats/{dataset}"
os.makedirs(save_dir, exist_ok=True)
bn_stats, rdp_norm = scatter_normalization(
train_loader,
scattering,
K,
device,
len(train_data),
len(train_data),
noise_multiplier=bn_noise_multiplier,
orders=ORDERS,
save_dir=save_dir)
model = ScatterLinear(K, (h, w), input_norm="BN", bn_stats=bn_stats)
else:
model = ScatterLinear(K, (h, w),
input_norm=input_norm,
num_groups=num_groups)
model.to(device)
trainable_params = sum(p.numel() for p in model.parameters()
if p.requires_grad)
print(f'model: {model}\n')
print(f'has {trainable_params / 1e6:.4f} million trainable parameters')
if augment:
model = nn.Sequential(scattering, model)
train_loader = torch.utils.data.DataLoader(train_data,
batch_size=mini_batch_size,
shuffle=True,
num_workers=1,
pin_memory=True,
drop_last=True)
preprocessor = None
else:
preprocessor = lambda x, y: (scattering(x), y)
# baseline Logistic Regression without privacy
if optim == "LR":
assert not augment
X_train = []
y_train = []
X_test = []
y_test = []
for data, target in train_loader:
with torch.no_grad():
data = data.to(device)
X_train.append(data.cpu().numpy().reshape(len(data), -1))
y_train.extend(target.cpu().numpy())
for data, target in test_loader:
with torch.no_grad():
data = data.to(device)
X_test.append(data.cpu().numpy().reshape(len(data), -1))
y_test.extend(target.cpu().numpy())
import numpy as np
X_train = np.concatenate(X_train, axis=0)
X_test = np.concatenate(X_test, axis=0)
y_train = np.asarray(y_train)
y_test = np.asarray(y_test)
print(X_train.shape, y_train.shape, X_test.shape, y_test.shape)
for idx, C in enumerate([0.01, 0.1, 1.0, 10, 100]):
clf = LogisticRegression(C=C, fit_intercept=True)
clf.fit(X_train, y_train)
train_acc = 100 * clf.score(X_train, y_train)
test_acc = 100 * clf.score(X_test, y_test)
print(f"C={C}, "
f"Acc train = {train_acc: .2f}, "
f"Acc test = {test_acc: .2f}")
logger.log_epoch(idx, 0, train_acc, 0, test_acc, None)
return
print(f"model has {get_num_params(model)} parameters")
if optim == "SGD":
optimizer = torch.optim.SGD(model.parameters(),
lr=lr,
momentum=momentum,
nesterov=nesterov)
else:
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
privacy_engine = PrivacyEngine(
model,
batch_size=bs,
sample_size=len(train_data),
alphas=ORDERS,
noise_multiplier=noise_multiplier,
max_grad_norm=max_grad_norm,
)
privacy_engine.attach(optimizer)
for epoch in range(0, epochs):
print(f"\nEpoch: {epoch}")
train_loss, train_acc = train(model,
train_loader,
optimizer,
n_acc_steps=n_acc_steps,
preprocessor=preprocessor)
test_loss, test_acc = test(model,
test_loader,
preprocessor=preprocessor)
if noise_multiplier > 0:
rdp_sgd = get_renyi_divergence(
privacy_engine.sample_rate,
privacy_engine.noise_multiplier) * privacy_engine.steps
epsilon, _ = get_privacy_spent(rdp_norm + rdp_sgd)
epsilon2, _ = get_privacy_spent(rdp_sgd)
print(f"ε = {epsilon:.3f} (sgd only: ε = {epsilon2:.3f})")
if max_epsilon is not None and epsilon >= max_epsilon:
return
else:
epsilon = None
logger.log_epoch(epoch, train_loss, train_acc, test_loss, test_acc,
epsilon)
logger.log_scalar("epsilon/train", epsilon, epoch)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# # set dataloader and model
if not opt.train == 3 or not opt.train == 4:
dataloader, dataloader_test = data_loader()
netG, netD, desc = define_model(device)
# config loss
if opt.coeff_color_loss:
print('use color loss')
gradient_critcriterion = MaskedGradient(opt).to(device)
# set noise
NZ = opt.imageSize // 2 ** nDep
noise = torch.FloatTensor(opt.batchSize, opt.zLoc, NZ, NZ)
fixnoise = torch.FloatTensor(opt.batchSize, opt.zLoc, NZ * 4, NZ * 4)
noise = noise.to(device)
fixnoise = fixnoise.to(device)
Noise = [NZ, noise, fixnoise]
# setup optimizer
optimizerD = optim.Adam(netD.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999)) # netD.parameters()
optimizerG = optim.Adam(netG.parameters(), lr=opt.lr, betas=(opt.beta1, 0.999))
if opt.train == 0:
test(opt, dataloader_test, device, netD, netG, Noise, desc)
elif opt.train == 1:
train(opt, dataloader, dataloader_test, device, netD, netG, desc, Noise, optimizerD, optimizerG)
elif opt.train == 2:
save_embedding(opt, dataloader_test, device, netD)
elif opt.train == 3:
interpolation(opt, device, netG, Noise)
elif opt.train == 4:
all_composition(opt, device, netG, Noise)
print('STARTING TRAINING')
for epoch in range(1, args.epochs + 1):
train(args,
model,
device,
train_loader,
optimizer,
epoch,
start_time=time.time())
f1 = get_mean_F1(model, validation_loader)
print('after epoch {} got f1 score of {}'.format(epoch, f1))
if f1 > info['highest F1']:
info['highest F1'] = np.copy(f1)
info['saved epoch'] = epoch
test(args, model, device, test_loader, epoch, trainDataset,
testDataset, path_submission)
torch.save(model, path_model)
print('currently best model --> saved')
print('TRAINING DONE')
print(info)
# ### Load Model
# In[ ]:
#if load_model:
# model = torch.load(path_model)
# In[ ]:
# start_epoch = chechpoint['epoch']
# model.load_state_dict(chechpoint['state_dict'])
# optimizer.load_state_dict(chechpoint['optimizer'])
# logger = Logger(os.path.join(checkpoint_dir, 'log.txt'), title=title, resume=True)
# else:
# logger = Logger(os.path.join(checkpoint_path, 'log.txt'), title=title)
# logger.set_names(['Learning Rate', 'Train Loss', 'Valid Loss', 'Train Acc.', 'Valid Acc.'])
# train ------------
for epoch in range(start_epoch, tot_epoch):
print('\nEpoch [%d | %d] LR: %f' %
(epoch + 1, tot_epoch, optimizer.param_groups[0]['lr']))
train_loss, train_acc, train_5 = train(train_dataloader, model, criterion,
optimizer, epoch, device, writer)
test_loss, test_acc, test_5 = test(val_dataloader, model, criterion, epoch,
device, writer)
# logger.append([ lr, train_loss, test_loss, train_acc, test_acc ])
print(
'train_loss:%f, val_loss:%f, train_acc:%f, train_5:%f, val_acc:%f, val_5:%f'
% (train_loss, test_loss, train_acc, train_5, test_acc, test_5))
is_best = test_acc > best_acc
best_acc = max(test_acc, best_acc)
save_checkpoint(
{
'fold': 1,
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'train_acc': train_acc,
def main(dataset,
augment=False,
use_scattering=False,
size=None,
batch_size=2048,
mini_batch_size=256,
sample_batches=False,
lr=1,
optim="SGD",
momentum=0.9,
nesterov=False,
noise_multiplier=1,
max_grad_norm=0.1,
epochs=100,
input_norm=None,
num_groups=None,
bn_noise_multiplier=None,
max_epsilon=None,
logdir=None,
early_stop=True,
seed=0):
torch.manual_seed(seed)
logger = Logger(logdir)
device = get_device()
train_data, test_data = get_data(dataset, augment=augment)
if use_scattering:
scattering, K, _ = get_scatter_transform(dataset)
scattering.to(device)
else:
scattering = None
K = 3 if len(train_data.data.shape) == 4 else 1
bs = batch_size
assert bs % mini_batch_size == 0
n_acc_steps = bs // mini_batch_size
# Batch accumulation and data augmentation with Poisson sampling isn't implemented
if sample_batches:
assert n_acc_steps == 1
assert not augment
train_loader = torch.utils.data.DataLoader(train_data,
batch_size=mini_batch_size,
shuffle=True,
num_workers=1,
pin_memory=True)
test_loader = torch.utils.data.DataLoader(test_data,
batch_size=mini_batch_size,
shuffle=False,
num_workers=1,
pin_memory=True)
rdp_norm = 0
if input_norm == "BN":
# compute noisy data statistics or load from disk if pre-computed
save_dir = f"bn_stats/{dataset}"
os.makedirs(save_dir, exist_ok=True)
bn_stats, rdp_norm = scatter_normalization(
train_loader,
scattering,
K,
device,
len(train_data),
len(train_data),
noise_multiplier=bn_noise_multiplier,
orders=ORDERS,
save_dir=save_dir)
model = CNNS[dataset](K, input_norm="BN", bn_stats=bn_stats, size=size)
else:
model = CNNS[dataset](K,
input_norm=input_norm,
num_groups=num_groups,
size=size)
model.to(device)
if use_scattering and augment:
model = nn.Sequential(scattering, model)
train_loader = torch.utils.data.DataLoader(train_data,
batch_size=mini_batch_size,
shuffle=True,
num_workers=1,
pin_memory=True,
drop_last=True)
else:
# pre-compute the scattering transform if necessery
train_loader = get_scattered_loader(train_loader,
scattering,
device,
drop_last=True,
sample_batches=sample_batches)
test_loader = get_scattered_loader(test_loader, scattering, device)
print(f"model has {get_num_params(model)} parameters")
if optim == "SGD":
optimizer = torch.optim.SGD(model.parameters(),
lr=lr,
momentum=momentum,
nesterov=nesterov)
else:
optimizer = torch.optim.Adam(model.parameters(), lr=lr)
privacy_engine = PrivacyEngine(
model,
batch_size=bs,
sample_size=len(train_data),
alphas=ORDERS,
noise_multiplier=noise_multiplier,
max_grad_norm=max_grad_norm,
)
privacy_engine.attach(optimizer)
best_acc = 0
flat_count = 0
results = dict(train_zeon=[],
train_xent=[],
test_zeon=[],
test_xent=[],
epoch=[])
for epoch in range(0, epochs):
print(f"\nEpoch: {epoch}")
train_loss, train_acc = train(model,
train_loader,
optimizer,
n_acc_steps=n_acc_steps)
test_loss, test_acc = test(model, test_loader)
results['train_zeon'].append(train_acc)
results['train_xent'].append(train_loss)
results['test_zeon'].append(test_acc)
results['test_xent'].append(test_loss)
results['epoch'].append(epoch)
if noise_multiplier > 0:
rdp_sgd = get_renyi_divergence(
privacy_engine.sample_rate,
privacy_engine.noise_multiplier) * privacy_engine.steps
epsilon, _ = get_privacy_spent(rdp_norm + rdp_sgd)
epsilon2, _ = get_privacy_spent(rdp_sgd)
print(f"ε = {epsilon:.3f} (sgd only: ε = {epsilon2:.3f})")
if max_epsilon is not None and epsilon >= max_epsilon:
return
else:
epsilon = None
logger.log_epoch(epoch, train_loss, train_acc, test_loss, test_acc,
epsilon)
logger.log_scalar("epsilon/train", epsilon, epoch)
# stop if we're not making progress
if test_acc > best_acc:
best_acc = test_acc
flat_count = 0
else:
flat_count += 1
if flat_count >= 20 and early_stop:
print("plateau...")
break
# Write to file.
record = {
**results,
**{
'best_acc': best_acc,
'seed': seed,
'dataset': dataset
}
}
record_path = os.path.join('.', 'record', f'{dataset}-{seed}.json')
os.makedirs(os.path.dirname(record_path), exist_ok=True)
with open(record_path, 'w') as f:
json.dump(record, f, indent=4)
import logging
logging.warning(f'Wrote to file: {record_path}')
def main():
parser = argparse.ArgumentParser()
# parser.add_argument('--DATASET_PATH', type=str, default='/data/davis/dataset/DUTS/')
# parser.add_argument('--EXPERIMENT', type=str, default='/data/davis/result/TrainNet/')
parser.add_argument('--DATASET_PATH',
type=str,
default='/home/yangle/DAVIS/dataset/DUTS/')
parser.add_argument('--EXPERIMENT',
type=str,
default='/home/yangle/DAVIS/result/TrainNet/')
parser.add_argument('--N_EPOCHS', type=int, default=200)
parser.add_argument('--MAX_PATIENCE', type=int, default=30)
parser.add_argument('--batch_size', type=int, default=32)
parser.add_argument('--seed', type=int, default=0)
parser.add_argument('--N_CLASSES', type=int, default=2)
parser.add_argument('--LEARNING_RATE', type=float, default=1e-4)
parser.add_argument('--LR_DECAY', type=float, default=0.995)
parser.add_argument('--DECAY_LR_EVERY_N_EPOCHS', type=int, default=1)
parser.add_argument('--WEIGHT_DECAY', type=float, default=0.0001)
parser.add_argument('--CUDNN', type=bool, default=True)
args = parser.parse_args()
torch.cuda.manual_seed(args.seed)
cudnn.benchmark = args.CUDNN
normalize = transforms.Normalize(mean=dataset.mean, std=dataset.std)
train_joint_transformer_img = transforms.Compose(
[joint_transforms.JointResize(224)])
mask_size_list = [28, 28, 28, 56, 112]
train_dset = dataset.Saliency(args.DATASET_PATH,
'train',
train_joint_transformer_img,
mask_size_list,
transform=transforms.Compose([
transforms.ToTensor(),
normalize,
]))
train_loader = torch.utils.data.DataLoader(train_dset,
batch_size=args.batch_size,
shuffle=True)
test_joint_transforms_img = transforms.Compose(
[joint_transforms.JointResize(224)])
val_dset = dataset.Saliency(args.DATASET_PATH,
'val',
test_joint_transforms_img,
mask_size_list,
transform=transforms.Compose(
[transforms.ToTensor(), normalize]))
val_loader = torch.utils.data.DataLoader(val_dset,
batch_size=args.batch_size,
shuffle=False)
print("TrainImages: %d" % len(train_loader.dataset.imgs))
print("ValImages: %d" % len(val_loader.dataset.imgs))
example_inputs, example_targets = next(iter(train_loader))
print("InputsBatchSize: ", example_inputs.size())
print("TargetsBatchSize: ", len(example_targets))
print("\nInput (size, max, min) ---")
# input
i = example_inputs[0]
print(i.size())
print(i.max())
print(i.min())
print("Target (size, max, min) ---")
# target
for mask in example_targets:
print(mask.size())
print(mask.max())
print(mask.min())
resnet50 = torchvision.models.resnet50(pretrained=True)
dict_resnet50 = resnet50.state_dict()
model = SegNet.resnet50()
# # initialize
model.apply(utils.weights_init)
SegNet_dict = model.state_dict()
pretrained_dict = {
k: v
for k, v in dict_resnet50.items() if k in SegNet_dict
}
# for k in pretrained_dict:
# print(k)
SegNet_dict.update(pretrained_dict)
model.load_state_dict(SegNet_dict)
# seperate layers, to set different lr
param_exist = []
param_add = []
for k, (name, module) in enumerate(model.named_children()):
# existing layers including: conv1 bn1 relu maxpool
# layer1 layer2 layer3 layer4
if k < 8:
for param in module.parameters():
param_exist.append(param)
# adding layers including: bottleneck skip3 skip2 skip1 skip0
# conv_end_1 bn_end_1 salmap Sigmoid mask0 mask4 mask3 mask2 mask1
else:
for param in module.parameters():
param_add.append(param)
model = torch.nn.DataParallel(model).cuda()
optimizer = optim.SGD([{
'params': param_exist,
'lr': 0.001
}, {
'params': param_add
}],
lr=0.01,
momentum=0.9,
weight_decay=0.0005)
print(' + Number of params: {}'.format(
sum([p.data.nelement() for p in model.parameters()])))
# optimizer = optim.RMSprop(model.parameters(), lr=args.LEARNING_RATE,
# weight_decay=args.WEIGHT_DECAY, eps=1e-12)
criterion = nn.BCELoss().cuda()
exp_dir = args.EXPERIMENT + 'ResNet50'
if os.path.exists(exp_dir):
shutil.rmtree(exp_dir)
exp = experiment.Experiment('ResNet50', args.EXPERIMENT)
exp.init()
START_EPOCH = exp.epoch
END_EPOCH = START_EPOCH + args.N_EPOCHS
for epoch in range(START_EPOCH, END_EPOCH):
since = time.time()
# ### Train ###
trn_loss, trn_err = utils.train(model, train_loader, optimizer,
criterion, epoch)
print('Epoch {:d}: Train - Loss: {:.4f}\tErr: {:.4f}'.format(
epoch, trn_loss, trn_err))
time_elapsed = time.time() - since
print('Train Time {:.0f}m {:.0f}s'.format(time_elapsed // 60,
time_elapsed % 60))
### Test ###
val_loss, val_err = utils.test(model, val_loader, criterion, epoch)
print('Val - Loss: {:.4f}, Error: {:.4f}'.format(val_loss, val_err))
time_elapsed = time.time() - since
print('Total Time {:.0f}m {:.0f}s\n'.format(time_elapsed // 60,
time_elapsed % 60))
### Save Metrics ###
exp.save_history('train', trn_loss, trn_err)
exp.save_history('val', val_loss, val_err)
### Checkpoint ###
exp.save_weights(model, trn_loss, val_loss, trn_err, val_err)
exp.save_optimizer(optimizer, val_loss)
## Early Stopping ##
if (epoch - exp.best_val_loss_epoch) > args.MAX_PATIENCE:
print(("Early stopping at epoch %d since no " +
"better loss found since epoch %.3").format(
epoch, exp.best_val_loss))
break
# Adjust Lr ###--old method
# utils.adjust_learning_rate(args.LEARNING_RATE, args.LR_DECAY, optimizer,
# epoch, args.DECAY_LR_EVERY_N_EPOCHS)
epoch_finish = epoch - 1
if epoch_finish > 0 and epoch % 2 == 0:
utils.adjust_learning_rate_SGD(optimizer, decay_rate=0.5)
exp.epoch += 1
