def main():
params = {'model': 'lenet',
'display_step': 250, # we decreased this value for HW7
'batch_size': 1024,
'iterations': 30_000, # 5000 or 30000
'initial_lr': 0.05, # we decreased this value for HW7
'lr_decay': 0.5,
'adjust_lr_step': 10_000,
'initial_momentum': 0.5,
'final_momentum': 0.95,
'momentum_change_steps': 20_000,
'adjust_momentum_step': 2_000,
'apply_weight_norm': True,
'weight_norm': 3.5,
'adjust_norm_step': 5_000,
'output_l2_decay': 0.001,
'pooling': 'max', # TODO try three options: 'max', 'avg','no'
'activation':'relu',
'random_seed': 42}
# we increased n_train for HW7
data = prepare_MNIST_data(n_train=20_000, n_val=10_000, n_test=5_000,
use_norm_shift=False, use_norm_scale=True, use_data_augmentation=False)
X_train, y_train, X_val, y_val, _, _ = data.values()
insample_dataloader = get_dataloader(X_train, y_train, batch_size=params['batch_size'])
outsample_dataloader = get_dataloader(X_val, y_val, batch_size=params['batch_size'])
clf = LeNet(params)
clf.fit(insample_dataloader, outsample_dataloader)
def train(config):
seed = config["seed"]
torch.manual_seed(seed)
torch.cuda.manual_seed(seed)
trainTransform, testTransform = data.get_transforms(config)
trainLoader = data.get_dataloader(config,
train=True,
transform=trainTransform)
testLoader = data.get_dataloader(config,
train=False,
transform=testTransform)
densenet = model.get_model(config).to(config["device"])
opt, scheduler = optimizer.get_scheduled_optimizer(config, densenet)
criterion = CrossEntropyLoss()
recorder = Recorder(config)
max_epoch = config["max_epoch"]
for epoch in range(max_epoch):
print("epoch:{:0>3}".format(epoch))
train_out = _train(densenet, criterion, trainLoader, config["device"],
opt)
recorder(epoch, train_out=train_out)
test_out = _test(densenet,
criterion,
testLoader,
config["device"],
need_output_y=False)
recorder(epoch, test_out=test_out)
scheduler.step()
torch.save(densenet, os.path.join(config["save_dir"], "latest.pth"))
print("train finished")
return None
def main():
data = prepare_MNIST_data(n_train=10_000, n_val=10_000, n_test=5_000,
use_norm_shift=False, use_norm_scale=True, use_data_augmentation=False)
X_train, y_train, X_val, y_val, _, _ = data.values()
insample_dataloader = get_dataloader(X_train, y_train, batch_size=params['batch_size'])
outsample_dataloader = get_dataloader(X_val, y_val, batch_size=params['batch_size'])
clf = DropoutClassifier(params)
clf.fit(insample_dataloader, outsample_dataloader)
def validate_dataset(model, split, tokenizer, dataset, topk=1):
assert split in ('dev', 'test')
fwd_dataloader = get_dataloader('xlnet', split, tokenizer, bwd=False, \
batch_size=16, num_workers=16, prefix=dataset)
bwd_dataloader = get_dataloader('xlnet', split, tokenizer, bwd=True, \
batch_size=16, num_workers=16, prefix=dataset)
em, f1, count = 0, 0, 0
model.start_n_top = topk
model.end_n_top = topk
model.eval()
for fwd_batch, bwd_batch in zip(fwd_dataloader, bwd_dataloader):
# Forward
input_ids, attention_mask, token_type_ids, fwd_input_tokens_no_unk, answers = fwd_batch
input_ids = input_ids.cuda(device=device)
attention_mask = attention_mask.cuda(device=device)
token_type_ids = token_type_ids.cuda(device=device)
with torch.no_grad():
outputs = model(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)
fwd_start_index = outputs[1]
fwd_end_index = outputs[3].view(-1, model.end_n_top, model.start_n_top).permute([0,2,1])[:,:,0]
# Backward
input_ids, attention_mask, token_type_ids, bwd_input_tokens_no_unk, answers = bwd_batch
input_ids = input_ids.cuda(device=device)
attention_mask = attention_mask.cuda(device=device)
token_type_ids = token_type_ids.cuda(device=device)
with torch.no_grad():
outputs = model(input_ids, attention_mask=attention_mask, token_type_ids=token_type_ids)
bwd_start_index = outputs[1]
bwd_end_index = outputs[3].view(-1, model.end_n_top, model.start_n_top).permute([0,2,1])[:,:,0] + 1
for i, answer in enumerate(answers):
preds = []
for k in range(model.start_n_top):
fwd_pred_tokens = fwd_input_tokens_no_unk[i][fwd_start_index[i][k]:fwd_end_index[i][k]]
bwd_pred_tokens = bwd_input_tokens_no_unk[i][bwd_start_index[i][k]:bwd_end_index[i][k]]
preds.append(tokenizer.convert_tokens_to_string(fwd_pred_tokens))
preds.append(tokenizer.convert_tokens_to_string(bwd_pred_tokens))
norm_preds_tokens = [norm_tokenizer.basic_tokenizer.tokenize(pred) for pred in preds]
norm_preds = [norm_tokenizer.convert_tokens_to_string(norm_pred_tokens) for norm_pred_tokens in norm_preds_tokens]
norm_answer_tokens = [norm_tokenizer.basic_tokenizer.tokenize(ans) for ans in answer]
norm_answer = [norm_tokenizer.convert_tokens_to_string(ans_tokens) for ans_tokens in norm_answer_tokens]
em += max(metric_max_over_ground_truths(exact_match_score, norm_pred, norm_answer) for norm_pred in norm_preds)
f1 += max(metric_max_over_ground_truths(f1_score, norm_pred, norm_answer) for norm_pred in norm_preds)
count += 1
del fwd_dataloader, bwd_dataloader
return em, f1, count
def main():
data = prepare_MNIST_data(n_train=10_000, n_val=10_000, n_test=5_000,
use_norm_shift=False, use_norm_scale=True, use_data_augmentation=False)
X_train, y_train, X_val, y_val, _, _ = data.values()
insample_dataloader = get_dataloader(X_train, y_train, batch_size=params['batch_size'])
outsample_dataloader = get_dataloader(X_val, y_val, batch_size=params['batch_size'])
clf = DropoutClassifier(params)
clf.fit(insample_dataloader, outsample_dataloader)
# To avoid unnecesary pain, we recommend you to save your classifiers
clf.save_weights('./your_results_path/')
clf.load_weights('./your_results_path/')
def test_simple_cnn_model():
args = update_args(cfg_file='simple_cnn')
device = 'cpu'
train_data_dict = get_dataset(args).train_data_dict
train_dataloader = get_dataloader(
batch_size=args.TRAIN.BATCH_SIZE,
dataset=train_data_dict['dataset'],
num_workers=args.DATA.NUM_WORKERS,
sampler=train_data_dict['train_sampler'],
)
validation_dataloader = get_dataloader(
batch_size=args.TRAIN.BATCH_SIZE,
dataset=train_data_dict['dataset'],
num_workers=args.DATA.NUM_WORKERS,
sampler=train_data_dict['validation_sampler'],
)
model = get_model(
args=args,
device=device,
hparams={
'learning rate': args.TRAIN.LR,
'batch size': args.TRAIN.BATCH_SIZE,
},
)
for data, labels in train_dataloader:
data, labels = data.to(device), labels.to(device)
outputs = model(data)
assert data.shape[0] == labels.shape[0] == outputs.shape[
0] == args.TRAIN.BATCH_SIZE
assert data.shape[1] == 3
assert outputs.shape[1] == len(args.DATA.CLASSES)
break
for data, labels in validation_dataloader:
data, labels = data.to(device), labels.to(device)
outputs = model(data)
assert data.shape[0] == labels.shape[0] == outputs.shape[
0] == args.TRAIN.BATCH_SIZE
assert data.shape[1] == 3
assert outputs.shape[1] == len(args.DATA.CLASSES)
break
def task1():
task_name = "base_dpn62_balance"
makedir(os.path.join(cfg.log_dir, task_name))
log = Logger(os.path.join(cfg.log_dir, task_name + '_log.txt'), mode="a")
log("\n\n" + '-' * 51 +
"[START %s]" % datetime.now().strftime('%Y-%m-%d %H:%M:%S') +
"-" * 51 + "\n\n")
print(cfg, file=log)
train_loader, val_loader, test_loader = get_dataloader()
model = DPN68()
model.cuda()
criterion1 = nn.BCEWithLogitsLoss().cuda()
criterion2 = BalanceLoss().cuda()
criterions = [criterion1, criterion2]
optimizer = optim.SGD(model.parameters(),
lr=cfg.lr,
momentum=0.9,
weight_decay=1e-5)
scheduler = lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.1)
model = train(task_name,
model,
optimizer,
criterions,
scheduler,
train_loader,
val_loader,
log=log)
submission_best_f1(task_name, model, test_loader, log=log)
def train_and_eval(cfg):
train_loader, val_loader, test_loader, num_classes = get_dataloader(
cfg.data.name, cfg.data.val_size, cfg.data.batch_size,
cfg.data.download, cfg.augment, False)
model = get_model(cfg.model.name, num_classes)
optimizer = optim.SGD(cfg.optim.model.lr,
momentum=0.9,
weight_decay=cfg.optim.model.weight_decay)
scheduler = lr_scheduler.MultiStepLR(cfg.optim.model.steps)
tq = reporters.TQDMReporter(range(cfg.optim.epochs), verb=cfg.verb)
callback = [
callbacks.AccuracyCallback(),
callbacks.LossCallback(),
reporters.TensorboardReporter("."),
reporters.IOReporter("."), tq
]
with trainers.SupervisedTrainer(model,
optimizer,
F.cross_entropy,
callbacks=callback,
scheduler=scheduler) as trainer:
for ep in tq:
trainer.train(train_loader)
trainer.test(val_loader, 'val')
trainer.test(test_loader)
def train(args):
#torch.manual_seed(args.seed)
#torch.cuda.manual_seed(args.seed)
dataset = data.get_dataset(args.dataset, training=True)
model = DFVE(args.image_channels, args.image_size, args.n_latent, args.lambda_, args.gamma).to(args.device)
optimizer = optim.Adam(model.parameters(),
lr=args.learning_rate,
betas=(args.adam_beta1, args.adam_beta2),
weight_decay=args.weight_decay)
model.train()
step = 0
epoch = 0
for _ in range(args.n_epochs):
epoch += 1
loader, _ = data.get_dataloader(dataset, args.batch_size)
for samples, labels in loader:
step += 1
x = samples.to(args.device).float()
z_mean, z_logvar = model(x)
loss, mmd_loss_from_prior, mmd_loss_for_mi = model.loss(z_mean, z_logvar, args.repeats)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if step % args.print_freq == 0:
print('[Epoch {:d}, Step {:d}] loss: {:.4f}, mmd_loss_from_prior: {:.4f}, mmd_loss_for_mi: {:.4f}'.format(
epoch, step, loss.item(), mmd_loss_from_prior.item(), mmd_loss_for_mi.item()))
monitor(z_mean, z_logvar, labels, epoch, step)
model.monitor()
def train(args):
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
dataset = data.get_dataset(args.dataset, training=True)
model = DFVE(args.image_channels, args.image_size, args.n_latent).to(args.device)
optimizer = optim.Adam(model.parameters(),
lr=args.learning_rate,
betas=(args.adam_beta1, args.adam_beta2),
weight_decay=args.weight_decay)
model.train()
step = 0
epoch = 0
for _ in range(args.n_epochs):
epoch += 1
loader, _ = data.get_dataloader(dataset, args.batch_size)
for samples, labels in loader:
step += 1
x = samples.to(args.device).float()
z = model(x, args.repeats, args.noise_sigma)
loss, mmd_loss_all, mmd_loss_avg = model.loss(z, args.gamma, args.kernel_gamma, args.kernel_power,
args.gnorm_mu, args.gnorm_sigma, args.gnorm_alpha)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if step % args.print_freq == 0:
print('[Epoch {:d}, Step {:d}] loss: {:.4f}, mmd_loss_all: {:.4f}, mmd_loss_avg: {:.4f}'.format(
epoch, step, loss.item(), mmd_loss_all.item(), mmd_loss_avg.item()))
if step % args.show_freq == 0:
monitor(z, labels, epoch, step)
model.monitor()
def test():
"""
train the model by the args
:param args:
:return:
"""
# initialize model
model = EfficientSeg(data_cfg.num_classes, compound_coef=cfg.compound_coef)
load_state_dict(model)
model = model.to(device)
# test model
model.eval()
transforms = CommonTransforms(trans_cfg, "val")
decode.device = device
if data_cfg.test_dir is not None:
# initialize the dataloader by dir
test_dataloader = data.get_dataloader(data_cfg.batch_size,
data_cfg.dataset,
data_cfg.test_dir,
with_label=False,
phase="test",
transforms=transforms)
# foreach the images
for iter_id, test_data in enumerate(test_dataloader):
# to device
inputs, infos = test_data
handle_output(inputs, infos, model, transforms)
else:
img_path = data_cfg.test_image
input_img = image.load_rgb_image(img_path)
input, _, info = transforms(input_img, img_path=img_path)
handle_output(input.unsqueeze(0), [info], model, transforms)
logger.close()
def train(name, gen, disc, gen_train_ratio=5, epochs=-1):
device = utils.get_device()
dataloader = data.get_dataloader()
loss_func = nn.BCELoss()
g_optimizer = to.Adam(gen.parameters(), lr=.0003, betas=(.5, .9))
d_optimizer = to.Adam(disc.parameters(), lr=.0003, betas=(.5, .9))
iter_axis = []
g_loss_axis = []
d_loss_axis = []
fixed_noise = torch.randn(64, gen.nz, device=device)
epoch_iterator = range(epochs) if epochs >= 0 else count(0)
iters = 0
for epoch in epoch_iterator:
for i, batch in enumerate(dataloader):
loss_disc, acc_real, acc_fake = train_discriminator(
gen, disc, batch, loss_func, d_optimizer)
for _ in range(gen_train_ratio):
loss_gen, acc_gen = train_generator(gen, disc, loss_func,
g_optimizer)
# Training stats
if i % 50 == 0:
print(
f'[{epoch:2d}/{epochs}][{i:3d}/{len(dataloader)}]\t' +\
f'Loss_D: {loss_disc:3.4f}\tLoss_G: {loss_gen:3.4f}\t' +\
f'D(x): {acc_real:3.4f}\tD(G(z)): {acc_fake:3.4f} / {acc_gen:3.4f}'
)
if (iters % 10 == 0) or ((epoch == epochs - 1) and
(i == len(dataloader) - 1)):
iter_axis.append(iters)
g_loss_axis.append(loss_gen)
d_loss_axis.append(loss_disc)
# Save output
if (iters % 500 == 0) or ((epoch == epochs - 1) and
(i == len(dataloader) - 1)):
print('Saving images...')
with torch.no_grad():
gen.mixing = False
fake = gen(fixed_noise).detach().cpu()
gen.mixing = True
utils.save_image(utils.make_torch_grid(fake),
f'{name}_{iters}.png')
# Save weights
if (iters % 1000 == 0) or ((epoch == epochs - 1) and
(i == len(dataloader) - 1)):
print('Saving weights...')
utils.save_weights(gen, f'{name}_gen.pt')
utils.save_weights(disc, f'{name}_disc.pt')
iters += 1
utils.plot_losses(iter_axis, g_loss_axis, d_loss_axis)
def train_encoding(args):
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
tag = 'latents_{:d}_alpha_{:d}'.format(args.n_latent,
int(args.gnorm_alpha))
save_dir = os.path.join(args.save_base, tag)
U.mkdir(save_dir)
dataset = data.get_dataset(args.dataset, training=True)
model = Model(args.image_channels, args.image_size, args.n_latent,
args.n_dims).to(args.device)
optimizer = optim.Adam(model.encoder.parameters(),
lr=args.learning_rate,
betas=(args.adam_beta1, args.adam_beta2),
weight_decay=args.weight_decay)
model.train()
step = 0
epoch = 0
examples = 0
while examples < args.max_examples:
epoch += 1
loader, _ = data.get_dataloader(dataset, args.batch_size)
for samples, labels in loader:
step += 1
x = samples.to(args.device).float() # B x C x H x W
z = model(x, args.repeats, args.noise_sigma,
'encoding') # B x repeats x n_latent
loss, mmd_loss_all, mmd_loss_avg = model.encoding_loss(
z, args.gamma, args.kernel_gamma, args.kernel_power,
args.gnorm_mu, args.gnorm_sigma, args.gnorm_alpha)
optimizer.zero_grad()
loss.backward()
optimizer.step()
prev_examples = examples
examples += x.size(0)
if examples // BASE_N > prev_examples // BASE_N:
print(
'[Epoch {:d}, Step {:d}, #Eg. {:d}] loss: {:.4f}, mmd_loss_all: {:.4f}, mmd_loss_avg: {:.4f}'
.format(epoch, step, examples, loss.item(),
mmd_loss_all.item(), mmd_loss_avg.item()))
if examples // BASE_N in args.save_points:
path = os.path.join(
save_dir, 'training_examples_{:d}_10k.ckpt'.format(
examples // BASE_N))
print('save {}'.format(path))
torch.save(
{
'examples': examples // BASE_N * BASE_N,
'loss': loss.item(),
'mmd_loss_all': mmd_loss_all.item(),
'mmd_loss_avg': mmd_loss_avg.item(),
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict()
}, path)
def __init__(self, cfg):
self.cfg = cfg
self.model = build_model(cfg, 0)
self.tng_dataloader, self.val_dataloader, self.num_classes, self.num_query = get_dataloader(
cfg)
self.model = self.model.cuda()
self.model.load_params_wo_fc(torch.load(cfg.TEST.WEIGHT))
self.get_distmat()
def get_dataloaders(args, config):
if args.dataset == 'mnist':
train_loader, test_loader = get_dataloader(args)
retrain_loader, _ = get_dataloader(
args, no_randomness=args.no_random_trainloaders)
elif args.dataset.lower()[0:7] == 'cifar10':
assert config is not None
args.cifar_init_lr = config['optimizer_learning_rate']
if args.second_model_name is not None:
assert second_config is not None
assert args.cifar_init_lr == second_config[
'optimizer_learning_rate']
# also the below things should be fine as it is just dataloader loading!
print('loading {} dataloaders'.format(args.dataset.lower()))
train_loader, test_loader = cifar_train.get_dataset(config)
retrain_loader, _ = cifar_train.get_dataset(
config, no_randomness=args.no_random_trainloaders)
return train_loader, test_loader, retrain_loader
def train(args):
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
tag = 'vae_latents_{:d}'.format(args.n_latent)
save_dir = os.path.join(args.save_base, tag)
U.mkdir(save_dir)
dataset = data.get_dataset(args.dataset, training=True)
model = Model(args.image_channels, args.image_size, args.n_latent,
args.n_dims).to(args.device)
optimizer = optim.Adam(model.parameters(),
lr=args.learning_rate,
betas=(args.adam_beta1, args.adam_beta2))
model.train()
step = 0
epoch = 0
examples = 0
while examples < args.max_examples:
epoch += 1
loader, _ = data.get_dataloader(dataset, args.batch_size)
for samples, labels in loader:
step += 1
x = samples.to(args.device).float() # B x C x H x W
z_mean, z_logvar, z, recon_x = model(x, args.repeats)
loss, recon_loss, kl_loss = model.loss(x, z_mean, z_logvar, z,
recon_x)
optimizer.zero_grad()
loss.backward()
optimizer.step()
prev_examples = examples
examples += x.size(0)
if examples // BASE_N > prev_examples // BASE_N:
print(
'[Epoch {:d}, Step {:d}, #Eg. {:d}] loss: {:.8f}, recon_loss: {:.8f}, kl_loss: {:.8f}'
.format(epoch, step, examples, loss.item(),
recon_loss.item(), kl_loss.item()))
if examples // BASE_N in args.save_points:
path = os.path.join(
save_dir, 'training_examples_{:d}_10k.ckpt'.format(
examples // BASE_N))
print('save {}'.format(path))
torch.save(
{
'examples': examples // BASE_N * BASE_N,
'loss': loss.item(),
'recon_loss': recon_loss.item(),
'kl_loss': kl_loss.item(),
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict()
}, path)
def __init__(self, model: nn.Module, train_set: SentiDataset, valid_set: SentiDataset, test_set: SentiDataset,
batch_size: int, optim_cls: Callable[..., Optimizer], lr: float):
"""
Learner for training binary sentiment analysis models.
Args:
model: Model.
train_set: Training dataset.
valid_set: Validation dataset.
test_set: Testing dataset.
batch_size: Batch size.
optim_cls: Optimizer class.
lr: Learning rate.
"""
self.model = model.to(self.device)
self.train_loader = get_dataloader(train_set, batch_size=batch_size)
self.valid_loader = get_dataloader(valid_set, batch_size=batch_size)
self.test_loader = get_dataloader(test_set, batch_size=batch_size)
self.loss_fn = nn.BCEWithLogitsLoss()
self.optimizer = optim_cls(self.model.parameters(), lr=lr)
def __init__(self, args):
super(Tester, self).__init__()
self.args = args
# dataloader
self.dataloader = get_dataloader(self.args)
# model
self.device = torch.device(
'cuda:0' if torch.cuda.is_available else 'cpu')
self.model = LstmPuncModel(self.args)
def execute(self, eval: bool) -> None:
"""Execution
Execute train or eval.
Args:
eval: For evaluation mode.
True: Execute eval.
False: Execute train.
"""
if not eval:
self.train_dataloader, self.val_dataloader = get_dataloader(
self.cfg, mode="trainval")
self.train()
else:
self.test_dataloader = get_dataloader(self.cfg, mode="test")
self.eval()
def train_encoding(args):
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
dataset = data.get_dataset(args.dataset, training=True)
model = Model(args.image_channels, args.image_size, args.n_latent,
args.n_dims).to(args.device)
optimizer = optim.Adam(model.encoder.parameters(),
lr=args.learning_rate,
betas=(args.adam_beta1, args.adam_beta2),
weight_decay=args.weight_decay)
line_plotter = analysis.LinePlotter(
title=
'Learning Curve (In terms of averaged standard deviation of z given x)',
legend=['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'All'],
env='train_mnist')
std_dict, count, examples = None, 0, 0
model.train()
step = 0
epoch = 0
for _ in range(args.n_epochs):
epoch += 1
loader, _ = data.get_dataloader(dataset, args.batch_size)
for samples, labels in loader:
step += 1
x = samples.to(args.device).float() # B x C x H x W
z = model(x, args.repeats, args.noise_sigma,
'encoding') # B x repeats x n_latent
loss, mmd_loss_all, mmd_loss_avg = model.encoding_loss(
z, args.gamma, args.kernel_gamma, args.kernel_power,
args.gnorm_mu, args.gnorm_sigma, args.gnorm_alpha)
optimizer.zero_grad()
loss.backward()
optimizer.step()
std_dict, count = analysis.stats_avg_per_x_std(
U.numpy(z),
U.numpy(labels).astype(np.int), std_dict, count)
examples += x.size(0)
if step % args.print_freq == 0:
print(
'[Epoch {:d}, Step {:d}] loss: {:.4f}, mmd_loss_all: {:.4f}, mmd_loss_avg: {:.4f}'
.format(epoch, step, loss.item(), mmd_loss_all.item(),
mmd_loss_avg.item()))
line_plotter.append(
examples,
[std_dict[l] / count for l in line_plotter.legend])
if epoch % args.show_freq == 0:
monitor(z, labels, epoch, step)
def train(cfg, local_rank):
# prepare dataset
tng_loader, val_loader, num_classes, num_query = get_dataloader(cfg)
do_train(
cfg,
local_rank,
tng_loader,
val_loader,
num_classes,
num_query,
)
def train():
"""
train the model by the args
:return:
"""
# initialize the dataloader by dir
train_transforms = CommonTransforms(trans_cfg, "train")
train_dataloader = data.get_dataloader(data_cfg.batch_size, data_cfg.dataset, data_cfg.train_dir,
phase="train", transforms=train_transforms)
eval_transforms = CommonTransforms(trans_cfg, "val")
eval_dataloader = data.get_dataloader(data_cfg.batch_size, data_cfg.dataset, data_cfg.train_dir,
phase="val", transforms=eval_transforms)
# initialize model, optimizer, loss_fn
model = EfficientSeg(data_cfg.num_classes, compound_coef=cfg.compound_coef,
ratios=eval(cfg.anchors_ratios), scales=eval(cfg.anchors_scales))
start_epoch, best_ap = load_state_dict(model, data_cfg.save_dir, cfg.pretrained_path)
model = model.to(device)
optimizer = get_optimizer(model, opt_cfg)
loss_fn = ComposeLoss(device)
# train model
# foreach epoch
for epoch in range(start_epoch, cfg.num_epochs):
# each epoch includes two phase: train,val
train_loss, train_loss_states = train_model_for_epoch(model, train_dataloader, loss_fn, optimizer, epoch)
write_metric(train_loss_states, epoch, "train")
executor.submit(save_checkpoint, model.state_dict(), epoch, best_ap, data_cfg.save_dir)
if epoch >= cfg.start_eval_epoch:
epoch, mAP, eval_results = evaluate_model(data_cfg, eval_dataloader, eval_transforms, model, epoch, data_cfg.dataset, decode_cfg, device, logger)
# judge the model. if model is greater than current best loss
if best_ap < mAP:
best_ap = mAP
logger.write("the best mAP:{}".format(best_ap))
logger.close()
executor.shutdown(wait=True)
def main(config):
config.device = torch.device(
'cuda:{}'.format(config.gpu) if torch.cuda.is_available() else 'cpu')
# load data_loader
data_loader = get_dataloader(config)
check_point = Checkpoint(config)
operator = Operator(config, check_point)
if config.is_train:
operator.train(data_loader)
else:
operator.test(data_loader)
def main(model, epochs, learning_rate, upscale_factor, save_period,
session_id):
model = models.get_model(model,
upscale_factor=upscale_factor,
batch_norms=False)
dataloader = data.get_dataloader(upscale_factor=upscale_factor)
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
crit = nn.L1Loss()
trainer = create_supervised_trainer(model, optimizer, crit)
trainer.attach(ProgressBar(persist=False))
trainer.attach(
utils.tensorboard_logger.TensorboardLogger(session_id=session_id))
trainer.attach(utils.saver.NetSaver(save_period, session_id))
_ = trainer.run(dataloader, max_epochs=epochs)
def test(config, testLoader=None):
# check if there's weight in work/* (* means the name of setting) before run test.
if testLoader is None:
trainTransform, testTransform = data.get_transforms(config)
testLoader = data.get_dataloader(config,
train=False,
transform=testTransform)
densenet = model.get_model(config)
densenet.load_state_dict(os.path.join(config["save_dir"], "latest.pth"))
criterion = CrossEntropyLoss()
test_out = _test(densenet, criterion, testLoader, need_output_y=True)
print("test finished")
return test_out
def train(args):
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
np.random.seed(args.seed)
loader, ds = data.get_dataloader(args.dataset,
args.batch_size,
max_steps=args.max_steps)
ds_sampler = data.DspritesFullSampler(ds.dataset)
model = DFVE(args.image_channels, args.n_latent, args.gamma,
args.kernel_gamma).to(args.device)
optimizer = optim.Adam(model.parameters(),
lr=args.learning_rate,
betas=(args.adam_beta1, args.adam_beta2),
weight_decay=args.weight_decay)
model.train()
step = 0
for batch in loader:
step += 1
_, samples = batch
x = samples.to(args.device).float()
z_mean, z_std = model(x)
loss, mmd_loss, kl_loss = model.loss(z_mean, z_std, args.c)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if step % args.print_freq == 0:
print(
'[Step {:d}] loss: {:.4f}, mmd_loss: {:.4f}, kl_loss: {:.4f}'.
format(step, loss.item(), mmd_loss.item(), kl_loss.item()))
print(' [z_mean (min)] {}'.format(
z_mean.min(0)[0].detach().cpu().numpy()))
print(' [z_mean (max)] {}'.format(
z_mean.max(0)[0].detach().cpu().numpy()))
print(' [z_mean (mean)] {}'.format(
z_mean.mean(0).detach().cpu().numpy()))
print(' [z_std (min)] {}'.format(
z_std.min(0)[0].detach().cpu().numpy()))
print(' [z_std (max)] {}'.format(
z_std.max(0)[0].detach().cpu().numpy()))
print(' [z_std (mean)] {}'.format(
z_std.mean(0).detach().cpu().numpy()))
if step % args.show_freq == 0:
with torch.no_grad():
show(ds_sampler, model, step)
def retrain_models(args, old_networks, train_loader, test_loader, config, tensorboard_obj=None, initial_acc=None, nicks=None):
accuracies = []
retrained_networks = []
# nicks = []
# assert len(old_networks) >= 4
for i in range(len(old_networks)):
nick = nicks[i]
# if i == len(old_networks) - 1:
# nick = 'naive_averaging'
# elif i == len(old_networks) - 2:
# nick = 'geometric'
# else:
# nick = 'model_' + str(i)
# nicks.append(nick)
print("Retraining model : ", nick)
if initial_acc is not None:
start_acc = initial_acc[i]
else:
start_acc = -1
if args.dataset.lower()[0:7] == 'cifar10':
if args.reinit_trainloaders:
print('reiniting trainloader')
retrain_loader, _ = cifar_train.get_dataset(config, no_randomness=args.no_random_trainloaders)
else:
retrain_loader = train_loader
output_root_dir = "{}/{}_models_ensembled/".format(args.baseroot, (args.dataset).lower())
output_root_dir = os.path.join(output_root_dir, args.exp_name, nick)
os.makedirs(output_root_dir, exist_ok=True)
retrained_network, acc = cifar_train.get_retrained_model(args, retrain_loader, test_loader, old_networks[i], config, output_root_dir, tensorboard_obj=tensorboard_obj, nick=nick, start_acc=initial_acc[i])
elif args.dataset.lower() == 'mnist':
if args.reinit_trainloaders:
print('reiniting trainloader')
retrain_loader, _ = get_dataloader(args, no_randomness=args.no_random_trainloaders)
else:
retrain_loader = train_loader
start_acc = initial_acc[i]
retrained_network, acc = get_retrained_model(args, retrain_loader, test_loader, old_network=old_networks[i], tensorboard_obj=tensorboard_obj, nick=nick, start_acc=start_acc, retrain_seed=args.retrain_seed)
retrained_networks.append(retrained_network)
accuracies.append(acc)
return retrained_networks, accuracies
def get_data(self, mode, size=None):
"Allow for custom mode and dataset size"
size = self.config.dataset_size if size is None else size
if self.config.preprocess_device == 'cpu':
dataset = get_dataset(self.config,
mode,
size,
preprocess=self.preprocess)
else:
dataset = get_dataset(self.config, mode, size)
print(f"Dataset Size: {len(dataset)}")
dataloader = get_dataloader(self.config, dataset)
return dataloader
def test_get_residual():
rain_path = 'E:\All_My_Files\All_My_DataSets\derain_datasets\Derain_ML_Proj\\final_testset\\rainy_image'
gt_path = 'E:\All_My_Files\All_My_DataSets\derain_datasets\Derain_ML_Proj\\final_testset\ground_truth'
batch = 3
dataloader = iter(
data.get_dataloader(gt_path=gt_path,
rain_path=rain_path,
batchSize=batch))
gt_image, rain_image = next(dataloader)
print(gt_image.size())
print(rain_image.size())
residual = get_residual_tensor(gt_image, rain_image)
print(residual.size())
save_image_from_tensor(residual, './test')
def __init__(self, cfg, logger, writer):
self.cfg, self.logger, self.writer = cfg, logger, writer
# Define dataloader
self.tng_dataloader, self.val_dataloader, self.num_classes, self.num_query = get_dataloader(
cfg)
# networks
self.model = build_model(cfg, self.num_classes)
# loss function
self.ce_loss = nn.CrossEntropyLoss()
self.triplet = TripletLoss(cfg.SOLVER.MARGIN)
# optimizer and scheduler
self.opt = make_optimizer(self.cfg, self.model)
self.lr_sched = make_lr_scheduler(self.cfg, self.opt)
self._construct()
def train(**kwargs):
opt = Config()
for k, v in kwargs.items():
setattr(opt, k, v)
device=t.device('cuda') if opt.use_gpu else t.device('cpu')
opt.caption_data_path = 'caption.pth' # 原始数据
opt.test_img = '' # 输入图片
# opt.model_ckpt='caption_0914_1947' # 预训练的模型
# 数据
vis = Visualizer(env=opt.env)
dataloader = get_dataloader(opt)
_data = dataloader.dataset._data
word2ix, ix2word = _data['word2ix'], _data['ix2word']
# 模型
model = CaptionModel(opt, word2ix, ix2word)
if opt.model_ckpt:
model.load(opt.model_ckpt)
optimizer = model.get_optimizer(opt.lr)
criterion = t.nn.CrossEntropyLoss()
model.to(device)
# 统计
loss_meter = meter.AverageValueMeter()
for epoch in range(opt.epoch):
loss_meter.reset()
for ii, (imgs, (captions, lengths), indexes) in tqdm.tqdm(enumerate(dataloader)):
# 训练
optimizer.zero_grad()
imgs = imgs.to(device)
captions = captions.to(device)
input_captions = captions[:-1]
target_captions = pack_padded_sequence(captions, lengths)[0]
score, _ = model(imgs, input_captions, lengths)
loss = criterion(score, target_captions)
loss.backward()
optimizer.step()
loss_meter.add(loss.item())
# 可视化
if (ii + 1) % opt.plot_every == 0:
if os.path.exists(opt.debug_file):
ipdb.set_trace()
vis.plot('loss', loss_meter.value()[0])
# 可视化原始图片 + 可视化人工的描述语句
raw_img = _data['ix2id'][indexes[0]]
img_path = opt.img_path + raw_img
raw_img = Image.open(img_path).convert('RGB')
raw_img = tv.transforms.ToTensor()(raw_img)
raw_caption = captions.data[:, 0]
raw_caption = ''.join([_data['ix2word'][ii] for ii in raw_caption])
vis.text(raw_caption, u'raw_caption')
vis.img('raw', raw_img, caption=raw_caption)
# 可视化网络生成的描述语句
results = model.generate(imgs.data[0])
vis.text('</br>'.join(results), u'caption')
model.save()