def test_basic_trainer():
model = nn.Linear(10, 10)
optimizer = optim.SGD()
scheduler = lr_scheduler.StepLR(9)
trainer = trainers.SupervisedTrainer(model,
optimizer,
F.cross_entropy,
scheduler=scheduler,
update_scheduler_by_epoch=False)
loader = [(torch.randn(2, 10), torch.zeros(2, dtype=torch.long))
for _ in range(10)]
for _ in trainer.epoch_range(1):
trainer.train(loader)
assert pytest.approx(trainer.optimizer.param_groups[0]["lr"], 0.01)
optimizer = torch.optim.SGD(model.parameters(), lr=1e-1)
trainer = trainers.SupervisedTrainer(model,
optimizer,
F.cross_entropy,
scheduler=scheduler,
update_scheduler_by_epoch=False)
for _ in trainer.epoch_range(1):
trainer.train(loader)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 9)
trainer = trainers.SupervisedTrainer(model,
optimizer,
F.cross_entropy,
scheduler=scheduler,
update_scheduler_by_epoch=False)
trainer.run(loader, loader, 15, 11)
assert trainer.step == 11 - 1
def main():
train_loader, test_loader = vision_loaders(args.data_name,
args.batch_size,
download=True,
num_workers=1)
if args.baseline:
model = resnet20()
else:
# model = se_resnet18(num_classes=1000)
# model = se_resnet50(num_classes=100)
model = resnet20_ASP(num_classes=10)
# model = resnet32_ASP(num_classes=10)
optimizer = optim.SGD(lr=1e-1, momentum=0.9, weight_decay=1e-4)
scheduler = lr_scheduler.StepLR(80, 0.1)
tqdm_rep = reporters.TQDMReporter(range(args.epochs))
_callbacks = [tqdm_rep, callbacks.AccuracyCallback()]
with Trainer(model,
optimizer,
F.cross_entropy,
scheduler=scheduler,
callbacks=_callbacks) as trainer:
for i in tqdm_rep:
trainer.train(train_loader)
trainer.test(test_loader)
def main():
train_loader, test_loader = cifar10_loaders(args.batch_size)
if args.model == "resnet":
model = resnet20()
elif args.model == "senet":
model = se_resnet20(num_classes=10, reduction=args.reduction)
elif args.model == "gcn":
model = resnet20_gcn()
else:
raise TypeError(f"{args.model} is not valid argument")
optimizer = optim.SGD(lr=1e-1, momentum=0.9, weight_decay=1e-4)
scheduler = lr_scheduler.StepLR(80, 0.1)
tqdm_rep = reporter.TQDMReporter(range(args.epochs),
callbacks=[callbacks.AccuracyCallback()],
save_dir='logs/',
report_freq=-1)
# tb_rep = reporter.TensorboardReporter(callbacks=[callbacks.AccuracyCallback(), callbacks.LossCallback()], save_dir='logs/')
trainer = Trainer(model,
optimizer,
F.cross_entropy,
scheduler=scheduler,
callbacks=tqdm_rep)
for _ in tqdm_rep:
trainer.train(train_loader)
trainer.test(test_loader)
def main():
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_transform = transforms.Compose([
transforms.Resize(256),
transforms.RandomCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(), normalize
])
valid_transform = transforms.Compose(
[transforms.Resize(224),
transforms.ToTensor(), normalize])
train_dataset = ImageFolder(args.data, train_transform)
valid_dataset = ImageFolder(args.data, valid_transform)
num_samples = int(len(train_dataset) / 10)
indices = list(range(num_samples))
split = int(np.floor(0.1 * num_samples))
np.random.shuffle(indices)
train_idx, valid_idx = indices[split:], indices[:split]
train_sampler = SubsetRandomSampler(train_idx)
valid_sampler = SubsetRandomSampler(valid_idx)
train_loader = DataLoader(train_dataset,
args.batch_size,
sampler=train_sampler,
num_workers=4)
valid_loader = DataLoader(valid_dataset,
args.batch_size,
sampler=valid_sampler,
num_workers=4)
print("num data:", num_samples)
print("num train batches:", len(train_loader))
print("num test batches:", len(valid_loader))
# return
# train_loader, test_loader = cifar10_loaders(args.batch_size)
model = se_resnet50(num_classes=42)
# model.load_state_dict(torch.load("seresnet50-60a8950a85b2b.pkl"))
optimizer = optim.SGD(lr=1e-1, momentum=0.9, weight_decay=1e-4)
scheduler = lr_scheduler.StepLR(80, 0.1)
tqdm_rep = reporters.TQDMReporter(range(args.epochs),
callbacks.AccuracyCallback())
_callbacks = [tqdm_rep, callbacks.AccuracyCallback()]
with Trainer(model,
optimizer,
F.cross_entropy,
scheduler=scheduler,
callbacks=_callbacks) as trainer:
for _ in tqdm_rep:
trainer.train(train_loader)
trainer.test(valid_loader)
torch.save(trainer.model.state_dict(), "se_resnet50.pkl")
def main():
train_loader, test_loader = get_dataloader(args.batch_size, args.root)
gpus = list(range(torch.cuda.device_count()))
se_resnet = nn.DataParallel(se_resnet50(num_classes=1000),
device_ids=gpus)
optimizer = optim.SGD(lr=0.6 / 1024 * args.batch_size, momentum=0.9, weight_decay=1e-4)
scheduler = lr_scheduler.StepLR(30, gamma=0.1)
weight_saver = callbacks.WeightSave("checkpoints")
tqdm_rep = reporter.TQDMReporter(range(args.epochs), callbacks=[callbacks.AccuracyCallback()])
trainer = Trainer(se_resnet, optimizer, F.cross_entropy, scheduler=scheduler,
callbacks=callbacks.CallbackList(weight_saver, tqdm_rep))
for _ in tqdm_rep:
trainer.train(train_loader)
trainer.test(test_loader)
def main():
train_loader, test_loader = cifar10_loaders(args.batch_size)
print(test_loader.sampler)
if args.baseline:
model = resnet20()
else:
model = se_resnet20(num_classes=10, reduction=args.reduction)
optimizer = optim.SGD(lr=1e-1, momentum=0.9, weight_decay=1e-4)
scheduler = lr_scheduler.StepLR(80, 0.1)
tqdm_rep = reporters.TQDMReporter(range(args.epochs), callbacks.AccuracyCallback())
_callbacks = [tqdm_rep, callbacks.AccuracyCallback()]
with Trainer(model, optimizer, F.cross_entropy, scheduler=scheduler, callbacks=_callbacks) as trainer:
for _ in tqdm_rep:
trainer.train(train_loader)
trainer.test(test_loader)
resnet = module_converter(resnet56(num_classes=10), keys=["conv1", "bn1", "relu", "layer1", "layer2", "layer3"])
aux = nn.ModuleDict(OrderedDict({k: v for k, v in {
# 32x32
"conv1": generate_aux(32, 16, 10, args.num_convs, args.num_fcs),
# 32x32
"layer1": generate_aux(32, 16, 10, args.num_convs, args.num_fcs),
# 16x16
"layer2": generate_aux(16, 32, 10, args.num_convs, args.num_fcs),
# 8x8
"layer3": generate_aux(8, 64, 10, args.num_convs, args.num_fcs),
}.items() if k in args.group}))
model = NaiveGreedyModule(resnet, aux=aux,
tail=nn.Sequential(nn.AdaptiveAvgPool2d(1), Flatten(), nn.Linear(64, 10)))
# import torch
# from homura.debug import module_debugger as simple_debugger
# simple_debugger(model, (torch.randn(1, 3, 32, 32), "layer1"), target=torch.tensor([1]),
# loss=lambda x, y: F.cross_entropy(x.pred, y))
print(args)
# print(model)
greedy_loss = [greedy_loss_by_name(name) for name in args.group]
tb = reporter.TensorboardReporter([_callbacks.LossCallback(), _callbacks.AccuracyCallback()] + greedy_loss,
"../results")
# tb.enable_report_params()
trainer = Trainer(model, optimizer, F.cross_entropy, callbacks=tb,
scheduler=lr_scheduler.StepLR(args.step, 0.2) if args.optimizer == "sgd" else None)
for _ in trange(args.epochs, ncols=80):
trainer.train(train_loader)
trainer.test(test_loader)