def main(args):
import sys
sys.path.append('/home/kp600168/semi')
from allinone import SEMI_DATASET_REGISTRY, SEMI_TRAINER_REGISTRY, SCHEDULER_REGISTRY
print(args)
cifar = SEMI_DATASET_REGISTRY('mix')(CIFAR10,
args.dataset,
10, [
tf.RandomResizedCrop((32, 32)),
tf.ToTensor(),
tf.Normalize(
(0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010))
],
semi_size=args.semi_size)
train_loader, test_loader, num_classes = cifar(
args.batch_size, num_workers=args.num_workers, return_num_classes=True)
model_dict = {
'generator_x': MODEL_REGISTRY('Generator_x')(),
'generator_z': MODEL_REGISTRY('Generator_z')(),
'discriminator_x': MODEL_REGISTRY('Discriminator_x')(),
'discriminator_z': MODEL_REGISTRY('Discriminator_z')(),
'discriminator_x_z': MODEL_REGISTRY('Discriminator_x_z')(num_classes),
}
kwargs = {
'model_dict':
model_dict,
'optimizer':
Adam(lr=args.lr_100 * args.batch_size / 100, betas=(0.5, 1 - 1e-3)),
'loss_f':
F.cross_entropy,
'consistency_weight':
SCHEDULER_REGISTRY('identity')(1.),
}
trainer = SEMI_TRAINER_REGISTRY('AdversariallyLearnedInference')(
**kwargs, reporters=[TQDMReporter(),
TensorboardReporter('.')])
for _ in trainer.epoch_range(args.epochs):
trainer.train(train_loader)
trainer.test(test_loader)
# trainer = SupervisedTrainer(lenet, SGD(lr=args.lr_256 * args.batch_size /
# 256, momentum=0.9), F.cross_entropy, reporters=[TQDMReporter()])
# for _ in trainer.epoch_range(args.epochs):
# trainer.train(train_loader)
# trainer.test(test_loader)
print(f"Max Accuracy={max(trainer.history['accuracy/test'])}")
def main(args):
import sys
sys.path.append('/home/kp600168/semi')
from allinone import SEMI_DATASET_REGISTRY, SEMI_TRAINER_REGISTRY, SCHEDULER_REGISTRY
print(args)
mnist = SEMI_DATASET_REGISTRY('mix')(MNIST,
args.dataset,
10, [
tf.Resize((32, 32)),
tf.ToTensor(),
tf.Normalize((0.1307, ),
(0.3081, ))
],
semi_size=args.semi_size)
train_loader, test_loader, num_classes = mnist(
args.batch_size, num_workers=args.num_workers, return_num_classes=True)
lenet = MODEL_REGISTRY(args.model)(num_classes=num_classes)
kwargs = {
'model':
MODEL_REGISTRY(args.model)(num_classes=num_classes),
'optimizer':
SGD(lr=args.lr_256 * args.batch_size / 256, momentum=0.9),
'loss_f':
F.cross_entropy,
'consistency_weight':
SCHEDULER_REGISTRY('identity')(1e-5),
'alpha':
SCHEDULER_REGISTRY('lambda')(lambda epoch: min(1 - 1 /
(1 + epoch), 0.99)),
'beta':
0.02
}
trainer = SEMI_TRAINER_REGISTRY('InterpolationConsistency')(
**kwargs, reporters=[TQDMReporter()])
for _ in trainer.epoch_range(args.epochs):
trainer.train(train_loader)
trainer.test(test_loader)
# trainer = SupervisedTrainer(lenet, SGD(lr=args.lr_256 * args.batch_size /
# 256, momentum=0.9), F.cross_entropy, reporters=[TQDMReporter()])
# for _ in trainer.epoch_range(args.epochs):
# trainer.train(train_loader)
# trainer.test(test_loader)
print(f"Max Accuracy={max(trainer.history['accuracy/test'])}")
def main(cfg):
if cfg.use_accimage:
enable_accimage()
model = MODEL_REGISTRY(cfg.model.name)(num_classes=10)
train_loader, test_loader = DATASET_REGISTRY("fast_cifar10" if cfg.use_fast_collate else "cifar10"
)(cfg.data.batch_size, num_workers=4,
use_prefetcher=cfg.use_prefetcher)
optimizer = None if cfg.bn_no_wd else optim.SGD(lr=1e-1, momentum=0.9, weight_decay=cfg.optim.weight_decay)
scheduler = lr_scheduler.MultiStepLR([100, 150], gamma=cfg.optim.lr_decay)
if cfg.bn_no_wd:
def set_optimizer(trainer):
bn_params = []
non_bn_parameters = []
for name, p in trainer.model.named_parameters():
if "bn" in name:
bn_params.append(p)
else:
non_bn_parameters.append(p)
optim_params = [
{"params": bn_params, "weight_decay": 0},
{"params": non_bn_parameters, "weight_decay": cfg.optim.weight_decay},
]
trainer.optimizer = torch.optim.SGD(optim_params, lr=1e-1, momentum=0.9)
trainers.SupervisedTrainer.set_optimizer = set_optimizer
if cfg.use_zerograd_none:
import types
def set_optimizer(trainer):
# see Apex for details
def zero_grad(self):
for group in self.param_groups:
for p in group['params']:
p.grad = None
trainer.optimizer = trainer.optimizer(trainer.model.parameters())
trainer.optimizer.zero_grad = types.MethodType(zero_grad, trainer.optimizer)
trainers.SupervisedTrainer.set_optimizer = set_optimizer
with trainers.SupervisedTrainer(model,
optimizer,
F.cross_entropy,
reporters=[reporters.TensorboardReporter('.')],
scheduler=scheduler,
use_amp=cfg.use_amp,
debug=cfg.debug
) as trainer:
for _ in trainer.epoch_range(cfg.optim.epochs):
trainer.train(train_loader)
trainer.test(test_loader)
print(f"Max Test Accuracy={max(trainer.reporter.history('accuracy/test')):.3f}")
def main(cfg):
if cfg.use_accimage:
enable_accimage()
model = MODEL_REGISTRY(cfg.name)(num_classes=10)
train_loader, test_loader = DATASET_REGISTRY(
"fast_cifar10" if cfg.use_fast_collate else "cifar10")(
cfg.batch_size, num_workers=4, use_prefetcher=cfg.use_prefetcher)
optimizer = None if cfg.bn_no_wd else optim.SGD(
lr=cfg.lr, momentum=0.9, weight_decay=cfg.weight_decay)
scheduler = lr_scheduler.CosineAnnealingWithWarmup(cfg.epochs, 4, 5)
if cfg.bn_no_wd:
def set_optimizer(trainer):
bn_params = []
non_bn_parameters = []
for name, p in trainer.model.named_parameters():
if "bn" in name:
bn_params.append(p)
else:
non_bn_parameters.append(p)
optim_params = [
{
"params": bn_params,
"weight_decay": 0
},
{
"params": non_bn_parameters,
"weight_decay": cfg.weight_decay
},
]
trainer.optimizer = torch.optim.SGD(optim_params,
lr=1e-1,
momentum=0.9)
trainers.SupervisedTrainer.set_optimizer = set_optimizer
with trainers.SupervisedTrainer(
model,
optimizer,
F.cross_entropy,
reporters=[reporters.TensorboardReporter('.')],
scheduler=scheduler,
use_amp=cfg.use_amp,
debug=cfg.debug) as trainer:
for _ in trainer.epoch_range(cfg.epochs):
trainer.train(train_loader)
trainer.test(test_loader)
trainer.scheduler.step()
print(
f"Max Test Accuracy={max(trainer.reporter.history('accuracy/test')):.3f}"
)
def main(cfg):
train_loader, test_loader, num_classes = DATASET_REGISTRY(cfg.data.name)(cfg.data.batch_size,
return_num_classes=True,
num_workers=4)
model = MODEL_REGISTRY(cfg.model.name)(num_classes=num_classes)
optimizer = optim.SGD(lr=1e-1, momentum=0.9, weight_decay=cfg.optim.weight_decay)
scheduler = lr_scheduler.CosineAnnealingWithWarmup(200, 4, 5)
with Trainer(model, optimizer, naive_cross_entropy_loss, scheduler=scheduler, cfg=cfg.model,
num_classes=num_classes) as trainer:
for _ in trainer.epoch_range(200):
trainer.train(train_loader)
trainer.test(test_loader)
print(f"Max Test Accuracy={max(trainer.reporter.history('accuracy/test')):.4f}")
class Config:
name: str = chika.choices(*MODEL_REGISTRY.choices())
batch_size: int = 128
epochs: int = 200
lr: float = 0.1
weight_decay: float = 1e-4
lr_decay: float = 0.1
bn_no_wd: bool = False
use_amp: bool = False
use_accimage: bool = False
use_prefetcher: bool = False
debug: bool = False
def main(args):
import sys
sys.path.append('/home/kp600168/semi')
from allinone import SEMI_DATASET_REGISTRY, SEMI_TRAINER_REGISTRY
print(args)
mnist = SEMI_DATASET_REGISTRY('split')(MNIST,
args.dataset,
10, [
tf.Resize((32, 32)),
tf.ToTensor(),
tf.Normalize((0.1307, ),
(0.3081, ))
],
semi_size=args.semi_size)
train_loader, test_loader, _, num_classes = mnist(args.batch_size,
num_workers=0,
return_num_classes=True)
lenet = MODEL_REGISTRY(args.model)(num_classes=num_classes)
kwargs = {
'bn_list': [lenet.bn1, lenet.bn2, lenet.bn3],
'sigma_list': [0.3, 0.3, 0.3],
'v_list':
[ConvTranspose2d(16, 6, 10, 2),
ConvTranspose2d(120, 16, 10)],
'lam_list': [0.1, 0.01, 0.01],
}
trainer = SEMI_TRAINER_REGISTRY('Ladder')(lenet,
SGD(lr=args.lr_256 *
args.batch_size / 256,
momentum=0.9),
F.cross_entropy,
**kwargs,
reporters=[TQDMReporter()])
# for _ in trainer.epoch_range(args.epochs):
# trainer.train(train_loader)
# trainer.test(test_loader)
trainer = SupervisedTrainer(lenet,
SGD(lr=args.lr_256 * args.batch_size / 256,
momentum=0.9),
F.cross_entropy,
reporters=[TQDMReporter()])
for _ in trainer.epoch_range(args.epochs):
trainer.train(train_loader)
trainer.test(test_loader)
print(f"Max Accuracy={max(trainer.history['accuracy/test'])}")
class Config:
model: str = chika.choices(*MODEL_REGISTRY.choices())
batch_size: int = 128
epochs: int = 200
lr: float = 0.1
weight_decay: float = 1e-4
data: str = chika.choices("cifar10", "cifar100", "svhn")
bn_no_wd: bool = False
use_amp: bool = False
use_accimage: bool = False
use_multi_tensor: bool = False
use_channel_last: bool = False
prefetch_factor: int = 2
persistent_workers: bool = False
debug: bool = False
download: bool = False
def main(args):
import sys
sys.path.append('/home/kp600168/semi')
from allinone import SEMI_DATASET_REGISTRY, SEMI_TRAINER_REGISTRY, SCHEDULER_REGISTRY, TRANSFORM_REGISTRY
print(args)
unlabel_transform = TRANSFORM_REGISTRY('ManyTimes')(tf.Compose([
tf.RandomResizedCrop(((32, 32)), (0.9, 1.0), (0.9, 1.1)),
tf.ToTensor(),
tf.Normalize((0.1307, ), (0.3081, ))
]), 4)
mnist = SEMI_DATASET_REGISTRY('split')(MNIST,
args.dataset,
10, [],
[unlabel_transform.transform], [
tf.Resize((32, 32)),
tf.ToTensor(),
tf.Normalize((0.1307, ),
(0.3081, ))
],
semi_size=args.semi_size,
unlabel_transform=unlabel_transform)
train_loader, test_loader, unlabel_loader, num_classes = mnist(
args.batch_size, num_workers=0, return_num_classes=True)
kwargs = {
'model': MODEL_REGISTRY(args.model)(num_classes=num_classes),
'optimizer': SGD(lr=args.lr_256 * args.batch_size / 256, momentum=0.9),
'loss_f': F.cross_entropy,
'temperature': 0.5,
'beta': 0.2,
'consistency_weight': SCHEDULER_REGISTRY('identity')(1.),
'dataset_type': 'split',
}
trainer = SEMI_TRAINER_REGISTRY('MixMatch')(**kwargs,
reporters=[TQDMReporter()])
for _ in trainer.epoch_range(args.epochs):
trainer.train(zip(train_loader, unlabel_loader))
trainer.test(test_loader)
print(f"Max Accuracy={max(trainer.history['accuracy/test'])}")
def _main(cfg):
model = MODEL_REGISTRY(cfg.model)(num_classes=10)
train_loader, test_loader = DATASET_REGISTRY("cifar10")(cfg.batch_size, num_workers=4, download=True)
optimizer = homura.optim.SGD(lr=cfg.optim.lr, momentum=0.9, weight_decay=cfg.optim.weight_decay)
scheduler = {
"cosine": lr_scheduler.CosineAnnealingWithWarmup(cfg.optim.epochs, 4, 5),
"abel": ABEL(cfg.optim.gamma),
"steps": lr_scheduler.MultiStepLR(milestones=cfg.optim.steps, gamma=cfg.optim.gamma),
}[cfg.optim.name]
with Trainer(
model,
optimizer,
F.cross_entropy,
reporters=[reporters.TensorboardReporter(".")],
scheduler=scheduler,
use_amp=cfg.use_amp,
) as trainer:
for _ in trainer.epoch_range(cfg.optim.epochs):
trainer.train(train_loader)
trainer.test(test_loader)
trainer.scheduler.step()
print(f"Max Test Accuracy={max(trainer.reporter.history('accuracy/test')):.3f}")
def main(args):
import sys
sys.path.append('/home/kp600168/semi')
from allinone import SEMI_DATASET_REGISTRY, SEMI_TRAINER_REGISTRY, TRANSFORM_REGISTRY, SCHEDULER_REGISTRY
print(args)
unlabel_transform = TRANSFORM_REGISTRY('twice')(tf.Compose([
tf.RandomResizedCrop(((32, 32)), (0.9, 1.0), (0.9, 1.1)),
tf.ToTensor(),
tf.Normalize((0.1307, ), (0.3081, ))
]))
mnist = SEMI_DATASET_REGISTRY('mix')(MNIST,
args.dataset,
10, [], [unlabel_transform], [
tf.Resize((32, 32)),
tf.ToTensor(),
tf.Normalize((0.1307, ),
(0.3081, ))
],
semi_size=args.semi_size)
train_loader, test_loader, num_classes = mnist(args.batch_size,
num_workers=4,
return_num_classes=True)
kwargs = {
'model':
MODEL_REGISTRY(args.model)(num_classes=num_classes),
'optimizer':
SGD(lr=args.lr_256 * args.batch_size / 256, momentum=0.9),
'loss_f':
F.cross_entropy,
'consistency_weight':
SCHEDULER_REGISTRY('identity')(0.01),
'alpha':
SCHEDULER_REGISTRY('lambda')(lambda epoch: min(1 - 1 /
(1 + epoch), 0.99)),
'dataset_type':
'mix',
}
trainer = SEMI_TRAINER_REGISTRY('meanteacher')(**kwargs,
reporters=[TQDMReporter()])
for _ in trainer.epoch_range(args.epochs):
trainer.train(train_loader)
trainer.test(test_loader)
# mnist = SEMI_DATASET_REGISTRY('split')(MNIST, args.dataset, 10, [], [unlabel_transform.transform], [tf.Resize(
# (32, 32)), tf.ToTensor(), tf.Normalize((0.1307,), (0.3081,))], semi_size=args.semi_size, unlabel_transform=unlabel_transform)
# train_loader, test_loader, unlabel_loader, num_classes = mnist(
# args.batch_size, num_workers=0, return_num_classes=True)
# kwargs = {
# 'model': MODEL_REGISTRY(args.model)(num_classes=num_classes),
# 'optimizer': SGD(lr=args.lr_256 * args.batch_size / 256, momentum=0.9),
# 'loss_f': F.cross_entropy,
# 'consistency_weight': SCHEDULER_REGISTRY('identity')(0.01),
# 'alpha': SCHEDULER_REGISTRY('lambda')(lambda epoch: min(1 - 1 / (1 + epoch), 0.99)),
# 'dataset_type': 'split',
# }
# trainer = SEMI_TRAINER_REGISTRY('meanteacher')(
# **kwargs, reporters=[TQDMReporter()])
# for _ in trainer.epoch_range(args.epochs):
# trainer.train(zip(train_loader, unlabel_loader))
# trainer.test(test_loader)
print(f"Max Accuracy={max(trainer.history['accuracy/test'])}")
def main(cfg):
if cfg.use_accimage:
enable_accimage()
data = DATASET_REGISTRY(cfg.data).setup(
cfg.batch_size,
num_workers=4,
download=cfg.download,
prefetch_factor=cfg.prefetch_factor,
persistent_workers=cfg.persistent_workers)
model = MODEL_REGISTRY(cfg.model)(num_classes=data.num_classes)
optimizer = None if cfg.bn_no_wd else optim.SGD(
lr=cfg.lr,
momentum=0.9,
weight_decay=cfg.weight_decay,
multi_tensor=cfg.use_multi_tensor)
scheduler = lr_scheduler.CosineAnnealingWithWarmup(cfg.epochs, 4, 5)
if cfg.bn_no_wd:
def set_optimizer(trainer):
bn_params = []
non_bn_parameters = []
for name, p in trainer.model.named_parameters():
if "norm" in name:
bn_params.append(p)
else:
non_bn_parameters.append(p)
optim_params = [
{
"params": bn_params,
"weight_decay": 0
},
{
"params": non_bn_parameters,
"weight_decay": cfg.weight_decay
},
]
trainer.optimizer = torch.optim.SGD(optim_params,
lr=1e-1,
momentum=0.9)
trainers.SupervisedTrainer.set_optimizer = set_optimizer
with trainers.SupervisedTrainer(
model,
optimizer,
F.cross_entropy,
reporters=[reporters.TensorboardReporter('.')],
scheduler=scheduler,
use_amp=cfg.use_amp,
use_channel_last=cfg.use_channel_last,
debug=cfg.debug) as trainer:
for _ in trainer.epoch_range(cfg.epochs):
trainer.train(data.train_loader)
trainer.test(data.test_loader)
trainer.scheduler.step()
print(
f"Max Test Accuracy={max(trainer.reporter.history('accuracy/test')):.3f}"
)
