def main():
if args.seed is None:
args.seed = random.randint(1, 10000)
print("Random Seed: ", args.seed)
random.seed(args.seed)
torch.manual_seed(args.seed)
if args.gpus:
torch.cuda.manual_seed_all(args.seed)
time_stamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
if args.evaluate:
args.results_dir = '/tmp'
if args.save is '':
args.save = time_stamp
save_path = os.path.join(args.results_dir, args.save)
if not os.path.exists(save_path):
os.makedirs(save_path)
if args.gpus is not None:
args.gpus = [int(i) for i in args.gpus.split(',')]
device = 'cuda:' + str(args.gpus[0])
cudnn.benchmark = True
else:
device = 'cpu'
if args.type == 'float64':
dtype = torch.float64
elif args.type == 'float32':
dtype = torch.float32
elif args.type == 'float16':
dtype = torch.float16
else:
raise ValueError('Wrong type!') # TODO int8
model = MobileNet2(input_size=args.input_size, scale=args.scaling)
num_parameters = sum([l.nelement() for l in model.parameters()])
print(model)
print('number of parameters: {}'.format(num_parameters))
print('FLOPs: {}'.format(
flops_benchmark.count_flops(MobileNet2,
args.batch_size // len(args.gpus) if args.gpus is not None else args.batch_size,
device, dtype, args.input_size, 3, args.scaling)))
# define loss function (criterion) and optimizer
criterion = torch.nn.CrossEntropyLoss()
if args.gpus is not None:
model = torch.nn.DataParallel(model, args.gpus)
model.to(device=device, dtype=dtype)
criterion.to(device=device, dtype=dtype)
optimizer = torch.optim.SGD(model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.decay,
nesterov=True)
best_test = 0
# optionally resume from a checkpoint
data = None
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume, map_location=device)
args.start_epoch = checkpoint['epoch'] - 1
best_test = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})"
.format(args.resume, checkpoint['epoch']))
elif os.path.isdir(args.resume):
checkpoint_path = os.path.join(args.resume, 'checkpoint.pth.tar')
csv_path = os.path.join(args.resume, 'results.csv')
print("=> loading checkpoint '{}'".format(checkpoint_path))
checkpoint = torch.load(checkpoint_path, map_location=device)
args.start_epoch = checkpoint['epoch'] - 1
best_test = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(checkpoint_path, checkpoint['epoch']))
data = []
with open(csv_path) as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
data.append(row)
else:
print("=> no checkpoint found at '{}'".format(args.resume))
if args.evaluate:
loss, top1, top5 = test(model, criterion, device, dtype) # TODO
return
csv_logger = CsvLogger(filepath=save_path, data=data)
csv_logger.save_params(sys.argv, args)
claimed_acc1 = None
claimed_acc5 = None
if args.input_size in claimed_acc_top1:
if args.scaling in claimed_acc_top1[args.input_size]:
claimed_acc1 = claimed_acc_top1[args.input_size][args.scaling]
claimed_acc5 = claimed_acc_top5[args.input_size][args.scaling]
csv_logger.write_text(
'Claimed accuracies are: {:.2f}% top-1, {:.2f}% top-5'.format(claimed_acc1 * 100., claimed_acc5 * 100.))
for epoch in trange(args.start_epoch, args.epochs + 1):
if epoch in args.schedule:
args.learning_rate *= args.gamma
for param_group in optimizer.param_groups:
param_group['lr'] = args.learning_rate
train_loss, train_accuracy1, train_accuracy5, = train(model, epoch, optimizer, criterion, device, dtype)
test_loss, test_accuracy1, test_accuracy5 = test(model, criterion, device, dtype)
csv_logger.write({'epoch': epoch + 1, 'val_error1': 1 - test_accuracy1, 'val_error5': 1 - test_accuracy5,
'val_loss': test_loss, 'train_error1': 1 - train_accuracy1,
'train_error5': 1 - train_accuracy5, 'train_loss': train_loss})
save_checkpoint({'epoch': epoch + 1, 'state_dict': model.state_dict(), 'best_prec1': best_test,
'optimizer': optimizer.state_dict()}, test_accuracy1 > best_test, filepath=save_path)
csv_logger.plot_progress(claimed_acc1=claimed_acc1, claimed_acc5=claimed_acc5)
if test_accuracy1 > best_test:
best_test = test_accuracy1
csv_logger.write_text('Best accuracy is {:.2f}% top-1'.format(best_test * 100.))
def main():
args = get_args()
device, dtype = args.device, args.dtype
train_loader, val_loader = get_loaders(args.dataroot, args.batch_size,
args.batch_size, args.input_size,
args.workers, args.world_size,
args.local_rank)
model = MnasNet(n_class=args.num_classes,
width_mult=args.scaling,
drop_prob=0.0,
num_steps=len(train_loader) * args.epochs)
num_parameters = sum([l.nelement() for l in model.parameters()])
flops = flops_benchmark.count_flops(MnasNet,
1,
device,
dtype,
args.input_size,
3,
width_mult=args.scaling)
if not args.child:
print(model)
print('number of parameters: {}'.format(num_parameters))
print('FLOPs: {}'.format(flops))
# define loss function (criterion) and optimizer
criterion = CrossEntropyLoss()
mixup = Mixup(args.num_classes, args.mixup, args.smooth_eps)
model, criterion = model.to(device=device,
dtype=dtype), criterion.to(device=device,
dtype=dtype)
if args.dtype == torch.float16:
for module in model.modules(): # FP batchnorm
if is_bn(module):
module.to(dtype=torch.float32)
if args.distributed:
args.device_ids = [args.local_rank]
dist.init_process_group(backend=args.dist_backend,
init_method=args.dist_init,
world_size=args.world_size,
rank=args.local_rank)
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[args.local_rank], output_device=args.local_rank)
print('Node #{}'.format(args.local_rank))
else:
model = torch.nn.parallel.DataParallel(model,
device_ids=[args.local_rank],
output_device=args.local_rank)
optimizer_class = torch.optim.SGD
optimizer_params = {
"lr": args.learning_rate,
"momentum": args.momentum,
"weight_decay": args.decay,
"nesterov": True
}
if args.find_clr:
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.decay,
nesterov=True)
find_bounds_clr(model,
train_loader,
optimizer,
criterion,
device,
dtype,
min_lr=args.min_lr,
max_lr=args.max_lr,
step_size=args.epochs_per_step * len(train_loader),
mode=args.mode,
save_path=args.save_path)
return
if args.sched == 'clr':
scheduler_class = CyclicLR
scheduler_params = {
"base_lr": args.min_lr,
"max_lr": args.max_lr,
"step_size": args.epochs_per_step * len(train_loader),
"mode": args.mode
}
elif args.sched == 'multistep':
scheduler_class = MultiStepLR
scheduler_params = {"milestones": args.schedule, "gamma": args.gamma}
elif args.sched == 'cosine':
scheduler_class = CosineLR
scheduler_params = {
"max_epochs": args.epochs,
"warmup_epochs": args.warmup,
"iter_in_epoch": len(train_loader)
}
elif args.sched == 'gamma':
scheduler_class = StepLR
scheduler_params = {"step_size": 30, "gamma": args.gamma}
else:
raise ValueError('Wrong scheduler!')
optim = OptimizerWrapper(model,
optimizer_class=optimizer_class,
optimizer_params=optimizer_params,
scheduler_class=scheduler_class,
scheduler_params=scheduler_params,
use_shadow_weights=args.dtype == torch.float16)
best_test = 0
# optionally resume from a checkpoint
data = None
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume, map_location=device)
args.start_epoch = checkpoint['epoch'] - 1
best_test = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
optim.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
args.resume, checkpoint['epoch']))
elif os.path.isdir(args.resume):
checkpoint_path = os.path.join(
args.resume, 'checkpoint{}.pth.tar'.format(args.local_rank))
csv_path = os.path.join(args.resume,
'results{}.csv'.format(args.local_rank))
print("=> loading checkpoint '{}'".format(checkpoint_path))
checkpoint = torch.load(checkpoint_path, map_location=device)
args.start_epoch = checkpoint['epoch'] - 1
best_test = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
optim.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
checkpoint_path, checkpoint['epoch']))
data = []
with open(csv_path) as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
data.append(row)
else:
print("=> no checkpoint found at '{}'".format(args.resume))
if args.evaluate:
loss, top1, top5 = test(model, val_loader, criterion, device, dtype,
args.child) # TODO
return
csv_logger = CsvLogger(filepath=args.save_path,
data=data,
local_rank=args.local_rank)
csv_logger.save_params(sys.argv, args)
claimed_acc1 = None
claimed_acc5 = None
if args.input_size in claimed_acc_top1:
if args.scaling in claimed_acc_top1[args.input_size]:
claimed_acc1 = claimed_acc_top1[args.input_size][args.scaling]
if not args.child:
csv_logger.write_text(
'Claimed accuracy is {:.2f}% top-1'.format(claimed_acc1 *
100.))
train_network(args.start_epoch, args.epochs, optim, model, train_loader,
val_loader, criterion, mixup, device, dtype, args.batch_size,
args.log_interval, csv_logger, args.save_path, claimed_acc1,
claimed_acc5, best_test, args.local_rank, args.child)
def main():
args = parser.parse_args()
if args.seed is None:
args.seed = random.randint(1, 10000)
print("Random Seed: ", args.seed)
random.seed(args.seed)
torch.manual_seed(args.seed)
if args.gpus:
torch.cuda.manual_seed_all(args.seed)
time_stamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
if args.evaluate:
args.results_dir = '/tmp'
if args.save is '':
args.save = time_stamp
save_path = os.path.join(args.results_dir, args.save)
if not os.path.exists(save_path):
os.makedirs(save_path)
if args.gpus is not None:
args.gpus = [int(i) for i in args.gpus.split(',')]
device = 'cuda:' + str(args.gpus[0])
cudnn.benchmark = True
else:
device = 'cpu'
if args.type == 'float64':
dtype = torch.float64
elif args.type == 'float32':
dtype = torch.float32
elif args.type == 'float16':
dtype = torch.float16
else:
raise ValueError('Wrong type!') # TODO int8
model = MobileNet2(input_size=args.input_size, scale=args.scaling)
num_parameters = sum([l.nelement() for l in model.parameters()])
print(model)
print('number of parameters: {}'.format(num_parameters))
print('FLOPs: {}'.format(
flops_benchmark.count_flops(MobileNet2,
args.batch_size // len(args.gpus) if args.gpus is not None else args.batch_size,
device, dtype, args.input_size, 3, args.scaling)))
train_loader, val_loader = get_loaders(args.dataroot, args.batch_size, args.batch_size, args.input_size,
args.workers)
# define loss function (criterion) and optimizer
criterion = torch.nn.CrossEntropyLoss()
if args.gpus is not None:
model = torch.nn.DataParallel(model, args.gpus)
model.to(device=device, dtype=dtype)
criterion.to(device=device, dtype=dtype)
optimizer = torch.optim.SGD(model.parameters(), args.learning_rate, momentum=args.momentum, weight_decay=args.decay,
nesterov=True)
if args.find_clr:
find_bounds_clr(model, train_loader, optimizer, criterion, device, dtype, min_lr=args.min_lr,
max_lr=args.max_lr, step_size=args.epochs_per_step * len(train_loader), mode=args.mode,
save_path=save_path)
return
if args.clr:
scheduler = CyclicLR(optimizer, base_lr=args.min_lr, max_lr=args.max_lr,
step_size=args.epochs_per_step * len(train_loader), mode=args.mode)
else:
scheduler = MultiStepLR(optimizer, milestones=args.schedule, gamma=args.gamma)
best_test = 0
# optionally resume from a checkpoint
data = None
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume, map_location=device)
args.start_epoch = checkpoint['epoch'] - 1
best_test = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})"
.format(args.resume, checkpoint['epoch']))
elif os.path.isdir(args.resume):
checkpoint_path = os.path.join(args.resume, 'checkpoint.pth.tar')
csv_path = os.path.join(args.resume, 'results.csv')
print("=> loading checkpoint '{}'".format(checkpoint_path))
checkpoint = torch.load(checkpoint_path, map_location=device)
args.start_epoch = checkpoint['epoch'] - 1
best_test = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(checkpoint_path, checkpoint['epoch']))
data = []
with open(csv_path) as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
data.append(row)
else:
print("=> no checkpoint found at '{}'".format(args.resume))
if args.evaluate:
loss, top1, top5 = test(model, val_loader, criterion, device, dtype) # TODO
return
csv_logger = CsvLogger(filepath=save_path, data=data)
csv_logger.save_params(sys.argv, args)
claimed_acc1 = None
claimed_acc5 = None
if args.input_size in claimed_acc_top1:
if args.scaling in claimed_acc_top1[args.input_size]:
claimed_acc1 = claimed_acc_top1[args.input_size][args.scaling]
claimed_acc5 = claimed_acc_top5[args.input_size][args.scaling]
csv_logger.write_text(
'Claimed accuracies are: {:.2f}% top-1, {:.2f}% top-5'.format(claimed_acc1 * 100., claimed_acc5 * 100.))
train_network(args.start_epoch, args.epochs, scheduler, model, train_loader, val_loader, optimizer, criterion,
device, dtype, args.batch_size, args.log_interval, csv_logger, save_path, claimed_acc1, claimed_acc5,
best_test)
start_epoch = 0
epochs = 400
epochs_per_step = 20
log_interval = 100
mode = 'triangular2'
evaluate = 'false'
model = MobileNet2(input_size=input_size, scale=scaling)
num_parameters = sum([l.nelement() for l in model.parameters()])
print(model)
print('number of parameters: {}'.format(num_parameters))
print('FLOPs: {}'.format(
flops_benchmark.count_flops(MobileNet2,
batch_size // len(gpus) if gpus is not None else batch_size,
device, dtype, input_size, 3, scaling)))
train_loader, val_loader = get_loaders(dataroot, batch_size, batch_size, input_size,
workers)
# define loss function (criterion) and optimizer
criterion = torch.nn.CrossEntropyLoss()
if gpus is not None:
model = torch.nn.DataParallel(model, gpus)
model.to(device=device, dtype=dtype)
criterion.to(device=device, dtype=dtype)
optimizer = torch.optim.SGD(model.parameters(), learning_rate, momentum=momentum, weight_decay=decay,
nesterov=True)
if args.find_clr:
find_bounds_clr(model, train_loader, optimizer, criterion, device, dtype, min_lr=min_lr,
def main():
args = parser.parse_args()
if args.seed is None:
args.seed = random.randint(1, 10000)
print("Random Seed: ", args.seed)
random.seed(args.seed)
torch.manual_seed(args.seed)
if args.gpus:
torch.cuda.manual_seed_all(args.seed)
time_stamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
if args.evaluate:
args.results_dir = '/tmp'
if args.save is '':
args.save = 'mar10_224_' + time_stamp
save_path = os.path.join(args.results_dir, args.save)
if not os.path.exists(save_path):
os.makedirs(save_path)
if args.gpus is not None:
args.gpus = [int(i) for i in args.gpus.split(',')]
device = 'cuda:' + str(args.gpus[0])
cudnn.benchmark = True
else:
device = 'cpu'
if args.type == 'float64':
dtype = torch.float64
elif args.type == 'float32':
dtype = torch.float32
elif args.type == 'float16':
dtype = torch.float16
else:
raise ValueError('Wrong type!') # TODO int8
model = STN_MobileNet2(input_size=args.input_size,
scale=args.scaling,
shearing=args.shearing)
# print(model.stnmod.fc_loc[0].bias.data)
num_parameters = sum([l.nelement() for l in model.parameters()])
print(model)
print('number of parameters: {}'.format(num_parameters))
print('FLOPs: {}'.format(
flops_benchmark.count_flops(
STN_MobileNet2, args.batch_size //
len(args.gpus) if args.gpus is not None else args.batch_size,
device, dtype, args.input_size, 3, args.scaling)))
train_loader, val_loader, test_loader = get_loaders(
args.dataroot, args.batch_size, args.batch_size, args.input_size,
args.workers, args.b_weights)
# define loss function (criterion) and optimizer
criterion = torch.nn.CrossEntropyLoss()
L1_criterion = torch.nn.L1Loss()
PW_criterion = torch.nn.CosineSimilarity(dim=2, eps=1e-6)
if args.gpus is not None:
model = torch.nn.DataParallel(model, args.gpus)
model.to(device=device, dtype=dtype)
criterion.to(device=device, dtype=dtype)
L1_criterion.to(device=device, dtype=dtype)
PW_criterion.to(device=device, dtype=dtype)
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.decay,
nesterov=True)
if args.find_clr:
find_bounds_clr(model,
train_loader,
optimizer,
PW_criterion,
device,
dtype,
min_lr=args.min_lr,
max_lr=args.max_lr,
step_size=args.epochs_per_step * len(train_loader),
mode=args.mode,
save_path=save_path)
return
if args.clr:
print('Use CLR')
scheduler = CyclicLR(optimizer,
base_lr=args.min_lr,
max_lr=args.max_lr,
step_size=args.epochs_per_step *
len(train_loader),
mode=args.mode)
else:
print('Use scheduler')
scheduler = MultiStepLR(optimizer,
milestones=args.schedule,
gamma=args.gamma)
best_val = 500
# optionally resume from a checkpoint
data = None
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume, map_location=device)
# args.start_epoch = checkpoint['epoch'] - 1
# best_val = checkpoint['best_prec1']
# best_test = checkpoint['best_prec1']
args.start_epoch = 0
best_val = 500
state_dict = checkpoint['state_dict']
# if weights from imagenet
new_state_dict = OrderedDict()
for k, v in state_dict.items():
# print(k, v.size())
name = k
if k == 'module.fc.bias':
new_state_dict[name] = torch.zeros(101)
continue
elif k == 'module.fc.weight':
new_state_dict[name] = torch.ones(101, 1280)
continue
else:
print('else:', name)
new_state_dict[name] = v
model.load_state_dict(new_state_dict, strict=False)
# optimizer.load_state_dict(checkpoint['optimizer'], strict=False)
print("=> loaded checkpoint '{}' (epoch {})".format(
args.resume, checkpoint['epoch']))
elif os.path.isdir(args.resume):
checkpoint_path = os.path.join(args.resume, 'checkpoint.pth.tar')
csv_path = os.path.join(args.resume, 'results.csv')
print("=> loading checkpoint '{}'".format(checkpoint_path))
checkpoint = torch.load(checkpoint_path, map_location=device)
args.start_epoch = checkpoint['epoch'] - 1
best_val = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
checkpoint_path, checkpoint['epoch']))
data = []
with open(csv_path) as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
data.append(row)
else:
print("=> no checkpoint found at '{}'".format(args.resume))
if args.evaluate:
loss, test_mae = test(model, predefined_points, 0, test_loader,
PW_criterion, device, dtype) # TODO
return
csv_logger = CsvLogger(filepath=save_path, data=data)
csv_logger.save_params(sys.argv, args)
# claimed_acc1 = None
# claimed_acc5 = None
# if args.input_size in claimed_acc_top1:
# if args.scaling in claimed_acc_top1[args.input_size]:
# claimed_acc1 = claimed_acc_top1[args.input_size][args.scaling]
# claimed_acc5 = claimed_acc_top5[args.input_size][args.scaling]
# csv_logger.write_text(
# 'Claimed accuracies are: {:.2f}% top-1, {:.2f}% top-5'.format(claimed_acc1 * 100., claimed_acc5 * 100.))
train_network(args.start_epoch, args.epochs, scheduler, model,
predefined_points, train_loader, val_loader, test_loader,
optimizer, PW_criterion, device, dtype, args.batch_size,
args.log_interval, csv_logger, save_path, best_val)
def main():
args = parser.parse_args()
if args.gpus is not None:
args.gpus = [int(i) for i in args.gpus.split(',')]
device = 'cuda:' + str(args.gpus[0])
cudnn.benchmark = True
else:
device = 'cpu'
if args.type == 'float64':
dtype = torch.float64
elif args.type == 'float32':
dtype = torch.float32
elif args.type == 'float16':
dtype = torch.float16
else:
raise ValueError('Wrong type!') # TODO int8
arch_setting = [(24, 1, 1), (24, 1, 1), (24, 1, 1), (24, 1, 1), (24, 1, 1),
(24, 1, 1), (32, 1, 2), (32, 1, 1), (32, 1, 1), (32, 1, 1),
(32, 1, 1), (32, 1, 1), (64, 1, 2), (64, 1, 1), (64, 1, 1),
(64, 1, 1), (64, 1, 1), (64, 1, 1), (96, 1, 1), (96, 1, 1),
(96, 1, 1), (96, 1, 1), (96, 1, 1), (96, 1, 1),
(160, 1, 2), (160, 1, 1), (160, 1, 1), (160, 1, 1),
(160, 1, 1), (160, 1, 1), (320, 1, 1), (320, 1, 1),
(320, 1, 1), (320, 1, 1), (320, 1, 1), (320, 1, 1)]
print("Evaluate RCNet on CIFAR100")
bin_setting = [
5, 3, 0, 1, 0, 0, 5, 2, 0, 3, 3, 5, 5, 5, 3, 3, 5, 6, 5, 1, 6, 5, 4, 6,
5, 4, 6, 5, 4, 1, 5, 0, 0, 0, 0, 0
]
cfg4net = create_arch4net(bin_setting, arch_setting)
model = RCNet(cfg4net)
file_handler = RCNet
num_parameters = sum([l.nelement() for l in model.parameters()])
print(model)
print('number of parameters: {}'.format(num_parameters))
print('FLOPs: {}'.format(
flops_benchmark.count_flops(
file_handler, args.batch_size //
len(args.gpus) if args.gpus is not None else args.batch_size,
device, dtype, args.input_size, 3, args.scaling, cfg4net)))
testset = torchvision.datasets.CIFAR100(root='./data',
train=False,
download=True,
transform=transform_test)
test_loader = torch.utils.data.DataLoader(testset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers)
# define loss function (criterion)
criterion = torch.nn.CrossEntropyLoss()
if args.gpus is not None:
model = torch.nn.DataParallel(model, args.gpus)
model.to(device=device, dtype=dtype)
criterion.to(device=device, dtype=dtype)
if os.path.isfile(args.load):
print("=> loading checkpoint '{}'".format(args.load))
checkpoint = torch.load(args.load, map_location=device)
model.load_state_dict(checkpoint['state_dict'])
loss, top1, top5 = test(model, test_loader, criterion, device, dtype)
print("=> Test loss: {}, Top 1 accu: {}, Top 5 accu: {}").format(
loss, top1, top5)
return
def main():
args = parser.parse_args()
if args.seed is None:
args.seed = random.randint(1, 10000)
print("Random Seed: ", args.seed)
random.seed(args.seed)
torch.manual_seed(args.seed)
if args.gpus:
torch.cuda.manual_seed_all(args.seed)
time_stamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
if args.evaluate:
args.results_dir = '/tmp'
if args.save is '':
args.save = time_stamp
save_path = os.path.join(args.results_dir, args.save)
if not os.path.exists(save_path):
os.makedirs(save_path)
if args.gpus is not None:
args.gpus = [int(i) for i in args.gpus.split(',')]
device = 'cuda:' + str(args.gpus[0])
cudnn.benchmark = True
else:
device = 'cpu'
if args.type == 'float64':
dtype = torch.float64
elif args.type == 'float32':
dtype = torch.float32
elif args.type == 'float16':
dtype = torch.float16
else:
raise ValueError('Wrong type!') # TODO int8
model = ShuffleNetV2(scale=args.scaling,
c_tag=args.c_tag,
SE=args.SE,
residual=args.residual,
groups=args.groups)
num_parameters = sum([l.nelement() for l in model.parameters()])
print(model)
print('number of parameters: {}'.format(num_parameters))
print('FLOPs: {}'.format(
flops_benchmark.count_flops(
ShuffleNetV2, args.batch_size //
len(args.gpus) if args.gpus is not None else args.batch_size,
device, dtype, args.input_size, 3, args.scaling, 3, args.c_tag,
1000, torch.nn.ReLU, args.SE, args.residual, args.groups)))
train_loader, val_loader = get_loaders(args.dataroot, args.batch_size,
args.batch_size, args.input_size,
args.workers)
# define loss function (criterion) and optimizer
criterion = torch.nn.CrossEntropyLoss()
if args.gpus is not None:
model = torch.nn.DataParallel(model, args.gpus)
model.to(device=device, dtype=dtype)
criterion.to(device=device, dtype=dtype)
optimizer = torch.optim.SGD(model.parameters(),
args.learning_rate,
momentum=args.momentum,
weight_decay=args.decay,
nesterov=True)
if args.find_clr:
find_bounds_clr(model,
train_loader,
optimizer,
criterion,
device,
dtype,
min_lr=args.min_lr,
max_lr=args.max_lr,
step_size=args.epochs_per_step * len(train_loader),
mode=args.mode,
save_path=save_path)
return
if args.clr:
scheduler = CyclicLR(optimizer,
base_lr=args.min_lr,
max_lr=args.max_lr,
step_size=args.epochs_per_step *
len(train_loader),
mode=args.mode)
else:
scheduler = MultiStepLR(optimizer,
milestones=args.schedule,
gamma=args.gamma)
best_test = 0
# optionally resume from a checkpoint
data = None
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume, map_location=device)
args.start_epoch = checkpoint['epoch'] - 1
best_test = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
args.resume, checkpoint['epoch']))
elif os.path.isdir(args.resume):
checkpoint_path = os.path.join(args.resume, 'checkpoint.pth.tar')
csv_path = os.path.join(args.resume, 'results.csv')
print("=> loading checkpoint '{}'".format(checkpoint_path))
checkpoint = torch.load(checkpoint_path, map_location=device)
args.start_epoch = checkpoint['epoch'] - 1
best_test = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
checkpoint_path, checkpoint['epoch']))
data = []
with open(csv_path) as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
data.append(row)
else:
print("=> no checkpoint found at '{}'".format(args.resume))
if args.evaluate:
loss, top1, top5 = test(model, val_loader, criterion, device,
dtype) # TODO
return
csv_logger = CsvLogger(filepath=save_path, data=data)
csv_logger.save_params(sys.argv, args)
claimed_acc1 = None
claimed_acc5 = None
if args.SE in claimed_acc_top1:
if args.scaling in claimed_acc_top1[args.SE]:
claimed_acc1 = 1 - claimed_acc_top1[args.SE][args.scaling]
csv_logger.write_text('Claimed accuracy is {:.2f}% top-1'.format(
claimed_acc1 * 100.))
train_network(args.start_epoch, args.epochs, scheduler, model,
train_loader, val_loader, optimizer, criterion, device,
dtype, args.batch_size, args.log_interval, csv_logger,
save_path, claimed_acc1, claimed_acc5, best_test)
def main():
import warnings
# filter out corrupted images warnings
warnings.filterwarnings("ignore", "(Possibly )?corrupt EXIF data", UserWarning)
args = get_args()
device, dtype = args.device, args.dtype
train_loader, val_loader = get_loaders(args.dataroot, args.batch_size, args.batch_size, args.input_size,
args.workers, args.world_size, args.local_rank)
args.num_batches = len(train_loader) * args.epochs
args.start_step = len(train_loader) * args.start_epoch
model = MobileNetV3(num_classes=args.num_classes, scale=args.scaling, in_channels=3, drop_prob=args.dp,
num_steps=args.num_batches, start_step=args.start_step, small=args.small)
num_parameters = sum([l.nelement() for l in model.parameters()])
flops = flops_benchmark.count_flops(MobileNetV3, 2, device, dtype, args.input_size, 3, num_classes=args.num_classes,
scale=args.scaling, drop_prob=args.dp, num_steps=args.num_batches,
start_step=args.start_step, small=args.small)
if not args.child:
print(model)
print('number of parameters: {}'.format(num_parameters))
print('FLOPs: {}'.format(flops))
print('Resuts saved to {}'.format(args.save_path))
# define loss function (criterion) and optimizer
criterion = CrossEntropyLoss()
model, criterion = model.to(device=device, dtype=dtype), criterion.to(device=device, dtype=dtype)
if args.dtype == torch.float16:
for module in model.modules(): # FP batchnorm
if is_bn(module):
module.to(dtype=torch.float32) # github.com/pytorch/pytorch/issues/20634
if args.distributed:
args.device_ids = [args.local_rank]
dist.init_process_group(backend=args.dist_backend, init_method=args.dist_init, world_size=args.world_size,
rank=args.local_rank)
if args.sync_bn:
model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model)
model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[args.local_rank],
output_device=args.local_rank)
print('Node #{}'.format(args.local_rank))
else:
model = torch.nn.parallel.DataParallel(model, device_ids=[args.local_rank], output_device=args.local_rank)
if args.find_clr:
optimizer = torch.optim.SGD(model.parameters(), args.learning_rate, momentum=args.momentum,
weight_decay=args.decay, nesterov=True)
find_bounds_clr(model, train_loader, optimizer, criterion, device, dtype, min_lr=args.min_lr,
max_lr=args.max_lr, step_size=args.epochs_per_step * len(train_loader), mode=args.mode,
save_path=args.save_path)
return
best_test = 0
# optionally resume from a checkpoint
data = None
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume, map_location=device)
args.start_epoch = checkpoint['epoch']
args.start_step = len(train_loader) * args.start_epoch
optim, mixup = init_optimizer_and_mixup(args, train_loader, model, checkpoint['optimizer'])
best_test = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
print("=> loaded checkpoint '{}' (epoch {})".format(args.resume, checkpoint['epoch']))
elif os.path.isdir(args.resume):
checkpoint_path = os.path.join(args.resume, 'checkpoint{}.pth.tar'.format(args.local_rank))
csv_path = os.path.join(args.resume, 'results{}.csv'.format(args.local_rank))
print("=> loading checkpoint '{}'".format(checkpoint_path))
checkpoint = torch.load(checkpoint_path, map_location=device)
args.start_epoch = checkpoint['epoch']
args.start_step = len(train_loader) * args.start_epoch
optim, mixup = init_optimizer_and_mixup(args, train_loader, model, checkpoint['optimizer'])
best_test = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
print("=> loaded checkpoint '{}' (epoch {})".format(checkpoint_path, checkpoint['epoch']))
data = []
with open(csv_path) as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
data.append(row)
else:
print("=> no checkpoint found at '{}'".format(args.resume))
else:
optim, mixup = init_optimizer_and_mixup(args, train_loader, model)
if args.evaluate:
if args.swa:
sd = swa_clr(args.swa, device)
model.load_state_dict(sd)
loss, top1, top5 = test(model, val_loader, criterion, device, dtype, args.child) # TODO
return
csv_logger = CsvLogger(filepath=args.save_path, data=data, local_rank=args.local_rank)
csv_logger.save_params(sys.argv, args)
claimed_acc1 = None
claimed_acc5 = None
ntype = 'small' if args.small else 'large'
if ntype in claimed_acc_top1:
if args.input_size in claimed_acc_top1[ntype]:
if args.scaling in claimed_acc_top1[ntype][args.input_size]:
claimed_acc1 = claimed_acc_top1[ntype][args.input_size][args.scaling]
if not args.child:
csv_logger.write_text('Claimed accuracy is {:.2f}% top-1'.format(claimed_acc1 * 100.))
train_network(args.start_epoch, args.epochs, optim, model, train_loader, val_loader, criterion, mixup,
device, dtype, args.batch_size, args.log_interval, csv_logger, args.save_path, claimed_acc1,
claimed_acc5, best_test, args.local_rank, args.child)