def get_model(self): image = fluid.layers.data( name='image', shape=[1, 28, 28], dtype='float32') label = fluid.layers.data(name='label', shape=[1], dtype='int64') model = MobileNet() out = model.net(input=image, class_dim=10) cost = fluid.layers.cross_entropy(input=out, label=label) avg_cost = fluid.layers.mean(x=cost) startup_prog = fluid.default_startup_program() train_prog = fluid.default_main_program() return startup_prog, train_prog
def get_model(self): image = paddle.static.data( name='image', shape=[None, 1, 28, 28], dtype='float32') label = paddle.static.data(name='label', shape=[None, 1], dtype='int64') model = MobileNet() out = model.net(input=image, class_dim=10) cost = paddle.nn.functional.loss.cross_entropy(input=out, label=label) avg_cost = paddle.mean(x=cost) startup_prog = paddle.static.default_startup_program() train_prog = paddle.static.default_main_program() return startup_prog, train_prog
def load_model(model_name): if model_name == 'deepyeast': from models import DeepYeast model = DeepYeast() elif model_name == 'resnet': from models import ResNet50 model = ResNet50() elif model_name == 'mobilenet': from models import MobileNet model = MobileNet() elif model_name == 'densenet': from models import DenseNet40_BC model = DenseNet40_BC() return model
def cfg2fitness(cfg): if args.local_rank == 0: print(str(cfg)) if str(cfg) in cfg2fit_dict.keys(): return cfg2fit_dict[str(cfg)] elif cfg == run_manager.net.module.config['cfg_base']: return 0. else: run_manager.run_config.n_epochs = run_manager.run_config.search_epoch if args.model == 'resnet18': run_manager.reset_model( ResNet_ImageNet(num_classes=1000, cfg=cfg, depth=18), net_origin.cpu()) elif args.model == 'resnet34': run_manager.reset_model( ResNet_ImageNet(num_classes=1000, cfg=cfg, depth=34), net_origin.cpu()) elif args.model == 'resnet50': run_manager.reset_model( ResNet_ImageNet(num_classes=1000, cfg=cfg, depth=50), net_origin.cpu()) elif args.model == 'mobilenet': run_manager.reset_model(MobileNet(num_classes=1000, cfg=cfg), net_origin.cpu()) elif args.model == 'mobilenetv2': run_manager.reset_model(MobileNetV2(num_classes=1000, cfg=cfg), net_origin.cpu()) elif args.model == 'vgg': run_manager.reset_model( VGG_CIFAR(cfg=cfg, cutout=False, num_classes=10), net_origin.cpu()) elif args.model == 'resnet56': run_manager.reset_model( ResNet_CIFAR(cfg=cfg, depth=56, num_classes=10, cutout=False), net_origin.cpu()) elif args.model == 'resnet110': run_manager.reset_model( ResNet_CIFAR(cfg=cfg, depth=110, num_classes=10, cutout=False), net_origin.cpu()) run_manager.start_epoch = 0 run_manager.train() _, acc1, _ = run_manager.validate(is_test=False, return_top5=True) cfg2fit_dict[str(cfg)] = acc1.item() return acc1.item()
def test_analysis_helper(self): image = paddle.static.data( name='image', shape=[None, 1, 28, 28], dtype='float32') label = paddle.static.data(name='label', shape=[None, 1], dtype='int64') model = MobileNet() out = model.net(input=image, class_dim=10) cost = paddle.nn.functional.loss.cross_entropy(input=out, label=label) avg_cost = paddle.mean(x=cost) acc_top1 = paddle.metric.accuracy(input=out, label=label, k=1) acc_top5 = paddle.metric.accuracy(input=out, label=label, k=5) optimizer = paddle.optimizer.Momentum( momentum=0.9, learning_rate=0.01, weight_decay=paddle.regularizer.L2Decay(4e-5)) optimizer.minimize(avg_cost) main_prog = paddle.static.default_main_program() places = paddle.static.cuda_places() if paddle.is_compiled_with_cuda( ) else paddle.static.cpu_places() exe = paddle.static.Executor(places[0]) def transform(x): return np.reshape(x, [1, 28, 28]) train_dataset = paddle.vision.datasets.MNIST( mode='train', backend='cv2', transform=transform) train_loader = paddle.io.DataLoader( train_dataset, places=places, feed_list=[image, label], drop_last=True, return_list=False, batch_size=64) exe.run(paddle.static.default_startup_program()) vars = ['conv2d_0.tmp_0', 'fc_0.tmp_0', 'fc_0.tmp_1', 'fc_0.tmp_2'] var_collector1 = VarCollector(main_prog, vars, use_ema=True) values = var_collector1.abs_max_run( train_loader, exe, step=None, loss_name=avg_cost.name) vars = [v.name for v in main_prog.list_vars() if v.persistable] var_collector2 = VarCollector(main_prog, vars, use_ema=False) values = var_collector2.run(train_loader, exe, step=None, loss_name=avg_cost.name) var_collector2.pdf(values)
def build_model(device, model_name, num_classes=10): """构建模型:vgg、vggnonorm、resnet、preactresnet、googlenet、densenet、 resnext、mobilenet、mobilenetv2、dpn、shufflenetg2、senet、shufflenetv2 :param device: 'cuda' if you have a GPU, 'cpu' otherwise :param model_name: One of the models available in the folder 'models' :param num_classes: 10 or 100 depending on the chosen dataset :return: The model architecture """ print('==> Building model..') model_name = model_name.lower() if model_name == 'vgg': net = VGG('VGG19', num_classes=num_classes) elif model_name == 'vggnonorm': net = VGG('VGG19', num_classes=num_classes, batch_norm=False) elif model_name == 'resnet': net = ResNet18(num_classes=num_classes) elif model_name == 'preactresnet': net = PreActResNet18() elif model_name == 'googlenet': net = GoogLeNet() elif model_name == 'densenet': net = DenseNet121() elif model_name == 'resnext': net = ResNeXt29_2x64d() elif model_name == 'mobilenet': net = MobileNet() elif model_name == 'mobilenetv2': net = MobileNetV2() elif model_name == 'dpn': net = DPN92() elif model_name == 'shufflenetg2': net = ShuffleNetG2() elif model_name == 'senet': net = SENet18() elif model_name == 'shufflenetv2': net = ShuffleNetV2(1) else: raise ValueError('Error: the specified model is incorrect ({})'.format(model_name)) net = net.to(device) if device == 'cuda': net = torch.nn.DataParallel(net) cudnn.benchmark = True return net
def test_analysis_helper(self): image = fluid.layers.data(name='image', shape=[1, 28, 28], dtype='float32') label = fluid.layers.data(name='label', shape=[1], dtype='int64') model = MobileNet() out = model.net(input=image, class_dim=10) cost = fluid.layers.cross_entropy(input=out, label=label) avg_cost = fluid.layers.mean(x=cost) acc_top1 = fluid.layers.accuracy(input=out, label=label, k=1) acc_top5 = fluid.layers.accuracy(input=out, label=label, k=5) optimizer = fluid.optimizer.Momentum( momentum=0.9, learning_rate=0.01, regularization=fluid.regularizer.L2Decay(4e-5)) optimizer.minimize(avg_cost) main_prog = fluid.default_main_program() places = fluid.cuda_places() if fluid.is_compiled_with_cuda( ) else fluid.cpu_places() exe = fluid.Executor(places[0]) train_reader = paddle.fluid.io.batch(paddle.dataset.mnist.train(), batch_size=64) train_loader = fluid.io.DataLoader.from_generator( feed_list=[image, label], capacity=512, use_double_buffer=True, iterable=True) train_loader.set_sample_list_generator(train_reader, places) exe.run(fluid.default_startup_program()) vars = ['conv2d_0.tmp_0', 'fc_0.tmp_0', 'fc_0.tmp_1', 'fc_0.tmp_2'] var_collector1 = VarCollector(main_prog, vars, use_ema=True) values = var_collector1.abs_max_run(train_loader, exe, step=None, loss_name=avg_cost.name) vars = [v.name for v in main_prog.list_vars() if v.persistable] var_collector2 = VarCollector(main_prog, vars, use_ema=False) values = var_collector2.run(train_loader, exe, step=None, loss_name=avg_cost.name) var_collector2.pdf(values)
def main(): global best_err1, best_err5 if config.train_params.use_seed: utils.set_seed(config.train_params.seed) imagenet = imagenet_data.ImageNet12(trainFolder=os.path.join(config.data.data_path, 'train'), testFolder=os.path.join(config.data.data_path, 'val'), num_workers=config.data.num_workers, type_of_data_augmentation=config.data.type_of_data_aug, data_config=config.data) train_loader, val_loader = imagenet.getTrainTestLoader(config.data.batch_size) if config.net_type == 'mobilenet': t_net = ResNet.resnet50(pretrained=True) s_net = Mov.MobileNet() elif config.net_type == 'resnet': t_net = ResNet.resnet34(pretrained=True) s_net = ResNet.resnet18(pretrained=False) else: print('undefined network type !!!') raise RuntimeError('%s does not support' % config.net_type) import knowledge_distiller d_net = knowledge_distiller.WSLDistiller(t_net, s_net) print('Teacher Net: ') print(t_net) print('Student Net: ') print(s_net) print('the number of teacher model parameters: {}'.format(sum([p.data.nelement() for p in t_net.parameters()]))) print('the number of student model parameters: {}'.format(sum([p.data.nelement() for p in s_net.parameters()]))) t_net = torch.nn.DataParallel(t_net) s_net = torch.nn.DataParallel(s_net) d_net = torch.nn.DataParallel(d_net) if config.optim.if_resume: checkpoint = torch.load(config.optim.resume_path) d_net.module.load_state_dict(checkpoint['train_state_dict']) best_err1 = checkpoint['best_err1'] best_err5 = checkpoint['best_err5'] start_epoch = checkpoint['epoch'] + 1 else: start_epoch = 0 t_net = t_net.cuda() s_net = s_net.cuda() d_net = d_net.cuda() ### choose optimizer parameters optimizer = torch.optim.SGD(list(s_net.parameters()), config.optim.init_lr, momentum=config.optim.momentum, weight_decay=config.optim.weight_decay, nesterov=True) cudnn.benchmark = True cudnn.enabled = True print('Teacher network performance') validate(val_loader, t_net, 0) for epoch in range(start_epoch, config.train_params.epochs + 1): adjust_learning_rate(optimizer, epoch) # train for one epoch train_with_distill(train_loader, d_net, optimizer, epoch) # evaluate on validation set err1, err5 = validate(val_loader, s_net, epoch) # remember best prec@1 and save checkpoint is_best = err1 <= best_err1 best_err1 = min(err1, best_err1) if is_best: best_err5 = err5 print('Current best accuracy (top-1 and 5 error):', best_err1, best_err5) save_checkpoint({ 'epoch': epoch, 'state_dict': s_net.module.state_dict(), 'train_state_dict': d_net.module.state_dict(), 'best_err1': best_err1, 'best_err5': best_err5, 'optimizer': optimizer.state_dict(), }, is_best) gc.collect() print('Best accuracy (top-1 and 5 error):', best_err1, best_err5)
def train_net(args): torch.manual_seed(7) np.random.seed(7) checkpoint = args.checkpoint start_epoch = 0 best_acc = 0 writer = SummaryWriter() epochs_since_improvement = 0 # Initialize / load checkpoint if checkpoint is None: if args.network == 'r18': model = resnet18(args) elif args.network == 'r34': model = resnet34(args) elif args.network == 'r50': model = resnet50(args) elif args.network == 'r101': model = resnet101(args) elif args.network == 'r152': model = resnet152(args) elif args.network == 'mobile': model = MobileNet(1.0) else: model = resnet_face18(args.use_se) model = nn.DataParallel(model) metric_fc = ArcMarginModel(args) metric_fc = nn.DataParallel(metric_fc) if args.optimizer == 'sgd': # optimizer = torch.optim.SGD([{'params': model.parameters()}, {'params': metric_fc.parameters()}], # lr=args.lr, momentum=args.mom, weight_decay=args.weight_decay) optimizer = InsightFaceOptimizer( torch.optim.SGD([{ 'params': model.parameters() }, { 'params': metric_fc.parameters() }], lr=args.lr, momentum=args.mom, weight_decay=args.weight_decay)) else: optimizer = torch.optim.Adam([{ 'params': model.parameters() }, { 'params': metric_fc.parameters() }], lr=args.lr, weight_decay=args.weight_decay) else: checkpoint = torch.load(checkpoint) start_epoch = checkpoint['epoch'] + 1 epochs_since_improvement = checkpoint['epochs_since_improvement'] model = checkpoint['model'] metric_fc = checkpoint['metric_fc'] optimizer = checkpoint['optimizer'] logger = get_logger() # Move to GPU, if available model = model.to(device) metric_fc = metric_fc.to(device) # Loss function if args.focal_loss: criterion = FocalLoss(gamma=args.gamma).to(device) else: criterion = nn.CrossEntropyLoss().to(device) # Custom dataloaders train_dataset = ArcFaceDataset('train') train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=8) # Epochs for epoch in range(start_epoch, args.end_epoch): # One epoch's training train_loss, train_top1_accs = train(train_loader=train_loader, model=model, metric_fc=metric_fc, criterion=criterion, optimizer=optimizer, epoch=epoch, logger=logger) print('\nCurrent effective learning rate: {}\n'.format(optimizer.lr)) print('Step num: {}\n'.format(optimizer.step_num)) writer.add_scalar('model/train_loss', train_loss, epoch) writer.add_scalar('model/train_accuracy', train_top1_accs, epoch) writer.add_scalar('model/learning_rate', optimizer.lr, epoch) # One epoch's validation megaface_acc = megaface_test(model) writer.add_scalar('model/megaface_accuracy', megaface_acc, epoch) # Check if there was an improvement is_best = megaface_acc > best_acc best_acc = max(megaface_acc, best_acc) if not is_best: epochs_since_improvement += 1 print("\nEpochs since last improvement: %d\n" % (epochs_since_improvement, )) else: epochs_since_improvement = 0 # Save checkpoint save_checkpoint(epoch, epochs_since_improvement, model, metric_fc, optimizer, best_acc, is_best)
def main(): parser = argparse.ArgumentParser() parser.add_argument("--mode", type=str, default="train") parser.add_argument("--model", type=str, default="mobilenet_v2") parser.add_argument("--dataset", type=str, default="cifar10") parser.add_argument("--dataroot", type=str, default="/tmp/data") parser.add_argument("--batch_size", type=int, default=128) parser.add_argument("--n_epochs", type=int, default=100) parser.add_argument("--lr", type=float, default=1e-3) parser.add_argument("--n_gpus", type=int, default=1) parser.add_argument("--checkpoint", type=str, default="/tmp/chkpt.pth.tar") parser.add_argument("--save_every", type=int, default=10) parser.add_argument("--pretrained", type=str, default=None) args = parser.parse_args() print(args) if torch.cuda.is_available(): print("cuda is available, use cuda") device = torch.device("cuda") else: print("cuda is not available, use cpu") device = torch.device("cpu") print("download dataset: {}".format(args.dataset)) train_loader, test_loader, n_classes = get_loaders( dataset=args.dataset, root=args.dataroot, batch_size=args.batch_size) print("build model: {}".format(args.model)) if args.model == "mobilenet": from models import MobileNet model = MobileNet(n_classes=n_classes) elif args.model == "mobilenet_v2": from models import MobileNet_v2 model = MobileNet_v2(n_classes=n_classes) elif args.model == "shufflenet": from models import ShuffleNet model = ShuffleNet(n_classes=n_classes) elif args.model == "shufflenet_v2": from models import ShuffleNet_v2 model = ShuffleNet_v2(n_classes=n_classes) elif args.model == "squeezenet": from models import SqueezeNet model = SqueezeNet(n_classes=n_classes) else: raise NotImplementedError model = model.to(device) if args.pretrained: model.load_state_dict(torch.load(args.checkpoint)) if args.n_gpus > 1: gpus = [] for i in range(args.n_gpus): gpus.append(i) model = nn.DataParallel(model, device_ids=gpus) optimizer = optim.Adam(model.parameters(), lr=args.lr) criterion = nn.CrossEntropyLoss() if args.mode == "train": for epoch in range(args.n_epochs): train(epoch, model, optimizer, criterion, train_loader, device) if (epoch + 1) % args.save_every == 0: print("saving model...") torch.save(the_model.state_dict(), args.checkpoint) elif args.mode == "test": test(model, criterion, test_loader, device) else: raise NotImplementedError
def test_accuracy(self): image = paddle.static.data( name='image', shape=[None, 1, 28, 28], dtype='float32') label = paddle.static.data(name='label', shape=[None, 1], dtype='int64') model = MobileNet() out = model.net(input=image, class_dim=10) cost = paddle.nn.functional.loss.cross_entropy(input=out, label=label) avg_cost = paddle.mean(x=cost) acc_top1 = paddle.metric.accuracy(input=out, label=label, k=1) acc_top5 = paddle.metric.accuracy(input=out, label=label, k=5) optimizer = paddle.optimizer.Momentum( momentum=0.9, learning_rate=0.01, weight_decay=paddle.regularizer.L2Decay(4e-5)) optimizer.minimize(avg_cost) main_prog = paddle.static.default_main_program() val_prog = main_prog.clone(for_test=True) place = paddle.CUDAPlace(0) if paddle.fluid.is_compiled_with_cuda( ) else paddle.CPUPlace() exe = paddle.static.Executor(place) exe.run(paddle.static.default_startup_program()) train_loader = paddle.io.DataLoader.from_generator( feed_list=[image, label], capacity=512, use_double_buffer=True, iterable=True) valid_loader = paddle.io.DataLoader.from_generator( feed_list=[image, label], capacity=512, use_double_buffer=True, iterable=True) train_reader = paddle.batch(paddle.dataset.mnist.train(), batch_size=64) eval_reader = paddle.batch(paddle.dataset.mnist.test(), batch_size=64) train_loader.set_sample_list_generator(train_reader, place) valid_loader.set_sample_list_generator(eval_reader, place) def train(program): iter = 0 for data in train_loader(): cost, top1, top5 = exe.run( program, feed=data, fetch_list=[avg_cost, acc_top1, acc_top5]) iter += 1 if iter % 100 == 0: print( 'train iter={}, avg loss {}, acc_top1 {}, acc_top5 {}'. format(iter, cost, top1, top5)) def test(program, outputs=[avg_cost, acc_top1, acc_top5]): iter = 0 result = [[], [], []] for data in valid_loader(): cost, top1, top5 = exe.run(program, feed=data, fetch_list=outputs) iter += 1 if iter % 100 == 0: print('eval iter={}, avg loss {}, acc_top1 {}, acc_top5 {}'. format(iter, cost, top1, top5)) result[0].append(cost) result[1].append(top1) result[2].append(top5) print(' avg loss {}, acc_top1 {}, acc_top5 {}'.format( np.mean(result[0]), np.mean(result[1]), np.mean(result[2]))) return np.mean(result[1]), np.mean(result[2]) train(main_prog) top1_1, top5_1 = test(val_prog) paddle.static.save_inference_model( dirname='./test_quant_post_dynamic', feeded_var_names=[image.name, label.name], target_vars=[avg_cost, acc_top1, acc_top5], main_program=val_prog, executor=exe, model_filename='model', params_filename='params') quant_post_dynamic( model_dir='./test_quant_post_dynamic', save_model_dir='./test_quant_post_inference', model_filename='model', params_filename='params', generate_test_model=True) quant_post_prog, feed_target_names, fetch_targets = paddle.static.load_inference_model( dirname='./test_quant_post_inference/test_model', executor=exe) top1_2, top5_2 = test(quant_post_prog, fetch_targets) print("before quantization: top1: {}, top5: {}".format(top1_1, top5_1)) print("after quantization: top1: {}, top5: {}".format(top1_2, top5_2))
]) # Create dataset train_dataset = MyDataset(filename=train_fpath, transform=train_transform) val_dataset = MyDataset(filename=train_fpath, transform=val_transform) # Split train and val data_len = len(train_dataset) indices = list(range(data_len)) val_len = int(VAL_RATIO * data_len) val_idx = np.random.choice(indices, size=val_len, replace=False) train_idx = list(set(indices) - set(val_idx)) train_sampler = SubsetRandomSampler(train_idx) val_sampler = SubsetRandomSampler(val_idx) train_loader = DataLoader(dataset=train_dataset, batch_size=BATCH_SIZE, sampler=train_sampler) val_loader = DataLoader(dataset=val_dataset, batch_size=BATCH_SIZE, sampler=val_sampler) # Train print('# [Info] Start training...') mobile = MobileNet() trainer = Trainer(model=mobile, train_loader=train_loader, val_loader=val_loader, weight_fpath=weight_fpath) trainer.train(epochs=EPOCHS) print('Done!')
num_workers=opt.n_workers, shuffle=True) test_data_loader = DataLoader(test_dataset, batch_size=opt.batch_size, num_workers=opt.n_workers, shuffle=False) backbone_network = opt.backbone_network n_layers = opt.n_layers if backbone_network == 'MobileNet': from models import MobileNet model = MobileNet(width_multiplier=opt.width_multiplier, attention=opt.attention_module, group_size=opt.group_size) if backbone_network == 'ResNet': from models import ResidualNetwork model = ResidualNetwork(n_layers=n_layers, dataset=opt.dataset, attention=opt.attention_module, group_size=opt.group_size) elif backbone_network == 'ResNext': from models import ResNext model = ResNext(n_layers=n_layers, n_groups=opt.n_groups,
def test_accuracy(self): image = fluid.layers.data(name='image', shape=[1, 28, 28], dtype='float32') label = fluid.layers.data(name='label', shape=[1], dtype='int64') model = MobileNet() out = model.net(input=image, class_dim=10) cost = fluid.layers.cross_entropy(input=out, label=label) avg_cost = fluid.layers.mean(x=cost) acc_top1 = fluid.layers.accuracy(input=out, label=label, k=1) acc_top5 = fluid.layers.accuracy(input=out, label=label, k=5) optimizer = fluid.optimizer.Momentum( momentum=0.9, learning_rate=0.01, regularization=fluid.regularizer.L2Decay(4e-5)) optimizer.minimize(avg_cost) main_prog = fluid.default_main_program() val_prog = main_prog.clone(for_test=True) place = fluid.CUDAPlace( 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) feeder = fluid.DataFeeder([image, label], place, program=main_prog) train_reader = paddle.fluid.io.batch(paddle.dataset.mnist.train(), batch_size=64) eval_reader = paddle.fluid.io.batch(paddle.dataset.mnist.test(), batch_size=64) def train(program): iter = 0 for data in train_reader(): cost, top1, top5 = exe.run( program, feed=feeder.feed(data), fetch_list=[avg_cost, acc_top1, acc_top5]) iter += 1 if iter % 100 == 0: print( 'train iter={}, avg loss {}, acc_top1 {}, acc_top5 {}'. format(iter, cost, top1, top5)) def test(program, outputs=[avg_cost, acc_top1, acc_top5]): iter = 0 result = [[], [], []] for data in train_reader(): cost, top1, top5 = exe.run(program, feed=feeder.feed(data), fetch_list=outputs) iter += 1 if iter % 100 == 0: print( 'eval iter={}, avg loss {}, acc_top1 {}, acc_top5 {}'. format(iter, cost, top1, top5)) result[0].append(cost) result[1].append(top1) result[2].append(top5) print(' avg loss {}, acc_top1 {}, acc_top5 {}'.format( np.mean(result[0]), np.mean(result[1]), np.mean(result[2]))) return np.mean(result[1]), np.mean(result[2]) train(main_prog) top1_1, top5_1 = test(val_prog) fluid.io.save_inference_model( dirname='./test_quant_post', feeded_var_names=[image.name, label.name], target_vars=[avg_cost, acc_top1, acc_top5], main_program=val_prog, executor=exe, model_filename='model', params_filename='params') quant_post_static(exe, './test_quant_post', './test_quant_post_inference', sample_generator=paddle.dataset.mnist.test(), model_filename='model', params_filename='params', batch_nums=10) quant_post_prog, feed_target_names, fetch_targets = fluid.io.load_inference_model( dirname='./test_quant_post_inference', executor=exe, model_filename='__model__', params_filename='__params__') top1_2, top5_2 = test(quant_post_prog, fetch_targets) print("before quantization: top1: {}, top5: {}".format(top1_1, top5_1)) print("after quantization: top1: {}, top5: {}".format(top1_2, top5_2))
ResNet_ImageNet(depth=18, num_classes=1000)) elif args.model == 'resnet34': assert args.dataset == 'imagenet', 'resnet34 only supports imagenet dataset' net = ResNet_ImageNet(num_classes=1000, cfg=None, depth=34) weight_path = 'Exp_base/resnet34_base/checkpoint/model_best.pth.tar' net_origin = nn.DataParallel( ResNet_ImageNet(depth=34, num_classes=1000)) elif args.model == 'resnet50': assert args.dataset == 'imagenet', 'resnet50 only supports imagenet dataset' net = ResNet_ImageNet(num_classes=1000, cfg=None, depth=50) weight_path = 'Exp_base/resnet50_base/checkpoint/model_best.pth.tar' net_origin = nn.DataParallel( ResNet_ImageNet(depth=50, num_classes=1000)) elif args.model == 'mobilenet': assert args.dataset == 'imagenet', 'mobilenet only supports imagenet dataset' net = MobileNet(num_classes=1000, cfg=None) weight_path = 'Exp_base/mobilenet_base/checkpoint/model_best.pth.tar' net_origin = nn.DataParallel(MobileNet(num_classes=1000)) elif args.model == 'mobilenetv2': assert args.dataset == 'imagenet', 'mobilenetv2 only supports imagenet dataset' net = MobileNetV2(num_classes=1000, cfg=None) weight_path = 'Exp_base/mobilenetv2_base/checkpoint/model_best.pth.tar' net_origin = nn.DataParallel(MobileNetV2(num_classes=1000)) elif args.model == 'vgg': assert args.dataset == 'cifar10', 'vgg only supports cifar10 dataset' net = VGG_CIFAR(cfg=None, cutout=False) weight_path = 'Exp_base/vgg_base/checkpoint/model_best.pth.tar' net_origin = nn.DataParallel(VGG_CIFAR(cutout=False)) elif args.model == "resnet56": assert args.dataset == 'cifar10', 'resnet56 only supports cifar10 dataset' net = ResNet_CIFAR(cfg=None, depth=56, num_classes=10, cutout=False)
net_origin = nn.DataParallel(ResNet_ImageNet(depth=34, num_classes=run_config.data_provider.n_classes)) elif args.model=="resnet50": assert args.dataset=='imagenet', 'resnet50 only supports imagenet dataset' net = ResNet_ImageNet(depth=50, num_classes=run_config.data_provider.n_classes, cfg=eval(args.cfg)) if args.base_path!=None: weight_path = args.base_path+'/checkpoint/model_best.pth.tar' net_origin = nn.DataParallel(ResNet_ImageNet(depth=50, num_classes=run_config.data_provider.n_classes)) elif args.model=="mobilenetv2": assert args.dataset=='imagenet', 'mobilenetv2 only supports imagenet dataset' net = MobileNetV2(num_classes=run_config.data_provider.n_classes, cfg=eval(args.cfg)) if args.base_path!=None: weight_path = args.base_path+'/checkpoint/model_best.pth.tar' net_origin = nn.DataParallel(MobileNetV2(num_classes=run_config.data_provider.n_classes)) elif args.model=="mobilenet": assert args.dataset=='imagenet', 'mobilenet only supports imagenet dataset' net = MobileNet(num_classes=run_config.data_provider.n_classes, cfg=eval(args.cfg)) if args.base_path!=None: weight_path = args.base_path+'/checkpoint/model_best.pth.tar' net_origin = nn.DataParallel(MobileNet(num_classes=run_config.data_provider.n_classes)) # build run manager run_manager = RunManager(args.path, net, run_config) if args.local_rank == 0: run_manager.save_config(print_info=True) # load checkpoints if args.base_path!=None: weight_path = args.base_path+'/checkpoint/model_best.pth.tar' if args.resume: run_manager.load_model() if args.train and run_manager.best_acc == 0:
num_workers=opt.n_workers, shuffle=True) test_data_loader = DataLoader(test_dataset, batch_size=opt.batch_size, num_workers=opt.n_workers, shuffle=False) backbone_network = opt.backbone_network n_layers = opt.n_layers if backbone_network == 'MobileNet': from models import MobileNet model = MobileNet(width_multiplier=opt.width_multiplier, attention=opt.attention_module, dataset=opt.dataset) if backbone_network == 'ResNet': from models import ResidualNetwork model = ResidualNetwork(n_layers=n_layers, dataset=opt.dataset, attention=opt.attention_module, group_size=opt.group_size) elif backbone_network == 'ResNext': from models import ResNext model = ResNext(n_layers=n_layers, n_groups=opt.n_groups,
def test_accuracy(self): image = paddle.static.data(name='image', shape=[None, 1, 28, 28], dtype='float32') label = paddle.static.data(name='label', shape=[None, 1], dtype='int64') model = MobileNet() out = model.net(input=image, class_dim=10) cost = paddle.nn.functional.loss.cross_entropy(input=out, label=label) avg_cost = paddle.mean(x=cost) acc_top1 = paddle.metric.accuracy(input=out, label=label, k=1) acc_top5 = paddle.metric.accuracy(input=out, label=label, k=5) optimizer = paddle.optimizer.Momentum( momentum=0.9, learning_rate=0.01, weight_decay=paddle.regularizer.L2Decay(4e-5)) optimizer.minimize(avg_cost) main_prog = paddle.static.default_main_program() val_prog = main_prog.clone(for_test=True) place = paddle.CUDAPlace( 0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() exe = paddle.static.Executor(place) exe.run(paddle.static.default_startup_program()) def transform(x): return np.reshape(x, [1, 28, 28]) train_dataset = paddle.vision.datasets.MNIST(mode='train', backend='cv2', transform=transform) test_dataset = paddle.vision.datasets.MNIST(mode='test', backend='cv2', transform=transform) train_loader = paddle.io.DataLoader(train_dataset, places=place, feed_list=[image, label], drop_last=True, batch_size=64, return_list=False) valid_loader = paddle.io.DataLoader(test_dataset, places=place, feed_list=[image, label], batch_size=64, return_list=False) def sample_generator_creator(): def __reader__(): for data in test_dataset: image, label = data yield image, label return __reader__ def train(program): iter = 0 for data in train_loader(): cost, top1, top5 = exe.run( program, feed=data, fetch_list=[avg_cost, acc_top1, acc_top5]) iter += 1 if iter % 100 == 0: print( 'train iter={}, avg loss {}, acc_top1 {}, acc_top5 {}'. format(iter, cost, top1, top5)) def test(program, outputs=[avg_cost, acc_top1, acc_top5]): iter = 0 result = [[], [], []] for data in valid_loader(): cost, top1, top5 = exe.run(program, feed=data, fetch_list=outputs) iter += 1 if iter % 100 == 0: print( 'eval iter={}, avg loss {}, acc_top1 {}, acc_top5 {}'. format(iter, cost, top1, top5)) result[0].append(cost) result[1].append(top1) result[2].append(top5) print(' avg loss {}, acc_top1 {}, acc_top5 {}'.format( np.mean(result[0]), np.mean(result[1]), np.mean(result[2]))) return np.mean(result[1]), np.mean(result[2]) train(main_prog) top1_1, top5_1 = test(val_prog) paddle.fluid.io.save_inference_model( dirname='./test_quant_post', feeded_var_names=[image.name, label.name], target_vars=[avg_cost, acc_top1, acc_top5], main_program=val_prog, executor=exe, model_filename='model', params_filename='params') quant_post_static(exe, './test_quant_post', './test_quant_post_inference', sample_generator=sample_generator_creator(), model_filename='model', params_filename='params', batch_nums=10) quant_post_prog, feed_target_names, fetch_targets = paddle.fluid.io.load_inference_model( dirname='./test_quant_post_inference', executor=exe, model_filename='__model__', params_filename='__params__') top1_2, top5_2 = test(quant_post_prog, fetch_targets) print("before quantization: top1: {}, top5: {}".format(top1_1, top5_1)) print("after quantization: top1: {}, top5: {}".format(top1_2, top5_2))
def train_net(args): torch.manual_seed(7) np.random.seed(7) checkpoint = args.checkpoint start_epoch = 0 best_acc = 0 writer = SummaryWriter() epochs_since_improvement = 0 # Initialize / load checkpoint if checkpoint is None: if args.network == 'r18': model = resnet18(args) elif args.network == 'r34': model = resnet34(args) elif args.network == 'r50': model = resnet50(args) elif args.network == 'r101': model = resnet101(args) elif args.network == 'r152': model = resnet152(args) elif args.network == 'mobile': model = MobileNet(1.0) elif args.network == 'mr18': print("mr18") model = myResnet18() else: model = resnet_face18(args.use_se) model = nn.DataParallel(model) metric_fc = ArcMarginModel(args) metric_fc = nn.DataParallel(metric_fc) if args.optimizer == 'sgd': optimizer = torch.optim.SGD([{ 'params': model.parameters() }, { 'params': metric_fc.parameters() }], lr=args.lr, momentum=args.mom, weight_decay=args.weight_decay) else: optimizer = torch.optim.Adam([{ 'params': model.parameters() }, { 'params': metric_fc.parameters() }], lr=args.lr, weight_decay=args.weight_decay) else: checkpoint = torch.load(checkpoint) start_epoch = checkpoint['epoch'] + 1 epochs_since_improvement = checkpoint['epochs_since_improvement'] model = checkpoint['model'] metric_fc = checkpoint['metric_fc'] optimizer = checkpoint['optimizer'] logger = get_logger() # Move to GPU, if available model = model.to(device) metric_fc = metric_fc.to(device) # Loss function if args.focal_loss: criterion = FocalLoss(gamma=args.gamma).to(device) else: criterion = nn.CrossEntropyLoss().to(device) # Custom dataloaders train_dataset = ArcFaceDataset('train') train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True) scheduler = StepLR(optimizer, step_size=args.lr_step, gamma=0.1) # Epochs for epoch in range(start_epoch, args.end_epoch): scheduler.step() if args.full_log: lfw_acc, threshold = lfw_test(model) writer.add_scalar('LFW_Accuracy', lfw_acc, epoch) full_log(epoch) start = datetime.now() # One epoch's training train_loss, train_top5_accs = train(train_loader=train_loader, model=model, metric_fc=metric_fc, criterion=criterion, optimizer=optimizer, epoch=epoch, logger=logger, writer=writer) writer.add_scalar('Train_Loss', train_loss, epoch) writer.add_scalar('Train_Top5_Accuracy', train_top5_accs, epoch) end = datetime.now() delta = end - start print('{} seconds'.format(delta.seconds)) # One epoch's validation lfw_acc, threshold = lfw_test(model) writer.add_scalar('LFW Accuracy', lfw_acc, epoch) # Check if there was an improvement is_best = lfw_acc > best_acc best_acc = max(lfw_acc, best_acc) if not is_best: epochs_since_improvement += 1 print("\nEpochs since last improvement: %d\n" % (epochs_since_improvement, )) else: epochs_since_improvement = 0 # Save checkpoint save_checkpoint(epoch, epochs_since_improvement, model, metric_fc, optimizer, best_acc, is_best)
def test_accuracy(self): image = paddle.static.data(name='image', shape=[None, 1, 28, 28], dtype='float32') label = paddle.static.data(name='label', shape=[None, 1], dtype='int64') model = MobileNet() out = model.net(input=image, class_dim=10) cost = paddle.nn.functional.loss.cross_entropy(input=out, label=label) avg_cost = paddle.mean(x=cost) acc_top1 = paddle.metric.accuracy(input=out, label=label, k=1) acc_top5 = paddle.metric.accuracy(input=out, label=label, k=5) optimizer = paddle.optimizer.Momentum( momentum=0.9, learning_rate=0.01, weight_decay=paddle.regularizer.L2Decay(4e-5)) optimizer.minimize(avg_cost) main_prog = paddle.static.default_main_program() val_prog = main_prog.clone(for_test=True) place = paddle.CUDAPlace(0) if paddle.fluid.is_compiled_with_cuda( ) else paddle.static.CPUPlace() exe = paddle.static.Executor(place) exe.run(paddle.static.default_startup_program()) train_loader = paddle.io.DataLoader.from_generator( feed_list=[image, label], capacity=512, use_double_buffer=True, iterable=True) valid_loader = paddle.io.DataLoader.from_generator( feed_list=[image, label], capacity=512, use_double_buffer=True, iterable=True) train_reader = paddle.batch(paddle.dataset.mnist.train(), batch_size=64) eval_reader = paddle.batch(paddle.dataset.mnist.test(), batch_size=64) train_loader.set_sample_list_generator(train_reader, place) valid_loader.set_sample_list_generator(eval_reader, place) def train(program): iter = 0 for data in train_loader(): cost, top1, top5 = exe.run( program, feed=data, fetch_list=[avg_cost, acc_top1, acc_top5]) iter += 1 if iter % 100 == 0: print( 'train iter={}, avg loss {}, acc_top1 {}, acc_top5 {}'. format(iter, cost, top1, top5)) def test(program): iter = 0 result = [[], [], []] for data in valid_loader(): cost, top1, top5 = exe.run( program, feed=data, fetch_list=[avg_cost, acc_top1, acc_top5]) iter += 1 if iter % 100 == 0: print( 'eval iter={}, avg loss {}, acc_top1 {}, acc_top5 {}'. format(iter, cost, top1, top5)) result[0].append(cost) result[1].append(top1) result[2].append(top5) print(' avg loss {}, acc_top1 {}, acc_top5 {}'.format( np.mean(result[0]), np.mean(result[1]), np.mean(result[2]))) return np.mean(result[1]), np.mean(result[2]) train(main_prog) top1_1, top5_1 = test(main_prog) config = { 'weight_quantize_type': 'channel_wise_abs_max', 'activation_quantize_type': 'moving_average_abs_max', 'quantize_op_types': ['depthwise_conv2d', 'mul', 'conv2d'], } quant_train_prog = quant_aware(main_prog, place, config, for_test=False) quant_eval_prog = quant_aware(val_prog, place, config, for_test=True) train(quant_train_prog) quant_eval_prog, int8_prog = convert(quant_eval_prog, place, config, save_int8=True) top1_2, top5_2 = test(quant_eval_prog) # values before quantization and after quantization should be close print("before quantization: top1: {}, top5: {}".format(top1_1, top5_1)) print("after quantization: top1: {}, top5: {}".format(top1_2, top5_2))
def main(): parser = argparse.ArgumentParser( description='Deep Learning SNN simulation') parser.add_argument('--config-file', default='config_1.ini') args = parser.parse_args() print(args) config = configparser.ConfigParser() config.read(args.config_file) pprint.pprint( {section: dict(config[section]) for section in config.sections()}) print defaults = config['DEFAULT'] device = defaults['device'] app_name = defaults['app_name'] print('[INFO] Simulating spiking {}'.format(app_name)) if not os.path.isdir(app_name): os.mkdir(app_name) out_dir = app_name org_model = config['original model'] " Load the original model " net = None if 'mobilenet_cifar10' in org_model['arch']: net = MobileNet() if org_model['arch'] == 'vgg_cifar10': net = VGG_16_cifar10 num_classes = org_model.getint('num_classes') model_path = org_model['model_path'] file_name = org_model['file_name'] state = None state, net = load_model(net, model_path, file_name) net = net.to(device) " Load the dataset " testloader, img_size = None, None data_config = config['dataset'] data_dir = data_config['data_dir'] if data_config['dataset'] == 'cifar10': _, testloader = load_cifar10(data_dir, org_model['arch']) img_size = [-1, 3, 32, 32] " Tasks to do " tasks = config['functions'] if tasks.getboolean('validate'): print(net) validate(net, testloader, device) new_net = None if tasks.getboolean('remove_bn'): if has_bn(net): new_net = MobileNet_nobn() new_net = merge_bn(net, new_net) new_net = new_net.to(device) print(new_net) print('Validating model after folding back BN layers...') validate(new_net, testloader, device) save_model(new_net, state, out_dir, 'nobn_' + file_name) else: print('model has no BN layers') if tasks.getboolean('use_nobn'): if 'mobilenet_cifar10' in org_model['arch']: net = MobileNet_nobn() state, net = load_model(net, out_dir, 'nobn_' + file_name) net = net.to(device) " quantize model " quant_config = config['quantization'] W = quant_config.getint('W') F = quant_config.getint('F') if tasks.getboolean('quantize'): net = quantize_model(net, W, F) net = net.to(device) print('Validating model after quantization...') state['acc'] = validate(net, testloader, device) save_model(net, state, out_dir, 'quant{}.{}_'.format(W, F) + file_name) " use quantized model " if tasks.getboolean('use_quant'): state, net = load_model(net, out_dir, 'quant{}.{}_'.format(W, F) + file_name) net = net.to(device) if tasks.getboolean('validate_nobn'): if not has_bn(net): print('Validating no_bn model...') validate(net, testloader) else: print('model has BN layers!! Exiting..') exit() " compute thresholds " spike_config = config['spiking'] percentile = spike_config.getfloat('percentile') if spike_config.getboolean('compute_thresholds'): compute_thresholds(net, testloader, out_dir, percentile, device) " convert ann to snn " spike_net, thresholds, max_acts, clamp_slope = None, None, None, None if spike_config.getboolean('convert_to_spike'): thresholds = np.loadtxt(os.path.join(out_dir, 'thresholds.txt')) max_acts = np.loadtxt(os.path.join(out_dir, 'max_acts.txt')) clamp_slope = spike_config.getfloat('clamp_slope') spike_net = createSpikingModel(net, org_model['arch'], num_classes, spike_config, thresholds, max_acts, device) print(spike_net) sanity_check(net, spike_net, max_acts) " simulate snn " if spike_config.getboolean('simulate_spiking'): thresholds = np.loadtxt(os.path.join(out_dir, 'thresholds.txt')) max_acts = np.loadtxt(os.path.join(out_dir, 'max_acts.txt')) simulate_spike_model(net, org_model['arch'], testloader, config, thresholds, max_acts, num_classes, img_size, device) " plot correlations " if spike_config.getboolean('plot_correlations'): corr = np.load(os.path.join(out_dir, 'layerwise_corr.npy')) plot_config = config['plotting'] #print('corr matrix shape: {}'.format(corr.shape)) plot_correlations(corr, out_dir, plot_config)
assert args.dataset == 'imagenet', 'resnet34 only supports imagenet dataset' net = ResNet_ImageNet(depth=34, num_classes=run_config.data_provider.n_classes, cfg=eval(args.cfg)) elif args.model == "resnet50": assert args.dataset == 'imagenet', 'resnet50 only supports imagenet dataset' net = ResNet_ImageNet(depth=50, num_classes=run_config.data_provider.n_classes, cfg=eval(args.cfg)) elif args.model == "mobilenetv2": assert args.dataset == 'imagenet', 'mobilenetv2 only supports imagenet dataset' net = MobileNetV2(num_classes=run_config.data_provider.n_classes, cfg=eval(args.cfg)) elif args.model == "mobilenet": assert args.dataset == 'imagenet', 'mobilenet only supports imagenet dataset' net = MobileNet(num_classes=run_config.data_provider.n_classes, cfg=eval(args.cfg)) # build run manager run_manager = RunManager(args.path, net, run_config) # load checkpoints best_model_path = '%s/checkpoint/model_best.pth.tar' % args.path assert os.path.isfile(best_model_path), 'wrong path' if torch.cuda.is_available(): checkpoint = torch.load(best_model_path) else: checkpoint = torch.load(best_model_path, map_location='cpu') if 'state_dict' in checkpoint: checkpoint = checkpoint['state_dict'] run_manager.net.load_state_dict(checkpoint)
def main(): global args, best_err1, best_err5 args = parser.parse_args() traindir = os.path.join(args.data_path, 'train') valdir = os.path.join(args.data_path, 'val') normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) train_dataset = datasets.ImageFolder(traindir, transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize]) ) train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True, sampler=None) val_dataset = datasets.ImageFolder(valdir, transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize]) ) val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True) if args.net_type == 'mobilenet': t_net = ResNet.resnet50(pretrained=True) s_net = Mov.MobileNet() elif args.net_type == 'resnet': t_net = ResNet.resnet152(pretrained=True) s_net = ResNet.resnet50(pretrained=False) else: print('undefined network type !!!') raise d_net = distiller.Distiller(t_net, s_net) print ('Teacher Net: ') print(t_net) print ('Student Net: ') print(s_net) print('the number of teacher model parameters: {}'.format(sum([p.data.nelement() for p in t_net.parameters()]))) print('the number of student model parameters: {}'.format(sum([p.data.nelement() for p in s_net.parameters()]))) t_net = torch.nn.DataParallel(t_net).cuda() s_net = torch.nn.DataParallel(s_net).cuda() d_net = torch.nn.DataParallel(d_net).cuda() # define loss function (criterion) and optimizer criterion_CE = nn.CrossEntropyLoss().cuda() optimizer = torch.optim.SGD(list(s_net.parameters()) + list(d_net.module.Connectors.parameters()), args.lr, momentum=args.momentum, weight_decay=args.weight_decay, nesterov=True) cudnn.benchmark = True print('Teacher network performance') validate(val_loader, t_net, criterion_CE, 0) for epoch in range(1, args.epochs+1): adjust_learning_rate(optimizer, epoch) # train for one epoch train_with_distill(train_loader, d_net, optimizer, criterion_CE, epoch) # evaluate on validation set err1, err5 = validate(val_loader, s_net, criterion_CE, epoch) # remember best prec@1 and save checkpoint is_best = err1 <= best_err1 best_err1 = min(err1, best_err1) if is_best: best_err5 = err5 print ('Current best accuracy (top-1 and 5 error):', best_err1, best_err5) save_checkpoint({ 'epoch': epoch, 'arch': args.net_type, 'state_dict': s_net.state_dict(), 'best_err1': best_err1, 'best_err5': best_err5, 'optimizer' : optimizer.state_dict(), }, is_best) gc.collect() print ('Best accuracy (top-1 and 5 error):', best_err1, best_err5)
from flask import Flask, render_template, request from models import MobileNet import os from math import floor app = Flask(__name__) app.config['UPLOAD_FOLDER'] = 'uploads' model = MobileNet() @app.route('/') def index(): return render_template('index.html') @app.route('/about') def about(): return render_template('about.html') @app.route('/infer', methods = ['POST']) def success(): if request.method == 'POST': f = request.files['file'] saveLocation = f.filename f.save(saveLocation) inference, confidence = model.infer(saveLocation) # make a percentage with 2 decimal points confidence = floor(confidence * 10000) / 100 # delete file after making an inference os.remove(saveLocation) # respond with the inference
def test_accuracy(self): image = fluid.layers.data(name='image', shape=[1, 28, 28], dtype='float32') image.stop_gradient = False label = fluid.layers.data(name='label', shape=[1], dtype='int64') model = MobileNet() out = model.net(input=image, class_dim=10) cost = fluid.layers.cross_entropy(input=out, label=label) avg_cost = fluid.layers.mean(x=cost) acc_top1 = fluid.layers.accuracy(input=out, label=label, k=1) acc_top5 = fluid.layers.accuracy(input=out, label=label, k=5) optimizer = fluid.optimizer.Momentum( momentum=0.9, learning_rate=0.01, regularization=fluid.regularizer.L2Decay(4e-5)) optimizer.minimize(avg_cost) main_prog = fluid.default_main_program() val_prog = main_prog.clone(for_test=True) place = fluid.CUDAPlace( 0) if fluid.is_compiled_with_cuda() else fluid.CPUPlace() exe = fluid.Executor(place) exe.run(fluid.default_startup_program()) feeder = fluid.DataFeeder([image, label], place, program=main_prog) train_reader = paddle.fluid.io.batch(paddle.dataset.mnist.train(), batch_size=64) eval_reader = paddle.fluid.io.batch(paddle.dataset.mnist.test(), batch_size=64) def train(program): iter = 0 for data in train_reader(): cost, top1, top5 = exe.run( program, feed=feeder.feed(data), fetch_list=[avg_cost, acc_top1, acc_top5]) iter += 1 if iter % 100 == 0: print( 'train iter={}, avg loss {}, acc_top1 {}, acc_top5 {}'. format(iter, cost, top1, top5)) def test(program): iter = 0 result = [[], [], []] for data in eval_reader(): cost, top1, top5 = exe.run( program, feed=feeder.feed(data), fetch_list=[avg_cost, acc_top1, acc_top5]) iter += 1 if iter % 100 == 0: print( 'eval iter={}, avg loss {}, acc_top1 {}, acc_top5 {}'. format(iter, cost, top1, top5)) result[0].append(cost) result[1].append(top1) result[2].append(top5) print(' avg loss {}, acc_top1 {}, acc_top5 {}'.format( np.mean(result[0]), np.mean(result[1]), np.mean(result[2]))) return np.mean(result[1]), np.mean(result[2]) train(main_prog) top1_1, top5_1 = test(main_prog) config = { 'weight_quantize_type': 'channel_wise_abs_max', 'activation_quantize_type': 'moving_average_abs_max', 'quantize_op_types': ['depthwise_conv2d', 'mul', 'conv2d'], } quant_train_prog_pact = quant_aware(main_prog, place, config, for_test=False, act_preprocess_func=pact, optimizer_func=get_optimizer, executor=exe) quant_eval_prog = quant_aware(val_prog, place, config, for_test=True) train(quant_train_prog_pact) quant_eval_prog, int8_prog = convert(quant_eval_prog, place, config, save_int8=True) top1_2, top5_2 = test(quant_eval_prog) # values before quantization and after quantization should be close print("before quantization: top1: {}, top5: {}".format(top1_1, top5_1)) print("after quantization: top1: {}, top5: {}".format(top1_2, top5_2))
def test_accuracy(self): image = paddle.static.data( name='image', shape=[None, 1, 28, 28], dtype='float32') image.stop_gradient = False label = paddle.static.data(name='label', shape=[None, 1], dtype='int64') model = MobileNet() out = model.net(input=image, class_dim=10) cost = paddle.nn.functional.loss.cross_entropy(input=out, label=label) avg_cost = paddle.mean(x=cost) acc_top1 = paddle.metric.accuracy(input=out, label=label, k=1) acc_top5 = paddle.metric.accuracy(input=out, label=label, k=5) optimizer = paddle.optimizer.Momentum( momentum=0.9, learning_rate=0.01, weight_decay=paddle.regularizer.L2Decay(4e-5)) optimizer.minimize(avg_cost) main_prog = paddle.static.default_main_program() val_prog = main_prog.clone(for_test=True) place = paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda( ) else paddle.CPUPlace() exe = paddle.static.Executor(place) exe.run(paddle.static.default_startup_program()) def transform(x): return np.reshape(x, [1, 28, 28]) train_dataset = paddle.vision.datasets.MNIST( mode='train', backend='cv2', transform=transform) test_dataset = paddle.vision.datasets.MNIST( mode='test', backend='cv2', transform=transform) train_loader = paddle.io.DataLoader( train_dataset, places=place, feed_list=[image, label], drop_last=True, return_list=False, batch_size=64) valid_loader = paddle.io.DataLoader( test_dataset, places=place, feed_list=[image, label], batch_size=64, return_list=False) def train(program): iter = 0 for data in train_loader(): cost, top1, top5 = exe.run( program, feed=data, fetch_list=[avg_cost, acc_top1, acc_top5]) iter += 1 if iter % 100 == 0: print( 'train iter={}, avg loss {}, acc_top1 {}, acc_top5 {}'. format(iter, cost, top1, top5)) def test(program): iter = 0 result = [[], [], []] for data in valid_loader(): cost, top1, top5 = exe.run( program, feed=data, fetch_list=[avg_cost, acc_top1, acc_top5]) iter += 1 if iter % 100 == 0: print('eval iter={}, avg loss {}, acc_top1 {}, acc_top5 {}'. format(iter, cost, top1, top5)) result[0].append(cost) result[1].append(top1) result[2].append(top5) print(' avg loss {}, acc_top1 {}, acc_top5 {}'.format( np.mean(result[0]), np.mean(result[1]), np.mean(result[2]))) return np.mean(result[1]), np.mean(result[2]) train(main_prog) top1_1, top5_1 = test(main_prog) config = { 'weight_quantize_type': 'channel_wise_abs_max', 'activation_quantize_type': 'moving_average_abs_max', 'quantize_op_types': ['depthwise_conv2d', 'mul', 'conv2d'], } quant_train_prog_pact = quant_aware( main_prog, place, config, for_test=False, act_preprocess_func=pact, optimizer_func=get_optimizer, executor=exe) quant_eval_prog = quant_aware(val_prog, place, config, for_test=True) train(quant_train_prog_pact) quant_eval_prog, int8_prog = convert( quant_eval_prog, place, config, save_int8=True) top1_2, top5_2 = test(quant_eval_prog) # values before quantization and after quantization should be close print("before quantization: top1: {}, top5: {}".format(top1_1, top5_1)) print("after quantization: top1: {}, top5: {}".format(top1_2, top5_2))
def train_net(args): torch.manual_seed(7) #torch的随机种子,在torch.randn使用 np.random.seed(7) checkpoint = args.checkpoint start_epoch = 0 best_acc = 0 writer = SummaryWriter() #tensorboard epochs_since_improvement = 0 # Initialize / load checkpoint if checkpoint is None: if args.network == 'r18': model = resnet18(args) elif args.network == 'r34': model = resnet34(args) elif args.network == 'r50': model = resnet50(args) elif args.network == 'r101': model = resnet101(args) elif args.network == 'r152': model = resnet152(args) elif args.network == 'mobile': model = MobileNet(1.0) else: model = resnet_face18(args.use_se) model = nn.DataParallel(model) metric_fc = ArcMarginModel(args) metric_fc = nn.DataParallel(metric_fc) if args.optimizer == 'sgd': # optimizer = torch.optim.SGD([{'params': model.parameters()}, {'params': metric_fc.parameters()}], # lr=args.lr, momentum=args.mom, weight_decay=args.weight_decay) optimizer = InsightFaceOptimizer( torch.optim.SGD([{ 'params': model.parameters() }, { 'params': metric_fc.parameters() }], lr=args.lr, momentum=args.mom, weight_decay=args.weight_decay)) else: optimizer = torch.optim.Adam([{ 'params': model.parameters() }, { 'params': metric_fc.parameters() }], lr=args.lr, weight_decay=args.weight_decay) else: checkpoint = torch.load(checkpoint) #这里还需要自己加载进去 start_epoch = checkpoint['epoch'] + 1 epochs_since_improvement = checkpoint['epochs_since_improvement'] model = checkpoint['model'] metric_fc = checkpoint['metric_fc'] optimizer = checkpoint['optimizer'] logger = get_logger() # Move to GPU, if available model = model.to(device) metric_fc = metric_fc.to(device) # Loss function if args.focal_loss: criterion = FocalLoss(gamma=args.gamma).to(device) else: criterion = nn.CrossEntropyLoss().to(device) # Custom dataloaders train_dataset = Dataset(root=args.train_path, phase='train', input_shape=(3, 112, 112)) train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=8) # Epochs for epoch in range(start_epoch, args.end_epoch): # One epoch's training # 这里写一个训练函数十分简练,值得学习 train_loss, train_top1_accs = train(train_loader=train_loader, model=model, metric_fc=metric_fc, criterion=criterion, optimizer=optimizer, epoch=epoch, logger=logger) print('\nCurrent effective learning rate: {}\n'.format(optimizer.lr)) print('Step num: {}\n'.format(optimizer.step_num)) writer.add_scalar('model/train_loss', train_loss, epoch) writer.add_scalar('model/train_accuracy', train_top1_accs, epoch) writer.add_scalar('model/learning_rate', optimizer.lr, epoch) # Save checkpoint if epoch % 10 == 0: save_checkpoint(epoch, epochs_since_improvement, model, metric_fc, optimizer, best_acc)
def test_accuracy(self): float_infer_model_path_prefix = "./mv1_float_inference" image = paddle.static.data(name='image', shape=[None, 1, 28, 28], dtype='float32') label = paddle.static.data(name='label', shape=[None, 1], dtype='int64') model = MobileNet() out = model.net(input=image, class_dim=10) cost = paddle.nn.functional.loss.cross_entropy(input=out, label=label) avg_cost = paddle.mean(x=cost) acc_top1 = paddle.metric.accuracy(input=out, label=label, k=1) acc_top5 = paddle.metric.accuracy(input=out, label=label, k=5) optimizer = paddle.optimizer.Momentum( momentum=0.9, learning_rate=0.01, weight_decay=paddle.regularizer.L2Decay(4e-5)) optimizer.minimize(avg_cost) main_prog = paddle.static.default_main_program() val_prog = main_prog.clone(for_test=True) #place = paddle.CPUPlace() place = paddle.CUDAPlace( 0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() exe = paddle.static.Executor(place) exe.run(paddle.static.default_startup_program()) def transform(x): return np.reshape(x, [1, 28, 28]) train_dataset = paddle.vision.datasets.MNIST(mode='train', backend='cv2', transform=transform) test_dataset = paddle.vision.datasets.MNIST(mode='test', backend='cv2', transform=transform) train_loader = paddle.io.DataLoader(train_dataset, places=place, feed_list=[image, label], drop_last=True, batch_size=64, return_list=False) valid_loader = paddle.io.DataLoader(test_dataset, places=place, feed_list=[image, label], batch_size=64, return_list=False) def sample_generator_creator(): def __reader__(): for data in test_dataset: image, label = data yield image, label return __reader__ def train(program): iter = 0 for data in train_loader(): cost, top1, top5 = exe.run( program, feed=data, fetch_list=[avg_cost, acc_top1, acc_top5]) iter += 1 if iter % 100 == 0: print( 'train iter={}, avg loss {}, acc_top1 {}, acc_top5 {}'. format(iter, cost, top1, top5)) def test(program, outputs=[avg_cost, acc_top1, acc_top5]): iter = 0 result = [[], [], []] for data in valid_loader(): cost, top1, top5 = exe.run(program, feed=data, fetch_list=outputs) iter += 1 if iter % 100 == 0: print( 'eval iter={}, avg loss {}, acc_top1 {}, acc_top5 {}'. format(iter, cost, top1, top5)) result[0].append(cost) result[1].append(top1) result[2].append(top5) print(' avg loss {}, acc_top1 {}, acc_top5 {}'.format( np.mean(result[0]), np.mean(result[1]), np.mean(result[2]))) return np.mean(result[1]), np.mean(result[2]) train(main_prog) top1_1, top5_1 = test(val_prog) paddle.static.save_inference_model( path_prefix=float_infer_model_path_prefix, feed_vars=[image, label], fetch_vars=[avg_cost, acc_top1, acc_top5], executor=exe, program=val_prog) quant_config = { 'weight_quantize_type': 'channel_wise_abs_max', 'activation_quantize_type': 'moving_average_abs_max', 'not_quant_pattern': ['skip_quant'], 'quantize_op_types': ['conv2d', 'depthwise_conv2d', 'mul'] } train_config = { "num_epoch": 1, # training epoch num "max_iter": 20, "save_iter_step": 10, "learning_rate": 0.0001, "weight_decay": 0.0001, "use_pact": False, "quant_model_ckpt_path": "./quantaware_with_infermodel_checkpoints/", "teacher_model_path_prefix": float_infer_model_path_prefix, "model_path_prefix": float_infer_model_path_prefix, "distill_node_pair": [ "teacher_fc_0.tmp_0", "fc_0.tmp_0", "teacher_batch_norm_24.tmp_4", "batch_norm_24.tmp_4", "teacher_batch_norm_22.tmp_4", "batch_norm_22.tmp_4", "teacher_batch_norm_18.tmp_4", "batch_norm_18.tmp_4", "teacher_batch_norm_13.tmp_4", "batch_norm_13.tmp_4", "teacher_batch_norm_5.tmp_4", "batch_norm_5.tmp_4" ] } def test_callback(compiled_test_program, feed_names, fetch_list, checkpoint_name): outputs = fetch_list iter = 0 result = [[], [], []] for data in valid_loader(): cost, top1, top5 = exe.run(compiled_test_program, feed=data, fetch_list=fetch_list) iter += 1 if iter % 100 == 0: print( 'eval iter={}, avg loss {}, acc_top1 {}, acc_top5 {}'. format(iter, cost, top1, top5)) result[0].append(cost) result[1].append(top1) result[2].append(top5) print("quant model checkpoint: " + checkpoint_name + ' avg loss {}, acc_top1 {}, acc_top5 {}'.format( np.mean(result[0]), np.mean(result[1]), np.mean( result[2]))) return np.mean(result[1]), np.mean(result[2]) def test_quant_aware_with_infermodel(exe, place): quant_aware_with_infermodel(exe, place, scope=None, train_reader=train_loader, quant_config=quant_config, train_config=train_config, test_callback=test_callback) def test_export_quant_infermodel(exe, place, checkpoint_path, quant_infermodel_save_path): export_quant_infermodel( exe, place, scope=None, quant_config=quant_config, train_config=train_config, checkpoint_path=checkpoint_path, export_inference_model_path_prefix=quant_infermodel_save_path) #place = paddle.CPUPlace() place = paddle.CUDAPlace( 0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() exe = paddle.static.Executor(place) test_quant_aware_with_infermodel(exe, place) checkpoint_path = "./quantaware_with_infermodel_checkpoints/epoch_0_iter_10" quant_infermodel_save_path = "./quantaware_with_infermodel_export" test_export_quant_infermodel(exe, place, checkpoint_path, quant_infermodel_save_path) train_config["use_pact"] = True test_quant_aware_with_infermodel(exe, place) train_config["use_pact"] = False checkpoint_path = "./quantaware_with_infermodel_checkpoints/epoch_0_iter_10" quant_infermodel_save_path = "./quantaware_with_infermodel_export" test_export_quant_infermodel(exe, place, checkpoint_path, quant_infermodel_save_path)
def __init__(self, architecture, # Only MobileNet for now input_size, # a tuple (height, width) labels, # class for classification max_box_per_image, anchors, debug, trainable=True): self.input_size = input_size self.debug = debug self.labels = list(labels) self.nb_class = len(self.labels) self.nb_box = 5 self.class_wt = np.ones(self.nb_class, dtype='float32') self.anchors = anchors self.n_cls = len(self.labels) self.max_box_per_image = max_box_per_image self.preprocess_input = preprocess_input ########################## # Make the model ########################## # make the feature extractor layers input_image = Input(shape=(self.input_size[1], self.input_size[0], 3)) self.true_boxes = Input(shape=(1, 1, 1, self.max_box_per_image, 4)) # self.feature_extractor = MobileNetFeature(self.input_size) if architecture == 'MobileNet': self.feature_extractor = MobileNet(input_shape=(self.input_size[1], self.input_size[0], 3), include_top=False, input_tensor=input_image, weights=None, trainable=trainable) last_layer = 'conv_pw_13_relu' elif architecture == 'Tiny Yolo': self.feature_extractor = DarkNet(416, input_tensor=input_image) last_layer = 'leaky_1' elif architecture == 'InceptionResNet': self.feature_extractor = InceptionResNetV2(include_top=False, input_shape=(self.input_size[1], self.input_size[0], 3), input_tensor=input_image, weights=None, trainable=trainable) last_layer = 'conv_7b_ac' else: raise ValueError('Architecture must be either MobileNet or InceptionResNet') if architecture == 'InceptionResNet': self.grid_h, self.grid_w = 11, 11 else: self.grid_h, self.grid_w = int(self.input_size[1] / 32), int(self.input_size[0] / 32) features = self.feature_extractor.get_layer(last_layer).output # make the object detection layer box_out = Conv2D(self.nb_box * (4 + 1 + self.nb_class), (1, 1), strides=(1, 1), padding='same', name='conv_output', kernel_initializer='lecun_normal')(features) box_out = Reshape((self.grid_h, self.grid_w, self.nb_box, 4 + 1 + self.nb_class))(box_out) box_out = Lambda(lambda args: args[0])([box_out, self.true_boxes]) class_out = GlobalAveragePooling2D()(features) class_out = Reshape((1, 1, 1024))(class_out) class_out = Conv2D(10, (1, 1), padding='same', name='conv_preds')(class_out) class_out = Activation('softmax')(class_out) class_out = Reshape((10,), name='classifier')(class_out) self.model = Model([input_image, self.true_boxes], [box_out, class_out]) # initialize the weights of the detection layer layer = self.model.layers[-6] weights = layer.get_weights() new_kernel = np.random.normal(size=weights[0].shape) / (self.grid_h * self.grid_w) new_bias = np.random.normal(size=weights[1].shape) / (self.grid_h * self.grid_w) layer.set_weights([new_kernel, new_bias]) # print a summary of the whole model self.model.summary()