def __init__(self, local_conv_out_channels=128, num_classes=None): super(Model, self).__init__() self.base = resnet50(pretrained=True) planes = 2048 self.local_conv = nn.Conv2d(planes, local_conv_out_channels, 1) self.local_bn = nn.BatchNorm2d(local_conv_out_channels) self.local_relu = nn.ReLU(inplace=True) if num_classes is not None: self.fc = nn.Linear(planes, num_classes) init.normal(self.fc.weight, std=0.001) init.constant(self.fc.bias, 0)
def model_modifier(m): m.layers[-1].activation = tf.keras.activations.linear imgs = [] labels = [] for index, label in index_label_map.items(): activation_maximization = ActivationMaximization(model, model_modifier) loss = lambda x: K.mean(x[:, index - 1]) activation = activation_maximization(loss, steps=itr, callbacks=[Print(interval=100)]) img = activation[0].astype(np.uint8) img = np.squeeze(img, 2) cv2.imwrite(label + ".png", img) imgs.append(img) labels.append(label) plot_images(imgs, labels, 400, layer_name) model = resnet50() model.compile(optimizer=keras.optimizers.Adam(), loss='categorical_crossentropy', metrics=['accuracy']) model.load_weights( 'logs\\HETERO RESNET DEFAULT 20200127-082347\\model_weights.h5') # visualize_conv_layer_filters(model, 'conv1') def visualize(model, itr): visualize_dense_layer(model, 'fc1000', index_emotrion, itr)
loss_func = self.mean(loss_func, (-1, )) return loss_func if __name__ == '__main__': if not args_opt.do_eval and args_opt.run_distribute: context.set_auto_parallel_context( device_num=args_opt.device_num, parallel_mode=ParallelMode.DATA_PARALLEL) context.set_auto_parallel_context( all_reduce_fusion_split_indices=[140]) init() context.set_context(mode=context.GRAPH_MODE) epoch_size = args_opt.epoch_size net = resnet50(args_opt.batch_size, args_opt.num_classes) loss = CrossEntropyLoss() opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), 0.01, 0.9) model = Model(net, loss_fn=loss, optimizer=opt, metrics={'acc'}) if args_opt.do_train: dataset = create_dataset(epoch_size) batch_num = dataset.get_dataset_size() config_ck = CheckpointConfig(save_checkpoint_steps=batch_num * 5, keep_checkpoint_max=10) ckpoint_cb = ModelCheckpoint(prefix="train_resnet_cifar10", directory="./", config=config_ck) loss_cb = LossMonitor()
img_dir = '/media/disk2/ljn/video_dataset/Mars/bbox_test/' test_dataset = dataset.videodataset(dataset_dir=img_dir, txt_path='list/list_test_seq_all.txt', new_height=256, new_width=128, frames=8, transform=transform) test_loader = torch.utils.data.DataLoader(dataset=test_dataset, batch_size=batch_size, shuffle=False, num_workers=8) ########### MODEL ########### pretrained = 'resnet50_3d_mars_00240.pth' model = resnet.resnet50(pretrained=pretrained, num_classes=625, train=False) model.cuda() model.eval() name = 'fea/fea' output = open(name, 'w') num = 0 for data in test_loader: num = num + batch_size images, label = data images = torch.transpose(images, 1, 2).contiguous() images = images.view( images.size(0) * images.size(1), images.size(2), images.size(3), images.size(4)) images = Variable(images).cuda()
def __init__(self, layers, norm_layer=None): super(Net, self).__init__() if norm_layer is None: norm_layer = nn.BatchNorm2d self._norm_layer = norm_layer resnet = None if args.model == 'resnet18': resnet = resnet18(False, True) print('use resnet18') elif args.model == 'resnet34': resnet = resnet34(False, True) print('use resnet34') elif args.model == 'resnet50': resnet = resnet50(False, True) print('use resnet50') elif args.model == 'resnet101': resnet = resnet101(False, True) print('use resnet101') resnet.to(device) criterion = nn.CrossEntropyLoss() optimizer = optim.SGD(resnet.parameters(), lr=lr, momentum=0.9) # train best = 999.99 best_state_dict = None best_optimizer = None print('Start training') pbar = tqdm(total=int(args.epoch * len(y_train) / batch_size), desc='training')
def __init__(self, model_path='resnet50.pth'): super(ResNet50, self).__init__() model = resnet50() model.load_state_dict(torch.load(model_path)) self.features = nn.Sequential(*list(model.children())[:-1])
def __init__(self, num_class, center_dim): super(Net, self).__init__() # self.backbone = resnet50(num_classes=2) self.ip1 = resnet50(num_classes=center_dim) self.ip2 = nn.Linear(center_dim, num_class, bias=False)
def resnet50(): """Load resnet50""" return resnet.resnet50(pretrained=True, progress=True)
def main_worker(gpu, ngpus_per_node, args): global best_acc1 args.gpu = gpu if args.gpu is not None: print("Use GPU: {} for training".format(args.gpu)) if args.distributed: if args.dist_url == "env://" and args.rank == -1: args.rank = int(os.environ["RANK"]) if args.multiprocessing_distributed: # For multiprocessing distributed training, rank needs to be the # global rank among all the processes args.rank = args.rank * ngpus_per_node + gpu dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, world_size=args.world_size, rank=args.rank) # create model # if args.pretrained: # print("=> using pre-trained model '{}'".format(args.arch)) # model = models.__dict__[args.arch](pretrained=True) # else: # print("=> creating model '{}'".format(args.arch)) # model = models.__dict__[args.arch]() # ---- or ---- model = resnet50(pretrained=True) if args.distributed: # For multiprocessing distributed, DistributedDataParallel constructor # should always set the single device scope, otherwise, # DistributedDataParallel will use all available devices. if args.gpu is not None: torch.cuda.set_device(args.gpu) model.cuda(args.gpu) # When using a single GPU per process and per # DistributedDataParallel, we need to divide the batch size # ourselves based on the total number of GPUs we have args.batch_size = int(args.batch_size / ngpus_per_node) args.workers = int( (args.workers + ngpus_per_node - 1) / ngpus_per_node) model = torch.nn.parallel.DistributedDataParallel( model, device_ids=[args.gpu]) else: model.cuda() # DistributedDataParallel will divide and allocate batch_size to all # available GPUs if device_ids are not set model = torch.nn.parallel.DistributedDataParallel(model) elif args.gpu is not None: torch.cuda.set_device(args.gpu) model = model.cuda(args.gpu) else: # DataParallel will divide and allocate batch_size to all available GPUs if args.arch.startswith('alexnet') or args.arch.startswith('vgg'): model.features = torch.nn.DataParallel(model.features) model.cuda() else: model = torch.nn.DataParallel(model).cuda() # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda(args.gpu) optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # optionally resume from a checkpoint if args.resume: if os.path.isfile(args.resume): print("=> loading checkpoint '{}'".format(args.resume)) if args.gpu is None: checkpoint = torch.load(args.resume) else: # Map model to be loaded to specified single gpu. loc = 'cuda:{}'.format(args.gpu) checkpoint = torch.load(args.resume, map_location=loc) args.start_epoch = checkpoint['epoch'] best_acc1 = checkpoint['best_acc1'] if args.gpu is not None: # best_acc1 may be from a checkpoint from a different GPU best_acc1 = best_acc1.to(args.gpu) model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) print("=> loaded checkpoint '{}' (epoch {})".format( args.resume, checkpoint['epoch'])) else: print("=> no checkpoint found at '{}'".format(args.resume)) cudnn.benchmark = True # Data loading code traindir = os.path.join(args.data, 'train') valdir = os.path.join(args.data, 'val') normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) train_dataset = datasets.ImageFolder( traindir, transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ])) if args.distributed: train_sampler = torch.utils.data.distributed.DistributedSampler( train_dataset) else: train_sampler = None train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), num_workers=args.workers, pin_memory=True, sampler=train_sampler) val_loader = torch.utils.data.DataLoader(datasets.ImageFolder( valdir, transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])), batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True) if args.evaluate: validate(val_loader, model, criterion, args) return for epoch in range(args.start_epoch, args.epochs): if args.distributed: train_sampler.set_epoch(epoch) adjust_learning_rate(optimizer, epoch, args) # train for one epoch train(train_loader, model, criterion, optimizer, epoch, args) # evaluate on validation set acc1 = validate(val_loader, model, criterion, args) # remember best acc@1 and save checkpoint is_best = acc1 > best_acc1 best_acc1 = max(acc1, best_acc1) if not args.multiprocessing_distributed or ( args.multiprocessing_distributed and args.rank % ngpus_per_node == 0): save_checkpoint( { 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'best_acc1': best_acc1, 'optimizer': optimizer.state_dict(), }, is_best)
transforms.RandomCrop(img_size), transforms.ToTensor(), normalize ]) print('==> Preparing data..') trainset = dataloader(train=True, transform=transform_train) trainloader = torch.utils.data.DataLoader(trainset, batch_size=1, num_workers=50, shuffle=True) model = args.model # Use args.model as pretrain model if model == 'resnet152': net = resnet.resnet152().to(device) elif model == 'resnet50': net = resnet.resnet50().to(device) else: sys.exit(-1) criterion = nn.CrossEntropyLoss() optimizer = optim.SGD( net.parameters(), lr=0.1, momentum=0.9, weight_decay=5e-4) # 优化方式为mini-batch momentum-SGD,并采用L2正则化(权重衰减) with open("acc.txt", "w") as f: with open("log.txt", "w") as f2: for epoch in range(start_epoch, n_epochs): print('\nEpoch: %d' % (epoch + 1)) net.train() sum_loss = 0.0 correct = 0.0 total = 0.0
def main(): model = resnet50(pretrained=True) fake_img = np.random.uniform(-1, 1, [2, 3, 224, 224]).astype("float32") fake_img = torch.tensor(fake_img) print(model(fake_img).shape)
default_clear_line=False, root_dir=os.path.normpath( os.path.dirname(os.path.realpath(__file__)))) mllogger = mllog.get_mllogger() # submission mllogger.event(key=mllog.constants.SUBMISSION_BENCHMARK, value="resnet") mllogger.event(key=mllog.constants.SUBMISSION_DIVISION, value="closed") mllogger.event(key=mllog.constants.SUBMISSION_ORG, value="SIAT") mllogger.event(key=mllog.constants.SUBMISSION_PLATFORM, value="Ascend 910") mllogger.event(key=mllog.constants.SUBMISSION_STATUS, value="cloud") mllogger.event(key=mllog.constants.CACHE_CLEAR) # init the distribute env init() # network net = resnet50(class_num=args.class_num) # evaluation network dist_eval_network = ClassifyCorrectCell(net) # loss if not args.use_label_smooth: args.label_smooth_factor = 0.0 loss = nn.SoftmaxCrossEntropyWithLogits( sparse=True, reduction="mean", smooth_factor=args.label_smooth_factor, num_classes=args.class_num) # train dataset epoch_size = args.max_epoch
res = [] for k in topk: correct_k = correct[:k].view(-1).float().sum(0, keepdim=True) res.append(correct_k.mul_(100.0 / batch_size)) return res def compute_softmax(i): i = i - i.min(axis=1, keepdims=True) log_sum = logsumexp(i, axis=1, keepdims=True) return i - log_sum NAME_TO_MODEL = { 'resnet50': resnet50(num_classes=100), 'ayangnet': AyangNet(), 'densenet': models.densenet161(num_classes=100), 'inceptionv3': inception_v3(num_classes=100), 'inceptionv4': inception_v4(num_classes=100), 'wideresnet': WideResNet(28, 100, widen_factor=10), 'widedensenet': DenseNet(60, (6, 6, 6, 6), 64, num_classes=100) } if __name__ == '__main__': default_path = './preprocess' noise_decay = 0.55 loss_fn = CrossEntropyLoss() # set up argument parser parser = argparse.ArgumentParser()
print("The batch tensor is:",images.shape) plt.show() # In[4]: get_ipython().system('wget -N https://obs.dualstack.cn-north-4.myhuaweicloud.com/mindspore-website/notebook/source-codes/resnet.py') # In[5]: from resnet import resnet50 net = resnet50(batch_size=32, num_classes=10) # In[6]: import mindspore.nn as nn from mindspore.nn import SoftmaxCrossEntropyWithLogits ls = SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean") opt = nn.Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), 0.01, 0.9) # In[7]:
def main(): os.system('shutdown -c') # cancel previous shutdown command log.console(args) tb.log('sizes/world', dist_utils.env_world_size()) # need to index validation directory before we start counting the time dataloader.sort_ar(args.data + '/validation') if args.distributed: log.console('Distributed initializing process group') torch.cuda.set_device(args.local_rank) dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, world_size=dist_utils.env_world_size()) assert (dist_utils.env_world_size() == dist.get_world_size()) log.console("Distributed: success (%d/%d)" % (args.local_rank, dist.get_world_size())) log.console("Loading model") model = resnet.resnet50(bn0=args.init_bn0).cuda() #if args.fp16: model = network_to_half(model) if args.distributed: model = dist_utils.DDP(model, device_ids=[args.local_rank], output_device=args.local_rank) best_top5 = 80 # only save models over 80%. Otherwise it stops to save every time global model_params, master_params #if args.fp16: model_params, master_params = prep_param_lists(model) #else: model_params = master_params = model.parameters() model_params = master_params = model.parameters() bparams, oparams = [], [] for name, param in model.named_parameters(): if 'bias' in name: bparams.append(param) else: oparams.append(param) optim_params = [{ 'params': bparams, 'weight_decay': 0. }, { 'params': oparams, 'weight_decay': args.weight_decay }] #model_params = master_params = model.parameters() #optim_params = experimental_utils.bnwd_optim_params(model, model_params, master_params) if args.no_bn_wd else master_params # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda() optimizer = torch.optim.SGD( optim_params, 0, momentum=args.momentum, weight_decay=args.weight_decay ) # start with 0 lr. Scheduler will change this later if args.resume: checkpoint = torch.load( args.resume, map_location=lambda storage, loc: storage.cuda(args.local_rank)) model.load_state_dict(checkpoint['state_dict']) args.start_epoch = checkpoint['epoch'] best_top5 = checkpoint['best_top5'] optimizer.load_state_dict(checkpoint['optimizer']) # save script so we can reproduce from logs shutil.copy2(os.path.realpath(__file__), f'{args.logdir}') log.console( "Creating data loaders (this could take up to 10 minutes if volume needs to be warmed up)" ) phases = eval(args.phases) dm = DataManager([copy.deepcopy(p) for p in phases if 'bs' in p]) scheduler = Scheduler(optimizer, [copy.deepcopy(p) for p in phases if 'lr' in p]) start_time = datetime.now() # Loading start to after everything is loaded if args.evaluate: return validate(dm.val_dl, model, criterion, 0, start_time) if args.distributed: log.console('Syncing machines before training') dist_utils.sum_tensor(torch.tensor([1.0]).float().cuda()) log.event("~~epoch\thours\ttop1\ttop5\n") for epoch in range(args.start_epoch, scheduler.tot_epochs): dm.set_epoch(epoch) train(dm.trn_dl, model, criterion, optimizer, scheduler, epoch) top1, top5 = validate(dm.val_dl, model, criterion, epoch, start_time) time_diff = (datetime.now() - start_time).total_seconds() / 3600.0 log.event(f'~~{epoch}\t{time_diff:.5f}\t\t{top1:.3f}\t\t{top5:.3f}\n') is_best = top5 > best_top5 best_top5 = max(top5, best_top5) if args.local_rank == 0: if is_best: save_checkpoint(epoch, model, best_top5, optimizer, is_best=True, filename='model_best.pth.tar') phase = dm.get_phase(epoch) if phase: save_checkpoint( epoch, model, best_top5, optimizer, filename=f'sz{phase["bs"]}_checkpoint.path.tar')
def resnet50_train(args_opt): epoch_size = args_opt.epoch_size batch_size = 32 class_num = 10 loss_scale_num = 1024 local_data_path = '/cache/data' # set graph mode and parallel mode context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=False) context.set_context(device_id=device_id) if device_num > 1: context.set_auto_parallel_context( device_num=device_num, parallel_mode=ParallelMode.DATA_PARALLEL, gradients_mean=True) init() local_data_path = os.path.join(local_data_path, str(device_id)) # data download print('Download data.') mox.file.copy_parallel(src_url=args_opt.data_url, dst_url=local_data_path) # create dataset print('Create train and evaluate dataset.') train_dataset = create_dataset(dataset_path=local_data_path, do_train=True, repeat_num=1, batch_size=batch_size) eval_dataset = create_dataset(dataset_path=local_data_path, do_train=False, repeat_num=1, batch_size=batch_size) train_step_size = train_dataset.get_dataset_size() print('Create dataset success.') # create model net = resnet50(class_num=class_num) # reduction='mean' means that apply reduction of mean to loss loss = SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean') lr = Tensor( get_lr(global_step=0, total_epochs=epoch_size, steps_per_epoch=train_step_size)) opt = Momentum(net.trainable_params(), lr, momentum=0.9, weight_decay=1e-4, loss_scale=loss_scale_num) loss_scale = FixedLossScaleManager(loss_scale_num, False) # amp_level="O2" means that the hybrid precision of O2 mode is used for training # the whole network except that batchnoram will be cast into float16 format and dynamic loss scale will be used # 'keep_batchnorm_fp32 = False' means that use the float16 format model = Model(net, amp_level="O2", keep_batchnorm_fp32=False, loss_fn=loss, optimizer=opt, loss_scale_manager=loss_scale, metrics={'acc'}) # define performance callback to show ips and loss callback to show loss for every epoch performance_cb = PerformanceCallback(batch_size) loss_cb = LossMonitor() cb = [performance_cb, loss_cb] print(f'Start run training, total epoch: {epoch_size}.') model.train(epoch_size, train_dataset, callbacks=cb) if device_num == 1 or device_id == 0: print(f'Start run evaluation.') output = model.eval(eval_dataset) print(f'Evaluation result: {output}.')
def resnet50_train(args_opt): epoch_size = args_opt.epoch_size batch_size = cfg.batch_size class_num = cfg.class_num loss_scale_num = cfg.loss_scale local_data_path = '/cache/data' local_ckpt_path = '/cache/ckpt_file' # set graph mode and parallel mode context.set_context(mode=context.GRAPH_MODE, device_target="Ascend", save_graphs=False) # data download print('Download data.') mox.file.copy_parallel(src_url=args_opt.data_url, dst_url=local_data_path) # create dataset print('Create train and evaluate dataset.') train_dataset = create_dataset(dataset_path=local_data_path, do_train=True, repeat_num=epoch_size, batch_size=batch_size) train_step_size = train_dataset.get_dataset_size() print('Create dataset success.') # create model net = resnet50(class_num=class_num) # reduction='mean' means that apply reduction of mean to loss loss = SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean') lr = Tensor( get_lr(global_step=0, total_epochs=epoch_size, steps_per_epoch=train_step_size)) opt = Momentum(net.trainable_params(), lr, momentum=0.9, weight_decay=1e-4, loss_scale=loss_scale_num) loss_scale = FixedLossScaleManager(loss_scale_num, False) # amp_level="O2" means that the hybrid precision of O2 mode is used for training # the whole network except that batchnorm will be cast into float16 format and dynamic loss scale will be used # 'keep_batchnorm_fp32 = False' means that use the float16 format model = Model(net, amp_level="O2", keep_batchnorm_fp32=False, loss_fn=loss, optimizer=opt, loss_scale_manager=loss_scale, metrics={'acc'}) # define performance callback to show ips and loss callback to show loss for every epoch time_cb = TimeMonitor(data_size=train_step_size) performance_cb = PerformanceCallback(batch_size) loss_cb = LossMonitor() cb = [time_cb, performance_cb, loss_cb] config_ck = CheckpointConfig( save_checkpoint_steps=cfg.save_checkpoint_epochs * train_step_size, keep_checkpoint_max=cfg.keep_checkpoint_max) ckpt_cb = ModelCheckpoint(prefix="resnet", directory=local_ckpt_path, config=config_ck) cb += [ckpt_cb] print(f'Start run training, total epoch: {epoch_size}.') model.train(epoch_size, train_dataset, callbacks=cb) # upload checkpoint files print('Upload checkpoint.') mox.file.copy_parallel(src_url=local_ckpt_path, dst_url=args_opt.train_url)
def __init__(self, backbone='resnet', layers=50, bins=(1, 2, 3, 6), dropout=0.1, classes=2, zoom_factor=8, use_ppm=False, use_softmax=True, use_aux=False, pretrained=True, syncbn=True, group_size=8, group=None): super(PSPNet, self).__init__() assert layers in [50, 101, 152] assert 2048 % len(bins) == 0 assert classes > 1 assert zoom_factor in [1, 2, 4, 8] self.zoom_factor = zoom_factor self.use_ppm = use_ppm self.use_softmax = use_softmax self.use_aux = use_aux if backbone == 'resnet': import resnet as models elif backbone == 'ibnnet_a': import ibnnet_a as models elif backbone == 'ibnnet_b': import ibnnet_b as models else: raise NameError('Backbone type not defined!') if syncbn: from torchE.nn import SyncBatchNorm2d # from lib.syncbn import SynchronizedBatchNorm2d as BatchNorm def BNFunc(*args, **kwargs): return SyncBatchNorm2d(*args, **kwargs, group_size=group_size, group=group, sync_stats=True) BatchNorm = BNFunc else: from torch.nn import BatchNorm2d as BatchNorm models.BatchNorm = BatchNorm if layers == 50: resnet = models.resnet50(pretrained=pretrained) elif layers == 101: resnet = models.resnet101(pretrained=pretrained) else: resnet = models.resnet152(pretrained=pretrained) if backbone == 'ibnnet_b': self.layer0 = nn.Sequential(resnet.conv1, resnet.INCat0, resnet.relu, resnet.maxpool) else: self.layer0 = nn.Sequential(resnet.conv1, resnet.bn1, resnet.relu, resnet.maxpool) self.layer1, self.layer2, self.layer3, self.layer4 = resnet.layer1, resnet.layer2, resnet.layer3, resnet.layer4 for n, m in self.layer4.named_modules(): if 'conv2' in n and not 'convbnin.conv2' in n: m.dilation, m.padding, m.stride = (2, 2), (2, 2), (1, 1) elif 'downsample.0' in n: m.stride = (1, 1) fea_dim = 2048 if use_ppm: self.ppm = PPM(fea_dim, int(fea_dim / len(bins)), bins, BatchNorm) fea_dim *= 2 self.cls = nn.Sequential( nn.AdaptiveAvgPool2d(1), nn.Conv2d(fea_dim, 20, kernel_size=1, bias=False), )
# 打印所使用的设备 print(f"use device is : {args.device_target}") #dataset_sink_mode = not args.device_target == "CPU" # 自动并行运算 if args.run_distribute: context.set_auto_parallel_context( device_num=args.device_num, parallel_mode=ParallelMode.DATA_PARALLEL) auto_parallel_context().set_all_reduce_fusion_split_indices([140]) init() # 定义训练过程中的一些参数 epoch_size = args.epoch_size # 反向传播计算迭代次数 net = resnet50(args.batch_size, args.num_classes) # 创建ResNet网络对象 net_loss = SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean') # 定义损失行数 opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), 0.01, 0.9) # 定义训练过程中的优化器 # 创建网络模型。metrics={"acc", "loss"}表示评估该网络模型的时候,评估准确率、损失值。 model = Model(net, loss_fn=net_loss, optimizer=opt, metrics={'acc'}) #sink_mode = not args.device_target == "CPU" # as for train, users could use model.train if args.do_train: train_dataset = create_dataset(dog_dataset_path, os.path.join(dog_dataset_path, "training.csv"), batch_size=args.batch_size,
def __init__(self, in_feats, num_classes): super(GCN_fast, self).__init__() self.resnet = resnet50(pretrained=True) self.resnet.avgpool = nn.AdaptiveAvgPool2d(1) self.resnet.fc = nn.Linear(512 * 4, in_feats) self.gcn1 = GCNLayer(in_feats, num_classes)
builder.set_int8_calibrator(int8_calibrator) builder.set_int8_mode(True) engine = builder.build_cuda_engine(trt_network) modelstream = engine.serialize() trt.utils.write_engine_to_file(args.trt8_model_name, modelstream) engine.destroy() builder.destroy() # engine = trt_engine('resnet',args.trt8_model_name).build_engine() # print ("Start INT8 Test...") # correct, total, use_time = do_test(engine) # print ('total images:',total,',time:',use_time,"s,Acc: {}".format(correct / float(total))) if __name__ == '__main__': if not os.path.exists(args.onnx_model_name): model = resnet50(pretrained=True) model.cuda() # Translate Pytorch Model into Onnx Model dummy_input = Variable(torch.randn(args.batch_size, args.input_channel, \ args.input_size, args.input_size, device='cuda')) output_names = ["output"] torch.onnx.export(model, dummy_input, args.onnx_model_name, verbose=False, output_names=output_names) onnx_2_float32() onnx_2_int8() # engine = trt_engine('resnet',args.trt32_model_name).build_engine()
def _get_model(args): if args.model.lower() == 'resnet50': model = resnet50() return model
help='window type for the stft') parser.add_argument('--max_len', default=101, type=int, help='maximal length of a window') parser.add_argument('--normalize', default=False, help='boolean, wheather or not to normalize the spect') args = parser.parse_args() setup(args) logger = get_logger(args, __file__) print(args, file=logger) path = glob.glob(os.path.join(args.model, '**', 'checkpoint.pt.best'))[0] model = myresnet.resnet50(num_classes=30) model = RobustnessModelInnerWrapper(model) d = argparse.Namespace() d.mean = torch.tensor(0) d.std = torch.tensor(1) model = AttackerModel(model, d) checkpoint = torch.load(path, pickle_module=dill) state_dict_path = 'model' if not ('model' in checkpoint): state_dict_path = 'state_dict' sd = checkpoint[state_dict_path] sd = {k[len('module.'):]: v for k, v in sd.items()} sd['normalizer.new_mean'] = torch.tensor([[0]]) sd['attacker.normalize.new_mean'] = torch.tensor([[0]]) sd['normalizer.new_std'] = torch.tensor([[1]])
def __init__(self, backbone='resnet', layers=50, bins=(1, 2, 3, 6), dropout=0.1, classes=2, zoom_factor=8, use_ppm=True, use_softmax=True, use_aux=True, pretrained=True, syncbn=True, group_size=8, group=None): super(PSPNet, self).__init__() assert layers in [50, 101, 152] assert 2048 % len(bins) == 0 assert classes > 1 assert zoom_factor in [1, 2, 4, 8] self.zoom_factor = zoom_factor self.use_ppm = use_ppm self.use_softmax = use_softmax self.use_aux = use_aux if backbone == 'resnet': import resnet as models elif backbone == 'ibnnet_a': import ibnnet_a as models elif backbone == 'ibnnet_b': import ibnnet_b as models else: raise NameError('Backbone type not defined!') if syncbn: # from lib.syncbn import SynchronizedBatchNorm2d as BatchNorm def BNFunc(*args, **kwargs): return SyncBatchNorm2d(*args, **kwargs, group_size=group_size, group=group, sync_stats=True) BatchNorm = BNFunc else: from torch.nn import BatchNorm2d as BatchNorm models.BatchNorm = BatchNorm if layers == 50: resnet = models.resnet50(pretrained=pretrained) elif layers == 101: resnet = models.resnet101(pretrained=pretrained) else: resnet = models.resnet152(pretrained=pretrained) if backbone == 'ibnnet_b': self.layer0 = nn.Sequential(resnet.conv1, resnet.INCat0, resnet.relu, resnet.maxpool) else: self.layer0 = nn.Sequential(resnet.conv1, resnet.bn1, resnet.relu, resnet.maxpool) self.layer1, self.layer2, self.layer3, self.layer4 = resnet.layer1, resnet.layer2, resnet.layer3, resnet.layer4 for n, m in self.layer3.named_modules(): if 'conv2' in n and not 'convbnin.conv2' in n: m.dilation, m.padding, m.stride = (2, 2), (2, 2), (1, 1) elif 'downsample.0' in n: m.stride = (1, 1) for n, m in self.layer4.named_modules(): if 'conv2' in n and not 'convbnin.conv2' in n: m.dilation, m.padding, m.stride = (4, 4), (4, 4), (1, 1) elif 'downsample.0' in n: m.stride = (1, 1) channel_4x = 256 # channel attention layer and spatial attention layer. self.cam_4x = ChannelAttentionLayer(channel_4x, reduction=1, multiply=True) self.sam_4x = SpatialAttentionLayer(channel_4x, reduction=1, multiply=True) channel_8x = 512 # channel attention layer and spatial attention layer. self.cam_8x = ChannelAttentionLayer(channel_8x, reduction=1, multiply=True) self.sam_8x = SpatialAttentionLayer(channel_8x, reduction=1, multiply=True) channel_1x = classes # final predict # channel attention layer and spatial attention layer. self.cam_1x = ChannelAttentionLayer(channel_1x, reduction=1, multiply=True) self.sam_1x = SpatialFCAttentionLayer(channel_1x, reduction=1, multiply=True) fea_dim = 2048 if use_ppm: self.ppm = PPM(fea_dim + 128, int(fea_dim / len(bins)), bins, BatchNorm) fea_dim *= 2 self.cls = nn.Sequential( nn.Conv2d(768, 256, kernel_size=3, padding=1, bias=False), BatchNorm(256), nn.ReLU(inplace=True), nn.Dropout2d(p=dropout), nn.Conv2d(256, classes, kernel_size=1)) self.cls_2 = nn.Sequential( nn.Conv2d(classes * 2, classes, kernel_size=1)) self.conv6 = nn.Sequential( nn.Conv2d(256 * 8, 512, kernel_size=1, padding=0, bias=True), BatchNorm(512), nn.ReLU(inplace=True)) self.conv1_1x1 = nn.Sequential( nn.Conv2d(256, 256, kernel_size=1, padding=0, bias=True), BatchNorm(256), nn.ReLU(inplace=True)) self.conv2_1x1 = nn.Sequential( nn.Conv2d(256, 256, kernel_size=1, padding=0, bias=True), BatchNorm(256), nn.ReLU(inplace=True)) if use_aux: self.aux = nn.Sequential( nn.Conv2d(1024, 256, kernel_size=3, padding=1, bias=False), BatchNorm(256), nn.ReLU(inplace=True), nn.Dropout2d(p=dropout), nn.Conv2d(256, classes, kernel_size=1)) # init_weights(self.aux) # comment to use default initialization init_weights(self.ppm)
ds = ds.batch(batch_size=args_opt.batch_size, drop_remainder=True) return ds if __name__ == '__main__': if args_opt.mode == 'train' and args_opt.run_distribute: context.set_auto_parallel_context( device_num=args_opt.device_num, parallel_mode=ParallelMode.DATA_PARALLEL, gradients_mean=True) auto_parallel_context().set_all_reduce_fusion_split_indices([140]) init() epoch_size = args_opt.epoch_size net = resnet50(args_opt.num_classes) ls = SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean") opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), 0.01, 0.9) model = Model(net, loss_fn=ls, optimizer=opt, metrics={'acc'}) if args_opt.mode == 'train': # train print("============== Starting Training ==============") dataset = create_dataset() batch_num = dataset.get_dataset_size() config_ck = CheckpointConfig(save_checkpoint_steps=batch_num, keep_checkpoint_max=10) checkpoint_path = args_opt.checkpoint_path if args_opt.checkpoint_path is not None else "./" ckpoint_cb = ModelCheckpoint(prefix="train_resnet_cifar10", directory=checkpoint_path,
trainset = torchvision.datasets.CIFAR100( root='~/data', train=True, download=True, transform=transform_train) testset = torchvision.datasets.CIFAR100( root='~/data', train=False, download=True, transform=transform_test) N_CLASSES = 100 trainloader = torch.utils.data.DataLoader( trainset, batch_size=args.train_bs, shuffle=True, num_workers=3) testloader = torch.utils.data.DataLoader( testset, batch_size=2000, shuffle=False, num_workers=3) truncloader = torch.utils.data.DataLoader( trainset, batch_size=args.prune_bs, num_workers=3) if args.resnet_type == 'resnet50': model = resnet.resnet50(num_classes=N_CLASSES) model = model.to(device) model.train() x, y = map(lambda x: x.to(device), next(iter(trainloader))) p = model(x) loss = F.cross_entropy(p, y) loss.backward() agg_tensor = [] for child in model.modules(): if isinstance(child, resnet.MaskedConv2d) or isinstance(child, resnet.MaskedLinear): agg_tensor += child.get_mask_grad() agg_tensor = torch.cat(agg_tensor, dim=0).cpu().numpy() value = np.percentile(agg_tensor, args.sparcity)
def main(): # os.system('shutdown -c') # cancel previous shutdown command log.console(args) tb.log('sizes/world', bps.size()) # need to index validation directory before we start counting the time dataloader.sort_ar(args.data + '/validation') # if args.distributed: # log.console('Distributed initializing process group') torch.cuda.set_device(bps.local_rank()) print(f'cuda device set to {bps.local_rank()}') log.console("cuda initialized (rank=%d)" % (bps.local_rank())) # dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, world_size=bps.size()) log.console("Distributed: success (%d/%d)" % (bps.rank(), bps.size())) log.console("Loading model (rank=%d)" % (bps.rank())) model = resnet.resnet50(bn0=args.init_bn0).cuda() # reuse the validate tensor global validate_tensor, dist_validate_tensor validate_tensor = torch.tensor([0, 0, 0, 0]).float().cuda() dist_validate_tensor = torch.tensor([0, 0, 0, 0, 0]).float().cuda() if args.fp16: model = network_to_half(model) best_top5 = 93 # only save models over 93%. Otherwise it stops to save every time global model_params, master_params if args.fp16: model_params, master_params = prep_param_lists(model) else: model_params = master_params = model.parameters() optim_params, name_list = experimental_utils.bnwd_optim_params( model, model_params, master_params) if args.no_bn_wd else master_params # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().cuda() optimizer = torch.optim.SGD( optim_params, 0, momentum=args.momentum, weight_decay=args.weight_decay ) # start with 0 lr. Scheduler will change this later named_param = [] for p in optim_params: tensors = p['params'] for tensor in tensors: named_param.append(tensor) # create bps_param (tuple) bps_param = [] for i, tensor in enumerate(named_param): name = name_list[i] bps_param.append((name, tensor)) # wrap with byteps optimizer optimizer = DistributedOptimizer( optimizer, named_parameters=bps_param, backward_passes_per_step=args.batches_per_pushpull, half=True, model=model, fp16_params=model_params, fp32_params=master_params, loss_scale=args.loss_scale) if args.resume: checkpoint = torch.load( args.resume, map_location=lambda storage, loc: storage.cuda(args.local_rank)) model.load_state_dict(checkpoint['state_dict']) args.start_epoch = checkpoint['epoch'] best_top5 = checkpoint['best_top5'] optimizer.load_state_dict(checkpoint['optimizer']) log.console( "Creating data loaders (this could take up to 10 minutes if volume needs to be warmed up)" ) num_machines = (bps.size() - 1) // 8 + 1 assert (num_machines in schedules) phases = schedules[num_machines] dm = DataManager([copy.deepcopy(p) for p in phases if 'bs' in p]) scheduler = Scheduler(optimizer, [copy.deepcopy(p) for p in phases if 'lr' in p]) # BytePS: broadcast parameters & optimizer state. broadcast_parameters([(name, p.detach()) for name, p in bps_param], root_rank=0) broadcast_optimizer_state(optimizer, root_rank=0) start_time = datetime.now() # Loading start to after everything is loaded if args.evaluate: return validate(dm.val_dl, model, criterion, 0, start_time) if args.distributed: log.console('Global Barrier: Syncing machines before training') tensor = torch.tensor([1.0]).float().cuda() barrier_handler = push_pull_async_inplace(tensor, average=True, name="init.barrier") while True: if poll(barrier_handler): synchronize(barrier_handler) break # do broadcast for validate tensor log.console('Broadcasting validate tensor') barrier_handler = push_pull_async_inplace(validate_tensor, average=True, name="validation_tensor") while True: if poll(barrier_handler): synchronize(barrier_handler) break barrier_handler = push_pull_async_inplace( dist_validate_tensor, average=True, name="distributed_validation_tensor") while True: if poll(barrier_handler): synchronize(barrier_handler) break log.event("~~epoch\thours\ttop1\ttop5\n") for epoch in range(args.start_epoch, scheduler.tot_epochs): dm.set_epoch(epoch) train(dm.trn_dl, model, criterion, optimizer, scheduler, epoch) top1, top5 = validate(dm.val_dl, model, criterion, epoch, start_time) time_diff = (datetime.now() - start_time).total_seconds() / 3600.0 log.event(f'~~{epoch}\t{time_diff:.5f}\t\t{top1:.3f}\t\t{top5:.3f}\n') is_best = top5 > best_top5 best_top5 = max(top5, best_top5) if args.local_rank == 0: if is_best: save_checkpoint(epoch, model, best_top5, optimizer, is_best=True, filename='model_best.pth.tar') phase = dm.get_phase(epoch) if phase: save_checkpoint( epoch, model, best_top5, optimizer, filename=f'sz{phase["bs"]}_checkpoint.path.tar')
def __init__(self, last_conv_stride=1, last_conv_dilation=1, num_stripes=6, local_conv_out_channels=256, num_classes=0): super(RelationModel, self).__init__() self.base = resnet50(pretrained=True, last_conv_stride=last_conv_stride, last_conv_dilation=last_conv_dilation) self.num_stripes = num_stripes self.num_classes = num_classes self.local_6_conv_list = nn.ModuleList() self.local_4_conv_list = nn.ModuleList() self.local_2_conv_list = nn.ModuleList() self.rest_6_conv_list = nn.ModuleList() self.rest_4_conv_list = nn.ModuleList() self.rest_2_conv_list = nn.ModuleList() self.relation_6_conv_list = nn.ModuleList() self.relation_4_conv_list = nn.ModuleList() self.relation_2_conv_list = nn.ModuleList() self.global_6_max_conv_list = nn.ModuleList() self.global_4_max_conv_list = nn.ModuleList() self.global_2_max_conv_list = nn.ModuleList() self.global_6_rest_conv_list = nn.ModuleList() self.global_4_rest_conv_list = nn.ModuleList() self.global_2_rest_conv_list = nn.ModuleList() self.global_6_pooling_conv_list = nn.ModuleList() self.global_4_pooling_conv_list = nn.ModuleList() self.global_2_pooling_conv_list = nn.ModuleList() for i in range(num_stripes): self.local_6_conv_list.append( nn.Sequential(nn.Conv2d(2048, local_conv_out_channels, 1), nn.BatchNorm2d(local_conv_out_channels), nn.ReLU(inplace=True))) for i in range(4): self.local_4_conv_list.append( nn.Sequential(nn.Conv2d(2048, local_conv_out_channels, 1), nn.BatchNorm2d(local_conv_out_channels), nn.ReLU(inplace=True))) for i in range(2): self.local_2_conv_list.append( nn.Sequential(nn.Conv2d(2048, local_conv_out_channels, 1), nn.BatchNorm2d(local_conv_out_channels), nn.ReLU(inplace=True))) for i in range(num_stripes): self.rest_6_conv_list.append( nn.Sequential(nn.Conv2d(2048, local_conv_out_channels, 1), nn.BatchNorm2d(local_conv_out_channels), nn.ReLU(inplace=True))) for i in range(4): self.rest_4_conv_list.append( nn.Sequential(nn.Conv2d(2048, local_conv_out_channels, 1), nn.BatchNorm2d(local_conv_out_channels), nn.ReLU(inplace=True))) for i in range(2): self.rest_2_conv_list.append( nn.Sequential(nn.Conv2d(2048, local_conv_out_channels, 1), nn.BatchNorm2d(local_conv_out_channels), nn.ReLU(inplace=True))) self.global_6_max_conv_list.append( nn.Sequential(nn.Conv2d(2048, local_conv_out_channels, 1), nn.BatchNorm2d(local_conv_out_channels), nn.ReLU(inplace=True))) self.global_4_max_conv_list.append( nn.Sequential(nn.Conv2d(2048, local_conv_out_channels, 1), nn.BatchNorm2d(local_conv_out_channels), nn.ReLU(inplace=True))) self.global_2_max_conv_list.append( nn.Sequential(nn.Conv2d(2048, local_conv_out_channels, 1), nn.BatchNorm2d(local_conv_out_channels), nn.ReLU(inplace=True))) self.global_6_rest_conv_list.append( nn.Sequential(nn.Conv2d(2048, local_conv_out_channels, 1), nn.BatchNorm2d(local_conv_out_channels), nn.ReLU(inplace=True))) self.global_4_rest_conv_list.append( nn.Sequential(nn.Conv2d(2048, local_conv_out_channels, 1), nn.BatchNorm2d(local_conv_out_channels), nn.ReLU(inplace=True))) self.global_2_rest_conv_list.append( nn.Sequential(nn.Conv2d(2048, local_conv_out_channels, 1), nn.BatchNorm2d(local_conv_out_channels), nn.ReLU(inplace=True))) for i in range(num_stripes): self.relation_6_conv_list.append( nn.Sequential( nn.Conv2d(local_conv_out_channels * 2, local_conv_out_channels, 1), nn.BatchNorm2d(local_conv_out_channels), nn.ReLU(inplace=True))) for i in range(4): self.relation_4_conv_list.append( nn.Sequential( nn.Conv2d(local_conv_out_channels * 2, local_conv_out_channels, 1), nn.BatchNorm2d(local_conv_out_channels), nn.ReLU(inplace=True))) for i in range(2): self.relation_2_conv_list.append( nn.Sequential( nn.Conv2d(local_conv_out_channels * 2, local_conv_out_channels, 1), nn.BatchNorm2d(local_conv_out_channels), nn.ReLU(inplace=True))) self.global_6_pooling_conv_list.append( nn.Sequential( nn.Conv2d(local_conv_out_channels * 2, local_conv_out_channels, 1), nn.BatchNorm2d(local_conv_out_channels), nn.ReLU(inplace=True))) self.global_4_pooling_conv_list.append( nn.Sequential( nn.Conv2d(local_conv_out_channels * 2, local_conv_out_channels, 1), nn.BatchNorm2d(local_conv_out_channels), nn.ReLU(inplace=True))) self.global_2_pooling_conv_list.append( nn.Sequential( nn.Conv2d(local_conv_out_channels * 2, local_conv_out_channels, 1), nn.BatchNorm2d(local_conv_out_channels), nn.ReLU(inplace=True))) if num_classes > 0: self.fc_local_6_list = nn.ModuleList() for _ in range(num_stripes): fc = nn.Linear(local_conv_out_channels, num_classes) init.normal_(fc.weight, std=0.001) init.constant_(fc.bias, 0) self.fc_local_6_list.append(fc) self.fc_local_4_list = nn.ModuleList() for _ in range(4): fc = nn.Linear(local_conv_out_channels, num_classes) init.normal_(fc.weight, std=0.001) init.constant_(fc.bias, 0) self.fc_local_4_list.append(fc) self.fc_local_2_list = nn.ModuleList() for _ in range(2): fc = nn.Linear(local_conv_out_channels, num_classes) init.normal_(fc.weight, std=0.001) init.constant_(fc.bias, 0) self.fc_local_2_list.append(fc) self.fc_rest_6_list = nn.ModuleList() for _ in range(num_stripes): fc = nn.Linear(local_conv_out_channels, num_classes) init.normal_(fc.weight, std=0.001) init.constant_(fc.bias, 0) self.fc_rest_6_list.append(fc) self.fc_rest_4_list = nn.ModuleList() for _ in range(4): fc = nn.Linear(local_conv_out_channels, num_classes) init.normal_(fc.weight, std=0.001) init.constant_(fc.bias, 0) self.fc_rest_4_list.append(fc) self.fc_rest_2_list = nn.ModuleList() for _ in range(2): fc = nn.Linear(local_conv_out_channels, num_classes) init.normal_(fc.weight, std=0.001) init.constant_(fc.bias, 0) self.fc_rest_2_list.append(fc) self.fc_local_rest_6_list = nn.ModuleList() for _ in range(num_stripes): fc = nn.Linear(local_conv_out_channels, num_classes) init.normal_(fc.weight, std=0.001) init.constant_(fc.bias, 0) self.fc_local_rest_6_list.append(fc) self.fc_local_rest_4_list = nn.ModuleList() for _ in range(4): fc = nn.Linear(local_conv_out_channels, num_classes) init.normal_(fc.weight, std=0.001) init.constant_(fc.bias, 0) self.fc_local_rest_4_list.append(fc) self.fc_local_rest_2_list = nn.ModuleList() for _ in range(2): fc = nn.Linear(local_conv_out_channels, num_classes) init.normal_(fc.weight, std=0.001) init.constant_(fc.bias, 0) self.fc_local_rest_2_list.append(fc) self.fc_global_6_list = nn.ModuleList() fc = nn.Linear(local_conv_out_channels, num_classes) init.normal_(fc.weight, std=0.001) init.constant_(fc.bias, 0) self.fc_global_6_list.append(fc) self.fc_global_4_list = nn.ModuleList() fc = nn.Linear(local_conv_out_channels, num_classes) init.normal_(fc.weight, std=0.001) init.constant_(fc.bias, 0) self.fc_global_4_list.append(fc) self.fc_global_2_list = nn.ModuleList() fc = nn.Linear(local_conv_out_channels, num_classes) init.normal_(fc.weight, std=0.001) init.constant_(fc.bias, 0) self.fc_global_2_list.append(fc) self.fc_global_max_6_list = nn.ModuleList() fc = nn.Linear(local_conv_out_channels, num_classes) init.normal_(fc.weight, std=0.001) init.constant_(fc.bias, 0) self.fc_global_max_6_list.append(fc) self.fc_global_max_4_list = nn.ModuleList() fc = nn.Linear(local_conv_out_channels, num_classes) init.normal_(fc.weight, std=0.001) init.constant_(fc.bias, 0) self.fc_global_max_4_list.append(fc) self.fc_global_max_2_list = nn.ModuleList() fc = nn.Linear(local_conv_out_channels, num_classes) init.normal_(fc.weight, std=0.001) init.constant_(fc.bias, 0) self.fc_global_max_2_list.append(fc) self.fc_global_rest_6_list = nn.ModuleList() fc = nn.Linear(local_conv_out_channels, num_classes) init.normal_(fc.weight, std=0.001) init.constant_(fc.bias, 0) self.fc_global_rest_6_list.append(fc) self.fc_global_rest_4_list = nn.ModuleList() fc = nn.Linear(local_conv_out_channels, num_classes) init.normal_(fc.weight, std=0.001) init.constant_(fc.bias, 0) self.fc_global_rest_4_list.append(fc) self.fc_global_rest_2_list = nn.ModuleList() fc = nn.Linear(local_conv_out_channels, num_classes) init.normal_(fc.weight, std=0.001) init.constant_(fc.bias, 0) self.fc_global_rest_2_list.append(fc)
def main(): global args, best_accu args = parser.parse_args() # Data loading code normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) transform_train = transforms.Compose([ transforms.Resize(size=(224, 224)), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize ]) transform_test = transforms.Compose( [transforms.Resize(size=(224, 224)), transforms.ToTensor(), normalize]) train_dataset = MultiLabelDataset(root='../dataset/', label='../list/train_g.txt', transform=transform_train) val_dataset = MultiLabelDataset(root='../dataset/', label='../list/test_g.txt', transform=transform_test) train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=4, pin_memory=False) val_loader = torch.utils.data.DataLoader(val_dataset, batch_size=args.batch_size, shuffle=False, num_workers=4, pin_memory=False) # create model model = resnet.resnet50(pretrained=True) # get the number of model parameters print('Number of model parameters: {}'.format( sum([p.data.nelement() for p in model.parameters()]))) # for training on multiple GPUs. # Use CUDA_VISIBLE_DEVICES=0,1 to specify which GPUs to use # model = torch.nn.DataParallel(model).cuda() model = model.cuda() # optionally resume from a checkpoint if args.resume: if os.path.isfile(args.resume): print("=> loading checkpoint '{}'".format(args.resume)) checkpoint = torch.load(args.resume) args.start_epoch = checkpoint['epoch'] best_accu = checkpoint['best_accu'] model.load_state_dict(checkpoint['state_dict']) print("=> loaded checkpoint '{}' (epoch {})".format( args.resume, checkpoint['epoch'])) else: print("=> no checkpoint found at '{}'".format(args.resume)) cudnn.benchmark = True # define loss function (criterion) and pptimizer criterion = nn.BCELoss().cuda() optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # iterative training for epoch in range(args.start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) # train for one epoch train(train_loader, model, criterion, optimizer, epoch) # evaluate on validation set accu = validate(val_loader, model, criterion, epoch) # remember best prec@1 and save checkpoint is_best = accu > best_accu best_accu = max(accu, best_accu) save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model.state_dict(), 'best_accu': best_accu, }, is_best) print('Best accuracy: ', best_accu)
]) logger = logging.getLogger() logging.info('ImageNet feature extraction') logging.info('Args: %s', args) hash_cmd = subprocess.run('git rev-parse --short HEAD', shell=True, check=True, stdout=subprocess.PIPE) git_hash = hash_cmd.stdout.decode('utf-8').strip() logging.info(f'Git commit: {git_hash}') # get model if args.model == 'resnet50': model = resnet50(pretrained=True).to(args.device) else: raise ValueError('Unkown model %s' % args.model) if args.device == 'cuda': model = DataParallel(model) # get input transform transform = transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) ]) def extract_features(split='val'):
testloader = DataLoader(testset, batch_size=512, shuffle=False, num_workers=4) # log columns = ['iteration', 'time', 'train_loss', 'train_acc', 'test_loss', 'test_acc', 'epoch'] log = pd.DataFrame(columns=columns) # model from resnet import resnet50 # from resnet import resnet50 model = resnet50(num_classes=200).to('cuda') model.train() # optimizer base_lr = .1 optimizer = optim.SGD(model.parameters(), lr=base_lr, momentum=.9, weight_decay=5e-4) criterion = nn.CrossEntropyLoss() def adjust_learning_rate(optimizer, iteration): """Sets the learning rate to the initial LR decayed by 10 every 30 epochs""" # with bs = 64, 1 epoch ~ 1500 sgd iterations