def __init__(self, local_conv_out_channels=128, num_classes=None):
super(Model, self).__init__()
self.base = resnet50(pretrained=True)
planes = 2048
self.local_conv = nn.Conv2d(planes, local_conv_out_channels, 1)
self.local_bn = nn.BatchNorm2d(local_conv_out_channels)
self.local_relu = nn.ReLU(inplace=True)
if num_classes is not None:
self.fc = nn.Linear(planes, num_classes)
init.normal(self.fc.weight, std=0.001)
init.constant(self.fc.bias, 0)
def model_modifier(m):
m.layers[-1].activation = tf.keras.activations.linear
imgs = []
labels = []
for index, label in index_label_map.items():
activation_maximization = ActivationMaximization(model, model_modifier)
loss = lambda x: K.mean(x[:, index - 1])
activation = activation_maximization(loss,
steps=itr,
callbacks=[Print(interval=100)])
img = activation[0].astype(np.uint8)
img = np.squeeze(img, 2)
cv2.imwrite(label + ".png", img)
imgs.append(img)
labels.append(label)
plot_images(imgs, labels, 400, layer_name)
model = resnet50()
model.compile(optimizer=keras.optimizers.Adam(),
loss='categorical_crossentropy',
metrics=['accuracy'])
model.load_weights(
'logs\\HETERO RESNET DEFAULT 20200127-082347\\model_weights.h5')
# visualize_conv_layer_filters(model, 'conv1')
def visualize(model, itr):
visualize_dense_layer(model, 'fc1000', index_emotrion, itr)
loss_func = self.mean(loss_func, (-1, ))
return loss_func
if __name__ == '__main__':
if not args_opt.do_eval and args_opt.run_distribute:
context.set_auto_parallel_context(
device_num=args_opt.device_num,
parallel_mode=ParallelMode.DATA_PARALLEL)
context.set_auto_parallel_context(
all_reduce_fusion_split_indices=[140])
init()
context.set_context(mode=context.GRAPH_MODE)
epoch_size = args_opt.epoch_size
net = resnet50(args_opt.batch_size, args_opt.num_classes)
loss = CrossEntropyLoss()
opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()),
0.01, 0.9)
model = Model(net, loss_fn=loss, optimizer=opt, metrics={'acc'})
if args_opt.do_train:
dataset = create_dataset(epoch_size)
batch_num = dataset.get_dataset_size()
config_ck = CheckpointConfig(save_checkpoint_steps=batch_num * 5,
keep_checkpoint_max=10)
ckpoint_cb = ModelCheckpoint(prefix="train_resnet_cifar10",
directory="./",
config=config_ck)
loss_cb = LossMonitor()
img_dir = '/media/disk2/ljn/video_dataset/Mars/bbox_test/'
test_dataset = dataset.videodataset(dataset_dir=img_dir,
txt_path='list/list_test_seq_all.txt',
new_height=256,
new_width=128,
frames=8,
transform=transform)
test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=8)
########### MODEL ###########
pretrained = 'resnet50_3d_mars_00240.pth'
model = resnet.resnet50(pretrained=pretrained, num_classes=625, train=False)
model.cuda()
model.eval()
name = 'fea/fea'
output = open(name, 'w')
num = 0
for data in test_loader:
num = num + batch_size
images, label = data
images = torch.transpose(images, 1, 2).contiguous()
images = images.view(
images.size(0) * images.size(1), images.size(2), images.size(3),
images.size(4))
images = Variable(images).cuda()
def __init__(self, layers, norm_layer=None):
super(Net, self).__init__()
if norm_layer is None:
norm_layer = nn.BatchNorm2d
self._norm_layer = norm_layer
resnet = None
if args.model == 'resnet18':
resnet = resnet18(False, True)
print('use resnet18')
elif args.model == 'resnet34':
resnet = resnet34(False, True)
print('use resnet34')
elif args.model == 'resnet50':
resnet = resnet50(False, True)
print('use resnet50')
elif args.model == 'resnet101':
resnet = resnet101(False, True)
print('use resnet101')
resnet.to(device)
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(resnet.parameters(), lr=lr, momentum=0.9)
# train
best = 999.99
best_state_dict = None
best_optimizer = None
print('Start training')
pbar = tqdm(total=int(args.epoch * len(y_train) / batch_size), desc='training')
def __init__(self, model_path='resnet50.pth'):
super(ResNet50, self).__init__()
model = resnet50()
model.load_state_dict(torch.load(model_path))
self.features = nn.Sequential(*list(model.children())[:-1])
def __init__(self, num_class, center_dim):
super(Net, self).__init__()
# self.backbone = resnet50(num_classes=2)
self.ip1 = resnet50(num_classes=center_dim)
self.ip2 = nn.Linear(center_dim, num_class, bias=False)
def resnet50():
"""Load resnet50"""
return resnet.resnet50(pretrained=True, progress=True)
def main_worker(gpu, ngpus_per_node, args):
global best_acc1
args.gpu = gpu
if args.gpu is not None:
print("Use GPU: {} for training".format(args.gpu))
if args.distributed:
if args.dist_url == "env://" and args.rank == -1:
args.rank = int(os.environ["RANK"])
if args.multiprocessing_distributed:
# For multiprocessing distributed training, rank needs to be the
# global rank among all the processes
args.rank = args.rank * ngpus_per_node + gpu
dist.init_process_group(backend=args.dist_backend,
init_method=args.dist_url,
world_size=args.world_size,
rank=args.rank)
# create model
# if args.pretrained:
# print("=> using pre-trained model '{}'".format(args.arch))
# model = models.__dict__[args.arch](pretrained=True)
# else:
# print("=> creating model '{}'".format(args.arch))
# model = models.__dict__[args.arch]()
# ---- or ----
model = resnet50(pretrained=True)
if args.distributed:
# For multiprocessing distributed, DistributedDataParallel constructor
# should always set the single device scope, otherwise,
# DistributedDataParallel will use all available devices.
if args.gpu is not None:
torch.cuda.set_device(args.gpu)
model.cuda(args.gpu)
# When using a single GPU per process and per
# DistributedDataParallel, we need to divide the batch size
# ourselves based on the total number of GPUs we have
args.batch_size = int(args.batch_size / ngpus_per_node)
args.workers = int(
(args.workers + ngpus_per_node - 1) / ngpus_per_node)
model = torch.nn.parallel.DistributedDataParallel(
model, device_ids=[args.gpu])
else:
model.cuda()
# DistributedDataParallel will divide and allocate batch_size to all
# available GPUs if device_ids are not set
model = torch.nn.parallel.DistributedDataParallel(model)
elif args.gpu is not None:
torch.cuda.set_device(args.gpu)
model = model.cuda(args.gpu)
else:
# DataParallel will divide and allocate batch_size to all available GPUs
if args.arch.startswith('alexnet') or args.arch.startswith('vgg'):
model.features = torch.nn.DataParallel(model.features)
model.cuda()
else:
model = torch.nn.DataParallel(model).cuda()
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().cuda(args.gpu)
optimizer = torch.optim.SGD(model.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# optionally resume from a checkpoint
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
if args.gpu is None:
checkpoint = torch.load(args.resume)
else:
# Map model to be loaded to specified single gpu.
loc = 'cuda:{}'.format(args.gpu)
checkpoint = torch.load(args.resume, map_location=loc)
args.start_epoch = checkpoint['epoch']
best_acc1 = checkpoint['best_acc1']
if args.gpu is not None:
# best_acc1 may be from a checkpoint from a different GPU
best_acc1 = best_acc1.to(args.gpu)
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
args.resume, checkpoint['epoch']))
else:
print("=> no checkpoint found at '{}'".format(args.resume))
cudnn.benchmark = True
# Data loading code
traindir = os.path.join(args.data, 'train')
valdir = os.path.join(args.data, 'val')
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_dataset = datasets.ImageFolder(
traindir,
transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
]))
if args.distributed:
train_sampler = torch.utils.data.distributed.DistributedSampler(
train_dataset)
else:
train_sampler = None
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=(train_sampler is None),
num_workers=args.workers,
pin_memory=True,
sampler=train_sampler)
val_loader = torch.utils.data.DataLoader(datasets.ImageFolder(
valdir,
transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
])),
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
if args.evaluate:
validate(val_loader, model, criterion, args)
return
for epoch in range(args.start_epoch, args.epochs):
if args.distributed:
train_sampler.set_epoch(epoch)
adjust_learning_rate(optimizer, epoch, args)
# train for one epoch
train(train_loader, model, criterion, optimizer, epoch, args)
# evaluate on validation set
acc1 = validate(val_loader, model, criterion, args)
# remember best acc@1 and save checkpoint
is_best = acc1 > best_acc1
best_acc1 = max(acc1, best_acc1)
if not args.multiprocessing_distributed or (
args.multiprocessing_distributed
and args.rank % ngpus_per_node == 0):
save_checkpoint(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'best_acc1': best_acc1,
'optimizer': optimizer.state_dict(),
}, is_best)
transforms.RandomCrop(img_size),
transforms.ToTensor(), normalize
])
print('==> Preparing data..')
trainset = dataloader(train=True, transform=transform_train)
trainloader = torch.utils.data.DataLoader(trainset,
batch_size=1,
num_workers=50,
shuffle=True)
model = args.model
# Use args.model as pretrain model
if model == 'resnet152':
net = resnet.resnet152().to(device)
elif model == 'resnet50':
net = resnet.resnet50().to(device)
else:
sys.exit(-1)
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(
net.parameters(), lr=0.1, momentum=0.9,
weight_decay=5e-4) # 优化方式为mini-batch momentum-SGD，并采用L2正则化（权重衰减）
with open("acc.txt", "w") as f:
with open("log.txt", "w") as f2:
for epoch in range(start_epoch, n_epochs):
print('\nEpoch: %d' % (epoch + 1))
net.train()
sum_loss = 0.0
correct = 0.0
total = 0.0
def main():
model = resnet50(pretrained=True)
fake_img = np.random.uniform(-1, 1, [2, 3, 224, 224]).astype("float32")
fake_img = torch.tensor(fake_img)
print(model(fake_img).shape)
default_clear_line=False,
root_dir=os.path.normpath(
os.path.dirname(os.path.realpath(__file__))))
mllogger = mllog.get_mllogger()
# submission
mllogger.event(key=mllog.constants.SUBMISSION_BENCHMARK, value="resnet")
mllogger.event(key=mllog.constants.SUBMISSION_DIVISION, value="closed")
mllogger.event(key=mllog.constants.SUBMISSION_ORG, value="SIAT")
mllogger.event(key=mllog.constants.SUBMISSION_PLATFORM, value="Ascend 910")
mllogger.event(key=mllog.constants.SUBMISSION_STATUS, value="cloud")
mllogger.event(key=mllog.constants.CACHE_CLEAR)
# init the distribute env
init()
# network
net = resnet50(class_num=args.class_num)
# evaluation network
dist_eval_network = ClassifyCorrectCell(net)
# loss
if not args.use_label_smooth:
args.label_smooth_factor = 0.0
loss = nn.SoftmaxCrossEntropyWithLogits(
sparse=True,
reduction="mean",
smooth_factor=args.label_smooth_factor,
num_classes=args.class_num)
# train dataset
epoch_size = args.max_epoch
res = []
for k in topk:
correct_k = correct[:k].view(-1).float().sum(0, keepdim=True)
res.append(correct_k.mul_(100.0 / batch_size))
return res
def compute_softmax(i):
i = i - i.min(axis=1, keepdims=True)
log_sum = logsumexp(i, axis=1, keepdims=True)
return i - log_sum
NAME_TO_MODEL = {
'resnet50': resnet50(num_classes=100),
'ayangnet': AyangNet(),
'densenet': models.densenet161(num_classes=100),
'inceptionv3': inception_v3(num_classes=100),
'inceptionv4': inception_v4(num_classes=100),
'wideresnet': WideResNet(28, 100, widen_factor=10),
'widedensenet': DenseNet(60, (6, 6, 6, 6), 64, num_classes=100)
}
if __name__ == '__main__':
default_path = './preprocess'
noise_decay = 0.55
loss_fn = CrossEntropyLoss()
# set up argument parser
parser = argparse.ArgumentParser()
print("The batch tensor is:",images.shape)
plt.show()
# In[4]:
get_ipython().system('wget -N https://obs.dualstack.cn-north-4.myhuaweicloud.com/mindspore-website/notebook/source-codes/resnet.py')
# In[5]:
from resnet import resnet50
net = resnet50(batch_size=32, num_classes=10)
# In[6]:
import mindspore.nn as nn
from mindspore.nn import SoftmaxCrossEntropyWithLogits
ls = SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean")
opt = nn.Momentum(filter(lambda x: x.requires_grad, net.get_parameters()), 0.01, 0.9)
# In[7]:
def main():
os.system('shutdown -c') # cancel previous shutdown command
log.console(args)
tb.log('sizes/world', dist_utils.env_world_size())
# need to index validation directory before we start counting the time
dataloader.sort_ar(args.data + '/validation')
if args.distributed:
log.console('Distributed initializing process group')
torch.cuda.set_device(args.local_rank)
dist.init_process_group(backend=args.dist_backend,
init_method=args.dist_url,
world_size=dist_utils.env_world_size())
assert (dist_utils.env_world_size() == dist.get_world_size())
log.console("Distributed: success (%d/%d)" %
(args.local_rank, dist.get_world_size()))
log.console("Loading model")
model = resnet.resnet50(bn0=args.init_bn0).cuda()
#if args.fp16: model = network_to_half(model)
if args.distributed:
model = dist_utils.DDP(model,
device_ids=[args.local_rank],
output_device=args.local_rank)
best_top5 = 80 # only save models over 80%. Otherwise it stops to save every time
global model_params, master_params
#if args.fp16: model_params, master_params = prep_param_lists(model)
#else: model_params = master_params = model.parameters()
model_params = master_params = model.parameters()
bparams, oparams = [], []
for name, param in model.named_parameters():
if 'bias' in name:
bparams.append(param)
else:
oparams.append(param)
optim_params = [{
'params': bparams,
'weight_decay': 0.
}, {
'params': oparams,
'weight_decay': args.weight_decay
}]
#model_params = master_params = model.parameters()
#optim_params = experimental_utils.bnwd_optim_params(model, model_params, master_params) if args.no_bn_wd else master_params
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().cuda()
optimizer = torch.optim.SGD(
optim_params,
0,
momentum=args.momentum,
weight_decay=args.weight_decay
) # start with 0 lr. Scheduler will change this later
if args.resume:
checkpoint = torch.load(
args.resume,
map_location=lambda storage, loc: storage.cuda(args.local_rank))
model.load_state_dict(checkpoint['state_dict'])
args.start_epoch = checkpoint['epoch']
best_top5 = checkpoint['best_top5']
optimizer.load_state_dict(checkpoint['optimizer'])
# save script so we can reproduce from logs
shutil.copy2(os.path.realpath(__file__), f'{args.logdir}')
log.console(
"Creating data loaders (this could take up to 10 minutes if volume needs to be warmed up)"
)
phases = eval(args.phases)
dm = DataManager([copy.deepcopy(p) for p in phases if 'bs' in p])
scheduler = Scheduler(optimizer,
[copy.deepcopy(p) for p in phases if 'lr' in p])
start_time = datetime.now() # Loading start to after everything is loaded
if args.evaluate:
return validate(dm.val_dl, model, criterion, 0, start_time)
if args.distributed:
log.console('Syncing machines before training')
dist_utils.sum_tensor(torch.tensor([1.0]).float().cuda())
log.event("~~epoch\thours\ttop1\ttop5\n")
for epoch in range(args.start_epoch, scheduler.tot_epochs):
dm.set_epoch(epoch)
train(dm.trn_dl, model, criterion, optimizer, scheduler, epoch)
top1, top5 = validate(dm.val_dl, model, criterion, epoch, start_time)
time_diff = (datetime.now() - start_time).total_seconds() / 3600.0
log.event(f'~~{epoch}\t{time_diff:.5f}\t\t{top1:.3f}\t\t{top5:.3f}\n')
is_best = top5 > best_top5
best_top5 = max(top5, best_top5)
if args.local_rank == 0:
if is_best:
save_checkpoint(epoch,
model,
best_top5,
optimizer,
is_best=True,
filename='model_best.pth.tar')
phase = dm.get_phase(epoch)
if phase:
save_checkpoint(
epoch,
model,
best_top5,
optimizer,
filename=f'sz{phase["bs"]}_checkpoint.path.tar')
def resnet50_train(args_opt):
epoch_size = args_opt.epoch_size
batch_size = 32
class_num = 10
loss_scale_num = 1024
local_data_path = '/cache/data'
# set graph mode and parallel mode
context.set_context(mode=context.GRAPH_MODE,
device_target="Ascend",
save_graphs=False)
context.set_context(device_id=device_id)
if device_num > 1:
context.set_auto_parallel_context(
device_num=device_num,
parallel_mode=ParallelMode.DATA_PARALLEL,
gradients_mean=True)
init()
local_data_path = os.path.join(local_data_path, str(device_id))
# data download
print('Download data.')
mox.file.copy_parallel(src_url=args_opt.data_url, dst_url=local_data_path)
# create dataset
print('Create train and evaluate dataset.')
train_dataset = create_dataset(dataset_path=local_data_path,
do_train=True,
repeat_num=1,
batch_size=batch_size)
eval_dataset = create_dataset(dataset_path=local_data_path,
do_train=False,
repeat_num=1,
batch_size=batch_size)
train_step_size = train_dataset.get_dataset_size()
print('Create dataset success.')
# create model
net = resnet50(class_num=class_num)
# reduction='mean' means that apply reduction of mean to loss
loss = SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean')
lr = Tensor(
get_lr(global_step=0,
total_epochs=epoch_size,
steps_per_epoch=train_step_size))
opt = Momentum(net.trainable_params(),
lr,
momentum=0.9,
weight_decay=1e-4,
loss_scale=loss_scale_num)
loss_scale = FixedLossScaleManager(loss_scale_num, False)
# amp_level="O2" means that the hybrid precision of O2 mode is used for training
# the whole network except that batchnoram will be cast into float16 format and dynamic loss scale will be used
# 'keep_batchnorm_fp32 = False' means that use the float16 format
model = Model(net,
amp_level="O2",
keep_batchnorm_fp32=False,
loss_fn=loss,
optimizer=opt,
loss_scale_manager=loss_scale,
metrics={'acc'})
# define performance callback to show ips and loss callback to show loss for every epoch
performance_cb = PerformanceCallback(batch_size)
loss_cb = LossMonitor()
cb = [performance_cb, loss_cb]
print(f'Start run training, total epoch: {epoch_size}.')
model.train(epoch_size, train_dataset, callbacks=cb)
if device_num == 1 or device_id == 0:
print(f'Start run evaluation.')
output = model.eval(eval_dataset)
print(f'Evaluation result: {output}.')
def resnet50_train(args_opt):
epoch_size = args_opt.epoch_size
batch_size = cfg.batch_size
class_num = cfg.class_num
loss_scale_num = cfg.loss_scale
local_data_path = '/cache/data'
local_ckpt_path = '/cache/ckpt_file'
# set graph mode and parallel mode
context.set_context(mode=context.GRAPH_MODE,
device_target="Ascend",
save_graphs=False)
# data download
print('Download data.')
mox.file.copy_parallel(src_url=args_opt.data_url, dst_url=local_data_path)
# create dataset
print('Create train and evaluate dataset.')
train_dataset = create_dataset(dataset_path=local_data_path,
do_train=True,
repeat_num=epoch_size,
batch_size=batch_size)
train_step_size = train_dataset.get_dataset_size()
print('Create dataset success.')
# create model
net = resnet50(class_num=class_num)
# reduction='mean' means that apply reduction of mean to loss
loss = SoftmaxCrossEntropyWithLogits(sparse=True, reduction='mean')
lr = Tensor(
get_lr(global_step=0,
total_epochs=epoch_size,
steps_per_epoch=train_step_size))
opt = Momentum(net.trainable_params(),
lr,
momentum=0.9,
weight_decay=1e-4,
loss_scale=loss_scale_num)
loss_scale = FixedLossScaleManager(loss_scale_num, False)
# amp_level="O2" means that the hybrid precision of O2 mode is used for training
# the whole network except that batchnorm will be cast into float16 format and dynamic loss scale will be used
# 'keep_batchnorm_fp32 = False' means that use the float16 format
model = Model(net,
amp_level="O2",
keep_batchnorm_fp32=False,
loss_fn=loss,
optimizer=opt,
loss_scale_manager=loss_scale,
metrics={'acc'})
# define performance callback to show ips and loss callback to show loss for every epoch
time_cb = TimeMonitor(data_size=train_step_size)
performance_cb = PerformanceCallback(batch_size)
loss_cb = LossMonitor()
cb = [time_cb, performance_cb, loss_cb]
config_ck = CheckpointConfig(
save_checkpoint_steps=cfg.save_checkpoint_epochs * train_step_size,
keep_checkpoint_max=cfg.keep_checkpoint_max)
ckpt_cb = ModelCheckpoint(prefix="resnet",
directory=local_ckpt_path,
config=config_ck)
cb += [ckpt_cb]
print(f'Start run training, total epoch: {epoch_size}.')
model.train(epoch_size, train_dataset, callbacks=cb)
# upload checkpoint files
print('Upload checkpoint.')
mox.file.copy_parallel(src_url=local_ckpt_path, dst_url=args_opt.train_url)
def __init__(self,
backbone='resnet',
layers=50,
bins=(1, 2, 3, 6),
dropout=0.1,
classes=2,
zoom_factor=8,
use_ppm=False,
use_softmax=True,
use_aux=False,
pretrained=True,
syncbn=True,
group_size=8,
group=None):
super(PSPNet, self).__init__()
assert layers in [50, 101, 152]
assert 2048 % len(bins) == 0
assert classes > 1
assert zoom_factor in [1, 2, 4, 8]
self.zoom_factor = zoom_factor
self.use_ppm = use_ppm
self.use_softmax = use_softmax
self.use_aux = use_aux
if backbone == 'resnet':
import resnet as models
elif backbone == 'ibnnet_a':
import ibnnet_a as models
elif backbone == 'ibnnet_b':
import ibnnet_b as models
else:
raise NameError('Backbone type not defined!')
if syncbn:
from torchE.nn import SyncBatchNorm2d
# from lib.syncbn import SynchronizedBatchNorm2d as BatchNorm
def BNFunc(*args, **kwargs):
return SyncBatchNorm2d(*args,
**kwargs,
group_size=group_size,
group=group,
sync_stats=True)
BatchNorm = BNFunc
else:
from torch.nn import BatchNorm2d as BatchNorm
models.BatchNorm = BatchNorm
if layers == 50:
resnet = models.resnet50(pretrained=pretrained)
elif layers == 101:
resnet = models.resnet101(pretrained=pretrained)
else:
resnet = models.resnet152(pretrained=pretrained)
if backbone == 'ibnnet_b':
self.layer0 = nn.Sequential(resnet.conv1, resnet.INCat0,
resnet.relu, resnet.maxpool)
else:
self.layer0 = nn.Sequential(resnet.conv1, resnet.bn1, resnet.relu,
resnet.maxpool)
self.layer1, self.layer2, self.layer3, self.layer4 = resnet.layer1, resnet.layer2, resnet.layer3, resnet.layer4
for n, m in self.layer4.named_modules():
if 'conv2' in n and not 'convbnin.conv2' in n:
m.dilation, m.padding, m.stride = (2, 2), (2, 2), (1, 1)
elif 'downsample.0' in n:
m.stride = (1, 1)
fea_dim = 2048
if use_ppm:
self.ppm = PPM(fea_dim, int(fea_dim / len(bins)), bins, BatchNorm)
fea_dim *= 2
self.cls = nn.Sequential(
nn.AdaptiveAvgPool2d(1),
nn.Conv2d(fea_dim, 20, kernel_size=1, bias=False),
)
# 打印所使用的设备
print(f"use device is : {args.device_target}")
#dataset_sink_mode = not args.device_target == "CPU"
# 自动并行运算
if args.run_distribute:
context.set_auto_parallel_context(
device_num=args.device_num,
parallel_mode=ParallelMode.DATA_PARALLEL)
auto_parallel_context().set_all_reduce_fusion_split_indices([140])
init()
# 定义训练过程中的一些参数
epoch_size = args.epoch_size # 反向传播计算迭代次数
net = resnet50(args.batch_size, args.num_classes) # 创建ResNet网络对象
net_loss = SoftmaxCrossEntropyWithLogits(sparse=True,
reduction='mean') # 定义损失行数
opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()),
0.01, 0.9) # 定义训练过程中的优化器
# 创建网络模型。metrics={"acc", "loss"}表示评估该网络模型的时候，评估准确率、损失值。
model = Model(net, loss_fn=net_loss, optimizer=opt, metrics={'acc'})
#sink_mode = not args.device_target == "CPU"
# as for train, users could use model.train
if args.do_train:
train_dataset = create_dataset(dog_dataset_path,
os.path.join(dog_dataset_path,
"training.csv"),
batch_size=args.batch_size,
def __init__(self, in_feats, num_classes):
super(GCN_fast, self).__init__()
self.resnet = resnet50(pretrained=True)
self.resnet.avgpool = nn.AdaptiveAvgPool2d(1)
self.resnet.fc = nn.Linear(512 * 4, in_feats)
self.gcn1 = GCNLayer(in_feats, num_classes)
builder.set_int8_calibrator(int8_calibrator)
builder.set_int8_mode(True)
engine = builder.build_cuda_engine(trt_network)
modelstream = engine.serialize()
trt.utils.write_engine_to_file(args.trt8_model_name, modelstream)
engine.destroy()
builder.destroy()
# engine = trt_engine('resnet',args.trt8_model_name).build_engine()
# print ("Start INT8 Test...")
# correct, total, use_time = do_test(engine)
# print ('total images:',total,',time:',use_time,"s,Acc: {}".format(correct / float(total)))
if __name__ == '__main__':
if not os.path.exists(args.onnx_model_name):
model = resnet50(pretrained=True)
model.cuda()
# Translate Pytorch Model into Onnx Model
dummy_input = Variable(torch.randn(args.batch_size, args.input_channel, \
args.input_size, args.input_size, device='cuda'))
output_names = ["output"]
torch.onnx.export(model,
dummy_input,
args.onnx_model_name,
verbose=False,
output_names=output_names)
onnx_2_float32()
onnx_2_int8()
# engine = trt_engine('resnet',args.trt32_model_name).build_engine()
def _get_model(args):
if args.model.lower() == 'resnet50':
model = resnet50()
return model
help='window type for the stft')
parser.add_argument('--max_len',
default=101,
type=int,
help='maximal length of a window')
parser.add_argument('--normalize',
default=False,
help='boolean, wheather or not to normalize the spect')
args = parser.parse_args()
setup(args)
logger = get_logger(args, __file__)
print(args, file=logger)
path = glob.glob(os.path.join(args.model, '**', 'checkpoint.pt.best'))[0]
model = myresnet.resnet50(num_classes=30)
model = RobustnessModelInnerWrapper(model)
d = argparse.Namespace()
d.mean = torch.tensor(0)
d.std = torch.tensor(1)
model = AttackerModel(model, d)
checkpoint = torch.load(path, pickle_module=dill)
state_dict_path = 'model'
if not ('model' in checkpoint):
state_dict_path = 'state_dict'
sd = checkpoint[state_dict_path]
sd = {k[len('module.'):]: v for k, v in sd.items()}
sd['normalizer.new_mean'] = torch.tensor([[0]])
sd['attacker.normalize.new_mean'] = torch.tensor([[0]])
sd['normalizer.new_std'] = torch.tensor([[1]])
def __init__(self,
backbone='resnet',
layers=50,
bins=(1, 2, 3, 6),
dropout=0.1,
classes=2,
zoom_factor=8,
use_ppm=True,
use_softmax=True,
use_aux=True,
pretrained=True,
syncbn=True,
group_size=8,
group=None):
super(PSPNet, self).__init__()
assert layers in [50, 101, 152]
assert 2048 % len(bins) == 0
assert classes > 1
assert zoom_factor in [1, 2, 4, 8]
self.zoom_factor = zoom_factor
self.use_ppm = use_ppm
self.use_softmax = use_softmax
self.use_aux = use_aux
if backbone == 'resnet':
import resnet as models
elif backbone == 'ibnnet_a':
import ibnnet_a as models
elif backbone == 'ibnnet_b':
import ibnnet_b as models
else:
raise NameError('Backbone type not defined!')
if syncbn:
# from lib.syncbn import SynchronizedBatchNorm2d as BatchNorm
def BNFunc(*args, **kwargs):
return SyncBatchNorm2d(*args,
**kwargs,
group_size=group_size,
group=group,
sync_stats=True)
BatchNorm = BNFunc
else:
from torch.nn import BatchNorm2d as BatchNorm
models.BatchNorm = BatchNorm
if layers == 50:
resnet = models.resnet50(pretrained=pretrained)
elif layers == 101:
resnet = models.resnet101(pretrained=pretrained)
else:
resnet = models.resnet152(pretrained=pretrained)
if backbone == 'ibnnet_b':
self.layer0 = nn.Sequential(resnet.conv1, resnet.INCat0,
resnet.relu, resnet.maxpool)
else:
self.layer0 = nn.Sequential(resnet.conv1, resnet.bn1, resnet.relu,
resnet.maxpool)
self.layer1, self.layer2, self.layer3, self.layer4 = resnet.layer1, resnet.layer2, resnet.layer3, resnet.layer4
for n, m in self.layer3.named_modules():
if 'conv2' in n and not 'convbnin.conv2' in n:
m.dilation, m.padding, m.stride = (2, 2), (2, 2), (1, 1)
elif 'downsample.0' in n:
m.stride = (1, 1)
for n, m in self.layer4.named_modules():
if 'conv2' in n and not 'convbnin.conv2' in n:
m.dilation, m.padding, m.stride = (4, 4), (4, 4), (1, 1)
elif 'downsample.0' in n:
m.stride = (1, 1)
channel_4x = 256
# channel attention layer and spatial attention layer.
self.cam_4x = ChannelAttentionLayer(channel_4x,
reduction=1,
multiply=True)
self.sam_4x = SpatialAttentionLayer(channel_4x,
reduction=1,
multiply=True)
channel_8x = 512
# channel attention layer and spatial attention layer.
self.cam_8x = ChannelAttentionLayer(channel_8x,
reduction=1,
multiply=True)
self.sam_8x = SpatialAttentionLayer(channel_8x,
reduction=1,
multiply=True)
channel_1x = classes # final predict
# channel attention layer and spatial attention layer.
self.cam_1x = ChannelAttentionLayer(channel_1x,
reduction=1,
multiply=True)
self.sam_1x = SpatialFCAttentionLayer(channel_1x,
reduction=1,
multiply=True)
fea_dim = 2048
if use_ppm:
self.ppm = PPM(fea_dim + 128, int(fea_dim / len(bins)), bins,
BatchNorm)
fea_dim *= 2
self.cls = nn.Sequential(
nn.Conv2d(768, 256, kernel_size=3, padding=1, bias=False),
BatchNorm(256), nn.ReLU(inplace=True), nn.Dropout2d(p=dropout),
nn.Conv2d(256, classes, kernel_size=1))
self.cls_2 = nn.Sequential(
nn.Conv2d(classes * 2, classes, kernel_size=1))
self.conv6 = nn.Sequential(
nn.Conv2d(256 * 8, 512, kernel_size=1, padding=0, bias=True),
BatchNorm(512), nn.ReLU(inplace=True))
self.conv1_1x1 = nn.Sequential(
nn.Conv2d(256, 256, kernel_size=1, padding=0, bias=True),
BatchNorm(256), nn.ReLU(inplace=True))
self.conv2_1x1 = nn.Sequential(
nn.Conv2d(256, 256, kernel_size=1, padding=0, bias=True),
BatchNorm(256), nn.ReLU(inplace=True))
if use_aux:
self.aux = nn.Sequential(
nn.Conv2d(1024, 256, kernel_size=3, padding=1, bias=False),
BatchNorm(256), nn.ReLU(inplace=True), nn.Dropout2d(p=dropout),
nn.Conv2d(256, classes, kernel_size=1))
# init_weights(self.aux)
# comment to use default initialization
init_weights(self.ppm)
ds = ds.batch(batch_size=args_opt.batch_size, drop_remainder=True)
return ds
if __name__ == '__main__':
if args_opt.mode == 'train' and args_opt.run_distribute:
context.set_auto_parallel_context(
device_num=args_opt.device_num,
parallel_mode=ParallelMode.DATA_PARALLEL,
gradients_mean=True)
auto_parallel_context().set_all_reduce_fusion_split_indices([140])
init()
epoch_size = args_opt.epoch_size
net = resnet50(args_opt.num_classes)
ls = SoftmaxCrossEntropyWithLogits(sparse=True, reduction="mean")
opt = Momentum(filter(lambda x: x.requires_grad, net.get_parameters()),
0.01, 0.9)
model = Model(net, loss_fn=ls, optimizer=opt, metrics={'acc'})
if args_opt.mode == 'train': # train
print("============== Starting Training ==============")
dataset = create_dataset()
batch_num = dataset.get_dataset_size()
config_ck = CheckpointConfig(save_checkpoint_steps=batch_num,
keep_checkpoint_max=10)
checkpoint_path = args_opt.checkpoint_path if args_opt.checkpoint_path is not None else "./"
ckpoint_cb = ModelCheckpoint(prefix="train_resnet_cifar10",
directory=checkpoint_path,
trainset = torchvision.datasets.CIFAR100(
root='~/data', train=True, download=True, transform=transform_train)
testset = torchvision.datasets.CIFAR100(
root='~/data', train=False, download=True, transform=transform_test)
N_CLASSES = 100
trainloader = torch.utils.data.DataLoader(
trainset, batch_size=args.train_bs, shuffle=True, num_workers=3)
testloader = torch.utils.data.DataLoader(
testset, batch_size=2000, shuffle=False, num_workers=3)
truncloader = torch.utils.data.DataLoader(
trainset, batch_size=args.prune_bs, num_workers=3)
if args.resnet_type == 'resnet50':
model = resnet.resnet50(num_classes=N_CLASSES)
model = model.to(device)
model.train()
x, y = map(lambda x: x.to(device), next(iter(trainloader)))
p = model(x)
loss = F.cross_entropy(p, y)
loss.backward()
agg_tensor = []
for child in model.modules():
if isinstance(child, resnet.MaskedConv2d) or isinstance(child, resnet.MaskedLinear):
agg_tensor += child.get_mask_grad()
agg_tensor = torch.cat(agg_tensor, dim=0).cpu().numpy()
value = np.percentile(agg_tensor, args.sparcity)
def main():
# os.system('shutdown -c') # cancel previous shutdown command
log.console(args)
tb.log('sizes/world', bps.size())
# need to index validation directory before we start counting the time
dataloader.sort_ar(args.data + '/validation')
# if args.distributed:
# log.console('Distributed initializing process group')
torch.cuda.set_device(bps.local_rank())
print(f'cuda device set to {bps.local_rank()}')
log.console("cuda initialized (rank=%d)" % (bps.local_rank()))
# dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, world_size=bps.size())
log.console("Distributed: success (%d/%d)" % (bps.rank(), bps.size()))
log.console("Loading model (rank=%d)" % (bps.rank()))
model = resnet.resnet50(bn0=args.init_bn0).cuda()
# reuse the validate tensor
global validate_tensor, dist_validate_tensor
validate_tensor = torch.tensor([0, 0, 0, 0]).float().cuda()
dist_validate_tensor = torch.tensor([0, 0, 0, 0, 0]).float().cuda()
if args.fp16: model = network_to_half(model)
best_top5 = 93 # only save models over 93%. Otherwise it stops to save every time
global model_params, master_params
if args.fp16: model_params, master_params = prep_param_lists(model)
else: model_params = master_params = model.parameters()
optim_params, name_list = experimental_utils.bnwd_optim_params(
model, model_params, master_params) if args.no_bn_wd else master_params
# define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().cuda()
optimizer = torch.optim.SGD(
optim_params,
0,
momentum=args.momentum,
weight_decay=args.weight_decay
) # start with 0 lr. Scheduler will change this later
named_param = []
for p in optim_params:
tensors = p['params']
for tensor in tensors:
named_param.append(tensor)
# create bps_param (tuple)
bps_param = []
for i, tensor in enumerate(named_param):
name = name_list[i]
bps_param.append((name, tensor))
# wrap with byteps optimizer
optimizer = DistributedOptimizer(
optimizer,
named_parameters=bps_param,
backward_passes_per_step=args.batches_per_pushpull,
half=True,
model=model,
fp16_params=model_params,
fp32_params=master_params,
loss_scale=args.loss_scale)
if args.resume:
checkpoint = torch.load(
args.resume,
map_location=lambda storage, loc: storage.cuda(args.local_rank))
model.load_state_dict(checkpoint['state_dict'])
args.start_epoch = checkpoint['epoch']
best_top5 = checkpoint['best_top5']
optimizer.load_state_dict(checkpoint['optimizer'])
log.console(
"Creating data loaders (this could take up to 10 minutes if volume needs to be warmed up)"
)
num_machines = (bps.size() - 1) // 8 + 1
assert (num_machines in schedules)
phases = schedules[num_machines]
dm = DataManager([copy.deepcopy(p) for p in phases if 'bs' in p])
scheduler = Scheduler(optimizer,
[copy.deepcopy(p) for p in phases if 'lr' in p])
# BytePS: broadcast parameters & optimizer state.
broadcast_parameters([(name, p.detach()) for name, p in bps_param],
root_rank=0)
broadcast_optimizer_state(optimizer, root_rank=0)
start_time = datetime.now() # Loading start to after everything is loaded
if args.evaluate:
return validate(dm.val_dl, model, criterion, 0, start_time)
if args.distributed:
log.console('Global Barrier: Syncing machines before training')
tensor = torch.tensor([1.0]).float().cuda()
barrier_handler = push_pull_async_inplace(tensor,
average=True,
name="init.barrier")
while True:
if poll(barrier_handler):
synchronize(barrier_handler)
break
# do broadcast for validate tensor
log.console('Broadcasting validate tensor')
barrier_handler = push_pull_async_inplace(validate_tensor,
average=True,
name="validation_tensor")
while True:
if poll(barrier_handler):
synchronize(barrier_handler)
break
barrier_handler = push_pull_async_inplace(
dist_validate_tensor,
average=True,
name="distributed_validation_tensor")
while True:
if poll(barrier_handler):
synchronize(barrier_handler)
break
log.event("~~epoch\thours\ttop1\ttop5\n")
for epoch in range(args.start_epoch, scheduler.tot_epochs):
dm.set_epoch(epoch)
train(dm.trn_dl, model, criterion, optimizer, scheduler, epoch)
top1, top5 = validate(dm.val_dl, model, criterion, epoch, start_time)
time_diff = (datetime.now() - start_time).total_seconds() / 3600.0
log.event(f'~~{epoch}\t{time_diff:.5f}\t\t{top1:.3f}\t\t{top5:.3f}\n')
is_best = top5 > best_top5
best_top5 = max(top5, best_top5)
if args.local_rank == 0:
if is_best:
save_checkpoint(epoch,
model,
best_top5,
optimizer,
is_best=True,
filename='model_best.pth.tar')
phase = dm.get_phase(epoch)
if phase:
save_checkpoint(
epoch,
model,
best_top5,
optimizer,
filename=f'sz{phase["bs"]}_checkpoint.path.tar')
def __init__(self,
last_conv_stride=1,
last_conv_dilation=1,
num_stripes=6,
local_conv_out_channels=256,
num_classes=0):
super(RelationModel, self).__init__()
self.base = resnet50(pretrained=True,
last_conv_stride=last_conv_stride,
last_conv_dilation=last_conv_dilation)
self.num_stripes = num_stripes
self.num_classes = num_classes
self.local_6_conv_list = nn.ModuleList()
self.local_4_conv_list = nn.ModuleList()
self.local_2_conv_list = nn.ModuleList()
self.rest_6_conv_list = nn.ModuleList()
self.rest_4_conv_list = nn.ModuleList()
self.rest_2_conv_list = nn.ModuleList()
self.relation_6_conv_list = nn.ModuleList()
self.relation_4_conv_list = nn.ModuleList()
self.relation_2_conv_list = nn.ModuleList()
self.global_6_max_conv_list = nn.ModuleList()
self.global_4_max_conv_list = nn.ModuleList()
self.global_2_max_conv_list = nn.ModuleList()
self.global_6_rest_conv_list = nn.ModuleList()
self.global_4_rest_conv_list = nn.ModuleList()
self.global_2_rest_conv_list = nn.ModuleList()
self.global_6_pooling_conv_list = nn.ModuleList()
self.global_4_pooling_conv_list = nn.ModuleList()
self.global_2_pooling_conv_list = nn.ModuleList()
for i in range(num_stripes):
self.local_6_conv_list.append(
nn.Sequential(nn.Conv2d(2048, local_conv_out_channels, 1),
nn.BatchNorm2d(local_conv_out_channels),
nn.ReLU(inplace=True)))
for i in range(4):
self.local_4_conv_list.append(
nn.Sequential(nn.Conv2d(2048, local_conv_out_channels, 1),
nn.BatchNorm2d(local_conv_out_channels),
nn.ReLU(inplace=True)))
for i in range(2):
self.local_2_conv_list.append(
nn.Sequential(nn.Conv2d(2048, local_conv_out_channels, 1),
nn.BatchNorm2d(local_conv_out_channels),
nn.ReLU(inplace=True)))
for i in range(num_stripes):
self.rest_6_conv_list.append(
nn.Sequential(nn.Conv2d(2048, local_conv_out_channels, 1),
nn.BatchNorm2d(local_conv_out_channels),
nn.ReLU(inplace=True)))
for i in range(4):
self.rest_4_conv_list.append(
nn.Sequential(nn.Conv2d(2048, local_conv_out_channels, 1),
nn.BatchNorm2d(local_conv_out_channels),
nn.ReLU(inplace=True)))
for i in range(2):
self.rest_2_conv_list.append(
nn.Sequential(nn.Conv2d(2048, local_conv_out_channels, 1),
nn.BatchNorm2d(local_conv_out_channels),
nn.ReLU(inplace=True)))
self.global_6_max_conv_list.append(
nn.Sequential(nn.Conv2d(2048, local_conv_out_channels, 1),
nn.BatchNorm2d(local_conv_out_channels),
nn.ReLU(inplace=True)))
self.global_4_max_conv_list.append(
nn.Sequential(nn.Conv2d(2048, local_conv_out_channels, 1),
nn.BatchNorm2d(local_conv_out_channels),
nn.ReLU(inplace=True)))
self.global_2_max_conv_list.append(
nn.Sequential(nn.Conv2d(2048, local_conv_out_channels, 1),
nn.BatchNorm2d(local_conv_out_channels),
nn.ReLU(inplace=True)))
self.global_6_rest_conv_list.append(
nn.Sequential(nn.Conv2d(2048, local_conv_out_channels, 1),
nn.BatchNorm2d(local_conv_out_channels),
nn.ReLU(inplace=True)))
self.global_4_rest_conv_list.append(
nn.Sequential(nn.Conv2d(2048, local_conv_out_channels, 1),
nn.BatchNorm2d(local_conv_out_channels),
nn.ReLU(inplace=True)))
self.global_2_rest_conv_list.append(
nn.Sequential(nn.Conv2d(2048, local_conv_out_channels, 1),
nn.BatchNorm2d(local_conv_out_channels),
nn.ReLU(inplace=True)))
for i in range(num_stripes):
self.relation_6_conv_list.append(
nn.Sequential(
nn.Conv2d(local_conv_out_channels * 2,
local_conv_out_channels, 1),
nn.BatchNorm2d(local_conv_out_channels),
nn.ReLU(inplace=True)))
for i in range(4):
self.relation_4_conv_list.append(
nn.Sequential(
nn.Conv2d(local_conv_out_channels * 2,
local_conv_out_channels, 1),
nn.BatchNorm2d(local_conv_out_channels),
nn.ReLU(inplace=True)))
for i in range(2):
self.relation_2_conv_list.append(
nn.Sequential(
nn.Conv2d(local_conv_out_channels * 2,
local_conv_out_channels, 1),
nn.BatchNorm2d(local_conv_out_channels),
nn.ReLU(inplace=True)))
self.global_6_pooling_conv_list.append(
nn.Sequential(
nn.Conv2d(local_conv_out_channels * 2, local_conv_out_channels,
1), nn.BatchNorm2d(local_conv_out_channels),
nn.ReLU(inplace=True)))
self.global_4_pooling_conv_list.append(
nn.Sequential(
nn.Conv2d(local_conv_out_channels * 2, local_conv_out_channels,
1), nn.BatchNorm2d(local_conv_out_channels),
nn.ReLU(inplace=True)))
self.global_2_pooling_conv_list.append(
nn.Sequential(
nn.Conv2d(local_conv_out_channels * 2, local_conv_out_channels,
1), nn.BatchNorm2d(local_conv_out_channels),
nn.ReLU(inplace=True)))
if num_classes > 0:
self.fc_local_6_list = nn.ModuleList()
for _ in range(num_stripes):
fc = nn.Linear(local_conv_out_channels, num_classes)
init.normal_(fc.weight, std=0.001)
init.constant_(fc.bias, 0)
self.fc_local_6_list.append(fc)
self.fc_local_4_list = nn.ModuleList()
for _ in range(4):
fc = nn.Linear(local_conv_out_channels, num_classes)
init.normal_(fc.weight, std=0.001)
init.constant_(fc.bias, 0)
self.fc_local_4_list.append(fc)
self.fc_local_2_list = nn.ModuleList()
for _ in range(2):
fc = nn.Linear(local_conv_out_channels, num_classes)
init.normal_(fc.weight, std=0.001)
init.constant_(fc.bias, 0)
self.fc_local_2_list.append(fc)
self.fc_rest_6_list = nn.ModuleList()
for _ in range(num_stripes):
fc = nn.Linear(local_conv_out_channels, num_classes)
init.normal_(fc.weight, std=0.001)
init.constant_(fc.bias, 0)
self.fc_rest_6_list.append(fc)
self.fc_rest_4_list = nn.ModuleList()
for _ in range(4):
fc = nn.Linear(local_conv_out_channels, num_classes)
init.normal_(fc.weight, std=0.001)
init.constant_(fc.bias, 0)
self.fc_rest_4_list.append(fc)
self.fc_rest_2_list = nn.ModuleList()
for _ in range(2):
fc = nn.Linear(local_conv_out_channels, num_classes)
init.normal_(fc.weight, std=0.001)
init.constant_(fc.bias, 0)
self.fc_rest_2_list.append(fc)
self.fc_local_rest_6_list = nn.ModuleList()
for _ in range(num_stripes):
fc = nn.Linear(local_conv_out_channels, num_classes)
init.normal_(fc.weight, std=0.001)
init.constant_(fc.bias, 0)
self.fc_local_rest_6_list.append(fc)
self.fc_local_rest_4_list = nn.ModuleList()
for _ in range(4):
fc = nn.Linear(local_conv_out_channels, num_classes)
init.normal_(fc.weight, std=0.001)
init.constant_(fc.bias, 0)
self.fc_local_rest_4_list.append(fc)
self.fc_local_rest_2_list = nn.ModuleList()
for _ in range(2):
fc = nn.Linear(local_conv_out_channels, num_classes)
init.normal_(fc.weight, std=0.001)
init.constant_(fc.bias, 0)
self.fc_local_rest_2_list.append(fc)
self.fc_global_6_list = nn.ModuleList()
fc = nn.Linear(local_conv_out_channels, num_classes)
init.normal_(fc.weight, std=0.001)
init.constant_(fc.bias, 0)
self.fc_global_6_list.append(fc)
self.fc_global_4_list = nn.ModuleList()
fc = nn.Linear(local_conv_out_channels, num_classes)
init.normal_(fc.weight, std=0.001)
init.constant_(fc.bias, 0)
self.fc_global_4_list.append(fc)
self.fc_global_2_list = nn.ModuleList()
fc = nn.Linear(local_conv_out_channels, num_classes)
init.normal_(fc.weight, std=0.001)
init.constant_(fc.bias, 0)
self.fc_global_2_list.append(fc)
self.fc_global_max_6_list = nn.ModuleList()
fc = nn.Linear(local_conv_out_channels, num_classes)
init.normal_(fc.weight, std=0.001)
init.constant_(fc.bias, 0)
self.fc_global_max_6_list.append(fc)
self.fc_global_max_4_list = nn.ModuleList()
fc = nn.Linear(local_conv_out_channels, num_classes)
init.normal_(fc.weight, std=0.001)
init.constant_(fc.bias, 0)
self.fc_global_max_4_list.append(fc)
self.fc_global_max_2_list = nn.ModuleList()
fc = nn.Linear(local_conv_out_channels, num_classes)
init.normal_(fc.weight, std=0.001)
init.constant_(fc.bias, 0)
self.fc_global_max_2_list.append(fc)
self.fc_global_rest_6_list = nn.ModuleList()
fc = nn.Linear(local_conv_out_channels, num_classes)
init.normal_(fc.weight, std=0.001)
init.constant_(fc.bias, 0)
self.fc_global_rest_6_list.append(fc)
self.fc_global_rest_4_list = nn.ModuleList()
fc = nn.Linear(local_conv_out_channels, num_classes)
init.normal_(fc.weight, std=0.001)
init.constant_(fc.bias, 0)
self.fc_global_rest_4_list.append(fc)
self.fc_global_rest_2_list = nn.ModuleList()
fc = nn.Linear(local_conv_out_channels, num_classes)
init.normal_(fc.weight, std=0.001)
init.constant_(fc.bias, 0)
self.fc_global_rest_2_list.append(fc)
def main():
global args, best_accu
args = parser.parse_args()
# Data loading code
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
transform_train = transforms.Compose([
transforms.Resize(size=(224, 224)),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(), normalize
])
transform_test = transforms.Compose(
[transforms.Resize(size=(224, 224)),
transforms.ToTensor(), normalize])
train_dataset = MultiLabelDataset(root='../dataset/',
label='../list/train_g.txt',
transform=transform_train)
val_dataset = MultiLabelDataset(root='../dataset/',
label='../list/test_g.txt',
transform=transform_test)
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=True,
num_workers=4,
pin_memory=False)
val_loader = torch.utils.data.DataLoader(val_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=4,
pin_memory=False)
# create model
model = resnet.resnet50(pretrained=True)
# get the number of model parameters
print('Number of model parameters: {}'.format(
sum([p.data.nelement() for p in model.parameters()])))
# for training on multiple GPUs.
# Use CUDA_VISIBLE_DEVICES=0,1 to specify which GPUs to use
# model = torch.nn.DataParallel(model).cuda()
model = model.cuda()
# optionally resume from a checkpoint
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
args.start_epoch = checkpoint['epoch']
best_accu = checkpoint['best_accu']
model.load_state_dict(checkpoint['state_dict'])
print("=> loaded checkpoint '{}' (epoch {})".format(
args.resume, checkpoint['epoch']))
else:
print("=> no checkpoint found at '{}'".format(args.resume))
cudnn.benchmark = True
# define loss function (criterion) and pptimizer
criterion = nn.BCELoss().cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# iterative training
for epoch in range(args.start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
# train for one epoch
train(train_loader, model, criterion, optimizer, epoch)
# evaluate on validation set
accu = validate(val_loader, model, criterion, epoch)
# remember best prec@1 and save checkpoint
is_best = accu > best_accu
best_accu = max(accu, best_accu)
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'best_accu': best_accu,
}, is_best)
print('Best accuracy: ', best_accu)
])
logger = logging.getLogger()
logging.info('ImageNet feature extraction')
logging.info('Args: %s', args)
hash_cmd = subprocess.run('git rev-parse --short HEAD',
shell=True,
check=True,
stdout=subprocess.PIPE)
git_hash = hash_cmd.stdout.decode('utf-8').strip()
logging.info(f'Git commit: {git_hash}')
# get model
if args.model == 'resnet50':
model = resnet50(pretrained=True).to(args.device)
else:
raise ValueError('Unkown model %s' % args.model)
if args.device == 'cuda':
model = DataParallel(model)
# get input transform
transform = transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
])
def extract_features(split='val'):
testloader = DataLoader(testset, batch_size=512, shuffle=False, num_workers=4)
# log
columns = ['iteration', 'time',
'train_loss', 'train_acc',
'test_loss', 'test_acc',
'epoch']
log = pd.DataFrame(columns=columns)
# model
from resnet import resnet50
# from resnet import resnet50
model = resnet50(num_classes=200).to('cuda')
model.train()
# optimizer
base_lr = .1
optimizer = optim.SGD(model.parameters(),
lr=base_lr,
momentum=.9,
weight_decay=5e-4)
criterion = nn.CrossEntropyLoss()
def adjust_learning_rate(optimizer, iteration):
"""Sets the learning rate to the initial LR decayed by 10 every 30 epochs"""
# with bs = 64, 1 epoch ~ 1500 sgd iterations
