def __init__(self, class_num, droprate=0.5, stride=2):
super(ft_net20, self).__init__()
self.add_module("module", resnet.resnet20())
weights_ = torch.load("weights_cifar10/resnet20-12fca82f.th")
self.load_state_dict(weights_['state_dict'])
self.module.linear = nn.Sequential()
self.classifier = ClassBlock(64, class_num, droprate)
def set_model(self,model_name=None):
if model_name is None:
model_name = self.FLAGS['model']['name']
if self.verbose: print(f'Setting model...{model_name}')
if model_name == 'resnet20':
self.model = resnet20(batch_size=self.FLAGS['train_params']['batch_size'],
l2_weight=self.FLAGS['weight_decay'],
activation_type=self.FLAGS['activation'],
certainty_variant=self.FLAGS['certainty_variant'],
model_variant=self.FLAGS['model_variant'],
logit_variant=self.FLAGS['logit_variant'])
elif model_name == 'resnet50':
self.model = resnet50v2(batch_size=self.FLAGS['train_params']['batch_size'],
l2_weight=self.FLAGS['weight_decay'],
activation_type=self.FLAGS['activation'],
certainty_variant=self.FLAGS['certainty_variant'],
model_variant=self.FLAGS['model_variant'],
logit_variant=self.FLAGS['logit_variant'])
# self.model = resnet50v2()
elif model_name == 'dummymodel':
self.model = dummymodel(batch_size=self.FLAGS['train_params']['batch_size'])
else:
raise ValueError(f'unknown model_name={model_name}')
def foramt_train(path="npy/", start=-1):
# 加载模型
model = resnet.resnet20()
load_model(model, "./save/save_0.1_resnet20/checkpoint.th")
#加载数据集
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
# 预训练LDA的数据集加载
train_loader = torch.utils.data.DataLoader(datasets.CIFAR10(
root='./data',
train=True,
transform=transforms.Compose([
transforms.ToTensor(),
normalize,
]),
download=True),
batch_size=1000,
shuffle=False,
pin_memory=False)
for batch, (X_input, y) in enumerate(train_loader):
if batch < start:
continue
print(">>>>>>saving for batch ", batch)
LDA_Xs = model.forward_all(X_input)
X_raw = tensor2np(X_input)
y = y.detach().numpy()
np.save(path + "{:0>2d}Y.npy".format(batch), y)
#np.save(path+"{:0>2d}X0.npy".format(batch), X_raw)
for layer, X in enumerate(LDA_Xs):
X2 = tensor2np(X)
name = "{:0>2d}X{}.npy".format(batch, layer)
np.save(path + name, X2)
del X_input, LDA_Xs, X_raw, X2, y
def main():
""" Main method. """
args = PARSER.parse_known_args()[0]
# sets up the backend for distributed training (optional)
device, local_rank = setup(distributed=args.distributed)
# retrieve the dataloaders for the chosen dataset
dataloaders, args = get_dataloaders(args)
# make dirs for current experiment logs, summaries etc
args = experiment_config(args)
# initialise the model
model = resnet.resnet20(args)
# place model onto GPU(s)
if args.distributed:
torch.cuda.set_device(device)
torch.set_num_threads(5) # n cpu threads / n processes per node
model = DistributedDataParallel(model.cuda(),
device_ids=[local_rank],
output_device=local_rank)
# only print stuff from process (rank) 0
args.print_progress = True if int(
os.environ.get('RANK')) == 0 else False
else:
if args.half_precision:
model.half() # convert to half precision
for layer in model.modules():
# keep batchnorm in 32 for convergence reasons
if isinstance(layer, nn.BatchNorm2d):
layer.float()
if torch.cuda.device_count() > 1:
model = nn.DataParallel(model)
print('\nUsing', torch.cuda.device_count(), 'GPU(s).\n')
model.to(device)
args.print_progress = True
if args.print_progress:
print_network(model, args) # prints out the network architecture etc
logging.info('\ntrain: {} - valid: {} - test: {}'.format(
len(dataloaders['train'].dataset),
len(dataloaders['valid'].dataset),
len(dataloaders['test'].dataset)))
# launch model training or inference
if not args.inference:
train(model, dataloaders, args)
if args.distributed: # cleanup
torch.distributed.destroy_process_group()
else:
model.load_state_dict(torch.load(args.load_checkpoint_dir))
test_loss, test_acc = evaluate(model, args, dataloaders['test'])
print('[Test] loss {:.4f} - acc {:.4f} - acc_topk {:.4f}'.format(
test_loss, test_acc[0], test_acc[1]))
def initial_model():
model = resnet20().to(device) #model
state_dict = torch.load("resnet20.th")['state_dict']
new_state_dict = OrderedDict()
for k, v in state_dict.items():
name = k[7:] # remove `module.`
#solve the problem of muti-gpus
new_state_dict[name] = v
model.load_state_dict(new_state_dict)
return model
def get_model(self):
if self.exp_type == 'pnml_cifar10':
model = load_pretrained_resnet20_cifar10_model(resnet20())
elif self.exp_type == 'random_labels':
model = WideResNet()
elif self.exp_type == 'out_of_dist_svhn':
model = load_pretrained_resnet20_cifar10_model(resnet20())
elif self.exp_type == 'out_of_dist_noise':
model = load_pretrained_resnet20_cifar10_model(resnet20())
elif self.exp_type == 'pnml_mnist':
model = Net()
elif self.exp_type == 'adversarial':
model = load_pretrained_resnet20_cifar10_model(resnet20())
elif self.exp_type == 'pnml_cifar10_lenet':
model = LeNet() # VGG('VGG16')
else:
raise NameError('No experiment type: %s' % self.exp_type)
return model
def create_model(num_classes=10):
# The number of channels in ResNet18 is divisible by 8.
# This is required for fast GEMM integer matrix multiplication.
# model = torchvision.models.resnet18(pretrained=False)
# model = resnet18(num_classes=num_classes, pretrained=False)
model = resnet20()
# We would use the pretrained ResNet18 as a feature extractor.
# for param in model.parameters():
# param.requires_grad = False
# Modify the last FC layer
# num_features = model.fc.in_features
# model.fc = nn.Linear(num_features, 10)
return model
def kd():
global args
args = parser.parse_args()
# Make dataset and loader
normalize = torchvision.transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_loader = torch.utils.data.DataLoader(torchvision.datasets.CIFAR10(
args.data_path,
train=True,
transform=torchvision.transforms.Compose([
torchvision.transforms.RandomHorizontalFlip(),
torchvision.transforms.RandomCrop(32, 4),
torchvision.transforms.ToTensor(), normalize
])),
batch_size=args.batch_size,
shuffle=True)
val_loader = torch.utils.data.DataLoader(torchvision.datasets.CIFAR10(
args.data_path,
train=False,
transform=torchvision.transforms.Compose(
[torchvision.transforms.ToTensor(), normalize])),
batch_size=args.test_batch_size)
# Load teacher (pretrained)
teacher = resnet56()
modify_properly(teacher, args.pretrained)
teacher.cuda()
# Make student
student = resnet20()
student.cuda()
t_embedding = TEmbedding().cuda()
s_embedding = SEmbedding().cuda()
criterion = {
'ce_loss': nn.CrossEntropyLoss().cuda(),
'ct_loss': ContrastiveLoss()
}
params = list(student.parameters()) + list(
t_embedding.parameters()) + list(s_embedding.parameters())
optimizer = torch.optim.Adam(params, lr=args.lr)
lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer,
milestones=[75, 150])
# Evaluate teacher
#if args.evaluate:
#validate(teacher, val_loader, criterion)
min_val_prec = 0.0
logger = {
'train/loss': [],
'train/accuracy': [],
'val/loss': [],
'val/accuracy': []
}
for epoch in range(args.num_epochs):
# training
tr_logger = train(train_loader, student, teacher, s_embedding,
t_embedding, criterion, optimizer, epoch)
# validating
val_logger = validate(val_loader, student, criterion)
logger['train/loss'].append(tr_logger['loss'].mean)
logger['train/accuracy'].append(tr_logger['prec'].mean)
logger['val/loss'].append(val_logger['loss'].mean)
logger['val/accuracy'].append(val_logger['prec'].mean)
lr_scheduler.step()
if min_val_prec < val_logger['prec'].mean:
min_val_prec = val_logger['prec'].mean
torch.save(student.state_dict(), 'ckpt/cwfd-resnet20-epochs' +
str(epoch) + '.pt') # TODO: add path variable
print("maximum of avg. val accuracy: {}".format(min_val_prec))
save_log(logger, 'logs/cwfd-resnet20.log')
def main():
global args, best_mAP
args = parser.parse_args()
useCuda = torch.cuda.is_available() and (args.gpu_id is not None or args.multiGPU)
torch.manual_seed(args.seed)
if useCuda:
torch.cuda.manual_seed(args.seed)
# Check the save_dir exists or not
if not os.path.exists(args.save_dir):
os.makedirs(args.save_dir)
# model = torch.nn.DataParallel(resnet.__dict__[args.arch]())
model = resnet.resnet20(useRRSVM=False)
# optionally resume from a checkpoint
if args.resume:
pretrained_file = os.path.join(default_model_directory, '{:s}.th'.format(args.arch))
if os.path.isfile(pretrained_file):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(pretrained_file)
# args.start_epoch = checkpoint['epoch'] if 'epoch' in checkpoint
best_prec1 = checkpoint['best_prec1'] if 'epoch' in checkpoint else 0
state_dict = {k.replace('module.', ''): v for k, v in checkpoint['state_dict'].items()}
model.load_state_dict(state_dict, strict=False)
print("=> loaded checkpoint {:s}".format(pretrained_file))
else:
print("=> no checkpoint found at '{}'".format(args.resume))
model = cuda_model.convertModel2Cuda(model, args.gpu_id, args.multiGpu)
# train_loader, val_loader = Cifar10.get_cifar10_datasets(args, train_portion=1.0) # using 10% of all data
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
#
train_loader = torch.utils.data.DataLoader(
datasets.CIFAR10(root=dataset_root, train=True, transform=transforms.Compose([
transforms.RandomHorizontalFlip(),
transforms.RandomCrop(32, 4),
transforms.ToTensor(),
normalize,
]), download=True),
batch_size=args.batch_size, shuffle=True,
num_workers=args.workers, pin_memory=True)
val_loader = torch.utils.data.DataLoader(
datasets.CIFAR10(root=dataset_root, train=False, transform=transforms.Compose([
transforms.ToTensor(),
normalize,
])),
batch_size=128, shuffle=False,
num_workers=args.workers, pin_memory=True)
#
# define loss function (criterion) and pptimizer
criterion = nn.CrossEntropyLoss().cuda()
# if args.half:
# model.half()
# criterion.half()
optimizer = torch.optim.SGD(model.parameters(), args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
args.start_epoch = 0
lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer,
milestones=[10, 20, 40])
# lr_scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer,
# milestones=[10, 20, 40], last_epoch=args.start_epoch - 1)
if args.arch in ['resnet1202', 'resnet110']:
# for resnet1202 original paper uses lr=0.01 for first 400 minibatches for warm-up
# then switch back. In this implementation it will correspond for first epoch.
for param_group in optimizer.param_groups:
param_group['lr'] = args.lr*0.1
if args.evaluate:
validate(val_loader, model, criterion)
return
is_best=False
for epoch in range(args.start_epoch, args.epochs):
# train for one epoch
lr_scheduler.step(epoch)
print('current lr {:.5e}'.format(optimizer.param_groups[0]['lr']))
train(train_loader, model, criterion, optimizer, epoch)
# evaluate on validation set
prec1 = validate(val_loader, model, criterion)
# remember best prec@1 and save checkpoint
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
if epoch > 0 and epoch % args.save_every == 0:
save_checkpoint({
'epoch': epoch + 1,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
}, is_best, filename=os.path.join(args.save_dir, 'checkpoint-{:04d}.th'.format(epoch)))
save_checkpoint({
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
}, is_best, filename=os.path.join(args.save_dir, 'checkpoint-{:04d}.th'.format(epoch)))
tot_test = len(indices_test)
TrainLoaderforExpert = torch.utils.data.DataLoader(
datasets.CIFAR10('./data', train=True, download=True,
transform=transform_train),
batch_size=args.batch_size, sampler = SubsetRandomSampler(indices_train))
TestLoaderforExpert = torch.utils.data.DataLoader(
datasets.CIFAR10('./data', train=False, transform=transform_test),
batch_size=args.test_batch_size, sampler = SubsetRandomSampler(indices_test))
# TestLoaderforExpert = torch.utils.data.DataLoader(
# datasets.CIFAR10('./data', train=False, transform=transform_train),
# batch_size=args.test_batch_size, shuffle=True)
model = resnet20()
if torch.cuda.is_available():
model = model.cuda()
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum,
weight_decay=5e-4, nesterov=True)
scheduler = StepLR(optimizer, step_size=35, gamma=0.1)
args.epochs = 1
val_scores = []
wt = "./weights/rr/random_injection_erasing/res22_retrained/rr_subset_" + str(sub) + ".pth.tar"
#wt = "./weights/router_resnet20_all_class.pth.tar"
#wt = "teacher_MLP_test_eresnet56_best_archi.pth.tar"
par = torch.load(wt)
model.load_state_dict(par)
print ("TEST SET performance on: {}".format(wt))
# # Pretrained 모델을 받아봅시다.
#
# 저는 구글링해보니 https://github.com/akamaster/pytorch_resnet_cifar10 에서 pretrained모델을 제공한다길래 받아봅니다.
# 우선 홈폴더에 ``git clone https://github.com/akamaster/pytorch_resnet_cifar10``
#
# 그리고 `resnet.py` 파일과 `pretrained_models`를 지금 경로로 옮겨옵니다.
#
# # 유의할점.
# - 1. 같은 preprocess 를 거치는가? Image Normalization 등.
# - 2. model.eval() 할 것. BatchNorm, Dropout 등의 특정 layer는 model.train() 과 model.eval() 모드에 따라 연산이 다름
# In[8]:
import resnet
resnet20 = torch.nn.DataParallel(resnet.resnet20())
# In[9]:
# resnet20 = torch.load('./resnet20.th') ## GPU 일때
checkpoint = torch.load('./pretrained_models/resnet20.th',
map_location=lambda storage, loc: storage) ## CPU 일때
resnet20.load_state_dict(checkpoint['state_dict'])
# # resnet20 pretrained 모델로 테스트를 돌려봅시다.
# In[10]:
from tqdm import tqdm
resnet20.eval()
correct = 0
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--batchSz', type=int, default=64)
parser.add_argument('--nEpochs', type=int, default=300)
parser.add_argument('--no-cuda', action='store_true')
parser.add_argument('--net')
parser.add_argument('--seed', type=int, default=1)
parser.add_argument('--opt',
type=str,
default='sgd',
choices=('sgd', 'adam', 'rmsprop'))
parser.add_argument('--gpu_id', type=str, default='0')
args = parser.parse_args()
args.cuda = not args.no_cuda and torch.cuda.is_available()
args.save = 'work/' + args.net
setproctitle.setproctitle(args.save)
torch.manual_seed(args.seed)
if args.cuda:
torch.cuda.manual_seed(args.seed)
if os.path.exists(args.save):
shutil.rmtree(args.save)
os.makedirs(args.save)
normMean = [0.49139968, 0.48215827, 0.44653124]
normStd = [0.24703233, 0.24348505, 0.26158768]
normTransform = transforms.Normalize(normMean, normStd)
trainTransform = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(), normTransform
])
testTransform = transforms.Compose([transforms.ToTensor(), normTransform])
kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {}
trainLoader = DataLoader(dset.CIFAR10(root='cifar',
train=True,
download=True,
transform=trainTransform),
batch_size=args.batchSz,
shuffle=True,
**kwargs)
testLoader = DataLoader(dset.CIFAR10(root='cifar',
train=False,
download=True,
transform=testTransform),
batch_size=args.batchSz,
shuffle=False,
**kwargs)
n_classes = 10
if args.net == 'resnet20':
net = resnet.resnet20(num_classes=n_classes)
elif args.net == 'resnet32':
net = resnet.resnet32(num_classes=n_classes)
elif args.net == 'resnet44':
net = resnet.resnet44(num_classes=n_classes)
elif args.net == 'resnet56':
net = resnet.resnet56(num_classes=n_classes)
elif args.net == 'resnet110':
net = resnet.resnet110(num_classes=n_classes)
elif args.net == 'resnetxt29':
net = resnetxt.resnetxt29(num_classes=n_classes)
elif args.net == 'deform_resnet32':
net = deformconvnet.deform_resnet32(num_classes=n_classes)
else:
net = densenet.DenseNet(growthRate=12,
depth=100,
reduction=0.5,
bottleneck=True,
nClasses=n_classes)
print(' + Number of params: {}'.format(
sum([p.data.nelement() for p in net.parameters()])))
if args.cuda:
net = net.cuda()
gpu_id = args.gpu_id
gpu_list = gpu_id.split(',')
gpus = [int(i) for i in gpu_list]
net = nn.DataParallel(net, device_ids=gpus)
if args.opt == 'sgd':
optimizer = optim.SGD(net.parameters(),
lr=1e-1,
momentum=0.9,
weight_decay=1e-4)
elif args.opt == 'adam':
optimizer = optim.Adam(net.parameters(), weight_decay=1e-4)
elif args.opt == 'rmsprop':
optimizer = optim.RMSprop(net.parameters(), weight_decay=1e-4)
trainF = open(os.path.join(args.save, 'train.csv'), 'w')
testF = open(os.path.join(args.save, 'test.csv'), 'w')
for epoch in range(1, args.nEpochs + 1):
adjust_opt(args.opt, optimizer, epoch)
train(args, epoch, net, trainLoader, optimizer, trainF)
test(args, epoch, net, testLoader, optimizer, testF)
torch.save(net, os.path.join(args.save, 'latest.pth'))
os.system('python plot.py {} &'.format(args.save))
trainF.close()
testF.close()
from resnet_imagenet import resnet18
from resnet import resnet20
# model = resnet18()
# print(model)
model = resnet20().cpu()
import copy
import torch
fused_model = copy.deepcopy(model)
# print(fused_model)
# fuse the layers in the frontend
fused_model = torch.quantization.fuse_modules(fused_model,
[["conv_1_3x3", "bn_1", "relu"]],
inplace=True)
# print(fused_model)
for module_name, module in fused_model.named_children():
if "stage" in module_name:
for basic_block_name, basic_block in module.named_children():
torch.quantization.fuse_modules(
basic_block,
[["conv_a", "bn_a", "relu_a"], ["conv_b", "bn_b"]],
inplace=True)
# for sub_block_name, sub_block in basic_block.named_children():
# if sub_block_name == "downsample":
# print(sub_block_name)
# print(fused_model)
test_accuracy = np.zeros((len(sensing_schemes), len(compression_factors)))
# Loop over sensing schemes and compression factors
for i, ss in enumerate(sensing_schemes):
for j, cf in enumerate(compression_factors):
# Define the data transformation for this network
sensing_transform = ss(cf, IM_DIM)
trans = transforms.Compose([transforms.ToTensor(), sensing_transform])
# Build the dataloaders
trainloader, valloader, testloader = get_dataloaders(
batch_size, val_split, trans, n_workers) # regular / proxy images
# (uncomment the line below if you want results for sparse recovered images)
# trainloader, valloader, testloader = get_sparse_recovered_dataloaders(sensing_transform, S, batch_size, val_split, n_workers)
# Construct the model
net = MNISTClassifier(resnet20(), lr, lr_milestones)
if torch.cuda.is_available():
trainer = pl.Trainer(gpus=2,
accelerator='ddp',
max_epochs=num_epochs,
progress_bar_refresh_rate=bar_refresh_rate)
else:
trainer = pl.Trainer(gpus=0,
max_epochs=num_epochs,
progress_bar_refresh_rate=bar_refresh_rate)
# Train the network
trainer.fit(net, trainloader, valloader)
# Test the network
trainer.test(model=net, test_dataloaders=testloader)
num_workers=16)
testset = torchvision.datasets.CIFAR10(root='./data',
train=False,
download=True,
transform=transforms.Compose([
transforms.ToTensor(),
normalize,
]))
testloader = torch.utils.data.DataLoader(testset,
batch_size=64,
shuffle=False,
num_workers=16)
model = resnet.resnet20(nclass=10, ds=args.ds)
model2 = resnet.resnet20(nclass=10, ds=args.ds)
if args.cuda:
model.cuda()
model = nn.DataParallel(model,
device_ids=range(torch.cuda.device_count()))
model2.cuda()
model2 = nn.DataParallel(model2,
device_ids=range(torch.cuda.device_count()))
#model = nn.DataParallel(model, device_ids=args.gpu)
print(model)
'''elif not args.combined:
pretrained_model = torch.load(args.pretrained)
best_acc = pretrained_model['acc']
def test():
router = resnet56()
#rweights = torch.load('./weights/router_resnet20_all_class.pth.tar')
#rweights = torch.load('./weights/suSan.pth.tar')
#rweights = torch.load('teacher_MLP_test_eresnet56_best_archi.pth.tar')
rweights = torch.load('./weights/resnet56_fmnist.pth.tar')
router.load_state_dict(rweights)
if torch.cuda.is_available():
router.cuda()
router.eval()
test_loss = 0
correct = 0
tt = 0
c = 0
for data, target in (test_loader):
if c == 50:
break
# c = c + 1
if args.cuda:
data, target = data.cuda(), target.cuda()
data, target = Variable(data, volatile=True), Variable(target)
output = router(data)
output = F.softmax(output)
rsoftmax = torch.sort(output, dim=1, descending=True)[0][0:,
0:args.topn]
pred = output.data.max(1, keepdim=True)[1]
tt += pred.eq(target.data.view_as(pred)).cpu().sum()
sortedsoftmax = torch.argsort(output, dim=1,
descending=True)[0:1, 0:args.topn]
sortedsoftmax = np.array(sortedsoftmax.cpu())
## reSet/call the predicitons
predictions = []
for i in SUBSET:
subset_flag[str(i)] = True
for i, pred in enumerate(sortedsoftmax):
for j in range(args.topn):
predictions.append(pred[j])
#print ("The top {} predictions of router: {}".format(args.topn, predictions))
rsm = []
for i, pred in enumerate(rsoftmax):
for j in range(args.topn):
rsm.append(pred[j])
sm = {}
#fout = torch.zeros([1,10])
for i, pred in enumerate(predictions):
sm[pred] = 0
tot = 0.0
expert = resnet20()
for sub in SUBSET:
if pred in sub and subset_flag[str(sub)] == True:
###### Load the saved weights for the experts #####
wt = "./weights/rr/random_injection_erasing/res20_fmnist/rr_subset_" + str(
sub) + ".pth.tar"
#wt = "./weights/latent_space_hardtraining/lp_subset_" + str(sub) + ".pth.tar"
wts = torch.load(wt)
expert.cuda()
expert.eval()
expert.load_state_dict(wts)
############################
### Inference part starts here ##########
output = F.softmax(expert(data))
#print (output)
output = torch.sort(output, dim=1,
descending=True)[0][0][0]
#print (pred, target, output)
sm[pred] += output.item() #* trust_factor(len(sub), 2)
tot += 1
subset_flag[str(sub)] = False
#sm[pred] += (rsm[i].item())
#if (tot != 0):
sm[pred] += (rsm[i].item() * 0.9)
#print (rsm[i].item())
# for pred in predictions:
# sm[pred] /= (tot)
ans = -0.99
prd = 0
# for p in predictions:
# print ("soft max for {} is {}".format(p, sm[p]))
# print ("the target value:", target)
for p in predictions:
if sm[p] >= ans:
ans = sm[p]
prd = p
if (prd == target.item()):
correct = correct + 1
# if (predictions[0] != prd):
# print ("The list of prediction: {} and the Target: {}".format(predictions, target))
# print ("The softmax score of expert prediction {} for {}".format(sm[2], prd))
# print ("The softmax score for acutally correct answer {}.".format(sm[target.item()]))
# print ("the softmax score by the router for correct answer {}.".format(rsm[2]))
# correct += pred.eq(target.data.view_as(pred)).cpu().sum()
test_loss /= len(test_loader.dataset)
print("Routers performance:", tt)
print(
'\nTest set: Average loss: {:.4f}, TOP 1 Accuracy: {}/{} ({:.4f}%)\n'.
format(test_loss, correct, len(test_loader.dataset),
100. * correct / len(test_loader.dataset)))
import numpy as np
from utils import plot_train_results
# Path to the checkpoint you want to use for testing
ckpt_path = "lightning_logs/version_10/checkpoints/epoch=19-step=4219.ckpt"
# Crude way of determining if we're on CIS machine or laptop
n_workers = 32 if torch.cuda.is_available() else 0
testset = torchvision.datasets.FashionMNIST(root="data",
train=False,
download=True,
transform=transforms.ToTensor())
testloader = torch.utils.data.DataLoader(testset,
batch_size=1000,
shuffle=False,
num_workers=n_workers)
ckpt = torch.load(ckpt_path, map_location=torch.device('cpu'))
net = MNISTClassifier(resnet20())
net.load_state_dict(ckpt['state_dict'])
if torch.cuda.is_available():
trainer = pl.Trainer(gpus=-1, accelerator='ddp')
else:
trainer = pl.Trainer(gpus=0)
trainer.test(model=net, test_dataloaders=testloader)
plot_train_results(net)
initial_learning_rate = BASE_LEARNING_RATE * BS_PER_GPU / 128
learning_rate = initial_learning_rate
for mult, start_epoch in LR_SCHEDULE:
if epoch >= start_epoch:
learning_rate = initial_learning_rate * mult
else:
break
tf.summary.scalar('learning rate', data=learning_rate, step=epoch)
return learning_rate
input_shape = (HEIGHT, WIDTH, NUM_CHANNELS)
img_input = tf.keras.layers.Input(shape=input_shape)
if NUM_GPUS == 1:
mood_resnet20 = resnet.resnet20(img_input=img_input, classes=NUM_CLASSES)
else:
mirrored_strategy = tf.distribute.MirroredStrategy()
with mirrored_strategy.scope():
mood_resnet20 = resnet.resnet20(img_input=img_input,
classes=NUM_CLASSES)
mood_resnet20.compile(optimizer=keras.optimizers.SGD(learning_rate=0.1,
momentum=0.9),
loss='sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'])
log_dir = "logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
file_writer = tf.summary.create_file_writer(log_dir + "/metrics")
file_writer.set_as_default()
def test():
router = resnet56()
#rweights = torch.load('./weights/router_resnet20_all_class.pth.tar')
#rweights = torch.load('./weights/suSan.pth.tar')
start_time = time.time()
#rweights = torch.load('./weights/best_so_far_res56.pth.tar')
rweights = torch.load('./weights/resnet56_fmnist.pth.tar')
router.load_state_dict(rweights)
if torch.cuda.is_available():
router.cuda()
router.eval()
test_loss = 0
correct = 0
tt = 0
c = 0
delta = []
for data, target in (test_loader):
# if c == 50:
# break
# c = c + 1
if (c % 20 == 0):
print(
"----- expert accuracy so far : {}/{}-----\n----- router accuracy so far : {}/{}-----"
.format(correct, c, tt, c))
print(
"The DELTA/improvement between router and expert: {}\n".format(
abs(correct - tt)))
if (c > 0):
print(
"Forcasting {:.2f}% (approx) accuracy at the end\n".format(
((10000.00 / c) * abs(tt - correct)) / 100.0 + 93.88))
c = c + 1
delta.append(abs(correct - tt))
if args.cuda:
data, target = data.cuda(), target.cuda()
data, target = Variable(data, volatile=True), Variable(target)
output = router(data)
output = F.softmax(output)
#print (output) #################### Remove this while running
rsoftmax = torch.sort(output, dim=1, descending=True)[0][0:,
0:args.topn]
pred = output.data.max(1, keepdim=True)[1]
pred2 = torch.argsort(output, dim=1, descending=True)[0][1]
tt += pred.eq(target.data.view_as(pred)).cpu().sum()
#tt += pred2.eq(target.data.view_as(pred)).cpu().sum()
sortedsoftmax = torch.argsort(output, dim=1,
descending=True)[0:1, 0:args.topn]
sortedsoftmax = np.array(sortedsoftmax.cpu())
## reSet/call the predicitons
predictions = []
for i in SUBSET:
subset_flag[str(i)] = True
for i, pred in enumerate(sortedsoftmax):
for j in range(args.topn):
predictions.append(pred[j])
#print ("The top {} predictions of router: {}".format(args.topn, predictions))
rsm = []
for i, pred in enumerate(rsoftmax):
for j in range(args.topn):
rsm.append(pred[j])
#sm = {}
fout = torch.zeros([1, 10], device='cuda') + (output * 0.7)
for i, pred in enumerate(predictions):
#sm[pred] = 0
tot = 0.0
expert = resnet20()
for sub in SUBSET:
if pred in sub and subset_flag[str(sub)] == True:
###### Load the saved weights for the experts #####
wt = "./weights/rr/random_injection_erasing/res20_fmnist/rr_subset_" + str(
sub) + ".pth.tar"
#wt = "./weights/latent_space_hardtraining/lp_subset_" + str(sub) + ".pth.tar"
wts = torch.load(wt)
expert.cuda()
expert.eval()
expert.load_state_dict(wts)
############################
### Inference part starts here ##########
output = F.softmax(expert(data))
#print (output)
#output = torch.sort(output, dim=1, descending=True)[0][0][0]
fout += output
#print (pred, target, output)
#sm[pred] += output.item() #* trust_factor(len(sub), 2)
tot += 1
subset_flag[str(sub)] = False
#fout = fout/tot
#print ("Fout:",fout)
prd = fout.data.max(1, keepdim=True)[1]
#
if (prd == target.item()):
correct = correct + 1
# correct += pred.eq(target.data.view_as(pred)).cpu().sum()
test_loss /= len(test_loader.dataset)
print("\nThe routers performance: {:4f}".format(
100.0 * (tt.data.item() / len(test_loader.dataset))))
print('EMNN (ours) accuracy: {:.4f}%)\n'.format(100. * correct /
len(test_loader.dataset)))
print("Total time taken {:.2f}.".format(time.time() - start_time))
delta = np.array(delta)
fl = "./inference_result/fmnist_delta_resnet56_[4_3].txt"
with open(fl, 'w') as f:
for item in delta:
f.write("%s\n" % item)
])
transforms_validate = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(cifar_norm_mean, cifar_norm_std)
])
trainset = datasets.CIFAR10(root='../data', train=True, download=True, transform=transforms_train) # 定义数据集
testset = datasets.CIFAR10(root='../data', train=False, download=False, transform=transforms_validate)
batch_size = args.batch
trainloader = DataLoader(trainset, batch_size=batch_size, shuffle=True,
pin_memory=True, drop_last=True, num_workers=2) # 定义数据集的loader
validateloader = DataLoader(testset, batch_size=batch_size, shuffle=True,
pin_memory=True, drop_last=True, num_workers=2)
# 定义模型
if torch.cuda.is_available():
cudnn.benchmark = True
resnet_model = resnet.resnet20()
# 恢复数据
start_epoch = 0
if args.resume:
checkpoint = torch.load(cp_dir)
start_epoch = checkpoint['epoch'] + 1
resnet_model.load_state_dict(checkpoint['model'])
print('resume model from '+cp_dir)
# 数据并行
dataParallel = False
if torch.cuda.device_count() > 1:
dataParallel = True
print('GPU amount: %d' % (torch.cuda.device_count()))
resnet_model = nn.DataParallel(resnet_model)
resnet_model = resnet_model.to(device)
initial_learning_rate = BASE_LEARNING_RATE * BS_PER_GPU / 128
learning_rate = initial_learning_rate
for mult, start_epoch in LR_SCHEDULE:
if epoch >= start_epoch:
learning_rate = initial_learning_rate * mult
else:
break
tf.summary.scalar('learning rate', data=learning_rate, step=epoch)
return learning_rate
input_shape = (HEIGHT, WIDTH, NUM_CHANNELS)
img_input = tf.keras.layers.Input(shape=input_shape)
if NUM_GPUS == 1:
model = resnet.resnet20(img_input=img_input, classes=NUM_CLASSES)
else:
mirrored_strategy = tf.distribute.MirroredStrategy()
with mirrored_strategy.scope():
model = resnet.resnet20(img_input=img_input, classes=NUM_CLASSES)
model.compile(optimizer=keras.optimizers.SGD(learning_rate=0.1, momentum=0.9),
loss='sparse_categorical_crossentropy',
metrics=['sparse_categorical_accuracy'])
log_dir = "logs/fit/" + datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
file_writer = tf.summary.create_file_writer(log_dir + "/metrics")
file_writer.set_as_default()
tensorboard_callback = TensorBoard(log_dir=log_dir,
update_freq='batch',
if __name__ == '__main__':
global args, best_prec1
best_prec1 = 0.
args = parser.parse_args()
save_fold_name = [
args.model,
str(args.depth), args.dataset,
'BS%d' % args.batch_size
]
if args.origin:
save_fold_name.insert(0, 'Origin')
if args.model == 'resnet':
if args.depth == 20:
network = resnet.resnet20()
if args.depth == 32:
network = resnet.resnet32()
if args.depth == 44:
network = resnet.resnet44()
if args.depth == 56:
network = resnet.resnet56()
if args.depth == 110:
network = resnet.resnet110()
if not args.origin:
print('Pruning the model in %s' % args.pruned_model_dir)
check_point = torch.load(args.pruned_model_dir + "model_best.pth.tar")
network.load_state_dict(check_point['state_dict'])
codebook_index_list = np.load(args.pruned_model_dir + "codebook.npy",
allow_pickle=True).tolist()
def draw_LDA():
# 超参数
# hyper parameters
# 加载模型
model = resnet.resnet20()
load_model(model, "./save/save_0.1_resnet20/checkpoint.th")
#加载数据集
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
val_loader = torch.utils.data.DataLoader(datasets.CIFAR10(
root='./data',
train=False,
transform=transforms.Compose([
transforms.ToTensor(),
normalize,
])),
batch_size=1000,
shuffle=True)
########################################
# 预训练LDA的数据集加载
#LDA预训练
LDAs = []
accs = []
for layer in range(7):
X2, y = load_layer_data(layer)
if (X2.shape[1] > 10000):
X2 = X2[:7000]
y = y[:7000]
LDA = discriminant_analysis.LinearDiscriminantAnalysis(n_components=3)
print("data shape", X2.shape, "label", y.shape, count_label(y))
LDA.fit(X2, y)
score = LDA.score(X2, y)
accs.append(score)
print(">>>>>>>layer", layer, "train acc:", score)
LDAs.append(LDA)
##############
print("==========LDA pretrain finished!========")
del X2
del y
################################################3
all_score = []
# 提取部分数据集
for batch, (X_input, y) in enumerate(val_loader):
batch_score = []
y = y.detach().numpy()
#预测
Xs = model.forward_all(X_input)
# 分类正确率
X_res = Xs[-1].detach().numpy()
print("resnet20 acc:", np.mean(np.argmax(X_res, axis=1) == y))
#分层画出结果
for layer, X in enumerate(Xs):
#print("===========layer:{}============".format(layer))
X = tensor2np(X)
#print(X.shape)
# 我们有了每层的数据 X 和 Y
vmodel = LDAs[layer]
X_ = vmodel.transform(X)
score = vmodel.score(X, y)
batch_score.append(score)
title = "LDA-{}".format(layer)
print(title, "[Accuracy:{:.5f}]".format(score))
if batch == 0:
y_ = y[not_outliers(X_)]
X_draw = X_[not_outliers(X_)]
fig = plt.figure()
plot_embedding_3D(X_draw,
y_,
savepath=SAVEDIR + title,
fig=fig,
sub="111")
fig = plt.figure()
plot_embedding_2D(X_draw[:, :2],
y_,
savepath=SAVEDIR + title,
fig=fig,
sub="111")
all_score.append(batch_score)
if batch >= 2:
break
print("==============final================")
print("train acc:\n", np.array(accs))
print("test acc:\n", np.array(all_score))
print("test acc mean\n", np.mean(all_score, axis=0))
def draw_TSNE():
# 超参数
# hyper parameters
#SAVEDIR = "vis13/"
# 加载模型
model = resnet.resnet20()
load_model(model, "./save/save_0.1_resnet20/checkpoint.th")
#加载数据集
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
val_loader = torch.utils.data.DataLoader(datasets.CIFAR10(
root='./data',
train=False,
transform=transforms.Compose([
transforms.ToTensor(),
normalize,
])),
batch_size=1000,
shuffle=True)
all_score = []
# 提取部分数据集
for batch, (X_input, y) in enumerate(val_loader):
batch_score = []
y = y.detach().numpy()
#预测
Xs = model.forward_all(X_input)
# 分类正确率
X_res = Xs[-1].detach().numpy()
print("resnet20 acc:", np.mean(np.argmax(X_res, axis=1) == y))
#分层画出结果
for layer, X in enumerate(Xs):
#print("===========layer:{}============".format(layer))
X = tensor2np(X)
#print(X.shape)
# 我们有了每层的数据 X 和 Y
vmodel = manifold.TSNE(n_components=3)
X_ = vmodel.fit_transform(X)
ARI, AMI = cluster_score(X_, y)
score = [ARI, AMI, vmodel.kl_divergence_]
batch_score.append(score)
title = "TSNE-{}".format(layer)
print(title, "ARI, AMI , vars :{}".format(score))
if batch == 0:
y_ = y[not_outliers(X_)]
X_draw = X_[not_outliers(X_)]
fig = plt.figure()
plot_embedding_3D(X_draw,
y_,
savepath=SAVEDIR + title,
fig=fig,
sub="111")
fig = plt.figure()
plot_embedding_2D(X_draw[:, :2],
y_,
savepath=SAVEDIR + title,
fig=fig,
sub="111")
all_score.append(batch_score)
if batch >= 2:
break
print("==============final================")
print("ARI, AMI, kl \n", np.array(all_score))
print("ARI, AMI, kl means\n", np.mean(all_score, axis=0))
def main(args):
#make sure cifar-10, cifar-100, STL-10 and Imagenet 32x32 downloaded in this path,
# otherwise change the download to True in load_dataset function
path = expanduser("~") + "/project/data" #the path of the data
train_loader, test_loader = load_dataset(args.label, args.batch_size, path, args.dataset, download=False)
cifarlike = (args.dataset=='cifar')
if args.model =="shake":
model = ShakeResNet(args.convBN, args.depth, args.factor, args.label, cifar=cifarlike)
elif args.model == "senet56":
model = se_resnet56(num_classes = args.label, reduction=8, convBN = args.convBN, cifar=cifarlike)
elif args.model == 'resnet20':
model = resnet20(num_classes = args.label, convBN = args.convBN, cifar=cifarlike)
elif args.model == 'resnet50':
model = resnet50(num_classes = args.label, convBN = args.convBN, cifar=cifarlike)
elif args.model == 'mobilenet':
if cifarlike and args.convBN.__name__=='conv3x3_bn':
print("Small image input or standard convolution is not supported in MobileNet!")
return
model = MobileNetV2(num_classes = args.label, convBN = args.convBN, large=False)
else:
print("Invalid model!")
return
model = torch.nn.DataParallel(model).cuda()
cudnn.benckmark = True
print('Number of model parameters: {}'.format(
sum([p.data.nelement() for p in model.parameters()])))
opt = optim.SGD(model.parameters(),
lr=args.lr,
momentum=0.9,
weight_decay=args.weight_decay,
nesterov=args.nesterov)
loss_func = nn.CrossEntropyLoss().cuda()
headers = ["Epoch", "LearningRate", "TrainLoss", "TestLoss", "TrainAcc.", "TestAcc."]
logger = utils.Logger(args.log_path, args.log_file, headers)
#let's train and test the model
for e in range(args.epochs):
lr = utils.cosine_lr(opt, args.lr, e, args.epochs)
model.train()
train_loss, train_acc, train_n = 0, 0, 0
for x, t in train_loader:
x, t = Variable(x.cuda()), Variable(t.cuda())
y = model(x)
loss = loss_func(y, t)
opt.zero_grad()
loss.backward()
opt.step()
train_acc += utils.accuracy(y, t).item()
train_loss += loss.item() * t.size(0)
train_n += t.size(0)
model.eval()
test_loss, test_acc, test_n = 0, 0, 0
for x, t in tqdm(test_loader, total=len(test_loader), leave=False):
with torch.no_grad():
x, t = Variable(x.cuda()), Variable(t.cuda())
y = model(x)
loss = loss_func(y, t)
test_loss += loss.item() * t.size(0)
test_acc += utils.accuracy(y, t).item()
test_n += t.size(0)
logger.write(e+1, lr, train_loss / train_n, test_loss / test_n,
train_acc / train_n * 100, test_acc / test_n * 100)
