def get_experiment_dataloaders(experiment_name: str,
data_dir: str = './data/'):
first_level = get_num_levels(experiment_name)[0]
config = get_experiment_config(experiment_name, first_level)
config = config_dict_to_namespace(config)
config.data_dir = os.path.join(data_dir, config.dataset)
torch.manual_seed(config.random_seed)
kwargs = {}
if not config.disable_cuda and torch.cuda.is_available():
use_gpu = True
torch.cuda.manual_seed_all(config.random_seed)
kwargs = {
'num_workers': config.num_workers,
'pin_memory': config.pin_memory
}
else:
use_gpu = False
data_dict = get_dataset(config.dataset, config.data_dir, 'test')
kwargs.update(data_dict)
config.num_classes = data_dict['num_classes']
test_loader = get_test_loader(batch_size=config.batch_size, **kwargs)
if 'cifar' in config.dataset:
valid_loader = test_loader
else:
valid_loader = get_test_loader(batch_size=config.batch_size, **kwargs)
if config.is_train:
data_dict = get_dataset(config.dataset, config.data_dir, 'train')
teachers = []
kwargs.update(data_dict)
train_loader = get_train_loader(batch_size=config.batch_size,
padding=config.padding,
padding_mode=config.padding_mode,
random_seed=config.random_seed,
shuffle=config.shuffle,
model_num=len(config.model_names),
teachers=teachers,
cuda=use_gpu,
**kwargs)
else:
train_loader = None
return train_loader, valid_loader, test_loader
def train(opt):
# Load models
print('----------- Network Initialization --------------')
student = select_model(dataset=opt.data_name,
model_name=opt.s_name,
pretrained=False,
pretrained_models_path=opt.s_model,
n_classes=opt.num_class).to(opt.device)
print('finished student model init...')
nets = {'snet': student}
# initialize optimizer
optimizer = torch.optim.SGD(student.parameters(),
lr=opt.lr,
momentum=opt.momentum,
weight_decay=opt.weight_decay,
nesterov=True)
# define loss functions
if opt.cuda:
criterionCls = nn.CrossEntropyLoss().cuda()
else:
criterionCls = nn.CrossEntropyLoss()
print('----------- DATA Initialization --------------')
train_loader = get_backdoor_loader(opt)
test_clean_loader, test_bad_loader = get_test_loader(opt)
print('----------- Train Initialization --------------')
for epoch in range(1, opt.epochs):
_adjust_learning_rate(optimizer, epoch, opt.lr)
# train every epoch
criterions = {'criterionCls': criterionCls}
train_step(opt, train_loader, nets, optimizer, criterions, epoch)
# evaluate on testing set
print('testing the models......')
acc_clean, acc_bad = test(opt, test_clean_loader, test_bad_loader,
nets, criterions, epoch)
# remember best precision and save checkpoint
if opt.save:
is_best = acc_bad[0] > opt.threshold_bad
opt.threshold_bad = min(acc_bad[0], opt.threshold_bad)
best_clean_acc = acc_clean[0]
best_bad_acc = acc_bad[0]
s_name = opt.s_name + '-S-model_best.pth'
save_checkpoint(
{
'epoch': epoch,
'state_dict': student.state_dict(),
'best_clean_acc': best_clean_acc,
'best_bad_acc': best_bad_acc,
'optimizer': optimizer.state_dict(),
}, is_best, opt.checkpoint_root, s_name)
def run(config):
kwargs = {}
if config.use_gpu:
# ensure reproducibility
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
torch.manual_seed(0)
np.random.seed(0)
kwargs = {'num_workers': config.num_workers}
# instantiate data loaders
if config.is_train:
data_loader = get_train_loader(config.data_dir,
config.batch_size,
is_shuffle=True**kwargs)
else:
data_loader = get_test_loader(config.data_dir,
config.batch_size,
is_shuffle=False,
**kwargs)
# instantiate trainer
trainer = Trainer(config, data_loader)
# either train
if config.is_train:
trainer.train()
# or load a pretrained model and test
else:
trainer.test()
def test(self):
# Load best model
model = SiameseNet()
_, _, _, model_state, _ = self.load_checkpoint(best=self.config.best)
model.load_state_dict(model_state)
if self.config.use_gpu:
model.cuda()
test_loader = get_test_loader(self.config.data_dir, self.config.way, self.config.test_trials,
self.config.seed, self.config.num_workers, self.config.pin_memory)
correct_sum = 0
num_test = test_loader.dataset.trials
print(f"[*] Test on {num_test} pairs.")
pbar = tqdm(enumerate(test_loader), total=num_test, desc="Test")
with torch.no_grad():
for i, (x1, x2, _) in pbar:
if self.config.use_gpu:
x1, x2 = x1.to(self.device), x2.to(self.device)
# compute log probabilities
out = model(x1, x2)
y_pred = torch.sigmoid(out)
y_pred = torch.argmax(y_pred)
if y_pred == 0:
correct_sum += 1
pbar.set_postfix_str(f"accuracy: {correct_sum / num_test}")
test_acc = (100. * correct_sum) / num_test
print(f"Test Acc: {correct_sum}/{num_test} ({test_acc:.2f}%)")
def main(config):
# ensure directories are setup
prepare_dirs(config)
# ensure reproducibility
torch.manual_seed(config.random_seed)
kwargs = {}
if torch.cuda.is_available():
torch.cuda.manual_seed(config.random_seed)
kwargs = {'num_workers': 4, 'pin_memory': False}
# instantiate data loaders
if config.is_train:
data_loader = get_train_valid_loader(config.data_dir, config.dataset,
config.batch_size,
config.random_seed, config.exp,
config.valid_size, config.shuffle,
**kwargs)
else:
data_loader = get_test_loader(config.data_dir, config.dataset,
config.batch_size, config.exp,
config.familiar, **kwargs)
# instantiate trainer
trainer = Trainer(config, data_loader)
if config.is_train:
trainer.train()
else:
if config.attack:
trainer.test_attack()
else:
trainer.test()
def main(config):
# ensure directories are setup
prepare_dirs(config)
# ensure reproducibility
torch.manual_seed(config.random_seed)
kwargs = {}
if config.use_gpu:
torch.cuda.manual_seed(config.random_seed)
kwargs = {'num_workers': 1, 'pin_memory': True}
# instantiate data loaders
if config.is_train:
data_loader = get_train_valid_loader(config.data_dir,
config.batch_size,
config.random_seed,
config.valid_size, config.shuffle,
config.show_sample, **kwargs)
else:
data_loader = get_test_loader(config.data_dir, config.batch_size,
**kwargs)
# instantiate trainer
trainer = Trainer(config, data_loader)
# either train
if config.is_train:
save_config(config)
trainer.train()
# or load a pretrained model and test
else:
trainer.test()
def train(model, early_stopping):
optimizer = optim.Adam(model.parameters(), lr=1e-3)
# data_iter = data_loader.get_loader(batch_size=args.batch_size)
data_iter = data_loader.get_train_loader(batch_size=args.batch_size)
for epoch in range(args.epochs):
model.train()
run_loss = 0.0
for idx, data in enumerate(data_iter):
data = utils.to_var(data)
ret = model.run_on_batch(data, optimizer, epoch)
run_loss += ret['loss'].item()
print('\r Progress epoch {}, {:.2f}%, average loss {}'.format(
epoch, (idx + 1) * 100.0 / len(data_iter),
run_loss / (idx + 1.0)))
test_data_iter = data_loader.get_test_loader(
batch_size=args.batch_size)
valid_loss = evaluate(model, test_data_iter)
# early stop
early_stopping(valid_loss, model)
if early_stopping.early_stop:
print("Early stopping")
break
def extract_feature(dir_path, net):
features = []
infos = []
# get test dataset
test_loader = get_test_loader(dir_path)
# change mode to test
net.eval()
for i, (images, info) in enumerate(test_loader):
if torch.cuda.is_available():
images = Variable(images, volatile=True).cuda()
else:
images = Variable(images, volatile=True)
feature, output = net(images)
feature = feature.cpu()
feature = feature.data.numpy()
feature = np.squeeze(feature)
features.append(feature)
# features.append(np.squeeze(feature))
person = int(np.squeeze(info[0].numpy()))
camera = int(np.squeeze(info[1].numpy()))
info = (person, camera)
infos.append(info)
features = np.asarray(features)
return features, infos
def main(config):
utils.prepare_dirs(config)
# ensure reproducibility
torch.manual_seed(config.random_seed)
kwargs = {}
if config.use_gpu:
torch.cuda.manual_seed(config.random_seed)
kwargs = {"num_workers": 1, "pin_memory": True}
# instantiate data loaders
if config.is_train:
dloader = data_loader.get_train_valid_loader(
config.data_dir,
config.batch_size,
config.random_seed,
config.valid_size,
config.shuffle,
config.show_sample,
**kwargs,
)
else:
dloader = data_loader.get_test_loader(
config.data_dir, config.batch_size, **kwargs,
)
trainer = Trainer(config, dloader)
# either train
if config.is_train:
utils.save_config(config)
trainer.train()
# or load a pretrained model and test
else:
trainer.test()
def main(config):
# ensure reproducibility
torch.manual_seed(config.random_seed)
kwargs = {}
if config.cuda:
torch.cuda.manual_seed(config.random_seed)
kwargs = {'num_workers': 1, 'pin_memory': True}
scores = []
# instantiate data loaders
count = 0
times = []
for i in [1, 2, 3]:
start = time.time()
count = i
train_data, test_data = load_dataset(config.data_dir, str(count))
# instantiate data loaders
data_loader = get_train_valid_loader(train_data, config.batch_size,
config.random_seed,
config.valid_size, config.shuffle,
config.show_sample, **kwargs)
test_loader = get_test_loader(test_data, config.batch_size, **kwargs)
# instantiate trainer
trainer = Trainer(config, count, data_loader, test_loader)
trainer.train()
result = trainer.test()
scores.append(result)
elapsed = time.time() - start
times.append(elapsed)
scores = np.array(scores)
times = np.array(times)
print('>>> scores:', scores)
print('aver time', times.mean())
# print('avg\tacc\tf1\tprec\trec\tauc')
print('acc:',
scores.mean(axis=0)[0], '\nf1',
scores.mean(axis=0)[1], '\nprec',
scores.mean(axis=0)[2], '\nrec',
scores.mean(axis=0)[3])
print('>>> std')
print('acc:',
scores.std(axis=0)[0], '\nf1',
scores.std(axis=0)[1], '\nprec',
scores.std(axis=0)[2], '\nrec',
scores.std(axis=0)[3])
def main():
args = get_args()
print('-' * 10, 'Args', '-' * 10)
print(args)
print('-' * 10, '----', '-' * 10)
root_dir = os.path.join(args.NAS_dir, 'data')
root_dir = os.path.join(root_dir, args.data_type)
model_name = os.path.basename(root_dir)+'_'+args.network+'_epochs:'+str(args.epochs)+\
'_slide_level-'+str(args.slide_level)+\
'_p_size-'+str(args.p_size)+'.md'
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
# print(device)
# sys.exit()
criterion = nn.CrossEntropyLoss()
if args.mode == 'train':
print('train mode start')
model_conv, _, _ = initialize_model(args.network, 2)
dataloaders, dataset_sizes = get_train_val_loader(
root_dir,
batch_size=args.batch_size,
slide_level=args.slide_level,
p_size=args.p_size)
model_conv = model_conv.to(device)
optimizer = optim.SGD(model_conv.parameters(),
lr=args.learning_rate,
momentum=0.9)
scheduler = lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)
model = train(model_conv,
dataloaders,
dataset_sizes,
device,
criterion,
optimizer,
scheduler,
num_epochs=args.epochs)
print('save model : {}'.format(os.path.join('model', model_name)))
save_model(model, 'model', model_name)
elif args.mode == 'test':
print('test mode start')
print('model name is:', model_name)
model_conv = torch.load(os.path.join('model', model_name))
dataloaders, dataset_sizes = get_test_loader(
root_dir, slide_level=args.slide_level, p_size=args.p_size)
test(model_conv, dataloaders, dataset_sizes, device, criterion)
def main(args):
# for fast training.
torch.backends.cudnn.benchmark = True
setup_seed(args.seed)
# create directories if not exist.
create_folder(args.save_root_dir, args.version, args.model_save_path)
create_folder(args.save_root_dir, args.version, args.sample_path)
create_folder(args.save_root_dir, args.version, args.log_path)
create_folder(args.save_root_dir, args.version, args.val_result_path)
create_folder(args.save_root_dir, args.version, args.test_result_path)
if args.mode == 'train':
loaders = Munch(ref=get_train_loader(root=args.train_img_dir,
img_size=args.image_size,
resize_size=args.resize_size,
batch_size=args.train_batch_size,
shuffle=args.shuffle,
num_workers=args.num_workers,
drop_last=args.drop_last),
val=get_test_loader(root=args.val_img_dir,
batch_size=args.val_batch_size,
shuffle=True,
num_workers=args.num_workers))
trainer = Trainer(loaders, args)
trainer.train()
elif args.mode == 'test':
loaders = Munch(tes=get_test_loader(root=args.test_img_dir,
img_size=args.test_img_size,
batch_size=args.val_batch_size,
shuffle=True,
num_workers=args.num_workers))
tester = Tester(loaders, args)
tester.test()
else:
raise NotImplementedError('Mode [{}] is not found'.format(args.mode))
def main(config):
# ensure directories are setup
prepare_dirs(config)
# ensure reproducibility
torch.manual_seed(config.random_seed)
kwargs = {}
if config.use_gpu:
torch.cuda.manual_seed(config.random_seed)
kwargs = {'num_workers': 1, 'pin_memory': True}
# instantiate data loaders
if config.is_train:
data_loader = get_train_valid_loader(
config.data_dir, config.batch_size,
config.random_seed, config.valid_size,
config.shuffle, config.show_sample, **kwargs
)
else:
data_loader = get_test_loader(
config.data_dir, config.batch_size, **kwargs
)
# for data, target in data_loader:
# print(data.size())
# print(target.size())
# break
#
# inputs, classes = next(iter(data_loader))
# print(data_loader)
# out = torchvision.utils.make_grid(inputs)
# class_names = np.arange(0,10)
# imshow(out, title=[class_names[x] for x in classes])
# instantiate trainer
trainer = Trainer(config, data_loader)
# either train
if config.is_train:
save_config(config)
trainer.train()
# or load a pretrained model and test
else:
trainer.test()
def __init__(self, arch, model_file, test_data, hyperparam, verbose):
################################################################
# cuda setting
################################################################
self.use_cuda = hyperparam['use_cuda'] and torch.cuda.is_available()
self.device = "cuda" if self.use_cuda else "cpu"
################################################################
# inference model and test loader
################################################################
self.inference_model = self.load_model(arch, model_file, hyperparam,
verbose)
self.test_loader = get_test_loader(
ibm_data=test_data,
to_tensor=hyperparam['to_tensor'],
batch_size=hyperparam['valid_batch_size'],
mean_for_normalize=hyperparam['train_mean'],
std_for_normalize=hyperparam['train_std'],
use_cuda=self.use_cuda,
shuffle=False)
def ensemble(model_name, file_name, tta=False):
preds_raw = []
model = create_model(model_name)
test_set = data_loader.get_test_set()
rounds = 1
if tta:
rounds = 20
loader = data_loader.get_test_loader(model, test_set, tta=True)
for index in range(rounds):
predictions = np.array(make_preds(model, loader))
preds_raw.append(predictions)
preds = np.mean(preds_raw, axis=0)
save_array(settings.PREDICT_DIR + os.sep + file_name, preds)
def main(config):
# ensure directories are setup
prepare_dirs(config)
# ensure reproducibility
#torch.manual_seed(config.random_seed)
kwargs = {}
if config.use_gpu:
#torch.cuda.manual_seed_all(config.random_seed)
kwargs = {
'num_workers': config.num_workers,
'pin_memory': config.pin_memory
}
#torch.backends.cudnn.deterministic = True
# instantiate data loaders
test_data_loader = get_test_loader(config.data_dir, config.batch_size,
**kwargs)
if config.is_train:
train_data_loader = get_train_loader(config.data_dir,
config.batch_size,
config.random_seed,
config.shuffle, **kwargs)
data_loader = (train_data_loader, test_data_loader)
else:
data_loader = test_data_loader
# instantiate trainer
trainer = Trainer(config, data_loader)
# either train
if config.is_train:
save_config(config)
trainer.train()
# or load a pretrained model and test
else:
trainer.test()
def main():
wandb.init("AVA")
config, unparsed = get_config()
# ensure reproducibility
torch.manual_seed(config.random_seed)
if config.use_gpu:
torch.cuda.manual_seed(config.random_seed)
# instantiate data loaders
if config.is_train:
data_loader = get_train_valid_loader(task='MNIST',
batch_size=config.batch_size,
random_seed=config.random_seed,
valid_size=config.valid_size)
else:
data_loader = get_test_loader(task='MNIST',
batch_size=config.batch_size)
wandb.config.update(config)
# instantiate trainer
trainer = Trainer(config, data_loader)
# either train
trainer.train()
def main(config):
# ensure reproducibility
torch.manual_seed(config.random_seed)
scores = []
# instantiate data loaders
count = 0
times = []
for i in range(1, 4):
start = time.time()
count = i
train_data, test_data = load_dataset(config.data_dir, str(count))
# instantiate data loaders
data_loader = get_train_loader(train_data, config.batch_size,
config.random_seed, config.shuffle)
test_loader = get_test_loader(test_data, config.batch_size)
# instantiate trainer
trainer = Trainer(config, count, data_loader, test_loader)
trainer.train()
result = trainer.test()
scores.append(result)
elapsed = time.time() - start
times.append(elapsed)
scores = np.array(scores)
times = np.array(times)
print('aver time', times.mean())
# print('avg\tacc\tf1\tprec\trec\tauc')
print('acc:',
scores.mean(axis=0)[0], '\nf1',
scores.mean(axis=0)[1], '\nprec',
scores.mean(axis=0)[2], '\nrec',
scores.mean(axis=0)[3])
def eval_(model_):
test_loader = data_loader.get_test_loader(data_dir='./data',
batch_size=16,
num_workers=4,
pin_memory=False)
net = model_
net = net.cuda()
optimizer, criterion = optimizer_(net)
correct = 0
total = 0
with torch.no_grad():
for data in test_loader:
images, labels = data
images, labels = images.cuda(), labels.cuda()
outputs = net(images)
_, predicted = torch.max(outputs.data, 1)
total += labels.size(0)
correct += (predicted == labels).sum().item()
print('Accuracy of the network on the 10000 test images: %d %%' %
(100 * correct / total))
def init_loaders(train_batch_size, test_batch_size):
import json
import h5py
from gensim.models.keyedvectors import KeyedVectors
with open('./data/datainfo-v1.1.json', 'r') as f:
data = json.load(f)
f = h5py.File('./data/resnet_features.hdf5', 'r')
img_features = f['resnet152_features'][()]
f.close()
wordvectors_file_vec = './data/fasttext-sbwc.vec'
# count = 1000
wordvectors = KeyedVectors.load_word2vec_format(
wordvectors_file_vec) #, limit=count)
train_loader = get_train_loader(wordvectors, data, img_features,
train_batch_size)
test_loader = get_test_loader(wordvectors, data, img_features,
test_batch_size)
return train_loader, test_loader
def main(**kwargs):
global args
lowest_error1 = 100
for arg, v in kwargs.items():
args.__setattr__(arg, v)
program_start_time = time.time()
instanceName = "Classification_Accuracy"
folder_path = os.path.dirname(
os.path.abspath(__file__)) + os.sep + args.model
timestamp = datetime.datetime.now()
ts_str = timestamp.strftime('%Y-%m-%d-%H-%M-%S')
path = folder_path + os.sep + instanceName + os.sep + args.model_name + os.sep + ts_str
tensorboard_folder = path + os.sep + "Graph"
os.makedirs(path)
args.savedir = path
writer = SummaryWriter(tensorboard_folder)
global logFile
logFile = path + os.sep + "log.txt"
args.filename = logFile
global num_outputs
print(args)
global device
device = 'cuda' if torch.cuda.is_available() else 'cpu'
if args.data == "cifar100" or args.data == "CIFAR100":
fig_title_str = " on CIFAR-100"
elif args.data == "cifar10" or args.data == "CIFAR10":
fig_title_str = " on CIFAR-10"
elif args.data == "tiny_imagenet":
fig_title_str = " on tiny_imagenet"
else:
LOG(
"ERROR =============================dataset should be CIFAR10 or CIFAR100",
logFile)
NotImplementedError
captionStrDict = {
"fig_title": fig_title_str,
"x_label": "epoch",
'elastic_final_layer_label': "Final_Layer_Output_Classifier",
"elastic_intermediate_layer_label": "Intermediate_Layer_Classifier_"
}
# save input parameters into log file
LOG("program start time: " + ts_str + "\n", logFile)
# if args.layers_weight_change == 1:
# LOG("weights for intermediate layers: 1/(34-Depth), giving different weights for different intermediate layers output, using the formula weigh = 1/(34-Depth)", logFile)
# elif args.layers_weight_change == 0:
# LOG("weights for intermediate layers: 1, giving same weights for different intermediate layers output as 1", logFile)
# else:
# print("Parameter --layers_weight_change, Error")
# sys.exit()
if args.model == "Elastic_ResNet18" or args.model == "Elastic_ResNet34" or args.model == "Elastic_ResNet50" or args.model == "Elastic_ResNet101" or args.model == "Elastic_ResNet152":
model = Elastic_ResNet(args, logFile)
elif args.model == "Elastic_InceptionV3":
args.target_size = (
299, 299, 3
) # since pytorch inceptionv3 pretrained accepts image size (299, 299, 3) instead of (224, 224, 3)
model = Elastic_InceptionV3(args, logFile)
elif args.model == "Elastic_MobileNet":
model = Elastic_MobileNet(args, logFile)
elif args.model == "Elastic_VGG16":
model = Elastic_VGG16_bn(args, logFile)
elif args.model == "Elastic_SqueezeNet":
model = Elastic_SqueezeNet(args, logFile)
elif args.model == "Elastic_DenseNet121" or args.model == "Elastic_DenseNet169" or args.model == "Elastic_DenseNet201":
model = Elastic_DenseNet(args, logFile)
else:
LOG(
"--model parameter should be in ResNet, InceptionV3, MobileNet, VGG16, SqueezeNet, DenseNet",
logFile)
exit()
num_outputs = model.num_outputs
# num_outputs = 1
LOG("num_outputs: " + str(num_outputs), logFile)
LOG("successfully create model: " + args.model, logFile)
args_str = str(args)
LOG(args_str, logFile)
model = model.to(device)
if device == 'cuda':
model = torch.nn.DataParallel(model).cuda()
cudnn.benchmark = True
# TUT thinkstation data folder path
data_folder = "/media/yi/e7036176-287c-4b18-9609-9811b8e33769/tiny_imagenet/tiny-imagenet-200"
# narvi data folder path
# data_folder = "/home/zhouy/data/tiny-imagenet-200"
# XPS 15 laptop data folder path
# data_folder = "D:\Elastic\data"
# args.batch_size = 1
summary(model, (3, 224, 224))
if args.data == "tiny_imagenet":
train_loader, test_loader = tiny_image_data_loader(data_folder, args)
else:
train_loader = get_train_loader(args.data,
data_dir=data_folder,
batch_size=args.batch_size,
augment=False,
target_size=args.target_size,
random_seed=20180614,
valid_size=0.2,
shuffle=True,
show_sample=False,
num_workers=4,
pin_memory=True,
debug=args.debug)
test_loader = get_test_loader(args.data,
data_dir=data_folder,
batch_size=args.batch_size,
shuffle=True,
target_size=args.target_size,
num_workers=4,
pin_memory=True,
debug=args.debug)
criterion = nn.CrossEntropyLoss().cuda()
if args.data != "tiny_imagenet":
pretrain_optimizer = torch.optim.SGD(filter(lambda p: p.requires_grad,
model.parameters()),
args.pretrain_learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
LOG("==> Pretraining for **1** epoches \n", logFile)
for pretrain_epoch in range(0, 1):
accs, losses, lr = train(train_loader, model, criterion,
pretrain_optimizer, pretrain_epoch)
epoch_result = " pretrain epoch: " + str(
pretrain_epoch) + ", pretrain error: " + str(
accs) + ", pretrain loss: " + str(
losses) + ", pretrain learning rate: " + str(
lr) + ", pretrain total train sum loss: " + str(
sum(losses))
LOG(epoch_result, logFile)
summary(model, (3, 224, 224))
LOG("==> Full training \n", logFile)
for param in model.parameters():
param.requires_grad = True
optimizers = []
childs = []
k = 0
for child in model.parameters():
childs.append(child)
k += 1
# childs_params = [childs[:9], childs[:15], childs[:21], childs[:27],
# childs[:33], childs[:39], childs[:45], childs[:51],
# childs[:57], childs[:63], childs[:69], childs[:75], childs]
childs_params = [childs[:25], childs[:43], childs[:61], childs]
for i in range(num_outputs):
optimizer = torch.optim.SGD(childs_params[i],
args.learning_rate,
momentum=args.momentum,
weight_decay=args.weight_decay)
optimizers.append(optimizer)
# optimizer = torch.optim.Adam(model.parameters(), lr=args.learning_rate, betas=(0.9, 0.999), eps=1e-08, weight_decay=args.weight_decay)
# summary(model, (3,224,224))
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
mode='min',
threshold=1e-4,
patience=10)
# implement early stop by own
EarlyStopping_epoch_count = 0
epochs_train_accs = []
epochs_train_top5_accs = []
epochs_train_losses = []
epochs_test_accs = []
epochs_test_losses = []
epochs_lr = []
epochs_test_top5_accs = []
for epoch in range(0, args.epochs):
epoch_str = "==================================== epoch %d ==============================" % epoch
LOG(epoch_str, logFile)
# Train for one epoch
accs, losses, lr, accs_top5 = train(train_loader, model, criterion,
optimizers, epoch)
epochs_train_accs.append(accs)
epochs_train_losses.append(losses)
epochs_lr.append(lr)
epochs_train_top5_accs.append(accs_top5)
writer.add_scalar(tensorboard_folder + os.sep + "data" + os.sep + 'lr',
lr, epoch)
for i, a, l, k in zip(range(len(accs)), accs, losses, accs_top5):
writer.add_scalar(
tensorboard_folder + os.sep + "data" + os.sep +
'train_error_' + str(i), a, epoch)
writer.add_scalar(
tensorboard_folder + os.sep + "data" + os.sep +
'train_losses_' + str(i), l, epoch)
writer.add_scalar(
tensorboard_folder + os.sep + "data" + os.sep +
'train_top5_error_' + str(i), k, epoch)
epoch_result = "\ntrain error: " + str(accs) + "top 5 error: " + str(
accs_top5) + ", \nloss: " + str(
losses) + ", \nlearning rate " + str(
lr) + ", \ntotal train sum loss " + str(sum(losses))
LOG(epoch_result, logFile)
if num_outputs > 1:
writer.add_scalar(
tensorboard_folder + os.sep + "data" + os.sep +
'train_total_sum_losses', sum(losses), epoch)
losses.append(sum(losses)) # add the total sum loss
LOG("train_total_sum_losses: " + str(sum(losses)), logFile)
# run on test dataset
LOG("==> test \n", logFile)
test_accs, test_losses, test_top5_accs = validate(
test_loader, model, criterion)
epochs_test_accs.append(test_accs)
epochs_test_losses.append(test_losses)
epochs_test_top5_accs.append(test_top5_accs)
for i, a, l, k in zip(range(len(test_accs)), test_accs, test_losses,
test_top5_accs):
writer.add_scalar(
tensorboard_folder + os.sep + "data" + os.sep + 'test_error_' +
str(i), a, epoch)
writer.add_scalar(
tensorboard_folder + os.sep + "data" + os.sep +
'test_losses_' + str(i), l, epoch)
writer.add_scalar(
tensorboard_folder + os.sep + "data" + os.sep +
'test_top5_losses_' + str(i), k, epoch)
test_result_str = "==> Test epoch: \nfinal output classifier error: " + str(
test_accs
) + "test top 5 error: " + str(test_top5_accs) + ", \ntest_loss" + str(
test_losses) + ", \ntotal test sum loss " + str(sum(test_losses))
LOG(test_result_str, logFile)
total_loss = sum(test_losses)
if num_outputs > 1:
writer.add_scalar(
tensorboard_folder + os.sep + "data" + os.sep +
'test_total_sum_losses', total_loss, epoch)
test_losses.append(total_loss) # add the total sum loss
LOG("test_total_sum_losses: " + str(total_loss), logFile)
log_stats(path, accs, losses, lr, test_accs, test_losses, accs_top5,
test_top5_accs)
# Remember best prec@1 and save checkpoint
is_best = test_accs[
-1] < lowest_error1 #error not accuracy, but i don't want to change variable names
if is_best:
lowest_error1 = test_accs[-1] #但是有个问题，有时是倒数第二个CLF取得更好的结果
save_checkpoint(
{
'epoch': epoch,
'model': args.model_name,
'state_dict': model.state_dict(),
'best_prec1': lowest_error1,
'optimizer': optimizer.state_dict(),
}, args)
# apply early_stop with monitoring val_loss
# EarlyStopping(patience=15, score_function=score_function(val_loss), trainer=model)
scheduler.step(total_loss) # adjust learning rate with test_loss
if epoch == 0:
prev_epoch_loss = total_loss # use all intemediate classifiers sum loss instead of only one classifier loss
else:
if total_loss >= prev_epoch_loss: # means this current epoch doesn't reduce test losses
EarlyStopping_epoch_count += 1
if EarlyStopping_epoch_count > 20:
LOG(
"No improving test_loss for more than 10 epochs, stop running model",
logFile)
break
# n_flops, n_params = measure_model(model, IMAGE_SIZE, IMAGE_SIZE)
# FLOPS_result = 'Finished training! FLOPs: %.2fM, Params: %.2fM' % (n_flops / 1e6, n_params / 1e6)
# LOG(FLOPS_result, logFile)
# print(FLOPS_result)
writer.close()
end_timestamp = datetime.datetime.now()
end_ts_str = end_timestamp.strftime('%Y-%m-%d-%H-%M-%S')
LOG("program end time: " + end_ts_str + "\n", logFile)
# here plot figures
plot_figs(epochs_train_accs, epochs_train_losses, epochs_test_accs,
epochs_test_losses, args, captionStrDict)
LOG("============Finish============", logFile)
def test(model, savepath):
model.load_state_dict(torch.load(savepath))
test_data_iter = data_loader.get_test_loader(batch_size=args.batch_size)
valid_loss = evaluate(model, test_data_iter)
args.span_range_height = args.span_range_width = args.span_range
args.grid_height = args.grid_width = args.grid_size
args.image_height = args.image_width = 28
torch.manual_seed(args.seed)
if args.cuda:
torch.cuda.manual_seed(args.seed)
model = mnist_model.get_model(args)
if args.cuda:
model.cuda()
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum)
train_loader = data_loader.get_train_loader(args)
test_loader = data_loader.get_test_loader(args)
def train(epoch):
model.train()
for batch_idx, (data, target) in enumerate(train_loader):
if args.cuda:
data, target = data.cuda(), target.cuda()
# print(data.shape)
data, target = Variable(data), Variable(target)
optimizer.zero_grad()
output = model(data)
loss = F.nll_loss(output, target)
loss.backward()
optimizer.step()
if batch_idx % args.log_interval == 0:
if args.is_train:
train_dataset, val_dataset = get_MNIST_train_val_dataset(args.data_dir)
# train_dataset, val_dataset = get_MNIST_train_val_dataset('./data/MNIST')
train_loader, val_loader = get_train_val_loader(
train_dataset,
val_dataset,
val_split=args.val_split,
random_split=args.random_split,
batch_size=args.batch_size,
**kwargs)
args.num_class = train_loader.dataset.num_class
args.num_channels = train_loader.dataset.num_channels
else:
test_dataset = get_MNIST_test_dataset(args.data_dir)
test_loader = get_test_loader(test_dataset, args.batch_size, **kwargs)
args.num_class = test_loader.dataset.num_class
args.num_channels = test_loader.dataset.num_channels
# build RAM model
model = RecurrentAttention(args)
if args.use_gpu:
model.cuda()
optimizer = torch.optim.SGD(model.parameters(),
lr=args.init_lr,
momentum=args.momentum)
logger.info('Number of model parameters: {:,}'.format(
sum([p.data.nelement() for p in model.parameters()])))
trainer = Trainer(model, optimizer, watch=['acc'], val_watch=['acc'])
# Initialize block_probs to be an even distribution(1/n)
block_probs = [[] for i in range(n_blocks)]
for i in range(n_blocks):
block_probs[i] = [(1/len(blocks[i])) for j in range(len(blocks[i]))]
# Initialize block_weights to all start at 0(used to be 1/n)
block_weights = [[] for i in range(n_blocks)]
for i in range(n_blocks):
block_weights[i] = [0 for j in range(len(blocks[i]))]
train_loader, valid_loader = custom_dl.get_train_valid_loader(data_dir=root+'/data/cifar10/',
batch_size=16,
augment=False,
random_seed=1)
test_loader = custom_dl.get_test_loader(data_dir=root+'/data/cifar10/', batch_size=16)
def save_grad(module, gradInput, gradOutput):
module.block_grad = gradInput[0]
def save_forward(module, forwardInput, output):
module.block_forward = output[0]
def train_layer(data, i, blocks):
net_blocks = []
# For each block layer, chose a random block to train
for block_layer in blocks:
block_i = np.random.choice(len(block_layer))
block = block_layer[block_i]
net_blocks.append(block)
net = ProxylessNasNet(first_conv, net_blocks, feature_mix_layer, classifier, ngpu=ngpu).to(device)
def main(config):
# ensure directories are setup
prepare_dirs(config)
# create Omniglot data loaders
torch.manual_seed(config.random_seed)
kwargs = {}
if config.use_gpu:
torch.cuda.manual_seed(config.random_seed)
kwargs = {'num_workers': 1, 'pin_memory': True}
if config.is_train:
data_loader = get_train_valid_loader(
config.data_dir, config.batch_size,
config.num_train, config.augment,
config.way, config.valid_trials,
config.shuffle, config.random_seed,
**kwargs
)
else:
data_loader = get_test_loader(
config.data_dir, config.way,
config.test_trials, config.random_seed,
**kwargs
)
# sample 3 layer wise hyperparams if first time training
if config.is_train and not config.resume:
print("[*] Sampling layer hyperparameters.")
layer_hyperparams = {
'layer_init_lrs': [],
'layer_end_momentums': [],
'layer_l2_regs': []
}
for i in range(6):
# sample
lr = random.uniform(1e-4, 1e-1)
mom = random.uniform(0, 1)
reg = random.uniform(0, 0.1)
# store
layer_hyperparams['layer_init_lrs'].append(lr)
layer_hyperparams['layer_end_momentums'].append(mom)
layer_hyperparams['layer_l2_regs'].append(reg)
try:
save_config(config, layer_hyperparams)
except ValueError:
print(
"[!] Samples already exist. Either change the model number,",
"or delete the json file and rerun.",
sep=' '
)
return
# else load it from config file
else:
try:
layer_hyperparams = load_config(config)
except FileNotFoundError:
print("[!] No previously saved config. Set resume to False.")
return
trainer = Trainer(config, data_loader, layer_hyperparams)
if config.is_train:
trainer.train()
else:
trainer.test()
img_var = Variable(img)
img_var = img_var.cuda() # send to GPU
op = model(img_var)
result = op.data[0].cpu().numpy()
value = np.amax(result)
index = np.argmax(result)
print('The image is: ', labels[index])
#%% TESTING ON LANDMASS-2 IMAGES:
landmass2_dir = '/home/yazeed/Documents/datasets/landmass-2/'
test_loader = get_test_loader(landmass2_dir,
batch_size,
shuffle=False,
num_workers=n_threads,
pin_memory=pin_memory)
# this only needs to be created once -- then reused:
target_onehot = torch.FloatTensor(test_loader.batch_size,len(test_loader.dataset.classes)).zero_()
if use_gpu:
target_onehot = target_onehot.cuda()
for batch_idx, (data, target) in enumerate(test_loader):
if use_gpu:
data, target = data.cuda(async=True), target.cuda(async=True) # On GPU
data, target = Variable(data), Variable(target)
def main(data_dir):
siamese_model = Siamese()
batch_size = 4
num_train = 30000
augment = True
way = 20
trials = 300
epochs = 50
train_loader, val_loader = get_train_valid_loader(data_dir,
batch_size,
num_train,
augment,
way,
trials,
pin_memory=True)
criterion = torch.nn.BCELoss()
optimizer = torch.optim.SGD(siamese_model.parameters(),
lr=1e-3,
momentum=0.9)
lambda1 = lambda epoch: 0.99**epoch
#scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lambda1)
writer = SummaryWriter()
siamese_model.cuda()
best_accuracy = 0.0
for i in range(epochs):
siamese_model.train()
#batch_count = 0
avg_train_loss = 0.0
for it, (img_1, img_2, labels) in enumerate(train_loader):
optimizer.zero_grad()
img_1 = img_1.cuda()
img_2 = img_2.cuda()
labels = labels.cuda()
preds = siamese_model(img_1, img_2)
loss = criterion(preds, labels)
avg_train_loss += loss.item()
writer.add_scalar('Loss_train', loss.item(),
len(train_loader) * i + it)
loss.backward()
optimizer.step()
#batch_count+=1
#print(batch_count)
siamese_model.eval()
count = 0
with torch.no_grad():
for ref_images, candidates in val_loader:
ref_images = ref_images.cuda()
candidates = candidates.cuda()
preds = siamese_model(ref_images, candidates)
if torch.argmax(preds) == 0:
count += 1
if count / len(val_loader) > best_accuracy:
best_accuracy = count / len(val_loader)
torch.save(siamese_model.state_dict(), 'best_model.pth')
writer.add_scalar('Accuracy_validation', count / trials, i)
print('Epoch {} | Train loss {} | Val accuracy {}'.format(
i, avg_train_loss / len(train_loader), count / trials))
#scheduler.step()
writer.flush()
best_model = Siamese().cuda()
best_model.load_state_dict(torch.load('best_model.pth'))
best_model.eval()
trials = 400
test_loader = get_test_loader(data_dir, way, trials)
test_count = 0
with torch.no_grad():
for ref_images, candidates in test_loader:
ref_images = ref_images.cuda()
candidates = candidates.cuda()
preds = best_model(ref_images, candidates)
if torch.argmax(preds) == 0:
test_count += 1
print('Test Accuracy {}'.format(test_count / len(test_loader)))
def train(opt):
# Load models
print('----------- Network Initialization --------------')
teacher = select_model(dataset=opt.data_name,
model_name=opt.t_name,
pretrained=True,
pretrained_models_path=opt.t_model,
n_classes=opt.num_class).to(opt.device)
print('finished teacher model init...')
student = select_model(dataset=opt.data_name,
model_name=opt.s_name,
pretrained=True,
pretrained_models_path=opt.s_model,
n_classes=opt.num_class).to(opt.device)
print('finished student model init...')
teacher.eval()
nets = {'snet': student, 'tnet': teacher}
for param in teacher.parameters():
param.requires_grad = False
# initialize optimizer
optimizer = torch.optim.SGD(student.parameters(),
lr=opt.lr,
momentum=opt.momentum,
weight_decay=opt.weight_decay,
nesterov=True)
# define loss functions
if opt.cuda:
criterionCls = nn.CrossEntropyLoss().cuda()
criterionAT = AT(opt.p)
else:
criterionCls = nn.CrossEntropyLoss()
criterionAT = AT(opt.p)
print('----------- DATA Initialization --------------')
train_loader = get_train_loader(opt)
test_clean_loader, test_bad_loader = get_test_loader(opt)
print('----------- Train Initialization --------------')
for epoch in range(0, opt.epochs):
adjust_learning_rate(optimizer, epoch, opt.lr)
# train every epoch
criterions = {'criterionCls': criterionCls, 'criterionAT': criterionAT}
if epoch == 0:
# before training test firstly
test(opt, test_clean_loader, test_bad_loader, nets, criterions,
epoch)
train_step(opt, train_loader, nets, optimizer, criterions, epoch + 1)
# evaluate on testing set
print('testing the models......')
acc_clean, acc_bad = test(opt, test_clean_loader, test_bad_loader,
nets, criterions, epoch + 1)
# remember best precision and save checkpoint
# save_root = opt.checkpoint_root + '/' + opt.s_name
if opt.save:
is_best = acc_clean[0] > opt.threshold_clean
opt.threshold_clean = min(acc_bad[0], opt.threshold_clean)
best_clean_acc = acc_clean[0]
best_bad_acc = acc_bad[0]
save_checkpoint(
{
'epoch': epoch,
'state_dict': student.state_dict(),
'best_clean_acc': best_clean_acc,
'best_bad_acc': best_bad_acc,
'optimizer': optimizer.state_dict(),
}, is_best, opt.checkpoint_root, opt.s_name)
traintransformSequence)
# train_dataset, val_dataset = get_MNIST_train_val_dataset('./data/MNIST')
train_loader, val_loader = get_train_val_loader(
train_dataset,
val_dataset,
val_split=args.val_split,
random_split=args.random_split,
batch_size=args.batch_size,
**kwargs)
args.num_class = train_loader.dataset.num_class
args.num_channels = train_loader.dataset.num_channels
else:
test_dataset = ImageFolder(os.path.join(args.data_dir, 'test'),
testtransformSequence)
test_loader = get_test_loader(test_dataset, args.batch_size, **kwargs)
args.num_class = test_loader.dataset.num_class
args.num_channels = test_loader.dataset.num_channels
# build RAM model
model = RecurrentAttention(args)
if args.use_gpu:
model.cuda()
# optimizer = torch.optim.SGD(model.parameters(), lr=args.init_lr, momentum=args.momentum)
optimizer = torch.optim.Adam(model.parameters(),
lr=0.0001,
betas=(0.9, 0.999),
eps=1e-08,
weight_decay=1e-5)
logger.info('Number of model parameters: {:,}'.format(
