def qmodel(name, device):
if name == "resnet18":
return ResNet18().to(device)
elif name == "resnet34":
return ResNet34().to(device)
elif name == "resnet50":
return ResNet50().to(device)
elif name == "resnet101":
return ResNet101().to(device)
elif name == "resnet152":
return ResNet152().to(device)
elif name == "vgg11":
return VGG("VGG11").to(device)
elif name == "vgg13":
return VGG("VGG13").to(device)
elif name == "vgg16":
return VGG("VGG16").to(device)
elif name == "vgg19":
return VGG("VGG19").to(device)
elif name == "densenet121":
return DenseNet121().to(device)
elif name == "densenet169":
return DenseNet169().to(device)
elif name == "densenet201":
return DenseNet201().to(device)
elif name == "resnext":
return ResNeXt29_8x64d().to(device)
def __init__(self, channel=32, alpha=0.5):
super(EFNet, self).__init__()
self.rgb_extractor = SCRN(channel=channel)
self.resnet_depth = ResNet18()
self.alpha = alpha
self.sd1 = CFC(channel)
self.sd2 = CFC(channel)
self.sd3 = CFC(channel)
self.sd4 = CFC(channel)
self.output_ds = ConcatOutput(3 * channel)
self.conv_upsample1 = BasicConv2d(2 * channel, channel, 3, padding=1)
self.conv_upsample2 = BasicConv2d(4 * channel, channel, 3, padding=1)
self.conv_upsample3 = BasicConv2d(8 * channel, channel, 3, padding=1)
self.conv_upsample4 = BasicConv2d(16 * channel, channel, 3, padding=1)
for m in self.modules():
if isinstance(m, nn.Conv2d):
m.weight.data.normal_(std=0.01)
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
def main(config_dict):
print("--- Loading dataset [{}] ---".format(config_dict["Dataset"]["name"])) # --------
if config_dict["Dataset"]["name"] == "SVHN":
svhn = SVHNData("Dataset/SVHN/train_32x32.mat", "Dataset/SVHN/test_32x32.mat")
train_images, train_labels = svhn.get_train_data()
test_images, test_labels = svhn.get_test_data()
elif config_dict["Dataset"]["name"] == "CIFAR10":
cifer10 = CiferData("Dataset/CIFAR10/")
train_images, train_labels, validation_images, validation_labels, test_images, test_labels = cifer10.training_data(7000)
else:
print("Not dataset. please check the toml file")
exit()
# -------------------------------------------------------------------------------------
train_data = train_images
train_label = train_labels
test_data = test_images
test_label = test_labels
print("--- Creating model [{}] ---".format(config_dict["Network"]["name"])) # ---------
if config_dict["Network"]["name"] == "ResNet50":
network = ResNet50(config_dict["Network"]["fig_size"], config_dict["Network"]["class"])
elif config_dict["Network"]["name"] == "ResNet18":
network = ResNet18(config_dict["Network"]["fig_size"], config_dict["Network"]["class"])
else:
network = ConvolutionalNeuralNetwork(config_dict["Network"]["fig_size"], config_dict["Network"]["class"])
network.set_model(config_dict["Network"]["lr"])
# -------------------------------------------------------------------------------------
#saver = tf.compat.v1.train.Saver
sess = tf.compat.v1.Session()
init = tf.compat.v1.global_variables_initializer()
sess.run(init)
epoch = config_dict["episode"]
batch_size = config_dict["batch_size"]
# train
accuracy_list = []
loss_list = []
with tqdm(range(epoch)) as pbar:
for i, ch in enumerate(pbar):
choice_id = np.random.choice(train_data.shape[0], batch_size, replace=False)
batch_data = train_data[choice_id]
batch_label = train_label[choice_id]
_, loss = network.train(sess, batch_data, batch_label)
loss_list.append(loss)
pbar.set_postfix(OrderedDict(loss=loss))
# test
accuracy = 0
for j in range(0, test_data.shape[0], 100):
data = test_data[j:j+100]
label = test_label[j:j+100]
accuracy += int(network.test(sess, data, label)[0]*data.shape[0])
print("test accuracy {}".format(accuracy/test_data.shape[0]*100.0))
def selectModel(modelName):
if modelName == 'resnet18':
print('modelName: ResNet18')
model = ResNet18()
elif modelName == 'mobilenetv1':
print('modelName: MobileNetV1')
model = MobileNet()
else:
raise ValueError
return model
def get_model(model_name):
if model_name == 'resnet':
from model.resnet import ResNet18
net = ResNet18(10)
elif model_name == 'lenet':
from model.lenet import LeNet
net = LeNet(10)
elif model_name == 'densenet':
from model.densenet import DenseNet
net = DenseNet(growthRate=12,
depth=40,
reduction=0.5,
bottleneck=True,
nClasses=10)
elif model_name == 'vgg':
from model.vgg import VGG
net = VGG('VGG16', num_classes=10)
return net
# Data Loader (Input Pipeline)
train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=32)
test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=10)
if args.model == 'resnet18':
cnn = ResNet18(num_classes=num_classes)
elif args.model == 'wideresnet':
if args.dataset == 'svhn':
cnn = WideResNet(depth=16,
num_classes=num_classes,
widen_factor=8,
dropRate=0.4)
else:
cnn = WideResNet(depth=28,
num_classes=num_classes,
widen_factor=10,
dropRate=0.3)
cnn = cnn.cuda()
criterion = nn.CrossEntropyLoss().cuda()
cnn_optimizer = torch.optim.SGD(cnn.parameters(),
# Data Loader (Input Pipeline)
train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
batch_size=args.batch_size,
shuffle=True,
pin_memory=True,
num_workers=2)
test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
if args.model == 'resnet18':
cnn = ResNet18(num_classes=num_classes, drop=args.drop)
elif args.model == 'wideresnet':
if args.dataset == 'svhn':
cnn = WideResNet(depth=16,
num_classes=num_classes,
widen_factor=4,
dropRate=args.drop)
else:
cnn = WideResNet(depth=28,
num_classes=num_classes,
widen_factor=10,
dropRate=args.drop)
cnn = cnn.cuda()
criterion = nn.CrossEntropyLoss().cuda()
cnn_optimizer = torch.optim.SGD(cnn.parameters(),
BATCH_SIZE = 128
LR = 0.1
N = 50000
valid = np.load('data/valid_data/valid.npz')['data']
clean = np.load('data/train_data/clean.npz')['data']
index = np.load('data/train_data/clean.npz')['index']
for i in range(2, 5):
print("Round " + str(i))
data = np.load('data/train_data/error_0.1_gaussian_0.' + str(i) +
'.npz')['data']
index = np.load('data/train_data/error_0.1_gaussian_0.' + str(i) +
'.npz')['index']
model = TrackedModel(ResNet18(), data, valid)
train_his, val_his, class_his, train_recall, val_recall = model.train(
optim.SGD(model.model.parameters(),
lr=LR,
momentum=0.9,
weight_decay=5e-4),
BATCH=BATCH_SIZE,
MAX_EPOCH=EPOCH,
USE_VAL=True)
his = np.zeros((N, EPOCH))
for j in range(N):
his[j] = train_his[np.where(class_his == j)]
np.savez('bin/record/baseline_cifar_error_0.1_gaussian_0.' + str(i) +
'_first_20',
his=his,
index=index,
num_workers=2) #生成一个个batch进行批训练，组成batch的时候顺序打乱取
testset = torchvision.datasets.CIFAR10(root='./data',
train=False,
download=True,
transform=transform_test)
testloader = torch.utils.data.DataLoader(testset,
batch_size=100,
shuffle=False,
num_workers=2)
# Cifar-10的标签
classes = ('plane', 'car', 'bird', 'cat', 'deer', 'dog', 'frog', 'horse',
'ship', 'truck')
# 模型定义-ResNet
net = ResNet18().to(device)
# 定义损失函数和优化方式
criterion = nn.CrossEntropyLoss() #损失函数为交叉熵，多用于多分类问题
optimizer = optim.SGD(
net.parameters(), lr=LR, momentum=0.9,
weight_decay=5e-4) #优化方式为mini-batch momentum-SGD，并采用L2正则化（权重衰减）
# 训练
if __name__ == "__main__":
best_acc = 85 #2 初始化best test accuracy
print("Start Training, Resnet-18!") # 定义遍历数据集的次数
with open("acc.txt", "w") as f:
with open("log.txt", "w") as f2:
for epoch in range(pre_epoch, EPOCH):
print('\nEpoch: %d' % (epoch + 1))
import torch.optim as optim
from trajectory_model.weighted_model import WeightedModel
from model.resnet import ResNet18
from sampler.sampler import Sampler
import random
import scipy.stats as st
COMP = 6
train_data = np.load('data/train_data/error_0.4_gaussian_0.1.npz')['data']
val_data = np.load('data/valid_data/valid.npz')['data']
index = np.load(
'bin/record/stats_baseline_cifar_error_0.4_gaussian_0.1_first.npz')['pred']
resnet_model = ResNet18().to(device=torch.device('cuda'))
#resnet_model.load_state_dict(torch.load('bin/saved_model/params_resnet_error_0.4_gaussian_0.1_first.pkl'))
weighted_sampler = Sampler(index, COMP)
model = WeightedModel(COMP, resnet_model, weighted_sampler, train_data,
val_data)
prev_state = model.model.state_dict()
NUM = 1
EPOCH = 30
LR = 0.01
train_loss_history, train_recall_history, val_loss_history, val_recall_history = \
model.train(optim.SGD(model.model.parameters(), lr=LR, momentum=0.9, weight_decay=5e-4), 30000, MODE='none', MAX_EPOCH=EPOCH)
def main():
print('==> Preparing data..')
transforms_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor()
])
transforms_test = transforms.Compose([transforms.ToTensor()])
mode = {'train': True, 'test': True}
rate = np.squeeze([0.2, 0.5, 0.8])
for iter in range(rate.size):
model = ResNet(num_classes=args.num_class)
if use_gpu:
model = model.cuda()
model = torch.nn.DataParallel(model)
optimizer = torch.optim.Adam(model.parameters(), lr=args.learning_rate)
image_datasets = {
'train':
Cifar10(root='./datasets',
train=True,
transform=None,
download=True),
'test':
Cifar10(root='./datasets',
train=False,
transform=None,
download=True)
}
trainData = image_datasets['train'].train_data
trainLabel = image_datasets['train'].train_labels
testData = image_datasets['test'].test_data
testLabel = image_datasets['test'].test_labels
true_label = np.squeeze(trainLabel).copy()
trainLabel, actual_noise_rate = GN.noisify(
nb_classes=args.num_class,
train_labels=np.squeeze(trainLabel),
noise_type='symmetric',
noise_rate=rate[iter])
trainData = np.array(trainData)
trainLabel = np.squeeze(trainLabel)
testData = np.array(testData)
testLabel = np.squeeze(testLabel)
train_data = DT(trainData=trainData,
trainLabel=trainLabel,
transform=transforms_train)
train_data_test = DT(trainData=trainData,
trainLabel=trainLabel,
transform=transforms_test)
test_data = DT(trainData=testData,
trainLabel=testLabel,
transform=transforms_test)
train_loader = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers)
train_loader_test = torch.utils.data.DataLoader(
train_data_test,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers)
test_loader = torch.utils.data.DataLoader(test_data,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers)
train(model, optimizer, train_loader, test_loader, train_loader_test,
true_label, rate[iter])
shuffle=True,
num_workers=8,
pin_memory=True)
testset = torchvision.datasets.STL10(root='stl10',
split='test',
download=False,
transform=test_transform)
test_loader = torch.utils.data.DataLoader(testset,
batch_size=50,
shuffle=False,
num_workers=8,
pin_memory=True)
if model_name == 'resnet':
from model.resnet import ResNet18
net = ResNet18(10)
elif model_name == 'lenet':
from model.lenet import LeNet
net = LeNet(10)
elif model_name == 'densenet':
from model.densenet import DenseNet
net = DenseNet(growthRate=12,
depth=40,
reduction=0.5,
bottleneck=True,
nClasses=10)
elif model_name == 'vgg':
from model.vgg import VGG
net = VGG('VGG16', num_classes=10)
if resume:
def main(args):
# Enter all arguments that you want to be in the filename of the saved output
ordered_args = [
'dataset',
'data_augmentation',
'seed',
'remove_percent',
'burn_in_epochs',
'remove_strategy',
'noise_percent',
'noise_labels',
'noise_pixels_percent',
'noise_pixels_std',
'optimizer',
'learning_rate',
]
save_fname = '__'.join('{}_{}'.format(arg, args_dict[arg])
for arg in ordered_args)
fname = os.path.join(args.output_dir, save_fname)
if os.path.exists(fname + '__stats_dict.pkl'):
redo = input(
"There exists experiment result already, continue? [yes/no] ")
if redo == 'no':
exit()
elif redo == 'yes':
pass
else:
raise ValueError('wrong answer')
os.makedirs(args.output_dir, exist_ok=True)
# Set appropriate devices
device = torch.device(args.device)
print('run on device: {0}'.format(device))
cudnn.benchmark = True # Should make training go faster for large models
# Set random seed for initialization
torch.manual_seed(args.seed)
if 'cuda' in args.device:
torch.cuda.manual_seed(args.seed)
npr.seed(args.seed)
train_ds, test_ds, num_classes = get_data(args.dataset)
if args.noise_percent > 0:
assert not (args.noise_labels and (args.noise_pixels_percent > 0))
if args.noise_labels:
train_ds, noise_indexes = noise_labels(train_ds,
args.noise_percent, fname)
if args.noise_pixels_percent:
train_ds, noise_indexes = noise_pixels(train_ds,
args.noise_percent,
args.noise_pixels_percent,
args.noise_pixels_std,
fname)
print('Training on ' + str(len(train_ds)) + ' examples')
# Setup model
if args.model == 'resnet18':
model = ResNet18(num_classes=num_classes)
elif args.model == 'wideresnet':
if args.dataset == 'svhn':
model = WideResNet(depth=16,
num_classes=num_classes,
widen_factor=8,
dropRate=0.4)
else:
model = WideResNet(depth=28,
num_classes=num_classes,
widen_factor=10,
dropRate=0.3)
elif args.model == 'cnn':
model = CNN(num_classes=num_classes)
else:
print(
'Specified model not recognized. Options are: resnet18 and wideresnet'
)
# Setup loss
model = model.to(args.device)
criterion = torch.nn.CrossEntropyLoss().cuda()
criterion.__init__(reduce=False)
# Setup optimizer
if args.optimizer == 'adam':
model_optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
elif args.optimizer == 'sgd':
model_optimizer = torch.optim.SGD(model.parameters(),
lr=args.learning_rate,
momentum=0.9,
nesterov=True,
weight_decay=5e-4)
scheduler = MultiStepLR(model_optimizer,
milestones=[60, 120, 160],
gamma=0.2)
elif args.optimizer == 'sgd-const-lr':
model_optimizer = torch.optim.SGD(model.parameters(),
lr=args.learning_rate,
momentum=0.9,
nesterov=True,
weight_decay=5e-4)
else:
print('Specified optimizer not recognized. Options are: adam and sgd')
save_point = os.path.join(args.output_dir, 'checkpoint', args.dataset)
os.makedirs(save_point, exist_ok=True)
checkpoint_fname = os.path.join(save_point, save_fname + '.t7')
# Initialize dictionary to save statistics for every example presentation
example_stats = {}
num_examples = len(train_ds)
example_weights = np.ones(num_examples)
elapsed_time = 0
# train_idx = np.array(range(0, len(train_ds)))
train_loader = DataLoader(train_ds,
batch_size=args.batch_size,
shuffle=True)
for epoch in range(args.epochs):
if args.remove_strategy != 'normal' and epoch >= args.burn_in_epochs:
if 'sampling' in args.remove_strategy:
# sampling by weight
normalized_weights = example_weights / example_weights.sum()
index_stats = example_stats.get('example_weights', [[], []])
index_stats[1].append(normalized_weights)
example_stats['example_weights'] = index_stats
choice_num = int(num_examples *
(1 - args.remove_percent / 100))
train_idx = np.random.choice(range(num_examples),
size=choice_num,
replace=False,
p=normalized_weights)
elif args.remove_strategy == 'low-acc':
remove_n = int(args.remove_percent * num_examples / 100)
losses = []
for idx in range(num_examples):
losses.append(example_stats[idx][0][epoch - 1])
losses = np.array(losses)
sorted_indexes = np.argsort(losses)
train_idx = sorted_indexes[:num_examples - remove_n]
elif args.remove_strategy == 'all-noise':
remove_n = int(args.remove_percent * num_examples / 100)
if args.remove_percent <= args.noise_percent_labels:
remove_indexes = npr.choice(noise_indexes,
remove_n,
replace=False)
train_idx = np.setdiff1d(range(num_examples),
remove_indexes)
else:
train_idx = np.setdiff1d(range(num_examples),
noise_indexes)
train_idx = npr.choice(train_idx,
num_examples - remove_n,
replace=False)
else:
# event method
_, unlearned_per_presentation, _, first_learned = compute_forgetting_statistics(
example_stats, epoch)
ordered_examples, ordered_values = sort_examples_by_forgetting(
[unlearned_per_presentation], [first_learned], epoch)
train_idx = sample_dataset_by_forgetting(
train_ds, ordered_examples, ordered_values,
args.remove_percent, args.remove_strategy)
sampler = torch.utils.data.SubsetRandomSampler(train_idx)
train_loader = DataLoader(train_ds,
batch_size=args.batch_size,
sampler=sampler)
start_time = time.time()
train(args, model, criterion, device, train_loader, model_optimizer,
epoch, example_stats)
test_loader = DataLoader(test_ds, batch_size=32, shuffle=True)
test(epoch, model, criterion, device, test_loader, example_stats,
checkpoint_fname)
if args.remove_strategy != 'normal' and epoch >= args.burn_in_epochs:
# evaluate on removed data
removed_idx = np.setdiff1d(range(num_examples), train_idx)
sampler = torch.utils.data.SubsetRandomSampler(removed_idx)
removed_loader = DataLoader(train_ds,
batch_size=args.batch_size,
sampler=sampler)
evaluate_on_removed(model, criterion, device, removed_loader,
epoch, example_stats)
if 'sampling' in args.remove_strategy:
example_weights = update_example_weights(example_weights,
example_stats, epoch,
args.remove_strategy)
epoch_time = time.time() - start_time
elapsed_time += epoch_time
print('| Elapsed time : %d:%02d:%02d' % (get_hms(elapsed_time)))
# Update optimizer step
if args.optimizer == 'sgd':
scheduler.step(epoch)
# Save the stats dictionary
fname = os.path.join(args.output_dir, save_fname)
with open(fname + "__stats_dict.pkl", "wb") as f:
pickle.dump(example_stats, f)
# Log the best train and test accuracy so far
with open(fname + "__best_acc.txt", "w") as f:
f.write('train test \n')
f.write(str(max(example_stats['train'][1])))
f.write(' ')
f.write(str(max(example_stats['test'][1])))
MAX_EPOCH = 150
MAX_ACCURACY = 1
BATCH_SIZE = 100
LR = 0.1
N = 50000
valid = np.load('data/valid_data/valid.npz')['data']
test = np.load('data/valid_data/test.npz')['data']
data = np.load('data/train_data/clean.npz')['data']
index = np.load('data/train_data/clean.npz')['index']
print('Stage 1 training starts!')
model = TrackedModel(ResNet18(), data, test)
train_his, val_his, class_his, train_recall, val_recall = model.train(
optim.SGD(model.model.parameters(), lr=LR, momentum=0.9, weight_decay=5e-4), BATCH=BATCH_SIZE, MAX_EPOCH=MAX_EPOCH, MAX_ACCURACY=MAX_ACCURACY, USE_VAL=True)
N, EPOCH_ONE = train_his.shape
his = np.zeros_like(train_his)
for i in range(N):
his[i] = train_his[np.where(class_his==i)]
np.savez('bin/record/baseline_cifar_test_first', his=his, index=index, train_recall=train_recall, val_recall=val_recall)
torch.save(model.model.state_dict(), 'bin/saved_model/params_resnet18_clean_first.pkl')
MAX_EPOCH = 50
MAX_ACCURACY = 1
LR = 0.01
print('Stage 2 training starts!')
args = parser.parse_args()
pprint(args)
# check and create directories
if not os.path.exists(args.checkpoint):
os.makedirs(args.checkpoint)
if not os.path.exists(args.log):
os.makedirs(args.log)
arch = 'resnet18_'
filename = arch + args.dataset + '_' + str(args.num_class)
checkpoint_filename = os.path.join(args.checkpoint, filename + '.pt')
model = ResNet(num_classes=args.num_class)
criterion = torch.nn.CrossEntropyLoss(size_average=True)
weight_criterion = CE(aggregate='sum')
use_gpu = torch.cuda.is_available()
if use_gpu:
model = model.cuda()
criterion = criterion.cuda()
weight_criterion.cuda()
torch.cuda.manual_seed(args.seed)
optimizer = torch.optim.Adam(model.parameters(), lr=args.learning_rate)
# Adjust learning rate and betas for Adam Optimizer
n_epoch = 200
def run_cutout(dataset="cifar10",
model="resnet18",
epochs=200,
batch_size=128,
learning_rate=0.1,
data_augmentation=False,
cutout=False,
n_holes=1,
length=8,
no_cuda=False,
seed=0):
cuda = not no_cuda and torch.cuda.is_available()
cudnn.benchmark = True # Should make training should go faster for large models
torch.manual_seed(seed)
if cuda:
torch.cuda.manual_seed(seed)
test_id = dataset + '_' + model
# Image Preprocessing
if dataset == 'svhn':
normalize = transforms.Normalize(
mean=[x / 255.0 for x in [109.9, 109.7, 113.8]],
std=[x / 255.0 for x in [50.1, 50.6, 50.8]])
else:
normalize = transforms.Normalize(
mean=[x / 255.0 for x in [125.3, 123.0, 113.9]],
std=[x / 255.0 for x in [63.0, 62.1, 66.7]])
train_transform = transforms.Compose([])
if data_augmentation:
train_transform.transforms.append(transforms.RandomCrop(32, padding=4))
train_transform.transforms.append(transforms.RandomHorizontalFlip())
train_transform.transforms.append(transforms.ToTensor())
train_transform.transforms.append(normalize)
if cutout:
train_transform.transforms.append(
Cutout(n_holes=n_holes, length=length))
test_transform = transforms.Compose([transforms.ToTensor(), normalize])
if dataset == 'cifar10':
num_classes = 10
train_dataset = datasets.CIFAR10(root='data/',
train=True,
transform=train_transform,
download=True)
test_dataset = datasets.CIFAR10(root='data/',
train=False,
transform=test_transform,
download=True)
elif dataset == 'cifar100':
num_classes = 100
train_dataset = datasets.CIFAR100(root='data/',
train=True,
transform=train_transform,
download=True)
test_dataset = datasets.CIFAR100(root='data/',
train=False,
transform=test_transform,
download=True)
elif dataset == 'svhn':
num_classes = 10
train_dataset = datasets.SVHN(root='data/',
split='train',
transform=train_transform,
download=True)
extra_dataset = datasets.SVHN(root='data/',
split='extra',
transform=train_transform,
download=True)
# Combine both training splits (https://arxiv.org/pdf/1605.07146.pdf)
data = np.concatenate([train_dataset.data, extra_dataset.data], axis=0)
labels = np.concatenate([train_dataset.labels, extra_dataset.labels],
axis=0)
train_dataset.data = data
train_dataset.labels = labels
test_dataset = datasets.SVHN(root='data/',
split='test',
transform=test_transform,
download=True)
# Data Loader (Input Pipeline)
train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
batch_size=batch_size,
shuffle=True,
pin_memory=True,
num_workers=2)
test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
batch_size=batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
if model == 'resnet18':
cnn = ResNet18(num_classes=num_classes)
elif model == 'wideresnet':
if dataset == 'svhn':
cnn = WideResNet(depth=16,
num_classes=num_classes,
widen_factor=8,
dropRate=0.4)
else:
cnn = WideResNet(depth=28,
num_classes=num_classes,
widen_factor=10,
dropRate=0.3)
cnn = cnn.cuda()
criterion = nn.CrossEntropyLoss().cuda()
cnn_optimizer = torch.optim.SGD(cnn.parameters(),
lr=learning_rate,
momentum=0.9,
nesterov=True,
weight_decay=5e-4)
if dataset == 'svhn':
scheduler = MultiStepLR(cnn_optimizer, milestones=[80, 120], gamma=0.1)
else:
scheduler = MultiStepLR(cnn_optimizer,
milestones=[60, 120, 160],
gamma=0.2)
#TODO: change path to relative path
filename = "/beegfs/work/workspace/ws/fr_mn119-augment-0/logs/{}.csv".format(
test_id)
# filename = 'logs/' + test_id + '.csv'
args = argparse.Namespace(
**{
"dataset": dataset,
"model": model,
"epochs": epochs,
"batch_size": batch_size,
"learning_rate": learning_rate,
"data_augmentation": data_augmentation,
"cutout": cutout,
"n_holes": n_holes,
"length": length,
"no_cuda": no_cuda,
"seed": seed
})
csv_logger = CSVLogger(args=args,
fieldnames=['epoch', 'train_acc', 'test_acc'],
filename=filename)
def test(loader):
cnn.eval() # Change model to 'eval' mode (BN uses moving mean/var).
correct = 0.
total = 0.
for images, labels in loader:
if dataset == 'svhn':
# SVHN labels are from 1 to 10, not 0 to 9, so subtract 1
labels = labels.type_as(torch.LongTensor()).view(-1) - 1
images = Variable(images, volatile=True).cuda()
labels = Variable(labels, volatile=True).cuda()
pred = cnn(images)
pred = torch.max(pred.data, 1)[1]
total += labels.size(0)
correct += (pred == labels.data).sum()
val_acc = correct / total
cnn.train()
return val_acc
for epoch in range(epochs):
xentropy_loss_avg = 0.
correct = 0.
total = 0.
progress_bar = tqdm(train_loader)
for i, (images, labels) in enumerate(progress_bar):
progress_bar.set_description('Epoch ' + str(epoch))
if dataset == 'svhn':
# SVHN labels are from 1 to 10, not 0 to 9, so subtract 1
labels = labels.type_as(torch.LongTensor()).view(-1) - 1
images = Variable(images).cuda(async=True)
labels = Variable(labels).cuda(async=True)
cnn.zero_grad()
pred = cnn(images)
xentropy_loss = criterion(pred, labels)
xentropy_loss.backward()
cnn_optimizer.step()
xentropy_loss_avg += xentropy_loss.data[0]
# Calculate running average of accuracy
_, pred = torch.max(pred.data, 1)
total += labels.size(0)
correct += (pred == labels.data).sum()
accuracy = correct / total
progress_bar.set_postfix(xentropy='%.3f' % (xentropy_loss_avg /
(i + 1)),
acc='%.3f' % accuracy)
test_acc = test(test_loader)
tqdm.write('test_acc: %.3f' % (test_acc))
scheduler.step(epoch)
row = {
'epoch': str(epoch),
'train_acc': str(accuracy),
'test_acc': str(test_acc)
}
csv_logger.writerow(row)
# torch.save(cnn.state_dict(), 'checkpoints/' + test_id + '.pt')
csv_logger.close()
results = {
'epoch': epoch,
'train_error': 1 - accuracy,
'test_error': 1 - test_acc
}
# validation error for hyperband
return results
def main():
# Training settings
parser = argparse.ArgumentParser(description='PyTorch MNIST Example')
parser.add_argument('--batch-size',
type=int,
default=128,
metavar='N',
help='input batch size for training (default: 128)')
parser.add_argument('--test-batch-size',
type=int,
default=1000,
metavar='N',
help='input batch size for testing (default: 1000)')
parser.add_argument('--epochs',
type=int,
default=350,
metavar='N',
help='number of epochs to train (default: 350)')
parser.add_argument('--lr',
type=float,
default=.1,
metavar='LR',
help='learning rate (default: 0.1)')
parser.add_argument('--gamma',
type=float,
default=0.1,
metavar='M',
help='Learning rate step gamma (default: 0.1)')
parser.add_argument('--no-cuda',
action='store_true',
default=False,
help='disables CUDA training')
parser.add_argument('--dry-run',
action='store_true',
default=False,
help='quickly check a single pass')
parser.add_argument('--seed',
type=int,
default=1,
metavar='S',
help='random seed (default: 1)')
parser.add_argument(
'--log-interval',
type=int,
default=10,
metavar='N',
help='how many batches to wait before logging training status')
parser.add_argument('--save-model',
action='store_true',
default=False,
help='For Saving the current Model')
args = parser.parse_args()
use_cuda = not args.no_cuda and torch.cuda.is_available()
torch.manual_seed(args.seed)
device = torch.device("cuda" if use_cuda else "cpu")
kwargs = {'batch_size': args.batch_size}
if use_cuda:
kwargs.update({
'num_workers': 1,
'pin_memory': True,
'shuffle': True
}, )
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010))
])
transform_test = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.4914, 0.4822, 0.4465),
(0.2023, 0.1994, 0.2010)),
])
dataset1 = datasets.CIFAR10('./data',
train=True,
download=True,
transform=transform_train)
dataset2 = datasets.CIFAR10('./data',
train=False,
transform=transform_test)
train_loader = torch.utils.data.DataLoader(dataset1, **kwargs)
test_loader = torch.utils.data.DataLoader(dataset2, **kwargs)
# model = models.resnet18().to(device)
model = ResNet18().to(device)
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(),
lr=args.lr,
momentum=0.9,
weight_decay=5e-4)
scheduler = MultiStepLR(optimizer, milestones=[150, 250], gamma=args.gamma)
for epoch in range(1, args.epochs + 1):
train(args, model, criterion, device, train_loader, optimizer, epoch)
test(model, criterion, device, test_loader)
scheduler.step()
if args.save_model:
torch.save(model.state_dict(), "mnist_cnn.pt")
train_loader = torch.utils.data.DataLoader(dataset=train_dataset,
batch_size=args.batch_size,
pin_memory=True,
num_workers=2,
sampler=get_sampler_classifier(train_loader, args.seed, args.prop))
test_loader = torch.utils.data.DataLoader(dataset=test_dataset,
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
num_workers=2)
if args.model == 'resnet18':
cnn = ResNet18(num_classes=num_classes, num_channels=num_channels)
elif args.model == 'wideresnet':
cnn = WideResNet(depth=28, num_classes=num_classes, num_channels=num_channels, widen_factor=10, dropRate=0.5)
cnn = cnn.cuda()
criterion = nn.CrossEntropyLoss().cuda()
cnn_optimizer = torch.optim.SGD(cnn.parameters(), lr=args.learning_rate,
momentum=0.9, nesterov=True, weight_decay=5e-4)
scheduler = MultiStepLR(cnn_optimizer, milestones=[round(0.4*args.epochs), round(0.8*args.epochs)], gamma=0.1)
def test(loader):
cnn.eval() # Change model to 'eval' mode (BN uses moving mean/var).
correct = 0.
total = 0.
for images, labels in loader: