def main():
model = model_dispatcher.MODEL_DISPATCHER[BASE_MODEL](True)
#model.to(DEVICE)
train_dataset = BengaliDatasetTrain(TRAINING_FOLDS, IMG_HEIGHT, IMG_WIDTH,
MODEL_MEAN, MODEL_STD)
train_loader = dataloader.DataLoader(dataset=train_dataset,
batch_size=TEST_BATCH_SIZE,
shuffle=True,
num_workers=4)
valid_dataset = BengaliDatasetTrain(VALIDATION_FOLDS, IMG_HEIGHT,
IMG_WIDTH, MODEL_MEAN, MODEL_STD)
valid_loader = dataloader.DataLoader(dataset=valid_dataset,
batch_size=TEST_BATCH_SIZE,
shuffle=False,
num_workers=4)
optimizer = torch.optim.Adam(model.parameters(), lr=1e-4)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
mode='min',
patience=5,
factor=0.3,
verbose=True)
for epoch in range(EPOCHS):
train(train_dataset, train_loader, model, optimizer)
val_score = evaluate(valid_dataset, valid_loader, model, optimizer)
scheduler.step(val_score)
torch.save(model.state_dict(),
f'{BASE_MODEL}_{VALIDATION_FOLDS[0]}.bin')
def __init__(self):
train_transform = transforms.Compose([
transforms.Resize((384, 128), interpolation=3),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
RandomErasing(probability=0.5, mean=[0.0, 0.0, 0.0])
])
test_transform = transforms.Compose([
transforms.Resize((384, 128), interpolation=3),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
])
# self.trainset = Market1501(train_transform, 'train', opt.data_path)
self.trainset = AlMarket1501(train_transform, 'train', opt.data_path)
self.testset = Market1501(test_transform, 'test', opt.data_path)
self.queryset = Market1501(test_transform, 'query', opt.data_path)
self.train_loader = dataloader.DataLoader(self.trainset,
sampler=RandomSampler(self.trainset, batch_id=opt.batchid,
batch_image=opt.batchimage),
batch_size=opt.batchid * opt.batchimage, num_workers=8,
pin_memory=True)
self.test_loader = dataloader.DataLoader(self.testset, batch_size=opt.batchtest, num_workers=8, pin_memory=True)
self.query_loader = dataloader.DataLoader(self.queryset, batch_size=opt.batchtest, num_workers=8,
pin_memory=True)
if opt.mode == 'vis':
self.query_image = test_transform(default_loader(opt.query_image))
def __init__(self, args):
self.loader_train = None
if not args.test_only and not args.demo:
data_train = SRdatasets(args, name=args.data_name, train=True)
# train_sampler = torch.utils.data.distributed.DistributedSampler(data_train)
self.loader_train = DataLoader.DataLoader(
data_train,
batch_size=args.batch_size,
num_workers=4,
shuffle=True)
if args.demo:
data_demo = Demo(args, name=args.demo_name, train=False)
self.loader_demo = DataLoader.DataLoader(data_demo,
batch_size=1,
shuffle=False)
self.loader_test = []
# Only Test one dataset
if not args.demo:
for test_data in args.test_name:
data_test = SRdatasets(args, name=test_data, train=False)
self.loader_test.append(
DataLoader.DataLoader(data_test,
batch_size=1,
num_workers=4,
shuffle=False))
def makeDataLoader(config):
# setup data_set, data_process instances
test_set = config.init_obj('test_set', module_data_process)
test_dataloader = module_dataloader.DataLoader(test_set, batch_size=test_set.batch_size,
num_workers=test_set.num_workers, collate_fn=test_set.collate_fn)
if not config['k_fold'] > 0:
train_set = config.init_obj('train_set', module_data_process)
valid_set = config.init_obj('valid_set', module_data_process)
train_dataloader = module_dataloader.DataLoader(train_set, batch_size=train_set.batch_size,
num_workers=train_set.num_workers, collate_fn=train_set.collate_fn)
valid_dataloader = module_dataloader.DataLoader(valid_set, batch_size=valid_set.batch_size,
num_workers=valid_set.num_workers, collate_fn=valid_set.collate_fn)
yield 0,train_dataloader, valid_dataloader, test_dataloader
else:
logger = config.get_logger('train')
logger.info('making {} fold data'.format(config['k_fold']))
all_set = config.init_obj('all_set', module_data_process)
for i,train_index,valid_index in all_set.make_k_fold_data(config['k_fold']):
train_set = torch.utils.data.dataset.Subset(all_set, train_index)
valid_set = torch.utils.data.dataset.Subset(all_set,valid_index)
train_dataloader = module_dataloader.DataLoader(train_set, batch_size=all_set.batch_size,
num_workers=all_set.num_workers,drop_last=True,
collate_fn=all_set.collate_fn)
valid_dataloader = module_dataloader.DataLoader(valid_set, batch_size=all_set.batch_size,
num_workers=all_set.num_workers,
collate_fn=all_set.collate_fn)
yield i,train_dataloader, valid_dataloader, test_dataloader
def __init__(self, model, training_set, testing_set, epochs, optimizer,
loss):
# Training variables
self.model = model
self.epochs = epochs
self.optimizer = optimizer
self.loss = loss
# Training and testing data as Dataset-objects
self.training_set = training_set
self.testing_set = testing_set
# Using DataLoader to load the datasets
self.training_data = dataloader.DataLoader(training_set)
self.testing_data = dataloader.DataLoader(testing_set)
# Containers for visualizing
self.train_loss = []
self.val_loss = []
self.val_accuracy = []
# Best model during training
self.best_model = copy.deepcopy(self.model)
self.best_loss = float("inf")
self.best_accuracy = 0.0 # 100.0 = 100%, 0.0 = 0%
self.best_results = [
] # list containing all results from best prediction
def __init__(self):
# paper is (384, 128)
train_transform = transforms.Compose([
transforms.Resize((256, 256), interpolation=3),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]),
RandomErasing(probability=0.5, mean=[0.0, 0.0, 0.0])
])
test_transform = transforms.Compose([
transforms.Resize((256, 256), interpolation=3),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
self.trainset = Car(transforms=train_transform, root=opt.data_path)
self.testset = Car(transforms=test_transform, root=opt.data_path)
self.train_loader = dataloader.DataLoader(
self.trainset,
sampler=RandomSampler(self.trainset,
batch_id=opt.batchid,
batch_image=opt.batchimage),
batch_size=opt.batchid * opt.batchimage,
num_workers=8,
pin_memory=True)
self.test_loader = dataloader.DataLoader(self.testset,
batch_size=opt.batchtest,
num_workers=1,
pin_memory=True)
def __init__(self, args, data_type):
self.args = args
# transform_list = [
# transforms.RandomChoice(
# [transforms.RandomHorizontalFlip(),
# transforms.RandomGrayscale(),
# transforms.RandomRotation(20),
# ]
# ),
# transforms.ToTensor(),
# transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[
# 0.229, 0.224, 0.225])
# ]
# transform = transforms.Compose(transform_list)
transform = transforms.Compose([
transforms.Resize((224, 224)),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
self.train_dataset = Dataset(args, data_type, transform)
self.train_loader = dataloader.DataLoader(
self.train_dataset,
shuffle=True,
batch_size=args.train_batch_size,
num_workers=args.nThread)
self.valid_loader = dataloader.DataLoader(
self.train_dataset,
shuffle=False,
batch_size=args.train_batch_size,
num_workers=args.nThread)
def __init__(self, args):
train_list = [
transforms.Resize((args.height, args.width), interpolation=3),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
]
if args.random_erasing:
train_list.append(
RandomErasing(probability=args.probability,
mean=[0.0, 0.0, 0.0]))
train_transform = transforms.Compose(train_list)
test_transform = transforms.Compose([
transforms.Resize((args.height, args.width), interpolation=3),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
if not args.test_only:
module_train = import_module('data.' + args.data_train.lower())
self.trainset = getattr(module_train,
args.data_train)(args, train_transform,
'train')
self.train_loader = dataloader.DataLoader(
self.trainset,
sampler=RandomSampler(self.trainset,
args.batchid,
batch_image=args.batchimage),
#shuffle=True,
batch_size=args.batchid * args.batchimage,
num_workers=args.nThread)
else:
self.train_loader = None
if args.data_test in [
'Market1501', 'Market1501_folder', 'MSMT17_folder',
'CUHK03_folder', 'SYSU30K_folder'
]:
module = import_module('data.' + args.data_train.lower())
self.testset = getattr(module,
args.data_test)(args, test_transform,
'test')
self.queryset = getattr(module,
args.data_test)(args, test_transform,
'query')
else:
raise Exception()
self.test_loader = dataloader.DataLoader(self.testset,
batch_size=args.batchtest,
num_workers=args.nThread)
self.query_loader = dataloader.DataLoader(self.queryset,
batch_size=args.batchtest,
num_workers=args.nThread)
def __init__(self, args):
self.loader_train = None
if not args.test_only:
datasets = []
for d in args.data_train:
module_name = d if d.find('DIV2K-Q') < 0 else 'DIV2KJPEG'
m = import_module('data.' + module_name.lower())
datasets.append(getattr(m, module_name)(args, name=d))
self.loader_train = dataloader.DataLoader(
MyConcatDataset(datasets),
batch_size=args.batch_size,
shuffle=True,
pin_memory=not args.cpu,
num_workers=args.n_threads,
)
self.loader_test = []
for d in args.data_test:
if d in ['SVNH', 'Set5', 'Set14', 'B100', 'Urban100']:
m = import_module('data.benchmark')
testset = getattr(m, 'Benchmark')(args, train=False, name=d)
else:
module_name = d if d.find('DIV2K-Q') < 0 else 'DIV2KJPEG'
m = import_module('data.' + module_name.lower())
testset = getattr(m, module_name)(args, train=False, name=d)
self.loader_test.append(
dataloader.DataLoader(
testset,
batch_size=1,
shuffle=False,
pin_memory=not args.cpu,
num_workers=args.n_threads,
))
def load_mnist_data(valid_size=0.1, shuffle=True, random_seed=2008, batch_size = 64,
num_workers = 1):
"""
We return train and test for plots and post-training experiments
"""
transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])
train = MNIST('data/MNIST', train=True, download=True, transform=transform)
test = MNIST('data/MNIST', train=False, download=True, transform=transform)
num_train = len(train)
indices = list(range(num_train))
split = int(np.floor(valid_size * num_train))
train_idx, valid_idx = indices[split:], indices[:split]
train_sampler = SubsetRandomSampler(train_idx)
valid_sampler = SubsetRandomSampler(valid_idx)
if shuffle == True:
np.random.seed(random_seed)
np.random.shuffle(indices)
# Create DataLoader
dataloader_args = dict(batch_size=batch_size,num_workers=num_workers)
train_loader = dataloader.DataLoader(train, sampler=train_sampler, **dataloader_args)
valid_loader = dataloader.DataLoader(train, sampler=valid_sampler, **dataloader_args)
dataloader_args['shuffle'] = False
test_loader = dataloader.DataLoader(test, **dataloader_args)
return train_loader, valid_loader, test_loader, train, test
def get_vqa_dataset(config):
transform = get_image_transform()
if 'clevr' in config.data.name:
train_dataset = ClevrDataset(image_root_path=config.data.train_image_root_path,
question_json_path=config.data.train_question_path,
question_vocab_path=config.data.question_vocab_path,
answer_vocab_path=config.data.answer_vocab_path,
transform=transform, data_nums=config.data.train_data_nums)
if config.data.valid_question_path is not None:
valid_dataset = ClevrDataset(image_root_path=config.data.valid_image_root_path,
question_json_path=config.data.valid_question_path,
question_vocab_path=config.data.question_vocab_path,
answer_vocab_path=config.data.answer_vocab_path,
transform=transform, data_nums=config.data.valid_data_nums)
else:
train_size = int((1 - config.train.valid_percent) * len(train_dataset))
val_size = len(train_dataset) - train_size
train_dataset, valid_dataset = random_split(train_dataset, [train_size, val_size])
train_dataset = train_dataset.dataset
valid_dataset = valid_dataset.dataset
else:
raise Exception(f'Not support data {config.data.name}')
train_loader = dataloader.DataLoader(dataset=train_dataset, batch_size=config.train.batch_size,
num_workers=config.train.num_workers,
shuffle=config.train.shuffle, collate_fn=train_dataset.collate_fn)
valid_loader = dataloader.DataLoader(dataset=valid_dataset, batch_size=config.train.batch_size,
num_workers=config.train.num_workers,
shuffle=config.train.shuffle, collate_fn=valid_dataset.collate_fn)
return train_dataset, train_loader, valid_dataset, valid_loader
def __init__(self):
train_transform = transforms.Compose([
RandomE(sig=0.05),
transforms.Resize((384, 144), interpolation=3),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
RandomErasing(probability=0.5, mean=[0.0, 0.0, 0.0])])
test_transform = transforms.Compose([
transforms.Resize((384, 144), interpolation=3),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])])
self.trainset = Market1501(train_transform, 'train',opt.data_path)
print('train_len:', len(self.trainset))
self.testset = Market1501(test_transform, 'test',opt.data_path)
print('test_len:', len(self.testset))
self.queryset = Market1501(test_transform, 'query',opt.data_path)
print('query_len:', len(self.queryset))
self.testset1 = Market1501(test_transform, 'test1', opt.data_path)
print('test_len:', len(self.testset1))
self.queryset1 = Market1501(test_transform, 'query1', opt.data_path)
print('query_len:', len(self.queryset1))
self.train_loader = dataloader.DataLoader(self.trainset,
sampler=RandomSampler(self.trainset, batch_id=opt.batchid, batch_image=opt.batchimage),
batch_size=opt.batchid*opt.batchimage, num_workers=8,pin_memory=True)
self.test_loader = dataloader.DataLoader(self.testset, batch_size=opt.batchtest, num_workers=8,pin_memory=True)
self.query_loader = dataloader.DataLoader(self.queryset, batch_size=opt.batchtest, num_workers=8,pin_memory=True)
self.test_loader1 = dataloader.DataLoader(self.testset1, batch_size=opt.batchtest, num_workers=8, pin_memory=True)
self.query_loader1 = dataloader.DataLoader(self.queryset1, batch_size=opt.batchtest, num_workers=8, pin_memory=True)
def get_loaders(train_batch_size=128,
test_batch_size=1024,
shuffle=False,
root='./data'):
train_set = MNIST(root=root,
train=True,
transform=transforms.ToTensor(),
download=True)
train_loader = dataloader.DataLoader(dataset=train_set,
batch_size=train_batch_size,
shuffle=shuffle,
num_workers=8,
pin_memory=True)
test_set = MNIST(root=root,
train=False,
transform=transforms.ToTensor(),
download=True)
test_loader = dataloader.DataLoader(dataset=test_set,
batch_size=test_batch_size,
shuffle=False,
num_workers=8,
pin_memory=True)
return train_loader, test_loader
def __init__(self, root):
# 创建tensorboard数据记录目录
self.summaryWriter = SummaryWriter("./logs")
# 加载训练数据集
self.train_dataset = MNISTDataset(root, True)
# 将60000份数据分成每批次100个数据
self.train_dataLoader = dataloader.DataLoader(self.train_dataset,
batch_size=100,
shuffle=True)
# 加载测试数据集
self.test_dataset = MNISTDataset(root, True)
# 将10000份数据分成每批次100个数据
self.test_dataLoader = dataloader.DataLoader(self.test_dataset,
batch_size=100,
shuffle=True)
# 创建模型
self.net = NetV3()
# 加载保存的参数
# self.net.load_state_dict(torch.load("./checkpoint/11.pkl"))
self.net.cuda()
# 创建优化器
self.opt = optim.Adam(self.net.parameters())
def main():
# 自适应使用GPU还是CPU
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = Net().to(device)
optimizer = torch.optim.Adam(model.parameters())
criterion = torch.nn.CrossEntropyLoss()
train_loader = Data.DataLoader(dataset=train_data,
batch_size=batch_size,
shuffle=True)
test_loader = Data.DataLoader(dataset=test_data, batch_size=batch_size)
adversary = FGSMAttack(epsilon=0.2)
for epoch in range(epochs):
for t, (x, y) in enumerate(train_loader):
x_var, y_var = to_var(x), to_var(y.long())
loss = criterion(model(x_var), y_var)
# adversarial training
if epoch + 1 > delay:
# use predicted label to prevent label leaking
y_pred = pred_batch(x, model)
x_adv = adv_train(x, y_pred, model, criterion, adversary)
x_adv_var = to_var(x_adv)
loss_adv = criterion(model(x_adv_var), y_var)
loss = (loss + loss_adv) / 2
if (t + 1) % 10 == 0:
print('t = %d, loss = %.8f' % (t + 1, loss.item()))
optimizer.zero_grad()
loss.backward()
optimizer.step()
# 每跑完一次epoch测试一下准确率 进入测试模式 禁止梯度传递
with torch.no_grad():
correct = 0
total = 0
sum_val_loss = 0
for data in test_loader:
images, labels = data
images, labels = images.to(device), labels.to(device)
outputs = model(images)
val_loss = criterion(outputs, labels)
sum_val_loss += val_loss.item()
# 取得分最高的那个类
_, predicted = torch.max(outputs.data, 1)
total += labels.size(0)
correct += (predicted == labels).sum()
print('epoch=%d accuracy=%.02f%% val_loss=%.02f%' %
(epoch + 1, (100 * correct / total), sum_val_loss))
sum_val_loss = 0.0
torch.save(model.state_dict(), './cifar-adv-pytorch/net.pth')
def main():
# 自适应使用GPU还是CPU
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
model = Net().to(device)
optimizer = torch.optim.Adam(model.parameters())
loss_func = torch.nn.CrossEntropyLoss()
train_loader = Data.DataLoader(dataset=train_data,
batch_size=batch_size,
shuffle=True)
test_loader = Data.DataLoader(dataset=test_data, batch_size=batch_size)
for epoch in range(epochs):
sum_loss = 0.0
for i, data in enumerate(train_loader):
inputs, labels = data
inputs, labels = inputs.to(device), labels.to(device)
# 梯度清零
optimizer.zero_grad()
# forward + backward
outputs = model(inputs)
loss = loss_func(outputs, labels)
loss.backward()
optimizer.step()
# 每训练100个batch打印一次平均loss
sum_loss += loss.item()
if i % 50 == 0:
print('epoch=%d, batch=%d loss: %.04f' %
(epoch + 1, i, sum_loss / 100))
sum_loss = 0.0
# 每跑完一次epoch测试一下准确率 进入测试模式 禁止梯度传递
with torch.no_grad():
correct = 0
total = 0
sum_val_loss = 0
for data in test_loader:
images, labels = data
images, labels = images.to(device), labels.to(device)
outputs = model(images)
val_loss = loss_func(outputs, labels)
sum_val_loss += val_loss.item()
# 取得分最高的那个类
_, predicted = torch.max(outputs.data, 1)
total += labels.size(0)
correct += (predicted == labels).sum()
print('epoch=%d accuracy=%.02f%% val_loss=%.02f%' %
(epoch + 1, (100 * correct / total), sum_val_loss))
sum_val_loss = 0.0
torch.save(model.state_dict(), './cifar-pytorch/net.pth')
def __init__(self, args):
print('[INFO] Making data...')
#train_list = [
# transforms.Resize((args.height, args.width), interpolation=3),
# transforms.RandomHorizontalFlip(),
# transforms.ToTensor(),
# transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
#]
#train_transform = transforms.Compose(train_list)
#test_transform = transforms.Compose([
# transforms.Resize((args.height, args.width), interpolation=3),
# transforms.ToTensor(),
# transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
#])
module_dataset = import_module('data.' + args.dataset.lower())
feature_data, feature_dim = None, None #getattr(module_dataset, 'import_feature')() #None, None
if not args.test_only:
self.trainset = getattr(module_dataset,
args.dataset)(subset='training',
feature_data=feature_data,
feature_dim=feature_dim)
self.train_loader = dataloader.DataLoader(
self.trainset,
#sampler=RandomSampler(self.trainset,args.batchid,batch_image=args.batchimage),
shuffle=True,
batch_size=args.batch_size,
num_workers=args.workers)
else:
self.train_loader = None
if args.dataset in ['ActivityNet']:
module = import_module('data.' + args.dataset.lower())
self.testset = getattr(module,
args.dataset)(subset='validation',
feature_data=feature_data,
feature_dim=feature_dim)
self.evaluateset = getattr(module,
args.dataset)(subset='validation',
feature_data=feature_data,
feature_dim=feature_dim,
output_meta=True)
else:
raise Exception()
self.test_loader = dataloader.DataLoader(self.testset,
batch_size=args.batch_size,
num_workers=args.workers)
# For evaluation, sometimes we just need pseudo labels
self.evaluate_loader = dataloader.DataLoader(
self.evaluateset,
batch_size=args.batch_size,
num_workers=args.workers)
print(len(self.trainset), len(self.testset), len(self.evaluateset))
def __init__(self):
super(MNIST_optest, self).__init__()
self.train_data = torchvision.datasets.MNIST(
'./mnist',
train=True,
transform=torchvision.transforms.ToTensor(),
download=True)
self.test_data = torchvision.datasets.MNIST(
'./mnist',
train=False,
transform=torchvision.transforms.ToTensor())
print("train_data:", self.train_data.train_data.size())
print("train_labels:", self.train_data.train_labels.size())
print("test_data:", self.test_data.test_data.size())
self.train_loader = Data.DataLoader(dataset=self.train_data,
batch_size=64,
shuffle=True)
self.test_loader = Data.DataLoader(dataset=self.test_data,
batch_size=64)
self.batch_size = 64
self.net = ResNet18().cuda()
self.old_net = ResNet18().cuda()
parameters = list(self.net.parameters())
old_parameters = list(self.old_net.parameters())
for i in range(len(parameters)):
old_parameters[i] = parameters[i].detach()
summary(self.net, input_size=(1, 28, 28))
self.beta = 0.9
self.beta_correction = 1
self.kp = 0
self.ki = 1
self.kd = 1
self.grad_saving = []
self.old_grad = []
self.optimizer = torch.optim.SGD(self.net.parameters(), 0.005)
self.old_optimizer = torch.optim.SGD(self.old_net.parameters(), 0.005)
self.loss_func = torch.nn.CrossEntropyLoss().cuda()
self.grad_saving = []
for batch_x, batch_y in self.train_loader:
batch_x, batch_y = Variable(batch_x).cuda(), Variable(
batch_y).cuda()
out = self.net(batch_x)
loss = self.loss_func(out, batch_y)
self.optimizer.zero_grad()
loss.backward()
a = list(self.net.parameters())
for i in range(len(a)):
self.grad_saving.append(a[i].grad / 10000)
self.old_grad.append(a[i].grad)
break
def __init__(self, args):
self.loader_train = None
if not args.test_only:
datasets = []
for d in args.data_train:
module_name = d if d.find('DIV2K-Q') < 0 else 'DIV2KJPEG'
m = import_module('data.' + module_name.lower())
datasets.append(getattr(m, module_name)(args, name=d))
# add distributed training dataset sampler according to
# https://github.com/horovod/horovod/blob/master/examples/pytorch/pytorch_imagenet_resnet50.py
self.train_sampler = torchDis.DistributedSampler(
# taking concatenated datasets
MyConcatDataset(datasets),
# number of processes participarting in distributed training
num_replicas=hvd.size(),
# Rank of the current process within num_replicas
rank=hvd.rank())
self.loader_train = dataloader.DataLoader(
# taking concatenated datasets
MyConcatDataset(datasets),
# how many samples per batch to load
batch_size=args.batch_size,
# data shuffling disabled when using sampler
shuffle=False,
pin_memory=not args.cpu,
# threads used in GPU for data loading
num_workers=args.n_threads,
# added the new distributed sampler
sampler=self.train_sampler)
self.loader_test = []
for d in args.data_test:
if d in ['Set5', 'Set14', 'B100', 'Urban100']:
m = import_module('data.benchmark')
testset = getattr(m, 'Benchmark')(args, train=False, name=d)
else:
module_name = d if d.find('DIV2K-Q') < 0 else 'DIV2KJPEG'
m = import_module('data.' + module_name.lower())
testset = getattr(m, module_name)(args, train=False, name=d)
self.loader_test.append(
dataloader.DataLoader(
testset,
batch_size=1,
shuffle=False,
pin_memory=not args.cpu,
num_workers=args.n_threads,
))
def __build_loader(data, batch_size=64):
loader = {}
loader['train'] = dataloader.DataLoader(data['train'],
batch_size=batch_size,
shuffle=True)
loader['test'] = dataloader.DataLoader(data['test'], batch_size=batch_size)
return loader
def __init__(self,
data="veri",
size=(288, 288),
sampler="triple",
mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]):
train_transform = transforms.Compose([
transforms.Resize(size, interpolation=3),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize(mean=mean, std=std),
RandomErasing(probability=0.5, mean=mean)
])
test_transform = transforms.Compose([
transforms.Resize(size, interpolation=3),
transforms.ToTensor(),
transforms.Normalize(mean=mean, std=std)
])
Dataset = {
'veri': VeRi,
"vehicleid": VehicleID,
'market': Market1501,
'msmt': MSMT17
}
self.trainset = Dataset[data](train_transform, 'train', opt.data_path)
self.testset = Dataset[data](test_transform, 'test', opt.data_path)
self.queryset = Dataset[data](test_transform, 'query', opt.data_path)
self.nums = self.trainset.len
if sampler == None:
self.train_loader = dataloader.DataLoader(self.trainset,
batch_size=opt.batchsize,
shuffle=True,
num_workers=4,
pin_memory=True)
elif sampler == "triple":
self.train_loader = dataloader.DataLoader(
self.trainset,
sampler=RandomSampler(self.trainset,
batch_id=opt.batchid,
batch_image=opt.batchimage),
batch_size=opt.batchid * opt.batchimage,
num_workers=4,
pin_memory=True)
self.test_loader = dataloader.DataLoader(self.testset,
batch_size=opt.batchtest,
shuffle=False,
num_workers=4,
pin_memory=True)
self.query_loader = dataloader.DataLoader(self.queryset,
batch_size=opt.batchtest,
shuffle=False,
num_workers=4,
pin_memory=True)
def prepare_data_loader():
print("train_data:", train_data.train_data.size())
print("train_labels:", train_data.train_labels.size())
print("test_data:", test_data.test_data.size())
train_loader = Data.DataLoader(dataset=train_data,
batch_size=64,
shuffle=True)
test_loader = Data.DataLoader(dataset=test_data, batch_size=64)
return train_loader, test_loader
def combine_datasources(dset, dset_extra, valid_size=0, shuffle=True, random_seed=2019,
maxsize=None, device='cpu'):
""" Combine two datasets.
Extends dataloader with additional data from other dataset(s). Note that we
add the examples in dset only to train (no validation)
Arguments:
dset (DataLoader): first dataloader
dset_extra (DataLoader): additional dataloader
valid_size (float): fraction of data use for validation fold
shiffle (bool): whether to shuffle train data
random_seed (int): random seed
maxsize (int): maximum number of examples in either train or validation loader
device (str): device for data loading
Returns:
train_loader_ext (DataLoader): train dataloader for combined data sources
valid_loader_ext (DataLoader): validation dataloader for combined data sources
"""
if shuffle == True and random_seed:
np.random.seed(random_seed)
## Convert both to TensorDataset
if isinstance(dset, torch.utils.data.DataLoader):
dataloader_args = {k:getattr(dset, k) for k in ['batch_size', 'num_workers']}
X, Y = load_full_dataset(dset, targets=True, device=device)
d = int(np.sqrt(X.shape[1]))
X = X.reshape(-1, 1, d, d)
dset = torch.utils.data.TensorDataset(X, Y)
logger.info(f'Main data size. X: {X.shape}, Y: {Y.shape}')
elif isinstance(dst, torch.utils.data.Dataset):
raise NotImplemented('Error: combine_datasources cant take Datasets yet.')
merged_dset = torch.utils.data.ConcatDataset([dset, dset_extra])
train_idx, valid_idx = random_index_split(len(dset), 1-valid_size, (maxsize, None)) # No maxsize for validation
train_idx = np.concatenate([train_idx, np.arange(len(dset_extra)) + len(dset)])
if shuffle:
train_sampler = SubsetRandomSampler(train_idx)
valid_sampler = SubsetRandomSampler(valid_idx)
else:
train_sampler = SubsetSampler(train_idx)
valid_sampler = SubsetSampler(valid_idx)
train_loader_ext = dataloader.DataLoader(merged_dset, sampler = train_sampler, **dataloader_args)
valid_loader_ext = dataloader.DataLoader(merged_dset, sampler = valid_sampler, **dataloader_args)
logger.info(f'Fold Sizes: {len(train_idx)}/{len(valid_idx)} (train/valid)')
return train_loader_ext, valid_loader_ext
def train():
dataset = subDataset()
# todo 创建DataLoader迭代器
dataloader = DataLoader.DataLoader(
dataset,
batch_size=16,
shuffle=True,
num_workers=1,
)
valloader = DataLoader.DataLoader(
dataset,
batch_size=32,
shuffle=True,
num_workers=1,
)
net = switchNet()
net.train()
lr = 0.001 # todo 学习率
criterion = torch.nn.CrossEntropyLoss()
optimizer = optim.Adam(net.parameters(), weight_decay=1e-5, lr=lr)
epoch = 2 # todo 迭代次数 （所有的数据训练一遍 称为 迭代一次）
for e in range(epoch):
for i, data in enumerate(dataloader):
input = data[0]
label = data[1]
input = torch.tensor(input, dtype=torch.float32).permute(
(0, 3, 1, 2))
label = torch.tensor(label)
optimizer.zero_grad()
out = net(input)
_, predicted = torch.max(out.data, 1)
correct = (predicted == label.long()).sum()
loss = criterion(out, label.long())
loss.backward()
optimizer.step()
for i, data in enumerate(valloader):
input = data[0]
label = data[1]
input = torch.tensor(input, dtype=torch.float32).permute(
(0, 3, 1, 2))
label = torch.tensor(label)
out = net(input)
_, predicted = torch.max(out.data, 1)
correct = (predicted == label.long()).sum()
print(correct.item())
break
torch.save(net.state_dict(), "./left_switch_net.pth") #todo 模型保存
def __init__(self, args):
train_transform_list = [
transforms.Resize((args.height, args.width), interpolation=3),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
]
if args.random_erasing:
train_transform_list.append(
RandomErasing(probability=0.5, mean=[0.0, 0.0, 0.0]))
train_transform = transforms.Compose(train_transform_list)
test_transform = transforms.Compose([
transforms.Resize((args.height, args.width), interpolation=3),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
# for market1501 dataset
if not args.test:
self.trainset = market1501.Market1501(args, train_transform,
'train')
self.train_loader = dataloader.DataLoader(
self.trainset,
batch_size=args.batchsize,
sampler=RandomSampler(self.trainset, args.batchid,
args.batchsize // args.batchid),
num_workers=args.worker,
drop_last=False,
pin_memory=True)
else:
self.trainset = None
self.train_loader = None
self.testset = market1501.Market1501(args, test_transform, 'test')
self.queryset = market1501.Market1501(args, test_transform, 'query')
self.test_loader = dataloader.DataLoader(self.testset,
batch_size=args.batchtest,
num_workers=args.worker,
drop_last=False,
pin_memory=True)
self.query_loader = dataloader.DataLoader(self.queryset,
batch_size=args.batchtest,
num_workers=args.worker,
drop_last=False,
pin_memory=True)
def __init__(self, args):
self.loader_train = None
# Go here if want to train
if not args.test_only:
datasets = []
for d in args.data_train:
module_name = d if d.find('DIV2K-Q') < 0 else 'DIV2KJPEG'
m = import_module('data.' + module_name.lower())
datasets.append(getattr(m, module_name)(args, name=d))
self.loader_train = dataloader.DataLoader(
MyConcatDataset(datasets),
batch_size=args.batch_size,
shuffle=True,
pin_memory=not args.cpu,
num_workers=args.n_threads,
)
self.loader_test = []
for d in args.data_test:
if d in ['Set5', 'Set14', 'B100', 'Urban100']:
m = import_module('data.benchmark')
testset = getattr(m, 'Benchmark')(args, train=False, name=d)
else:
# GO here since data_test == 'Demo
module_name = d if d.find('DIV2K-Q') < 0 else 'DIV2KJPEG'
# It imports all the modules from the file in the directory corresponding to d
# here - d = demo.py --> imports files from src/data.py
# Can also import from other files in directory e.g. common.py or div2k.py!
m = import_module('data.' + module_name.lower())
# Assign to testset all attributes in demo.Demo class
# testset type is data.demo.Demo
# passed into it we have the idx_scale and other args
testset = getattr(m, module_name)(args, train=False, name=d)
self.loader_test.append(
# Call Data loader - Load demo data into data loader
# Built in pytorch function
dataloader.DataLoader(
testset,
batch_size=1,
shuffle=False,
pin_memory=not args.cpu,
num_workers=args.n_threads,
)
)
def load_cifar_data(valid_size=0.1,
shuffle=True,
resize=None,
random_seed=2008,
batch_size=64,
num_workers=1):
"""
We return train and test for plots and post-training experiments
"""
transf_seq = [
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
]
if resize and (resize[0] != 32 or resize[1] != 32):
transf_seq.insert(0, transforms.Resize(resize))
transform = transforms.Compose(transf_seq)
# normalized according to pytorch torchvision guidelines https://chsasank.github.io/vision/models.html
train = CIFAR10('data/CIFAR',
train=True,
download=True,
transform=transform)
test = CIFAR10('data/CIFAR',
train=False,
download=True,
transform=transform)
num_train = len(train)
indices = list(range(num_train))
split = int(np.floor(valid_size * num_train))
train_idx, valid_idx = indices[split:], indices[:split]
train_sampler = SubsetRandomSampler(train_idx)
valid_sampler = SubsetRandomSampler(valid_idx)
if shuffle == True:
np.random.seed(random_seed)
np.random.shuffle(indices)
# Create DataLoader
dataloader_args = dict(batch_size=batch_size, num_workers=num_workers)
train_loader = dataloader.DataLoader(train,
sampler=train_sampler,
**dataloader_args)
valid_loader = dataloader.DataLoader(train,
sampler=valid_sampler,
**dataloader_args)
dataloader_args['shuffle'] = False
test_loader = dataloader.DataLoader(test, **dataloader_args)
return train_loader, valid_loader, test_loader, train, test
def __init__(self, args):
transform = {
'train':
transforms.Compose([
transforms.Resize((args.height, args.width), interpolation=3),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
]),
'test':
transforms.Compose([
transforms.Resize((args.height, args.width), interpolation=3),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
]),
'query':
transforms.Compose([
transforms.Resize((args.height, args.width), interpolation=3),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
])
}
self.transforms = transform
self.dataset = {
name: Dataset(args, transform[name], name)
for name in ['train', 'test', 'query']
}
self.dataloader = {}
self.dataloader['train'] = dataloader.DataLoader(
self.dataset['train'],
shuffle=True,
batch_size=args.batchtrain,
num_workers=args.nThread)
self.dataloader['test'] = dataloader.DataLoader(
self.dataset['test'],
shuffle=False,
batch_size=args.batchtest,
num_workers=args.nThread)
self.dataloader['query'] = dataloader.DataLoader(
self.dataset['query'],
shuffle=False,
batch_size=args.batchtest,
num_workers=args.nThread)
def loadData(train, test):
""""Loading the dataset from memory"""
train_loader = dataloader.DataLoader(train,
batch_size=16,
shuffle=True,
num_workers=2)
test_loader = dataloader.DataLoader(test,
batch_size=16,
shuffle=False,
num_workers=2)
return train_loader, test_loader
def makeDataLoader(config):
# setup data_set, data_process instances
train_set = config.init_obj('train_set', module_data_process)
valid_set = config.init_obj('valid_set', module_data_process)
test_set = config.init_obj('test_set', module_data_process)
# print('train num:{}\t valid num:{}\t test num:{}'.format(len(train_set),len(valid_set),len(test_set)))
train_dataloader = module_dataloader.DataLoader(train_set, batch_size=train_set.batch_size,
num_workers=train_set.num_workers, collate_fn=train_set.collate_fn)
valid_dataloader = module_dataloader.DataLoader(valid_set, batch_size=valid_set.batch_size,
num_workers=valid_set.num_workers, collate_fn=valid_set.collate_fn)
test_dataloader = module_dataloader.DataLoader(test_set, batch_size=test_set.batch_size,
num_workers=test_set.num_workers, collate_fn=test_set.collate_fn)
return train_dataloader, valid_dataloader,test_dataloader
