def get_data(name, data_dir, height, width, batch_size, workers, pose_aug):
root = osp.join(data_dir, name)
dataset = datasets.create(name, root)
# use combined trainval set for training as default
train_loader = DataLoader(Preprocessor(dataset.trainval,
root=dataset.images_dir,
with_pose=True,
pose_root=dataset.poses_dir,
pid_imgs=dataset.trainval_query,
height=height,
width=width,
pose_aug=pose_aug),
sampler=RandomPairSampler(dataset.trainval,
neg_pos_ratio=3),
batch_size=batch_size,
num_workers=workers,
pin_memory=False)
test_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
test_loader = DataLoader(Preprocessor(
list(set(dataset.query) | set(dataset.gallery)),
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=False)
return dataset, train_loader, test_loader
def get_test_data(dataname, data_dir, height, width, workers=8, test_batch=64):
root = osp.join(data_dir, dataname)
dataset = datasets.create(dataname, root, combine_all=False)
test_transformer = T.Compose([
T.Resize((height, width), interpolation=3),
T.ToTensor(),
])
query_loader = DataLoader(Preprocessor(dataset.query,
root=osp.join(
dataset.images_dir,
dataset.query_path),
transform=test_transformer),
batch_size=test_batch,
num_workers=workers,
shuffle=False,
pin_memory=True)
gallery_loader = DataLoader(Preprocessor(dataset.gallery,
root=osp.join(
dataset.images_dir,
dataset.gallery_path),
transform=test_transformer),
batch_size=test_batch,
num_workers=workers,
shuffle=False,
pin_memory=True)
return dataset, query_loader, gallery_loader
def get_data(name, split_id, data_dir, height, width, batch_size, workers,
combine_trainval):
root = osp.join(data_dir, name)
dataset = datasets.create(name, root, split_id=split_id)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_set = dataset.trainval if combine_trainval else dataset.train
num_classes = (dataset.num_trainval_ids
if combine_trainval else dataset.num_train_ids)
train_transformer = T.Compose([
T.RandomSizedRectCrop(height, width),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalizer,
])
test_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
normalizer,
])
train_loader = DataLoader(Preprocessor(train_set,
root=dataset.images_dir,
transform=train_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=True,
pin_memory=True,
drop_last=True)
val_loader = DataLoader(Preprocessor(dataset.val,
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
test_loader = DataLoader(Preprocessor(
list(set(dataset.query) | set(dataset.gallery)),
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
pid_train = np.array(list(pid for _, pid, _ in train_set))
class_weight = np.array([(pid_train == i).sum()
for i in range(num_classes)])
assert np.all(class_weight != 0)
class_weight = pid_train.shape[0] / num_classes / class_weight
class_weight = torch.Tensor(class_weight).cuda()
return dataset, num_classes, class_weight, train_loader, val_loader, test_loader
def get_data(name, data_dir, height, width, batch_size,
workers):
root = osp.join(data_dir, name)
dataset = datasets.create(name, root, num_val=0.1)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
# use all training and validation images in target dataset
train_set = dataset.trainval
num_classes = dataset.num_trainval_ids
transformer = T.Compose([
T.Resize((height,width)),
T.ToTensor(),
normalizer,
])
extfeat_loader = DataLoader(
Preprocessor(train_set, root=dataset.images_dir,
transform=transformer),
batch_size=batch_size, num_workers=workers,
shuffle=False, pin_memory=True)
test_loader = DataLoader(
Preprocessor(list(set(dataset.query) | set(dataset.gallery)),
root=dataset.images_dir, transform=transformer),
batch_size=batch_size//2, num_workers=workers,
shuffle=False, pin_memory=True)
return dataset, num_classes, extfeat_loader, test_loader
def get_data(dataname,
data_dir,
height,
width,
batch_size,
combine_all=False,
min_size=0.,
max_size=0.8,
workers=8,
test_batch=64):
root = osp.join(data_dir, dataname)
dataset = datasets.create(dataname, root, combine_all=combine_all)
num_classes = dataset.num_train_ids
train_transformer = T.Compose([
T.RandomHorizontalFlip(),
T.Resize((height, width), interpolation=3),
T.RandomOcclusion(min_size, max_size),
T.ToTensor(),
])
test_transformer = T.Compose([
T.Resize((height, width), interpolation=3),
T.ToTensor(),
])
train_loader = DataLoader(Preprocessor(dataset.train,
root=osp.join(
dataset.images_dir,
dataset.train_path),
transform=train_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=True,
pin_memory=True,
drop_last=True)
query_loader = DataLoader(Preprocessor(dataset.query,
root=osp.join(
dataset.images_dir,
dataset.query_path),
transform=test_transformer),
batch_size=test_batch,
num_workers=32,
shuffle=False,
pin_memory=True)
gallery_loader = DataLoader(Preprocessor(dataset.gallery,
root=osp.join(
dataset.images_dir,
dataset.gallery_path),
transform=test_transformer),
batch_size=test_batch,
num_workers=32,
shuffle=False,
pin_memory=True)
return dataset, num_classes, train_loader, query_loader, gallery_loader
def get_data(name, split_id, data_dir, height, width, batch_size,
num_instances, workers, combine_trainval, batch_id):
root = osp.join(data_dir, name)
if name == 'synthetic':
dataset = datasets.create(name,
root,
split_id=split_id,
batch_id=batch_id)
else:
dataset = datasets.create(name, root, split_id=split_id)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_set = dataset.trainval if combine_trainval else dataset.train
num_classes = (dataset.num_trainval_ids
if combine_trainval else dataset.num_train_ids)
train_transformer = T.Compose([
T.RandomSizedRectCrop(height, width),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalizer,
])
test_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
normalizer,
])
train_loader = DataLoader(Preprocessor(train_set,
root=dataset.images_dir,
transform=train_transformer),
batch_size=batch_size,
num_workers=workers,
sampler=RandomIdentitySampler(
train_set, num_instances),
pin_memory=True,
drop_last=True)
val_loader = DataLoader(Preprocessor(dataset.val,
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
test_loader = DataLoader(Preprocessor(
list(set(dataset.query) | set(dataset.gallery)),
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
return dataset, num_classes, train_loader, val_loader, test_loader
def get_data(name, data_dir, height, width, batch_size, workers, pose_aug,
skip, rate, eraser):
root = osp.join(data_dir, name)
dataset = datasets.create(name, root)
video_dict = None
if osp.isfile(osp.join(root, 'video.json')):
video_dict = read_json(osp.join(root, 'video.json'))
if eraser:
train_transformer = T.Compose([
T.RectScale(height, width),
T.RandomSizedEarser(),
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
else:
train_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
# use combined trainval set for training as default
train_loader = DataLoader(Preprocessor(dataset.trainval,
name,
root=dataset.images_dir,
with_pose=True,
pose_root=dataset.poses_dir,
pid_imgs=dataset.trainval_query,
height=height,
width=width,
pose_aug=pose_aug,
transform=train_transformer),
sampler=RandomTripletSampler(
dataset.trainval,
video_dict=video_dict,
skip_frames=skip,
inter_rate=rate),
batch_size=batch_size,
num_workers=workers,
pin_memory=False)
test_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
])
test_loader = DataLoader(Preprocessor(
list(set(dataset.query) | set(dataset.gallery)),
name,
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=False)
return dataset, train_loader, test_loader
def get_data(
name,
split_id,
data_dir,
height,
width,
batch_size,
workers,
):
root = osp.join(data_dir, name)
dataset = datasets.create(name, root, split_id=split_id)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_set = dataset.trainval
num_classes = dataset.num_trainval_ids
train_transformer = T.Compose([
T.Resize((height, width), interpolation=3),
T.ToTensor(),
normalizer,
])
test_transformer = T.Compose([
T.Resize((height, width), interpolation=3),
T.ToTensor(),
normalizer,
])
train_loader = DataLoader(Preprocessor(train_set,
root=dataset.images_dir,
transform=train_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=False,
drop_last=False)
val_loader = DataLoader(Preprocessor(dataset.val,
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=False)
test_loader = DataLoader(Preprocessor(
list(set(dataset.query) | set(dataset.gallery)),
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=False)
return dataset, num_classes, train_loader, val_loader, test_loader
def get_data(
name,
split_id,
data_dir,
height,
width,
batch_size,
workers,
):
root = osp.join(data_dir, name)
dataset = datasets.create(name, root, split_id=split_id)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406], #for each channel
std=[0.229, 0.224, 0.225])
train_set = dataset.trainval # accessing through an object i.e., dataset (as object is being created at line 40)
num_classes = dataset.num_trainval_ids
train_transformer = T.Compose([
T.Resize((height, width), interpolation=3),
T.ToTensor(),
normalizer,
])
test_transformer = T.Compose([
T.Resize((height, width), interpolation=3),
T.ToTensor(),
normalizer,
])
train_loader = DataLoader(Preprocessor(train_set, # It represents a Python iterable over a dataset
root=dataset.images_dir,
transform=train_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=False, # https://discuss.pytorch.org/t/when-to-set-pin-memory-to-true/19723
drop_last=False) # https://discuss.pytorch.org/t/usage-of-drop-last-on-data-loader/66741
val_loader = DataLoader(Preprocessor(dataset.val,
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=False)
test_loader = DataLoader(Preprocessor(
list(set(dataset.query) | set(dataset.gallery)),
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=False)
return dataset, num_classes, train_loader, val_loader, test_loader
def get_data(name, data_dir, height, width, batch_size, workers):
root = osp.join(data_dir, name)
root = data_dir
dataset = datasets.create(name, root)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
num_classes = dataset.num_train_ids
train_transformer = T.Compose([
T.RectScale(height, width),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalizer,
])
test_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
normalizer,
])
train_loader = DataLoader(Preprocessor(dataset.train,
root=osp.join(
dataset.images_dir,
dataset.train_path),
transform=train_transformer,
random_mask=False),
batch_size=batch_size,
num_workers=workers,
shuffle=True,
pin_memory=True,
drop_last=True)
query_loader = DataLoader(Preprocessor(dataset.query,
root=osp.join(
dataset.images_dir,
dataset.query_path),
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
gallery_loader = DataLoader(Preprocessor(dataset.gallery,
root=osp.join(
dataset.images_dir,
dataset.gallery_path),
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
return dataset, num_classes, train_loader, query_loader, gallery_loader
def test(dataset, net, perturbation, args, evaluator, epoch, name,saveRank=False):
print(">> Evaluating network on test datasets...")
net = net.cuda()
net.eval()
normalize = T.Normalize(
mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]
)
mean = torch.Tensor(normalize.mean).view(1, 3, 1, 1).cuda()
std = torch.Tensor(normalize.std).view(1, 3, 1, 1).cuda()
test_transformer = T.Compose([
T.RectScale(args.height, args.width),
T.ToTensor()
])
query_loader = DataLoader(
Preprocessor(dataset.query, name, root=dataset.images_dir, transform=test_transformer),
batch_size=args.batch_size, num_workers=0, shuffle=False, pin_memory=True
)
gallery_loader = DataLoader(
Preprocessor(dataset.gallery, name, root=dataset.images_dir, transform=test_transformer),
batch_size=args.batch_size, num_workers=8, shuffle=False, pin_memory=True
)
qFeats, gFeats, testQImage,noiseQIamge, qnames, gnames = [], [], [], [], [],[]
qCams, gCams = [], []
import random
with torch.no_grad():
for batch_index, (inputs, qname, _, qCam) in enumerate(query_loader):
inputs = inputs.cuda()
perturted_input = perturbation(inputs)
successful_diffs = ((perturted_input - inputs) * 3 + 0.5).clamp(0, 1)
if len(testQImage) < 8:
testQImage.append(perturted_input[0, ...])
noiseQIamge.append(successful_diffs[0, ...])
perturted_input = torch.clamp(perturted_input, 0, 1)
###normalization
norm_perturted_input = (perturted_input - mean) / std
perturbed_feature = net(norm_perturted_input)[0]
qFeats.append(perturbed_feature)
qnames.extend(qname)
qCams.append(qCam.cuda())
qFeats = torch.cat(qFeats, 0)
for (inputs, gname, _, gCam) in gallery_loader:
###normalize####
inputs = inputs.cuda()
inputs = (inputs - mean) / std
gFeats.append(net(inputs)[0])
gnames.extend(gname)
gCams.append(gCam.cuda())
gFeats = torch.cat(gFeats, 0)
gCams, qCams = torch.cat(gCams).view(1, -1), torch.cat(qCams).view(-1, 1)
distMat = calDist(qFeats, gFeats)
# evaluate on test datasets
s= evaluator.evaMat(distMat, dataset.query, dataset.gallery)
return testQImage, noiseQIamge,s
def get_data(name, data_dir, height, width, ratio, batch_size, workers, num_instances=8):
root = osp.join(data_dir, name)
root = data_dir
dataset = datasets.create(name, root)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
listnormalizer = T.ListNormalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
num_classes = dataset.num_train_ids + 1 # plus 1 more label for the zero-padded feature
train_transformer = T.Compose([
T.RectScale(height, width),
T.RandomHorizontalFlip(),
T.RandomVerticalCropCont(height,width),
T.ListToTensor(),
listnormalizer,
])
test_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
normalizer,
])
query_transformer = T.Compose([
T.ContVerticalCropDiscret(height,width, ratio),
T.ToTensor(),
normalizer,
])
train_loader = DataLoader(
Preprocessor(dataset.train, root=osp.join(dataset.images_dir,dataset.train_path),
transform=train_transformer),
batch_size=batch_size, num_workers=workers,
sampler=RandomIdentitySampler(dataset.train, num_instances),
# shuffle=True,
pin_memory=True, drop_last=True)
query_loader = DataLoader(
Preprocessor(dataset.query, root=osp.join(dataset.images_dir,dataset.query_path),
transform=query_transformer),
batch_size=batch_size, num_workers=workers,
shuffle=False, pin_memory=True)
gallery_loader = DataLoader(
Preprocessor(dataset.gallery, root=osp.join(dataset.images_dir,dataset.gallery_path),
transform=test_transformer),
batch_size=batch_size, num_workers=workers,
shuffle=False, pin_memory=True)
return dataset, num_classes, train_loader, query_loader, gallery_loader
def get_data(name,
split_id,
data_dir,
height,
width,
crop_height,
crop_width,
batch_size,
caffe_sampler=False,
workers=4):
root = osp.join(data_dir, name)
dataset = datasets.create(name, root, split_id=split_id)
train_set = dataset.trainval
num_classes = dataset.num_trainval_ids
# transforms
train_transformer = T.Compose([
T.RectScale(height, width),
# T.CenterCrop((crop_height, crop_width)),
T.RandomHorizontalFlip(),
T.ToTensor(),
T.RGB_to_BGR(),
T.NormalizeBy(255),
])
test_transformer = T.Compose([
T.RectScale(height, width),
# T.CenterCrop((crop_height, crop_width)),
T.ToTensor(),
T.RGB_to_BGR(),
T.NormalizeBy(255),
])
# dataloaders
sampler = caffeSampler(train_set, name, batch_size=batch_size, root=dataset.images_dir) if caffe_sampler else \
RandomIdentitySampler(train_set, 10) #TODO
train_loader = DataLoader(Preprocessor(train_set,
root=dataset.images_dir,
transform=train_transformer),
batch_size=batch_size,
num_workers=workers,
sampler=sampler,
pin_memory=True,
drop_last=True)
test_loader = DataLoader(Preprocessor(
list(set(dataset.query) | set(dataset.gallery)),
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
return dataset, num_classes, train_loader, test_loader
def get_data(dataset_name, split_id, data_dir, batch_size, workers,
num_instances, combine_trainval=False):
root = osp.join(data_dir, dataset_name)
dataset = get_dataset(dataset_name, root,
split_id=split_id, num_val=1, download=True)
normalizer = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_set = dataset.trainval if combine_trainval else dataset.train
num_classes = (dataset.num_trainval_ids if combine_trainval
else dataset.num_train_ids)
train_processor = Preprocessor(train_set, root=dataset.images_dir,
transform=transforms.Compose([
transforms.RandomSizedRectCrop(256, 128),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalizer,
]))
if num_instances > 0:
train_loader = DataLoader(
train_processor, batch_size=batch_size, num_workers=workers,
sampler=RandomIdentitySampler(train_set, num_instances),
pin_memory=True)
else:
train_loader = DataLoader(
train_processor, batch_size=batch_size, num_workers=workers,
shuffle=True, pin_memory=True)
val_loader = DataLoader(
Preprocessor(dataset.val, root=dataset.images_dir,
transform=transforms.Compose([
transforms.RectScale(256, 128),
transforms.ToTensor(),
normalizer,
])),
batch_size=batch_size, num_workers=workers,
shuffle=False, pin_memory=True)
test_loader = DataLoader(
Preprocessor(list(set(dataset.query) | set(dataset.gallery)),
root=dataset.images_dir,
transform=transforms.Compose([
transforms.RectScale(256, 128),
transforms.ToTensor(),
normalizer,
])),
batch_size=batch_size, num_workers=workers,
shuffle=False, pin_memory=True)
return dataset, num_classes, train_loader, val_loader, test_loader
def get_data(name, split_id, data_dir, height, width, batch_size, workers,
combine_trainval, np_ratio):
root = osp.join(data_dir, name)
dataset = datasets.create(name, root, split_id=split_id)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_set = dataset.trainval if combine_trainval else dataset.train
train_transformer = T.Compose([
T.RandomSizedRectCrop(height, width),
T.RandomSizedEarser(),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalizer,
])
test_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
normalizer,
])
train_loader = DataLoader(Preprocessor(train_set,
root=dataset.images_dir,
transform=train_transformer),
sampler=RandomPairSampler(
train_set, neg_pos_ratio=np_ratio),
batch_size=batch_size,
num_workers=workers,
pin_memory=False)
val_loader = DataLoader(Preprocessor(dataset.val,
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=0,
shuffle=False,
pin_memory=False)
test_loader = DataLoader(Preprocessor(
list(set(dataset.query) | set(dataset.gallery)),
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=0,
shuffle=False,
pin_memory=False)
return dataset, train_loader, val_loader, test_loader
def get_data(dataname, data_dir, height, width, batch_size, camstyle=0, re=0, workers=8):
root = osp.join(data_dir, dataname)
dataset = datasets.create(dataname, root)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
num_classes = dataset.num_train_ids
train_transformer = T.Compose([
T.RandomSizedRectCrop(height, width),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalizer,
T.RandomErasing(EPSILON=re),
])
test_transformer = T.Compose([
T.Resize((height, width), interpolation=3),
T.ToTensor(),
normalizer,
])
train_loader = DataLoader(
Preprocessor(dataset.train, root=osp.join(dataset.images_dir, dataset.train_path),
transform=train_transformer),
batch_size=batch_size, num_workers=workers,
shuffle=True, pin_memory=True, drop_last=True)
query_loader = DataLoader(
Preprocessor(dataset.query,
root=osp.join(dataset.images_dir, dataset.query_path), transform=test_transformer),
batch_size=batch_size, num_workers=workers,
shuffle=False, pin_memory=True)
gallery_loader = DataLoader(
Preprocessor(dataset.gallery,
root=osp.join(dataset.images_dir, dataset.gallery_path), transform=test_transformer),
batch_size=batch_size, num_workers=workers,
shuffle=False, pin_memory=True)
if camstyle <= 0:
camstyle_loader = None
else:
camstyle_loader = DataLoader(
Preprocessor(dataset.camstyle, root=osp.join(dataset.images_dir, dataset.camstyle_path),
transform=train_transformer),
batch_size=camstyle, num_workers=workers,
shuffle=True, pin_memory=True, drop_last=True)
return dataset, num_classes, train_loader, query_loader, gallery_loader, camstyle_loader
def get_data(data_dir, source, target, height, width, batch_size, re=0, workers=8):
dataset = DA(data_dir, source, target)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
num_classes = dataset.num_train_ids
train_transformer = T.Compose([
T.RandomSizedRectCrop(height, width),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalizer,
T.RandomErasing(EPSILON=re),
])
test_transformer = T.Compose([
T.Resize((height, width), interpolation=3),
T.ToTensor(),
normalizer,
])
source_train_loader = DataLoader(
Preprocessor(dataset.source_train, root=osp.join(dataset.source_images_dir, dataset.source_train_path),
transform=train_transformer),
batch_size=batch_size, num_workers=workers,
shuffle=True, pin_memory=True, drop_last=True)
target_train_loader = DataLoader(
UnsupervisedCamStylePreprocessor(dataset.target_train,
root=osp.join(dataset.target_images_dir, dataset.target_train_path),
camstyle_root=osp.join(dataset.target_images_dir,
dataset.target_train_camstyle_path),
num_cam=dataset.target_num_cam, transform=train_transformer),
batch_size=batch_size, num_workers=workers,
shuffle=True, pin_memory=True, drop_last=True)
query_loader = DataLoader(
Preprocessor(dataset.query,
root=osp.join(dataset.target_images_dir, dataset.query_path), transform=test_transformer),
batch_size=batch_size, num_workers=workers,
shuffle=False, pin_memory=True)
gallery_loader = DataLoader(
Preprocessor(dataset.gallery,
root=osp.join(dataset.target_images_dir, dataset.gallery_path), transform=test_transformer),
batch_size=batch_size, num_workers=workers,
shuffle=False, pin_memory=True)
return dataset, num_classes, source_train_loader, target_train_loader, query_loader, gallery_loader
def get_data(split_id, data_dir, height, width, batchSize, workers, combine_trainval, train_list, \
val_list, query_list, gallery_list):
root = data_dir
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225]) # RGB imagenet
train_set = train_list + val_list if combine_trainval else train_list # a list
train_transformer = T.Compose([
T.RectScale(height, width),
T.RandomHorizontalFlip(),
T.ToTensor(), # [0, 255] to [0.0, 1.0]
normalizer, # normalize each channel of the input
])
test_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
normalizer,
])
train_loader = DataLoader(
Preprocessor(train_set, root=root, transform=train_transformer),
batch_size=batchSize,
num_workers=workers,
sampler=RandomSampler(train_set),
# shuffle=True,
pin_memory=True,
drop_last=True)
val_loader = DataLoader(Preprocessor(val_list,
root=root,
transform=test_transformer),
batch_size=batchSize,
num_workers=workers,
shuffle=False,
pin_memory=True)
test_loader = DataLoader(Preprocessor(
list(set(query_list) | set(gallery_list)),
root=root,
transform=test_transformer),
batch_size=batchSize,
num_workers=workers,
shuffle=False,
pin_memory=True)
return train_loader, val_loader, test_loader
def create_test_data_loader(args, name, dataset):
train_transformer = T.Compose([
T.RectScale(args.height, args.width),
T.ToTensor(),
T.Normalize(mean=[0.486, 0.459, 0.408], std=[0.229, 0.224, 0.225])
])
test_transformer = T.Compose([
T.RectScale(args.height, args.width),
T.ToTensor(),
T.Normalize(mean=[0.486, 0.459, 0.408], std=[0.229, 0.224, 0.225])
])
train_loader = DataLoader(Preprocessor(dataset.train,
root=os.path.join(
dataset.images_dir,
dataset.train_path),
transform=train_transformer),
batch_size=args.batch_size,
num_workers=args.workers,
shuffle=False,
pin_memory=True,
drop_last=False)
query_loader = DataLoader(Preprocessor(dataset.query,
root=os.path.join(
dataset.images_dir,
dataset.query_path),
transform=test_transformer),
batch_size=args.batch_size * 4,
num_workers=args.workers,
shuffle=False,
pin_memory=True,
drop_last=False)
gallery_loader = DataLoader(Preprocessor(dataset.gallery,
root=os.path.join(
dataset.images_dir,
dataset.gallery_path),
transform=test_transformer),
batch_size=args.batch_size * 4,
num_workers=args.workers,
shuffle=False,
pin_memory=True,
drop_last=False)
print('{} Datasets Has beed loaded.'.format(name))
return train_loader, query_loader, gallery_loader
def get_dataloader(dataset,data_dir,
training=False, height=256,
width=128, batch_size=64, workers=1):
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
if training:
transformer = T.Compose([
T.RandomSizedRectCrop(height, width),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalizer,
])
else:
transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
normalizer,
])
data_loader = DataLoader(
Preprocessor(dataset, root=data_dir,
transform=transformer),
batch_size=batch_size, num_workers=workers,
shuffle=training, pin_memory=True, drop_last=training)
return data_loader
def get_train_loader(dataset, height, width, batch_size, workers,
num_instances, iters, trainset=None):
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_transformer = T.Compose([
T.Resize((height, width), interpolation=3),
T.RandomHorizontalFlip(p=0.5),
T.Pad(10),
T.RandomCrop((height, width)),
T.ToTensor(),
normalizer,
T.RandomErasing(probability=0.5, mean=[0.485, 0.456, 0.406])
])
train_set = dataset.train if trainset is None else trainset
rmgs_flag = num_instances > 0
if rmgs_flag:
sampler = RandomMultipleGallerySampler(train_set, num_instances)
else:
sampler = None
train_loader = IterLoader(
DataLoader(Preprocessor(train_set, root=dataset.images_dir,
transform=train_transformer, mutual=False),
batch_size=batch_size, num_workers=workers, sampler=sampler,
shuffle=not rmgs_flag, pin_memory=True, drop_last=True), length=iters)
return train_loader
def get_loader(data,
root,
height=256,
width=128,
batch_size=32,
workers=0,
training=False):
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
if training:
transformer = T.Compose([
T.RandomSizedRectCrop(height, width), # 对图像进行随机裁剪并缩放.
T.RandomHorizontalFlip(), # 对给定的PIL.Image进行随机水平翻转,概率为0.5,属于数据增强.
T.ToTensor(), # 将numpy图像转换为torch图像.
normalizer,
])
else:
transformer = T.Compose([
T.RectScale(height, width), # 缩放图像.
T.ToTensor(),
normalizer,
])
batch_size = batch_size * 8
data_loader = DataLoader(Preprocessor(data,
root=root,
transform=transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=training,
pin_memory=True)
return data_loader
def get_data(sourceName, mteName, targetName, split_id, data_dir, height, width,
batch_size, workers, combine_trainval):
root = osp.join(data_dir, sourceName)
rootMte = osp.join(data_dir, mteName)
rootTgt = osp.join(data_dir, targetName)
sourceSet = datasets.create(sourceName, root, num_val=0.1, split_id=split_id)
mteSet = datasets.create(mteName, rootMte, num_val=0.1, split_id=split_id)
tgtSet = datasets.create(targetName, rootTgt, num_val=0.1, split_id=split_id)
num_classes = (sourceSet.num_trainval_ids if combine_trainval else sourceSet.num_train_ids)
class_tgt = (tgtSet.num_trainval_ids if combine_trainval else tgtSet.num_train_ids)
train_transformer = T.Compose([
Resize((height, width)),
T.ToTensor(),
])
# generate dataset
curSet = [(osp.join(sourceSet.images_dir, val[0]), val[1], val[2]) for val in sourceSet.trainval]
for val in mteSet.trainval:
curSet.append((osp.join(mteSet.images_dir, val[0]), num_classes + int(val[1]), int(val[2])))
meta_train = DataLoader(
Preprocessor(curSet, root=None, transform=train_transformer),
batch_size=batch_size, num_workers=workers, shuffle=False, pin_memory=True)
return sourceSet, tgtSet, mteSet, num_classes, class_tgt, meta_train
def get_dataloader(dataset,data_dir,
training=False, height=256,
width=128, batch_size=64, workers=1):
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])#(0.1307,), (0.3081,)
# normalizer = T.Normalize(mean=[0.1307, 0.1307, 0.1307],
# std=[0.3081, 0.3081, 0.3081])#, (0.3081,)
if training:
transformer = T.Compose([
T.RectScale(config.input_size + 32, config.input_size + 32),
torchvision.transforms.RandomCrop(config.input_size),
torchvision.transforms.RandomHorizontalFlip(),
# RandomRotation(degrees=20),
T.ToTensor(),
normalizer,
])
else:
transformer = T.Compose([
T.RectScale(config.input_size, config.input_size),
T.ToTensor(),
normalizer,
])
data_loader = DataLoader(
Preprocessor(dataset, root=data_dir,
transform=transformer),
batch_size=batch_size, num_workers=workers,
shuffle=training, pin_memory=True, drop_last=training)
return data_loader
def get_data(sourceName, targetName, split_id, data_dir, height, width,
batch_size, workers, combine):
root = osp.join(data_dir, sourceName)
rootTgt = osp.join(data_dir, targetName)
sourceSet = datasets.create(sourceName,
root,
num_val=0.1,
split_id=split_id)
tgtSet = datasets.create(targetName,
rootTgt,
num_val=0.1,
split_id=split_id)
num_classes = sourceSet.num_trainval_ids if combine else sourceSet.num_train_ids
class_tgt = tgtSet.num_trainval_ids if combine else tgtSet.num_train_ids
train_transformer = T.Compose([
Resize((height, width)),
T.ToTensor(),
])
train_loader = DataLoader(Preprocessor(sourceSet.trainval,
root=sourceSet.images_dir,
transform=train_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
return sourceSet, tgtSet, num_classes, class_tgt, train_loader
def get_dataloader(self, dataset, training=False) :
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
if training:
transformer = T.Compose([
T.RandomSizedRectCrop(self.data_height, self.data_width),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalizer,
])
batch_size = self.batch_size
else:
transformer = T.Compose([
T.RectScale(self.data_height, self.data_width),
T.ToTensor(),
normalizer,
])
batch_size = self.eval_bs
data_loader = DataLoader(
Preprocessor(dataset, root=self.data_dir,
transform=transformer, is_training=training, max_frames=self.max_frames),
batch_size=batch_size, num_workers=self.data_workers,
shuffle=training, pin_memory=True, drop_last=training)
current_status = "Training" if training else "Test"
print("create dataloader for {} with batch_size {}".format(current_status, batch_size))
return data_loader
def get_source_data(name, data_dir, height, width, batch_size,
workers):
root = osp.join(data_dir, name)
dataset = datasets.create(name, root, num_val=0.1)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_set = dataset.trainval # use all training image to train
num_classes = dataset.num_trainval_ids
transformer = T.Compose([
Resize((height,width)),
T.ToTensor(),
normalizer,
])
extfeat_loader = DataLoader(
Preprocessor(train_set, root=dataset.images_dir,
transform=transformer),
batch_size=batch_size, num_workers=workers,
shuffle=False, pin_memory=True)
return dataset, extfeat_loader
def get_data(sourceName, mteName, split_id, data_dir, height, width,
batch_size, workers, combine,num_instances=8):
root = osp.join(data_dir, sourceName)
rootMte = osp.join(data_dir, mteName)
sourceSet = datasets.create(sourceName, root, num_val=0.1, split_id=split_id)
mteSet = datasets.create(mteName, rootMte, num_val=0.1, split_id=split_id)
num_classes = sourceSet.num_trainval_ids if combine else sourceSet.num_train_ids
class_meta = mteSet.num_trainval_ids if combine else mteSet.num_train_ids
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
test_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
normalizer,
])
defen_train_transformer = T.Compose([
Resize((height, width)),
T.RandomSizedRectCrop(height, width),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalizer,
T.RandomErasing(probability=0.5, sh=0.2, r1=0.3)
])
meta_train_loader = DataLoader(
Preprocessor(sourceSet.trainval, root=sourceSet.images_dir,
transform=defen_train_transformer),
batch_size=batch_size, num_workers=workers,
sampler=RandomIdentitySampler(sourceSet.trainval, num_instances),
pin_memory=True, drop_last=True)
meta_test_loader=DataLoader(
Preprocessor(mteSet.trainval, root=mteSet.images_dir,
transform=defen_train_transformer),
batch_size=batch_size, num_workers=workers,
sampler=RandomIdentitySampler(mteSet.trainval, num_instances),
pin_memory=True, drop_last=True)
sc_test_loader = DataLoader(
Preprocessor(list(set(sourceSet.query) | set(sourceSet.gallery)),
root=sourceSet.images_dir, transform=test_transformer),
batch_size=batch_size, num_workers=workers,
shuffle=False, pin_memory=True)
return sourceSet, mteSet, num_classes, meta_train_loader, meta_test_loader,sc_test_loader,class_meta
def get_data(name, split_id, data_dir, height, width, batch_size, workers,
combine_trainval):
root = os.path.join(data_dir, name)
dataset = datasets.create(name, root, split_id=split_id)
train_set = dataset.trainval if combine_trainval else dataset.train
num_classes = (dataset.num_trainval_ids
if combine_trainval else dataset.num_train_ids)
train_transformer = t.Compose([
t.RandomSizedRectCrop(height, width),
t.RandomHorizontalFlip(),
ColorHistograms()
])
test_transformer = t.Compose(
[t.RectScale(height, width),
ColorHistograms()])
train_loader = DataLoader(Preprocessor(train_set,
root=dataset.images_dir,
transform=train_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=True,
pin_memory=True,
drop_last=True)
val_loader = DataLoader(Preprocessor(dataset.val,
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
test_loader = DataLoader(Preprocessor(
list(set(dataset.query) | set(dataset.gallery)),
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
return dataset, num_classes, train_loader, val_loader, test_loader
def get_data(data_dir, target, height, width, batch_size, re=0, workers=8):
dataset = DA(data_dir, target, generate_propagate_data=True)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
num_classes = dataset.num_train_ids
test_transformer = T.Compose([
T.Resize((height, width), interpolation=3),
T.ToTensor(),
normalizer,
])
propagate_loader = DataLoader(UnsupervisedTargetPreprocessor(
dataset.target_train_original,
root=osp.join(dataset.target_images_dir, dataset.target_train_path),
num_cam=dataset.target_num_cam,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
query_loader = DataLoader(Preprocessor(dataset.query,
root=osp.join(
dataset.target_images_dir,
dataset.query_path),
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
gallery_loader = DataLoader(Preprocessor(dataset.gallery,
root=osp.join(
dataset.target_images_dir,
dataset.gallery_path),
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
return dataset, num_classes, query_loader, gallery_loader, propagate_loader
