def get_transformer(config):
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
base_transformer = [T.ToTensor(), normalizer]
if config.training is False:
return T.Compose([T.Resize((config.height, config.width))] + base_transformer)
if config.img_translation is None:
return T.Compose([T.RandomSizedRectCrop(config.height, config.width),
T.RandomHorizontalFlip()] + base_transformer)
return T.Compose([T.RandomTranslateWithReflect(config.img_translation),
T.RandomSizedRectCrop(config.height, config.width),
T.RandomHorizontalFlip()] + base_transformer)
def get_data(name, split_id, data_dir, height, width, batch_size, workers,
combine_trainval):
root = osp.join(data_dir, name)
dataset = datasets.create(name, root, split_id=split_id)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_set = dataset.trainval if combine_trainval else dataset.train
num_classes = (dataset.num_trainval_ids
if combine_trainval else dataset.num_train_ids)
train_transformer = T.Compose([
T.RandomSizedRectCrop(height, width),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalizer,
])
test_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
normalizer,
])
train_loader = DataLoader(Preprocessor(train_set,
root=dataset.images_dir,
transform=train_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=True,
pin_memory=True,
drop_last=True)
val_loader = DataLoader(Preprocessor(dataset.val,
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
test_loader = DataLoader(Preprocessor(
list(set(dataset.query) | set(dataset.gallery)),
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
pid_train = np.array(list(pid for _, pid, _ in train_set))
class_weight = np.array([(pid_train == i).sum()
for i in range(num_classes)])
assert np.all(class_weight != 0)
class_weight = pid_train.shape[0] / num_classes / class_weight
class_weight = torch.Tensor(class_weight).cuda()
return dataset, num_classes, class_weight, train_loader, val_loader, test_loader
def get_data(name, split_id, data_dir, height, width, batch_size,
num_instances, workers, combine_trainval, batch_id):
root = osp.join(data_dir, name)
if name == 'synthetic':
dataset = datasets.create(name,
root,
split_id=split_id,
batch_id=batch_id)
else:
dataset = datasets.create(name, root, split_id=split_id)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_set = dataset.trainval if combine_trainval else dataset.train
num_classes = (dataset.num_trainval_ids
if combine_trainval else dataset.num_train_ids)
train_transformer = T.Compose([
T.RandomSizedRectCrop(height, width),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalizer,
])
test_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
normalizer,
])
train_loader = DataLoader(Preprocessor(train_set,
root=dataset.images_dir,
transform=train_transformer),
batch_size=batch_size,
num_workers=workers,
sampler=RandomIdentitySampler(
train_set, num_instances),
pin_memory=True,
drop_last=True)
val_loader = DataLoader(Preprocessor(dataset.val,
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
test_loader = DataLoader(Preprocessor(
list(set(dataset.query) | set(dataset.gallery)),
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
return dataset, num_classes, train_loader, val_loader, test_loader
def get_dataloader(dataset,data_dir,
training=False, height=256,
width=128, batch_size=64, workers=1):
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
if training:
transformer = T.Compose([
T.RandomSizedRectCrop(height, width),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalizer,
])
else:
transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
normalizer,
])
data_loader = DataLoader(
Preprocessor(dataset, root=data_dir,
transform=transformer),
batch_size=batch_size, num_workers=workers,
shuffle=training, pin_memory=True, drop_last=training)
return data_loader
def get_loader(data,
root,
height=256,
width=128,
batch_size=32,
workers=0,
training=False):
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
if training:
transformer = T.Compose([
T.RandomSizedRectCrop(height, width), # 对图像进行随机裁剪并缩放.
T.RandomHorizontalFlip(), # 对给定的PIL.Image进行随机水平翻转,概率为0.5,属于数据增强.
T.ToTensor(), # 将numpy图像转换为torch图像.
normalizer,
])
else:
transformer = T.Compose([
T.RectScale(height, width), # 缩放图像.
T.ToTensor(),
normalizer,
])
batch_size = batch_size * 8
data_loader = DataLoader(Preprocessor(data,
root=root,
transform=transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=training,
pin_memory=True)
return data_loader
def get_dataloader(self, dataset, training=False) :
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
if training:
transformer = T.Compose([
T.RandomSizedRectCrop(self.data_height, self.data_width),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalizer,
])
batch_size = self.batch_size
else:
transformer = T.Compose([
T.RectScale(self.data_height, self.data_width),
T.ToTensor(),
normalizer,
])
batch_size = self.eval_bs
data_loader = DataLoader(
Preprocessor(dataset, root=self.data_dir,
transform=transformer, is_training=training, max_frames=self.max_frames),
batch_size=batch_size, num_workers=self.data_workers,
shuffle=training, pin_memory=True, drop_last=training)
current_status = "Training" if training else "Test"
print("create dataloader for {} with batch_size {}".format(current_status, batch_size))
return data_loader
def get_data(dataset_name, split_id, data_dir, batch_size, workers,
num_instances, combine_trainval=False):
root = osp.join(data_dir, dataset_name)
dataset = get_dataset(dataset_name, root,
split_id=split_id, num_val=1, download=True)
normalizer = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_set = dataset.trainval if combine_trainval else dataset.train
num_classes = (dataset.num_trainval_ids if combine_trainval
else dataset.num_train_ids)
train_processor = Preprocessor(train_set, root=dataset.images_dir,
transform=transforms.Compose([
transforms.RandomSizedRectCrop(256, 128),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalizer,
]))
if num_instances > 0:
train_loader = DataLoader(
train_processor, batch_size=batch_size, num_workers=workers,
sampler=RandomIdentitySampler(train_set, num_instances),
pin_memory=True)
else:
train_loader = DataLoader(
train_processor, batch_size=batch_size, num_workers=workers,
shuffle=True, pin_memory=True)
val_loader = DataLoader(
Preprocessor(dataset.val, root=dataset.images_dir,
transform=transforms.Compose([
transforms.RectScale(256, 128),
transforms.ToTensor(),
normalizer,
])),
batch_size=batch_size, num_workers=workers,
shuffle=False, pin_memory=True)
test_loader = DataLoader(
Preprocessor(list(set(dataset.query) | set(dataset.gallery)),
root=dataset.images_dir,
transform=transforms.Compose([
transforms.RectScale(256, 128),
transforms.ToTensor(),
normalizer,
])),
batch_size=batch_size, num_workers=workers,
shuffle=False, pin_memory=True)
return dataset, num_classes, train_loader, val_loader, test_loader
def get_data(name, split_id, data_dir, height, width, batch_size,
num_instances, workers, combine_trainval):
root = osp.join(data_dir, name)
dataset = datasets.create(name, root, split_id=split_id)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
# load train_set, query_set, gallery_set
mt_train_set = dataset.train
mt_num_classes = dataset.num_train_tids_sub
query_set = dataset.query
gallery_set = dataset.gallery
train_transformer = T.Compose([
T.RandomSizedRectCrop(height, width),
T.RandomHorizontalFlip(),
T.ToTensor(), normalizer
])
test_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
normalizer,
])
# Random ID
mt_train_set = flatten_dataset(mt_train_set)
num_task = len(
mt_num_classes) # num_task equals camera number, each camera is a task
mt_train_loader = DataLoader(
Preprocessor_Image(mt_train_set,
root=dataset.dataset_dir,
transform=train_transformer),
batch_size=batch_size,
num_workers=workers,
sampler=RandomIdentitySampler(
mt_train_set, num_instances,
num_task), # Here is different between softmax_loss
pin_memory=True,
drop_last=True)
query_set = flatten_dataset(query_set)
gallery_set = flatten_dataset(gallery_set)
test_set = list(set(query_set) | set(gallery_set))
test_loader = DataLoader(Preprocessor_Image(test_set,
root=dataset.dataset_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
return mt_train_loader, mt_num_classes, test_loader, query_set, gallery_set
def get_data(dataname, data_dir, height, width, batch_size, camstyle=0, re=0, workers=8):
root = osp.join(data_dir, dataname)
dataset = datasets.create(dataname, root)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
num_classes = dataset.num_train_ids
train_transformer = T.Compose([
T.RandomSizedRectCrop(height, width),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalizer,
T.RandomErasing(EPSILON=re),
])
test_transformer = T.Compose([
T.Resize((height, width), interpolation=3),
T.ToTensor(),
normalizer,
])
train_loader = DataLoader(
Preprocessor(dataset.train, root=osp.join(dataset.images_dir, dataset.train_path),
transform=train_transformer),
batch_size=batch_size, num_workers=workers,
shuffle=True, pin_memory=True, drop_last=True)
query_loader = DataLoader(
Preprocessor(dataset.query,
root=osp.join(dataset.images_dir, dataset.query_path), transform=test_transformer),
batch_size=batch_size, num_workers=workers,
shuffle=False, pin_memory=True)
gallery_loader = DataLoader(
Preprocessor(dataset.gallery,
root=osp.join(dataset.images_dir, dataset.gallery_path), transform=test_transformer),
batch_size=batch_size, num_workers=workers,
shuffle=False, pin_memory=True)
if camstyle <= 0:
camstyle_loader = None
else:
camstyle_loader = DataLoader(
Preprocessor(dataset.camstyle, root=osp.join(dataset.images_dir, dataset.camstyle_path),
transform=train_transformer),
batch_size=camstyle, num_workers=workers,
shuffle=True, pin_memory=True, drop_last=True)
return dataset, num_classes, train_loader, query_loader, gallery_loader, camstyle_loader
def get_data(name, split_id, data_dir, height, width, batch_size, workers,
combine_trainval, np_ratio):
root = osp.join(data_dir, name)
dataset = datasets.create(name, root, split_id=split_id)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_set = dataset.trainval if combine_trainval else dataset.train
train_transformer = T.Compose([
T.RandomSizedRectCrop(height, width),
T.RandomSizedEarser(),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalizer,
])
test_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
normalizer,
])
train_loader = DataLoader(Preprocessor(train_set,
root=dataset.images_dir,
transform=train_transformer),
sampler=RandomPairSampler(
train_set, neg_pos_ratio=np_ratio),
batch_size=batch_size,
num_workers=workers,
pin_memory=False)
val_loader = DataLoader(Preprocessor(dataset.val,
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=0,
shuffle=False,
pin_memory=False)
test_loader = DataLoader(Preprocessor(
list(set(dataset.query) | set(dataset.gallery)),
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=0,
shuffle=False,
pin_memory=False)
return dataset, train_loader, val_loader, test_loader
def get_data(data_dir, source, target, height, width, batch_size, re=0, workers=8):
dataset = DA(data_dir, source, target)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
num_classes = dataset.num_train_ids
train_transformer = T.Compose([
T.RandomSizedRectCrop(height, width),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalizer,
T.RandomErasing(EPSILON=re),
])
test_transformer = T.Compose([
T.Resize((height, width), interpolation=3),
T.ToTensor(),
normalizer,
])
source_train_loader = DataLoader(
Preprocessor(dataset.source_train, root=osp.join(dataset.source_images_dir, dataset.source_train_path),
transform=train_transformer),
batch_size=batch_size, num_workers=workers,
shuffle=True, pin_memory=True, drop_last=True)
target_train_loader = DataLoader(
UnsupervisedCamStylePreprocessor(dataset.target_train,
root=osp.join(dataset.target_images_dir, dataset.target_train_path),
camstyle_root=osp.join(dataset.target_images_dir,
dataset.target_train_camstyle_path),
num_cam=dataset.target_num_cam, transform=train_transformer),
batch_size=batch_size, num_workers=workers,
shuffle=True, pin_memory=True, drop_last=True)
query_loader = DataLoader(
Preprocessor(dataset.query,
root=osp.join(dataset.target_images_dir, dataset.query_path), transform=test_transformer),
batch_size=batch_size, num_workers=workers,
shuffle=False, pin_memory=True)
gallery_loader = DataLoader(
Preprocessor(dataset.gallery,
root=osp.join(dataset.target_images_dir, dataset.gallery_path), transform=test_transformer),
batch_size=batch_size, num_workers=workers,
shuffle=False, pin_memory=True)
return dataset, num_classes, source_train_loader, target_train_loader, query_loader, gallery_loader
def get_dataloader(self, dataset, training=False):
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
if training: # import transforms as T
transformer = T.Compose([
T.RandomSizedRectCrop(
self.data_height,
self.data_width), # data_height = 256 data_width = 128
T.RandomHorizontalFlip(), # 随机水平翻转
T.ToTensor(
), # Convert a ``PIL Image`` or ``numpy.ndarray`` to tensor.
normalizer, # normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
# std=[0.229, 0.224, 0.225])
])
batch_size = self.batch_size # batch_size = 16
else:
transformer = T.Compose([
T.RectScale(self.data_height,
self.data_width), # RectScale():三角缩放(?)
T.ToTensor(
), # Convert a ``PIL Image`` or ``numpy.ndarray`` to tensor.
normalizer, # normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
]) # std=[0.229, 0.224, 0.225])
batch_size = self.eval_bs # batch_size = 64
data_dir = self.data_dir # data_dir = dataset_all.image_dir
data_loader = DataLoader( # DataLoader()
Preprocessor(
dataset,
root=data_dir,
num_samples=self.
frames_per_video, # root = dataset_all.image_dir num_samples = 1
transform=transformer,
is_training=training,
max_frames=self.max_frames
), # transform = T.compose()返回值 is_training = False max_frames = 900
batch_size=batch_size,
num_workers=self.
data_workers, # batch_size = 16 data_workers = 6
shuffle=training,
pin_memory=True,
drop_last=training) # shuffle = True drop_last = True
current_status = "Training" if training else "Testing" # current_status = 'Training'
print("Create dataloader for {} with batch_size {}".format(
current_status,
batch_size)) # Create dataloader for Training with batch_size 16
return data_loader
def get_data(name, split_id, data_dir, height, width, batch_size, workers,
combine_trainval):
root = os.path.join(data_dir, name)
dataset = datasets.create(name, root, split_id=split_id)
train_set = dataset.trainval if combine_trainval else dataset.train
num_classes = (dataset.num_trainval_ids
if combine_trainval else dataset.num_train_ids)
train_transformer = t.Compose([
t.RandomSizedRectCrop(height, width),
t.RandomHorizontalFlip(),
ColorHistograms()
])
test_transformer = t.Compose(
[t.RectScale(height, width),
ColorHistograms()])
train_loader = DataLoader(Preprocessor(train_set,
root=dataset.images_dir,
transform=train_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=True,
pin_memory=True,
drop_last=True)
val_loader = DataLoader(Preprocessor(dataset.val,
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
test_loader = DataLoader(Preprocessor(
list(set(dataset.query) | set(dataset.gallery)),
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
return dataset, num_classes, train_loader, val_loader, test_loader
def get_data(sourceName, mteName, split_id, data_dir, height, width,
batch_size, workers, combine,num_instances=8):
root = osp.join(data_dir, sourceName)
rootMte = osp.join(data_dir, mteName)
sourceSet = datasets.create(sourceName, root, num_val=0.1, split_id=split_id)
mteSet = datasets.create(mteName, rootMte, num_val=0.1, split_id=split_id)
num_classes = sourceSet.num_trainval_ids if combine else sourceSet.num_train_ids
class_meta = mteSet.num_trainval_ids if combine else mteSet.num_train_ids
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
test_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
normalizer,
])
defen_train_transformer = T.Compose([
Resize((height, width)),
T.RandomSizedRectCrop(height, width),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalizer,
T.RandomErasing(probability=0.5, sh=0.2, r1=0.3)
])
meta_train_loader = DataLoader(
Preprocessor(sourceSet.trainval, root=sourceSet.images_dir,
transform=defen_train_transformer),
batch_size=batch_size, num_workers=workers,
sampler=RandomIdentitySampler(sourceSet.trainval, num_instances),
pin_memory=True, drop_last=True)
meta_test_loader=DataLoader(
Preprocessor(mteSet.trainval, root=mteSet.images_dir,
transform=defen_train_transformer),
batch_size=batch_size, num_workers=workers,
sampler=RandomIdentitySampler(mteSet.trainval, num_instances),
pin_memory=True, drop_last=True)
sc_test_loader = DataLoader(
Preprocessor(list(set(sourceSet.query) | set(sourceSet.gallery)),
root=sourceSet.images_dir, transform=test_transformer),
batch_size=batch_size, num_workers=workers,
shuffle=False, pin_memory=True)
return sourceSet, mteSet, num_classes, meta_train_loader, meta_test_loader,sc_test_loader,class_meta
def get_dataloader(self, dataset, training=False):
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
if training:
transformer = T.Compose([
T.RandomSizedRectCrop(self.data_height, self.data_width),
T.RandomHorizontalFlip(),
T.ToTensor(), normalizer,
T.RandomErasing(probability=0.5, sh=0.2, r1=0.3)
])
else:
transformer = T.Compose([
T.Resize((self.data_height, self.data_width)),
T.ToTensor(),
normalizer,
])
if training and self.num_classes == 0:
data_loader = DataLoader(Preprocessor(dataset,
root='',
transform=transformer),
batch_size=self.batch_size,
num_workers=self.data_workers,
sampler=RandomIdentitySampler(
dataset, self.num_instances),
pin_memory=True,
drop_last=training)
else:
data_loader = DataLoader(Preprocessor(dataset,
root='',
transform=transformer),
batch_size=self.batch_size,
num_workers=self.data_workers,
shuffle=training,
pin_memory=True,
drop_last=training)
current_status = "Training" if training else "Test"
print("create dataloader for {} with batch_size {}".format(
current_status, self.batch_size))
return data_loader
def get_data(name, split_id, data_dir, height, width, batch_size,
num_instances, workers, combine_trainval):
root = osp.join(data_dir, name)
dataset = datasets.create(name, root, split_id=split_id)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
# load train_set, query_set, gallery_set
mt_train_set = dataset.train
mt_num_classes = dataset.num_train_tids_sub
query_set = dataset.query
gallery_set = dataset.gallery
train_transformer = T.Compose([
T.RandomSizedRectCrop(height, width),
T.RandomHorizontalFlip(),
T.ToTensor(), normalizer
])
test_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
normalizer,
])
# Random ID
mt_train_set = flatten_dataset(mt_train_set)
num_task = len(
mt_num_classes) # num_task equals camera number, each camera is a task
mt_train_loader = DataLoader(
Preprocessor_Image(mt_train_set,
root=dataset.dataset_dir,
transform=train_transformer),
batch_size=batch_size,
num_workers=workers,
sampler=RandomIdentitySampler(
mt_train_set, num_instances,
num_task), # Here is different between softmax_loss
pin_memory=True,
drop_last=True)
# correct format conflict
query_set_new = []
for index in range(len(query_set)):
img_paths = tuple(query_set[index][0])
tid = query_set[index][1]
pid = query_set[index][2]
tid_sub = query_set[index][3]
pid_sub = query_set[index][4]
camid = query_set[index][5]
query_set_new.append((img_paths, tid, pid, tid_sub, pid_sub, camid))
gallery_set_new = []
for index in range(len(gallery_set)):
img_paths = tuple(gallery_set[index][0])
tid = gallery_set[index][1]
pid = gallery_set[index][2]
tid_sub = gallery_set[index][3]
pid_sub = gallery_set[index][4]
camid = gallery_set[index][5]
query_set_new.append((img_paths, tid, pid, tid_sub, pid_sub, camid))
test_set = list(set(query_set_new) | set(gallery_set_new))
seq_len = 16
test_loader = DataLoader(Preprocessor_Video(test_set,
transform=test_transformer,
seq_len=seq_len,
sample='random'),
batch_size=int(batch_size / seq_len),
num_workers=workers,
shuffle=False,
pin_memory=True)
return mt_train_loader, mt_num_classes, test_loader, query_set, gallery_set
def get_data(name, split_id, data_dir, height, width, batch_size,
num_instances, workers, combine_trainval):
root = osp.join(data_dir, name)
dataset = datasets.create(name, root, split_id=split_id)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_set = dataset.trainval if combine_trainval else dataset.train
num_classes = (dataset.num_trainval_ids
if combine_trainval else dataset.num_train_ids)
query = dataset.query
query_ids = [pid for _, pid, _ in query]
query_fnames = [fname for fname, _, _ in query]
query_cams = [cam for _, _, cam in query]
query_ids_unique = list(set(query_ids))
query_fnames_new, query_ids_new, query_cams_new = [], [], []
gallery_fnames_new, gallery_ids_new, gallery_cams_new = [], [], []
for k in query_ids_unique:
idx = query_ids.index(k)
query_ids_new.append(k)
query_fnames_new.append(query_fnames[idx])
query_cams_new.append(query_cams[idx])
new_idx = idx + 1
while query_cams[idx] == query_cams[new_idx]:
new_idx += 1
gallery_ids_new.append(k)
gallery_fnames_new.append(query_fnames[new_idx])
gallery_cams_new.append(query_cams[new_idx])
query_num = len(query_ids_unique)
query_test_num = 100 # 2 GPU
split_num = query_num // query_test_num
test_set = []
tmp = []
for k in range(split_num):
for i in range(2):
for j in range(k * query_test_num, (k + 1) * query_test_num):
if i == 0:
tmp.extend((query_fnames_new[j], query_ids_new[j],
query_cams_new[j]))
else:
tmp.extend((gallery_fnames_new[j], gallery_ids_new[j],
gallery_cams_new[j]))
test_set.append(tmp)
tmp = []
train_transformer = T.Compose([
T.RandomSizedRectCrop(height, width),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalizer,
])
test_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
normalizer,
])
train_loader = DataLoader(Preprocessor(train_set,
root=dataset.images_dir,
transform=train_transformer),
batch_size=batch_size,
num_workers=workers,
sampler=RandomIdentitySampler(
train_set, num_instances),
pin_memory=True,
drop_last=True)
val_loader = DataLoader(Preprocessor(dataset.val,
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
"""
test_loader = DataLoader(
Preprocessor(test_set, root=dataset.images_dir,
transform=test_transformer),
batch_size=2*query_test_num, num_workers=workers,
shuffle=False, pin_memory=True)
"""
test_loader = DataLoader(Preprocessor(
list(set(dataset.query) | set(dataset.gallery)),
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
return dataset, num_classes, train_loader, val_loader, test_loader
def main(args):
np.random.seed(args.seed)
torch.manual_seed(args.seed)
cudnn.benchmark = True
# Redirect print to both console and log file
if not args.evaluate:
sys.stdout = Logger(osp.join(args.logs_dir, 'log.txt'))
# Create data loaders
assert args.num_instances > 1, "num_instances should be greater than 1"
assert args.batch_size % args.num_instances == 0, \
'num_instances should divide batch_size'
if args.height is None or args.width is None:
args.height, args.width = (144, 56) if args.arch == 'inception' else \
(256, 128)
train, val, trainval = [], [], []
numbers = [0, 0, 0]
dataset_cuhk03 = merge('cuhk03', train, val, trainval, numbers, args.data_dir, args.split)
dataset_market1501 = merge('market1501', train, val, trainval, numbers, args.data_dir, args.split)
merge('cuhksysu', train, val, trainval, numbers, args.data_dir, args.split)
merge('mars', train, val, trainval, numbers, args.data_dir, args.split)
num_train_ids, num_val_ids, num_trainval_ids = numbers
assert num_val_ids == dataset_cuhk03.num_val_ids + dataset_market1501.num_val_ids
print("============================================")
print("JSTL dataset loaded")
print(" subset | # ids | # images")
print(" ---------------------------")
print(" train | {:5d} | {:8d}"
.format(num_train_ids, len(train)))
print(" val | {:5d} | {:8d}"
.format(num_val_ids, len(val)))
print(" trainval | {:5d} | {:8d}"
.format(num_trainval_ids, len(trainval)))
query_cuhk03, gallery_cuhk03 = dataset_cuhk03.query, dataset_cuhk03.gallery
query_market1501, gallery_market1501 = dataset_market1501.query, dataset_market1501.gallery
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_set = trainval if args.combine_trainval else train
num_classes = (num_trainval_ids if args.combine_trainval
else num_train_ids)
train_transformer = T.Compose([
T.RandomSizedRectCrop(args.height, args.width),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalizer,
])
test_transformer = T.Compose([
T.RectScale(args.height, args.width),
T.ToTensor(),
normalizer,
])
train_loader = DataLoader(
Preprocessor(train_set, root=args.data_dir,
transform=train_transformer),
batch_size=args.batch_size, num_workers=args.workers,
sampler=RandomIdentitySampler(train_set, args.num_instances),
pin_memory=True, drop_last=True)
val_loader = DataLoader(
Preprocessor(val, root=args.data_dir,
transform=test_transformer),
batch_size=args.batch_size, num_workers=args.workers,
shuffle=False, pin_memory=True)
test_loader_cuhk03 = DataLoader(
Preprocessor(list(set(query_cuhk03) | set(gallery_cuhk03)),
root=dataset_cuhk03.images_dir, transform=test_transformer),
batch_size=args.batch_size, num_workers=args.workers,
shuffle=False, pin_memory=True)
test_loader_market1501 = DataLoader(
Preprocessor(list(set(query_market1501) | set(gallery_market1501)),
root=dataset_market1501.images_dir, transform=test_transformer),
batch_size=args.batch_size, num_workers=args.workers,
shuffle=False, pin_memory=True)
# Create model
# Hacking here to let the classifier be the last feature embedding layer
# Net structure: avgpool -> FC(1024) -> FC(args.features)
model = models.create(args.arch, num_features=1024,
dropout=args.dropout, num_classes=args.features)
# Load from checkpoint
start_epoch = best_top1 = 0
if args.resume:
checkpoint = load_checkpoint(args.resume)
model.load_state_dict(checkpoint['state_dict'])
start_epoch = checkpoint['epoch']
best_top1 = checkpoint['best_top1']
print("=> Start epoch {} best top1 {:.1%}"
.format(start_epoch, best_top1))
model = nn.DataParallel(model).cuda()
# Distance metric
metric = DistanceMetric(algorithm=args.dist_metric)
# Evaluator
evaluator = Evaluator(model)
if args.evaluate:
metric.train(model, train_loader)
print("Validation:")
evaluator.evaluate(val_loader, val, val, metric)
print("Test(cuhk03):")
evaluator.evaluate(test_loader_cuhk03, query_cuhk03, gallery_cuhk03, metric)
print("Test(market1501):")
evaluator.evaluate(test_loader_market1501, query_market1501, gallery_market1501, metric)
return
# Criterion
criterion = TripletLoss(margin=args.margin).cuda()
# Optimizer
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr,
weight_decay=args.weight_decay)
# Trainer
trainer = Trainer(model, criterion)
# Schedule learning rate
def adjust_lr(epoch):
lr = args.lr if epoch <= 100 else \
args.lr * (0.001 ** ((epoch - 100) / 50.0))
for g in optimizer.param_groups:
g['lr'] = lr * g.get('lr_mult', 1)
# Start training
for epoch in range(start_epoch, args.epochs):
adjust_lr(epoch)
trainer.train(epoch, train_loader, optimizer)
if epoch < args.start_save:
continue
top1 = evaluator.evaluate(val_loader, val, val)
is_best = top1 > best_top1
best_top1 = max(top1, best_top1)
save_checkpoint({
'state_dict': model.module.state_dict(),
'epoch': epoch + 1,
'best_top1': best_top1,
}, is_best, fpath=osp.join(args.logs_dir, 'checkpoint.pth.tar'))
print('\n * Finished epoch {:3d} top1: {:5.1%} best: {:5.1%}{}\n'.
format(epoch, top1, best_top1, ' *' if is_best else ''))
# Final test
print('Test with best model:')
checkpoint = load_checkpoint(osp.join(args.logs_dir, 'model_best.pth.tar'))
model.module.load_state_dict(checkpoint['state_dict'])
metric.train(model, train_loader)
print("Test(cuhk03):")
evaluator.evaluate(test_loader_cuhk03, query_cuhk03, gallery_cuhk03, metric)
print("Test(market1501):")
evaluator.evaluate(test_loader_market1501, query_market1501, gallery_market1501, metric)
def get_data(name, split_id, data_dir, height, width, batch_size, workers,
combine_trainval):
root = osp.join(data_dir, name)
dataset = datasets.create(name, root, split_id=split_id)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_set = dataset.trainval if combine_trainval else dataset.train
num_classes = (dataset.num_trainval_ids
if combine_trainval else dataset.num_train_ids)
train_transformer = T.Compose([
T.RandomSizedRectCrop(height, width),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalizer,
])
test_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
normalizer,
])
train_loader = DataLoader(
#images_dir = osp.join(self.root, 'images')
Preprocessor(train_set,
root=dataset.images_dir,
transform=train_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=True,
pin_memory=True,
drop_last=True)
train_loader_head = DataLoader(Preprocessor(
train_set,
root="/home/bfs/zty/reid_market/examples/data/cuhk03/images_head",
transform=train_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=True,
pin_memory=True,
drop_last=True)
train_loader_upper = DataLoader(Preprocessor(
train_set,
root="/home/bfs/zty/reid_market/examples/data/cuhk03/images_upper",
transform=train_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=True,
pin_memory=True,
drop_last=True)
train_loader_lower = DataLoader(Preprocessor(
train_set,
root="/home/bfs/zty/reid_market/examples/data/cuhk03/images_lower",
transform=train_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=True,
pin_memory=True,
drop_last=True)
val_loader = DataLoader(Preprocessor(dataset.val,
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
val_loader_head = DataLoader(Preprocessor(
dataset.val,
root="/home/bfs/zty/reid_market/examples/data/cuhk03/images_head",
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
val_loader_upper = DataLoader(Preprocessor(
dataset.val,
root="/home/bfs/zty/reid_market/examples/data/cuhk03/images_upper",
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
val_loader_lower = DataLoader(Preprocessor(
dataset.val,
root="/home/bfs/zty/reid_market/examples/data/cuhk03/images_lower",
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
test_loader = DataLoader(Preprocessor(
list(set(dataset.query) | set(dataset.gallery)),
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
test_loader_head = DataLoader(Preprocessor(
list(set(dataset.query) | set(dataset.gallery)),
root="/home/bfs/zty/reid_market/examples/data/cuhk03/images_head",
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
test_loader_upper = DataLoader(Preprocessor(
list(set(dataset.query) | set(dataset.gallery)),
root="/home/bfs/zty/reid_market/examples/data/cuhk03/images_upper",
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
test_loader_lower = DataLoader(Preprocessor(
list(set(dataset.query) | set(dataset.gallery)),
root="/home/bfs/zty/reid_market/examples/data/cuhk03/images_lower",
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
return dataset, num_classes, train_loader, train_loader_head, train_loader_upper, train_loader_lower,\
val_loader, val_loader_head, val_loader_upper, val_loader_lower, test_loader, test_loader_head, \
test_loader_upper, test_loader_lower
def get_data(name, split_id, data_dir, height, width, batch_size,
num_instances, workers, combine_trainval, make_data):
root = osp.join(data_dir, name)
if make_data:
from_dir1, from_dir2, num_eval, test, single = make_data
if test == 'test':
build_test = True
test = '_test'
elif test == 'train':
build_test = False
test = 'train'
else:
build_test = None
test = ''
to_dir = '{}{}{}{}{}'.format(from_dir1, from_dir2, num_eval, test,
single)
if single == 'single':
single = True
elif single == 'many':
single = False
else:
print('must be either single or many')
root = osp.join(data_dir, 'select')
dataset = datasets.create(name,
root,
from_dir1=from_dir1,
from_dir2=from_dir2,
to_dir=to_dir,
num_eval=int(num_eval),
make_test=build_test,
single=single,
split_id=split_id)
data_path = osp.join(root, 'datasets', to_dir)
else:
dataset = datasets.create(name, root, split_id=split_id)
data_path = osp.join(root, 'datasets', name)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_set = dataset.trainval if combine_trainval else dataset.train
num_classes = (dataset.num_trainval_ids
if combine_trainval else dataset.num_train_ids)
train_transformer = T.Compose([
T.RectCrop(height, width),
# T.RectCrop(height / 2, width / 2), # effectively blurring
# T.RectScale(int(height/2), int(width/2)),
# T.RectScale(height, width),
T.RandomSizedRectCrop(height, width, 0.4),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalizer,
])
test_transformer = T.Compose([
T.RectCrop(height, width),
# T.RectScale(int(height / 2), int(width / 2
T.RectScale(height, width),
T.ToTensor(),
normalizer,
])
train_loader = DataLoader(Preprocessor(train_set,
root=dataset.images_dir,
transform=train_transformer),
batch_size=batch_size,
num_workers=workers,
sampler=RandomIdentitySampler(
train_set, num_instances),
pin_memory=True,
drop_last=True)
val_loader = DataLoader(
Preprocessor(
dataset.val, root=dataset.images_dir,
transform=test_transformer), # todo originally test_transformer
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
test_loader = DataLoader(Preprocessor(
list(set(dataset.query) | set(dataset.gallery)),
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=True)
return dataset, num_classes, train_loader, val_loader, test_loader, data_path
def get_data(name, split_id, data_dir, height, width, batch_size, workers,
combine_trainval, np_ratio, model, instance_mode, eraser):
root = osp.join(data_dir, name)
dataset = datasets.create(name, root, split_id=split_id)
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_set = dataset.trainval if combine_trainval else dataset.train
if eraser:
train_transformer = T.Compose([
T.RandomSizedRectCrop(height, width),
T.RandomSizedEarser(),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalizer,
])
else:
train_transformer = T.Compose([
T.RandomSizedRectCrop(height, width),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalizer,
])
test_transformer = T.Compose([
T.RectScale(height, width),
T.ToTensor(),
normalizer,
])
if (model == 'Single'):
video_dict = None
if osp.isfile(osp.join(root, 'video.json')):
video_dict = read_json(osp.join(root, 'video.json'))
sampler = RandomTripletSampler(train_set,
video_dict=None,
skip_frames=10,
inter_rate=0.9,
inst_sample=instance_mode)
elif (model == 'Siamese'):
sampler = RandomPairSampler(train_set, neg_pos_ratio=np_ratio)
else:
raise ValueError('unrecognized mode')
train_loader = DataLoader(Preprocessor(train_set,
name,
root=dataset.images_dir,
transform=train_transformer),
sampler=sampler,
batch_size=batch_size,
num_workers=workers,
pin_memory=False)
val_loader = DataLoader(Preprocessor(dataset.val,
name,
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=False)
test_loader = DataLoader(Preprocessor(
list(set(dataset.query) | set(dataset.gallery)),
name,
root=dataset.images_dir,
transform=test_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=False,
pin_memory=False)
return dataset, train_loader, val_loader, test_loader
epoch = 0
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
test_transformer = T.Compose([
T.RectScale(data_height, data_width),
T.ToTensor(),
normalizer,
])
# Get 'train_loader'.
normalizer = T.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])
train_transformer = T.Compose([
T.RandomSizedRectCrop(data_height, data_width),
T.RandomHorizontalFlip(),
T.ToTensor(),
normalizer,
])
train_loader = DataLoader(Preprocessor(dataset.train,
root=dataset.images_dir,
transform=train_transformer),
batch_size=batch_size,
num_workers=workers,
shuffle=True,
pin_memory=True,
drop_last=True)
# Get 'optimizer'.
if hasattr(model.module, 'base'):
base_param_ids = set(map(id, model.module.base.parameters()))