def combine_trainval_sets(im_dirs, partition_files, save_dir): new_im_dir = ospj(save_dir, 'trainval_images') may_make_dir(new_im_dir) new_im_names = [] new_start_id = 0 occluded = [] for im_dir, partition_file in zip(im_dirs, partition_files): partitions = load_pickle(partition_file) im_paths = [ospj(im_dir, n) for n in partitions['trainval_im_names']] im_paths.sort() occluded_ = [n for n in partitions['occluded']] occluded += occluded_ new_im_names_, id_mapping = move_ims(im_paths, new_im_dir, parse_im_name, new_im_name_tmpl, new_start_id, occluded) new_start_id += len(id_mapping) new_im_names += new_im_names_ new_ids = range(new_start_id) partitions = { 'trainval_im_names': new_im_names, 'trainval_ids2labels': dict(zip(new_ids, new_ids)), 'occluded': occluded } partition_file = ospj(save_dir, 'partitions.pkl') save_pickle(partitions, partition_file) print('Partition file saved to {}'.format(partition_file))
def save_images(zip_file, save_dir=None, train_test_split_file=None): """Rename and move all used images to a directory.""" print("Extracting zip file") root = osp.dirname(osp.abspath(zip_file)) if save_dir is None: save_dir = root may_make_dir(save_dir) with ZipFile(zip_file) as z: z.extractall(path=save_dir) print("Extracting zip file done") new_im_dir = osp.join(save_dir, 'images') may_make_dir(new_im_dir) raw_dir = osp.join(save_dir, osp.basename(zip_file)[:-4]) im_paths = [] nums = [] for dir_name in ['bounding_box_train', 'bounding_box_test', 'query']: im_paths_ = get_im_names(osp.join(raw_dir, dir_name), return_path=True, return_np=False) im_paths_.sort() im_paths += list(im_paths_) nums.append(len(im_paths_)) im_names = move_ims(im_paths, new_im_dir, parse_original_im_name, new_im_name_tmpl) split = dict() keys = ['trainval_im_names', 'gallery_im_names', 'q_im_names'] inds = [0] + nums inds = np.cumsum(inds) for i, k in enumerate(keys): split[k] = im_names[inds[i]:inds[i + 1]] save_pickle(split, train_test_split_file) print('Saving images done.') return split
def save_images(zip_file, save_dir=None, train_test_split_file=None): """Rename and move all used images to a directory.""" print("Extracting zip file") root = osp.dirname(osp.abspath(zip_file)) if save_dir is None: save_dir = root may_make_dir(save_dir) with ZipFile(zip_file) as z: z.extractall(path=save_dir) print("Extracting zip file done") new_im_dir = osp.join(save_dir, 'images') may_make_dir(new_im_dir) raw_dir = osp.join(save_dir, osp.basename(zip_file)[:-4]) im_paths = [] nums = [] for dir_name in ['bounding_box_train', 'bounding_box_test', 'query']: im_paths_ = get_im_names(osp.join(raw_dir, dir_name), return_path=True, return_np=False) im_paths_.sort() im_paths += list(im_paths_) nums.append(len(im_paths_)) im_names = move_ims( im_paths, new_im_dir, parse_original_im_name, new_im_name_tmpl) split = dict() keys = ['trainval_im_names', 'gallery_im_names', 'q_im_names'] inds = [0] + nums inds = np.cumsum(inds) for i, k in enumerate(keys): split[k] = im_names[inds[i]:inds[i + 1]] save_pickle(split, train_test_split_file) print('Saving images done.') return split
def combine_trainval_sets( im_dirs, partition_files, save_dir): new_im_dir = ospj(save_dir, 'trainval_images') may_make_dir(new_im_dir) new_im_names = [] new_start_id = 0 for im_dir, partition_file in zip(im_dirs, partition_files): partitions = load_pickle(partition_file) im_paths = [ospj(im_dir, n) for n in partitions['trainval_im_names']] im_paths.sort() new_im_names_, id_mapping = move_ims( im_paths, new_im_dir, parse_im_name, new_im_name_tmpl, new_start_id) new_start_id += len(id_mapping) new_im_names += new_im_names_ new_ids = range(new_start_id) partitions = {'trainval_im_names': new_im_names, 'trainval_ids2labels': dict(zip(new_ids, new_ids)), } partition_file = ospj(save_dir, 'partitions.pkl') save_pickle(partitions, partition_file) print('Partition file saved to {}'.format(partition_file))
def save_images(mat_file, save_dir, new_im_name_tmpl): def deref(mat, ref): return mat[ref][:].T def dump(mat, refs, pid, cam, im_dir): """Save the images of a person under one camera.""" for i, ref in enumerate(refs): im = deref(mat, ref) if im.size == 0 or im.ndim < 2: break fname = new_im_name_tmpl.format(pid, cam, i) imsave(osp.join(im_dir, fname), im) mat = h5py.File(mat_file, 'r') labeled_im_dir = osp.join(save_dir, 'labeled/images') detected_im_dir = osp.join(save_dir, 'detected/images') all_im_dir = osp.join(save_dir, 'all/images') may_make_dir(labeled_im_dir) may_make_dir(detected_im_dir) may_make_dir(all_im_dir) # loop through camera pairs pid = 0 for labeled, detected in zip(mat['labeled'][0], mat['detected'][0]): labeled, detected = deref(mat, labeled), deref(mat, detected) assert labeled.shape == detected.shape # loop through ids in a camera pair for i in range(labeled.shape[0]): # We don't care about whether different persons are under same cameras, # we only care about the same person being under different cameras or not. dump(mat, labeled[i, :5], pid, 0, labeled_im_dir) dump(mat, labeled[i, 5:], pid, 1, labeled_im_dir) dump(mat, detected[i, :5], pid, 0, detected_im_dir) dump(mat, detected[i, 5:], pid, 1, detected_im_dir) dump(mat, chain(detected[i, :5], labeled[i, :5]), pid, 0, all_im_dir) dump(mat, chain(detected[i, 5:], labeled[i, 5:]), pid, 1, all_im_dir) pid += 1 if pid % 100 == 0: sys.stdout.write('\033[F\033[K') print('Saving images {}/{}'.format(pid, 1467))
def save_images(mat_file, save_dir, new_im_name_tmpl): def deref(mat, ref): return mat[ref][:].T def dump(mat, refs, pid, cam, im_dir): """Save the images of a person under one camera.""" for i, ref in enumerate(refs): im = deref(mat, ref) if im.size == 0 or im.ndim < 2: break fname = new_im_name_tmpl.format(pid, cam, i) imsave(osp.join(im_dir, fname), im) mat = h5py.File(mat_file, 'r') labeled_im_dir = osp.join(save_dir, 'labeled/images') detected_im_dir = osp.join(save_dir, 'detected/images') all_im_dir = osp.join(save_dir, 'all/images') may_make_dir(labeled_im_dir) may_make_dir(detected_im_dir) may_make_dir(all_im_dir) # loop through camera pairs pid = 0 for labeled, detected in zip(mat['labeled'][0], mat['detected'][0]): labeled, detected = deref(mat, labeled), deref(mat, detected) assert labeled.shape == detected.shape # loop through ids in a camera pair for i in range(labeled.shape[0]): # We don't care about whether different persons are under same cameras, # we only care about the same person being under different cameras or not. dump(mat, labeled[i, :5], pid, 0, labeled_im_dir) dump(mat, labeled[i, 5:], pid, 1, labeled_im_dir) dump(mat, detected[i, :5], pid, 0, detected_im_dir) dump(mat, detected[i, 5:], pid, 1, detected_im_dir) dump(mat, chain(detected[i, :5], labeled[i, :5]), pid, 0, all_im_dir) dump(mat, chain(detected[i, 5:], labeled[i, 5:]), pid, 1, all_im_dir) pid += 1 if pid % 100 == 0: sys.stdout.write('\033[F\033[K') print('Saving images {}/{}'.format(pid, 1467))
def save_images(zip_file, save_dir=None, train_test_split_file=None): """Rename and move all used images to a directory.""" print("Extracting zip file") root = osp.dirname(osp.abspath(zip_file)) if save_dir is None: save_dir = root may_make_dir(osp.abspath(save_dir)) with ZipFile(zip_file) as z: z.extractall(path=save_dir) print("Extracting zip file done") new_im_dir = osp.join(save_dir, 'images') may_make_dir(osp.abspath(new_im_dir)) raw_dir = osp.join(save_dir, osp.basename(zip_file)[:-4]) im_paths = [] nums = [] im_paths_ = get_im_names(osp.join(raw_dir, 'bounding_box_train'), return_path=True, return_np=False) im_paths_.sort() im_paths += list(im_paths_) nums.append(len(im_paths_)) im_paths_ = get_im_names(osp.join(raw_dir, 'bounding_box_test'), return_path=True, return_np=False) im_paths_.sort() im_paths_ = [p for p in im_paths_ if not osp.basename(p).startswith('-1')] im_paths += list(im_paths_) nums.append(len(im_paths_)) im_paths_ = get_im_names(osp.join(raw_dir, 'query'), return_path=True, return_np=False) im_paths_.sort() im_paths += list(im_paths_) nums.append(len(im_paths_)) q_ids_cams = set([(parse_original_im_name(osp.basename(p), 'id'), parse_original_im_name(osp.basename(p), 'cam')) for p in im_paths_]) im_paths_ = get_im_names(osp.join(raw_dir, 'gt_bbox'), return_path=True, return_np=False) im_paths_.sort() # Only gather images for those ids and cams used in testing. im_paths_ = [ p for p in im_paths_ if (parse_original_im_name(osp.basename(p), 'id'), parse_original_im_name(osp.basename(p), 'cam')) in q_ids_cams ] im_paths += list(im_paths_) nums.append(len(im_paths_)) im_names = move_ims(im_paths, new_im_dir, parse_original_im_name, new_im_name_tmpl) split = dict() keys = [ 'trainval_im_names', 'gallery_im_names', 'q_im_names', 'mq_im_names' ] inds = [0] + nums inds = np.cumsum(np.array(inds)) for i, k in enumerate(keys): split[k] = im_names[inds[i]:inds[i + 1]] save_pickle(split, train_test_split_file) print('Saving images done.') return split
default=ospeu('~/Dataset/duke/images')) parser.add_argument( '--duke_partition_file', type=str, default=ospeu('~/Dataset/duke/partitions.pkl') ) parser.add_argument( '--save_dir', type=str, default=ospeu('~/Dataset/market1501_cuhk03_duke') ) args = parser.parse_args() im_dirs = [ ospap(ospeu(args.market1501_im_dir)), # ospap(ospeu(args.cuhk03_im_dir)), ospap(ospeu(args.duke_im_dir)) ] partition_files = [ ospap(ospeu(args.market1501_partition_file)), #ospap(ospeu(args.cuhk03_partition_file)), ospap(ospeu(args.duke_partition_file)) ] save_dir = ospap(ospeu(args.save_dir)) may_make_dir(save_dir) combine_trainval_sets(im_dirs, partition_files, save_dir)
default=ospeu('~/Dataset/duke/images')) parser.add_argument( '--duke_partition_file', type=str, default=ospeu('~/Dataset/duke/partitions.pkl') ) parser.add_argument( '--save_dir', type=str, default=ospeu('~/Dataset/market1501_cuhk03_duke') ) args = parser.parse_args() im_dirs = [ ospap(ospeu(args.market1501_im_dir)), ospap(ospeu(args.cuhk03_im_dir)), ospap(ospeu(args.duke_im_dir)) ] partition_files = [ ospap(ospeu(args.market1501_partition_file)), ospap(ospeu(args.cuhk03_partition_file)), ospap(ospeu(args.duke_partition_file)) ] save_dir = ospap(ospeu(args.save_dir)) may_make_dir(save_dir) combine_trainval_sets(im_dirs, partition_files, save_dir)
def transform(zip_file, train_test_partition_file, save_dir=None): """Save images and partition the train/val/test set. """ print("Extracting zip file") root = osp.dirname(osp.abspath(zip_file)) if save_dir is None: save_dir = root may_make_dir(save_dir) with ZipFile(zip_file) as z: z.extractall(path=save_dir) print("Extracting zip file done") mat_file = osp.join(save_dir, osp.basename(zip_file)[:-4], 'cuhk-03.mat') save_images(mat_file, save_dir, new_im_name_tmpl) if osp.exists(train_test_partition_file): train_test_partition = load_pickle(train_test_partition_file) else: raise RuntimeError('Train/test partition file should be provided.') for im_type in ['detected', 'labeled']: trainval_im_names = train_test_partition[im_type]['train_im_names'] trainval_ids = list( set([parse_im_name(n, 'id') for n in trainval_im_names])) # Sort ids, so that id-to-label mapping remains the same when running # the code on different machines. trainval_ids.sort() trainval_ids2labels = dict(zip(trainval_ids, range(len(trainval_ids)))) train_val_partition = \ partition_train_val_set(trainval_im_names, parse_im_name, num_val_ids=100) train_im_names = train_val_partition['train_im_names'] train_ids = list( set([ parse_im_name(n, 'id') for n in train_val_partition['train_im_names'] ])) # Sort ids, so that id-to-label mapping remains the same when running # the code on different machines. train_ids.sort() train_ids2labels = dict(zip(train_ids, range(len(train_ids)))) # A mark is used to denote whether the image is from # query (mark == 0), or # gallery (mark == 1), or # multi query (mark == 2) set val_marks = [0, ] * len(train_val_partition['val_query_im_names']) \ + [1, ] * len(train_val_partition['val_gallery_im_names']) val_im_names = list(train_val_partition['val_query_im_names']) \ + list(train_val_partition['val_gallery_im_names']) test_im_names = list(train_test_partition[im_type]['query_im_names']) \ + list(train_test_partition[im_type]['gallery_im_names']) test_marks = [0, ] * len(train_test_partition[im_type]['query_im_names']) \ + [1, ] * len( train_test_partition[im_type]['gallery_im_names']) partitions = { 'trainval_im_names': trainval_im_names, 'trainval_ids2labels': trainval_ids2labels, 'train_im_names': train_im_names, 'train_ids2labels': train_ids2labels, 'val_im_names': val_im_names, 'val_marks': val_marks, 'test_im_names': test_im_names, 'test_marks': test_marks } partition_file = osp.join(save_dir, im_type, 'partitions.pkl') save_pickle(partitions, partition_file) print('Partition file for "{}" saved to {}'.format( im_type, partition_file))
def save_im(im, save_path): """im: shape [3, H, W]""" may_make_dir(ospdn(save_path)) im = im.transpose(1, 2, 0) Image.fromarray(im).save(save_path)
def save_images(zip_file, save_dir=None, train_test_split_file=None): """Rename and move all used images to a directory.""" print("Extracting zip file") root = osp.dirname(osp.abspath(zip_file)) if save_dir is None: save_dir = root may_make_dir(osp.abspath(save_dir)) with ZipFile(zip_file) as z: z.extractall(path=save_dir) print("Extracting zip file done") new_im_dir = osp.join(save_dir, 'images') may_make_dir(osp.abspath(new_im_dir)) raw_dir = osp.join(save_dir, osp.basename(zip_file)[:-4]) im_paths = [] nums = [] im_paths_ = get_im_names(osp.join(raw_dir, 'bounding_box_train'), return_path=True, return_np=False) im_paths_.sort() im_paths += list(im_paths_) nums.append(len(im_paths_)) im_paths_ = get_im_names(osp.join(raw_dir, 'bounding_box_test'), return_path=True, return_np=False) im_paths_.sort() im_paths_ = [p for p in im_paths_ if not osp.basename(p).startswith('-1')] im_paths += list(im_paths_) nums.append(len(im_paths_)) im_paths_ = get_im_names(osp.join(raw_dir, 'query'), return_path=True, return_np=False) im_paths_.sort() im_paths += list(im_paths_) nums.append(len(im_paths_)) q_ids_cams = set([(parse_original_im_name(osp.basename(p), 'id'), parse_original_im_name(osp.basename(p), 'cam')) for p in im_paths_]) im_paths_ = get_im_names(osp.join(raw_dir, 'gt_bbox'), return_path=True, return_np=False) im_paths_.sort() # Only gather images for those ids and cams used in testing. im_paths_ = [p for p in im_paths_ if (parse_original_im_name(osp.basename(p), 'id'), parse_original_im_name(osp.basename(p), 'cam')) in q_ids_cams] im_paths += list(im_paths_) nums.append(len(im_paths_)) im_names = move_ims( im_paths, new_im_dir, parse_original_im_name, new_im_name_tmpl) split = dict() keys = ['trainval_im_names', 'gallery_im_names', 'q_im_names', 'mq_im_names'] inds = [0] + nums inds = np.cumsum(np.array(inds)) for i, k in enumerate(keys): split[k] = im_names[inds[i]:inds[i + 1]] save_pickle(split, train_test_split_file) print('Saving images done.') return split
def transform(zip_file, train_test_partition_file, save_dir=None): """Save images and partition the train/val/test set. """ print("Extracting zip file") root = osp.dirname(osp.abspath(zip_file)) if save_dir is None: save_dir = root may_make_dir(save_dir) with ZipFile(zip_file) as z: z.extractall(path=save_dir) print("Extracting zip file done") mat_file = osp.join(save_dir, osp.basename(zip_file)[:-4], 'cuhk-03.mat') save_images(mat_file, save_dir, new_im_name_tmpl) if osp.exists(train_test_partition_file): train_test_partition = load_pickle(train_test_partition_file) else: raise RuntimeError('Train/test partition file should be provided.') for im_type in ['detected', 'labeled']: trainval_im_names = train_test_partition[im_type]['train_im_names'] trainval_ids = list(set([parse_im_name(n, 'id') for n in trainval_im_names])) # Sort ids, so that id-to-label mapping remains the same when running # the code on different machines. trainval_ids.sort() trainval_ids2labels = dict(zip(trainval_ids, range(len(trainval_ids)))) train_val_partition = \ partition_train_val_set(trainval_im_names, parse_im_name, num_val_ids=100) train_im_names = train_val_partition['train_im_names'] train_ids = list(set([parse_im_name(n, 'id') for n in train_val_partition['train_im_names']])) # Sort ids, so that id-to-label mapping remains the same when running # the code on different machines. train_ids.sort() train_ids2labels = dict(zip(train_ids, range(len(train_ids)))) # A mark is used to denote whether the image is from # query (mark == 0), or # gallery (mark == 1), or # multi query (mark == 2) set val_marks = [0, ] * len(train_val_partition['val_query_im_names']) \ + [1, ] * len(train_val_partition['val_gallery_im_names']) val_im_names = list(train_val_partition['val_query_im_names']) \ + list(train_val_partition['val_gallery_im_names']) test_im_names = list(train_test_partition[im_type]['query_im_names']) \ + list(train_test_partition[im_type]['gallery_im_names']) test_marks = [0, ] * len(train_test_partition[im_type]['query_im_names']) \ + [1, ] * len( train_test_partition[im_type]['gallery_im_names']) partitions = {'trainval_im_names': trainval_im_names, 'trainval_ids2labels': trainval_ids2labels, 'train_im_names': train_im_names, 'train_ids2labels': train_ids2labels, 'val_im_names': val_im_names, 'val_marks': val_marks, 'test_im_names': test_im_names, 'test_marks': test_marks} partition_file = osp.join(save_dir, im_type, 'partitions.pkl') save_pickle(partitions, partition_file) print('Partition file for "{}" saved to {}'.format(im_type, partition_file))