def create_debug_ca_dataset():
'''
Create a mini subset of Category_and_Attribute data. Assume standard CA index file and label files already exist.
'''
num_train = 10
num_test = 10
same_train_test = True
samples = io.load_json(design_root + 'Label/ca_samples.json')
attr_label = io.load_data(design_root + 'Label/ca_attr_label.pkl')
bbox_label = io.load_data(design_root + 'Label/ca_bbox_label_256.pkl')
lm_label = io.load_data(design_root + 'Label/ca_landmark_label_256.pkl')
if same_train_test:
id_list = samples.keys()[0:num_train]
split = {'train': id_list, 'test': id_list}
else:
id_list = samples.keys()[0:(num_train + num_test)]
split = {'train': id_list[0:num_train], 'test': id_list[num_train::]}
samples = {s_id:samples[s_id] for s_id in id_list}
attr_label = {s_id:attr_label[s_id] for s_id in id_list}
bbox_label = {s_id:bbox_label[s_id] for s_id in id_list}
lm_label = {s_id:lm_label[s_id] for s_id in id_list}
io.save_json(samples, design_root + 'Label/debugca_samples.json')
io.save_data(attr_label, design_root + 'Label/debugca_attr_label.pkl')
io.save_data(bbox_label, design_root + 'Label/debugca_bbox_label.pkl')
io.save_data(lm_label, design_root + 'Label/debugca_landmark_label.pkl')
io.save_json(split, design_root + 'Split/debugca_split.json')
def initialize(self, opt, split):
self.opt = opt
self.root = opt.data_root
self.split = split
print('loading data ...')
samples = io.load_json(os.path.join(opt.data_root, opt.fn_sample))
attr_label = io.load_data(os.path.join(opt.data_root, opt.fn_label))
attr_entry = io.load_json(os.path.join(opt.data_root, opt.fn_entry))
attr_split = io.load_json(os.path.join(opt.data_root, opt.fn_split))
lm_label = io.load_data(os.path.join(opt.data_root, opt.fn_landmark))
self.id_list = attr_split[split]
if opt.max_dataset_size != float('inf'):
self.id_list = self.id_list[0:opt.max_dataset_size]
self.sample_list = [samples[s_id] for s_id in self.id_list]
self.attr_label_list = [attr_label[s_id] for s_id in self.id_list]
self.lm_list = [lm_label[s_id] for s_id in self.id_list]
self.attr_entry = attr_entry
# check data
assert len(self.attr_entry) == len(
self.attr_label_list[0]
) == opt.n_attr, 'Attribute number not match!'
print('dataset created (%d samples)' % len(self))
# get transform
self.to_tensor = transforms.ToTensor()
if opt.image_normalize == 'imagenet':
self.tensor_normalize = transforms.Normalize([0.485, 0.456, 0.406],
[0.229, 0.224, 0.225])
else:
self.tensor_normalize = transforms.Normalize([0.5, 0.5, 0.5],
[0.5, 0.5, 0.5])
def initialize(self, opt, split):
self.opt = opt
self.root = opt.data_root
# get transform
transform_list = []
if opt.resize_or_crop == 'resize':
# only resize image
transform_list.append(transforms.Resize(opt.fine_size, Image.BICUBIC))
elif opt.resize_or_crop == 'resize_and_crop':
# scale and crop
transform_list.append(transforms.Resize(opt.load_size, Image.BICUBIC))
if split == 'train':
transform_list.append(transforms.RandomCrop(opt.fine_size))
transform_list.append(transforms.RandomHorizontalFlip())
else:
transform_list.append(transforms.CenterCrop(opt.fine_size))
transform_list.append(transforms.ToTensor())
if opt.image_normalize == 'imagenet':
transform_list.append(transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]))
else:
transform_list.append(transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5]))
self.transform = transforms.Compose(transform_list)
# load sample list
print('loading data ...')
samples = io.load_json(os.path.join(opt.data_root, opt.fn_sample))
attr_label = io.load_data(os.path.join(opt.data_root, opt.fn_label))
attr_entry = io.load_json(os.path.join(opt.data_root, opt.fn_entry))
attr_split = io.load_json(os.path.join(opt.data_root, opt.fn_split))
self.id_list = attr_split[split]
if opt.max_dataset_size != float('inf'):
self.id_list = self.id_list[0:opt.max_dataset_size]
self.sample_list = [samples[s_id] for s_id in self.id_list]
self.attr_label_list = [attr_label[s_id] for s_id in self.id_list]
self.attr_entry = attr_entry
if opt.joint_cat:
cat_label = io.load_data(os.path.join(opt.data_root, opt.fn_cat))
self.cat_list = [cat_label[s_id] for s_id in self.id_list]
if opt.unmatch:
np.random.shuffle(self.sample_list)
# check data
assert len(self.attr_entry) == len(self.attr_label_list[0]) == opt.n_attr, 'Attribute number not match!'
print('dataset created (%d samples)' % len(self))
def _unit_func(idx, pair_list, bidirectional_corr):
for id_1, id_2 in tqdm.tqdm(pair_list, position=idx):
pred_1 = io.load_data(hmr_pred_dir1 + id_1 + '.pkl')
pred_2 = io.load_data(hmr_pred_dir2 + id_2 + '.pkl')
corr_2to1, vis_mask_2 = calc_correspondence_from_smpl_internal(
pred_2, pred_1)
flow_util.write_corr(output_dir + '%s_%s.corr' % (id_2, id_1),
corr_2to1, vis_mask_2)
if bidirectional_corr:
corr_1to2, vis_mask_1 = calc_correspondence_from_smpl_internal(
pred_1, pred_2)
flow_util.write_corr(output_dir + '%s_%s.corr' % (id_1, id_2),
corr_1to2, vis_mask_1)
def __init__(self, opt, split='train'):
assert split in {'train', 'val', 'test'}
super(Dataset, self).__init__()
self.opt = opt
self.split = split
self.label = io.load_data(opt.fn_label)['label']
if split in {'train', 'val'}:
self.id_list = io.load_json(
opt.fn_split)['train' if split == 'train' else 'test']
self.image_dir = opt.img_dir
else:
self.image_dir = opt.test_dir
self.id_list = [fn[0:-4] for fn in os.listdir(self.image_dir)]
if split == 'train':
self.transform = transforms.Compose([
transforms.Resize(opt.rescale_size),
transforms.RandomCrop(opt.crop_size),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
])
else:
self.transform = transforms.Compose([
transforms.Resize(opt.rescale_size),
transforms.CenterCrop(opt.crop_size),
transforms.ToTensor(),
transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5])
])
if opt.debug:
self.id_list = self.id_list[0:64]
def initialize(self, opt, split):
self.opt = opt
self.root = opt.data_root
if opt.debug:
split = 'debug'
self.split = split
#############################
# load data
#############################
print('loading data ...')
# data split
data_split = io.load_json(os.path.join(opt.data_root, opt.fn_split))
self.pose_label = io.load_data(os.path.join(opt.data_root,
opt.fn_pose))
self.img_dir = os.path.join(opt.data_root, opt.img_dir)
self.seg_dir = os.path.join(opt.data_root, opt.seg_dir)
self.edge_dir = os.path.join(opt.data_root, opt.edge_dir)
#############################
# create index list
#############################
self.id_list = data_split[split]
#############################
# other
#############################
self.tensor_normalize_std = transforms.Normalize([0.5, 0.5, 0.5],
[0.5, 0.5, 0.5])
self.color_jitter = transforms.ColorJitter(brightness=0.3,
contrast=0.3,
saturation=0.3,
hue=0.3)
self.to_pil_image = transforms.ToPILImage()
def gather_pose_estimation_result():
'''
We use the state-of-the-art human pose estimation method (https://github.com/tensorboy/pytorch_Realtime_Multi-Person_Pose_Estimation) to get key points
This function is for gathering estimation results.
'''
num_key_p = 18
rst_dir = 'datasets/DeepFashion/Fashion_design/Temp/pose_pkl/'
split = io.load_json(
'datasets/DeepFashion/Fashion_design/Split/ca_gan_split_trainval_upper.json'
)
id_list = split['train'] + split['test']
pose_label = {}
n_fail = 0
for idx, s_id in enumerate(id_list):
print('%d/%d : %s' % (idx, len(id_list), s_id))
fn_pose = rst_dir + s_id + '.pkl'
if not os.path.isfile(fn_pose):
pose_label[s_id] = [[-1, -1] for _ in range(num_key_p)]
n_fail += 1
else:
pose = io.load_data(fn_pose)
assert len(pose) == num_key_p
# p[i][j] = (x, y, score, id) is the j-th keypoints of i-th type.
# we assume that j <= 1, because our image contains single person
pose_label[s_id] = [[p[0][0], p[0][1]] if len(p) > 0 else [-1, -1]
for p in pose]
io.save_data(
pose_label,
'datasets/DeepFashion/Fashion_design/Label/ca_gan_pose_label_256.pkl')
print('%d (out of %d) samples failed' % (n_fail, len(id_list)))
def initialize(self, opt, split):
self.opt = opt
self.data_root = opt.data_root
self.split = split
#############################
# set path / load label
#############################
data_split = io.load_json(os.path.join(opt.data_root, opt.fn_split))
self.img_dir = os.path.join(opt.data_root, opt.img_dir)
self.seg_dir = os.path.join(opt.data_root, opt.seg_dir)
self.corr_dir = os.path.join(opt.data_root, opt.corr_dir)
self.pose_label = io.load_data(os.path.join(opt.data_root,
opt.fn_pose))
#############################
# create index list
#############################
self.id_list = data_split[split]
#############################
# other
#############################
if opt.debug:
self.id_list = self.id_list[0:32]
self.tensor_normalize_std = transforms.Normalize([0.5, 0.5, 0.5],
[0.5, 0.5, 0.5])
self.to_pil_image = transforms.ToPILImage()
self.pil_to_tensor = transforms.ToTensor()
self.color_jitter = transforms.ColorJitter(brightness=0.0,
contrast=0.0,
saturation=0.0,
hue=0.2)
def create_pose_label():
'''
create 18-keypoint pose label. follow the setting in VITON
'''
pose = io.load_data(zalando_root + 'Source/pose.pkl')
split = io.load_json(zalando_root + 'Split/zalando_split.json')
id_list = split['train'] + split['test']
pose_label = {}
for idx, s_id in enumerate(id_list):
print('%d / %d' % (idx, len(id_list)))
subset = pose[s_id + '_0']['subset'] # [i1, i2, ..., in, totial_score, n]
candidate = pose[s_id + '_0']['candidate'] # [[x_i, y_i, score_i, id_i]]
label = []
for i in subset[0][0:-2]:
i = int(i)
if i == -1:
label.append([-1, -1])
else:
x = candidate[i][0]
y = candidate[i][1]
label.append([x, y])
pose_label[s_id] = label
io.save_data(pose_label, zalando_root + 'Label/zalando_pose_label.pkl')
def create_silhouette():
# DeepFashion
#smpl_pred_dir = 'datasets/DF_Pose/3d/hmr_dfm_v2/pred/'
#output_dir = 'datasets/DF_Pose/Img/silhouette24/'
#image_split = io.load_json('datasets/DF_Pose/Label/image_split_dfm.json')
# Market-1501
smpl_pred_dir = 'datasets/market1501/3d/hmr/pred/'
output_dir = 'datasets/market1501/Images/silhouette24/'
image_split = io.load_json('datasets/market1501/Label/image_split.json')
faces = np.load('scripts/3d/smpl_faces.npy')
vert2part = io.load_json('scripts/3d/smpl_vert_to_bodypart.json')
def _func(face_id):
if face_id == 4294967295:
return 0
else:
verts = faces[face_id]
part_id = vert2part[verts[0]] + 1
return part_id
_vfunc = np.vectorize(_func)
io.mkdir_if_missing(output_dir)
id_list = image_split['train'] + image_split['test']
for sid in tqdm.tqdm(id_list):
pred = io.load_data(smpl_pred_dir + '%s.pkl' % sid)
vis = pred['visibility']
silh = _vfunc(vis).astype(np.uint8)
silh = cv2.medianBlur(silh, 5)
imageio.imwrite(output_dir + '%s.bmp' % sid, silh)
def create_attr_entry():
'''
Create attribute entry list, which contains original 1000 attributes used in Category_and_Attribute benchmark
'''
print('loading data...')
attr_entry_list = io.load_str_list(ca_root +
'Anno/list_attr_cloth.txt')[2::]
attr_label = io.load_data(design_root + 'Label/ca_attr_label.pkl')
split = io.load_json(design_root + 'Split/ca_split.json')
train_ids = set(split['train'])
attr_mat = np.array(
[v for k, v in attr_label.iteritems() if k in train_ids],
dtype=np.float32)
print('computing positive rates')
num_sample = len(train_ids)
pos_rate = attr_mat.sum(axis=0) / num_sample
attr_entry = []
for idx, s in enumerate(attr_entry_list):
s = s.split()
attr_name = ' '.join(s[0:-1])
attr_type = int(s[-1])
attr_entry.append({
'entry': attr_name,
'type': attr_type,
'pos_rate': pos_rate[idx]
})
io.save_json(attr_entry, design_root + 'Label/attr_entry.json')
def visualize_samples():
num_sample = 10
dir_out = 'temp/attr_example'
io.mkdir_if_missing(dir_out)
samples = io.load_json(design_root + 'Label/ca_samples.json')
attr_label = io.load_data(design_root + 'Label/ca_attr_label.pkl')
attr_entry = io.load_json(design_root + 'Label/attr_entry.json')
id_set = set(samples.keys())
for i, att in enumerate(attr_entry):
print('attribute %d / %d: %s' % (i, len(attr_entry), att['entry']))
dir_att = os.path.join(dir_out, att['entry'])
io.mkdir_if_missing(dir_att)
pos_id_list = [
s_id for s_id, label in attr_label.iteritems() if label[i] == 1
]
np.random.shuffle(pos_id_list)
for s_id in pos_id_list[0:num_sample]:
fn_src = samples[s_id]['img_path']
fn_tar = os.path.join(dir_att, 'pos_' + s_id + '.jpg')
shutil.copyfile(fn_src, fn_tar)
neg_id_list = list(id_set - set(pos_id_list))
np.random.shuffle(neg_id_list)
for s_id in neg_id_list[0:num_sample]:
fn_src = samples[s_id]['img_path']
fn_tar = os.path.join(dir_att, 'neg_' + s_id + '.jpg')
shutil.copyfile(fn_src, fn_tar)
def keypoint_guided_tps():
num_sample = 64
pair_list = io.load_json(
'datasets/DF_Pose/Label/pair_split.json')['test'][0:num_sample]
pose_label = io.load_data('datasets/DF_Pose/Label/pose_label.pkl')
image_dir = 'datasets/DF_Pose/Img/img_df/'
seg_dir = 'datasets/DF_Pose/Img/seg-lip_df_revised/'
output_dir = 'temp/patch_matching/output/tps_keypoint/'
io.mkdir_if_missing(output_dir)
tps = cv2.createThinPlateSplineShapeTransformer()
for i, (id_1, id_2) in enumerate(tqdm.tqdm(pair_list)):
kp_1 = np.array(pose_label[id_1][1:14],
dtype=np.float64).reshape(1, -1, 2)
kp_2 = np.array(pose_label[id_2][1:14],
dtype=np.float64).reshape(1, -1, 2)
kp_matches = []
for j in range(kp_1.shape[1]):
if (kp_1[0, j] >= 0).all() and (kp_2[0, j] >= 0).all():
kp_matches.append(cv2.DMatch(j, j, 0))
if len(kp_matches) == 0:
continue
tps.estimateTransformation(kp_2, kp_1, kp_matches)
img_1 = cv2.imread(image_dir + id_1 + '.jpg')
img_2 = cv2.imread(image_dir + id_2 + '.jpg')
img_w = tps.warpImage(img_1)
seg = cv2.imread(seg_dir + id_2 + '.bmp', cv2.IMREAD_GRAYSCALE)
mask = ((seg == 3) | (seg == 7)).astype(img_w.dtype)[:, :, np.newaxis]
img_out = img_w * mask + img_2 * (1 - mask)
cv2.imwrite(output_dir + '%d_%s_%s.jpg' % (i, id_1, id_2), img_out)
cv2.imwrite(output_dir + 'w%d_%s_%s.jpg' % (i, id_1, id_2), img_w)
def create_test_pair():
np.random.rand(0)
split = io.load_json(design_root +
'Split/ca_gan_split_trainval_upper.json')
cat_label = io.load_data(design_root + 'Label/ca_cat_label.pkl')
cat_entry = io.load_json(design_root + 'Label/cat_entry.json')
# group samples by category label
cat_to_ids = defaultdict(lambda: [])
for s_id in split['test']:
c = cat_label[s_id]
cat_to_ids[c].append(s_id)
n = 0
pair_list = []
for c, s_list in cat_to_ids.iteritems():
print('[%d/%d] %s: %d samples...' %
(n, len(cat_to_ids), cat_entry[c]['entry'], len(s_list)))
n += 1
s_list_org = [s_id for s_id in s_list]
for i in range(len(s_list) - 1):
j = np.random.randint(i + 1, len(s_list))
temp = s_list[i]
s_list[i] = s_list[j]
s_list[j] = temp
pair_list += zip(s_list_org, s_list)
pair_dict = {s_tar: s_src for s_tar, s_src in pair_list}
io.save_json(pair_dict, design_root + 'Temp/ca_test_tps_pair.json')
io.save_str_list(pair_dict.keys(),
design_root + 'Temp/ca_test_tps_tar.txt')
io.save_str_list(pair_dict.values(),
design_root + 'Temp/ca_test_tps_src.txt')
def run_logreg(quality='high'):
"""
Runs a simple logistic regression model; first fits the model
on the training data (70 percent of the total data) and tests on
the rest of the data.
Args:
none
Returns:
none
"""
data = io.load_data(quality=quality)
X, y, class_names = preprocessing.create_data_tensor(data)
X_train, y_train, X_test, y_test = preprocessing.create_train_test_split(
X, y, test_size=0.3, shuffle=True)
# flatten data
flattened_Xtrain = preprocessing.flatten_matrix(X_train)
flattened_Xtest = preprocessing.flatten_matrix(X_test)
# fit logistic regression model
logreg_model = linear_model.LogisticRegression(multi_class='ovr')
logreg_model.fit(flattened_Xtrain, y_train)
y_predict_train = logreg_model.predict(flattened_Xtrain)
y_predict = logreg_model.predict(flattened_Xtest)
# print metrics and confusion plot
analysis.run_analyses(y_predict_train, y_train, y_predict, y_test,
class_names)
def create_hmr_pose_label_adapt():
'''
This is to create a version of hmr_pose joint, which is adapted to openpose joint:
- compute "neck" using a regressor, trained useing openpose joints (due to the different definition of neck point"
- invalidate the joint points which is invalid in dfm_pose
'''
from sklearn.linear_model import RidgeCV
# DeepFashion
#joint_label = io.load_data('datasets/DF_Pose/Label/pose_label_dfm.pkl')
#joint_label_hmr = io.load_data('datasets/DF_Pose/Label/pose_label_hmr.pkl')
#fn_out = 'datasets/DF_Pose/Label/pose_label_hmr_adapt.pkl'
# Market-1501
joint_label = io.load_data('datasets/market1501/Label/pose_label.pkl')
joint_label_hmr = io.load_data(
'datasets/market1501/Label/pose_label_hmr.pkl')
fn_out = 'datasets/market1501/Label/pose_label_hmr_adapt.pkl'
# train a linear regressor, which predict neck point location from left/right shoulder locations
print('training regressor...')
pts_dfm = np.array(joint_label.values()) #(N,18,2)
v = (pts_dfm[:, [1, 2, 5], :].reshape(-1, 6) >= 0).all(axis=1)
x_train = (pts_dfm[v])[:, [2, 5]].reshape(-1, 4) #shoulder points
y_train = (pts_dfm[v])[:, 1].reshape(-1, 2) #neck points
reg = RidgeCV(normalize=False)
reg.fit(x_train, y_train)
pts_hmr = np.array(joint_label_hmr.values())
x_test = pts_hmr[:, [2, 5], :].reshape(-1, 4)
y_test = reg.predict(x_test).reshape(-1, 2)
# generate adapted joint label
joint_label_adapt = {}
for idx, sid in enumerate(tqdm.tqdm(joint_label_hmr.keys())):
p_h = np.array(joint_label_hmr[sid])
p_d = np.array(joint_label[sid])
if (p_h[[2, 5], :] >= 0).all():
p_h[1, :] = y_test[idx]
inv = (p_d < 0).any(axis=1) | (p_h < 0).any(axis=1) | (p_h > 255).any(
axis=1
) # invalid joint points in joint_dfm will also be marked as invalid in joint_hmr
p_h[inv, :] = -1
joint_label_adapt[sid] = p_h.tolist()
io.save_data(joint_label_adapt, fn_out)
def pad_image_for_segmentation():
'''
resize and padding image for segmentation (using fashionGAN code)
Todo: add inshop version
'''
sz_tar = 256
output_dir = 'datasets/DeepFashion/Fashion_design/Img/img_ca_pad'
io.mkdir_if_missing(output_dir)
samples = io.load_json(design_root + 'Label/ca_samples.json')
split = io.load_json(design_root + 'Split/ca_gan_split_trainval.json')
id_list = split['train'] + split['test']
# update landmark and bbox
lm_label = io.load_data(design_root + 'Label/ca_landmark_label.pkl')
bbox_label = io.load_data(design_root + 'Label/ca_bbox_label.pkl')
lm_label_pad = {}
bbox_label_pad = {}
io.save_str_list(id_list, os.path.join(output_dir, 'img_ca_pad.txt'))
for i, s_id in enumerate(id_list):
img_org = image.imread(samples[s_id]['img_path_org'])
h, w = img_org.shape[0:2]
if h > w:
img = image.resize(img_org, (-1, sz_tar))
scale = 1. * sz_tar / h
else:
img = image.resize(img_org, (sz_tar, -1))
scale = 1. * sz_tar / w
# img = image.pad_square(img, sz_tar, padding_value = 255, mode = 'lefttop')
# image.imwrite(img, os.path.join(output_dir, s_id + '.jpg'))
bbox_label_pad[s_id] = [c * scale for c in bbox_label[s_id]]
lm_label_pad[s_id] = []
for x, y, v in lm_label[s_id]:
lm_label_pad[s_id].append([x * scale, y * scale, v])
print('padding image %d / %d' % (i, len(id_list)))
io.save_data(lm_label_pad,
design_root + 'Label/ca_landmark_label_pad_%d.pkl' % sz_tar)
io.save_data(bbox_label_pad,
design_root + 'Label/ca_bbox_label_pad_%d.pkl' % sz_tar)
def load_attr_data(self):
opt = self.opt
self.samples = io.load_json(os.path.join(opt.data_root, opt.fn_sample))
self.attr_label = io.load_data(
os.path.join(opt.data_root, opt.fn_label))
self.attr_entry = io.load_json(
os.path.join(opt.data_root, opt.fn_entry))
self.data_loaded = True
def calc_correspondence_from_smpl():
'''
Compute pixel-wise correspondence between image pair by SMPL model (http://smpl.is.tue.mpg.de/).
The SMPL fit result is predicted by HMR(https://github.com/akanazawa/hmr), with following format:
pred = {
'id': sid,
'theta': theta,
'proc_param': proc_param,
'verts2d': verts2d,
'verts_z': verts_z,
'visibility': visibility, # a map with same size of img, each pixel is its corresponding SMPL face index (or 4294967295 if it's corresponding to no face)
}
See '/data2/ynli/human3d/hmr/run_hmr.py' for more details
'''
# num_pair = 64
# pair_split_fn = 'datasets/DF_Pose/Label/pair_split.json'
# hmr_pred_dir = 'temp/3d_hmr/hmr_df_openpose/pred/'
# output_dir = 'temp/3d_hmr/corr/'
num_pair = -1
pair_split_fn = 'datasets/DF_Pose/Label/pair_split_dfm.json'
hmr_pred_dir = 'datasets/DF_Pose/3d/hmr_dfm/pred/'
output_dir = 'datasets/DF_Pose/3d/hmr_dfm/corr/'
io.mkdir_if_missing(output_dir)
# load pair ids
pairs = io.load_json(pair_split_fn)
pair_list = pairs['test'] + pairs['train']
if num_pair > 0:
pair_list = pair_list[:num_pair]
for id_1, id_2 in tqdm.tqdm(pair_list):
pred_1 = io.load_data(hmr_pred_dir + id_1 + '.pkl')
pred_2 = io.load_data(hmr_pred_dir + id_2 + '.pkl')
corr_2to1, vis_mask_2 = calc_correspondence_from_smpl_internal(
pred_2, pred_1)
flow_util.write_corr(output_dir + '%s_%s.corr' % (id_2, id_1),
corr_2to1, vis_mask_2)
corr_1to2, vis_mask_1 = calc_correspondence_from_smpl_internal(
pred_1, pred_2)
flow_util.write_corr(output_dir + '%s_%s.corr' % (id_1, id_2),
corr_1to2, vis_mask_1)
def main():
data = io.load_data(quality="low")
X, y, class_names = preprocessing.create_data_tensor(data)
X_train, y_train, X_test, y_test = preprocessing.create_train_test_split(
X, y)
X = preprocessing.scale_spatially(X)
# for i in class_names:
# # print i
# # print class_names[i]
# # print np.where(y==i)
# # print np.where(y==i)[0]
# # print np.where(y==i)[0].size
# print class_names[i], i, np.where(y==i)[0].size
# analysis.plot_signals_two_column(data[class_names[y[0]]][0][:, 0:3],
# X[0, 0:3, :].T,
# ['Raw X', 'Raw Y', 'Raw Z'],
# ['Resampled X', 'Resampled Y', 'Resampled Z'])
shop_idx = np.where(y == 3)[0]
shop_idx = shop_idx[0:6]
# print X[shop_idx, 0, :].shape
NUM = 3
C1 = 0
C2 = 3
d1 = X[np.where(y == C1)[0][0:NUM], 2, :].T
d2 = X[np.where(y == C2)[0][0:NUM], 2, :].T
d1p = np.roll(d1, -1, 0) - d1
# d1p = d1p[0:d1.shape[0]-1]
d1p[-1, :] = d1p[-2, :]
d2p = np.roll(d2, -1, 0) - d2
d2p[-1, :] = d2p[-2, :]
labels1 = [str(class_names[C1]) + ' ' + str(i) for i in xrange(NUM)] + [
str(class_names[C1]) + '\' ' + str(i) for i in xrange(NUM)
]
labels2 = [str(class_names[C2]) + ' ' + str(i) for i in xrange(NUM)] + [
str(class_names[C2]) + '\' ' + str(i) for i in xrange(NUM)
]
print d1.shape, d1p.shape
print d2.shape, d2p.shape
print np.concatenate((d1, d1p), 1).shape
# analysis.plot_signals_two_column(np.concatenate((d1, d1p), 1),
# np.concatenate((d2, d2p), 1),
# labels1,
# labels2)
print class_names[4]
d2 = X[np.where(y == 4)[0][0:2], 2, :].T
analysis.plot_signals(d2, labels2)
def run_svm(quality="high", ablation=False, concat=True):
"""
Runs a simple SVM model with a linear kernel; first fits the model
on the training data (70 percent of the total data) and tests on
the rest of the data.
Args:
None
Returns:
None
"""
data = io.load_data(quality=quality)
X, y, class_names = preprocessing.create_data_tensor(data)
if ablation:
run_ablation_svm(X, y, class_names, quality)
return
X_train, y_train, X_test, y_test = preprocessing.create_train_test_split(X, y, test_size=0.3, shuffle=False)
flattened_Xtrain = preprocessing.flatten_matrix(X_train)
flattened_Xtest = preprocessing.flatten_matrix(X_test)
if concat:
X_train_400 = np.load('../data/seq_mining_features/k_2-w_2/X_train-400.npy')
X_test_400 = np.load('../data/seq_mining_features/k_2-w_2/X_test-400.npy')
y_train_400 = np.load('../data/seq_mining_features/k_2-w_2/y_train-400.npy')
y_test_400 = np.load('../data/seq_mining_features/k_2-w_2/y_test-400.npy')
print X_train_400.shape, y_train_400.shape
print flattened_Xtrain.shape, y_train.shape
print '-----------'
print X_test_400.shape, y_test_400.shape
print flattened_Xtest.shape, y_test.shape
flattened_Xtrain_concatenated = np.hstack((flattened_Xtrain, X_train_400[:, 0:75]))
flattened_Xtest_concatenated = np.hstack((flattened_Xtest, X_test_400[:, 0:75]))
print '-----------'
print flattened_Xtrain.shape, y_train.shape
print flattened_Xtest.shape, y_test.shape
C = 0.01
# fit svm model
svm_model = svm.SVC(kernel="linear", C=C, decision_function_shape='ovr')
svm_model.fit(flattened_Xtrain, y_train)
y_predict_train = svm_model.predict(flattened_Xtrain)
y_predict = svm_model.predict(flattened_Xtest)
analysis.run_analyses(y_predict_train, y_train, y_predict, y_test, class_names, ablation=False, confusion=False)
print '-----------==================-----------'
svm_model = svm.SVC(kernel="linear", C=C, decision_function_shape='ovr')
svm_model.fit(flattened_Xtrain_concatenated, y_train)
y_predict_train = svm_model.predict(flattened_Xtrain_concatenated)
y_predict = svm_model.predict(flattened_Xtest_concatenated)
analysis.run_analyses(y_predict_train, y_train, y_predict, y_test, class_names, ablation=False, confusion=False)
print '-----------==================-----------'
def divide_vert_into_bodypart():
'''
Devide 6890 verts of a SMPL model into 24 parts, each part corresponding to a joint. A vert will be assigned to the joint with the largest vert-to-angle weight.
May need to run this function under HMR environment
'''
smpl_dict = io.load_data('scripts/3d/neutral_smpl_with_cocoplus_reg.pkl')
weights = smpl_dict['weights']
vert2part = weights.argmax(axis=1).tolist()
io.save_json(vert2part, 'scripts/3d/smpl_vert_to_bodypart.json')
def resize_and_pad():
'''
resize the image that its longer side equals to new_size. Then pad the image to have the size [new_size, new_size]
create new pose label at the same time
'''
# config
new_size = 256
img_root = zalando_root + 'Img/img_zalando/'
output_dir = zalando_root + 'Img/img_zalando_%d/' % new_size
split = io.load_json(zalando_root + 'Split/zalando_split.json')
pose_label = io.load_data(zalando_root + 'Label/zalando_pose_label.pkl')
io.mkdir_if_missing(output_dir)
id_list = split['train'] + split['test']
# id_list = id_list[0:10]
new_pose_label = {}
for i, s_id in enumerate(id_list):
print('%d / %d' % (i, len(id_list)))
# resize image
img = cv2.imread(img_root + s_id + '_0.jpg')
w, h = img.shape[1], img.shape[0]
if w < h:
top = 0
bottom = 0
left = (h-w)//2
right = h-w-left
ratio = new_size/h
else:
top = (w-h)//2
bottom = w-h-top
left = 0
right = 0
ratio = new_size/w
img = cv2.copyMakeBorder(img, top, bottom, left, right, borderType=cv2.BORDER_REPLICATE)
img = cv2.resize(img, dsize=(new_size, new_size), interpolation=cv2.INTER_LINEAR)
cv2.imwrite(output_dir + s_id + '_0.jpg', img)
# resize clothing image
img1 = cv2.imread(img_root + s_id + '_1.jpg')
if not (img1.shape[0] == h and img1.shape[1] == w):
img1 = cv2.resize(img1, dsize=(w,h))
img1 = cv2.copyMakeBorder(img1, top, bottom, left, right, borderType=cv2.BORDER_REPLICATE)
img1 = cv2.resize(img1, dsize=(new_size, new_size), interpolation=cv2.INTER_LINEAR)
cv2.imwrite(output_dir + s_id + '_1.jpg', img1)
# modify pose label
pose = pose_label[s_id]
new_pose = [[(p[0]+left)*ratio, (p[1]+top)*ratio] if p != [-1,-1] else [-1,-1] for p in pose]
new_pose_label[s_id] = new_pose
io.save_data(new_pose_label, zalando_root + 'Label/zalando_pose_label_%d.pkl' % new_size)
def load_test_pair_index():
num_pair = 12800
pair_index = io.load_data(
'datasets/DF_Pose/Anno/NIPS17-test/p_pair_test.p'
) # test index used in NIPS17 paper: Pose Guided Person Image Generation
pair_split = io.load_json('datasets/DF_Pose/Label/pair_split.json')
# store previous generated pairs
pair_split['test_disordered_pair'] = pair_split['test']
# use pair indexes provided in NIPS17 paper
pair_split['test'] = [[s1[0:-4], s2[0:-4]]
for s1, s2 in pair_index[0:num_pair]]
np.random.shuffle(pair_split['test'])
io.save_json(pair_split, 'datasets/DF_Pose/Label/pair_split.json')
def create_dug_ca_gan_dataset():
num_train = 10
num_test = 10
same_train_test = True
samples = io.load_json(design_root + 'Label/ca_samples.json')
attr_label = io.load_data(design_root + 'Label/ca_attr_label.pkl')
bbox_label = io.load_data(design_root + 'Label/ca_bbox_label_256.pkl')
lm_label = io.load_data(design_root + 'Label/ca_landmark_label_256.pkl')
seg_path_list = io.load_json(design_root + 'Label/ca_seg_paths.json')
ca_split = io.load_json(design_root + 'Split/ca_gan_split_trainval_upper.json')
if same_train_test:
split = {
'train': ca_split['train'][0:num_train],
'test': ca_split['train'][0:num_train]
}
id_list = split['train']
else:
split = {
'train': ca_split['train'][0:num_train],
'test': ca_split['test'][0:num_test]
}
id_list = split['train'] + split['test']
samples = {s_id: samples[s_id] for s_id in id_list}
attr_label = {s_id:attr_label[s_id] for s_id in id_list}
bbox_label = {s_id:bbox_label[s_id] for s_id in id_list}
lm_label = {s_id:lm_label[s_id] for s_id in id_list}
seg_path_list = {s_id: seg_path_list[s_id] for s_id in id_list}
io.save_json(samples, design_root + 'Label/debugca_gan_samples.json')
io.save_data(attr_label, design_root + 'Label/debugca_gan_attr_label.pkl')
io.save_data(bbox_label, design_root + 'Label/debugca_gan_bbox_label.pkl')
io.save_data(lm_label, design_root + 'Label/debugca_gan_landmark_label.pkl')
io.save_json(seg_path_list, design_root + 'Label/debugca_seg_paths.json')
io.save_json(split, design_root + 'Split/debugca_gan_split.json')
def extract_feature(opt, save_feat=True):
fn_feat = os.path.join(
os.path.join('checkpoints', opt.id, 'feat',
'%s.pkl' % opt.which_epoch))
if os.path.isfile(fn_feat):
print('loading feature from %s' % fn_feat)
feat_data = io.load_data(fn_feat)
# feat_data['feat_train']
# feat_data['feat_test']
# feat_data['id_list_train']
# feat_data['id_list_test']
else:
# create model
model = AttributeEncoder()
model.initialize(opt)
model.eval()
feat_data = {
'feat_train': [],
'feat_test': [],
'id_list_train': [],
'id_list_test': []
}
for split in ['train', 'test']:
loader = CreateDataLoader(opt, split=split)
for i, data in enumerate(loader):
model.set_input(data)
model.extract_feat()
feat_data['feat_%s' %
split].append(model.output['feat'].data.cpu().numpy(
)) # size: batch_size * feat_size
feat_data['id_list_%s' %
split] += data['id'] # list of length batch_size
print('\r[%s] extract feature from %s samples %d/%d' %
(opt.id, split, i, len(loader)),
end='')
print('\n')
feat_data['feat_%s' % split] = np.vstack(feat_data['feat_%s' %
split])
if save_feat:
io.mkdir_if_missing(os.path.join('checkpoints', opt.id, 'feat'))
io.save_data(feat_data, fn_feat)
return feat_data
def create_image_info():
'''
create a .mat file containing:
- id_1
- id_2
- image_1
- image_2
- image_gen (generated by PoseTranfer_x)
'''
image_dir = '/data2/ynli/datasets/DF_Pose/Img/img_df/'
model_id = 'PoseTransfer_7.5'
image_gen_dir = '/data2/ynli/Fashion/fashionHD/checkpoints/%s/test/' % model_id
pair_indices = io.load_json(
'datasets/DF_Pose/Label/pair_split.json')['test'][0:num_sample]
pose_label = io.load_data('datasets/DF_Pose/Label/pose_label.pkl')
id_1 = [p[0] for p in pair_indices]
id_2 = [p[1] for p in pair_indices]
image_1 = []
image_2 = []
image_gen = []
scale_2over1 = []
for i in range(num_sample):
image_1.append(image_dir + id_1[i] + '.jpg')
image_2.append(image_dir + id_2[i] + '.jpg')
image_gen.append(image_gen_dir + '%s_%s.jpg' % (id_1[i], id_2[i]))
pose_1 = np.array(pose_label[id_1[i]])
pose_2 = np.array(pose_label[id_2[i]])
scale_2over1.append(pose_util.relative_scale_from_pose(pose_2, pose_1))
image_info = {
'id_1': id_1,
'id_2': id_2,
'image_1': image_1,
'image_2': image_2,
'image_gen': image_gen,
'model_id': model_id,
'scale_2over1': scale_2over1
}
data_dict = {
k: np.array(v, dtype=np.object)
for k, v in image_info.iteritems()
}
io.save_json(image_info, 'temp/patch_matching/label/image_info.json'
) # for other functions in this script
scipy.io.matlab.savemat(
'temp/patch_matching/label/image_info.mat', data_dict
) # for PatchMatch matlab tools and other matlab implementions
def initialize(self, opt, split):
self.opt = opt
self.root = opt.data_root
self.split = split
print('loading data ...')
samples = io.load_json(os.path.join(opt.data_root, opt.fn_sample))
# attr_label = io.load_data(os.path.join(opt.data_root, opt.fn_label))
# attr_entry = io.load_json(os.path.join(opt.data_root, opt.fn_entry))
data_split = io.load_json(os.path.join(opt.data_root, opt.fn_split))
lm_label = io.load_data(os.path.join(opt.data_root, opt.fn_landmark))
seg_paths = io.load_json(os.path.join(opt.data_root, opt.fn_seg_path))
edge_paths = io.load_json(os.path.join(opt.data_root,
opt.fn_edge_path))
# color_paths = io.load_json(os.path.join(opt.data_root, opt.fn_color_path))
flx_seg_paths = io.load_json(
os.path.join(opt.data_root, opt.fn_flx_seg_path))
self.id_list = data_split[split]
# self.attr_entry = attr_entry
if opt.max_dataset_size != float('inf'):
self.id_list = self.id_list[0:opt.max_dataset_size]
self.sample_list = [samples[s_id] for s_id in self.id_list]
# self.attr_label_list = [attr_label[s_id] for s_id in self.id_list]
self.lm_list = [lm_label[s_id] for s_id in self.id_list]
self.seg_path_list = [seg_paths[s_id] for s_id in self.id_list]
self.edge_path_list = [edge_paths[s_id] for s_id in self.id_list]
# self.color_path_list = [color_paths[s_id] for s_id in self.id_list]
self.flx_seg_path_list = [flx_seg_paths[s_id] for s_id in self.id_list]
# check data
# assert len(self.attr_entry) == len(self.attr_label_list[0]) == opt.n_attr, 'Attribute number not match!'
print('dataset created (%d samples)' % len(self))
# get transform
self.to_tensor = transforms.ToTensor()
# use standard normalization, which is different from attribute dataset
# image will be normalized again (under imagenet distribution) before fed into attribute encoder in GAN model
self.tensor_normalize_std = transforms.Normalize([0.5, 0.5, 0.5],
[0.5, 0.5, 0.5])
self.tensor_normalize_imagenet = transforms.Normalize(
[0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
if self.opt.color_jitter:
self.color_jitter = transforms.ColorJitter(brightness=0.3,
contrast=0.3,
saturation=0.3,
hue=0.3)
self.to_pil_image = transforms.ToPILImage()
def create_hmr_pose_label():
'''
name: [openpose_index, hmr_index]:
nose: 0,14
neck: 1, 12
right_shoulder: 2, 8
right_elow: 3, 7
right_wrist: 4, 6
left_shoulder: 5, 9
left_elbow: 6, 10
left_wrist: 7, 11
right_hip: 8, 2
right_knee: 9, 1
right_ankle: 10, 0
left_hip: 11, 3
left_knee: 12, 4
left_ankle: 13, 5
right_eye: 14, 16
left_eye: 15, 15
right_ear: 16, 18
left_ear: 17, 17
head_top: -, 13
'''
# DeepFashion
#image_split = io.load_json('datasets/DF_Pose/Label/image_split_dfm.json')
#smpl_pred_dir = 'datasets/DF_Pose/3d/hmr_dfm_v3/pred/'
#fn_out = 'datasets/DF_pose/Label/pose_label_hmr.pkl'
# Market-1501
image_split = io.load_json('datasets/market1501/Label/image_split.json')
smpl_pred_dir = 'datasets/market1501/3d/hmr/pred/'
fn_out = 'datasets/market1501/Label/pose_label_hmr.pkl'
id_list = image_split['train'] + image_split['test']
kp_order_map = [
14, 12, 8, 7, 6, 9, 10, 11, 2, 1, 0, 3, 4, 5, 16, 15, 18, 17
]
pose_label_hmr = {}
for sid in tqdm.tqdm(id_list):
pred = io.load_data(smpl_pred_dir + '%s.pkl' % sid)
joints = pred['joints']
pts = joints[kp_order_map]
pts[(pts[:, 0] < 0) | (pts[:, 0] > 255) | (pts[:, 1] < 0) |
(pts[:, 1] > 255)] = -1
pose_label_hmr[sid] = pts.tolist()
io.save_data(pose_label_hmr, fn_out)
def initialize(self, opt, split):
self.opt = opt
self.root = opt.data_root
if opt.debug:
split = 'debug'
self.split = split
#############################
# load data
#############################
print('loading data ...')
data_split = io.load_json(os.path.join(opt.data_root, opt.fn_split))
self.img_dir = os.path.join(opt.data_root, opt.img_dir)
self.seg_dir = os.path.join(opt.data_root, opt.seg_dir)
self.pose_label = io.load_data(os.path.join(opt.data_root,
opt.fn_pose))
#############################
# create index list
#############################
self.id_list = data_split[split]
#############################
# other
#############################
self.tensor_normalize_std = transforms.Normalize([0.5, 0.5, 0.5],
[0.5, 0.5, 0.5])
# self.color_jitter = transforms.ColorJitter(brightness=0.3, contrast=0.3, saturation=0.3, hue=0.3)
# self.to_pil_image = transforms.ToPILImage()
#############################
# define body limbs
#############################
# self.bparts = [
# ['lshoulder', 'lhip', 'rhip', 'rshoulder'],
# ['lshoulder', 'rshoulder', 'nose'],
# ['lshoulder', 'lelbow'],
# ['lelbow', 'lwrist'],
# ['rshoulder', 'relbow'],
# ['relbow', 'rwrist'],
# ['lhip', 'lknee'],
# ['rhip', 'rknee']]
self.bparts = [['rshoulder', 'rhip', 'lhip', 'lshoulder'],
['rshoulder', 'lshoulder', 'nose'],
['rshoulder', 'relbow'], ['relbow', 'rwrist'],
['lshoulder', 'lelbow'], ['lelbow', 'lwrist'],
['rhip', 'rknee'], ['lhip', 'lknee']]
