def create_cloth_edge_map():
'''
create edge map that only contains cloth edge (inside the cloth mask)
'''
# config
mask_dilate = 5
seg_dir = design_root + 'Img/seg_ca_syn_256/'
edge_dir = design_root + 'Img/edge_ca_256/'
output_dir = design_root + 'Img/edge_ca_256_cloth/'
io.mkdir_if_missing(output_dir)
split = io.load_json(design_root +
'Split/ca_gan_split_trainval_upper.json')
id_list = split['train'] + split['test']
for i, s_id in enumerate(id_list):
print('%d/%d' % (i, len(id_list)))
seg_map = image.imread(seg_dir + s_id + '.bmp', 'grayscale')
edge_map = image.imread(edge_dir + s_id + '.jpg', 'grayscale')
assert seg_map.shape == edge_map.shape
mask = ((seg_map == 3) | (seg_map == 4)).astype(np.uint8)
mask = cv2.dilate(mask, kernel=np.ones((mask_dilate, mask_dilate)))
edge_map_cloth = edge_map * mask
image.imwrite(edge_map_cloth, output_dir + s_id + '.jpg')
def test_affine_augmentation():
img = image.imread(
'datasets/DeepFashion/Fashion_design/Img/img_ca_256/ca_9.jpg')
assert img is not None
output_dir = 'temp/affine_augmentation'
io.mkdir_if_missing(output_dir)
# config
scale = [0.05, 0.1, 0.15]
num_per_scale = 10
w, h = img.shape[1], img.shape[0]
keypoint_src = np.array([[0, 0], [w, 0], [0, h]], dtype=np.float32)
for s in scale:
for i in range(num_per_scale):
offset = (np.random.rand(3, 2) * 2 - 1) * np.array([w, h]) * s
offset = offset.astype(np.float32)
keypoint_dst = keypoint_src + offset
m = cv2.getAffineTransform(keypoint_src, keypoint_dst)
img_trans = cv2.warpAffine(img,
m,
dsize=(w, h),
flags=cv2.INTER_LINEAR,
borderMode=cv2.BORDER_REPLICATE)
img_trans = img_trans * 0.8 + img * 0.2
image.imwrite(
img_trans, os.path.join(output_dir,
'affine_%f_%d.jpg' % (s, i)))
def create_color_map():
#### test
img = image.imread(
'datasets/DeepFashion/Fashion_design/Img/img_ca_256/ca_9.jpg')
assert img is not None
output_dir = 'temp/color_map_generation_test'
io.mkdir_if_missing(output_dir)
# color map by gaussian blur
kernel_size = 21
for sigma in [1, 2, 5, 10, 20]:
img_blur = cv2.GaussianBlur(img, (kernel_size, kernel_size), sigma)
image.imwrite(
img_blur,
os.path.join(output_dir,
'gaussian_%d_%f.jpg' % (kernel_size, sigma)))
# color map by downsampling
for scale in [2, 4, 8, 16, 32]:
w, h = img.shape[1], img.shape[0]
dw, dh = w // scale, h // scale
img_blur = cv2.resize(cv2.resize(img, (dw, dh),
interpolation=cv2.INTER_LINEAR),
(w, h),
interpolation=cv2.INTER_LINEAR)
image.imwrite(img_blur,
os.path.join(output_dir, 'downsample_%d.jpg') % scale)
def visualize_seg_map():
num_sample = 1000
output_dir = 'temp/seg_map'
io.mkdir_if_missing(output_dir)
samples = io.load_json(design_root + 'Label/ca_samples.json')
split = io.load_json(design_root + 'Split/ca_gan_split_trainval.json')
id_list = split['train'] + split['test']
id_list = [s_id for s_id in id_list if samples[s_id]['cloth_type'] == 3]
seg_dir_list = [
design_root + 'Img/seg_ca_256/', design_root + 'Img/seg_ca_syn_256/'
]
for i, s_id in enumerate(id_list[0:num_sample]):
s = samples[s_id]
img = image.imread(s['img_path'])
imgs = [img]
for seg_dir in seg_dir_list:
seg = image.imread(seg_dir + s_id + '.bmp') * 20
mask = img * (seg > 0).astype(np.float)
imgs += [seg, mask]
img = image.stitch(imgs, 0)
image.imwrite(img, os.path.join(output_dir, s_id + '.jpg'))
print(i)
def _align_and_resize_image_unit(worker_idx, id_list, samples, bbox_label,
lm_label, img_size, region_rate,
interp_method, aligned_bbox_label,
aligned_lm_label):
'''
Parallel helper function of align_and_resize_image()
'''
x_c = 0.5 * img_size
y_c = 0.5 * img_size
rg_size = region_rate * img_size
num_sample = len(id_list)
for idx, s_id in enumerate(id_list):
s = samples[s_id]
x1, y1, x2, y2 = bbox_label[s_id]
w = x2 - x1
h = y2 - y1
if w > h:
t_w = rg_size
t_h = rg_size * h / w
else:
t_w = rg_size * w / h
t_h = rg_size
t_x1 = x_c - 0.5 * t_w
t_x2 = x_c + 0.5 * t_w
t_y1 = y_c - 0.5 * t_h
t_y2 = y_c + 0.5 * t_h
# apply image transform
p_src = [(x1, y1), (x1, y2), (x2, y1), (x2, y2)]
p_tar = [(t_x1, t_y1), (t_x1, t_y2), (t_x2, t_y1), (t_x2, t_y2)]
img = image.imread(s['img_path_org'])
img_out, trans_mat = image.align_image(img,
p_src,
p_tar,
sz_tar=(img_size, img_size),
flags=interp_method)
image.imwrite(img_out, s['img_path'])
if aligned_bbox_label is not None and aligned_lm_label is not None:
# tranform bbox and landmarks
lm_src = np.array(lm_label[s_id])
lm_p_src = lm_src[:, 0:2] # landmark coordinates
lm_v = lm_src[:, 2:3] # visibility
lm_p_tar = image.transform_coordinate(lm_p_src, trans_mat)
lm_tar = np.hstack((lm_p_tar, lm_v)).tolist()
aligned_bbox_label[s_id] = [t_x1, t_y1, t_x2, t_y2]
aligned_lm_label[s_id] = lm_tar
print('[align and resize image] worker-%2d: %d / %d' %
(worker_idx, idx, num_sample))
def create_inner_edge_map():
'''
extract the edges inside the clothing regions
'''
# config
kernel_size = 7
threshold = 0
split = io.load_json(design_root + 'Split/ca_gan_split_trainval.json')
id_list = split['train'] + split['test']
edge_root = design_root + 'Img/edge_ca_256'
seg_root = design_root + 'Img/seg_ca_256'
output_dir = design_root + 'Img/edge_ca_256_inner'
io.mkdir_if_missing(output_dir)
kernel = np.zeros((kernel_size, kernel_size), np.uint8)
k = (kernel_size - 1) / 2
for i in range(kernel_size):
for j in range(kernel_size):
if np.abs(i - k) + np.abs(j - k) <= k:
kernel[i, j] = 1
for i, s_id in enumerate(id_list):
edge = image.imread(os.path.join(edge_root, s_id + '.jpg'),
'grayscale')
seg = image.imread(os.path.join(seg_root, s_id + '.bmp'), 'grayscale')
mask_upper = cv2.erode((seg == 3).astype(np.uint8), kernel)
mask_lower = cv2.erode((seg == 4).astype(np.uint8), kernel)
mask = mask_upper | mask_lower
edge_inner = edge * mask
edge_inner = (edge_inner >= threshold).astype(np.uint8) * edge_inner
image.imwrite(edge_inner, os.path.join(output_dir, s_id + '.jpg'))
print('extracting inner edge %d / %d' % (i, len(id_list)))
# create labels
edge_paths = {
s_id: os.path.join(output_dir, s_id + '.jpg')
for s_id in id_list
}
split_debug = io.load_json(design_root + 'Split/debugca_gan_split.json')
edge_paths_debug = {
s_id: p
for s_id, p in edge_paths.iteritems()
if s_id in split_debug['train'] + split_debug['test']
}
io.save_json(edge_paths, design_root + 'Label/ca_edge_inner_paths.json')
io.save_json(edge_paths_debug,
design_root + 'Label/debugca_edge_inner_paths.json')
def merge_seg_map():
'''
input seg map:
0-background, 1-hair, 2-head, 3-upperbody, 4-lowerbody, 5-leg, 6-arm
'''
# config
seg_root = '/data2/ynli/Fashion/ICCV17-fashionGAN/complete_demo/output/img_ca_256/seg_7'
tar_root = 'datasets/DeepFashion/Fashion_design/Img/seg_ca_256'
io.mkdir_if_missing(tar_root)
samples = io.load_json(
'datasets/DeepFashion/Fashion_design/Label/ca_samples.json')
seg_map_paths = {}
for i, (s_id, s) in enumerate(samples.items()):
seg_org = image.imread(os.path.join(seg_root, s_id + '.bmp'),
mode='grayscale')
# assert seg_org
if s['cloth_type'] == 1:
seg_mrg = (seg_org == 3).astype(np.uint8)
elif s['cloth_type'] == 2:
seg_mrg = (seg_org == 4).astype(np.uint8)
else:
seg_mrg = np.logical_or(seg_org == 3,
seg_org == 4).astype(np.uint8)
fn_out = os.path.join(tar_root, s_id + '.bmp')
image.imwrite(seg_mrg, fn_out)
seg_map_paths[s_id] = fn_out
print('\rmerge segmentation map: %d / %d' % (i, len(samples)))
print('\n')
io.save_json(
seg_map_paths,
'datasets/DeepFashion/Fashion_design/Label/ca_seg_paths.json')
def create_aligned_index():
'''
create (tar_id, edge_src_id, color_src_id) tuplets
'''
if False:
split = io.load_json(design_root +
'Split/ca_gan_split_trainval_upper.json')
############### for train/test ###############
edge_pair = io.load_json(design_root + 'Label/ca_tps_pair.json')
# attach color src index (random select) for each target
np.random.seed(0)
color_pair = {}
for set_name in ['train', 'test']:
id_list = split[set_name]
id_list_shuffle = [s_id for s_id in id_list]
np.random.shuffle(id_list_shuffle)
for tar_id, src_id in zip(id_list, id_list_shuffle):
color_pair[tar_id] = src_id
# create index file
aligned_index = {}
for s_id in split['train'] + split['test']:
aligned_index[s_id] = {
'id': s_id,
'edge_ids': [edge_pair[s_id]],
'color_ids': [color_pair[s_id]]
}
io.save_json(
aligned_index,
design_root + 'Label/ca_gan_trainval_upper_aligned_index.json')
############### for vis ###############
edge_vis_group = io.load_json(design_root +
'Label/ca_vis_tps_group.json')
vis_id_list = edge_vis_group.keys()
# check
assert set(vis_id_list).issubset(set(split['test']))
# attach color src candidate set (random select) for each target
num_color = 6
id_list = split['test']
color_vis_group = {}
for tar_id in vis_id_list:
id_list_shuffle = [s_id for s_id in id_list if s_id != tar_id]
np.random.shuffle(id_list_shuffle)
color_vis_group[tar_id] = id_list_shuffle[0:num_color]
# create index file
vis_aligned_index = {}
for s_id in vis_id_list:
vis_aligned_index[s_id] = {
'id': s_id,
'edge_ids': edge_vis_group[s_id],
'color_ids': color_vis_group[s_id]
}
io.save_json(vis_aligned_index,
design_root + 'Label/ca_gan_vis_upper_aligned_index.json')
############### visualize ###############
vis_aligned_index = io.load_json(
design_root + 'Label/ca_gan_vis_upper_aligned_index.json')
num_visual = 10
img_dir = design_root + 'Img/img_ca_256/'
output_dir = 'temp/aligned_index/'
io.mkdir_if_missing(output_dir)
for tar_id, index in vis_aligned_index.items()[0:num_visual]:
img_edge = [
image.imread(img_dir + s_id + '.jpg')
for s_id in [tar_id] + index['edge_ids']
]
img_color = [
image.imread(img_dir + s_id + '.jpg')
for s_id in [tar_id] + index['color_ids']
]
img_edge = np.concatenate(img_edge, axis=1)
img_color = np.concatenate(img_color, axis=1)
img_out = np.concatenate((img_edge, img_color), axis=0)
image.imwrite(img_out, output_dir + tar_id + '.jpg')
def create_flexible_segmap():
'''
reset the region label of a segmap (orignal 7 channel, 0-background, 1-hair, 2-head, 3-upper, 4-lower, 5-arm, 6-leg):
0-background
1-head
2-hair
3-cloth (both upper and lower)
4-cloth_flexible (boundary area of cloth region 3)
5-arm_flexible (arm region + sleeve_points region)
6-leg
'''
# config
cloth_mefilt_size = 9
cloth_dilate_size = 9
cloth_dilate_neck_size = 21
cloth_erode_size = 9
lower_arm_width = 15
upper_arm_width = 18
img_size = 256
# load data
seg_dir = design_root + 'Img/seg_ca_syn_256/'
output_dir = design_root + 'Img/seg_ca_syn_256_flexible/'
io.mkdir_if_missing(output_dir)
split = io.load_json(design_root +
'Split/ca_gan_split_trainval_upper.json')
lm_label = io.load_data(design_root + 'Label/ca_landmark_label_256.pkl')
pose_label = io.load_data(design_root + 'Label/ca_gan_pose_label_256.pkl')
id_list = split['train'] + split['test']
# create grid
gx, gy = np.meshgrid(range(img_size), range(img_size))
# subfunctions
def _get_rect_region(p1, p2, width, gx, gy):
'''return the mask of an overlap region between a rectangle and a cycle'''
rect = np.abs((p2[1] - p1[1]) * gx -
(p2[0] - p1[0]) * gy + p2[0] * p1[1] -
p2[1] * p1[0]) / np.sqrt((p2[1] - p1[1])**2 +
(p2[0] - p1[0])**2) <= width
cycle = (gx -
(p1[0] + p2[0]) / 2)**2 + (gy - (p1[1] + p2[1]) / 2)**2 <= (
np.sqrt((p1[0] - p2[0])**2 / 4 +
(p1[1] - p2[1])**2 / 4) + width)**2
return rect & cycle
def _get_cos_score(p0, p1, p2):
'''should be positive if p1 and p2 are at the same side of p0'''
v1 = p1 - p0
v2 = p2 - p0
return v1.dot(v2) / np.sqrt(v1.dot(v1) * v2.dot(v2))
for i, s_id in enumerate(id_list):
print('%d/%d: %s' % (i, len(id_list), s_id))
seg_map = image.imread(seg_dir + s_id + '.bmp', 'grayscale')
# unchanged part: head, hair and leg
hair_mask = seg_map == 1
head_mask = seg_map == 2
leg_mask = seg_map == 5
# cloth
cloth_mask = ((seg_map == 3) | (seg_map == 4)).astype(np.uint8)
base_cloth_mask = cv2.medianBlur(cloth_mask, ksize=cloth_mefilt_size)
# cloth flexible
cloth_flx_mask_outer = cv2.dilate(
cloth_mask,
kernel=np.ones((cloth_dilate_size, cloth_dilate_size), np.uint8))
cloth_flx_mask_inner = cv2.erode(
cloth_mask,
kernel=np.ones((cloth_erode_size, cloth_erode_size), np.uint8))
cloth_flx_mask_neck = cv2.dilate(
cloth_flx_mask_outer,
kernel=np.ones((cloth_dilate_neck_size, cloth_dilate_neck_size),
np.uint8)) * (head_mask.astype(np.uint8))
cloth_flx_mask = cloth_flx_mask_outer - cloth_flx_mask_inner + cloth_flx_mask_neck
cloth_mask = cloth_mask.astype(np.bool)
cloth_flx_mask = cloth_flx_mask.astype(np.bool)
# arm flexible
arm_flx_mask = (seg_map == 6)
pose = np.array(pose_label[s_id])
p_sleeve = np.array(
lm_label[s_id][2:4])[:,
0:2] # sleeve points [[x_l, y_l], [x_r, y_r]]
v_sleeve = np.array(lm_label[s_id][2:4])[:, 2]
p_hand = pose[[
4, 7
]] # hand points [[x_l, y_l], [x_r, y_r]]. the same below
p_elbow = pose[[3, 6]]
p_shoulder = pose[[2, 5]]
# clr_points = np.array(lm_label[s_id][0:2]) # collar
for j in range(2):
# if the sleeve point is visible: add arm region (defined by pose point, not fashion landmark) to arm_flx_mask
# otherwise, only add original arm regsion into arm_flx_mask
if v_sleeve[j] == 0 and (p_hand[j] != -1).all() and (
p_elbow[j] != -1).all() and (p_shoulder[j] != -1).all():
# case one: sleeve point on lower arm
if _get_cos_score(p_sleeve[j], p_hand[j],
p_elbow[j]) < _get_cos_score(
p_sleeve[j], p_elbow[j], p_shoulder[j]):
upper_arm = _get_rect_region(p_elbow[j], p_shoulder[j],
upper_arm_width, gx, gy)
lower_arm = _get_rect_region(p_sleeve[j], p_elbow[j],
lower_arm_width, gx, gy)
arm_flx_mask = arm_flx_mask | (
(upper_arm | lower_arm)
& cloth_flx_mask_outer.astype(np.bool))
else:
upper_arm = _get_rect_region(p_sleeve[j], p_shoulder[j],
upper_arm_width, gx, gy)
arm_flx_mask = arm_flx_mask | (
upper_arm & cloth_flx_mask_outer.astype(np.bool))
# combine channels
seg_org = seg_map.copy()
seg_map[:] = 0
seg_map[hair_mask] = 1
seg_map[head_mask] = 2
seg_map[cloth_mask] = 3
seg_map[cloth_flx_mask] = 4
seg_map[arm_flx_mask] = 5
seg_map[leg_mask] = 6
# save
# image.imwrite(np.concatenate((seg_org, arm_flx_mask.astype(np.uint8)),axis=0)*20, output_dir + s_id + '.bmp')
# image.imwrite(np.concatenate((seg_org, seg_map, arm_flx_mask.astype(np.uint8)),axis=0)*10, output_dir + s_id + '.bmp')
image.imwrite(seg_map, output_dir + s_id + '.bmp')
def visualize_attr_pred(self, model, num_top_attr=5):
'''
This method visualize attribute prediction result of each sample in an image:
- original image
- top-k predicted attributes
- annotated attributes
- top-k predicted attribute spatial maps (if available)
Input:
model: AttributeEncoderModel instance (to get model output)
num_top_attr
'''
if not self.data_loaded:
self.load_attr_data()
opt = self.opt
dir_output = os.path.join('checkpoints', opt.id, 'vis_attr')
io.mkdir_if_missing(dir_output)
for idx, s_id in enumerate(model.input['id']):
prob = model.output['prob'][idx].data.cpu().numpy().flatten()
if 'map' in model.output:
prob_map = model.output['map'][idx].data.cpu().numpy()
else:
prob_map = None
top_pred_attr = (-prob).argsort()[0:num_top_attr]
gt_attr = [
i for i, l in enumerate(self.attr_label[s_id]) if l == 1
]
img = image.imread(self.samples[s_id]['img_path'])
if prob_map is None:
img_out = img
else:
img_out = [img]
h, w = img.shape[0:2]
for i_att in top_pred_attr:
p = prob[i_att]
m = prob_map[i_att]
m = (m - m.min()) / (m.max() - m.min())
m = image.resize(m, (w, h))[:, :, np.newaxis]
img_out.append(img * m)
img_out = image.stitch(img_out, 0)
tag_list = [s_id, self.samples[s_id]['img_path_org']]
tag_list.append('prediction: ' + ', '.join([
'%s (%.3f)' % (self.attr_entry[i]['entry'], prob[i])
for i in top_pred_attr
]))
tag_list.append(
'annotation: ' +
', '.join([self.attr_entry[i]['entry'] for i in gt_attr]))
img_out = image.add_tag(img_out, tag_list, ['k', 'k', 'b', 'r'])
fn_output = os.path.join(dir_output, s_id + '.jpg')
image.imwrite(img_out, fn_output)