def batch_generate_pymeta(data_root_folder, force_to_generate=False):
"""
This function will convert batches into pymeta style which can be loaded by convnet
"""
allfolder = iu.getfolderlist(data_root_folder)
print "Get %d folders" % len(allfolder)
l = []
import sys
for fn in allfolder:
a = DHMLPE()
fp = iu.fullfile(data_root_folder, fn, "matlab_meta.mat")
if iu.exists(fp, "file"):
save_fp = iu.fullfile(data_root_folder, fn, "batches.meta")
print "-----------------------------"
print "Processing ", fp
if iu.exists(save_fp, "file") and not force_to_generate:
print "Ha ha, it exists!"
else:
meta = a.get_convnet_meta(fp)
mio.pickle(save_fp, meta)
print "Saved %s" % save_fp
else:
l = l + [fp]
print "=============\n"
print "Here is what I cannot find (%d in total)" % len(l)
print l
def batch_generate_pymeta(data_root_folder, force_to_generate=False):
"""
This function will convert batches into pymeta style which can be loaded by convnet
"""
allfolder = iu.getfolderlist(data_root_folder)
print 'Get %d folders' % len(allfolder)
l = []
import sys
for fn in allfolder:
a = DHMLPE()
fp = iu.fullfile(data_root_folder, fn, 'matlab_meta.mat')
if iu.exists(fp, 'file'):
save_fp = iu.fullfile(data_root_folder, fn, 'batches.meta')
print '-----------------------------'
print 'Processing ', fp
if iu.exists(save_fp, 'file') and not force_to_generate:
print 'Ha ha, it exists!'
else:
meta = a.get_convnet_meta(fp)
mio.pickle(save_fp, meta)
print 'Saved %s' % save_fp
else:
l = l + [fp]
print '=============\n'
print 'Here is what I cannot find (%d in total)' % len(l)
print l
def show_bm_cmp(ndata, gt_target, mv_target, bm_targets, mv_score, gt_score,
bm_score, solver=None, save_path=None):
mv_margin = MMLSSolver.calc_margin(gt_target - mv_target).flatten() * 1200
bm_margin = MMLSSolver.calc_margin(gt_target - bm_targets).flatten() * 1200
save_path = '/public/sijinli2/ibuffer/2015-02-04/bm_cmp_SP_t004_act_14_graph_0010'
save_path = '/public/sijinli2/ibuffer/2015-02-14/bm_cmp_FCJ0_act_14_graph_0020_test_'
d = {'gt_target':gt_target, 'mv_target':mv_target, 'bm_target':bm_targets,
'mv_score':mv_score, 'gt_score':gt_score, 'bm_score':bm_score
}
if save_path:
mio.pickle(save_path, d)
ndata = mv_margin.size
pt_size = 15
pl.subplot(4,1,1)
avg = np.mean(bm_margin)
pl.scatter(range(ndata), bm_margin, s=pt_size,c='r',label='mpjpe between max_score and best match (avg={:.3f})'.format(avg))
pl.legend()
pl.subplot(4,1,2)
pl.scatter(range(ndata), mv_score - bm_score, s=pt_size,c='b',label='max score - best_match_score')
pl.legend()
pl.subplot(4,1,3)
pl.scatter(range(ndata), (mv_score - bm_score)/np.abs(mv_score), c='b',label='(max_score - best_match_score)/abs(max_score)')
pl.legend()
pl.subplot(4,1,4)
pl.scatter(range(ndata), mv_score, s=pt_size, c='r',label='max score')
pl.scatter(range(ndata), bm_score, s=pt_size,c='b',label='best match score')
pl.legend()
pl.show()
def ReadDataToHMLPEDic(imgdir,example_path, data_category, max_per_batch,save_dir):
"""
Read all data in 'data_category'
into HMLPE dictionary
There is no need to generating training data, since they can be generated in
hmlpe.py
"""
import scipy.io as sio
import iutils as iu
import iread.myio as mio
import iread.hmlpe as hmlpe
import imgproc
from PIL import Image
if data_category != 'istest':
print 'Warn: The correctness of data type %s is not guaranteed' % data_category
all_example = sio.loadmat(example_path)['examples']
examples = ExtractSubExample(all_example, data_category)
ndata = examples.shape[-1]
iu.ensure_dir(save_dir)
buf_size = min(ndata, max_per_batch)
dimdic = {'data':(112,112,3), 'part_indmap':(8,8), 'joint_indmap':(8,8)}
nparts = 7
njoints = 8
d = hmlpe.HMLPE.prepare_savebuffer(dimdic, buf_size, nparts, njoints)
d['oridet'] = np.zeros((4,buf_size), dtype=np.int)
d['coords'] = np.ndarray((2,29, buf_size), dtype=np.float32)
tdsize = dimdic['data'][0]
dsize = dimdic['data'][0] * dimdic['data'][1] * dimdic['data'][2]
d['data'] = d['data'].reshape((dsize, -1),order='F')
d['is_positive'][:] = True
d['is_mirror'][:] = False
bid = 1
j = 0
for i in range(ndata):
if j == max_per_batch:
mio.pickle(iu.fullfile(save_dir, 'data_batch_%d' % bid), d)
bid = bid + 1
if ndata - i < max_per_batch:
d = hmlpe.HMLPE.prepare_savebuffer(dimdic, buf_size, nparts, njoints)
fp = iu.fullfile(imgdir, str(examples[i]['filepath'][0]))
img = Image.open(fp)
tbox = examples[i]['torsobox'][0].reshape((4))
d['filenames'][j] = fp
d['coords'][...,j] = examples[i]['coords']
d['oribbox'][...,j] = bbox = ExtendBndbox(tbox, img.size)
orijoints8 = CvtCoordsToJoints(examples[i]['coords']).reshape((8,2),order='C') - 1 # to python stype 0-idx
d['joints8'][...,j] = TransformPoints(orijoints8, bbox, dimdic['data']).reshape((8,2),order='C')
imgarr = imgproc.ensure_rgb(np.asarray(img))
sub_img = Image.fromarray(imgarr[bbox[1]:bbox[3], bbox[0]:bbox[2],:])
data_img = np.asarray(sub_img.resize((dimdic['data'][0], dimdic['data'][1]))).reshape((dsize),order='F')
d['data'][...,j] = data_img
d['indmap'][...,j] = hmlpe.HMLPE.create_part_indicatormap(d['joints8'][...,j], hmlpe.part_idx, dimdic['part_indmap'], 0.3, 30.0, 12.0)
d['joint_indmap'][...,j] = hmlpe.HMLPE.create_joint_indicatormap(d['joints8'][...,j], dimdic['joint_indmap'], 30.0, 12.0)
d['jointmasks'][...,j] = hmlpe.HMLPE.makejointmask(dimdic['data'], d['joints8'][...,j])
j = j + 1
mio.pickle(iu.fullfile(save_dir, 'data_batch_%d' % bid), d)
def show_what_is_best_all(train_dp, test_dp, solver):
"""
In this funciton, I will show what is the upper bound for the KNN search like methods
"""
candidates = train_dp.data_dic['feature_list'][0][..., train_dp.data_range]
eval_data = train_dp.data_dic['feature_list'][0][..., test_dp.data_range]
print 'Begin to calc the best mpjpe candidate shape {} eval shape'.format(candidates.shape, eval_data.shape)
res = what_is_the_best_match(candidates, eval_data, solver)
save_path = '/public/sijinli2/ibuffer/2015-01-16/best_match_act_14'
mio.pickle(save_path, {'best_mpjpe':res})
def show_what_is_best_all(train_dp, test_dp, solver):
"""
In this funciton, I will show what is the upper bound for the KNN search like methods
"""
candidates = train_dp.data_dic["feature_list"][0][..., train_dp.data_range]
eval_data = train_dp.data_dic["feature_list"][0][..., test_dp.data_range]
print "Begin to calc the best mpjpe candidate shape {} eval shape".format(candidates.shape, eval_data.shape)
res = what_is_the_best_match(candidates, eval_data, solver)
save_path = "/public/sijinli2/ibuffer/2015-01-16/best_match_act_14"
mio.pickle(save_path, {"best_mpjpe": res})
def ReadCropImageToHMLPEDic(dataset_dir, save_dir, istrain = False, isOC = True):
"""
This function will be used for generating testing data
Because training and testing data has different format in oribbox
For generating trainign samples,
please use
create_lsp_regression_data.m
(dataset_dir, type=3, opt)
opt.OC = ?
and hmlpe.py
"""
import iutils as iu
import iread.hmlpe as hmlpe
import iread.myio as mio
import scipy.io as sio
from PIL import Image
ndata = 1000
if istrain:
s_idx = 0
else:
s_idx = 1000
imgdir = iu.fullfile(dataset_dir, 'images-crop')
if isOC:
dmat = sio.loadmat(iu.fullfile(dataset_dir, 'jointsOC.mat'))
else:
dmat = sio.loadmat(iu.fullfile(dataset_dir, 'joints-crop.mat'))
lsp_jt = dmat['joints']
dimdic = {'data':(112,112,3), 'part_indmap':(8,8), 'joint_indmap': (8,8)}
nparts = 7
njoints = 8
d = hmlpe.HMLPE.prepare_savebuffer(dimdic, ndata, nparts, njoints)
d['data'] = d['data'].reshape((-1,ndata),order='F')
d['is_mirror'][:] = False
d['is_positive'][:] = True
for idx in range(s_idx, s_idx + ndata):
imgpath = iu.fullfile(imgdir, 'im%04d.jpg' % (idx + 1))
img = Image.open(imgpath)
i = idx - s_idx
orijoints8, isvisible = GetJoints8(lsp_jt[...,idx])
bbox = GetUpperBodyBox(img.size)
img_arr = np.asarray(img)[bbox[1]:bbox[3], bbox[0]:bbox[2],:]
s = np.asarray([(dimdic['data'][1]-1.0)/(bbox[2] - bbox[0]),(dimdic['data'][0]-1.0)/(bbox[3]-bbox[1])]).reshape((1,2))
tjoints = (orijoints8 - bbox[0:2,:].reshape((1,2)))*s
masks = hmlpe.HMLPE.makejointmask(dimdic['data'], tjoints)
d['data'][...,i] = np.asarray(Image.fromarray(img_arr).resize((dimdic['data'][0], dimdic['data'][1]))).reshape((-1,1),order='F').flatten()
d['joints8'][...,i] = tjoints
d['jointmasks'][...,i] = np.logical_and(masks, isvisible)
d['filenames'][i] = imgpath
d['oribbox'][...,i] = bbox.flatten()
d['indmap'][...,i] = hmlpe.HMLPE.create_part_indicatormap(tjoints, hmlpe.part_idx, dimdic['part_indmap'], 0.3, 30.0, 12.0)
d['joint_indmap'][...,i] = hmlpe.HMLPE.create_joint_indicatormap(tjoints, dimdic['joint_indmap'], 30.0, 12.0)
mio.pickle(iu.fullfile(save_dir, 'data_batch_1'), d)
def save_again():
"""
change net_list objects to saved dict
"""
d = Solver.get_saved_model("/public/sijinli2/ibuffer/2015-01-13/itheano_test")
save_path = "/public/sijinli2/ibuffer/2015-01-13/saved_haha"
for net_name in d["net_list"]:
net = d["net_list"][net_name]
res_dic = dict()
for e in net["layers"]:
l = net["layers"][e]
res_dic[e] = [l[0], l[1], l[2].save_to_dic()]
d["net_list"][net_name]["layers"] = res_dic
print d["net_list"]["eval_net"].keys()
mio.pickle(save_path, d)
def save_again():
"""
change net_list objects to saved dict
"""
d = Solver.get_saved_model(
'/public/sijinli2/ibuffer/2015-01-13/itheano_test')
save_path = '/public/sijinli2/ibuffer/2015-01-13/saved_haha'
for net_name in d['net_list']:
net = d['net_list'][net_name]
res_dic = dict()
for e in net['layers']:
l = net['layers'][e]
res_dic[e] = [l[0], l[1], l[2].save_to_dic()]
d['net_list'][net_name]['layers'] = res_dic
print d['net_list']['eval_net'].keys()
mio.pickle(save_path, d)
def show_bm_cmp(ndata, gt_target, mv_target, bm_targets, mv_score, gt_score, bm_score, solver=None, save_path=None):
mv_margin = MMLSSolver.calc_margin(gt_target - mv_target).flatten() * 1200
bm_margin = MMLSSolver.calc_margin(gt_target - bm_targets).flatten() * 1200
save_path = "/public/sijinli2/ibuffer/2015-01-22/saved_batch_data_test_net4_K_rbf_correct_test"
d = {
"gt_target": gt_target,
"mv_target": mv_target,
"bm_target": bm_targets,
"mv_score": mv_score,
"gt_score": gt_score,
"bm_score": bm_score,
}
if save_path:
mio.pickle(save_path, d)
ndata = mv_margin.size
pt_size = 15
pl.subplot(4, 1, 1)
avg = np.mean(bm_margin)
pl.scatter(
range(ndata),
bm_margin,
s=pt_size,
c="r",
label="mpjpe between max_score and best match (avg={:.3f})".format(avg),
)
pl.legend()
pl.subplot(4, 1, 2)
pl.scatter(range(ndata), mv_score - bm_score, s=pt_size, c="b", label="max score - best_match_score")
pl.legend()
pl.subplot(4, 1, 3)
pl.scatter(
range(ndata),
(mv_score - bm_score) / np.abs(mv_score),
c="b",
label="(max_score - best_match_score)/abs(max_score)",
)
pl.legend()
pl.subplot(4, 1, 4)
pl.scatter(range(ndata), mv_score, s=pt_size, c="r", label="max score")
pl.scatter(range(ndata), bm_score, s=pt_size, c="b", label="best match score")
pl.legend()
pl.show()
def eval_mpjpe(self, op):
"""
This function will simple evaluate mpjpe
"""
train = False
net = self.feature_net
output_layer_name = self.prediction_layer_name
pred_outputs = net.get_layer_by_names([output_layer_name])[0].outputs
pred_func = theano.function(inputs=net.inputs,
outputs=pred_outputs,
on_unused_input='ignore')
itm = iu.itimer()
itm.restart()
cur_data = self.get_next_batch(train)
pred_mpjpe_l = []
pred_target_l = []
save_folder = op.get_value('save_res_path')
assert (save_folder is not None)
iu.ensure_dir(save_folder)
save_path = iu.fullfile(save_folder, 'pose_prediction')
net.set_train_mode(
False) # added in Oct 27, 2015 After ICCV submission
while True:
self.print_iteration()
ndata = cur_data[2][0].shape[-1]
input_data = self.prepare_data(cur_data[2])
gt_target = input_data[1].T
print 'gt_target.shape = {}'.format(gt_target.shape)
preds = pred_func(*[input_data[0]])[0].T
print 'Prediction.shape = {}'.format(preds.shape)
pred_mpjpe = dutils.calc_mpjpe_from_residual(
preds - gt_target, 17) * 1200
print 'pred_mpjpe = {}'.format(pred_mpjpe.mean())
self.epoch, self.batchnum = self.test_dp.epoch, self.test_dp.batchnum
pred_mpjpe_l.append(pred_mpjpe)
pred_target_l.append(preds)
if self.epoch == 1:
break
cur_data = self.get_next_batch(train)
mio.pickle(save_path, {'preds': pred_target_l, 'mpjpe': pred_mpjpe_l})
preds = np.concatenate(pred_target_l, axis=1)
mpjpe = np.concatenate(pred_mpjpe_l, axis=1)
mio.pickle(save_path, {'preds': pred_target_l, 'mpjpe': pred_mpjpe_l})
def cvt1(source_exp_name, target_exp_name):
print '''
SP_t004_act_14:
source meta [rel_gt, img_feature_accv_fc_j0, relskel_feature_t004]
Raw_SP_t004_act_14:
target meta [rel_gt, img_feature_accv_fc_j0, rel_gt]
'''
base_path = '/opt/visal/tmp/for_sijin/Data/H36M/H36MExp/'
source_meta = mio.unpickle(
iu.fullfile(base_path, 'folder_%s' % source_exp_name, 'batches.meta'))
target_meta_folder = iu.fullfile(base_path, 'folder_%s' % target_exp_name)
target_meta_path = iu.fullfile(target_meta_folder, 'batches.meta')
d = source_meta.copy()
print d.keys()
d['feature_list'] = [source_meta['feature_list'][k] for k in [0, 1, 0]]
d['feature_dim'] = [source_meta['feature_dim'][k] for k in [0, 1, 0]]
# print d['info']
print 'folder :{}\n path {}'.format(target_meta_folder, target_meta_path)
iu.ensure_dir(target_meta_folder)
mio.pickle(target_meta_path, d)
def cvt1(source_exp_name, target_exp_name):
print '''
SP_t004_act_14:
source meta [rel_gt, img_feature_accv_fc_j0, relskel_feature_t004]
Raw_SP_t004_act_14:
target meta [rel_gt, img_feature_accv_fc_j0, rel_gt]
'''
base_path = '/opt/visal/tmp/for_sijin/Data/H36M/H36MExp/'
source_meta = mio.unpickle(iu.fullfile(base_path, 'folder_%s' % source_exp_name,
'batches.meta'))
target_meta_folder = iu.fullfile(base_path, 'folder_%s' % target_exp_name)
target_meta_path = iu.fullfile(target_meta_folder, 'batches.meta')
d = source_meta.copy()
print d.keys()
d['feature_list'] = [source_meta['feature_list'][k] for k in [0, 1, 0]]
d['feature_dim'] = [source_meta['feature_dim'][k] for k in [0, 1, 0]]
# print d['info']
print 'folder :{}\n path {}'.format(target_meta_folder, target_meta_path)
iu.ensure_dir(target_meta_folder)
mio.pickle(target_meta_path, d)
def eval_mpjpe(self, op):
"""
This function will simple evaluate mpjpe
"""
train=False
net = self.feature_net
output_layer_name = self.prediction_layer_name
pred_outputs = net.get_layer_by_names([output_layer_name])[0].outputs
pred_func = theano.function(inputs=net.inputs, outputs=pred_outputs,on_unused_input='ignore')
itm = iu.itimer()
itm.restart()
cur_data = self.get_next_batch(train)
pred_mpjpe_l = []
pred_target_l = []
save_folder = op.get_value('save_res_path')
assert(save_folder is not None)
iu.ensure_dir(save_folder)
save_path = iu.fullfile(save_folder, 'pose_prediction')
net.set_train_mode(False) # added in Oct 27, 2015 After ICCV submission
while True:
self.print_iteration()
ndata = cur_data[2][0].shape[-1]
input_data = self.prepare_data(cur_data[2])
gt_target = input_data[1].T
print 'gt_target.shape = {}'.format(gt_target.shape)
preds = pred_func(*[input_data[0]])[0].T
print 'Prediction.shape = {}'.format(preds.shape)
pred_mpjpe = dutils.calc_mpjpe_from_residual(preds - gt_target, 17) * 1200
print 'pred_mpjpe = {}'.format(pred_mpjpe.mean())
self.epoch, self.batchnum = self.test_dp.epoch, self.test_dp.batchnum
pred_mpjpe_l.append(pred_mpjpe)
pred_target_l.append(preds)
if self.epoch == 1:
break
cur_data = self.get_next_batch(train)
mio.pickle(save_path, {'preds':pred_target_l, 'mpjpe':pred_mpjpe_l})
preds = np.concatenate(pred_target_l, axis=1)
mpjpe = np.concatenate(pred_mpjpe_l, axis=1)
mio.pickle(save_path, {'preds':pred_target_l, 'mpjpe':pred_mpjpe_l})
def ReadDataToHMLPEDic(imgdir, example_path, data_category, max_per_batch,
save_dir):
"""
Read all data in 'data_category'
into HMLPE dictionary
There is no need to generating training data, since they can be generated in
hmlpe.py
"""
import scipy.io as sio
import iutils as iu
import iread.myio as mio
import iread.hmlpe as hmlpe
import imgproc
from PIL import Image
if data_category != 'istest':
print 'Warn: The correctness of data type %s is not guaranteed' % data_category
all_example = sio.loadmat(example_path)['examples']
examples = ExtractSubExample(all_example, data_category)
ndata = examples.shape[-1]
iu.ensure_dir(save_dir)
buf_size = min(ndata, max_per_batch)
dimdic = {
'data': (112, 112, 3),
'part_indmap': (8, 8),
'joint_indmap': (8, 8)
}
nparts = 7
njoints = 8
d = hmlpe.HMLPE.prepare_savebuffer(dimdic, buf_size, nparts, njoints)
d['oridet'] = np.zeros((4, buf_size), dtype=np.int)
d['coords'] = np.ndarray((2, 29, buf_size), dtype=np.float32)
tdsize = dimdic['data'][0]
dsize = dimdic['data'][0] * dimdic['data'][1] * dimdic['data'][2]
d['data'] = d['data'].reshape((dsize, -1), order='F')
d['is_positive'][:] = True
d['is_mirror'][:] = False
bid = 1
j = 0
for i in range(ndata):
if j == max_per_batch:
mio.pickle(iu.fullfile(save_dir, 'data_batch_%d' % bid), d)
bid = bid + 1
if ndata - i < max_per_batch:
d = hmlpe.HMLPE.prepare_savebuffer(dimdic, buf_size, nparts,
njoints)
fp = iu.fullfile(imgdir, str(examples[i]['filepath'][0]))
img = Image.open(fp)
tbox = examples[i]['torsobox'][0].reshape((4))
d['filenames'][j] = fp
d['coords'][..., j] = examples[i]['coords']
d['oribbox'][..., j] = bbox = ExtendBndbox(tbox, img.size)
orijoints8 = CvtCoordsToJoints(examples[i]['coords']).reshape(
(8, 2), order='C') - 1 # to python stype 0-idx
d['joints8'][..., j] = TransformPoints(orijoints8, bbox,
dimdic['data']).reshape(
(8, 2), order='C')
imgarr = imgproc.ensure_rgb(np.asarray(img))
sub_img = Image.fromarray(imgarr[bbox[1]:bbox[3], bbox[0]:bbox[2], :])
data_img = np.asarray(
sub_img.resize((dimdic['data'][0], dimdic['data'][1]))).reshape(
(dsize), order='F')
d['data'][..., j] = data_img
d['indmap'][..., j] = hmlpe.HMLPE.create_part_indicatormap(
d['joints8'][..., j], hmlpe.part_idx, dimdic['part_indmap'], 0.3,
30.0, 12.0)
d['joint_indmap'][..., j] = hmlpe.HMLPE.create_joint_indicatormap(
d['joints8'][..., j], dimdic['joint_indmap'], 30.0, 12.0)
d['jointmasks'][...,
j] = hmlpe.HMLPE.makejointmask(dimdic['data'],
d['joints8'][..., j])
j = j + 1
mio.pickle(iu.fullfile(save_dir, 'data_batch_%d' % bid), d)
def ReadTestDataToHMLPEDic(imgdir, annot_dir, tech_report_path, save_path=None, iou_thresh=0.5):
"""
This function will use detection window provided by the author
Also, erro detection window are not included
For image without detection, all the joint points will be zero
"""
import Stickmen
import scipy.io as sio
import PIL
import iutils as iu
from PIL import Image
import matplotlib.pyplot as plt
import iread.hmlpe as hp
import iread.myio as mio
import ipyml.geometry as igeo
rp = sio.loadmat(tech_report_path)['techrep2010_buffy']
d_annot = ExtractAnnotation(annot_dir)
ndata = rp.shape[1]
data = []
tsize = 112 # the size of test image
part_ind_dim = (8,8)
joint_ind_dim = (8,8)
filter_size = 32.0
stride = 12.0
indrate = 0.3
joint_filter_size = 32.0
joint_stride = 12.0
njoints = 8
nparts = 7
ndet = ndata # One test point for one image
newdim = (tsize,tsize,3)
data = hp.HMLPE.prepare_savebuffer({'data':newdim,\
'part_indmap': part_ind_dim,\
'joint_indmap':joint_ind_dim },\
ndet, nparts, njoints)
# add Buffy specific field
add_buffy_specific_field(data, ndet, imgdir)
idx = 0
f_calc_area = lambda rec: (rec[1][1]-rec[0][1]) * (rec[1][0]-rec[0][0]) if len(rec)==2 else 0
f_mk_rec = lambda det: ((det[0],det[1]), (det[2], det[3]))
for i in range(ndata):
ep = rp[0,i][1][0][0]
fr = rp[0,i][2][0][0]
imgpath = iu.fullfile(imgdir, 'buffy_s5e'+str(ep)+'_original', \
('%06d' % fr) + '.jpg')
img = Image.open(imgpath)
data['ep'][...,i] = ep
data['fr'][...,i] = fr
data['filenames'][i] = imgpath
data['is_positive'][...,i] = True
data['annotation'][...,i] = Stickmen.ReorderStickmen(d_annot[ep][fr])
data['gt_det'][...,i] = np.asarray(Stickmen.EstDetFromStickmen( data['annotation'][...,i]))
gt_rec = f_mk_rec(data['gt_det'][...,i]) #
gt_area = f_calc_area(gt_rec)
if rp[0,i][0].size == 0: # No detection are found in this image
# imgdata will also be all zero
# oridet will be all zero
# oribbox will be all zero
# joints8 will be all zero
# jointmasks wiil be all zero
# indmap will be all zero
# joint_indmap will be all zero
# nothing need to be done, since they are default value
pass
else:
m = -1
for j in range(rp[0,i][0].size):
det = rp[0,i][0][0,j]['det'][0] # det = (x,y,x1,y1)
cur_rec = f_mk_rec(det)
int_area = f_calc_area(igeo.RectIntersectRect(cur_rec, gt_rec))
if int_area > ( gt_area - int_area ) * iou_thresh:
m = j
break
if m != -1:
oribbox = ExtendBndbox(det, img.size)
arr_img = np.asarray(img)[oribbox[1]:oribbox[3]+1, \
oribbox[0]:oribbox[2]+1,:]
res_img = Image.fromarray(arr_img).resize((tsize,tsize))
data['data'][...,i] =np.asarray(res_img)
tmppoints = convert_AHEcoor_to_joints8(data['annotation'][...,i])
data['joints8'][...,i] = TransformPoints(tmppoints, oribbox, np.asarray(res_img.size) -1)
data['jointmasks'][...,i] = hp.HMLPE.makejointmask(newdim, data['joints8'][...,i])
data['indmap'][...,i] = hp.HMLPE.create_part_indicatormap(data['joints8'][...,i], hp.part_idx, part_ind_dim, indrate, filter_size, stride)
data['joint_indmap'][...,i] = hp.HMLPE.create_joint_indicatormap(data['joints8'][...,i], joint_ind_dim, joint_filter_size, joint_stride)
data['oribbox'][...,i] = oribbox
data['data'] = data['data'].reshape((-1,ndet),order='F')
if save_path is not None:
mio.pickle(save_path, data)
return data
def ReadCropImageToHMLPEDic(dataset_dir, save_dir, istrain=False, isOC=True):
"""
This function will be used for generating testing data
Because training and testing data has different format in oribbox
For generating trainign samples,
please use
create_lsp_regression_data.m
(dataset_dir, type=3, opt)
opt.OC = ?
and hmlpe.py
"""
import iutils as iu
import iread.hmlpe as hmlpe
import iread.myio as mio
import scipy.io as sio
from PIL import Image
ndata = 1000
if istrain:
s_idx = 0
else:
s_idx = 1000
imgdir = iu.fullfile(dataset_dir, 'images-crop')
if isOC:
dmat = sio.loadmat(iu.fullfile(dataset_dir, 'jointsOC.mat'))
else:
dmat = sio.loadmat(iu.fullfile(dataset_dir, 'joints-crop.mat'))
lsp_jt = dmat['joints']
dimdic = {
'data': (112, 112, 3),
'part_indmap': (8, 8),
'joint_indmap': (8, 8)
}
nparts = 7
njoints = 8
d = hmlpe.HMLPE.prepare_savebuffer(dimdic, ndata, nparts, njoints)
d['data'] = d['data'].reshape((-1, ndata), order='F')
d['is_mirror'][:] = False
d['is_positive'][:] = True
for idx in range(s_idx, s_idx + ndata):
imgpath = iu.fullfile(imgdir, 'im%04d.jpg' % (idx + 1))
img = Image.open(imgpath)
i = idx - s_idx
orijoints8, isvisible = GetJoints8(lsp_jt[..., idx])
bbox = GetUpperBodyBox(img.size)
img_arr = np.asarray(img)[bbox[1]:bbox[3], bbox[0]:bbox[2], :]
s = np.asarray([(dimdic['data'][1] - 1.0) / (bbox[2] - bbox[0]),
(dimdic['data'][0] - 1.0) / (bbox[3] - bbox[1])
]).reshape((1, 2))
tjoints = (orijoints8 - bbox[0:2, :].reshape((1, 2))) * s
masks = hmlpe.HMLPE.makejointmask(dimdic['data'], tjoints)
d['data'][..., i] = np.asarray(
Image.fromarray(img_arr).resize(
(dimdic['data'][0], dimdic['data'][1]))).reshape(
(-1, 1), order='F').flatten()
d['joints8'][..., i] = tjoints
d['jointmasks'][..., i] = np.logical_and(masks, isvisible)
d['filenames'][i] = imgpath
d['oribbox'][..., i] = bbox.flatten()
d['indmap'][..., i] = hmlpe.HMLPE.create_part_indicatormap(
tjoints, hmlpe.part_idx, dimdic['part_indmap'], 0.3, 30.0, 12.0)
d['joint_indmap'][..., i] = hmlpe.HMLPE.create_joint_indicatormap(
tjoints, dimdic['joint_indmap'], 30.0, 12.0)
mio.pickle(iu.fullfile(save_dir, 'data_batch_1'), d)
def ReadTestDataToHMLPEDic(imgdir,
annot_dir,
tech_report_path,
save_path=None,
iou_thresh=0.5):
"""
This function will use detection window provided by the author
Also, erro detection window are not included
For image without detection, all the joint points will be zero
"""
import Stickmen
import scipy.io as sio
import PIL
import iutils as iu
from PIL import Image
import matplotlib.pyplot as plt
import iread.hmlpe as hp
import iread.myio as mio
import ipyml.geometry as igeo
rp = sio.loadmat(tech_report_path)['techrep2010_buffy']
d_annot = ExtractAnnotation(annot_dir)
ndata = rp.shape[1]
data = []
tsize = 112 # the size of test image
part_ind_dim = (8, 8)
joint_ind_dim = (8, 8)
filter_size = 32.0
stride = 12.0
indrate = 0.3
joint_filter_size = 32.0
joint_stride = 12.0
njoints = 8
nparts = 7
ndet = ndata # One test point for one image
newdim = (tsize, tsize, 3)
data = hp.HMLPE.prepare_savebuffer({'data':newdim,\
'part_indmap': part_ind_dim,\
'joint_indmap':joint_ind_dim },\
ndet, nparts, njoints)
# add Buffy specific field
add_buffy_specific_field(data, ndet, imgdir)
idx = 0
f_calc_area = lambda rec: (rec[1][1] - rec[0][1]) * (rec[1][0] - rec[0][
0]) if len(rec) == 2 else 0
f_mk_rec = lambda det: ((det[0], det[1]), (det[2], det[3]))
for i in range(ndata):
ep = rp[0, i][1][0][0]
fr = rp[0, i][2][0][0]
imgpath = iu.fullfile(imgdir, 'buffy_s5e'+str(ep)+'_original', \
('%06d' % fr) + '.jpg')
img = Image.open(imgpath)
data['ep'][..., i] = ep
data['fr'][..., i] = fr
data['filenames'][i] = imgpath
data['is_positive'][..., i] = True
data['annotation'][..., i] = Stickmen.ReorderStickmen(d_annot[ep][fr])
data['gt_det'][..., i] = np.asarray(
Stickmen.EstDetFromStickmen(data['annotation'][..., i]))
gt_rec = f_mk_rec(data['gt_det'][..., i]) #
gt_area = f_calc_area(gt_rec)
if rp[0, i][0].size == 0: # No detection are found in this image
# imgdata will also be all zero
# oridet will be all zero
# oribbox will be all zero
# joints8 will be all zero
# jointmasks wiil be all zero
# indmap will be all zero
# joint_indmap will be all zero
# nothing need to be done, since they are default value
pass
else:
m = -1
for j in range(rp[0, i][0].size):
det = rp[0, i][0][0, j]['det'][0] # det = (x,y,x1,y1)
cur_rec = f_mk_rec(det)
int_area = f_calc_area(igeo.RectIntersectRect(cur_rec, gt_rec))
if int_area > (gt_area - int_area) * iou_thresh:
m = j
break
if m != -1:
oribbox = ExtendBndbox(det, img.size)
arr_img = np.asarray(img)[oribbox[1]:oribbox[3]+1, \
oribbox[0]:oribbox[2]+1,:]
res_img = Image.fromarray(arr_img).resize((tsize, tsize))
data['data'][..., i] = np.asarray(res_img)
tmppoints = convert_AHEcoor_to_joints8(data['annotation'][...,
i])
data['joints8'][..., i] = TransformPoints(
tmppoints, oribbox,
np.asarray(res_img.size) - 1)
data['jointmasks'][..., i] = hp.HMLPE.makejointmask(
newdim, data['joints8'][..., i])
data['indmap'][..., i] = hp.HMLPE.create_part_indicatormap(
data['joints8'][..., i], hp.part_idx, part_ind_dim,
indrate, filter_size, stride)
data['joint_indmap'][...,
i] = hp.HMLPE.create_joint_indicatormap(
data['joints8'][...,
i], joint_ind_dim,
joint_filter_size, joint_stride)
data['oribbox'][..., i] = oribbox
data['data'] = data['data'].reshape((-1, ndet), order='F')
if save_path is not None:
mio.pickle(save_path, data)
return data
