def show_db(pr, num_sample=None, num_per_object=5, n=None):
db_files = ut.read_lines(pj(pr.dsdir, 'db_files.txt'))[:n]
db_files = ut.shuffled_with_seed(db_files)
counts = {}
#db_files = ut.parfilter(db_ok, db_files)
names = ut.parmap(name_from_file, db_files)
table = []
for name, db_file in zip(names, db_files[:num_sample]):
if counts.get(name, 0) < num_per_object:
counts[name] = 1 + counts.get(name, 0)
row = vis_example(db_file)
table.append(row)
ig.show(table)
def test(path, match_str=None):
train_dir = pj(path, 'training')
check_path = tf.train.latest_checkpoint(train_dir)
print 'Restoring from:', check_path
net = NetClf(check_path, gpu)
examples = []
for line in ut.read_lines(pj(path, 'test.csv')):
s = line.split(',')
#print match_str, s[3]
print s
if (match_str is not None) and (match_str not in s[3]):
print 'skipping'
continue
examples.append((s[0], s[1], int(s[2]), s[3]))
print 'Testing on:', len(examples), 'examples'
labels = []
probs = []
accs = []
table = []
for i, ex in enumerate(examples):
im_after = ig.load(ex[0])
im_prev = ig.load(ex[1])
label = ex[2]
prob = net.predict(im_after, im_prev)
#print prob, label
pred = int(prob >= 0.5)
labels.append(label)
probs.append(prob)
accs.append(pred == label)
if i < 50:
color = '#00DD00' if pred == label else '#DD0000'
row = [
im_after, im_prev,
ut.font_color_html('pred = %.3f' % prob, color),
'gt = %d' % label
]
table.append(row)
labels = np.array(labels, 'bool')
probs = np.array(probs, 'float32')
accs = np.array(accs)
print 'Accuracy:', np.mean(accs)
print 'mAP:', sklearn.metrics.average_precision_score(labels, probs)
ig.show(table)
def analyze(pr):
eval_exs = ut.load(pj(pr.resdir, 'eval.pk'))
# accuracy by object
by_name = ut.accum_dict((ex.object_name, ex) for ex in eval_exs)
accs, labels = [], []
for name in by_name:
exs = by_name[name]
accs.append(np.mean(ut.mapattr(exs).acc))
labels.append(np.mean(ut.mapattr(exs).label))
print name, ut.f4(accs[-1]), ut.f4(labels[-1])
print 'Object-averaged accuracy:', ut.f4(np.mean(accs))
print 'Object-averaged base:', ut.f4(np.mean(labels))
chosen = set()
table = []
for ex in sorted(exs, key=lambda x: x.prob)[::-1]:
if ex.object_name not in chosen:
chosen.add(ex.object_name)
print ex.object_name
row = vis_example(ex.db_file)
row = ['Prob:', ex.prob, 'Label:', ex.label] + row
table.append(row)
ig.show(table, rows_per_page=25)
pr.samp_sr)), 'src:',
imtable.Video(full_ims, pr.fps, Sound(full_samples_src,
pr.samp_sr))]
if arg.out is not None:
ut.mkdir(arg.out)
vid_s = arg.vid_file.split('/')[-1].split('.mp4')[0]
mask_s = '' if arg.mask is None else '_%s' % arg.mask
cam_s = '' if not arg.cam else '_cam'
suffix_s = '' if arg.suffix == '' else '_%s' % arg.suffix
name = '%s%s%s_%s' % (suffix_s, mask_s, cam_s, vid_s)
def snd(x):
x = Sound(x, pr.samp_sr)
x.samples = np.clip(x.samples, -1., 1.)
return x
print 'Writing to:', arg.out
ut.save(pj(arg.out, 'ret%s.pk' % name), ret)
ut.make_video(full_ims, pr.fps, pj(arg.out, 'fg%s.mp4' % name),
snd(full_samples_fg))
ut.make_video(full_ims, pr.fps, pj(arg.out, 'bg%s.mp4' % name),
snd(full_samples_bg))
ut.make_video(full_ims, pr.fps, pj(arg.out, 'src%s.mp4' % name),
snd(full_samples_src))
else:
print 'Not writing, since --out was not set'
print 'Video results:'
ig.show(table)
def show_results(ims,
samples_mix,
samples_gt,
spec_mix,
spec_gt,
spec_pred_fg,
spec_pred0,
samples_pred,
samples_pred0,
samples_gt_auto,
samples_mix_auto,
ytids,
pr=None,
table=[],
min_before_show=1,
n=None):
def make_vid(ims, samples):
samples = np.clip(samples, -1, 1).astype('float64')
snd = sound.Sound(samples, pr.samp_sr)
return imtable.Video(ims, pr.fps, snd)
def vis_spec(spec):
return ut.jet(spec.T, pr.spec_min, pr.spec_max * 0.75)
for i in range(spec_mix.shape[0])[:n]:
row = [
'mix:',
make_vid(ims[i], samples_mix[i]), 'pred:',
make_vid(ims[i], samples_pred[i])
]
# if pr.use_decoder:
# row += ['pred (before):', make_vid(ims[i], samples_pred0[i])]
row += [
'gt:',
make_vid(ims[i], samples_gt[i]), 'gt autoencoded:',
make_vid(ims[i], samples_gt_auto[i]), 'mix autoencoded:',
make_vid(ims[i], samples_mix_auto[i]),
ut.link('https://youtube.com/watch?v=%s' % ytids[i], ytids[i])
]
table.append(row)
row = [
'mix:',
vis_spec(spec_mix[i]), 'pred:',
vis_spec(spec_pred_fg[i])
]
# if pr.use_decoder:
# row += ['pred (before):', make_vid(ims[i], samples_pred0[i])]
row += [
'gt:',
vis_spec(spec_gt[i]), 'gt autoencoded',
vis_spec(spec_gt[i]), 'mix autoencoded',
vis_spec(spec_mix[i]), ''
]
table.append(row)
if len(table) >= min_before_show * 2:
ig.show(table)
table[:] = []
return np.array([1], np.int64)
def fit_cylinder(depth0,
depth,
plane_thresh=0.02,
new_thresh=0.02,
inlier_frac=0.9925,
show=True):
# compute occupancy map for the first frame (should precompute when using many frames...)
plane = fit_ground_plane(depth0)
ptm0 = ptm_from_depth(depth0)
pts0 = pts_from_ptm(ptm0)
on_plane0 = planefit.dist_from_plane_ptm(ptm0, plane) < plane_thresh
on_plane0 = in_largest_cc(on_plane0)
proj_plane0 = planefit.project_onto_plane(plane, ok_pts(ptm0[on_plane0]))
ptm = ptm_from_depth(depth)
has_pt = ptm[:, :, 3] != 0
# find points that are very close to the table's surface
pts, inds = pts_from_ptm(ptm, inds=True)
proj_plane = planefit.project_onto_plane(plane, pts)
ok = ut.knnsearch(proj_plane0, proj_plane,
method='kd')[0].flatten() <= 0.005
near_surface = np.zeros(ptm.shape[:2], 'bool')
near_surface[inds[0][ok], inds[1][ok]] = True
# find points that are not in the original point cloud
dist = ut.knnsearch(pts0, pts, method='kd')[0].flatten()
new_pt = np.zeros_like(near_surface)
ok = dist > new_thresh
new_pt[inds[0][ok], inds[1][ok]] = True
# todo: probably want to filter out the robot's gripper, e.g. with a height check
occ_before_cc = new_pt & near_surface & has_pt
occ = in_largest_cc(occ_before_cc)
# segment object and find height
pts = ptm[occ]
pts = pts[pts[:, 3] > 0, :3]
height = np.percentile(np.abs(planefit.dist_to_plane(plane, pts)), 99.7)
print 'Height: %.3fm' % height
# find an ellipse that covers the occupied points
# todo: want to deal with self-occlusion here (kinect won't see the backside of the object)
pts = ok_pts(ptm[occ])
proj = planefit.project_onto_plane(plane, pts)
# add a slight bias toward choosing farther z points, since these will often be self-occluded
#center = np.array(list(np.median(proj[:, :2], axis = 0)) + [np.percentile(proj[:, 2], 70.)])
center = np.array(
list(np.median(proj[:, :2], axis=0)) +
[np.percentile(proj[:, 2], 75.)])
d = np.sqrt(np.percentile(np.sum((proj - center)**2, 1), 95.)) + 0.01
scales = np.array([d, d])
# show ellipse
if show:
# show points in/out of cylinder
#nrm_vis = color_normals(ptm)
ptm_vis = ut.clip_rescale_im(ptm[:, :, 2], 0., 2.)
oval_vis1 = ptm_vis.copy()
oval_missing_vis = ptm_vis.copy()
for y in xrange(ptm.shape[0]):
for x in xrange(ptm.shape[1]):
if ptm[y, x, 3]:
pt = ptm[y, x, :3]
proj = (planefit.project_onto_plane(plane, pt[None]) -
center)[0]
ok = ((proj[:2] / scales[:2])**2).sum() <= 1.
ok = ok and planefit.dist_to_plane(
plane, pt[None], signed=True)[0] <= height
if ok and occ[y, x]:
oval_vis1[y, x] = 255
elif occ[y, x]:
# not covered
oval_missing_vis[y, x] = 255
# show cylinder
axes = parallel_axes(plane)
pts_lo, pts_hi = [], []
for theta in np.linspace(0., 2 * np.pi, 100):
x = np.array([np.cos(theta), np.sin(theta)])
pt = axes.T.dot(x * scales[:2]) + center
pts_lo.append(pt.flatten())
pts_hi.append(pt + height * plane[:3])
print plane[:3]
pts_lo, pts_hi = map(np.array, [pts_lo, pts_hi])
proj_lo = ut.inhomog(camera_matrix().dot(pts_lo.T)).T
proj_hi = ut.inhomog(camera_matrix().dot(pts_hi.T)).T
oval_vis = ptm_vis.copy()
c1 = (128, 0, 0)
c2 = (0, 0, 128)
c3 = (0, 128, 0)
oval_vis = ig.draw_lines(oval_vis,
proj_lo[:-1],
proj_lo[1:],
colors=c1,
width=2)
oval_vis = ig.draw_lines(oval_vis,
proj_hi[:-1],
proj_hi[1:],
colors=c2,
width=2)
oval_vis = ig.draw_lines(oval_vis, [proj_hi[0]], [proj_lo[0]],
colors=c3,
width=2)
oval_vis = ig.draw_lines(oval_vis, [proj_hi[len(proj_hi) / 2]],
[proj_lo[len(proj_hi) / 2]],
colors=c3,
width=2)
def make_vis(x):
v = ptm_vis.copy()
v[x] = 255
return np.flipud(v)
ig.show([
'parametric ellipse:',
np.flipud(oval_vis),
'ellipse:',
np.flipud(oval_vis1),
'missing:',
np.flipud(oval_missing_vis),
'occ:',
make_vis(occ),
'occ_before_cc:',
make_vis(occ_before_cc),
'near_surface',
make_vis(near_surface),
'new_pt',
make_vis(new_pt),
'has_pt',
make_vis(has_pt),
'on_plane0',
make_vis(on_plane0),
'input:',
np.flipud(ptm_vis),
])
ut.toplevel_locals()
