def analyze(pr):
eval_exs = ut.load(pj(pr.resdir, 'eval.pk'))
# accuracy by object
by_name = ut.accum_dict((ex.object_name, ex) for ex in eval_exs)
accs, labels = [], []
for name in by_name:
exs = by_name[name]
accs.append(np.mean(ut.mapattr(exs).acc))
labels.append(np.mean(ut.mapattr(exs).label))
print name, ut.f4(accs[-1]), ut.f4(labels[-1])
print 'Object-averaged accuracy:', ut.f4(np.mean(accs))
print 'Object-averaged base:', ut.f4(np.mean(labels))
chosen = set()
table = []
for ex in sorted(exs, key=lambda x: x.prob)[::-1]:
if ex.object_name not in chosen:
chosen.add(ex.object_name)
print ex.object_name
row = vis_example(ex.db_file)
row = ['Prob:', ex.prob, 'Label:', ex.label] + row
table.append(row)
ig.show(table, rows_per_page=25)
def test(pr, gpu, test_on_train=False, center_crop=True):
[gpu] = set_gpus([gpu])
if pr.inputs == ['press']:
net = PressClf(pr)
else:
#check_path = tf.train.latest_checkpoint(pr.train_dir)
check_path = pj(pr.train_dir, 'net.tf-%d' % pr.model_iter)
print 'Restoring from:', check_path
net = NetClf(pr, check_path, gpu)
if test_on_train:
print 'Testing on train!'
data = ut.load(pj(pr.dsdir, 'train.pk'))
else:
data = ut.load(pj(pr.dsdir, 'test.pk'))
labels, probs, accs, vals = [], [], [], []
for i in xrange(len(data)):
ex = data[i]
label = ex['is_gripping']
def load_im(k, v):
if k.startswith('gel') or k.startswith('im'):
im = ig.uncompress(v)
elif k.startswith('depth'):
#v = np.tile(v, (1, 1, 3))
im = v.astype('float32')
else:
raise RuntimeError()
if center_crop:
im = ut.crop_center(im, 224)
return im
inputs = {k: load_im(k, ex[k]) for k in im_names}
inputs['initial_press_prob'] = ex['initial_press_prob']
inputs['ee'] = ex['end_effector']
pred, prob = net.predict(**inputs)
#print prob, pred, label
labels.append(label)
probs.append(prob)
accs.append(pred == label)
if i % 50 == 0:
print 'running average acc:', ut.f3(np.mean(accs))
vals.append(
ut.Struct(label=label,
prob=prob,
acc=accs[-1],
idx=i,
db_file=ex['db_file'],
object_name=ex['object_name']))
labels = np.array(labels, 'bool')
probs = np.array(probs, 'float32')
accs = np.array(accs)
acc = np.mean(accs)
ap = sklearn.metrics.average_precision_score(labels, probs)
print 'Accuracy:', acc
print 'mAP:', ap
print 'Base rate:', ut.f3(
np.array(ut.mapattr(vals).label).astype('float32').mean())
ut.save(pj(pr.resdir, 'eval_results.pk'),
dict(acc=acc, ap=ap, results=(labels, probs)))
ut.save(pj(pr.resdir, 'eval.pk'), vals)
return acc
def test(pr, gpu, test_on_train=False, crop_type='center'):
[gpu] = set_gpus([gpu])
if ut.hastrue(pr, 'use_clf'):
net = SVMClf(pr)
else:
#check_path = tf.train.latest_checkpoint(pr.train_dir)
check_path = pj(pr.train_dir, 'net.tf-%d' % pr.model_iter)
print 'Restoring from:', check_path
net = NetClf(pr, check_path, gpu)
if test_on_train:
print 'Testing on train!'
data = ut.load(pj(pr.dsdir, 'train.pk'))
else:
data = ut.load(pj(pr.dsdir, 'test.pk'))
labels, probs, accs, vals = [], [], [], []
for i in xrange(len(data)):
ex = data[i]
label = ex['is_gripping']
def load_im(k, v):
if k.startswith('gel') or k.startswith('im'):
im = ig.uncompress(v)
elif k.startswith('depth'):
#v = np.tile(v, (1, 1, 3))
im = v.astype('float32')
else:
raise RuntimeError()
if crop_type == 'center':
crops = [ut.crop_center(im, 224)]
elif crop_type == 'multi':
crops = []
dh = (im.shape[0] - crop_dim)
num_dim_samples = 3
for y in np.linspace(0, dh, num_dim_samples).astype('l'):
dw = (im.shape[1] - crop_dim)
for x in np.linspace(0, dw, num_dim_samples).astype('l'):
crops.append(im[y:y + crop_dim, x:x + crop_dim])
return ut.shuffled_with_seed(crops, k.split('_')[0] + str(i))
all_inputs = {k: load_im(k, ex[k]) for k in im_names}
ps = []
for j in xrange(len(all_inputs['gel0_pre'])):
inputs = {k: all_inputs[k][j] for k in im_names}
inputs['initial_press_prob'] = ex['initial_press_prob']
inputs['ee'] = ex['end_effector']
_, prob = net.predict(**inputs)
ps.append(prob)
prob = np.mean(ps)
pred = int(prob >= net.thresh)
print prob, pred, label
labels.append(label)
probs.append(prob)
accs.append(pred == label)
print 'running average acc:', np.mean(accs)
vals.append(
ut.Struct(label=label,
prob=prob,
acc=accs[-1],
idx=i,
db_file=ex['db_file'],
object_name=ex['object_name']))
labels = np.array(labels, 'bool')
probs = np.array(probs, 'float32')
accs = np.array(accs)
acc = np.mean(accs)
ap = sklearn.metrics.average_precision_score(labels, probs)
print 'Accuracy:', acc
print 'mAP:', ap
print 'Base rate:', ut.f3(
np.array(ut.mapattr(vals).label).astype('float32').mean())
ut.save(pj(pr.resdir, 'eval_results.pk'),
dict(acc=acc, ap=ap, results=(labels, probs)))
ut.save(pj(pr.resdir, 'eval.pk'), vals)
