def write_tf(): data = ut.load('../results/grasp-dset/v5/train.pk') tf_file = '../results/grasp-dset/v5/train.tf' assert not os.path.exists(tf_file) writer = tf.python_io.TFRecordWriter(tf_file) for d in ut.time_est(data): fbl = lambda x: tf.train.Feature(bytes_list=tf.train.BytesList(value= [x])) fl = lambda x: tf.train.Feature(float_list=tf.train.FloatList( value=map(float, x.flatten()))) il = lambda x: tf.train.Feature(int64_list=tf.train.Int64List(value=x)) feat = { 'gel0_pre': fbl(d['gel0_pre']), 'gel1_pre': fbl(d['gel1_pre']), 'gel0_post': fbl(d['gel0_post']), 'gel1_post': fbl(d['gel1_post']), 'im0_pre': fbl(d['im0_pre']), 'im0_post': fbl(d['im0_post']), 'im1_pre': fbl(d['im1_pre']), 'im1_post': fbl(d['im1_post']), 'depth0_pre': fl(d['depth0_pre']), 'depth0_post': fl(d['depth0_post']), 'end_effector': fl(d['end_effector']), 'initial_press_prob': fl(d['initial_press_prob']), 'is_gripping': il([d['is_gripping']]) } ex = tf.train.Example(features=tf.train.Features(feature=feat)) writer.write(ex.SerializeToString()) writer.close()
def test(path): train_dir = pj(path, 'training') check_path = tf.train.latest_checkpoint(train_dir) print 'Restoring from:', check_path net = NetClf(check_path, gpu) data = ut.load(pj(path, 'test.pk')) labels = [] probs = [] accs = [] for i in xrange(len(data)): ex = data[i] label = ex['is_gripping'] ex = {k: ig.uncompress(ex[k]) for k in im_names} prob = net.predict(**ex) print prob, label pred = int(prob >= 0.5) labels.append(label) probs.append(prob) accs.append(pred == label) labels = np.array(labels, 'bool') probs = np.array(probs, 'float32') accs = np.array(accs) print 'Accuracy:', np.mean(accs) print 'mAP:', sklearn.metrics.average_precision_score(labels, probs)
def train_press(pr): data = ut.load(pj(pr.dsdir, 'train.pk')) xs, ys = [], [] for ex in data: xs.append([ex['initial_press_prob']]) ys.append(ex['is_gripping']) xs = np.array(xs, 'float32') ys = np.array(ys, 'int64') clf = sklearn.svm.SVC(C=1., kernel='linear') clf.fit(xs, ys) ut.save(pj(pr.resdir, 'clf.pk'), clf)
def eval_all(run=True, use_gpu=False,test_on_train=False): print """\\begin{tabular}""" for pr in all_params(): if run: if use_gpu: grasp_net.test(pr, gpus[0], test_on_train = test_on_train) else: grasp_net.test(pr, None, test_on_train = test_on_train) out_file = pj(pr.resdir, 'eval_results.pk') if os.path.exists(out_file): r = ut.load(out_file) print '%s & %s%% & %s%% \\\\' % (pr.description, ut.f2(100*r['acc']), ut.f2(100*r['ap'])) print """\\end{tabular}"""
def train_clf(pr): data = ut.load(pj(pr.dsdir, 'train.pk')) xs, ys = [], [] for ex in data: ex = copy.copy(ex) for k, v in ex.items(): if k.startswith('gel'): ex[k] = ut.crop_center(ig.uncompress(v), 224) xs.append(example_feats(ex, pr)) ys.append(ex['is_gripping']) xs = np.array(xs, 'float32') ys = np.array(ys, 'int64') clf = sklearn.pipeline.Pipeline([ ('scale', sklearn.preprocessing.StandardScaler(with_mean=True, with_std=True)), ('svm', sklearn.svm.SVC(C=1., kernel='linear')) ]) clf.fit(xs, ys) ut.save(pj(pr.resdir, 'clf.pk'), clf)
def analyze(pr): eval_exs = ut.load(pj(pr.resdir, 'eval.pk')) # accuracy by object by_name = ut.accum_dict((ex.object_name, ex) for ex in eval_exs) accs, labels = [], [] for name in by_name: exs = by_name[name] accs.append(np.mean(ut.mapattr(exs).acc)) labels.append(np.mean(ut.mapattr(exs).label)) print name, ut.f4(accs[-1]), ut.f4(labels[-1]) print 'Object-averaged accuracy:', ut.f4(np.mean(accs)) print 'Object-averaged base:', ut.f4(np.mean(labels)) chosen = set() table = [] for ex in sorted(exs, key=lambda x: x.prob)[::-1]: if ex.object_name not in chosen: chosen.add(ex.object_name) print ex.object_name row = vis_example(ex.db_file) row = ['Prob:', ex.prob, 'Label:', ex.label] + row table.append(row) ig.show(table, rows_per_page=25)
def test(pr, gpu, test_on_train=False, center_crop=True): [gpu] = set_gpus([gpu]) if pr.inputs == ['press']: net = PressClf(pr) else: #check_path = tf.train.latest_checkpoint(pr.train_dir) check_path = pj(pr.train_dir, 'net.tf-%d' % pr.model_iter) print 'Restoring from:', check_path net = NetClf(pr, check_path, gpu) if test_on_train: print 'Testing on train!' data = ut.load(pj(pr.dsdir, 'train.pk')) else: data = ut.load(pj(pr.dsdir, 'test.pk')) labels, probs, accs, vals = [], [], [], [] for i in xrange(len(data)): ex = data[i] label = ex['is_gripping'] def load_im(k, v): if k.startswith('gel') or k.startswith('im'): im = ig.uncompress(v) elif k.startswith('depth'): #v = np.tile(v, (1, 1, 3)) im = v.astype('float32') else: raise RuntimeError() if center_crop: im = ut.crop_center(im, 224) return im inputs = {k: load_im(k, ex[k]) for k in im_names} inputs['initial_press_prob'] = ex['initial_press_prob'] inputs['ee'] = ex['end_effector'] pred, prob = net.predict(**inputs) #print prob, pred, label labels.append(label) probs.append(prob) accs.append(pred == label) if i % 50 == 0: print 'running average acc:', ut.f3(np.mean(accs)) vals.append( ut.Struct(label=label, prob=prob, acc=accs[-1], idx=i, db_file=ex['db_file'], object_name=ex['object_name'])) labels = np.array(labels, 'bool') probs = np.array(probs, 'float32') accs = np.array(accs) acc = np.mean(accs) ap = sklearn.metrics.average_precision_score(labels, probs) print 'Accuracy:', acc print 'mAP:', ap print 'Base rate:', ut.f3( np.array(ut.mapattr(vals).label).astype('float32').mean()) ut.save(pj(pr.resdir, 'eval_results.pk'), dict(acc=acc, ap=ap, results=(labels, probs))) ut.save(pj(pr.resdir, 'eval.pk'), vals) return acc
def __init__(self, pr): self.clf = ut.load(pj(pr.resdir, 'clf.pk'))
def test(pr, gpu, test_on_train=False, crop_type='center'): [gpu] = set_gpus([gpu]) if ut.hastrue(pr, 'use_clf'): net = SVMClf(pr) else: #check_path = tf.train.latest_checkpoint(pr.train_dir) check_path = pj(pr.train_dir, 'net.tf-%d' % pr.model_iter) print 'Restoring from:', check_path net = NetClf(pr, check_path, gpu) if test_on_train: print 'Testing on train!' data = ut.load(pj(pr.dsdir, 'train.pk')) else: data = ut.load(pj(pr.dsdir, 'test.pk')) labels, probs, accs, vals = [], [], [], [] for i in xrange(len(data)): ex = data[i] label = ex['is_gripping'] def load_im(k, v): if k.startswith('gel') or k.startswith('im'): im = ig.uncompress(v) elif k.startswith('depth'): #v = np.tile(v, (1, 1, 3)) im = v.astype('float32') else: raise RuntimeError() if crop_type == 'center': crops = [ut.crop_center(im, 224)] elif crop_type == 'multi': crops = [] dh = (im.shape[0] - crop_dim) num_dim_samples = 3 for y in np.linspace(0, dh, num_dim_samples).astype('l'): dw = (im.shape[1] - crop_dim) for x in np.linspace(0, dw, num_dim_samples).astype('l'): crops.append(im[y:y + crop_dim, x:x + crop_dim]) return ut.shuffled_with_seed(crops, k.split('_')[0] + str(i)) all_inputs = {k: load_im(k, ex[k]) for k in im_names} ps = [] for j in xrange(len(all_inputs['gel0_pre'])): inputs = {k: all_inputs[k][j] for k in im_names} inputs['initial_press_prob'] = ex['initial_press_prob'] inputs['ee'] = ex['end_effector'] _, prob = net.predict(**inputs) ps.append(prob) prob = np.mean(ps) pred = int(prob >= net.thresh) print prob, pred, label labels.append(label) probs.append(prob) accs.append(pred == label) print 'running average acc:', np.mean(accs) vals.append( ut.Struct(label=label, prob=prob, acc=accs[-1], idx=i, db_file=ex['db_file'], object_name=ex['object_name'])) labels = np.array(labels, 'bool') probs = np.array(probs, 'float32') accs = np.array(accs) acc = np.mean(accs) ap = sklearn.metrics.average_precision_score(labels, probs) print 'Accuracy:', acc print 'mAP:', ap print 'Base rate:', ut.f3( np.array(ut.mapattr(vals).label).astype('float32').mean()) ut.save(pj(pr.resdir, 'eval_results.pk'), dict(acc=acc, ap=ap, results=(labels, probs))) ut.save(pj(pr.resdir, 'eval.pk'), vals)
def __init__(self, pr): self.pr = pr self.clf = ut.load(pj(pr.resdir, 'clf.pk')) self.thresh = 0.