def train_press(pr):
data = ut.load(pj(pr.dsdir, 'train.pk'))
xs, ys = [], []
for ex in data:
xs.append([ex['initial_press_prob']])
ys.append(ex['is_gripping'])
xs = np.array(xs, 'float32')
ys = np.array(ys, 'int64')
clf = sklearn.svm.SVC(C=1., kernel='linear')
clf.fit(xs, ys)
ut.save(pj(pr.resdir, 'clf.pk'), clf)
def train_clf(pr):
data = ut.load(pj(pr.dsdir, 'train.pk'))
xs, ys = [], []
for ex in data:
ex = copy.copy(ex)
for k, v in ex.items():
if k.startswith('gel'):
ex[k] = ut.crop_center(ig.uncompress(v), 224)
xs.append(example_feats(ex, pr))
ys.append(ex['is_gripping'])
xs = np.array(xs, 'float32')
ys = np.array(ys, 'int64')
clf = sklearn.pipeline.Pipeline([
('scale',
sklearn.preprocessing.StandardScaler(with_mean=True, with_std=True)),
('svm', sklearn.svm.SVC(C=1., kernel='linear'))
])
clf.fit(xs, ys)
ut.save(pj(pr.resdir, 'clf.pk'), clf)
pr.samp_sr)), 'src:',
imtable.Video(full_ims, pr.fps, Sound(full_samples_src,
pr.samp_sr))]
if arg.out is not None:
ut.mkdir(arg.out)
vid_s = arg.vid_file.split('/')[-1].split('.mp4')[0]
mask_s = '' if arg.mask is None else '_%s' % arg.mask
cam_s = '' if not arg.cam else '_cam'
suffix_s = '' if arg.suffix == '' else '_%s' % arg.suffix
name = '%s%s%s_%s' % (suffix_s, mask_s, cam_s, vid_s)
def snd(x):
x = Sound(x, pr.samp_sr)
x.samples = np.clip(x.samples, -1., 1.)
return x
print 'Writing to:', arg.out
ut.save(pj(arg.out, 'ret%s.pk' % name), ret)
ut.make_video(full_ims, pr.fps, pj(arg.out, 'fg%s.mp4' % name),
snd(full_samples_fg))
ut.make_video(full_ims, pr.fps, pj(arg.out, 'bg%s.mp4' % name),
snd(full_samples_bg))
ut.make_video(full_ims, pr.fps, pj(arg.out, 'src%s.mp4' % name),
snd(full_samples_src))
else:
print 'Not writing, since --out was not set'
print 'Video results:'
ig.show(table)
def test(pr, gpu, test_on_train=False, center_crop=True):
[gpu] = set_gpus([gpu])
if pr.inputs == ['press']:
net = PressClf(pr)
else:
#check_path = tf.train.latest_checkpoint(pr.train_dir)
check_path = pj(pr.train_dir, 'net.tf-%d' % pr.model_iter)
print 'Restoring from:', check_path
net = NetClf(pr, check_path, gpu)
if test_on_train:
print 'Testing on train!'
data = ut.load(pj(pr.dsdir, 'train.pk'))
else:
data = ut.load(pj(pr.dsdir, 'test.pk'))
labels, probs, accs, vals = [], [], [], []
for i in xrange(len(data)):
ex = data[i]
label = ex['is_gripping']
def load_im(k, v):
if k.startswith('gel') or k.startswith('im'):
im = ig.uncompress(v)
elif k.startswith('depth'):
#v = np.tile(v, (1, 1, 3))
im = v.astype('float32')
else:
raise RuntimeError()
if center_crop:
im = ut.crop_center(im, 224)
return im
inputs = {k: load_im(k, ex[k]) for k in im_names}
inputs['initial_press_prob'] = ex['initial_press_prob']
inputs['ee'] = ex['end_effector']
pred, prob = net.predict(**inputs)
#print prob, pred, label
labels.append(label)
probs.append(prob)
accs.append(pred == label)
if i % 50 == 0:
print 'running average acc:', ut.f3(np.mean(accs))
vals.append(
ut.Struct(label=label,
prob=prob,
acc=accs[-1],
idx=i,
db_file=ex['db_file'],
object_name=ex['object_name']))
labels = np.array(labels, 'bool')
probs = np.array(probs, 'float32')
accs = np.array(accs)
acc = np.mean(accs)
ap = sklearn.metrics.average_precision_score(labels, probs)
print 'Accuracy:', acc
print 'mAP:', ap
print 'Base rate:', ut.f3(
np.array(ut.mapattr(vals).label).astype('float32').mean())
ut.save(pj(pr.resdir, 'eval_results.pk'),
dict(acc=acc, ap=ap, results=(labels, probs)))
ut.save(pj(pr.resdir, 'eval.pk'), vals)
return acc
def write_data(out_dir,
rebalance_data=True,
train_frac=0.75,
val_frac=0.0,
n=None,
seed=0):
#def write_data(out_dir, rebalance_data = True, train_frac = 0.75, val_frac = 0.0, n = 10):
assert not os.path.exists(out_dir)
ut.mkdir(out_dir)
base_data = '../data/grasp/'
ut.sys_check('find -L %s -name "*.hdf5" > %s/all_db_files.txt' %
(base_data, out_dir))
all_db_files = map(os.path.abspath,
ut.read_lines(pj(out_dir, 'all_db_files.txt'))[:n])
all_db_files = ut.shuffled_with_seed(all_db_files, seed)
all_db_files = filter(db_ok, all_db_files)
ut.write_lines(pj(out_dir, 'db_files.txt'), all_db_files)
by_name = ut.accum_dict((name_from_file(x), x) for x in all_db_files)
names = ut.shuffled_with_seed(sorted(by_name.keys()), seed)
num_names = len(names)
num_train = int(train_frac * num_names)
num_val = int(val_frac * num_names)
i = 0
train_names = names[i:num_train]
i += num_train
val_names = names[i:i + num_val]
i += num_val
test_names = names[i:]
print num_train, num_val, len(test_names)
splits = [('train', train_names), ('val', val_names), ('test', test_names)]
print 'Number of objects in each split:'
for s, o in splits:
print s, '->', len(o)
#press_clf = press.NetClf(press_model_file, gpu = write_data_gpu)
press_clf = None #press.NetClf(press_model_file, gpu = write_data_gpu)
for dset_name, names in splits:
ut.write_lines(pj(out_dir, '%s_objects.txt' % dset_name), names)
tf_file = pj(out_dir, '%s.tf' % dset_name)
pk_file = pj(out_dir, '%s.pk' % dset_name)
full_pk_file = pj(out_dir, 'full_%s.pk' % dset_name)
if os.path.exists(tf_file):
os.remove(tf_file)
writer = tf.python_io.TFRecordWriter(tf_file)
split_db_files = ut.flatten(by_name[name] for name in names)
split_db_files = ut.shuffled_with_seed(split_db_files, dset_name)
data = []
for db_file in ut.time_est(split_db_files):
with h5py.File(db_file, 'r') as db:
#print 'keys =', db.keys()
def im(x, crop=False, compress=True):
x = ig.uncompress(x)
x = np.array(x)
if crop:
x = crop_kinect(x)
#ig.show(x)
x = ig.scale(x, (256, 256), 1)
if compress:
x = ig.compress(x)
return x
def depth(x):
x = np.array(x).astype('float32')
x = ig.scale(x, (256, 256), 1)
return x
def parse_ee(x):
names = [
'angle_of_EE_at_grasping', 'location_of_EE_at_grasping'
]
vs = [x[name].value for name in names]
ee = np.concatenate([np.array(v).flatten()
for v in vs]).astype('float32')
return ee
label_file = pj(
label_path,
db_file.split('/')[-1].replace('.hdf5', '.txt'))
if os.path.exists(label_file):
print 'Reading label from file'
is_gripping = bool(ut.read_file(label_file))
else:
is_gripping = int(np.array(db['is_gripping']))
pre, mid, _ = milestone_frames(db)
# Estimate the probability that the robot is initially gripping the object
if 0:
press_a = press_clf.predict(
im(db['/GelSightA_image'].value[mid], compress=False),
im(db['/GelSightA_image'].value[pre], compress=False))
press_b = press_clf.predict(
im(db['/GelSightB_image'].value[mid], compress=False),
im(db['/GelSightB_image'].value[pre], compress=False))
initial_press_prob = 0.5 * (press_a + press_b)
else:
initial_press_prob = np.float32(-1.)
#print initial_press_prob, ig.show(im(db['/GelSightA_image'].value[mid], compress = False))
d = dict(
gel0_pre=im(db['/GelSightA_image'].value[pre]),
gel1_pre=im(db['/GelSightB_image'].value[pre]),
gel0_post=im(db['/GelSightA_image'].value[mid]),
gel1_post=im(db['/GelSightB_image'].value[mid]),
im0_pre=im(db['/color_image_KinectA'].value[pre],
crop=True),
im0_post=im(db['/color_image_KinectA'].value[mid],
crop=True),
im1_pre=im(db['/color_image_KinectB'].value[pre],
crop=True),
im1_post=im(db['/color_image_KinectB'].value[mid],
crop=True),
depth0_pre=depth(
crop_kinect(db['/depth_image_KinectA'].value[pre])),
depth0_post=depth(
crop_kinect(db['/depth_image_KinectA'].value[mid])),
initial_press_prob=initial_press_prob,
is_gripping=int(is_gripping),
end_effector=parse_ee(db),
object_name=str(np.array(db['object_name'].value)[0]),
db_file=db_file)
data.append(d)
# for db files
ut.save(full_pk_file, data)
# rebalance data?
if rebalance_data:
by_label = [[], []]
for x in ut.shuffled_with_seed(data, 'rebalance1'):
by_label[x['is_gripping']].append(x)
n = min(map(len, by_label))
print len(data), 'before rebalance'
data = ut.shuffled_with_seed(by_label[0][:n] + by_label[1][:n],
'rebalance2')
print len(data), 'after rebalance'
writer = tf.python_io.TFRecordWriter(tf_file)
for d in data:
fbl = lambda x: tf.train.Feature(bytes_list=tf.train.BytesList(
value=[x]))
fl = lambda x: tf.train.Feature(float_list=tf.train.FloatList(
value=map(float, x.flatten())))
il = lambda x: tf.train.Feature(int64_list=tf.train.Int64List(value
=x))
feat = {
'gel0_pre': fbl(d['gel0_pre']),
'gel1_pre': fbl(d['gel1_pre']),
'gel0_post': fbl(d['gel0_post']),
'gel1_post': fbl(d['gel1_post']),
'im0_pre': fbl(d['im0_pre']),
'im0_post': fbl(d['im0_post']),
'im1_pre': fbl(d['im1_pre']),
'im1_post': fbl(d['im1_post']),
'depth0_pre': fl(d['depth0_pre']),
'depth0_post': fl(d['depth0_post']),
'end_effector': fl(d['end_effector']),
'initial_press_prob': fl(d['initial_press_prob']),
'is_gripping': il([d['is_gripping']])
}
ex = tf.train.Example(features=tf.train.Features(feature=feat))
writer.write(ex.SerializeToString())
writer.close()
ut.save(pk_file, data)
print dset_name, '->', len(data), 'examples'
def write_data(out_dir, train_frac=0.75, val_frac=0.05):
ut.mkdir(out_dir)
base_data = '../data/grasp/'
ut.sys_check('find %s -name "*.hdf5" > %s/db_files.txt' %
(base_data, out_dir))
all_db_files = ut.read_lines(pj(out_dir, 'db_files.txt'))
all_db_files = ut.shuffled_with_seed(all_db_files)
name_from_file = lambda x: '_'.join(x.split('/')[-1].split('_')[2:])
by_name = ut.accum_dict((name_from_file(x), x) for x in all_db_files)
names = ut.shuffled_with_seed(sorted(by_name.keys()))
num_names = len(all_db_files)
num_train = int(train_frac * num_names)
num_val = int(val_frac * num_names)
i = 0
train_names = names[i:num_train]
i += num_train
val_names = names[i:i + num_val]
i += num_val
test_names = names[i:]
for dset_name, names in [('train', train_names), ('val', val_names),
('test', test_names)]:
ut.write_lines(pj(out_dir, '%s_objects.txt' % dset_name), names)
tf_file = pj(out_dir, '%s.tf' % dset_name)
pk_file = pj(out_dir, '%s.pk' % dset_name)
if os.path.exists(tf_file):
os.remove(tf_file)
writer = tf.python_io.TFRecordWriter(tf_file)
data = []
for name in names:
for db_file in by_name[name]:
with h5py.File(db_file, 'r') as db:
def im(x):
x = np.array(x)
x = ig.scale(x, (256, 256), 1)
return ig.compress(x)
if 'is_gripping' in db:
label = int(np.array(db['is_gripping']))
elif 'Is gripping?' in db:
label = int(np.array(db['Is gripping?']))
else:
print 'Skipping: %s. Missing is_gripping' % db_file
print 'Keys:', ' '.join(db.keys())
continue
data.append({
'gel0_pre':
im(db['GelSightA_image_pre_gripping']),
'gel1_pre':
im(db['GelSightB_image_pre_gripping']),
'gel0_post':
im(db['GelSightA_image_post_gripping']),
'gel1_post':
im(db['GelSightB_image_post_gripping']),
'is_gripping':
label
})
fbl = lambda x: tf.train.Feature(bytes_list=tf.train.
BytesList(value=[x]))
feat = {
'gel0_pre':
fbl(im(db['GelSightA_image_pre_gripping'])),
'gel1_pre':
fbl(im(db['GelSightB_image_pre_gripping'])),
'gel0_post':
fbl(im(db['GelSightA_image_post_gripping'])),
'gel1_post':
fbl(im(db['GelSightB_image_post_gripping'])),
'is_gripping':
tf.train.Feature(int64_list=tf.train.Int64List(
value=[label]))
}
ex = tf.train.Example(features=tf.train.Features(
feature=feat))
writer.write(ex.SerializeToString())
writer.close()
ut.save(pk_file, data)
print dset_name, '->', len(data), 'examples'
def test(pr, gpu, test_on_train=False, crop_type='center'):
[gpu] = set_gpus([gpu])
if ut.hastrue(pr, 'use_clf'):
net = SVMClf(pr)
else:
#check_path = tf.train.latest_checkpoint(pr.train_dir)
check_path = pj(pr.train_dir, 'net.tf-%d' % pr.model_iter)
print 'Restoring from:', check_path
net = NetClf(pr, check_path, gpu)
if test_on_train:
print 'Testing on train!'
data = ut.load(pj(pr.dsdir, 'train.pk'))
else:
data = ut.load(pj(pr.dsdir, 'test.pk'))
labels, probs, accs, vals = [], [], [], []
for i in xrange(len(data)):
ex = data[i]
label = ex['is_gripping']
def load_im(k, v):
if k.startswith('gel') or k.startswith('im'):
im = ig.uncompress(v)
elif k.startswith('depth'):
#v = np.tile(v, (1, 1, 3))
im = v.astype('float32')
else:
raise RuntimeError()
if crop_type == 'center':
crops = [ut.crop_center(im, 224)]
elif crop_type == 'multi':
crops = []
dh = (im.shape[0] - crop_dim)
num_dim_samples = 3
for y in np.linspace(0, dh, num_dim_samples).astype('l'):
dw = (im.shape[1] - crop_dim)
for x in np.linspace(0, dw, num_dim_samples).astype('l'):
crops.append(im[y:y + crop_dim, x:x + crop_dim])
return ut.shuffled_with_seed(crops, k.split('_')[0] + str(i))
all_inputs = {k: load_im(k, ex[k]) for k in im_names}
ps = []
for j in xrange(len(all_inputs['gel0_pre'])):
inputs = {k: all_inputs[k][j] for k in im_names}
inputs['initial_press_prob'] = ex['initial_press_prob']
inputs['ee'] = ex['end_effector']
_, prob = net.predict(**inputs)
ps.append(prob)
prob = np.mean(ps)
pred = int(prob >= net.thresh)
print prob, pred, label
labels.append(label)
probs.append(prob)
accs.append(pred == label)
print 'running average acc:', np.mean(accs)
vals.append(
ut.Struct(label=label,
prob=prob,
acc=accs[-1],
idx=i,
db_file=ex['db_file'],
object_name=ex['object_name']))
labels = np.array(labels, 'bool')
probs = np.array(probs, 'float32')
accs = np.array(accs)
acc = np.mean(accs)
ap = sklearn.metrics.average_precision_score(labels, probs)
print 'Accuracy:', acc
print 'mAP:', ap
print 'Base rate:', ut.f3(
np.array(ut.mapattr(vals).label).astype('float32').mean())
ut.save(pj(pr.resdir, 'eval_results.pk'),
dict(acc=acc, ap=ap, results=(labels, probs)))
ut.save(pj(pr.resdir, 'eval.pk'), vals)