def record_template(bc, template=None):
# records the process of placing blocks of the template
if template is None:
template = cv2.imread("/home/annal/Izzy/vision_amt/scripts/objects/template.png")
template[:,:,1] = template[:,:,2]
template[:,:,0] = np.zeros((420, 420))
# template[:,:,2] = np.zeros((420, 420))
# template = cv2.resize(template, (250, 250))
frames = []
while 1:
frame = bc.read_frame()
frame = inputdata.im2tensor(frame, channels = 3)
final = np.abs(-frame + template/255.0)
frames.append(final * 255)
cv2.imshow('camera', final)
a = cv2.waitKey(30)
if a == 27:
cv2.destroyAllWindows()
break
elif a == ord(' '):
return 'next'
time.sleep(.1)
rollout_path = AMTOptions.supervised_dir+ 'supervised_template/'
os.makedirs(rollout_path)
i = 0
for frame in frames:
name = 'template_' + str(i) + '.jpg'
cv2.imwrite(rollout_path + name, frame)
i += 1
print "saved to: ", rollout_path
def generate_rollout_inputs(net,
model_path,
conv_layer,
rollout,
preview=False):
np.set_printoptions(threshold=np.nan)
sess = net.load(var_path=model_path)
with sess.as_default():
for j in range(0, 100):
test_image_path = opt.data_dir + 'amt/colors/rollout' + str(
rollout) + '_frame_' + str(j) + '.jpg'
test_image = cv2.imread(test_image_path)
im = inputdata.im2tensor(test_image, channels=3)
shape = im.shape
im = np.reshape(im, (-1, shape[0], shape[1], shape[2]))
filter = sess.run(conv_layer, feed_dict={net.x: im})
for i in range(filter.shape[-1]):
filter_image = filter[0, :, :, i] * 255.0
filter_path = net.name + '_frame' + str(j) + '_input' + str(
i) + '.jpg'
cv2.imwrite(
'filters/nrollout' + str(rollout) + '/' + filter_path,
np.array(filter_image))
cv2.imwrite(
'filters/nrollout' + str(rollout) + '/' + net.name + '_frame' +
str(j) + '.jpg', test_image)
sess.close()
def vectorize_image(image): img = cv2.resize(cv2.imread(image), (250,250)) im = inputdata.im2tensor(img, channels = 3) imshape = im.shape im = np.reshape(im, (-1, imshape[0], imshape[1], imshape[2])) return im
def run_samples(n):
net = net6_c.NetSix_C()
path = '/media/1tb/Izzy/nets/net6_05-11-2016_12h09m12s.ckpt'
sess = net.load(path)
bc = BinaryCamera('./meta.txt')
bc.open()
for i in range(4):
bc.read_frame()
num_samples = n
samples = []
images = []
for i in range(num_samples):
print i
try:
while True:
frame = bc.read_frame()
frame = inputdata.im2tensor(frame, channels=3)
cv2.imshow('preview', frame)
cv2.waitKey(30)
except:
pass
frame = bc.read_frame()
frame = similarity.color(frame)
images.append(frame)
print "Saved sample"
np.save('samples_images.npy', images)
def generate_inputs(net, model_path, conv_layer, test_image, preview=False):
sess = net.load(var_path=model_path)
im = inputdata.im2tensor(test_image, channels=3)
shape = im.shape
im = np.reshape(im, (-1, shape[0], shape[1], shape[2]))
with sess.as_default():
filter = sess.run(conv_layer, feed_dict={net.x: im})
for i in range(filter.shape[-1]):
filter_image = filter[0, :, :, i] * 255.0
filter_path = net.name + '_input' + str(i) + '.jpg'
cv2.imwrite('filters/' + filter_path, np.array(filter_image))
cv2.imwrite('filters/' + net.name + '_sample.jpg', test_image)
sess.close()
def generate_inputs(net, model_path, conv_layer, test_image, preview=False):
sess = net.load(var_path=model_path)
im = inputdata.im2tensor(test_image, channels=3)
shape = im.shape
im = np.reshape(im, (-1, shape[0], shape[1], shape[2]))
with sess.as_default():
filter = sess.run(conv_layer, feed_dict={net.x:im})
for i in range(filter.shape[-1]):
filter_image = filter[0,:,:,i] * 255.0
filter_path = net.name + '_input' + str(i) + '.jpg'
cv2.imwrite('filters/' + filter_path, np.array(filter_image))
cv2.imwrite('filters/' + net.name +'_sample.jpg', test_image)
sess.close()
def display_template(self, template=None):
if template is None:
template = cv2.imread("/home/annal/Izzy/vision_amt/scripts/objects/template.png")
template[:,:,1] = template[:,:,2]
template[:,:,0] = np.zeros((420, 420))
# template[:,:,2] = np.zeros((420, 420))
# template = cv2.resize(template, (250, 250))
while 1:
frame = self.bc.read_frame()
frame = inputdata.im2tensor(frame, channels = 3)
final = np.abs(-frame + template/255.0)
cv2.imshow('camera', final)
a = cv2.waitKey(30)
if a == 27:
cv2.destroyWindow('camera')
break
time.sleep(.005)
def generate_rollout_inputs(net, model_path, conv_layer, rollout, preview=False):
np.set_printoptions(threshold=np.nan)
sess = net.load(var_path=model_path)
with sess.as_default():
for j in range(0, 100):
test_image_path = opt.data_dir + 'amt/colors/rollout' + str(rollout) + '_frame_' + str(j) + '.jpg'
test_image = cv2.imread(test_image_path)
im = inputdata.im2tensor(test_image, channels=3)
shape = im.shape
im = np.reshape(im, (-1, shape[0], shape[1], shape[2]))
filter = sess.run(conv_layer, feed_dict={net.x:im})
for i in range(filter.shape[-1]):
filter_image = filter[0,:,:,i] * 255.0
filter_path = net.name + '_frame' + str(j) + '_input' + str(i) + '.jpg'
cv2.imwrite('filters/nrollout' + str(rollout) + '/' + filter_path, np.array(filter_image))
cv2.imwrite('filters/nrollout' + str(rollout) + '/' + net.name + '_frame' + str(j) + '.jpg', test_image)
sess.close()
def rollout_tf(self, num_frames=100):
net = self.options.tf_net
path = self.options.tf_net_path
sess = net.load(var_path=self.options.tf_net_path)
recording = []
# policy_net = None # Placeholder
# learner = PolicyGradient(net_dims = policy_net, 'tanh')
# reward_obj = RL_reward()
# self.qc = query_cam(self.bc)
# #Clear Buffer ... NEED TO TEST
# # self.qc.start()
# while(self.qc.read_frame() is None):
# print self.qc.frame
# pass # wait until images start coming through
#
for i in range(4):
self.bc.vc.grab()
try:
for i in range(num_frames):
# traj_states = []
# traj_actions = []
# rewards = []
# Read from the most updated frame
for i in range(4):
self.bc.vc.grab()
frame = self.bc.read_frame()
#frame = self.qc.read_frame()
# done reading
if (False):
gray_frame = self.gray(frame)
elif (False):
gray_frame = self.segment(frame)
elif (True):
gray_frame = self.color(frame)
gray_frame = np.reshape(gray_frame, (250, 250, 3))
test_frame = inputdata.im2tensor(gray_frame, channels=3)
print test_frame.shape
cv2.imshow('preview', test_frame)
#cv2.imshow("camera",gray_frame)
cv2.waitKey(30)
current_state = self.long2short_state(
self.state(self.izzy.getState()),
self.state(self.turntable.getState()))
im = bc.read_frame()
# reward = reward_obj.reward_function(im, current_state)
# delta_state = learner.get_action(current_state)
# if i != (num_frames-1):
# traj_states.append(current_state)
# traj_actions.append(delta_state)
# rewards.append(reward)
delta_state = self.rescale(
net.output(sess, gray_frame, channels=3))
#delta_state = net.output(sess, gray_frame,channels=3)
delta_state = self.deltaSafetyLimites(delta_state)
delta_state[2] = 0.0
recording.append((frame, current_state, delta_state))
new_izzy, new_t = self.apply_deltas(delta_state)
# TODO: uncomment these to update izzy and t
print "DELTA STATE ", delta_state
self.izzy._zeke._queueState(ZekeState(new_izzy))
self.turntable.gotoState(TurntableState(new_t), .25, .25)
time.sleep(.005)
# learner.gradient_update(traj_states, traj_actions, rewards, 'sgd')
except KeyboardInterrupt:
pass
# stop querying the camera
# self.qc.terminate()
# terminated
sess.close()
# self.izzy._zeke.steady(True)
self.prompt_save(recording)
from Net.tensor import net5, inputdata
from options import AMTOptions as opt
import tensorflow as tf
import numpy as np
import cv2
path = opt.tf_dir + 'net5/net5_02-15-2016_13h22m34s.ckpt'
net = net5.NetFive()
sess = net.load(var_path=path)
filter_summaries = []
slices = tf.split(3, 5, net.w_conv1)
for i, slice in enumerate(slices):
print slice.get_shape().as_list()
scope = 'jonathan' + str(i)
filter_summaries.append(tf.image_summary(scope, slice, max_images=1))
summary_writer = tf.train.SummaryWriter('/tmp/logs', sess.graph_def)
test_image_path = opt.colors_dir + 'rollout54_frame_16.jpg'
test_image = cv2.imread(test_image_path)
im = inputdata.im2tensor(test_image, 3)
shape = np.shape(im)
im = np.reshape(im, (-1, shape[0], shape[1], shape[2]))
feed_dict = {net.x: im}
with sess.as_default():
for filtsum in filter_summaries:
filtsum = filtsum.eval(feed_dict=feed_dict)
summary_writer.add_summary(filtsum, 0)
import cv2
import time
from options import Options
from Net.tensor import inputdata
bc = BinaryCamera("./meta.txt")
bc.open()
frame = bc.read_frame()
frames = []
fourcc = cv2.cv.CV_FOURCC(*'mp4v')
writer = cv2.VideoWriter("hardware_reset.mov", fourcc, 10.0, (420,420))
try:
while True:
frame = bc.read_frame()
frame = inputdata.im2tensor(frame, channels=3)
cv2.imshow("camera",frame)
cv2.waitKey(30)
frames.append(frame)
#time.sleep(0.08)
except KeyboardInterrupt:
pass
#for frame in frames:
# writer.write(frame)
#writer.release()
def teleop(c, izzy, t, bc, name):
target_state_i = get_state(izzy.getState())
# target_state_t = get_state(t.getState())
print "Start"
i = 0
frames = []
deltas_lst = []
states = []
avg_angd = 0.0
max_angd = 0.0
max_ford = 0.0
avg_ford = 0.0
past_deltas = []
length_pass = 3
try:
# for i in range(4):
# bc.vc.grab()
while i < 100:
start = time.time()
controls = c.getUpdates()
deltas, stop = controls2deltas(controls)
print stop
if stop:
break
for _ in range(4):
bc.vc.grab()
# cv2.imshow("camera", frame)
# cv2.waitKey(30)
if not all(d == 0.0 for d in deltas):
print i
i += 1
frame = bc.read_frame()
disp_frame = inputdata.im2tensor(frame, channels = 3)
# vis_frame = detectSingulatedCam.highlight(disp_frame * 255)
frames.append(frame)
cv2.imshow("camera", disp_frame)
cv2.waitKey(30)
deltas_lst.append(deltas)
# print "Current: ", target_state_i, target_state_t
# true_cur = izzy.getState().state
print "Current: ", target_state_i#, "true current: ", true_cur
# target_state_i = true_cur
# avg_angd += target_state_i[0] - true_cur[0]
# max_angd = max(max_angd, target_state_i[0] - true_cur[0])
# max_ford = max(max_ford, target_state_i[2] - true_cur[2])
# avg_ford += target_state_i[2] - true_cur[2]
states.append(target_state_i[:])
# new_izzy, new_t = apply_deltas(deltas, target_state_i, target_state_t)
past_deltas.append((deltas[0], deltas[1]))
if len(past_deltas) > length_pass:
past_deltas.pop(0)
deltas[0] = np.sum([past_delta[0] for past_delta in past_deltas])/len(past_deltas)
deltas[1] = np.sum([past_delta[1] for past_delta in past_deltas])/len(past_deltas)
new_izzy = apply_deltas(deltas, target_state_i)
print "Deltas: ", deltas
# target_state_i, target_state_t = new_izzy, new_t
target_state_i = new_izzy
# print "Teleop: ", new_izzy, new_t
print "Teleop: ", new_izzy
izzy._zeke._queueState(ZekeState(new_izzy))
# t.gotoState(TurntableState(new_t), .25, .25)
offset = max(0, .3 - (time.time() - start))
print "offset time: ", offset
time.sleep(offset)
print "total time: ", time.time() - start
#if i == 501:
# break
except KeyboardInterrupt:
pass
stop = False
return_to_start(izzy)
# print avg_angd/100.0, max_angd, avg_ford/100.0, max_ford
return prompt_save(frames, deltas_lst, states, name)
from Net.tensor import net5, inputdata
from options import AMTOptions as opt
import tensorflow as tf
import numpy as np
import cv2
path = opt.tf_dir + "net5/net5_02-15-2016_13h22m34s.ckpt"
net = net5.NetFive()
sess = net.load(var_path=path)
filter_summaries = []
slices = tf.split(3, 5, net.w_conv1)
for i, slice in enumerate(slices):
print slice.get_shape().as_list()
scope = "jonathan" + str(i)
filter_summaries.append(tf.image_summary(scope, slice, max_images=1))
summary_writer = tf.train.SummaryWriter("/tmp/logs", sess.graph_def)
test_image_path = opt.colors_dir + "rollout54_frame_16.jpg"
test_image = cv2.imread(test_image_path)
im = inputdata.im2tensor(test_image, 3)
shape = np.shape(im)
im = np.reshape(im, (-1, shape[0], shape[1], shape[2]))
feed_dict = {net.x: im}
with sess.as_default():
for filtsum in filter_summaries:
filtsum = filtsum.eval(feed_dict=feed_dict)
summary_writer.add_summary(filtsum, 0)
def rollout_tf(self, num_frames=100):
net = self.options.tf_net
path = self.options.tf_net_path
sess = net.load(var_path=self.options.tf_net_path)
recording = []
# policy_net = None # Placeholder
# learner = PolicyGradient(net_dims = policy_net, 'tanh')
# reward_obj = RL_reward()
# self.qc = query_cam(self.bc)
# #Clear Buffer ... NEED TO TEST
# # self.qc.start()
# while(self.qc.read_frame() is None):
# print self.qc.frame
# pass # wait until images start coming through
#
for i in range(4):
self.bc.vc.grab()
try:
target_state_i = self.state(self.izzy.getState())
target_state_t = self.state(self.turntable.getState())
for i in range(num_frames):
# traj_states = []
# traj_actions = []
# rewards = []
# Read from the most updated frame
for i in range(4):
self.bc.vc.grab()
frame = self.bc.read_frame()
#frame = self.qc.read_frame()
# done reading
if(False):
gray_frame = self.gray(frame)
elif(False):
gray_frame = self.segment(frame)
elif(True):
gray_frame = self.color(frame)
gray_frame = np.reshape(gray_frame, (250, 250, 3))
test_frame = inputdata.im2tensor(gray_frame, channels=3)
print test_frame.shape
cv2.imshow('preview', test_frame)
#cv2.imshow("camera",gray_frame)
cv2.waitKey(30)
target_state_i = self.state(self.izzy.getState())
target_state_t = self.state(self.turntable.getState())
current_state = self.long2short_state(target_state_i, target_state_t)
delta_state = self.rescale(net.output(sess, gray_frame,channels=3))
dists = net.class_dist(sess, gray_frame)
plt.clf()
x = np.array([-1,-0.5,0,0.5,1])
plt.subplot(2,1,1)
plt.plot(x,dists[0,:])
plt.xlabel('Rotation')
plt.subplot(2,1,2)
plt.plot(x,dists[1,:])
plt.xlabel('Extension')
plt.draw()
plt.show(block=False)
#delta_state = net.output(sess, gray_frame,channels=3)
delta_state = self.deltaSafetyLimites(delta_state)
delta_state[2] = 0.0
recording.append((frame, current_state,delta_state))
new_izzy, new_t = self.apply_deltas(delta_state,target_state_i,target_state_t)
# TODO: uncomment these to update izzy and t
print "DELTA STATE ",delta_state
self.izzy._zeke._queueState(ZekeState(new_izzy))
self.turntable.gotoState(TurntableState(new_t), .25, .25)
time.sleep(.005)
# learner.gradient_update(traj_states, traj_actions, rewards, 'sgd')
except KeyboardInterrupt:
pass
# stop querying the camera
# self.qc.terminate()
# terminated
sess.close()
# self.izzy._zeke.steady(True)
self.prompt_save(recording)
