def record_template(bc, template=None): # records the process of placing blocks of the template if template is None: template = cv2.imread("/home/annal/Izzy/vision_amt/scripts/objects/template.png") template[:,:,1] = template[:,:,2] template[:,:,0] = np.zeros((420, 420)) # template[:,:,2] = np.zeros((420, 420)) # template = cv2.resize(template, (250, 250)) frames = [] while 1: frame = bc.read_frame() frame = inputdata.im2tensor(frame, channels = 3) final = np.abs(-frame + template/255.0) frames.append(final * 255) cv2.imshow('camera', final) a = cv2.waitKey(30) if a == 27: cv2.destroyAllWindows() break elif a == ord(' '): return 'next' time.sleep(.1) rollout_path = AMTOptions.supervised_dir+ 'supervised_template/' os.makedirs(rollout_path) i = 0 for frame in frames: name = 'template_' + str(i) + '.jpg' cv2.imwrite(rollout_path + name, frame) i += 1 print "saved to: ", rollout_path
def generate_rollout_inputs(net, model_path, conv_layer, rollout, preview=False): np.set_printoptions(threshold=np.nan) sess = net.load(var_path=model_path) with sess.as_default(): for j in range(0, 100): test_image_path = opt.data_dir + 'amt/colors/rollout' + str( rollout) + '_frame_' + str(j) + '.jpg' test_image = cv2.imread(test_image_path) im = inputdata.im2tensor(test_image, channels=3) shape = im.shape im = np.reshape(im, (-1, shape[0], shape[1], shape[2])) filter = sess.run(conv_layer, feed_dict={net.x: im}) for i in range(filter.shape[-1]): filter_image = filter[0, :, :, i] * 255.0 filter_path = net.name + '_frame' + str(j) + '_input' + str( i) + '.jpg' cv2.imwrite( 'filters/nrollout' + str(rollout) + '/' + filter_path, np.array(filter_image)) cv2.imwrite( 'filters/nrollout' + str(rollout) + '/' + net.name + '_frame' + str(j) + '.jpg', test_image) sess.close()
def vectorize_image(image): img = cv2.resize(cv2.imread(image), (250,250)) im = inputdata.im2tensor(img, channels = 3) imshape = im.shape im = np.reshape(im, (-1, imshape[0], imshape[1], imshape[2])) return im
def run_samples(n): net = net6_c.NetSix_C() path = '/media/1tb/Izzy/nets/net6_05-11-2016_12h09m12s.ckpt' sess = net.load(path) bc = BinaryCamera('./meta.txt') bc.open() for i in range(4): bc.read_frame() num_samples = n samples = [] images = [] for i in range(num_samples): print i try: while True: frame = bc.read_frame() frame = inputdata.im2tensor(frame, channels=3) cv2.imshow('preview', frame) cv2.waitKey(30) except: pass frame = bc.read_frame() frame = similarity.color(frame) images.append(frame) print "Saved sample" np.save('samples_images.npy', images)
def generate_inputs(net, model_path, conv_layer, test_image, preview=False): sess = net.load(var_path=model_path) im = inputdata.im2tensor(test_image, channels=3) shape = im.shape im = np.reshape(im, (-1, shape[0], shape[1], shape[2])) with sess.as_default(): filter = sess.run(conv_layer, feed_dict={net.x: im}) for i in range(filter.shape[-1]): filter_image = filter[0, :, :, i] * 255.0 filter_path = net.name + '_input' + str(i) + '.jpg' cv2.imwrite('filters/' + filter_path, np.array(filter_image)) cv2.imwrite('filters/' + net.name + '_sample.jpg', test_image) sess.close()
def generate_inputs(net, model_path, conv_layer, test_image, preview=False): sess = net.load(var_path=model_path) im = inputdata.im2tensor(test_image, channels=3) shape = im.shape im = np.reshape(im, (-1, shape[0], shape[1], shape[2])) with sess.as_default(): filter = sess.run(conv_layer, feed_dict={net.x:im}) for i in range(filter.shape[-1]): filter_image = filter[0,:,:,i] * 255.0 filter_path = net.name + '_input' + str(i) + '.jpg' cv2.imwrite('filters/' + filter_path, np.array(filter_image)) cv2.imwrite('filters/' + net.name +'_sample.jpg', test_image) sess.close()
def display_template(self, template=None): if template is None: template = cv2.imread("/home/annal/Izzy/vision_amt/scripts/objects/template.png") template[:,:,1] = template[:,:,2] template[:,:,0] = np.zeros((420, 420)) # template[:,:,2] = np.zeros((420, 420)) # template = cv2.resize(template, (250, 250)) while 1: frame = self.bc.read_frame() frame = inputdata.im2tensor(frame, channels = 3) final = np.abs(-frame + template/255.0) cv2.imshow('camera', final) a = cv2.waitKey(30) if a == 27: cv2.destroyWindow('camera') break time.sleep(.005)
def generate_rollout_inputs(net, model_path, conv_layer, rollout, preview=False): np.set_printoptions(threshold=np.nan) sess = net.load(var_path=model_path) with sess.as_default(): for j in range(0, 100): test_image_path = opt.data_dir + 'amt/colors/rollout' + str(rollout) + '_frame_' + str(j) + '.jpg' test_image = cv2.imread(test_image_path) im = inputdata.im2tensor(test_image, channels=3) shape = im.shape im = np.reshape(im, (-1, shape[0], shape[1], shape[2])) filter = sess.run(conv_layer, feed_dict={net.x:im}) for i in range(filter.shape[-1]): filter_image = filter[0,:,:,i] * 255.0 filter_path = net.name + '_frame' + str(j) + '_input' + str(i) + '.jpg' cv2.imwrite('filters/nrollout' + str(rollout) + '/' + filter_path, np.array(filter_image)) cv2.imwrite('filters/nrollout' + str(rollout) + '/' + net.name + '_frame' + str(j) + '.jpg', test_image) sess.close()
def rollout_tf(self, num_frames=100): net = self.options.tf_net path = self.options.tf_net_path sess = net.load(var_path=self.options.tf_net_path) recording = [] # policy_net = None # Placeholder # learner = PolicyGradient(net_dims = policy_net, 'tanh') # reward_obj = RL_reward() # self.qc = query_cam(self.bc) # #Clear Buffer ... NEED TO TEST # # self.qc.start() # while(self.qc.read_frame() is None): # print self.qc.frame # pass # wait until images start coming through # for i in range(4): self.bc.vc.grab() try: for i in range(num_frames): # traj_states = [] # traj_actions = [] # rewards = [] # Read from the most updated frame for i in range(4): self.bc.vc.grab() frame = self.bc.read_frame() #frame = self.qc.read_frame() # done reading if (False): gray_frame = self.gray(frame) elif (False): gray_frame = self.segment(frame) elif (True): gray_frame = self.color(frame) gray_frame = np.reshape(gray_frame, (250, 250, 3)) test_frame = inputdata.im2tensor(gray_frame, channels=3) print test_frame.shape cv2.imshow('preview', test_frame) #cv2.imshow("camera",gray_frame) cv2.waitKey(30) current_state = self.long2short_state( self.state(self.izzy.getState()), self.state(self.turntable.getState())) im = bc.read_frame() # reward = reward_obj.reward_function(im, current_state) # delta_state = learner.get_action(current_state) # if i != (num_frames-1): # traj_states.append(current_state) # traj_actions.append(delta_state) # rewards.append(reward) delta_state = self.rescale( net.output(sess, gray_frame, channels=3)) #delta_state = net.output(sess, gray_frame,channels=3) delta_state = self.deltaSafetyLimites(delta_state) delta_state[2] = 0.0 recording.append((frame, current_state, delta_state)) new_izzy, new_t = self.apply_deltas(delta_state) # TODO: uncomment these to update izzy and t print "DELTA STATE ", delta_state self.izzy._zeke._queueState(ZekeState(new_izzy)) self.turntable.gotoState(TurntableState(new_t), .25, .25) time.sleep(.005) # learner.gradient_update(traj_states, traj_actions, rewards, 'sgd') except KeyboardInterrupt: pass # stop querying the camera # self.qc.terminate() # terminated sess.close() # self.izzy._zeke.steady(True) self.prompt_save(recording)
from Net.tensor import net5, inputdata from options import AMTOptions as opt import tensorflow as tf import numpy as np import cv2 path = opt.tf_dir + 'net5/net5_02-15-2016_13h22m34s.ckpt' net = net5.NetFive() sess = net.load(var_path=path) filter_summaries = [] slices = tf.split(3, 5, net.w_conv1) for i, slice in enumerate(slices): print slice.get_shape().as_list() scope = 'jonathan' + str(i) filter_summaries.append(tf.image_summary(scope, slice, max_images=1)) summary_writer = tf.train.SummaryWriter('/tmp/logs', sess.graph_def) test_image_path = opt.colors_dir + 'rollout54_frame_16.jpg' test_image = cv2.imread(test_image_path) im = inputdata.im2tensor(test_image, 3) shape = np.shape(im) im = np.reshape(im, (-1, shape[0], shape[1], shape[2])) feed_dict = {net.x: im} with sess.as_default(): for filtsum in filter_summaries: filtsum = filtsum.eval(feed_dict=feed_dict) summary_writer.add_summary(filtsum, 0)
import cv2 import time from options import Options from Net.tensor import inputdata bc = BinaryCamera("./meta.txt") bc.open() frame = bc.read_frame() frames = [] fourcc = cv2.cv.CV_FOURCC(*'mp4v') writer = cv2.VideoWriter("hardware_reset.mov", fourcc, 10.0, (420,420)) try: while True: frame = bc.read_frame() frame = inputdata.im2tensor(frame, channels=3) cv2.imshow("camera",frame) cv2.waitKey(30) frames.append(frame) #time.sleep(0.08) except KeyboardInterrupt: pass #for frame in frames: # writer.write(frame) #writer.release()
def teleop(c, izzy, t, bc, name): target_state_i = get_state(izzy.getState()) # target_state_t = get_state(t.getState()) print "Start" i = 0 frames = [] deltas_lst = [] states = [] avg_angd = 0.0 max_angd = 0.0 max_ford = 0.0 avg_ford = 0.0 past_deltas = [] length_pass = 3 try: # for i in range(4): # bc.vc.grab() while i < 100: start = time.time() controls = c.getUpdates() deltas, stop = controls2deltas(controls) print stop if stop: break for _ in range(4): bc.vc.grab() # cv2.imshow("camera", frame) # cv2.waitKey(30) if not all(d == 0.0 for d in deltas): print i i += 1 frame = bc.read_frame() disp_frame = inputdata.im2tensor(frame, channels = 3) # vis_frame = detectSingulatedCam.highlight(disp_frame * 255) frames.append(frame) cv2.imshow("camera", disp_frame) cv2.waitKey(30) deltas_lst.append(deltas) # print "Current: ", target_state_i, target_state_t # true_cur = izzy.getState().state print "Current: ", target_state_i#, "true current: ", true_cur # target_state_i = true_cur # avg_angd += target_state_i[0] - true_cur[0] # max_angd = max(max_angd, target_state_i[0] - true_cur[0]) # max_ford = max(max_ford, target_state_i[2] - true_cur[2]) # avg_ford += target_state_i[2] - true_cur[2] states.append(target_state_i[:]) # new_izzy, new_t = apply_deltas(deltas, target_state_i, target_state_t) past_deltas.append((deltas[0], deltas[1])) if len(past_deltas) > length_pass: past_deltas.pop(0) deltas[0] = np.sum([past_delta[0] for past_delta in past_deltas])/len(past_deltas) deltas[1] = np.sum([past_delta[1] for past_delta in past_deltas])/len(past_deltas) new_izzy = apply_deltas(deltas, target_state_i) print "Deltas: ", deltas # target_state_i, target_state_t = new_izzy, new_t target_state_i = new_izzy # print "Teleop: ", new_izzy, new_t print "Teleop: ", new_izzy izzy._zeke._queueState(ZekeState(new_izzy)) # t.gotoState(TurntableState(new_t), .25, .25) offset = max(0, .3 - (time.time() - start)) print "offset time: ", offset time.sleep(offset) print "total time: ", time.time() - start #if i == 501: # break except KeyboardInterrupt: pass stop = False return_to_start(izzy) # print avg_angd/100.0, max_angd, avg_ford/100.0, max_ford return prompt_save(frames, deltas_lst, states, name)
from Net.tensor import net5, inputdata from options import AMTOptions as opt import tensorflow as tf import numpy as np import cv2 path = opt.tf_dir + "net5/net5_02-15-2016_13h22m34s.ckpt" net = net5.NetFive() sess = net.load(var_path=path) filter_summaries = [] slices = tf.split(3, 5, net.w_conv1) for i, slice in enumerate(slices): print slice.get_shape().as_list() scope = "jonathan" + str(i) filter_summaries.append(tf.image_summary(scope, slice, max_images=1)) summary_writer = tf.train.SummaryWriter("/tmp/logs", sess.graph_def) test_image_path = opt.colors_dir + "rollout54_frame_16.jpg" test_image = cv2.imread(test_image_path) im = inputdata.im2tensor(test_image, 3) shape = np.shape(im) im = np.reshape(im, (-1, shape[0], shape[1], shape[2])) feed_dict = {net.x: im} with sess.as_default(): for filtsum in filter_summaries: filtsum = filtsum.eval(feed_dict=feed_dict) summary_writer.add_summary(filtsum, 0)
def rollout_tf(self, num_frames=100): net = self.options.tf_net path = self.options.tf_net_path sess = net.load(var_path=self.options.tf_net_path) recording = [] # policy_net = None # Placeholder # learner = PolicyGradient(net_dims = policy_net, 'tanh') # reward_obj = RL_reward() # self.qc = query_cam(self.bc) # #Clear Buffer ... NEED TO TEST # # self.qc.start() # while(self.qc.read_frame() is None): # print self.qc.frame # pass # wait until images start coming through # for i in range(4): self.bc.vc.grab() try: target_state_i = self.state(self.izzy.getState()) target_state_t = self.state(self.turntable.getState()) for i in range(num_frames): # traj_states = [] # traj_actions = [] # rewards = [] # Read from the most updated frame for i in range(4): self.bc.vc.grab() frame = self.bc.read_frame() #frame = self.qc.read_frame() # done reading if(False): gray_frame = self.gray(frame) elif(False): gray_frame = self.segment(frame) elif(True): gray_frame = self.color(frame) gray_frame = np.reshape(gray_frame, (250, 250, 3)) test_frame = inputdata.im2tensor(gray_frame, channels=3) print test_frame.shape cv2.imshow('preview', test_frame) #cv2.imshow("camera",gray_frame) cv2.waitKey(30) target_state_i = self.state(self.izzy.getState()) target_state_t = self.state(self.turntable.getState()) current_state = self.long2short_state(target_state_i, target_state_t) delta_state = self.rescale(net.output(sess, gray_frame,channels=3)) dists = net.class_dist(sess, gray_frame) plt.clf() x = np.array([-1,-0.5,0,0.5,1]) plt.subplot(2,1,1) plt.plot(x,dists[0,:]) plt.xlabel('Rotation') plt.subplot(2,1,2) plt.plot(x,dists[1,:]) plt.xlabel('Extension') plt.draw() plt.show(block=False) #delta_state = net.output(sess, gray_frame,channels=3) delta_state = self.deltaSafetyLimites(delta_state) delta_state[2] = 0.0 recording.append((frame, current_state,delta_state)) new_izzy, new_t = self.apply_deltas(delta_state,target_state_i,target_state_t) # TODO: uncomment these to update izzy and t print "DELTA STATE ",delta_state self.izzy._zeke._queueState(ZekeState(new_izzy)) self.turntable.gotoState(TurntableState(new_t), .25, .25) time.sleep(.005) # learner.gradient_update(traj_states, traj_actions, rewards, 'sgd') except KeyboardInterrupt: pass # stop querying the camera # self.qc.terminate() # terminated sess.close() # self.izzy._zeke.steady(True) self.prompt_save(recording)