class RCDriverNNOnly(object): def __init__(self, model_path): print('initing...') # load trained neural network if model_num == 1: import model_b as model self.model = model elif model_num == 0: load_model_start = time.time() from model import NeuralNetwork self.model = NeuralNetwork(model_path) load_model_end = time.time() print('loading model costs {:02f} second(s)'.format(load_model_end - load_model_start)) self.cap = cv2.VideoCapture(0) self.servo = PyServo.Servo(SerialID,Baudrate) def drive(self): step = 0 try: while True: ret, img = self.cap.read() if ret: #print(img.shape) img = cv2.resize(img,(320,240),interpolation=cv2.INTER_CUBIC) #img = cv2.remap(img,map1,map2, interpolation= cv2.INTER_LINEAR, borderMode=cv2.BORDER_CONSTANT) gray = cv2.cvtColor(img,cv2.COLOR_BGR2GRAY) height, width = gray.shape roi_img = gray[roi[0]:roi[1], :] # Apply GuassianBlur (reduces noise) blurred = cv2.GaussianBlur(roi_img, (3, 3), 0) # Apply Canny filter auto = self.model.auto_canny(blurred) #print(auto.shape) global count if self.servo.isLinsten(): if timer: timer_prediction = time.time() prediction_english = None prediction_english_proba = None # neural network makes prediction print("predicting...") prediction, lines = self.model.predict(auto) if timer: print('prediction takes:{} seconds'.format(time.time()-timer_prediction)) timer_action = time.time() #proba_left, proba_right, proba_forward = probas[0] if np.all(prediction == [ 0., 0., 1.]): prediction_english = 'FORWARD' self.servo.RunGroup(Move_Forward,1) elif np.all(prediction == [ 1., 0., 0.]): prediction_english = 'LEFT' self.servo.RunGroup(Move_Left,1) elif np.all(prediction == [ 0., 1., 0.]): prediction_english = 'RIGHT' self.servo.RunGroup(Move_Right,1) print(prediction_english) if write_temp_file: if model_num ==1: #auto = self.model.visualize_lines(cv2.cvtColor(roi_img,cv2.COLOR_GRAY2BGR),lines) pass cv2.putText(auto, "Prediction: {}".format(prediction_english, ), (10, 30), cv2.FONT_HERSHEY_SIMPLEX, .45, (255, 255, 0), 1) step += 1 cv2.imwrite('./test_images/step{}.jpg'.format(step), auto) print('writing images of step{}'.format(step)) if timer: print('Action takes:{} seconds'.format(time.time()-timer_action)) print(count) count = 0 else: while True: if self.servo.isLinsten(): self.servo.Group_complete = True break count+=1 except Exception as e: print(e) finally: print('total {} frames predicted'.format(step))
class RCDriverNNOnly(object): def __init__(self, host, port, _host, _port, model_path): # load trained neural network self.nn = NeuralNetwork(model_path) self.cap = cv2.VideoCapture(0) self.rc_car = RCControl(_h, _p) def drive(self): stream_bytes = b' ' frame = 0 try: # stream video frames one by one while True: ret, frame = self.cap.read() if ret: frame = cv2.resize(frame, (320, 240), interpolation=cv2.INTER_CUBIC) gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) image = frame height, width = gray.shape roi = gray[int(height / 2):height, :] # Apply GuassianBlur (reduces noise) blurred = cv2.GaussianBlur(roi, (3, 3), 0) # Apply Canny filter auto = self.nn.auto_canny(blurred) cv2.imshow('image', image) cv2.imshow('What the model sees', auto) # cv2.imshow('mlp_image', roi) # Neural network model makes prediciton # prediction = self.model.predict(auto) # reshape image #image_array = roi.reshape(1, int(height/2) * width).astype(np.float32) global count frame += 1 if count >= 5: prediction_english = None prediction_english_proba = None # neural network makes prediction prediction, probas = self.nn.predict(auto) proba_left, proba_right, proba_forward = probas[0] if np.all(prediction == [0., 0., 1.]): prediction_english = 'FORWARD' prediction_english_proba = proba_forward elif np.all(prediction == [1., 0., 0.]): prediction_english = 'LEFT' prediction_english_proba = proba_left elif np.all(prediction == [0., 1., 0.]): prediction_english = 'RIGHT' prediction_english_proba = proba_right count = 0 self.rc_car.steer(prediction) else: count += 1 if cv2.waitKey(1) & 0xFF == ord('q'): print("car stopped") self.rc_car.stop() break finally: cv2.destroyAllWindows()
class RCDriverNNOnly(object): def __init__(self, host, port, _host, _port, model_path): #tcp host port #udp _host _port self.server_socket = socket.socket() self.server_socket.bind((host, port)) self.server_socket.listen(0) # accept a single connection self.connection = self.server_socket.accept()[0].makefile('rb') # load trained neural network self.nn = NeuralNetwork(model_path) self.rc_car = RCControl(_h, _p) def drive(self): stream_bytes = b' ' frame = 0 try: # stream video frames one by one while True: stream_bytes += self.connection.read(1024) first = stream_bytes.find(b'\xff\xd8') last = stream_bytes.find(b'\xff\xd9') if first != -1 and last != -1: jpg = stream_bytes[first:last + 2] stream_bytes = stream_bytes[last + 2:] gray = cv2.imdecode(np.frombuffer(jpg, dtype=np.uint8), cv2.IMREAD_GRAYSCALE) image = cv2.imdecode(np.frombuffer(jpg, dtype=np.uint8), cv2.IMREAD_COLOR) # lower half of the image height, width = gray.shape roi = gray[int(height / 2):height, :] # Apply GuassianBlur (reduces noise) blurred = cv2.GaussianBlur(roi, (3, 3), 0) # Apply Canny filter auto = self.nn.auto_canny(blurred) cv2.imshow('image', image) cv2.imshow('What the model sees', auto) # cv2.imshow('mlp_image', roi) image = cv2.imdecode(np.frombuffer(jpg, dtype=np.uint8), cv2.IMREAD_GRAYSCALE) # Neural network model makes prediciton # prediction = self.model.predict(auto) # reshape image #image_array = roi.reshape(1, int(height/2) * width).astype(np.float32) prediction_english = None prediction_english_proba = None # neural network makes prediction prediction, probas = self.nn.predict(auto) proba_left, proba_right, proba_forward = probas[0] if np.all(prediction == [0., 0., 1.]): prediction_english = 'FORWARD' prediction_english_proba = proba_forward elif np.all(prediction == [1., 0., 0.]): prediction_english = 'LEFT' prediction_english_proba = proba_left elif np.all(prediction == [0., 1., 0.]): prediction_english = 'RIGHT' prediction_english_proba = proba_right global count frame += 1 cv2.putText( gray, "Prediction (sig={}): {}, {:>05}".format( SIGMA, prediction_english, prediction_english_proba), (10, 30), cv2.FONT_HERSHEY_SIMPLEX, .45, (255, 255, 0), 1) if write_temp_file: cv2.imwrite( 'test_frames_temp/frame{:>05}.jpg'.format(frame), gray) if count >= 10: self.rc_car.steer(prediction) count = 0 else: count += 1 if cv2.waitKey(1) & 0xFF == ord('q'): print("car stopped") self.rc_car.stop() break finally: cv2.destroyAllWindows() self.connection.close() self.server_socket.close()