def _get_observation(self): measurements, snesor_data = self.client.read_data() carla_im = snesor_data.get('CameraSegmentation', None) '''For RGB image''' #self.observation_image = image_converter.to_rgb_array(carla_im) '''For Semantic Segmentation Image''' #print(type(carla_im)) self.observation_image = np.uint8(image_converter.labels_to_cityscapes_palette(carla_im)) cv2.imshow('im', self.observation_image) cv2.waitKey(1) stats = measurements.player_measurements pos = measurements.player_measurements.transform movement = self.get_movemet(pos) # print('Movemene ', movement) self.percentage_intersection_other_lane = abs(stats.intersection_otherlane) self.percentage_offroad = stats.intersection_offroad collision = stats.collision_vehicles or stats.collision_pedestrians or stats.collision_other '''Sometimes, the collision sensor doesn't register, so I check for distnce moved between each step, after 100 timesteps because at lower time steps the distance moved is pretty low''' car_stuck = self.is_car_stuck(movement) done = self._is_game_over(collision, car_stuck) #print(type(done)) observation = self.vae_observation(observation_image=self.observation_image) return observation, done
def sensor_process(self, sensor_data): self.image_rgb = np.transpose( image_converter.to_rgb_array(sensor_data['CameraRGB']).swapaxes( 0, 1), (1, 0, 2)) self.image_depth = np.transpose( image_converter.depth_to_logarithmic_grayscale( sensor_data['CameraDepth']).swapaxes(0, 1), (1, 0, 2)) self.image_depth = np.array(self.image_depth, dtype=np.uint8) self.image_seg = np.transpose( image_converter.labels_to_cityscapes_palette( sensor_data['CameraSemSeg']).swapaxes(0, 1), (1, 0, 2)) self.image_seg = np.array(self.image_seg, dtype=np.uint8) if self.image_rgb is not None: image_rgb_frame = CvBridge().cv2_to_imgmsg(self.image_rgb, "rgb8") self.image_rgb_pub.publish(image_rgb_frame) if self.image_depth is not None: image_depth_frame = CvBridge().cv2_to_imgmsg( self.image_depth, "rgb8") self.image_depth_pub.publish(image_depth_frame) if self.image_seg is not None: image_seg_frame = CvBridge().cv2_to_imgmsg(self.image_seg, "rgb8") self.image_seg_pub.publish(image_seg_frame)
def seg_cb(self, data): try: cv_image = self.bridge.imgmsg_to_cv2(data) except CvBridgeError as e: print(e) new_img = ic.labels_to_cityscapes_palette(np.asarray(cv_image)) print(cv_image) cv2.imshow("test", new_img) cv2.waitKey(3)
def __compute_mean_iou(self, ground_frame, segmented_frame): # Transfrom the ground frame to Cityscapes palette; the segmented # frame is transformed by segmentation operators. ground_frame = labels_to_cityscapes_palette(ground_frame) (mean_iou, class_iou) = compute_semantic_iou(ground_frame, segmented_frame) self._logger.info('IoU class scores: {}'.format(class_iou)) self._logger.info('mean IoU score: {}'.format(mean_iou)) self._csv_logger.info('{},{},{},{}'.format( time_epoch_ms(), self.name, self._flags.eval_segmentation_metric, mean_iou))
def _on_render(self): gap_x = (WINDOW_WIDTH - 2 * MINI_WINDOW_WIDTH) / 3 mini_image_y = WINDOW_HEIGHT - MINI_WINDOW_HEIGHT - gap_x if self._main_image is not None: array = image_converter.to_rgb_array(self._main_image) surface = pygame.surfarray.make_surface(array.swapaxes(0, 1)) self._display.blit(surface, (0, 0)) if self._mini_view_image1 is not None: array = image_converter.depth_to_logarithmic_grayscale( self._mini_view_image1) surface = pygame.surfarray.make_surface(array.swapaxes(0, 1)) self._display.blit(surface, (gap_x, mini_image_y)) if self._mini_view_image2 is not None: array = image_converter.labels_to_cityscapes_palette( self._mini_view_image2) surface = pygame.surfarray.make_surface(array.swapaxes(0, 1)) self._display.blit(surface, (2 * gap_x + MINI_WINDOW_WIDTH, mini_image_y)) if self._map_view is not None: array = self._map_view array = array[:, :, :3] new_window_width = (float(WINDOW_HEIGHT) / float( self._map_shape[0])) * float(self._map_shape[1]) surface = pygame.surfarray.make_surface(array.swapaxes(0, 1)) w_pos = int(self._position[0] * (float(WINDOW_HEIGHT) / float(self._map_shape[0]))) h_pos = int(self._position[1] * (new_window_width / float(self._map_shape[1]))) pygame.draw.circle(surface, [255, 0, 0, 255], (w_pos, h_pos), 6, 0) for agent in self._agent_positions: if agent.HasField('vehicle'): agent_position = self._map.get_position_on_map([ agent.vehicle.transform.location.x, agent.vehicle.transform.location.y, agent.vehicle.transform.location.z ]) w_pos = int( agent_position[0] * (float(WINDOW_HEIGHT) / float(self._map_shape[0]))) h_pos = int(agent_position[1] * (new_window_width / float(self._map_shape[1]))) pygame.draw.circle(surface, [255, 0, 255, 255], (w_pos, h_pos), 4, 0) self._display.blit(surface, (WINDOW_WIDTH, 0)) pygame.display.flip()
def _on_render(self): gap_x = (WINDOW_WIDTH - 2 * MINI_WINDOW_WIDTH) / 3 mini_image_y = WINDOW_HEIGHT - MINI_WINDOW_HEIGHT - gap_x global COUNT global frame #COUNT+=1 if self._main_image is not None: array = image_converter.to_rgb_array(self._main_image) surface = pygame.surfarray.make_surface(array.swapaxes(0, 1)) self._display.blit(surface, (0, 0)) if((COUNT%frame)==0): pygame.image.save(surface,"E:/RGB/rgb"+format(COUNT)+".png") # print("RGB saved") if self._mini_view_image2 is not None: array = image_converter.labels_to_cityscapes_palette( self._mini_view_image2) surface = pygame.surfarray.make_surface(array.swapaxes(0, 1)) if(COUNT%frame==0): pygame.image.save(surface,"E:/Semanticdata/Semantic"+format(COUNT)+".png") # print("Semantic saved") self._display.blit( surface, (2 * gap_x + MINI_WINDOW_WIDTH, mini_image_y)) if self._map_view is not None: array = self._map_view array = array[:, :, :3] new_window_width = \ (float(WINDOW_HEIGHT) / float(self._map_shape[0])) * \ float(self._map_shape[1]) surface = pygame.surfarray.make_surface(array.swapaxes(0, 1)) w_pos = int(self._position[0]*(float(WINDOW_HEIGHT)/float(self._map_shape[0]))) h_pos = int(self._position[1] *(new_window_width/float(self._map_shape[1]))) pygame.draw.circle(surface, [255, 0, 0, 255], (w_pos, h_pos), 6, 0) for agent in self._agent_positions: if agent.HasField('vehicle'): agent_position = self._map.convert_to_pixel([ agent.vehicle.transform.location.x, agent.vehicle.transform.location.y, agent.vehicle.transform.location.z]) w_pos = int(agent_position[0]*(float(WINDOW_HEIGHT)/float(self._map_shape[0]))) h_pos = int(agent_position[1] *(new_window_width/float(self._map_shape[1]))) pygame.draw.circle(surface, [255, 0, 255, 255], (w_pos, h_pos), 4, 0) self._display.blit(surface, (WINDOW_WIDTH, 0)) pygame.display.flip()
def on_segmented_frame(self, msg): # Ensure we didn't skip a frame. if self._last_segmented_timestamp != -1: assert self._last_segmented_timestamp + 1 == msg.timestamp.coordinates[ 1] self._last_segmented_timestamp = msg.timestamp.coordinates[1] if self._last_bgr_timestamp % self._flags.log_every_nth_frame != 0: return # Write the segmented image. frame_array = labels_to_cityscapes_palette(msg.data) img = Image.fromarray(np.uint8(frame_array)) file_name = '{}carla-segmented-{}.png'.format( self._flags.data_path, self._last_segmented_timestamp) img.save(file_name)
def display_frame(self, msg): if self._last_seq_num + 1 != msg.timestamp.coordinates[1]: self._logger.error( 'Expected msg with seq num {} but received {}'.format( (self._last_seq_num + 1), msg.timestamp.coordinates[1])) if self._flags.fail_on_message_loss: assert self._last_seq_num + 1 == msg.timestamp.coordinates[1] self._last_seq_num = msg.timestamp.coordinates[1] frame_array = labels_to_cityscapes_palette(msg.data) img = Image.fromarray(np.uint8(frame_array)) open_cv_image = rgb_to_bgr(np.array(img)) cv2.imshow(self.name, open_cv_image) cv2.waitKey(1)
def _get_camera_images(self): sensor_data = self._sensor_data image_object = { "rgb_center": ic.to_bgra_array(sensor_data.get("RGBCameraCenter", None)), "rgb_left": ic.to_bgra_array(sensor_data.get("RGBCameraLeft", None)), "rgb_right": ic.to_bgra_array(sensor_data.get("RGBCameraRight", None)), "depth": ic.depth_to_logarithmic_grayscale( sensor_data.get("DepthCamera", None) ), "sem_seg": ic.labels_to_cityscapes_palette( sensor_data.get("SemSegCamera", None) ), } return image_object
def _on_render(self): gap_x = (WINDOW_WIDTH - 2 * MINI_WINDOW_WIDTH) / 3 mini_image_y = WINDOW_HEIGHT - MINI_WINDOW_HEIGHT - gap_x if self._main_image is not None: array = image_converter.to_rgb_array(self._main_image) surface = pygame.surfarray.make_surface(array.swapaxes(0, 1)) self._display.blit(surface, (0, 0)) if self._mini_view_image1 is not None: array = image_converter.depth_to_logarithmic_grayscale( self._mini_view_image1) surface = pygame.surfarray.make_surface(array.swapaxes(0, 1)) self._display.blit(surface, (gap_x, mini_image_y)) if self._mini_view_image2 is not None: array = image_converter.labels_to_cityscapes_palette( self._mini_view_image2) surface = pygame.surfarray.make_surface(array.swapaxes(0, 1)) self._display.blit(surface, (2 * gap_x + MINI_WINDOW_WIDTH, mini_image_y)) if self._lidar_measurement is not None: lidar_data = np.array(self._lidar_measurement.data[:, :2]) lidar_data *= 2.0 lidar_data += 100.0 lidar_data = np.fabs(lidar_data) lidar_data = lidar_data.astype(np.int32) lidar_data = np.reshape(lidar_data, (-1, 2)) #draw lidar lidar_img_size = (200, 200, 3) lidar_img = np.zeros(lidar_img_size) lidar_img[tuple(lidar_data.T)] = (255, 255, 255) surface = pygame.surfarray.make_surface(lidar_img) self._display.blit(surface, (10, 10)) if self._map_view is not None: array = self._map_view array = array[:, :, :3] new_window_width = \ (float(WINDOW_HEIGHT) / float(self._map_shape[0])) * \ float(self._map_shape[1]) surface = pygame.surfarray.make_surface(array.swapaxes(0, 1)) w_pos = int(self._position[0] * (float(WINDOW_HEIGHT) / float(self._map_shape[0]))) h_pos = int(self._position[1] * (new_window_width / float(self._map_shape[1]))) pygame.draw.circle(surface, [255, 0, 0, 255], (w_pos, h_pos), 6, 0) for agent in self._agent_positions: if agent.HasField('vehicle'): agent_position = self._map.convert_to_pixel([ agent.vehicle.transform.location.x, agent.vehicle.transform.location.y, agent.vehicle.transform.location.z ]) w_pos = int( agent_position[0] * (float(WINDOW_HEIGHT) / float(self._map_shape[0]))) h_pos = int(agent_position[1] * (new_window_width / float(self._map_shape[1]))) pygame.draw.circle(surface, [255, 0, 255, 255], (w_pos, h_pos), 4, 0) if agent.HasField('pedestrian'): agent_position = self._map.convert_to_pixel([ agent.pedestrian.transform.location.x, agent.pedestrian.transform.location.y, agent.pedestrian.transform.location.z ]) w_pos = int( agent_position[0] * (float(WINDOW_HEIGHT) / float(self._map_shape[0]))) h_pos = int(agent_position[1] * (new_window_width / float(self._map_shape[1]))) pygame.draw.circle(surface, [0, 0, 0, 0], (w_pos, h_pos), 4, 0) self._display.blit(surface, (WINDOW_WIDTH, 0)) pygame.display.flip()
def _on_render(self, meas): gap_x = (WINDOW_WIDTH - 2 * MINI_WINDOW_WIDTH) / 3 mini_image_y = WINDOW_HEIGHT - MINI_WINDOW_HEIGHT - gap_x #meas.pl if self._main_image is not None: array = image_converter.to_rgb_array(self._main_image) surface = pygame.surfarray.make_surface(array.swapaxes(0, 1)) self._display.blit(surface, (0, 0)) if mixed_reality: if self._mini_view_image2 is not None: array1 = image_converter.labels_to_cityscapes_palette( self._mini_view_image2) surface1 = pygame.surfarray.make_surface(array1.swapaxes(0, 1)) surface1.set_alpha(128) self._display.blit(surface1, (0, 0)) else: if self._mini_view_image1 is not None: array = image_converter.depth_to_logarithmic_grayscale( self._mini_view_image1) surface = pygame.surfarray.make_surface(array.swapaxes(0, 1)) self._display.blit(surface, (gap_x, mini_image_y)) if self._mini_view_image2 is not None: array = image_converter.labels_to_cityscapes_palette( self._mini_view_image2) surface = pygame.surfarray.make_surface(array.swapaxes(0, 1)) self._display.blit( surface, (2 * gap_x + MINI_WINDOW_WIDTH, mini_image_y)) myfont = pygame.font.SysFont("sans", 25, bold=True) label = myfont.render(("Current Speed: %f km/h" % meas.player_measurements.forward_speed), 1, (255, 255, 0)) self._display.blit(label, (WINDOW_WIDTH / 2, WINDOW_HEIGHT - 40)) neighborAgents = [agent for agent in meas.non_player_agents if scipy.spatial.distance.cdist([[meas.player_measurements.transform.location.x, \ meas.player_measurements.transform.location.y]], \ [[agent.vehicle.transform.location.x,agent.vehicle.transform.location.y]]) < 50e2] myfont = pygame.font.SysFont("sans", 20, bold=True) for agent in neighborAgents: if agent.HasField('vehicle'): # print(agent.vehicle.transform.location) label = myfont.render(("Vehicle nearby"), 1, (255, 0, 255)) self._display.blit(label, (WINDOW_WIDTH / 2, 40)) if agent.HasField('pedestrian'): # print(agent.vehicle.transform.location) label = myfont.render(("Pedestrian nearby"), 1, (255, 0, 255)) self._display.blit(label, (WINDOW_WIDTH / 2, 80)) if self._map_view is not None: array = self._map_view array = array[:, :, :3] new_window_width = (float(WINDOW_HEIGHT) / float( self._map_shape[0])) * float(self._map_shape[1]) surface = pygame.surfarray.make_surface(array.swapaxes(0, 1)) w_pos = int(self._position[0] * (float(WINDOW_HEIGHT) / float(self._map_shape[0]))) h_pos = int(self._position[1] * (new_window_width / float(self._map_shape[1]))) pygame.draw.circle(surface, [255, 0, 0, 255], (w_pos, h_pos), 6, 0) for agent in self._agent_positions: if agent.HasField('vehicle'): print("found non-player agent") agent_position = self._map.get_position_on_map([ agent.vehicle.transform.location.x, agent.vehicle.transform.location.y, agent.vehicle.transform.location.z ]) w_pos = int( agent_position[0] * (float(WINDOW_HEIGHT) / float(self._map_shape[0]))) h_pos = int(agent_position[1] * (new_window_width / float(self._map_shape[1]))) pygame.draw.circle(surface, [255, 0, 255, 255], (w_pos, h_pos), 4, 0) self._display.blit(surface, (WINDOW_WIDTH, 0)) pygame.display.flip()
def _on_render(self): gap_x = (WINDOW_WIDTH - 2 * MINI_WINDOW_WIDTH) / 3 mini_image_y = WINDOW_HEIGHT - MINI_WINDOW_HEIGHT - gap_x if self._main_image is not None: array = image_converter.to_rgb_array(self._main_image) surface = pygame.surfarray.make_surface(array.swapaxes(0, 1)) self._display.blit(surface, (0, 0)) if self._mini_view_image1 is not None: array = image_converter.depth_to_logarithmic_grayscale(self._mini_view_image1) surface = pygame.surfarray.make_surface(array.swapaxes(0, 1)) self._display.blit(surface, (gap_x, mini_image_y)) if self._mini_view_image2 is not None: array = image_converter.labels_to_cityscapes_palette( self._mini_view_image2) surface = pygame.surfarray.make_surface(array.swapaxes(0, 1)) self._display.blit( surface, (2 * gap_x + MINI_WINDOW_WIDTH, mini_image_y)) if self._lidar_measurement is not None: lidar_points = np.array(self._lidar_measurement.data['points'][:, :, :2]) lidar_points /= 50.0 lidar_points += 100.0 lidar_points = np.fabs(lidar_points) lidar_points = lidar_points.astype(np.int32) lidar_points = np.reshape(lidar_points, (-1, 2)) lidar_points_labels = self._lidar_measurement.data['labels'] lidar_points_labels = np.reshape(lidar_points_labels, (-1)) #draw lidar lidar_img_size = (200, 200, 3) lidar_img = np.zeros(lidar_img_size) for class_id, color in image_converter.semantic_segmentation_classes_to_colors.items(): lidar_img[tuple(lidar_points[lidar_points_labels == class_id].T)] = color surface = pygame.surfarray.make_surface( lidar_img ) self._display.blit(surface, (10, 10)) if self._map_view is not None: array = self._map_view array = array[:, :, :3] new_window_width =(float(WINDOW_HEIGHT)/float(self._map_shape[0]))*float(self._map_shape[1]) surface = pygame.surfarray.make_surface(array.swapaxes(0, 1)) w_pos = int(self._position[0]*(float(WINDOW_HEIGHT)/float(self._map_shape[0]))) h_pos =int(self._position[1] *(new_window_width/float(self._map_shape[1]))) pygame.draw.circle(surface, [255, 0, 0, 255], (w_pos,h_pos), 6, 0) for agent in self._agent_positions: if agent.HasField('vehicle'): agent_position = self._map.get_position_on_map([ agent.vehicle.transform.location.x, agent.vehicle.transform.location.y, agent.vehicle.transform.location.z]) w_pos = int(agent_position[0]*(float(WINDOW_HEIGHT)/float(self._map_shape[0]))) h_pos =int(agent_position[1] *(new_window_width/float(self._map_shape[1]))) pygame.draw.circle(surface, [255, 0, 255, 255], (w_pos,h_pos), 4, 0) self._display.blit(surface, (WINDOW_WIDTH, 0)) pygame.display.flip()
def _on_render(self): measurements, sensor_data = self.client.read_data() temp_array = np.zeros((240, 320, 1)) gap_x = (WINDOW_WIDTH - 2 * MINI_WINDOW_WIDTH) / 3 mini_image_y = WINDOW_HEIGHT - MINI_WINDOW_HEIGHT - gap_x # array = image_converter.labels_to_cityscapes_palette(self._mini_view_image2) # sem_return = array # print("sem return is " + str(np.shape(np.array(sem_return)))) if self._main_image is not None: array = image_converter.to_rgb_array(self._main_image) surface = pygame.surfarray.make_surface(array.swapaxes(0, 1)) self._display.blit(surface, (0, 0)) if self._mini_view_image1 is not None: array = image_converter.depth_to_logarithmic_grayscale( self._mini_view_image1) surface = pygame.surfarray.make_surface(array.swapaxes(0, 1)) self._display.blit(surface, (gap_x, mini_image_y)) if self._mini_view_image2 is not None: # print("Entered") array = image_converter.labels_to_cityscapes_palette( self._mini_view_image2) # temp_array = array[:,:,0] # plt.imshow(temp_array[:,:,0]) if self._enable_imitation: print("Size of input is " + str(np.shape(np.asarray(array)))) control = measurements.player_measurements.autopilot_control # img = np.asarray(array[:,:,0]) img = array # np.reshape(img, img.shape + (1,)) print("Shape of img is " + str(np.shape(img))) # plt.imshow(img[:,:,0]) # plt.show() img = cv2.resize(img[:, :, :], (320, 240)) img = img.reshape(1, 240, 320, 3) print("Shape of img is " + str(np.shape(img))) img = img[:, :, :, 0] img = img.reshape(1, 240, 320, 1) print("Shape of img is " + str(np.shape(img))) plt.imshow(img[0, :, :, 0]) # # try: control.steer = float( self.model.predict(img[:, :, :], batch_size=1)) / 10.0 control.throttle = 0.5 # # except: # # control.steer = 0.0 print("Steer is " + str(control.steer)) self.client.send_control(control) surface = pygame.surfarray.make_surface(array.swapaxes(0, 1)) self._display.blit(surface, (2 * gap_x + MINI_WINDOW_WIDTH, mini_image_y)) if self._lidar_measurement is not None: lidar_data = np.array(self._lidar_measurement.data[:, :2]) lidar_data *= 2.0 lidar_data += 100.0 lidar_data = np.fabs(lidar_data) lidar_data = lidar_data.astype(np.int32) lidar_data = np.reshape(lidar_data, (-1, 2)) #draw lidar lidar_img_size = (200, 200, 3) lidar_img = np.zeros(lidar_img_size) lidar_img[tuple(lidar_data.T)] = (255, 255, 255) surface = pygame.surfarray.make_surface(lidar_img) self._display.blit(surface, (10, 10)) if self._map_view is not None: array = self._map_view array = array[:, :, :3] new_window_width = \ (float(WINDOW_HEIGHT) / float(self._map_shape[0])) * \ float(self._map_shape[1]) surface = pygame.surfarray.make_surface(array.swapaxes(0, 1)) w_pos = int(self._position[0] * (float(WINDOW_HEIGHT) / float(self._map_shape[0]))) h_pos = int(self._position[1] * (new_window_width / float(self._map_shape[1]))) pygame.draw.circle(surface, [255, 0, 0, 255], (w_pos, h_pos), 6, 0) for agent in self._agent_positions: if agent.HasField('vehicle'): agent_position = self._map.convert_to_pixel([ agent.vehicle.transform.location.x, agent.vehicle.transform.location.y, agent.vehicle.transform.location.z ]) w_pos = int( agent_position[0] * (float(WINDOW_HEIGHT) / float(self._map_shape[0]))) h_pos = int(agent_position[1] * (new_window_width / float(self._map_shape[1]))) pygame.draw.circle(surface, [255, 0, 255, 255], (w_pos, h_pos), 4, 0) self._display.blit(surface, (WINDOW_WIDTH, 0)) pygame.display.flip()
fonte = pygame.font.SysFont("Arial", 28) white = (255, 255, 255) texto = fonte.render(texto, 1, white) display.blit(texto, (10, 40)) # Sensor Data: imagem_camera1 = sensor_data.get("Camera1", None) if imagem_camera1 is not None: array = image_converter.to_rgb_array(imagem_camera1) surface = pygame.surfarray.make_surface( array.swapaxes(0, 1)) display.blit(surface, (0, 70)) imagem_camera2 = sensor_data.get("Camera2", None) if imagem_camera2 is not None: array = image_converter.labels_to_cityscapes_palette( imagem_camera2) surface = pygame.surfarray.make_surface( array.swapaxes(0, 1)) display.blit(surface, (500, 70)) imagem_camera3 = sensor_data.get("Camera3", None) if imagem_camera3 is not None: array = image_converter.depth_to_logarithmic_grayscale( imagem_camera3) surface = pygame.surfarray.make_surface( array.swapaxes(0, 1)) display.blit(surface, (0, 570)) lidar_measurement = sensor_data.get('Lidar32', None) if lidar_measurement is not None: lidar_data = np.array(lidar_measurement.data[:, :2])