def _reset(self):
self.num_steps = 0
self.total_reward = 0
self.prev_measurement = None
self.prev_image = None
self.episode_id = datetime.today().strftime("%Y-%m-%d_%H-%M-%S_%f")
self.measurements_file = None
# Create a CarlaSettings object. This object is a wrapper around
# the CarlaSettings.ini file. Here we set the configuration we
# want for the new episode.
settings = CarlaSettings()
self.scenario = random.choice(self.config["scenarios"])
assert self.scenario["city"] == self.city, (self.scenario, self.city)
self.weather = random.choice(self.scenario["weather_distribution"])
settings.set(
SynchronousMode=True,
SendNonPlayerAgentsInfo=True,
NumberOfVehicles=self.scenario["num_vehicles"],
NumberOfPedestrians=self.scenario["num_pedestrians"],
WeatherId=self.weather)
settings.randomize_seeds()
if self.config["use_depth_camera"]:
camera1 = Camera("CameraDepth", PostProcessing="Depth")
camera1.set_image_size(self.config["render_x_res"],
self.config["render_y_res"])
camera1.set_position(30, 0, 130)
settings.add_sensor(camera1)
camera2 = Camera("CameraRGB")
camera2.set_image_size(self.config["render_x_res"],
self.config["render_y_res"])
camera2.set_position(30, 0, 130)
settings.add_sensor(camera2)
# Setup start and end positions
scene = self.client.load_settings(settings)
positions = scene.player_start_spots
self.start_pos = positions[self.scenario["start_pos_id"]]
self.end_pos = positions[self.scenario["end_pos_id"]]
self.start_coord = [
self.start_pos.location.x // 100, self.start_pos.location.y // 100
]
self.end_coord = [
self.end_pos.location.x // 100, self.end_pos.location.y // 100
]
print("Start pos {} ({}), end {} ({})".format(
self.scenario["start_pos_id"], self.start_coord,
self.scenario["end_pos_id"], self.end_coord))
# Notify the server that we want to start the episode at the
# player_start index. This function blocks until the server is ready
# to start the episode.
print("Starting new episode...")
self.client.start_episode(self.scenario["start_pos_id"])
image, py_measurements = self._read_observation()
self.prev_measurement = py_measurements
return self.encode_obs(self.preprocess_image(image), py_measurements)
def run_carla_client(args):
with make_connection(CarlaClient, args.host, args.port, timeout=15) as client:
logging.info('CarlaClient connected')
filename = '_images/episode_{:0>3d}/image_{:0>5d}.png'
frames_per_episode = 300
episode = 0
while True:
episode += 1
settings = CarlaSettings()
settings.set(SendNonPlayerAgentsInfo=True,SynchronousMode=args.synchronous)
settings.randomize_seeds()
camera = Camera('DefaultCamera')
camera.set_image_size(300, 200) # Do not change this, hard-coded in test.
settings.add_camera(camera)
logging.debug('sending CarlaSettings:\n%s', settings)
logging.info('new episode requested')
scene = client.request_new_episode(settings)
number_of_player_starts = len(scene.player_start_spots)
player_start = random.randint(0, max(0, number_of_player_starts - 1))
logging.info(
'start episode at %d/%d player start (%d frames)',
player_start,
number_of_player_starts,
frames_per_episode)
client.start_episode(player_start)
use_ai_control = (random.random() < 0.5)
reverse = (random.random() < 0.2)
for frame in range(0, frames_per_episode):
logging.debug('reading measurements...')
measurements, images = client.read_measurements()
logging.debug('received data of %d agents', len(measurements.non_player_agents))
assert len(images) == 1
assert (images[0].width, images[0].height) == (camera.ImageSizeX, camera.ImageSizeY)
if args.images_to_disk:
images[0].save_to_disk(filename.format(episode, frame))
logging.debug('sending control...')
control = measurements.player_measurements.ai_control
if not use_ai_control:
control.steer = random.uniform(-1.0, 1.0)
control.throttle = 0.3
control.hand_brake = False
control.reverse = reverse
client.send_control(
steer=control.steer,
throttle=control.throttle,
brake=control.brake,
hand_brake=control.hand_brake,
reverse=control.reverse)
def build_experiments(self):
"""
Creates the whole set of experiment objects,
The experiments created depends on the selected Town.
"""
# We check the town, based on that we define the town related parameters
# The size of the vector is related to the number of tasks, inside each
# task there is also multiple poses ( start end, positions )
if self._city_name == 'Town01':
poses_tasks = [[[7, 3]], [[138, 17]], [[140, 134]], [[140, 134]]]
vehicles_tasks = [0, 0, 0, 20]
pedestrians_tasks = [0, 0, 0, 50]
else:
poses_tasks = [[[4, 2]], [[37, 76]], [[19, 66]], [[19, 66]]]
vehicles_tasks = [0, 0, 0, 15]
pedestrians_tasks = [0, 0, 0, 50]
# We set the camera
# This single RGB camera is used on every experiment
camera = Camera('CameraRGB')
camera.set(FOV=100)
camera.set_image_size(800, 600)
camera.set_position(2.0, 0.0, 1.4)
camera.set_rotation(-15.0, 0, 0)
# Based on the parameters, creates a vector with experiment objects.
experiments_vector = []
for weather in self.weathers:
for iteration in range(len(poses_tasks)):
poses = poses_tasks[iteration]
vehicles = vehicles_tasks[iteration]
pedestrians = pedestrians_tasks[iteration]
conditions = CarlaSettings()
conditions.set(
SendNonPlayerAgentsInfo=True,
NumberOfVehicles=vehicles,
NumberOfPedestrians=pedestrians,
WeatherId=weather
)
# Add all the cameras that were set for this experiments
conditions.add_sensor(camera)
experiment = Experiment()
experiment.set(
Conditions=conditions,
Poses=poses,
Task=iteration,
Repetitions=1
)
experiments_vector.append(experiment)
return experiments_vector
def run(self):
settings = CarlaSettings()
settings.set(
SynchronousMode=True,
SendNonPlayerAgentsInfo=True,
NumberOfVehicles=60,
NumberOfPedestrians=90)
settings.add_camera(Camera('DefaultCamera'))
self.run_carla_client(settings, 3, 100)
def build_experiments(self):
"""
Creates the whole set of experiment objects,
The experiments created depend on the selected Town.
"""
# We set the camera
# This single RGB camera is used on every experiment
camera = Camera('CameraRGB')
camera.set(FOV=100)
camera.set_image_size(800, 600)
camera.set_position(2.0, 0.0, 1.4)
camera.set_rotation(-15.0, 0, 0)
if self._city_name == 'Town01':
poses_tasks = self._poses_town01()
vehicles_tasks = [0, 0, 0, 20]
pedestrians_tasks = [0, 0, 0, 50]
else:
poses_tasks = self._poses_town02()
vehicles_tasks = [0, 0, 0, 15]
pedestrians_tasks = [0, 0, 0, 50]
experiments_vector = []
for weather in self.weathers:
for iteration in range(len(poses_tasks)):
poses = poses_tasks[iteration]
vehicles = vehicles_tasks[iteration]
pedestrians = pedestrians_tasks[iteration]
conditions = CarlaSettings()
conditions.set(
SendNonPlayerAgentsInfo=True,
NumberOfVehicles=vehicles,
NumberOfPedestrians=pedestrians,
WeatherId=weather
)
# Add all the cameras that were set for this experiments
conditions.add_sensor(camera)
experiment = Experiment()
experiment.set(
Conditions=conditions,
Poses=poses,
Task=iteration,
Repetitions=1
)
experiments_vector.append(experiment)
return experiments_vector
def make_carla_settings(args):
"""Make a CarlaSettings object with the settings we need."""
settings = CarlaSettings()
settings.set(
SynchronousMode=True,
SendNonPlayerAgentsInfo=False,
NumberOfVehicles=30,
NumberOfPedestrians=0,
WeatherId=1,
QualityLevel='Epic')
settings.randomize_seeds()
camera0 = sensor.Camera('CameraRGB')
camera0.set_image_size(CAM_WINDOW_WIDTH, CAM_WINDOW_HEIGHT)
camera0.set_position(2.0, 0.0, 1.4)
camera0.set_rotation(-15.0, 0.0, 0.0)
settings.add_sensor(camera0)
camera1 = sensor.Camera('CameraDepth', PostProcessing='Depth')
camera1.set_image_size(MINI_WINDOW_WIDTH, MINI_WINDOW_HEIGHT)
camera1.set_position(2.0, 0.0, 1.4)
camera1.set_rotation(0.0, 0.0, 0.0)
settings.add_sensor(camera1)
camera2 = sensor.Camera('CameraSemSeg', PostProcessing='SemanticSegmentation')
camera2.set_image_size(MINI_WINDOW_WIDTH, MINI_WINDOW_HEIGHT)
camera2.set_position(2.0, 0.0, 1.4)
camera2.set_rotation(0.0, 0.0, 0.0)
settings.add_sensor(camera2)
lidar = sensor.Lidar('Lidar32')
lidar.set_position(0, 0, 2.5)
lidar.set_rotation(0, 0, 0)
lidar.set(
Channels=0,
Range=50,
PointsPerSecond=100000,
RotationFrequency=10,
UpperFovLimit=0,
LowerFovLimit=0)
settings.add_sensor(lidar)
return settings
def run_carla_client(args):
# Here we will run 3 episodes with 300 frames each.
number_of_episodes = 15
frames_per_episode = 10030
# [0 , 1 , 2 , 3 , 4 , 5 , 6 , 7, 8 , 9 , 10, 11, 12, 13, 14]
# vehicles_num = [60, 60, 70, 50, 60, 60, 80, 60, 60, 60, 50, 70, 60, 50, 50]
vehicles_num = [35, 35, 30, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35]
# vehicles_num = [60, 60, 70, 50, 60, 60, 80, 60, 60, 60, 50, 70, 60, 50, 50]
# We assume the CARLA server is already waiting for a client to connect at
# host:port. To create a connection we can use the `make_carla_client`
# context manager, it creates a CARLA client object and starts the
# connection. It will throw an exception if something goes wrong. The
# context manager makes sure the connection is always cleaned up on exit.
with make_carla_client(args.host, args.port) as client:
print('CarlaClient connected')
# list_of_episodes = [11]
list_of_episodes = [9, 11, 14]
# list_of_episodes = [0, 1, 2, 3, 4, 5, 6, 7, 9, 11, 14]
# list_of_episodes = [6, 7, 9, 11, 14]
# list_of_episodes = [2, 3, 4, 5, 6, 7, 9, 11, 14]
# list_of_episodes = [2, 7]
# list_of_episodes = [5, 6, 7, 11, 14]
# list_of_episodes = [6, 11]
# list_of_episodes = [7]
#list(range(0, number_of_episodes))
#list_of_episodes.pop(13)
#list_of_episodes.pop(12)
#list_of_episodes.pop(10)
#list_of_episodes.pop(8)
for episode in list_of_episodes:
# Start a new episode.
if args.settings_filepath is None:
# Create a CarlaSettings object. This object is a wrapper around
# the CarlaSettings.ini file. Here we set the configuration we
# want for the new episode.
settings = CarlaSettings()
settings.set(
SynchronousMode=True,
SendNonPlayerAgentsInfo=False,
NumberOfVehicles= vehicles_num[episode],#random.choice([0, 20, 15, 20, 25, 21, 24, 18, 40, 35, 25, 30]), #25,
NumberOfPedestrians=25,
DisableTwoWheeledVehicles=False,
WeatherId= episode, #1, #random.choice([1, 3, 7, 8, 14]),
QualityLevel=args.quality_level)
settings.randomize_seeds()
# Now we want to add a couple of cameras to the player vehicle.
# We will collect the images produced by these cameras every
# frame.
#### Cameras aligned across the y-axis
#### Horizontally shifted in the following Range
# [-1.62, -1.08, -0.54, 0.0, 0.54, 1.08, 1.62]
# LEFT RGB CAMERA
# y_locs = [-1.62, -1.08, -0.54, 0.0, 0.54, 1.08, 1.62]
# x_locs = [1.3, 1.84, 2.38, 2.92, 3.46, 4.0, 4.54]
# y_locs_left = [-1.08, -0.54, 0.0, 0.54, 1.08]
y_locs_left = -1.08
x_locs_left = [1.84, 2.38, 2.92]
LeftSideCameras = {}
for i,x_position in enumerate(x_locs_left):
# COLOR
camera_rgb = Camera('LeftSideCameras{0}RGB'.format(i),
PostProcessing='SceneFinal')
camera_rgb.set_image_size(800, 600)
camera_rgb.set_position(x_position, y_locs_left, 1.50)
camera_rgb.set_rotation(0, -90.0, 0)
LeftSideCameras['LeftSideCameras{0}RGB'.format(i)] = camera_rgb
settings.add_sensor(camera_rgb)
# DEPTH
camera_depth = Camera('LeftSideCameras{0}Depth'.format(i),
PostProcessing='Depth')
camera_depth.set_image_size(800, 600)
camera_depth.set_position(x_position, y_locs_left, 1.50)
camera_depth.set_rotation(0, -90.0, 0)
LeftSideCameras['LeftSideCameras{0}Depth'.format(i)] = camera_depth
settings.add_sensor(camera_depth)
# SEGMENTATION
camera_seg = Camera('LeftSideCameras{0}Seg'.format(i),
PostProcessing='SemanticSegmentation')
camera_seg.set_image_size(800, 600)
camera_seg.set_position(x_position, y_locs_left, 1.50)
camera_seg.set_rotation(0, -90.0, 0)
LeftSideCameras['LeftSideCameras{0}Seg'.format(i)] = camera_seg
settings.add_sensor(camera_seg)
y_locs_right = 1.08
x_locs_right = [1.84, 2.38, 2.92]
RightSideCameras = {}
for i,x_position in enumerate(x_locs_right):
# COLOR
camera_rgb = Camera('RightSideCameras{0}RGB'.format(i),
PostProcessing='SceneFinal')
camera_rgb.set_image_size(800, 600)
camera_rgb.set_position(x_position, y_locs_right, 1.50)
camera_rgb.set_rotation(0, 90.0, 0)
RightSideCameras['RightSideCameras{0}RGB'.format(i)] = camera_rgb
settings.add_sensor(camera_rgb)
# DEPTH
camera_depth = Camera('RightSideCameras{0}Depth'.format(i),
PostProcessing='Depth')
camera_depth.set_image_size(800, 600)
camera_depth.set_position(x_position, y_locs_right, 1.50)
camera_depth.set_rotation(0, 90.0, 0)
RightSideCameras['RightSideCameras{0}Depth'.format(i)] = camera_depth
settings.add_sensor(camera_depth)
# SEGMENTATION
camera_seg = Camera('RightSideCameras{0}Seg'.format(i),
PostProcessing='SemanticSegmentation')
camera_seg.set_image_size(800, 600)
camera_seg.set_position(x_position, y_locs_right, 1.50)
camera_seg.set_rotation(0, 90.0, 0)
RightSideCameras['RightSideCameras{0}Seg'.format(i)] = camera_seg
settings.add_sensor(camera_seg)
else:
with open(args.settings_filepath, 'r') as fp:
settings = fp.read()
scene = client.load_settings(settings)
# Choose one player start at random.
number_of_player_starts = len(scene.player_start_spots)
player_start = random.randint(0, max(0, number_of_player_starts - 1))
# Notify the server that we want to start the episode at the
# player_start index. This function blocks until the server is ready
# to start the episode.
print('Starting new episode...')
client.start_episode(player_start)
LeftSideCameras_to_car = []
for i in range(len(x_locs_right)):
LeftSideCameras_to_car.append(LeftSideCameras['LeftSideCameras{0}RGB'.format(i)].get_transform())
RightSideCameras_to_car = []
for i in range(len(x_locs_right)):
RightSideCameras_to_car.append(RightSideCameras['RightSideCameras{0}RGB'.format(i)].get_transform())
# camera_90_p_l_to_car_transform = camera_90_p_l.get_transform()
# camera_90_p_r_to_car_transform = camera_90_p_r.get_transform()
# Create a folder for saving episode data
if not os.path.isdir("/data/teddy/Datasets/carla_left_and_right/Town2/episode_{:0>5d}".format(episode)):
os.makedirs("/data/teddy/Datasets/carla_left_and_right/Town2/episode_{:0>5d}".format(episode))
# Iterate every frame in the episode.
for frame in range(0, frames_per_episode):
# Read the data produced by the server this frame.
measurements, sensor_data = client.read_data()
# player_measurements = measurements.player_measurements
world_transform = Transform(measurements.player_measurements.transform)
# Compute the final transformation matrix.
LeftSideCameras_to_world = [world_transform*cam_to_car \
for cam_to_car in LeftSideCameras_to_car]
RightSideCameras_to_world = [world_transform*cam_to_car \
for cam_to_car in RightSideCameras_to_car]
# for i in range(len()):
# LeftSideCameras_to_world.append()
# RightSideCameras_to_world.append(world_transform * RightSideCameras_to_car[i])
# Save the images to disk if requested.
if frame >= 30 and (frame % 2 == 0):
if args.save_images_to_disk:
for name, measurement in sensor_data.items():
filename = args.out_filename_format.format(episode, name, (frame-30)/2)
measurement.save_to_disk(filename)
# Save Transform matrix of each camera to separated files
for cam_num in range(len(x_locs_left)):
line = ""
filename = "{}episode_{:0>5d}/LeftSideCameras{}".format(args.root_path, episode, cam_num) + ".txt"
with open(filename, 'a+') as myfile:
for x in np.asarray(LeftSideCameras_to_world[cam_num].matrix[:3, :]).reshape(-1):
line += "{:.8e} ".format(x)
line = line[:-1]
line += "\n"
myfile.write(line)
line = ""
# Forward Cameras
# forward_cam_ids = list(range(len(x_locs_right)))
# forward_cam_ids.pop(mid_cam)
for i, cam_num in enumerate(x_locs_right):
line = ""
filename = "{}episode_{:0>5d}/RightSideCameras{}".format(args.root_path, episode, cam_num) + ".txt"
with open(filename, 'a+') as myfile:
for x in np.asarray(RightSideCameras_to_world[i].matrix[:3, :]).reshape(-1):
line += "{:.8e} ".format(x)
line = line[:-1]
line += "\n"
myfile.write(line)
line = ""
if not args.autopilot:
client.send_control(
steer=random.uniform(-1.0, 1.0),
throttle=0.5,
brake=0.0,
hand_brake=False,
reverse=False)
else:
# Together with the measurements, the server has sent the
# control that the in-game autopilot would do this frame. We
# can enable autopilot by sending back this control to the
# server. We can modify it if wanted, here for instance we
# will add some noise to the steer.
control = measurements.player_measurements.autopilot_control
#control.steer += random.uniform(-0.1, 0.1)
client.send_control(control)
class CarlaEnvironmentWrapper(EnvironmentWrapper):
def __init__(self, num_speedup_steps = 30, require_explicit_reset=True, is_render_enabled=False, early_termination_enabled=False, run_offscreen=False, cameras=['SceneFinal'], save_screens=False):
EnvironmentWrapper.__init__(self, is_render_enabled, save_screens)
self.episode_max_time = 1000000
self.allow_braking = True
self.log_path = 'logs/CarlaLogs.txt'
self.num_speedup_steps = num_speedup_steps
self.is_game_ready_for_input = False
self.run_offscreen = run_offscreen
self.kill_when_connection_lost = True
# server configuration
self.port = get_open_port()
self.host = 'localhost'
self.level = 'town2' #Why town2: https://github.com/carla-simulator/carla/issues/10#issuecomment-342483829
self.map = CarlaLevel().get(self.level)
# client configuration
self.verbose = True
self.observation = None
self.camera_settings = dict(
ImageSizeX=carla_config.server_width,
ImageSizeY=carla_config.server_height,
FOV=110.0,
PositionX=2.0, # 200 for Carla 0.7
PositionY=0.0,
PositionZ=1.40, # 140 for Carla 0.7
RotationPitch = 0.0,
RotationRoll = 0.0,
RotationYaw = 0.0,
)
self.rgb_camera_name = 'CameraRGB'
self.segment_camera_name = 'CameraSegment'
self.depth_camera_name = 'CameraDepth'
self.rgb_camera = 'SceneFinal' in cameras
self.segment_camera = 'SemanticSegmentation' in cameras
self.depth_camera = 'Depth' in cameras
self.class_grouping = carla_config.class_grouping or [(i, ) for i in range(carla_config.no_of_classes)]
self.autocolor_the_segments = False
self.color_the_depth_map = False
self.enable_coalesced_output = False
self.max_depth_value = 1.0 #255.0 for CARLA 7.0
self.min_depth_value = 0.0
self.config = None #'Environment/CarlaSettings.ini'
if self.config:
# load settings from file
with open(self.config, 'r') as fp:
self.settings = CarlaSettings(fp.read())
else:
# hard coded settings
#print("CarlaSettings.ini not found; using default settings")
self.settings = CarlaSettings()
self.settings.set(
SynchronousMode=True,
SendNonPlayerAgentsInfo=False,
NumberOfVehicles=15,
NumberOfPedestrians=30,
WeatherId=1,
SeedVehicles = 123456789,
SeedPedestrians = 123456789)
#self.settings.randomize_seeds()
# add cameras
if self.rgb_camera: self.settings.add_sensor(self.create_camera(self.rgb_camera_name, 'SceneFinal'))
if self.segment_camera: self.settings.add_sensor(self.create_camera(self.segment_camera_name, 'SemanticSegmentation'))
if self.depth_camera: self.settings.add_sensor(self.create_camera(self.depth_camera_name, 'Depth'))
self.car_speed = 0
self.is_game_setup = False # Will be true only when setup_client_and_server() is called, either explicitly, or by reset()
# action space
self.discrete_controls = True
self.action_space_size = 2
self.action_space_high = [1, 1]
self.action_space_low = [-1, -1]
self.action_space_abs_range = np.maximum(np.abs(self.action_space_low), np.abs(self.action_space_high))
self.steering_strength = 0.35
self.gas_strength = 1.0
self.brake_strength = 0.6
self.actions = {0: [0., 0.],
1: [0., -self.steering_strength],
2: [0., self.steering_strength],
3: [self.gas_strength-0.15, 0.],
4: [-self.brake_strength, 0],
5: [self.gas_strength-0.3, -self.steering_strength],
6: [self.gas_strength-0.3, self.steering_strength],
7: [-self.brake_strength, -self.steering_strength],
8: [-self.brake_strength, self.steering_strength]}
self.actions_description = ['NO-OP', 'TURN_LEFT', 'TURN_RIGHT', 'GAS', 'BRAKE',
'GAS_AND_TURN_LEFT', 'GAS_AND_TURN_RIGHT',
'BRAKE_AND_TURN_LEFT', 'BRAKE_AND_TURN_RIGHT']
for idx, action in enumerate(self.actions_description):
for key in key_map.keys():
if action == key:
self.key_to_action[key_map[key]] = idx
# measurements
self.measurements_size = (1,)
self.autopilot = None
self.kill_if_unmoved_for_n_steps = 15
self.unmoved_steps = 0.0
self.early_termination_enabled = early_termination_enabled
if self.early_termination_enabled:
self.max_neg_steps = 70
self.cur_neg_steps = 0
self.early_termination_punishment = 20.0
# env initialization
if not require_explicit_reset: self.reset(True)
# render
if self.automatic_render:
self.init_renderer()
if self.save_screens:
create_dir(self.images_path)
self.rgb_img_path = self.images_path+"/rgb/"
create_dir(self.rgb_img_path)
self.segmented_img_path = self.images_path+"/segmented/"
create_dir(self.segmented_img_path)
self.depth_img_path = self.images_path+"/depth/"
create_dir(self.depth_img_path)
def create_camera(self, camera_name, PostProcessing):
#camera = Camera('CameraRGB')
#camera.set_image_size(carla_config.server_width, carla_config.server_height)
#camera.set_position(200, 0, 140)
#camera.set_rotation(0, 0, 0)
#self.settings.add_sensor(camera)
camera = Camera(camera_name, **dict(self.camera_settings, PostProcessing=PostProcessing))
return camera
def setup_client_and_server(self, reconnect_client_only = False):
# open the server
if not reconnect_client_only:
self.server = self._open_server()
self.server_pid = self.server.pid # To kill incase child process gets lost
# open the client
self.game = CarlaClient(self.host, self.port, timeout=99999999)
self.game.connect(connection_attempts=100) #It's taking a very long time for the server process to spawn, so the client needs to wait or try sufficient no. of times lol
scene = self.game.load_settings(self.settings)
# get available start positions
positions = scene.player_start_spots
self.num_pos = len(positions)
self.iterator_start_positions = 0
self.is_game_setup = self.server and self.game
return
def close_client_and_server(self):
self._close_server()
print("Disconnecting the client")
self.game.disconnect()
self.game = None
self.server = None
self.is_game_setup = False
return
def _open_server(self):
# Note: There is no way to disable rendering in CARLA as of now
# https://github.com/carla-simulator/carla/issues/286
# decrease the window resolution if you want to see if performance increases
# Command: $CARLA_ROOT/CarlaUE4.sh /Game/Maps/Town02 -benchmark -carla-server -fps=15 -world-port=9876 -windowed -ResX=480 -ResY=360 -carla-no-hud
# To run off_screen: SDL_VIDEODRIVER=offscreen SDL_HINT_CUDA_DEVICE=0 <command> #https://github.com/carla-simulator/carla/issues/225
my_env = None
if self.run_offscreen:
my_env = {**os.environ, 'SDL_VIDEODRIVER': 'offscreen', 'SDL_HINT_CUDA_DEVICE':'0'}
with open(self.log_path, "wb") as out:
cmd = [path.join(environ.get('CARLA_ROOT'), 'CarlaUE4.sh'), self.map,
"-benchmark", "-carla-server", "-fps=20", "-world-port={}".format(self.port),
"-windowed -ResX={} -ResY={}".format(carla_config.server_width, carla_config.server_height),
"-carla-no-hud", "-quality-level=Low"]
if self.config:
cmd.append("-carla-settings={}".format(self.config))
p = subprocess.Popen(cmd, stdout=out, stderr=out, env=my_env)
return p
def _close_server(self):
if self.kill_when_connection_lost:
os.killpg(os.getpgid(self.server.pid), signal.SIGKILL)
return
no_of_attempts = 0
while is_process_alive(self.server_pid):
print("Trying to close Carla server with pid %d" % self.server_pid)
if no_of_attempts<5:
self.server.terminate()
elif no_of_attempts<10:
self.server.kill()
elif no_of_attempts<15:
os.kill(self.server_pid, signal.SIGTERM)
else:
os.kill(self.server_pid, signal.SIGKILL)
time.sleep(10)
no_of_attempts += 1
def check_early_stop(self, player_measurements, immediate_reward):
if player_measurements.intersection_offroad>0.95 or immediate_reward < -1 or (self.control.throttle == 0.0 and player_measurements.forward_speed < 0.1 and self.control.brake != 0.0):
self.cur_neg_steps += 1
early_done = (self.cur_neg_steps > self.max_neg_steps)
if early_done:
print("Early kill the mad car")
return early_done, self.early_termination_punishment
else:
self.cur_neg_steps /= 2 #Exponentially decay
return False, 0.0
def _update_state(self):
# get measurements and observations
measurements = []
while type(measurements) == list:
try:
measurements, sensor_data = self.game.read_data()
except:
# Connection between cli and server lost; reconnect
if self.kill_when_connection_lost: raise
print("Connection to server lost while reading state. Reconnecting...........")
self.close_client_and_server()
self.setup_client_and_server(reconnect_client_only=False)
self.done = True
self.location = (measurements.player_measurements.transform.location.x,
measurements.player_measurements.transform.location.y,
measurements.player_measurements.transform.location.z)
is_collision = measurements.player_measurements.collision_vehicles != 0 \
or measurements.player_measurements.collision_pedestrians != 0 \
or measurements.player_measurements.collision_other != 0
# CARLA doesn't recognize if collision occured and colliding speed is less than 5 km/h (Around 0.7 m/s)
# Ref: https://github.com/carla-simulator/carla/issues/13
# Recognize that as a collision
self.car_speed = measurements.player_measurements.forward_speed
if self.control.throttle > 0 and self.car_speed < 0.75 and self.control.brake==0.0 and self.is_game_ready_for_input:
self.unmoved_steps += 1.0
if self.unmoved_steps > self.kill_if_unmoved_for_n_steps:
is_collision = True
print("Car stuck somewhere lol")
elif self.unmoved_steps>0: self.unmoved_steps -= 0.50 #decay slowly, since it may be stuck and not accelerate few times
if is_collision: print("Collision occured")
speed_reward = self.car_speed - 1
if speed_reward > 30.:
speed_reward = 30.
self.reward = speed_reward*1.2 \
- (measurements.player_measurements.intersection_otherlane * (self.car_speed+1.5)*1.2) \
- (measurements.player_measurements.intersection_offroad * (self.car_speed+2.5)*1.5) \
- is_collision * 250 \
- np.abs(self.control.steer) * 2
# Scale down the reward by a factor
self.reward /= 10
if self.early_termination_enabled:
early_done, punishment = self.check_early_stop(measurements.player_measurements, self.reward)
if early_done:
self.done = True
self.reward -= punishment
# update measurements
self.observation = {
#'observation': sensor_data['CameraRGB'].data,
'acceleration': measurements.player_measurements.acceleration,
'forward_speed': measurements.player_measurements.forward_speed,
'intersection_otherlane': measurements.player_measurements.intersection_otherlane,
'intersection_offroad': measurements.player_measurements.intersection_offroad
}
if self.rgb_camera:
self.observation['rgb_image'] = sensor_data[self.rgb_camera_name].data
if self.segment_camera:
self.observation['segmented_image'] = sensor_data[self.segment_camera_name].data
if self.depth_camera:
self.observation['depth_map'] = sensor_data[self.depth_camera_name].data
if self.segment_camera and self.depth_camera and self.enable_coalesced_output:
self.observation['coalesced_data'] = coalesce_depth_and_segmentation(
self.observation['segmented_image'], self.class_grouping, self.observation['depth_map'], self.max_depth_value)
if self.segment_camera and (self.autocolor_the_segments or self.is_render_enabled):
self.observation['colored_segmented_image'] = convert_segmented_to_rgb(carla_config.colors_segment, self.observation['segmented_image'])
self.autopilot = measurements.player_measurements.autopilot_control
# action_p = ['%.2f' % member for member in [self.control.throttle, self.control.steer]]
# screen.success('REWARD: %.2f, ACTIONS: %s' % (self.reward, action_p))
if (measurements.game_timestamp >= self.episode_max_time) or is_collision:
# screen.success('EPISODE IS DONE. GameTime: {}, Collision: {}'.format(str(measurements.game_timestamp),
# str(is_collision)))
self.done = True
def _take_action(self, action_idx):
if not self.is_game_setup:
print("Reset the environment before calling step")
sys.exit(1)
if type(action_idx) == int:
action = self.actions[action_idx]
else:
action = action_idx
self.control = VehicleControl()
if self.car_speed>35.0 and action[0]>0:
action[0] -= 0.20*(self.car_speed/35.0)
self.control.throttle = np.clip(action[0], 0, 1)
self.control.steer = np.clip(action[1], -1, 1)
self.control.brake = np.abs(np.clip(action[0], -1, 0))
if not self.allow_braking:
self.control.brake = 0
self.control.hand_brake = False
self.control.reverse = False
controls_sent = False
while not controls_sent:
try:
self.game.send_control(self.control)
controls_sent = True
except:
if self.kill_when_connection_lost: raise
print("Connection to server lost while sending controls. Reconnecting...........")
self.close_client_and_server()
self.setup_client_and_server(reconnect_client_only=False)
self.done = True
return
def init_renderer(self):
self.num_cameras = 0
if self.rgb_camera: self.num_cameras += 1
if self.segment_camera: self.num_cameras += 1
if self.depth_camera: self.num_cameras += 1
self.renderer.create_screen(carla_config.render_width, carla_config.render_height*self.num_cameras)
def _restart_environment_episode(self, force_environment_reset=True):
if not force_environment_reset and not self.done and self.is_game_setup:
print("Can't reset dude, episode ain't over yet")
return None #User should handle this
self.is_game_ready_for_input = False
if not self.is_game_setup:
self.setup_client_and_server()
if self.is_render_enabled:
self.init_renderer()
else:
self.iterator_start_positions += 1
if self.iterator_start_positions >= self.num_pos:
self.iterator_start_positions = 0
try:
self.game.start_episode(self.iterator_start_positions)
except:
self.game.connect()
self.game.start_episode(self.iterator_start_positions)
self.unmoved_steps = 0.0
if self.early_termination_enabled:
self.cur_neg_steps = 0
# start the game with some initial speed
self.car_speed = 0
observation = None
for i in range(self.num_speedup_steps):
observation, reward, done, _ = self.step([1.0, 0])
self.observation = observation
self.is_game_ready_for_input = True
return observation
def get_rendered_image(self):
temp = []
if self.rgb_camera: temp.append(self.observation['rgb_image'])
if self.segment_camera:
temp.append(self.observation['colored_segmented_image'])
if self.depth_camera:
if self.color_the_depth_map: temp.append(depthmap_to_rgb(self.observation['depth_map']))
else: temp.append(depthmap_to_grey(self.observation['depth_map']))
return np.vstack((img for img in temp))
def save_screenshots(self):
if not self.save_screens:
print("save_screens is set False")
return
filename = str(int(time.time()*100))
if self.rgb_camera:
save_image(self.rgb_img_path+filename+".png", self.observation['rgb_image'])
if self.segment_camera:
np.save(self.segmented_img_path+filename, self.observation['segmented_image'])
if self.depth_camera:
save_depthmap_as_16bit_png(self.depth_img_path+filename+".png",self.observation['depth_map'],self.max_depth_value,0.95) #Truncating sky as 0
def run_carla_client(args):
# Here we will run 3 episodes with 300 frames each.
number_of_episodes = 3
frames_per_episode = 300
# We assume the CARLA server is already waiting for a client to connect at
# host:port. To create a connection we can use the `make_carla_client`
# context manager, it creates a CARLA client object and starts the
# connection. It will throw an exception if something goes wrong. The
# context manager makes sure the connection is always cleaned up on exit.
with make_carla_client(args.host, args.port) as client:
print('CarlaClient connected')
for episode in range(0, number_of_episodes):
# Start a new episode.
if args.settings_filepath is None:
# Create a CarlaSettings object. This object is a wrapper around
# the CarlaSettings.ini file. Here we set the configuration we
# want for the new episode.
settings = CarlaSettings()
settings.set(SynchronousMode=True,
SendNonPlayerAgentsInfo=True,
NumberOfVehicles=20,
NumberOfPedestrians=40,
WeatherId=random.choice([1, 3, 7, 8, 14]),
QualityLevel=args.quality_level)
settings.randomize_seeds()
# Now we want to add a couple of cameras to the player vehicle.
# We will collect the images produced by these cameras every
# frame.
# The default camera captures RGB images of the scene.
camera0 = Camera('CameraRGB')
# Set image resolution in pixels.
camera0.set_image_size(800, 600)
# Set its position relative to the car in meters.
camera0.set_position(0.30, 0, 1.30)
settings.add_sensor(camera0)
# Let's add another camera producing ground-truth depth.
camera1 = Camera('CameraDepth', PostProcessing='Depth')
camera1.set_image_size(800, 600)
camera1.set_position(0.30, 0, 1.30)
settings.add_sensor(camera1)
if args.lidar:
lidar = Lidar('Lidar32')
lidar.set_position(0, 0, 2.50)
lidar.set_rotation(0, 0, 0)
lidar.set(Channels=32,
Range=50,
PointsPerSecond=100000,
RotationFrequency=10,
UpperFovLimit=10,
LowerFovLimit=-30)
settings.add_sensor(lidar)
else:
# Alternatively, we can load these settings from a file.
with open(args.settings_filepath, 'r') as fp:
settings = fp.read()
# Now we load these settings into the server. The server replies
# with a scene description containing the available start spots for
# the player. Here we can provide a CarlaSettings object or a
# CarlaSettings.ini file as string.
scene = client.load_settings(settings)
# Choose one player start at random.
number_of_player_starts = len(scene.player_start_spots)
player_start = random.randint(0, max(0,
number_of_player_starts - 1))
# Notify the server that we want to start the episode at the
# player_start index. This function blocks until the server is ready
# to start the episode.
print('Starting new episode...')
client.start_episode(player_start)
# Iterate every frame in the episode.
for frame in range(0, frames_per_episode):
# Read the data produced by the server this frame.
measurements, sensor_data = client.read_data()
# Print some of the measurements.
print_measurements(measurements)
# Save the images to disk if requested.
if args.save_images_to_disk:
for name, measurement in sensor_data.items():
filename = args.out_filename_format.format(
episode, name, frame)
measurement.save_to_disk(filename)
# We can access the encoded data of a given image as numpy
# array using its "data" property. For instance, to get the
# depth value (normalized) at pixel X, Y
#
# depth_array = sensor_data['CameraDepth'].data
# value_at_pixel = depth_array[Y, X]
#
# Now we have to send the instructions to control the vehicle.
# If we are in synchronous mode the server will pause the
# simulation until we send this control.
if not args.autopilot:
client.send_control(steer=random.uniform(-1.0, 1.0),
throttle=0.5,
brake=0.0,
hand_brake=False,
reverse=False)
else:
# Together with the measurements, the server has sent the
# control that the in-game autopilot would do this frame. We
# can enable autopilot by sending back this control to the
# server. We can modify it if wanted, here for instance we
# will add some noise to the steer.
control = measurements.player_measurements.autopilot_control
control.steer += random.uniform(-0.1, 0.1)
client.send_control(control)
def run_carla_clients(args):
filename = '_images_repeatability/server{:d}/{:0>6d}.png'
with make_carla_client(args.host1, args.port1) as client1:
logging.info('1st client connected')
with make_carla_client(args.host2, args.port2) as client2:
logging.info('2nd client connected')
settings = CarlaSettings()
settings.set(
SynchronousMode=True,
SendNonPlayerAgentsInfo=True,
NumberOfVehicles=50,
NumberOfPedestrians=50,
WeatherId=random.choice([1, 3, 7, 8, 14]))
settings.randomize_seeds()
if args.images_to_disk:
camera = Camera('DefaultCamera')
camera.set_image_size(800, 600)
settings.add_sensor(camera)
scene1 = client1.load_settings(settings)
scene2 = client2.load_settings(settings)
number_of_player_starts = len(scene1.player_start_spots)
assert number_of_player_starts == len(scene2.player_start_spots)
player_start = random.randint(0, max(0, number_of_player_starts - 1))
logging.info(
'start episode at %d/%d player start (run forever, press ctrl+c to cancel)',
player_start,
number_of_player_starts)
client1.start_episode(player_start)
client2.start_episode(player_start)
frame = 0
while True:
frame += 1
meas1, sensor_data1 = client1.read_data()
meas2, sensor_data2 = client2.read_data()
player1 = meas1.player_measurements
player2 = meas2.player_measurements
images1 = [x for x in sensor_data1.values() if isinstance(x, Image)]
images2 = [x for x in sensor_data2.values() if isinstance(x, Image)]
control1 = player1.autopilot_control
control2 = player2.autopilot_control
try:
assert len(images1) == len(images2)
assert len(meas1.non_player_agents) == len(meas2.non_player_agents)
assert player1.transform.location.x == player2.transform.location.x
assert player1.transform.location.y == player2.transform.location.y
assert player1.transform.location.z == player2.transform.location.z
assert control1.steer == control2.steer
assert control1.throttle == control2.throttle
assert control1.brake == control2.brake
assert control1.hand_brake == control2.hand_brake
assert control1.reverse == control2.reverse
except AssertionError:
logging.exception('assertion failed')
if args.images_to_disk:
assert len(images1) == 1
images1[0].save_to_disk(filename.format(1, frame))
images2[0].save_to_disk(filename.format(2, frame))
client1.send_control(control1)
client2.send_control(control2)
def run_carla_client(args):
with make_connection(CarlaClient, args.host, args.port,
timeout=15) as client:
logging.info('CarlaClient connected')
filename = '_images/episode_{:0>3d}/image_{:0>5d}.png'
frames_per_episode = 300
episode = 0
while True:
episode += 1
settings = CarlaSettings()
settings.set(SendNonPlayerAgentsInfo=True,
SynchronousMode=args.synchronous)
settings.randomize_seeds()
camera = Camera('DefaultCamera')
camera.set_image_size(
300, 200) # Do not change this, hard-coded in test.
settings.add_sensor(camera)
logging.debug('sending CarlaSettings:\n%s', settings)
logging.info('new episode requested')
scene = client.load_settings(settings)
number_of_player_starts = len(scene.player_start_spots)
player_start = random.randint(0, max(0,
number_of_player_starts - 1))
logging.info('start episode at %d/%d player start (%d frames)',
player_start, number_of_player_starts,
frames_per_episode)
client.start_episode(player_start)
use_autopilot_control = (random.random() < 0.5)
reverse = (random.random() < 0.2)
for frame in range(0, frames_per_episode):
logging.debug('reading measurements...')
measurements, sensor_data = client.read_data()
images = [
x for x in sensor_data.values() if isinstance(x, Image)
]
logging.debug('received data of %d agents',
len(measurements.non_player_agents))
assert len(images) == 1
assert (images[0].width,
images[0].height) == (camera.ImageSizeX,
camera.ImageSizeY)
if args.images_to_disk:
images[0].save_to_disk(filename.format(episode, frame))
logging.debug('sending control...')
control = measurements.player_measurements.autopilot_control
if not use_autopilot_control:
control.steer = random.uniform(-1.0, 1.0)
control.throttle = 0.3
control.hand_brake = False
control.reverse = reverse
client.send_control(steer=control.steer,
throttle=control.throttle,
brake=control.brake,
hand_brake=control.hand_brake,
reverse=control.reverse)
def run_carla_client(args):
frames_per_episode = 27500 #2700
spline_points = 1724
report = {
'num_episodes':
args.num_episodes, #num of episodes for done on this run
'controller_name':
args.controller_name, # name of controller used in this run
'distances':
[], # distanc traveld by car in this controller successfully
'target_speed':
args.target_speed, # the taarget speed set to this controller
}
track_DF = pd.read_csv('racetrack{}.txt'.format(args.racetrack),
header=None)
#print(track_DF.head)
# The track waypoint data (trajectory to follow) are rescaled by 100x with relation to Carla measurements
track_DF = track_DF / 100.0
pts_2D = track_DF.loc[:, [0, 1]].values
tck, u = splprep(pts_2D.T, u=None, s=2.0, per=1, k=3)
u_new = np.linspace(u.min(), u.max(), spline_points)
x_new, y_new = splev(u_new, tck, der=0)
pts_2D = np.c_[x_new, y_new]
steer = 0.0
throttle = 0.5
depth_array = None
if args.controller_name == 'mpc':
weather_id = 2
controller = MPCController(args.target_speed)
elif args.controller_name == 'pd':
weather_id = 1
controller = PDController(args.target_speed)
elif args.controller_name == 'pad':
weather_id = 5
controller = PadController()
elif args.controller_name == 'auto':
weather_id = 5
controller = PsudoAutoController(args.target_speed)
elif args.controller_name == 'nn':
from neural_network_controller import NNController
weather_id = 1
controller = NNController(
args.target_speed,
args.model_dir_name,
args.which_model,
args.throttle_coeff_A,
args.throttle_coeff_B,
args.ensemble_prediction,
)
report['model_dir_name'] = args.model_dir_name
report['which_model'] = args.which_model
report['throttle_coeff_A'] = args.throttle_coeff_A
report['throttle_coeff_B'] = args.throttle_coeff_B
report['ensemble_prediction'] = args.ensemble_prediction
with make_carla_client(args.host, args.port) as client:
print('CarlaClient connected')
episode = 0
num_fails = 0
while episode < args.num_episodes:
# Start a new episode
if args.store_data:
depth_storage = np.zeros(
((IMAGE_CLIP_LOWER - IMAGE_CLIP_UPPER) // IMAGE_DECIMATION,
IMAGE_SIZE[1] // IMAGE_DECIMATION,
frames_per_episode)).astype(DTYPE)
log_dicts = frames_per_episode * [None]
else:
depth_storage = None
log_dicts = None
#if I'm not passing config file
if args.settings_filepath is None:
settings = CarlaSettings()
settings.set(SynchronousMode=True,
SendNonPlayerAgentsInfo=False,
NumberOfVehicles=1,
NumberOfPedestrians=0,
WeatherId=weather_id,
QualityLevel=args.quality_level)
# Now we will add a depth camera to the vehicle.
camera = Camera('CameraDepth',
PostProcessing='Depth',
FOV=69.4)
camera.set_image_size(IMAGE_SIZE[1], IMAGE_SIZE[0])
camera.set_position(2.30, 0, 1.30)
settings.add_sensor(camera)
else:
# Alternatively, we can load these settings from a file.
with open(args.settings_filepath, 'r') as fp:
settings = fp.read()
# Now we load these settings into the server.
scene = client.load_settings(settings)
# Choose one player start at random (we only have one player-car-).
num_of_player_starts = len(scene.player_start_spots)
player_start = random.randint(0, max(0, num_of_player_starts - 1))
# Notify the server that we want to start the episode at the player_start index.
# This function blocks until the server is ready to start the episode.
print('Starting new episode...', )
client.start_episode(player_start)
status, depth_storage, one_log_dict, log_dicts, distance_travelled = run_episode(
client, controller, pts_2D, depth_storage, log_dicts,
frames_per_episode, args.controller_name, args.store_data)
# In case of collision or stop episod fails and we don't sore data and restart it.
if 'FAIL' in status:
num_fails += 1
print(status)
continue
else:
print('SUCCESS: ' + str(episode))
report['distances'].append(distance_travelled)
if args.store_data:
np.save(
'depth_data/{}_racetrack{}_depth_data{}.npy'.format(
args.controller_name, args.racetrack, episode),
depth_storage)
pd.DataFrame(log_dicts).to_csv(
'logs/{}_racetrack{}_log{}.csv'.format(
args.controller_name, args.racetrack, episode),
index=False)
episode += 1
report['num_fails'] = num_fails
report_output = os.path.join('reports', args.report_filename)
pd.to_pickle(report, report_output)
def run_carla_clients(args):
filename = '_images_repeatability/server{:d}/{:0>6d}.png'
with make_carla_client(args.host1, args.port1) as client1:
logging.info('1st client connected')
with make_carla_client(args.host2, args.port2) as client2:
logging.info('2nd client connected')
settings = CarlaSettings()
settings.set(SynchronousMode=True,
SendNonPlayerAgentsInfo=True,
NumberOfVehicles=50,
NumberOfPedestrians=50,
WeatherId=random.choice([1, 3, 7, 8, 14]))
settings.randomize_seeds()
if args.images_to_disk:
camera = Camera('DefaultCamera')
camera.set_image_size(800, 600)
settings.add_sensor(camera)
scene1 = client1.load_settings(settings)
scene2 = client2.load_settings(settings)
number_of_player_starts = len(scene1.player_start_spots)
assert number_of_player_starts == len(scene2.player_start_spots)
player_start = random.randint(0, max(0,
number_of_player_starts - 1))
logging.info(
'start episode at %d/%d player start (run forever, press ctrl+c to cancel)',
player_start, number_of_player_starts)
client1.start_episode(player_start)
client2.start_episode(player_start)
frame = 0
while True:
frame += 1
meas1, sensor_data1 = client1.read_data()
meas2, sensor_data2 = client2.read_data()
player1 = meas1.player_measurements
player2 = meas2.player_measurements
images1 = [
x for x in sensor_data1.values() if isinstance(x, Image)
]
images2 = [
x for x in sensor_data2.values() if isinstance(x, Image)
]
control1 = player1.autopilot_control
control2 = player2.autopilot_control
try:
assert len(images1) == len(images2)
assert len(meas1.non_player_agents) == len(
meas2.non_player_agents)
assert player1.transform.location.x == player2.transform.location.x
assert player1.transform.location.y == player2.transform.location.y
assert player1.transform.location.z == player2.transform.location.z
assert control1.steer == control2.steer
assert control1.throttle == control2.throttle
assert control1.brake == control2.brake
assert control1.hand_brake == control2.hand_brake
assert control1.reverse == control2.reverse
except AssertionError:
logging.exception('assertion failed')
if args.images_to_disk:
assert len(images1) == 1
images1[0].save_to_disk(filename.format(1, frame))
images2[0].save_to_disk(filename.format(2, frame))
client1.send_control(control1)
client2.send_control(control2)
class CarlaEnvironment(Environment):
def __init__(self, level: LevelSelection,
seed: int, frame_skip: int, human_control: bool, custom_reward_threshold: Union[int, float],
visualization_parameters: VisualizationParameters,
server_height: int, server_width: int, camera_height: int, camera_width: int,
verbose: bool, experiment_suite: ExperimentSuite, config: str, episode_max_time: int,
allow_braking: bool, quality: CarlaEnvironmentParameters.Quality,
cameras: List[CameraTypes], weather_id: List[int], experiment_path: str,
separate_actions_for_throttle_and_brake: bool,
num_speedup_steps: int, max_speed: float, **kwargs):
super().__init__(level, seed, frame_skip, human_control, custom_reward_threshold, visualization_parameters)
# server configuration
self.server_height = server_height
self.server_width = server_width
self.port = get_open_port()
self.host = 'localhost'
self.map_name = CarlaLevel[level.upper()].value['map_name']
self.map_path = CarlaLevel[level.upper()].value['map_path']
self.experiment_path = experiment_path
# client configuration
self.verbose = verbose
self.quality = quality
self.cameras = cameras
self.weather_id = weather_id
self.episode_max_time = episode_max_time
self.allow_braking = allow_braking
self.separate_actions_for_throttle_and_brake = separate_actions_for_throttle_and_brake
self.camera_width = camera_width
self.camera_height = camera_height
# setup server settings
self.experiment_suite = experiment_suite
self.config = config
if self.config:
# load settings from file
with open(self.config, 'r') as fp:
self.settings = fp.read()
else:
# hard coded settings
self.settings = CarlaSettings()
self.settings.set(
SynchronousMode=True,
SendNonPlayerAgentsInfo=False,
NumberOfVehicles=15,
NumberOfPedestrians=30,
WeatherId=random.choice(force_list(self.weather_id)),
QualityLevel=self.quality.value,
SeedVehicles=seed,
SeedPedestrians=seed)
if seed is None:
self.settings.randomize_seeds()
self.settings = self._add_cameras(self.settings, self.cameras, self.camera_width, self.camera_height)
# open the server
self.server = self._open_server()
logging.disable(40)
# open the client
self.game = CarlaClient(self.host, self.port, timeout=99999999)
self.game.connect()
if self.experiment_suite:
self.current_experiment_idx = 0
self.current_experiment = self.experiment_suite.get_experiments()[self.current_experiment_idx]
self.scene = self.game.load_settings(self.current_experiment.conditions)
else:
self.scene = self.game.load_settings(self.settings)
# get available start positions
self.positions = self.scene.player_start_spots
self.num_positions = len(self.positions)
self.current_start_position_idx = 0
self.current_pose = 0
# state space
self.state_space = StateSpace({
"measurements": VectorObservationSpace(4, measurements_names=["forward_speed", "x", "y", "z"])
})
for camera in self.scene.sensors:
self.state_space[camera.name] = ImageObservationSpace(
shape=np.array([self.camera_height, self.camera_width, 3]),
high=255)
# action space
if self.separate_actions_for_throttle_and_brake:
self.action_space = BoxActionSpace(shape=3, low=np.array([-1, 0, 0]), high=np.array([1, 1, 1]),
descriptions=["steer", "gas", "brake"])
else:
self.action_space = BoxActionSpace(shape=2, low=np.array([-1, -1]), high=np.array([1, 1]),
descriptions=["steer", "gas_and_brake"])
# human control
if self.human_control:
# convert continuous action space to discrete
self.steering_strength = 0.5
self.gas_strength = 1.0
self.brake_strength = 0.5
# TODO: reverse order of actions
self.action_space = PartialDiscreteActionSpaceMap(
target_actions=[[0., 0.],
[0., -self.steering_strength],
[0., self.steering_strength],
[self.gas_strength, 0.],
[-self.brake_strength, 0],
[self.gas_strength, -self.steering_strength],
[self.gas_strength, self.steering_strength],
[self.brake_strength, -self.steering_strength],
[self.brake_strength, self.steering_strength]],
descriptions=['NO-OP', 'TURN_LEFT', 'TURN_RIGHT', 'GAS', 'BRAKE',
'GAS_AND_TURN_LEFT', 'GAS_AND_TURN_RIGHT',
'BRAKE_AND_TURN_LEFT', 'BRAKE_AND_TURN_RIGHT']
)
# map keyboard keys to actions
for idx, action in enumerate(self.action_space.descriptions):
for key in key_map.keys():
if action == key:
self.key_to_action[key_map[key]] = idx
self.num_speedup_steps = num_speedup_steps
self.max_speed = max_speed
# measurements
self.autopilot = None
self.planner = Planner(self.map_name)
# env initialization
self.reset_internal_state(True)
# render
if self.is_rendered:
image = self.get_rendered_image()
self.renderer.create_screen(image.shape[1], image.shape[0])
def _add_cameras(self, settings, cameras, camera_width, camera_height):
# add a front facing camera
if CameraTypes.FRONT in cameras:
camera = Camera(CameraTypes.FRONT.value)
camera.set(FOV=100)
camera.set_image_size(camera_width, camera_height)
camera.set_position(2.0, 0, 1.4)
camera.set_rotation(-15.0, 0, 0)
settings.add_sensor(camera)
# add a left facing camera
if CameraTypes.LEFT in cameras:
camera = Camera(CameraTypes.LEFT.value)
camera.set(FOV=100)
camera.set_image_size(camera_width, camera_height)
camera.set_position(2.0, 0, 1.4)
camera.set_rotation(-15.0, -30, 0)
settings.add_sensor(camera)
# add a right facing camera
if CameraTypes.RIGHT in cameras:
camera = Camera(CameraTypes.RIGHT.value)
camera.set(FOV=100)
camera.set_image_size(camera_width, camera_height)
camera.set_position(2.0, 0, 1.4)
camera.set_rotation(-15.0, 30, 0)
settings.add_sensor(camera)
# add a front facing depth camera
if CameraTypes.DEPTH in cameras:
camera = Camera(CameraTypes.DEPTH.value)
camera.set_image_size(camera_width, camera_height)
camera.set_position(0.2, 0, 1.3)
camera.set_rotation(8, 30, 0)
camera.PostProcessing = 'Depth'
settings.add_sensor(camera)
# add a front facing semantic segmentation camera
if CameraTypes.SEGMENTATION in cameras:
camera = Camera(CameraTypes.SEGMENTATION.value)
camera.set_image_size(camera_width, camera_height)
camera.set_position(0.2, 0, 1.3)
camera.set_rotation(8, 30, 0)
camera.PostProcessing = 'SemanticSegmentation'
settings.add_sensor(camera)
return settings
def _get_directions(self, current_point, end_point):
"""
Class that should return the directions to reach a certain goal
"""
directions = self.planner.get_next_command(
(current_point.location.x,
current_point.location.y, 0.22),
(current_point.orientation.x,
current_point.orientation.y,
current_point.orientation.z),
(end_point.location.x, end_point.location.y, 0.22),
(end_point.orientation.x, end_point.orientation.y, end_point.orientation.z))
return directions
def _open_server(self):
log_path = path.join(self.experiment_path if self.experiment_path is not None else '.', 'logs',
"CARLA_LOG_{}.txt".format(self.port))
if not os.path.exists(os.path.dirname(log_path)):
os.makedirs(os.path.dirname(log_path))
with open(log_path, "wb") as out:
cmd = [path.join(environ.get('CARLA_ROOT'), 'CarlaUE4.sh'), self.map_path,
"-benchmark", "-carla-server", "-fps={}".format(30 / self.frame_skip),
"-world-port={}".format(self.port),
"-windowed -ResX={} -ResY={}".format(self.server_width, self.server_height),
"-carla-no-hud"]
if self.config:
cmd.append("-carla-settings={}".format(self.config))
p = subprocess.Popen(cmd, stdout=out, stderr=out)
return p
def _close_server(self):
os.killpg(os.getpgid(self.server.pid), signal.SIGKILL)
def _update_state(self):
# get measurements and observations
measurements = []
while type(measurements) == list:
measurements, sensor_data = self.game.read_data()
self.state = {}
for camera in self.scene.sensors:
self.state[camera.name] = sensor_data[camera.name].data
self.location = [measurements.player_measurements.transform.location.x,
measurements.player_measurements.transform.location.y,
measurements.player_measurements.transform.location.z]
self.distance_from_goal = np.linalg.norm(np.array(self.location[:2]) -
[self.current_goal.location.x, self.current_goal.location.y])
is_collision = measurements.player_measurements.collision_vehicles != 0 \
or measurements.player_measurements.collision_pedestrians != 0 \
or measurements.player_measurements.collision_other != 0
speed_reward = measurements.player_measurements.forward_speed - 1
if speed_reward > 30.:
speed_reward = 30.
self.reward = speed_reward \
- (measurements.player_measurements.intersection_otherlane * 5) \
- (measurements.player_measurements.intersection_offroad * 5) \
- is_collision * 100 \
- np.abs(self.control.steer) * 10
# update measurements
self.measurements = [measurements.player_measurements.forward_speed] + self.location
self.autopilot = measurements.player_measurements.autopilot_control
# The directions to reach the goal (0 Follow lane, 1 Left, 2 Right, 3 Straight)
directions = int(self._get_directions(measurements.player_measurements.transform, self.current_goal) - 2)
self.state['high_level_command'] = directions
if (measurements.game_timestamp >= self.episode_max_time) or is_collision:
self.done = True
self.state['measurements'] = np.array(self.measurements)
def _take_action(self, action):
self.control = VehicleControl()
if self.separate_actions_for_throttle_and_brake:
self.control.steer = np.clip(action[0], -1, 1)
self.control.throttle = np.clip(action[1], 0, 1)
self.control.brake = np.clip(action[2], 0, 1)
else:
# transform the 2 value action (steer, throttle - brake) into a 3 value action (steer, throttle, brake)
self.control.steer = np.clip(action[0], -1, 1)
self.control.throttle = np.clip(action[1], 0, 1)
self.control.brake = np.abs(np.clip(action[1], -1, 0))
# prevent braking
if not self.allow_braking or self.control.brake < 0.1 or self.control.throttle > self.control.brake:
self.control.brake = 0
# prevent over speeding
if hasattr(self, 'measurements') and self.measurements[0] * 3.6 > self.max_speed and self.control.brake == 0:
self.control.throttle = 0.0
self.control.hand_brake = False
self.control.reverse = False
self.game.send_control(self.control)
def _load_experiment(self, experiment_idx):
self.current_experiment = self.experiment_suite.get_experiments()[experiment_idx]
self.scene = self.game.load_settings(self.current_experiment.conditions)
self.positions = self.scene.player_start_spots
self.num_positions = len(self.positions)
self.current_start_position_idx = 0
self.current_pose = 0
def _restart_environment_episode(self, force_environment_reset=False):
# select start and end positions
if self.experiment_suite:
# if an expeirent suite is available, follow its given poses
if self.current_pose >= len(self.current_experiment.poses):
# load a new experiment
self.current_experiment_idx = (self.current_experiment_idx + 1) % len(self.experiment_suite.get_experiments())
self._load_experiment(self.current_experiment_idx)
self.current_start_position_idx = self.current_experiment.poses[self.current_pose][0]
self.current_goal = self.positions[self.current_experiment.poses[self.current_pose][1]]
self.current_pose += 1
else:
# go over all the possible positions in a cyclic manner
self.current_start_position_idx = (self.current_start_position_idx + 1) % self.num_positions
# choose a random goal destination
self.current_goal = random.choice(self.positions)
try:
self.game.start_episode(self.current_start_position_idx)
except:
self.game.connect()
self.game.start_episode(self.current_start_position_idx)
# start the game with some initial speed
for i in range(self.num_speedup_steps):
self.control = VehicleControl(throttle=1.0, brake=0, steer=0, hand_brake=False, reverse=False)
self.game.send_control(VehicleControl())
def get_rendered_image(self) -> np.ndarray:
"""
Return a numpy array containing the image that will be rendered to the screen.
This can be different from the observation. For example, mujoco's observation is a measurements vector.
:return: numpy array containing the image that will be rendered to the screen
"""
image = [self.state[camera.name] for camera in self.scene.sensors]
image = np.vstack(image)
return image
def run_carla_client(args):
# Here we will run 3 episodes with 300 frames each.
numcams = 5
framestart = 20 # To compensate for the lag while starting an episode
total_frames = args.total_frames
datasize = total_frames / numcams
# turn points in Town01
chosen = [94, 97, 99, 102, 42, 70, 85, 44, 46, 67, 69]
# # turn points in Town01 for reversed
# reverse = [95, 98, 100, 101, 103, 39, 49, 71, 84, 45, 47, 66, 68, 78]
# chosen = chosen + reverse
record_weathers = [5]
number_of_episodes = len(chosen)
frames_per_episode = datasize / number_of_episodes + framestart
# chosen = [10]
# number_of_episodes = 1
# frames_per_episode = 3000
# We assume the CARLA server is already waiting for a client to connect at
# host:port. To create a connection we can use the `make_carla_client`
# context manager, it creates a CARLA client object and starts the
# connection. It will throw an exception if something goes wrong. The
# context manager makes sure the connection is always cleaned up on exit.
if args.humandriver:
driver = HumanDriver()
with make_carla_client(args.host, args.port) as client:
print('CarlaClient connected')
for weather in record_weathers:
if args.record:
folder_name = str(datetime.datetime.today().day) + '_' + 'Carla_' + '_' + args.experiment_name \
+ 'Weather_' + str(weather)
camera_dict, angles = get_camera_dict(args.settings_filepath,
args.cameras)
res = [int(i) for i in args.resolution.split(',')]
image_cut = [int(i) for i in args.image_cut.split(',')]
print(res, image_cut)
recorder = Recorder(args.path_to_save + folder_name + '/', res,\
image_cut=image_cut,camera_dict=camera_dict, angles=angles)
start = 0
for episode in range(0, number_of_episodes):
# Start a new episode.
if args.settings_filepath is None:
# Create a CarlaSettings object. This object is a wrapper around
# the CarlaSettings.ini file. Here we set the configuration we
# want for the new episode.
settings = CarlaSettings()
settings.set(SynchronousMode=True,
SendNonPlayerAgentsInfo=True,
NumberOfVehicles=0,
NumberOfPedestrians=4,
WeatherId=random.choice([1, 3, 7, 8, 14]),
QualityLevel=args.quality_level)
settings.randomize_seeds()
else:
# Alternatively, we can load these settings from a file.
with open(args.settings_filepath, 'r') as fp:
settings = fp.read()
print(settings)
settings = settings[:-2] + "\nWeatherId=" + str(weather)
settings = settings + '\n\n[CARLA/QualitySettings]\nQualityLevel=' + args.quality_level
# Now we load these settings into the server. The server replies
# with a scene description containing the available start spots for
# the player. Here we can provide a CarlaSettings object or a
# CarlaSettings.ini file as string.
scene = client.load_settings(settings)
# Choose one player start at random.
number_of_player_starts = len(scene.player_start_spots)
player_start = chosen[
start] #random.randint(0, max(0, number_of_player_starts - 1))
print("Iter is ", start, "Start used is ", player_start)
start += 1
start = start % len(chosen)
# Notify the server that we want to start the episode at the
# player_start index. This function blocks until the server is ready
# to start the episode.
print('Starting new episode...')
client.start_episode(player_start)
# Iterate every frame in the episode.
ct = 0
recordFrame = 0
# for frame in range(0, frames_per_episode):
while recordFrame < frames_per_episode:
# Read the data produced by the server this frame.
measurements, sensor_data = client.read_data()
# for agent in measurements.non_player_agents:
# if agent.HasField('traffic_light'):
# agent.traffic_light.state = 0
# im = Image.frombytes(mode='RGBA', size=(200, 88), data=sensor_data['Camera0'].raw_data, decoder_name="raw")
# im.save('test/test.png')
# Save the images to disk if requested.
if args.save_images_to_disk:
for name, measurement in sensor_data.items():
filename = args.out_filename_format.format(
episode, name, recordFrame)
measurement.save_to_disk(filename)
# We can access the encodnearested data of a given image as numpy
# array using its "data" property. For instance, to get the
# depth value (normalized) at pixel X, Y
#
# depth_array = sensor_data['CameraDepth'].data
# value_at_pixel = depth_array[Y, X]
#
# Now we have to send the instructions to control the vehicle.
# If we are in synchronous mode the server will pause the
# simulation until we send this control.
control = Control()
if args.humandriver:
control = driver.computeControl()
client.send_control(control)
elif args.autopilot:
# Together with the measurements, the server has sent the
# control that the in-game autopilot would do this frame. We
# can enable autopilot by sending back this control to the
# server. We can modify it if wanted, here for instance we
# will add some noise to the steer.
control = measurements.player_measurements.autopilot_control
# control.steer += random.uniform(-0.1, 0.1)
client.send_control(control)
else:
control.steer = random.uniform(-1.0, 1.0)
control.throttle = 0.5
control.brake = 0.0
control.hand_brake = False
control.reverse = False
client.send_control(control)
if args.record:
if recordFrame > framestart:
direction = 2
actions = control
action_noisy, drifting_time, will_drift = recorder._noiser.compute_noise(
actions,
measurements.player_measurements.forward_speed)
print("Noise diff",
actions.steer - action_noisy.steer,
actions.throttle - action_noisy.throttle,
actions.brake - action_noisy.brake)
if measurements.player_measurements.forward_speed <= 0.1:
print("No move Frame no: ", recordFrame)
else:
print("Yes!! move Frame no: ", recordFrame)
recorder.record(measurements, sensor_data,
actions, action_noisy,
direction)
recordFrame += 1
else:
recordFrame += 1
def run_carla_client(args):
# Here we will run 3 episodes with 300 frames each.
number_of_episodes = 5
cut_per_episode = 40
frames_per_cut = 200
# We assume the CARLA server is already waiting for a client to connect at
# host:port. To create a connection we can use the `make_carla_client`
# context manager, it creates a CARLA client object and starts the
# connection. It will throw an exception if something goes wrong. The
# context manager makes sure the connection is always cleaned up on exit.
with make_carla_client(args.host, args.port) as client:
print('CarlaClient connected')
for episode in range(0, number_of_episodes):
print("input any key to continue...")
start = input()
# each episode dir store a set of traindata. if dir not existed, then make it
pathdir = '/home/kadn/dataTrain/episode_{:0>3}'.format(episode)
mkdir(pathdir)
# Create a CarlaSettings object. This object is a wrapper around
# the CarlaSettings.ini file. Here we set the configuration we
# want for the new episode.
settings = CarlaSettings()
settings.set(
SynchronousMode=True,
SendNonPlayerAgentsInfo=True,
NumberOfVehicles=20,
NumberOfPedestrians=40,
# WeatherId=random.choice([1, 3, 7, 8, 14]),
WeatherId = 1,
QualityLevel=args.quality_level)
settings.randomize_seeds()
# Now we want to add a couple of cameras to the player vehicle.
# We will collect the images produced by these cameras every
# frame.
# The default camera captures RGB images of the scene.
camera0 = Camera('CameraRGB')
# Set image resolution in pixels.
camera0.set_image_size(88, 200)
# Set its position relative to the car in meters.
camera0.set_position(0.30, 0, 1.30)
settings.add_sensor(camera0)
# Let's add another camera producing ground-truth depth.
camera1 = Camera('CameraDepth', PostProcessing='Depth')
camera1.set_image_size(200, 88)
camera1.set_position(0.30, 0, 1.30)
settings.add_sensor(camera1)
camera2 = Camera('CameraSemSeg', PostProcessing='SemanticSegmentation')
camera2.set_image_size(88, 200)
camera2.set_position(2.0, 0.0, 1.4)
camera2.set_rotation(0.0, 0.0, 0.0)
settings.add_sensor(camera2)
# if args.lidar:
lidar = Lidar('Lidar32')
lidar.set_position(0, 0, 2.50)
lidar.set_rotation(0, 0, 0)
lidar.set(
Channels=0,
Range=30,
PointsPerSecond=200000,
RotationFrequency=10,
UpperFovLimit=0,
LowerFovLimit=0)
settings.add_sensor(lidar)
# else:
#
# # Alternatively, we can load these settings from a file.
# with open(args.settings_filepath, 'r') as fp:
# settings = fp.read()
# Now we load these settings into the server. The server replies
# with a scene description containing the available start spots for
# the player. Here we can provide a CarlaSettings object or a
# CarlaSettings.ini file as string.
scene = client.load_settings(settings)
# Choose one player start at random.
# number_of_player_starts = len(scene.player_start_spots)
# player_start = random.randint(0, max(0, number_of_player_starts - 1))
player_start = 1
# Notify the server that we want to start the episode at the
# player_start index. This function blocks until the server is ready
# to start the episode.
print('Starting new episode at %r...' % scene.map_name)
# Start a new episode.
client.start_episode(player_start)
for cut_per_episode in range(0,cut_per_episode):
num = fileCounter(pathdir)
filename = "data_{:0>6}.h5".format(num)
filepath = pathdir + '/' + filename
f = h5py.File(filepath, "w")
rgb_file = f.create_dataset("rgb", (200, 88, 200), np.uint8)
seg_file = f.create_dataset("seg", (200, 88, 200), np.uint8)
lidar_file = f.create_dataset('lidar',(200,200,200),np.uint8)
startendpoint = f.create_dataset('startend',(1,2),np.float32)
targets_file = f.create_dataset("targets", (200, 28), np.float32)
index_file = 0
# Iterate every frame in the episode.
for frame in range(0, frames_per_cut):
# Read the data produced by the server this frame.
measurements, sensor_data = client.read_data()
# get data and store
sensors, targets_data = record_train_data(measurements,sensor_data)
rgb_file[:,:,index_file] = sensors['rgb']
seg_file[:,:,index_file] = sensors['seg']
lidar_file[:,:,index_file] = sensors['lidar']
targets_file[index_file,:] = targets_data
index_file += 1
# We can access the encoded data of a given image as numpy
# array using its "data" property. For instance, to get the
# depth value (normalized) at pixel X, Y
#
# depth_array = sensor_data['CameraDepth'].data
# value_at_pixel = depth_array[Y, X]
#
# Now we have to send the instructions to control the vehicle.
# If we are in synchronous mode the server will pause the
# simulation until we send this control.
if args.autopilot:
client.send_control(
steer=0,
throttle=0.8,
brake=0.0,
hand_brake=False,
reverse=False)
else:
# Together with the measurements, the server has sent the
# control that the in-game autopilot would do this frame. We
# can enable autopilot by sending back this control to the
# server. We can modify it if wanted, here for instance we
# will add some noise to the steer.
control = measurements.player_measurements.autopilot_control
control.steer += random.uniform(-0.05, 0.05)
client.send_control(control)
def build_experiments(self):
"""
Creates the whole set of experiment objects,
The experiments created depend on the selected Town.
"""
# We set the camera
# This single RGB camera is used on every experiment
camera = Camera('CameraMiddle')
camera.set(FOV=100)
camera.set_image_size(800, 600)
camera.set_position(2.5, 0.0, 1.0)
camera.set_rotation(0, 0, 0)
if self._city_name == 'Town01':
poses_tasks = self._poses_town01()
vehicles_tasks = [0, 0, 0, 100]
pedestrians_tasks = [0, 0, 0, 300]
n_samples = [2, 2, 2, 0]
#n_samples = [3, 6, 6, 9]
else:
poses_tasks = self._poses_town02()
vehicles_tasks = [0, 0, 0, 50]
pedestrians_tasks = [0, 0, 0, 150]
n_samples = [2, 2, 2, 0]
#n_samples = [3, 6, 6, 9]
experiments_vector = []
random.seed(1)
for weather in self.weathers:
for iteration in range(len(poses_tasks)):
poses = poses_tasks[iteration]
vehicles = vehicles_tasks[iteration]
pedestrians = pedestrians_tasks[iteration]
nsample = n_samples[iteration]
if nsample == 0:
continue
poses = random.sample(poses, nsample)
conditions = CarlaSettings()
conditions.set(SendNonPlayerAgentsInfo=True,
NumberOfVehicles=vehicles,
NumberOfPedestrians=pedestrians,
WeatherId=weather)
# Add all the cameras that were set for this experiments
conditions.add_sensor(camera)
experiment = Experiment()
experiment.set(Conditions=conditions,
Poses=poses,
Task=iteration,
Repetitions=1)
experiments_vector.append(experiment)
return experiments_vector
def run_carla_client(args):
frames_per_episode = 10000
spline_points = 10000
track_DF = pd.read_csv('racetrack{}.txt'.format(args.racetrack), header=None)
# The track data are rescaled by 100x with relation to Carla measurements
track_DF = track_DF / 100
pts_2D = track_DF.loc[:, [0, 1]].values # Speed & Steer values
#splprep(array , u=these values are calculated automatically as M = len(x[0]), s=A smoothing condition per =1 : data points are considered periodic)
#tck :tuple () u:array(values of parameters)
tck, u = splprep(pts_2D.T, u=None, s=2.0, per=1)
u_new = np.linspace(u.min(), u.max(), spline_points)
x_new, y_new = splev(u_new, tck, der=0)
pts_2D = np.c_[x_new, y_new]
steer = 0.0
throttle = 0.5
depth_array = None
# 'Pad' collect data mode
if args.controller_name == 'pad':
weather_id = 4
controller = PadController(args.target_speed)
# 'nn' training mode
elif args.controller_name == 'nn':
# Import it here because importing TensorFlow is time consuming
from neural_network_controller import NNController
weather_id = 4
controller = NNController(
args.target_speed,
args.model_dir_name,
args.which_model,
args.ensemble_prediction,
)
# connect to server
with make_carla_client(args.host, args.port) as client:
print('CarlaClient connected')
episode = 0
num_fails = 0
while episode < args.num_episodes:
# Start a new episode
depth_storage = None
log_dicts = None
if args.settings_filepath is None:
# Here we set the configuration we want for the new episode.
settings = CarlaSettings()
settings.set(
SynchronousMode=True,
SendNonPlayerAgentsInfo=True,
NumberOfVehicles=40,
NumberOfPedestrians=150,
WeatherId=weather_id,
QualityLevel= 'Epic' # lower quality
)
settings.randomize_seeds()
# Now we want to add a couple of cameras to the player vehicle.
# FOV : is the extent of the observable world that is seen at any given moment
# 69.4 defualt in intel data sheet
camera = Camera('CameraDepth', PostProcessing='Depth', FOV=69.4)
camera.set_image_size(IMAGE_SIZE[1], IMAGE_SIZE[0])
camera.set_position(2.30, 0, 1.30)
settings.add_sensor(camera)
else:
# Alternatively, we can load these settings from a file.
with open(args.settings_filepath, 'r') as fp:
settings = fp.read()
# Now we load these settings into the server.
scene = client.load_settings(settings)
# Choose one player start at random.
num_of_player_starts = len(scene.player_start_spots)
player_start = random.randint(0, max(0, num_of_player_starts - 1))
print('Starting new episode...', )
client.start_episode(player_start)
status, depth_storage, one_log_dict, distance_travelled = run_episode(
client,
controller,
pts_2D,
depth_storage,
log_dicts,
frames_per_episode,
args.controller_name
)
if 'FAIL' in status:
num_fails += 1
print(status)
continue
else:
print('SUCCESS: ' + str(episode))
episode += 1
def run_carla_client(args):
# Here we will run 3 episodes with 300 frames each.
number_of_episodes = 500000
frames_per_episode = 300000000000000
global k
# We assume the CARLA server is already waiting for a client to connect at
# host:port. To create a connection we can use the `make_carla_client`
# context manager, it creates a CARLA client object and starts the
# connection. It will throw an exception if something goes wrong. The
# context manager makes sure the connection is always cleaned up on exit.
with make_carla_client(args.host, args.port) as client:
print('CarlaClient connected')
#########################
for episode in range(350, number_of_episodes):
# Start a new episode.
print(episode)
if args.settings_filepath is None:
# Create a CarlaSettings object. This object is a wrapper around
# the CarlaSettings.ini file. Here we set the configuration we
# want for the new episode.
settings = CarlaSettings()
settings.set(
SynchronousMode=True,
SendNonPlayerAgentsInfo=True,
NumberOfVehicles=0,
NumberOfPedestrians=0,
# WeatherId=random.choice([1, 3, 7, 8, 14]),
WeatherId=1,
QualityLevel=args.quality_level)
settings.randomize_seeds()
# Now we want to add a couple of cameras to the player vehicle.
# We will collect the images produced by these cameras every
# frame.
# The default camera captures RGB images of the scene.
camera0 = Camera('CameraRGB')
# Set image resolution in pixels.
camera0.set_image_size(210, 160)
# Set its position relative to the car in meters.
camera0.set_position(0.30, 0, 1.30)
settings.add_sensor(camera0)
# Let's add another camera producing ground-truth depth.
# camera1 = Camera('CameraDepth', PostProcessing='Depth')
# camera1.set_image_size(210, 160)
# camera1.set_position(0.30, 0, 1.30)
# settings.add_sensor(camera1)
#
# if args.lidar:
# lidar = Lidar('Lidar32')
# lidar.set_position(0, 0, 2.50)
# lidar.set_rotation(0, 0, 0)
# lidar.set(
# Channels=32,
# Range=50,
# PointsPerSecond=100000,
# RotationFrequency=10,
# UpperFovLimit=10,
# LowerFovLimit=-30)
# settings.add_sensor(lidar)
#
# else:
#
# # Alternatively, we can load these settings from a file.
# with open(args.settings_filepath, 'r') as fp:
# settings = fp.read()
# Now we load these settings into the server. The server replies
# with a scene description containing the available start spots for
# the player. Here we can provide a CarlaSettings object or a
# CarlaSettings.ini file as string.
scene = client.load_settings(settings)
# Choose one player start at random.
#从哪里开始
# number_of_player_starts = len(scene.player_start_spots)
# player_start = random.randint(0, max(0, number_of_player_starts - 1))
# Notify the server that we want to start the episode at the
# player_start index. This function blocks until the server is ready
# to start the episode.
print('Starting new episode at %r...' % scene.map_name)
#client.start_episode(player_start)
client.start_episode(0)
#--------------------------------------------------------------------
# Iterate every frame in the episode.
# Read the data produced by the server this frame.
# 这个应该是获取到的数据
measurements, sensor_data = client.read_data()
# Print some of the measurements.
image_RGB = to_rgb_array(sensor_data['CameraRGB'])
image_RGB_real=image_RGB.flatten()
print(image_RGB_real)
print(image_RGB_real.shape[0])
###############################################################################################
if k==1:
# player_measurements = measurements.player_measurements
# loc = np.array([player_measurements.transform.location.x, player_measurements.transform.location.y])
RL = DeepQNetwork(n_actions=7,
n_features=image_RGB_real.shape[0],
learning_rate=0.0001, e_greedy=0.9,
replace_target_iter=1000, memory_size=2000,
e_greedy_increment=0.000001, )
total_steps = 0
else:
print("rl运行完毕")
k=2
while True:
measurements, sensor_data = client.read_data()
player_measurements = measurements.player_measurements
other_lane = 100 * player_measurements.intersection_otherlane
offroad = 100 * player_measurements.intersection_offroad
# loc1=np.array([player_measurements.transform.location.x, player_measurements.transform.location.y])
# print(offroad)
# print(other_lane)
# print(loc1)
image_RGB = to_rgb_array(sensor_data['CameraRGB'])
image_RGB_real = image_RGB.flatten()
print_measurements(measurements)
observation = image_RGB_real
ep_r = 0
done = False
# while True:
# # env.render()
action = RL.choose_action(observation)
if not args.autopilot:
brake1=0.0
steer1=0.0
if (action > 4):
brake1 = actions[action]
else:
steer1 = actions[action]
client.send_control(
# steer=random.uniform(-1.0, 1.0),
steer=steer1,
throttle=0.6,
brake=brake1,
hand_brake=False,
reverse=False)
else:
# Together with the measurements, the server has sent the
# control that the in-game autopilot would do this frame. We
# can enable autopilot by sending back this control to the
# server. We can modify it if wanted, here for instance we
# will add some noise to the steer.
control = measurements.player_measurements.autopilot_control
# control.steer += random.uniform(-0.1, 0.1)
client.send_control(control)
# loc2=np.array([player_measurements.transform.location.x, player_measurements.transform.location.y])
image_RGB = to_rgb_array(sensor_data['CameraRGB'])
image_RGB_rea2 = image_RGB.flatten()
observation_ = image_RGB_rea2
reward = -other_lane - offroad+np.sqrt(np.square(player_measurements.transform.location.x-271.0)+np.square(player_measurements.transform.location.y-129.5))
print(offroad)
col=player_measurements.collision_other
speed=player_measurements.forward_speed * 3.6
print(col)
if offroad > 10 or other_lane>10 or col > 0 :
print(111111111111111111111)
done = True
RL.store_transition(observation, action, reward, observation_)
ep_r += reward
if total_steps > 100:
RL.learn(do_train=1)
if done:
print('episode: ',
'ep_r: ', round(ep_r, 2),
' epsilon: ', round(RL.epsilon, 2))
# client.start_episode(0)
break
observation = observation_
total_steps += 1
# RL.plot_cost()
# Save the images to disk if requested.
if args.save_images_to_disk:
for name, measurement in sensor_data.items():
filename = args.out_filename_format.format(episode, name, frame)
measurement.save_to_disk(filename)
continue
class CarlaEnvironment(Environment):
def __init__(self, level: LevelSelection, seed: int, frame_skip: int,
human_control: bool, custom_reward_threshold: Union[int,
float],
visualization_parameters: VisualizationParameters,
server_height: int, server_width: int, camera_height: int,
camera_width: int, verbose: bool, config: str,
episode_max_time: int, allow_braking: bool,
quality: CarlaEnvironmentParameters.Quality,
cameras: List[CameraTypes], weather_id: List[int],
experiment_path: str, **kwargs):
super().__init__(level, seed, frame_skip, human_control,
custom_reward_threshold, visualization_parameters)
# server configuration
self.server_height = server_height
self.server_width = server_width
self.port = get_open_port()
self.host = 'localhost'
self.map = self.env_id
self.experiment_path = experiment_path
# client configuration
self.verbose = verbose
self.quality = quality
self.cameras = cameras
self.weather_id = weather_id
self.episode_max_time = episode_max_time
self.allow_braking = allow_braking
self.camera_width = camera_width
self.camera_height = camera_height
# state space
self.state_space = StateSpace({
"measurements":
VectorObservationSpace(
4, measurements_names=["forward_speed", "x", "y", "z"])
})
for camera in self.cameras:
self.state_space[camera.value] = ImageObservationSpace(
shape=np.array([self.camera_height, self.camera_width, 3]),
high=255)
# setup server settings
self.config = config
if self.config:
# load settings from file
with open(self.config, 'r') as fp:
self.settings = fp.read()
else:
# hard coded settings
self.settings = CarlaSettings()
self.settings.set(SynchronousMode=True,
SendNonPlayerAgentsInfo=False,
NumberOfVehicles=15,
NumberOfPedestrians=30,
WeatherId=random.choice(
force_list(self.weather_id)),
QualityLevel=self.quality.value,
SeedVehicles=seed,
SeedPedestrians=seed)
if seed is None:
self.settings.randomize_seeds()
self.settings = self._add_cameras(self.settings, self.cameras,
self.camera_width,
self.camera_height)
# open the server
self.server = self._open_server()
logging.disable(40)
# open the client
self.game = CarlaClient(self.host, self.port, timeout=99999999)
self.game.connect()
scene = self.game.load_settings(self.settings)
# get available start positions
positions = scene.player_start_spots
self.num_pos = len(positions)
self.iterator_start_positions = 0
# action space
self.action_space = BoxActionSpace(shape=2,
low=np.array([-1, -1]),
high=np.array([1, 1]))
# human control
if self.human_control:
# convert continuous action space to discrete
self.steering_strength = 0.5
self.gas_strength = 1.0
self.brake_strength = 0.5
self.action_space = PartialDiscreteActionSpaceMap(
target_actions=[[0., 0.], [0., -self.steering_strength],
[0., self.steering_strength],
[self.gas_strength, 0.],
[-self.brake_strength, 0],
[self.gas_strength, -self.steering_strength],
[self.gas_strength, self.steering_strength],
[self.brake_strength, -self.steering_strength],
[self.brake_strength, self.steering_strength]],
descriptions=[
'NO-OP', 'TURN_LEFT', 'TURN_RIGHT', 'GAS', 'BRAKE',
'GAS_AND_TURN_LEFT', 'GAS_AND_TURN_RIGHT',
'BRAKE_AND_TURN_LEFT', 'BRAKE_AND_TURN_RIGHT'
])
# map keyboard keys to actions
for idx, action in enumerate(self.action_space.descriptions):
for key in key_map.keys():
if action == key:
self.key_to_action[key_map[key]] = idx
self.num_speedup_steps = 30
# measurements
self.autopilot = None
# env initialization
self.reset_internal_state(True)
# render
if self.is_rendered:
image = self.get_rendered_image()
self.renderer.create_screen(image.shape[1], image.shape[0])
def _add_cameras(self, settings, cameras, camera_width, camera_height):
# add a front facing camera
if CameraTypes.FRONT in cameras:
camera = Camera(CameraTypes.FRONT.value)
camera.set_image_size(camera_width, camera_height)
camera.set_position(0.2, 0, 1.3)
camera.set_rotation(8, 0, 0)
settings.add_sensor(camera)
# add a left facing camera
if CameraTypes.LEFT in cameras:
camera = Camera(CameraTypes.LEFT.value)
camera.set_image_size(camera_width, camera_height)
camera.set_position(0.2, 0, 1.3)
camera.set_rotation(8, -30, 0)
settings.add_sensor(camera)
# add a right facing camera
if CameraTypes.RIGHT in cameras:
camera = Camera(CameraTypes.RIGHT.value)
camera.set_image_size(camera_width, camera_height)
camera.set_position(0.2, 0, 1.3)
camera.set_rotation(8, 30, 0)
settings.add_sensor(camera)
# add a front facing depth camera
if CameraTypes.DEPTH in cameras:
camera = Camera(CameraTypes.DEPTH.value)
camera.set_image_size(camera_width, camera_height)
camera.set_position(0.2, 0, 1.3)
camera.set_rotation(8, 30, 0)
camera.PostProcessing = 'Depth'
settings.add_sensor(camera)
# add a front facing semantic segmentation camera
if CameraTypes.SEGMENTATION in cameras:
camera = Camera(CameraTypes.SEGMENTATION.value)
camera.set_image_size(camera_width, camera_height)
camera.set_position(0.2, 0, 1.3)
camera.set_rotation(8, 30, 0)
camera.PostProcessing = 'SemanticSegmentation'
settings.add_sensor(camera)
return settings
def _open_server(self):
log_path = path.join(
self.experiment_path if self.experiment_path is not None else '.',
'logs', "CARLA_LOG_{}.txt".format(self.port))
if not os.path.exists(os.path.dirname(log_path)):
os.makedirs(os.path.dirname(log_path))
with open(log_path, "wb") as out:
cmd = [
path.join(environ.get('CARLA_ROOT'),
'CarlaUE4.sh'), self.map, "-benchmark",
"-carla-server", "-fps={}".format(30 / self.frame_skip),
"-world-port={}".format(self.port),
"-windowed -ResX={} -ResY={}".format(self.server_width,
self.server_height),
"-carla-no-hud"
]
if self.config:
cmd.append("-carla-settings={}".format(self.config))
p = subprocess.Popen(cmd, stdout=out, stderr=out)
return p
def _close_server(self):
os.killpg(os.getpgid(self.server.pid), signal.SIGKILL)
def _update_state(self):
# get measurements and observations
measurements = []
while type(measurements) == list:
measurements, sensor_data = self.game.read_data()
self.state = {}
for camera in self.cameras:
self.state[camera.value] = sensor_data[camera.value].data
self.location = [
measurements.player_measurements.transform.location.x,
measurements.player_measurements.transform.location.y,
measurements.player_measurements.transform.location.z
]
is_collision = measurements.player_measurements.collision_vehicles != 0 \
or measurements.player_measurements.collision_pedestrians != 0 \
or measurements.player_measurements.collision_other != 0
speed_reward = measurements.player_measurements.forward_speed - 1
if speed_reward > 30.:
speed_reward = 30.
self.reward = speed_reward \
- (measurements.player_measurements.intersection_otherlane * 5) \
- (measurements.player_measurements.intersection_offroad * 5) \
- is_collision * 100 \
- np.abs(self.control.steer) * 10
# update measurements
self.measurements = [measurements.player_measurements.forward_speed
] + self.location
self.autopilot = measurements.player_measurements.autopilot_control
# action_p = ['%.2f' % member for member in [self.control.throttle, self.control.steer]]
# screen.success('REWARD: %.2f, ACTIONS: %s' % (self.reward, action_p))
if (measurements.game_timestamp >=
self.episode_max_time) or is_collision:
# screen.success('EPISODE IS DONE. GameTime: {}, Collision: {}'.format(str(measurements.game_timestamp),
# str(is_collision)))
self.done = True
self.state['measurements'] = self.measurements
def _take_action(self, action):
self.control = VehicleControl()
self.control.throttle = np.clip(action[0], 0, 1)
self.control.steer = np.clip(action[1], -1, 1)
self.control.brake = np.abs(np.clip(action[0], -1, 0))
if not self.allow_braking:
self.control.brake = 0
self.control.hand_brake = False
self.control.reverse = False
self.game.send_control(self.control)
def _restart_environment_episode(self, force_environment_reset=False):
self.iterator_start_positions += 1
if self.iterator_start_positions >= self.num_pos:
self.iterator_start_positions = 0
try:
self.game.start_episode(self.iterator_start_positions)
except:
self.game.connect()
self.game.start_episode(self.iterator_start_positions)
# start the game with some initial speed
for i in range(self.num_speedup_steps):
self._take_action([1.0, 0])
def get_rendered_image(self) -> np.ndarray:
"""
Return a numpy array containing the image that will be rendered to the screen.
This can be different from the observation. For example, mujoco's observation is a measurements vector.
:return: numpy array containing the image that will be rendered to the screen
"""
image = [self.state[camera.value] for camera in self.cameras]
image = np.vstack(image)
return image
def run_carla_client(host, port, far):
# Here we will run a single episode with 300 frames.
number_of_frames = 3000
frame_step = 100 # Save one image every 100 frames
output_folder = '_out'
image_size = [800, 600]
camera_local_pos = [0.3, 0.0, 1.3] # [X, Y, Z]
camera_local_rotation = [0, 0, 0] # [pitch(Y), yaw(Z), roll(X)]
fov = 70
# Connect with the server
with make_carla_client(host, port) as client:
print('CarlaClient connected')
# Here we load the settings.
settings = CarlaSettings()
settings.set(
SynchronousMode=True,
SendNonPlayerAgentsInfo=False,
NumberOfVehicles=20,
NumberOfPedestrians=40,
WeatherId=random.choice([1, 3, 7, 8, 14]))
settings.randomize_seeds()
camera1 = Camera('CameraDepth', PostProcessing='Depth', FOV=fov)
camera1.set_image_size(*image_size)
camera1.set_position(*camera_local_pos)
camera1.set_rotation(*camera_local_rotation)
settings.add_sensor(camera1)
camera2 = Camera('CameraRGB', PostProcessing='SceneFinal', FOV=fov)
camera2.set_image_size(*image_size)
camera2.set_position(*camera_local_pos)
camera2.set_rotation(*camera_local_rotation)
settings.add_sensor(camera2)
client.load_settings(settings)
# Start at location index id '0'
client.start_episode(0)
# Compute the camera transform matrix
camera_to_car_transform = camera2.get_unreal_transform()
# Iterate every frame in the episode except for the first one.
for frame in range(1, number_of_frames):
# Read the data produced by the server this frame.
measurements, sensor_data = client.read_data()
# Save one image every 'frame_step' frames
if not frame % frame_step:
# Start transformations time mesure.
timer = StopWatch()
# RGB image [[[r,g,b],..[r,g,b]],..[[r,g,b],..[r,g,b]]]
image_RGB = to_rgb_array(sensor_data['CameraRGB'])
# 2d to (camera) local 3d
# We use the image_RGB to colorize each 3D point, this is optional.
# "max_depth" is used to keep only the points that are near to the
# camera, meaning 1.0 the farest points (sky)
point_cloud = depth_to_local_point_cloud(
sensor_data['CameraDepth'],
image_RGB,
max_depth=far
)
# (Camera) local 3d to world 3d.
# Get the transform from the player protobuf transformation.
world_transform = Transform(
measurements.player_measurements.transform
)
# Compute the final transformation matrix.
car_to_world_transform = world_transform * camera_to_car_transform
# Car to World transformation given the 3D points and the
# transformation matrix.
point_cloud.apply_transform(car_to_world_transform)
# End transformations time mesure.
timer.stop()
# Save PLY to disk
# This generates the PLY string with the 3D points and the RGB colors
# for each row of the file.
point_cloud.save_to_disk(os.path.join(
output_folder, '{:0>5}.ply'.format(frame))
)
print_message(timer.milliseconds(), len(point_cloud), frame)
client.send_control(
measurements.player_measurements.autopilot_control
)
def build_experiments(self):
"""
Creates the whole set of experiment objects,
The experiments created depends on the selected Town.
"""
# We check the town, based on that we define the town related parameters
# The size of the vector is related to the number of tasks, inside each
# task there is also multiple poses ( start end, positions )
if self._city_name == 'Town01':
#poses_tasks = [[[7, 3]], [[138, 17]], [[140, 134]], [[140, 134]]]
poses_tasks = [[[138, 134]]]
vehicles_tasks = [20]
pedestrians_tasks = [10]
else:
right_curves = [[[1, 56], [65, 69], [78, 51], [44, 61], [40, 17],
[71, 16], [74, 38], [46, 12], [19, 18], [26, 74],
[37, 76], [11, 44], [20, 6], [10, 22], [28, 2],
[5, 15], [14, 33], [34, 8]]]
left_curves = [[[57, 82], [72, 43], [52, 79], [70, 66], [43, 14],
[11, 47], [79, 32], [37, 75], [75, 16], [26, 73],
[39, 5], [2, 37], [34, 13], [6, 35], [10, 19],
[23, 6], [5, 30], [16, 2]]]
special_test = [[[19, 66], [79, 14], [42, 13], [31, 71], [54, 30],
[10, 61], [66, 3], [27, 12], [2, 29], [16, 14],
[70, 73], [46, 67], [51, 81], [56, 65], [43, 54]]]
corl_task2 = [[[19, 66], [79, 14], [19, 57], [23, 1], [53, 76],
[42, 13], [31, 71], [33, 5], [54, 30], [10, 61],
[66, 3], [27, 12], [79, 19], [2, 29], [16, 14],
[5, 57], [70, 73], [46, 67], [57, 50], [61, 49],
[21, 12], [51, 81], [77, 68], [56, 65], [43, 54]]]
#poses_tasks = corl_task2
poses_tasks = [[[10, 19]]]
vehicles_tasks = [0] #*len(poses_tasks[0])
pedestrians_tasks = [0] #*len(poses_tasks[0])
# We set the camera
# This single RGB camera is used on every experiment
'''camera = Camera('CameraRGB')
camera.set(FOV=90)
camera.set_image_size(800, 600)
camera.set_position(1.44, 0.0, 1.2)
camera.set_rotation(0, 0, 0)'''
camera0 = Camera('CameraRGB0')
camera0.set(FOV=90)
camera0.set_image_size(400, 300)
camera0.set_position(1.7, 0.0, 1.3)
camera0.set_rotation(0, 0, 0)
camera1 = Camera('CameraRGB1')
camera1.set(FOV=90)
camera1.set_image_size(400, 300)
camera1.set_position(1.7, 0.0, 1.3)
camera1.set_rotation(0, -45, 0)
camera2 = Camera('CameraRGB2')
camera2.set(FOV=90)
camera2.set_image_size(400, 300)
camera2.set_position(1.7, 0.0, 1.3)
camera2.set_rotation(0, +45, 0)
# Based on the parameters, creates a vector with experiment objects.
experiments_vector = []
for weather in self.weathers:
for iteration in range(len(poses_tasks)):
poses = poses_tasks[iteration]
vehicles = vehicles_tasks[iteration]
pedestrians = pedestrians_tasks[iteration]
conditions = CarlaSettings()
conditions.set(SendNonPlayerAgentsInfo=True,
NumberOfVehicles=vehicles,
NumberOfPedestrians=pedestrians,
WeatherId=weather)
# Add all the cameras that were set for this experiments
conditions.add_sensor(camera0)
conditions.add_sensor(camera1)
conditions.add_sensor(camera2)
experiment = Experiment()
experiment.set(Conditions=conditions,
Poses=poses,
Task=iteration,
Repetitions=1)
experiments_vector.append(experiment)
'''path0=os.listdir('CameraRGB0')
path1=os.listdir('CameraRGB1')
path2=os.listdir('CameraRGB2')
os.mkdir('CameraRGB')
width=400
height=300
total_width = width*3
max_height = height
i = 0
images=[]
images.append(Image.open('CameraRGB1/'+path1[i]))
images.append(Image.open('CameraRGB0/'+path0[i]))
images.append(Image.open('CameraRGB2/'+path2[i]))
new_im = Image.new('RGB', (total_width, max_height))
x_offset = 0
for im in images:
new_im.paste(im, (x_offset,0))
x_offset += im.size[0]
new_im.save('CameraRGB/'+path0[i])
i = i+1'''
return experiments_vector
def run_carla_client(args):
# Here we will run 1 episodes with 1000 frames each.
number_of_episodes = 1
frames_per_episode = 1000
# We assume the CARLA server is already waiting for a client to connect at
# host:port. To create a connection we can use the `make_carla_client`
# context manager, it creates a CARLA client object and starts the
# connection. It will throw an exception if something goes wrong. The
# context manager makes sure the connection is always cleaned up on exit.
with make_carla_client(args.host, args.port) as client:
print('CarlaClient connected')
for episode in range(0, number_of_episodes):
# Start a new episode.
if args.settings_filepath is None:
# Create a CarlaSettings object. This object is a wrapper around
# the CarlaSettings.ini file. Here we set the configuration we
# want for the new episode.
settings = CarlaSettings()
settings.set(
SynchronousMode=True,
SendNonPlayerAgentsInfo=True,
NumberOfVehicles=0,
NumberOfPedestrians=0,
WeatherId=args.weatherId, #bbescos
QualityLevel=args.quality_level)
# Now we want to add a few cameras to the player vehicle.
# We will collect the images produced by these cameras every
# frame.
# The default camera captures RGB images of the scene.
camera0 = Camera('RGB')
# Set image resolution in pixels.
camera0.set_image_size(800, 600)
# Set its position relative to the car in meters.
camera0.set_position(1.80, 0, 1.30)
settings.add_sensor(camera0)
# Let's add another RGB camera in the back of the car
camera0_b = Camera('RGB_back')
camera0_b.set_image_size(800, 600)
camera0_b.set_position(-1.70, 0, 1.30) #bbescos
camera0_b.set_rotation(0, 180, 0)
settings.add_sensor(camera0_b)
# Let's add another camera producing ground-truth depth.
camera1 = Camera('Depth', PostProcessing='Depth')
camera1.set_image_size(800, 600)
camera1.set_position(1.80, 0, 1.30)
settings.add_sensor(camera1)
# Let's add another camera producing ground-truth depth for the back.
camera1_b = Camera('Depth_back', PostProcessing='Depth')
camera1_b.set_image_size(800, 600)
camera1_b.set_position(-1.70, 0, 1.30)
camera1_b.set_rotation(0, 180, 0)
settings.add_sensor(camera1_b)
# Let's add another camera producing ground-truth semantic segmentation.
camera2 = Camera('SemanticSegmentation',
PostProcessing='SemanticSegmentation')
camera2.set_image_size(800, 600)
camera2.set_position(1.80, 0, 1.30)
settings.add_sensor(camera2)
# Let's add another camera producing ground-truth semantic segmentation for the back.
camera2_b = Camera('SemanticSegmentation_back',
PostProcessing='SemanticSegmentation')
camera2_b.set_image_size(800, 600)
camera2_b.set_position(-1.70, 0, 1.30)
camera2_b.set_rotation(0, 180, 0)
settings.add_sensor(camera2_b)
if args.lidar:
lidar = Lidar('Lidar32')
lidar.set_position(0, 0, 2.50)
lidar.set_rotation(0, 0, 0)
lidar.set(Channels=32,
Range=50,
PointsPerSecond=100000,
RotationFrequency=10,
UpperFovLimit=10,
LowerFovLimit=-30)
settings.add_sensor(lidar)
else:
# Alternatively, we can load these settings from a file.
with open(args.settings_filepath, 'r') as fp:
settings = fp.read()
# Now we load these settings into the server. The server replies
# with a scene description containing the available start spots for
# the player. Here we can provide a CarlaSettings object or a
# CarlaSettings.ini file as string.
scene = client.load_settings(settings)
# Choose one player start.
number_of_player_starts = len(scene.player_start_spots)
player_start = args.playerStart
# Notify the server that we want to start the episode at the
# player_start index. This function blocks until the server is ready
# to start the episode.
print('Starting new episode...')
client.start_episode(player_start)
# Iterate every frame in the episode.
for frame in range(0, frames_per_episode):
print('Frame : ', frame)
save_bool = True
# Read the data produced by the server this frame.
measurements, sensor_data = client.read_data()
# Save Trajectory
save_trajectory(frame, measurements)
# Read trajectory to check if it is the same #bbescos
file = open(args.trajectoryFile, 'r')
lines = file.readlines()
line = lines[frame]
words = line.split()
pos_x_s = float(words[1])
if abs(pos_x_s -
measurements.player_measurements.transform.location.x
) > 0.1:
save_bool = False
print(
pos_x_s,
measurements.player_measurements.transform.location.x)
pos_y_s = float(words[2])
if abs(pos_y_s -
measurements.player_measurements.transform.location.y
) > 0.1:
save_bool = False
print(
pos_y_s,
measurements.player_measurements.transform.location.y)
file.close()
# Save the images to disk if requested.
if args.save_images_to_disk and frame % 10 == 0 and frame > 29 and save_bool:
for name, measurement in sensor_data.items():
filename = args.out_filename_format.format(
episode, name, frame)
measurement.save_to_disk(filename)
# Now we have to send the instructions to control the vehicle.
# If we are in synchronous mode the server will pause the
# simulation until we send this control.
if not args.autopilot:
client.send_control(steer=random.uniform(-1.0, 1.0),
throttle=0.5,
brake=0.0,
hand_brake=False,
reverse=False)
else:
# Together with the measurements, the server has sent the
# control that the in-game autopilot would do this frame. We
# can enable autopilot by sending back this control to the
# server.
control = measurements.player_measurements.autopilot_control
# Read control file
file = open(args.controlFile, 'r')
lines = file.readlines()
line = lines[frame]
words = line.split()
steer = float(words[1])
throttle = float(words[2])
brake = float(words[3])
hand_brake = (words[4] == 'True')
reverse = (words[5] == 'True')
file.close()
control.steer = steer
control.throttle = throttle
control.brake = brake
control.hand_brake = hand_brake
control.reverse = reverse
save_control(frame, control)
client.send_control(control)
def build_experiments(self):
"""
Creates the whole set of experiment objects,
The experiments created depend on the selected Town.
"""
# We set the camera
# This single RGB camera is used on every experiment
camera = Camera('CameraRGB')
camera.set(FOV=100)
camera.set_image_size(800, 600)
camera.set_position(2.0, 0.0, 1.4)
camera.set_rotation(-15.0, 0, 0)
if self._task == 'go-straight':
task_select = 0
elif self._task == 'right_turn':
task_select = 1
elif self._task == 'left_turn':
task_select = 2
elif self._task == 'intersection-straight':
task_select = 3
elif self._task == 'intersection-right':
task_select = 4
elif self._task == 'intersection-left':
task_select = 5
if self._city_name == 'Town01':
poses_tasks = self._poses_town01()
vehicles_tasks = [0, 0, 0, 20]
pedestrians_tasks = [0, 0, 0, 50]
elif self._city_name == 'Town02':
poses_tasks = self._poses_town02()
vehicles_tasks = [0, 0, 0, 20]
pedestrians_tasks = [0, 0, 0, 50]
elif 'Town01_nemesis' in self._city_name:
poses_tasks = [self._poses_town01_nemesis()[task_select]]
vehicles_tasks = [0, 0, 0, 0, 0, 0]
pedestrians_tasks = [0, 0, 0, 0, 0, 0]
elif 'town01' in self._city_name:
poses_tasks = [self._poses_town01_nemesis()[task_select]]
vehicles_tasks = [0, 0, 0, 0, 0, 0]
pedestrians_tasks = [0, 0, 0, 0, 0, 0]
elif 'Town02_nemesis' in self._city_name:
poses_tasks = [self._poses_town02_nemesis()[task_select]]
vehicles_tasks = [0, 0, 0, 0, 0, 0]
pedestrians_tasks = [0, 0, 0, 0, 0, 0]
experiments_vector = []
"""repeating experiment"""
num_iters = 1
weathers_iters = [self._weather] * num_iters
# 0 - Default
# 1 - ClearNoon
# 2 - CloudyNoon
# 3 - WetNoon
# 4 - WetCloudyNoon
# 5 - MidRainyNoon
# 6 - HardRainNoon
# 7 - SoftRainNoon
# 8 - ClearSunset
# 9 - CloudySunset
# 10 - WetSunset
# 11 - WetCloudySunset
# 12 - MidRainSunset
# 13 - HardRainSunset
# 14 - SoftRainSunset
for weather in weathers_iters:
# for iteration in range(len(poses_tasks)):
for iteration in range(0, 1):
poses = poses_tasks[iteration]
vehicles = vehicles_tasks[iteration]
pedestrians = pedestrians_tasks[iteration]
conditions = CarlaSettings()
conditions.set(SendNonPlayerAgentsInfo=True,
NumberOfVehicles=vehicles,
NumberOfPedestrians=pedestrians,
WeatherId=weather)
# Add all the cameras that were set for this experiments
conditions.add_sensor(camera)
experiment = Experiment()
experiment.set(Conditions=conditions,
Poses=poses,
Task=iteration,
Repetitions=1)
experiments_vector.append(experiment)
return experiments_vector
class CarlaEnvironmentWrapper(EnvironmentWrapper):
def __init__(self, tuning_parameters):
EnvironmentWrapper.__init__(self, tuning_parameters)
self.tp = tuning_parameters
# server configuration
self.server_height = self.tp.env.server_height
self.server_width = self.tp.env.server_width
self.port = get_open_port()
self.host = 'localhost'
self.map = CarlaLevel().get(self.tp.env.level)
# client configuration
self.verbose = self.tp.env.verbose
self.depth = self.tp.env.depth
self.stereo = self.tp.env.stereo
self.semantic_segmentation = self.tp.env.semantic_segmentation
self.height = self.server_height * (1 + int(self.depth) +
int(self.semantic_segmentation))
self.width = self.server_width * (1 + int(self.stereo))
self.size = (self.width, self.height)
self.config = self.tp.env.config
if self.config:
# load settings from file
with open(self.config, 'r') as fp:
self.settings = fp.read()
else:
# hard coded settings
self.settings = CarlaSettings()
self.settings.set(SynchronousMode=True,
SendNonPlayerAgentsInfo=False,
NumberOfVehicles=15,
NumberOfPedestrians=30,
WeatherId=1)
self.settings.randomize_seeds()
# add cameras
camera = Camera('CameraRGB')
camera.set_image_size(self.width, self.height)
camera.set_position(200, 0, 140)
camera.set_rotation(0, 0, 0)
self.settings.add_sensor(camera)
# open the server
self.server = self._open_server()
logging.disable(40)
# open the client
self.game = CarlaClient(self.host, self.port, timeout=99999999)
self.game.connect()
scene = self.game.load_settings(self.settings)
# get available start positions
positions = scene.player_start_spots
self.num_pos = len(positions)
self.iterator_start_positions = 0
# action space
self.discrete_controls = False
self.action_space_size = 2
self.action_space_high = [1, 1]
self.action_space_low = [-1, -1]
self.action_space_abs_range = np.maximum(
np.abs(self.action_space_low), np.abs(self.action_space_high))
self.steering_strength = 0.5
self.gas_strength = 1.0
self.brake_strength = 0.5
self.actions = {
0: [0., 0.],
1: [0., -self.steering_strength],
2: [0., self.steering_strength],
3: [self.gas_strength, 0.],
4: [-self.brake_strength, 0],
5: [self.gas_strength, -self.steering_strength],
6: [self.gas_strength, self.steering_strength],
7: [self.brake_strength, -self.steering_strength],
8: [self.brake_strength, self.steering_strength]
}
self.actions_description = [
'NO-OP', 'TURN_LEFT', 'TURN_RIGHT', 'GAS', 'BRAKE',
'GAS_AND_TURN_LEFT', 'GAS_AND_TURN_RIGHT', 'BRAKE_AND_TURN_LEFT',
'BRAKE_AND_TURN_RIGHT'
]
for idx, action in enumerate(self.actions_description):
for key in key_map.keys():
if action == key:
self.key_to_action[key_map[key]] = idx
self.num_speedup_steps = 30
# measurements
self.measurements_size = (1, )
self.autopilot = None
# env initialization
self.reset(True)
# render
if self.is_rendered:
image = self.get_rendered_image()
self.renderer.create_screen(image.shape[1], image.shape[0])
def _open_server(self):
log_path = path.join(logger.experiments_path,
"CARLA_LOG_{}.txt".format(self.port))
with open(log_path, "wb") as out:
cmd = [
path.join(environ.get('CARLA_ROOT'),
'CarlaUE4.sh'), self.map, "-benchmark",
"-carla-server", "-fps=10", "-world-port={}".format(self.port),
"-windowed -ResX={} -ResY={}".format(self.server_width,
self.server_height),
"-carla-no-hud"
]
if self.config:
cmd.append("-carla-settings={}".format(self.config))
p = subprocess.Popen(cmd, stdout=out, stderr=out)
return p
def _close_server(self):
os.killpg(os.getpgid(self.server.pid), signal.SIGKILL)
def _update_state(self):
# get measurements and observations
measurements = []
while type(measurements) == list:
measurements, sensor_data = self.game.read_data()
self.location = (measurements.player_measurements.transform.location.x,
measurements.player_measurements.transform.location.y,
measurements.player_measurements.transform.location.z)
is_collision = measurements.player_measurements.collision_vehicles != 0 \
or measurements.player_measurements.collision_pedestrians != 0 \
or measurements.player_measurements.collision_other != 0
speed_reward = measurements.player_measurements.forward_speed - 1
if speed_reward > 30.:
speed_reward = 30.
self.reward = speed_reward \
- (measurements.player_measurements.intersection_otherlane * 5) \
- (measurements.player_measurements.intersection_offroad * 5) \
- is_collision * 100 \
- np.abs(self.control.steer) * 10
# update measurements
self.state = {
'observation': sensor_data['CameraRGB'].data,
'measurements': [measurements.player_measurements.forward_speed],
}
self.autopilot = measurements.player_measurements.autopilot_control
# action_p = ['%.2f' % member for member in [self.control.throttle, self.control.steer]]
# screen.success('REWARD: %.2f, ACTIONS: %s' % (self.reward, action_p))
if (measurements.game_timestamp >=
self.tp.env.episode_max_time) or is_collision:
# screen.success('EPISODE IS DONE. GameTime: {}, Collision: {}'.format(str(measurements.game_timestamp),
# str(is_collision)))
self.done = True
def _take_action(self, action_idx):
if type(action_idx) == int:
action = self.actions[action_idx]
else:
action = action_idx
self.last_action_idx = action
self.control = VehicleControl()
self.control.throttle = np.clip(action[0], 0, 1)
self.control.steer = np.clip(action[1], -1, 1)
self.control.brake = np.abs(np.clip(action[0], -1, 0))
if not self.tp.env.allow_braking:
self.control.brake = 0
self.control.hand_brake = False
self.control.reverse = False
self.game.send_control(self.control)
def _restart_environment_episode(self, force_environment_reset=False):
self.iterator_start_positions += 1
if self.iterator_start_positions >= self.num_pos:
self.iterator_start_positions = 0
try:
self.game.start_episode(self.iterator_start_positions)
except:
self.game.connect()
self.game.start_episode(self.iterator_start_positions)
# start the game with some initial speed
state = None
for i in range(self.num_speedup_steps):
state = self.step([1.0, 0])['state']
self.state = state
return state
def run_carla_client(args):
# Here we will run args._episode episodes with args._frames frames each.
number_of_episodes = args._episode
frames_per_episode = args._frames
# set speed and yaw rate variable to default
speedyawrate = np.nan
#call init in eval file to load model
checkpoint_dir_loc = args.chckpt_loc
prefix = args.model_name
tf_carla_eval.init(checkpoint_dir_loc, prefix)
# create the carla client
with make_carla_client(args.host, args.port, timeout=100000) as client:
print('CarlaClient connected')
for episode in range(0, number_of_episodes):
if args.settings_filepath is None:
# if same starting position arg set use same weather
if args._sameStart > -1:
choice = 1
nrVehicles = 0
nrPedestrians = 0
else:
choice = random.choice([1, 3, 7, 8, 14])
nrVehicles = 150
nrPedestrians = 100
settings = CarlaSettings()
settings.set(SynchronousMode=args.synchronous_mode,
SendNonPlayerAgentsInfo=True,
NumberOfVehicles=nrVehicles,
NumberOfPedestrians=nrPedestrians,
WeatherId=choice,
QualityLevel=args.quality_level)
settings.randomize_seeds()
camera0 = Camera('CameraRGB')
camera0.set_image_size(1920, 640)
camera0.set_position(2.00, 0, 1.30)
settings.add_sensor(camera0)
camera1 = Camera('CameraDepth', PostProcessing='Depth')
camera1.set_image_size(1920, 640)
camera1.set_position(2.00, 0, 1.30)
settings.add_sensor(camera1)
camera2 = Camera('CameraSegmentation',
PostProcessing='SemanticSegmentation')
camera2.set_image_size(1920, 640)
camera2.set_position(2.00, 0, 1.30)
settings.add_sensor(camera2)
if args._save:
camera3 = Camera('CameraRGB_top',
PostProcessing='SceneFinal')
camera3.set_image_size(800, 800)
camera3.set_position(-6.0, 0, 4.0)
camera3.set_rotation(yaw=0, roll=0, pitch=-10)
settings.add_sensor(camera3)
else:
with open(args.settings_filepath, 'r') as fp:
settings = fp.read()
scene = client.load_settings(settings)
# Choose one player start at random.
number_of_player_starts = len(scene.player_start_spots)
if args._sameStart > -1:
player_start = args._sameStart
else:
player_start = random.randint(
0, max(0, number_of_player_starts - 1))
print('Starting new episode at %r...' % scene.map_name)
client.start_episode(player_start)
if args._save:
# initialize stuck variable. if car does not move after colliding for x frames we restart episode.
nrStuck = 0
# maximum number of times we can get stuck before restarting
maxStuck = 5
# last location variable to keep track if car is changing position
last_loc = namedtuple('last_loc', 'x y z')
last_loc.__new__.__defaults__ = (0.0, 0.0, 0.0)
last_loc.x = -1
last_loc.y = -1
# delete frames of previous run to not interfere with video creation
for rmf in glob.glob(args.file_loc + 'tmpFrameFolder/frame*'):
os.remove(rmf)
# Iterate every frame in the episode.
for frame in range(0, frames_per_episode):
measurements, sensor_data = client.read_data()
# if we are saving video of episode move to next episode if we get stuck
if args._save:
player_measurements = measurements.player_measurements
col_other = player_measurements.collision_other
col_cars = player_measurements.collision_vehicles
curr_loc = player_measurements.transform.location
if player_measurements.forward_speed <= 0.05 and sqrdist(
last_loc, curr_loc) < 2 and frame > 30:
if nrStuck == maxStuck:
print(
"\nWe are stuck! Writing to video and restarting."
)
#args._sameStart += 1
break
else:
print("Stuck: " + str(nrStuck) + "/" +
str(maxStuck))
nrStuck += 1
last_loc = curr_loc
# Skip first couple of images due to setup time.
if frame > 19:
player_measurements = measurements.player_measurements
for name, curr_measurements in sensor_data.items():
if name == 'CameraDepth':
depth = curr_measurements.return_depth_map()
if name == 'CameraSegmentation':
segmentation = curr_measurements.return_segmentation_map(
)
if name == 'CameraRGB':
rgb = curr_measurements.return_rgb()
yaw, yaw_rate, speed, prev_time = process_odometry(
measurements, yaw_shift, yaw_old, prev_time)
yaw_old = yaw
im = Image.new('L', (256 * 6, 256 * 6), (127))
shift = 256 * 6 / 2
draw = ImageDraw.Draw(im)
gmTime = time.time()
for agent in measurements.non_player_agents:
if agent.HasField('vehicle') or agent.HasField(
'pedestrian'):
participant = agent.vehicle if agent.HasField(
'vehicle') else agent.pedestrian
if not participantInRange(
participant.transform.location,
player_measurements.transform.location):
continue
angle = cart2pol(
participant.transform.orientation.x,
participant.transform.orientation.y)
polygon = agent2world(participant, angle)
polygon = world2player(
polygon, math.radians(-yaw),
player_measurements.transform)
polygon = player2image(polygon,
shift,
multiplier=25)
polygon = [tuple(row) for row in polygon.T]
draw.polygon(polygon, 0, 0)
im = im.resize((256, 256), Image.ANTIALIAS)
im = im.rotate(imrotate)
gmTime = time.time() - gmTime
if not args.all_data:
print("only all data supported for now")
exit()
else:
if args._opd:
ppTime, evTime, imTime, tkTime, speedyawrate, direction = tf_carla_eval.eval_only_drive(
im, rgb, depth, segmentation, -yaw_rate, speed)
else:
ppTime, evTime, imTime, tkTime, speedyawrate, direction, imReady, im = tf_carla_eval.eval(
im, rgb, depth, segmentation, -yaw_rate, speed)
if args._save and imReady:
# append frame to array for video output
for name, curr_measurements in sensor_data.items(
):
if name == 'CameraRGB_top':
filename = args.file_loc + "tmpFrameFolder/frame" + str(
frame).zfill(4) + ".jpg"
Image.fromarray(
np.concatenate([
im,
curr_measurements.return_rgb()
], 1)).save(filename)
break
#printTimePerEval(gmTime, ppTime, evTime, imTime, tkTime)
else:
# get first values
yaw_shift = measurements.player_measurements.transform.rotation.yaw
yaw_old = ((yaw_shift - 180) % 360) - 180
imrotate = round(yaw_old) + 90
shift_x = measurements.player_measurements.transform.location.x
shift_y = measurements.player_measurements.transform.location.y
prev_time = np.int64(measurements.game_timestamp)
if not args.autopilot:
control = getKeyboardControl()
else:
# values are nan while script is gathering frames before first prediction
if np.any(np.isnan(speedyawrate)):
control = measurements.player_measurements.autopilot_control
control.steer += random.uniform(-0.01, 0.01)
else:
control = VehicleControl()
# speedyawrate contains T entries, first entry is first prediction
dirAvgEncoding = np.mean(
np.squeeze(np.array(direction)), 0)
speedyawrate = np.asarray(speedyawrate)
steerAvgEncoding = np.mean(np.squeeze(speedyawrate), 0)
control.throttle = mapThrottle(player_measurements,
speedyawrate)
control.brake = int(control.throttle == -1)
control.steer = mapSteer(steerAvgEncoding[1])
#printSteering(measurements.player_measurements.forward_speed,
# measurements.player_measurements.autopilot_control.throttle,
# measurements.player_measurements.autopilot_control.steer,
# speedyawrate, control.throttle, control.steer,
# dirAvgEncoding, steerAvgEncoding)
client.send_control(control)
if args._save:
print("\nConverting frames to video...")
os.system(
"ffmpeg -r 10 -f image2 -start_number 20 -i {}frame%04d.jpg -vcodec libx264 -crf 15 -pix_fmt yuv420p {}"
.format(
args.file_loc + "tmpFrameFolder/", args.file_loc +
"video" + str(time.time() * 1000) + ".mp4"))
print("Finished conversion.")
def make_carla_settings():
"""Make a CarlaSettings object with the settings we need."""
settings = CarlaSettings()
settings.set(SendNonPlayerAgentsInfo=True,
SynchronousMode=True,
NumberOfVehicles=30,
NumberOfPedestrians=50,
WeatherId=1)
settings.set(DisableTwoWheeledVehicles=True)
settings.randomize_seeds() # IMPORTANT TO RANDOMIZE THE SEEDS EVERY TIME
camera0 = sensor.Camera('CentralSemanticSeg',
PostProcessing='SemanticSegmentation')
camera0.set_image_size(WINDOW_WIDTH, WINDOW_HEIGHT)
camera0.set(FOV=FOV)
camera0.set_position(2.0, 0.0, 1.4)
camera0.set_rotation(-15.0, 0, 0)
settings.add_sensor(camera0)
camera0 = sensor.Camera('CentralRGB')
camera0.set_image_size(WINDOW_WIDTH, WINDOW_HEIGHT)
camera0.set(FOV=FOV)
camera0.set_position(2.0, 0.0, 1.4)
camera0.set_rotation(-15.0, 0, 0)
settings.add_sensor(camera0)
camera0 = sensor.Camera('CentralDepth', PostProcessing='Depth')
camera0.set_image_size(WINDOW_WIDTH, WINDOW_HEIGHT)
camera0.set(FOV=FOV)
camera0.set_position(2.0, 0.0, 1.4)
camera0.set_rotation(-15.0, 0, 0)
settings.add_sensor(camera0)
camera0 = sensor.Camera('LeftSemanticSeg',
PostProcessing='SemanticSegmentation')
camera0.set_image_size(WINDOW_WIDTH, WINDOW_HEIGHT)
camera0.set(FOV=FOV)
camera0.set_position(2.0, 0.0, 1.4)
camera0.set_rotation(-15.0, -30.0, 0)
settings.add_sensor(camera0)
camera0 = sensor.Camera('LeftRGB')
camera0.set_image_size(WINDOW_WIDTH, WINDOW_HEIGHT)
camera0.set(FOV=FOV)
camera0.set_position(2.0, 0.0, 1.4)
camera0.set_rotation(-15.0, -30.0, 0)
settings.add_sensor(camera0)
camera0 = sensor.Camera('LeftDepth', PostProcessing='Depth')
camera0.set_image_size(WINDOW_WIDTH, WINDOW_HEIGHT)
camera0.set(FOV=FOV)
camera0.set_position(2.0, 0.0, 1.4)
camera0.set_rotation(-15.0, -30.0, 0)
settings.add_sensor(camera0)
camera0 = sensor.Camera('RightSemanticSeg',
PostProcessing='SemanticSegmentation')
camera0.set_image_size(WINDOW_WIDTH, WINDOW_HEIGHT)
camera0.set(FOV=FOV)
camera0.set_position(2.0, 0.0, 1.4)
camera0.set_rotation(-15.0, 30.0, 0)
settings.add_sensor(camera0)
camera0 = sensor.Camera('RightRGB')
camera0.set_image_size(WINDOW_WIDTH, WINDOW_HEIGHT)
camera0.set(FOV=FOV)
camera0.set_position(2.0, 0.0, 1.4)
camera0.set_rotation(-15.0, 30.0, 0)
settings.add_sensor(camera0)
camera0 = sensor.Camera('RightDepth', PostProcessing='Depth')
camera0.set_image_size(WINDOW_WIDTH, WINDOW_HEIGHT)
camera0.set(FOV=FOV)
camera0.set_position(2.0, 0.0, 1.4)
camera0.set_rotation(-15.0, 30.0, 0)
settings.add_sensor(camera0)
lidar = sensor.Lidar('Lidar32')
lidar.set_position(0, 0, 2.5)
lidar.set_rotation(0, 0, 0)
lidar.set(Channels=32,
Range=50,
PointsPerSecond=100000,
RotationFrequency=10,
UpperFovLimit=10,
LowerFovLimit=-30)
settings.add_sensor(lidar)
return settings
def run_carla_client(args, classifier, plt, plt_index):
global global_steering_indicator
# Here we will run 3 episodes with 300 frames each.
frames_per_episode = args.frames
if args.position:
number_of_episodes = 1
else:
number_of_episodes = 3
# We assume the CARLA server is already waiting for a client to connect at
# host:port. To create a connection we can use the `make_carla_client`
# context manager, it creates a CARLA client object and starts the
# connection. It will throw an exception if something goes wrong. The
# context manager makes sure the connection is always cleaned up on exit.
with make_carla_client(args.host, args.port) as client:
print('CarlaClient connected')
for episode in range(0, number_of_episodes):
# Start a new episode.
if args.settings_filepath is None:
# Create a CarlaSettings object. This object is a wrapper around
# the CarlaSettings.ini file. Here we set the configuration we
# want for the new episode.
settings = CarlaSettings()
settings.set(
SynchronousMode=True,
SendNonPlayerAgentsInfo=True,
NumberOfVehicles=0,
NumberOfPedestrians=0,
WeatherId=random.choice([2]),
QualityLevel=args.quality_level)
settings.randomize_seeds()
# Now we want to add a couple of cameras to the player vehicle.
# We will collect the images produced by these cameras every
# frame.
# # The default camera captures RGB images of the scene.
# camera0 = Camera('CameraRGB')
# # Set image resolution in pixels.
# camera0.set_image_size(256, 256)
# # Set its position relative to the car in meters.
# camera0.set_position(2.2, 0, 1.30)
# settings.add_sensor(camera0)
if args.lidar:
# Adding a lidar sensor
lidar = Lidar('Lidar32')
lidar.set_position(0, 0, 2.50)
lidar.set_rotation(0, 0, 0)
lidar.set(
Channels=32,
Range=50,
PointsPerSecond=args.lidar_pps,
RotationFrequency=10,
UpperFovLimit=10,
LowerFovLimit=-30)
settings.add_sensor(lidar)
else:
# Let's add another camera producing ground-truth depth.
camera1 = Camera('CameraDepth', PostProcessing='Depth')
camera1.set_image_size(256, 256)
camera1.set_position(2.2, 0, 1.30)
camera1.set(FOV=90.0)
#camera1.set_rotation(pitch=-8, yaw=0, roll=0)
settings.add_sensor(camera1)
camera2 = Camera('CameraSeg', PostProcessing='SemanticSegmentation')
camera2.set_image_size(256, 256)
camera2.set_position(2.2, 0, 1.30)
camera2.set(FOV=90.0)
#camera2.set_rotation(pitch=-8, yaw=0, roll=0)
settings.add_sensor(camera2)
if args.capture:
camera3 = Camera('CameraRGB')
camera3.set_image_size(512, 256)
camera3.set_position(-8, 0, 5)
camera3.set(FOV=90.0)
camera3.set_rotation(pitch=-20, yaw=0, roll=0)
settings.add_sensor(camera3)
else:
# Alternatively, we can load these settings from a file.
with open(args.settings_filepath, 'r') as fp:
settings = fp.read()
# Now we load these settings into the server. The server replies
# with a scene description containing the available start spots for
# the player. Here we can provide a CarlaSettings object or a
# CarlaSettings.ini file as string.
scene = client.load_settings(settings)
# Choose one player start at random.
number_of_player_starts = len(scene.player_start_spots)
if args.position:
player_start = args.position
else:
player_start = random.choice([42,67,69,79,94,97, 70, 44,85,102])
# Notify the server that we want to start the episode at the
# player_start index. This function blocks until the server is ready
# to start the episode.
print('Starting new episode at %r...' % scene.map_name)
""" Begin added code """
if args.capture:
directory = '_capture/pos{}'.format(player_start)
if not os.path.exists(directory):
os.makedirs(directory)
print("Capturing point clouds to {}".format(directory))
""" End added code """
client.start_episode(player_start)
# Iterate every frame in the episode.
for frame in range(0, frames_per_episode):
# Read the data produced by the server this frame.
measurements, sensor_data = client.read_data()
""" Begin added code """
# dict of steering directions of available curves
# [1,0] if the next curve will be a left one
# [0,1] if the next curve will be a right one
directions = {
67: [[1,0]], # straight
42: [[1,0]], # straight or left
69: [[0,1]], # right
79: [[0,1]], # right
94: [[0,1]], # right
97: [[0,1]], # right
70: [[1,0]], # left
44: [[1,0]], # left
85: [[1,0]], # left
102: [[1,0]] # left
}
#dynamically set the global steering indicator during the game
if(args.key_control):
set_steering_indicator_from_keypress()
steering_direction = args.steering_indicator or global_steering_indicator
if args.use_steering_indicator:
if steering_direction == "left":
steering_indicator = [[1,0]]
elif steering_direction == "right":
steering_indicator = [[0,1]]
else:
steering_indicator = [[0,0]]
steering_indicator = torch.Tensor(steering_indicator)
if cuda_available:
steering_indicator = steering_indicator.cuda()
if args.lidar:
point_cloud = sensor_data['Lidar32'].data
if args.lidar_fov == 180:
print("FOV 180")
print("Before", point_cloud.shape)
point_cloud = filter_fov(point_cloud)
print("After", point_cloud.shape)
else:
# Get depth and seg as numpy array for further processing
depth_obj = sensor_data['CameraDepth']
depth = depth_obj.data
fov = depth_obj.fov
# Filter seg to get intersection points
seg = sensor_data['CameraSeg'].data
filtered_seg = pp.filter_seg_image(seg)
# Add following lines to measure performance of generating pointcloud
# def f():
# return pp.depth_to_local_point_cloud(depth, fov, filtered_seg)
# print(timeit.timeit(f, number=1000) / 1000)
# Create pointcloud from seg and depth (but only take intersection points)
point_cloud = pp.depth_to_local_point_cloud(depth, fov, filtered_seg)
# Save point cloud for later visualization
if args.capture:
pp.save_to_disk(point_cloud, "{}/point_clouds/point_cloud_{:0>5d}.ply".format(directory, frame))
sensor_data['CameraRGB'].save_to_disk('{}/images/image_{:0>5d}.png'.format(directory, frame))
# Normalizing the point cloud
if not args.no_center or not args.no_norm:
point_cloud = point_cloud - np.expand_dims(np.mean(point_cloud, axis=0), 0) # center
if not args.no_norm:
dist = np.max(np.sqrt(np.sum(point_cloud ** 2, axis=1)), 0)
point_cloud = point_cloud / dist # scale
#pp.save_point_cloud(point_cloud, 'test')
# pp.save_seg_image(filtered_seg)
""" End added code """
# Print some of the measurements.
#print_measurements(measurements)
# # Save the images to disk if requested.
# if args.save_images_to_disk:
# for name, measurement in sensor_data.items():
# filename = args.out_filename_format.format(episode, name, frame)
# measurement.save_to_disk(filename)
# We can access the encoded data of a given image as numpy
# array using its "data" property. For instance, to get the
# depth value (normalized) at pixel X, Y
#
# depth_array = sensor_data['CameraDepth'].data
# value_at_pixel = depth_array[Y, X]
#
# Now we have to send the instructions to control the vehicle.
# If we are in synchronous mode the server will pause the
# simulation until we send this control.
if not args.autopilot:
client.send_control(
steer=random.uniform(-1.0, 1.0),
throttle=0.5,
brake=0.0,
hand_brake=False,
reverse=False)
else:
# Together with the measurements, the server has sent the
# control that the in-game autopilot would do this frame. We
# can enable autopilot by sending back this control to the
# server. We can modify it if wanted, here for instance we
# will add some noise to the steer.
""" Beginn added code """
control = measurements.player_measurements.autopilot_control
#steer = control.steer
#print(control)
point_cloud = point_cloud.transpose()
points = np.expand_dims(point_cloud, axis=0)
points = torch.from_numpy(points.astype(np.float32))
#print(points)
if cuda_available:
points = points.cuda()
classifier = classifier.eval()
if args.use_steering_indicator:
#print("Using PointNetReg2")
steer, _, _ = classifier((points, steering_indicator))
else:
#print("Using PointNetReg")
steer, _, _ = classifier(points)
steer = steer.item()
if args.use_steering_indicator:
print("Using steering indicator: {} / {}".format(steering_direction, steering_indicator))
print("Autopilot steer: ", control.steer)
print("Pointnet steer: ", steer)
print("Relative difference: ", steer / control.steer if control.steer != 0.0 else 'NaN')
print("Absolute difference: ", control.steer - steer)
print("FOV:", args.lidar_fov)
# Plot some visualizations to visdom
if args.visdom:
plt.plot_point_cloud(points, var_name='pointcloud')
plt.plot('steering', 'PointNet', 'Steering', plt_index, steer)
plt.plot('steering', 'Autopilot', 'Steering', plt_index, control.steer)
plt_index += 1
# Let pointnet take over steering control
if True:
print("Who's in command: POINTNET")
control.steer = steer
if args.ignore_red_lights or args.use_steering_indicator:
control.throttle = 0.5
control.brake=0.0
hand_brake=False
else:
print("Who's in command: AUTOPILOT")
print("_________")
#pdb.set_trace()
""" End added code """
client.send_control(control)
def run_carla_client(host, port, autopilot_on, save_images_to_disk,
image_filename_format):
# Here we will run 3 episodes with 300 frames each.
number_of_episodes = 3
frames_per_episode = 300
# We assume the CARLA server is already waiting for a client to connect at
# host:port. To create a connection we can use the `make_carla_client`
# context manager, it creates a CARLA client object and starts the
# connection. It will throw an exception if something goes wrong. The
# context manager makes sure the connection is always cleaned up on exit.
with make_carla_client(host, port) as client:
print('CarlaClient connected')
for episode in range(0, number_of_episodes):
# Start a new episode.
# Create a CarlaSettings object. This object is a handy wrapper
# around the CarlaSettings.ini file. Here we set the configuration
# we want for the new episode.
settings = CarlaSettings()
settings.set(SynchronousMode=True,
NumberOfVehicles=30,
NumberOfPedestrians=50,
WeatherId=random.choice([1, 3, 7, 8, 14]))
settings.randomize_seeds()
# Now we want to add a couple of cameras to the player vehicle. We
# will collect the images produced by these cameras every frame.
# The default camera captures RGB images of the scene.
camera0 = Camera('CameraRGB')
# Set image resolution in pixels.
camera0.set_image_size(800, 600)
# Set its position relative to the car in centimeters.
camera0.set_position(30, 0, 130)
settings.add_camera(camera0)
# Let's add another camera producing ground-truth depth.
camera1 = Camera('CameraDepth', PostProcessing='Depth')
camera1.set_image_size(800, 600)
camera1.set_position(30, 0, 130)
settings.add_camera(camera1)
print('Requesting new episode...')
# Now we request a new episode with these settings. The server
# replies with a scene description containing the available start
# spots for the player. Here instead of a CarlaSettings object we
# could also provide a CarlaSettings.ini file as string.
scene = client.request_new_episode(settings)
# Choose one player start at random.
number_of_player_starts = len(scene.player_start_spots)
player_start = random.randint(0, max(0,
number_of_player_starts - 1))
# Notify the server that we want to start the episode at
# `player_start`. This function blocks until the server is ready to
# start the episode.
client.start_episode(player_start)
# Iterate every frame in the episode.
for frame in range(0, frames_per_episode):
# Read the measurements and images produced by the server this
# frame.
measurements, images = client.read_measurements()
# Print some of the measurements we received.
print_player_measurements(measurements.player_measurements)
# Save the images to disk if requested.
if save_images_to_disk:
for n, image in enumerate(images):
image.save_to_disk(
image_filename_format.format(episode, n, frame))
# Now we have to send the instructions to control the vehicle.
# If we are in synchronous mode the server will pause the
# simulation until we send this control.
if not autopilot_on:
client.send_control(steer=random.uniform(-1.0, 1.0),
throttle=0.3,
brake=False,
hand_brake=False,
reverse=False)
else:
# Together with the measurements, the server has sent the
# control that the in-game AI would do this frame. We can
# enable autopilot by sending back this control to the
# server. Here we will also add some noise to the steer.
control = measurements.player_measurements.ai_control
control.steer += random.uniform(-0.1, 0.1)
client.send_control(control)
print('Done.')
return True
class CarlaRosBridge(object):
"""
Carla Ros bridge
"""
def __init__(self, client, params):
"""
:param params: dict of parameters, see settings.yaml
:param rate: rate to query data from carla in Hz
"""
self.setup_carla_client(client=client, params=params)
self.frames_per_episode = params['Framesperepisode']
self.tf_to_publish = []
self.msgs_to_publish = []
self.publishers = {}
# definitions useful for time
self.cur_time = rospy.Time.from_sec(
0) # at the beginning of simulation
self.carla_game_stamp = 0
self.carla_platform_stamp = 0
# creating handler to handle vehicles messages
self.player_handler = PlayerAgentHandler(
"player_vehicle", process_msg_fun=self.process_msg)
self.non_players_handler = NonPlayerAgentsHandler(
"vehicles", process_msg_fun=self.process_msg)
# creating handler for sensors
self.sensors = {}
for name, _ in self.param_sensors.items():
self.add_sensor(name)
# creating input controller listener
self.input_controller = InputController()
def setup_carla_client(self, client, params):
self.client = client
self.param_sensors = params.get('sensors', {})
self.carla_settings = CarlaSettings()
self.carla_settings.set(
SendNonPlayerAgentsInfo=params.get('SendNonPlayerAgentsInfo',
True),
NumberOfVehicles=params.get('NumberOfVehicles', 20),
NumberOfPedestrians=params.get('NumberOfPedestrians', 40),
WeatherId=params.get('WeatherId', random.choice([1, 3, 7, 8, 14])),
SynchronousMode=params.get('SynchronousMode', True),
QualityLevel=params.get('QualityLevel', 'Low'))
self.carla_settings.randomize_seeds()
def add_sensor(self, name):
rospy.loginfo("Adding sensor {}".format(name))
sensor_type = self.param_sensors[name]['SensorType']
params = self.param_sensors[name]['carla_settings']
sensor_handler = None
if sensor_type == 'LIDAR_RAY_CAST':
sensor_handler = LidarHandler
if sensor_type == 'CAMERA':
sensor_handler = CameraHandler
if sensor_handler:
self.sensors[name] = sensor_handler(
name,
params,
carla_settings=self.carla_settings,
process_msg_fun=self.process_msg)
else:
rospy.logerr(
"Unable to handle sensor {name} of type {sensor_type}".format(
sensor_type=sensor_type, name=name))
def on_shutdown(self):
rospy.loginfo("Shutdown requested")
def process_msg(self, topic=None, msg=None):
"""
Function used to process message
Here we create publisher if not yet created
Store the message in a list (waiting for their publication) with their associated publisher
Messages for /tf topics are handle differently in order to publish all transform in the same message
:param topic: topic to publish the message on
:param msg: ros message
"""
if topic not in self.publishers:
if topic == 'tf':
self.publishers[topic] = rospy.Publisher(topic,
TFMessage,
queue_size=100)
else:
self.publishers[topic] = rospy.Publisher(topic,
type(msg),
queue_size=10)
if topic == 'tf':
# transform are merged in same message
self.tf_to_publish.append(msg)
else:
self.msgs_to_publish.append((self.publishers[topic], msg))
def send_msgs(self):
for publisher, msg in self.msgs_to_publish:
publisher.publish(msg)
self.msgs_to_publish = []
tf_msg = TFMessage(self.tf_to_publish)
self.publishers['tf'].publish(tf_msg)
self.tf_to_publish = []
def compute_cur_time_msg(self):
self.process_msg('clock', Clock(self.cur_time))
def run(self):
self.publishers['clock'] = rospy.Publisher("clock",
Clock,
queue_size=10)
# load settings into the server
scene = self.client.load_settings(self.carla_settings)
# Choose one player start at random.
number_of_player_starts = len(scene.player_start_spots)
player_start = random.randint(0, max(0, number_of_player_starts - 1))
# Send occupancy grid to rivz
map_handler = MapHandler(scene.map_name)
map_handler.send_map()
self.client.start_episode(player_start)
for frame in count():
if (frame == self.frames_per_episode) or rospy.is_shutdown():
break
measurements, sensor_data = self.client.read_data()
# handle time
self.carla_game_stamp = measurements.game_timestamp
self.cur_time = rospy.Time.from_sec(self.carla_game_stamp * 1e-3)
self.compute_cur_time_msg()
# handle agents
self.player_handler.process_msg(measurements.player_measurements,
cur_time=self.cur_time)
self.non_players_handler.process_msg(
measurements.non_player_agents, cur_time=self.cur_time)
# handle sensors
for name, data in sensor_data.items():
self.sensors[name].process_sensor_data(data, self.cur_time)
# publish all messages
self.send_msgs()
# handle control
if rospy.get_param('carla_autopilot', True):
control = measurements.player_measurements.autopilot_control
self.client.send_control(control)
else:
control = self.input_controller.cur_control
self.client.send_control(**control)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
rospy.loginfo("Exiting Bridge")
return None
def run_carla_client(args):
# Here we will run 3 episodes with 300 frames each.
number_of_episodes = 15
frames_per_episode = 1400
# [0 , 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10, 11, 12, 13, 14]
vehicles_num = [
30, 60, 50, 80, 120, 100, 100, 120, 110, 100, 80, 70, 70, 80, 100
]
# We assume the CARLA server is already waiting for a client to connect at
# host:port. To create a connection we can use the `make_carla_client`
# context manager, it creates a CARLA client object and starts the
# connection. It will throw an exception if something goes wrong. The
# context manager makes sure the connection is always cleaned up on exit.
with make_carla_client(args.host, args.port) as client:
print('CarlaClient connected')
for episode in range(10, number_of_episodes):
# Start a new episode.
if args.settings_filepath is None:
# Create a CarlaSettings object. This object is a wrapper around
# the CarlaSettings.ini file. Here we set the configuration we
# want for the new episode.
settings = CarlaSettings()
settings.set(
SynchronousMode=True,
SendNonPlayerAgentsInfo=False,
NumberOfVehicles=vehicles_num[
episode], #random.choice([0, 20, 15, 20, 25, 21, 24, 18, 40, 35, 25, 30]), #25,
NumberOfPedestrians=80,
DisableTwoWheeledVehicles=False,
WeatherId=episode, #1, #random.choice([1, 3, 7, 8, 14]),
QualityLevel=args.quality_level)
settings.randomize_seeds()
# Now we want to add a couple of cameras to the player vehicle.
# We will collect the images produced by these cameras every
# frame.
# LEFT RGB CAMERA
camera_l = Camera('LeftCameraRGB', PostProcessing='SceneFinal')
camera_l.set_image_size(800, 600)
camera_l.set_position(1.30, -0.27, 1.50)
settings.add_sensor(camera_l)
# LEFT DEPTH
camera_ld = Camera('LeftCameraDepth', PostProcessing='Depth')
camera_ld.set_image_size(800, 600)
camera_ld.set_position(1.30, -0.27, 1.50)
settings.add_sensor(camera_ld)
# LEFT SEGMENTATION
camera_ls = Camera('LeftCameraSeg',
PostProcessing='SemanticSegmentation')
camera_ls.set_image_size(800, 600)
camera_ls.set_position(1.30, -0.27, 1.50)
settings.add_sensor(camera_ls)
# RIGHT RGB CAMERA
camera_r = Camera('RightCameraRGB',
PostProcessing='SceneFinal')
camera_r.set_image_size(800, 600)
camera_r.set_position(1.30, 0.27, 1.50)
settings.add_sensor(camera_r)
# RIGHT DEPTH
camera_rd = Camera('RightCameraDepth', PostProcessing='Depth')
camera_rd.set_image_size(800, 600)
camera_rd.set_position(1.30, 0.27, 1.50)
settings.add_sensor(camera_rd)
# RIGHT SEGMENTATION
camera_rs = Camera('RightCameraSeg',
PostProcessing='SemanticSegmentation')
camera_rs.set_image_size(800, 600)
camera_rs.set_position(1.30, 0.27, 1.50)
settings.add_sensor(camera_rs)
'''
# LEFT 15 DEGREE RGB CAMERA
camera_l_15 = Camera('15_LeftCameraRGB', PostProcessing='SceneFinal')
camera_l_15.set_image_size(800, 600)
camera_l_15.set_position(1.30, -0.7, 1.50) # [X, Y, Z]
camera_l_15.set_rotation(0, -15.0, 0) # [pitch(Y), yaw(Z), roll(X)]
settings.add_sensor(camera_l_15)
# LEFT 15 DEGREE DEPTH
camera_ld_15 = Camera('15_LeftCameraDepth', PostProcessing='Depth')
camera_ld_15.set_image_size(800, 600)
camera_ld_15.set_position(1.30, -0.7, 1.50)
camera_ld_15.set_rotation(0, -15.0, 0)
settings.add_sensor(camera_ld_15)
# LEFT 15 DEGREE SEGMENTATION
camera_ls_15 = Camera('15_LeftCameraSeg', PostProcessing='SemanticSegmentation')
camera_ls_15.set_image_size(800, 600)
camera_ls_15.set_position(1.30, -0.7, 1.50)
camera_ls_15.set_rotation(0, -15.0, 0)
settings.add_sensor(camera_ls_15)
# Right 15 DEGREE RGB CAMERA
camera_r_15 = Camera('15_RightCameraRGB', PostProcessing='SceneFinal')
camera_r_15.set_image_size(800, 600)
camera_r_15.set_position(1.30, 0.7, 1.50)
camera_r_15.set_rotation(0, 15.0, 0)
settings.add_sensor(camera_r_15)
# Right 15 DEGREE DEPTH
camera_rd_15 = Camera('15_RightCameraDepth', PostProcessing='Depth')
camera_rd_15.set_image_size(800, 600)
camera_rd_15.set_position(1.30, 0.7, 1.50)
camera_rd_15.set_rotation(0, 15.0, 0)
settings.add_sensor(camera_rd_15)
# RIGHT 15 DEGREE SEGMENTATION
camera_ls_15 = Camera('15_RightCameraSeg', PostProcessing='SemanticSegmentation')
camera_ls_15.set_image_size(800, 600)
camera_ls_15.set_position(1.30, 0.7, 1.50)
camera_ls_15.set_rotation(0, 15.0, 0)
settings.add_sensor(camera_ls_15)
# LEFT 30 DEGREE RGB CAMERA
camera_l_30 = Camera('30_LeftCameraRGB', PostProcessing='SceneFinal')
camera_l_30.set_image_size(800, 600)
camera_l_30.set_position(1.30, -0.7, 1.50)
camera_l_30.set_rotation(0, -30.0, 0)
settings.add_sensor(camera_l_30)
# LEFT 30 DEGREE DEPTH
camera_ld_30 = Camera('30_LeftCameraDepth', PostProcessing='Depth')
camera_ld_30.set_image_size(800, 600)
camera_ld_30.set_position(1.30, -0.7, 1.50)
camera_ld_30.set_rotation(0, -30.0, 0)
settings.add_sensor(camera_ld_30)
# LEFT 30 DEGREE SEGMENTATION
camera_ls_30 = Camera('30_LeftCameraSeg', PostProcessing='SemanticSegmentation')
camera_ls_30.set_image_size(800, 600)
camera_ls_30.set_position(1.30, -0.7, 1.50)
camera_ls_30.set_rotation(0, -30.0, 0)
settings.add_sensor(camera_ls_30)
# RIGHT 30 DEGREE RGB CAMERA
camera_r_30 = Camera('30_RightCameraRGB', PostProcessing='SceneFinal')
camera_r_30.set_image_size(800, 600)
camera_r_30.set_position(1.30, 0.7, 1.50)
camera_r_30.set_rotation(0, 30.0, 0)
settings.add_sensor(camera_r_30)
# RIGHT 30 DEGREE DEPTH
camera_rd_30 = Camera('30_RightCameraDepth', PostProcessing='Depth')
camera_rd_30.set_image_size(800, 600)
camera_rd_30.set_position(1.30, 0.7, 1.50)
camera_rd_30.set_rotation(0, 30.0, 0)
settings.add_sensor(camera_rd_30)
# RIGHT 30 DEGREE SEGMENTATION
camera_rs_30 = Camera('30_RightCameraSeg', PostProcessing='SemanticSegmentation')
camera_rs_30.set_image_size(800, 600)
camera_rs_30.set_position(1.30, 0.7, 1.50)
camera_rs_30.set_rotation(0, 30.0, 0)
settings.add_sensor(camera_rs_30)
'''
'''
if args.lidar:
lidar = Lidar('Lidar32')
lidar.set_position(0, 0, 2.50)
lidar.set_rotation(0, 0, 0)
lidar.set(
Channels=32,
Range=50,
PointsPerSecond=100000,
RotationFrequency=10,
UpperFovLimit=10,
LowerFovLimit=-30)
settings.add_sensor(lidar)
'''
else:
# Alternatively, we can load these settings from a file.
with open(args.settings_filepath, 'r') as fp:
settings = fp.read()
# Now we load these settings into the server. The server replies
# with a scene description containing the available start spots for
# the player. Here we can provide a CarlaSettings object or a
# CarlaSettings.ini file as string.
scene = client.load_settings(settings)
# Choose one player start at random.
number_of_player_starts = len(scene.player_start_spots)
player_start = random.randint(0, max(0,
number_of_player_starts - 1))
# Notify the server that we want to start the episode at the
# player_start index. This function blocks until the server is ready
# to start the episode.
print('Starting new episode...')
client.start_episode(player_start)
if not os.path.isdir(
"/data/khoshhal/Dataset/episode_{:0>4d}".format(episode)):
os.makedirs(
"/data/khoshhal/Dataset/episode_{:0>4d}".format(episode))
# Iterate every frame in the episode.
for frame in range(0, frames_per_episode):
# Read the data produced by the server this frame.
measurements, sensor_data = client.read_data()
#image = sensor_data.get('LeftCameraSeg', None)
# array = image_converter.depth_to_logarithmic_grayscale(image)
#array = image_converter.labels_to_cityscapes_palette(image)
#filename = '{:0>6d}'.format(frame)
#filename += '.png'
#cv2.imwrite(filename, array)
#image.save_to_disk("/data/khoshhal/Dataset/episode_{:0>4d}/{:s}/{:0>6d}".format(episode, "LeftCameraSeg", frame))
# Print some of the measurements.
# print_measurements(measurements)
# Save the images to disk if requested.
if frame > 199:
if args.save_images_to_disk:
for name, measurement in sensor_data.items():
filename = args.out_filename_format.format(
episode, name, frame - 200)
measurement.save_to_disk(filename)
player_measurements = measurements.player_measurements
with open(
"/data/khoshhal/Dataset/episode_{:0>4d}".format(
episode) + ".txt", 'a') as myfile:
line = "{},{:.1f},{:.1f},{:.1f},{:.1f},{:.1f},{:.1f}\n".format(
frame - 200,
player_measurements.transform.location.x,
player_measurements.transform.location.y,
player_measurements.transform.location.z,
player_measurements.transform.rotation.pitch,
player_measurements.transform.rotation.yaw,
player_measurements.transform.rotation.roll)
myfile.write(line)
# We can access the encoded data of a given image as numpy
# array using its "data" property. For instance, to get the
# depth value (normalized) at pixel X, Y
#
# depth_array = sensor_data['CameraDepth'].data
# value_at_pixel = depth_array[Y, X]
#
# Now we have to send the instructions to control the vehicle.
# If we are in synchronous mode the server will pause the
# simulation until we send this control.
if not args.autopilot:
client.send_control(steer=random.uniform(-1.0, 1.0),
throttle=0.5,
brake=0.0,
hand_brake=False,
reverse=False)
else:
# Together with the measurements, the server has sent the
# control that the in-game autopilot would do this frame. We
# can enable autopilot by sending back this control to the
# server. We can modify it if wanted, here for instance we
# will add some noise to the steer.
control = measurements.player_measurements.autopilot_control
#control.steer += random.uniform(-0.1, 0.1)
client.send_control(control)
#time.sleep(1)
myfile.close()
def _reset(self):
self.num_steps = 0
self.total_reward = 0
self.prev_measurement = None
self.prev_image = None
self.episode_id = datetime.today().strftime("%Y-%m-%d_%H-%M-%S_%f")
self.measurements_file = None
# Create a CarlaSettings object. This object is a wrapper around
# the CarlaSettings.ini file. Here we set the configuration we
# want for the new episode.
settings = CarlaSettings()
self.scenario = random.choice(self.config["scenarios"])
assert self.scenario["city"] == self.city, (self.scenario, self.city)
self.weather = random.choice(self.scenario["weather_distribution"])
settings.set(
SynchronousMode=True,
# ServerTimeOut=10000, # CarlaSettings: no key named 'ServerTimeOut'
SendNonPlayerAgentsInfo=True,
NumberOfVehicles=self.scenario["num_vehicles"],
NumberOfPedestrians=self.scenario["num_pedestrians"],
WeatherId=self.weather)
settings.randomize_seeds()
if self.config["use_depth_camera"]:
camera1 = Camera("CameraDepth", PostProcessing="Depth")
camera1.set_image_size(self.config["render_x_res"],
self.config["render_y_res"])
# camera1.set_position(30, 0, 130)
camera1.set(FOV=120)
camera1.set_position(2.0, 0.0, 1.4)
camera1.set_rotation(0.0, 0.0, 0.0)
settings.add_sensor(camera1)
camera2 = Camera("CameraRGB")
camera2.set_image_size(self.config["render_x_res"],
self.config["render_y_res"])
# camera2.set_position(30, 0, 130)
# camera2.set_position(0.3, 0.0, 1.3)
camera2.set(FOV=120)
camera2.set_position(2.0, 0.0, 1.4)
camera2.set_rotation(0.0, 0.0, 0.0)
settings.add_sensor(camera2)
# Setup start and end positions
scene = self.client.load_settings(settings)
positions = scene.player_start_spots
self.start_pos = positions[self.scenario["start_pos_id"]]
self.end_pos = positions[self.scenario["end_pos_id"]]
self.start_coord = [
self.start_pos.location.x // 100, self.start_pos.location.y // 100
]
self.end_coord = [
self.end_pos.location.x // 100, self.end_pos.location.y // 100
]
print("Start pos {} ({}), end {} ({})".format(
self.scenario["start_pos_id"], self.start_coord,
self.scenario["end_pos_id"], self.end_coord))
# Notify the server that we want to start the episode at the
# player_start index. This function blocks until the server is ready
# to start the episode.
print("Starting new episode...")
self.client.start_episode(self.scenario["start_pos_id"])
# Process observations: self._read_observation() returns image and py_measurements.
image, py_measurements = self._read_observation()
self.prev_measurement = py_measurements
return self.encode_obs(self.preprocess_image(image), py_measurements)
def run_carla_client(host, port, far):
# Here we will run a single episode with 300 frames.
number_of_frames = 3000
frame_step = 10 # Save one image every 100 frames
output_folder = '_out'
image_size = [1280, 720]
camera_local_pos = [0.3, 0.0, 1.3] # [X, Y, Z]
camera_local_rotation = [0, 0, 0] # [pitch(Y), yaw(Z), roll(X)]
fov = 59
WINDOW_WIDTH = 1280
WINDOW_HEIGHT = 720
MINI_WINDOW_WIDTH = 320
MINI_WINDOW_HEIGHT = 180
WINDOW_WIDTH_HALF = WINDOW_WIDTH / 2
WINDOW_HEIGHT_HALF = WINDOW_HEIGHT / 2
k = np.identity(3)
k[0, 2] = WINDOW_WIDTH_HALF
k[1, 2] = WINDOW_HEIGHT_HALF
k[0, 0] = k[1, 1] = WINDOW_WIDTH / \
(2.0 * math.tan(59.0 * math.pi / 360.0))
intrinsic = k
# Connect with the server
with make_carla_client(host, port) as client:
print('CarlaClient connected')
# Here we load the settings.
settings = CarlaSettings()
settings.set(SynchronousMode=True,
SendNonPlayerAgentsInfo=False,
NumberOfVehicles=20,
NumberOfPedestrians=40,
WeatherId=1)
settings.randomize_seeds()
camera1 = Camera('CameraDepth', PostProcessing='Depth', FOV=fov)
camera1.set_image_size(*image_size)
camera1.set_position(*camera_local_pos)
camera1.set_rotation(*camera_local_rotation)
settings.add_sensor(camera1)
camera2 = Camera('CameraRGB', PostProcessing='SceneFinal', FOV=fov)
camera2.set_image_size(*image_size)
camera2.set_position(*camera_local_pos)
camera2.set_rotation(*camera_local_rotation)
settings.add_sensor(camera2)
client.load_settings(settings)
# Start at location index id '0'
client.start_episode(0)
# Compute the camera transform matrix
camera_to_car_transform = camera2.get_unreal_transform()
print("camera_to_car_transform", camera_to_car_transform)
# Iterate every frame in the episode except for the first one.
for frame in range(1, number_of_frames):
# Read the data produced by the server this frame.
measurements, sensor_data = client.read_data()
# Save one image every 'frame_step' frames
if not frame % frame_step:
# Start transformations time mesure.
# RGB image [[[r,g,b],..[r,g,b]],..[[r,g,b],..[r,g,b]]]
image_RGB = to_rgb_array(sensor_data['CameraRGB'])
image_depth = to_rgb_array(sensor_data['CameraDepth'])
#measurements.player_measurements.transform.location.x-=40
#measurements.player_measurements.transform.location.z-=2
world_transform = Transform(
measurements.player_measurements.transform)
im_bgr = cv2.cvtColor(image_RGB, cv2.COLOR_RGB2BGR)
pos_vector = np.array(
[[measurements.player_measurements.transform.location.x],
[measurements.player_measurements.transform.location.y],
[measurements.player_measurements.transform.location.z],
[1.0]])
carla_utils_obj = segmentation.carla_utils(
intrinsic, camera_to_car_transform, world_transform)
get_2d_point = carla_utils_obj.run_seg(frame)
fdfd = str(world_transform)
sdsd = fdfd[1:-1].split('\n')
new = []
for i in sdsd:
dd = i[:-1].lstrip('[ ')
ff = re.sub("\s+", ",", dd.strip())
gg = ff.split(',')
new.append([float(i) for i in gg])
newworld = np.array(new)
fdfd = str(camera_to_car_transform)
sdsd = fdfd[1:-1].split('\n')
new = []
for i in sdsd:
dd = i[:-1].lstrip('[ ')
ff = re.sub("\s+", ",", dd.strip())
gg = ff.split(',')
new.append([float(i) for i in gg])
newcam = np.array(new)
extrinsic = np.matmul(newworld, newcam)
depth = image_depth[int(get_2d_point[1]), int(get_2d_point[0])]
scale = depth[0] + depth[1] * 256 + depth[2] * 256 * 256
scale = scale / ((256 * 256 * 256) - 1)
depth = scale * 1000
pos2d = np.array([(WINDOW_WIDTH - get_2d_point[0]) * depth,
(WINDOW_HEIGHT - get_2d_point[1]) * depth,
depth])
first = np.dot(np.linalg.inv(intrinsic), pos2d)
first = np.append(first, 1)
sec = np.matmul((extrinsic), first)
print('goal-', sec)
client.send_control(
measurements.player_measurements.autopilot_control)
class CarlaEnv(Env):
def __init__(self, city_name="Town01", ep_length=400, z_size=512):
self.z_size = z_size
self.HEIGHT = 160
self.WIDTH = 80
self.NUM_CHANNEL = 3
# self.action_space = Box(low=np.array([-1.0]), high=np.array([-1.0]), dtype=np.float32) # Throttle, steer, break
# self.observation_space = Box(low=0, high=255, shape=(self.HEIGHT, self.WIDTH, self.NUM_CHANNEL), dtype=np.int32)
self.action_space = Box(low=np.array([-0.5]), high=np.array([0.5]), dtype=np.float32)
self.observation_space = Box(low=np.finfo(np.float32).min,
high=np.finfo(np.float32).max,
shape=(1, self.z_size), dtype=np.float32)
self.ep_length = ep_length
self.current_step = 0
self.dim = self.observation_space.shape # The shape of the input on which we are learning
# Carla Settings
self.MAX_ALLOWED_OFFROAD = 0.3
self.carla_settings = CarlaSettings()
self.carla_settings.set(SynchronousMode=True,
SendNonPlayerAgentsInfo=True,
NumberOfVehicles=20,
NumberOfPedestrians=40,
QualityLevel='Low',
)
def client_init(self, client):
self.client = client
self.carla_settings = CarlaSettings()
self.carla_settings.set(SynchronousMode=True,
SendNonPlayerAgentsInfo=True,
NumberOfVehicles=20,
NumberOfPedestrians=40,
QualityLevel='Low',
)
camera0 = Camera('CameraRGB')
camera0.set_image_size(self.HEIGHT, self.WIDTH)
camera0.set_position(2.0, 0.0, 1.4)
camera0.set_rotation(0.0, 0.0, 0.0)
camera1 = Camera('CameraSegmentation', PostProcessing='SemanticSegmentation')
camera1.set_image_size(self.HEIGHT, self.WIDTH)
camera1.set_position(2.0, 0, 1.4)
self.settings = CarlaSettings()
self.settings.add_sensor(camera0)
self.settings.add_sensor(camera1)
scene = self.client.load_settings(self.settings)
start_spots = scene.player_start_spots
self.target = start_spots[26]
self.player_start = 26
cv2.namedWindow('im', cv2.WINDOW_NORMAL)
def reset(self):
self.current_step = 0
while True:
'''Sometimes the client looses connection to the server. So we retry connection until connected'''
try:
self.client.start_episode(self.player_start)
observation, done = self._get_observation()
break
except:
print('could not connect')
self.client.disconnect()
time.sleep(5)
self.client.connect()
time.sleep(10)
return observation
def step(self, action):
self.current_step += 1
self.action = action
control = Control()
control.throttle = 0.4 # action[0]
control.steer = action[0]
control.brake = 0 # action[2]
try:
self.client.send_control(control)
observation, done = self._get_observation()
except:
print('Lost Connection')
self.reset()
self.client.send_control(control)
observation, done = self._get_observation()
reward = self._get_reward(done=done)
info = {}
#print("reward: ", reward)
#print(observation, reward, done, info)
observation = np.random.random((1, 512))
reward = np.random.random()
done = False
info = {}
return observation, reward, done, info
def _get_observation(self):
measurements, snesor_data = self.client.read_data()
carla_im = snesor_data.get('CameraSegmentation', None)
'''For RGB image'''
#self.observation_image = image_converter.to_rgb_array(carla_im)
'''For Semantic Segmentation Image'''
#print(type(carla_im))
self.observation_image = np.uint8(image_converter.labels_to_cityscapes_palette(carla_im))
cv2.imshow('im', self.observation_image)
cv2.waitKey(1)
stats = measurements.player_measurements
pos = measurements.player_measurements.transform
movement = self.get_movemet(pos)
# print('Movemene ', movement)
self.percentage_intersection_other_lane = abs(stats.intersection_otherlane)
self.percentage_offroad = stats.intersection_offroad
collision = stats.collision_vehicles or stats.collision_pedestrians or stats.collision_other
'''Sometimes, the collision sensor doesn't register, so I check for distnce moved between each step, after 100
timesteps because at lower time steps the distance moved is pretty low'''
car_stuck = self.is_car_stuck(movement)
done = self._is_game_over(collision, car_stuck)
#print(type(done))
observation = self.vae_observation(observation_image=self.observation_image)
return observation, done
def vae_observation(self, observation_image):
"""Appending for the vae buffer here. vae.optimize() uses this buffer to train """
self.vae.buffer_append(observation_image)
ob = self.vae.encode(observation_image)
# print(ob)
return ob
def get_movemet(self, pos):
if self.current_step == 0:
self.old_pos = pos
return 1e-7
else:
dist_moved = np.sqrt(
(self.old_pos.location.x - pos.location.x) ** 2 + (self.old_pos.location.y - pos.location.y) ** 2)
self.old_pos = pos
return dist_moved
def is_car_stuck(self, dist_moved):
if dist_moved < 0.01 and self.current_step > 100:
return True
else:
return False
def _is_game_over(self, collision, car_stuck):
''' Game is over after a collision '''
if collision or car_stuck:
return True
else:
return False
def _get_reward(self, done):
''' Here we try to follow the road divider, so ideally the lane intersection should be 50% '''
if done:
return 0.0
if self.percentage_offroad > self.MAX_ALLOWED_OFFROAD:
return 0.0
return 1 - abs(self.action)
def set_vae(self, vae):
self.vae = vae
def render(self, mode='human'):
cv2.imshow('im', self.observation_image)
cv2.waitKey(1)
def run_carla_client(args):
# Here we will run args._episode episodes with args._frames frames each.
number_of_episodes = args._episode
frames_per_episode = args._frames
# create the carla client
with make_carla_client(args.host, args.port, timeout=100000) as client:
print('CarlaClient connected')
for episode in range(0, number_of_episodes):
if args.settings_filepath is None:
settings = CarlaSettings()
settings.set(SynchronousMode=args.synchronous_mode,
SendNonPlayerAgentsInfo=True,
NumberOfVehicles=150,
NumberOfPedestrians=100,
WeatherId=random.choice([1, 3, 7, 8, 14]),
QualityLevel=args.quality_level)
settings.randomize_seeds()
camera0 = Camera('CameraRGB')
camera0.set_image_size(1920, 640)
camera0.set_position(2.00, 0, 1.30)
settings.add_sensor(camera0)
camera1 = Camera('CameraDepth', PostProcessing='Depth')
camera1.set_image_size(1920, 640)
camera1.set_position(2.00, 0, 1.30)
settings.add_sensor(camera1)
camera2 = Camera('CameraSegmentation',
PostProcessing='SemanticSegmentation')
camera2.set_image_size(1920, 640)
camera2.set_position(2.00, 0, 1.30)
settings.add_sensor(camera2)
else:
# Alternatively, we can load these settings from a file.
with open(args.settings_filepath, 'r') as fp:
settings = fp.read()
scene = client.load_settings(settings)
# Choose one player start at random.
number_of_player_starts = len(scene.player_start_spots)
player_start = random.randint(0, max(0,
number_of_player_starts - 1))
# create folder for current episode
file_loc = args.file_loc
if not os.path.exists(file_loc):
os.makedirs(file_loc)
#set destination path for the tfrecords
preprocessing_situ_all_data.set_dest_path(
file_loc, args._samples_per_record, args._K, args._T,
args._image_size, args._seq_length, args._step_size)
print('Starting new episode at %r...' % scene.map_name)
client.start_episode(player_start)
# take unique time snapshot used for naming
prefix = str(int(round(args.start_time))) + '_' + str(episode)
#update the prefix of the tfrecord name and reset the globals when starting a new episode
preprocessing_situ_all_data.update_episode_reset_globals(prefix)
# Iterate every frame in the episode.
for frame in range(0, frames_per_episode):
measurements, sensor_data = client.read_data()
#discard episode if misbehaviour flag is set and a form of collision is detected
if args.no_misbehaviour:
player_measurements = measurements.player_measurements
col_cars = player_measurements.collision_vehicles
col_ped = player_measurements.collision_pedestrians
col_other = player_measurements.collision_other
if col_cars + col_ped + col_other > 0:
print(
"MISBEHAVIOUR DETECTED! Discarding Episode... \n")
break
print_measurements(measurements)
# Save the images to disk if requested. Skip first couple of images due to setup time.
if args.save_images_to_disk and frame > 19:
player_measurements = measurements.player_measurements
for name, measurement in sensor_data.items():
if name == 'CameraDepth':
depth = measurement.return_depth_map()
if name == 'CameraSegmentation':
segmentation = measurement.return_segmentation_map(
)
if name == 'CameraRGB':
rgb = measurement.return_rgb()
yaw, yaw_rate, speed, prev_time = process_odometry(
measurements, yaw_shift, yaw_old, prev_time)
yaw_old = yaw
#generate image with bounding boxes, will be transformed to gridmaps in preprocessing
im = Image.new('L', (256 * 6, 256 * 6), (127))
shift = 256 * 6 / 2
draw = ImageDraw.Draw(im)
for agent in measurements.non_player_agents:
if agent.HasField('vehicle') or agent.HasField(
'pedestrian'):
participant = agent.vehicle if agent.HasField(
'vehicle') else agent.pedestrian
angle = cart2pol(
participant.transform.orientation.x,
participant.transform.orientation.y)
polygon = agent2world(participant, angle)
polygon = world2player(
polygon, math.radians(-yaw),
player_measurements.transform)
polygon = player2image(polygon,
shift,
multiplier=25)
polygon = [tuple(row) for row in polygon.T]
draw.polygon(polygon, 0, 0)
im = im.resize((256, 256), Image.ANTIALIAS)
im = im.rotate(imrotate)
#pass frame by frame to the preprocessor, it buffers these and stores them as tfrecords
if not args.all_data:
preprocessing_situ.main(im, -yaw_rate, speed)
else:
preprocessing_situ_all_data.main(
im, rgb, depth, segmentation, -yaw_rate, speed)
else:
# get values while frame < 20 to be used when frame == 20
yaw_shift = measurements.player_measurements.transform.rotation.yaw
yaw_old = ((yaw_shift - 180) % 360) - 180
imrotate = round(yaw_old) + 90
shift_x = measurements.player_measurements.transform.location.x
shift_y = measurements.player_measurements.transform.location.y
prev_time = np.int64(measurements.game_timestamp)
if not args.autopilot:
client.send_control(steer=random.uniform(-1.0, 1.0),
throttle=0.5,
brake=0.0,
hand_brake=False,
reverse=False)
else:
# Together with the measurements, the server has sent the
# control that the in-game autopilot would do this frame. We
# can enable autopilot by sending back this control to the
# server. We can modify it if wanted, here for instance we
# will add some noise to the steer.
control = measurements.player_measurements.autopilot_control
control.steer += random.uniform(-0.01, 0.01)
client.send_control(control)
def _reset(self):
self.num_steps = 0
self.total_reward = 0
self.prev_measurement = None
self.prev_image = None
self.episode_id = datetime.today().strftime("%Y-%m-%d_%H-%M-%S_%f")
self.measurements_file = None
self.enable_autopilot = np.random.random() < expert_probability
self.pre_intersection = np.array([0.0, 0.0])
# Create a CarlaSettings object. This object is a wrapper around
# the CarlaSettings.ini file. Here we set the configuration we
# want for the new episode.
settings = CarlaSettings()
self.scenario = random.choice(self.config["scenarios"])
assert self.scenario["city"] == self.city, (self.scenario, self.city)
self.weather = random.choice(self.scenario["weather_distribution"])
settings.set(
SynchronousMode=True,
# ServerTimeOut=10000, # CarlaSettings: no key named 'ServerTimeOut'
SendNonPlayerAgentsInfo=True,
NumberOfVehicles=self.scenario["num_vehicles"],
NumberOfPedestrians=self.scenario["num_pedestrians"],
WeatherId=self.weather)
settings.randomize_seeds()
if ENV_CONFIG["use_radar"]:
camera_radar = Camera("Camera_radar", PostProcessing="Depth")
camera_radar.set_image_size(self.config["render_x_res"],
self.config["render_y_res"])
camera_radar.set_position(2.15, 0.0, 0.5)
settings.add_sensor(camera_radar)
if self.config["use_sensor"] == 'use_semantic':
camera0 = Camera("CameraSemantic",
PostProcessing="SemanticSegmentation")
camera0.set_image_size(self.config["render_x_res"],
self.config["render_y_res"])
# camera0.set_position(30, 0, 130)
camera0.set(FOV=120)
camera0.set_position(2.0, 0.0, 1.4)
camera0.set_rotation(0.0, 0.0, 0.0)
settings.add_sensor(camera0)
if self.config["use_sensor"] in ['use_depth', 'use_logdepth']:
camera1 = Camera("CameraDepth", PostProcessing="Depth")
camera1.set_image_size(self.config["render_x_res"],
self.config["render_y_res"])
# camera1.set_position(30, 0, 130)
camera1.set(FOV=120)
camera1.set_position(2.0, 0.0, 1.4)
camera1.set_rotation(0.0, 0.0, 0.0)
settings.add_sensor(camera1)
if self.config["use_sensor"] == 'use_rgb':
camera2 = Camera("CameraRGB")
camera2.set_image_size(self.config["render_x_res"],
self.config["render_y_res"])
# camera2.set_position(30, 0, 130)
# camera2.set_position(0.3, 0.0, 1.3)
camera2.set_position(1.0, 0.0, 1.5)
# camera2.set_position(2.15, 0.0, 0.5) # for radar
# camera2.set(FOV=110)
# camera2.set_position(1.5, 0.0, 1.4)
# camera2.set_rotation(0.0, 0.0, 0.0)
settings.add_sensor(camera2)
if self.config["use_sensor"] == 'use_2rgb':
camera_l = Camera("CameraRGB_L")
camera_l.set_image_size(self.config["render_x_res"],
self.config["render_y_res"])
camera_l.set_position(1.0, -0.1, 1.5)
# camera_l.set(FOV=120)
# camera_l.set_position(2.0, -0.1, 1.4)
# camera_l.set_rotation(0.0, 0.0, 0.0)
settings.add_sensor(camera_l)
camera_r = Camera("CameraRGB_R")
camera_r.set_image_size(self.config["render_x_res"],
self.config["render_y_res"])
camera_r.set_position(1.0, 0.1, 1.5)
# camera_r.set(FOV=120)
# camera_r.set_position(2.0, 0.1, 1.4)
# camera_r.set_rotation(0.0, 0.0, 0.0)
settings.add_sensor(camera_r)
if self.config["use_sensor"] == 'use_rgb_semantic':
camera_rgb = Camera("Camera_rgb")
camera_rgb.set_image_size(self.config["render_x_res"],
self.config["render_y_res"])
camera_rgb.set_position(1.0, 0.0, 1.3)
settings.add_sensor(camera_rgb)
camera_semantic = Camera("Camera_semantic",
PostProcessing="SemanticSegmentation")
camera_semantic.set_image_size(self.config["render_x_res"],
self.config["render_y_res"])
camera_semantic.set_position(1.0, 0.0, 1.3)
settings.add_sensor(camera_semantic)
# Setup start and end positions
scene = self.client.load_settings(settings)
self.positions = positions = scene.player_start_spots
self.start_pos = positions[self.scenario["start_pos_id"]]
self.pre_pos = self.start_pos.location
self.cnt1 = 0
self.displacement = 1000.0
self.end_pos = positions[self.scenario["end_pos_id"]]
self.start_coord = [
self.start_pos.location.x // 1, self.start_pos.location.y // 1
]
self.end_coord = [
self.end_pos.location.x // 1, self.end_pos.location.y // 1
]
print("Start pos {} ({}), end {} ({})".format(
self.scenario["start_pos_id"], self.start_coord,
self.scenario["end_pos_id"], self.end_coord))
# Notify the server that we want to start the episode at the
# player_start index. This function blocks until the server is ready
# to start the episode.
print("Starting new episode...")
self.client.start_episode(self.scenario["start_pos_id"])
# remove the vehicle dropping when starting a new episode.
cnt = 1
z1 = 0
zero_control = VehicleControl()
while (cnt < 3):
self.client.send_control(
zero_control
) # VehicleControl().steer = 0, VehicleControl().throttle = 0, VehicleControl().reverse = False
z0 = z1
z1 = self.client.read_data(
)[0].player_measurements.transform.location.z
print(z1)
if z0 - z1 == 0:
cnt += 1
print('Starting new episode done.\n')
# Process observations: self._read_observation() returns image and py_measurements.
image, py_measurements = self._read_observation()
py_measurements["collided"] = np.array(0.0, dtype=np.float32)
self.prev_measurement = py_measurements
# self.current_heading = self.pre_heading = np.array([py_measurements["x_orient"], py_measurements["y_orient"]])
# self.angular_speed = 0.0
self.pre_heading_degree = self.current_heading_degree = py_measurements[
"current_heading_degree"]
self.angular_speed_degree = np.array(0.0)
return self.encode_obs(self.preprocess_image(image),
py_measurements), py_measurements
def run_carla_client(args):
# Settings
# Town 1
## Natural turns: [42,67,69,79,94,97,70,44,85,102], [0.0, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8]
## T-intersection: [88,90,107,133,136]
## Side intersection left: [ 66, 16, 25 ]
## Side intersection right: [ 138, 19, 26 ]
## Staight: [14, 149, 78, 71, 89]
# Town 2
## Natural turns right: [65, 78, 44, 1]
## Natural turns left: [49, 50, 57, 70]
## T-intersection [2, 5, 10, 11, 19, 26, 34, 37]
## Side intersection left: [7, 23, 16, 39, 6, 43, 79]
## Side intersection right: [40, 20, 28, 32, 34, 46, 71, 74]
## Straight: [61, 64, 77, 51]
positions = args.positions or [65, 78, 49, 50, 2, 5, 10, 7, 23, 40, 20, 61]
levels_of_randomness = args.randomness or [0.0, 0.2, 0.4, 0.6]
frames_per_level = args.frames or [
200, 200, 200, 200, 200, 200, 200, 200, 200, 200, 200
]
print("Positions: ", positions)
print("Levels of randomness: ", levels_of_randomness)
print("Frames per level: ", frames_per_level)
# We assume the CARLA server is already waiting for a client to connect at
# host:port. To create a connection we can use the `make_carla_client`
# context manager, it creates a CARLA client object and starts the
# connection. It will throw an exception if something goes wrong. The
# context manager makes sure the connection is always cleaned up on exit.
with make_carla_client(args.host, args.port) as client:
print('CarlaClient connected')
directions = ["_left", "_right"
] if args.force_left_and_right else ["_"]
for direction in directions:
print("Direction ", direction)
# Create a CarlaSettings object. This object is a wrapper around
# the CarlaSettings.ini file. Here we set the configuration we
# want for the new episode.
settings = CarlaSettings()
direction_var = 6 if direction == "_right" else 0
settings.set(SynchronousMode=True,
SendNonPlayerAgentsInfo=True,
NumberOfVehicles=0,
NumberOfPedestrians=0,
WeatherId=random.choice([2]),
QualityLevel=args.quality_level,
SeedVehicles=direction_var)
#settings.randomize_seeds()
if args.capture:
# To create visualisation of the current run
camera3 = Camera('CameraRGB')
camera3.set_image_size(512, 256)
camera3.set_position(-8, 0, 5)
camera3.set(FOV=args.fov)
camera3.set_rotation(pitch=-20, yaw=0, roll=0)
settings.add_sensor(camera3)
# Now we want to add a couple of cameras to the player vehicle.
# We will collect the images produced by these cameras every
# frame.
if args.point_cloud:
# Camera to produce point Cloud
camera1 = Camera('CameraDepth', PostProcessing='Depth')
camera1.set_image_size(256, 256)
camera1.set_position(2.2, 0, 1.30)
camera1.set(FOV=args.fov)
settings.add_sensor(camera1)
camera2 = Camera('CameraSeg',
PostProcessing='SemanticSegmentation')
camera2.set_image_size(256, 256)
camera2.set_position(2.2, 0, 1.30)
camera2.set(FOV=args.fov)
settings.add_sensor(camera2)
if args.lidar:
lidar = Lidar('Lidar32')
lidar.set_position(0, 0, 2.50)
lidar.set_rotation(0, 0, 0)
lidar.set(Channels=32,
Range=50,
PointsPerSecond=100000,
RotationFrequency=10,
UpperFovLimit=10,
LowerFovLimit=-30)
settings.add_sensor(lidar)
# Now we load these settings into the server. The server replies
# with a scene description containing the available start spots for
# the player. Here we can provide a CarlaSettings object or a
# CarlaSettings.ini file as string.
scene = client.load_settings(settings)
# Notify the server that we want to start the episode at the
# player_start index. This function blocks until the server is ready
# to start the episode.
for episode, position in enumerate(positions):
for randomness, frames in zip(levels_of_randomness,
frames_per_level):
crashed = True
while crashed:
crashed = False
# Start a new episode.
print('Starting new episode at %r...' % scene.map_name)
print('Episode {}, Position {}, Randomness {}'.format(
episode, position, randomness))
client.start_episode(position)
if args.capture:
# Make sure directories exist
directory = '_out/pos{}{}/randomness_{}'.format(
position, direction, randomness)
ply_dir = '{}/point_clouds'.format(directory)
ply_dir_full = '{}/point_clouds_full'.format(
directory)
lidar_dir = '{}/lidar'.format(directory)
img_dir = '{}/images'.format(directory)
if not os.path.exists(img_dir):
os.makedirs(img_dir)
if args.point_cloud and not os.path.exists(
ply_dir):
os.makedirs(ply_dir)
if args.point_cloud and not os.path.exists(
ply_dir_full):
os.makedirs(ply_dir_full)
if args.lidar and not os.path.exists(lidar_dir):
os.makedirs(lidar_dir)
# Write steering data to csv file
if args.point_cloud:
csv = open(
"{}/driving_data.csv".format(ply_dir), "w")
elif args.lidar:
csv = open(
"{}/driving_data.csv".format(lidar_dir),
"w")
csv.write(",name,speed,throttle,steer\n")
# Iterate every frame in the episode
for frame in tqdm(range(frames)):
# Read the data produced by the server this frame.
measurements, sensor_data = client.read_data()
if args.capture:
if args.point_cloud:
# Save processed point clouds and autopilot steering to disk if requested
# Get depth and seg as numpy array for further processing
depth_obj = sensor_data['CameraDepth']
depth = depth_obj.data
fov = depth_obj.fov
# Filter seg to get intersection points
seg = sensor_data['CameraSeg'].data
filtered_seg = pp.filter_seg_image(seg)
if args.full_point_cloud:
# Converting depth image to grayscale
point_cloud_full = pp.depth_to_local_point_cloud(
depth,
fov,
seg,
max_depth=0.05,
full=True)
filename_full = "point_cloud_full_{:0>5d}".format(
frame)
pp.save_to_disk(
point_cloud_full,
"{}/{}.ply".format(
ply_dir_full, filename_full))
# Create pointcloud from seg and depth (but only take intersection points)
point_cloud = pp.depth_to_local_point_cloud(
depth,
fov,
filtered_seg,
max_depth=0.05)
filename = "point_cloud_{:0>5d}".format(
frame)
pp.save_to_disk(
point_cloud,
"{}/{}.ply".format(ply_dir, filename))
if args.lidar:
sensor_data['Lidar32'].save_to_disk(
'{}/point_cloud_{:0>5d}'.format(
lidar_dir, frame))
# Save steering data of this frame
control = measurements.player_measurements.autopilot_control
csv.write("0,image_{:0>5d},0,0,{}\n".format(
frame, control.steer))
# Save rgb image to visualize later
sensor_data['CameraRGB'].save_to_disk(
'{}/image_{:0>5d}.png'.format(
img_dir, frame))
# Now we have to send the instructions to control the vehicle.
# If we are in synchronous mode the server will pause the
# simulation until we send this control.
control = measurements.player_measurements.autopilot_control
speed = measurements.player_measurements.forward_speed
old_steer = control.steer
control.steer += random.uniform(
-randomness, randomness)
if args.ignore_red_lights:
control.throttle = 0.5
control.brake = 0.0
control.hand_brake = False
control.reverse = False
client.send_control(control)
crashed = False
if speed < 1 and abs(old_steer) > 0.5:
print(
"\nSeems like we crashed.\nTrying again..."
)
crashed = True
break
def run_carla_client(host, port, autopilot_on, save_images_to_disk, image_filename_format):
# Here we will run 3 episodes with 300 frames each.
number_of_episodes = 3
frames_per_episode = 300
# We assume the CARLA server is already waiting for a client to connect at
# host:port. To create a connection we can use the `make_carla_client`
# context manager, it creates a CARLA client object and starts the
# connection. It will throw an exception if something goes wrong. The
# context manager makes sure the connection is always cleaned up on exit.
with make_carla_client(host, port) as client:
print('CarlaClient connected')
for episode in range(0, number_of_episodes):
# Start a new episode.
# Create a CarlaSettings object. This object is a handy wrapper
# around the CarlaSettings.ini file. Here we set the configuration
# we want for the new episode.
settings = CarlaSettings()
settings.set(
SynchronousMode=True,
NumberOfVehicles=30,
NumberOfPedestrians=50,
WeatherId=random.choice([1, 3, 7, 8, 14]))
settings.randomize_seeds()
# Now we want to add a couple of cameras to the player vehicle. We
# will collect the images produced by these cameras every frame.
# The default camera captures RGB images of the scene.
camera0 = Camera('CameraRGB')
# Set image resolution in pixels.
camera0.set_image_size(800, 600)
# Set its position relative to the car in centimeters.
camera0.set_position(30, 0, 130)
settings.add_camera(camera0)
# Let's add another camera producing ground-truth depth.
camera1 = Camera('CameraDepth', PostProcessing='Depth')
camera1.set_image_size(800, 600)
camera1.set_position(30, 0, 130)
settings.add_camera(camera1)
print('Requesting new episode...')
# Now we request a new episode with these settings. The server
# replies with a scene description containing the available start
# spots for the player. Here instead of a CarlaSettings object we
# could also provide a CarlaSettings.ini file as string.
scene = client.request_new_episode(settings)
# Choose one player start at random.
number_of_player_starts = len(scene.player_start_spots)
player_start = random.randint(0, max(0, number_of_player_starts - 1))
# Notify the server that we want to start the episode at
# `player_start`. This function blocks until the server is ready to
# start the episode.
client.start_episode(player_start)
# Iterate every frame in the episode.
for frame in range(0, frames_per_episode):
# Read the measurements and images produced by the server this
# frame.
measurements, images = client.read_measurements()
# Print some of the measurements we received.
print_player_measurements(measurements.player_measurements)
# Save the images to disk if requested.
if save_images_to_disk:
for n, image in enumerate(images):
image.save_to_disk(image_filename_format.format(episode, n, frame))
# Now we have to send the instructions to control the vehicle.
# If we are in synchronous mode the server will pause the
# simulation until we send this control.
if not autopilot_on:
client.send_control(
steer=random.uniform(-1.0, 1.0),
throttle=0.3,
brake=False,
hand_brake=False,
reverse=False)
else:
# Together with the measurements, the server has sent the
# control that the in-game AI would do this frame. We can
# enable autopilot by sending back this control to the
# server. Here we will also add some noise to the steer.
control = measurements.player_measurements.ai_control
control.steer += random.uniform(-0.1, 0.1)
client.send_control(control)
print('Done.')
return True
def run_carla_client(args):
frames_per_episode = 10000
spline_points = 10000
report = {
'num_episodes': args.num_episodes,
'controller_name': args.controller_name,
'distances': [],
'target_speed': args.target_speed,
}
track_DF = pd.read_csv('racetrack{}.txt'.format(args.racetrack),
header=None)
# The track data are rescaled by 100x with relation to Carla measurements
track_DF = track_DF / 100
pts_2D = track_DF.loc[:, [0, 1]].values
tck, u = splprep(pts_2D.T, u=None, s=2.0, per=1, k=3)
u_new = np.linspace(u.min(), u.max(), spline_points)
x_new, y_new = splev(u_new, tck, der=0)
pts_2D = np.c_[x_new, y_new]
steer = 0.0
throttle = 0.5
depth_array = None
if args.controller_name == 'mpc':
weather_id = 2
controller = MPCController(args.target_speed)
elif args.controller_name == 'pd':
weather_id = 1
controller = PDController(args.target_speed)
elif args.controller_name == 'pad':
weather_id = 5
controller = PadController()
elif args.controller_name == 'nn':
# Import it here because importing TensorFlow is time consuming
from neural_network_controller import NNController # noqa
weather_id = 11
controller = NNController(
args.target_speed,
args.model_dir_name,
args.which_model,
args.throttle_coeff_A,
args.throttle_coeff_B,
args.ensemble_prediction,
)
report['model_dir_name'] = args.model_dir_name
report['which_model'] = args.which_model
report['throttle_coeff_A'] = args.throttle_coeff_A
report['throttle_coeff_B'] = args.throttle_coeff_B
report['ensemble_prediction'] = args.ensemble_prediction
with make_carla_client(args.host, args.port) as client:
print('CarlaClient connected')
episode = 0
num_fails = 0
while episode < args.num_episodes:
# Start a new episode
if args.store_data:
depth_storage = np.zeros(
((IMAGE_CLIP_LOWER - IMAGE_CLIP_UPPER) // IMAGE_DECIMATION,
IMAGE_SIZE[1] // IMAGE_DECIMATION,
frames_per_episode)).astype(DTYPE)
log_dicts = frames_per_episode * [None]
else:
depth_storage = None
log_dicts = None
if args.settings_filepath is None:
# Create a CarlaSettings object. This object is a wrapper around
# the CarlaSettings.ini file. Here we set the configuration we
# want for the new episode.
settings = CarlaSettings()
settings.set(SynchronousMode=True,
SendNonPlayerAgentsInfo=False,
NumberOfVehicles=0,
NumberOfPedestrians=0,
WeatherId=weather_id,
QualityLevel=args.quality_level)
settings.randomize_seeds()
# Now we want to add a couple of cameras to the player vehicle.
# We will collect the images produced by these cameras every
# frame.
# Let's add another camera producing ground-truth depth.
camera = Camera('CameraDepth',
PostProcessing='Depth',
FOV=69.4)
# MD: I got the 69.4 from here: https://click.intel.com/intelr-realsensetm-depth-camera-d435.html
camera.set_image_size(IMAGE_SIZE[1], IMAGE_SIZE[0])
camera.set_position(2.30, 0, 1.30)
settings.add_sensor(camera)
else:
# Alternatively, we can load these settings from a file.
with open(args.settings_filepath, 'r') as fp:
settings = fp.read()
# Now we load these settings into the server. The server replies
# with a scene description containing the available start spots for
# the player. Here we can provide a CarlaSettings object or a
# CarlaSettings.ini file as string.
scene = client.load_settings(settings)
# Choose one player start at random.
num_of_player_starts = len(scene.player_start_spots)
player_start = random.randint(0, max(0, num_of_player_starts - 1))
# Notify the server that we want to start the episode at the
# player_start index. This function blocks until the server is ready
# to start the episode.
print('Starting new episode...', )
client.start_episode(player_start)
status, depth_storage, one_log_dict, log_dicts, distance_travelled = run_episode(
client, controller, pts_2D, depth_storage, log_dicts,
frames_per_episode, args.controller_name, args.store_data)
if 'FAIL' in status:
num_fails += 1
print(status)
continue
else:
print('SUCCESS: ' + str(episode))
report['distances'].append(distance_travelled)
if args.store_data:
np.save(
'depth_data/{}_racetrack{}_depth_data{}.npy'.format(
args.controller_name, args.racetrack, episode),
depth_storage)
pd.DataFrame(log_dicts).to_csv(
'logs/{}_racetrack{}_log{}.txt'.format(
args.controller_name, args.racetrack, episode),
index=False)
episode += 1
report['num_fails'] = num_fails
report_output = os.path.join('reports', args.report_filename)
pd.to_pickle(report, report_output)
def run_carla_client(host, port, far):
# Here we will run a single episode with 300 frames.
number_of_frames = 3000
frame_step = 10 # Save one image every 100 frames
output_folder = '_out'
image_size = [1280, 720]
camera_local_pos = [0.3, 0.0, 1.3] # [X, Y, Z]
camera_local_rotation = [0, 0, 0] # [pitch(Y), yaw(Z), roll(X)]
fov = 59
# Connect with the server
with make_carla_client(host, port) as client:
print('CarlaClient connected')
# Here we load the settings.
settings = CarlaSettings()
settings.set(SynchronousMode=True,
SendNonPlayerAgentsInfo=False,
NumberOfVehicles=20,
NumberOfPedestrians=40,
WeatherId=7)
settings.randomize_seeds()
camera1 = Camera('CameraDepth', PostProcessing='Depth', FOV=fov)
camera1.set_image_size(*image_size)
camera1.set_position(*camera_local_pos)
camera1.set_rotation(*camera_local_rotation)
settings.add_sensor(camera1)
camera2 = Camera('CameraRGB', PostProcessing='SceneFinal', FOV=fov)
camera2.set_image_size(*image_size)
camera2.set_position(*camera_local_pos)
camera2.set_rotation(*camera_local_rotation)
settings.add_sensor(camera2)
client.load_settings(settings)
# Start at location index id '0'
client.start_episode(0)
# Compute the camera transform matrix
camera_to_car_transform = camera2.get_unreal_transform()
print("camera_to_car_transform", camera_to_car_transform)
# Iterate every frame in the episode except for the first one.
for frame in range(1, number_of_frames):
# Read the data produced by the server this frame.
measurements, sensor_data = client.read_data()
# Save one image every 'frame_step' frames
if not frame % frame_step:
# Start transformations time mesure.
# RGB image [[[r,g,b],..[r,g,b]],..[[r,g,b],..[r,g,b]]]
image_RGB = to_rgb_array(sensor_data['CameraRGB'])
world_transform = Transform(
measurements.player_measurements.transform)
# Compute the final transformation matrix.
car_to_world_transform = world_transform * camera_to_car_transform
print("{}.png".format(str(frame)))
print([
measurements.player_measurements.transform.location.x,
measurements.player_measurements.transform.location.y,
measurements.player_measurements.transform.location.z
])
print("world_transform\n", world_transform)
#print("car_to_world_transform\n",car_to_world_transform)
print('\n')
im_bgr = cv2.cvtColor(image_RGB, cv2.COLOR_RGB2BGR)
cv2.imwrite("./{}/{}.png".format(output_folder, str(frame)),
im_bgr)
# 2d to (camera) local 3d
# We use the image_RGB to colorize each 3D point, this is optional.
# "max_depth" is used to keep only the points that are near to the
# camera, meaning 1.0 the farest points (sky)
# Save PLY to disk
# This generates the PLY string with the 3D points and the RGB colors
# for each row of the file.
client.send_control(
measurements.player_measurements.autopilot_control)
def run_carla_client(args):
# Here we will run 3 episodes with 300 frames each.
number_of_episodes = 3
frames_per_episode = 300
# We assume the CARLA server is already waiting for a client to connect at
# host:port. To create a connection we can use the `make_carla_client`
# context manager, it creates a CARLA client object and starts the
# connection. It will throw an exception if something goes wrong. The
# context manager makes sure the connection is always cleaned up on exit.
with make_carla_client(args.host, args.port) as client:
print('CarlaClient connected')
for episode in range(0, number_of_episodes):
# Start a new episode.
if args.settings_filepath is None:
# Create a CarlaSettings object. This object is a wrapper around
# the CarlaSettings.ini file. Here we set the configuration we
# want for the new episode.
settings = CarlaSettings()
settings.set(
SynchronousMode=True,
SendNonPlayerAgentsInfo=True,
NumberOfVehicles=20,
NumberOfPedestrians=40,
WeatherId=random.choice([1, 3, 7, 8, 14]),
QualityLevel=args.quality_level)
settings.randomize_seeds()
# Now we want to add a couple of cameras to the player vehicle.
# We will collect the images produced by these cameras every
# frame.
# The default camera captures RGB images of the scene.
camera0 = Camera('CameraRGB')
# Set image resolution in pixels.
camera0.set_image_size(800, 600)
# Set its position relative to the car in meters.
camera0.set_position(0.30, 0, 1.30)
settings.add_sensor(camera0)
# Let's add another camera producing ground-truth depth.
camera1 = Camera('CameraDepth', PostProcessing='Depth')
camera1.set_image_size(800, 600)
camera1.set_position(0.30, 0, 1.30)
settings.add_sensor(camera1)
if args.lidar:
lidar = Lidar('Lidar32')
lidar.set_position(0, 0, 2.50)
lidar.set_rotation(0, 0, 0)
lidar.set(
Channels=32,
Range=50,
PointsPerSecond=100000,
RotationFrequency=10,
UpperFovLimit=10,
LowerFovLimit=-30)
settings.add_sensor(lidar)
else:
# Alternatively, we can load these settings from a file.
with open(args.settings_filepath, 'r') as fp:
settings = fp.read()
# Now we load these settings into the server. The server replies
# with a scene description containing the available start spots for
# the player. Here we can provide a CarlaSettings object or a
# CarlaSettings.ini file as string.
scene = client.load_settings(settings)
# Choose one player start at random.
number_of_player_starts = len(scene.player_start_spots)
player_start = random.randint(0, max(0, number_of_player_starts - 1))
# Notify the server that we want to start the episode at the
# player_start index. This function blocks until the server is ready
# to start the episode.
print('Starting new episode at %r...' % scene.map_name)
client.start_episode(player_start)
# Iterate every frame in the episode.
for frame in range(0, frames_per_episode):
# Read the data produced by the server this frame.
measurements, sensor_data = client.read_data()
# Print some of the measurements.
print_measurements(measurements)
# Save the images to disk if requested.
if args.save_images_to_disk:
for name, measurement in sensor_data.items():
filename = args.out_filename_format.format(episode, name, frame)
measurement.save_to_disk(filename)
# We can access the encoded data of a given image as numpy
# array using its "data" property. For instance, to get the
# depth value (normalized) at pixel X, Y
#
# depth_array = sensor_data['CameraDepth'].data
# value_at_pixel = depth_array[Y, X]
#
# Now we have to send the instructions to control the vehicle.
# If we are in synchronous mode the server will pause the
# simulation until we send this control.
if not args.autopilot:
client.send_control(
steer=random.uniform(-1.0, 1.0),
throttle=0.5,
brake=0.0,
hand_brake=False,
reverse=False)
else:
# Together with the measurements, the server has sent the
# control that the in-game autopilot would do this frame. We
# can enable autopilot by sending back this control to the
# server. We can modify it if wanted, here for instance we
# will add some noise to the steer.
control = measurements.player_measurements.autopilot_control
control.steer += random.uniform(-0.1, 0.1)
client.send_control(control)