def transforms_from_agent(agent):
""" Returns the KITTI object type and transforms, locations and extension of the given agent """
if agent.HasField('pedestrian'):
obj_type = 'Pedestrian'
agent_transform = Transform(agent.pedestrian.transform)
bbox_transform = Transform(agent.pedestrian.bounding_box.transform)
ext = agent.pedestrian.bounding_box.extent
location = agent.pedestrian.transform.location
elif agent.HasField('vehicle'):
obj_type = 'Car'
agent_transform = Transform(agent.vehicle.transform)
bbox_transform = Transform(agent.vehicle.bounding_box.transform)
ext = agent.vehicle.bounding_box.extent
location = agent.vehicle.transform.location
else:
return (None, None, None, None, None)
return obj_type, agent_transform, bbox_transform, ext, location
def get_extrinsic(measurement_data, camera2):
camera_to_car_transform = camera2.get_unreal_transform()
world_transform = Transform(measurement_data.player_measurements.transform)
newworld = np.array(convert_transform(world_transform))
newcam = np.array(convert_transform(camera_to_car_transform))
return np.matmul(newworld, newcam), newcam
def make_carla_settings(args):
"""Make a CarlaSettings object with the settings we need."""
settings = CarlaSettings()
settings.set(
SynchronousMode=False,
SendNonPlayerAgentsInfo=True,
NumberOfVehicles=NUM_VEHICLES,
NumberOfPedestrians=NUM_PEDESTRIANS,
DisableTwoWheeledVehicles=NO_BIKE,
# WeatherId=random.choice([1, 3, 7, 8, 14]),
WeatherId=1,
QualityLevel=args.quality_level)
settings.randomize_seeds()
camera0 = sensor.Camera('CameraRGB')
camera0.set_image_size(WINDOW_WIDTH, WINDOW_HEIGHT)
camera0.set_position(0, 0.0, CAMERA_HEIGHT_POS)
camera0.set_rotation(0.0, 0.0, 0.0)
settings.add_sensor(camera0)
lidar = sensor.Lidar('Lidar32')
lidar.set_position(0, 0.0, LIDAR_HEIGHT_POS)
lidar.set_rotation(0, 0, 0)
lidar.set(Channels=40,
Range=MAX_RENDER_DEPTH_IN_METERS,
PointsPerSecond=720000,
RotationFrequency=10,
UpperFovLimit=7,
LowerFovLimit=-16)
settings.add_sensor(lidar)
""" Depth camera for filtering out occluded vehicles """
depth_camera = sensor.Camera('DepthCamera', PostProcessing='Depth')
depth_camera.set(FOV=90.0)
depth_camera.set_image_size(WINDOW_WIDTH, WINDOW_HEIGHT)
depth_camera.set_position(0, 0, CAMERA_HEIGHT_POS)
depth_camera.set_rotation(0, 0, 0)
settings.add_sensor(depth_camera)
# (Intrinsic) K Matrix
# | f 0 Cu
# | 0 f Cv
# | 0 0 1
# (Cu, Cv) is center of image
k = np.identity(3)
k[0, 2] = WINDOW_WIDTH_HALF
k[1, 2] = WINDOW_HEIGHT_HALF
f = WINDOW_WIDTH / \
(2.0 * math.tan(90.0 * math.pi / 360.0))
k[0, 0] = k[1, 1] = f
camera_to_car_transform = camera0.get_unreal_transform()
lidar_to_car_transform = lidar.get_transform() * Transform(
Rotation(yaw=90), Scale(z=-1))
return settings, k, camera_to_car_transform, lidar_to_car_transform
def find_pts(update_pts, measurement_data, newcam):
pts_2d_ls_new = []
for i in range(len(update_pts)):
world_transform = Transform(
measurement_data.player_measurements.transform)
newworld = np.array(convert_transform(world_transform))
extrinsic = np.matmul(newworld, newcam)
world_coord = np.asarray(update_pts[i]).reshape(4, -1)
conversion = _3d_to_2d(extrinsic, world_coord, intrinsic)
pts_2d_ls_new.append(conversion)
return pts_2d_ls_new
def map_ground_bounding_box_to_2D(distance_img, world_transform,
obstacle_transform, bounding_box,
rgb_transform, rgb_intrinsic, rgb_img_size):
(image_width, image_height) = rgb_img_size
extrinsic_mat = world_transform * rgb_transform
obj_transform = Transform(obstacle_transform.to_transform_pb2())
bbox_pb2 = bounding_box.to_bounding_box_pb2()
bbox_transform = Transform(bbox_pb2.transform)
ext = bbox_pb2.extent
# 8 bounding box vertices relative to (0,0,0)
bbox = np.array([[ext.x, ext.y, ext.z], [ext.x, -ext.y, ext.z],
[ext.x, ext.y, -ext.z], [ext.x, -ext.y, -ext.z],
[-ext.x, ext.y, ext.z], [-ext.x, -ext.y, ext.z],
[-ext.x, ext.y, -ext.z], [-ext.x, -ext.y, -ext.z]])
# Transform the vertices with respect to the bounding box transform.
bbox = bbox_transform.transform_points(bbox)
# The bounding box transform is with respect to the object transform.
# Transform the points relative to its transform.
bbox = obj_transform.transform_points(bbox)
# Object's transform is relative to the world. Thus, the bbox contains
# the 3D bounding box vertices relative to the world.
coords = []
for vertex in bbox:
pos_vector = np.array([
[vertex[0, 0]], # [[X,
[vertex[0, 1]], # Y,
[vertex[0, 2]], # Z,
[1.0] # 1.0]]
])
# Transform the points to camera.
transformed_3d_pos = np.dot(inv(extrinsic_mat.matrix), pos_vector)
# Transform the points to 2D.
pos2d = np.dot(rgb_intrinsic, transformed_3d_pos[:3])
# Normalize the 2D points.
pos2d = np.array([pos2d[0] / pos2d[2], pos2d[1] / pos2d[2], pos2d[2]])
# Add the points to the image.
if pos2d[2] > 0: # If the point is in front of the camera.
x_2d = float(image_width - pos2d[0])
y_2d = float(image_height - pos2d[1])
if (x_2d >= 0 or y_2d >= 0) and (x_2d < image_width
or y_2d < image_height):
coords.append((x_2d, y_2d, pos2d[2]))
return coords
def run_carla_client(host, port, far):
# Here we will run a single episode with 300 frames.
number_of_frames = 30
frame_step = 10 # Save one image every 100 frames
image_size = [800, 600]
camera_local_pos = [0.3, 0.0, 1.3] # [X, Y, Z]
camera_local_rotation = [0, 0, 0] # [pitch(Y), yaw(Z), roll(X)]
fov = 70
autopilot = False
control = VehicleControl()
for start_i in range(150):
output_folder = '/home2/mariap/Packages/CARLA_0.8.2/PythonClient/_out/pos_' + str(
start_i)
if not os.path.exists(output_folder):
os.mkdir(output_folder)
print("make " + str(output_folder))
# Connect with the server
with make_carla_client(host, port) as client:
print('CarlaClient connected')
for start_i in range(150):
output_folder = '/home2/mariap/Packages/CARLA_0.8.2/PythonClient/_out/pos_' + str(
start_i)
print(output_folder)
# Here we load the settings.
settings = CarlaSettings()
settings.set(SynchronousMode=True,
SendNonPlayerAgentsInfo=True,
NumberOfVehicles=100,
NumberOfPedestrians=500,
WeatherId=random.choice([1, 3, 7, 8, 14]))
settings.randomize_seeds()
camera1 = Camera('CameraDepth', PostProcessing='Depth', FOV=fov)
camera1.set_image_size(*image_size)
camera1.set_position(*camera_local_pos)
camera1.set_rotation(*camera_local_rotation)
settings.add_sensor(camera1)
camera2 = Camera('CameraRGB', PostProcessing='SceneFinal', FOV=fov)
camera2.set_image_size(*image_size)
camera2.set_position(*camera_local_pos)
camera2.set_rotation(*camera_local_rotation)
settings.add_sensor(camera2)
camera3 = Camera('CameraSeg',
PostProcessing='SemanticSegmentation')
camera3.set_image_size(*image_size)
camera3.set_position(*camera_local_pos)
camera3.set_rotation(*camera_local_rotation)
settings.add_sensor(camera3)
client.load_settings(settings)
# Start at location index id '0'
client.start_episode(start_i)
cameras_dict = {}
pedestrians_dict = {}
cars_dict = {}
# Compute the camera transform matrix
camera_to_car_transform = camera2.get_unreal_transform()
# (Intrinsic) (3, 3) K Matrix
K = np.identity(3)
K[0, 2] = image_size[0] / 2.0
K[1, 2] = image_size[1] / 2.0
K[0,
0] = K[1,
1] = image_size[0] / (2.0 * np.tan(fov * np.pi / 360.0))
with open(output_folder + '/camera_intrinsics.p', 'w') as camfile:
pickle.dump(K, camfile)
# Iterate every frame in the episode except for the first one.
for frame in range(1, number_of_frames):
# Read the data produced by the server this frame.
measurements, sensor_data = client.read_data()
# Save one image every 'frame_step' frames
if not frame % frame_step:
for name, measurement in sensor_data.items():
filename = '{:s}/{:0>6d}'.format(name, frame)
measurement.save_to_disk(
os.path.join(output_folder, filename))
# Start transformations time mesure.
timer = StopWatch()
# RGB image [[[r,g,b],..[r,g,b]],..[[r,g,b],..[r,g,b]]]
image_RGB = to_rgb_array(sensor_data['CameraRGB'])
image_seg = np.tile(
labels_to_array(sensor_data['CameraSeg']), (1, 1, 3))
# 2d to (camera) local 3d
# We use the image_RGB to colorize each 3D point, this is optional.
# "max_depth" is used to keep only the points that are near to the
# camera, meaning 1.0 the farest points (sky)
point_cloud = depth_to_local_point_cloud(
sensor_data['CameraDepth'], image_RGB, max_depth=far)
point_cloud_seg = depth_to_local_point_cloud(
sensor_data['CameraDepth'], image_seg, max_depth=far)
# (Camera) local 3d to world 3d.
# Get the transform from the player protobuf transformation.
world_transform = Transform(
measurements.player_measurements.transform)
# Compute the final transformation matrix.
car_to_world_transform = world_transform * camera_to_car_transform
# Car to World transformation given the 3D points and the
# transformation matrix.
point_cloud.apply_transform(car_to_world_transform)
point_cloud_seg.apply_transform(car_to_world_transform)
Rt = car_to_world_transform.matrix
Rt_inv = car_to_world_transform.inverse(
).matrix # Rt_inv is the world to camera matrix !!
#R_inv=world_transform.inverse().matrix
cameras_dict[frame] = {}
cameras_dict[frame]['inverse_rotation'] = Rt_inv[:]
cameras_dict[frame]['rotation'] = Rt[:]
cameras_dict[frame]['translation'] = Rt_inv[0:3, 3]
cameras_dict[frame][
'camera_to_car'] = camera_to_car_transform.matrix
# Get non-player info
vehicles = {}
pedestrians = {}
for agent in measurements.non_player_agents:
# check if the agent is a vehicle.
if agent.HasField('vehicle'):
pos = agent.vehicle.transform.location
pos_vector = np.array([[pos.x], [pos.y], [pos.z],
[1.0]])
trnasformed_3d_pos = np.dot(Rt_inv, pos_vector)
pos2d = np.dot(K, trnasformed_3d_pos[:3])
# Normalize the point
norm_pos2d = np.array([
pos2d[0] / pos2d[2], pos2d[1] / pos2d[2],
pos2d[2]
])
# Now, pos2d contains the [x, y, d] values of the image in pixels (where d is the depth)
# You can use the depth to know the points that are in front of the camera (positive depth).
x_2d = image_size[0] - norm_pos2d[0]
y_2d = image_size[1] - norm_pos2d[1]
vehicles[agent.id] = {}
vehicles[agent.id]['transform'] = [
agent.vehicle.transform.location.x,
agent.vehicle.transform.location.y,
agent.vehicle.transform.location.z
]
vehicles[agent.id][
'bounding_box.transform'] = agent.vehicle.bounding_box.transform.location.z
vehicles[agent.id][
'yaw'] = agent.vehicle.transform.rotation.yaw
vehicles[agent.id]['bounding_box'] = [
agent.vehicle.bounding_box.extent.x,
agent.vehicle.bounding_box.extent.y,
agent.vehicle.bounding_box.extent.z
]
vehicle_transform = Transform(
agent.vehicle.bounding_box.transform)
pos = agent.vehicle.transform.location
bbox3d = agent.vehicle.bounding_box.extent
# Compute the 3D bounding boxes
# f contains the 4 points that corresponds to the bottom
f = np.array([[
pos.x + bbox3d.x, pos.y - bbox3d.y,
pos.z - bbox3d.z +
agent.vehicle.bounding_box.transform.location.z
],
[
pos.x + bbox3d.x,
pos.y + bbox3d.y,
pos.z - bbox3d.z + agent.vehicle.
bounding_box.transform.location.z
],
[
pos.x - bbox3d.x,
pos.y + bbox3d.y,
pos.z - bbox3d.z + agent.vehicle.
bounding_box.transform.location.z
],
[
pos.x - bbox3d.x,
pos.y - bbox3d.y,
pos.z - bbox3d.z + agent.vehicle.
bounding_box.transform.location.z
],
[
pos.x + bbox3d.x,
pos.y - bbox3d.y,
pos.z + bbox3d.z + agent.vehicle.
bounding_box.transform.location.z
],
[
pos.x + bbox3d.x,
pos.y + bbox3d.y,
pos.z + bbox3d.z + agent.vehicle.
bounding_box.transform.location.z
],
[
pos.x - bbox3d.x,
pos.y + bbox3d.y,
pos.z + bbox3d.z + agent.vehicle.
bounding_box.transform.location.z
],
[
pos.x - bbox3d.x,
pos.y - bbox3d.y,
pos.z + bbox3d.z + agent.vehicle.
bounding_box.transform.location.z
]])
f_rotated = vehicle_transform.transform_points(f)
f_2D_rotated = []
vehicles[
agent.id]['bounding_box_coord'] = f_rotated
for i in range(f.shape[0]):
point = np.array([[f_rotated[i, 0]],
[f_rotated[i, 1]],
[f_rotated[i, 2]], [1]])
transformed_2d_pos = np.dot(
Rt_inv,
point) # 3d Position in camera space
pos2d = np.dot(
K, transformed_2d_pos[:3]
) # Conversion to camera frustum space
norm_pos2d = np.array([
pos2d[0] / pos2d[2], pos2d[1] / pos2d[2],
pos2d[2]
])
#print([image_size[0] - (pos2d[0] / pos2d[2]), image_size[1] - (pos2d[1] / pos2d[2])])
f_2D_rotated.append(
np.array([
image_size[0] - norm_pos2d[0],
image_size[1] - norm_pos2d[1]
]))
vehicles[
agent.id]['bounding_box_2D'] = f_2D_rotated
elif agent.HasField('pedestrian'):
pedestrians[agent.id] = {}
pedestrians[agent.id]['transform'] = [
agent.pedestrian.transform.location.x,
agent.pedestrian.transform.location.y,
agent.pedestrian.transform.location.z
]
pedestrians[agent.id][
'yaw'] = agent.pedestrian.transform.rotation.yaw
pedestrians[agent.id]['bounding_box'] = [
agent.pedestrian.bounding_box.extent.x,
agent.pedestrian.bounding_box.extent.y,
agent.pedestrian.bounding_box.extent.z
]
vehicle_transform = Transform(
agent.pedestrian.bounding_box.transform)
pos = agent.pedestrian.transform.location
bbox3d = agent.pedestrian.bounding_box.extent
# Compute the 3D bounding boxes
# f contains the 4 points that corresponds to the bottom
f = np.array(
[[pos.x + bbox3d.x, pos.y - bbox3d.y, pos.z],
[pos.x + bbox3d.x, pos.y + bbox3d.y, pos.z],
[pos.x - bbox3d.x, pos.y + bbox3d.y, pos.z],
[pos.x - bbox3d.x, pos.y - bbox3d.y, pos.z],
[
pos.x + bbox3d.x, pos.y - bbox3d.y,
pos.z + bbox3d.z
],
[
pos.x + bbox3d.x, pos.y + bbox3d.y,
pos.z + bbox3d.z
],
[
pos.x - bbox3d.x, pos.y + bbox3d.y,
pos.z + bbox3d.z
],
[
pos.x - bbox3d.x, pos.y - bbox3d.y,
pos.z + bbox3d.z
]])
f_rotated = vehicle_transform.transform_points(f)
pedestrians[
agent.id]['bounding_box_coord'] = f_rotated
f_2D_rotated = []
for i in range(f.shape[0]):
point = np.array([[f_rotated[i, 0]],
[f_rotated[i, 1]],
[f_rotated[i, 2]], [1]])
transformed_2d_pos = np.dot(
Rt_inv, point) # See above for cars
pos2d = np.dot(K, transformed_2d_pos[:3])
norm_pos2d = np.array([
pos2d[0] / pos2d[2], pos2d[1] / pos2d[2],
pos2d[2]
])
f_2D_rotated.append([
image_size[0] - norm_pos2d[0],
image_size[1] - norm_pos2d[1]
])
pedestrians[
agent.id]['bounding_box_2D'] = f_2D_rotated
cars_dict[frame] = vehicles
pedestrians_dict[frame] = pedestrians
# End transformations time mesure.
timer.stop()
# Save PLY to disk
# This generates the PLY string with the 3D points and the RGB colors
# for each row of the file.
point_cloud.save_to_disk(
os.path.join(output_folder,
'{:0>5}.ply'.format(frame)))
point_cloud_seg.save_to_disk(
os.path.join(output_folder,
'{:0>5}_seg.ply'.format(frame)))
print_message(timer.milliseconds(), len(point_cloud),
frame)
if autopilot:
client.send_control(
measurements.player_measurements.autopilot_control)
else:
control.hand_brake = True
client.send_control(control)
with open(output_folder + '/cameras.p', 'w') as camerafile:
pickle.dump(cameras_dict, camerafile)
print(output_folder + "cameras.p")
with open(output_folder + '/people.p', 'w') as peoplefile:
pickle.dump(pedestrians_dict, peoplefile)
with open(output_folder + '/cars.p', 'w') as carfile:
pickle.dump(cars_dict, carfile)
def run_carla_client(host, port, far):
# Here we will run a single episode with 300 frames.
number_of_frames = 3000
frame_step = 10 # Save one image every 100 frames
output_folder = '_out'
image_size = [1280, 720]
camera_local_pos = [0.7, 0.0, 1.3] # [X, Y, Z]
camera_local_rotation = [0, 0, 0] # [pitch(Y), yaw(Z), roll(X)]
fov = 59
WINDOW_WIDTH = 1280
WINDOW_HEIGHT = 720
MINI_WINDOW_WIDTH = 320
MINI_WINDOW_HEIGHT = 180
WINDOW_WIDTH_HALF = WINDOW_WIDTH / 2
WINDOW_HEIGHT_HALF = WINDOW_HEIGHT / 2
k = np.identity(3)
k[0, 2] = WINDOW_WIDTH_HALF
k[1, 2] = WINDOW_HEIGHT_HALF
k[0, 0] = k[1, 1] = WINDOW_WIDTH / \
(2.0 * math.tan(59.0 * math.pi / 360.0))
intrinsic = k
# Connect with the server
with make_carla_client(host, port) as client:
print('CarlaClient connected')
# Here we load the settings.
settings = CarlaSettings()
settings.set(SynchronousMode=True,
SendNonPlayerAgentsInfo=False,
NumberOfVehicles=20,
NumberOfPedestrians=40,
WeatherId=1)
settings.randomize_seeds()
camera1 = Camera('CameraDepth', PostProcessing='Depth', FOV=fov)
camera1.set_image_size(*image_size)
camera1.set_position(*camera_local_pos)
camera1.set_rotation(*camera_local_rotation)
settings.add_sensor(camera1)
camera2 = Camera('CameraRGB', PostProcessing='SceneFinal', FOV=fov)
camera2.set_image_size(*image_size)
camera2.set_position(*camera_local_pos)
camera2.set_rotation(*camera_local_rotation)
settings.add_sensor(camera2)
#scene = client.load_settings(settings)
client.load_settings(settings)
#print("sjdsjhdjshdjshdjshgds",scene.player_start_spots[0].location.x)
# Start at location index id '0'
client.start_episode(0)
# Compute the camera transform matrix
camera_to_car_transform = camera2.get_unreal_transform()
print("camera_to_car_transform", camera_to_car_transform)
carla_utils_obj = segmentation.carla_utils(intrinsic)
# Iterate every frame in the episode except for the first one.
for frame in range(1, number_of_frames):
# Read the data produced by the server this frame.
measurements, sensor_data = client.read_data()
# Save one image every 'frame_step' frames
if not frame % frame_step:
# Start transformations time mesure.
# RGB image [[[r,g,b],..[r,g,b]],..[[r,g,b],..[r,g,b]]]
image_RGB = to_rgb_array(sensor_data['CameraRGB'])
image_depth = to_rgb_array(sensor_data['CameraDepth'])
#measurements.player_measurements.transform.location.x-=40
#measurements.player_measurements.transform.location.z-=2
world_transform = Transform(
measurements.player_measurements.transform)
im_bgr = cv2.cvtColor(image_RGB, cv2.COLOR_RGB2BGR)
pos_vector = ([
measurements.player_measurements.transform.location.x,
measurements.player_measurements.transform.location.y,
measurements.player_measurements.transform.location.z
])
fdfd = str(world_transform)
sdsd = fdfd[1:-1].split('\n')
new = []
for i in sdsd:
dd = i[:-1].lstrip('[ ')
ff = re.sub("\s+", ",", dd.strip())
gg = ff.split(',')
new.append([float(i) for i in gg])
newworld = np.array(new)
fdfd = str(camera_to_car_transform)
sdsd = fdfd[1:-1].split('\n')
new = []
for i in sdsd:
dd = i[:-1].lstrip('[ ')
ff = re.sub("\s+", ",", dd.strip())
gg = ff.split(',')
new.append([float(i) for i in gg])
newcam = np.array(new)
extrinsic = np.matmul(newworld, newcam)
get_2d_point = carla_utils_obj.run_seg(im_bgr, extrinsic,
pos_vector)
print(get_2d_point)
depth = image_depth[int(get_2d_point[0]), int(get_2d_point[1])]
scale = depth[0] + depth[1] * 256 + depth[2] * 256 * 256
scale = scale / ((256 * 256 * 256) - 1)
depth = scale * 1000
pos2d = np.array([(WINDOW_WIDTH - get_2d_point[1]) * depth,
(WINDOW_HEIGHT - get_2d_point[0]) * depth,
depth])
first = np.dot(np.linalg.inv(intrinsic), pos2d)
first = np.append(first, 1)
sec = np.matmul((extrinsic), first)
print("present", pos_vector)
print('goal-', sec)
client.send_control(
measurements.player_measurements.autopilot_control)
def run_carla_client(args):
# Here we will run 3 episodes with 300 frames each.
number_of_episodes = 15
frames_per_episode = 1230
# [0 , 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10, 11, 12, 13, 14]
vehicles_num = [
140, 100, 120, 80, 90, 100, 80, 90, 110, 100, 80, 70, 70, 80, 100
]
# We assume the CARLA server is already waiting for a client to connect at
# host:port. To create a connection we can use the `make_carla_client`
# context manager, it creates a CARLA client object and starts the
# connection. It will throw an exception if something goes wrong. The
# context manager makes sure the connection is always cleaned up on exit.
with make_carla_client(args.host, args.port) as client:
print('CarlaClient connected')
for episode in range(0, number_of_episodes):
# Start a new episode.
if args.settings_filepath is None:
# Create a CarlaSettings object. This object is a wrapper around
# the CarlaSettings.ini file. Here we set the configuration we
# want for the new episode.
settings = CarlaSettings()
settings.set(
SynchronousMode=True,
SendNonPlayerAgentsInfo=False,
NumberOfVehicles=vehicles_num[
episode], #random.choice([0, 20, 15, 20, 25, 21, 24, 18, 40, 35, 25, 30]), #25,
NumberOfPedestrians=120,
DisableTwoWheeledVehicles=False,
WeatherId=episode, #1, #random.choice([1, 3, 7, 8, 14]),
QualityLevel=args.quality_level)
settings.randomize_seeds()
# Now we want to add a couple of cameras to the player vehicle.
# We will collect the images produced by these cameras every
# frame.
# LEFT RGB CAMERA
camera_l = Camera('LeftCameraRGB', PostProcessing='SceneFinal')
camera_l.set_image_size(800, 600)
camera_l.set_position(1.30, -0.27, 1.50)
settings.add_sensor(camera_l)
# LEFT DEPTH
camera_ld = Camera('LeftCameraDepth', PostProcessing='Depth')
camera_ld.set_image_size(800, 600)
camera_ld.set_position(1.30, -0.27, 1.50)
settings.add_sensor(camera_ld)
# LEFT SEGMENTATION
camera_ls = Camera('LeftCameraSeg',
PostProcessing='SemanticSegmentation')
camera_ls.set_image_size(800, 600)
camera_ls.set_position(1.30, -0.27, 1.50)
settings.add_sensor(camera_ls)
# RIGHT RGB CAMERA
camera_r = Camera('RightCameraRGB',
PostProcessing='SceneFinal')
camera_r.set_image_size(800, 600)
camera_r.set_position(1.30, 0.27, 1.50)
settings.add_sensor(camera_r)
# RIGHT DEPTH
camera_rd = Camera('RightCameraDepth', PostProcessing='Depth')
camera_rd.set_image_size(800, 600)
camera_rd.set_position(1.30, 0.27, 1.50)
settings.add_sensor(camera_rd)
# RIGHT SEGMENTATION
camera_rs = Camera('RightCameraSeg',
PostProcessing='SemanticSegmentation')
camera_rs.set_image_size(800, 600)
camera_rs.set_position(1.30, 0.27, 1.50)
settings.add_sensor(camera_rs)
# LEFT 15 DEGREE RGB CAMERA
camera_l_15 = Camera('15_LeftCameraRGB',
PostProcessing='SceneFinal')
camera_l_15.set_image_size(800, 600)
camera_l_15.set_position(1.30, -0.7, 1.50) # [X, Y, Z]
camera_l_15.set_rotation(0, -15.0,
0) # [pitch(Y), yaw(Z), roll(X)]
settings.add_sensor(camera_l_15)
# LEFT 15 DEGREE DEPTH
camera_ld_15 = Camera('15_LeftCameraDepth',
PostProcessing='Depth')
camera_ld_15.set_image_size(800, 600)
camera_ld_15.set_position(1.30, -0.7, 1.50)
camera_ld_15.set_rotation(0, -15.0, 0)
settings.add_sensor(camera_ld_15)
# LEFT 15 DEGREE SEGMENTATION
camera_ls_15 = Camera('15_LeftCameraSeg',
PostProcessing='SemanticSegmentation')
camera_ls_15.set_image_size(800, 600)
camera_ls_15.set_position(1.30, -0.7, 1.50)
camera_ls_15.set_rotation(0, -15.0, 0)
settings.add_sensor(camera_ls_15)
# Right 15 DEGREE RGB CAMERA
camera_r_15 = Camera('15_RightCameraRGB',
PostProcessing='SceneFinal')
camera_r_15.set_image_size(800, 600)
camera_r_15.set_position(1.30, 0.7, 1.50)
camera_r_15.set_rotation(0, 15.0, 0)
settings.add_sensor(camera_r_15)
# Right 15 DEGREE DEPTH
camera_rd_15 = Camera('15_RightCameraDepth',
PostProcessing='Depth')
camera_rd_15.set_image_size(800, 600)
camera_rd_15.set_position(1.30, 0.7, 1.50)
camera_rd_15.set_rotation(0, 15.0, 0)
settings.add_sensor(camera_rd_15)
# RIGHT 15 DEGREE SEGMENTATION
camera_ls_15 = Camera('15_RightCameraSeg',
PostProcessing='SemanticSegmentation')
camera_ls_15.set_image_size(800, 600)
camera_ls_15.set_position(1.30, 0.7, 1.50)
camera_ls_15.set_rotation(0, 15.0, 0)
settings.add_sensor(camera_ls_15)
# LEFT 30 DEGREE RGB CAMERA
camera_l_30 = Camera('30_LeftCameraRGB',
PostProcessing='SceneFinal')
camera_l_30.set_image_size(800, 600)
camera_l_30.set_position(1.30, -0.7, 1.50)
camera_l_30.set_rotation(0, -30.0, 0)
settings.add_sensor(camera_l_30)
# LEFT 30 DEGREE DEPTH
camera_ld_30 = Camera('30_LeftCameraDepth',
PostProcessing='Depth')
camera_ld_30.set_image_size(800, 600)
camera_ld_30.set_position(1.30, -0.7, 1.50)
camera_ld_30.set_rotation(0, -30.0, 0)
settings.add_sensor(camera_ld_30)
# LEFT 30 DEGREE SEGMENTATION
camera_ls_30 = Camera('30_LeftCameraSeg',
PostProcessing='SemanticSegmentation')
camera_ls_30.set_image_size(800, 600)
camera_ls_30.set_position(1.30, -0.7, 1.50)
camera_ls_30.set_rotation(0, -30.0, 0)
settings.add_sensor(camera_ls_30)
# RIGHT 30 DEGREE RGB CAMERA
camera_r_30 = Camera('30_RightCameraRGB',
PostProcessing='SceneFinal')
camera_r_30.set_image_size(800, 600)
camera_r_30.set_position(1.30, 0.7, 1.50)
camera_r_30.set_rotation(0, 30.0, 0)
settings.add_sensor(camera_r_30)
# RIGHT 30 DEGREE DEPTH
camera_rd_30 = Camera('30_RightCameraDepth',
PostProcessing='Depth')
camera_rd_30.set_image_size(800, 600)
camera_rd_30.set_position(1.30, 0.7, 1.50)
camera_rd_30.set_rotation(0, 30.0, 0)
settings.add_sensor(camera_rd_30)
# RIGHT 30 DEGREE SEGMENTATION
camera_rs_30 = Camera('30_RightCameraSeg',
PostProcessing='SemanticSegmentation')
camera_rs_30.set_image_size(800, 600)
camera_rs_30.set_position(1.30, 0.7, 1.50)
camera_rs_30.set_rotation(0, 30.0, 0)
settings.add_sensor(camera_rs_30)
else:
# Alternatively, we can load these settings from a file.
with open(args.settings_filepath, 'r') as fp:
settings = fp.read()
# Now we load these settings into the server. The server replies
# with a scene description containing the available start spots for
# the player. Here we can provide a CarlaSettings object or a
# CarlaSettings.ini file as string.
scene = client.load_settings(settings)
# Choose one player start at random.
number_of_player_starts = len(scene.player_start_spots)
player_start = random.randint(0, max(0,
number_of_player_starts - 1))
# Notify the server that we want to start the episode at the
# player_start index. This function blocks until the server is ready
# to start the episode.
print('Starting new episode...')
client.start_episode(player_start)
camera_l_to_car_transform = camera_l.get_unreal_transform()
camera_r_to_car_transform = camera_r.get_unreal_transform()
camera_l_15_to_car_transform = camera_l_15.get_unreal_transform()
camera_r_15_to_car_transform = camera_r_15.get_unreal_transform()
camera_l_30_to_car_transform = camera_l_30.get_unreal_transform()
camera_r_30_to_car_transform = camera_r_30.get_unreal_transform()
if not os.path.isdir(args.dataset_path +
"/episode_{:0>4d}".format(episode)):
os.makedirs(args.dataset_path +
"/episode_{:0>4d}".format(episode))
# Iterate every frame in the episode.
for frame in range(0, frames_per_episode):
# Read the data produced by the server this frame.
measurements, sensor_data = client.read_data()
#image = sensor_data.get('LeftCameraSeg', None)
# array = image_converter.depth_to_logarithmic_grayscale(image)
#array = image_converter.labels_to_cityscapes_palette(image)
#filename = '{:0>6d}'.format(frame)
#filename += '.png'
#cv2.imwrite(filename, array)
#image.save_to_disk("/data/khoshhal/Dataset/episode_{:0>4d}/{:s}/{:0>6d}".format(episode, "LeftCameraSeg", frame))
# Print some of the measurements.
# print_measurements(measurements)
#player_measurements = measurements.player_measurements
world_transform = Transform(
measurements.player_measurements.transform)
# Compute the final transformation matrix.
camera_l_to_world_transform = world_transform * camera_l_to_car_transform
camera_r_to_world_transform = world_transform * camera_r_to_car_transform
camera_l_15_to_world_transform = world_transform * camera_l_15_to_car_transform
camera_r_15_to_world_transform = world_transform * camera_r_15_to_car_transform
camera_l_30_to_world_transform = world_transform * camera_l_30_to_car_transform
camera_r_30_to_world_transform = world_transform * camera_r_30_to_car_transform
args.out_filename_format = dataset_path + '/episode_{:0>4d}/{:s}/{:0>6d}'
# Save the images to disk if requested.
if frame >= 30:
if args.save_images_to_disk:
for name, measurement in sensor_data.items():
filename = args.out_filename_format.format(
episode, name, frame - 30)
measurement.save_to_disk(filename)
# Save Transform matrix of each camera to separated files
line = ""
filename = "{}/episode_{:0>4d}/LeftCamera".format(
args.dataset_path, episode) + ".txt"
with open(filename, 'a') as LeftCamera:
for x in np.asarray(camera_l_to_world_transform.
matrix[:3, :]).reshape(-1):
line += "{:.8e} ".format(x)
line = line[:-1]
line += "\n"
LeftCamera.write(line)
line = ""
filename = "{}/episode_{:0>4d}/RightCamera".format(
args.dataset_path, episode) + ".txt"
with open(filename, 'a') as RightCamera:
for x in np.asarray(camera_r_to_world_transform.
matrix[:3, :]).reshape(-1):
line += "{:.8e} ".format(x)
line = line[:-1]
line += "\n"
RightCamera.write(line)
line = ""
filename = "{}/episode_{:0>4d}/15_LeftCamera".format(
args.dataset_path, episode) + ".txt"
with open(filename, 'a') as myfile:
for x in np.asarray(camera_l_15_to_world_transform.
matrix[:3, :]).reshape(-1):
line += "{:.8e} ".format(x)
line = line[:-1]
line += "\n"
myfile.write(line)
line = ""
filename = "{}/episode_{:0>4d}/15_RightCamera".format(
args.dataset_path, episode) + ".txt"
with open(filename, 'a') as myfile:
for x in np.asarray(camera_r_15_to_world_transform.
matrix[:3, :]).reshape(-1):
line += "{:.8e} ".format(x)
line = line[:-1]
line += "\n"
myfile.write(line)
line = ""
filename = "{}/episode_{:0>4d}/30_LeftCamera".format(
args.dataset_path, episode) + ".txt"
with open(filename, 'a') as myfile:
for x in np.asarray(camera_l_30_to_world_transform.
matrix[:3, :]).reshape(-1):
line += "{:.8e} ".format(x)
line = line[:-1]
line += "\n"
myfile.write(line)
line = ""
filename = "{}/episode_{:0>4d}/30_RightCamera".format(
args.dataset_path, episode) + ".txt"
with open(filename, 'a') as myfile:
for x in np.asarray(camera_r_30_to_world_transform.
matrix[:3, :]).reshape(-1):
line += "{:.8e} ".format(x)
line = line[:-1]
line += "\n"
myfile.write(line)
line = ""
# We can access the encoded data of a given image as numpy
# array using its "data" property. For instance, to get the
# depth value (normalized) at pixel X, Y
#
# depth_array = sensor_data['CameraDepth'].data
# value_at_pixel = depth_array[Y, X]
#
# Now we have to send the instructions to control the vehicle.
# If we are in synchronous mode the server will pause the
# simulation until we send this control.
if not args.autopilot:
client.send_control(steer=random.uniform(-1.0, 1.0),
throttle=0.5,
brake=0.0,
hand_brake=False,
reverse=False)
else:
# Together with the measurements, the server has sent the
# control that the in-game autopilot would do this frame. We
# can enable autopilot by sending back this control to the
# server. We can modify it if wanted, here for instance we
# will add some noise to the steer.
control = measurements.player_measurements.autopilot_control
#control.steer += random.uniform(-0.1, 0.1)
client.send_control(control)
#time.sleep(1)
myfile.close()
LeftCamera.close()
RightCamera.close()
def _on_loop(self):
self._timer.tick()
measurements, sensor_data = self.client.read_data()
# logging.info("Frame no: {}, = {}".format(self.captured_frame_no,
# (self.captured_frame_no + 1) % NUM_RECORDINGS_BEFORE_RESET))
# Reset the environment if the agent is stuck or can't find any agents or if we have captured enough frames in this one
is_stuck = self._frames_since_last_capture >= NUM_EMPTY_FRAMES_BEFORE_RESET
is_enough_datapoints = (self._captured_frames_since_restart +
1) % NUM_RECORDINGS_BEFORE_RESET == 0
if (is_stuck or is_enough_datapoints) and GEN_DATA:
logging.warning("Is stucK: {}, is_enough_datapoints: {}".format(
is_stuck, is_enough_datapoints))
self._on_new_episode()
# If we dont sleep, the client will continue to render
return True
# (Extrinsic) Rt Matrix
# (Camera) local 3d to world 3d.
# Get the transform from the player protobuf transformation.
world_transform = Transform(measurements.player_measurements.transform)
# Compute the final transformation matrix.
self._extrinsic = world_transform * self._camera_to_car_transform
self._measurements = measurements
self._last_player_location = measurements.player_measurements.transform.location
self._main_image = sensor_data.get('CameraRGB', None)
self._lidar_measurement = sensor_data.get('Lidar32', None)
self._depth_image = sensor_data.get('DepthCamera', None)
# Print measurements every second.
if self._timer.elapsed_seconds_since_lap() > 1.0:
if self._city_name is not None:
# Function to get car position on map.
map_position = self._map.convert_to_pixel([
measurements.player_measurements.transform.location.x,
measurements.player_measurements.transform.location.y,
measurements.player_measurements.transform.location.z
])
# Function to get orientation of the road car is in.
lane_orientation = self._map.get_lane_orientation([
measurements.player_measurements.transform.location.x,
measurements.player_measurements.transform.location.y,
measurements.player_measurements.transform.location.z
])
MeasurementsDisplayHelper.print_player_measurements_map(
measurements.player_measurements, map_position,
lane_orientation, self._timer)
else:
MeasurementsDisplayHelper.print_player_measurements(
measurements.player_measurements, self._timer)
# Plot position on the map as well.
self._timer.lap()
control = self._get_keyboard_control(pygame.key.get_pressed())
# Set the player position
if self._city_name is not None:
self._position = self._map.convert_to_pixel([
measurements.player_measurements.transform.location.x,
measurements.player_measurements.transform.location.y,
measurements.player_measurements.transform.location.z
])
self._agent_positions = measurements.non_player_agents
if control is None:
self._on_new_episode()
elif self._enable_autopilot:
self.client.send_control(
measurements.player_measurements.autopilot_control)
else:
self.client.send_control(control)
def run_carla_client(args):
# Here we will run 3 episodes with 300 frames each.
number_of_episodes = 15
frames_per_episode = 10050
# [0 , 1 , 2 , 3 , 4 , 5 , 6 , 7, 8 , 9 , 10, 11, 12, 13, 14]
# vehicles_num = [60, 60, 70, 50, 60, 60, 80, 60, 60, 60, 50, 70, 60, 50, 50]
vehicles_num = [35, 35, 30, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35]
# vehicles_num = [60, 60, 70, 50, 60, 60, 80, 60, 60, 60, 50, 70, 60, 50, 50]
# We assume the CARLA server is already waiting for a client to connect at
# host:port. To create a connection we can use the `make_carla_client`
# context manager, it creates a CARLA client object and starts the
# connection. It will throw an exception if something goes wrong. The
# context manager makes sure the connection is always cleaned up on exit.
with make_carla_client(args.host, args.port) as client:
print('CarlaClient connected')
# list_of_episodes = [11]
# list_of_episodes = [9, 11, 14]
list_of_episodes = [0, 1, 2, 3, 4, 5, 6, 7, 9, 11, 14]
# list_of_episodes = [6, 7, 9, 11, 14]
# list_of_episodes = [2, 3, 4, 5, 6, 7, 9, 11, 14]
# list_of_episodes = [2, 7]
# list_of_episodes = [5, 6, 7, 11, 14]
# list_of_episodes = [6, 11]
# list_of_episodes = [7]
#list(range(0, number_of_episodes))
#list_of_episodes.pop(13)
#list_of_episodes.pop(12)
#list_of_episodes.pop(10)
#list_of_episodes.pop(8)
for episode in list_of_episodes:
# Start a new episode.
if args.settings_filepath is None:
# Create a CarlaSettings object. This object is a wrapper around
# the CarlaSettings.ini file. Here we set the configuration we
# want for the new episode.
settings = CarlaSettings()
settings.set(
SynchronousMode=True,
SendNonPlayerAgentsInfo=False,
NumberOfVehicles=vehicles_num[
episode], #random.choice([0, 20, 15, 20, 25, 21, 24, 18, 40, 35, 25, 30]), #25,
NumberOfPedestrians=4,
DisableTwoWheeledVehicles=False,
WeatherId=episode, #1, #random.choice([1, 3, 7, 8, 14]),
QualityLevel=args.quality_level)
settings.randomize_seeds()
# Now we want to add a couple of cameras to the player vehicle.
# We will collect the images produced by these cameras every
# frame.
#### Cameras aligned across the y-axis
#### Horizontally shifted in the following Range
# [-1.62, -1.08, -0.54, 0.0, 0.54, 1.08, 1.62]
# LEFT RGB CAMERA
# y_locs = [-1.62, -1.08, -0.54, 0.0, 0.54, 1.08, 1.62]
# z_locs = [1.3, 1.84, 2.38, 2.92, 3.46, 4.0, 4.54]
# y_locs_left = [-1.08, -0.54, 0.0, 0.54, 1.08]
z_locs = [
i.item() - 7 * 0.5 for i in np.linspace(0, 14 * 0.5, 15)
] # Up
y_locs = [
i.item() - 7 * 0.5 for i in np.linspace(0, 14 * 0.5, 15)
] # Right
rotation_y = 0
CameraArray = {}
for j, y_position in enumerate(y_locs):
for i, z_position in enumerate(z_locs):
# COLOR
camera_rgb = Camera('RGB_{:0>2d}_{:0>2d}'.format(j, i),
PostProcessing='SceneFinal')
camera_rgb.set_image_size(800, 600)
camera_rgb.set_position(2.0, y_position,
2.00 - z_position)
camera_rgb.set_rotation(0, rotation_y, 0)
CameraArray['RGB_{:0>2d}_{:0>2d}'.format(
j, i)] = camera_rgb
settings.add_sensor(camera_rgb)
# DEPTH
camera_depth = Camera('Depth_{:0>2d}_{:0>2d}'.format(
j, i),
PostProcessing='Depth')
camera_depth.set_image_size(800, 600)
camera_depth.set_position(2.0, y_position,
2.0 - z_position)
camera_depth.set_rotation(0, rotation_y, 0)
CameraArray['Depth_{:0>2d}_{:0>2d}'.format(
j, i)] = camera_depth
settings.add_sensor(camera_depth)
# SEGMENTATION
camera_seg = Camera(
'SEM_{:0>2d}_{:0>2d}'.format(j, i),
PostProcessing='SemanticSegmentation')
camera_seg.set_image_size(800, 600)
camera_seg.set_position(2.0, y_position,
2.0 - z_position)
camera_seg.set_rotation(0, rotation_y, 0)
CameraArray['SEM_{:0>2d}_{:0>2d}'.format(
j, i)] = camera_seg
settings.add_sensor(camera_seg)
else:
with open(args.settings_filepath, 'r') as fp:
settings = fp.read()
scene = client.load_settings(settings)
# Choose one player start at random.
number_of_player_starts = len(scene.player_start_spots)
player_start = random.randint(0, max(0,
number_of_player_starts - 1))
# Notify the server that we want to start the episode at the
# player_start index. This function blocks until the server is ready
# to start the episode.
print('Starting new episode...')
client.start_episode(player_start)
print('Created a new episode...')
Cameras_to_car = []
for j in range(len(y_locs)):
for i in range(len(z_locs)):
Cameras_to_car.append(
CameraArray['RGB_{:0>2d}_{:0>2d}'.format(
j, i)].get_transform())
print('created a camera array')
# Create a folder for saving episode data
episode_dir = "{}/Town{:0>2d}/episode_{:0>5d}".format(
args.data_path, args.town_id, episode)
if not os.path.isdir(episode_dir):
os.makedirs(episode_dir)
print('created destination folder')
# Iterate every frame in the episode.
for frame in range(0, frames_per_episode):
# Read the data produced by the server this frame.
time.sleep(1)
measurements, sensor_data = client.read_data()
print('got measurements')
# player_measurements = measurements.player_measurements
world_transform = Transform(
measurements.player_measurements.transform)
# Compute the final transformation matrix.
print('debug point 1')
Cameras_to_world = [world_transform*cam_to_car \
for cam_to_car in Cameras_to_car]
if frame >= 50 and (frame % 100 == 0):
print('debug point 2')
if args.save_images_to_disk:
output_dir = os.path.join(
episode_dir, "{:0>6d}".format((frame - 50) / 100))
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
for name, measurement in sensor_data.items():
# filename = args.out_filename_format.format(episode, name, (frame-50)/100)
filename = os.path.join(output_dir, name)
print('writing:', filename)
measurement.save_to_disk(filename)
cam_2_world_file = os.path.join(
output_dir, 'cam_2_world.txt')
with open(cam_2_world_file, 'w') as myfile:
for cam_num in range(len(Cameras_to_world)):
line = ""
for x in np.asarray(Cameras_to_world[cam_num].
matrix[:3, :]).reshape(-1):
line += "{:.8e} ".format(x)
line = line[:-1]
line += "\n"
myfile.write(line)
if not args.autopilot:
client.send_control(steer=random.uniform(-1.0, 1.0),
throttle=0.5,
brake=0.0,
hand_brake=False,
reverse=False)
else:
# Together with the measurements, the server has sent the
# control that the in-game autopilot would do this frame. We
# can enable autopilot by sending back this control to the
# server. We can modify it if wanted, here for instance we
# will add some noise to the steer.
control = measurements.player_measurements.autopilot_control
#control.steer += random.uniform(-0.1, 0.1)
client.send_control(control)
def run_carla_client(args):
# Here we will run 3 episodes with 300 frames each.
number_of_episodes = 15
frames_per_episode = 10030
# [0 , 1 , 2 , 3 , 4 , 5 , 6 , 7, 8 , 9 , 10, 11, 12, 13, 14]
# vehicles_num = [60, 60, 70, 50, 60, 60, 80, 60, 60, 60, 50, 70, 60, 50, 50]
vehicles_num = [35, 35, 30, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35, 35]
# vehicles_num = [60, 60, 70, 50, 60, 60, 80, 60, 60, 60, 50, 70, 60, 50, 50]
# We assume the CARLA server is already waiting for a client to connect at
# host:port. To create a connection we can use the `make_carla_client`
# context manager, it creates a CARLA client object and starts the
# connection. It will throw an exception if something goes wrong. The
# context manager makes sure the connection is always cleaned up on exit.
with make_carla_client(args.host, args.port) as client:
print('CarlaClient connected')
# list_of_episodes = [11]
list_of_episodes = [9, 11, 14]
# list_of_episodes = [0, 1, 2, 3, 4, 5, 6, 7, 9, 11, 14]
# list_of_episodes = [6, 7, 9, 11, 14]
# list_of_episodes = [2, 3, 4, 5, 6, 7, 9, 11, 14]
# list_of_episodes = [2, 7]
# list_of_episodes = [5, 6, 7, 11, 14]
# list_of_episodes = [6, 11]
# list_of_episodes = [7]
#list(range(0, number_of_episodes))
#list_of_episodes.pop(13)
#list_of_episodes.pop(12)
#list_of_episodes.pop(10)
#list_of_episodes.pop(8)
for episode in list_of_episodes:
# Start a new episode.
if args.settings_filepath is None:
# Create a CarlaSettings object. This object is a wrapper around
# the CarlaSettings.ini file. Here we set the configuration we
# want for the new episode.
settings = CarlaSettings()
settings.set(
SynchronousMode=True,
SendNonPlayerAgentsInfo=False,
NumberOfVehicles= vehicles_num[episode],#random.choice([0, 20, 15, 20, 25, 21, 24, 18, 40, 35, 25, 30]), #25,
NumberOfPedestrians=25,
DisableTwoWheeledVehicles=False,
WeatherId= episode, #1, #random.choice([1, 3, 7, 8, 14]),
QualityLevel=args.quality_level)
settings.randomize_seeds()
# Now we want to add a couple of cameras to the player vehicle.
# We will collect the images produced by these cameras every
# frame.
#### Cameras aligned across the y-axis
#### Horizontally shifted in the following Range
# [-1.62, -1.08, -0.54, 0.0, 0.54, 1.08, 1.62]
# LEFT RGB CAMERA
# y_locs = [-1.62, -1.08, -0.54, 0.0, 0.54, 1.08, 1.62]
# x_locs = [1.3, 1.84, 2.38, 2.92, 3.46, 4.0, 4.54]
# y_locs_left = [-1.08, -0.54, 0.0, 0.54, 1.08]
y_locs_left = -1.08
x_locs_left = [1.84, 2.38, 2.92]
LeftSideCameras = {}
for i,x_position in enumerate(x_locs_left):
# COLOR
camera_rgb = Camera('LeftSideCameras{0}RGB'.format(i),
PostProcessing='SceneFinal')
camera_rgb.set_image_size(800, 600)
camera_rgb.set_position(x_position, y_locs_left, 1.50)
camera_rgb.set_rotation(0, -90.0, 0)
LeftSideCameras['LeftSideCameras{0}RGB'.format(i)] = camera_rgb
settings.add_sensor(camera_rgb)
# DEPTH
camera_depth = Camera('LeftSideCameras{0}Depth'.format(i),
PostProcessing='Depth')
camera_depth.set_image_size(800, 600)
camera_depth.set_position(x_position, y_locs_left, 1.50)
camera_depth.set_rotation(0, -90.0, 0)
LeftSideCameras['LeftSideCameras{0}Depth'.format(i)] = camera_depth
settings.add_sensor(camera_depth)
# SEGMENTATION
camera_seg = Camera('LeftSideCameras{0}Seg'.format(i),
PostProcessing='SemanticSegmentation')
camera_seg.set_image_size(800, 600)
camera_seg.set_position(x_position, y_locs_left, 1.50)
camera_seg.set_rotation(0, -90.0, 0)
LeftSideCameras['LeftSideCameras{0}Seg'.format(i)] = camera_seg
settings.add_sensor(camera_seg)
y_locs_right = 1.08
x_locs_right = [1.84, 2.38, 2.92]
RightSideCameras = {}
for i,x_position in enumerate(x_locs_right):
# COLOR
camera_rgb = Camera('RightSideCameras{0}RGB'.format(i),
PostProcessing='SceneFinal')
camera_rgb.set_image_size(800, 600)
camera_rgb.set_position(x_position, y_locs_right, 1.50)
camera_rgb.set_rotation(0, 90.0, 0)
RightSideCameras['RightSideCameras{0}RGB'.format(i)] = camera_rgb
settings.add_sensor(camera_rgb)
# DEPTH
camera_depth = Camera('RightSideCameras{0}Depth'.format(i),
PostProcessing='Depth')
camera_depth.set_image_size(800, 600)
camera_depth.set_position(x_position, y_locs_right, 1.50)
camera_depth.set_rotation(0, 90.0, 0)
RightSideCameras['RightSideCameras{0}Depth'.format(i)] = camera_depth
settings.add_sensor(camera_depth)
# SEGMENTATION
camera_seg = Camera('RightSideCameras{0}Seg'.format(i),
PostProcessing='SemanticSegmentation')
camera_seg.set_image_size(800, 600)
camera_seg.set_position(x_position, y_locs_right, 1.50)
camera_seg.set_rotation(0, 90.0, 0)
RightSideCameras['RightSideCameras{0}Seg'.format(i)] = camera_seg
settings.add_sensor(camera_seg)
else:
with open(args.settings_filepath, 'r') as fp:
settings = fp.read()
scene = client.load_settings(settings)
# Choose one player start at random.
number_of_player_starts = len(scene.player_start_spots)
player_start = random.randint(0, max(0, number_of_player_starts - 1))
# Notify the server that we want to start the episode at the
# player_start index. This function blocks until the server is ready
# to start the episode.
print('Starting new episode...')
client.start_episode(player_start)
LeftSideCameras_to_car = []
for i in range(len(x_locs_right)):
LeftSideCameras_to_car.append(LeftSideCameras['LeftSideCameras{0}RGB'.format(i)].get_transform())
RightSideCameras_to_car = []
for i in range(len(x_locs_right)):
RightSideCameras_to_car.append(RightSideCameras['RightSideCameras{0}RGB'.format(i)].get_transform())
# camera_90_p_l_to_car_transform = camera_90_p_l.get_transform()
# camera_90_p_r_to_car_transform = camera_90_p_r.get_transform()
# Create a folder for saving episode data
if not os.path.isdir("/data/teddy/Datasets/carla_left_and_right/Town2/episode_{:0>5d}".format(episode)):
os.makedirs("/data/teddy/Datasets/carla_left_and_right/Town2/episode_{:0>5d}".format(episode))
# Iterate every frame in the episode.
for frame in range(0, frames_per_episode):
# Read the data produced by the server this frame.
measurements, sensor_data = client.read_data()
# player_measurements = measurements.player_measurements
world_transform = Transform(measurements.player_measurements.transform)
# Compute the final transformation matrix.
LeftSideCameras_to_world = [world_transform*cam_to_car \
for cam_to_car in LeftSideCameras_to_car]
RightSideCameras_to_world = [world_transform*cam_to_car \
for cam_to_car in RightSideCameras_to_car]
# for i in range(len()):
# LeftSideCameras_to_world.append()
# RightSideCameras_to_world.append(world_transform * RightSideCameras_to_car[i])
# Save the images to disk if requested.
if frame >= 30 and (frame % 2 == 0):
if args.save_images_to_disk:
for name, measurement in sensor_data.items():
filename = args.out_filename_format.format(episode, name, (frame-30)/2)
measurement.save_to_disk(filename)
# Save Transform matrix of each camera to separated files
for cam_num in range(len(x_locs_left)):
line = ""
filename = "{}episode_{:0>5d}/LeftSideCameras{}".format(args.root_path, episode, cam_num) + ".txt"
with open(filename, 'a+') as myfile:
for x in np.asarray(LeftSideCameras_to_world[cam_num].matrix[:3, :]).reshape(-1):
line += "{:.8e} ".format(x)
line = line[:-1]
line += "\n"
myfile.write(line)
line = ""
# Forward Cameras
# forward_cam_ids = list(range(len(x_locs_right)))
# forward_cam_ids.pop(mid_cam)
for i, cam_num in enumerate(x_locs_right):
line = ""
filename = "{}episode_{:0>5d}/RightSideCameras{}".format(args.root_path, episode, cam_num) + ".txt"
with open(filename, 'a+') as myfile:
for x in np.asarray(RightSideCameras_to_world[i].matrix[:3, :]).reshape(-1):
line += "{:.8e} ".format(x)
line = line[:-1]
line += "\n"
myfile.write(line)
line = ""
if not args.autopilot:
client.send_control(
steer=random.uniform(-1.0, 1.0),
throttle=0.5,
brake=0.0,
hand_brake=False,
reverse=False)
else:
# Together with the measurements, the server has sent the
# control that the in-game autopilot would do this frame. We
# can enable autopilot by sending back this control to the
# server. We can modify it if wanted, here for instance we
# will add some noise to the steer.
control = measurements.player_measurements.autopilot_control
#control.steer += random.uniform(-0.1, 0.1)
client.send_control(control)
def read_carla_data(self):
read_start_time = time.time()
measurements, sensor_data = self.client.read_data()
measure_time = time.time()
self._logger.info(
'Got readings for game time {} and platform time {}'.format(
measurements.game_timestamp, measurements.platform_timestamp))
# Send measurements
player_measurements = measurements.player_measurements
vehicle_pos = ((player_measurements.transform.location.x,
player_measurements.transform.location.y,
player_measurements.transform.location.z),
(player_measurements.transform.orientation.x,
player_measurements.transform.orientation.y,
player_measurements.transform.orientation.z))
world_transform = Transform(player_measurements.transform)
timestamp = Timestamp(
coordinates=[measurements.game_timestamp, self.message_num])
self.message_num += 1
ray.register_custom_serializer(Message, use_pickle=True)
ray.register_custom_serializer(WatermarkMessage, use_pickle=True)
watermark = WatermarkMessage(timestamp)
self.get_output_stream('world_transform').send(
Message(world_transform, timestamp))
self.get_output_stream('world_transform').send(watermark)
self.get_output_stream('vehicle_pos').send(
Message(vehicle_pos, timestamp))
self.get_output_stream('vehicle_pos').send(watermark)
acceleration = (player_measurements.acceleration.x,
player_measurements.acceleration.y,
player_measurements.acceleration.z)
self.get_output_stream('acceleration').send(
Message(acceleration, timestamp))
self.get_output_stream('acceleration').send(watermark)
self.get_output_stream('forward_speed').send(
Message(player_measurements.forward_speed, timestamp))
self.get_output_stream('forward_speed').send(watermark)
self.get_output_stream('vehicle_collisions').send(
Message(player_measurements.collision_vehicles, timestamp))
self.get_output_stream('vehicle_collisions').send(watermark)
self.get_output_stream('pedestrian_collisions').send(
Message(player_measurements.collision_pedestrians, timestamp))
self.get_output_stream('pedestrian_collisions').send(watermark)
self.get_output_stream('other_collisions').send(
Message(player_measurements.collision_other, timestamp))
self.get_output_stream('other_collisions').send(watermark)
self.get_output_stream('other_lane').send(
Message(player_measurements.intersection_otherlane, timestamp))
self.get_output_stream('other_lane').send(watermark)
self.get_output_stream('offroad').send(
Message(player_measurements.intersection_offroad, timestamp))
self.get_output_stream('offroad').send(watermark)
vehicles = []
pedestrians = []
traffic_lights = []
speed_limit_signs = []
for agent in measurements.non_player_agents:
if agent.HasField('vehicle'):
pos = messages.Transform(agent.vehicle.transform)
bb = messages.BoundingBox(agent.vehicle.bounding_box)
forward_speed = agent.vehicle.forward_speed
vehicle = messages.Vehicle(pos, bb, forward_speed)
vehicles.append(vehicle)
elif agent.HasField('pedestrian'):
if not self.agent_id_map.get(agent.id):
self.pedestrian_count += 1
self.agent_id_map[agent.id] = self.pedestrian_count
pedestrian_index = self.agent_id_map[agent.id]
pos = messages.Transform(agent.pedestrian.transform)
bb = messages.BoundingBox(agent.pedestrian.bounding_box)
forward_speed = agent.pedestrian.forward_speed
pedestrian = messages.Pedestrian(pedestrian_index, pos, bb,
forward_speed)
pedestrians.append(pedestrian)
elif agent.HasField('traffic_light'):
transform = messages.Transform(agent.traffic_light.transform)
traffic_light = messages.TrafficLight(
transform, agent.traffic_light.state)
traffic_lights.append(traffic_light)
elif agent.HasField('speed_limit_sign'):
transform = messages.Transform(
agent.speed_limit_sign.transform)
speed_sign = messages.SpeedLimitSign(
transform, agent.speed_limit_sign.speed_limit)
speed_limit_signs.append(speed_sign)
vehicles_msg = Message(vehicles, timestamp)
self.get_output_stream('vehicles').send(vehicles_msg)
self.get_output_stream('vehicles').send(watermark)
pedestrians_msg = Message(pedestrians, timestamp)
self.get_output_stream('pedestrians').send(pedestrians_msg)
self.get_output_stream('pedestrians').send(watermark)
traffic_lights_msg = Message(traffic_lights, timestamp)
self.get_output_stream('traffic_lights').send(traffic_lights_msg)
self.get_output_stream('traffic_lights').send(watermark)
traffic_sings_msg = Message(speed_limit_signs, timestamp)
self.get_output_stream('traffic_signs').send(traffic_sings_msg)
self.get_output_stream('traffic_signs').send(watermark)
# Send sensor data
for name, measurement in sensor_data.items():
data_stream = self.get_output_stream(name)
if data_stream.get_label('camera_type') == 'SceneFinal':
# Transform the Carla RGB images to BGR.
data_stream.send(
Message(
pylot_utils.bgra_to_bgr(to_bgra_array(measurement)),
timestamp))
else:
data_stream.send(Message(measurement, timestamp))
data_stream.send(watermark)
self.client.send_control(**self.control)
end_time = time.time()
measurement_runtime = (measure_time - read_start_time) * 1000
total_runtime = (end_time - read_start_time) * 1000
self._logger.info('Carla measurement time {}; total time {}'.format(
measurement_runtime, total_runtime))
self._csv_logger.info('{},{},{},{}'.format(time_epoch_ms(), self.name,
measurement_runtime,
total_runtime))
def exec_waypoint_nav_demo(args):
""" Executes waypoint navigation demo.
"""
with make_carla_client(args.host, args.port) as client:
print('Carla client connected.')
settings, camera2 = make_carla_settings(args)
# Now we load these settings into the server. The server replies
# with a scene description containing the available start spots for
# the player. Here we can provide a CarlaSettings object or a
# CarlaSettings.ini file as string.
scene = client.load_settings(settings)
# Refer to the player start folder in the WorldOutliner to see the
# player start information
player_start = PLAYER_START_INDEX
# Notify the server that we want to start the episode at the
# player_start index. This function blocks until the server is ready
# to start the episode.
print('Starting new episode at %r...' % scene.map_name)
client.start_episode(player_start)
#############################################
# Load Configurations
#############################################
# Load configuration file (options.cfg) and then parses for the various
# options. Here we have two main options:
# live_plotting and live_plotting_period, which controls whether
# live plotting is enabled or how often the live plotter updates
# during the simulation run.
config = configparser.ConfigParser()
config.read(
os.path.join(os.path.dirname(os.path.realpath(__file__)),
'options.cfg'))
demo_opt = config['Demo Parameters']
# Get options
enable_live_plot = demo_opt.get('live_plotting', 'true').capitalize()
enable_live_plot = enable_live_plot == 'True'
live_plot_period = float(demo_opt.get('live_plotting_period', 0))
# Set options
live_plot_timer = Timer(live_plot_period)
#############################################
# Load Waypoints
#############################################
# Opens the waypoint file and stores it to "waypoints"
camera_to_car_transform = camera2.get_unreal_transform()
carla_utils_obj = segmentationobj.carla_utils(intrinsic)
measurement_data, sensor_data = client.read_data()
measurements = measurement_data
image_RGB = to_rgb_array(sensor_data['CameraRGB'])
image_depth = to_rgb_array(sensor_data['CameraDepth'])
world_transform = Transform(measurements.player_measurements.transform)
im_bgr = cv2.cvtColor(image_RGB, cv2.COLOR_RGB2BGR)
pos_vector = ([
measurements.player_measurements.transform.location.x,
measurements.player_measurements.transform.location.y,
measurements.player_measurements.transform.location.z
])
fdfd = str(world_transform)
sdsd = fdfd[1:-1].split('\n')
new = []
for i in sdsd:
dd = i[:-1].lstrip('[ ')
ff = re.sub("\s+", ",", dd.strip())
gg = ff.split(',')
new.append([float(i) for i in gg])
newworld = np.array(new)
fdfd = str(camera_to_car_transform)
sdsd = fdfd[1:-1].split('\n')
new = []
for i in sdsd:
dd = i[:-1].lstrip('[ ')
ff = re.sub("\s+", ",", dd.strip())
gg = ff.split(',')
new.append([float(i) for i in gg])
newcam = np.array(new)
extrinsic = np.matmul(newworld, newcam)
#print("dfjdfjkjfdkf",extrinsic)
get_2d_point, pointsmid, res_img = carla_utils_obj.run_seg(
im_bgr, extrinsic, pos_vector)
#print(get_2d_point)
#dis1=((get_2d_point[1]-pointsmid[0][1]),(get_2d_point[0]-pointsmid[0][0]))
#dis2=((get_2d_point[1]-pointsmid[1][1]),(get_2d_point[0]-pointsmid[1][0]))
#dis3=((get_2d_point[1]-pointsmid[2][1]),(get_2d_point[0]-pointsmid[2][0]))
#dis4=((get_2d_point[1]-pointsmid[3][1]),(get_2d_point[0]-pointsmid[3][0]))
#flagbox=darknet_proper_fps.performDetect(im_bgr,thresh,configPath, weightPath, metaPath, showImage, makeImageOnly, initOnly,pointsmid)
depth1 = image_depth[int(pointsmid[0][0]), int(pointsmid[0][1])]
depth2 = image_depth[int(pointsmid[1][0]), int(pointsmid[1][1])]
depth3 = image_depth[int(pointsmid[2][0]), int(pointsmid[2][1])]
depth4 = image_depth[int(pointsmid[3][0]), int(pointsmid[3][1])]
scale1 = depth1[0] + depth1[1] * 256 + depth1[2] * 256 * 256
scale1 = scale1 / ((256 * 256 * 256) - 1)
depth1 = scale1 * 1000
pos2d1 = np.array([(WINDOW_WIDTH - pointsmid[0][1]) * depth1,
(WINDOW_HEIGHT - pointsmid[0][0]) * depth1, depth1])
first1 = np.dot(np.linalg.inv(intrinsic), pos2d1)
first1 = np.append(first1, 1)
sec1 = np.matmul((extrinsic), first1)
scale2 = depth2[0] + depth2[1] * 256 + depth2[2] * 256 * 256
scale2 = scale2 / ((256 * 256 * 256) - 1)
depth2 = scale2 * 1000
pos2d2 = np.array([(WINDOW_WIDTH - pointsmid[1][1]) * depth2,
(WINDOW_HEIGHT - pointsmid[1][0]) * depth2, depth2])
first2 = np.dot(np.linalg.inv(intrinsic), pos2d2)
first2 = np.append(first2, 1)
sec2 = np.matmul((extrinsic), first2)
scale3 = depth3[0] + depth3[1] * 256 + depth3[2] * 256 * 256
scale3 = scale3 / ((256 * 256 * 256) - 1)
depth3 = scale3 * 1000
pos2d3 = np.array([(WINDOW_WIDTH - pointsmid[2][1]) * depth3,
(WINDOW_HEIGHT - pointsmid[2][0]) * depth3, depth3])
first3 = np.dot(np.linalg.inv(intrinsic), pos2d3)
first3 = np.append(first3, 1)
sec3 = np.matmul((extrinsic), first3)
scale4 = depth4[0] + depth4[1] * 256 + depth4[2] * 256 * 256
scale4 = scale4 / ((256 * 256 * 256) - 1)
depth4 = scale4 * 1000
pos2d4 = np.array([(WINDOW_WIDTH - pointsmid[3][1]) * depth4,
(WINDOW_HEIGHT - pointsmid[3][0]) * depth4, depth4])
first4 = np.dot(np.linalg.inv(intrinsic), pos2d4)
first4 = np.append(first4, 1)
sec4 = np.matmul((extrinsic), first4)
depth = image_depth[int(get_2d_point[0]), int(get_2d_point[1])]
scale = depth[0] + depth[1] * 256 + depth[2] * 256 * 256
scale = scale / ((256 * 256 * 256) - 1)
depth = scale * 1000
pos2d = np.array([(WINDOW_WIDTH - get_2d_point[1]) * depth,
(WINDOW_HEIGHT - get_2d_point[0]) * depth, depth])
first = np.dot(np.linalg.inv(intrinsic), pos2d)
first = np.append(first, 1)
sec = np.matmul((extrinsic), first)
print("present", pos_vector)
print('goal-', sec)
pos_vector[2] = 4.0
ini = pos_vector
goal = list(sec)
goal[2] = 1.08
aa = ini[0]
dec = abs(ini[1] - goal[1]) / abs(aa - (goal[0]))
f1 = open(WAYPOINTS_FILENAME, 'a+')
for i in range(int(goal[0]), int(aa)):
# print(int(goal[0])-i)
if abs((int(aa) - i)) < 10:
ini = [ini[0] - 1, ini[1] + dec, ini[2] - 0.03]
else:
ini = [ini[0] - 1, ini[1] + dec, ini[2]]
#if i<int(aa)-1:
f1.write(
str(ini[0]) + ',' + str(ini[1]) + ',' + str(ini[2]) + '\n')
#else:
#f1.write(str(ini[0])+','+str(ini[1])+','+str(ini[2]))
waypoints_file = WAYPOINTS_FILENAME
waypoints_np = None
with open(waypoints_file) as waypoints_file_handle:
waypoints = list(
csv.reader(waypoints_file_handle,
delimiter=',',
quoting=csv.QUOTE_NONNUMERIC))
waypoints_np = np.array(waypoints)
#print("dfjdjfhjdhfjdfhjdfdjdfdufh",waypoints_np)
print((waypoints_np))
# Because the waypoints are discrete and our controller performs better
# with a continuous path, here we will send a subset of the waypoints
# within some lookahead distance from the closest point to the vehicle.
# Interpolating between each waypoint will provide a finer resolution
# path and make it more "continuous". A simple linear interpolation
# is used as a preliminary method to address this issue, though it is
# better addressed with better interpolation methods (spline
# interpolation, for example).
# More appropriate interpolation methods will not be used here for the
# sake of demonstration on what effects discrete paths can have on
# the controller. It is made much more obvious with linear
# interpolation, because in a way part of the path will be continuous
# while the discontinuous parts (which happens at the waypoints) will
# show just what sort of effects these points have on the controller.
# Can you spot these during the simulation? If so, how can you further
# reduce these effects?
# Linear interpolation computations
# Compute a list of distances between waypoints
wp_distance = [] # distance array
for i in range(1, waypoints_np.shape[0]):
wp_distance.append(
np.sqrt((waypoints_np[i, 0] - waypoints_np[i - 1, 0])**2 +
(waypoints_np[i, 1] - waypoints_np[i - 1, 1])**2))
wp_distance.append(0) # last distance is 0 because it is the distance
# from the last waypoint to the last waypoint
# Linearly interpolate between waypoints and store in a list
wp_interp = [] # interpolated values
# (rows = waypoints, columns = [x, y, v])
wp_interp_hash = [] # hash table which indexes waypoints_np
# to the index of the waypoint in wp_interp
interp_counter = 0 # counter for current interpolated point index
for i in range(waypoints_np.shape[0] - 1):
# Add original waypoint to interpolated waypoints list (and append
# it to the hash table)
wp_interp.append(list(waypoints_np[i]))
wp_interp_hash.append(interp_counter)
interp_counter += 1
# Interpolate to the next waypoint. First compute the number of
# points to interpolate based on the desired resolution and
# incrementally add interpolated points until the next waypoint
# is about to be reached.
num_pts_to_interp = int(np.floor(wp_distance[i] /\
float(INTERP_DISTANCE_RES)) - 1)
wp_vector = waypoints_np[i + 1] - waypoints_np[i]
wp_uvector = wp_vector / np.linalg.norm(wp_vector)
for j in range(num_pts_to_interp):
next_wp_vector = INTERP_DISTANCE_RES * float(j +
1) * wp_uvector
wp_interp.append(list(waypoints_np[i] + next_wp_vector))
interp_counter += 1
# add last waypoint at the end
wp_interp.append(list(waypoints_np[-1]))
wp_interp_hash.append(interp_counter)
interp_counter += 1
#############################################
# Controller 2D Class Declaration
#############################################
# This is where we take the controller2d.py class
# and apply it to the simulator
controller = controller2d.Controller2D(waypoints)
#############################################
# Determine simulation average timestep (and total frames)
#############################################
# Ensure at least one frame is used to compute average timestep
num_iterations = ITER_FOR_SIM_TIMESTEP
if (ITER_FOR_SIM_TIMESTEP < 1):
num_iterations = 1
# Gather current data from the CARLA server. This is used to get the
# simulator starting game time. Note that we also need to
# send a command back to the CARLA server because synchronous mode
# is enabled.
sim_start_stamp = measurement_data.game_timestamp / 1000.0
# Send a control command to proceed to next iteration.
#print("dddddddddddddddddddddddddddddddddddddddddddddd",sim_start_stamp)
# This mainly applies for simulations that are in synchronous mode.
send_control_command(client, throttle=0.0, steer=0, brake=1.0)
# Computes the average timestep based on several initial iterations
sim_duration = 0
for i in range(num_iterations):
# Gather current data
measurement_data, sensor_data = client.read_data()
# Send a control command to proceed to next iteration
send_control_command(client, throttle=0.0, steer=0, brake=1.0)
# Last stamp
if i == num_iterations - 1:
sim_duration = measurement_data.game_timestamp / 1000.0 -\
sim_start_stamp
# Outputs average simulation timestep and computes how many frames
# will elapse before the simulation should end based on various
# parameters that we set in the beginning.
SIMULATION_TIME_STEP = sim_duration / float(num_iterations)
print("SERVER SIMULATION STEP APPROXIMATION: " + \
str(SIMULATION_TIME_STEP))
TOTAL_EPISODE_FRAMES = int((TOTAL_RUN_TIME + WAIT_TIME_BEFORE_START) /\
SIMULATION_TIME_STEP) + TOTAL_FRAME_BUFFER
#############################################
# Frame-by-Frame Iteration and Initialization
#############################################
# Store pose history starting from the start position
measurement_data, sensor_data = client.read_data()
start_x, start_y, start_yaw = get_current_pose(measurement_data)
send_control_command(client, throttle=0.0, steer=0, brake=1.0)
x_history = [start_x]
y_history = [start_y]
yaw_history = [start_yaw]
time_history = [0]
speed_history = [0]
#############################################
# Vehicle Trajectory Live Plotting Setup
#############################################
# Uses the live plotter to generate live feedback during the simulation
# The two feedback includes the trajectory feedback and
# the controller feedback (which includes the speed tracking).
lp_traj = lv.LivePlotter(tk_title="Trajectory Trace")
lp_1d = lv.LivePlotter(tk_title="Controls Feedback")
###
# Add 2D position / trajectory plot
###
trajectory_fig = lp_traj.plot_new_dynamic_2d_figure(
title='Vehicle Trajectory',
figsize=(FIGSIZE_X_INCHES, FIGSIZE_Y_INCHES),
edgecolor="black",
rect=[PLOT_LEFT, PLOT_BOT, PLOT_WIDTH, PLOT_HEIGHT])
trajectory_fig.set_invert_x_axis() # Because UE4 uses left-handed
# coordinate system the X
# axis in the graph is flipped
trajectory_fig.set_axis_equal() # X-Y spacing should be equal in size
# Add waypoint markers
trajectory_fig.add_graph("waypoints",
window_size=waypoints_np.shape[0],
x0=waypoints_np[:, 0],
y0=waypoints_np[:, 1],
linestyle="-",
marker="",
color='g')
# Add trajectory markers
trajectory_fig.add_graph("trajectory",
window_size=TOTAL_EPISODE_FRAMES,
x0=[start_x] * TOTAL_EPISODE_FRAMES,
y0=[start_y] * TOTAL_EPISODE_FRAMES,
color=[1, 0.5, 0])
# Add lookahead path
trajectory_fig.add_graph("lookahead_path",
window_size=INTERP_MAX_POINTS_PLOT,
x0=[start_x] * INTERP_MAX_POINTS_PLOT,
y0=[start_y] * INTERP_MAX_POINTS_PLOT,
color=[0, 0.7, 0.7],
linewidth=4)
# Add starting position marker
trajectory_fig.add_graph("start_pos",
window_size=1,
x0=[start_x],
y0=[start_y],
marker=11,
color=[1, 0.5, 0],
markertext="Start",
marker_text_offset=1)
# Add end position marker
trajectory_fig.add_graph("end_pos",
window_size=1,
x0=[waypoints_np[-1, 0]],
y0=[waypoints_np[-1, 1]],
marker="D",
color='r',
markertext="End",
marker_text_offset=1)
# Add car marker
trajectory_fig.add_graph("car",
window_size=1,
marker="s",
color='b',
markertext="Car",
marker_text_offset=1)
###
# Add 1D speed profile updater
###
forward_speed_fig =\
lp_1d.plot_new_dynamic_figure(title="Forward Speed (m/s)")
forward_speed_fig.add_graph("forward_speed",
label="forward_speed",
window_size=TOTAL_EPISODE_FRAMES)
forward_speed_fig.add_graph("reference_signal",
label="reference_Signal",
window_size=TOTAL_EPISODE_FRAMES)
# Add throttle signals graph
throttle_fig = lp_1d.plot_new_dynamic_figure(title="Throttle")
throttle_fig.add_graph("throttle",
label="throttle",
window_size=TOTAL_EPISODE_FRAMES)
# Add brake signals graph
brake_fig = lp_1d.plot_new_dynamic_figure(title="Brake")
brake_fig.add_graph("brake",
label="brake",
window_size=TOTAL_EPISODE_FRAMES)
# Add steering signals graph
steer_fig = lp_1d.plot_new_dynamic_figure(title="Steer")
steer_fig.add_graph("steer",
label="steer",
window_size=TOTAL_EPISODE_FRAMES)
# live plotter is disabled, hide windows
if not enable_live_plot:
lp_traj._root.withdraw()
lp_1d._root.withdraw()
# Iterate the frames until the end of the waypoints is reached or
# the TOTAL_EPISODE_FRAMES is reached. The controller simulation then
# ouptuts the results to the controller output directory.
reached_the_end = False
skip_first_frame = True
closest_index = 0 # Index of waypoint that is currently closest to
# the car (assumed to be the first index)
closest_distance = 0 # Closest distance of closest waypoint to car
counter = 0
#print("dssssssssssssssssssssssssssssssssssssssssssssssssssssssss",TOTAL_EPISODE_FRAMES)
for frame in range(TOTAL_EPISODE_FRAMES):
# Gather current data from the CARLA server
measurement_data, sensor_data = client.read_data()
#print("lllllllllllllllllllllllllll",len(waypoints_np),waypoints_np[-1])
#update_pts=list(waypoints_np[-1])
# Update pose, timestamp
current_x, current_y, current_yaw = \
get_current_pose(measurement_data)
current_speed = measurement_data.player_measurements.forward_speed
current_timestamp = float(measurement_data.game_timestamp) / 1000.0
#print("xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",current_timestamp)
# Wait for some initial time before starting the demo
if current_timestamp <= WAIT_TIME_BEFORE_START:
send_control_command(client, throttle=0.0, steer=0, brake=1.0)
counter += 1
#flagbox=darknet_proper_fps.performDetect(res_img,thresh,configPath, weightPath, metaPath, showImage, makeImageOnly, initOnly,pointsmid)
continue
else:
current_timestamp = current_timestamp - WAIT_TIME_BEFORE_START
#print("sdmskdkfjdkfjdjksf",counter)
#for i in range(1,5):
update_pts = [list(sec1), list(sec2), list(sec3), list(sec4)]
pts_2d_ls = []
for i in range(len(update_pts)):
world_coord = np.asarray(update_pts[i]).reshape(4, -1)
# print("world_coord.shape",world_coord.shape)
# world_coord = np.array([[250.0 ,129.0 ,38.0 ,1.0]]).reshape(4,-1)
world_transform = Transform(
measurement_data.player_measurements.transform)
fdfd = str(world_transform)
sdsd = fdfd[1:-1].split('\n')
new = []
for i in sdsd:
dd = i[:-1].lstrip('[ ')
ff = re.sub("\s+", ",", dd.strip())
gg = ff.split(',')
new.append([float(i) for i in gg])
newworld = np.array(new)
extrinsic = np.matmul(newworld, newcam)
cam_coord = np.linalg.inv(extrinsic) @ world_coord
img_coord = intrinsic @ cam_coord[:3]
img_coord = np.array([
img_coord[0] / img_coord[2], img_coord[1] / img_coord[2],
img_coord[2]
])
if (img_coord[2] > 0):
x_2d = WINDOW_WIDTH - img_coord[0]
y_2d = WINDOW_HEIGHT - img_coord[1]
pts_2d_ls.append([x_2d, y_2d])
#x_diff=(pts_2d_ls[0]-get_2d_point[1])
#y_diff=(pts_2d_ls[1]-get_2d_point[0])
#print("sdsdjsdsjdksjdskdjk",x_diff,y_diff,pts_2d_ls,get_2d_point)
#get_2d_point=[pts_2d_ls[1],pts_2d_ls[0]]
image_RGB = to_rgb_array(sensor_data['CameraRGB'])
im_bgr = cv2.cvtColor(image_RGB, cv2.COLOR_RGB2BGR)
image_depth = to_rgb_array(sensor_data['CameraDepth'])
counter += 1
if counter == 0:
flagbox = False
else:
#pointsmid[0][1]=pointsmid[0][1]+x_diff
#pointsmid[0][0]=pointsmid[0][0]+y_diff
#pointsmid[1][1]=pointsmid[1][1]+x_diff
#pointsmid[1][0]=pointsmid[1][0]+y_diff
#pointsmid[2][1]=pointsmid[2][1]+x_diff
#pointsmid[2][0]=pointsmid[2][0]+y_diff
#pointsmid[3][1]=pointsmid[3][1]+x_diff
#pointsmid[3][0]=pointsmid[3][0]+y_diff
try:
pointsmid[0][1] = pts_2d_ls[0][0]
pointsmid[0][0] = pts_2d_ls[0][1]
pointsmid[1][1] = pts_2d_ls[1][0]
pointsmid[1][0] = pts_2d_ls[1][1]
pointsmid[2][1] = pts_2d_ls[2][0]
pointsmid[2][0] = pts_2d_ls[2][1]
pointsmid[3][1] = pts_2d_ls[3][0]
pointsmid[3][0] = pts_2d_ls[3][1]
disbox = False
flagbox = darknet_proper_fps.performDetect(
im_bgr, thresh, configPath, weightPath, metaPath,
showImage, makeImageOnly, initOnly, pointsmid, disbox)
except Exception as e:
disbox = True
flagbox = darknet_proper_fps.performDetect(
im_bgr, thresh, configPath, weightPath, metaPath,
showImage, makeImageOnly, initOnly, pointsmid, disbox)
if flagbox != False:
midofpts = [(pointsmid[1][1] + pointsmid[0][1]) / 2,
(pointsmid[1][0] + pointsmid[0][0]) / 2]
depthflag = image_depth[int(flagbox[1]),
int(flagbox[0])]
depthpts = image_depth[int(midofpts[1]),
int(midofpts[0])]
print(depthflag, depthpts)
#cv2.circle(im_bgr,(int(flagbox[0]),int(flagbox[1])), 5, (255,0,255), -1)
#cv2.circle(im_bgr,(int(midofpts[0]),int(midofpts[1])), 5, (0,0,255), -1)
cv2.imwrite('./seg_out/{}_zz.jpg'.format(frame),
im_bgr)
scalenew = depthflag[
0] + depthflag[1] * 256 + depthflag[2] * 256 * 256
scalenew = scalenew / ((256 * 256 * 256) - 1)
depthflag = scalenew * 1000
scalenew = depthpts[
0] + depthpts[1] * 256 + depthpts[2] * 256 * 256
scalenew = scalenew / ((256 * 256 * 256) - 1)
depthpts = scalenew * 1000
diff = abs(depthflag - depthpts)
print("entereeeeeeeeeeeeeeeeeeeeeeeeeeeeherree", diff)
if diff < 10:
flagbox = True
else:
flagbox = False
print(e)
print("fffffffffffffffffff", flagbox)
x_history.append(current_x)
y_history.append(current_y)
yaw_history.append(current_yaw)
speed_history.append(current_speed)
time_history.append(current_timestamp)
if flagbox == False:
# Store history
###
# Controller update (this uses the controller2d.py implementation)
###
# To reduce the amount of waypoints sent to the controller,
# provide a subset of waypoints that are within some
# lookahead distance from the closest point to the car. Provide
# a set of waypoints behind the car as well.
# Find closest waypoint index to car. First increment the index
# from the previous index until the new distance calculations
# are increasing. Apply the same rule decrementing the index.
# The final index should be the closest point (it is assumed that
# the car will always break out of instability points where there
# are two indices with the same minimum distance, as in the
# center of a circle)
closest_distance = np.linalg.norm(
np.array([
waypoints_np[closest_index, 0] - current_x,
waypoints_np[closest_index, 1] - current_y
]))
new_distance = closest_distance
new_index = closest_index
while new_distance <= closest_distance:
closest_distance = new_distance
closest_index = new_index
new_index += 1
if new_index >= waypoints_np.shape[0]: # End of path
break
new_distance = np.linalg.norm(
np.array([
waypoints_np[new_index, 0] - current_x,
waypoints_np[new_index, 1] - current_y
]))
new_distance = closest_distance
new_index = closest_index
while new_distance <= closest_distance:
closest_distance = new_distance
closest_index = new_index
new_index -= 1
if new_index < 0: # Beginning of path
break
new_distance = np.linalg.norm(
np.array([
waypoints_np[new_index, 0] - current_x,
waypoints_np[new_index, 1] - current_y
]))
# Once the closest index is found, return the path that has 1
# waypoint behind and X waypoints ahead, where X is the index
# that has a lookahead distance specified by
# INTERP_LOOKAHEAD_DISTANCE
waypoint_subset_first_index = closest_index - 1
if waypoint_subset_first_index < 0:
waypoint_subset_first_index = 0
waypoint_subset_last_index = closest_index
total_distance_ahead = 0
while total_distance_ahead < INTERP_LOOKAHEAD_DISTANCE:
total_distance_ahead += wp_distance[
waypoint_subset_last_index]
waypoint_subset_last_index += 1
if waypoint_subset_last_index >= waypoints_np.shape[0]:
waypoint_subset_last_index = waypoints_np.shape[0] - 1
break
# Use the first and last waypoint subset indices into the hash
# table to obtain the first and last indicies for the interpolated
# list. Update the interpolated waypoints to the controller
# for the next controller update.
new_waypoints = \
wp_interp[wp_interp_hash[waypoint_subset_first_index]:\
wp_interp_hash[waypoint_subset_last_index] + 1]
controller.update_waypoints(new_waypoints)
# Update the other controller values and controls
controller.update_values(current_x, current_y, current_yaw,
current_speed, current_timestamp,
frame)
controller.update_controls()
cmd_throttle, cmd_steer, cmd_brake = controller.get_commands()
# Skip the first frame (so the controller has proper outputs)
if skip_first_frame and frame == 0:
pass
else:
# Update live plotter with new feedback
trajectory_fig.roll("trajectory", current_x, current_y)
trajectory_fig.roll("car", current_x, current_y)
# When plotting lookahead path, only plot a number of points
# (INTERP_MAX_POINTS_PLOT amount of points). This is meant
# to decrease load when live plotting
new_waypoints_np = np.array(new_waypoints)
path_indices = np.floor(
np.linspace(0, new_waypoints_np.shape[0] - 1,
INTERP_MAX_POINTS_PLOT))
trajectory_fig.update(
"lookahead_path",
new_waypoints_np[path_indices.astype(int), 0],
new_waypoints_np[path_indices.astype(int), 1],
new_colour=[0, 0.7, 0.7])
forward_speed_fig.roll("forward_speed", current_timestamp,
current_speed)
forward_speed_fig.roll("reference_signal",
current_timestamp,
controller._desired_speed)
throttle_fig.roll("throttle", current_timestamp,
cmd_throttle)
brake_fig.roll("brake", current_timestamp, cmd_brake)
steer_fig.roll("steer", current_timestamp, cmd_steer)
# Refresh the live plot based on the refresh rate
# set by the options
if enable_live_plot and \
live_plot_timer.has_exceeded_lap_period():
lp_traj.refresh()
lp_1d.refresh()
live_plot_timer.lap()
# Output controller command to CARLA server
send_control_command(client,
throttle=cmd_throttle,
steer=cmd_steer,
brake=cmd_brake)
# Find if reached the end of waypoint. If the car is within
# DIST_THRESHOLD_TO_LAST_WAYPOINT to the last waypoint,
# the simulation will end.
dist_to_last_waypoint = np.linalg.norm(
np.array([
waypoints[-1][0] -
current_x, ###########changed waypoints[-1]
waypoints[-1][1] - current_y
]))
if dist_to_last_waypoint < DIST_THRESHOLD_TO_LAST_WAYPOINT:
reached_the_end = True
if reached_the_end:
break
else:
send_control_command(client,
throttle=0.0,
steer=0.0,
brake=1.0)
break
# End of demo - Stop vehicle and Store outputs to the controller output
# directory.
if reached_the_end:
print("Reached the end of path. Writing to controller_output...")
send_control_command(client, throttle=0.0, steer=0.0, brake=1.0)
else:
print("stop!!!!.")
# Stop the car
#send_control_command(client, throttle=0.0, steer=0.0, brake=1.0)
# Store the various outputs
store_trajectory_plot(trajectory_fig.fig, 'trajectory.png')
store_trajectory_plot(forward_speed_fig.fig, 'forward_speed.png')
store_trajectory_plot(throttle_fig.fig, 'throttle_output.png')
store_trajectory_plot(brake_fig.fig, 'brake_output.png')
store_trajectory_plot(steer_fig.fig, 'steer_output.png')
write_trajectory_file(x_history, y_history, speed_history,
time_history)
def run_carla_client(host, port, far):
# Here we will run a single episode with 300 frames.
number_of_frames = 3000
frame_step = 100 # Save one image every 100 frames
output_folder = '_out'
image_size = [800, 600]
camera_local_pos = [0.3, 0.0, 1.3] # [X, Y, Z]
camera_local_rotation = [0, 0, 0] # [pitch(Y), yaw(Z), roll(X)]
fov = 70
# Connect with the server
with make_carla_client(host, port) as client:
print('CarlaClient connected')
# Here we load the settings.
settings = CarlaSettings()
settings.set(SynchronousMode=True,
SendNonPlayerAgentsInfo=False,
NumberOfVehicles=20,
NumberOfPedestrians=40,
WeatherId=random.choice([1, 3, 7, 8, 14]))
settings.randomize_seeds()
camera1 = Camera('CameraDepth', PostProcessing='Depth', FOV=fov)
camera1.set_image_size(*image_size)
camera1.set_position(*camera_local_pos)
camera1.set_rotation(*camera_local_rotation)
settings.add_sensor(camera1)
camera2 = Camera('CameraRGB', PostProcessing='SceneFinal', FOV=fov)
camera2.set_image_size(*image_size)
camera2.set_position(*camera_local_pos)
camera2.set_rotation(*camera_local_rotation)
settings.add_sensor(camera2)
client.load_settings(settings)
# Start at location index id '0'
client.start_episode(0)
# Compute the camera transform matrix
camera_to_car_transform = camera2.get_unreal_transform()
# Iterate every frame in the episode except for the first one.
for frame in range(1, number_of_frames):
# Read the data produced by the server this frame.
measurements, sensor_data = client.read_data()
# Save one image every 'frame_step' frames
if not frame % frame_step:
# Start transformations time mesure.
timer = StopWatch()
# RGB image [[[r,g,b],..[r,g,b]],..[[r,g,b],..[r,g,b]]]
image_RGB = to_rgb_array(sensor_data['CameraRGB'])
# 2d to (camera) local 3d
# We use the image_RGB to colorize each 3D point, this is optional.
# "max_depth" is used to keep only the points that are near to the
# camera, meaning 1.0 the farest points (sky)
point_cloud = depth_to_local_point_cloud(
sensor_data['CameraDepth'], image_RGB, max_depth=far)
# (Camera) local 3d to world 3d.
# Get the transform from the player protobuf transformation.
world_transform = Transform(
measurements.player_measurements.transform)
# Compute the final transformation matrix.
car_to_world_transform = world_transform * camera_to_car_transform
# Car to World transformation given the 3D points and the
# transformation matrix.
point_cloud.apply_transform(car_to_world_transform)
# End transformations time mesure.
timer.stop()
# Save PLY to disk
# This generates the PLY string with the 3D points and the RGB colors
# for each row of the file.
point_cloud.save_to_disk(
os.path.join(output_folder, '{:0>5}.ply'.format(frame)))
print_message(timer.milliseconds(), len(point_cloud), frame)
client.send_control(
measurements.player_measurements.autopilot_control)
def run_carla_client(args):
number_of_episodes = 15
frames_per_episode = 5030
# [0 , 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10, 11, 12, 13, 14]
vehicles_num = [25, 20, 30, 20, 15, 45, 45, 50, 25, 25, 25, 20, 25, 25, 25]
# We assume the CARLA server is already waiting for a client to connect at
# host:port. To create a connection we can use the `make_carla_client`
# context manager, it creates a CARLA client object and starts the
# connection. It will throw an exception if something goes wrong. The
# context manager makes sure the connection is always cleaned up on exit.
with make_carla_client(args.host, args.port) as client:
print('CarlaClient connected')
for episode in [1, 4, 11]:
# Start a new episode.
if args.settings_filepath is None:
# Create a CarlaSettings object. This object is a wrapper around
# the CarlaSettings.ini file. Here we set the configuration we
# want for the new episode.
settings = CarlaSettings()
settings.set(
SynchronousMode=True,
SendNonPlayerAgentsInfo=False,
NumberOfVehicles= vehicles_num[episode],#random.choice([0, 20, 15, 20, 25, 21, 24, 18, 40, 35, 25, 30]), #25,
NumberOfPedestrians=50,
DisableTwoWheeledVehicles=False,
WeatherId= episode, #1, #random.choice([1, 3, 7, 8, 14]),
QualityLevel=args.quality_level)
settings.randomize_seeds()
# Now we want to add a couple of cameras to the player vehicle.
# We will collect the images produced by these cameras every
# frame.
#### FRONT STEREO ####
# LEFT RGB CAMERA
camera_l = Camera('LeftCameraRGB', PostProcessing='SceneFinal')
camera_l.set_image_size(800, 600)
camera_l.set_position(1.50, -0.27, 1.50)
settings.add_sensor(camera_l)
# LEFT DEPTH
camera_ld = Camera('LeftCameraDepth', PostProcessing='Depth')
camera_ld.set_image_size(800, 600)
camera_ld.set_position(1.50, -0.27, 1.50)
settings.add_sensor(camera_ld)
# LEFT SEGMENTATION
camera_ls = Camera('LeftCameraSeg', PostProcessing='SemanticSegmentation')
camera_ls.set_image_size(800, 600)
camera_ls.set_position(1.50, -0.27, 1.50)
settings.add_sensor(camera_ls)
# RIGHT RGB CAMERA
camera_r = Camera('RightCameraRGB', PostProcessing='SceneFinal')
camera_r.set_image_size(800, 600)
camera_r.set_position(1.50, 0.27, 1.50)
settings.add_sensor(camera_r)
# RIGHT DEPTH
camera_rd = Camera('RightCameraDepth', PostProcessing='Depth')
camera_rd.set_image_size(800, 600)
camera_rd.set_position(1.50, 0.27, 1.50)
settings.add_sensor(camera_rd)
# RIGHT SEGMENTATION
camera_rs = Camera('RightCameraSeg', PostProcessing='SemanticSegmentation')
camera_rs.set_image_size(800, 600)
camera_rs.set_position(1.50, 0.27, 1.50)
settings.add_sensor(camera_rs)
#### -45 DEGREE STEREO CAMERA ####
# LEFT STEREO -45 DEGREE RGB
camera_45_n_l = Camera('45_N_LeftCameraRGB', PostProcessing='SceneFinal')
camera_45_n_l.set_image_size(800, 600)
camera_45_n_l.set_position(0.8, -1.0, 1.50) # [X, Y, Z]
camera_45_n_l.set_rotation(0, -45.0, 0) # [pitch(Y), yaw(Z), roll(X)]
settings.add_sensor(camera_45_n_l)
# RIGHT STEREO -45 DEGREE RGB
camera_45_n_r = Camera('45_N_RightCameraRGB', PostProcessing='SceneFinal')
camera_45_n_r.set_image_size(800, 600)
camera_45_n_r.set_position(1.2, -0.6, 1.50)
camera_45_n_r.set_rotation(0, -45.0, 0)
settings.add_sensor(camera_45_n_r)
# LEFT STEREO -45 DEGREE DEPTH
camera_45_n_ld = Camera('45_N_LeftCameraDepth', PostProcessing='Depth')
camera_45_n_ld.set_image_size(800, 600)
camera_45_n_ld.set_position(0.8, -1.0, 1.50)
camera_45_n_ld.set_rotation(0, -45.0, 0)
settings.add_sensor(camera_45_n_ld)
# RIGHT STEREO -45 DEGREE DEPTH
camera_45_n_rd = Camera('45_N_RightCameraDepth', PostProcessing='Depth')
camera_45_n_rd.set_image_size(800, 600)
camera_45_n_rd.set_position(1.2, -0.6, 1.50)
camera_45_n_rd.set_rotation(0, -45.0, 0)
settings.add_sensor(camera_45_n_rd)
# LEFT STEREO -45 DEGREE SEGMENTATION
camera_45_n_ls = Camera('45_N_LeftCameraSeg', PostProcessing='SemanticSegmentation')
camera_45_n_ls.set_image_size(800, 600)
camera_45_n_ls.set_position(0.8, -1.0, 1.50)
camera_45_n_ls.set_rotation(0, -45.0, 0)
settings.add_sensor(camera_45_n_ls)
# RIGHT STEREO -45 DEGREE SEGMENTATION
camera_45_n_rs = Camera('45_N_RightCameraSeg', PostProcessing='SemanticSegmentation')
camera_45_n_rs.set_image_size(800, 600)
camera_45_n_rs.set_position(1.2, -0.6, 1.50)
camera_45_n_rs.set_rotation(0, -45.0, 0)
settings.add_sensor(camera_45_n_rs)
#### +45 DEGREE STEREO CAMERA ####
# LEFT STEREO +45 DEGREE RGB
camera_45_p_l = Camera('45_P_LeftCameraRGB', PostProcessing='SceneFinal')
camera_45_p_l.set_image_size(800, 600)
camera_45_p_l.set_position(1.2, 0.6, 1.50) # [X, Y, Z]
camera_45_p_l.set_rotation(0, 45.0, 0) # [pitch(Y), yaw(Z), roll(X)]
settings.add_sensor(camera_45_p_l)
# RIGHT STEREO +45 DEGREE RGB
camera_45_p_r = Camera('45_P_RightCameraRGB', PostProcessing='SceneFinal')
camera_45_p_r.set_image_size(800, 600)
camera_45_p_r.set_position(0.8, 1.0, 1.50)
camera_45_p_r.set_rotation(0, 45.0, 0)
settings.add_sensor(camera_45_p_r)
# LEFT STEREO +45 DEGREE DEPTH
camera_45_p_ld = Camera('45_P_LeftCameraDepth', PostProcessing='Depth')
camera_45_p_ld.set_image_size(800, 600)
camera_45_p_ld.set_position(1.2, 0.6, 1.50)
camera_45_p_ld.set_rotation(0, 45.0, 0)
settings.add_sensor(camera_45_p_ld)
# RIGHT STEREO +45 DEGREE DEPTH
camera_45_p_rd = Camera('45_P_RightCameraDepth', PostProcessing='Depth')
camera_45_p_rd.set_image_size(800, 600)
camera_45_p_rd.set_position(0.8, 1.0, 1.50)
camera_45_p_rd.set_rotation(0, 45.0, 0)
settings.add_sensor(camera_45_p_rd)
# LEFT STEREO +45 DEGREE SEGMENTATION
camera_45_p_ls = Camera('45_P_LeftCameraSeg', PostProcessing='SemanticSegmentation')
camera_45_p_ls.set_image_size(800, 600)
camera_45_p_ls.set_position(1.2, 0.6, 1.50)
camera_45_p_ls.set_rotation(0, 45.0, 0)
settings.add_sensor(camera_45_p_ls)
# RIGHT STEREO +45 DEGREE SEGMENTATION
camera_45_p_rs = Camera('45_P_RightCameraSeg', PostProcessing='SemanticSegmentation')
camera_45_p_rs.set_image_size(800, 600)
camera_45_p_rs.set_position(0.8, 1.0, 1.50)
camera_45_p_rs.set_rotation(0, 45.0, 0)
settings.add_sensor(camera_45_p_rs)
'''
#### -90 DEGREE STEREO CAMERA ####
# LEFT STEREO -90 DEGREE RGB
camera_90_n_l = Camera('90_N_LeftCameraRGB', PostProcessing='SceneFinal')
camera_90_n_l.set_image_size(800, 600)
camera_90_n_l.set_position(-0.27, -1.0, 1.50) # [X, Y, Z]
camera_90_n_l.set_rotation(0, -90.0, 0) # [pitch(Y), yaw(Z), roll(X)]
settings.add_sensor(camera_90_n_l)
# RIGHT STEREO -90 DEGREE RGB
camera_90_n_r = Camera('90_N_RightCameraRGB', PostProcessing='SceneFinal')
camera_90_n_r.set_image_size(800, 600)
camera_90_n_r.set_position(0.27, -1.0, 1.50)
camera_90_n_r.set_rotation(0, -90.0, 0)
settings.add_sensor(camera_90_n_r)
# LEFT STEREO -90 DEGREE DEPTH
camera_90_n_ld = Camera('90_N_LeftCameraDepth', PostProcessing='Depth')
camera_90_n_ld.set_image_size(800, 600)
camera_90_n_ld.set_position(-0.27, -1.0, 1.50)
camera_90_n_ld.set_rotation(0, -90.0, 0)
settings.add_sensor(camera_90_n_ld)
# RIGHT STEREO -90 DEGREE DEPTH
camera_90_n_rd = Camera('90_N_RightCameraDepth', PostProcessing='Depth')
camera_90_n_rd.set_image_size(800, 600)
camera_90_n_rd.set_position(0.27, -1.0, 1.50)
camera_90_n_rd.set_rotation(0, -90.0, 0)
settings.add_sensor(camera_90_n_rd)
# LEFT STEREO -90 DEGREE SEGMENTATION
camera_90_n_ls = Camera('90_N_LeftCameraSeg', PostProcessing='SemanticSegmentation')
camera_90_n_ls.set_image_size(800, 600)
camera_90_n_ls.set_position(-0.27, -1.0, 1.50)
camera_90_n_ls.set_rotation(0, -90.0, 0)
settings.add_sensor(camera_90_n_ls)
# RIGHT STEREO -90 DEGREE SEGMENTATION
camera_90_n_rs = Camera('90_N_RightCameraSeg', PostProcessing='SemanticSegmentation')
camera_90_n_rs.set_image_size(800, 600)
camera_90_n_rs.set_position(0.27, -1.0, 1.50)
camera_90_n_rs.set_rotation(0, -90.0, 0)
settings.add_sensor(camera_90_n_rs)
#### +90 DEGREE STEREO CAMERA ####
# LEFT STEREO +90 DEGREE RGB
camera_90_p_l = Camera('90_P_LeftCameraRGB', PostProcessing='SceneFinal')
camera_90_p_l.set_image_size(800, 600)
camera_90_p_l.set_position(0.27, 1.0, 1.50) # [X, Y, Z]
camera_90_p_l.set_rotation(0, 90.0, 0) # [pitch(Y), yaw(Z), roll(X)]
settings.add_sensor(camera_90_p_l)
# RIGHT STEREO +90 DEGREE RGB
camera_90_p_r = Camera('90_P_RightCameraRGB', PostProcessing='SceneFinal')
camera_90_p_r.set_image_size(800, 600)
camera_90_p_r.set_position(-0.27, 1.0, 1.50)
camera_90_p_r.set_rotation(0, 90.0, 0)
settings.add_sensor(camera_90_p_r)
# LEFT STEREO +90 DEGREE DEPTH
camera_90_p_ld = Camera('90_P_LeftCameraDepth', PostProcessing='Depth')
camera_90_p_ld.set_image_size(800, 600)
camera_90_p_ld.set_position(0.27, 1.0, 1.50)
camera_90_p_ld.set_rotation(0, 90.0, 0)
settings.add_sensor(camera_90_p_ld)
# RIGHT STEREO +90 DEGREE DEPTH
camera_90_p_rd = Camera('90_P_RightCameraDepth', PostProcessing='Depth')
camera_90_p_rd.set_image_size(800, 600)
camera_90_p_rd.set_position(-0.27, 1.0, 1.50)
camera_90_p_rd.set_rotation(0, 90.0, 0)
settings.add_sensor(camera_90_p_rd)
# LEFT STEREO +90 DEGREE SEGMENTATION
camera_90_p_ls = Camera('90_P_LeftCameraSeg', PostProcessing='SemanticSegmentation')
camera_90_p_ls.set_image_size(800, 600)
camera_90_p_ls.set_position(0.27, 1.0, 1.50)
camera_90_p_ls.set_rotation(0, 90.0, 0)
settings.add_sensor(camera_90_p_ls)
# RIGHT STEREO +90 DEGREE SEGMENTATION
camera_90_p_rs = Camera('90_P_RightCameraSeg', PostProcessing='SemanticSegmentation')
camera_90_p_rs.set_image_size(800, 600)
camera_90_p_rs.set_position(-0.27, 1.0, 1.50)
camera_90_p_rs.set_rotation(0, 90.0, 0)
settings.add_sensor(camera_90_p_rs)
'''
'''
if args.lidar:
lidar = Lidar('Lidar32')
lidar.set_position(0, 0, 2.50)
lidar.set_rotation(0, 0, 0)
lidar.set(
Channels=32,
Range=50,
PointsPerSecond=100000,
RotationFrequency=10,
UpperFovLimit=10,
LowerFovLimit=-30)
settings.add_sensor(lidar)
'''
else:
# Alternatively, we can load these settings from a file.
with open(args.settings_filepath, 'r') as fp:
settings = fp.read()
# Now we load these settings into the server. The server replies
# with a scene description containing the available start spots for
# the player. Here we can provide a CarlaSettings object or a
# CarlaSettings.ini file as string.
scene = client.load_settings(settings)
# Choose one player start at random.
number_of_player_starts = len(scene.player_start_spots)
player_start = random.randint(0, max(0, number_of_player_starts - 1))
# Notify the server that we want to start the episode at the
# player_start index. This function blocks until the server is ready
# to start the episode.
print('Starting new episode...')
client.start_episode(player_start)
camera_l_to_car_transform = camera_l.get_unreal_transform()
camera_r_to_car_transform = camera_r.get_unreal_transform()
camera_45_n_l_to_car_transform = camera_45_n_l.get_unreal_transform()
camera_45_n_r_to_car_transform = camera_45_n_r.get_unreal_transform()
camera_45_p_l_to_car_transform = camera_45_p_l.get_unreal_transform()
camera_45_p_r_to_car_transform = camera_45_p_r.get_unreal_transform()
'''
camera_90_n_l_to_car_transform = camera_90_n_l.get_unreal_transform()
camera_90_n_r_to_car_transform = camera_90_n_r.get_unreal_transform()
camera_90_p_l_to_car_transform = camera_90_p_l.get_unreal_transform()
camera_90_p_r_to_car_transform = camera_90_p_r.get_unreal_transform()
'''
# Create a folder for saving episode data
if not os.path.isdir("/data/khoshhal/Dataset/episode_{:0>4d}".format(episode)):
os.makedirs("/data/khoshhal/Dataset/episode_{:0>4d}".format(episode))
# Iterate every frame in the episode.
for frame in range(0, frames_per_episode):
# Read the data produced by the server this frame.
measurements, sensor_data = client.read_data()
# player_measurements = measurements.player_measurements
world_transform = Transform(measurements.player_measurements.transform)
# Compute the final transformation matrix.
camera_l_to_world_transform = world_transform * camera_l_to_car_transform
camera_r_to_world_transform = world_transform * camera_r_to_car_transform
camera_45_n_l_to_world_transform = world_transform * camera_45_n_l_to_car_transform
camera_45_n_r_to_world_transform = world_transform * camera_45_n_r_to_car_transform
camera_45_p_l_to_world_transform = world_transform * camera_45_p_l_to_car_transform
camera_45_p_r_to_world_transform = world_transform * camera_45_p_r_to_car_transform
'''
camera_90_n_l_to_world_transform = world_transform * camera_90_n_l_to_car_transform
camera_90_n_r_to_world_transform = world_transform * camera_90_n_r_to_car_transform
camera_90_p_l_to_world_transform = world_transform * camera_90_p_l_to_car_transform
camera_90_p_r_to_world_transform = world_transform * camera_90_p_r_to_car_transform
'''
# Save the images to disk if requested.
if frame >= 30:# and (frame % 2 == 0):
if args.save_images_to_disk:
for name, measurement in sensor_data.items():
filename = args.out_filename_format.format(episode, name, (frame-30))
measurement.save_to_disk(filename)
# Save Transform matrix of each camera to separated files
line = ""
filename = "{}episode_{:0>4d}/LeftCamera".format(args.root_path, episode) + ".txt"
with open(filename, 'a') as myfile:
for x in np.asarray(camera_l_to_world_transform.matrix[:3, :]).reshape(-1):
line += "{:.8e} ".format(x)
line = line[:-1]
line += "\n"
myfile.write(line)
line = ""
filename = "{}episode_{:0>4d}/RightCamera".format(args.root_path, episode) + ".txt"
with open(filename, 'a') as myfile:
for x in np.asarray(camera_r_to_world_transform.matrix[:3, :]).reshape(-1):
line += "{:.8e} ".format(x)
line = line[:-1]
line += "\n"
myfile.write(line)
line = ""
filename = "{}episode_{:0>4d}/45_N_LeftCamera".format(args.root_path, episode) + ".txt"
with open(filename, 'a') as myfile:
for x in np.asarray(camera_45_n_l_to_world_transform.matrix[:3, :]).reshape(-1):
line += "{:.8e} ".format(x)
line = line[:-1]
line += "\n"
myfile.write(line)
line = ""
filename = "{}episode_{:0>4d}/45_N_RightCamera".format(args.root_path, episode) + ".txt"
with open(filename, 'a') as myfile:
for x in np.asarray(camera_45_n_r_to_world_transform.matrix[:3, :]).reshape(-1):
line += "{:.8e} ".format(x)
line = line[:-1]
line += "\n"
myfile.write(line)
line = ""
filename = "{}episode_{:0>4d}/45_P_LeftCamera".format(args.root_path, episode) + ".txt"
with open(filename, 'a') as myfile:
for x in np.asarray(camera_45_p_l_to_world_transform.matrix[:3, :]).reshape(-1):
line += "{:.8e} ".format(x)
line = line[:-1]
line += "\n"
myfile.write(line)
line = ""
filename = "{}episode_{:0>4d}/45_P_RightCamera".format(args.root_path, episode) + ".txt"
with open(filename, 'a') as myfile:
for x in np.asarray(camera_45_p_r_to_world_transform.matrix[:3, :]).reshape(-1):
line += "{:.8e} ".format(x)
line = line[:-1]
line += "\n"
myfile.write(line)
line = ""
'''
filename = "{}episode_{:0>4d}/90_N_LeftCamera".format(args.root_path, episode) + ".txt"
with open(filename, 'a') as myfile:
for x in np.asarray(camera_90_n_l_to_world_transform.matrix[:3, :]).reshape(-1):
line += "{:.8e} ".format(x)
line = line[:-1]
line += "\n"
myfile.write(line)
line = ""
filename = "{}episode_{:0>4d}/90_N_RightCamera".format(args.root_path, episode) + ".txt"
with open(filename, 'a') as myfile:
for x in np.asarray(camera_90_n_r_to_world_transform.matrix[:3, :]).reshape(-1):
line += "{:.8e} ".format(x)
line = line[:-1]
line += "\n"
myfile.write(line)
line = ""
filename = "{}episode_{:0>4d}/90_P_LeftCamera".format(args.root_path, episode) + ".txt"
with open(filename, 'a') as myfile:
for x in np.asarray(camera_90_p_l_to_world_transform.matrix[:3, :]).reshape(-1):
line += "{:.8e} ".format(x)
line = line[:-1]
line += "\n"
myfile.write(line)
line = ""
filename = "{}episode_{:0>4d}/90_P_RightCamera".format(args.root_path, episode) + ".txt"
with open(filename, 'a') as myfile:
for x in np.asarray(camera_90_p_r_to_world_transform.matrix[:3, :]).reshape(-1):
line += "{:.8e} ".format(x)
line = line[:-1]
line += "\n"
myfile.write(line)
line = ""
'''
# We can access the encoded data of a given image as numpy
# array using its "data" property. For instance, to get the
# depth value (normalized) at pixel X, Y
#
# depth_array = sensor_data['CameraDepth'].data
# value_at_pixel = depth_array[Y, X]
#
# Now we have to send the instructions to control the vehicle.
# If we are in synchronous mode the server will pause the
# simulation until we send this control.
if not args.autopilot:
client.send_control(
steer=random.uniform(-1.0, 1.0),
throttle=0.5,
brake=0.0,
hand_brake=False,
reverse=False)
else:
# Together with the measurements, the server has sent the
# control that the in-game autopilot would do this frame. We
# can enable autopilot by sending back this control to the
# server. We can modify it if wanted, here for instance we
# will add some noise to the steer.
control = measurements.player_measurements.autopilot_control
#control.steer += random.uniform(-0.1, 0.1)
client.send_control(control)
def get_bbox(self, measurement, seg):
global
width = WIDTH
height = HEIGHT
extrinsic = Transform(measurement.player_measurements.transform) * self.obs_to_car_transform
bbox_list = []
orientation_list = []
distance_list = []
# main_rotation = measurement.player_measurements.transform.rotation
player_location = measurement.player_measurements.transform.location
player_location = np.array([player_location.x, player_location.y, player_location.z])
# collect the 2bbox generated from the 3d-bbox of non-player agents
for agent in measurement.non_player_agents:
if agent.HasField("vehicle"):
# veh_id = agent.id
# idx = self.nonplayer_ids[veh_id]
vehicle_transform = Transform(agent.vehicle.transform)
bbox_transform = Transform(agent.vehicle.bounding_box.transform)
ext = agent.vehicle.bounding_box.extent
bbox = np.array([
[ ext.x, ext.y, ext.z],
[- ext.x, ext.y, ext.z],
[ ext.x, - ext.y, ext.z],
[- ext.x, - ext.y, ext.z],
[ ext.x, ext.y, - ext.z],
[- ext.x, ext.y, - ext.z],
[ ext.x, - ext.y, - ext.z],
[- ext.x, - ext.y, - ext.z]
])
bbox = bbox_transform.transform_points(bbox)
bbox = vehicle_transform.transform_points(bbox)
orientation = agent.vehicle.transform.orientation
vehicle_location = agent.vehicle.transform.location
cur_location = np.array([vehicle_location.x, vehicle_location.y, vehicle_location.z])
distance = np.linalg.norm(player_location - cur_location)
vertices = []
for vertex in bbox:
pos_vector = np.array([
[vertex[0,0]], # [[X,
[vertex[0,1]], # Y,
[vertex[0,2]], # Z,
[1.0] # 1.0]]
])
transformed_3d_pos = np.dot(inv(extrinsic.matrix), pos_vector)
pos2d = np.dot(self.intrinsic, transformed_3d_pos[:3])
pos2d = np.array([
pos2d[0] / pos2d[2], pos2d[1] / pos2d[2], pos2d[2]
])
if pos2d[2] > 0:
x_2d = width - pos2d[0]
y_2d = height - pos2d[1]
vertices.append([x_2d, y_2d])
if len(vertices) > 1:
# vehicle_rotation = agent.vehicle.transform.rotation
vertices = np.array(vertices)
bbox_list.append([np.min(vertices[:, 0]), np.min(vertices[:, 1]),
np.max(vertices[:, 0]), np.max(vertices[:, 1])])
orientation_list.append(orientation)
distance_list.append(distance)
seg_bboxes = seg_to_bbox(seg)
final_bboxes = []
final_directions = []
final_distances = []
assert(len(bbox_list) == len(orientation_list))
for i in range(len(bbox_list)):
bbox = bbox_list[i]
direction = orientation_list[i]
xmin, ymin, xmax, ymax = bbox
x1, y1, x2, y2 = width, height, 0, 0
for segbbox in seg_bboxes:
xmin0, ymin0, xmax0, ymax0 = segbbox
if xmin0 >= xmin - 5 and ymin0 >= ymin - 5 and xmax0 < xmax + 5 and ymax0 < ymax + 5:
x1 = min(x1, xmin0)
y1 = min(y1, ymin0)
x2 = max(x2, xmax0)
y2 = max(y2, ymax0)
if x2 > x1 and y2 > y1 and [int(x1), int(y1), int(x2), int(y2)] not in final_bboxes:
final_bboxes.append([int(x1), int(y1), int(x2), int(y2)])
relative_orientation = get_angle(direction.x, direction.y, self.orientation.x, self.orientation.y)
final_directions.append(relative_orientation)
final_distances.append(distance_list[i])
# for angle in final_directions:
# self.angle_logger.write("timestep {}: {}\n".format(self.timestep, angle))
# self.angle_logger.flush()
final_distances = np.array(final_distances)
visible_coll_num = min(coll_veh_num, final_distances.size)
coll_idx = np.argpartition(final_distances, visible_coll_num - 1)[:visible_coll_num]
final_colls = [1 if i in coll_idx else 0 for i in range(final_distances.size)]
return final_bboxes, final_directions, final_colls
def run_carla_client( args):
# Here we will run 3 episodes with 300 frames each.
number_of_episodes = 150
frames_per_episode = 500
# for start_i in range(150):
# if start_i%4==0:
# output_folder = 'Packages/CARLA_0.8.2/PythonClient/new_data-viz/test_' + str(start_i)
# if not os.path.exists(output_folder):
# os.mkdir(output_folder)
# print( "make "+str(output_folder))
# ./CarlaUE4.sh -carla-server -benchmark -fps=17 -windowed
# carla-server "/usr/local/carla/Unreal/CarlaUE4/CarlaUE4.uproject" /usr/local/carla/Maps/Town03 -benchmark -fps=10 -windowed
# We assume the CARLA server is already waiting for a client to connect at
# host:port. To create a connection we can use the `make_carla_client`
# context manager, it creates a CARLA client object and starts the
# connection. It will throw an exception if something goes wrong. The
# context manager makes sure the connection is always cleaned up on exit.
with make_carla_client(args.host, args.port) as client:
print('CarlaClient connected')
global episode_nbr
print (episode_nbr)
for episode in range(0,number_of_episodes):
if episode % 1 == 0:
output_folder = 'Datasets/carla-sync/train/test_' + str(episode)
if not os.path.exists(output_folder+"/cameras.p"):
# Start a new episode.
episode_nbr=episode
frame_step = 1 # Save one image every 100 frames
pointcloud_step=50
image_size = [800, 600]
camera_local_pos = [0.3, 0.0, 1.3] # [X, Y, Z]
camera_local_rotation = [0, 0, 0] # [pitch(Y), yaw(Z), roll(X)]
fov = 70
# Create a CarlaSettings object. This object is a wrapper around
# the CarlaSettings.ini file. Here we set the configuration we
# want for the new episode.
settings = CarlaSettings()
settings.set(
SynchronousMode=True,
SendNonPlayerAgentsInfo=True,
NumberOfVehicles=50,
NumberOfPedestrians=200,
WeatherId=random.choice([1, 3, 7, 8, 14]),
QualityLevel=args.quality_level)
settings.randomize_seeds()
# Now we want to add a couple of cameras to the player vehicle.
# We will collect the images produced by these cameras every
# frame.
camera1 = Camera('CameraDepth', PostProcessing='Depth', FOV=fov)
camera1.set_image_size(*image_size)
camera1.set_position(*camera_local_pos)
camera1.set_rotation(*camera_local_rotation)
settings.add_sensor(camera1)
camera2 = Camera('CameraRGB', PostProcessing='SceneFinal', FOV=fov)
camera2.set_image_size(*image_size)
camera2.set_position(*camera_local_pos)
camera2.set_rotation(*camera_local_rotation)
settings.add_sensor(camera2)
camera3 = Camera('CameraSeg', PostProcessing='SemanticSegmentation', FOV=fov)
camera3.set_image_size(*image_size)
camera3.set_position(*camera_local_pos)
camera3.set_rotation(*camera_local_rotation)
settings.add_sensor(camera3)
# Now we load these settings into the server. The server replies
# with a scene description containing the available start spots for
# the player. Here we can provide a CarlaSettings object or a
# CarlaSettings.ini file as string.
scene = client.load_settings(settings)
# Choose one player start at random.
number_of_player_starts = len(scene.player_start_spots)
player_start = episode#random.randint(0, max(0, number_of_player_starts - 1))
output_folder = 'Datasets/carla-sync/train/test_' + str(episode)
# Notify the server that we want to start the episode at the
# player_start index. This function blocks until the server is ready
# to start the episode.
print('Starting new episode...')
client.start_episode(player_start)
cameras_dict = {}
pedestrians_dict = {}
cars_dict = {}
# Compute the camera transform matrix
camera_to_car_transform = camera2.get_unreal_transform()
# (Intrinsic) (3, 3) K Matrix
K = np.identity(3)
K[0, 2] = image_size[0] / 2.0
K[1, 2] = image_size[1] / 2.0
K[0, 0] = K[1, 1] = image_size[0] / (2.0 * np.tan(fov * np.pi / 360.0))
with open(output_folder + '/camera_intrinsics.p', 'w') as camfile:
pickle.dump(K, camfile)
# Iterate every frame in the episode.
for frame in range(0, frames_per_episode):
# Read the data produced by the server this frame.
measurements, sensor_data = client.read_data()
# Print some of the measurements.
#print_measurements(measurements)
if not frame % frame_step:
# Save the images to disk if requested.
for name, measurement in sensor_data.items():
filename = args.out_filename_format.format(episode, name, frame)
print (filename)
measurement.save_to_disk(filename)
# We can access the encoded data of a given image as numpy
# array using its "data" property. For instance, to get the
# depth value (normalized) at pixel X, Y
#
# depth_array = sensor_data['CameraDepth'].data
# value_at_pixel = depth_array[Y, X]
#
# Now we have to send the instructions to control the vehicle.
# If we are in synchronous mode the server will pause the
# simulation until we send this control.
# RGB image [[[r,g,b],..[r,g,b]],..[[r,g,b],..[r,g,b]]]
image_RGB = to_rgb_array(sensor_data['CameraRGB'])
labels=labels_to_array(sensor_data['CameraSeg'])[:,:,np.newaxis]
image_seg = np.tile(labels, (1, 1, 3))
depth_array = sensor_data['CameraDepth'].data*1000
# 2d to (camera) local 3d
# We use the image_RGB to colorize each 3D point, this is optional.
# "max_depth" is used to keep only the points that are near to the
# camera, meaning 1.0 the farest points (sky)
if not frame % pointcloud_step:
point_cloud = depth_to_local_point_cloud(
sensor_data['CameraDepth'],
image_RGB,
max_depth=args.far
)
point_cloud_seg = depth_to_local_point_cloud(
sensor_data['CameraDepth'],
segmentation=image_seg,
max_depth=args.far
)
# (Camera) local 3d to world 3d.
# Get the transform from the player protobuf transformation.
world_transform = Transform(
measurements.player_measurements.transform
)
# Compute the final transformation matrix.
car_to_world_transform = world_transform * camera_to_car_transform
# Car to World transformation given the 3D points and the
# transformation matrix.
point_cloud.apply_transform(car_to_world_transform)
point_cloud_seg.apply_transform(car_to_world_transform)
Rt = car_to_world_transform.matrix
Rt_inv = car_to_world_transform.inverse().matrix
# R_inv=world_transform.inverse().matrix
cameras_dict[frame] = {}
cameras_dict[frame]['inverse_rotation'] = Rt_inv[:]
cameras_dict[frame]['rotation'] = Rt[:]
cameras_dict[frame]['translation'] = Rt_inv[0:3, 3]
cameras_dict[frame]['camera_to_car'] = camera_to_car_transform.matrix
# Get non-player info
vehicles = {}
pedestrians = {}
for agent in measurements.non_player_agents:
# check if the agent is a vehicle.
if agent.HasField('vehicle'):
pos = agent.vehicle.transform.location
pos_vector = np.array([[pos.x], [pos.y], [pos.z], [1.0]])
trnasformed_3d_pos = np.dot(Rt_inv, pos_vector)
pos2d = np.dot(K, trnasformed_3d_pos[:3])
# Normalize the point
norm_pos2d = np.array([
pos2d[0] / pos2d[2],
pos2d[1] / pos2d[2],
pos2d[2]])
# Now, pos2d contains the [x, y, d] values of the image in pixels (where d is the depth)
# You can use the depth to know the points that are in front of the camera (positive depth).
x_2d = image_size[0] - norm_pos2d[0]
y_2d = image_size[1] - norm_pos2d[1]
vehicles[agent.id] = {}
vehicles[agent.id]['transform'] = [agent.vehicle.transform.location.x,
agent.vehicle.transform.location.y,
agent.vehicle.transform.location.z]
vehicles[agent.id][
'bounding_box.transform'] = agent.vehicle.bounding_box.transform.location.z
vehicles[agent.id]['yaw'] = agent.vehicle.transform.rotation.yaw
vehicles[agent.id]['bounding_box'] = [agent.vehicle.bounding_box.extent.x,
agent.vehicle.bounding_box.extent.y,
agent.vehicle.bounding_box.extent.z]
vehicle_transform = Transform(agent.vehicle.bounding_box.transform)
pos = agent.vehicle.transform.location
bbox3d = agent.vehicle.bounding_box.extent
# Compute the 3D bounding boxes
# f contains the 4 points that corresponds to the bottom
f = np.array([[pos.x + bbox3d.x, pos.y - bbox3d.y,
pos.z - bbox3d.z + agent.vehicle.bounding_box.transform.location.z],
[pos.x + bbox3d.x, pos.y + bbox3d.y,
pos.z - bbox3d.z + agent.vehicle.bounding_box.transform.location.z],
[pos.x - bbox3d.x, pos.y + bbox3d.y,
pos.z - bbox3d.z + agent.vehicle.bounding_box.transform.location.z],
[pos.x - bbox3d.x, pos.y - bbox3d.y,
pos.z - bbox3d.z + agent.vehicle.bounding_box.transform.location.z],
[pos.x + bbox3d.x, pos.y - bbox3d.y,
pos.z + bbox3d.z + agent.vehicle.bounding_box.transform.location.z],
[pos.x + bbox3d.x, pos.y + bbox3d.y,
pos.z + bbox3d.z + agent.vehicle.bounding_box.transform.location.z],
[pos.x - bbox3d.x, pos.y + bbox3d.y,
pos.z + bbox3d.z + agent.vehicle.bounding_box.transform.location.z],
[pos.x - bbox3d.x, pos.y - bbox3d.y,
pos.z + bbox3d.z + agent.vehicle.bounding_box.transform.location.z]])
f_rotated = vehicle_transform.transform_points(f)
f_2D_rotated = []
vehicles[agent.id]['bounding_box_coord'] = f_rotated
for i in range(f.shape[0]):
point = np.array([[f_rotated[i, 0]], [f_rotated[i, 1]], [f_rotated[i, 2]], [1]])
transformed_2d_pos = np.dot(Rt_inv, point)
pos2d = np.dot(K, transformed_2d_pos[:3])
norm_pos2d = np.array([
pos2d[0] / pos2d[2],
pos2d[1] / pos2d[2],
pos2d[2]])
# print([image_size[0] - (pos2d[0] / pos2d[2]), image_size[1] - (pos2d[1] / pos2d[2])])
f_2D_rotated.append(
np.array([image_size[0] - norm_pos2d[0], image_size[1] - norm_pos2d[1]]))
vehicles[agent.id]['bounding_box_2D'] = f_2D_rotated
elif agent.HasField('pedestrian'):
pedestrians[agent.id] = {}
pedestrians[agent.id]['transform'] = [agent.pedestrian.transform.location.x,
agent.pedestrian.transform.location.y,
agent.pedestrian.transform.location.z]
pedestrians[agent.id]['yaw'] = agent.pedestrian.transform.rotation.yaw
pedestrians[agent.id]['bounding_box'] = [agent.pedestrian.bounding_box.extent.x,
agent.pedestrian.bounding_box.extent.y,
agent.pedestrian.bounding_box.extent.z]
# get the needed transformations
# remember to explicitly make it Transform() so you can use transform_points()
pedestrian_transform = Transform(agent.pedestrian.transform)
bbox_transform = Transform(agent.pedestrian.bounding_box.transform)
# get the box extent
ext = agent.pedestrian.bounding_box.extent
# 8 bounding box vertices relative to (0,0,0)
bbox = np.array([
[ ext.x, ext.y, ext.z],
[- ext.x, ext.y, ext.z],
[ ext.x, - ext.y, ext.z],
[- ext.x, - ext.y, ext.z],
[ ext.x, ext.y, - ext.z],
[- ext.x, ext.y, - ext.z],
[ ext.x, - ext.y, - ext.z],
[- ext.x, - ext.y, - ext.z]
])
# transform the vertices respect to the bounding box transform
bbox = bbox_transform.transform_points(bbox)
# the bounding box transform is respect to the pedestrian transform
# so let's transform the points relative to it's transform
bbox = pedestrian_transform.transform_points(bbox)
# pedestrian's transform is relative to the world, so now,
# bbox contains the 3D bounding box vertices relative to the world
pedestrians[agent.id]['bounding_box_coord'] =copy.deepcopy(bbox)
# Additionally, you can print these vertices to check that is working
f_2D_rotated=[]
ys=[]
xs=[]
zs=[]
for vertex in bbox:
pos_vector = np.array([
[vertex[0,0]], # [[X,
[vertex[0,1]], # Y,
[vertex[0,2]], # Z,
[1.0] # 1.0]]
])
# transform the points to camera
transformed_3d_pos =np.dot(Rt_inv, pos_vector)# np.dot(inv(self._extrinsic.matrix), pos_vector)
zs.append(transformed_3d_pos[2])
# transform the points to 2D
pos2d =np.dot(K, transformed_3d_pos[:3]) #np.dot(self._intrinsic, transformed_3d_pos[:3])
# normalize the 2D points
pos2d = np.array([
pos2d[0] / pos2d[2],
pos2d[1] / pos2d[2],
pos2d[2]
])
# print the points in the screen
if pos2d[2] > 0: # if the point is in front of the camera
x_2d = image_size[0]-pos2d[0]#WINDOW_WIDTH - pos2d[0]
y_2d = image_size[1]-pos2d[1]#WINDOW_HEIGHT - pos2d[1]
ys.append(y_2d)
xs.append(x_2d)
f_2D_rotated.append( (y_2d, x_2d))
if len(xs)>1:
bbox=[[int(min(xs)), int(max(xs))],[int(min(ys)), int(max(ys))]]
clipped_seg=labels[bbox[1][0]:bbox[1][1],bbox[0][0]:bbox[0][1]]
recounted = Counter(clipped_seg.flatten())
if 4 in recounted.keys() and recounted[4]>0.1*len(clipped_seg.flatten()):
clipped_depth=depth_array[bbox[1][0]:bbox[1][1],bbox[0][0]:bbox[0][1]]
#print (clipped_depth.shape)
people_indx=np.where(clipped_seg==4)
masked_depth=[]
for people in zip(people_indx[0],people_indx[1] ):
#print(people)
masked_depth.append(clipped_depth[people])
#masked_depth=clipped_depth[np.where(clipped_seg==4)]
#print (masked_depth)
#print ("Depth "+ str(min(zs))+" "+ str(max(zs)))
#recounted = Counter(masked_depth)
#print(recounted)
avg_depth=np.mean(masked_depth)
if avg_depth<700 and avg_depth>=min(zs)-10 and avg_depth<= max(zs)+10:
#print("Correct depth")
pedestrians[agent.id]['bounding_box_2D'] = f_2D_rotated
pedestrians[agent.id]['bounding_box_2D_size']=recounted[4]
pedestrians[agent.id]['bounding_box_2D_avg_depth']=avg_depth
pedestrians[agent.id]['bounding_box_2D_depths']=zs
#print ( pedestrians[agent.id].keys())
#else:
# print(recounted)
# print ("Depth "+ str(min(zs))+" "+ str(max(zs)))
#if sum(norm(depth_array-np.mean(zs))<1.0):
# pedestrians[agent.id] = {}
# pedestrians[agent.id]['transform'] = [agent.pedestrian.transform.location.x,
# agent.pedestrian.transform.location.y,
# agent.pedestrian.transform.location.z]
# pedestrians[agent.id]['yaw'] = agent.pedestrian.transform.rotation.yaw
# pedestrians[agent.id]['bounding_box'] = [agent.pedestrian.bounding_box.extent.x,
# agent.pedestrian.bounding_box.extent.y,
# agent.pedestrian.bounding_box.extent.z]
# vehicle_transform = Transform(agent.pedestrian.bounding_box.transform)
# pos = agent.pedestrian.transform.location
#
# bbox3d = agent.pedestrian.bounding_box.extent
#
# # Compute the 3D bounding boxes
# # f contains the 4 points that corresponds to the bottom
# f = np.array([[pos.x + bbox3d.x, pos.y - bbox3d.y, pos.z- bbox3d.z ],
# [pos.x + bbox3d.x, pos.y + bbox3d.y, pos.z- bbox3d.z ],
# [pos.x - bbox3d.x, pos.y + bbox3d.y, pos.z- bbox3d.z ],
# [pos.x - bbox3d.x, pos.y - bbox3d.y, pos.z- bbox3d.z ],
# [pos.x + bbox3d.x, pos.y - bbox3d.y, pos.z + bbox3d.z],
# [pos.x + bbox3d.x, pos.y + bbox3d.y, pos.z + bbox3d.z],
# [pos.x - bbox3d.x, pos.y + bbox3d.y, pos.z + bbox3d.z],
# [pos.x - bbox3d.x, pos.y - bbox3d.y, pos.z + bbox3d.z]])
#
# f_rotated = pedestrian_transform.transform_points(f)
# pedestrians[agent.id]['bounding_box_coord'] = f_rotated
# f_2D_rotated = []
#
# for i in range(f.shape[0]):
# point = np.array([[f_rotated[i, 0]], [f_rotated[i, 1]], [f_rotated[i, 2]], [1]])
# transformed_2d_pos = np.dot(Rt_inv, point)
# pos2d = np.dot(K, transformed_2d_pos[:3])
# norm_pos2d = np.array([
# pos2d[0] / pos2d[2],
# pos2d[1] / pos2d[2],
# pos2d[2]])
# f_2D_rotated.append([image_size[0] - norm_pos2d[0], image_size[1] - norm_pos2d[1]])
# pedestrians[agent.id]['bounding_box_2D'] = f_2D_rotated
cars_dict[frame] = vehicles
pedestrians_dict[frame] = pedestrians
#print("End of Episode")
#print(len(pedestrians_dict[frame]))
# Save PLY to disk
# This generates the PLY string with the 3D points and the RGB colors
# for each row of the file.
if not frame % pointcloud_step:
point_cloud.save_to_disk(os.path.join(
output_folder, '{:0>5}.ply'.format(frame))
)
point_cloud_seg.save_to_disk(os.path.join(
output_folder, '{:0>5}_seg.ply'.format(frame))
)
if not args.autopilot:
client.send_control(
hand_brake=True)
else:
# Together with the measurements, the server has sent the
# control that the in-game autopilot would do this frame. We
# can enable autopilot by sending back this control to the
# server. We can modify it if wanted, here for instance we
# will add some noise to the steer.
control = measurements.player_measurements.autopilot_control
control.steer += random.uniform(-0.1, 0.1)
client.send_control(control)
print ("Start pickle save")
with open(output_folder + '/cameras.p', 'w') as camerafile:
pickle.dump(cameras_dict, camerafile)
with open(output_folder + '/people.p', 'w') as peoplefile:
pickle.dump(pedestrians_dict, peoplefile)
with open(output_folder + '/cars.p', 'w') as carfile:
pickle.dump(cars_dict, carfile)
print ("Episode done")
def run_carla_client(host, port, far):
# Here we will run a single episode with 300 frames.
number_of_frames = 3000
frame_step = 10 # Save one image every 100 frames
output_folder = '_out'
image_size = [1280, 720]
camera_local_pos = [0.3, 0.0, 1.3] # [X, Y, Z]
camera_local_rotation = [0, 0, 0] # [pitch(Y), yaw(Z), roll(X)]
fov = 59
# Connect with the server
with make_carla_client(host, port) as client:
print('CarlaClient connected')
# Here we load the settings.
settings = CarlaSettings()
settings.set(SynchronousMode=True,
SendNonPlayerAgentsInfo=False,
NumberOfVehicles=20,
NumberOfPedestrians=40,
WeatherId=7)
settings.randomize_seeds()
camera1 = Camera('CameraDepth', PostProcessing='Depth', FOV=fov)
camera1.set_image_size(*image_size)
camera1.set_position(*camera_local_pos)
camera1.set_rotation(*camera_local_rotation)
settings.add_sensor(camera1)
camera2 = Camera('CameraRGB', PostProcessing='SceneFinal', FOV=fov)
camera2.set_image_size(*image_size)
camera2.set_position(*camera_local_pos)
camera2.set_rotation(*camera_local_rotation)
settings.add_sensor(camera2)
client.load_settings(settings)
# Start at location index id '0'
client.start_episode(0)
# Compute the camera transform matrix
camera_to_car_transform = camera2.get_unreal_transform()
print("camera_to_car_transform", camera_to_car_transform)
# Iterate every frame in the episode except for the first one.
for frame in range(1, number_of_frames):
# Read the data produced by the server this frame.
measurements, sensor_data = client.read_data()
# Save one image every 'frame_step' frames
if not frame % frame_step:
# Start transformations time mesure.
# RGB image [[[r,g,b],..[r,g,b]],..[[r,g,b],..[r,g,b]]]
image_RGB = to_rgb_array(sensor_data['CameraRGB'])
world_transform = Transform(
measurements.player_measurements.transform)
# Compute the final transformation matrix.
car_to_world_transform = world_transform * camera_to_car_transform
print("{}.png".format(str(frame)))
print([
measurements.player_measurements.transform.location.x,
measurements.player_measurements.transform.location.y,
measurements.player_measurements.transform.location.z
])
print("world_transform\n", world_transform)
#print("car_to_world_transform\n",car_to_world_transform)
print('\n')
im_bgr = cv2.cvtColor(image_RGB, cv2.COLOR_RGB2BGR)
cv2.imwrite("./{}/{}.png".format(output_folder, str(frame)),
im_bgr)
# 2d to (camera) local 3d
# We use the image_RGB to colorize each 3D point, this is optional.
# "max_depth" is used to keep only the points that are near to the
# camera, meaning 1.0 the farest points (sky)
# Save PLY to disk
# This generates the PLY string with the 3D points and the RGB colors
# for each row of the file.
client.send_control(
measurements.player_measurements.autopilot_control)
