def log_responses(responses, idx):
for response_idx, response in enumerate(responses):
filename = os.path.join(
tmp_dir, "{}_{}_{}".format(idx, response.image_type, response_idx))
if response.pixels_as_float:
print("Type %d, size %d" %
(response.image_type, len(response.image_data_float)))
airsim.write_pfm(os.path.normpath(filename + '.pfm'),
airsim.get_pfm_array(response))
elif response.compress: #png format
print("Type %d, size %d" %
(response.image_type, len(response.image_data_uint8)))
airsim.write_file(os.path.normpath(filename + '.png'),
response.image_data_uint8)
else: #uncompressed array
print("Type %d, size %d" %
(response.image_type, len(response.image_data_uint8)))
img1d = np.fromstring(response.image_data_uint8,
dtype=np.uint8) # get numpy array
img_rgb = img1d.reshape(
response.height, response.width,
3) # reshape array to 3 channel image array H X W X 3
cv2.imwrite(os.path.normpath(filename + '.png'),
img_rgb) # write to png
def save_image(client, tmp_dir, count, id):
vehicle_name = "Drone" + str(id)
responses = client.simGetImages ([airsim.ImageRequest ("1", airsim.ImageType.Scene, False, False)], \
vehicle_name=vehicle_name)
filename = os.path.join(tmp_dir, str(count))
for idx, response in enumerate(responses):
if response.pixels_as_float:
print("Type %d, size %d" %
(response.image_type, len(response.image_data_float)))
airsim.write_pfm(os.path.normpath(filename + '.pfm'),
airsim.get_pfm_array(response))
elif response.compress: # png format
print("Type %d, size %d" %
(response.image_type, len(response.image_data_uint8)))
airsim.write_file(os.path.normpath(filename + '.png'),
response.image_data_uint8)
else: # uncompressed array
print("Type %d, size %d" %
(response.image_type, len(response.image_data_uint8)))
img1d = np.fromstring(response.image_data_uint8,
dtype=np.uint8) # get numpy array
img_rgb = img1d.reshape(
response.height, response.width,
3) # reshape array to 4 channel image array H X W X 3
cv2.imwrite(os.path.normpath(filename + '.jpg'),
img_rgb) # write to png
save_imu_pose(client, filename, id)
def save_images(responses, prefix=""):
for i, response in enumerate(responses):
filename = os.path.join(tmp_dir, prefix + "_" + str(i))
if response.pixels_as_float:
print(
f"Type {response.image_type}, size {len(response.image_data_float)}, pos {response.camera_position}"
)
airsim.write_pfm(os.path.normpath(filename + '.pfm'),
airsim.get_pfm_array(response))
else:
print(
f"Type {response.image_type}, size {len(response.image_data_uint8)}, pos {response.camera_position}"
)
airsim.write_file(os.path.normpath(filename + '.png'),
response.image_data_uint8)
def save_as_pfm(self,
scale=100,
file_name="PFM " + str(datetime.datetime.now()),
path="./"):
"""
Saves the image to the specified location with .pfm format.
Args:
cam_type: specifies the location of the camera in the drone.
file_name: gets the day and time the image was received by default,.
path: specifies the location you want to save. It saves in the directory where it is located by default.
scale: ??????????????????
"""
airsim.write_pfm(os.path.join(path, file_name + ".pfm"),
np.flipud(self.fetch_single_img()),
scale=scale)
return
def getDroneImages(client):
# get camera images from the car
responses = client.simGetImages([
airsim.ImageRequest(
"0", airsim.ImageType.DepthVis), #depth visualization image
airsim.ImageRequest("1", airsim.ImageType.DepthPerspective,
True), #depth in perspective projection
airsim.ImageRequest(
"1", airsim.ImageType.Scene), #scene vision image in png format
airsim.ImageRequest("1", airsim.ImageType.Scene, False, False)
]) #scene vision image in uncompressed RGBA array
print('Retrieved images: %d' % len(responses))
tmp_dir = os.path.join(tempfile.gettempdir(), "airsim_drone")
print("Saving images to %s" % tmp_dir)
try:
os.makedirs(tmp_dir)
except OSError:
if not os.path.isdir(tmp_dir):
raise
for idx, response in enumerate(responses):
filename = os.path.join(tmp_dir, str(str(idx) + random_generator(4)))
if response.pixels_as_float:
print("Type %d, size %d" %
(response.image_type, len(response.image_data_float)))
airsim.write_pfm(os.path.normpath(filename + '.pfm'),
airsim.get_pfm_array(response))
elif response.compress: #png format
print("Type %d, size %d" %
(response.image_type, len(response.image_data_uint8)))
airsim.write_file(os.path.normpath(filename + '.png'),
response.image_data_uint8)
else: #uncompressed array
print("Type %d, size %d" %
(response.image_type, len(response.image_data_uint8)))
img1d = np.fromstring(response.image_data_uint8,
dtype=np.uint8) # get numpy array
img_rgb = img1d.reshape(
response.height, response.width,
3) # reshape array to 4 channel image array H X W X 3
cv2.imwrite(os.path.normpath(filename + '.png'),
img_rgb) # write to png
cv2.IMREAD_UNCHANGED)
img = np.concatenate((imgL_0, imgR_0), axis=1)
cv2.imshow('***', img)
key = cv2.waitKey(1)
if key == ord('s'):
loc = [[p.x_val, p.y_val, p.z_val],
[o.w_val, o.x_val, o.y_val, o.z_val]]
trajactory.append(loc)
fname = str(id).zfill(5)
airsim.write_file((saveDir + fname + '_L.png'),
rLeft[0].image_data_uint8)
airsim.write_file((saveDir + fname + '_R.png'),
rRight[0].image_data_uint8)
airsim.write_pfm(((saveDir + fname + '_D.pfm')),
airsim.get_pfm_array(rLeft[1]))
airsim.write_file((saveDir + fname + '_M.png'),
rLeft[2].image_data_uint8)
id = id + 1
# cv2.imshow('***', img), cv2.waitKey(), cv2.destroyAllWindows()
...
with open(saveDir + 'path.json', 'w') as write_file:
json.dump(trajactory, write_file)
cv2.destroyAllWindows()
if __name__ == "__main1__":
v_p = []
v_p.append(airsim.Vector3r(138, 0, -2))
v_p.append(airsim.Vector3r(138.5, 0, -2))
v_p.append(airsim.Vector3r(138.5, 0.5, -2))
def training_data_collector(vehicle_client, car_client):
depth_dir = check_dir_exist("./Data/Depth/")
rgb_dir = check_dir_exist("./Data/RGB/")
seg_dir = check_dir_exist("./Data/Seg/")
timestamp = time.time()
responses = vehicle_client.simGetImages([
airsim.ImageRequest("0", airsim.ImageType.Segmentation, pixels_as_float=False, compress=False),
# segmentation image in int
airsim.ImageRequest("1", airsim.ImageType.DepthPerspective, pixels_as_float=True, compress=False),
# depth in perspective projection
airsim.ImageRequest("2", airsim.ImageType.Scene, pixels_as_float=False, compress=False)
# scene vision image in uncompressed RGBA array
])
np.shape(responses[0])
r0 = responses[0]
r1 = responses[1]
r2 = responses[2]
if not (r0.height > 0 and r1.height > 0 and r2.height > 0):
return "BAD_DATA"
(x_m, y_m) = np.meshgrid(range(0, r1.width), range(0, r1.height))
if len(r0.image_data_float) > 1:
img_seg = np.array(r0.image_data_float)
img_seg = img_seg.reshape(r0.height, r0.width)
else:
img_seg = np.frombuffer(r0.image_data_uint8, dtype=np.uint8) # get numpy array
img_seg = img_seg.reshape(r0.height, r0.width, 4)
if len(r1.image_data_float) > 1:
img_depth = np.array(r1.image_data_float)
img_depth = img_depth.reshape(r1.height, r1.width)
# img_depth = img_depth * 200
# img_depth[img_depth > 255] = 255
else:
img_depth = np.frombuffer(r1.image_data_uint8, dtype=np.uint8) # get numpy array
img_depth = img_depth.reshape(r1.height, r1.width, 4)
if len(r2.image_data_float) > 1:
img_rgb = np.array(r2.image_data_float)
img_rgb = img_rgb.reshape(r2.height, r2.width)
else:
img_rgb = np.frombuffer(r2.image_data_uint8, dtype=np.uint8) # get numpy array
img_rgb = img_rgb.reshape(r2.height, r2.width, 4)
# write the depth file
depth_fname = os.path.join(depth_dir, str(timestamp) + '.pfm')
airsim.write_pfm(depth_fname, airsim.get_pfm_array(r1))
# write the rgb file
# notice that the color of airsim published is BGRA you need to convert to RGB explicitly
# the color is still 4 channels.
rgb_fname = os.path.join(rgb_dir, str(timestamp) + '.png')
img_rgb = cv2.cvtColor(img_rgb, cv2.COLOR_BGRA2RGBA)
# write to file
cv2.imwrite(rgb_fname, img_rgb)
# airsim.write_file(rgb_fname, img_rgb[:, :, :3])
# write the segmentation file
# note that the segmentation file has 4 channels, we need to filter out that redundent channel
img_seg = img_seg[:, :, :3]
segmask_img = segrgb2segmask(img_seg)
seg_fname = os.path.join(seg_dir, str(timestamp) + '.png')
cv2.imwrite(seg_fname, segmask_img)
pd = car_client.getCarState().kinematics_estimated.position
heading = car_client.getCarState().kinematics_estimated.orientation
xyzw = heading.to_numpy_array()
ypr = quaternion_to_euler(xyzw[0], xyzw[1], xyzw[2], xyzw[3])
# car_point = np.array([pd.x_val, pd.y_val, pd.z_val])
log_data_str = "{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\n".format(timestamp, pd.x_val, pd.y_val, pd.z_val, ypr[2],
ypr[1], ypr[0], rgb_fname, seg_fname, depth_fname)
return log_data_str
os.makedirs(os.path.join(tmp_dir, str(n)))
except OSError:
if not os.path.isdir(tmp_dir):
raise
for x in range(50): # do few times
#xn = 1 + x*5 # some random number
client.simSetVehiclePose(airsim.Pose(airsim.Vector3r(x, 0, -2), airsim.to_quaternion(0, 0, 0)), True)
time.sleep(0.1)
responses = client.simGetImages([
airsim.ImageRequest("0", airsim.ImageType.Scene),
airsim.ImageRequest("1", airsim.ImageType.Scene),
airsim.ImageRequest("2", airsim.ImageType.Scene)])
for i, response in enumerate(responses):
if response.pixels_as_float:
print("Type %d, size %d, pos %s" % (response.image_type, len(response.image_data_float), pprint.pformat(response.camera_position)))
airsim.write_pfm(os.path.normpath(os.path.join(tmp_dir, str(x) + "_" + str(i) + '.pfm')), airsim.get_pfm_array(response))
else:
print("Type %d, size %d, pos %s" % (response.image_type, len(response.image_data_uint8), pprint.pformat(response.camera_position)))
airsim.write_file(os.path.normpath(os.path.join(tmp_dir, str(i), str(x) + "_" + str(i) + '.png')), response.image_data_uint8)
pose = client.simGetVehiclePose()
pp.pprint(pose)
time.sleep(3)
# currently reset() doesn't work in CV mode. Below is the workaround
client.simSetVehiclePose(airsim.Pose(airsim.Vector3r(0, 0, 0), airsim.to_quaternion(0, 0, 0)), True)
import os
# connect to the AirSim simulator
client = airsim.MultirotorClient()
client.confirmConnection()
client.enableApiControl(True)
client.armDisarm(True)
# Async methods returns Future. Call join() to wait for task to complete.
client.takeoffAsync().join()
client.moveToPositionAsync(-10, 10, -10, 5).join()
# take images
responses = client.simGetImages([
airsim.ImageRequest("0", airsim.ImageType.DepthVis),
airsim.ImageRequest("1", airsim.ImageType.DepthPlanar, True)
])
print('Retrieved images: %d', len(responses))
# do something with the images
for response in responses:
if response.pixels_as_float:
print("Type %d, size %d" %
(response.image_type, len(response.image_data_float)))
airsim.write_pfm(os.path.normpath('/temp/py1.pfm'),
airsim.getPfmArray(response))
else:
print("Type %d, size %d" %
(response.image_type, len(response.image_data_uint8)))
airsim.write_file(os.path.normpath('/temp/py1.png'),
response.image_data_uint8)
print("CameraInfo %d: %s" % (camera_name, pp.pprint(camera_info)))
airsim.wait_key('Press any key to get images')
for x in range(3): # do few times
z = x * -20 - 5 # some random number
client.simSetVehiclePose(airsim.Pose(airsim.Vector3r(z, z, z), airsim.to_quaternion(x / 3.0, 0, x / 3.0)), True)
responses = client.simGetImages([
airsim.ImageRequest("0", airsim.ImageType.DepthVis),
airsim.ImageRequest("1", airsim.ImageType.DepthPerspective, True),
airsim.ImageRequest("2", airsim.ImageType.Segmentation),
airsim.ImageRequest("3", airsim.ImageType.Scene),
airsim.ImageRequest("4", airsim.ImageType.DisparityNormalized),
airsim.ImageRequest("4", airsim.ImageType.SurfaceNormals)])
for i, response in enumerate(responses):
if response.pixels_as_float:
print("Type %d, size %d, pos %s" % (response.image_type, len(response.image_data_float), pprint.pformat(response.camera_position)))
airsim.write_pfm(os.path.normpath('/temp/cv_mode_' + str(x) + "_" + str(i) + '.pfm'), airsim.get_pfm_array(response))
else:
print("Type %d, size %d, pos %s" % (response.image_type, len(response.image_data_uint8), pprint.pformat(response.camera_position)))
airsim.write_file(os.path.normpath('/temp/cv_mode_' + str(x) + "_" + str(i) + '.png'), response.image_data_uint8)
pose = client.simGetVehiclePose()
pp.pprint(pose)
time.sleep(3)
# currently reset() doesn't work in CV mode. Below is the workaround
client.simSetVehiclePose(airsim.Pose(airsim.Vector3r(0, 0, 0), airsim.to_quaternion(0, 0, 0)), True)
time.sleep(0.1)
responses = client.simGetImages([
airsim.ImageRequest("0", airsim.ImageType.Scene),
airsim.ImageRequest("1", airsim.ImageType.Scene),
airsim.ImageRequest("2", airsim.ImageType.Scene)
])
for i, response in enumerate(responses):
if response.pixels_as_float:
print("Type %d, size %d, pos %s" %
(response.image_type, len(response.image_data_float),
pprint.pformat(response.camera_position)))
airsim.write_pfm(
os.path.normpath(
os.path.join(tmp_dir,
str(x) + "_" + str(i) + '.pfm')),
airsim.get_pfm_array(response))
else:
print("Type %d, size %d, pos %s" %
(response.image_type, len(response.image_data_uint8),
pprint.pformat(response.camera_position)))
airsim.write_file(
os.path.normpath(
os.path.join(tmp_dir, str(i),
str(x) + "_" + str(i) + '.png')),
response.image_data_uint8)
pose = client.simGetVehiclePose()
pp.pprint(pose)
client = airsim.MultirotorClient()
client.confirmConnection()
client.enableApiControl(True)
client.armDisarm(True)
# Async methods returns Future. Call join() to wait for task to complete.
client.takeoffAsync().join()
client.moveToPositionAsync(-10, 10, -10, 5).join()
# take images
responses = client.simGetImages([
airsim.ImageRequest("0", airsim.ImageType.DepthVis),
airsim.ImageRequest("1", airsim.ImageType.DepthPlanner, True)
])
print('Retrieved images: %d', len(responses))
# do something with the images
for response in responses:
if response.pixels_as_float:
print("Type %d, size %d" %
(response.image_type, len(response.image_data_float)))
# airsim.write_pfm(os.path.normpath('/temp/py1.pfm'), airsim.get_pfm_array(response))
airsim.write_pfm(os.path.normpath('py1.pfm'),
airsim.get_pfm_array(response))
else:
print("Type %d, size %d" %
(response.image_type, len(response.image_data_uint8)))
# airsim.write_file(os.path.normpath('/temp/py1.png'), response.image_data_uint8)
airsim.write_file(os.path.normpath('py1.png'),
response.image_data_uint8)
airsim.ImageRequest("1", airsim.ImageType.DepthPerspective,
True), # depth in perspective projection
airsim.ImageRequest(
"1",
airsim.ImageType.Scene), # scene vision image in png format
airsim.ImageRequest(
"1", airsim.ImageType.Scene, False,
False), # scene vision image in uncompressed RGBA array
airsim.ImageRequest(1, airsim.ImageType.DepthPlanner, True)
], )
print('Retrieved images: %d', len(responses))
# do something with images
for response in responses:
if response.pixels_as_float:
print("Type %d, size %d" %
(response.image_type, len(response.image_data_float)))
airsim.write_pfm('py1.pfm', airsim.get_pfm_array(response))
else:
print("Type %d, size %d" %
(response.image_type, len(response.image_data_uint8)))
airsim.write_file('py1.png', response.image_data_uint8)
print("Type %d, size %d" %
(response.image_type, len(response.image_data_uint8)))
img1d = np.fromstring(response.image_data_uint8,
dtype=np.uint8) # get numpy array
img_rgb = img1d.reshape(
response.height, response.width,
3) # reshape array to 4 channel image array H X W X 3
cv2.imwrite(os.path.normpath("car_img" + '.png'),
img_rgb) # write to png
def record(client, start_pos, end_pos, start_quat, end_quat, N):
# generate positions
pos_diff = end_pos - start_pos
pos_diff = np.repeat(np.reshape(pos_diff, [1, 3]), N, 0)
inc = np.reshape(np.linspace(0, 1, num=N), [N, 1])
pos_diff *= inc
start_pos = np.repeat(np.reshape(start_pos, [1, 3]), N, 0)
pos = start_pos + pos_diff
# generate orientations
qs = Quaternion.intermediates(start_quat,
end_quat,
N,
include_endpoints=False)
# combine
for i, q in enumerate(qs):
print(pos[i, :], q.rotation_matrix)
responses = get_image_and_depth(client, pos[i, :], q)
for j, response in enumerate(responses):
if response.pixels_as_float:
airsim.write_pfm(
os.path.normpath('depth{:07d}.pfm'.format(i + 1)),
airsim.get_pfm_array(response))
else:
airsim.write_file(
os.path.normpath('image{:07d}.png'.format(i + 1)),
response.image_data_uint8)
R = q.rotation_matrix
rx = R[1, :]
ry = -R[2, :]
rz = -R[0, :]
def rotm(rx, ry, rz):
return np.array([-rz, rx, -ry])
savemat("pose{:07d}.mat".format(i + 1), {'C': pos[i, :], 'R': R})
thetas = [-20, -10, 10, 20]
for theta in thetas:
th = np.radians(theta)
R = rotm(rx * cos(th) + ry * sin(th), -rx * sin(th) + ry * cos(th),
rz)
response = get_image(client, pos[i, :], Quaternion(matrix=R))
airsim.write_file(
os.path.normpath('image{:07d}_rotz{}.png'.format(i + 1,
theta)),
response.image_data_uint8)
savemat("pose{:07d}_rotz{}.mat".format(i + 1, theta), {
'C': pos[i, :],
'R': R
})
R = rotm(-rz * sin(th) + rx * cos(th), ry,
rz * cos(th) + rx * sin(th))
response = get_image(client, pos[i, :], Quaternion(matrix=R))
airsim.write_file(
os.path.normpath('image{:07d}_roty{}.png'.format(i + 1,
theta)),
response.image_data_uint8)
savemat("pose{:07d}_roty{}.mat".format(i + 1, theta), {
'C': pos[i, :],
'R': R
})
R = rotm(rx,
ry * cos(th) + rz * sin(th), -ry * sin(th) + rz * cos(th))
response = get_image(client, pos[i, :], Quaternion(matrix=R))
airsim.write_file(
os.path.normpath('image{:07d}_rotx{}.png'.format(i + 1,
theta)),
response.image_data_uint8)
savemat("pose{:07d}_rotx{}.mat".format(i + 1, theta), {
'C': pos[i, :],
'R': R
})
def collect_data(x_increment, y_increment, z_increment, rot_increment):
"""
params:
...
rot_increment = number of increments as camera rotates
around yaw axis in a fixed spot
"""
# get point data
pts = load_points(x_increment, y_increment, z_increment)
rot_coords = np.linspace(0, 2 * np.pi, num=rot_increment)
for [x, y, z] in pts:
for yaw in rot_coords:
# TODO: figure out what second param does
client.simSetVehiclePose(
airsim.Pose(airsim.Vector3r(x, y, z),
airsim.to_quaternion(0, 0, yaw)),
True) # PRY in radians
time.sleep(0.2)
# save data with desired channel
if segmentation:
success = client.simSetSegmentationObjectID(object_name, 20)
responses = client.simGetImages([
airsim.ImageRequest(
"0", airsim.ImageType.Scene), # REGULAR PICTURE
airsim.ImageRequest("0", airsim.ImageType.Segmentation,
False, False)
])
else:
responses = client.simGetImages(
[airsim.ImageRequest("0", airsim.ImageType.Scene)])
if not demo:
for i, response in enumerate(responses):
# save images to file
if response.pixels_as_float:
print("Type %d, size %d, pos %s" %
(response.image_type,
len(response.image_data_float),
pprint.pformat(response.camera_position)))
airsim.write_pfm(
os.path.normpath(
os.path.join(
tmp_dir, "Segmentation",
str(round(x, 2)) + "_" + str(round(y, 2)) +
"_" + str(round(z, 2)) + "_" +
str(round(yaw, 2)) + "_" + str(i) +
'.pfm')), airsim.get_pfm_array(response))
elif response.compress: # png format
print("Type %d, size %d, pos %s" %
(response.image_type,
len(response.image_data_uint8),
pprint.pformat(response.camera_position)))
airsim.write_file(
os.path.normpath(
os.path.join(
tmp_dir, "Normal",
str(round(x, 2)) + "_" + str(round(y, 2)) +
"_" + str(round(z, 2)) + "_" +
str(round(yaw, 2)) + "_" + str(i) +
'.png')), response.image_data_uint8)
else: # uncompressed array - numpy demo
print("Type %d, size %d" %
(response.image_type,
len(response.image_data_uint8)))
img1d = np.fromstring(
response.image_data_uint8,
dtype=np.uint8) # get numpy array
img_rgb = img1d.reshape(
response.height, response.width, 3
) # reshape array to 3 channel image array H X W X 3
cv2.imwrite(
os.path.join(
tmp_dir, "Segmentation",
str(round(x, 2)) + "_" + str(round(y, 2)) +
"_" + str(round(z, 2)) + "_" +
str(round(yaw, 2)) + "_" + str(i) + '.pfm'),
img_rgb) # write to png
if i == 1:
pp.pprint(client.simGetVehiclePose().position.x_val)
drone_state = client.simGetVehiclePose
# log relevent state information
pitchRollYaw = airsim.utils.to_eularian_angles(
drone_state().orientation)
with open(os.path.join(tmp_dir, file_name), 'a') as f:
f.write("{},{},{},{},{},{},{},{},{}\n".format(
drone_state().position.x_val,
drone_state().position.y_val,
drone_state().position.z_val,
drone_state().position.x_val -
unreal_object[0],
drone_state().position.y_val -
unreal_object[1],
drone_state().position.z_val -
unreal_object[2], pitchRollYaw[0],
pitchRollYaw[1], pitchRollYaw[2]))
# sanity check
pose = client.simGetVehiclePose()
pp.pprint(pose)
def get_camtrap_images(env_num, animal_class, num_animals):
output_folder = r"C:\Users\t-sabeer\Documents\AirSimImages\\"
pp = pprint.PrettyPrinter(indent=4)
client = connect_to_airsim()
object_lookup = set_segmentation_ids(client, animal_class, num_animals)
time.sleep(5) #env needs time to generate
camera_position_list = [(-45, -45, 0, -.1, 0.8), (-45, 0, 0, -.1, -.7),
(0, -45, 0, -.1, 2.0), (0, 0, 0, -.1, -2.3)]
for cam_num, cam_pos in enumerate(camera_position_list):
client.simSetVehiclePose(
airsim.Pose(airsim.Vector3r(cam_pos[0], cam_pos[1], cam_pos[2]),
airsim.to_quaternion(cam_pos[3], 0, cam_pos[4])), True)
#client.simSetCameraOrientation("0", airsim.to_quaternion(-0.161799, 0, 0)); #radians
#pose = client.simGetVehiclePose()
#pp.pprint(pose)
#print('Pose ' + str(cam_num))
#print(pose)
time.sleep(1) #new camera position needs time to update
for x in range(3): # create sequence of 3
responses = client.simGetImages([
airsim.ImageRequest("0", airsim.ImageType.Scene),
airsim.ImageRequest("0", airsim.ImageType.Segmentation, False,
False)
])
for i, response in enumerate(responses):
if response.pixels_as_float:
#print("Type %d, size %d, pos %s" % (response.image_type, len(response.image_data_float), pprint.pformat(response.camera_position)))
airsim.write_pfm(
os.path.normpath(output_folder + str(env_num) +
'_cam_' + str(cam_num) + '_frame_' +
str(x) + "_" + str(i) + '.pfm'),
airsim.get_pfm_array(response))
if response.compress:
#print("Type %d, size %d, pos %s" % (response.image_type, len(response.image_data_uint8), pprint.pformat(response.camera_position)))
airsim.write_file(
os.path.normpath(output_folder + str(env_num) +
'_cam_' + str(cam_num) + '_frame_' +
str(x) + "_" + str(i) + '.jpg'),
response.image_data_uint8)
else:
img = np.fromstring(response.image_data_uint8,
dtype=np.uint8) #get numpy array
img_rgba = img.reshape(response.height, response.width, 4)
img_rgba = np.flipud(
img_rgba) #original image is flipped vertically
airsim.write_png(
os.path.normpath(output_folder + str(env_num) +
'_cam_' + str(cam_num) + '_frame_' +
str(x) + "_" + str(i) + '.png'),
img_rgba) #write to png
time.sleep(1) #1fps frame rate
#pose = client.simGetVehiclePose()
#pp.pprint(pose)
return
# get camera images from the car
responses = client.simGetImages([
airsim.ImageRequest("0", airsim.ImageType.DepthVis), #depth visualization image
airsim.ImageRequest("1", airsim.ImageType.DepthPerspective, True), #depth in perspective projection
airsim.ImageRequest("1", airsim.ImageType.Scene), #scene vision image in png format
airsim.ImageRequest("1", airsim.ImageType.Scene, False, False)]) #scene vision image in uncompressed RGBA array
print('Retrieved images: %d', len(responses))
for response in responses:
filename = 'c:/temp/py' + str(idx)
if not os.path.exists('c:/temp/'):
os.makedirs('c:/temp/')
if response.pixels_as_float:
print("Type %d, size %d" % (response.image_type, len(response.image_data_float)))
airsim.write_pfm(os.path.normpath(filename + '.pfm'), airsim.get_pfm_array(response))
elif response.compress: #png format
print("Type %d, size %d" % (response.image_type, len(response.image_data_uint8)))
airsim.write_file(os.path.normpath(filename + '.png'), response.image_data_uint8)
else: #uncompressed array
print("Type %d, size %d" % (response.image_type, len(response.image_data_uint8)))
img1d = np.fromstring(response.image_data_uint8, dtype=np.uint8) #get numpy array
img_rgba = img1d.reshape(response.height, response.width, 4) #reshape array to 4 channel image array H X W X 4
img_rgba = np.flipud(img_rgba) #original image is flipped vertically
img_rgba[:,:,1:2] = 100 #just for fun add little bit of green in all pixels
airsim.write_png(os.path.normpath(filename + '.greener.png'), img_rgba) #write to png
#restore to original state
client.reset()
success = client.simSetSegmentationObjectID(object_name, 0)
responses = client.simGetImages([
airsim.ImageRequest("0", airsim.ImageType.Scene), # REGULAR PICTURE
airsim.ImageRequest("0", airsim.I mageType.Segmentation, False, False)])
else:
responses = client.simGetImages([
airsim.ImageRequest("0", airsim.ImageType.Scene)])
if not demo:
for i, response in enumerate(responses):
# Save images to file
if response.pixels_as_float:
print("Type %d, size %d, pos %s" % (response.image_type, len(response.image_data_float), pprint.pformat(response.camera_position)))
airsim.write_pfm(os.path.normpath(os.path.join(tmp_dir, "Segmentation", str(round(x,2)) + "_" + str(round(y,2)) + "_" + str(round(z,2))+ "_" + str(i) + '.pfm')), airsim.get_pfm_array(response))
elif response.compress: #png format
print("Type %d, size %d, pos %s" % (response.image_type, len(response.image_data_uint8), pprint.pformat(response.camera_position)))
airsim.write_file(os.path.normpath(os.path.join(tmp_dir, "Normal", str(round(x,2)) + "_" + str(round(y,2)) + "_" + str(round(z,2))+ "_" + str(i) + '.png')), response.image_data_uint8)
else: #uncompressed array - numpy demo
print("Type %d, size %d" % (response.image_type, len(response.image_data_uint8)))
img1d = np.fromstring(response.image_data_uint8, dtype=np.uint8) #get numpy array
img_rgb = img1d.reshape(response.height, response.width, 3) #reshape array to 3 channel image array H X W X 3
cv2.imwrite(os.path.join(tmp_dir, "Segmentation", str(round(x,2)) + "_" + str(round(y,2)) + "_" + str(round(z,2))+ "_" + str(i) + '.pfm'), img_rgb) # write to png
if i == 1:
pp.pprint(client.simGetVehiclePose().position.x_val)
drone_state = client.simGetVehiclePose
# Log relevent state information
pitchRollYaw = airsim.utils.to_eularian_angles(drone_state().orientation)
def main():
# create data directory
data_directory = os.path.join(
DATA_PATH, datetime.datetime.now().strftime("%Y-%m-%d-%H-%M"),
)
if not os.path.exists(data_directory):
os.makedirs(data_directory)
# connect to the AirSim simulator
client = airsim.MultirotorClient()
client.confirmConnection()
# make the multirotor take off
client.enableApiControl(True)
client.armDisarm(True)
client.takeoffAsync().join()
try:
# initialize collision record
collision_record = []
while len(collision_record) < COLLISION_COUNT:
# initialize flying position and orientation
while True:
position = [
random.uniform(start_range[0], start_range[1])
for start_range in FLYING_START
]
orientation = random.uniform(-math.pi, math.pi)
client.simSetVehiclePose(
airsim.Pose(
airsim.Vector3r(*position),
airsim.Quaternionr(
0, 0, math.sin(orientation / 2), math.cos(orientation / 2)
),
),
ignore_collison=True,
)
if not client.simGetCollisionInfo().has_collided:
break
# initialize flying deadline
deadline = datetime.datetime.now() + datetime.timedelta(
seconds=FLYING_DURATION
)
# initialize image queue
frame_queue = collections.deque(maxlen=COLLISION_FRAME)
# start flying
client.moveByVelocityZAsync(
vx=FLYING_SPEED * math.cos(orientation),
vy=FLYING_SPEED * math.sin(orientation),
z=position[2],
drivetrain=airsim.DrivetrainType.ForwardOnly,
duration=FLYING_DURATION,
)
while True:
# gather images from the camera
responses = client.simGetImages(
[
airsim.ImageRequest(
"front_center",
airsim.ImageType.Scene,
pixels_as_float=False,
),
airsim.ImageRequest(
"front_center",
airsim.ImageType.DepthPlanner,
pixels_as_float=True,
),
airsim.ImageRequest(
"front_center",
airsim.ImageType.DepthPerspective,
pixels_as_float=True,
),
airsim.ImageRequest(
"front_center",
airsim.ImageType.Segmentation,
pixels_as_float=False,
),
]
)
# push images into queue
frame_queue.appendleft(
{
"scene": responses[0].image_data_uint8,
"depth_planner": airsim.get_pfm_array(responses[1]),
"depth_perspective": airsim.get_pfm_array(responses[2]),
"segmentation": responses[3].image_data_uint8,
}
)
# check if a collision occured
collision_info = client.simGetCollisionInfo()
if collision_info.has_collided:
# check if the number of pictures is enough
if len(frame_queue) < COLLISION_FRAME:
print(
f"not enough picture before this collision: "
f"{len(frame_queue)}"
)
break
# save the collision info with custom timestamp
timestamp = str(
datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
)
collision_record.append(
{
"normal": collision_info.normal.to_numpy_array().tolist(),
"impact_point": collision_info.impact_point.to_numpy_array().tolist(),
"position": collision_info.position.to_numpy_array().tolist(),
"penetration_depth": collision_info.penetration_depth,
"time_stamp": timestamp,
"object_name": collision_info.object_name,
"object_id": collision_info.object_id,
}
)
with open(
os.path.join(data_directory, "collision_record.json"), "w"
) as record_file:
json.dump(collision_record, record_file)
# create image directories
scene_directory = os.path.join(data_directory, "scene", timestamp)
depth_planner_directory = os.path.join(
data_directory, "depth_planner", timestamp
)
depth_perspective_directory = os.path.join(
data_directory, "depth_perspective", timestamp
)
segmentation_directory = os.path.join(
data_directory, "segmentation", timestamp
)
os.makedirs(scene_directory)
os.makedirs(depth_planner_directory)
os.makedirs(depth_perspective_directory)
os.makedirs(segmentation_directory)
# save the captured images
frame_index = 1
while len(frame_queue) > 0:
frame = frame_queue.pop()
base_name = str(frame_index).zfill(3)
airsim.write_file(
os.path.join(scene_directory, base_name + ".png"),
frame["scene"],
)
airsim.write_pfm(
os.path.join(depth_planner_directory, base_name + ".pfm"),
frame["depth_planner"],
)
airsim.write_pfm(
os.path.join(depth_planner_directory, base_name + ".pfm"),
frame["depth_perspective"],
)
airsim.write_file(
os.path.join(segmentation_directory, base_name + ".png"),
frame["segmentation"],
)
frame_index += 1
# finish this collision
print(f"collision record #{len(collision_record)} is saved")
break
elif datetime.datetime.now() > deadline:
print("overdue the deadline")
break
print(f"{COLLISION_COUNT} collision datasets are saved.")
print("returning to origin state...")
time.sleep(5)
except:
logging.exception("")
client.reset()
for pitch in range(-5, 5, 5):
for yaw in range(-10, 10, 2):
q = airsim.to_quaternion(pitch/10.0, 0, yaw/10.0)
client.simCharSetHeadRotation(q)
time.sleep(0.1)
airsim.wait_key('Press any key to get images')
for x in range(3): # do few times
responses = client.simGetImages([
airsim.ImageRequest("0", airsim.ImageType.DepthVis),
airsim.ImageRequest("0", airsim.ImageType.Segmentation),
airsim.ImageRequest("0", airsim.ImageType.Scene),
airsim.ImageRequest("0", airsim.ImageType.SurfaceNormals)])
for i, response in enumerate(responses):
if response.pixels_as_float:
print("Type %d, size %d, pos %s" % (response.image_type, len(response.image_data_float), pprint.pformat(response.camera_position)))
airsim.write_pfm(os.path.normpath('/temp/cv_mode_' + str(x) + "_" + str(i) + '.pfm'), airsim.get_pfm_array(response))
else:
print("Type %d, size %d, pos %s" % (response.image_type, len(response.image_data_uint8), pprint.pformat(response.camera_position)))
airsim.write_file(os.path.normpath('/temp/cv_mode_' + str(x) + "_" + str(i) + '.png'), response.image_data_uint8)
pose = client.simGetVehiclePose()
pose.position.x_val = pose.position.x_val + 1
pose.position.y_val = pose.position.y_val - 0.5
pose.position.z_val = pose.position.z_val - 0.5
client.simSetVehiclePose(pose, True)
time.sleep(3)
client.reset()
responses = client.simGetImages(generateImageRequestList(camera_name='0'))
print('Retrieved images: %d' % len(responses))
tmp_dir = os.path.join('.', "airsim_drone")
try:
os.makedirs(tmp_dir)
except OSError:
if not os.path.isdir(tmp_dir):
raise
for idx, response in enumerate(responses):
filename = os.path.join(tmp_dir, str(idx))
if response.pixels_as_float:
print("Type %d, size %d" %
(response.image_type, len(response.image_data_float)))
airsim.write_pfm(os.path.normpath(filename + '.pfm'),
airsim.get_pfm_array(response))
elif response.compress: #png format
print("Type %d, size %d" %
(response.image_type, len(response.image_data_uint8)))
airsim.write_file(os.path.normpath(filename + '.png'),
response.image_data_uint8)
else: #uncompressed array
print("Type %d, size %d" %
(response.image_type, len(response.image_data_uint8)))
img1d = np.fromstring(response.image_data_uint8,
dtype=np.uint8) # get numpy array
img_rgb = img1d.reshape(
response.height, response.width,
3) # reshape array to 4 channel image array H X W X 3
cv2.imwrite(os.path.normpath(filename + '.png'),
img_rgb) # write to png
# actually, 3 = scene, 5 = depth, 0 = segmentation
responses = client.simGetImages([
airsim.ImageRequest(0, 3),
airsim.ImageRequest(0, 5),
airsim.ImageRequest(0, 0),
])
'''
responses = client.simGetImages([
airsim.ImageRequest(0, airsim.ImageType.Scene),
airsim.ImageRequest(0, airsim.ImageType.DepthPlanner),
airsim.ImageRequest(0, airsim.ImageType.DepthPerspective),
airsim.ImageRequest(0, airsim.ImageType.DepthVis),
airsim.ImageRequest(0, airsim.ImageType.DisparityNormalized),
airsim.ImageRequest(0, airsim.ImageType.Segmentation),
airsim.ImageRequest(0, airsim.ImageType.SurfaceNormals),
airsim.ImageRequest(0, airsim.ImageType.Infrared),
])
'''
for response in responses:
if response.pixels_as_float:
print("Type %d, size %d" %
(response.image_type, len(response.image_data_float)))
airsim.write_pfm('py %d.pfm' % response.image_type,
airsim.get_pfm_array(response))
else:
print("Type %d, size %d" %
(response.image_type, len(response.image_data_uint8)))
airsim.write_file('py%d.png' % response.image_type,
response.image_data_uint8)