def take_snapshot(self):
if not os.path.exists(self.image_dir):
os.makedirs(self.image_dir)
# first hold our current position so drone doesn't try and keep flying while we take the picture.
pos = self.client.getMultirotorState(
vehicle_name=self.id).kinematics_estimated.position
self.client.moveToPositionAsync(
pos.x_val,
pos.y_val,
self.z,
0.25,
3,
airsim.DrivetrainType.MaxDegreeOfFreedom,
airsim.YawMode(False, self.camera_heading),
vehicle_name=self.id)
responses = self.client.simGetImages(
[airsim.ImageRequest(0, airsim.ImageType.Scene)],
vehicle_name=self.id) # scene vision image in png format
response = responses[0]
filename = self.photo_prefix + \
str(self.snapshot_index) + "_" + str(int(time.time()))
self.snapshot_index += 1
airsim.write_file(os.path.normpath(self.image_dir + filename + '.png'),
response.image_data_uint8)
print("Saved snapshot: {}".format(filename))
# cause smooth ramp up to happen again after photo is taken.
self.start_time = time.time()
def take_snapshot(self) -> None:
if not os.path.exists(self.image_dir):
os.makedirs(self.image_dir)
# first hold our current position so drone doesn't try and keep flying while we take the picture
pos = self.client.getMultirotorState().kinematics_estimated.position
self.client.moveToPositionAsync(
pos.x_val,
pos.y_val,
self.z,
velocity=0.25,
timeout_sec=3,
drivetrain=airsim.DrivetrainType.MaxDegreeOfFreedom,
yaw_mode=airsim.YawMode(is_rate=False,
yaw_or_rate=self.camera_heading))
png_image = self.client.simGetImage("0", airsim.ImageType.Scene)
filename = "{}{}_{:.0f}".format(self.photo_prefix, self.snapshot_index,
time.time())
self.snapshot_index += 1
airsim.write_file(os.path.normpath(
os.path.join(self.image_dir, filename + '.png')),
bstr=png_image)
print("Saved snapshot: {}".format(filename))
# cause smooth ramp up to happen again after photo is taken
self.start_time = time.time()
def _take_pictures_loop(client):
print("[ff] Press [space] to take pictures (or any other key to stop)")
record = ""
img_count = 0
while True:
if msvcrt.kbhit():
if msvcrt.getch() != b" ":
break # https://stackoverflow.com/a/13207813
response, *_ = client.simGetImages(
[airsim.ImageRequest(ff.CameraName.front_center, airsim.ImageType.Scene)]
)
img_count += 1
print(f" {img_count} pictures taken", end="\r")
timestamp, position, orientation = get_record_line_from(response)
tx, ty, tz = position
qx, qy, qz, qw = orientation
record += "\n" + " ".join([str(_) for _ in [timestamp, tx, ty, tz, qx, qy, qz, qw]])
airsim.write_file(
os.path.join(args.output_folder, f"out_{timestamp}.png"),
response.image_data_uint8,
)
print("***\n" + record + "\n***")
print()
def checkSafe(drone,yawVal,droneNetwork):
imagequeue = []
tmp_dir = r"C:\Users\Ernest\Pictures\DronePic"
drone.moveByAngleZAsync(0,0,-1,yaw=yawVal,duration=1)
time.sleep(4)
responses = drone.simGetImages([airsim.ImageRequest(0,airsim.ImageType.Scene)])
imagequeue.append(responses[0].image_data_uint8)
filename = os.path.join(tmp_dir, "current")
airsim.write_file(os.path.normpath(filename + '.png'),imagequeue[0])
img_addr = r"C:\Users\Ernest\Pictures\DronePic\current.png"
img = image.load_img(img_addr, target_size = (80,80))
img_test = image.img_to_array(img)
img_test = np.expand_dims(img_test, axis=0)
img_test = preprocess_input(img_test)
result = droneNetwork.predict(img_test)
if result == [[1.]]:
return True
else:
return False
"""yah = input(" :").lower()
def captureImages(dataPath, ctr, filename):
# Check if the file already exists or not
# If it exists then skip the retrieval of responses from the simulator
# NOTE: Should only be used only when dataset collection has to be resumed
# flag = 0 means that the files are not present/missing, hence data has to be recorded
flag = 0
for i in range(0, len(dataPath)):
filepath = dataPath[i] + '/' + filename
if (os.path.isfile(os.path.normpath(filepath + '.png')) or os.path.isfile(os.path.normpath(filepath + '.pfm'))):
flag = 1
print("File exists")
break
else:
flag = 0
if (flag == 0):
# Retrive responses from AirSim client
responses = client.simGetImages([
airsim.ImageRequest("front_left", airsim.ImageType.Scene),
airsim.ImageRequest("front_right", airsim.ImageType.Scene),
airsim.ImageRequest("front_left", airsim.ImageType.DepthPlanner, True),
airsim.ImageRequest("front_right", airsim.ImageType.DepthPlanner, True)])
for i, response in enumerate(responses):
if response.pixels_as_float:
depth_raw = airsim.get_pfm_array(response)
save_depth_float_as_uchar16(os.path.normpath(dataPath[i] + '/' + filename + '.png'), depth_raw)
else:
airsim.write_file(os.path.normpath(dataPath[i] + '/' + filename + '.png'), response.image_data_uint8)
def do_screenshot(self, is_display=False):
""" get camera images from the car
Returns:
image_paths (list): filepaths of the screenshots
"""
responses = self.client.simGetImages([
airsim.ImageRequest("front_center", airsim.ImageType.Scene),
airsim.ImageRequest("bottom_center", airsim.ImageType.Scene),
])
log.debug('Retrieved images: %d' % len(responses))
filename_prefix = self.image_folder + "{}-".format(self.image_id)
self.image_id += 1
image_paths = []
for idx, response in enumerate(responses):
filename = filename_prefix + str(idx)
if response.compress: #png format
log.debug('image type {}, size {}'.format(
response.image_type, len(response.image_data_uint8)))
filename = os.path.normpath(filename + '.png')
airsim.write_file(filename, response.image_data_uint8)
else:
log.error('error: image format not support')
image_paths.append(filename)
if is_display:
self.display(image_paths)
return image_paths
def save_image(client, tmp_dir, count, id):
vehicle_name = "Drone" + str(id)
responses = client.simGetImages ([airsim.ImageRequest ("1", airsim.ImageType.Scene, False, False)], \
vehicle_name=vehicle_name)
filename = os.path.join(tmp_dir, str(count))
for idx, response in enumerate(responses):
if response.pixels_as_float:
print("Type %d, size %d" %
(response.image_type, len(response.image_data_float)))
airsim.write_pfm(os.path.normpath(filename + '.pfm'),
airsim.get_pfm_array(response))
elif response.compress: # png format
print("Type %d, size %d" %
(response.image_type, len(response.image_data_uint8)))
airsim.write_file(os.path.normpath(filename + '.png'),
response.image_data_uint8)
else: # uncompressed array
print("Type %d, size %d" %
(response.image_type, len(response.image_data_uint8)))
img1d = np.fromstring(response.image_data_uint8,
dtype=np.uint8) # get numpy array
img_rgb = img1d.reshape(
response.height, response.width,
3) # reshape array to 4 channel image array H X W X 3
cv2.imwrite(os.path.normpath(filename + '.jpg'),
img_rgb) # write to png
save_imu_pose(client, filename, id)
def log_responses(responses, idx):
for response_idx, response in enumerate(responses):
filename = os.path.join(
tmp_dir, "{}_{}_{}".format(idx, response.image_type, response_idx))
if response.pixels_as_float:
print("Type %d, size %d" %
(response.image_type, len(response.image_data_float)))
airsim.write_pfm(os.path.normpath(filename + '.pfm'),
airsim.get_pfm_array(response))
elif response.compress: #png format
print("Type %d, size %d" %
(response.image_type, len(response.image_data_uint8)))
airsim.write_file(os.path.normpath(filename + '.png'),
response.image_data_uint8)
else: #uncompressed array
print("Type %d, size %d" %
(response.image_type, len(response.image_data_uint8)))
img1d = np.fromstring(response.image_data_uint8,
dtype=np.uint8) # get numpy array
img_rgb = img1d.reshape(
response.height, response.width,
3) # reshape array to 3 channel image array H X W X 3
cv2.imwrite(os.path.normpath(filename + '.png'),
img_rgb) # write to png
def genImg(self, im_num, vehicle_name):
responses = self.getImg(vehicle_name)
# pprint.pprint(responses[0].camera_position)
# pprint.pprint(responses[0].camera_orientation)
filename_img = self.cam_folder + ("%.6d.png" % im_num)
filename_ts = self.pose_folder + ("cam_ts%.6d.txt" % im_num)
for idx, response in enumerate(responses):
if self.compress_img: #png format
airsim.write_file(os.path.normpath(filename_img),
response.image_data_uint8)
else: #uncompressed array
img1d = np.fromstring(response.image_data_uint8,
dtype=np.uint8)
img_rgb = img1d.reshape(response.height, response.width, 3)
cv2.imwrite(os.path.normpath(filename_img), img_rgb)
output = open(os.path.normpath(filename_ts), 'wb')
pickle.dump(response.time_stamp, output)
output.close()
camera_info = self.client.simGetCameraInfo("0",
vehicle_name=vehicle_name)
filename_cam_pose = self.pose_folder + vehicle_name + (
"campose%.6d.txt" % im_num)
op1 = open(os.path.normpath(filename_cam_pose), 'wb')
pickle.dump(camera_info, op1)
op1.close()
return camera_info
def saveImage(response, filename):
if response.pixels_as_float:
print(
f"""Type {response.image_type}, size {len(response.image_data_float)}, "
height {response.height}, width {response.width}""")
# airsim.write_pfm(os.path.normpath(filename + '.pfm'), airsim.get_pfm_array(response))
depth = np.array(response.image_data_float, dtype=np.float64)
depth = depth.reshape(response.height, response.width, -1)
depth = np.array(depth * 255, dtype=np.uint8)
# save pic
cv2.imwrite('{}.png'.format(filename), depth)
elif response.compress: #png format
print("Type %d, size %d, height %d, width %d" %
(response.image_type, len(
response.image_data_uint8), response.height, response.width))
airsim.write_file(os.path.normpath(filename + '.png'),
response.image_data_uint8)
else: #uncompressed array
print(
f"""Type {response.image_type}, size {len(response.image_data_uint8)}, "
height {response.height}, width {response.width}""")
img1d = np.fromstring(response.image_data_uint8,
dtype=np.uint8) # get numpy array
img_rgb = img1d.reshape(
response.height, response.width,
3) # reshape array to 3 channel image array H X W X 3
cv2.imwrite(os.path.normpath(filename + '.png'),
img_rgb) # write to png
def grab_images(self,idx2):
responses = self.client.simGetImages([
airsim.ImageRequest("0", airsim.ImageType.DepthVis), #depth visualization image
airsim.ImageRequest("1", airsim.ImageType.DisparityNormalized, True), #depth in perspective projection
#airsim.ImageRequest("1", airsim.ImageType.DepthPerspective, True), #depth in perspective projection
airsim.ImageRequest("1", airsim.ImageType.Scene), #scene vision image in png format
airsim.ImageRequest("1", airsim.ImageType.Scene, False, False)]) #scene vision image in uncompressed RGBA array
print('Retrieved images: %d', len(responses))
for response in responses:
filename = self.fold+'py' + str(idx2)
if not os.path.exists(self.fold):
os.makedirs(self.fold)
if response.pixels_as_float:
print("Type %d, size %d" % (response.image_type, len(response.image_data_float)))
depth_img=response
dimg_array=np.array(response.image_data_float).reshape(144,256)
dimg2=copy.deepcopy(((dimg_array/(dimg_array.max()))*255).astype(np.uint8))
dimg= Image.fromarray(dimg2)
if dimg.mode != 'RGB':
dimg = dimg.convert('RGB')
print("Saving Depth Image!")
dimg.save(os.path.normpath(filename+'_depth.png'))
dimg = dimg.convert('I')
rawImage = self.client.simGetImage(0, airsim.ImageType.DepthPerspective,"")
#png= np.array(rawImage[0].image_data_float,dtype=np.uint8).reshape(144,256)
png = cv2.imdecode(np.frombuffer(rawImage, np.uint8) , cv2.IMREAD_UNCHANGED)
gray = cv2.cvtColor(png, cv2.COLOR_BGR2GRAY)
Image3D = cv2.reprojectImageTo3D(np.array(gray), self.projectionMatrix)
print("Saving Point Cloud!")
self.savePointCloud(np.array(Image3D), filename+'_cloudpoint.asc')
#airsim.write_file(os.path.normpath(file_name+'_depth.png'),)
elif response.compress: #png format
print("Type %d, size %d" % (response.image_type, len(response.image_data_uint8)))
#print(type(response.image_data_uint8))
print("Saving RGB Image!")
airsim.write_file(os.path.normpath(filename + '.png'), response.image_data_uint8)
img_data=Image.open(io.BytesIO(response.image_data_uint8))
if img_data.mode != 'RGB':
img_data = img_data.convert('RGB')
#img_data.show()
else: #uncompressed array
pass
# print("Type %d, size %d" % (response.image_type, len(response.image_data_uint8)))
# img1d = np.fromstring(response.image_data_uint8, dtype=np.uint8) #get numpy array
# img_rgba = img1d.reshape(response.height, response.width, 4) #reshape array to 4 channel image array H X W X 4
# img_rgba = np.flipud(img_rgba) #original image is flipped vertically
# img_rgba[:,:,1:2] = 100 #just for fun add little bit of green in all pixels
# airsim.write_png(os.path.normpath(filename + '.greener.png'), img_rgba) #write to png
self.img_data=img_data
self.dimg=dimg
def fly(client: airsim.MultirotorClient, args: argparse.Namespace) -> None:
if args.verbose:
print(
f"[ff] HomeGeoPoint: {Vec3.from_GeoPoint(client.getHomeGeoPoint())}\n"
)
print(
f"[ff] VehiclePose.position: {Vec3.from_Vector3r(client.simGetVehiclePose().position)}\n"
)
initial_state = client.getMultirotorState()
if initial_state.landed_state == airsim.LandedState.Landed:
print(f"[ff] Taking off")
client.takeoffAsync(timeout_sec=8).join()
else:
client.hoverAsync().join() # airsim.LandedState.Flying
#__move_on_path(client, args.flight_path, args.flight_velocity)
#__move_on_box(client, z=-20, side=20, velocity=args.flight_velocity)
if not args.use_viewpoints:
future = client.moveOnPathAsync([
airsim.Vector3r(coord.x, coord.y, coord.z)
for coord in args.flight_path
], args.flight_velocity)
else:
import viewpoints
future = client.moveOnPathAsync(
[airsim.Vector3r(*position) for position in viewpoints.Positions],
args.flight_velocity)
print(f"[ff] Press [space] to take pictures")
ch, img_count, img_responses = msvcrt.getch(), 0, []
while ch == b' ':
img_responses.extend(
client.simGetImages([
airsim.ImageRequest(
ff.CameraName.bottom_center,
airsim.ImageType.Scene,
False,
True # compressed PNG image
)
]))
img_count += 1
print(f" {img_count} pictures taken", end="\r")
ch = msvcrt.getch()
print()
print(f"[ff] Waiting for drone to finish path...", end=" ", flush=True)
future.join()
print(f"done.")
for i, response in enumerate(img_responses):
airsim.write_file(os.path.join(args.output_folder, f"out_{i}.png"),
response.image_data_uint8)
time.sleep(4)
print(f"[ff] Drone reset")
client.reset()
def save_image(client, camera_list, iter):
for camera in camera_list:
if not os.path.exists('c:/temp/{}'.format(camera)):
os.makedirs('c:/temp/{}'.format(camera))
#client.simGetImagesでやるとよいかも
for camera in camera_list:
filename = 'c:/temp/{}/{}'.format(camera, str(iter))
png_image = client.simGetImage(camera, airsim.ImageType.Scene)
airsim.write_file(os.path.normpath(filename + ".png"), png_image)
def take_snapshot(self):
# first hold our current position so drone doesn't try and keep flying while we take the picture.
pos = self.client.getMultirotorState().kinematics_estimated.position
self.client.moveToPositionAsync(pos.x_val, pos.y_val, self.z, 0.5, 10, airsim.DrivetrainType.MaxDegreeOfFreedom,
airsim.YawMode(False, self.camera_heading)).join()
responses = self.client.simGetImages([airsim.ImageRequest(1, airsim.ImageType.Scene)]) #scene vision image in png format
response = responses[0]
filename = "photo_" + str(self.snapshot_index)
self.snapshot_index += 1
airsim.write_file(os.path.normpath(filename + '.png'), response.image_data_uint8)
print("Saved snapshot: {}".format(filename))
self.start_time = time.time() # cause smooth ramp up to happen again after photo is taken.
def fly(client: airsim.MultirotorClient, args: argparse.Namespace) -> None:
initial_pose = client.simGetVehiclePose()
initial_state = client.getMultirotorState()
if args.verbose:
ff.print_pose(initial_pose,
to_eularian_angles=airsim.to_eularian_angles)
if initial_state.landed_state == airsim.LandedState.Landed:
print("[ff] Taking off")
client.takeoffAsync(timeout_sec=8).join()
else:
client.hoverAsync().join() # airsim.LandedState.Flying
print("[ff] Flying viewpoints...")
print("[ff] Press [space] to take pictures (or any other key to stop)"
) # TODO
future = _move_by_path(client, args)
img_count = 0
while True:
if msvcrt.kbhit():
if msvcrt.getch() != b" ":
break # https://stackoverflow.com/a/13207813
response, *_ = client.simGetImages([
airsim.ImageRequest(
ff.CameraName.front_center,
airsim.ImageType.Scene,
False,
True, # compressed PNG image
)
])
img_count += 1
print(f" {img_count} pictures taken", end="\r")
airsim.write_file(
os.path.join(args.output_folder, f"out_{img_count}.png"),
response.image_data_uint8,
)
# TODO save poses to .log file
print("camera_position:", response.camera_position) # Vector3r
print("camera_orientation:",
response.camera_orientation) # Quaternionr
print()
print(f"[ff] Waiting for drone to finish path...", end=" ", flush=True)
future.join()
print("done")
time.sleep(4)
client.reset()
print("[ff] Drone reset")
def generate_image(self, v_identifier):
"""
Generate Image
"""
responses = self.client.simGetImages([
airsim.ImageRequest(
"front_center",
airsim.ImageType.Scene) # scene vision image in png format
])
image_filename = os.path.normpath(
os.path.join(self.data_dir, str(v_identifier)) + '.png')
airsim.write_file(image_filename, responses[0].image_data_uint8)
# log_file("Image saved", str(image_filename))
return image_filename
def save_images(responses, prefix=""):
for i, response in enumerate(responses):
filename = os.path.join(tmp_dir, prefix + "_" + str(i))
if response.pixels_as_float:
print(
f"Type {response.image_type}, size {len(response.image_data_float)}, pos {response.camera_position}"
)
airsim.write_pfm(os.path.normpath(filename + '.pfm'),
airsim.get_pfm_array(response))
else:
print(
f"Type {response.image_type}, size {len(response.image_data_uint8)}, pos {response.camera_position}"
)
airsim.write_file(os.path.normpath(filename + '.png'),
response.image_data_uint8)
def take_snapshot(self):
# first hold our current position so drone doesn't try and keep flying while we take the picture.
pos = self.client.getMultirotorState(self.name).kinematics_estimated.position
self.client.moveToPositionAsync(pos.x_val, pos.y_val, self.z,
velocity=0.5, timeout_sec=10,
drivetrain=airsim.DrivetrainType.MaxDegreeOfFreedom,
yaw_mode=airsim.YawMode(False, self.camera_heading),
vehicle_name=self.name).join()
responses = self.client.simGetImages([airsim.ImageRequest(1, airsim.ImageType.Scene)],
self.name) #scene vision image in png format
response = responses[0]
filename = f"photo_{self.snapshot_index}_{self.name}"
self.snapshot_index += 1
airsim.write_file(os.path.normpath(filename + '.png'), response.image_data_uint8)
print("Saved snapshot: {}".format(filename))
self.start_time = time.time() # cause smooth ramp up to happen again after photo is taken.
def getDroneImages(client):
# get camera images from the car
responses = client.simGetImages([
airsim.ImageRequest(
"0", airsim.ImageType.DepthVis), #depth visualization image
airsim.ImageRequest("1", airsim.ImageType.DepthPerspective,
True), #depth in perspective projection
airsim.ImageRequest(
"1", airsim.ImageType.Scene), #scene vision image in png format
airsim.ImageRequest("1", airsim.ImageType.Scene, False, False)
]) #scene vision image in uncompressed RGBA array
print('Retrieved images: %d' % len(responses))
tmp_dir = os.path.join(tempfile.gettempdir(), "airsim_drone")
print("Saving images to %s" % tmp_dir)
try:
os.makedirs(tmp_dir)
except OSError:
if not os.path.isdir(tmp_dir):
raise
for idx, response in enumerate(responses):
filename = os.path.join(tmp_dir, str(str(idx) + random_generator(4)))
if response.pixels_as_float:
print("Type %d, size %d" %
(response.image_type, len(response.image_data_float)))
airsim.write_pfm(os.path.normpath(filename + '.pfm'),
airsim.get_pfm_array(response))
elif response.compress: #png format
print("Type %d, size %d" %
(response.image_type, len(response.image_data_uint8)))
airsim.write_file(os.path.normpath(filename + '.png'),
response.image_data_uint8)
else: #uncompressed array
print("Type %d, size %d" %
(response.image_type, len(response.image_data_uint8)))
img1d = np.fromstring(response.image_data_uint8,
dtype=np.uint8) # get numpy array
img_rgb = img1d.reshape(
response.height, response.width,
3) # reshape array to 4 channel image array H X W X 3
cv2.imwrite(os.path.normpath(filename + '.png'),
img_rgb) # write to png
def _observe_at_view(self, view, img_path):
"""Fly to given position and take a photo at given orientation."""
position = view["position"]
position = (Vector3r(position[0], position[1], position[2])
if type(position) == list else position)
print("Observe (%f, %f, %f) with yaw %f and pitch %f" % (
position.x_val,
position.y_val,
position.z_val,
view["yaw"] / math.pi * 180,
view["pitch"] / math.pi * 180,
))
self._move_to(position, yaw=view["yaw"] / math.pi * 180)
# self._uav.hoverAsync().join()
# time.sleep(5)
self._uav.simSetCameraOrientation(
"0", airsim.to_quaternion(view["pitch"], 0.0, 0.0))
png = self._uav.simGetImage("0", airsim.ImageType.Scene)
ReconstructionNavigator.mkdir(os.path.dirname(img_path))
airsim.write_file(img_path, png)
def take_picture(drone_name, is_save=True):
responses = client.simGetImages([
airsim.ImageRequest("front_center", airsim.ImageType.Scene),
# airsim.ImageRequest("bottom_center", airsim.ImageType.Scene),
], vehicle_name=drone_name)
if is_save:
drone_image_folder = '{}\\{}'.format(image_folder, drone_name)
if not os.path.isdir(drone_image_folder):
os.makedirs(drone_image_folder)
for idx, response in enumerate(responses):
if response.compress: #png format
print('image type {}, size {}'.format(
response.image_type, len(response.image_data_uint8)))
filename = '{}\\{}-{}.png'.format(
drone_image_folder, image_id[drone_name], idx)
image_id[drone_name] += 1
airsim.write_file(filename, response.image_data_uint8)
print('save image: {}'.format(filename))
else:
print('error: image format not support')
return responses
def saveImage(response, filename):
if response.pixels_as_float:
# airsim.write_pfm(os.path.normpath(filename + '.pfm'), airsim.get_pfm_array(response))
depth = np.array(response.image_data_float, dtype=np.float64)
depth = depth.reshape((response.height, response.width, -1))
depth = np.array(depth * 255, dtype=np.uint8)
# save pic
cv2.imwrite(os.path.normpath(filename + '.png'), depth)
elif response.compress: #png format
airsim.write_file(os.path.normpath(filename + '.png'),
response.image_data_uint8)
else: #uncompressed array
img1d = np.fromstring(response.image_data_uint8,
dtype=np.uint8) # get numpy array
img_rgb = img1d.reshape(
response.height, response.width,
3) # reshape array to 3 channel image array H X W X 3
cv2.imwrite(os.path.normpath(filename + '.png'),
img_rgb) # write to png
def query_sensors(client, unique_path_str):
folder_name = 'sensors'
if not (os.path.isdir(folder_name)):
os.makedirs(folder_name);
# Query three cameras
images = client.airsim_client.simGetImages([
airsim.ImageRequest("Front", airsim.ImageType.Scene),
airsim.ImageRequest("Follow", airsim.ImageType.Scene),
airsim.ImageRequest("Wheel", airsim.ImageType.Scene)])
# Query both accelerometers
sensor_values = client.airsim_client.readSensors()
# Get the current bot state
bot_state = client.airsim_client.getUrdfBotState()
airsim.write_file(os.path.join(folder_name, '{0}_front.png'.format(unique_path_str)), images[0].image_data_uint8)
airsim.write_file(os.path.join(folder_name, '{0}_follow.png'.format(unique_path_str)), images[1].image_data_uint8)
airsim.write_file(os.path.join(folder_name, '{0}_wheel.png'.format(unique_path_str)), images[2].image_data_uint8)
with open(os.path.join(folder_name, '{0}_state.json'.format(unique_path_str),), 'w') as f:
f.write(jsonpickle.encode(bot_state))
for sensor_name in [n for n in sensor_values['readings']]:
data_file = os.path.join(folder_name, '{0}_data.csv'.format(sensor_name))
data_dict = sensor_values['readings'][sensor_name]
keys = sorted([k for k in data_dict])
if not (os.path.isfile(data_file)):
with open(data_file, 'w') as f:
f.write('{0}\n'.format(','.join(keys)))
with open(data_file, 'a') as f:
f.write('{0}\n'.format(','.join([str(data_dict[k]) for k in keys])))
def take_photo(airsim_client,
pose_client,
current_state,
file_manager,
viewpoint=""):
"""
Description:
Calls simulator to take picture and return GT values simultaneously.
Writes picture file.
Inputs:
airsim_client: an object of type VehicleClient or DroneFlightClient
pose_client: an object of type PoseEstimationClient
current_state: an object of type State
file_manager: object of type FileManager
viewpoint: int. (default: ""), the viewpoint from which the drone is looking at the person
Returns:
photo: photo taken at simulation step
"""
if not airsim_client.is_using_airsim:
viewpoint = airsim_client.chosen_cam_view
photo = airsim_retrieve_gt(airsim_client, pose_client, current_state,
file_manager)
if airsim_client.is_using_airsim:
loc = file_manager.update_photo_loc(linecount=airsim_client.linecount,
viewpoint=viewpoint)
airsim.write_file(os.path.normpath(loc), photo)
if airsim_client.linecount > 3 and pose_client.quiet:
loc_rem = file_manager.take_photo_loc + '/img_' + str(
airsim_client.linecount - 2) + '.png'
if os.path.isfile(loc_rem):
os.remove(loc_rem)
else:
file_manager.update_photo_loc(
linecount=airsim_client.get_photo_index(), viewpoint=viewpoint)
return photo
imagequeue = []
while True:
# get RGBA camera images from the car
responses = client.simGetImages(
[airsim.ImageRequest(1, airsim.ImageType.Scene)])
# add image to queue
imagequeue.append(responses[0].image_data_uint8)
# dump queue when it gets full
if len(imagequeue) == QUEUESIZE:
for i in range(QUEUESIZE):
airsim.write_file(
os.path.normpath(IMAGEDIR + '/image%03d.png' % i),
imagequeue[i])
imagequeue.pop(0)
collision_info = client.simGetCollisionInfo()
if collision_info.has_collided:
print(
"Collision at pos %s, normal %s, impact pt %s, penetration %f, name %s, obj id %d"
% (pprint.pformat(collision_info.position),
pprint.pformat(collision_info.normal),
pprint.pformat(collision_info.impact_point),
collision_info.penetration_depth, collision_info.object_name,
collision_info.object_id))
break
print("CameraInfo %d: %s" % (camera_name, pp.pprint(camera_info)))
airsim.wait_key('Press any key to get images')
for x in range(3): # do few times
z = x * -20 - 5 # some random number
client.simSetVehiclePose(airsim.Pose(airsim.Vector3r(z, z, z), airsim.to_quaternion(x / 3.0, 0, x / 3.0)), True)
responses = client.simGetImages([
airsim.ImageRequest("0", airsim.ImageType.DepthVis),
airsim.ImageRequest("1", airsim.ImageType.DepthPerspective, True),
airsim.ImageRequest("2", airsim.ImageType.Segmentation),
airsim.ImageRequest("3", airsim.ImageType.Scene),
airsim.ImageRequest("4", airsim.ImageType.DisparityNormalized),
airsim.ImageRequest("4", airsim.ImageType.SurfaceNormals)])
for i, response in enumerate(responses):
if response.pixels_as_float:
print("Type %d, size %d, pos %s" % (response.image_type, len(response.image_data_float), pprint.pformat(response.camera_position)))
airsim.write_pfm(os.path.normpath('/temp/cv_mode_' + str(x) + "_" + str(i) + '.pfm'), airsim.get_pfm_array(response))
else:
print("Type %d, size %d, pos %s" % (response.image_type, len(response.image_data_uint8), pprint.pformat(response.camera_position)))
airsim.write_file(os.path.normpath('/temp/cv_mode_' + str(x) + "_" + str(i) + '.png'), response.image_data_uint8)
pose = client.simGetVehiclePose()
pp.pprint(pose)
time.sleep(3)
# currently reset() doesn't work in CV mode. Below is the workaround
client.simSetVehiclePose(airsim.Pose(airsim.Vector3r(0, 0, 0), airsim.to_quaternion(0, 0, 0)), True)
for pitch in range(-5, 5, 5):
for yaw in range(-10, 10, 2):
q = airsim.to_quaternion(pitch/10.0, 0, yaw/10.0)
client.simCharSetHeadRotation(q)
time.sleep(0.1)
airsim.wait_key('Press any key to get images')
for x in range(3): # do few times
responses = client.simGetImages([
airsim.ImageRequest("0", airsim.ImageType.DepthVis),
airsim.ImageRequest("0", airsim.ImageType.Segmentation),
airsim.ImageRequest("0", airsim.ImageType.Scene),
airsim.ImageRequest("0", airsim.ImageType.SurfaceNormals)])
for i, response in enumerate(responses):
if response.pixels_as_float:
print("Type %d, size %d, pos %s" % (response.image_type, len(response.image_data_float), pprint.pformat(response.camera_position)))
airsim.write_pfm(os.path.normpath('/temp/cv_mode_' + str(x) + "_" + str(i) + '.pfm'), airsim.get_pfm_array(response))
else:
print("Type %d, size %d, pos %s" % (response.image_type, len(response.image_data_uint8), pprint.pformat(response.camera_position)))
airsim.write_file(os.path.normpath('/temp/cv_mode_' + str(x) + "_" + str(i) + '.png'), response.image_data_uint8)
pose = client.simGetVehiclePose()
pose.position.x_val = pose.position.x_val + 1
pose.position.y_val = pose.position.y_val - 0.5
pose.position.z_val = pose.position.z_val - 0.5
client.simSetVehiclePose(pose, True)
time.sleep(3)
client.reset()
airsim.ImageRequest("0", airsim.ImageType.DepthVis), #depth visualization image
airsim.ImageRequest("1", airsim.ImageType.DepthPerspective, True), #depth in perspective projection
airsim.ImageRequest("1", airsim.ImageType.Scene), #scene vision image in png format
airsim.ImageRequest("1", airsim.ImageType.Scene, False, False)]) #scene vision image in uncompressed RGBA array
print('Retrieved images: %d', len(responses))
for response in responses:
filename = 'c:/temp/py' + str(idx)
if not os.path.exists('c:/temp/'):
os.makedirs('c:/temp/')
if response.pixels_as_float:
print("Type %d, size %d" % (response.image_type, len(response.image_data_float)))
airsim.write_pfm(os.path.normpath(filename + '.pfm'), airsim.get_pfm_array(response))
elif response.compress: #png format
print("Type %d, size %d" % (response.image_type, len(response.image_data_uint8)))
airsim.write_file(os.path.normpath(filename + '.png'), response.image_data_uint8)
else: #uncompressed array
print("Type %d, size %d" % (response.image_type, len(response.image_data_uint8)))
img1d = np.fromstring(response.image_data_uint8, dtype=np.uint8) #get numpy array
img_rgba = img1d.reshape(response.height, response.width, 4) #reshape array to 4 channel image array H X W X 4
img_rgba = np.flipud(img_rgba) #original image is flipped vertically
img_rgba[:,:,1:2] = 100 #just for fun add little bit of green in all pixels
airsim.write_png(os.path.normpath(filename + '.greener.png'), img_rgba) #write to png
#restore to original state
client.reset()
client.enableApiControl(False)
def get_camtrap_images(env_num, animal_class, num_animals):
output_folder = r"C:\Users\t-sabeer\Documents\AirSimImages\\"
pp = pprint.PrettyPrinter(indent=4)
client = connect_to_airsim()
object_lookup = set_segmentation_ids(client, animal_class, num_animals)
time.sleep(5) #env needs time to generate
camera_position_list = [(-45, -45, 0, -.1, 0.8), (-45, 0, 0, -.1, -.7),
(0, -45, 0, -.1, 2.0), (0, 0, 0, -.1, -2.3)]
for cam_num, cam_pos in enumerate(camera_position_list):
client.simSetVehiclePose(
airsim.Pose(airsim.Vector3r(cam_pos[0], cam_pos[1], cam_pos[2]),
airsim.to_quaternion(cam_pos[3], 0, cam_pos[4])), True)
#client.simSetCameraOrientation("0", airsim.to_quaternion(-0.161799, 0, 0)); #radians
#pose = client.simGetVehiclePose()
#pp.pprint(pose)
#print('Pose ' + str(cam_num))
#print(pose)
time.sleep(1) #new camera position needs time to update
for x in range(3): # create sequence of 3
responses = client.simGetImages([
airsim.ImageRequest("0", airsim.ImageType.Scene),
airsim.ImageRequest("0", airsim.ImageType.Segmentation, False,
False)
])
for i, response in enumerate(responses):
if response.pixels_as_float:
#print("Type %d, size %d, pos %s" % (response.image_type, len(response.image_data_float), pprint.pformat(response.camera_position)))
airsim.write_pfm(
os.path.normpath(output_folder + str(env_num) +
'_cam_' + str(cam_num) + '_frame_' +
str(x) + "_" + str(i) + '.pfm'),
airsim.get_pfm_array(response))
if response.compress:
#print("Type %d, size %d, pos %s" % (response.image_type, len(response.image_data_uint8), pprint.pformat(response.camera_position)))
airsim.write_file(
os.path.normpath(output_folder + str(env_num) +
'_cam_' + str(cam_num) + '_frame_' +
str(x) + "_" + str(i) + '.jpg'),
response.image_data_uint8)
else:
img = np.fromstring(response.image_data_uint8,
dtype=np.uint8) #get numpy array
img_rgba = img.reshape(response.height, response.width, 4)
img_rgba = np.flipud(
img_rgba) #original image is flipped vertically
airsim.write_png(
os.path.normpath(output_folder + str(env_num) +
'_cam_' + str(cam_num) + '_frame_' +
str(x) + "_" + str(i) + '.png'),
img_rgba) #write to png
time.sleep(1) #1fps frame rate
#pose = client.simGetVehiclePose()
#pp.pprint(pose)
return
def image_handle(self, collision_info): #Saves images stored in im_list from flight and logs times in nanoseconds #Depth image handling: https://github.com/microsoft/AirSim/issues/921 gap_time = math.floor(self.gap / self.speed) #Determine time needed from collision to give danger reading del_ind = math.floor(gap_time / self.time_step) + self.num_frames #Number of frames to delete (add num_frames because need full set of frames when giving result del self.cam_im_list[-del_ind:] del self.state_list[-del_ind:] if len(self.cam_im_list) >= 2*self.num_frames: #Check if enough frames to generate safe and danger images #Define range of indexes to sample from for safe images min_ind = 0 max_ind = len(self.cam_im_list) - 2*self.num_frames + 1 start_ind = self.generate_index(min_ind, max_ind) #Generate start index for 5 safe images #Store collision time collision_time = collision_info.time_stamp #Store relevant safe images and data associated for idx, im in enumerate(self.cam_im_list[start_ind:start_ind + self.num_frames]): #Generate time to collision for current frame time_to_collision = collision_time - im[0].time_stamp #Save rgb image to file airsim.write_file(os.path.normpath(os.path.join(self.fold_path, 'flight' + "_" + "rgb" + "_" +"safe" + '_' + str(self.flight_num) + '_' + str(idx) + '.png')), im[0].image_data_uint8) #Generate depth image depth = im[1] depth_im = self.generate_depth_image(depth) #Save depth image cv2.imwrite(os.path.normpath(os.path.join(self.fold_path, 'flight' + "_" + "depth" + "_" +"safe" + '_' + str(self.flight_num) + '_' + str(idx)+ '.png')), depth_im) #find index of state being used, relative to initial list so can get state info state_ind = start_ind + idx #Store state information and save to csv linear_velocity = self.state_list[state_ind].kinematics_estimated.linear_velocity angular_velocity = self.state_list[state_ind].kinematics_estimated.angular_velocity x_lin_vel, y_lin_vel, z_lin_vel, x_ang_vel, y_ang_vel, z_ang_vel = linear_velocity.x_val, linear_velocity.y_val, linear_velocity.z_val, angular_velocity.x_val, angular_velocity.y_val, angular_velocity.z_val row = [str(self.flight_num), 'safe', str(idx),str(x_lin_vel),str(y_lin_vel), str(z_lin_vel), str(x_ang_vel), str(y_ang_vel), str(z_ang_vel), str(np.sqrt(x_lin_vel**2 + y_lin_vel**2 + z_lin_vel**2)), str(self.speed), str(time_to_collision), str(start_ind), str(len(self.cam_im_list))] with open(self.csv_path, 'a') as csvFile: writer = csv.writer(csvFile) writer.writerow(row) #Store relevant danger images and data associated for idx, im in enumerate(self.cam_im_list[len(self.cam_im_list) - self.num_frames:]): #Generate time to collision for current frame time_to_collision = collision_time - im[0].time_stamp #time stamp the same for im[0] and im[1] #Save rgb image to file airsim.write_file(os.path.normpath(os.path.join(self.fold_path,'flight' + "_" + "rgb" + "_" + "danger" + '_' + str(self.flight_num) + '_' + str(idx) + '.png')), im[0].image_data_uint8) #Generate depth image depth = im[1] depth_im = self.generate_depth_image(depth) #Save depth image cv2.imwrite(os.path.normpath(os.path.join(self.fold_path,'flight' "_" + "depth" + "_" +"danger" + '_' + str(self.flight_num) + '_' + str(idx) + '.png')), depth_im) #find index of state being used, relative to initial list so can get state info state_ind = len(self.state_list) - self.num_frames + idx #Want state corresponding to #Store state information linear_velocity = self.state_list[state_ind].kinematics_estimated.linear_velocity angular_velocity = self.state_list[state_ind].kinematics_estimated.angular_velocity x_lin_vel, y_lin_vel, z_lin_vel, x_ang_vel, y_ang_vel, z_ang_vel = linear_velocity.x_val, linear_velocity.y_val, linear_velocity.z_val, angular_velocity.x_val, angular_velocity.y_val, angular_velocity.z_val row = [str(self.flight_num), 'danger', str(idx),str(x_lin_vel),str(y_lin_vel), str(z_lin_vel), str(x_ang_vel), str(y_ang_vel), str(z_ang_vel), str(np.sqrt(x_lin_vel**2 + y_lin_vel**2 + z_lin_vel**2)), str(self.speed), str(time_to_collision), str(start_ind), str(len(self.cam_im_list))] with open(self.csv_path, 'a') as csvFile: writer = csv.writer(csvFile) writer.writerow(row) self.flight_num += 1 #Increase fligh number for saving images
imgL_0 = cv2.imdecode(
airsim.string_to_uint8_array(rLeft[0].image_data_uint8),
cv2.IMREAD_UNCHANGED)
imgR_0 = cv2.imdecode(
airsim.string_to_uint8_array(rRight[0].image_data_uint8),
cv2.IMREAD_UNCHANGED)
img = np.concatenate((imgL_0, imgR_0), axis=1)
cv2.imshow('***', img)
key = cv2.waitKey(1)
if key == ord('s'):
loc = [[p.x_val, p.y_val, p.z_val],
[o.w_val, o.x_val, o.y_val, o.z_val]]
trajactory.append(loc)
fname = str(id).zfill(5)
airsim.write_file((saveDir + fname + '_L.png'),
rLeft[0].image_data_uint8)
airsim.write_file((saveDir + fname + '_R.png'),
rRight[0].image_data_uint8)
airsim.write_pfm(((saveDir + fname + '_D.pfm')),
airsim.get_pfm_array(rLeft[1]))
airsim.write_file((saveDir + fname + '_M.png'),
rLeft[2].image_data_uint8)
id = id + 1
# cv2.imshow('***', img), cv2.waitKey(), cv2.destroyAllWindows()
...
with open(saveDir + 'path.json', 'w') as write_file:
json.dump(trajactory, write_file)
cv2.destroyAllWindows()
import os
# connect to the AirSim simulator
client = airsim.MultirotorClient()
client.confirmConnection()
client.enableApiControl(True)
client.armDisarm(True)
# Async methods returns Future. Call join() to wait for task to complete.
client.takeoffAsync().join()
client.moveToPositionAsync(-10, 10, -10, 5).join()
# take images
responses = client.simGetImages([
airsim.ImageRequest("0", airsim.ImageType.DepthVis),
airsim.ImageRequest("1", airsim.ImageType.DepthPlanar, True)
])
print('Retrieved images: %d', len(responses))
# do something with the images
for response in responses:
if response.pixels_as_float:
print("Type %d, size %d" %
(response.image_type, len(response.image_data_float)))
airsim.write_pfm(os.path.normpath('/temp/py1.pfm'),
airsim.getPfmArray(response))
else:
print("Type %d, size %d" %
(response.image_type, len(response.image_data_uint8)))
airsim.write_file(os.path.normpath('/temp/py1.png'),
response.image_data_uint8)
os.makedirs(tmp_dir)
except OSError:
if not os.path.isdir(tmp_dir):
raise
for idx, response in enumerate(responses):
filename = os.path.join(tmp_dir, str(idx))
if response.pixels_as_float:
print("Type %d, size %d" %
(response.image_type, len(response.image_data_float)))
airsim.write_pfm(os.path.normpath(filename + '.pfm'),
airsim.get_pfm_array(response))
elif response.compress: #png format
print("Type %d, size %d" %
(response.image_type, len(response.image_data_uint8)))
airsim.write_file(os.path.normpath(filename + '.png'),
response.image_data_uint8)
else: #uncompressed array
print("Type %d, size %d" %
(response.image_type, len(response.image_data_uint8)))
img1d = np.fromstring(response.image_data_uint8,
dtype=np.uint8) # get numpy array
img_rgb = img1d.reshape(
response.height, response.width,
3) # reshape array to 4 channel image array H X W X 3
cv2.imwrite(os.path.normpath(filename + '.png'),
img_rgb) # write to png
airsim.wait_key('Press any key to reset to original state')
client.armDisarm(False)
client.reset()
client.enableApiControl(False)
os.makedirs(os.path.join(tmp_dir, str(n)))
except OSError:
if not os.path.isdir(tmp_dir):
raise
for x in range(50): # do few times
#xn = 1 + x*5 # some random number
client.simSetVehiclePose(airsim.Pose(airsim.Vector3r(x, 0, -2), airsim.to_quaternion(0, 0, 0)), True)
time.sleep(0.1)
responses = client.simGetImages([
airsim.ImageRequest("0", airsim.ImageType.Scene),
airsim.ImageRequest("1", airsim.ImageType.Scene),
airsim.ImageRequest("2", airsim.ImageType.Scene)])
for i, response in enumerate(responses):
if response.pixels_as_float:
print("Type %d, size %d, pos %s" % (response.image_type, len(response.image_data_float), pprint.pformat(response.camera_position)))
airsim.write_pfm(os.path.normpath(os.path.join(tmp_dir, str(x) + "_" + str(i) + '.pfm')), airsim.get_pfm_array(response))
else:
print("Type %d, size %d, pos %s" % (response.image_type, len(response.image_data_uint8), pprint.pformat(response.camera_position)))
airsim.write_file(os.path.normpath(os.path.join(tmp_dir, str(i), str(x) + "_" + str(i) + '.png')), response.image_data_uint8)
pose = client.simGetVehiclePose()
pp.pprint(pose)
time.sleep(3)
# currently reset() doesn't work in CV mode. Below is the workaround
client.simSetVehiclePose(airsim.Pose(airsim.Vector3r(0, 0, 0), airsim.to_quaternion(0, 0, 0)), True)
