def step(self, quad_offset, t):
# move with given velocity
self.client.simPause(False)
has_collided = False
landed = False
self.client.moveByVelocityAsync(quad_offset[0], quad_offset[1],
quad_offset[2], timeslice)
start_time = time.time()
i = 0
while time.time() - start_time < timeslice:
sleep(0.05)
# get quadrotor states
quad_pos = self.client.getMultirotorState(
).kinematics_estimated.position
# decide whether collision occured
collided = self.client.simGetCollisionInfo().has_collided
landed = quad_pos.y_val > 10 and self.client.getMultirotorState(
).landed_state == airsim.LandedState.Landed
landed = landed or quad_pos.z_val > floorZ
collision = collided or landed
if collision:
i += 1
if i > COLISION_THRESHOLD:
has_collided = True
if i > 0:
print('colision cnt: ', i)
self.client.simPause(True)
# observe with depth camera
responses = self.client.simGetImages(
[airsim.ImageRequest(1, airsim.ImageType.DepthVis, True)])
# get quadrotor states
quad_pos = self.client.getMultirotorState(
).kinematics_estimated.position
quad_vel = self.client.getMultirotorState(
).kinematics_estimated.linear_velocity
# decide whether done
dead = has_collided or quad_pos.y_val <= outY
done = dead or quad_pos.y_val >= goalY
# compute reward
if done and t < 2:
done = False
reward = self.compute_reward(quad_pos, quad_vel, dead)
if reward == config.reward['slow']:
done = True
# log info
info = {}
info['Y'] = quad_pos.y_val
info['level'] = self.level
if landed:
info['status'] = 'landed'
elif has_collided:
info['status'] = 'collision'
elif quad_pos.y_val <= outY:
info['status'] = 'out'
elif quad_pos.y_val >= goalY:
info['status'] = 'goal'
elif reward == config.reward['slow']:
info['status'] = 'slow'
else:
info['status'] = 'going'
quad_vel = np.array([quad_vel.x_val, quad_vel.y_val, quad_vel.z_val])
observation = (responses, quad_vel)
return observation, reward, done, info
print("Done: %r" % (found))
##for neighborhood environment
#set object ID for sky
found = client.simSetSegmentationObjectID("SkySphere", 42, True)
print("Done: %r" % (found))
#below doesn't work yet. You must set CustomDepthStencilValue in Unreal Editor for now
airsim.wait_key('Press any key to set Landscape object ID to 128')
found = client.simSetSegmentationObjectID("[\w]*", 128, True)
print("Done: %r" % (found))
#get segmentation image in various formats
responses = client.simGetImages([
airsim.ImageRequest("0", airsim.ImageType.Segmentation,
True), #depth in perspective projection
airsim.ImageRequest("0", airsim.ImageType.Segmentation, False, False)
]) #scene vision image in uncompressed RGBA array
print('Retrieved images: %d', len(responses))
#save segmentation images in various formats
for idx, response in enumerate(responses):
filename = 'c:/temp/py_seg_' + str(idx)
if response.pixels_as_float:
print("Type %d, size %d" %
(response.image_type, len(response.image_data_float)))
#airsim.write_pfm(os.path.normpath(filename + '.pfm'), airsim.get_pfm_array(response))
elif response.compress: #png format
print("Type %d, size %d" %
(response.image_type, len(response.image_data_uint8)))
client.setCarControls(car_controls)
print("Go reverse, steer right")
time.sleep(3) # let car drive a bit
car_controls.is_manual_gear = False; # change back gear to auto
car_controls.manual_gear = 0
# apply breaks
car_controls.brake = 1
client.setCarControls(car_controls)
print("Apply break")
time.sleep(3) # let car drive a bit
car_controls.brake = 0 #remove break
# get camera images from the car
responses = client.simGetImages([
airsim.ImageRequest("0", airsim.ImageType.DepthVis), #depth visualization image
airsim.ImageRequest("1", airsim.ImageType.DepthPerspective, True), #depth in perspective projection
airsim.ImageRequest("1", airsim.ImageType.Scene), #scene vision image in png format
airsim.ImageRequest("1", airsim.ImageType.Scene, False, False)]) #scene vision image in uncompressed RGBA array
print('Retrieved images: %d', len(responses))
for response in responses:
filename = 'c:/temp/py' + str(idx)
if response.pixels_as_float:
print("Type %d, size %d" % (response.image_type, len(response.image_data_float)))
airsim.write_pfm(os.path.normpath(filename + '.pfm'), airsim.get_pfm_array(response))
elif response.compress: #png format
print("Type %d, size %d" % (response.image_type, len(response.image_data_uint8)))
airsim.write_file(os.path.normpath(filename + '.png'), response.image_data_uint8)
else: #uncompressed array
# ready to run example: PythonClient/multirotor/hello_drone.py
import airsim
# connect to the AirSim simulator
client = airsim.MultirotorClient()
client.confirmConnection()
client.enableApiControl(True)
client.armDisarm(True)
# Async methods returns Future. Call join() to wait for task to complete.
client.takeoffAsync().join()
client.moveToPositionAsync(-10, 10, -10, 5).join()
# take images
responses = client.simGetImages([
airsim.ImageRequest("0", airsim.ImageType.DepthVis),
airsim.ImageRequest("1", airsim.ImageType.DepthPlanner, True)
])
print('Retrieved images: %d', len(responses))
# do something with the images
for response in responses:
if response.pixels_as_float:
print("Type %d, size %d" %
(response.image_type, len(response.image_data_float)))
airsim.write_pfm(os.path.normpath('/temp/py1.pfm'),
airsim.getPfmArray(response))
else:
print("Type %d, size %d" %
(response.image_type, len(response.image_data_uint8)))
airsim.write_file(os.path.normpath('/temp/py1.png'),
"Pitch": 0.0, "Roll": 0.0, "Yaw": 0.0
}
}
}
}
}
'''
client = airsim.VehicleClient()
client.confirmConnection()
framecounter = 1
prevtimestamp = datetime.now()
while (framecounter <= 500):
if framecounter % 150 == 0:
client.simGetImages([
airsim.ImageRequest("high_res", airsim.ImageType.Scene, False,
False)
])
print("High resolution image captured.")
if framecounter % 30 == 0:
now = datetime.now()
print(f"Time spent for 30 frames: {now-prevtimestamp}")
prevtimestamp = now
client.simGetImages(
[airsim.ImageRequest("low_res", airsim.ImageType.Scene, False, False)])
framecounter += 1
import airsim
import cv2
import numpy as np
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D
client = airsim.VehicleClient()
client.confirmConnection()
responses = client.simGetImages([
airsim.ImageRequest("0",
airsim.ImageType.Segmentation,
pixels_as_float=False,
compress=False), # segmentation image in int
airsim.ImageRequest("1",
airsim.ImageType.DepthPerspective,
pixels_as_float=True,
compress=False), # depth in perspective projection
airsim.ImageRequest(
"2", airsim.ImageType.Scene, pixels_as_float=False,
compress=False) # scene vision image in uncompressed RGBA array
])
np.shape(responses[0])
r0 = responses[0]
r1 = responses[1]
r2 = responses[2]
(x_m, y_m) = np.meshgrid(range(0, r1.width), range(0, r1.height))
if len(r0.image_data_float) > 1:
img0 = np.array(r0.image_data_float)
img0 = img0.reshape(r0.height, r0.width)
else:
#radians
airsim.wait_key('Press any key to get camera parameters')
for camera_name in range(5):
camera_info = client.simGetCameraInfo(str(camera_name))
print("CameraInfo %d: %s" % (camera_name, pp.pprint(camera_info)))
airsim.wait_key('Press any key to get images')
for x in range(3): # do few times
z = x * -20 - 5 # some random number
client.simSetVehiclePose(
airsim.Pose(airsim.Vector3r(z, z, z),
airsim.to_quaternion(x / 3.0, 0, x / 3.0)), True)
responses = client.simGetImages([
airsim.ImageRequest("0", airsim.ImageType.DepthVis),
airsim.ImageRequest("1", airsim.ImageType.DepthPerspective, True),
airsim.ImageRequest("2", airsim.ImageType.Segmentation),
airsim.ImageRequest("3", airsim.ImageType.Scene),
airsim.ImageRequest("4", airsim.ImageType.DisparityNormalized),
airsim.ImageRequest("4", airsim.ImageType.SurfaceNormals)
])
for i, response in enumerate(responses):
if response.pixels_as_float:
print("Type %d, size %d, pos %s" %
(response.image_type, len(response.image_data_float),
pprint.pformat(response.camera_position)))
airsim.write_pfm(
os.path.normpath('/temp/cv_mode_' + str(x) + "_" + str(i) +
'.pfm'), airsim.get_pfm_array(response))
def im_store(self):
#Store images on timer from thread
cam_im, state_info = self.client.simGetImages([airsim.ImageRequest("front_center", airsim.ImageType.Scene), airsim.ImageRequest("front_center", airsim.ImageType.DepthVis, True)]), self.client.getMultirotorState()#RGB png image
self.cam_im_list.append(cam_im)
self.state_list.append(state_info)
initial_pose_y = pose.position.y_val
initial_pose_z = pose.position.z_val
for imgs in range(1, 100, 1):
pose.position.x_val = initial_pose_x + random.randrange(0, 10,
1) / 40 - 0.15
pose.position.y_val = initial_pose_y + random.randrange(0, 10,
1) / 40 - 0.15
pose.position.z_val = initial_pose_z + random.randrange(0, 10,
1) / 40 - 0.15
client.simSetVehiclePose(pose, True)
# For skintone in range (1, 31, 5):
skintone = random.randrange(1, 31, 5)
client.simCharSetSkinDarkness(float(skintone) / 10)
# For age in range (1, 22, 10):
age = random.randrange(1, 21, 5)
client.simCharSetSkinAgeing(float(age) / 10)
# For pitch and yaw:
pitch = random.randrange(-8, 8, 2)
yaw = random.randrange(-12, 12, 2)
q = airsim.to_quaternion(pitch / 10.0, 0, yaw / 10.0)
client.simCharSetHeadRotation(q)
for x in range(1):
responses = client.simGetImages(
[airsim.ImageRequest("0", airsim.ImageType.Scene)])
def _step(self, action):
assert self.action_space.contains(action), "%r (%s) invalid"%(action, type(action))
time.sleep(0.05)
car_state = airsimClient.getCarState()
self.car_state = car_state
speed = car_state.speed
self.stepN += 1
steer = (action-2)/2
gas = max(min(20,(speed-20)/-15),0)
airsimClient.setCarControls(gas, steer)
self.steer = steer
collision_info = airsimClient.simGetCollisionInfo()
if collision_info.time_stamp != self.last_collision and collision_info.time_stamp != 0:
done = True
else:
done = False
'''
elif speed < 0.5:
self.stallCount += 1
else:
self.stallCount = 0
if self.stallCount > 6:
done = False
else:
done = False
'''
self.last_collision = collision_info.time_stamp
self.sensors = airsimClient.getSensorStates()
cdepth = self.sensors[1]
self.state = self.sensors
self.state.append(action)
self.addToLog('speed', speed)
self.addToLog('steer', steer)
steerLookback = 17
steerAverage = np.average(self.allLogs['steer'][-steerLookback:])
self.steerAverage = steerAverage
responses2 = airsimClient.simGetImages([airsim.ImageRequest("0", airsim.ImageType.Scene, False, False)])
cam_image = responses2[0]
img1d = np.fromstring(cam_image.image_data_uint8, dtype=np.uint8)
img_rgb = img1d.reshape(cam_image.height, cam_image.width, 3)
self.yolores, self.closeness = self.yolo(img_rgb,self.net,0.5,0.5)
print()
print(closeness)
# Training using the Roaming mode
reward, dSpeed = self.computeReward('roam')
self.addToLog('reward', reward)
rewardSum = np.sum(self.allLogs['reward'])
# Terminate the episode on large cumulative amount penalties,
# since car probably got into an unexpected loop of some sort
if rewardSum < -1000:
done = True
sys.stdout.write("\r\x1b[K{}/{}==>reward/depth/steer/speed: {:.0f}/{:.0f} \t({:.1f}/{:.1f}/{:.1f}) \t{:.1f}/{:.1f} \t{:.2f}/{:.2f} ".format(self.episodeN, self.stepN, reward, rewardSum, self.state[0], self.state[1], self.state[2], steer, steerAverage, speed, dSpeed))
sys.stdout.flush()
# placeholder for additional logic
if done:
pass
return np.array(self.state), reward, done, {}
s = pprint.pformat(state1)
print("Drone1: State: %s" % s)
# f1 = client_1.moveToPositionAsync(-5, 5, -10, 5, vehicle_name="Drone1")
# car_controls1.throttle = 0.5
# car_controls1.steering = 0.5
# client_2.setCarControls(car_controls1, "Car1")
# print("Car1: Go Forward")
# f1.join()
# time.sleep(2)
airsim.wait_key('Press any key to take images')
# get camera images from the car
responses1 = client_1.simGetImages(
[
airsim.ImageRequest(
"0", airsim.ImageType.DepthVis), #depth visualization image
airsim.ImageRequest("1", airsim.ImageType.Scene, False, False)
],
vehicle_name="Drone1") #scene vision image in uncompressed RGBA array
print('Drone1: Retrieved images: %d' % len(responses1))
responses2 = client_2.simGetImages(
[
airsim.ImageRequest(
"0", airsim.ImageType.Segmentation), #depth visualization image
airsim.ImageRequest("1", airsim.ImageType.Scene, False, False)
],
"Car1") #scene vision image in uncompressed RGBA array
print('Car1: Retrieved images: %d' % (len(responses2)))
tmp_dir = os.path.join(tempfile.gettempdir(), "airsim_drone")
print("Saving images to %s" % tmp_dir)
def parse_camera(in_image, droneId):
img1d = np.fromstring(in_image.image_data_uint8,
dtype=np.uint8) # get numpy array
image = img1d.reshape(
in_image.height, in_image.width,
3) # reshape array to 3 channel image array H X W X 3
#print(img1d.shape, image.shape, end='\r')
# camera_publisher.publish(bridge.cv2_to_imgmsg(image))
#print(cv2.__file__)
if droneId == 1:
cv2.imshow('image1', image)
else:
cv2.imshow('image2', image)
cv2.waitKey(1)
while not rospy.is_shutdown():
parse_camera(
client.simGetImages([
airsim.ImageRequest("front_center_custom", airsim.ImageType.Scene,
False, False)
], "drone_1")[0], 1)
parse_camera(
client.simGetImages([
airsim.ImageRequest("front_center_custom", airsim.ImageType.Scene,
False, False)
], "drone_2")[0], 2)
camera_publishing_rate.sleep()
cv2.destroyAllWindows()
def get_image():
image = client.simGetImages([airsim.ImageRequest("0", airsim.ImageType.Scene, False, False)])[0]
image1d = np.fromstring(image.image_data_uint8, dtype=np.uint8)
image_rgba = image1d.reshape(image.height, image.width, 4)
image_rgba = np.flipud(image_rgba)
return image_rgba[:, :, 0:3]
import numpy as np
import airsim
import os
import cv2
client = airsim.VehicleClient()
client.confirmConnection()
filename = "test_uncompressed"
filename2 = "test_compressed"
responses = client.simGetImages([
airsim.ImageRequest("front_center", airsim.ImageType.Scene, False, False),
airsim.ImageRequest("front_center", airsim.ImageType.Scene, False, True)
])
response = responses[0]
print(
f"Len: {len(response.image_data_uint8)}, height: {response.height}, width: {response.width}"
)
# get numpy array
img1d = np.fromstring(response.image_data_uint8, dtype=np.uint8)
# reshape array to 3 channel image array H X W X 3
img_rgb = img1d.reshape(response.height, response.width, 3)
# Display image for confirmation
cv2.imshow("Image", img_rgb)
cv2.waitKey(0)
# original image is fliped vertically
client = airsim.MultirotorClient()
# try to exclude all objects from the segmentation image as per the documentation
result = client.simSetSegmentationObjectID("[\w]*", -1, True)
print("set all object segmentation ids to -1: {}".format(result))
# set the ID of the symbol cube to 99 instead of -1 to include it in the segmentation image
result = client.simSetSegmentationObjectID("symbol_cube", 99, True)
# result = client.simSetSegmentationObjectID("OrangeBall", 99, True)
symbol_cube_id = client.simGetSegmentationObjectID("symbol_cube")
print("set symbol_cube id: {} -> {}".format(result, symbol_cube_id))
# request Scene and Segmentation images
requests = [
airsim.ImageRequest(camera_name=0,
image_type=airsim.ImageType.Scene,
pixels_as_float=False,
compress=False),
airsim.ImageRequest(camera_name=0,
image_type=airsim.ImageType.Segmentation,
pixels_as_float=False,
compress=False)
]
responses = client.simGetImages(requests)
print("response length: {}".format(len(responses)))
# for each returned image
for i in range(len(requests)):
# make the image as a 1-dimensional numpy array
img1d = numpy.frombuffer(responses[i].image_data_uint8, dtype=numpy.uint8)
img_rgb = img1d.reshape(responses[i].height, responses[i].width, 3)
jaw_pose = airsim.Pose()
#jaw_pose.position = airsim.Vector3r(0.002, 0.001, -0.003)
#jaw_pose.orientation = airsim.to_quaternion(0, 0, -.15)
client.simCharSetBonePose("Jaw", jaw_pose)
airsim.wait_key('Press any key to turn head around')
for pitch in range(-15, 15, 3):
for yaw in range(-30, 30, 2):
q = airsim.to_quaternion(pitch / 10.0, 0, yaw / 10.0)
client.simCharSetHeadRotation(q)
time.sleep(0.1)
airsim.wait_key('Press any key to get images')
for x in range(3): # do few times
responses = client.simGetImages([
airsim.ImageRequest("0", airsim.ImageType.DepthVis),
airsim.ImageRequest("0", airsim.ImageType.Segmentation),
airsim.ImageRequest("0", airsim.ImageType.Scene),
airsim.ImageRequest("0", airsim.ImageType.SurfaceNormals)
])
for i, response in enumerate(responses):
if response.pixels_as_float:
print("Type %d, size %d, pos %s" %
(response.image_type, len(response.image_data_float),
pprint.pformat(response.camera_position)))
airsim.write_pfm(
os.path.normpath('/temp/cv_mode_' + str(x) + "_" + str(i) +
'.pfm'), airsim.get_pfm_array(response))
else:
print("Type %d, size %d, pos %s" %
barometer_data = client.getBarometerData()
s = pprint.pformat(barometer_data)
print("barometer_data: %s" % s)
magnetometer_data = client.getMagnetometerData()
s = pprint.pformat(magnetometer_data)
print("magnetometer_data: %s" % s)
gps_data = client.getGpsData()
s = pprint.pformat(gps_data)
print("gps_data: %s" % s)
landed = client.getMultirotorState().landed_state
if landed == airsim.LandedState.Landed:
print("taking off...")
client.takeoffAsync().join()
else:
print("already flying...")
client.hoverAsync().join()
im_req = airsim.ImageRequest("1", airsim.ImageType.Scene, False, False)
response = client.simGetImages([im_req])[0]
im = np.frombuffer(response.image_data_uint8, dtype=np.uint8).reshape(
[response.height, response.width, 3])
cv2.imwrite('im.png', im)
client.moveByRollPitchYawThrottleAsync(0, np.deg2rad(45), 0, 0.5, 2.)
client.moveByMotorPWMsAsync()
k = 0
INITIAL_EPSILON = 0.5
REPLAY_MEMORY = 10000
BATCH = 32 # Make process batch after observe for adqn
FRAME_PER_ACTION = 1
filename = "./logs/log_file_score.csv"
throttle = 1
brake = 0
steering_angle = 0
handbrake = 0
data_collect = []
image_response = client.simGetImages(
[airsim.ImageRequest("0", airsim.ImageType.Scene, False, False)])[0]
image1d = np.fromstring(image_response.image_data_uint8, dtype=np.uint8)
image_bgr = image1d.reshape(image_response.height, image_response.width, 3)
#image_grey = cv2.cvtColor(image_bgr, cv2.COLOR_BGR2GRAY)
b, g, r = cv2.split(image_bgr)
x_t = cv2.merge((r, g, b))
x_t = x_t[40:101, 36:220, 0:3].astype(float)
image_buf = x_t
image_buf /= 255 # Normalization
s_t = image_buf #np.stack((x_t, x_t, x_t, x_t), axis=2)
s_t1 = s_t
im_action = 1
#s_tlm = data_buf
client.enableApiControl(True)
car_controls = airsim.CarControls()
# Make RL agent
NumBufferFrames = 4
SizeRows = 84
SizeCols = 84
NumActions = 6
agent = DeepQAgent((NumBufferFrames, SizeRows, SizeCols), NumActions, monitor=True)
# Train
epoch = 100
current_step = 0
max_steps = epoch * 250000
responses = client.simGetImages([airsim.ImageRequest("0", airsim.ImageType.DepthPerspective, True, False)])
current_state = transform_input(responses)
try:
while True:
action = agent.act(current_state)
car_controls = interpret_action(action)
client.setCarControls(car_controls)
car_state = client.getCarState()
reward = compute_reward(car_state)
done = isDone(car_state, car_controls, reward)
if done == 1:
reward = -10
def telemetry():
while True:
global x_t
global reward
global episode_reward
global terminal
global im_action
#global time_train
#global time_total
#global time_image
global episodes
while True:
try:
#client.simPause(True)
image_response = client.simGetImages([
airsim.ImageRequest("0", airsim.ImageType.Scene, False,
False)
])[0]
image1d = np.fromstring(image_response.image_data_uint8,
dtype=np.uint8)
image_bgr = image1d.reshape(image_response.height,
image_response.width, 3)
#image_grey = cv2.cvtColor(image_bgr, cv2.COLOR_BGR2GRAY)
b, g, r = cv2.split(image_bgr)
image_rgb = cv2.merge((r, g, b))
x_t = image_rgb[40:101, 36:220, 0:3].astype(float)
image_buf = x_t
image_buf /= 255 # Normalization
#from PIL import Image
#img = Image.fromarray(image_rgb[40:101,36:220,0:3])
#img.save('my.png')
#img.show()
#x_t = np.reshape(x_t, (61, 184, 1))
#client.simContinueForTime(0.2)
break
except:
print("Image Error")
car_state = client.getCarState()
#alive_reward += 5
#print (s_t1)
reward = max(0, (car_state.speed)) / 33 #+ alive_reward
episode_reward += reward
#s_tlm = round(abs(car_state.speed),1), round(car_state.kinematics_estimated.linear_acceleration.y_val,1)
#client.simPause(False)
car_controls = interpret_action(action_index)
client.setCarControls(car_controls)
collision_info = client.simGetCollisionInfo()
im_action = im_replay.get_im_response()
#print (car_state.kinematics_estimated.linear_velocity.x_val)
#?????? fix crash when map change
terminal = False
if collision_info.has_collided:
reward = -1
episode_reward = 0
terminal = True
client.reset()
episodes += 1
if car_state.kinematics_estimated.position.z_val <= -1:
client.reset()
def main(args):
client = airsim.VehicleClient()
client.confirmConnection()
ts = datetime.datetime.now().isoformat()[:-7].replace(':', '')
tmp_dir = os.path.join(args.out_path, args.env, ts)
print("Saving images to %s" % tmp_dir)
try:
os.makedirs(tmp_dir)
except OSError:
if not os.path.isdir(tmp_dir):
raise
nseqs = 3600
h5file = tables.open_file(os.path.join(tmp_dir, 'output.h5'),
mode="w",
title="Flythroughs")
seq_len = 40
short_seq_len = 10
nominal_fps = 30
labels_table = h5file.create_table('/',
'labels',
Particle,
expectedrows=nseqs)
video_table = h5file.create_earray('/',
'videos',
tables.UInt8Atom(),
shape=(0, seq_len, 3, 112, 112),
expectedrows=nseqs)
short_video_table = h5file.create_earray('/',
'short_videos',
tables.UInt8Atom(),
shape=(0, short_seq_len, 3, 112,
112),
expectedrows=nseqs)
depth_table = h5file.create_earray('/',
'depth',
tables.Float64Atom(),
shape=(0, 112, 112),
expectedrows=nseqs)
for i in tqdm(range(nseqs)): # do few times
# For flat environments, start ground plane localization not too high.
ground_from = 5
# 3 meters/s -> jogging speed
MAX_SPEED = 3
collisions = True
pause = 0
# Manually define areas of interest. Note that inside one of the airsim
# environments, you can pull up coordinates using `;`. However, the
# coordinates listed are multiplied by 100 (i.e. the numbers are in cm
# rather than meters); divide by 100 to define reasonable boundaries
# for the capture area.
if args.env == 'blocks':
x = np.random.uniform(-50, 50)
y = np.random.uniform(-50, 50)
pause = .05 # blocks is too fast sometimes
elif args.env.startswith('nh'):
x = np.random.uniform(-150, 150)
y = np.random.uniform(-150, 150)
client.simEnableWeather(True)
for k in range(8):
client.simSetWeatherParameter(k, 0.0)
if args.env == 'nh_fall':
client.simSetWeatherParameter(
airsim.WeatherParameter.MapleLeaf, 1.0)
client.simSetWeatherParameter(airsim.WeatherParameter.RoadLeaf,
1.0)
if args.env == 'nh_winter':
client.simSetWeatherParameter(airsim.WeatherParameter.Snow,
1.0)
client.simSetWeatherParameter(airsim.WeatherParameter.RoadSnow,
1.0)
elif args.env == 'mountains':
# Most of the interesting stuff (e.g. the lake, roads) is on the
# diagonal of this very large environment (several kilometers!).
xy = np.random.uniform(0, 2500)
xmy = np.random.uniform(-100, 100)
x = xy + xmy
y = xy - xmy
# This environment varies a lot in height, start from higher
ground_from = 100
elif args.env == 'trap':
x = np.random.uniform(-10, 10)
y = np.random.uniform(-10, 10)
# This one has lots of branches, keep sequences that have collisions
collisions = False
else:
raise NotImplementedError(args.env)
# Time of day effects works in blocks and trap only.
time_of_day = np.random.randint(5, 21)
client.simSetTimeOfDay(
True,
start_datetime=f"2020-03-21 {time_of_day:02}:00:00",
is_start_datetime_dst=True,
celestial_clock_speed=0,
update_interval_secs=60,
move_sun=True)
if pause > 0:
time.sleep(pause)
pitch = np.random.uniform(
-.25,
.25) # Sometimes we look a little up, sometimes a little down
roll = 0 # Should be 0 unless something is wrong
# 360 degrees lookaround
yaw = np.random.uniform(-np.pi, np.pi)
heading_yaw = draw_von_mises(2.5)
heading_pitch = draw_von_mises(16)
# Max ~90 degrees per second head rotation
rotation_yaw = 30 * np.pi / 180 * np.random.randn()
rotation_pitch = 10 * np.pi / 180 * np.random.randn()
speed = MAX_SPEED * np.random.rand()
# Figure out the height of the ground. Move the camera way far above
# ground, and get the distance to the ground via the depth buffer
# from the bottom camera.
client.simSetVehiclePose(
airsim.Pose(airsim.Vector3r(x, y, -ground_from),
airsim.to_quaternion(0, 0, 0)), True)
if pause > 0:
time.sleep(pause)
responses = client.simGetImages([
airsim.ImageRequest("bottom_center",
airsim.ImageType.DepthPlanner,
pixels_as_float=True)
])
response = responses[0]
the_arr = airsim.list_to_2d_float_array(response.image_data_float,
response.width,
response.height)
# Then move to ~5.5 feet above the ground
rgy = range(int(.4 * the_arr.shape[0]), int(.6 * the_arr.shape[0]))
rgx = range(int(.4 * the_arr.shape[0]), int(.6 * the_arr.shape[0]))
the_min = np.median(the_arr[rgy, rgx])
z = the_min - ground_from - np.random.uniform(1.4, 2)
if z > 50:
# More than 50 meters below sea level, bad draw.
continue
#client.startRecording()
z = z.item()
depths = []
frames = []
booped = False
for k in range(seq_len):
if booped:
continue
# In nominal seconds
t = (k - (seq_len - 1) / 2) / nominal_fps
d = t * speed
client.simSetVehiclePose(
airsim.Pose(
airsim.Vector3r(
x + d * np.cos(yaw + heading_yaw) *
np.cos(pitch + heading_pitch),
y + d * np.sin(yaw + heading_yaw) *
np.cos(pitch + heading_pitch),
z + d * np.sin(pitch + heading_pitch)),
airsim.to_quaternion(pitch + t * rotation_pitch, roll,
yaw + t * rotation_yaw)), True)
responses = client.simGetImages([
airsim.ImageRequest("front_center", airsim.ImageType.Scene,
False, False),
airsim.ImageRequest("front_center",
airsim.ImageType.DepthPlanner, True,
False),
])
for j, response in enumerate(responses):
if j == 0:
frames.append(
airsim.string_to_uint8_array(
response.image_data_uint8).reshape(
response.height, response.width,
3)[:, :, ::-1])
if j == 1:
zbuff = airsim.list_to_2d_float_array(
response.image_data_float, response.width,
response.height)
depths.append(zbuff)
# Did we bump into something?
if collisions:
closest = zbuff[zbuff.shape[0] // 4:-zbuff.shape[0] //
4, zbuff.shape[1] //
4:-zbuff.shape[1] // 4].min()
if closest < .5:
# oops I booped it again.
booped = True
if j == 0 and args.save_images:
filename = os.path.join(tmp_dir, f"{i:02}_{k:02}.png")
matplotlib.image.imsave(filename, frames[-1])
if pause > 0:
time.sleep(pause)
row = labels_table.row
if not booped and not args.save_images:
# Let's save this!
row['x'] = x
row['y'] = y
row['z'] = z
row['yaw'] = yaw
row['pitch'] = pitch
row['speed'] = speed
row['heading_yaw'] = heading_yaw
row['heading_pitch'] = heading_pitch
row['rotation_yaw'] = rotation_yaw
row['rotation_pitch'] = rotation_pitch
row.append()
V = np.stack(frames, axis=0).transpose((0, 3, 1, 2))
V = V.reshape((1, ) + V.shape)
video_table.append(V)
# Take a subset of the data.
mid_seq = range((seq_len - short_seq_len) // 2,
(seq_len + short_seq_len) // 2)
assert V.shape[1] == seq_len
short_video_table.append(V[:, mid_seq, :, :, :])
# Only record the mid-point z buffer
n = seq_len // 2
D = depths[n]
D = D.reshape((1, ) + D.shape)
depth_table.append(D)
h5file.close()
# currently reset() doesn't work in CV mode. Below is the workaround
client.simSetVehiclePose(
airsim.Pose(airsim.Vector3r(0, 0, 0), airsim.to_quaternion(0, 0, 0)),
True)
print("state: %s" % pprint.pformat(state))
#airsim.wait_key('Press any key to move vehicle to (-10, 10, -10) at 5 m/s')
#client.moveToPositionAsync(-10, 10, -10, 5).join()
#client.hoverAsync().join()
state = client.getMultirotorState()
print("state: %s" % pprint.pformat(state))
caminfo = client.simGetCameraInfo("0")
print("cam info: ", caminfo, caminfo.fov, type(caminfo.fov))
airsim.wait_key('Press any key to take images')
# get camera images from the car
responses = client.simGetImages([
airsim.ImageRequest("0", airsim.ImageType.DepthVis, True), #depth visualization image
airsim.ImageRequest("1", airsim.ImageType.DepthPerspective, True), #depth in perspective projection
airsim.ImageRequest("1", airsim.ImageType.DepthPlanner,True), #scene vision image in png format
airsim.ImageRequest("1", airsim.ImageType.DepthPlanner), #scene vision image in png format
airsim.ImageRequest("1", airsim.ImageType.Scene, False, False)
]) #scene vision image in uncompressed RGBA array
print('Retrieved images: %d' % len(responses))
tmp_dir = os.path.join(tempfile.gettempdir(), "airsim_drone")
print ("Saving images to %s" % tmp_dir)
try:
os.makedirs(tmp_dir)
except OSError:
if not os.path.isdir(tmp_dir):
raise
