def do_stuff_at_uavmvs_viewpoint(i, pose):
nonlocal client, camera_poses, airsim_record
log_string = f"({i}/{len(camera_poses)})"
p, q = pose.position, pose.orientation
if args.debug:
log_string += f" position = {to_xyz_str(p)}"
client.simPlotTransforms([pose], scale=100, is_persistent=True)
# client.simPlotArrows([p], [LOOK_AT_TARGET], Rgba.White, thickness=3.0, duration=10)
elif args.capture_dir:
path = f"{args.prefix}pose{args.suffix}_{i:0{len(str(len(camera_poses)))}}.png"
path = os.path.join(args.capture_dir, path)
airsim.write_png(path, AirSimImage.get_mono(client, CAPTURE_CAMERA))
log_string += f' saved image to "{path}"'
record_line = AirSimRecord.make_line_string(p, q, time_stamp=str(i), image_file=path)
airsim_record.append(record_line)
ff.log(log_string)
def get_img():
# get state of the car
car_state = client.getCarState()
#print(car_state)
#print("Speed %d, Gear %d" % (car_state.speed, car_state.gear))
# set the controls for car
#car_controls.throttle = 1
#car_controls.steering = 0
#client.setCarControls(car_controls)
# let car drive a bit
time.sleep(1)
# get camera images Let me tell you, i setup a software on the adult vids (p**n material) website and guess what, you visited this website to experience fun (you know what i mean). When you were watching videos, your internet browser initiated functioning as a Remote Desktop that has a keylogger which gave me access to your display as well as cam. Just after that, my software obtained your entire contacts from your Messenger, Facebook, as well as emailaccount. and then i made a double-screen video. First part shows the video you were viewing (you've got a fine taste lol . . .), and 2nd part shows the view of your webcam, and it is u. from the car
responses = client.simGetImages(
[airsim.ImageRequest("0", airsim.ImageType.Scene, False, False)])
response = responses[0]
# get numpy array
img1d = np.fromstring(response.image_data_uint8, dtype=np.uint8)
# reshape array to 4 channel image array H X W X 4
img_rgba = img1d.reshape(response.height, response.width, 4)
# original image is fliped vertically
img_rgba = np.flipud(img_rgba)
# just for fun add little bit of green in all pixels
#img_rgba[:,:,1:2] = 100
# write to png
airsim.write_png(os.path.normpath("pic.png"), img_rgba)
img = cv2.imread("pic.png")
image = img_preprocess(img)
image = np.array([image])
#model = build_model()
steering_angle = float(model.predict(image))
print(steering_angle)
car_controls.throttle = 1
car_controls.steering = steering_angle
client.setCarControls(car_controls)
return img
def saveImage(self):
responses = self.client.simGetImages([ImageRequest("0", airsim.ImageType.Scene, False, False)],
vehicle_name=self.nombre)
response = responses[0]
# get numpy array
img1d = np.fromstring(response.image_data_uint8, dtype=np.uint8)
print(img1d)
# reshape array to 4 channel image array H X W X 4
img_rgb = img1d.reshape(response.height, response.width, 3)
print(img_rgb)
# img = Image.fromarray(img_rgb, 'RGB')
# print(img)
# original image is fliped vertically
img_rgb = np.flipud(img_rgb)
# write to png
dt = datetime.now()
airsim.write_png(os.path.normpath('./Logs/' +
str(self.nombre) +
str(dt.date()) +
str(dt.hour) +
str(dt.minute) +
str(dt.second) +
str(dt.microsecond) + '.png'), img_rgb)
print("Type %d, size %d" %
(response.image_type, len(response.image_data_float)))
airsim.write_pfm(os.path.normpath(filename + '.pfm'),
airsim.get_pfm_array(response))
elif response.compress: #png format
print("Type %d, size %d" %
(response.image_type, len(response.image_data_uint8)))
airsim.write_file(os.path.normpath(filename + '.png'),
response.image_data_uint8)
else: #uncompressed array
print("Type %d, size %d" %
(response.image_type, len(response.image_data_uint8)))
img1d = np.fromstring(response.image_data_uint8,
dtype=np.uint8) #get numpy array
img_rgba = img1d.reshape(
response.height, response.width,
4) #reshape array to 4 channel image array H X W X 4
img_rgba = np.flipud(img_rgba) #original image is flipped vertically
img_rgba[:, :, 1:
2] = 100 #just for fun add little bit of green in all pixels
airsim.write_png(os.path.normpath(filename + '.greener.png'),
img_rgba) #write to png
airsim.wait_key('Press any key to reset to original state')
client.armDisarm(False)
client.reset()
# that's enough fun for now. let's quit cleanly
client.enableApiControl(False)
airsim.ImageRequest("1", airsim.ImageType.Scene), #scene vision image in png format
airsim.ImageRequest("1", airsim.ImageType.Scene, False, False)]) #scene vision image in uncompressed RGBA array
print('Retrieved images: %d', len(responses))
for response in responses:
filename = 'c:/temp/py' + str(idx)
if not os.path.exists('c:/temp/'):
os.makedirs('c:/temp/')
if response.pixels_as_float:
print("Type %d, size %d" % (response.image_type, len(response.image_data_float)))
airsim.write_pfm(os.path.normpath(filename + '.pfm'), airsim.get_pfm_array(response))
elif response.compress: #png format
print("Type %d, size %d" % (response.image_type, len(response.image_data_uint8)))
airsim.write_file(os.path.normpath(filename + '.png'), response.image_data_uint8)
else: #uncompressed array
print("Type %d, size %d" % (response.image_type, len(response.image_data_uint8)))
img1d = np.fromstring(response.image_data_uint8, dtype=np.uint8) #get numpy array
img_rgba = img1d.reshape(response.height, response.width, 4) #reshape array to 4 channel image array H X W X 4
img_rgba = np.flipud(img_rgba) #original image is flipped vertically
img_rgba[:,:,1:2] = 100 #just for fun add little bit of green in all pixels
airsim.write_png(os.path.normpath(filename + '.greener.png'), img_rgba) #write to png
#restore to original state
client.reset()
client.enableApiControl(False)
def get_camtrap_images(env_num, animal_class, num_animals):
output_folder = r"C:\Users\t-sabeer\Documents\AirSimImages\\"
pp = pprint.PrettyPrinter(indent=4)
client = connect_to_airsim()
object_lookup = set_segmentation_ids(client, animal_class, num_animals)
time.sleep(5) #env needs time to generate
camera_position_list = [(-45, -45, 0, -.1, 0.8), (-45, 0, 0, -.1, -.7),
(0, -45, 0, -.1, 2.0), (0, 0, 0, -.1, -2.3)]
for cam_num, cam_pos in enumerate(camera_position_list):
client.simSetVehiclePose(
airsim.Pose(airsim.Vector3r(cam_pos[0], cam_pos[1], cam_pos[2]),
airsim.to_quaternion(cam_pos[3], 0, cam_pos[4])), True)
#client.simSetCameraOrientation("0", airsim.to_quaternion(-0.161799, 0, 0)); #radians
#pose = client.simGetVehiclePose()
#pp.pprint(pose)
#print('Pose ' + str(cam_num))
#print(pose)
time.sleep(1) #new camera position needs time to update
for x in range(3): # create sequence of 3
responses = client.simGetImages([
airsim.ImageRequest("0", airsim.ImageType.Scene),
airsim.ImageRequest("0", airsim.ImageType.Segmentation, False,
False)
])
for i, response in enumerate(responses):
if response.pixels_as_float:
#print("Type %d, size %d, pos %s" % (response.image_type, len(response.image_data_float), pprint.pformat(response.camera_position)))
airsim.write_pfm(
os.path.normpath(output_folder + str(env_num) +
'_cam_' + str(cam_num) + '_frame_' +
str(x) + "_" + str(i) + '.pfm'),
airsim.get_pfm_array(response))
if response.compress:
#print("Type %d, size %d, pos %s" % (response.image_type, len(response.image_data_uint8), pprint.pformat(response.camera_position)))
airsim.write_file(
os.path.normpath(output_folder + str(env_num) +
'_cam_' + str(cam_num) + '_frame_' +
str(x) + "_" + str(i) + '.jpg'),
response.image_data_uint8)
else:
img = np.fromstring(response.image_data_uint8,
dtype=np.uint8) #get numpy array
img_rgba = img.reshape(response.height, response.width, 4)
img_rgba = np.flipud(
img_rgba) #original image is flipped vertically
airsim.write_png(
os.path.normpath(output_folder + str(env_num) +
'_cam_' + str(cam_num) + '_frame_' +
str(x) + "_" + str(i) + '.png'),
img_rgba) #write to png
time.sleep(1) #1fps frame rate
#pose = client.simGetVehiclePose()
#pp.pprint(pose)
return
import airsim #pip install airsim
import numpy as np
import os
# for car use CarClient()
client = airsim.CarClient()
responses = client.simGetImages(
[airsim.ImageRequest("0", airsim.ImageType.Scene, False, False)])
response = responses[0]
# get numpy array
img1d = np.fromstring(response.image_data_uint8, dtype=np.uint8)
# reshape array to 4 channel image array H X W X 4
img_rgb = img1d.reshape(response.height, response.width, 3)
# original image is fliped vertically
img_rgb = np.flipud(img_rgb)
# just for fun add little bit of green in all pixels
img_rgb[:, :, 1:2] = 100
# write to png
filename = 'd:/airsim'
airsim.write_png(os.path.normpath(filename + '.png'), img_rgb)
response = client.simGetImages(request)
# format Scene image
img1d = numpy.frombuffer(response[SCENE].image_data_uint8,
dtype=numpy.uint8)
img_rgb = img1d.reshape(response[SCENE].height, response[SCENE].width, 3)
# generate a unique filename
id_encoding = "{},{},{},{},{},{},{}".format(scenario_name, x, y, z, pitch,
roll, yaw)
filename = str(uuid.uuid3(uuid.NAMESPACE_X500, id_encoding)).replace(
"-", "") + ".png"
full_filename = "{}/{}".format(image_directory, filename)
# write Scene image to png
airsim.write_png(os.path.normpath(full_filename), img_rgb)
# format Segmentation image
img1d = numpy.frombuffer(response[SEGMENTATION].image_data_uint8,
dtype=numpy.uint8)
img_rgb = img1d.reshape(response[SEGMENTATION].height,
response[SEGMENTATION].width, 3)
# slice the Segmentation image to get rid of RGB
# we know that this works with segmentation id of 99 (should work generally but depends on color palette)
mask_image = img_rgb[:, :, 0]
# get the mask
mask = get_rle(mask_image)
# show reconstructed mask (for debugging)
accZ.append(state.kinematics_estimated.linear_acceleration.z_val)
angX.append(state.kinematics_estimated.angular_velocity.x_val)
angY.append(state.kinematics_estimated.angular_velocity.y_val)
angZ.append(state.kinematics_estimated.angular_velocity.z_val)
oriX.append(state.kinematics_estimated.orientation.x_val)
oriY.append(state.kinematics_estimated.orientation.y_val)
oriZ.append(state.kinematics_estimated.orientation.z_val)
oriW.append(state.kinematics_estimated.orientation.w_val)
Cls.append(0)
# get numpy array
img1d = np.fromstring(response.image_data_uint8, dtype=np.uint8)
# reshape array to 4 channel image array H X W X 4
img_rgb = img1d.reshape(response.height, response.width, 3)
# write to png
airsim.write_png(os.path.normpath(new_dataset_images_directory + '/im' + str(idx) + '.png'), img_rgb)
idx = idx + 1
# stop recorder after set time
t = time.time()
if (t-t0) > sim_seconds:
break
# save IMU data to csv
df = pd.DataFrame(list(zip(T, accX, accY, accZ, angX, angY, angZ, oriX, oriY, oriZ, oriW, Cls)),
columns=['Time', 'AccelX', 'AccelY', 'AccelZ', 'GyroX', 'GyroY', 'GyroZ',
'OriX', 'OriY', 'OriZ', 'OriW', 'class'])
df.to_csv(new_dataset_csv_name)
print("Recorded data " + str(idx) + " instances")
import pprint
from cv_bridge import CvBridge
client = airsim.VehicleClient()
client.confirmConnection()
found = client.simSetSegmentationObjectID("Ball[\w]*", 22, True)
print("Found balls: %r" % (found))
found = client.simSetSegmentationObjectID("Cube[\w]*", 32, True)
print("Found balls: %r" % (found))
found = client.simSetSegmentationObjectID("Pyramid[\w]*", 42, True)
print("Found balls: %r" % (found))
found = client.simSetSegmentationObjectID("Sky[\w]*", 0, True)
print("Found sky: %r" % (found))
found = client.simSetSegmentationObjectID("Ground[\w]*", 1, True)
print("Found ground: %r" % (found))
responses = client.simGetImages(
[airsim.ImageRequest("0", airsim.ImageType.Segmentation, False, False)])
response = responses[0]
print("Type %d, size %d" %
(response.image_type, len(response.image_data_uint8)))
img1d = np.fromstring(response.image_data_uint8, dtype=np.uint8)
img_rgba = img1d.reshape(response.height, response.width, 4)
img_rgba = np.flipud(img_rgba)
airsim.write_png('/sim/ml_scripts/images/segmentation.png', img_rgba)
"""
import airsim #pip install airsim
import numpy as np
import os
import matplotlib.pyplot as plt
filename = "C:/temp/"
client = airsim.CarClient()
client.confirmConnection()
client.enableApiControl(True)
responses = client.simGetImages(
[airsim.ImageRequest("0", airsim.ImageType.Scene, False, False)])
response = responses[0]
# get numpy array
img1d = np.fromstring(response.image_data_uint8, dtype=np.uint8)
# reshape array to 4 channel image array H X W X 4
img_rgba = img1d.reshape(response.height, response.width, 4)
# original image is fliped vertically
img_rgba = np.flipud(img_rgba)
print(img_rgba)
# just for fun add little bit of green in all pixels
# write to png
airsim.write_png(os.path.normpath(filename + 'inputDetectLine.png'), img_rgba)
def main(camera_name, align_to_left_camera, image_width, image_heightm,
requests, level_name, human_name, item_name, num_capture):
timestamp = int(time.time())
client = airsim.VehicleClient()
camera_info = camera_info_list[camera_name]
level_info = level_info_list[level_name]
item_info = item_info_list[item_name]
# camera config
# TODO: we might add random lense distortion for getting more realistic images.
lc, rc = (0.0, 1.0) if align_to_left_camera else (-0.5, 0.5)
client.simSetCameraPose(
'front_left',
airsim.Pose(airsim.Vector3r(0., lc * camera_info.baseline, 0.),
airsim.Quaternionr()).UE4ToAirSim())
client.simSetCameraFov('front_left', camera_info.HFOV)
client.simSetCameraImageSize('front_left', image_width, image_height)
client.simSetCameraPose(
'front_right',
airsim.Pose(airsim.Vector3r(0., rc * camera_info.baseline, 0.),
airsim.Quaternionr()).UE4ToAirSim())
client.simSetCameraFov('front_right', camera_info.HFOV)
client.simSetCameraImageSize('front_right', image_width, image_height)
# load level
#client.simLoadLevel(level_name) # TODO: after loading different level, position reference differs from what we expect. fix it.
# swapn humans/items
# TODO: find a way to immediately reflect segID change
human_actor_name = 'human_{}'.format(timestamp)
human_BP_name = metahumans_bp_path_template.format(human_name)
client.simSpawnObject(human_actor_name,
human_BP_name,
physics_enabled=False,
from_actorBP=True)
client.simSetSegmentationObjectID(human_actor_name, 1)
if item_name != 'none':
item_actor_name = 'item_{}'.format(timestamp)
client.simSpawnObject(item_actor_name,
item_name,
physics_enabled=False,
from_actorBP=False)
client.simSetSegmentationObjectID(item_actor_name, 2)
# sometimes the metahuman's hands are splited away from the body in the first frame... wait a little to avoid it
time.sleep(0.5)
# generate data
capture_folder = './capture_{}_{}_{}_{}/'.format(level_name, human_name,
item_name, timestamp)
os.makedirs(capture_folder, exist_ok=True)
for idx in range(num_capture):
# TODO: some metahumans's face are unnaturally lightened in Room environment. find how to fix it!
# configure objects in the scene.
# we follow the UE4's coordinates/units convention for ease of working with the UE4 environment.
# airsim uses NED coordinates with unit 'm' while UE4 uses NEU coordinates with unit 'cm'.
# base position/horizontal rotation where we arragnge the stereo camera, humans, items.
# here we set the base position to near the bottom center of the booth shelf.
base_position = generate_random_position(level_info.position_range)
base_rotation = generate_random_rotation((0, 0, 1), (0, 360))
#base_rotation = airsim.Quaternionr()
#base_rotation = generate_random_rotation((0, 0, 1), (90, 90))
world_to_base = airsim.Pose(base_position, base_rotation)
# derive the stereo camera position
camera_roll_error = 0 #5
camera_pitch_error = 0 #3
camera_height_error = 0 #5
camera_horiz_error = 0 #3
#camera_position = generate_random_position((80, 80, 40, 40, 160, 160))
#camera_rotation = generate_random_rotation((0, 0, 1), (-150, -150))
camera_position = generate_random_position(
(-17 - camera_horiz_error, -17 + camera_horiz_error,
0 - camera_horiz_error, 0 + camera_horiz_error,
212.5 - camera_height_error, 212.5 + camera_height_error))
camera_rotation = generate_random_rotation(
(0, 0, 1), (-camera_roll_error,
camera_roll_error)) * generate_random_rotation(
(0, 1, 0),
(90 - camera_pitch_error, 90 + camera_pitch_error))
base_to_camera = airsim.Pose(camera_position, camera_rotation)
world_to_camera = base_to_camera * world_to_base
client.simSetVehiclePose(world_to_camera.UE4ToAirSim())
# derive the human position
# TODO: add human skelton scaling variation
crouching_diff = 0 # TODO: impl
human_position = generate_random_position(
(-70, -30, -20, 20, -crouching_diff, -crouching_diff))
#human_position = generate_random_position((-40, -40, 0, 0, -crouching_diff, -crouching_diff))
human_rotation_diff = 20 #20#60
human_rotation = generate_random_rotation(
(0, 0, 1), (-90 - human_rotation_diff, -90 + human_rotation_diff))
base_to_human = airsim.Pose(human_position, human_rotation)
world_to_human = base_to_human * world_to_base
client.simSetObjectPose(human_actor_name, world_to_human.UE4ToAirSim())
# derive the human pose
left_hand_IKposition = airsim.Vector3r(
50, 0, -100) # relative to the clavicle
left_hand_rotation = generate_random_position((-30, -30, 0, 0, 0, 0))
# TODO: find natural range
right_hand_IKposition = generate_random_hand_reach(
(60, 100)) * -1 # relative to the clavicle
right_hand_rotation = generate_random_position(
metahumans_hand_rotation_range) # relative to the bone
# TODO: find a way to make crouching pose, natural foots, waist pose
client.simSetMetahumanPose(human_actor_name, left_hand_IKposition,
left_hand_rotation, right_hand_IKposition,
right_hand_rotation)
# derive the item pose
if item_name != 'none':
hand_pose_world = client.simGetMetahumanBonePose(
human_actor_name, 'middle_01_r').AirSimToUE4()
hand_to_item = airsim.Pose(
airsim.Vector3r(0, item_info.grab_offset, 0),
generate_random_rotation((0, 0, 1), (0, 360)))
world_to_item = hand_to_item * hand_pose_world
client.simSetObjectPose(item_actor_name,
world_to_item.UE4ToAirSim())
# request image captures
exposure_bias = generate_random_scalar(
level_info.auto_exposure_bias_range)
for camera_name in ["front_left", "front_right"]:
client.simSetCameraPostProcess(
camera_name,
auto_exposure_bias=exposure_bias,
auto_exposure_max_brightness=1.0,
auto_exposure_min_brightness=1.0,
lens_flare_intensity=0.0,
motion_blur_amount=0.0,
)
responses = client.simGetImages(requests)
# request attribute data
# TODO: gather data useful for machine learning (camera params, key points, etc)
# save images and attribute data
for response, request in zip(responses, requests):
img = airsim.decode_image_response(response)
if img.dtype == np.float16:
img = np.asarray(
(img * 1000).clip(0, 65535), dtype=np.uint16
) # convert to an uint16 depth image with unit mm. (we are expecting that the depth camera range is up to 65m)
if request.image_type == airsim.ImageType.Scene:
name = 'rgb'
elif request.image_type == airsim.ImageType.DepthPlanar:
name = 'depth' # be careful that simulated depth values are not so accurate at far range due to the limited bit-depth of rengdering depth buffer.
elif request.image_type == airsim.ImageType.Segmentation:
name = 'mask'
else:
raise Exception('no impl')
if request.camera_name == 'front_left':
name += 'L'
elif request.camera_name == 'front_right':
name += 'R'
else:
raise Exception('no impl')
airsim.write_png(
'{}{:0>8}_{}.png'.format(capture_folder, idx, name), img)
if item_name != 'none':
client.simDestroyObject(item_actor_name)
client.simDestroyObject(human_actor_name)
requests = [
airsim.ImageRequest(camera_name=0,
image_type=airsim.ImageType.Scene,
pixels_as_float=False,
compress=False),
airsim.ImageRequest(camera_name=0,
image_type=airsim.ImageType.Segmentation,
pixels_as_float=False,
compress=False)
]
responses = client.simGetImages(requests)
print("response length: {}".format(len(responses)))
# for each returned image
for i in range(len(requests)):
# make the image as a 1-dimensional numpy array
img1d = numpy.frombuffer(responses[i].image_data_uint8, dtype=numpy.uint8)
img_rgb = img1d.reshape(responses[i].height, responses[i].width, 3)
# chatter about the image
print("response dimensions: {} x {}".format(responses[i].height,
responses[i].width))
print("dimensions: {} x {}".format(img_rgb.shape[0], img_rgb.shape[1]))
# save the image
filename = "image_{}.png".format(i)
airsim.write_png(filename, img_rgb)
if (i == 1):
print(img1d)
action = 19 * np.random.random_sample((1, 20)) + 1
client.moveByVelocityAsync(action[0, 0], action[0, 1], 0, 5.0).join()
# Grabbing an image API
responses = client.simGetImages([
airsim.ImageRequest("front_center", airsim.ImageType.Scene, False,
False)
])
response = responses[0]
img1d = np.fromstring(response.image_data_uint8, dtype=np.uint8)
img_rgba = img1d.reshape(response.height, response.width, 4)
img_rgba = np.flipud(img_rgba)
# Saving image API
file = "pic" + str(done) + ".png"
name = os.path.join(dir_path, "temp_images", file)
airsim.write_png(name, img_rgba)
# Finding position api
#api Pose simGetVehiclePose(const std::string& vehicle_name = "") const;
vehicle_pose = client.simGetVehiclePose().position
vehicle_orientation = client.simGetVehiclePose().orientation
print("Pose is x {}, y {}, z {}".format(vehicle_pose.x_val,
vehicle_pose.y_val,
vehicle_pose.z_val))
# Find velocity API
vehicle_linear_vel = client.simGetGroundTruthKinematics().linear_velocity
vehicle_vel = sqrt(vehicle_linear_vel.x_val**2 +
vehicle_linear_vel.y_val**2)
#print("Velocity is {}".format(vehicle_vel))
# Resetting to origin api
