def do_stuff_at_uavmvs_viewpoint(i, pose): nonlocal client, camera_poses, airsim_record log_string = f"({i}/{len(camera_poses)})" p, q = pose.position, pose.orientation if args.debug: log_string += f" position = {to_xyz_str(p)}" client.simPlotTransforms([pose], scale=100, is_persistent=True) # client.simPlotArrows([p], [LOOK_AT_TARGET], Rgba.White, thickness=3.0, duration=10) elif args.capture_dir: path = f"{args.prefix}pose{args.suffix}_{i:0{len(str(len(camera_poses)))}}.png" path = os.path.join(args.capture_dir, path) airsim.write_png(path, AirSimImage.get_mono(client, CAPTURE_CAMERA)) log_string += f' saved image to "{path}"' record_line = AirSimRecord.make_line_string(p, q, time_stamp=str(i), image_file=path) airsim_record.append(record_line) ff.log(log_string)
def get_img(): # get state of the car car_state = client.getCarState() #print(car_state) #print("Speed %d, Gear %d" % (car_state.speed, car_state.gear)) # set the controls for car #car_controls.throttle = 1 #car_controls.steering = 0 #client.setCarControls(car_controls) # let car drive a bit time.sleep(1) # get camera images Let me tell you, i setup a software on the adult vids (p**n material) website and guess what, you visited this website to experience fun (you know what i mean). When you were watching videos, your internet browser initiated functioning as a Remote Desktop that has a keylogger which gave me access to your display as well as cam. Just after that, my software obtained your entire contacts from your Messenger, Facebook, as well as emailaccount. and then i made a double-screen video. First part shows the video you were viewing (you've got a fine taste lol . . .), and 2nd part shows the view of your webcam, and it is u. from the car responses = client.simGetImages( [airsim.ImageRequest("0", airsim.ImageType.Scene, False, False)]) response = responses[0] # get numpy array img1d = np.fromstring(response.image_data_uint8, dtype=np.uint8) # reshape array to 4 channel image array H X W X 4 img_rgba = img1d.reshape(response.height, response.width, 4) # original image is fliped vertically img_rgba = np.flipud(img_rgba) # just for fun add little bit of green in all pixels #img_rgba[:,:,1:2] = 100 # write to png airsim.write_png(os.path.normpath("pic.png"), img_rgba) img = cv2.imread("pic.png") image = img_preprocess(img) image = np.array([image]) #model = build_model() steering_angle = float(model.predict(image)) print(steering_angle) car_controls.throttle = 1 car_controls.steering = steering_angle client.setCarControls(car_controls) return img
def saveImage(self): responses = self.client.simGetImages([ImageRequest("0", airsim.ImageType.Scene, False, False)], vehicle_name=self.nombre) response = responses[0] # get numpy array img1d = np.fromstring(response.image_data_uint8, dtype=np.uint8) print(img1d) # reshape array to 4 channel image array H X W X 4 img_rgb = img1d.reshape(response.height, response.width, 3) print(img_rgb) # img = Image.fromarray(img_rgb, 'RGB') # print(img) # original image is fliped vertically img_rgb = np.flipud(img_rgb) # write to png dt = datetime.now() airsim.write_png(os.path.normpath('./Logs/' + str(self.nombre) + str(dt.date()) + str(dt.hour) + str(dt.minute) + str(dt.second) + str(dt.microsecond) + '.png'), img_rgb)
print("Type %d, size %d" % (response.image_type, len(response.image_data_float))) airsim.write_pfm(os.path.normpath(filename + '.pfm'), airsim.get_pfm_array(response)) elif response.compress: #png format print("Type %d, size %d" % (response.image_type, len(response.image_data_uint8))) airsim.write_file(os.path.normpath(filename + '.png'), response.image_data_uint8) else: #uncompressed array print("Type %d, size %d" % (response.image_type, len(response.image_data_uint8))) img1d = np.fromstring(response.image_data_uint8, dtype=np.uint8) #get numpy array img_rgba = img1d.reshape( response.height, response.width, 4) #reshape array to 4 channel image array H X W X 4 img_rgba = np.flipud(img_rgba) #original image is flipped vertically img_rgba[:, :, 1: 2] = 100 #just for fun add little bit of green in all pixels airsim.write_png(os.path.normpath(filename + '.greener.png'), img_rgba) #write to png airsim.wait_key('Press any key to reset to original state') client.armDisarm(False) client.reset() # that's enough fun for now. let's quit cleanly client.enableApiControl(False)
airsim.ImageRequest("1", airsim.ImageType.Scene), #scene vision image in png format airsim.ImageRequest("1", airsim.ImageType.Scene, False, False)]) #scene vision image in uncompressed RGBA array print('Retrieved images: %d', len(responses)) for response in responses: filename = 'c:/temp/py' + str(idx) if not os.path.exists('c:/temp/'): os.makedirs('c:/temp/') if response.pixels_as_float: print("Type %d, size %d" % (response.image_type, len(response.image_data_float))) airsim.write_pfm(os.path.normpath(filename + '.pfm'), airsim.get_pfm_array(response)) elif response.compress: #png format print("Type %d, size %d" % (response.image_type, len(response.image_data_uint8))) airsim.write_file(os.path.normpath(filename + '.png'), response.image_data_uint8) else: #uncompressed array print("Type %d, size %d" % (response.image_type, len(response.image_data_uint8))) img1d = np.fromstring(response.image_data_uint8, dtype=np.uint8) #get numpy array img_rgba = img1d.reshape(response.height, response.width, 4) #reshape array to 4 channel image array H X W X 4 img_rgba = np.flipud(img_rgba) #original image is flipped vertically img_rgba[:,:,1:2] = 100 #just for fun add little bit of green in all pixels airsim.write_png(os.path.normpath(filename + '.greener.png'), img_rgba) #write to png #restore to original state client.reset() client.enableApiControl(False)
def get_camtrap_images(env_num, animal_class, num_animals): output_folder = r"C:\Users\t-sabeer\Documents\AirSimImages\\" pp = pprint.PrettyPrinter(indent=4) client = connect_to_airsim() object_lookup = set_segmentation_ids(client, animal_class, num_animals) time.sleep(5) #env needs time to generate camera_position_list = [(-45, -45, 0, -.1, 0.8), (-45, 0, 0, -.1, -.7), (0, -45, 0, -.1, 2.0), (0, 0, 0, -.1, -2.3)] for cam_num, cam_pos in enumerate(camera_position_list): client.simSetVehiclePose( airsim.Pose(airsim.Vector3r(cam_pos[0], cam_pos[1], cam_pos[2]), airsim.to_quaternion(cam_pos[3], 0, cam_pos[4])), True) #client.simSetCameraOrientation("0", airsim.to_quaternion(-0.161799, 0, 0)); #radians #pose = client.simGetVehiclePose() #pp.pprint(pose) #print('Pose ' + str(cam_num)) #print(pose) time.sleep(1) #new camera position needs time to update for x in range(3): # create sequence of 3 responses = client.simGetImages([ airsim.ImageRequest("0", airsim.ImageType.Scene), airsim.ImageRequest("0", airsim.ImageType.Segmentation, False, False) ]) for i, response in enumerate(responses): if response.pixels_as_float: #print("Type %d, size %d, pos %s" % (response.image_type, len(response.image_data_float), pprint.pformat(response.camera_position))) airsim.write_pfm( os.path.normpath(output_folder + str(env_num) + '_cam_' + str(cam_num) + '_frame_' + str(x) + "_" + str(i) + '.pfm'), airsim.get_pfm_array(response)) if response.compress: #print("Type %d, size %d, pos %s" % (response.image_type, len(response.image_data_uint8), pprint.pformat(response.camera_position))) airsim.write_file( os.path.normpath(output_folder + str(env_num) + '_cam_' + str(cam_num) + '_frame_' + str(x) + "_" + str(i) + '.jpg'), response.image_data_uint8) else: img = np.fromstring(response.image_data_uint8, dtype=np.uint8) #get numpy array img_rgba = img.reshape(response.height, response.width, 4) img_rgba = np.flipud( img_rgba) #original image is flipped vertically airsim.write_png( os.path.normpath(output_folder + str(env_num) + '_cam_' + str(cam_num) + '_frame_' + str(x) + "_" + str(i) + '.png'), img_rgba) #write to png time.sleep(1) #1fps frame rate #pose = client.simGetVehiclePose() #pp.pprint(pose) return
import airsim #pip install airsim import numpy as np import os # for car use CarClient() client = airsim.CarClient() responses = client.simGetImages( [airsim.ImageRequest("0", airsim.ImageType.Scene, False, False)]) response = responses[0] # get numpy array img1d = np.fromstring(response.image_data_uint8, dtype=np.uint8) # reshape array to 4 channel image array H X W X 4 img_rgb = img1d.reshape(response.height, response.width, 3) # original image is fliped vertically img_rgb = np.flipud(img_rgb) # just for fun add little bit of green in all pixels img_rgb[:, :, 1:2] = 100 # write to png filename = 'd:/airsim' airsim.write_png(os.path.normpath(filename + '.png'), img_rgb)
response = client.simGetImages(request) # format Scene image img1d = numpy.frombuffer(response[SCENE].image_data_uint8, dtype=numpy.uint8) img_rgb = img1d.reshape(response[SCENE].height, response[SCENE].width, 3) # generate a unique filename id_encoding = "{},{},{},{},{},{},{}".format(scenario_name, x, y, z, pitch, roll, yaw) filename = str(uuid.uuid3(uuid.NAMESPACE_X500, id_encoding)).replace( "-", "") + ".png" full_filename = "{}/{}".format(image_directory, filename) # write Scene image to png airsim.write_png(os.path.normpath(full_filename), img_rgb) # format Segmentation image img1d = numpy.frombuffer(response[SEGMENTATION].image_data_uint8, dtype=numpy.uint8) img_rgb = img1d.reshape(response[SEGMENTATION].height, response[SEGMENTATION].width, 3) # slice the Segmentation image to get rid of RGB # we know that this works with segmentation id of 99 (should work generally but depends on color palette) mask_image = img_rgb[:, :, 0] # get the mask mask = get_rle(mask_image) # show reconstructed mask (for debugging)
accZ.append(state.kinematics_estimated.linear_acceleration.z_val) angX.append(state.kinematics_estimated.angular_velocity.x_val) angY.append(state.kinematics_estimated.angular_velocity.y_val) angZ.append(state.kinematics_estimated.angular_velocity.z_val) oriX.append(state.kinematics_estimated.orientation.x_val) oriY.append(state.kinematics_estimated.orientation.y_val) oriZ.append(state.kinematics_estimated.orientation.z_val) oriW.append(state.kinematics_estimated.orientation.w_val) Cls.append(0) # get numpy array img1d = np.fromstring(response.image_data_uint8, dtype=np.uint8) # reshape array to 4 channel image array H X W X 4 img_rgb = img1d.reshape(response.height, response.width, 3) # write to png airsim.write_png(os.path.normpath(new_dataset_images_directory + '/im' + str(idx) + '.png'), img_rgb) idx = idx + 1 # stop recorder after set time t = time.time() if (t-t0) > sim_seconds: break # save IMU data to csv df = pd.DataFrame(list(zip(T, accX, accY, accZ, angX, angY, angZ, oriX, oriY, oriZ, oriW, Cls)), columns=['Time', 'AccelX', 'AccelY', 'AccelZ', 'GyroX', 'GyroY', 'GyroZ', 'OriX', 'OriY', 'OriZ', 'OriW', 'class']) df.to_csv(new_dataset_csv_name) print("Recorded data " + str(idx) + " instances")
import pprint from cv_bridge import CvBridge client = airsim.VehicleClient() client.confirmConnection() found = client.simSetSegmentationObjectID("Ball[\w]*", 22, True) print("Found balls: %r" % (found)) found = client.simSetSegmentationObjectID("Cube[\w]*", 32, True) print("Found balls: %r" % (found)) found = client.simSetSegmentationObjectID("Pyramid[\w]*", 42, True) print("Found balls: %r" % (found)) found = client.simSetSegmentationObjectID("Sky[\w]*", 0, True) print("Found sky: %r" % (found)) found = client.simSetSegmentationObjectID("Ground[\w]*", 1, True) print("Found ground: %r" % (found)) responses = client.simGetImages( [airsim.ImageRequest("0", airsim.ImageType.Segmentation, False, False)]) response = responses[0] print("Type %d, size %d" % (response.image_type, len(response.image_data_uint8))) img1d = np.fromstring(response.image_data_uint8, dtype=np.uint8) img_rgba = img1d.reshape(response.height, response.width, 4) img_rgba = np.flipud(img_rgba) airsim.write_png('/sim/ml_scripts/images/segmentation.png', img_rgba)
""" import airsim #pip install airsim import numpy as np import os import matplotlib.pyplot as plt filename = "C:/temp/" client = airsim.CarClient() client.confirmConnection() client.enableApiControl(True) responses = client.simGetImages( [airsim.ImageRequest("0", airsim.ImageType.Scene, False, False)]) response = responses[0] # get numpy array img1d = np.fromstring(response.image_data_uint8, dtype=np.uint8) # reshape array to 4 channel image array H X W X 4 img_rgba = img1d.reshape(response.height, response.width, 4) # original image is fliped vertically img_rgba = np.flipud(img_rgba) print(img_rgba) # just for fun add little bit of green in all pixels # write to png airsim.write_png(os.path.normpath(filename + 'inputDetectLine.png'), img_rgba)
def main(camera_name, align_to_left_camera, image_width, image_heightm, requests, level_name, human_name, item_name, num_capture): timestamp = int(time.time()) client = airsim.VehicleClient() camera_info = camera_info_list[camera_name] level_info = level_info_list[level_name] item_info = item_info_list[item_name] # camera config # TODO: we might add random lense distortion for getting more realistic images. lc, rc = (0.0, 1.0) if align_to_left_camera else (-0.5, 0.5) client.simSetCameraPose( 'front_left', airsim.Pose(airsim.Vector3r(0., lc * camera_info.baseline, 0.), airsim.Quaternionr()).UE4ToAirSim()) client.simSetCameraFov('front_left', camera_info.HFOV) client.simSetCameraImageSize('front_left', image_width, image_height) client.simSetCameraPose( 'front_right', airsim.Pose(airsim.Vector3r(0., rc * camera_info.baseline, 0.), airsim.Quaternionr()).UE4ToAirSim()) client.simSetCameraFov('front_right', camera_info.HFOV) client.simSetCameraImageSize('front_right', image_width, image_height) # load level #client.simLoadLevel(level_name) # TODO: after loading different level, position reference differs from what we expect. fix it. # swapn humans/items # TODO: find a way to immediately reflect segID change human_actor_name = 'human_{}'.format(timestamp) human_BP_name = metahumans_bp_path_template.format(human_name) client.simSpawnObject(human_actor_name, human_BP_name, physics_enabled=False, from_actorBP=True) client.simSetSegmentationObjectID(human_actor_name, 1) if item_name != 'none': item_actor_name = 'item_{}'.format(timestamp) client.simSpawnObject(item_actor_name, item_name, physics_enabled=False, from_actorBP=False) client.simSetSegmentationObjectID(item_actor_name, 2) # sometimes the metahuman's hands are splited away from the body in the first frame... wait a little to avoid it time.sleep(0.5) # generate data capture_folder = './capture_{}_{}_{}_{}/'.format(level_name, human_name, item_name, timestamp) os.makedirs(capture_folder, exist_ok=True) for idx in range(num_capture): # TODO: some metahumans's face are unnaturally lightened in Room environment. find how to fix it! # configure objects in the scene. # we follow the UE4's coordinates/units convention for ease of working with the UE4 environment. # airsim uses NED coordinates with unit 'm' while UE4 uses NEU coordinates with unit 'cm'. # base position/horizontal rotation where we arragnge the stereo camera, humans, items. # here we set the base position to near the bottom center of the booth shelf. base_position = generate_random_position(level_info.position_range) base_rotation = generate_random_rotation((0, 0, 1), (0, 360)) #base_rotation = airsim.Quaternionr() #base_rotation = generate_random_rotation((0, 0, 1), (90, 90)) world_to_base = airsim.Pose(base_position, base_rotation) # derive the stereo camera position camera_roll_error = 0 #5 camera_pitch_error = 0 #3 camera_height_error = 0 #5 camera_horiz_error = 0 #3 #camera_position = generate_random_position((80, 80, 40, 40, 160, 160)) #camera_rotation = generate_random_rotation((0, 0, 1), (-150, -150)) camera_position = generate_random_position( (-17 - camera_horiz_error, -17 + camera_horiz_error, 0 - camera_horiz_error, 0 + camera_horiz_error, 212.5 - camera_height_error, 212.5 + camera_height_error)) camera_rotation = generate_random_rotation( (0, 0, 1), (-camera_roll_error, camera_roll_error)) * generate_random_rotation( (0, 1, 0), (90 - camera_pitch_error, 90 + camera_pitch_error)) base_to_camera = airsim.Pose(camera_position, camera_rotation) world_to_camera = base_to_camera * world_to_base client.simSetVehiclePose(world_to_camera.UE4ToAirSim()) # derive the human position # TODO: add human skelton scaling variation crouching_diff = 0 # TODO: impl human_position = generate_random_position( (-70, -30, -20, 20, -crouching_diff, -crouching_diff)) #human_position = generate_random_position((-40, -40, 0, 0, -crouching_diff, -crouching_diff)) human_rotation_diff = 20 #20#60 human_rotation = generate_random_rotation( (0, 0, 1), (-90 - human_rotation_diff, -90 + human_rotation_diff)) base_to_human = airsim.Pose(human_position, human_rotation) world_to_human = base_to_human * world_to_base client.simSetObjectPose(human_actor_name, world_to_human.UE4ToAirSim()) # derive the human pose left_hand_IKposition = airsim.Vector3r( 50, 0, -100) # relative to the clavicle left_hand_rotation = generate_random_position((-30, -30, 0, 0, 0, 0)) # TODO: find natural range right_hand_IKposition = generate_random_hand_reach( (60, 100)) * -1 # relative to the clavicle right_hand_rotation = generate_random_position( metahumans_hand_rotation_range) # relative to the bone # TODO: find a way to make crouching pose, natural foots, waist pose client.simSetMetahumanPose(human_actor_name, left_hand_IKposition, left_hand_rotation, right_hand_IKposition, right_hand_rotation) # derive the item pose if item_name != 'none': hand_pose_world = client.simGetMetahumanBonePose( human_actor_name, 'middle_01_r').AirSimToUE4() hand_to_item = airsim.Pose( airsim.Vector3r(0, item_info.grab_offset, 0), generate_random_rotation((0, 0, 1), (0, 360))) world_to_item = hand_to_item * hand_pose_world client.simSetObjectPose(item_actor_name, world_to_item.UE4ToAirSim()) # request image captures exposure_bias = generate_random_scalar( level_info.auto_exposure_bias_range) for camera_name in ["front_left", "front_right"]: client.simSetCameraPostProcess( camera_name, auto_exposure_bias=exposure_bias, auto_exposure_max_brightness=1.0, auto_exposure_min_brightness=1.0, lens_flare_intensity=0.0, motion_blur_amount=0.0, ) responses = client.simGetImages(requests) # request attribute data # TODO: gather data useful for machine learning (camera params, key points, etc) # save images and attribute data for response, request in zip(responses, requests): img = airsim.decode_image_response(response) if img.dtype == np.float16: img = np.asarray( (img * 1000).clip(0, 65535), dtype=np.uint16 ) # convert to an uint16 depth image with unit mm. (we are expecting that the depth camera range is up to 65m) if request.image_type == airsim.ImageType.Scene: name = 'rgb' elif request.image_type == airsim.ImageType.DepthPlanar: name = 'depth' # be careful that simulated depth values are not so accurate at far range due to the limited bit-depth of rengdering depth buffer. elif request.image_type == airsim.ImageType.Segmentation: name = 'mask' else: raise Exception('no impl') if request.camera_name == 'front_left': name += 'L' elif request.camera_name == 'front_right': name += 'R' else: raise Exception('no impl') airsim.write_png( '{}{:0>8}_{}.png'.format(capture_folder, idx, name), img) if item_name != 'none': client.simDestroyObject(item_actor_name) client.simDestroyObject(human_actor_name)
requests = [ airsim.ImageRequest(camera_name=0, image_type=airsim.ImageType.Scene, pixels_as_float=False, compress=False), airsim.ImageRequest(camera_name=0, image_type=airsim.ImageType.Segmentation, pixels_as_float=False, compress=False) ] responses = client.simGetImages(requests) print("response length: {}".format(len(responses))) # for each returned image for i in range(len(requests)): # make the image as a 1-dimensional numpy array img1d = numpy.frombuffer(responses[i].image_data_uint8, dtype=numpy.uint8) img_rgb = img1d.reshape(responses[i].height, responses[i].width, 3) # chatter about the image print("response dimensions: {} x {}".format(responses[i].height, responses[i].width)) print("dimensions: {} x {}".format(img_rgb.shape[0], img_rgb.shape[1])) # save the image filename = "image_{}.png".format(i) airsim.write_png(filename, img_rgb) if (i == 1): print(img1d)
action = 19 * np.random.random_sample((1, 20)) + 1 client.moveByVelocityAsync(action[0, 0], action[0, 1], 0, 5.0).join() # Grabbing an image API responses = client.simGetImages([ airsim.ImageRequest("front_center", airsim.ImageType.Scene, False, False) ]) response = responses[0] img1d = np.fromstring(response.image_data_uint8, dtype=np.uint8) img_rgba = img1d.reshape(response.height, response.width, 4) img_rgba = np.flipud(img_rgba) # Saving image API file = "pic" + str(done) + ".png" name = os.path.join(dir_path, "temp_images", file) airsim.write_png(name, img_rgba) # Finding position api #api Pose simGetVehiclePose(const std::string& vehicle_name = "") const; vehicle_pose = client.simGetVehiclePose().position vehicle_orientation = client.simGetVehiclePose().orientation print("Pose is x {}, y {}, z {}".format(vehicle_pose.x_val, vehicle_pose.y_val, vehicle_pose.z_val)) # Find velocity API vehicle_linear_vel = client.simGetGroundTruthKinematics().linear_velocity vehicle_vel = sqrt(vehicle_linear_vel.x_val**2 + vehicle_linear_vel.y_val**2) #print("Velocity is {}".format(vehicle_vel)) # Resetting to origin api