def render_from_file(meta_data, max_indx=5): clear_env() car_shape, car_rot, car_trans, cam_location, cam_rotation, img_fov, img_size, kpt_dict = load_meta_data( meta_data) request('vset /camera/1/location {:.6f} {:.6f} {:.6f}'.format( cam_location[0], cam_location[1], cam_location[2])) request('vset /camera/1/rotation {:.6f} {:.6f} {:.6f}'.format( cam_rotation[0], cam_rotation[1], cam_rotation[2])) request('vset /camera/1/fov {:.6f}'.format(img_fov)) car_group = Car_Manager() if max_indx <= 0: max_indx = max_indx + len(car_shape) car_color = [[i, 0, 0] for i in range(len(car_trans))] print(len(car_shape)) for i, shape in enumerate(car_shape): if i >= max_indx: break car_group.add_car(shape) car_group.trans_car(car_trans[i], "car{}".format(i)) #car_group.trans_car([0,0,0], "car{}".format(i)) car_group.rot_car(car_rot[i], "car{}".format(i)) car_group.annotate_car(car_color[i], "car{}".format(i)) car_group.flush() cwd_root, _ = os.path.split(meta_data) base_dir = join(cwd_root, "render_res") if DEBUG: print("Finished write object pose data ") time.sleep(0.1) if DEBUG: print("Acquiring image ...") img = read_png(request('vget /camera/1/lit png')) cv2.imwrite(join(base_dir, "car_arrangment.png"), img[:, :, 2::-1]) mask = read_png(request('vget /camera/1/object_mask png')) cv2.imwrite(join(base_dir, "car_mask.png"), mask[:, :, 2::-1]) png = read_png(request('vget /camera/0/lit png')) cv2.imwrite(join(base_dir, "overview.png"), png[:, :, 2::-1]) for i in range(len(car_color)): obj_mask = udb.get_mask(mask, car_color[i]) [ys, xs] = np.where(obj_mask) bbox = [min(xs), max(xs), min(ys), max(ys)] print(bbox) obj_img = udb.mask_img(img, obj_mask) bbox_img = img[min(ys):max(ys), min(xs):max(xs), :] cv2.imwrite(join(base_dir, "car%d_seg.png" % i), obj_img[:, :, 2::-1]) cv2.imwrite(join(base_dir, "car%d_bbox.png" % i), bbox_img[:, :, 2::-1]) if DEBUG: print("Finished write image to files") return True
def get_image(obj, dist, az, el): client.request( 'vset /animal/{obj}/camera {dist} {az} {el}'.format(**locals())) lit = client.request('vget /animal/{obj}/image'.format(**locals())) seg = client.request('vget /animal/{obj}/seg'.format(**locals())) depth = client.request('vget /animal/{obj}/depth'.format(**locals())) lit = read_png(lit) seg = read_png(seg) depth = read_npy(depth) return lit, seg, depth
def onpress(event): rot_offset = 10 # Rotate 5 degree for each key press loc_offset = 10 # Move 5.0 when press a key # Up and Down in cam-plane if event.key == 'w': loc[0] += loc_offset if event.key == 's': loc[0] -= loc_offset # Left and Right movement in cam-plane if event.key == 'a': loc[1] -= loc_offset if event.key == 'd': loc[1] += loc_offset # In and Out movement into cam-plane if event.key == 'q': loc[2] += loc_offset if event.key == 'e': loc[2] -= loc_offset # cmd = 'vset /camera/0/rotation %s' % ' '.join([str(v) for v in rot]) # client.request(cmd) cmd = 'vset /camera/0/moveto %s' % ' '.join([str(v) for v in loc]) client.request(cmd) print(client.request('vget /camera/0/location')) print(client.request('vget /camera/0/rotation')) res = client.request('vget /camera/2/lit png') img = read_png(res) ax.imshow(img) fig.canvas.draw()
def onpress(event): rot_offset = 10 # Rotate 5 degree for each key press loc_offset = 10 # Move 5.0 when press a key if event.key == 'a': rot[1] -= rot_offset if event.key == 'd': rot[1] += rot_offset if event.key == 'q': loc[2] += loc_offset # Move up if event.key == 'e': loc[2] -= loc_offset # Move down if event.key == 'w': loc[1] -= loc_offset if event.key == 's': loc[1] += loc_offset if event.key == 'up': loc[1] -= loc_offset if event.key == 'down': loc[1] += loc_offset if event.key == 'left': loc[0] -= loc_offset if event.key == 'right': loc[0] += loc_offset cmd = 'vset /camera/0/rotation %s' % ' '.join([str(v) for v in rot]) client.request(cmd) cmd = 'vset /camera/0/location %s' % ' '.join([str(v) for v in loc]) client.request(cmd) res = client.request('vget /camera/0/lit png') img = read_png(res) # print(event.key) # print('Requested image %s' % str(img.shape)) ax.imshow(img) fig.canvas.draw()
def onpress(event): rot_offset = 10 # Rotate 5 degree for each key press loc_offset = 10 # Move 5.0 when press a key if event.key == 'a': rot[1] -= rot_offset if event.key == 'd': rot[1] += rot_offset if event.key == 'q': loc[2] += loc_offset # Move up if event.key == 'e': loc[2] -= loc_offset # Move down if event.key == 'w': loc[1] -= loc_offset if event.key == 's': loc[1] += loc_offset if event.key == 'up': loc[1] -= loc_offset if event.key == 'down': loc[1] += loc_offset if event.key == 'left': loc[0] -= loc_offset if event.key == 'right': loc[0] += loc_offset cmd = 'vset /camera/0/rotation %s' % ' '.join([str(v) for v in rot]) client.request(cmd) cmd = 'vset /camera/0/location %s' % ' '.join([str(v) for v in loc]) client.request(cmd) res = client.request('vget /camera/0/lit png') img = read_png(res) # print(event.key) # print('Requested image %s' % str(img.shape)) ax.imshow(img) fig.canvas.draw()
def rotate_around_object(obj_loc: tuple, d: tuple, angle: tuple, y_max: int, view_modes: list)-> dict: """ Rotates camera around obj_loc and return images from 360 view around for each view mode. Client has to be connected. :param obj_loc: 3-tuple, coordinates of the object location (x,y,z) :param d: 2-tuple, distance of camera from obj_loc (distance on x axis, distance on z axis) :param angle: 2-tuple, angle step by which is camera rotated around z and y axis (z step, y step) :param y_max: top y angle value by which can camera be rotated above object :param view_modes: list of strings representing requested view modes. Allowed are lit, normal, depth, object_mask. :return: Dictionary with given view modes as keys and lists of images as values. """ output = {} for view_mode in view_modes: output[view_mode] = [] t = get_translation_matrix([-obj_loc[0], -obj_loc[1], -obj_loc[2]]) t_neg = get_translation_matrix([obj_loc[0], obj_loc[1], obj_loc[2]]) cam_start_loc = [obj_loc[0] - d[0], obj_loc[1], obj_loc[2] + d[1]] loc_orig = np.matmul(t, np.array([[cam_start_loc[0]], [cam_start_loc[1]], [cam_start_loc[2]], [1.0]])) h_rot = list( zip([get_rotation_z_axis(x) for x in range(0, 360, angle[0])], [x for x in range(0, 360, angle[0])])) y_angles = [y for y in range(0, -y_max, -angle[1])] v_rot = list(zip([get_rotation_y_axis(y) for y in range(0, len(y_angles) * angle[1], angle[1])], y_angles)) for v_params in v_rot: for h_params in h_rot: new_loc = np.round(np.matmul(t_neg, np.matmul(h_params[0], np.matmul(v_params[0], loc_orig)))) assert client.request('vset /camera/0/rotation {} {} {}'.format(v_params[1], h_params[1] % 360, 0)) == 'ok', \ 'Did not get \'ok\' response from urealcv server for setting camera rotation' assert client.request('vset /camera/0/location {} {} {}'.format(new_loc[0][0], new_loc[1][0], new_loc[2][0])) == 'ok', \ 'Did not get \'ok\' response from urealcv server for setting camera location' for view_mode in view_modes: output[view_mode].append(read_png(client.request('vget /camera/0/' + view_mode + ' png'))) return output
def run_commands_demo(logger): time.sleep(10) logger.request('vget /unrealcv/status') logger.request('vget /unrealcv/version') logger.request('vget /scene/name') res = logger.request('vget /camera/0/location') res = logger.request('vget /camera/0/rotation') # res = logger.request('vset /camera/0/location -162 -126 90') # res = logger.request('vset /camera/0/rotation 348 60 0') res = logger.request('vget /camera/0/lit png', log=False) img = read_png(res) res = logger.request('vget /camera/0/depth npy', log=False) depth = read_npy(res) clip_far = np.median(depth) * 5 # use median instead of mean logger.writelines('Before clip, max=%f, mean=%f' % (depth.max(), depth.mean())) depth[depth > clip_far] = clip_far logger.writelines('Before clip, max=%f, mean=%f' % (depth.max(), depth.mean())) # Trancate the depth of the window and sky to make it easier to see res = logger.request('vget /camera/0/normal npy', log=False) normal = read_npy(res) res = logger.request('vget /camera/0/object_mask png', log=False) object_mask = read_png(res) logger.save_image("img.png", img) logger.save_image("depth.png", normalize(depth)) logger.save_image("surface_normal.png", normalize(normal)) logger.save_image("object_mask.png", object_mask) res = logger.request('vget /camera/0/lit lit.png') res = logger.request('vget /camera/0/depth depth.png') res = logger.request('vget /camera/0/object_mask object_mask.png') res = logger.request('vget /objects', log=False) object_names = res.split(' ') logger.writelines(object_names[:5])
def run_commands_demo(logger): time.sleep(10) logger.request('vget /unrealcv/status') logger.request('vget /unrealcv/version') logger.request('vget /scene/name') res = logger.request('vget /camera/0/location') res = logger.request('vget /camera/0/rotation') # res = logger.request('vset /camera/0/location -162 -126 90') # res = logger.request('vset /camera/0/rotation 348 60 0') res = logger.request('vget /camera/0/lit png', log = False) img = read_png(res) res = logger.request('vget /camera/0/depth npy', log = False) depth = read_npy(res) clip_far = np.median(depth) * 5 # use median instead of mean logger.writelines('Before clip, max=%f, mean=%f' % (depth.max(), depth.mean())) depth[depth > clip_far] = clip_far logger.writelines('Before clip, max=%f, mean=%f' % (depth.max(), depth.mean())) # Trancate the depth of the window and sky to make it easier to see res = logger.request('vget /camera/0/normal npy', log = False) normal = read_npy(res) res = logger.request('vget /camera/0/object_mask png', log = False) object_mask = read_png(res) logger.save_image("img.png", img) logger.save_image("depth.png", normalize(depth)) logger.save_image("surface_normal.png", normalize(normal)) logger.save_image("object_mask.png", object_mask) res = logger.request('vget /camera/0/lit lit.png') res = logger.request('vget /camera/0/depth depth.png') res = logger.request('vget /camera/0/object_mask object_mask.png') res = logger.request('vget /objects', log = False) object_names = res.split(' ') logger.writelines(object_names[:5])
def get_rgb_and_depth(self, loc, rot, verbose=False): # set camera to loc and rot self.vset(loc, rot, verbose=verbose) res = client.request('vget /camera/0/lit png') img = read_png(res) if verbose: print(res) res = client.request('vget /camera/0/depth npy') depth = read_npy(res) if verbose: print(res) return img, depth
def sample_anim_frames(self, obj_id, mesh_paths, anim_paths, ratios): # Render images using cartesian product of parameters self.make_obj('CvCharacter', obj_id, [0, 0, 138]) self.cam_track_obj(obj_id, 0, 30, 200) ims = [] params = list(itertools.product(mesh_paths, anim_paths, ratios)) for param in tqdm(params): mesh_path, anim_path, ratio = param self.request( 'vset /human/{obj_id}/mesh {mesh_path}'.format(**locals())) self.request( 'vset /human/{obj_id}/animation/ratio {anim_path} {ratio}'. format(**locals())) res = self.request('vget /camera/0/lit png') im = read_png(res) ims.append(im) return ims
rgba = tuple([int(color[2:]) for color in tmp_color]) actor_color_dict[actor_name] = rgba for target_name in target_name_list: if actor_name.startswith(target_name): if target_name not in target_actor_dict: target_actor_dict[target_name] = [actor_name] else: target_actor_dict[target_name].append(actor_name) break # save to png file render_file_path = output_folder + 'render' + img_file_id + '.png' mask_file_path = output_folder + 'mask' + img_file_id + '.png' client.request('vget /camera/' + camera_id + '/lit ' + render_file_path) mask_res = client.request('vget /camera/' + camera_id + '/object_mask png') mask_array = read_png(mask_res) cv2.imwrite(mask_file_path, mask_array) for target in target_actor_dict: target_rect_dict[target] = [] actor_list = target_actor_dict[target] for actor_name in actor_list: current_target = np.all((mask_array == actor_color_dict[actor_name]), axis=-1).astype(np.uint8) _, contours, _ = cv2.findContours(current_target, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) # top_left x, y, width, height rect_info = cv2.boundingRect(contours[0]) target_rect_dict[target].append(rect_info) full_info_dict = {}
def capture_img(self): res = self.client.request('vget /camera/{id}/lit png'.format(id = self.id)) img = read_png(res) return img
def get_mask(cam_ind=0): ## TODO: 052319, Currently not working. res = client.request('vget /camera/{cam_ind}/object_mask png'.format(**locals())) object_mask = read_png(res) return object_mask
def get_rgb(cam_ind=0): res = client.request('vget /camera/{:d}/lit png'.format(cam_ind)) img = read_png(res) return img
from unrealcv import client from unrealcv.util import read_png, read_npy import argparse import imageio import numpy as np parser = argparse.ArgumentParser() parser.add_argument('-n', type=int, help='Number of iterations') parser.add_argument('--cam_id', type=int, help='Camera id for data capture') args = parser.parse_args() client.connect() if args.cam_id: cam_ids = [args.cam_id] else: cams = client.request('vget /cameras').strip().split(' ') print(cams) cam_ids = range(len(cams)) for i in range(args.n): for cam_id in cam_ids: res = client.request('vget /camera/{cam_id}/lit png'.format(**locals())) imageio.imsave('tmp/lit_{cam_id}.png'.format(**locals()), read_png(res)) # client.request('vget /camera/{cam_id}/depth npy'.format(**locals())) # np.save('tmp/depth_{cam_id}.npy'.format(**locals()), read_npy(res)) res = client.request('vget /camera/{cam_id}/object_mask png'.format(**locals())) imageio.imsave('tmp/seg_{cam_id}.png'.format(**locals()), read_png(res))
import imageio import os import argparse parser = argparse.ArgumentParser() parser.add_argument('img_filename') args = parser.parse_args() fname = args.img_filename client.connect() client.request('vset /arm/owi535/random_pose') # Set the arm to a random pose # Or set the arm to a specific pose with # client.request('vset /arm/owi535/pose 20 20 20 20 20') client.request('vset /camera/0/location 320 0 300') client.request('vset /camera/0/rotation -30 180 0') res = client.request('vget /camera/0/lit png') im = read_png(res) texture_filename = os.path.abspath('./000000000139.jpg') client.request('vset /env/sky/texture %s' % texture_filename) client.request('vset /env/floor/texture %s' % texture_filename) client.request('vset /env/random_lighting') imageio.imwrite(fname, im) keypoints = client.request('vget /arm/owi535/keypoints') print(keypoints) # Dense 3D keypoint in the world space.
def get_image(dist, az, el): client.request('vset /car/0/camera {dist} {az} {el}'.format(**locals())) lit = client.request('vget /car/0/image'.format(**locals())) lit = read_png(lit) return lit
def get_rgb(self): id = self.id res = self.request('vget /camera/{id}/lit png'.format(**locals())) return read_png(res)
def capture_frame(func=None): res = client.request('vget /camera/0/lit png') im = read_png(res) return im
def get_img_from_preds(preds): set_camrea(preds) i = 1 data = client.request(f'vget /camera/{i}/lit png') im = read_png(data) return im
from unrealcv import client client.connect() from unrealcv.util import read_png import imageio client.request('vset /car/0/mesh/id suv') # The id can be selected from # [suv, sedan2door, sedan4door, hybrid, hatchback] client.request('vset /car/0/door/angles 30 30 30 30 30 30') # The six values are associated to fl (front left), fr, bl, br, hood, trunk img = read_png(client.request('vget /car/0/image')) imageio.imwrite('car.png', img)
def get_img_from_preds(preds): set_camrea(preds) i = 1 data = client.request('vget /camera/{i}/lit png'.format(**locals())) im = read_png(data) return im
def request_image(self): image_data = ue4.request(f"vget /camera/0/lit png") return read_png(image_data)
def get_img(self): obj_id = self.id img = self.request('vget /car/{obj_id}/image'.format(**locals())) img = read_png(img) return img
def get_seg(): seg = client.request('vget /car/0/seg'.format(**locals())) seg = read_png(seg) return seg
def get_seg(self): obj_id = self.id seg = self.request('vget /car/{obj_id}/seg'.format(**locals())) seg = read_png(seg) return seg
x=camera_traj[i]['location'][0], y=camera_traj[i]['location'][1], z=camera_traj[i]['location'][2])) client.request('vset /camera/0/rotation {x} {y} {z}'.format( x=camera_traj[i]['rotation'][0], y=camera_traj[i]['rotation'][1], z=camera_traj[i]['rotation'][2])) res_depth = client.request('vget /camera/0/depth npy') res_lit = client.request('vget /camera/0/lit png') res_object = client.request('vget /camera/0/object_mask png') res_normal = client.request('vget /camera/0/normal png') im_depth = read_npy(res_depth) #由於深度會有超過5000以上的值,圖會有白色的部分,因此把他指定為0,會轉為黑色 im_depth[im_depth > 5000] = 0 im_lit = read_png(res_lit) im_object = read_png(res_object) im_normal = read_png(res_normal) pil_img_depth = Image.fromarray(im_depth) pil_img_lit = Image.fromarray(im_lit) pil_img_object = Image.fromarray(im_object) pil_img_normal = Image.fromarray(im_normal) pil_img_depth = pil_img_depth.convert('RGB') # pil_img_depth.save('./data_arch/depth_image/depth_image_{i}.png'.format(i=i)) pil_img_depth.save(depth_image_path + '/depth_image_{i}.png'.format(i=i)) # np.save('./data_arch/depth/depth_{i}'.format(i=i),im_depth) np.save(depth_path + '/dapth_{i}'.format(i=i), im_depth) # pil_img_lit.save('./data_arch/lit/lit_{i}.png'.format(i=i)) pil_img_lit.save(lit_path + '/lit_{i}.png'.format(i=i))
def get_mask(self): id = self.id res = self.request( 'vget /camera/{id}/object_mask png'.format(**locals())) return read_png(res)
def render_from_file(meta_data, max_indx=5): clear_env() car_shape, car_rot, car_trans, cam_location, cam_rotation, img_fov, img_size, kpt_dict = load_meta_data( meta_data) request('vset /camera/1/location {:.6f} {:.6f} {:.6f}'.format( cam_location[0], cam_location[1], cam_location[2])) request('vset /camera/1/rotation {:.6f} {:.6f} {:.6f}'.format( cam_rotation[0], cam_rotation[1], cam_rotation[2])) request('vset /camera/1/fov {:.6f}'.format(img_fov)) car_group = Car_Manager() # if max_indx <= 0: # max_indx = max_indx + len(car_shape) #num_obj = min([len(car_trans), max_indx, 2]) num_obj = 2 car_color = [[i, 0, 0] for i in range(num_obj)] shape = car_shape[0] # for i in range(num_obj): # car_group.add_car(shape, shape_lib="ShapenetKeypoint") # #car_group.trans_car(car_trans[i], "car{}".format(i)) # car_group.trans_car([0, 100*i,150*i+100], "car{}".format(i)) # car_group.rot_car([0,0,0], "car{}".format(i)) # car_group.annotate_car(car_color[i], "car{}".format(i)) with open("example/car_activity/render_shape.json", "r") as f: data_render = json.load(f) trans_model = data_render[shape]["trans"] i = 0 car_group.add_car(shape, scale=(-1, 1, 1)) print(trans_model) car_group.trans_car([ trans_model[0] * 500, trans_model[2] * 500, 200 + trans_model[1] * 500 ], "car{}".format(i)) car_group.rot_car([0, 90, 0], "car{}".format(i)) car_group.annotate_car(car_color[i], "car{}".format(i)) i = 1 car_group.add_car(shape, shape_lib="ShapenetKeypoint") car_group.trans_car([0, 0, 200], "car{}".format(i)) car_group.rot_car([0, 0, 0], "car{}".format(i)) car_group.annotate_car(car_color[i], "car{}".format(i)) car_group.flush() cwd_root, _ = os.path.split(meta_data) base_dir = join(cwd_root, "render_res") if DEBUG: print("Finished write object pose data ") time.sleep(0.1) if DEBUG: print("Acquiring image ...") img = read_png(request('vget /camera/1/lit png')) cv2.imwrite(join(base_dir, "car_arrangment.png"), img[:, :, 2::-1]) mask = read_png(request('vget /camera/1/object_mask png')) cv2.imwrite(join(base_dir, "car_mask.png"), mask[:, :, 2::-1]) png = read_png(request('vget /camera/0/lit png')) cv2.imwrite(join(base_dir, "overview.png"), png[:, :, 2::-1]) for i in range(len(car_color)): obj_mask = udb.get_mask(mask, car_color[i]) [ys, xs] = np.where(obj_mask) bbox = [min(xs), max(xs), min(ys), max(ys)] print(bbox) obj_img = udb.mask_img(img, obj_mask) bbox_img = img[min(ys):max(ys), min(xs):max(xs), :] cv2.imwrite(join(base_dir, "car%d_seg.png" % i), obj_img[:, :, 2::-1]) cv2.imwrite(join(base_dir, "car%d_bbox.png" % i), bbox_img[:, :, 2::-1]) if DEBUG: print("Finished write image to files") return True
def capture_frame(func=None): res = client.request('vget /camera/0/lit png') im = read_png(res) return im
def get_im(self): res = self.request('vget /camera/0/lit png') return read_png(res)
from unrealcv import clientt from unrealcv.util import read_png import imageio client.connect() im = read_png(client.request('vget /camera/0/lit png')) imageio.imwrite('hello_world.png', im)
def get_seg(self): res = self.request('vget /camera/0/object_mask png') return read_png(res)