def __init__(self,
setting_file = 'search_quadcopter.json',
reset_type = 'random', # random
action_type = 'discrete', # 'discrete', 'continuous'
observation_type = 'color', # 'color', 'depth', 'rgbd'
reward_type = 'distance', # distance
docker = False,
resolution=(84, 84)
):
print(setting_file)
setting = self.load_env_setting(setting_file)
self.docker = docker
self.reset_type = reset_type
self.roll = 0
self.pitch = 0
# start unreal env
self.unreal = env_unreal.RunUnreal(ENV_BIN=setting['env_bin'])
env_ip, env_port = self.unreal.start(docker,resolution)
# connect UnrealCV
self.unrealcv = Navigation(cam_id=self.cam_id,
port= env_port,
ip=env_ip,
env=self.unreal.path2env,
resolution=resolution)
# define action
self.action_type = action_type
assert self.action_type == 'discrete' or self.action_type == 'continuous'
if self.action_type == 'discrete':
self.action_space = spaces.Discrete(len(self.discrete_actions))
elif self.action_type == 'continuous':
self.action_space = spaces.Box(low = np.array(self.continous_actions['low']),high = np.array(self.continous_actions['high']))
self.count_steps = 0
# define observation space,
# color, depth, rgbd,...
self.observation_type = observation_type
assert self.observation_type == 'color' or self.observation_type == 'depth' or self.observation_type == 'rgbd'
self.observation_shape = self.unrealcv.define_observation(self.cam_id,self.observation_type)
# define reward type
# distance, bbox, bbox_distance,
self.reward_type = reward_type
self.reward_function = reward.Reward(setting)
self.rendering = False
# init augment env
if 'random' in self.reset_type or 'hide' in self.reset_type:
self.show_list = self.objects_env
self.hiden_list = random.sample(self.objects_env, min(15,len(self.objects_env)))
for x in self.hiden_list:
self.show_list.remove(x)
self.unrealcv.hide_obj(x)
def __init__(self,
setting_file = 'search_rr_plant78.json',
reset_type = 'waypoint', # testpoint, waypoint,
augment_env = None, #texture, target, light
test = True, # if True will use the test_xy as start point
action_type = 'discrete', # 'discrete', 'continuous'
observation_type = 'rgbd', # 'color', 'depth', 'rgbd'
reward_type = 'bbox', # distance, bbox, bbox_distance,
docker = False,
resolution = (160,120)
):
setting = self.load_env_setting(setting_file)
self.cam_id = 0
print('env_bin: ', setting.env_bin)
# run virtual enrionment in docker container
# self.docker = run_docker.RunDocker()
# env_ip, env_dir = self.docker.start(ENV_NAME=ENV_NAME)
# start unreal env
docker = False
# self.unreal = env_unreal.RunUnreal(ENV_BIN="house_withfloor/MyProject2/Binaries/Linux/MyProject2")
self.unreal = env_unreal.RunUnreal(ENV_BIN=setting.env_bin)
env_ip,env_port = self.unreal.start(docker,resolution)
# connect unrealcv client to server
# self.unrealcv = UnrealCv(self.cam_id, ip=env_ip, env=env_dir)
# connect UnrealCV
self.unrealcv = Navigation(cam_id=self.cam_id,
port= env_port,
ip=env_ip,
targets=self.target_list,
env=self.unreal.path2env,
resolution=resolution)
self.unrealcv.pitch = self.pitch
self.startpose = self.unrealcv.get_pose(self.cam_id)
print('start_pose: ', self.startpose)
#try hardcode start pose
self.startpose = [750.0, 295.0, 212.3,356.5,190.273, 0.0]
# ACTION: (linear velocity ,angle velocity)
self.ACTION_LIST = [
(20, 0),
(20, 15),
(20,-15),
(20, 30),
(20,-30),
]
self.count_steps = 0
self.max_steps = 100
self.target_pos = ( -60, 0, 50)
self.action_space = gym.spaces.Discrete(len(self.ACTION_LIST))
state = self.unrealcv.read_image(self.cam_id, 'lit')
self.observation_space = gym.spaces.Box(low=0, high=255, shape=state.shape)
def __init__(
self,
setting_file='depth_fusion.json',
reset_type='waypoint', # testpoint, waypoint,
augment_env=None, #texture, target, light
test=True, # if True will use the test_xy as start point
action_type='discrete', # 'discrete', 'continuous'
observation_type='rgbd', # 'color', 'depth', 'rgbd'
reward_type='bbox', # distance, bbox, bbox_distance,
docker=False,
# resolution = (84,84)
# resolution = (640,480),
resolution=(640, 480),
log_dir='log/'):
setting = self.load_env_setting(setting_file)
self.cam_id = 0
# self.reset_type = 'random'
self.reset_type = 'test'
self.log_dir = log_dir
# gt_pcl = pcl.load('house-000024-gt.ply')
# run virtual enrionment in docker container
# self.docker = run_docker.RunDocker()
# env_ip, env_dir = self.docker.start(ENV_NAME=ENV_NAME)
# start unreal env
docker = False
# self.unreal = env_unreal.RunUnreal(ENV_BIN="house_withfloor/MyProject2/Binaries/Linux/MyProject2")
self.unreal = env_unreal.RunUnreal(ENV_BIN=self.env_bin)
env_ip, env_port = self.unreal.start(docker, resolution)
# connect unrealcv client to server
# self.unrealcv = UnrealCv(self.cam_id, ip=env_ip, env=env_dir)
# connect UnrealCV
self.unrealcv = Navigation(cam_id=self.cam_id,
port=env_port,
ip=env_ip,
targets=self.target_list,
env=self.unreal.path2env,
resolution=resolution)
self.unrealcv.pitch = self.pitch
# define action
self.action_type = action_type
assert self.action_type == 'discrete' or self.action_type == 'continuous'
if self.action_type == 'discrete':
self.action_space = spaces.Discrete(len(self.discrete_actions))
elif self.action_type == 'continuous':
self.action_space = spaces.Box(
low=np.array(self.continous_actions['low']),
high=np.array(self.continous_actions['high']))
self.observation_type = observation_type
assert self.observation_type == 'color' or self.observation_type == 'depth' or self.observation_type == 'rgbd' or self.observation_type == 'gray'
self.observation_shape = self.unrealcv.define_observation(
self.cam_id, self.observation_type)
self.startpose = self.unrealcv.get_pose(self.cam_id)
#try hardcode start pose
# self.startpose = [750.0, 295.0, 212.3,356.5,190.273, 0.0]
# self.startpose = [0.0, 707.1068, 707.1067,0.0,270.0, -45.0] # [0,45,1000]
# self.startpose = [0.0,99.6,8.72,0.0,270.0,-5.0] #[for depth fusion] [0,5,100]
# self.startpose = [0.0,70.7,70.7,0.0,270.0,-45.0]
azimuth, elevation, distance = self.start_pose_rel
# print('start pose rel', azimuth,elevation,distance)
self.startpose = poseRelToAbs(azimuth, elevation, distance)
# print('start_pose: ', self.startpose)
''' create base frame '''
poseOrigin(self.log_dir + 'frame-{:06}.pose.txt'.format(1000))
# ACTION: (Azimuth, Elevation, Distance)
self.count_steps = 0
self.count_house_frames = 0
# self.max_steps = 35
self.target_pos = (-60, 0, 50)
self.gt_pclpose_prev = np.array(self.start_pose_rel)
# self.action_space = gym.spaces.Discrete(len(self.ACTION_LIST))
# state = self.unrealcv.read_image(self.cam_id, 'lit')
# self.observation_space = gym.spaces.Box(low=0, high=255, shape=state.shape)
self.nn_distance_module = tf.load_op_library(
'/home/daryl/gym-unrealcv/gym_unrealcv/envs/utils/tf_nndistance_so.so'
)
self.total_distance = 0
objects = self.unrealcv.get_objects()
# print('objects', objects)
self.houses = [(obj) for obj in objects if obj.startswith('BAT6_')]
# print('houses', self.houses)
for house in self.houses:
self.unrealcv.hide_obj(house)
self.house_id = 0
self.unrealcv.show_obj(self.houses[self.house_id])
gt_dir = '/hdd/AIRSCAN/datasets/house38_44/groundtruth/'
self.gt_pcl = []
for i in range(len(self.houses)):
gt_fn = gt_dir + self.houses[i] + '_sampled_10k.ply'
# print('gt', gt_fn)
# gt_pcl = pcl.load(gt_fn)
gt_pcl = o3d.read_point_cloud(gt_fn)
gt_pcl = np.asarray(gt_pcl.points, dtype=np.float32)
# gt_pcl = pcl.load('/home/daryl/datasets/BAT6_SETA_HOUSE8_WTR_sampled_10k.ply')
# gt_pcl = np.asarray(gt_pcl)
self.gt_pcl.append(np.expand_dims(gt_pcl, axis=0))
def __init__(
self,
setting_file='sfmHouseTarget.json',
reset_type='waypoint', # testpoint, waypoint,
augment_env=None, #texture, target, light
test=True, # if True will use the test_xy as start point
action_type='discrete', # 'discrete', 'continuous'
observation_type='rgbd', # 'color', 'depth', 'rgbd'
reward_type='bbox', # distance, bbox, bbox_distance,
docker=False,
# resolution = (320,240)
resolution=(640, 480)):
setting = self.load_env_setting(setting_file)
self.cam_id = 0
# run virtual enrionment in docker container
# self.docker = run_docker.RunDocker()
# env_ip, env_dir = self.docker.start(ENV_NAME=ENV_NAME)
# start unreal env
docker = False
# self.unreal = env_unreal.RunUnreal(ENV_BIN="house_withfloor/MyProject2/Binaries/Linux/MyProject2")
self.unreal = env_unreal.RunUnreal(ENV_BIN=self.env_bin)
env_ip, env_port = self.unreal.start(docker, resolution)
# connect unrealcv client to server
# self.unrealcv = UnrealCv(self.cam_id, ip=env_ip, env=env_dir)
# connect UnrealCV
self.unrealcv = Navigation(cam_id=self.cam_id,
port=env_port,
ip=env_ip,
targets=self.target_list,
env=self.unreal.path2env,
resolution=resolution)
self.unrealcv.pitch = self.pitch
# define action
self.action_type = action_type
assert self.action_type == 'discrete' or self.action_type == 'continuous'
if self.action_type == 'discrete':
self.action_space = spaces.Discrete(len(self.discrete_actions))
elif self.action_type == 'continuous':
self.action_space = spaces.Box(
low=np.array(self.continous_actions['low']),
high=np.array(self.continous_actions['high']))
self.startpose = self.unrealcv.get_pose(self.cam_id)
print('start_pose: ', self.startpose)
#try hardcode start pose
# self.startpose = [750.0, 295.0, 212.3,356.5,190.273, 0.0]
self.startpose = [0.0, 707.1068, 707.1067, 0.0, 270.0,
-45.0] # [0,45,1000]
# ACTION: (Azimuth, Elevation, Distance)
print("start")
object_mask = self.unrealcv.read_image(self.cam_id,
'object_mask',
mode='file')
cv2.imshow('object_mask', object_mask)
cv2.waitKey(0)
boxes = self.unrealcv.get_bboxes(object_mask, self.target_list)
# ACTION: (linear velocity ,angle velocity)
# self.ACTION_LIST = [
# (20, 0),
# (20, 15),
# (20,-15),
# (20, 30),
# (20,-30),
# ]
self.count_steps = 0
# self.max_steps = 35
self.target_pos = (-60, 0, 50)
# self.action_space = gym.spaces.Discrete(len(self.ACTION_LIST))
state = self.unrealcv.read_image(self.cam_id, 'lit')
self.observation_space = gym.spaces.Box(low=0,
high=255,
shape=state.shape)
self.reward_type = reward_type
self.reward_function = reward.Reward(setting)
def __init__(self,
setting_file,
category,
reset_type='waypoint', # testpoint, waypoint,
augment_env=None, # texture, target, light
action_type='Discrete', # 'Discrete', 'Continuous'
observation_type='Rgbd', # 'color', 'depth', 'rgbd'
reward_type='bbox', # distance, bbox, bbox_distance,
docker=False,
resolution=(160, 120)
):
#print(setting_file)
setting = self.load_env_setting(setting_file)
self.cam_id = setting['cam_id']
self.target_list = setting['targets'][category]
self.trigger_th = setting['trigger_th']
self.height = setting['height']
self.pitch = setting['pitch']
self.discrete_actions = setting['discrete_actions']
self.continous_actions = setting['continous_actions']
self.docker = docker
self.reset_type = reset_type
self.augment_env = augment_env
# start unreal env
self.unreal = env_unreal.RunUnreal(ENV_BIN=setting['env_bin'])
env_ip, env_port = self.unreal.start(docker, resolution)
# connect UnrealCV
self.unrealcv = Navigation(cam_id=self.cam_id,
port=env_port,
ip=env_ip,
targets=self.target_list,
env=self.unreal.path2env,
resolution=resolution)
self.unrealcv.pitch = self.pitch
# define action
self.action_type = action_type
assert self.action_type == 'Discrete' or self.action_type == 'Continuous'
if self.action_type == 'Discrete':
self.action_space = spaces.Discrete(len(self.discrete_actions))
elif self.action_type == 'Continuous':
self.action_space = spaces.Box(low=np.array(self.continous_actions['low']),
high=np.array(self.continous_actions['high']))
# define observation space,
# color, depth, rgbd,...
self.observation_type = observation_type
assert self.observation_type == 'Color' or self.observation_type == 'Depth' or self.observation_type == 'Rgbd'
self.observation_space = self.unrealcv.define_observation(self.cam_id, self.observation_type, 'direct')
# define reward type
# distance, bbox, bbox_distance,
self.reward_type = reward_type
self.reward_function = reward.Reward(setting)
# set start position
self.trigger_count = 0
current_pose = self.unrealcv.get_pose(self.cam_id)
current_pose[2] = self.height
self.unrealcv.set_location(self.cam_id, current_pose[:3])
self.count_steps = 0
self.targets_pos = self.unrealcv.build_pose_dic(self.target_list)
# for reset point generation and selection
self.reset_module = reset_point.ResetPoint(setting, reset_type, current_pose)
def __init__(self,
setting_file = 'depth_fusion.json',
reset_type = 'waypoint', # testpoint, waypoint,
augment_env = None, #texture, target, light
test = True, # if True will use the test_xy as start point
action_type = 'discrete', # 'discrete', 'continuous'
observation_type = 'rgbd', # 'color', 'depth', 'rgbd'
reward_type = 'bbox', # distance, bbox, bbox_distance,
docker = False,
resolution = (640,480),
log_dir='log/'
):
setting = self.load_env_setting(setting_file)
self.cam_id = 0
# self.reset_type = 'random'
self.reset_type = 'test'
self.log_dir = log_dir
gt_pcl = o3d.read_point_cloud('/home/daryl/gym-unrealcv/BAT6_SETA_HOUSE44_OBJ_No_InnerMesh_sampled_10k.ply')
self.gt_pcl = np.asarray(gt_pcl.points, dtype=np.float32)
# self.gt_pcl = np.asarray(gt_pcl)
self.gt_pcl = np.expand_dims(self.gt_pcl,axis=0)
# start unreal env
docker = False
self.unreal = env_unreal.RunUnreal(ENV_BIN=self.env_bin)
env_ip,env_port = self.unreal.start(docker,resolution)
# connect UnrealCV
self.unrealcv = Navigation(cam_id=self.cam_id,
port= env_port,
ip=env_ip,
targets=self.target_list,
env=self.unreal.path2env,
resolution=resolution)
self.unrealcv.pitch = self.pitch
# define action
self.action_type = action_type
assert self.action_type == 'discrete' or self.action_type == 'continuous'
if self.action_type == 'discrete':
self.action_space = spaces.Discrete(len(self.discrete_actions))
elif self.action_type == 'continuous':
self.action_space = spaces.Box(low = np.array(self.continous_actions['low']),high = np.array(self.continous_actions['high']))
self.observation_type = observation_type
assert self.observation_type == 'color' or self.observation_type == 'depth' or self.observation_type == 'rgbd' or self.observation_type == 'gray'
self.observation_shape = self.unrealcv.define_observation(self.cam_id,self.observation_type)
self.startpose = self.unrealcv.get_pose(self.cam_id)
azimuth, elevation, distance = self.start_pose_rel
# print('start pose rel', azimuth,elevation,distance)
self.startpose = poseRelToAbs(azimuth, elevation, distance)
# print('start_pose: ', self.startpose)
''' create base frame '''
poseOrigin(self.log_dir+'frame-{:06}.pose.txt'.format(1000))
# ACTION: (Azimuth, Elevation, Distance)
self.count_steps = 0
self.depth_count = 0
self.depth_thresh = 0.5
self.total_distance = 0
# self.max_steps = 35
self.target_pos = ( -60, 0, 50)
self.pose_prev = np.array(self.start_pose_rel)
self.nn_distance_module =tf.load_op_library(self.nn_distance_path)
def __init__(self,
setting_file = 'search_rr_plant78.json',
reset_type = 'waypoint', # testpoint, waypoint,
test = True, # if True will use the test_xy as start point
action_type = 'discrete', # 'discrete', 'continuous'
observation_type = 'rgbd', # 'color', 'depth', 'rgbd'
reward_type = 'bbox', # distance, bbox, bbox_distance,
docker = False,
):
setting = self.load_env_setting(setting_file)
self.test = test
self.docker = docker
self.reset_type = reset_type
# start unreal env
self.unreal = env_unreal.RunUnreal(ENV_BIN=setting['env_bin'])
env_ip,env_port = self.unreal.start(docker)
# connect UnrealCV
self.unrealcv = Navigation(cam_id=self.cam_id,
port= env_port,
ip=env_ip,
targets=self.target_list,
env=self.unreal.path2env)
self.unrealcv.pitch = self.pitch
# define action
self.action_type = action_type
assert self.action_type == 'discrete' or self.action_type == 'continuous'
if self.action_type == 'discrete':
self.action_space = spaces.Discrete(len(self.discrete_actions))
elif self.action_type == 'continuous':
self.action_space = spaces.Box(low = np.array(self.continous_actions['low']),high = np.array(self.continous_actions['high']))
self.count_steps = 0
self.targets_pos = self.unrealcv.build_pose_dic(self.target_list)
# define observation space,
# color, depth, rgbd,...
self.observation_type = observation_type
assert self.observation_type == 'color' or self.observation_type == 'depth' or self.observation_type == 'rgbd'
if self.observation_type == 'color':
state = self.unrealcv.read_image(self.cam_id,'lit')
self.observation_space = spaces.Box(low=0, high=255, shape=state.shape)
elif self.observation_type == 'depth':
state = self.unrealcv.read_depth(self.cam_id)
self.observation_space = spaces.Box(low=0, high=10, shape=state.shape)
elif self.observation_type == 'rgbd':
state = self.unrealcv.get_rgbd(self.cam_id)
s_high = state
s_high[:,:,-1] = 10.0 #max_depth
s_high[:,:,:-1] = 255 #max_rgb
s_low = np.zeros(state.shape)
self.observation_space = spaces.Box(low=s_low, high=s_high)
# define reward type
# distance, bbox, bbox_distance,
self.reward_type = reward_type
# set start position
self.trigger_count = 0
current_pose = self.unrealcv.get_pose(self.cam_id)
current_pose[2] = self.height
self.unrealcv.set_location(self.cam_id,current_pose[:3])
self.trajectory = []
# for reset point generation and selection
self.reset_module = reset_point.ResetPoint(setting, reset_type, test, current_pose)
self.reward_function = reward.Reward(setting)
self.reward_function.dis2target_last, self.targetID_last = self.select_target_by_distance(current_pose, self.targets_pos)
self.rendering = False