def rollout(env,
agents,
max_path_length=np.inf,
animated=False,
speedup=1,
always_return_paths=False):
##############################################
# SEEK AGENT
seek_observations = []
seek_actions = []
seek_rewards = []
seek_agent_infos = []
seek_env_infos = []
obs = env.reset()
agents['seek'].reset()
seek_path_length = 0
if animated:
env.render()
while seek_path_length < max_path_length:
# print('rollout: obs shape = ', obs[0].shape)
a, agent_info = agents['seek'].get_action(obs)
if animated:
env.render()
obs_next, r, d, env_info = env.step(a)
seek_observations.append(obs)
seek_rewards.append(r)
seek_actions.append(env.action_space.flatten(a))
seek_agent_infos.append(agent_info)
seek_env_infos.append(env_info)
seek_path_length += 1
obs = obs_next
if d:
break
if animated and not always_return_paths:
return
seek_paths = dict(
observations=e2e_tensor_utils.stack_tensor_list(seek_observations),
actions=tensor_utils.stack_tensor_list(seek_actions),
rewards=tensor_utils.stack_tensor_list(seek_rewards),
agent_infos=tensor_utils.stack_tensor_dict_list(seek_agent_infos),
env_infos=tensor_utils.stack_tensor_dict_list(seek_env_infos),
)
seek_paths['actions'] = seek_paths['actions'].astype(glob_config.dtype)
seek_paths['rewards'] = seek_paths['rewards'].astype(glob_config.dtype)
return {'seek': seek_paths}
def rollout_hide(env,
agents,
mode,
max_path_length=np.inf,
init_state=None,
init_goal=None):
# Reset the model configuration
env.reset()
obs = env.reload_model(pose=init_state, goal=init_goal)
hide_observations = []
hide_states = []
hide_actions = []
hide_rewards = []
hide_agent_infos = []
hide_env_infos = []
print("rollout : HIDE")
agents['hide'].reset()
hide_path_length = 0
while hide_path_length < max_path_length:
env.render()
a, agent_info = agents['hide'].get_action(obs)
hide_states.append(env.unwrapped.get_all_pose())
#print('-------> state: ', obs[0:2])
obs_next, r, d, env_info = env.step(a)
hide_observations.append(obs)
hide_rewards.append(r)
hide_actions.append(a)
hide_agent_infos.append(agent_info)
hide_env_infos.append(env_info)
hide_path_length += 1
obs = obs_next
if d:
break
hide_paths = dict(
observations=e2e_tensor_utils.stack_tensor_list(hide_observations),
actions=tensor_utils.stack_tensor_list(hide_actions),
rewards=tensor_utils.stack_tensor_list(hide_rewards),
agent_infos=tensor_utils.stack_tensor_dict_list(hide_agent_infos),
env_infos=tensor_utils.stack_tensor_dict_list(hide_env_infos),
states=hide_states,
)
print('Episode done:', hide_path_length)
return hide_paths
def rollout_seek(env, agents, mode, max_path_length=np.inf):
seek_observations = []
seek_actions = []
seek_rewards = []
seek_agent_infos = []
seek_env_infos = []
print("rollout : SEEK")
obs = env.reset()
agents['seek'].reset()
seek_path_length = 0
while seek_path_length < max_path_length:
env.render()
a, agent_info = agents['seek'].get_action(obs)
obs_next, r, d, env_info = env.step(a)
seek_observations.append(obs)
seek_rewards.append(r)
seek_actions.append(a)
seek_agent_infos.append(agent_info)
seek_env_infos.append(env_info)
seek_path_length += 1
obs = obs_next
if d:
break
seek_paths = dict(
observations=e2e_tensor_utils.stack_tensor_list(seek_observations),
actions=tensor_utils.stack_tensor_list(seek_actions),
rewards=tensor_utils.stack_tensor_list(seek_rewards),
agent_infos=tensor_utils.stack_tensor_dict_list(seek_agent_infos),
env_infos=tensor_utils.stack_tensor_dict_list(seek_env_infos),
)
seek_paths['actions'] = seek_paths['actions'].astype(glob_config.dtype)
seek_paths['rewards'] = seek_paths['rewards'].astype(glob_config.dtype)
return {'seek': seek_paths}
def rollout_brownian(env,
agents,
max_path_length=np.inf,
animated=False,
speedup=1,
always_return_paths=False,
mode=None,
hide_tmax=None):
##############################################
## HIDE AGENT
env.reset()
# 현재 agent['hide'].starts에서 random하게 하나를 뽑는다.
start_pose, start_pose_id = agents['hide'].sample_one_start()
start_pose = np.array(start_pose)
# 현재 agent['hide'].starts에서 p의 확률로 agent['hide'].starts_old에서 1-p의 확률로 random하게 하나를 뽑는다.
goal, goal_id = agents['hide'].sample_one_goal()
obs = env.env.env.reload_model(pose=start_pose, goal=goal)
# print("++++++++++++++++++++++++++++++++++++")
# print('start_pose:', start_pose[0][0:2], ' goal:', goal[0][0:2])
# print('start_pose:', np.array(obs[0][0:2]) * 2.4, ' goal:',np.array(obs[0][-2:]) * 2.4)
##############################################
## SEEK AGENT
# print('rollout: Student')
if animated:
env.render()
# env.env.unwrapped.frame_skip = 10
if mode is not None:
if mode == 'seek_force_only':
env.env.env.use_stop = False
env.env.env.add_mnist_reward(True)
env.env.env.use_mnist_stop_criteria(True)
elif mode == 'reach_center_and_stop':
env.env.env.use_stop = False
env.env.env.use_distance2center_stop_criteria = True
seek_observations = []
seek_actions = []
seek_rewards = []
seek_agent_infos = []
seek_env_infos = []
agents['seek'].reset()
seek_path_length = 0
if animated:
env.render()
while seek_path_length < max_path_length:
# if seek_path_length < 2: print('seek obs: ', obs)
a, agent_info = agents['seek'].get_action(obs)
# print('action:',a)
if animated:
env.render()
obs_next, r, d, env_info = env.step(a)
seek_observations.append(obs)
seek_rewards.append(r)
seek_actions.append(env.action_space.flatten(a))
seek_agent_infos.append(agent_info)
seek_env_infos.append(env_info)
seek_path_length += 1
if d:
print('SEEK| path_length:', len(seek_rewards))
break
obs = obs_next
# print('step seek')
## Here we assigning if the goal was reached for a particular goal
step_limit = env.env.unwrapped.step_limit
goal_reached = float(seek_path_length < step_limit)
# starts값에 대한 reward값 저장
if agents['hide'].reverse_mode:
agents['hide'].rewards[start_pose_id].append(goal_reached)
else:
agents['hide'].rewards[goal_id].append(goal_reached)
if animated and not always_return_paths:
return
seek_paths = dict(
observations=e2e_tensor_utils.stack_tensor_list(seek_observations),
actions=tensor_utils.stack_tensor_list(seek_actions),
rewards=tensor_utils.stack_tensor_list(seek_rewards),
agent_infos=tensor_utils.stack_tensor_dict_list(seek_agent_infos),
env_infos=tensor_utils.stack_tensor_dict_list(seek_env_infos),
)
seek_paths['actions'] = seek_paths['actions'].astype(glob_config.dtype)
seek_paths['rewards'] = seek_paths['rewards'].astype(glob_config.dtype)
return {'seek': seek_paths}
def rollout_debug(env,
agents,
max_path_length=np.inf,
animated=False,
speedup=1,
always_return_paths=False):
##############################################
# SEEK AGENT
# env.env.unwrapped.reload_model(pose=last_pose)
animated = True
always_return_paths = True
seek_observations = []
seek_actions = []
seek_rewards = []
seek_agent_infos = []
seek_env_infos = []
env.env.env.use_stop = False
env.env.env.use_mnist_reward(True)
env.env.env.use_mnist_stop_criteria(True)
obs = env.reset()
agents['seek'].reset()
seek_path_length = 0
if animated:
env.render()
while seek_path_length < max_path_length:
a, agent_info = agents['seek'].get_action(obs)
if animated:
env.render()
obs_next, r, d, env_info = env.step(a)
seek_observations.append(obs)
seek_rewards.append(r)
seek_actions.append(env.action_space.flatten(a))
seek_agent_infos.append(agent_info)
seek_env_infos.append(env_info)
seek_path_length += 1
obs = obs_next
print('Distance = ', env_info['act_min_dist'], ' Max_dist = ',
env_info['act_dist_max'])
time.sleep(0.5)
if d:
break
# print('step seek')
if animated and not always_return_paths:
return
seek_paths = dict(
observations=e2e_tensor_utils.stack_tensor_list(seek_observations),
actions=tensor_utils.stack_tensor_list(seek_actions),
rewards=tensor_utils.stack_tensor_list(seek_rewards),
agent_infos=tensor_utils.stack_tensor_dict_list(seek_agent_infos),
env_infos=tensor_utils.stack_tensor_dict_list(seek_env_infos),
)
seek_paths['actions'] = seek_paths['actions'].astype(glob_config.dtype)
seek_paths['rewards'] = seek_paths['rewards'].astype(glob_config.dtype)
return {'seek': seek_paths}
def rollout_seek(env,
agents,
max_path_length=np.inf,
animated=False,
speedup=1,
always_return_paths=False,
mode=None):
##############################################
# SEEK AGENT
# env.env.unwrapped.reload_model(pose=last_pose)
seek_observations = []
seek_actions = []
seek_rewards = []
seek_agent_infos = []
seek_env_infos = []
if mode == 'mnist_stop':
env.env.env.use_stop = False
env.env.env.use_mnist_reward(True)
env.env.env.use_mnist_stop_criteria(True)
else:
env.env.env.use_stop = False
obs = env.reset()
agents['seek'].reset()
seek_path_length = 0
# print('obs: ', obs[1])
if animated:
env.render()
while seek_path_length < max_path_length:
a, agent_info = agents['seek'].get_action(obs)
if animated:
env.render()
obs_next, r, d, env_info = env.step(a)
seek_observations.append(obs)
seek_rewards.append(r)
seek_actions.append(env.action_space.flatten(a))
seek_agent_infos.append(agent_info)
seek_env_infos.append(env_info)
seek_path_length += 1
obs = obs_next
if d:
break
print('SEEK Test | path_length:', seek_path_length)
if animated and not always_return_paths:
return
seek_paths = dict(
observations=e2e_tensor_utils.stack_tensor_list(seek_observations),
actions=tensor_utils.stack_tensor_list(seek_actions),
rewards=tensor_utils.stack_tensor_list(seek_rewards),
agent_infos=tensor_utils.stack_tensor_dict_list(seek_agent_infos),
env_infos=tensor_utils.stack_tensor_dict_list(seek_env_infos),
)
seek_paths['actions'] = seek_paths['actions'].astype(glob_config.dtype)
seek_paths['rewards'] = seek_paths['rewards'].astype(glob_config.dtype)
return {'seek': seek_paths}
def rollout_hide_seek(env,
agents,
max_path_length=np.inf,
animated=False,
speedup=1,
always_return_paths=False,
mode=None,
hide_tmax=None):
# animated = True
## HIDE AGENT
#Reset the model configuration
env.reset()
obs = env.env.env.reload_model()
last_goal = env.env.unwrapped.get_all_pose()
# print('-----------------------------------------------------')
# print('goal hide: ', env.env.env.goal, 'obs:', obs)
# if animated:
# print('rollout: HIDE')
# print('Frame skip = ', env.env.unwrapped.frame_skip)
# frame_skip_prev = env.env.unwrapped.frame_skip
# env.env.unwrapped.frame_skip = 20
hide_observations = []
hide_actions = []
hide_rewards = []
hide_agent_infos = []
hide_env_infos = []
# Hide is capable of stopping so let's set stop if available
# WARNING: It is important to do all this stuff after reset, since
# blocks dependent stuff could be reset from config file as well
if mode is not None:
if mode == 'seek_force_only':
env.env.env.use_stop = True
env.env.env.add_mnist_reward(False)
env.env.env.use_mnist_stop_criteria(False)
elif mode == 'reach_center_and_stop':
env.env.env.use_stop = True
env.env.env.use_distance2center_stop_criteria = False
prev_set_limit = env.env.unwrapped.step_limit
if hide_tmax is not None:
env.env.unwrapped.step_limit = hide_tmax
# print('rollout: hide step_limit = ', env.env.unwrapped.step_limit)
agents['hide'].reset()
hide_path_length = 0
if animated:
env.render()
# print('rollout: HIDE')
while hide_path_length < max_path_length:
a, agent_info = agents['hide'].get_action(obs)
# print('hide action: ', a)
if animated:
env.render()
obs_next, r, d, env_info = env.step(a)
hide_observations.append(obs)
hide_rewards.append(r)
hide_actions.append(env.action_space.flatten(a))
hide_agent_infos.append(agent_info)
hide_env_infos.append(env_info)
hide_path_length += 1
last_pose = env.env.unwrapped.get_all_pose()
# last_goal = copy.deepcopy(env.env.env.goal)
obs = obs_next
# print('hide obs: ', obs_next)
# time.sleep(0.5)
# if r > 0:
# print('!!!!!!!!!!!!!! r:', r, 'stop crit: ', env.env.unwrapped.use_distance2center_stop_criteria)
if d:
break
# print('step hide')
# print('-------------------------')
# print('goal hide last: ', env.env.env.goal, 'obs:', obs[1])
hide_paths = dict(
observations=e2e_tensor_utils.stack_tensor_list(hide_observations),
actions=tensor_utils.stack_tensor_list(hide_actions),
rewards=tensor_utils.stack_tensor_list(hide_rewards),
agent_infos=tensor_utils.stack_tensor_dict_list(hide_agent_infos),
env_infos=tensor_utils.stack_tensor_dict_list(hide_env_infos),
)
if animated:
time.sleep(1)
##############################################
# SEEK AGENT
# print('last obs: ', obs[1])
if env.spec.id[:6] != 'Blocks' or env.spec.id[:12] == 'BlocksSimple':
# Avoiding randomization for blocks env
env.reset() #must do reset for reacher otherwise it feaks out
obs = env.env.env.reload_model(pose=last_pose, goal=last_goal)
# print('goal seek: ', env.env.env.goal, 'obs:', obs)
# print('Timelen max = ', env.env.unwrapped.step_limit)
# print('Prev limit = ', prev_set_limit)
if animated:
print('rollout: SEEK')
# env.env.unwrapped.frame_skip = 10
# print('rollout: SEEK')
if mode is not None:
if mode == 'seek_force_only':
env.env.env.use_stop = False
env.env.env.add_mnist_reward(True)
env.env.env.use_mnist_stop_criteria(True)
elif mode == 'reach_center_and_stop':
env.env.env.use_stop = False
env.env.env.use_distance2center_stop_criteria = True
if hide_tmax is not None:
env.env.unwrapped.step_limit = prev_set_limit
seek_observations = []
seek_actions = []
seek_rewards = []
seek_agent_infos = []
seek_env_infos = []
# obs = env.reset()
agents['seek'].reset()
seek_path_length = 0
if animated:
env.render()
while seek_path_length < max_path_length:
# if seek_path_length < 2: print('seek obs: ', obs)
a, agent_info = agents['seek'].get_action(obs)
if animated:
# print('Seek obs: ', obs, 'action:', a)
# print('action:', a)
env.render()
obs_next, r, d, env_info = env.step(a)
seek_observations.append(obs)
seek_rewards.append(r)
seek_actions.append(env.action_space.flatten(a))
seek_agent_infos.append(agent_info)
seek_env_infos.append(env_info)
seek_path_length += 1
if d:
# print('break ...')
break
obs = obs_next
# print('step seek')
# if animated:
# env.env.unwrapped.frame_skip = frame_skip_prev
if animated and not always_return_paths:
return
seek_paths = dict(
observations=e2e_tensor_utils.stack_tensor_list(seek_observations),
actions=tensor_utils.stack_tensor_list(seek_actions),
rewards=tensor_utils.stack_tensor_list(seek_rewards),
agent_infos=tensor_utils.stack_tensor_dict_list(seek_agent_infos),
env_infos=tensor_utils.stack_tensor_dict_list(seek_env_infos),
)
hide_paths['actions'] = hide_paths['actions'].astype(glob_config.dtype)
seek_paths['actions'] = seek_paths['actions'].astype(glob_config.dtype)
hide_paths['rewards'] = hide_paths['rewards'].astype(glob_config.dtype)
seek_paths['rewards'] = seek_paths['rewards'].astype(glob_config.dtype)
return {'hide': hide_paths, 'seek': seek_paths}
def rollout_hide(env,
agents,
max_path_length=np.inf,
animated=False,
speedup=1,
always_return_paths=False,
mode=None,
hide_tmax=None,
init_state=None,
init_goal=None,
return_states_as_list=False):
## HIDE AGENT
# animated = True
# Reset the model configuration
# print('Init goal: ', init_goal)
if env.spec.id[:6] == 'Blocks':
env.reset()
obs = env.env.env.reload_model(pose=init_state, goal=init_goal)
else:
env.reset()
obs = env.env.env.reload_model(pose=init_state, goal=init_goal)
# time.sleep(1)
# if animated:
# print('rollout: HIDE')
# frame_skip_prev = env.env.unwrapped.frame_skip
# env.env.unwrapped.frame_skip = 20
hide_observations = []
hide_states = []
hide_actions = []
hide_rewards = []
hide_agent_infos = []
hide_env_infos = []
# Hide is capable of stopping so let's set stop if available
# WARNING: It is important to do all this stuff after reset, since
# blocks dependent stuff could be reset from config file as well
if mode is not None:
if mode == 'seek_force_only':
env.env.env.use_stop = True
env.env.env.add_mnist_reward(False)
env.env.env.use_mnist_stop_criteria(False)
elif mode == 'reach_center_and_stop':
env.env.env.use_stop = True
env.env.env.use_distance2center_stop_criteria = False
prev_set_limit = env.env.unwrapped.step_limit
if hide_tmax is not None:
env.env.unwrapped.step_limit = hide_tmax
agents['hide'].reset()
hide_path_length = 0
if animated:
env.render()
while hide_path_length < max_path_length:
a, agent_info = agents['hide'].get_action(obs)
if animated:
env.render()
# need to do it before the step, to match states to observations in the vector
hide_states.append(env.env.unwrapped.get_all_pose())
obs_next, r, d, env_info = env.step(a)
# print('action:', a)
hide_observations.append(obs)
hide_rewards.append(r)
hide_actions.append(env.action_space.flatten(a))
hide_agent_infos.append(agent_info)
hide_env_infos.append(env_info)
hide_path_length += 1
obs = obs_next
if d:
print('Hide | path_length:', hide_path_length)
break
if mode is not None:
if mode == 'seek_force_only':
env.env.env.use_stop = False
env.env.env.add_mnist_reward(True)
env.env.env.use_mnist_stop_criteria(True)
elif mode == 'reach_center_and_stop':
env.env.env.use_stop = False
env.env.env.use_distance2center_stop_criteria = True
if hide_tmax is not None:
env.env.unwrapped.step_limit = prev_set_limit
if not return_states_as_list:
hide_states = tensor_utils.stack_tensor_list(hide_states)
hide_paths = dict(
observations=e2e_tensor_utils.stack_tensor_list(hide_observations),
actions=tensor_utils.stack_tensor_list(hide_actions),
rewards=tensor_utils.stack_tensor_list(hide_rewards),
agent_infos=tensor_utils.stack_tensor_dict_list(hide_agent_infos),
env_infos=tensor_utils.stack_tensor_dict_list(hide_env_infos),
states=hide_states,
)
# print('Episode done:', hide_path_length)
return hide_paths