def run(cls, model, resnet, task_queue, args, lock, successes, failures,
results):
'''
evaluation loop
'''
# start THOR
env = ThorEnv()
while True:
if task_queue.qsize() == 0:
break
task = task_queue.get()
try:
traj = model.load_task_json(task)
r_idx = task['repeat_idx']
print("Evaluating: %s" % (traj['root']))
print("No. of trajectories left: %d" % (task_queue.qsize()))
cls.evaluate(env, model, r_idx, resnet, traj, args, lock,
successes, failures, results)
except Exception as e:
import traceback
traceback.print_exc()
print("Error: " + repr(e))
# stop THOR
env.stop()
def run(cls, model, resnet, task_queue, args, lock, successes, failures,
results):
'''
evaluation loop
'''
# start THOR
env = ThorEnv()
# make subgoals list
subgoals_to_evaluate = cls.ALL_SUBGOALS if args.subgoals.lower(
) == "all" else args.subgoals.split(',')
subgoals_to_evaluate = [
sg for sg in subgoals_to_evaluate if sg in cls.ALL_SUBGOALS
]
print("Subgoals to evaluate: %s" % str(subgoals_to_evaluate))
# create empty stats per subgoal
for sg in subgoals_to_evaluate:
successes[sg] = list()
failures[sg] = list()
while True:
if task_queue.qsize() == 0:
break
task = task_queue.get()
try:
traj = model.load_task_json(model.args, task)[0]
if args.modular_subgoals:
filtered_traj_by_subgoal = {
subgoal: model.load_task_json(task, subgoal)
for subgoal in subgoals_to_evaluate
}
else:
filtered_traj_by_subgoal = None
r_idx = task['repeat_idx']
subgoals_and_idxs = [
(sg['discrete_action']['action'], sg['high_idx'])
for sg in traj['plan']['high_pddl']
if sg['discrete_action']['action'] in subgoals_to_evaluate
]
for subgoal, eval_idx in subgoals_and_idxs:
print("No. of trajectories left: %d" %
(task_queue.qsize()))
if filtered_traj_by_subgoal is not None:
subgoal_filtered_traj_data = filtered_traj_by_subgoal[
subgoal]
else:
subgoal_filtered_traj_data = None
cls.evaluate(env, model, eval_idx, r_idx, resnet, traj,
args, lock, successes, failures, results,
subgoal_filtered_traj_data)
except Exception as e:
import traceback
traceback.print_exc()
print("Error: " + repr(e))
# stop THOR
env.stop()
def run():
'''
replay loop
'''
# start THOR env
env = ThorEnv(player_screen_width=IMAGE_WIDTH,
player_screen_height=IMAGE_HEIGHT)
skipped_files = []
while len(traj_list) > 0:
lock.acquire()
json_file = traj_list.pop()
lock.release()
print("Augmenting: " + json_file)
try:
augment_traj(env, json_file)
except Exception as e:
import traceback
traceback.print_exc()
print("Error: " + repr(e))
print("Skipping " + json_file)
skipped_files.append(json_file)
env.stop()
print("Finished.")
# skipped files
if len(skipped_files) > 0:
print("Skipped Files:")
print(skipped_files)
def run():
'''
replay loop
'''
# start THOR env
env = ThorEnv(player_screen_width=IMAGE_WIDTH,
player_screen_height=IMAGE_HEIGHT)
skipped_files = []
finished = []
cache_file = os.path.join(args.save_path, "cache.json")
while len(traj_list) > 0:
lock.acquire()
json_file = traj_list.pop()
lock.release()
print("(%d Left) Augmenting: %s" % (len(traj_list), json_file))
try:
augment_traj(env, json_file)
finished.append(json_file)
with open(cache_file, 'w') as f:
json.dump({'finished': finished}, f)
except Exception as e:
import traceback
traceback.print_exc()
print("Error: " + repr(e))
print("Skipping " + json_file)
skipped_files.append(json_file)
env.stop()
print("Finished.")
# skipped files
if len(skipped_files) > 0:
print("Skipped Files:")
print(skipped_files)
def run():
print(all_scene_numbers)
# create env and agent
env = ThorEnv()
while len(all_scene_numbers) > 0:
lock.acquire()
scene_num = all_scene_numbers.pop()
lock.release()
fn = os.path.join('layouts', ('FloorPlan%d-layout.npy') % scene_num)
if os.path.isfile(fn):
print("file %s already exists; skipping this floorplan" % fn)
continue
openable_json_file = os.path.join(
'layouts', ('FloorPlan%d-openable.json') % scene_num)
scene_objs_json_file = os.path.join(
'layouts', ('FloorPlan%d-objects.json') % scene_num)
scene_name = ('FloorPlan%d') % scene_num
print('Running ' + scene_name)
event = env.reset(scene_name,
render_image=False,
render_depth_image=False,
render_class_image=False,
render_object_image=True)
agent_height = event.metadata['agent']['position']['y']
scene_objs = list(
set([obj['objectType'] for obj in event.metadata['objects']]))
with open(scene_objs_json_file, 'w') as sof:
json.dump(scene_objs, sof, sort_keys=True, indent=4)
# Get all the reachable points through Unity for this step size.
event = env.step(
dict(action='GetReachablePositions',
gridSize=constants.AGENT_STEP_SIZE /
constants.RECORD_SMOOTHING_FACTOR))
if event.metadata['actionReturn'] is None:
print("ERROR: scene %d 'GetReachablePositions' returns None" %
scene_num)
else:
reachable_points = set()
for point in event.metadata['actionReturn']:
reachable_points.add((point['x'], point['z']))
print("scene %d got %d reachable points, now checking" %
(scene_num, len(reachable_points)))
# Pick up a small object to use in testing whether points are good for openable objects.
open_test_objs = {
'ButterKnife', 'CD', 'CellPhone', 'Cloth', 'CreditCard',
'DishSponge', 'Fork', 'KeyChain', 'Pen', 'Pencil', 'SoapBar',
'Spoon', 'Watch'
}
good_obj_point = None
good_obj_point = get_obj(env, open_test_objs, reachable_points,
agent_height, scene_name, good_obj_point)
best_open_point = {
} # map from object names to the best point from which they can be successfully opened
best_sem_coverage = {
} # number of pixels in the semantic map of the receptacle at the existing best openpt
checked_points = set()
scene_receptacles = set()
for point in reachable_points:
point_is_valid = True
action = {
'action': 'TeleportFull',
'x': point[0],
'y': agent_height,
'z': point[1],
}
event = env.step(action)
if event.metadata['lastActionSuccess']:
for horizon in [-30, 0, 30]:
action = {
'action': 'TeleportFull',
'x': point[0],
'y': agent_height,
'z': point[1],
'rotateOnTeleport': True,
'rotation': 0,
'horizon': horizon
}
event = env.step(action)
if not event.metadata['lastActionSuccess']:
point_is_valid = False
break
for rotation in range(3):
action = {'action': 'RotateLeft'}
event = env.step(action)
if not event.metadata['lastActionSuccess']:
point_is_valid = False
break
if not point_is_valid:
break
if point_is_valid:
checked_points.add(point)
else:
continue
# Check whether we can open objects from here in any direction with any tilt.
for rotation in range(4):
# First try up, then down, then return to the horizon before moving again.
for horizon in [-30, 0, 30]:
action = {
'action': 'TeleportFull',
'x': point[0],
'y': agent_height,
'z': point[1],
'rotateOnTeleport': True,
'rotation': rotation * 90,
'horizon': horizon
}
event = env.step(action)
for obj in event.metadata['objects']:
if (obj['visible'] and obj['objectId']
and obj['receptacle']
and not obj['pickupable']
and obj['objectType']
in constants.VAL_RECEPTACLE_OBJECTS):
obj_name = obj['objectId']
obj_point = (obj['position']['x'],
obj['position']['y'])
scene_receptacles.add(obj_name)
# Go ahead and attempt to close the object from this position if it's open.
if obj['openable'] and obj['isOpen']:
close_action = {
'action': 'CloseObject',
'objectId': obj['objectId']
}
event = env.step(close_action)
point_to_recep = np.linalg.norm(
np.array(point) - np.array(obj_point))
if len(env.last_event.
metadata['inventoryObjects']) > 0:
inv_obj = env.last_event.metadata[
'inventoryObjects'][0]['objectId']
else:
inv_obj = None
# Heuristic implemented in task_game_state has agent 0.5 or farther in agent space.
heuristic_far_enough_from_recep = 0.5 < point_to_recep
# Ensure this point affords a larger view according to the semantic segmentation
# of the receptacle than the existing.
point_sem_coverage = get_mask_of_obj(
env, obj['objectId'])
if point_sem_coverage is None:
use_sem_heuristic = False
better_sem_covereage = False
else:
use_sem_heuristic = True
better_sem_covereage = (
obj_name not in best_sem_coverage
or
best_sem_coverage[obj_name] is None
or point_sem_coverage >
best_sem_coverage[obj_name])
# Ensure that this point is farther away than our existing best candidate.
# We'd like to open each receptacle from as far away as possible while retaining
# the ability to pick/place from it.
farther_than_existing_good_point = (
obj_name not in best_open_point
or point_to_recep > np.linalg.norm(
np.array(point) - np.array(
best_open_point[obj_name][:2]))
)
# If we don't have an inventory object, though, we'll fall back to the heuristic
# of being able to open/close as _close_ as possible.
closer_than_existing_good_point = (
obj_name not in best_open_point
or point_to_recep < np.linalg.norm(
np.array(point) - np.array(
best_open_point[obj_name][:2]))
)
# Semantic segmentation heuristic.
if ((use_sem_heuristic
and heuristic_far_enough_from_recep
and better_sem_covereage) or
(not use_sem_heuristic and
# Distance heuristics.
(heuristic_far_enough_from_recep and
(inv_obj and
farther_than_existing_good_point) or
(not inv_obj and
closer_than_existing_good_point)))):
if obj['openable']:
action = {
'action': 'OpenObject',
'objectId': obj['objectId']
}
event = env.step(action)
if not obj[
'openable'] or event.metadata[
'lastActionSuccess']:
# We can open the object, so try placing our small inventory obj inside.
# If it can be placed inside and retrieved, then this is a safe point.
action = {
'action':
'PutObject',
'objectId':
inv_obj,
'receptacleObjectId':
obj['objectId'],
'forceAction':
True,
'placeStationary':
True
}
if inv_obj:
event = env.step(action)
if inv_obj is None or event.metadata[
'lastActionSuccess']:
action = {
'action': 'PickupObject',
'objectId': inv_obj
}
if inv_obj:
event = env.step(action)
if inv_obj is None or event.metadata[
'lastActionSuccess']:
# Finally, ensure we can also close the receptacle.
if obj['openable']:
action = {
'action':
'CloseObject',
'objectId':
obj['objectId']
}
event = env.step(
action)
if not obj['openable'] or event.metadata[
'lastActionSuccess']:
# We can put/pick our inv object into the receptacle from here.
# We have already ensured this point is farther than any
# existing best, so this is the new best.
best_open_point[
obj_name] = [
point[0],
point[1],
rotation * 90,
horizon
]
best_sem_coverage[
obj_name] = point_sem_coverage
# We could not retrieve our inv object, so we need to go get another one
else:
good_obj_point = get_obj(
env, open_test_objs,
reachable_points,
agent_height,
scene_name,
good_obj_point)
action = {
'action':
'TeleportFull',
'x': point[0],
'y': agent_height,
'z': point[1],
'rotateOnTeleport':
True,
'rotation':
rotation * 90,
'horizon': horizon
}
event = env.step(action)
# Regardless of what happened up there, try to close the receptacle again if
# it remained open.
if obj['isOpen']:
action = {
'action': 'CloseObject',
'objectId': obj['objectId']
}
event = env.step(action)
essential_objs = []
if scene_num in constants.SCENE_TYPE["Kitchen"]:
essential_objs.extend(["Microwave", "Fridge"])
for obj in essential_objs:
if not np.any([obj in obj_key for obj_key in best_open_point]):
print(
"WARNING: Essential object %s has no open points in scene %d"
% (obj, scene_num))
print(
"scene %d found open/pick/place/close positions for %d/%d receptacle objects"
% (scene_num, len(best_open_point), len(scene_receptacles)))
with open(openable_json_file, 'w') as f:
json.dump(best_open_point, f, sort_keys=True, indent=4)
print("scene %d reachable %d, checked %d; taking intersection" %
(scene_num, len(reachable_points), len(checked_points)))
points = np.array(list(checked_points))[:, :2]
points = points[np.lexsort((points[:, 0], points[:, 1])), :]
np.save(fn, points)
env.stop()
print('Done')
def run_rollouts(cls, model, task_queue, results, args, validation=False):
env = ThorEnv()
while True:
if validation:
task, seen = task_queue.get()
else:
task = task_queue.get()
if task is None:
break
# reset model
model.reset()
# setup scene
traj_data = model.load_task_json(task)
r_idx = task['repeat_idx']
cls.setup_scene(env, traj_data, r_idx, args)
feat = model.featurize([traj_data],
load_frames=False,
load_mask=False)
curr_rollout = []
done = False
fails = 0
total_reward = 0
num_steps = 0
while not done and num_steps < args.max_steps:
# extract visual features
curr_image = Image.fromarray(np.uint8(env.last_event.frame))
feat['frames'] = model.resnet.featurize([curr_image],
batch=1).unsqueeze(0)
# forward model
out = model.step(feat)
pred = model.sample_pred(out, greedy=validation)
# monitor resource usage
monitor = start_monitor(
path=args.dout,
note="validation" if validation else "rollout" +
f" step={num_steps}")
# # check if <<stop>> was predicted
# if pred['action_low'] == "<<stop>>":
# print("\tpredicted STOP")
# break
# get action and mask
action = pred['action_low']
mask = pred['action_low_mask'] if cls.has_interaction(
action) else None
# use predicted action and mask (if available) to interact with the env
t_success, _, _, err, _ = env.va_interact(
action,
interact_mask=mask,
smooth_nav=args.smooth_nav,
debug=args.debug)
if not t_success:
fails += 1
if fails >= args.max_fails:
break
# next time-step
reward, done = env.get_transition_reward()
total_reward += reward
num_steps += 1
if not validation:
curr_rollout.append({
'frames':
feat['frames'].cpu().detach().numpy(),
'lang_goal_instr_data':
feat['lang_goal_instr'].data.cpu().detach().numpy(),
'lang_goal_instr_batch':
feat['lang_goal_instr'].batch_sizes.cpu().detach(
).numpy(),
'lang_goal_instr_sorted':
feat['lang_goal_instr'].sorted_indices.cpu().detach(
).numpy() if feat['lang_goal_instr'].sorted_indices
is not None else None,
'lang_goal_instr_unsorted':
feat['lang_goal_instr'].unsorted_indices.cpu().detach(
).numpy() if feat['lang_goal_instr'].unsorted_indices
is not None else None,
'action_dist':
pred['action_low_dist'].cpu().detach().numpy(),
'action_mask_dist':
pred['action_low_mask_dist'].cpu().detach().numpy(),
'action_idx':
pred['action_low_idx'].cpu().detach().numpy(),
'action_mask_idx':
pred['action_low_mask_idx'].cpu().detach().numpy(),
'reward':
np.array([reward])
})
stop_monitor(monitor)
if validation:
# check if goal was satisfied
goal_satisfied = env.get_goal_satisfied()
# goal_conditions
pcs = env.get_goal_conditions_met()
goal_condition_success_rate = pcs[0] / float(pcs[1])
# SPL
path_len_weight = len(traj_data['plan']['low_actions'])
s_spl = (1 if goal_satisfied else 0) * min(
1., path_len_weight / float(num_steps))
pc_spl = goal_condition_success_rate * min(
1., path_len_weight / float(num_steps))
# path length weighted SPL
plw_s_spl = s_spl * path_len_weight
plw_pc_spl = pc_spl * path_len_weight
# log success/fails
log_entry = {
'trial':
traj_data['task_id'],
'type':
traj_data['task_type'],
'repeat_idx':
int(r_idx),
'seen':
seen,
'goal_instr':
traj_data['turk_annotations']['anns'][r_idx]['task_desc'],
'goal_satisfied':
goal_satisfied,
'completed_goal_conditions':
int(pcs[0]),
'total_goal_conditions':
int(pcs[1]),
'goal_condition_success':
float(goal_condition_success_rate),
'success_spl':
float(s_spl),
'path_len_weighted_success_spl':
float(plw_s_spl),
'goal_condition_spl':
float(pc_spl),
'path_len_weighted_goal_condition_spl':
float(plw_pc_spl),
'path_len_weight':
int(path_len_weight),
'reward':
float(total_reward)
}
results.put(log_entry)
else:
results.put(curr_rollout)
env.stop()