def get_true_contain_question(episode, things, receptacles):
receptacles = list(receptacles)
random.shuffle(receptacles)
for parent in receptacles:
parents = game_util.get_objects_of_type(parent,
episode.event.metadata)
random.shuffle(parents)
for parent in parents:
if len(parent['pivotSimObjs']) == 0:
continue
obj = random.choice(parent['pivotSimObjs'])
obj_type = obj['objectId'].split('|')[0]
if obj_type in things:
return ExistenceQuestion(obj_type, parent['objectType'])
return None
def step(self, action_or_ind):
self.reward = -0.01
action, teleport_failure, should_fail = self.get_action(action_or_ind)
t_start = time.time()
if should_fail or teleport_failure:
self.event.metadata['lastActionSuccess'] = False
else:
if action[
'action'] != 'Teleport' or not constants.USE_NAVIGATION_AGENT:
self.event = self.env.step(action)
else:
# Action is teleport and I should do low level navigation.
pass
new_pose = game_util.get_pose(self.event)
point_dists = np.sum(np.abs(self.graph.points -
np.array(new_pose)[:2]),
axis=1)
if np.min(point_dists) > 0.0001:
print('Point teleport failure')
closest_point = self.graph.points[np.argmin(point_dists)]
self.event = self.env.step({
'action': 'Teleport',
'x': closest_point[0] * constants.AGENT_STEP_SIZE,
'y': self.agent_height,
'z': closest_point[1] * constants.AGENT_STEP_SIZE,
'rotateOnTeleport': True,
'rotation': self.pose[2] * 90,
})
else:
closest_point = np.argmin(point_dists)
if closest_point not in self.visited_locations:
self.visited_locations.add(closest_point)
self.times[2, 0] += time.time() - t_start
self.times[2, 1] += 1
if self.times[2, 1] % 100 == 0:
print('env step time %.3f' % (self.times[2, 0] / self.times[2, 1]))
if self.event.metadata['lastActionSuccess']:
self.process_frame()
if action['action'] == 'OpenObject':
if self.question_type_ind == 2 and action['objectId'].split(
'|')[0] != constants.OBJECTS[self.question_target[1]]:
self.reward -= 1.0
elif action['objectId'] not in self.opened_receptacles:
if self.question_type_ind == 2 and action[
'objectId'].split('|')[0] == constants.OBJECTS[
self.question_target[1]]:
self.reward += 5.0
else:
self.reward += 0.1
self.opened_receptacles.add(action['objectId'])
elif action[
'action'] == 'CloseObject' and self.question_type_ind != 2:
if action['objectId'] not in self.closed_receptacles:
self.reward += 0.1
self.closed_receptacles.add(action['objectId'])
# Update seen objects related to question
objs = game_util.get_objects_of_type(
constants.OBJECTS[self.object_target], self.event.metadata)
objs = [
obj for obj in objs
if (obj['objectId'] in self.event.instance_detections2D
and game_util.check_object_size(
self.event.instance_detections2D[obj['objectId']]))
]
for obj in objs:
self.seen_obj1.add(obj['objectId'])
if self.question_type_ind in {2, 3}:
objs = game_util.get_objects_of_type(
constants.OBJECTS[self.question_target[1]],
self.event.metadata)
objs = [
obj for obj in objs
if (obj['objectId'] in self.event.instance_detections2D
and game_util.check_object_size(
self.event.instance_detections2D[obj['objectId']]))
]
for obj in objs:
self.seen_obj2.add(obj['objectId'])
if not self.can_end:
if self.question_type_ind == 0:
self.can_end = len(self.seen_obj1) > 0
elif self.question_type_ind == 1:
self.can_end = len(self.seen_obj1) == self.answer
elif self.question_type_ind == 2:
objs = game_util.get_objects_of_type(
constants.OBJECTS[self.question_target[1]],
self.event.metadata)
if not self.answer:
if objs[0]['openable']:
if all([
obj['objectId'] in self.opened_receptacles
for obj in objs
]):
self.can_end = True
else:
if all([
obj['objectId'] in self.seen_obj2
for obj in objs
]):
self.can_end = True
else:
objs = [
obj for obj in objs
if (obj['objectId'] in
self.event.instance_detections2D
and game_util.check_object_size(
self.event.instance_detections2D[
obj['objectId']]))
]
for obj in objs:
for contained_obj in obj['pivotSimObjs']:
if contained_obj['objectId'] in self.seen_obj1:
self.can_end = True
else:
self.reward -= 0.05
def reset(self, seed=None, test_ind=None):
self.board = None
self.seen_object = False
self.terminal = False
self.opened_receptacles = set()
self.closed_receptacles = set()
self.seen_obj1 = set()
self.seen_obj2 = set()
self.visited_locations = set()
self.can_end = False
if seed is not None:
self.local_random.seed(seed)
# Do equal number of each question type in train.
question_type_ind = self.local_random.sample(
constants.USED_QUESTION_TYPES, 1)[0]
# get random row
if test_ind is not None:
question_row, question_type_ind = test_ind
question_type = self.question_types[question_type_ind]
question_data = self.test_datasets[question_type_ind][
question_row, :]
test_ind = (question_row, question_type_ind)
else:
question_type = self.question_types[question_type_ind]
question_row = self.local_random.randint(
0,
len(self.datasets[question_type_ind]) - 1)
question_data = self.datasets[question_type_ind][question_row, :]
container_ind = None
if question_type_ind == 0 or question_type_ind == 1:
scene_num, scene_seed, object_ind, answer = question_data
self.question_target = object_ind
if question_type_ind == 0:
answer = bool(answer)
elif question_type_ind == 2:
scene_num, scene_seed, object_ind, container_ind, answer = question_data
answer = bool(answer)
self.question_target = (object_ind, container_ind)
else:
raise Exception('No question type found for type %d' %
question_type_ind)
self.scene_seed = scene_seed
self.scene_num = scene_num
self.object_target = object_ind
self.parent_target = container_ind
self.container_target = np.zeros(constants.NUM_CLASSES)
self.direction_target = np.zeros(4)
if container_ind is not None:
self.container_target[container_ind] = 1
self.question_type_ind = question_type_ind
self.scene_name = 'FloorPlan%d' % scene_num
grid_file = 'layouts/%s-layout.npy' % self.scene_name
self.graph = graph_obj.Graph(grid_file, use_gt=False)
self.xray_graph = graph_obj.Graph(grid_file, use_gt=True)
self.bounds = [
self.graph.xMin, self.graph.yMin,
self.graph.xMax - self.graph.xMin + 1,
self.graph.yMax - self.graph.yMin + 1
]
max_num_repeats = 1
remove_prob = 0.5
if question_type == 'existence':
max_num_repeats = 10
remove_prob = 0.25
elif question_type == 'counting':
max_num_repeats = constants.MAX_COUNTING_ANSWER + 1
remove_prob = 0.5
elif question_type == 'contains':
max_num_repeats = 10
remove_prob = 0.25
self.event = game_util.reset(self.env, self.scene_name)
self.agent_height = self.event.metadata['agent']['position']['y']
self.camera_height = self.agent_height + constants.CAMERA_HEIGHT_OFFSET
self.event = self.env.random_initialize(
self.scene_seed,
max_num_repeats=max_num_repeats,
remove_prob=remove_prob)
print('Type:', question_type, 'Row: ', question_row, 'Scene',
self.scene_name, 'seed', scene_seed)
print(
'Question:',
game_util.get_question_str(question_type_ind, object_ind,
container_ind))
if self.question_type_ind == 2:
print('Answer:', constants.OBJECTS[object_ind], 'in',
constants.OBJECTS[container_ind], 'is', answer)
else:
print('Answer:', constants.OBJECTS[object_ind], 'is', answer)
self.answer = answer
# Verify answer
if self.question_type_ind == 0:
objs = game_util.get_objects_of_type(constants.OBJECTS[object_ind],
self.event.metadata)
computed_answer = len(objs) > 0
requires_interaction = True
for obj in objs:
parent = obj['parentReceptacle'].split('|')[0]
if parent not in {'Fridge', 'Cabinet', 'Microwave'}:
requires_interaction = False
break
elif self.question_type_ind == 1:
objs = game_util.get_objects_of_type(constants.OBJECTS[object_ind],
self.event.metadata)
computed_answer = len(objs)
requires_interaction = True
elif self.question_type_ind == 2:
objs = game_util.get_objects_of_type(constants.OBJECTS[object_ind],
self.event.metadata)
if len(objs) == 0:
computed_answer = False
requires_interaction = constants.OBJECTS[
self.question_target[1]] in {
'Fridge', 'Cabinet', 'Microwave'
}
else:
obj = objs[0]
computed_answer = False
for obj in objs:
requires_interaction = True
parent = obj['parentReceptacle'].split('|')[0]
if parent in constants.OBJECT_CLASS_TO_ID:
parent_ind = constants.OBJECT_CLASS_TO_ID[parent]
computed_answer = parent_ind == self.question_target[1]
if computed_answer:
if parent not in {
'Fridge', 'Cabinet', 'Microwave'
}:
requires_interaction = False
break
else:
computed_answer = False
self.requires_interaction = requires_interaction
try:
assert self.answer == computed_answer, 'Answer does not match scene metadata'
except AssertionError:
print('Type:', question_type, 'Row: ', question_row, 'Scene',
self.scene_name, 'seed', scene_seed)
print('Answer', computed_answer, 'does not match expected value',
self.answer, ', did randomization process change?')
pdb.set_trace()
self.answer = computed_answer
if constants.NUM_CLASSES > 1:
self.hidden_items = set()
objects = self.event.metadata['objects']
for obj in objects:
if obj['receptacle'] and obj['openable'] and not obj['isopen']:
for inside_obj in obj['receptacleObjectIds']:
self.hidden_items.add(inside_obj)
objects = self.event.metadata['objects']
for obj in objects:
if obj['objectType'] not in constants.OBJECT_CLASS_TO_ID:
continue
obj_bounds = game_util.get_object_bounds(obj, self.bounds)
self.xray_graph.memory[
obj_bounds[1]:obj_bounds[3], obj_bounds[0]:obj_bounds[2],
constants.OBJECT_CLASS_TO_ID[obj['objectType']] + 1] = 1
start_point = self.local_random.randint(0,
self.graph.points.shape[0] - 1)
start_point = self.graph.points[start_point, :].copy()
self.start_point = (start_point[0], start_point[1],
self.local_random.randint(0, 3))
action = {
'action': 'TeleportFull',
'x': self.start_point[0] * constants.AGENT_STEP_SIZE,
'y': self.agent_height,
'z': self.start_point[1] * constants.AGENT_STEP_SIZE,
'rotateOnTeleport': True,
'rotation': self.start_point[2] * 90,
'horizon': 30,
}
self.event = self.env.step(action)
self.process_frame()
self.reward = 0
self.end_point = []
def setup_problem(self, seed=None, info=None, scene=None, objs=None):
'''
setup goal with sampled objects or with the objects specified
note: must be used with `initialize_random_scene`
'''
self.terminal = False
print(
'setup random goal ----------------------------------------------')
print('seed', seed)
print('info', info)
print(
'--------------------------------------------------------------------\n\n'
)
super(TaskGameState, self).setup_problem(seed)
info, max_num_repeats, remove_prob = self.get_random_task_vals(
scene=scene)
dataset_type, task_row = info
print('Type:', dataset_type, 'Row: ', task_row, 'Scene',
self.scene_name, 'seed', self.scene_seed)
perform_sanity_check = True
# pickupable, receptacle, toggleable candidates
pickupable_objects = [
o for o in self.env.last_event.metadata['objects']
if (o['pickupable'] and o['objectType'] in constants.OBJECTS)
]
receptacle_objects = [
o for o in self.env.last_event.metadata['objects']
if (o['receptacle'] and o['objectType'] in constants.RECEPTACLES
and o['objectType'] not in constants.MOVABLE_RECEPTACLES_SET)
]
toggle_objects = [
o for o in self.env.last_event.metadata['objects']
if o['toggleable']
and not o['isToggled'] # must be one we can turn on.
and o['objectType'] in constants.VAL_ACTION_OBJECTS["Toggleable"]
]
if len(pickupable_objects) == 0:
print("Task Failed - %s" % constants.pddl_goal_type)
raise Exception("No pickupable objects in the scene")
# filter based on crit
obj_crit, _ = self.get_filter_crit(constants.pddl_goal_type)
pickupable_objects = list(filter(obj_crit, pickupable_objects))
if len(pickupable_objects) == 0:
print("Task Failed - %s" % constants.pddl_goal_type)
raise Exception("No pickupable objects related to the goal '%s'" %
constants.pddl_goal_type)
# choose a pickupable object
if constants.FORCED_SAMPLING or objs is None:
self.rand_chosen_object = random.choice(pickupable_objects)
else:
chosen_class_available = [
obj for obj in pickupable_objects
if obj['objectType'] == constants.OBJ_PARENTS[objs['pickup']]
]
if len(chosen_class_available) == 0:
print("Task Failed - %s" % constants.pddl_goal_type)
raise Exception(
"Couldn't find a valid parent '%s' for pickup object with class '%s'"
% (constants.OBJ_PARENTS[objs['pickup']], objs['pickup']))
self.rand_chosen_object = random.choice(chosen_class_available)
self.rand_chosen_object_class = self.rand_chosen_object['objectType']
self.object_target = constants.OBJECTS.index(
self.rand_chosen_object_class)
# for now, any obj differing from its parent is obtained via slicing, but that may change in the future.
if constants.OBJ_PARENTS[objs['pickup']] != objs['pickup']:
requires_slicing = True
else:
requires_slicing = False
# choose a movable receptacle
if "movable_recep" in constants.pddl_goal_type:
val_movable_receps = [
o for o in self.env.last_event.metadata['objects']
if (o['objectType'] in constants.MOVABLE_RECEPTACLES_SET) and (
self.rand_chosen_object_class in
constants.VAL_RECEPTACLE_OBJECTS[o['objectType']])
]
if len(val_movable_receps) == 0:
print("Task Failed - %s" % constants.pddl_goal_type)
raise Exception(
"Couldn't find a valid moveable receptacle for the chosen object class"
)
if objs is not None:
val_movable_receps = [
o for o in val_movable_receps
if o['objectType'] == objs['mrecep']
]
self.rand_chosen_val_moveable_recep_class = random.choice(
val_movable_receps)['objectType']
self.mrecep_target = constants.OBJECTS.index(
self.rand_chosen_val_moveable_recep_class)
else:
self.mrecep_target = None
# if slicing is required, make sure a knife is available in the scene
if requires_slicing:
knife_objs = [
o for o in self.env.last_event.metadata['objects']
if ("Knife" in o['objectType'])
]
if len(knife_objs) == 0:
print("Task Failed - %s" % constants.pddl_goal_type)
raise Exception("Couldn't find knife in the scene to cut with")
if constants.pddl_goal_type == "look_at_obj_in_light":
# choose a toggleable object
self.parent_target = None
if constants.FORCED_SAMPLING or objs is None:
rand_chosen_toggle_object = random.choice(toggle_objects)
else:
toggle_class_available = [
obj for obj in toggle_objects
if obj['objectType'] == objs['toggle']
]
if len(toggle_class_available) == 0:
print("Task Failed - %s" % constants.pddl_goal_type)
raise Exception(
"Couldn't find a valid toggle object of class '%s'" %
objs['toggle'])
rand_chosen_toggle_object = random.choice(
toggle_class_available)
rand_chosen_toggle_class = rand_chosen_toggle_object['objectType']
self.toggle_target = constants.OBJECTS.index(
rand_chosen_toggle_class)
else:
self.toggle_target = None
# find all valid receptacles in which rand_chosen object or rand chosen moveable receptacle can be placed
val_receptacle_objects_orig = [
r for r in receptacle_objects
if (self.rand_chosen_val_moveable_recep_class if self.
mrecep_target != None else self.rand_chosen_object_class
) in constants.VAL_RECEPTACLE_OBJECTS[r['objectType']]
]
_, recep_crit = self.get_filter_crit(constants.pddl_goal_type)
val_receptacle_objects = list(
filter(recep_crit, val_receptacle_objects_orig))
if len(val_receptacle_objects) == 0:
print("Task Failed - %s" % constants.pddl_goal_type)
raise Exception(
"Couldn't find a valid receptacle object according to constraints specified"
)
# choose a receptacle object
if constants.FORCED_SAMPLING or objs is None:
rand_chosen_receptacle_object = random.choice(
val_receptacle_objects)
else:
receptacle_class_available = [
obj for obj in val_receptacle_objects
if obj['objectType'] == objs['receptacle']
]
if len(receptacle_class_available) == 0:
print("Task Failed - %s" % constants.pddl_goal_type)
raise Exception(
"Couldn't find a valid receptacle object of class '%s'"
% objs['receptacle'])
rand_chosen_receptacle_object = random.choice(
receptacle_class_available)
rand_chosen_receptacle_object_class = rand_chosen_receptacle_object[
'objectType']
self.parent_target = constants.OBJECTS.index(
rand_chosen_receptacle_object_class)
# pddl_param dict
constants.data_dict['pddl_params'][
'object_target'] = constants.OBJECTS[
self.object_target] if self.object_target is not None else ""
constants.data_dict['pddl_params']['object_sliced'] = requires_slicing
constants.data_dict['pddl_params'][
'parent_target'] = constants.OBJECTS[
self.parent_target] if self.parent_target is not None else ""
constants.data_dict['pddl_params'][
'toggle_target'] = constants.OBJECTS[
self.toggle_target] if self.toggle_target is not None else ""
constants.data_dict['pddl_params'][
'mrecep_target'] = constants.OBJECTS[
self.mrecep_target] if self.mrecep_target is not None else ""
self.task_target = (self.object_target, self.parent_target,
self.toggle_target, self.mrecep_target)
if self.parent_target is None:
validity_check = True
else:
objs = game_util.get_objects_of_type(
constants.OBJECTS[self.object_target],
self.env.last_event.metadata)
available_receptacles = game_util.get_objects_of_type(
constants.OBJECTS[self.parent_target],
self.env.last_event.metadata)
validity_check = (len(objs) > 0
and len([obj for obj in available_receptacles]))
if perform_sanity_check:
try:
assert validity_check, 'Task does not seem valid according to scene metadata'
except AssertionError:
raise Exception(
str(('Row: ', task_row, 'Scene', self.scene_name, 'seed',
self.scene_seed)))
templated_task_desc = self.get_task_str()
print('problem id', self.problem_id)
print('Task:', templated_task_desc)
constants.data_dict['template']['task_desc'] = templated_task_desc
def does_have_atleast_n_duplicates(x, n):
num_instances = len(
game_util.get_objects_of_type(x['objectType'],
self.env.last_event.metadata))
return num_instances >= n