def __init__(self, active=False):
super().__init__()
self._active = active
self._valid_detector = LearnedValidDetector()
self.query1 = "yes or n"
self.query2 = "y/n"
self.query3 = "(y or n)"
def __init__(self, active=False):
super().__init__()
self._active = active
self._valid_detector = LearnedValidDetector()
self._affordance_extractor = LmAffordanceExtractor()
self.best_action = None
self._eagerness = 0.
self.actions_that_caused_death = {}
def __init__(self, active=False):
super().__init__()
self._active = active
self._valid_detector = LearnedValidDetector()
self._entity_detector = SpacyEntityDetector()
self._to_examine = {} # Location : ['entity1', 'entity2']
self._validation_threshold = 0.5 # Best threshold over 16 seeds, but not very sensitive.
self._high_eagerness = 0.9
self._low_eagerness = 0.11
def __init__(self, active=False):
super().__init__()
self._active = active
self._valid_detector = LearnedValidDetector()
self.regexps = [
re.compile(
".*(Perhaps you should|You should|You'll have to|You'd better|You\'re not going anywhere until you) (.*) first.*"
),
re.compile(".*(You\'re not going anywhere until you) (.*)\..*"),
re.compile(".*(You need to) (.*) before.*")
]
def __init__(self, active=False, p_retry=.3):
super().__init__()
self._active = active
self._nav_actions = [North, South, West, East, NorthWest,
SouthWest, NorthEast, SouthEast, Up,
Down, Enter, Exit]
self._p_retry = p_retry
self._valid_detector = LearnedValidDetector()
self._suggested_directions = []
self._default_eagerness = 0.1
self._low_eagerness = 0.01
class Darkness(DecisionModule):
"""
The Darkness module listens for phrases like 'it's pitch black' and tries to turn on a light.
"""
def __init__(self, active=False):
super().__init__()
self._active = active
self._valid_detector = LearnedValidDetector()
self.queries = ['pitch black', 'too dark to see']
def process_event(self, event):
""" Process an event from the event stream. """
if not self._active:
return
if type(event) is NewTransitionEvent:
if any(query in event.new_obs for query in self.queries):
self._eagerness = 1.
def take_control(self):
""" Performs the previously extracted action """
obs = yield
action = StandaloneAction('turn on')
response = yield action
p_valid = self._valid_detector.action_valid(action, first_sentence(response))
success = (p_valid > 0.5)
self.record(success)
self._eagerness = 0.
class YesNo(DecisionModule):
"""
The YesNo module listens for Yes/No questions and always outputs Yes.
"""
def __init__(self, active=False):
super().__init__()
self._active = active
self._valid_detector = LearnedValidDetector()
self.query1 = "yes or n"
self.query2 = "y/n"
self.query3 = "(y or n)"
def process_event(self, event):
""" Process an event from the event stream. """
if not self._active:
return
if type(event) is NewTransitionEvent:
obs = event.new_obs.lower()
if (self.query1 in obs) or (self.query2 in obs) or (self.query3 in obs):
self._eagerness = 1.
else:
self._eagerness = 0.
def take_control(self):
""" Always answers yes """
obs = yield
action = rng.choice([Yes, No])
response = yield action
p_valid = self._valid_detector.action_valid(action, first_sentence(response))
success = (p_valid > 0.5)
self.record(success)
self._eagerness = 0.
def __init__(self, seed, env, rom_name, output_subdir='.'):
self.setup_logging(rom_name, output_subdir)
rng.seed(seed)
dbg("RandomSeed: {}".format(seed))
self.knowledge_graph = gv.kg
self.knowledge_graph.__init__() # Re-initialize KnowledgeGraph
gv.event_stream.clear()
self.modules = [
Examiner(True),
Hoarder(True),
Navigator(True),
Interactor(True),
Idler(True),
YesNo(True),
YouHaveTo(True),
Darkness(True)
]
self.active_module = None
self.action_generator = None
self.first_step = True
self._valid_detector = LearnedValidDetector()
if env and rom_name:
self.env = env
self.step_num = 0
class YouHaveTo(DecisionModule):
"""
The YouHaveTo module listens for phrases of the type You'll have to X first.
"""
def __init__(self, active=False):
super().__init__()
self._active = active
self._valid_detector = LearnedValidDetector()
self.regexps = [
re.compile(
".*(Perhaps you should|You should|You'll have to|You'd better|You\'re not going anywhere until you) (.*) first.*"
),
re.compile(".*(You\'re not going anywhere until you) (.*)\..*"),
re.compile(".*(You need to) (.*) before.*")
]
def match(self, text):
""" Returns the matching text if a regexp matches or empty string. """
for regexp in self.regexps:
match = regexp.match(text)
if match:
return match.group(2)
return ''
def process_event(self, event):
""" Process an event from the event stream. """
if not self._active:
return
if type(event) is NewTransitionEvent:
match = self.match(event.new_obs)
if match:
self.act_to_do = StandaloneAction(match)
if self.act_to_do.recognized():
self._eagerness = 1.
def take_control(self):
""" Performs the previously extracted action """
obs = yield
response = yield self.act_to_do
dbg("[YouHaveTo] {} --> {}".format(self.act_to_do, response))
p_valid = self._valid_detector.action_valid(self.act_to_do,
first_sentence(response))
success = (p_valid > 0.5)
self.record(success)
self._eagerness = 0.
class NailAgent():
"""
NAIL Agent: Navigate, Acquire, Interact, Learn
NAIL has a set of decision modules which compete for control over low-level
actions. Changes in world-state and knowledge_graph stream events to the
decision modules. The modules then update how eager they are to take control.
"""
def __init__(self, seed, env, rom_name, output_subdir='.'):
self.setup_logging(rom_name, output_subdir)
rng.seed(seed)
dbg("RandomSeed: {}".format(seed))
self.knowledge_graph = gv.kg
self.knowledge_graph.__init__() # Re-initialize KnowledgeGraph
gv.event_stream.clear()
self.modules = [
Examiner(True),
Hoarder(True),
Navigator(True),
Interactor(True),
Idler(True),
YesNo(True),
YouHaveTo(True),
Darkness(True)
]
self.active_module = None
self.action_generator = None
self.first_step = True
self._valid_detector = LearnedValidDetector()
if env and rom_name:
self.env = env
self.step_num = 0
def setup_logging(self, rom_name, output_subdir):
""" Configure the logging facilities. """
for handler in logging.root.handlers[:]:
handler.close()
logging.root.removeHandler(handler)
self.logpath = os.path.join(output_subdir, 'nail_logs')
if not os.path.exists(self.logpath):
os.mkdir(self.logpath)
self.kgs_dir_path = os.path.join(output_subdir, 'kgs')
if not os.path.exists(self.kgs_dir_path):
os.mkdir(self.kgs_dir_path)
self.logpath = os.path.join(self.logpath, rom_name)
logging.basicConfig(format='%(message)s',
filename=self.logpath + '.log',
level=logging.DEBUG,
filemode='w')
def elect_new_active_module(self):
""" Selects the most eager module to take control. """
most_eager = 0.
for module in self.modules:
eagerness = module.get_eagerness()
if eagerness >= most_eager:
self.active_module = module
most_eager = eagerness
dbg("[NAIL](elect): {} Eagerness: {}"\
.format(type(self.active_module).__name__, most_eager))
self.action_generator = self.active_module.take_control()
self.action_generator.send(None)
def generate_next_action(self, observation):
"""Returns the action selected by the current active module and
selects a new active module if the current one is finished.
"""
next_action = None
while not next_action:
try:
next_action = self.action_generator.send(observation)
except StopIteration:
self.consume_event_stream()
self.elect_new_active_module()
return next_action.text()
def consume_event_stream(self):
""" Each module processes stored events then the stream is cleared. """
for module in self.modules:
module.process_event_stream()
event_stream.clear()
def take_action(self, observation):
if self.env:
# Add true locations to the .log file.
loc = self.env.get_player_location()
if loc and hasattr(loc, 'num') and hasattr(
loc, 'name') and loc.num and loc.name:
dbg("[TRUE_LOC] {} \"{}\"".format(loc.num, loc.name))
# Output a snapshot of the kg.
# with open(os.path.join(self.kgs_dir_path, str(self.step_num) + '.kng'), 'w') as f:
# f.write(str(self.knowledge_graph)+'\n\n')
# self.step_num += 1
observation = observation.strip()
if self.first_step:
dbg("[NAIL] {}".format(observation))
self.first_step = False
return 'look' # Do a look to get rid of intro text
if not kg.player_location:
loc = Location(observation)
kg.add_location(loc)
kg.player_location = loc
kg._init_loc = loc
self.consume_event_stream()
if not self.active_module:
self.elect_new_active_module()
next_action = self.generate_next_action(observation)
return next_action
def observe(self, obs, action, score, new_obs, terminal):
""" Observe will be used for learning from rewards. """
p_valid = self._valid_detector.action_valid(action, new_obs)
dbg("[VALID] p={:.3f} {}".format(p_valid, clean(new_obs)))
if kg.player_location:
dbg("[EAGERNESS] {}".format(' '.join(
[str(module.get_eagerness()) for module in self.modules[:5]])))
event_stream.push(
NewTransitionEvent(obs, action, score, new_obs, terminal))
action_recognized(action, new_obs) # Update the unrecognized words
if terminal:
kg.reset()
def finalize(self):
with open(self.logpath + '.kng', 'w') as f:
f.write(str(self.knowledge_graph) + '\n\n')
def __init__(self, active=False):
super().__init__()
self._active = active
self._valid_detector = LearnedValidDetector()
self.queries = ['pitch black', 'too dark to see']
class Interactor(DecisionModule):
"""
The Interactor creates actions designed to interact with objects
at the current location.
"""
def __init__(self, active=False):
super().__init__()
self._active = active
self._valid_detector = LearnedValidDetector()
self._affordance_extractor = LmAffordanceExtractor()
self.best_action = None
self._eagerness = 0.
self.actions_that_caused_death = {}
def process_event(self, event):
pass
def get_eagerness(self):
if not self._active:
return 0.
self.best_action = None
self._eagerness = 0.
max_eagerness = 0.
# Consider single-object actions.
for entity in kg.player_location.entities + kg.inventory.entities:
for action, prob in self._affordance_extractor.extract_single_object_actions(
entity):
if prob <= max_eagerness:
break
if entity.has_action_record(action) or \
(not action.recognized()) or \
(action in self.actions_that_caused_death) or \
((action.verb == 'take') and (entity in kg.inventory.entities)): # Need to promote to Take.
continue
max_eagerness = prob
self.best_action = action
break
# Consider double-object actions.
for entity1 in kg.player_location.entities + kg.inventory.entities:
for entity2 in kg.player_location.entities + kg.inventory.entities:
if entity1 != entity2:
for action, prob in self._affordance_extractor.extract_double_object_actions(
entity1, entity2):
if prob <= max_eagerness:
break
if entity1.has_action_record(action) or \
(not action.recognized()) or \
(action in self.actions_that_caused_death):
continue
max_eagerness = prob
self.best_action = action
break
self._eagerness = max_eagerness
return self._eagerness
def take_control(self):
obs = yield
# Failsafe checks
if self._eagerness == 0.: # Should never happen anyway.
self.get_eagerness() # But if it does, try finding a best action.
if self._eagerness == 0.:
return # If no good action can be found, simply return without yielding.
action = self.best_action
self.best_action = None
self._eagerness = 0.
response = yield action
p_valid = action.validate(response)
if p_valid is None:
p_valid = self._valid_detector.action_valid(
action, first_sentence(response))
if isinstance(action, SingleAction):
action.entity.add_action_record(action, p_valid, response)
elif isinstance(action, DoubleAction):
action.entity1.add_action_record(action, p_valid, response)
success = (p_valid > 0.5)
self.record(success)
if success:
action.apply()
dbg("[INT]({}) p={:.2f} {} --> {}".format("val" if success else "inv",
p_valid, action, response))
if ('RESTART' in response and 'RESTORE' in response
and 'QUIT' in response) or ('You have died' in response):
if action not in self.actions_that_caused_death:
self.actions_that_caused_death[
action] = True # Remember actions that cause death.
def __init__(self, active=False):
super().__init__()
self._active = active
self._valid_detector = LearnedValidDetector()
self._eagerness = .05
class Idler(DecisionModule):
"""
The Idler module accepts control when no others are willing to.
"""
def __init__(self, active=False):
super().__init__()
self._active = active
self._valid_detector = LearnedValidDetector()
self._eagerness = .05
def process_event(self, event):
""" Process an event from the event stream. """
pass
def get_random_entity(self):
""" Returns a random entity from the location or inventory. """
if kg.player_location.entities or kg.inventory.entities:
return rng.choice(kg.player_location.entities +
kg.inventory.entities)
return None
def get_standalone_action(self):
return StandaloneAction(rng.choice(standalone_verbs))
def get_single_object_action(self):
entity = self.get_random_entity()
if not entity:
return None
verb = rng.choice(single_object_verbs)
return SingleAction(verb, entity)
def get_double_action(self):
if len(kg.player_location.entities) + len(kg.inventory.entities) <= 1:
return None
entity1 = None
entity2 = None
count = 0
while id(entity1) == id(entity2):
if count == 100:
return None # Failsafe
else:
count += 1
entity1 = self.get_random_entity()
entity2 = self.get_random_entity()
verb, prep = rng.choice(complex_verbs)
return DoubleAction(verb, entity1, prep, entity2)
def get_action(self):
if not self._active:
return StandaloneAction('look')
n = rng.random()
if n < .1:
return self.get_standalone_action()
elif n < .8:
return self.get_single_object_action()
else:
return self.get_double_action()
def take_control(self):
obs = yield
action = self.get_action()
while action is None or not action.recognized():
action = self.get_action()
response = yield action
p_valid = self._valid_detector.action_valid(action,
first_sentence(response))
if isinstance(action, StandaloneAction):
kg.player_location.add_action_record(action, p_valid, response)
elif isinstance(action, SingleAction):
action.entity.add_action_record(action, p_valid, response)
elif isinstance(action, DoubleAction):
action.entity1.add_action_record(action, p_valid, response)
success = (p_valid > 0.5)
self.record(success)
dbg("[IDLER]({}) p={:.2f} {} --> {}".format(
"val" if success else "inv", p_valid, action, response))
class Navigator(DecisionModule):
"""
The Navigator is responsible for choosing a navigation action and recording
the effects of that action.
Args:
p_retry: Probability of re-trying failed actions
eagerness: Default eagerness for this module
"""
def __init__(self, active=False, p_retry=.3):
super().__init__()
self._active = active
self._nav_actions = [North, South, West, East, NorthWest,
SouthWest, NorthEast, SouthEast, Up,
Down, Enter, Exit]
self._p_retry = p_retry
self._valid_detector = LearnedValidDetector()
self._suggested_directions = []
self._default_eagerness = 0.1
self._low_eagerness = 0.01
def get_mentioned_directions(self, description):
""" Returns the nav actions mentioned in a description. """
tokens = tokenize(description)
return [act for act in self._nav_actions if act.text() in tokens]
def process_event(self, event):
""" Process an event from the event stream. """
pass
def get_eagerness(self):
if not self._active:
return 0.
if self.get_unexplored_actions(kg.player_location):
return self._default_eagerness
return rng.choice([self._low_eagerness, self._default_eagerness])
def get_unexplored_actions(self, location):
""" Returns a list of nav actions not yet attempted from a given location. """
return [act for act in self._nav_actions if act not in location.action_records \
and act.recognized()]
def get_successful_nav_actions(self, location):
""" Returns a list of nav actions that have been successful from the location. """
return [c.action for c in kg.connections.outgoing(location) if c.action.recognized()]
def get_failed_nav_actions(self, location):
""" Returns a list of nav actions that have failed from the location. """
successful_actions = self.get_successful_nav_actions(location)
return [act for act in self._nav_actions if act in location.action_records \
and act not in successful_actions and act.recognized()]
def get_action(self):
"""
First try to take an unexplored nav action. If none exist, sample one
of the successful or failed nav actions.
"""
loc = kg.player_location
# If there was a previously suggested direction, try it
if self._suggested_directions:
act = rng.choice(self._suggested_directions)
del self._suggested_directions[:]
dbg("[NAV] Trying suggested action: {}".format(act))
return act
# First try to move in one of the directions mentioned in the description.
likely_nav_actions = self.get_mentioned_directions(loc.description)
for act in likely_nav_actions:
if not loc.has_action_record(act):
dbg("[NAV] Trying mentioned action: {}".format(act))
return act
# Then try something new
unexplored = self.get_unexplored_actions(loc)
if unexplored:
act = rng.choice(unexplored)
dbg("[NAV] Trying unexplored action: {}".format(act))
return act
# Try a previously successful action
if rng.random() > self._p_retry:
successful_actions = self.get_successful_nav_actions(loc)
if successful_actions:
act = rng.choice(successful_actions)
dbg("[NAV] Trying previously successful action: {}".format(act))
return act
# Finally, just try something random
act = rng.choice(self._nav_actions)
dbg("[NAV] Trying random action: {}".format(act))
return act
def find_most_similar_loc(self, description, loc_list):
"""Returns the location from loc_list with that best matches the
provided description."""
most_similar = None
best_similarity = 0
for loc in loc_list:
similarity = fuzz.partial_ratio(loc.description, description)
if similarity > best_similarity:
best_similarity = similarity
most_similar = loc
return most_similar
def relocalize(self, description):
"""Resets the player's location to location best matching the
provided description, creating a new location if needed. """
loc = kg.most_similar_location(description)
if loc:
dbg("[NAV](relocalizing) \"{}\" to {}".format(description, loc))
kg.player_location = loc
else:
dbg("[NAV](relocalizing aborted) \"{}\" to {}".format(description, loc))
def take_control(self):
"""
Takes a navigational action and records the resulting transition.
"""
obs = yield
curr_loc = kg.player_location
action = self.get_action()
response = yield action
p_valid = self._valid_detector.action_valid(action, response)
# Check if we've tried this action before
tried_before = False
if curr_loc.has_action_record(action):
prev_valid, result = curr_loc.action_records[action]
if result.startswith(response):
tried_before = True
curr_loc.add_action_record(action, p_valid, response)
self._suggested_directions = self.get_mentioned_directions(response)
if self._suggested_directions:
dbg("[NAV] Suggested Directions: {}".format(self._suggested_directions))
if action in self._suggested_directions: # Don't try the same nav action again
self._suggested_directions.remove(action)
# If an existing locations matches the response, then we're done
possible_loc_name = Location.extract_name(response)
existing_locs = kg.locations_with_name(possible_loc_name)
if existing_locs:
# If multiple locations match, we need the most similar
if len(existing_locs) > 1:
look = yield Look
existing_loc = self.find_most_similar_loc(look, existing_locs)
else:
existing_loc = existing_locs[0]
dbg("[NAV](revisited-location) {}".format(existing_loc.name))
kg.add_connection(Connection(curr_loc, action, existing_loc))
kg.player_location = existing_loc
return
# This is either a new location or a failed action
if tried_before:
known_destination = kg.connections.navigate(curr_loc, action)
if known_destination:
# We didn't reach the expected destination. Likely mislocalized.
look = yield Look
self.relocalize(look)
else: # This has failed previously
dbg("[NAV-fail] p={:.2f} Response: {}".format(p_valid, response))
else:
# This is a new response: do a look to see if we've moved.
if p_valid < .1:
dbg("[NAV](Suspected-Invalid) {}".format(response))
return
look = yield Look
p_stay = fuzz.ratio(look, curr_loc.description) / 100.
p_move = fuzz.ratio(look, response) / 100.
moved = p_move > p_stay
dbg("[NAV]({}) p={} {} --> {}".format(
'val' if moved else 'inv', p_move, action, response))
self.record(moved)
if moved:
# Check if we've moved to an existing location
possible_loc_name = Location.extract_name(look)
existing_locs = kg.locations_with_name(possible_loc_name)
if existing_locs:
if len(existing_locs) > 1:
existing_loc = self.find_most_similar_loc(look, existing_locs)
else:
existing_loc = existing_locs[0]
dbg("[NAV](revisited-location) {}".format(existing_loc.name))
kg.add_connection(Connection(curr_loc, action, existing_loc))
kg.player_location = existing_loc
return
# Finally, create a new location
new_loc = Location(look)
to_loc = kg.add_location(new_loc)
kg.add_connection(Connection(curr_loc, action, new_loc))
kg.player_location = new_loc
class Examiner(DecisionModule):
"""
Examiner is responsible for gathering information from the environment
by issuing the examine command on objects present at a location.
"""
def __init__(self, active=False):
super().__init__()
self._active = active
self._valid_detector = LearnedValidDetector()
self._entity_detector = SpacyEntityDetector()
self._to_examine = {} # Location : ['entity1', 'entity2']
self._validation_threshold = 0.5 # Best threshold over 16 seeds, but not very sensitive.
self._high_eagerness = 0.9
self._low_eagerness = 0.11
def detect_entities(self, message):
""" Returns a list of detected candidate entities as strings. """
return self._entity_detector.detect(message)
def get_event_info(self, event):
""" Returns the location and information contained by a new event. """
message = ''
location = kg.player_location
if type(event) is NewLocationEvent:
location = event.new_location
message = event.new_location.description
elif type(event) is NewEntityEvent:
message = event.new_entity.description
elif type(event) is NewActionRecordEvent:
message = event.result_text
elif type(event) is LocationChangedEvent:
location = event.new_location
return location, message
def process_event(self, event):
""" Process an event from the event stream. """
location, message = self.get_event_info(event)
if location not in self._to_examine:
self._to_examine[location] = []
if not message:
return
candidate_entities = self.detect_entities(message)
dbg("[EXM](detect) {} --> {}".format(clean(message),
candidate_entities))
self.filter(candidate_entities)
def get_eagerness(self):
""" If we are located at an unexamined location, this module is very eager."""
if not self._active:
return 0.
if self.get_descriptionless_entities():
return self._high_eagerness
elif self._to_examine[kg.player_location]:
return self._low_eagerness
else:
return 0.
def get_descriptionless_entities(self):
l = [e for e in kg.player_location.entities if not e.description]
l.extend([e for e in kg.inventory if not e.description])
return l
def filter(self, candidate_entities):
""" Filters candidate entities. """
curr_loc = kg.player_location
for entity_name in candidate_entities:
action = gv.Examine(entity_name)
if curr_loc.has_entity_with_name(entity_name) or \
action in curr_loc.action_records or \
not action.recognized() or \
entity_name in self._to_examine[curr_loc]:
continue
self._to_examine[curr_loc].append(entity_name)
def take_control(self):
"""
1) Detect candidate Entities from current location.
2) Examine entities to get detailed descriptions
3) Extract nested entities from detailed descriptions
"""
obs = yield
curr_loc = kg.player_location
undescribed_entities = self.get_descriptionless_entities()
if undescribed_entities:
entity = undescribed_entities[0]
action = gv.Examine(entity.name)
response = yield action
entity.description = response
p_valid = self._valid_detector.action_valid(action, response)
dbg("[EXM] p={:.2f} {} --> {}".format(p_valid, action,
clean(response)))
curr_loc.add_action_record(action, 1., response)
else:
entity_name = self._to_examine[curr_loc].pop()
action = gv.Examine(entity_name)
response = yield action
p_valid = self._valid_detector.action_valid(
action, first_sentence(response))
success = (p_valid > self._validation_threshold)
self.record(success)
dbg("[EXM]({}) p={:.2f} {} --> {}".format(
"val" if success else "inv", p_valid, action, clean(response)))
curr_loc.add_action_record(action, p_valid, response)
if success:
entity = curr_loc.get_entity_by_description(response)
if entity is None:
entity = Entity(entity_name,
curr_loc,
description=response)
# TODO: incorrect for entities discovered inside other entities
curr_loc.add_entity(entity)
else:
dbg("[EXM](val) Discovered alternate name "\
"\'{}\' for \'{}\'".format(entity_name, entity.name))
entity.add_name(entity_name)
if success:
entity = curr_loc.get_entity_by_description(response)
inv_entity = kg.inventory.get_entity_by_description(response)
if entity is None and inv_entity is None:
entity = Entity(entity_name,
curr_loc,
description=response)
# TODO: incorrect for entities discovered inside other entities
curr_loc.add_entity(entity)
else:
if entity:
dbg("[EXM](val) Discovered alternate name " \
"\'{}\' for \'{}\'".format(entity_name, entity.name))
entity.add_name(entity_name)
if inv_entity:
dbg("[EXM](val) Discovered alternate name " \
"\'{}\' for inventory item \'{}\'".format(entity_name, inv_entity.name))
inv_entity.add_name(entity_name)