def evaluate(self) -> None:
db_api = DatabaseAPI()
demonstration_templates = db_api.get_demonstration_templates()
uim = UserInputManager()
uim.say("Actions learned from demonstration:")
uim.show([t.name for t in demonstration_templates])
def evaluate(self) -> None:
is_default = False
uim = UserInputManager()
uim.say(f"Do you want to make {self.area_name} your default area? (y/n)")
if uim.confirm_user():
is_default = True
self.db_api = DatabaseAPI()
self.db_api.add_area(area_name=self.area_name,
corners=(self.corner_0, self.corner_1, self.corner_2, self.corner_3),
is_default=is_default)
self.db_api.set_state(State.DEFINE_AREA)
class TemplateDetector(CrowModule):
"""
First part of the NL pipeline. Preliminary template detection in the text.
"""
namespace = db.onto
def __init__(self, language='en'):
self.lang = language
self.ui = UserInputManager(language=language)
def detect_templates(self, tagged_text: TaggedText) -> List[tt]:
"""
Tries to guess which template should be used to represent robot instructions for the chunk of
text. Can detect more templates, but only the first one which matches will be used later.
Parameters
----------
tagged_text a tagged text in which the template should be detected
Returns
-------
a list of guessed templates for the tagged text sorted by their probability
"""
self.templ_det = self.ui.load_file('templates_detection.json')
self.guidance_file = self.ui.load_file('guidance_dialogue.json')
templates = []
detect_fns = [
self.detect_pick, self.detect_apply_glue, self.detect_put,
self.detect_tidy, self.detect_learn, self.detect_tower,
self.detect_demonstration_list, self.detect_define_area
]
# try to find custom templates (compound actions) first
custom_templates = self.detect_custom_templates(tagged_text)
if custom_templates:
templates += custom_templates
# add detected basic templates (actions)
for detect_fn in detect_fns:
res = detect_fn(tagged_text)
if res:
templates += res
return templates
def detect_custom_templates(self, tagged_text: TaggedText):
"""
Retrieves learned custom templates (compound actions) from the database
and tries to detect them in the text.
Parameters
----------
tagged_text a tagged text in which a custom template should be detected
Returns
-------
a list of custom templates detected in the text, an empty list if no template is detected
"""
all_custom_templates = db_api.get_custom_templates()
custom_templates = []
for custom_template in all_custom_templates:
custom_template_name = custom_template.name[1:]
if custom_template_name.lower() in tagged_text.get_text().lower():
custom_templates.append(custom_template)
return custom_templates
def detect_pick(self, tagged_text: TaggedText) -> List[tt]:
"""
Detector for PickTask
"""
if tagged_text.contains_pos_token(
self.templ_det[self.lang]['take'],
"VB") or tagged_text.contains_pos_token(
self.templ_det[self.lang]['pick'], "VB"):
self.ui.say(self.guidance_file[self.lang]["template_match"] +
self.templ_det[self.lang]['pick'])
#if tagged_text.contains_pos_token("take", "VB") or \
# tagged_text.contains_pos_token("pick", "VB"):
return [tt.PICK_TASK]
def detect_apply_glue(self, tagged_text: TaggedText) -> List[tt]:
"""
Detector for ApplyGlueTask
"""
if tagged_text.contains_pos_token(
self.templ_det[self.lang]['glue'],
"VB") or tagged_text.contains_pos_token(
self.templ_det[self.lang]['glue'],
"NNS") or tagged_text.contains_pos_token(
self.templ_det[self.lang]['glue'], "NNP"):
self.ui.say(self.guidance_file[self.lang]["template_match"] +
self.templ_det[self.lang]['glue'])
return [tt.APPLY_GLUE]
def detect_learn(self, tagged_text: TaggedText) -> List[tt]:
"""
Detector for LearnNewTask
"""
if tagged_text.contains_text("learn") and tagged_text.contains_text(
"new task"):
self.ui.say(self.guidance_file[self.lang]["template_match"] +
self.templ_det[self.lang]['learn_new_task'])
return [tt.LEARN_NEW_TASK]
def detect_put(self, tagged_text: TaggedText) -> List[tt]:
"""
Detector for PutTask
"""
#if tagged_text.contains_text("put"):
if tagged_text.contains_text(self.templ_det[self.lang]['put']):
self.ui.say(self.guidance_file[self.lang]["template_match"] +
self.templ_det[self.lang]['put'])
return [tt.PUT_TASK]
def detect_tidy(self, tagged_text: TaggedText) -> List[tt]:
"""
Detector for PutTask
"""
#if tagged_text.contains_text("put"):
if tagged_text.contains_text(self.templ_det[self.lang]['tidy']):
self.ui.say(self.guidance_file[self.lang]["template_match"] +
self.templ_det[self.lang]['tidy'])
return [tt.TIDY_TASK]
def detect_tower(self, tagged_text: TaggedText) -> List[tt]:
"""
Detector for LearnTowerFromDemonstration
"""
if tagged_text.contains_text("learn") and tagged_text.contains_text(
"tower"):
self.ui.say(self.guidance_file[self.lang]["template_match"] +
self.templ_det[self.lang]['learn_new_tower'])
return [tt.LEARN_TOWER]
def detect_demonstration_list(self, tagged_text: TaggedText) -> List[tt]:
"""
Detector for DemonstrationList
"""
if tagged_text.contains_text("show") and tagged_text.contains_text(
"demonstration"):
self.ui.say(self.guidance_file[self.lang]["template_match"] +
self.templ_det[self.lang]['demonstration'])
return [tt.DEMONSTRATION_LIST]
def detect_define_area(self, tagged_text: TaggedText) -> List[tt]:
"""
Detector for DefineArea
"""
if tagged_text.contains_text("define") and tagged_text.contains_text(
"area"):
self.ui.say(self.guidance_file[self.lang]["template_match"] +
self.templ_det[self.lang]['define_area'])
return [tt.DEFINE_AREA]
class ObjectGrounder:
namespace = db.onto
#class Flags(Enum):
# CAN_BE_PICKED = 1
def __init__(self, language = 'en'):
self.lang = language
self.db_api = DatabaseAPI()
self.cd = ColorDetector()
self.ar = UserInputManager(language = self.lang)
self.logger = logging.getLogger(__name__)
self.templ_file = self.ar.load_file('templates_detection.json')
self.guidance_file = self.ar.load_file('guidance_dialogue.json')
def ground_object(self, obj_placeholder, flags=()) -> Any:
# try to find out the class of the object
# the 0th item of an is_a list should be always ObjectPlaceholder
# if the item is bound to a real class, the class will be the 1st item
cls = obj_placeholder.is_a[-1]
if "last_mentioned" in obj_placeholder.flags:
objs = [self.db_api.get_last_mentioned_object()]
else:
props = obj_placeholder.get_properties()
props.discard(db.onto.hasFlags) # this is internal property of the object placeholder
# put together a dictionary of properties required from the object
props_vals = {self.get_prop_name(prop): getattr(obj_placeholder, self.get_prop_name(prop)) for prop in props}
# find if there is an object with such properties in the workspace
objs = self.db_api.get_by_properties(cls=cls, properties=props_vals)
# if self.Flags.CAN_BE_PICKED in flags:
# objs = list(filter(lambda x: x._can_be_picked, objs))
# only one object found / any object can be selected
if len(objs) == 1 or ("any" in obj_placeholder.flags and len(objs) > 0):
obj = objs[0]
self.db_api.set_last_mentioned_object(obj)
# more objects -> ask the user to select one
elif len(objs) > 1:
obj = self.ar.ask_to_select_from_class_objects(objs)
self.db_api.set_last_mentioned_object(obj)
else:
self.logger.warning(f"No object of type {cls} in the workspace.")
self.ar.say(self.guidance_file[self.lang]["no_object_workspace"], f"{cls}")
obj = None
if obj:
self.logger.debug(f"Object found for {obj_placeholder}: {obj}")
#self.ar.say(self.guidance_file[self.lang]["object_found"],f"{obj_placeholder}")
return obj
def get_prop_name(self, prop : ow.DataPropertyClass):
# we need to use the python name of properties whenever it is defined
if hasattr(prop, "python_name"):
return prop.python_name
return prop.name
class ObjectDetector(CrowModule):
"""
Detects an object in text.
"""
namespace = db.onto
def __init__(self,language = 'en'):
self.logger = logging.getLogger(__name__)
self.lang = language
self.db_api = DatabaseAPI()
self.ui = UserInputManager(language = language)
self.class_map = {
"screwdriver": db.onto.Screwdriver,
"hammer": db.onto.Hammer,
"pliers": db.onto.Pliers,
"glue" : db.onto.Glue,
"panel" : db.onto.Panel,
"cube" : db.onto.Cube
}
self.templ_det = self.ui.load_file('templates_detection.json')
self.guidance_file = self.ui.load_file('guidance_dialogue.json')
def detect_object(self, tagged_text : TaggedText) -> db.onto.Object:
"""
Detects an object mentioned in the input text, extracts its properties and saves it
in the object placeholder.
Parameters
----------
tagged_text an input text
Returns
-------
an object placeholder to be grounded later
"""
obj = None
text = tagged_text.get_text()
# try to detect one of the known objects in text
for obj_str in self.class_map.keys():
try:
#TODO should be only NN, but we have a problem that kostka is detected as VB/VBD
obj_str_lang = self.templ_det[self.lang][obj_str]
if tagged_text.contains_pos_token(obj_str_lang, "NN") or tagged_text.contains_pos_token(obj_str_lang, "VBD") or tagged_text.contains_pos_token(obj_str_lang, "VB") or tagged_text.contains_pos_token(obj_str_lang, "NNS") :
obj = self.detect_explicit_object(tagged_text, obj_str)
break
except:
pass
# try to detect a coreference to an object
if obj is None and tagged_text.contains_pos_token("it", "PRP"):
obj = self.detect_coreferenced_object()
self.logger.debug(f"Object detected for \"{text}\": {obj}")
self.ui.say(self.guidance_file[self.lang]["object_matched"]+" "+ text)
return obj
def detect_explicit_object(self, tagged_text, obj_str):
"""
Detect an object which is mentioned explicitly.
"""
cls = self.class_map[obj_str]
obj = db.onto.ObjectPlaceholder()
obj.is_a.append(cls)
obj_str_lang = self.templ_det[self.lang][obj_str]
if tagged_text.contains_text(self.templ_det[self.lang]['any']+" " + obj_str_lang):
# the "any" flag will be used to select any object without asking the user
obj.flags.append("any")
self.detect_object_color(obj, tagged_text)
self.detect_object_id(obj, tagged_text)
self.detect_object_location(obj, obj_str, tagged_text)
return obj
# def detect_known_object(self, tagged_text, obj_str):
# cls = self.class_map[obj_str]
# obj = db.onto.ObjectPlaceholder()
# obj.is_a.append(cls)
#
# if tagged_text.contains_text("any " + obj_str):
# obj.flags.append("any")
#
# self.detect_object_color(obj, tagged_text)
# self.detect_object_location(obj, obj_str, tagged_text)
#
# return obj
def detect_coreferenced_object(self):
"""
Detect that the text is referencing an object mentioned earlier.
"""
obj = db.onto.ObjectPlaceholder()
obj.flags.append("last_mentioned")
return obj
def detect_object_location(self, obj, obj_str, tagged_text):
# cut the part of the text that is sent into the location detector to avoid infinite loop
# TODO: all of this is only a temporary solution, not intended to be used in the final product
end_index = tagged_text.get_text().find(obj_str) + len(obj_str)
new_tagged_text = tagged_text.cut(end_index, None)
ld = LocationDetector.LocationDetector(language = self.lang)
location = ld.detect_location(new_tagged_text)
if location:
obj.location = location
def detect_object_color(self, obj, tagged_text):
cd = ColorDetector.ColorDetector(language = self.lang)
color = cd.detect_color(tagged_text)
if color:
obj.color.append(db.onto.NamedColor(color))
def detect_object_id(self, obj, tagged_text):
idet = IdDetector.IdDetector()
id = idet.detect_id(tagged_text)
if id is not None:
obj.aruco_id = id
class NLProcessor():
def __init__(self, language="en"):
self.gp = GrammarParser(language=language)
self.td = TemplateDetector(language=language)
self.tf = TemplateFactory()
self.lang = language
self.ui = UserInputManager(language=language)
self.logger = logging.getLogger(__name__)
self.guidance_file = self.ui.load_file('guidance_dialogue.json')
def process_text(self, sentence: str) -> RobotProgram:
"""
Turns an input text into a program template which can be used for creating instructions for the robot
(after grounding).
The program template is dependent only on the content of the sentence and not
on the current state of the workspace.
Parameters
----------
sentence an input sentence as string
Returns
-------
a program template - formalized instructions for the robot with placeholders for real objects and locations
(right now, the behavior is undefined in case the sentence does not allow creating a valid program)
"""
parsed_text = self.gp.parse(sentence)
root = parsed_text.parse_tree
db_api = DatabaseAPI()
state = db_api.get_state()
if state == State.LEARN_FROM_INSTRUCTIONS:
program = RobotCustomProgram()
else:
program = RobotProgram()
# hardcoded program structure: all subprograms are located directly under the root node "AND"
program.root = RobotProgramOperator(operator_type="AND")
for subnode in root.subnodes:
if type(subnode) is ParseTreeNode:
# create a single robot instructon
program_node = self.process_node(subnode)
program.root.add_child(program_node)
return program
def process_node(self, subnode: ParseTreeNode) -> RobotProgramOperand:
node = RobotProgramOperand()
node.parsed_text = subnode
# working with flat tagged text (without any tree structure)
tagged_text = subnode.flatten()
# using TemplateDetector to get a list of templates sorted by probability
template_types = self.td.detect_templates(tagged_text)
self.logger.debug(
f"Templates detected for \"{tagged_text.get_text()}\": {[t.name for t in template_types]}"
)
# try to sequentially match each template
for template_type in template_types:
# custom template cannot be parametrized yet -> no matching required
# TODO the condition looks kind of stupid
if type(template_type) is not TemplateType:
# custom template is already a valid program
node = template_type.root
template = None
break
# get an object representing the template
template = self.tf.get_template(template_type)
# try to match all the template parameters
template.match(tagged_text, language=self.lang)
# check if the template is matched successfully
if template.is_filled():
break
else:
self.logger.error("No template match for \"{}\"".format(
tagged_text.get_text()))
self.ui.say(self.guidance_file[self.lang]["no_template_match"] +
tagged_text.get_text())
template = None
# save the filled template in the program node
node.template = template
return node