Ejemplo n.º 1
0
class RegisterIntentParserTests(unittest.TestCase):
    """All tests related to the DomainIntentDeterminationEngine."""
    def setUp(self):
        """Setting up testing env."""
        self.engine = DomainIntentDeterminationEngine()

    def test_register_intent_parser(self):
        """Test to make sure that intent parser is working with correct data."""
        parser = IntentBuilder("Intent").build()
        self.engine.register_intent_parser(parser)
        self.assertEqual(len(self.engine.intent_parsers), 1)

    def test_register_intent_parser_defult_is_empty(self):
        """Test to make sure that the intent parser is empty when first created."""
        assert len(self.engine.intent_parsers) == 0

    def test_register_intent_parser_throws_error_if_not_intent_parsers(self):
        """
        Test register_intent_parser throws error if IntentBuilder object is not passed in.
        This should throw a ValueError.
        """
        with self.assertRaises(ValueError):
            self.engine.register_intent_parser("NOTAPARSER")
Ejemplo n.º 2
0
class RegisterIntentParserTests(unittest.TestCase):
    """All tests related to the DomainIntentDeterminationEngine."""

    def setUp(self):
        """Setting up testing env."""
        self.engine = DomainIntentDeterminationEngine()

    def test_register_intent_parser(self):
        """Test to make sure that intent parser is working with correct data."""
        parser = IntentBuilder("Intent").build()
        self.engine.register_intent_parser(parser)
        self.assertEqual(len(self.engine.intent_parsers), 1)

    def test_register_intent_parser_defult_is_empty(self):
        """Test to make sure that the intent parser is empty when first created."""
        assert len(self.engine.intent_parsers) == 0

    def test_register_intent_parser_throws_error_if_not_intent_parsers(self):
        """
        Test register_intent_parser throws error if IntentBuilder object is not passed in.
        This should throw a ValueError.
        """
        with self.assertRaises(ValueError):
            self.engine.register_intent_parser("NOTAPARSER")
Ejemplo n.º 3
0
class SelectBestIntentTests(unittest.TestCase):
    """All tests related to the DomainIntentDeterminationEngine."""

    def setUp(self):
        """Setting up testing env."""
        self.engine = DomainIntentDeterminationEngine()

    def test_select_best_intent(self):
        """
        Test to make sure that best intent is being returned.
        This test is to make sure that best intent works identicly to its counter part
        in the IntentEngine.
        """
        parser1 = IntentBuilder("Parser1").require("Entity1").build()
        self.engine.register_intent_parser(parser1)
        self.engine.register_entity("tree", "Entity1")

        utterance = "go to the tree house"
        intent = next(self.engine.determine_intent(utterance))
        assert intent
        self.assertEqual(intent['intent_type'], 'Parser1')

        parser2 = IntentBuilder("Parser2").require("Entity1").require("Entity2").build()
        self.engine.register_intent_parser(parser2)
        self.engine.register_entity("house", "Entity2")
        intent = next(self.engine.determine_intent(utterance))
        assert intent
        self.assertEqual(intent['intent_type'], 'Parser2')

    def test_select_best_intent_with_domain(self):
        """Test to make sure that best intent is working with domains."""
        self.engine.register_domain('Domain1')
        self.engine.register_domain('Domain2')

        # Creating first intent domain
        parser1 = IntentBuilder("Parser1").require("Entity1").build()
        self.engine.register_intent_parser(parser1, domain='Domain1')
        self.engine.register_entity("tree", "Entity1", domain='Domain1')

        # Creating second intent domain
        parser2 = IntentBuilder("Parser1").require("Entity2").build()
        self.engine.register_intent_parser(parser2, domain="Domain2")
        self.engine.register_entity("house", "Entity2", domain="Domain2")

        utterance = "Entity1 Entity2 go to the tree house"
        intents = self.engine.determine_intent(utterance, 2)

        intent = next(intents)
        assert intent
        self.assertEqual(intent['intent_type'], 'Parser1')

        intent = next(intents)
        assert intent
        self.assertEqual(intent['intent_type'], 'Parser1')

    def test_select_best_intent_enuse_enitities_dont_register_in_multiple_domains(self):
        """Test to make sure that 1 entity does not end up in multiple domains."""
        self.engine.register_domain('Domain1')
        self.engine.register_domain('Domain2')

        # Creating first intent domain
        parser1 = IntentBuilder("Parser1").require("Entity1").build()
        self.engine.register_intent_parser(parser1, domain='Domain1')
        self.engine.register_entity("tree", "Entity1", domain='Domain1')

        # Creating second intent domain
        parser2 = IntentBuilder("Parser2").require("Entity2").build()
        self.engine.register_intent_parser(parser2, domain="Domain2")
        self.engine.register_entity("house", "Entity2", domain="Domain2")

        utterance = "go to the house"
        intents = self.engine.determine_intent(utterance, 1)
        for intent in intents:
            self.assertNotEqual(intent['intent_type'], 'Parser1')

        utterance = "go to the tree"
        intents = self.engine.determine_intent(utterance, 1)
        for intent in intents:
            self.assertNotEqual(intent['intent_type'], 'Parser2')
Ejemplo n.º 4
0
artists = [
    "third eye blind", "the who", "the clash", "john mayer", "kings of leon",
    "adelle"
]

for a in artists:
    engine.register_entity(a, "Artist", domain='Domain2')

music_verbs = ["listen", "hear", "play"]

for mv in music_verbs:
    engine.register_entity(mv, "MusicVerb", domain='Domain2')

music_keywords = ["songs", "music"]

for mk in music_keywords:
    engine.register_entity(mk, "MusicKeyword", domain='Domain2')

music_intent = IntentBuilder("MusicIntent")\
    .require("MusicVerb")\
    .optionally("MusicKeyword")\
    .optionally("Artist")\
    .build()

engine.register_intent_parser(weather_intent, domain='Domain1')
engine.register_intent_parser(music_intent, domain='Domain2')

if __name__ == "__main__":
    for intents in engine.determine_intent(' '.join(sys.argv[1:])):
        print(intents)
Ejemplo n.º 5
0
music_verbs = [
    "listen",
    "hear",
    "play"
]

for mv in music_verbs:
    engine.register_entity(mv, "MusicVerb", domain='Domain2')

music_keywords = [
    "songs",
    "music"
]

for mk in music_keywords:
    engine.register_entity(mk, "MusicKeyword", domain='Domain2')

music_intent = IntentBuilder("MusicIntent")\
    .require("MusicVerb")\
    .optionally("MusicKeyword")\
    .optionally("Artist")\
    .build()

engine.register_intent_parser(weather_intent, domain='Domain1')
engine.register_intent_parser(music_intent, domain='Domain2')


if __name__ == "__main__":
    for intents in engine.determine_intent(' '.join(sys.argv[1:])):
        print(intents)
Ejemplo n.º 6
0
class Skill(object):
    def __init__(self, root_dir, name, nlp, active):
        self._root_dir = root_dir
        self._name = name
        self._nlp = nlp
        self._active = active

        self._intents_container = None
        self._adapt_intent_engine = None

        self.initialize_intent_parser()

    def is_active(self):
        return self._active

    def get_name(self):
        return self._name

    def initialize_intent_parser(self):

        self._intents_container = IntentContainer("%s_cache" % self._name)

        self._adapt_intent_engine = DomainIntentDeterminationEngine()
        self._adapt_intent_engine.register_domain(self._name)

        for intent_name, intent_file_path in self.get_intent_names():
            #print ("###### IntentBuilder: %s, %s" % (intent_name, intent_file_path))
            adapt_intent_builder = IntentBuilder(intent_name)
            for intent_name, intent_example_sentences_array in self.intent_training_file_content(
                    intent_file_path, 'intent'):
                #print ("add intent %s, %s" % (intent_name, intent_example_sentences_array))
                self._intents_container.add_intent(
                    intent_name, intent_example_sentences_array)

            for entity_name, entities_array in self.intent_training_file_content(
                    intent_file_path, 'entities'):
                #print ("add entity %s, %s " % (entity_name, entities_array))
                self._intents_container.add_entity(entity_name, entities_array)

                # adapt
                if entity_name.endswith("_keyword"):
                    for k in entities_array:
                        #print ("add keyword %s to %s" % (k, intent_name))
                        self._adapt_intent_engine.register_entity(
                            k, entity_name, domain=self._name)

                    adapt_intent_builder.require(entity_name)

            adapt_intent = adapt_intent_builder.build()
            self._adapt_intent_engine.register_intent_parser(adapt_intent,
                                                             domain=self._name)

        self._intents_container.train(debug=False)

    def get_intent_file_content(self, skill_file_path):
        content_array = []
        with open(skill_file_path, 'r', encoding='utf-8') as skill_file:
            for entry in skill_file:
                content_array.append(entry)
        return content_array

    def get_entities_file_content(self, skill_file_path, allow_variations):
        content_array = []
        with open(skill_file_path, 'r', encoding='utf-8') as skill_file:
            for entry in skill_file:
                entries, variations = entry.strip().split('|'), []
                content_array.append(entries[0])
                if allow_variations:
                    if len(entries) > 1:
                        content_array.extend(entries[1].split(','))
        return content_array

    def get_intent_names(self):
        intent_root_file_path = os.path.join(self._root_dir, self._name,
                                             'intents')
        for intent_name in os.listdir(intent_root_file_path):
            intent_file_path = os.path.join(intent_root_file_path, intent_name)
            yield intent_name, intent_file_path

    def intent_training_file_content(self,
                                     artefacts_root_dir,
                                     artefact_file_extension,
                                     allow_variations=True):
        for artefact_file_path in os.listdir(artefacts_root_dir):
            if artefact_file_path.endswith('.' + artefact_file_extension):
                artefact_name = artefact_file_path.replace(
                    '.' + artefact_file_extension, '')
                if artefact_file_extension is 'entities':
                    artefact_file_lines = self.get_entities_file_content(
                        os.path.join(artefacts_root_dir, artefact_file_path),
                        allow_variations)
                elif artefact_file_extension is 'intent':
                    artefact_file_lines = self.get_intent_file_content(
                        os.path.join(artefacts_root_dir, artefact_file_path))
                yield artefact_name, artefact_file_lines

    def expand_intents(self, include_additional_entities=False):
        # load entities first in the file and build a dictionary
        result = dict()
        entities_dict = dict()

        for intent_name, intent_file_path in self.get_intent_names():

            for entity_type, entities_array in self.intent_training_file_content(
                    intent_file_path, 'entities', False):
                entities_dict[entity_type] = entities_array

            # load intents again from file
            for intent_type, intent_array in self.intent_training_file_content(
                    intent_file_path, 'intent'):
                intent_sentences = set()
                for line in intent_array:
                    line_tokens = self._nlp.tokenization.tokenize(line)
                    expanded = expand_parentheses(line_tokens)
                    for sentence_tokens in expanded:
                        sentence = self._nlp.tokenization.detokenize(
                            sentence_tokens)
                        fieldnames = [
                            fname
                            for _, fname, _, _ in Formatter().parse(sentence)
                            if fname
                        ]
                        fields_dict = dict()
                        for fieldname in fieldnames:
                            if fieldname in entities_dict:
                                fields_dict[fieldname] = entities_dict[
                                    fieldname].copy()
                            else:
                                if include_additional_entities:
                                    field_values = self.get_additional_entities(
                                        fieldname)
                                    if len(field_values) > 0:
                                        fields_dict[fieldname] = field_values

                        if len(fields_dict) > 0:
                            keys, values = zip(*fields_dict.items())
                            permutations = [
                                dict(zip(keys, v))
                                for v in itertools.product(*values)
                            ]
                            for p in permutations:
                                entities_dict_permutation = EntitiesDict(p)
                                intent_sentences.add(
                                    sentence.format(
                                        **entities_dict_permutation))
                        else:
                            intent_sentences.add(sentence)

                result[intent_type] = list(intent_sentences)

        return result

    def get_additional_entities(self, fieldname):
        return []

    def calculate_intent(self, text):
        text = self._nlp.preprocess(text)

        # example result
        # {'intent_type': 'beth.fydd.y.tywydd', 'confidence': 1.0, 'target': None, 'keyword': 'tywydd'}
        #
        #print ("evaluating: %s with adapt:" % text)
        adapt_best_confidence = 0.0
        adapt_result = self._adapt_intent_engine.determine_intent(text)
        for a in adapt_result:
            # print (a)
            if a["confidence"] > adapt_best_confidence:
                adapt_best_confidence = a["confidence"]

        # example result
        # {'sent': "beth yw ' r tywydd", 'name': 'beth.ywr.tywydd', 'conf': 1.0, 'matches': {'tywydd_keyword': 'tywydd?'}}
        #
        #print ("evaluating: %s with padatious:" % text)
        padatious_result = self._intents_container.calc_intent(text)

        return adapt_best_confidence, padatious_result

    def handle(self, intent, latitude, longitude):
        pass
Ejemplo n.º 7
0
class SelectBestIntentTests(unittest.TestCase):
    """All tests related to the DomainIntentDeterminationEngine."""
    def setUp(self):
        """Setting up testing env."""
        self.engine = DomainIntentDeterminationEngine()

    def test_select_best_intent(self):
        """
        Test to make sure that best intent is being returned.
        This test is to make sure that best intent works identicly to its counter part
        in the IntentEngine.
        """
        parser1 = IntentBuilder("Parser1").require("Entity1").build()
        self.engine.register_intent_parser(parser1)
        self.engine.register_entity("tree", "Entity1")

        utterance = "go to the tree house"
        intent = next(self.engine.determine_intent(utterance))
        assert intent
        self.assertEqual(intent['intent_type'], 'Parser1')

        parser2 = IntentBuilder("Parser2").require("Entity1").require(
            "Entity2").build()
        self.engine.register_intent_parser(parser2)
        self.engine.register_entity("house", "Entity2")
        intent = next(self.engine.determine_intent(utterance))
        assert intent
        self.assertEqual(intent['intent_type'], 'Parser2')

    def test_select_best_intent_with_domain(self):
        """Test to make sure that best intent is working with domains."""
        self.engine.register_domain('Domain1')
        self.engine.register_domain('Domain2')

        # Creating first intent domain
        parser1 = IntentBuilder("Parser1").require("Entity1").build()
        self.engine.register_intent_parser(parser1, domain='Domain1')
        self.engine.register_entity("tree", "Entity1", domain='Domain1')

        # Creating second intent domain
        parser2 = IntentBuilder("Parser1").require("Entity2").build()
        self.engine.register_intent_parser(parser2, domain="Domain2")
        self.engine.register_entity("house", "Entity2", domain="Domain2")

        utterance = "Entity1 Entity2 go to the tree house"
        intents = self.engine.determine_intent(utterance, 2)

        intent = next(intents)
        assert intent
        self.assertEqual(intent['intent_type'], 'Parser1')

        intent = next(intents)
        assert intent
        self.assertEqual(intent['intent_type'], 'Parser1')

    def test_select_best_intent_enuse_enitities_dont_register_in_multiple_domains(
            self):
        """Test to make sure that 1 entity does not end up in multiple domains."""
        self.engine.register_domain('Domain1')
        self.engine.register_domain('Domain2')

        # Creating first intent domain
        parser1 = IntentBuilder("Parser1").require("Entity1").build()
        self.engine.register_intent_parser(parser1, domain='Domain1')
        self.engine.register_entity("tree", "Entity1", domain='Domain1')

        # Creating second intent domain
        parser2 = IntentBuilder("Parser2").require("Entity2").build()
        self.engine.register_intent_parser(parser2, domain="Domain2")
        self.engine.register_entity("house", "Entity2", domain="Domain2")

        utterance = "go to the house"
        intents = self.engine.determine_intent(utterance, 1)
        for intent in intents:
            self.assertNotEqual(intent['intent_type'], 'Parser1')

        utterance = "go to the tree"
        intents = self.engine.determine_intent(utterance, 1)
        for intent in intents:
            self.assertNotEqual(intent['intent_type'], 'Parser2')

    def test_drop_intent_from_domain(self):
        """Test that intent is dropped from the correct domain."""
        self.engine.register_domain('Domain1')
        self.engine.register_domain('Domain2')

        # Creating first intent domain
        parser1 = IntentBuilder("Parser1").require("Entity1").build()
        self.engine.register_intent_parser(parser1, domain='Domain1')
        self.engine.register_entity("tree", "Entity1", domain='Domain1')

        # Creating second intent domain
        parser2 = IntentBuilder("Parser2").require("Entity2").build()
        self.engine.register_intent_parser(parser2, domain="Domain2")
        self.engine.register_entity("house", "Entity2", domain="Domain2")

        self.engine.drop_intent_parser(domain="Domain2",
                                       parser_names=['Parser2'])
        self.assertEqual(len(self.engine.domains['Domain2'].intent_parsers), 0)

    def test_drop_entity_from_domain(self):
        """Test that entity is dropped from domain."""
        self.engine.register_domain('Domain1')
        self.engine.register_domain('Domain2')

        # Creating first intent domain
        parser1 = IntentBuilder("Parser1").require("Entity1").build()
        self.engine.register_intent_parser(parser1, domain='Domain1')
        self.engine.register_entity("tree", "Entity1", domain='Domain1')

        # Creating second intent domain
        parser2 = IntentBuilder("Parser2").require("Entity2").build()
        self.engine.register_intent_parser(parser2, domain="Domain2")
        self.engine.register_entity("house", "Entity2", domain="Domain2")

        self.assertTrue(
            self.engine.drop_entity(domain="Domain2", entity_type='Entity2'))

    def testDropRegexEntity(self):
        self.engine.register_domain("Domain1")
        self.engine.register_domain("Domain2")

        self.engine.register_regex_entity(r"the dog (?P<Dog>.*)", "Domain1")
        self.engine.register_regex_entity(r"the cat (?P<Cat>.*)", "Domain2")
        self.assertTrue(
            self.engine.drop_regex_entity(domain='Domain2', entity_type='Cat'))
        self.assertFalse(
            self.engine.drop_regex_entity(domain='Domain1', entity_type='Cat'))
Ejemplo n.º 8
0
class _IntentParser:
    def __init__(self):

        self.engine = DomainIntentDeterminationEngine()
        self.exact_matches = {}

        # Load all files in ./intents
        matches = []
        for root, dirnames, filenames in os.walk(
                os.path.join(os.path.dirname(__file__), 'intents')):
            for filename in fnmatch.filter(filenames, '*.json'):
                path = os.path.join(root, filename)
                matches.append(path)

        if len(matches) == 0:
            raise Exception("No intent specifications found at ",
                            os.path.join(os.path.dirname(__file__), 'intents'))

        for path in matches:
            with open(path) as file:
                if not self.loadIntentSpec(
                        root.split('/')[-1], json.load(file)):
                    logging.warning(
                        "Something went wrong during parsing of ",
                        os.path.join(os.path.dirname(__file__), 'intents'))

    def loadIntentSpec(self, domain, data):

        if not 'intent' in data:
            return False

        if not domain in self.engine.domains:
            self.engine.register_domain(domain)

        new_intent = IntentBuilder(data['intent'])

        if 'exact' in data:
            for term in data['exact']:
                self.exact_matches[term] = data['intent']

        if 'required' in data:
            for entity in data['required']:
                for key, value in entity.iteritems():
                    if isinstance(value, basestring):
                        self.engine.register_regex_entity(value, domain=domain)
                        new_intent = new_intent.require(key)
                        continue

                    for keyword in value:
                        self.engine.register_entity(keyword,
                                                    key,
                                                    domain=domain)

                    new_intent = new_intent.require(key)

        if 'optionally' in data:
            for entity in data['optionally']:
                for key, value in entity.iteritems():
                    for keyword in value:
                        self.engine.register_entity(keyword,
                                                    key,
                                                    domain=domain)

                new_intent = new_intent.optionally(key)

        new_intent = new_intent.build()
        self.engine.register_intent_parser(new_intent, domain=domain)

        return True

    def parse(self, text, expected_intents=None):
        """
        Returns best match for intent.
        """
        tokenized = ' '.join(tokenize_string(text)).lower()
        if tokenized in self.exact_matches:
            return {
                'intent_type': self.exact_matches[tokenized],
                'confidence': 1.0,
                'Keyword': tokenized
            }

        for intent in self.engine.determine_intent(text):
            if intent and intent.get('confidence') > 0.0:
                if expected_intents and intent.get(
                        'intent_type') in expected_intents:
                    return intent
                elif not expected_intents:
                    return intent
                else:
                    continue

        return None