Example #1
0
def parseMsg(msg):
    container = IntentContainer('intent_cache')

    container.add_intent('greetings',
                         ['hi.', 'hellow', '(waz|wazz|wad) up?', 'hey', 'yoo', 'buddy', '(you|hey) there?'])
    container.add_intent('bye', ['bye.', 'tata', 'thank you', 'adios', 'thanks', 'allah hafez', 'done'])
    # container.add_intent('goal', ['i want to go (|to) {goal}', '(destination|i wanna go) {goal}',
    #                               'destination is {goal}'])
    # container.add_intent('start', ['from {start}', '(i am (in|at){start}'
    #                                ])
    container.add_intent('search',
                         ['I want to go from {start} to {goal}.', 'I want to (go to|go) {goal} from {start}.'])
    container.add_intent('ajaira', ['lala', 'kchdskfhsk', 'iwurhb', 'uerwyvdsvjjkc', 'sufgbsdjc'])

    container.train()
    result = container.calc_intent(msg)
    print(result)
    print(result.name)
    # print(str(result.matches['start']))
    # print(result.matches['start'])
    # print(result.matches['goal'])
    return result
class PadatiousExtractor(IntentExtractor):
    keyword_based = False

    def __init__(self, cache_dir=None, *args, **kwargs):
        super().__init__(*args, **kwargs)
        # TODO xdg data_dir
        data_dir = expanduser(self.config.get("data_dir", "~/.padatious"))
        cache_dir = cache_dir or join(data_dir, "padatious")
        self.lock = Lock()
        self.container = IntentContainer(cache_dir)
        self.registered_intents = []

    def detach_intent(self, intent_name):
        if intent_name in self.registered_intents:
            LOG.debug("Detaching padatious intent: " + intent_name)
            with self.lock:
                self.container.remove_intent(intent_name)
            self.registered_intents.remove(intent_name)

    def detach_skill(self, skill_id):
        LOG.debug("Detaching padatious skill: " + str(skill_id))
        remove_list = [i for i in self.registered_intents if skill_id in i]
        for i in remove_list:
            self.detach_intent(i)

    def register_entity(self, entity_name, samples=None, reload_cache=True):
        samples = samples or [entity_name]
        with self.lock:
            self.container.add_entity(entity_name,
                                      samples,
                                      reload_cache=reload_cache)

    def register_intent(self, intent_name, samples=None, reload_cache=True):
        samples = samples or [intent_name]
        if intent_name not in self._intent_samples:
            self._intent_samples[intent_name] = samples
        else:
            self._intent_samples[intent_name] += samples
        with self.lock:
            self.container.add_intent(intent_name,
                                      samples,
                                      reload_cache=reload_cache)
        self.registered_intents.append(intent_name)

    def register_entity_from_file(self,
                                  entity_name,
                                  file_name,
                                  reload_cache=True):
        with self.lock:
            self.container.load_entity(entity_name,
                                       file_name,
                                       reload_cache=reload_cache)

    def register_intent_from_file(self,
                                  intent_name,
                                  file_name,
                                  single_thread=True,
                                  timeout=120,
                                  reload_cache=True,
                                  force_training=True):
        try:
            with self.lock:
                self.container.load_intent(intent_name,
                                           file_name,
                                           reload_cache=reload_cache)
            self.registered_intents.append(intent_name)
            success = self._train(single_thread=single_thread,
                                  timeout=timeout,
                                  force_training=force_training)
            if success:
                LOG.debug(file_name + " trained successfully")
            else:
                LOG.error(file_name + " FAILED TO TRAIN")

        except Exception as e:
            LOG.exception(e)

    def _get_remainder(self, intent, utterance):
        if intent["name"] in self.intent_samples:
            return get_utterance_remainder(
                utterance, samples=self.intent_samples[intent["name"]])
        return utterance

    def calc_intent(self, utterance, min_conf=None):
        min_conf = min_conf or self.config.get("padatious_min_conf", 0.65)
        utterance = utterance.strip().lower()
        with self.lock:
            intent = self.container.calc_intent(utterance).__dict__
        if intent["conf"] < min_conf:
            return {
                "intent_type": "unknown",
                "entities": {},
                "conf": 0,
                "intent_engine": "padatious",
                "utterance": utterance,
                "utterance_remainder": utterance
            }
        intent["utterance_remainder"] = self._get_remainder(intent, utterance)
        intent["entities"] = intent.pop("matches")
        intent["intent_engine"] = "padatious"
        intent["intent_type"] = intent.pop("name")
        intent["utterance"] = intent.pop("sent")

        if isinstance(intent["utterance"], list):
            intent["utterance"] = " ".join(intent["utterance"])
        return intent

    def intent_scores(self, utterance):
        utterance = utterance.strip().lower()
        intents = [i.__dict__ for i in self.container.calc_intents(utterance)]
        for idx, intent in enumerate(intents):
            intent["utterance_remainder"] = self._get_remainder(
                intent, utterance)
            intents[idx]["entities"] = intents[idx].pop("matches")
            intents[idx]["intent_type"] = intents[idx].pop("name")
            intent["intent_engine"] = "padatious"
            intent["utterance"] = intent.pop("sent")
            if isinstance(intents[idx]["utterance"], list):
                intents[idx]["utterance"] = " ".join(intents[idx]["utterance"])
        return intents

    def calc_intents(self, utterance, min_conf=None):
        min_conf = min_conf or self.config.get("padatious_min_conf", 0.65)
        utterance = utterance.strip().lower()
        bucket = {}
        for ut in self.segmenter.segment(utterance):
            intent = self.calc_intent(ut)
            if intent["conf"] < min_conf:
                bucket[ut] = None
            else:
                bucket[ut] = intent
        return bucket

    def calc_intents_list(self, utterance):
        utterance = utterance.strip().lower()
        bucket = {}
        for ut in self.segmenter.segment(utterance):
            bucket[ut] = self.filter_intents(ut)
        return bucket

    def manifest(self):
        # TODO vocab, skill ids, intent_data
        return {"intent_names": self.registered_intents}

    def _train(self, single_thread=True, timeout=120, force_training=True):
        with self.lock:
            return self.container.train(single_thread=single_thread,
                                        timeout=timeout,
                                        force=force_training,
                                        debug=True)
Example #3
0
def determine_intent(msg):
    container = IntentContainer('intent_cache')
    container.add_intent('time', ['What is the time now.', 'Tell me the time'])
    container.add_intent(
        'goodbye', ['See you!', 'Goodbye!', 'Bye', 'Bye Bye', 'Go to sleep'])
    container.add_intent('search', [
        'Search for {query} (using|on) (internet|web|google).',
        'Lookup {query} (using|on) the (internet|web|google).',
        'Search {addons} {query}.'
    ])
    container.add_intent('device', [
        'Turn the {location} {device} {state}.',
        'Turn {state} the {location} {device}.', 'Turn the {device} {state}.'
    ])
    #container.add_intent('fans', ['Turn the {location} (fan|fans) {state}.', 'Turn {state} the {location} (fan|fans).'])
    container.add_intent('Wolfram', [
        'What (is|are|was|were) {query}.',
        'Who (is|was|were|invented|did|scored|will be) {query}.',
        'When (is|are|was|were) {query}.', 'How (is|are|was|were) {query}.'
    ])
    container.add_intent(
        'Creator',
        ['Who created you.', 'Who made you.', 'By whom were you created.'])
    container.add_intent(
        'news',
        ['Tell me the news.', 'What is the news.', 'Get me the news update'])
    container.add_intent(
        'weather', ['Tell me about the weather.', 'What is the weather like.'])
    container.add_intent('Joke', [
        'Tell me a joke.', 'Tell me a {type} joke.',
        'Can you (say|tell) a joke.', 'Can you (say|tell) a {type} joke.'
    ])
    container.add_intent('Me',
                         ['Tell me about yourself.', '(Who|What) are you.'],
                         'What is your name.')
    container.add_intent(
        'Praise', ['I am great.', 'I (did|completed) {task}.'])  #Sarcasm

    container.train()
    data = container.calc_intent(msg)
    return data
class ColossalCaveAdventureSkill(MycroftSkill):
    save_file = expanduser("~/cave_adventure.save")
    playing = False
    container = None

    def initialize(self):
        self.game = Game()
        load_advent_dat(self.game)
        self.last_interaction = time.time()
        self._init_padatious()
        self.disable_intent("save.intent")

    def _init_padatious(self):
        # i want to check in converse method if some intent by this skill will trigger
        # however there is no mechanism to query the intent parser
        # PR incoming
        intent_cache = expanduser(
            self.config_core['padatious']['intent_cache'])
        self.container = IntentContainer(intent_cache)
        for intent in ["restore.intent", "play.intent", "credits.intent"]:
            name = str(self.skill_id) + ':' + intent
            filename = self.find_resource(intent, 'vocab')
            if filename is not None:
                with open(filename, "r") as f:
                    self.container.add_intent(name, f.readlines())
        self.container.train()

    def will_trigger(self, utterance):
        # check if self will trigger for given utterance
        # adapt match
        if self.voc_match(utterance, "save"):
            return True
        # padatious match
        intent = self.container.calc_intent(utterance)
        if intent.conf < 0.5:
            return False
        return True

    def get_intro_message(self):
        """ Get a message to speak on first load of the skill.

        Useful for post-install setup instructions.

        Returns:
            str: message that will be spoken to the user
        """
        self.speak_dialog("thank.you")
        return None

    def speak_output(self, line):
        # dont speak parts of the intro
        # replace type with say because its voice game
        # re join words split across lines
        # reformat \n and separate by sentence
        line = line.lower().replace("type", "say").replace("-\n", "")
        line = line.replace(
            '  i should warn\nyou that i look at only the first five letters of each word, so you\'ll\nhave to enter "northeast" as "ne" to distinguish it from "north".',
            "")
        line = line.replace(
            "- - this program was originally developed by willie crowther.  most of the\nfeatures of the current program were added by don woods (don @ su-ai).\ncontact don if you have any questions, comments, etc.",
            "")
        line = line.replace("\n", " ").replace("(",
                                               "").replace(")", "").replace(
                                                   "etc.", "etc")
        lines = line.split(".")
        for line in lines:
            self.speak(line.strip(), expect_response=True, wait=True)
        self.last_interaction = time.time()
        self.maybe_end_game()

    @intent_file_handler("credits.intent")
    def handle_credits(self, message=None):
        self.speak_dialog("credits")

    @intent_file_handler("play.intent")
    def handle_play(self, message=None):
        self.playing = True
        self.enable_intent("save.intent")
        self.game.start()
        self.speak_output(self.game.output)

    @intent_handler(
        IntentBuilder("Save").require("save").optionally("cave").optionally(
            "adventure"))
    def handle_save(self, message=None):
        if not self.playing:
            self.speak_dialog("save.not.found")
        else:
            with open(self.save_file, "wb") as f:
                self.game.t_suspend("save", f)
                self.speak_dialog("game.saved")

    @intent_file_handler("restore.intent")
    def handle_restore(self, message):
        if exists(self.save_file):
            self.playing = True
            self.game = Game.resume(self.save_file)
            self.speak_dialog("restore.game")
        else:
            self.speak_dialog("save.not.found")
            new_game = self.ask_yesno("new.game")
            if new_game:
                self.handle_play()

    def maybe_end_game(self):
        # end game if no interaction for 10 mins
        if self.playing:
            timed_out = time.time() - self.last_interaction > 10 * 3600
            # disable save and gameplay
            if self.game.is_finished or timed_out:
                self.disable_intent("Save")
                self.playing = False
                self.game = Game()
                load_advent_dat(self.game)
            # save game to allow restoring if timedout
            if timed_out:
                self.handle_save()

    def converse(self, utterances, lang="en-us"):
        """ Handle conversation.

        This method gets a peek at utterances before the normal intent
        handling process after a skill has been invoked once.

        To use, override the converse() method and return True to
        indicate that the utterance has been handled.

        Args:
            utterances (list): The utterances from the user
            lang:       language the utterance is in

        Returns:
            bool: True if an utterance was handled, otherwise False
        """
        # check if game was abandoned midconversation and we should clean it up
        self.maybe_end_game()
        if self.playing:
            ut = utterances[0]
            # if self will trigger do nothing and let intents handle it
            if self.will_trigger(ut):
                # save / restore will trigger
                return False
            # capture speech and pipe to the game
            words = ut.split(" ")
            if words:
                self.speak_output(self.game.do_command(words))
                return True
        return False
Example #5
0
class Skill(object):
    def __init__(self, root_dir, name, nlp, active):
        self._root_dir = root_dir
        self._name = name
        self._nlp = nlp
        self._active = active

        self._intents_container = None
        self._adapt_intent_engine = None

        self.initialize_intent_parser()

    def is_active(self):
        return self._active

    def get_name(self):
        return self._name

    def initialize_intent_parser(self):

        self._intents_container = IntentContainer("%s_cache" % self._name)

        self._adapt_intent_engine = DomainIntentDeterminationEngine()
        self._adapt_intent_engine.register_domain(self._name)

        for intent_name, intent_file_path in self.get_intent_names():
            #print ("###### IntentBuilder: %s, %s" % (intent_name, intent_file_path))
            adapt_intent_builder = IntentBuilder(intent_name)
            for intent_name, intent_example_sentences_array in self.intent_training_file_content(
                    intent_file_path, 'intent'):
                #print ("add intent %s, %s" % (intent_name, intent_example_sentences_array))
                self._intents_container.add_intent(
                    intent_name, intent_example_sentences_array)

            for entity_name, entities_array in self.intent_training_file_content(
                    intent_file_path, 'entities'):
                #print ("add entity %s, %s " % (entity_name, entities_array))
                self._intents_container.add_entity(entity_name, entities_array)

                # adapt
                if entity_name.endswith("_keyword"):
                    for k in entities_array:
                        #print ("add keyword %s to %s" % (k, intent_name))
                        self._adapt_intent_engine.register_entity(
                            k, entity_name, domain=self._name)

                    adapt_intent_builder.require(entity_name)

            adapt_intent = adapt_intent_builder.build()
            self._adapt_intent_engine.register_intent_parser(adapt_intent,
                                                             domain=self._name)

        self._intents_container.train(debug=False)

    def get_intent_file_content(self, skill_file_path):
        content_array = []
        with open(skill_file_path, 'r', encoding='utf-8') as skill_file:
            for entry in skill_file:
                content_array.append(entry)
        return content_array

    def get_entities_file_content(self, skill_file_path, allow_variations):
        content_array = []
        with open(skill_file_path, 'r', encoding='utf-8') as skill_file:
            for entry in skill_file:
                entries, variations = entry.strip().split('|'), []
                content_array.append(entries[0])
                if allow_variations:
                    if len(entries) > 1:
                        content_array.extend(entries[1].split(','))
        return content_array

    def get_intent_names(self):
        intent_root_file_path = os.path.join(self._root_dir, self._name,
                                             'intents')
        for intent_name in os.listdir(intent_root_file_path):
            intent_file_path = os.path.join(intent_root_file_path, intent_name)
            yield intent_name, intent_file_path

    def intent_training_file_content(self,
                                     artefacts_root_dir,
                                     artefact_file_extension,
                                     allow_variations=True):
        for artefact_file_path in os.listdir(artefacts_root_dir):
            if artefact_file_path.endswith('.' + artefact_file_extension):
                artefact_name = artefact_file_path.replace(
                    '.' + artefact_file_extension, '')
                if artefact_file_extension is 'entities':
                    artefact_file_lines = self.get_entities_file_content(
                        os.path.join(artefacts_root_dir, artefact_file_path),
                        allow_variations)
                elif artefact_file_extension is 'intent':
                    artefact_file_lines = self.get_intent_file_content(
                        os.path.join(artefacts_root_dir, artefact_file_path))
                yield artefact_name, artefact_file_lines

    def expand_intents(self, include_additional_entities=False):
        # load entities first in the file and build a dictionary
        result = dict()
        entities_dict = dict()

        for intent_name, intent_file_path in self.get_intent_names():

            for entity_type, entities_array in self.intent_training_file_content(
                    intent_file_path, 'entities', False):
                entities_dict[entity_type] = entities_array

            # load intents again from file
            for intent_type, intent_array in self.intent_training_file_content(
                    intent_file_path, 'intent'):
                intent_sentences = set()
                for line in intent_array:
                    line_tokens = self._nlp.tokenization.tokenize(line)
                    expanded = expand_parentheses(line_tokens)
                    for sentence_tokens in expanded:
                        sentence = self._nlp.tokenization.detokenize(
                            sentence_tokens)
                        fieldnames = [
                            fname
                            for _, fname, _, _ in Formatter().parse(sentence)
                            if fname
                        ]
                        fields_dict = dict()
                        for fieldname in fieldnames:
                            if fieldname in entities_dict:
                                fields_dict[fieldname] = entities_dict[
                                    fieldname].copy()
                            else:
                                if include_additional_entities:
                                    field_values = self.get_additional_entities(
                                        fieldname)
                                    if len(field_values) > 0:
                                        fields_dict[fieldname] = field_values

                        if len(fields_dict) > 0:
                            keys, values = zip(*fields_dict.items())
                            permutations = [
                                dict(zip(keys, v))
                                for v in itertools.product(*values)
                            ]
                            for p in permutations:
                                entities_dict_permutation = EntitiesDict(p)
                                intent_sentences.add(
                                    sentence.format(
                                        **entities_dict_permutation))
                        else:
                            intent_sentences.add(sentence)

                result[intent_type] = list(intent_sentences)

        return result

    def get_additional_entities(self, fieldname):
        return []

    def calculate_intent(self, text):
        text = self._nlp.preprocess(text)

        # example result
        # {'intent_type': 'beth.fydd.y.tywydd', 'confidence': 1.0, 'target': None, 'keyword': 'tywydd'}
        #
        #print ("evaluating: %s with adapt:" % text)
        adapt_best_confidence = 0.0
        adapt_result = self._adapt_intent_engine.determine_intent(text)
        for a in adapt_result:
            # print (a)
            if a["confidence"] > adapt_best_confidence:
                adapt_best_confidence = a["confidence"]

        # example result
        # {'sent': "beth yw ' r tywydd", 'name': 'beth.ywr.tywydd', 'conf': 1.0, 'matches': {'tywydd_keyword': 'tywydd?'}}
        #
        #print ("evaluating: %s with padatious:" % text)
        padatious_result = self._intents_container.calc_intent(text)

        return adapt_best_confidence, padatious_result

    def handle(self, intent, latitude, longitude):
        pass
Example #6
0
from padatious import IntentContainer

container = IntentContainer('intent_cache')
container.add_intent('greetings', ['Hi there!', 'Hello.'])
container.add_intent('goodbye', ['See you!', 'Goodbye!'])
container.add_intent('search', ['Search for {query} (using|on) {engine}.'])
container.train()

print(container.calc_intent('Hi!'))
print(container.calc_intent('Search for cats on CatTube.'))

result=container.calc_intent('search mamamamam')
print(result.name)


if __name__ == '__main__':

 if (result.name=="search"):
     print("you are searching")
 elif (result.name=="greetings"):
     print("ami greet")
 else:
     print("ami kisui na")




Example #7
0
along with this program.  If not, see <https://www.gnu.org/licenses/>.
"""

import client
import time
import html2text
import nimiqx
import currencies
from padatious import IntentContainer

h = html2text.HTML2Text()
h.ignore_links = True

container = IntentContainer("intent_cache")

container.add_intent("price_currency",
                     ["price {currency}", "price in {currency}"])


def sendReply(message: str, originalToot: tuple):
    newToot = client.mastodon.status_post(
        message,
        in_reply_to_id=originalToot,
        visibility="direct",
    )
    print("Reply sent: {toot_id}".format(toot_id=newToot.id))


if __name__ == "__main__":
    print("Bot started")
    while True:
        notifications = client.mastodon.notifications()
Example #8
0
class TNaLaGmesConstruct(object):
    cache_dir = join(expanduser("~"), "tnalagmes", "intent_cache")
    if not exists(cache_dir):
        makedirs(cache_dir)

    def __init__(self, object_type="tnalagmes_object", adapt=None):
        self.adapt = adapt or IntentDeterminationEngine()
        self.context_manager = ContextManager(self.adapt)
        self.object_type = object_type
        self.container = IntentContainer(self.cache_dir)
        self.intents = {}
        self.register_default_intents()
        self.register_core_intents()
        self.container.train()
        self.waiting_for_user = False
        self._output = ""
        self.input = ""

    @staticmethod
    def fuzzy_match(x, against):
        """Perform a 'fuzzy' comparison between two strings.
        Returns:
            float: match percentage -- 1.0 for perfect match,
                   down to 0.0 for no match at all.
        """
        return SequenceMatcher(None, x, against).ratio()

    @staticmethod
    def match_one(query, choices):
        """
            Find best match from a list or dictionary given an input

            Arguments:
                query:   string to test
                choices: list or dictionary of choices

            Returns: tuple with best match, score
        """
        if isinstance(choices, dict):
            _choices = list(choices.keys())
        elif isinstance(choices, list):
            _choices = choices
        else:
            raise ValueError('a list or dict of choices must be provided')

        best = (_choices[0],
                TNaLaGmesConstruct.fuzzy_match(query, _choices[0]))
        for c in _choices[1:]:
            score = TNaLaGmesConstruct.fuzzy_match(query, c)
            if score > best[1]:
                best = (c, score)

        if isinstance(choices, dict):
            return choices[best[0]], best[1]
        else:
            return best

    @staticmethod
    def extract_number(text, short_scale=True, ordinals=False, lang="en-us"):
        """Takes in a string and extracts a number.

        Args:
            text (str): the string to extract a number from
            short_scale (bool): Use "short scale" or "long scale" for large
                numbers -- over a million.  The default is short scale, which
                is now common in most English speaking countries.
                See https://en.wikipedia.org/wiki/Names_of_large_numbers
            ordinals (bool): consider ordinal numbers, e.g. third=3 instead of 1/3
            lang (str): the BCP-47 code for the language to use
        Returns:
            (int, float or False): The number extracted or False if the input
                                   text contains no numbers
        """
        lang_lower = str(lang).lower()
        if lang_lower.startswith("en"):
            return extract_number_en(text,
                                     short_scale=short_scale,
                                     ordinals=ordinals)
        elif lang_lower.startswith("pt"):
            return extractnumber_pt(text)
        elif lang_lower.startswith("it"):
            return extractnumber_it(text)
        elif lang_lower.startswith("fr"):
            return extractnumber_fr(text)
        elif lang_lower.startswith("sv"):
            return extractnumber_sv(text)
        # elif lang_lower.startswith("de"):
        #    return extractnumber_de(text)
        # TODO: extractnumber_xx for other languages
        return text

    @staticmethod
    def extract_datetime(text, anchor=None, lang="en-us"):
        """
        Extracts date and time information from a sentence.  Parses many of the
        common ways that humans express dates and times, including relative dates
        like "5 days from today", "tomorrow', and "Tuesday".

        Vague terminology are given arbitrary values, like:
            - morning = 8 AM
            - afternoon = 3 PM
            - evening = 7 PM

        If a time isn't supplied or implied, the function defaults to 12 AM

        Args:
            text (str): the text to be interpreted
            anchor (:obj:`datetime`, optional): the date to be used for
                relative dating (for example, what does "tomorrow" mean?).
                Defaults to the current local date/time.
            lang (string): the BCP-47 code for the language to use

        Returns:
            [:obj:`datetime`, :obj:`str`]: 'datetime' is the extracted date
                as a datetime object in the user's local timezone.
                'leftover_string' is the original phrase with all date and time
                related keywords stripped out. See examples for further
                clarification

                Returns 'None' if the input string is empty.

        Examples:

            extract_datetime(
            ... "What is the weather like the day after tomorrow?",
            ... datetime(2017, 06, 30, 00, 00)
            ... )
            [datetime.datetime(2017, 7, 2, 0, 0), 'what is weather like']

            extract_datetime(
            ... "Set up an appointment 2 weeks from Sunday at 5 pm",
            ... datetime(2016, 02, 19, 00, 00)
            ... )
            [datetime.datetime(2016, 3, 6, 17, 0), 'set up appointment']
        """

        lang_lower = str(lang).lower()

        if not anchor:
            anchor = now_local()

        if lang_lower.startswith("en"):
            return extract_datetime_en(text, anchor)
        elif lang_lower.startswith("pt"):
            return extract_datetime_pt(text, anchor)
        elif lang_lower.startswith("it"):
            return extract_datetime_it(text, anchor)
        elif lang_lower.startswith("fr"):
            return extract_datetime_fr(text, anchor)
        elif lang_lower.startswith("sv"):
            return extract_datetime_sv(text, anchor)
        # TODO: extract_datetime for other languages
        return text

    @staticmethod
    def normalize(text, lang="en-us", remove_articles=True):
        """Prepare a string for parsing

        This function prepares the given text for parsing by making
        numbers consistent, getting rid of contractions, etc.
        Args:
            text (str): the string to normalize
            lang (str): the code for the language text is in
            remove_articles (bool): whether to remove articles (like 'a', or
                                    'the'). True by default.
        Returns:
            (str): The normalized string.
        """

        lang_lower = str(lang).lower()
        if lang_lower.startswith("en"):
            return normalize_en(text, remove_articles)
        elif lang_lower.startswith("es"):
            return normalize_es(text, remove_articles)
        elif lang_lower.startswith("pt"):
            return normalize_pt(text, remove_articles)
        elif lang_lower.startswith("it"):
            return normalize_it(text, remove_articles)
        elif lang_lower.startswith("fr"):
            return normalize_fr(text, remove_articles)
        elif lang_lower.startswith("sv"):
            return normalize_sv(text, remove_articles)
        # TODO: Normalization for other languages
        return text

    @staticmethod
    def word_gender(word, input_string="", lang="en-us"):
        '''
        guess gender of word, optionally use raw input text for context
        returns "m" if the word is male, "f" if female, False if unknown
        '''
        if "pt" in lang or "es" in lang:
            # spanish follows same rules
            return get_gender_pt(word, input_string)
        elif "it" in lang:
            return get_gender_it(word, input_string)

        return False

    @property
    def output(self):
        out = self._output
        self._output = ""
        return out

    def manual_fix_parse(self, text):
        # TODO replace vars
        return text

    @output.setter
    def output(self, text=""):
        if isinstance(text, list):
            text = [t.strip() for t in text if t.strip()]
            text = random.choice(text)
        else:
            if not text.strip():
                return
        self._output += self.manual_fix_parse(text) + "\n"

    def ask_yes_no(self, prompt):
        self.output = prompt
        self.waiting_for_user = True
        while self.waiting_for_user:
            sleep(0.1)
        response = self.normalize(self.input)
        if response[0] == 'y':
            return True
        if response[0] == 'n':
            return False
        else:
            return self.ask_yes_no(prompt)

    def ask_numeric(self, prompt, lower_bound=None, upper_bound=None):
        self.output = prompt
        self.waiting_for_user = True
        while self.waiting_for_user:
            sleep(0.1)
        response = self.extract_number(self.input)
        try:
            value = int(response)
        except ValueError:
            self.output = "impossible!"
            return self.ask_numeric(prompt, lower_bound, upper_bound)
        if lower_bound is not None:
            if value < lower_bound:
                self.output = str(response) + " is too low"
                return self.ask_numeric(prompt, lower_bound, upper_bound)
        if upper_bound is not None:
            if value > upper_bound:
                self.output = str(response) + " is too high"
                return self.ask_numeric(prompt, lower_bound, upper_bound)
        return value

    def ask_with_timeout(self, prompt="say BANG", timeout=7):
        self.output = prompt
        self.waiting_for_user = True
        while self.waiting_for_user:
            sleep(0.1)
        response = self.input.lower().strip()
        # TODO measure mic level or type speed
        return response, random.randint(1, 7)

    def register_default_intents(self):
        pass

    def handle_yes(self, intent):
        return "you sound positive"

    def handle_no(self, intent):
        return "you sound negative"

    def register_core_intents(self):
        self.register_keyword_intent("yes", handler=self.handle_yes)
        self.register_keyword_intent("no", handler=self.handle_no)

    def calc_intent(self, utterance, lang="en-us"):
        # check adapt
        best_intent = None
        utterances = utterance
        if isinstance(utterance, str):
            utterances = [utterance]
        for utterance in utterances:
            try:
                # normalize() changes "it's a boy" to "it is boy", etc.
                ut = self.normalize(utterance, lang)

                if not ut:
                    continue
                best_intent = next(
                    self.adapt.determine_intent(
                        ut,
                        100,
                        include_tags=True,
                        context_manager=self.context_manager))
                # TODO - Should Adapt handle this?
                best_intent['utterance'] = utterance
                best_intent['normalized_utterance'] = ut
            except StopIteration:
                # don't show error in log
                continue
            except Exception as e:
                LOG.exception(e)
                continue
        if best_intent and best_intent.get('confidence', 0.0) > 0.0:
            return best_intent
        LOG.debug("unknown adapt command: " + str(utterances))
        # check padatious
        return self.container.calc_intent(utterance)

    def register_intent(self, name, samples, handler=None):
        self.container.add_intent(name, samples)
        self.intents[name] = handler

    @staticmethod
    def load_resource(name, sep="\n", is_json=False):
        path = resolve_resource_file(name)
        if path and exists(path):
            with open(path, "r") as f:
                lines = f.read()
                if is_json:
                    return json.loads(lines)
                return lines.split(sep)
        return None

    def register_keyword_intent(self,
                                name,
                                samples=None,
                                optionals=None,
                                handler=None,
                                ignore_default_kw=False):
        optionals = optionals or []
        samples = samples or [name]
        intent_name = self.object_type + ':' + name
        intent = IntentBuilder(intent_name)

        if samples and isinstance(samples, list):
            samples = {samples[0]: samples}
        if optionals and isinstance(optionals, list):
            optionals = {optionals[0]: optionals}

        if not ignore_default_kw:
            data = self.load_resource(name)
            if data:
                # merge file data
                for new in data:
                    if new not in samples[name]:
                        samples[name].append(new)

            for optional in optionals:
                data = self.load_resource(optional)
                if data:
                    # merge file data
                    for new in data:
                        optionals[optional] = optionals[optional] or []
                        if new not in optionals[optional]:
                            optionals[optional].append(new)

        for required in samples:
            for kw in samples[required]:
                self.adapt.register_entity(required, kw)
            intent.require(required)
        for optional in optionals:
            for kw in optionals[optional]:
                self.adapt.register_entity(optional, kw)
            intent.optionally(optional)
        self.adapt.register_intent_parser(intent.build())
        self.intents[intent_name] = handler

    def parse_command(self, utterance):
        # parse intent
        intent = self.calc_intent(utterance)
        intent_name = intent.get("intent_type", "")
        if intent_name in self.intents:
            return self.intents[intent_name](intent)
        return "?"