def __init__(self, resource_name, backend, language_name):
self.intent_examples = []
self.entity_examples = []
self.resource_name = resource_name
self.files = util.recursively_find_files(resource_name)
self.fformat = self.guess_format(self.files)
self.tokenizer = None
self.language_name = language_name
if backend in ['mitie', 'mitie_sklearn']:
from rasa_nlu.tokenizers.mitie_tokenizer import MITIETokenizer
self.tokenizer = MITIETokenizer()
elif backend in ['spacy_sklearn']:
from rasa_nlu.tokenizers.spacy_tokenizer import SpacyTokenizer
self.tokenizer = SpacyTokenizer(language_name)
else:
from rasa_nlu.tokenizers.whitespace_tokenizer import WhitespaceTokenizer
self.tokenizer = WhitespaceTokenizer()
warnings.warn(
"backend not recognised by TrainingData : defaulting to tokenizing by splitting on whitespace"
)
if self.fformat == 'luis':
self.load_luis_data(self.files[0])
elif self.fformat == 'wit':
self.load_wit_data(self.files[0])
elif self.fformat == 'api':
self.load_api_data(self.files)
elif self.fformat == 'rasa_nlu':
self.load_data(self.files[0])
else:
raise ValueError("unknown training file format : {0}".format(
self.fformat))
def __init__(self, intent_classifier_file=None, entity_extractor_file=None, feature_extractor_file=None, **kwargs):
if entity_extractor_file:
self.extractor = named_entity_extractor(entity_extractor_file) # ,metadata["feature_extractor"])
with open(intent_classifier_file, 'rb') as f:
self.classifier = cloudpickle.load(f)
self.featurizer = MITIEFeaturizer(feature_extractor_file)
self.tokenizer = MITIETokenizer()
def __init__(self,
intent_classifier=None,
entity_extractor=None,
entity_synonyms=None):
self.extractor = entity_extractor
self.classifier = intent_classifier
self.ent_synonyms = entity_synonyms
self.tokenizer = MITIETokenizer()
def test_mitie():
from rasa_nlu.tokenizers.mitie_tokenizer import MITIETokenizer
tk = MITIETokenizer()
tk.tokenize(u"Hi. My name is rasa") == [
u'Hi', u'My', u'name', u'is', u'rasa'
]
tk.tokenize(u"ὦ ἄνδρες ᾿Αθηναῖοι.") == [u'ὦ', u'ἄνδρες', u'᾿Αθηναῖοι']
def __init__(self,
intent_classifier=None,
entity_extractor=None,
feature_extractor=None,
**kwargs):
self.extractor = named_entity_extractor(entity_extractor,
feature_extractor)
self.classifier = text_categorizer(intent_classifier,
feature_extractor)
self.tokenizer = MITIETokenizer()
def test_mitie():
from rasa_nlu.tokenizers.mitie_tokenizer import MITIETokenizer
tk = MITIETokenizer()
tk.tokenize(u"Hi. My name is rasa") == [
u'Hi', u'My', u'name', u'is', u'rasa'
]
tk.tokenize(u"ὦ ἄνδρες ᾿Αθηναῖοι.") == [u'ὦ', u'ἄνδρες', u'᾿Αθηναῖοι']
tk.tokenize_with_offsets(u"Forecast for lunch") == ([
u'Forecast', u'for', u'lunch'
], [0, 9, 13])
def find_entity(cls, ent, text):
tk = MITIETokenizer()
tokens, offsets = tk.tokenize_with_offsets(text)
if ent["start"] not in offsets:
message = u"invalid entity {0} in example {1}:".format(ent, text) + \
u" entities must span whole tokens"
raise ValueError(message)
start = offsets.index(ent["start"])
_slice = text[ent["start"]:ent["end"]]
val_tokens = tokenize(_slice)
end = start + len(val_tokens)
return start, end
def init_tokenizer(self, backend, nlp):
if backend in [mitie.MITIE_BACKEND_NAME, mitie.MITIE_SKLEARN_BACKEND_NAME]:
from rasa_nlu.tokenizers.mitie_tokenizer import MITIETokenizer
self.tokenizer = MITIETokenizer()
elif backend in [spacy.SPACY_BACKEND_NAME]:
from rasa_nlu.tokenizers.spacy_tokenizer import SpacyTokenizer
self.tokenizer = SpacyTokenizer(nlp)
else:
from rasa_nlu.tokenizers.whitespace_tokenizer import WhitespaceTokenizer
self.tokenizer = WhitespaceTokenizer()
warnings.warn(
"backend not recognised by TrainingData : defaulting to tokenizing by splitting on whitespace")
def test_mitie_featurizer():
from rasa_nlu.featurizers.mitie_featurizer import MITIEFeaturizer
filename = os.environ.get('MITIE_FILE')
if not filename or not os.path.isfile(filename):
filename = "data/total_word_feature_extractor.dat"
ftr = MITIEFeaturizer(filename)
sentence = "Hey how are you today"
tokens = MITIETokenizer().tokenize(sentence)
vecs = ftr.features_for_tokens(tokens)
assert np.allclose(
vecs[:5],
np.array([0., -4.4551446, 0.26073121, -1.46632245, -1.84205751]),
atol=1e-5)
def test_mitie():
from rasa_nlu.tokenizers.mitie_tokenizer import MITIETokenizer
tk = MITIETokenizer()
assert tk.tokenize(u"Hi. My name is rasa") == [
u'Hi', u'My', u'name', u'is', u'rasa'
]
assert tk.tokenize(u"ὦ ἄνδρες ᾿Αθηναῖοι") == [
u'ὦ', u'ἄνδρες', u'᾿Αθηναῖοι'
]
assert tk.tokenize_with_offsets(u"Forecast for lunch") == ([
u'Forecast', u'for', u'lunch'
], [0, 9, 13])
assert tk.tokenize_with_offsets(u"hey ńöñàśçií how're you?") == ([
u'hey', u'ńöñàśçií', u'how', u'\'re', 'you', '?'
], [0, 4, 13, 16, 20, 23])
def __init__(self,
intent_classifier=None,
entity_extractor=None,
feature_extractor=None,
entity_synonyms=None,
**kwargs):
self.extractor = None
self.classifier = None
if entity_extractor:
self.extractor = named_entity_extractor(entity_extractor, feature_extractor)
if intent_classifier:
self.classifier = text_categorizer(intent_classifier, feature_extractor)
self.tokenizer = MITIETokenizer()
self.ent_synonyms = None
if entity_synonyms:
Interpreter.load_synonyms(entity_synonyms)
def __init__(self,
intent_classifier=None,
entity_extractor=None,
feature_extractor=None,
entity_synonyms=None,
**kwargs):
self.extractor = None
self.classifier = None
if entity_extractor:
self.extractor = named_entity_extractor(entity_extractor,
feature_extractor)
if intent_classifier:
with open(intent_classifier, 'rb') as f:
self.classifier = cloudpickle.load(f)
self.featurizer = MITIEFeaturizer(feature_extractor)
self.tokenizer = MITIETokenizer()
self.ent_synonyms = None
if entity_synonyms:
self.ent_synonyms = Interpreter.load_synonyms(entity_synonyms)
def __init__(self, metadata):
self.extractor = named_entity_extractor(
metadata["entity_extractor"]) # ,metadata["feature_extractor"])
self.classifier = text_categorizer(
metadata["intent_classifier"]) # ,metadata["feature_extractor"])
self.tokenizer = MITIETokenizer()