Esempio n. 1
0
def do_train(cfg,  # type: RasaNLUModelConfig
             data,  # type: Text
             path=None,  # type: Text
             project=None,  # type: Optional[Text]
             fixed_model_name=None,  # type: Optional[Text]
             storage=None,  # type: Text
             component_builder=None,  # type: Optional[ComponentBuilder]
             **kwargs   # type: Any
             ):
    # type: (...) -> Tuple[Trainer, Interpreter, Text]
    """Loads the trainer and the data and runs the training of the model."""

    # Ensure we are training a model that we can save in the end
    # WARN: there is still a race condition if a model with the same name is
    # trained in another subprocess
    trainer = Trainer(cfg, component_builder)
    persistor = create_persistor(storage)
    training_data = load_data(data, cfg.language)
    interpreter = trainer.train(training_data, **kwargs)

    if path:
        persisted_path = trainer.persist(path,
                                         persistor,
                                         project,
                                         fixed_model_name)
    else:
        persisted_path = None

    return trainer, interpreter, persisted_path
Esempio n. 2
0
def test_load_and_persist_without_train(component_builder):
    _config = utilities.base_test_conf("all_components")
    trainer = Trainer(_config, component_builder)
    persistor = create_persistor(_config)
    persisted_path = trainer.persist(_config['path'], persistor, model_name=_config['name'])
    loaded = utilities.load_interpreter_for_model(_config, persisted_path, component_builder)
    assert loaded.pipeline
    assert loaded.parse("hello", time=None) is not None
Esempio n. 3
0
    def train(cfg_name, project_name):
        from rasa_nlu import training_data

        cfg = config.load(cfg_name)
        trainer = Trainer(cfg, component_builder)
        training_data = training_data.load_data(data)

        trainer.train(training_data)
        trainer.persist("test_projects", project_name=project_name)
Esempio n. 4
0
def test_train_with_empty_data(component_builder):
    _config = utilities.base_test_conf("all_components")
    trainer = Trainer(_config, component_builder)
    trainer.train(TrainingData())
    persistor = create_persistor(_config)
    persisted_path = trainer.persist(_config['path'], persistor, model_name=_config['name'])
    loaded = utilities.load_interpreter_for_model(_config, persisted_path, component_builder)
    assert loaded.pipeline
    assert loaded.parse("hello") is not None
    assert loaded.parse("Hello today is Monday, again!") is not None
Esempio n. 5
0
def test_train_with_empty_data(language, pipeline, component_builder, tmpdir):
    _config = RasaNLUModelConfig({"pipeline": pipeline, "language": language})
    trainer = Trainer(_config, component_builder)
    trainer.train(TrainingData())
    persistor = create_persistor(_config)
    persisted_path = trainer.persist(tmpdir.strpath, persistor,
                                     project_name="my_project")
    loaded = Interpreter.load(persisted_path, component_builder)
    assert loaded.pipeline
    assert loaded.parse("hello") is not None
    assert loaded.parse("Hello today is Monday, again!") is not None
Esempio n. 6
0
def train_nlu():
    from rasa_nlu.converters import load_data
    from rasa_nlu.config import RasaNLUConfig
    from rasa_nlu.model import Trainer

    training_data = load_data("data/mobile_nlu_data.json")
    trainer = Trainer(RasaNLUConfig("mobile_nlu_model_config.json"))
    trainer.train(training_data)
    model_directory = trainer.persist("models/", project_name="ivr", fixed_model_name="demo")

    return model_directory
Esempio n. 7
0
def train_nlu():
    from rasa_nlu.training_data import load_data
    from rasa_nlu import config
    from rasa_nlu.model import Trainer

    training_data = load_data('data/nlu_data/')
    trainer = Trainer(config.load("nlu_model_config.yml"))
    trainer.train(training_data)
    model_directory = trainer.persist('models/nlu', fixed_model_name="current")

    return model_directory
Esempio n. 8
0
    def train(cfg_name, model_name):
        from rasa_nlu.train import create_persistor
        from rasa_nlu.converters import load_data

        config = RasaNLUConfig(cfg_name)
        trainer = Trainer(config)
        training_data = load_data(config['data'])

        trainer.train(training_data)
        persistor = create_persistor(config)
        trainer.persist("test_models", persistor, model_name=model_name)
Esempio n. 9
0
def do_train(config, component_builder=None):
    # type: (RasaNLUConfig, Optional[ComponentBuilder]) -> Tuple[Trainer, Interpreter, Text]
    """Loads the trainer and the data and runs the training of the specified model."""

    # Ensure we are training a model that we can save in the end
    # WARN: there is still a race condition if a model with the same name is trained in another subprocess
    trainer = Trainer(config, component_builder)
    persistor = create_persistor(config)
    training_data = load_data(config['data'])
    interpreter = trainer.train(training_data)
    persisted_path = trainer.persist(config['path'], persistor, model_name=config['name'])
    return trainer, interpreter, persisted_path
Esempio n. 10
0
def train_nlu_gao():
    from rasa_nlu_gao.training_data import load_data
    from rasa_nlu_gao import config
    from rasa_nlu_gao.model import Trainer

    training_data = load_data('data/rasa_dataset_training.json')
    trainer = Trainer(config.load("config_embedding_bilstm.yml"))
    trainer.train(training_data)
    model_directory = trainer.persist('models/nlu_gao/',
                                      fixed_model_name="current")

    return model_directory
Esempio n. 11
0
def run_cv_evaluation(data, n_folds, nlu_config):
    from sklearn import metrics
    from sklearn.model_selection import StratifiedKFold
    from collections import defaultdict
    # type: (List[rasa_nlu.training_data.Message], int, RasaNLUConfig) -> Dict[Text, List[float]]
    """Stratified cross validation on data

    :param data: list of rasa_nlu.training_data.Message objects
    :param n_folds: integer, number of cv folds
    :param nlu_config: nlu config file
    :return: dictionary with key, list structure, where each entry in list
              corresponds to the relevant result for one fold

    """
    trainer = Trainer(nlu_config)
    results = defaultdict(list)

    y_true = [e.get("intent") for e in data]

    skf = StratifiedKFold(n_splits=n_folds, random_state=11, shuffle=True)
    counter = 1
    logger.info("Evaluation started")
    for train_index, test_index in skf.split(data, y_true):

        train = [data[i] for i in train_index]
        test = [data[i] for i in test_index]

        logger.debug("Fold: {}".format(counter))
        logger.debug("Training ...")
        trainer.train(TrainingData(training_examples=train))
        model_directory = trainer.persist("projects/")  # Returns the directory the model is stored in

        logger.debug("Evaluation ...")
        interpreter = Interpreter.load(model_directory, nlu_config)
        test_y = [e.get("intent") for e in test]

        preds = []
        for e in test:
            res = interpreter.parse(e.text)
            if res.get('intent'):
                preds.append(res['intent'].get('name'))
            else:
                preds.append(None)

        # compute fold metrics
        results["Accuracy"].append(metrics.accuracy_score(test_y, preds))
        results["F1-score"].append(metrics.f1_score(test_y, preds, average='weighted'))
        results["Precision"] = metrics.precision_score(test_y, preds, average='weighted')

        # increase fold counter
        counter += 1

    return dict(results)
Esempio n. 12
0
def train_models(languages):
    """Generate your trained model."""
    utils.check_languages(languages)
    config = utils.load_config()
    for language in languages:
        click.echo(_("================== Processing {lang} ==================").format(lang=language))
        training_data = load_data(utils.get_training_data_path(language, config))
        trainer = Trainer(RasaNLUConfig(cmdline_args=config))
        click.echo(_("Training data for language {lang}.").format(lang=language))
        trainer.train(training_data)
        click.echo(_("Persisting trained data for {lang}.").format(lang=language))
        model_dir = trainer.persist(utils.get_model_base_dir(language))
        click.echo(_("Stored data for {lang} in {path}.").format(lang=language, path=model_dir))
    click.echo(_("================ Finished Training ================"))
Esempio n. 13
0
def cross_validate(data: TrainingData, n_folds: int,
                   nlu_config: Union[RasaNLUModelConfig, Text]
                   ) -> CVEvaluationResult:
    """Stratified cross validation on data.

    Args:
        data: Training Data
        n_folds: integer, number of cv folds
        nlu_config: nlu config file

    Returns:
        dictionary with key, list structure, where each entry in list
              corresponds to the relevant result for one fold
    """
    from collections import defaultdict
    import tempfile

    if isinstance(nlu_config, str):
        nlu_config = config.load(nlu_config)

    trainer = Trainer(nlu_config)
    train_results = defaultdict(list)
    test_results = defaultdict(list)
    entity_train_results = defaultdict(lambda: defaultdict(list))
    entity_test_results = defaultdict(lambda: defaultdict(list))
    tmp_dir = tempfile.mkdtemp()

    for train, test in generate_folds(n_folds, data):
        interpreter = trainer.train(train)

        # calculate train accuracy
        train_results = combine_intent_result(train_results, interpreter,
                                              train)
        test_results = combine_intent_result(test_results, interpreter, test)
        # calculate test accuracy
        entity_train_results = combine_entity_result(entity_train_results,
                                                     interpreter, train)
        entity_test_results = combine_entity_result(entity_test_results,
                                                    interpreter, test)

    shutil.rmtree(tmp_dir, ignore_errors=True)

    return (CVEvaluationResult(dict(train_results), dict(test_results)),
            CVEvaluationResult(dict(entity_train_results),
                               dict(entity_test_results)))
Esempio n. 14
0
def zipped_nlu_model():
    spacy_config_path = "sample_configs/config_pretrained_embeddings_spacy.yml"

    cfg = config.load(spacy_config_path)
    trainer = Trainer(cfg)
    td = training_data.load_data(DEFAULT_DATA_PATH)

    trainer.train(td)
    trainer.persist("test_models",
                    project_name="test_model_pretrained_embeddings")

    model_dir_list = os.listdir(TEST_MODEL_PATH)

    # directory name of latest model
    model_dir = sorted(model_dir_list)[-1]

    # path of that directory
    model_path = os.path.join(TEST_MODEL_PATH, model_dir)

    zip_path = zip_folder(model_path)

    return zip_path
Esempio n. 15
0
def run_cv_evaluation(data, n_folds, nlu_config):
    # type: (TrainingData, int, RasaNLUModelConfig) -> CVEvaluationResult
    """Stratified cross validation on data

    :param data: Training Data
    :param n_folds: integer, number of cv folds
    :param nlu_config: nlu config file
    :return: dictionary with key, list structure, where each entry in list
              corresponds to the relevant result for one fold
    """
    from collections import defaultdict
    import tempfile

    trainer = Trainer(nlu_config)
    train_results = defaultdict(list)
    test_results = defaultdict(list)
    entity_train_results = defaultdict(lambda: defaultdict(list))
    entity_test_results = defaultdict(lambda: defaultdict(list))
    tmp_dir = tempfile.mkdtemp()

    for train, test in generate_folds(n_folds, data):
        interpreter = trainer.train(train)

        # calculate train accuracy
        train_results = combine_intent_result(train_results, interpreter,
                                              train)
        test_results = combine_intent_result(test_results, interpreter, test)
        # calculate test accuracy
        entity_train_results = combine_entity_result(entity_train_results,
                                                     interpreter, train)
        entity_test_results = combine_entity_result(entity_test_results,
                                                    interpreter, test)

    shutil.rmtree(tmp_dir, ignore_errors=True)

    return (CVEvaluationResult(dict(train_results), dict(test_results)),
            CVEvaluationResult(dict(entity_train_results),
                               dict(entity_test_results)))
Esempio n. 16
0
    def create_and_train(cls, project_name, training_data):
        """
        Train a model

        The training data is an object with this structure:
        name: <Optional name of the model>
        description: <Optional description of the model>
        tags: <Optional list of tags for the model>
        config:
            language: <Language supported by the model>
            pipeline:
                An array of items with this structure:
                    - name: <Name of the pipeline step>
                    - <arg>: <Optional step-dependent argument>
        data:
            A json-formatted array of training data
        """
        # Try to create the model
        model_name = training_data.get('name', "model_{:%Y%m%d-%H%M%S}".format(timestamp))
        training_data['name'] = model_name
        # This step may fail with a ModelExistsError or a ProjectNotFoundError
        MLModel.create(project_name, training_data)
        model = DATA[project_name][model_name]
        # Load and validate the configuration and training data
        conf = config.load(**json.loads(training_data['config']))
        data = RasaReader().read_from_json(json.loads(training_data['data']))
        # Initialize a trainer and run training
        trainer = Trainer(conf)
        trainer.train(td)
        # Save the results
        model_directory = trainer.persist(
            os.path.join(THIS_DIR, 'projects'),
            project_name=project_name,
            fixed_model_name=model_name)
        # Save model properties
        model['model_directory'] = model_directory
        model['config'] = training_data['config']
        return model
Esempio n. 17
0
def train_nlu(data, config, model_dir):
    """Trains the model to extract intents and entities.

    Parameters
    ----------
    data : json file
        Training data in data.json
    config : .config file
        configuration file containing the pipeline used for the model.
        We specify the use of space_sklearn and ner_crf.
    model_dir : path
        Path to the directory where the model is saved after training

    Returns
    -------
    Nothing

    """
    training_data = load_data(data)
    trainer = Trainer(RasaNLUConfig(config))
    trainer.train(training_data)
    model_directory = trainer.persist(model_dir,
                                      fixed_model_name='winereviewnlu')
Esempio n. 18
0
class NLU:
    def __init__(self):

        self.interpreter = None
        self.trainer = Trainer(config)

    def __call__(self):
        pass

    def train(self, db_name, uri=None):

        training_data = load_db_data(db_name, uri)
        self.interpreter = self.trainer.train(training_data)

    def classify(self, sent, user):

        if self.interpreter is None:
            print("train if you got new.. using old one ")
            try:
                model_directory = self.trainer.persist('./models/')
                metadata = Metadata.load(model_directory)
                self.interpreter = Interpreter.load(metadata, config)
            except:
                raise ("Need to train model. No pre-trained model found.")

        result = self.interpreter.parse(sent)

        probable_module = result["intent"]['name'].split("-")

        is_calc = len(probable_module) - 1
        print(is_calc)

        if is_calc:
            print('here')
            return eval(probable_module[0])(sent, user)

        return best_match(sent, probable_module[0])
Esempio n. 19
0
def run_cv_evaluation(data, n_folds, nlu_config):
    # type: (TrainingData, int, RasaNLUModelConfig) -> CVEvaluationResult
    """Stratified cross validation on data

    :param data: Training Data
    :param n_folds: integer, number of cv folds
    :param nlu_config: nlu config file
    :return: dictionary with key, list structure, where each entry in list
              corresponds to the relevant result for one fold
    """
    from collections import defaultdict
    import tempfile

    trainer = Trainer(nlu_config)
    train_results = defaultdict(list)
    test_results = defaultdict(list)
    entity_train_results = defaultdict(lambda: defaultdict(list))
    entity_test_results = defaultdict(lambda: defaultdict(list))
    tmp_dir = tempfile.mkdtemp()

    for train, test in generate_folds(n_folds, data):
        interpreter = trainer.train(train)

        # calculate train accuracy
        train_results = combine_intent_result(train_results, interpreter, train)
        test_results = combine_intent_result(test_results, interpreter, test)
        # calculate test accuracy
        entity_train_results = combine_entity_result(entity_train_results,
                                                     interpreter, train)
        entity_test_results = combine_entity_result(entity_test_results,
                                                    interpreter, test)

    shutil.rmtree(tmp_dir, ignore_errors=True)

    return (CVEvaluationResult(dict(train_results), dict(test_results)),
            CVEvaluationResult(dict(entity_train_results),
                               dict(entity_test_results)))
Esempio n. 20
0
    def train(cfg_name, project_name):
        from rasa_nlu import training_data

        cfg = config.load(cfg_name)
        trainer = Trainer(cfg, component_builder)
        training_data = training_data.load_data(data)

        trainer.train(training_data)
        trainer.persist("test_projects", project_name=project_name)
Esempio n. 21
0
def train_nlu(nlu_training_data):
    from rasa_nlu.config import RasaNLUConfig
    from rasa_nlu.converters import load_data
    from rasa_nlu.model import Trainer

    training_data = load_data(nlu_training_data_path + nlu_training_data)
    trainer = Trainer(RasaNLUConfig('nlu_model_config.json'))
    trainer.train(training_data)
    trainer.persist('models/nlu/', fixed_model_name='current')
Esempio n. 22
0
def train(nlu_config: Union[Text, RasaNLUModelConfig],
          data: Text,
          path: Optional[Text] = None,
          project: Optional[Text] = None,
          fixed_model_name: Optional[Text] = None,
          storage: Optional[Text] = None,
          component_builder: Optional[ComponentBuilder] = None,
          training_data_endpoint: Optional[EndpointConfig] = None,
          **kwargs: Any
          ) -> Tuple[Trainer, Interpreter, Text]:
    """Loads the trainer and the data and runs the training of the model."""

    if isinstance(nlu_config, str):
        nlu_config = config.load(nlu_config)

    # Ensure we are training a model that we can save in the end
    # WARN: there is still a race condition if a model with the same name is
    # trained in another subprocess
    trainer = Trainer(nlu_config, component_builder)
    persistor = create_persistor(storage)
    if training_data_endpoint is not None:
        training_data = load_data_from_endpoint(training_data_endpoint,
                                                nlu_config.language)
    else:
        training_data = load_data(data, nlu_config.language)
    interpreter = trainer.train(training_data, **kwargs)

    if path:
        persisted_path = trainer.persist(path,
                                         persistor,
                                         project,
                                         fixed_model_name)
    else:
        persisted_path = None

    return trainer, interpreter, persisted_path
Esempio n. 23
0
def train_nlu():
    from rasa_nlu import utils, config
    from rasa_nlu.training_data.loading import load_data
    from rasa_nlu.model import Trainer
    from rasa_nlu.config import RasaNLUModelConfig
    nlu_config = config.load(os.path.join(current_dir, "sample/nlu_config.yml"))
    nlu_trainer = Trainer(nlu_config)
    nlu_training_data = load_data(os.path.join(current_dir, "sample/nlu"))
    nlu_trainer.train(nlu_training_data)
    nlu_trainer.persist(os.path.join(current_dir, "sample/models/current/nlu"), fixed_model_name="default")
Esempio n. 24
0
def test_train():
    print(datetime.datetime.now())
    # data_set generated by Chatito
    training_data = load_data(data_set)
    # load rasa pipeline
    trainer = Trainer(config.load("sample_configs/config_spacy.yml"))
    trainer.train(training_data)
    # save model
    trainer.persist('./projects/')
    print(datetime.datetime.now())
Esempio n. 25
0
def train_nlu():
    LOGGER.info("[NLU] Loading training data from %s", NLU_DATA)
    training_data = load_data(NLU_DATA)

    LOGGER.info("[NLU] Loading config from %s", NLU_CONFIG)
    trainer = Trainer(config.load(NLU_CONFIG))

    LOGGER.info("[NLU] Training NLU...")
    trainer.train(training_data)
    trainer.persist(NLU_MODEL, fixed_model_name="nlu", project_name="current")
Esempio n. 26
0
def do_train(config):
    # type: (RasaNLUConfig) -> (Trainer, str)
    """Loads the trainer and the data and runs the training of the specified model."""

    trainer = Trainer(config)
    persistor = create_persistor(config)
    training_data = load_data(
        config['data'],
        config['language'],
        luis_data_tokenizer=config['luis_data_tokenizer'])
    trainer.validate()
    trainer.train(training_data)
    persisted_path = trainer.persist(config['path'], persistor)
    return trainer, persisted_path
Esempio n. 27
0
def train_nlu(data_file, config_file, model_dir):
    #assign nlu
    training_data = load_data(data_file)
    # Create a config that uses this pipeline
    configuration = config.load(config_file)
    # Create a trainer that uses this config
    trainer = Trainer(configuration)
    # Create an interpreter by training the model
    trainer.train(training_data)
    #create model folder and store nlu
    trainer.persist(model_dir, fixed_model_name="nlu")
Esempio n. 28
0
    def train(cfg_name, model_name):
        from rasa_nlu.train import create_persistor
        from rasa_nlu.converters import load_data

        config = RasaNLUConfig(cfg_name)
        trainer = Trainer(config)
        training_data = load_data(config['data'])

        trainer.train(training_data)
        persistor = create_persistor(config)
        trainer.persist("test_models", persistor, model_name=model_name)
Esempio n. 29
0
def train(data, config_file, model_dir):
    """
    Function to train the bot
    """
    training_data = load_data(data)
    configuration = config.load(config_file)
    trainer = Trainer(configuration)
    trainer.train(training_data)
    trainer.persist(model_dir, fixed_model_name=const.MODEL_NAME)
    print("Training complete.")
    return
Esempio n. 30
0
    def train(cfg_name, project_name):
        from rasa_nlu.train import create_persistor
        from rasa_nlu import training_data

        config = RasaNLUConfig(cfg_name)
        trainer = Trainer(config, component_builder)
        training_data = training_data.load_data(config['data'])

        trainer.train(training_data)
        persistor = create_persistor(config)
        trainer.persist("test_projects", persistor, project_name)
Esempio n. 31
0
def train_nlu(data, nlu_config, directory):
  training_data = load_data(data)

  # Creates an instance of the Trainer using the nlu_config
  trainer = Trainer(config.load(nlu_config))

  # Starts training
  trainer.train(training_data)

  # Tells the training data where to go
  trainer.persist(directory, fixed_model_name = 'nlu')
Esempio n. 32
0
def train_model(dataset_directory, model_name,
                model_name_directory):  #need to told model name directory
    train_data = load_data(
        dataset_directory)  #load data file and assign in train_data
    trainer = Trainer(config.load("config_spacy.yaml"))  #load config file
    trainer.train(train_data)  # Training Data
    trainer.persist(model_name_directory, fixed_model_name=model_name)
    # Returns the directory the model is stored in (Creat a folder to store model in) and you can fixed the model name
    # metadata = read_json_file('metadata.json')
    # metadata['model_name'] = model_name
    # save_json_file(metadata,'metadata.json')
    return "function train_model success"
Esempio n. 33
0
def train_nlu(data, nlu_config, directory):
  # Loads the intent training data
  training_data = load_data(data)

  # Creates an instance of the Trainer using the nlu_config
  trainer = Trainer(config.load(nlu_config))

  # Starts the training
  trainer.train(training_data)

  # Saves the trained model in the specified directory with the specified name
  trainer.persist(directory, fixed_model_name = 'nlu')
Esempio n. 34
0
def test_train():
    print(datetime.datetime.now())
    # data_set generated by Chatito
    training_data = load_data(data_set)
    # load rasa pipeline
    trainer = Trainer(
        config.load(
            "/Users/guolei/Documents/EIT/GUOLEI/ContextManager/sample_configs/config_spacy.yml"
        ))
    trainer.train(training_data)
    # save model
    trainer.persist('./projects/')
    print(datetime.datetime.now())
Esempio n. 35
0
    def train(cfg_name, model_name):
        from rasa_nlu.train import create_persistor
        from rasa_nlu.converters import load_data

        config = RasaNLUConfig(cfg_name)
        trainer = Trainer(config)
        training_data = load_data(config['data'])

        trainer.train(training_data)
        persistor = create_persistor(config)
        trainer.persist(os.path.join("test_models", model_name),
                        persistor,
                        create_unique_subfolder=False)
Esempio n. 36
0
def construct_persistent_model_from_training_data(training_file,
                                                  configuration_file,
                                                  output_directory):
    with open(file, 'r', encoding='utf8') as f:
        content = f.readlines()
    train = data_Extraction.build_full_train_data_from_lines(content)
    #print(train)
    training_data = data_Extraction.load_rasa_data_from_string(train)

    trainer = Trainer(RasaNLUConfig(configuration_file))
    trainer.train(training_data)
    #print(model.parse(',,,D,Theater am Neumarkt,Die Kleinbürgerhochzeit,,,,,STS Leiter Dokumentation,04/09/2017,,,,0,Ambrosius Humm,Michel Seigner,,,"Henning Heers, Iris Erdmann, Hildegard Pintgen, Verena Reichhardt, Urs Bihler, Klaus Henner Russius, Nikola Weisse, Bernd Spitzer, Daniel Plancherel",,,,Christian Schneeberger,01/04/2003,0,S,Isolde Hahn,,,,,1,1,,6,,,,15/05/1976,1,CH: Zürich: Theater am Neumarkt,,Philippe Pilliod,,80003197605151,Zürich: Theater am Neumarkt Theater am Neumarkt,1975/76,,Bertolt Brecht,80003,,,,'.replace(',',' ')))
    trainer.persist(output_directory)
Esempio n. 37
0
class RasaNlp(Nlp):
    def __init__(self, file_name, lang='en'):
        from rasa_nlu.training_data import load_data
        from rasa_nlu.config import RasaNLUConfig
        from rasa_nlu.model import Trainer

        self.data = load_data(file_name)
        self._trainer = Trainer(
            RasaNLUConfig(
                cmdline_args=dict(pipeline="spacy_sklearn", language=lang)))
        self._interpreter = self._trainer.train(self.data)

    def predict(self, text):
        r = self._interpreter.parse(text)
        r['intents'] = r.pop('intent_ranking')
        for intent in r['intents']:
            intent['intent'] = intent.pop('name')

        return r
Esempio n. 38
0
def train_nlu():
    print("=> Importing rasa_nlu...")
    from rasa_nlu.training_data import load_data
    from rasa_nlu import config
    from rasa_nlu.model import Trainer
    shutil.rmtree(botpath.NLU_MODEL_PATH, ignore_errors=True)

    print("=> Training NLU...%s - %s" %
          (botpath.NLU_DATA_FILE, botpath.CONFIG_FILE))
    training_data = load_data(botpath.NLU_DATA_FILE)
    train_config = config.load(botpath.CONFIG_FILE)
    trainer = Trainer(train_config, skip_validation=True)
    trainer.train(training_data)

    print("=> Saving Result...%s" % botpath.NLU_MODEL_PATH)
    trainer.persist(botpath.NLU_MODEL_PATH, fixed_model_name=botpath.PROJECT)
Esempio n. 39
0
def zipped_nlu_model():
    spacy_config_path = "sample_configs/config_spacy.yml"

    cfg = config.load(spacy_config_path)
    trainer = Trainer(cfg)
    td = training_data.load_data(DEFAULT_DATA_PATH)

    trainer.train(td)
    trainer.persist("test_models", project_name="test_model_spacy_sklearn")

    model_dir_list = os.listdir(TEST_MODEL_PATH)

    # directory name of latest model
    model_dir = sorted(model_dir_list)[-1]

    # path of that directory
    model_path = os.path.join(TEST_MODEL_PATH, model_dir)

    zip_path = zip_folder(model_path)

    return zip_path
Esempio n. 40
0
def train_update(update, by):
    update.start_training(by)

    examples = [
        Message.build(
            text=example.get_text(update.language),
            intent=example.intent,
            entities=[
                example_entity.rasa_nlu_data
                for example_entity in example.get_entities(update.language)
            ]) for example in update.examples
    ]

    label_examples_query = update.examples \
        .filter(entities__entity__label__isnull=False) \
        .annotate(entities_count=models.Count('entities')) \
        .filter(entities_count__gt=0)

    label_examples = [
        Message.build(
            text=example.get_text(update.language),
            entities=[
                example_entity.get_rasa_nlu_data(label_as_entity=True)
                for example_entity in filter(
                    lambda ee: ee.entity.label,
                    example.get_entities(update.language))
            ]) for example in label_examples_query
    ]

    rasa_nlu_config = get_rasa_nlu_config_from_update(update)
    trainer = Trainer(rasa_nlu_config, ComponentBuilder(use_cache=False))
    training_data = BothubTrainingData(label_training_examples=label_examples,
                                       training_examples=examples)
    trainer.train(training_data)
    persistor = BothubPersistor(update)
    trainer.persist(mkdtemp(),
                    persistor=persistor,
                    project_name=str(update.repository.uuid),
                    fixed_model_name=str(update.id))
Esempio n. 41
0
    def __init__(self, model_path="config_spacy.yml", data='train.md'):
        # Create a trainer that uses this config
        trainer = Trainer(config.load("config_spacy.yml"))
        # Load the training data
        training_data = load_data('train.md')
        # Create an interpreter by training the model
        self.interpreter = trainer.train(training_data)
        trainer = Trainer(config.load("config_spacy.yml"))
        self.tieba_interpreter = trainer.train(load_data("tieba_train.md"))
        self.tieba = Tieba()
        self.respond_dict = {
            "TIEBA": self.respond_tieba,
            "default": self.respond_default,
            "get_posts": self.tieba.get_posts,
            "turn_to_post": self.tieba.turn_to_post,
            "LAUNCH": self.launch,
            "QUERY": self.query,
            "ROUTE": self.route
        }

        self.state = State.FREE
        self.message_trace = []
Esempio n. 42
0
def train(data='./intents',
          config_file='config.yaml',
          model_dir='models/',
          project='default'):
    """ Trains the nlu on given config and data

    Parameters:
    ----------
    data: str
          path to training data
    config_file: str
                 path to config file
    model_dir: str
               path ot output model directory
    project: str
             name of the project, needed for model generation

    """
    training_data = load_data(data)
    trainer = Trainer(config.load(config_file))
    trainer.train(training_data)
    trainer.persist(model_dir, project_name=project)
Esempio n. 43
0
def trainbot(projid):
    db = getMongoDBConnection('pkasy')
    t_data = {
        'rasa_nlu_data': {
            'regex_features': [],
            'entity_synonyms': [],
            'common_examples': []
        }
    }
    collection = db['training_data_' + projid]
    for intent in collection.find():
        for text in intent['text']:
            # data = t_data['rasa_nlu_data']
            t_data['rasa_nlu_data']['common_examples'].append({
                'intent':
                intent['intent'],
                'text':
                text['value'],
                'entities':
                text['entities']
            })

    pprint.pprint(t_data)
    #f= open("F://chatbots/"+projid+"/training_data.json","w+")
    f = open("./chatbots/" + projid + "/training_data.json", "w+")
    f.write(json.dumps(t_data))
    f.close()

    #t1_data = load_data("F://chatbots/"+projid+"/training_data.json")
    t1_data = load_data("./chatbots/" + projid + "/training_data.json")
    trainer = Trainer(config.load("./chatbots/config.yaml"))
    # trainer = Trainer(RasaNLUConfig("F://chatbots/"+projid+"/config.yaml"))
    trainer.train(t1_data)
    #trainer.persist('F://chatbots/'+projid+'/models/nlu/', fixed_model_name="current")
    trainer.persist('./chatbots/' + projid + '/models/nlu/',
                    fixed_model_name="current")
    return jsonify({'status': 'sucess'})
Esempio n. 44
0
    def trainning(self, force=False):
        '''
        Creates a new trainning for the bot. This method only makes a new training if the .trainning_hash file does not
        exist, or, if the data_file hash is changed.

        When force is True we force a new trainning. Even though there is already one for the current data file.
        '''
        model_dir = Path(self.model_dir)
        data_file = Path(self.data_file)
        trainning_hash_file = model_dir.child('.trainning_hash')
        new_trainning = True

        # The data_file has been modified?
        if trainning_hash_file.exists() and not force:
            new_trainning_hash = calc_hash(data_file)
            current_trainning_hash = trainning_hash_file.read_file()
            new_trainning = new_trainning_hash != current_trainning_hash

        if new_trainning or force:
            training_data = load_data(data_file)
            trainer = Trainer(RasaNLUConfig(self.config_file))
            trainer.train(training_data)
            trainer.persist(model_dir)  # Returns the directory the model is stored in
            trainning_hash_file.write_file(calc_hash(data_file))
def train_nlu(data, config_file, model_dir):
    training_data = load_data(data)
    trainer = Trainer(config.load(config_file), builder)
    trainer.train(training_data)
    model_directory = trainer.persist(model_dir,
                                      fixed_model_name='restaurantnlu')
Esempio n. 46
0
def train_nlu():
    training_data = load_data(NLU_DATA)
    trainer = Trainer(RasaNLUConfig(CONFIG_PATH))
    trainer.train(training_data)
    model_directory = trainer.persist('../models/nlu', fixed_model_name="current")