def test_markdown_empty_section():
data = load_data("data/test/markdown_single_sections/empty_section.md")
assert data.regex_features == [{"name": "greet", "pattern": r"hey[^\s]*"}]
assert not data.entity_synonyms
assert len(data.lookup_tables) == 1
assert data.lookup_tables[0]["name"] == "chinese"
assert "Chinese" in data.lookup_tables[0]["elements"]
assert "Chines" in data.lookup_tables[0]["elements"]
def edit():
if request.method == 'POST':
#Create new file
if 'create' in request.form:
new = request.form['new_name'] + '.md'
return redirect(url_for('content', name=new, content=""))
#Open existing file
elif 'open' in request.form:
#Save at local directory
curr = request.files['file']
curr.save(os.path.join(app.config['UPLOAD_FOLDER'],secure_filename(curr.filename)))
return redirect(url_for('content', name=secure_filename(curr.filename)))
#Convert file type (md <-> json)
elif 'convert' in request.form:
#Save at local directory
curr = request.files['convert_file']
curr.save(os.path.join(app.config['UPLOAD_FOLDER'],secure_filename(curr.filename)))
return redirect(url_for('convert', name=secure_filename(curr.filename)))
#Merge training data
elif 'merge' in request.form:
from rasa.nlu import training_data, load_data
#Save at local directory
curr_1 = request.files['merge_file_1']
curr_1.save(os.path.join(app.config['UPLOAD_FOLDER'],secure_filename(curr_1.filename)))
curr_2 = request.files['merge_file_2']
curr_2.save(os.path.join(app.config['UPLOAD_FOLDER'],secure_filename(curr_2.filename)))
file1 = os.path.join(app.config['UPLOAD_FOLDER'],secure_filename(curr_1.filename))
#app.config['UPLOAD_FOLDER'] + secure_filename(curr_1.filename)
file2 = os.path.join(app.config['UPLOAD_FOLDER'],secure_filename(curr_2.filename))
#app.config['UPLOAD_FOLDER'] + secure_filename(curr_2.filename)
#Perform Merge
td = training_data.load_data(file1)
td = td.merge(load_data(file2))
content = td.nlu_as_markdown() # always save as markdown
new_name = request.form['merge_name']
return redirect(url_for('content', name=new_name, content=content, merge=True))
#Train-test split
else:
from rasa.nlu import training_data, load_data
curr = request.files['train_test_split']
curr.save(os.path.join(app.config['UPLOAD_FOLDER'],secure_filename(curr.filename)))
fraction = int(request.form['rangeSlider'])/100
file = os.path.join(app.config['UPLOAD_FOLDER'],secure_filename(curr.filename))
#app.config['UPLOAD_FOLDER'] + secure_filename(curr.filename)
td = training_data.load_data(file)
#Peform Train Test Split
data_train, data_test = td.train_test_split(train_frac=fraction)
train_name = os.path.splitext(os.path.basename(curr.filename))[0] + "_train"
test_name = os.path.splitext(os.path.basename(curr.filename))[0] + "_test"
data_train = data_train.nlu_as_markdown()
data_test = data_test.nlu_as_markdown()
return redirect(url_for('split', train_name=train_name, test_name=test_name, data_train=data_train, data_test=data_test))
else:
return render_template('/edit/index.html')
def test_validate_required_components_from_data(
config_path: Text, data_path: Text,
expected_warning_excerpts: List[Text]):
loaded_config = config.load(config_path)
trainer = Trainer(loaded_config)
training_data = load_data(data_path)
with pytest.warns(UserWarning) as record:
components.validate_required_components_from_data(
trainer.pipeline, training_data)
assert len(record) == 1
assert all([excerpt in record[0].message.args[0]]
for excerpt in expected_warning_excerpts)
def __init__(self, languages=None):
if languages is None:
languages = ['en', 'de']
pipeline = [{"name": "WhitespaceTokenizer"},
{"name": "CRFEntityExtractor"},
{"name": "EntitySynonymMapper"},
{"name": "CountVectorsFeaturizer"},
{"name": "EmbeddingIntentClassifier"}]
self.interpreters = {}
for lang in languages:
filepath = resource_filename(__name__, f'nlu_{lang}.md')
training_data = load_data(filepath)
trainer = Trainer(RasaNLUModelConfig({"pipeline": pipeline}))
self.interpreters[lang] = trainer.train(training_data)
def test_section_value_with_delimiter():
td_section_with_delimiter = load_data(
"data/test/markdown_single_sections/section_with_delimiter.md"
)
assert td_section_with_delimiter.entity_synonyms == {"10:00 am": "10:00"}
def test_markdown_not_existing_section():
with pytest.raises(ValueError):
load_data("data/test/markdown_single_sections/not_existing_section.md")
