def test_filter_messages():
filename = get_project_root() / 'data' / 'askubuntu' / 'askubuntu.tsv'
train = get_filtered_messages(filename, training=True)
assert 53 == len(train)
test = get_filtered_messages(filename, training=False)
assert 109 == len(test)
assert 'Software Recommendation' == test[0].data['intent']
def test_train_evaluate():
reduce_output()
validate_debug_params()
hparams = get_debug_hparams()
hparams = hparams._replace(output_dir=str(get_project_root() / 'tmp' /
'test_my_classifier_train_eval'))
_, estimator = get_model_fn_and_estimator(hparams)
train_evaluate(hparams, estimator)
def test_main():
reduce_output()
model_dir = get_project_root() / 'tmp' / 'custom_estimator'
if model_dir.is_dir():
clean_folder(model_dir)
args = [None, '--train_steps=10', '--model_dir={}'.format(model_dir)]
tf.logging.set_verbosity(tf.logging.WARN)
main(args)
def test_clean_folder():
folder = get_project_root() / 'tmp' / 'test_clean_folder'
if folder.is_dir():
shutil.rmtree(str(folder))
else:
tf.gfile.MakeDirs(str(folder))
tf.gfile.MakeDirs(str(folder / 'some_dir'))
clean_folder(folder)
sub_files = os.listdir(str(folder))
assert 0 == len(list(sub_files))
def test_train():
reduce_output()
validate_debug_params()
hparams = get_debug_hparams()
hparams = hparams._replace(output_dir=str(get_project_root() / 'tmp' /
'test_my_classifier_train'))
_, estimator = get_model_fn_and_estimator(hparams)
train(hparams, estimator, max_steps=2)
clean_folder(hparams.output_dir)
def classify_sentence():
rc.FLAGS.task_name = TASK
rc.FLAGS.do_train = True
rc.FLAGS.do_eval = True
rc.FLAGS.data_dir = str(get_project_root() / 'data' / TASK)
rc.FLAGS.vocab_file = str(BERT_BASE_DIR / 'vocab.txt')
rc.FLAGS.bert_config_file = str(BERT_BASE_DIR / 'bert_config.json')
rc.FLAGS.init_checkpoint = str(BERT_BASE_DIR / 'bert_model.ckpt')
rc.FLAGS.max_seq_length = 128
rc.FLAGS.train_batch_size = 16 # using too much memory at 32 batches
rc.FLAGS.learning_rate = 2e-5
rc.FLAGS.num_train_epochs = 1
rc.FLAGS.output_dir = str(get_project_root() / 'output' / '1_epochs')
start_time = time.time()
rc.main('')
running_time = round(((time.time() - start_time) / 60), 2)
print('Execution took {} minutes'.format(running_time))
def get_debug_hparams() -> HParams:
"""Parameters for lightweight BERT execution for debug purposes."""
from improv.utils import get_project_root
task_name = '_chatbot'
bert_model = 'uncased_L-12_H-768_A-12'
bert_pretrained_dir = Path.home() / 'Downloads' / bert_model
output_dir_name = 'debug'
return HParams(
data_dir=get_project_root() / 'data' / task_name,
bert_config_file=bert_pretrained_dir / 'bert_config.json',
task='ner_intent',
task_name=task_name,
vocab_file=bert_pretrained_dir / 'vocab.txt',
output_dir=str(get_project_root() / 'tmp' / task_name /
output_dir_name),
local_dir=str(get_project_root() / 'tmp' / task_name /
output_dir_name),
init_checkpoint=bert_pretrained_dir / 'bert_model.ckpt',
do_lower_case=bert_model.startswith('uncased'),
max_seq_length=128,
do_train_eval=False,
do_train=True,
do_eval=True,
do_predict=True,
train_batch_size=1,
eval_batch_size=8,
predict_batch_size=8,
learning_rate=2e-5,
num_train_steps=2,
warmup_proportion=0.1,
save_checkpoints_steps=1,
save_summary_steps=1,
iterations_per_loop=1,
use_tpu=False,
tpu_name='', # is used as tpu_address in Colab script
tpu_zone=None,
gcp_project=None,
master=None,
num_tpu_cores=8)
from improv.utils import get_project_root
from improv.read_ner import get_ner_lines, get_unique_labels, get_interesting_labels_indexes
data_dir = get_project_root() / 'data' / 'chatbot'
def test_get_ner_lines():
lines = get_ner_lines(data_dir / 'test.txt')
assert len(lines) == 105
assert lines[0][0] == 'C O O O O B-StationDest'
assert lines[0][1] == 'FindConnection i want to go marienplatz'
def test_get_unique_labels():
labels = get_unique_labels(data_dir)
# order might change for each execution
expected = ['Criterion', 'Line', 'StationDest', 'StationStart', 'Vehicle']
for label in expected:
# it might be that some I-<label> is missing, this could be caused by (small) dataset.
full_label = 'B-{}'.format(label)
assert full_label in labels
assert '[CLS]' in labels
assert '[SEP]' in labels
assert 'X' in labels
assert len(
labels) > 16 # changed by the intent examples, two intents occur
assert 'DepartureTime' in labels # this is a sentence intent example
def test_get_interesting_labels_indexes():
def test_find_tf_events():
folder = get_project_root() / 'tests'
assert 'events.out.tfevents.000000' == find_tf_events(folder).name
def test_get_y_true():
file = get_project_root() / 'data' / 'askubuntu' / 'askubuntu.tsv'
y_true = get_y_true(file, training=False)
assert 'Software Recommendation' == y_true[0]
assert 109 == len(y_true)
from improv.utils import get_project_root
from improv.evaluate import parse_file, print_scores
# this filename should be updated when output format changes (which possibly will happen)
filename = get_project_root() / 'runs' / '2018-12-20 chatbot' / 'results.txt'
def test_parse_file():
ner_datas = parse_file(filename)
for ner_data in ner_datas:
print(ner_data)
def test_evaluate():
print_scores(filename)
def get_debug_filename() -> Path:
return get_project_root() / get_filename(Corpus.ASKUBUNTU)
def get_filename(corpus: Corpus) -> Path:
"""Returns filename for some Corpus. This avoids re-defining corpus location all over the place."""
from improv.utils import get_project_root
task = corpus.name.lower()
return get_project_root() / 'data' / task / (task + '.tsv')
return round(f1_score(y_true, y_pred, average='weighted'), 3)
def is_ner_not_empty(ner_datas: Tuple[NERData]) -> bool:
y_true = map(lambda ner_data: ner_data.true[1:], ner_datas)
y_true = chain(*y_true)
def is_not_empty(token: str) -> bool:
return token != 'O'
return any(map(is_not_empty, y_true))
def print_scores(filename: Path):
"""Determine intent and NER accuracy (weighted f1 score)."""
ner_datas = tuple(parse_file(filename))
if ner_datas[0].text[0] == 'INTENT':
y_true, y_pred = get_intents(ner_datas)
print('intents weighted f1: {}'.format(rounded_f1(y_true, y_pred)))
if is_ner_not_empty(ner_datas):
y_true, y_pred = get_entities(ner_datas)
print('entities weighted f1: {}'.format(rounded_f1(y_true, y_pred)))
if __name__ == '__main__':
fn = get_project_root(
) / 'runs' / '2018-12-20 chatbot' / '2018-12-20 chatbot joint.txt'
print_scores(fn)