def train_intent_model(int_ds, random_selection: bool, limit_num_sents: bool, num_samples=None): if random_selection: selection = get_intents_selection(int_ds['train'], num_samples) # selected intent labels: (num_samples, ) np.ndarray filt_train = get_filtered_lst(int_ds['train'], selection) # almost the same as int_ds['train'] but filtered according to selection filt_test = get_filtered_lst(int_ds['test'], selection) mod_int_ds = copy.deepcopy(int_ds) # deepcopy in order to not modify the original dict mod_int_ds['train'] = filt_train mod_int_ds['test'] = filt_test dataset = mod_int_ds else: dataset = int_ds split = Split() X_int_train, y_int_train = split.get_X_y(dataset['train'], fit=True, limit_num_sents=limit_num_sents, set_type='train') # fit only on first dataset X_int_test, y_int_test = split.get_X_y(dataset['test'] + dataset['oos_test'], fit=False, limit_num_sents=limit_num_sents, set_type='test') svc_int = svm.SVC(C=1, kernel='linear').fit(X_int_train, y_int_train) return svc_int, X_int_test, y_int_test, split
def train_intent_model(int_ds, random_selection: bool, dim: int, limit_num_sents: bool, num_samples=None): if random_selection: selection = get_intents_selection( int_ds['train'], num_samples) # selected intent labels: (num_samples, ) np.ndarray filt_train = get_filtered_lst( int_ds['train'], selection ) # almost the same as int_ds['train'] but filtered according to selection mod_int_ds = copy.deepcopy( int_ds) # deepcopy in order to not modify the original dict mod_int_ds['train'] = filt_train dataset = mod_int_ds else: dataset = int_ds train_str_int = dataset_2_string(dataset['train'], limit_num_sents=limit_num_sents, set_type='train') X_int_test, y_int_test = get_X_y_fasttext(dataset['test'] + dataset['oos_test'], limit_num_sents=limit_num_sents, set_type='test') with NamedTemporaryFile() as f: f.write(train_str_int.encode('utf8')) f.seek(0) # Train model for in-scope queries model_int = fasttext.train_supervised( input=f.name, dim=dim, pretrainedVectors=f'{PRETRAINED_VECTORS_PATH}/cc.en.{dim}.vec') return model_int, X_int_test, y_int_test
def train_intent_model(int_ds, random_selection: bool, limit_num_sents: bool, num_samples=None): if random_selection: selection = get_intents_selection( int_ds['train'], num_samples) # selected intent labels: (num_samples, ) np.ndarray filt_train = get_filtered_lst( int_ds['train'], selection ) # almost the same as int_ds['train'] but filtered according to selection mod_int_ds = copy.deepcopy( int_ds) # deepcopy in order to not modify the original dict mod_int_ds['train'] = filt_train dataset = mod_int_ds else: dataset = int_ds train_str_int = dataset_2_string_rasa(dataset['train'], limit_num_sents=limit_num_sents, set_type='train') X_int_test, y_int_test = get_X_y_rasa(dataset['test'] + dataset['oos_test'], limit_num_sents=limit_num_sents, set_type='test') with NamedTemporaryFile(suffix='.yml') as f: f.write(train_str_int.encode('utf8')) f.seek(0) training_data = rasa.shared.nlu.training_data.loading.load_data(f.name) config = rasa.nlu.config.load('config.yml') trainer = rasa.nlu.model.Trainer(config) model_int = trainer.train(training_data) return model_int, X_int_test, y_int_test
def train_intent_model(int_ds, random_selection: bool, limit_num_sents: bool, num_samples=None): if random_selection: selection = get_intents_selection( int_ds['train'], num_samples) # selected intent labels: (num_samples, ) np.ndarray filt_train = get_filtered_lst( int_ds['train'], selection ) # almost the same as int_ds['train'] but filtered according to selection filt_test = get_filtered_lst(int_ds['test'], selection) mod_int_ds = copy.deepcopy( int_ds) # deepcopy in order to not modify the original dict mod_int_ds['train'] = filt_train mod_int_ds['test'] = filt_test dataset = mod_int_ds else: dataset = int_ds # Split and tokenize dataset split = Split_BERT() tokenizer = BertTokenizer.from_pretrained('bert-base-uncased') X_int_train, y_int_train = split.get_X_y(dataset['train'], limit_num_sents=limit_num_sents, set_type='train') X_int_val, y_int_val = split.get_X_y(dataset['val'], limit_num_sents=limit_num_sents, set_type='val') X_int_test, y_int_test = split.get_X_y(dataset['test'] + dataset['oos_test'], limit_num_sents=limit_num_sents, set_type='test') train_int_ids, train_int_attention_masks, train_int_labels = tokenize_BERT( X_int_train, y_int_train, tokenizer) val_int_ids, val_int_attention_masks, val_int_labels = tokenize_BERT( X_int_val, y_int_val, tokenizer) num_labels = len(split.intents_dct.keys( )) - 1 # minus 1 because 'oos' label isn't used in training # Train model model_int = TFBertForSequenceClassification.from_pretrained( 'bert-base-uncased', num_labels=num_labels) # we have to adjust the number of labels print('\nBert Model', model_int.summary()) log_dir = 'tensorboard_data/tb_bert' model_save_path = './models/bert_model.h5' callbacks = [ tf.keras.callbacks.ModelCheckpoint(filepath=model_save_path, save_weights_only=True, monitor='val_loss', mode='min', save_best_only=True), tf.keras.callbacks.TensorBoard(log_dir=log_dir) ] loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True) metric = tf.keras.metrics.SparseCategoricalAccuracy('accuracy') optimizer = tf.keras.optimizers.Adam(learning_rate=4e-5) model_int.compile(loss=loss, optimizer=optimizer, metrics=[metric]) history = model_int.fit( [train_int_ids, train_int_attention_masks], train_int_labels, batch_size=32, epochs=5, validation_data=([val_int_ids, val_int_attention_masks], val_int_labels), callbacks=callbacks) return model_int, X_int_test, y_int_test, split
for num_samples in [3, 6, 9, 12]: # choose only a certain number of samples print( f'{repetitions} times random selection {num_samples} intents' ) accuracy_lst, recall_lst = [], [] far_lst, frr_lst = [], [] for i in range(repetitions): selection = get_intents_selection( int_ds['train'], num_samples ) # selected intent labels: (num_samples, ) np.ndarray filt_train = get_filtered_lst( int_ds['train'], selection ) # almost the same as int_ds['train'] but filtered according to selection filt_test = get_filtered_lst(int_ds['test'], selection) mod_int_ds = copy.deepcopy( int_ds ) # deepcopy in order to not modify the original dict mod_int_ds['train'] = filt_train mod_int_ds['test'] = filt_test temp_res = evaluate(mod_int_ds, LIMIT_NUM_SENTS) # temporary results accuracy_lst.append(temp_res['accuracy']) recall_lst.append(temp_res['recall']) far_lst.append(temp_res['far'])