def create_datasets(path_to_data_folders):
data_root_folder = Path(path_to_data_folders)
ref_train_path = data_root_folder / 'train.csv'
ref_train_dataset = read_csv(ref_train_path)
for t in data_root_folder.iterdir(): # iterate over train size
if not t.is_dir(): # ignore reference train
continue
train_size = int(str(t).split('_')[-1])
LOGGER.info("Processing datasets of size: %s" % train_size)
for s in t.iterdir(): # iterate over seeds
if not s.is_dir(): # ignore pkl
continue
LOGGER.info("Processing seed: %s" % str(s.name))
# validate
val_csv_data = read_csv(s / 'validate.csv')
val_intents, val_entities = extract_intents_entities(
val_csv_data, ENTITY_MAPPING)
val_dataset = make_dataset_dict(val_intents, val_entities)
dump_json(val_dataset, s / 'validate.json'.format(train_size))
# train
tr_csv_data = read_csv(s / 'train_{}.csv'.format(train_size))
train_intents, train_entities = extract_intents_entities(
tr_csv_data, ENTITY_MAPPING)
enriched_train_entities = merge_entity_dict(
train_entities, val_entities)
train_dataset = make_dataset_dict(train_intents,
enriched_train_entities)
dump_json(train_dataset, s / 'train_{}.json'.format(train_size))
# augmented
csv_data = read_csv(s / 'train_{}_aug_2000.csv'.format(train_size))
augmented_utterances = csv_data[-2000:]
process_and_dump_augmentation(
current_path=s,
train_data=tr_csv_data,
train_entities=enriched_train_entities,
augmentation_data=augmented_utterances,
ref_data=ref_train_dataset,
augmentation_ratio=0.5,
train_size=train_size)
process_and_dump_augmentation(
current_path=s,
train_data=tr_csv_data,
train_entities=enriched_train_entities,
augmentation_data=augmented_utterances,
ref_data=ref_train_dataset,
augmentation_ratio=1,
train_size=train_size)
def add_nones(self,
sentences,
none_folder,
none_size=None,
none_intents=None,
pseudolabels=None,
none_idx=None):
none_path = none_folder / 'train.csv'
none_sentences = read_csv(none_path)
if none_intents is not None:
none_sentences = self.filter_intents(none_sentences, none_intents)
random.shuffle(none_sentences)
for row in none_sentences[:none_size]:
if 'snips' in str(none_folder): # if the none class is also on
# the snips format
new_row = row
if pseudolabels is not None:
new_row[3] = pseudolabels[row[3]]
else:
new_row[3] = 'None'
else:
utterance = row[none_idx]
new_row = [
utterance, 'O ' * len(word_tokenize(utterance)), utterance,
'None'
]
if pseudolabels is not None:
new_row[3] = pseudolabels[row[3]]
sentences.append(new_row)
return sentences
def add_nones(sentences, none_folder, none_idx, none_size):
none_path = none_folder / 'train.csv'
none_sentences = read_csv(none_path)
random.shuffle(none_sentences)
for row in none_sentences[:none_size]:
utterance = row[none_idx]
new_row = ["", "", "", "None", "", utterance, "", "", ""]
sentences.append(new_row)
return sentences
def save_augmented_dataset(generated_sentences, n_generated, train_path,
output_dir):
dataset = read_csv(train_path)
for s, l, d, i in zip(generated_sentences['utterances'],
generated_sentences['labellings'],
generated_sentences['delexicalised'],
generated_sentences['intents']):
dataset.append([s, l, d, i])
augmented_path = output_dir / Path(
train_path.name.replace('.csv', '_aug_{}.csv'.format(n_generated)))
write_csv(dataset, augmented_path)
def csv2json(csv_path_in, output_dir):
print('Starting csv2json conversion...')
jsondic = {'language': 'en'}
csv_data = read_csv(Path(csv_path_in))
intents, entities = extract_intents_entities(csv_data)
jsondic['intents'] = intents
jsondic['entities'] = entities
filename = str(Path(csv_path_in).stem) + '.json'
path_out = Path(output_dir) / filename
dump_json(jsondic, path_out)
print('Successfully converted csv2json !')
def embed_dataset(dataset_path, infersent_path, force_cpu=False):
"""
To make this work, first run ./get_infersent.sh
"""
MODEL_PATH = infersent_path / "encoder/infersent1.pkl"
params_model = {'bsize': 64, 'word_emb_dim': 300, 'enc_lstm_dim': 2048,
'pool_type': 'max', 'dpout_model': 0.0, 'version': 1}
model = InferSent(params_model)
if force_cpu:
model.load_state_dict(torch.load(MODEL_PATH, map_location='cpu'))
else:
model.load_state_dict(torch.load(MODEL_PATH))
model.cuda()
W2V_PATH = infersent_path / 'GloVe/glove.840B.300d.txt'
model.set_w2v_path(W2V_PATH)
model.build_vocab_k_words(K=100000)
csv_data = read_csv(dataset_path / 'train.csv')
csv_data = csv_data[1:] # skip header
data = defaultdict(list)
for irow, row in enumerate(csv_data):
if 'snips' in str(dataset_path):
utterance, labels, delexicalised, intent = row
else:
raise TypeError(
"Unknown dataset type. Implement your own first. See the "
"README")
data[intent].append(utterance)
vectors = {}
for i, (intent, sentences) in enumerate(data.items()):
print('{}/{} done'.format(i, len(data.items())))
embeddings = model.encode(sentences)
avg_embedding = np.mean(embeddings, axis=0)
vectors[intent] = avg_embedding
return vectors
def add_nones(sentences, none_folder, none_idx, none_size):
none_path = none_folder / 'train.csv'
none_sentences = read_csv(none_path)
random.shuffle(none_sentences)
new_rows = [[row[none_idx]] for row in none_sentences[:none_size]]
return sentences + new_rows
def build_data_files(self,
dataset_folder,
dataset_size=None,
restrict_intents=None,
none_folder=None,
none_size=None,
none_intents=None,
none_idx=None,
infersent_selection="no_infersent_selection",
cosine_threshold=None,
output_folder=None,
skip_header=True):
original_train_path = dataset_folder / 'train.csv'
original_test_path = dataset_folder / 'validate.csv'
new_train = read_csv(original_train_path)
new_test = read_csv(original_test_path)
if skip_header:
header_train = new_train[0]
header_test = new_test[0]
new_train = new_train[1:]
new_test = new_test[1:]
# filter intents
filter_prefix = ''
if restrict_intents is not None:
filter_prefix = '_filtered'
new_train = self.filter_intents(new_train, restrict_intents)
new_test = self.filter_intents(new_test, restrict_intents)
# trim_dataset
trim_prefix = ''
if dataset_size is not None:
trim_prefix = '_{}'.format(dataset_size)
original_dataset_size = len(new_train)
keep_fraction = dataset_size / original_dataset_size
intents = self.get_intents(new_train)
sss = StratifiedShuffleSplit(n_splits=1,
test_size=1 - keep_fraction)
keep_indices = list(sss.split(intents, intents))[0][0]
new_train = [new_train[i] for i in keep_indices]
# new_train = random.sample(new_train, dataset_size)
# add nones
train_none_prefix = ''
test_none_prefix = ''
if none_size is not None:
train_none_prefix = '_none_{}'.format(none_size)
test_none_prefix = '_with_none'
pseudolabels = None
if infersent_selection != NO_INFERSENT_SELECTION:
assert (none_intents is None)
none_intents, pseudolabels = self.select_none_intents(
new_train, restrict_intents, none_folder, cosine_threshold)
if infersent_selection == 'unsupervised':
pseudolabels = None # ignore pseudolabels
new_train = self.add_nones(new_train,
none_folder,
none_size=none_size,
none_intents=none_intents,
pseudolabels=pseudolabels,
none_idx=none_idx)
new_test = self.add_nones(new_test,
none_folder,
none_size=200,
none_intents=none_intents,
none_idx=none_idx)
if output_folder is not None:
new_train_path = output_folder / 'train{}{}{}.csv'.format(
trim_prefix, train_none_prefix, filter_prefix)
new_test_path = output_folder / 'validate{}{}.csv'.format(
test_none_prefix, filter_prefix)
else:
new_train_path = dataset_folder / 'train{}{}{}.csv'.format(
trim_prefix, train_none_prefix, filter_prefix)
new_test_path = dataset_folder / 'validate{}{}.csv'.format(
test_none_prefix, filter_prefix)
if skip_header:
new_train = [header_train] + new_train
new_test = [header_test] + new_test
write_csv(new_test, new_test_path)
write_csv(new_train, new_train_path)
return new_train_path, new_test_path
from automatic_data_generation.utils.io import read_csv, write_csv
from pathlib import Path
from sklearn.model_selection import StratifiedShuffleSplit
data = read_csv(Path('data.csv'))
header = [data[0]]
data = data[1:]
intents = [row[3] for row in data]
test_fraction = 0.2
test_size = int(test_fraction * len(data))
sss = StratifiedShuffleSplit(n_splits=1,
test_size=test_size)
train_indices, test_indices = list(sss.split(intents, intents))[0]
train = header + [data[i] for i in train_indices]
validate = header + [data[i] for i in test_indices]
write_csv(validate, Path('validate.csv'))
write_csv(train, Path('train.csv'))