def process_dialogflow(infold, outfold, dev_split=0.1):
if not os.path.exists(infold):
link = 'www.dialogflow.com'
raise ValueError(f'Data not found at {infold}. '
f'Export your dialogflow data from'
f'{link} and unzip at {infold}.')
if if_exist(outfold, [f'{mode}.tsv' for mode in ['train', 'test']]):
logging.info(DATABASE_EXISTS_TMP.format('mturk', outfold))
return
os.makedirs(outfold, exist_ok=True)
files = get_intent_query_files_dialogflow(infold)
slot_labels = get_slots_dialogflow(files)
intent_queries, intent_names, slot_tags = get_intents_slots_dialogflow(
files, slot_labels)
train_queries, train_slots, test_queries, test_slots = partition_data(
intent_queries, slot_tags, split=dev_split)
write_files(train_queries, f'{outfold}/train.tsv')
write_files(train_slots, f'{outfold}/train_slots.tsv')
write_files(test_queries, f'{outfold}/test.tsv')
write_files(test_slots, f'{outfold}/test_slots.tsv')
write_files(slot_labels, f'{outfold}/dict.slots.csv')
write_files(intent_names, f'{outfold}/dict.intents.csv')
def process_assistant(infold, outfold, modes=['train', 'test']):
"""
https://github.com/xliuhw/NLU-Evaluation-Data - this dataset includes
about 25 thousand examples with 66 various multi-domain intents and 57 entity types.
"""
if if_exist(outfold, [f'{mode}_slots.tsv' for mode in modes]):
logging.info(DATABASE_EXISTS_TMP.format('robot', outfold))
return outfold
logging.info(
f'Processing assistant commands dataset and store at {outfold}')
os.makedirs(outfold, exist_ok=True)
# copy train/test files to the convenient directory to work with
copy_input_files(infold)
infold += "/dataset"
# get list of intents from train folder (test folder supposed to be the same)
intent_names = get_intents(infold + "/trainset")
write_files(intent_names, f'{outfold}/dict.intents.csv')
# get all train and test queries with their intent
for mode in modes:
intent_queries = get_intent_queries(infold, intent_names, mode)
write_files(intent_queries, f'{outfold}/{mode}.tsv')
# get list of all unique slots in training and testing files
slot_types = get_slots(infold, modes)
write_files(slot_types, f'{outfold}/dict.slots.csv')
# create files of slot queries
slot_dict = {k: v for v, k in enumerate(slot_types)}
for mode in modes:
slot_queries = get_slot_queries(infold, slot_dict, mode, intent_names)
write_files(slot_queries, f'{outfold}/{mode}_slots.tsv')
def create_vocab_mlm(
self,
data_dir,
vocab_size,
sample_size,
special_tokens=['[PAD]', '[UNK]', '[CLS]', '[SEP]', '[MASK]'],
train_file='',
):
vocab = special_tokens[:]
bert_dir = f'{data_dir}/bert'
if if_exist(bert_dir, ['tokenizer.model']):
logging.info(DATABASE_EXISTS_TMP.format('WikiText_BERT', bert_dir))
return data_dir, f'{bert_dir}/tokenizer.model'
logging.info(f'Processing WikiText dataset and store at {bert_dir}')
os.makedirs(bert_dir, exist_ok=True)
if not train_file:
files = glob.glob(f'{data_dir}/*.txt')
train_file = f'{bert_dir}/merged.txt'
logging.info(f"Merging {len(files)} txt files into {train_file}")
with open(train_file, "w") as merged:
for file in tqdm(files):
with open(file, 'r') as inf:
content = inf.read().strip()
merged.write(content + '\n\n\n')
else:
train_file = f'{data_dir}/{train_file}'
cmd = (f"--input={train_file} --model_prefix={bert_dir}/tokenizer "
f"--vocab_size={vocab_size - len(vocab)} "
f"--input_sentence_size={sample_size} "
f"--shuffle_input_sentence=true --hard_vocab_limit=false "
f"--bos_id=-1 --eos_id=-1")
SPT.Train(cmd)
# Add BERT control symbols
tokens = []
with open(f"{bert_dir}/tokenizer.vocab", "r") as f:
f.readline() # skip first <unk> token
# Read tokens from each line and parse for vocab
for line in f:
piece = line.split("\t")[0]
token = piece[1:] if piece.startswith("▁") else f"##{piece}"
tokens.append(token)
vocab.extend(tokens)
# Save vocabulary to output file
with open(f'{bert_dir}/vocab.txt', "w") as f:
for token in vocab:
f.write(f"{token}\n".format())
return data_dir, f'{bert_dir}/tokenizer.model'
def process_thucnews(data_dir):
modes = ['train', 'test']
train_size = 0.8
if not os.path.exists(data_dir):
link = 'thuctc.thunlp.org/'
raise ValueError(f'Data not found at {data_dir}. '
f'Please download THUCNews from {link}.')
outfold = f'{data_dir}/nemo-processed-thucnews'
if if_exist(outfold, [f'{mode}.tsv' for mode in modes]):
logging.info(DATABASE_EXISTS_TMP.format('THUCNews', outfold))
return outfold
logging.info(f'Processing THUCNews dataset and store at {outfold}')
os.makedirs(outfold, exist_ok=True)
outfiles = {}
for mode in modes:
outfiles[mode] = open(os.path.join(outfold, mode + '.tsv'),
'a+',
encoding='utf-8')
outfiles[mode].write('sentence\tlabel\n')
categories = [
'体育', '娱乐', '家居', '彩票', '房产', '教育', '时尚', '时政', '星座', '游戏', '社会', '科技',
'股票', '财经'
]
for category in categories:
label = categories.index(category)
category_files = glob.glob(f'{data_dir}/{category}/*.txt')
test_num = int(len(category_files) * (1 - train_size))
test_files = category_files[:test_num]
train_files = category_files[test_num:]
for mode in modes:
logging.info(f'Processing {mode} data of the category {category}')
if mode == 'test':
files = test_files
else:
files = train_files
if len(files) == 0:
logging.info(f'Skipping category {category} for {mode} mode')
continue
for file in tqdm.tqdm(files):
with open(file, 'r', encoding='utf-8') as f:
news = f.read().strip().replace('\r', '')
news = news.replace('\n', '').replace('\t', ' ')
outfiles[mode].write(f'{news}\t{label}\n')
for mode in modes:
outfiles[mode].close()
return outfold
def process_snips(data_dir, uncased, modes=['train', 'test'], dev_split=0.1):
if not os.path.exists(data_dir):
link = 'www.github.com/snipsco/spoken-language'
'-understanding-research-datasets'
raise ValueError(
f'Data not found at {data_dir}. '
f'Resquest to download the SNIPS dataset from {link}.')
outfold = f'{data_dir}/nemo-processed'
if uncased:
outfold = f'{outfold}-uncased'
exist = True
for dataset in ['light', 'speak', 'all']:
if if_exist(f'{outfold}/{dataset}', [f'{mode}.tsv' for mode in modes]):
logging.info(
DATABASE_EXISTS_TMP.format('SNIPS-' + dataset.upper(),
outfold))
else:
exist = False
if exist:
return outfold
logging.info(f'Processing SNIPS dataset and store at {outfold}')
os.makedirs(outfold, exist_ok=True)
speak_dir = 'smart-speaker-en-close-field'
light_dir = 'smart-lights-en-close-field'
light_files = [f'{data_dir}/{light_dir}/dataset.json']
speak_files = [f'{data_dir}/{speak_dir}/training_dataset.json']
speak_files.append(f'{data_dir}/{speak_dir}/test_dataset.json')
light_train, light_dev, light_slots, light_intents = get_dataset(
light_files, dev_split)
speak_train, speak_dev, speak_slots, speak_intents = get_dataset(
speak_files)
create_dataset(light_train, light_dev, light_slots, light_intents, uncased,
f'{outfold}/light')
create_dataset(speak_train, speak_dev, speak_slots, speak_intents, uncased,
f'{outfold}/speak')
create_dataset(
light_train + speak_train,
light_dev + speak_dev,
light_slots | speak_slots,
light_intents | speak_intents,
uncased,
f'{outfold}/all',
)
return outfold
def process_atis(infold, uncased, modes=['train', 'test'], dev_split=0):
""" MSFT's dataset, processed by Kaggle
https://www.kaggle.com/siddhadev/atis-dataset-from-ms-cntk
"""
outfold = f'{infold}/nemo-processed'
vocab = get_vocab(f'{infold}/atis.dict.vocab.csv')
if uncased:
outfold = f'{outfold}-uncased'
if if_exist(outfold, [f'{mode}.tsv' for mode in modes]):
logging.info(DATABASE_EXISTS_TMP.format('ATIS', outfold))
return outfold
logging.info(f'Processing ATIS dataset and store at {outfold}')
os.makedirs(outfold, exist_ok=True)
outfiles = {}
for mode in modes:
outfiles[mode] = open(os.path.join(outfold, mode + '.tsv'), 'w')
outfiles[mode].write('sentence\tlabel\n')
outfiles[mode + '_slots'] = open(f'{outfold}/{mode}_slots.tsv', 'w')
queries = open(f'{infold}/atis.{mode}.query.csv', 'r').readlines()
intents = open(f'{infold}/atis.{mode}.intent.csv', 'r').readlines()
slots = open(f'{infold}/atis.{mode}.slots.csv', 'r').readlines()
for i, query in enumerate(queries):
sentence = ids2text(query.strip().split()[1:-1], vocab)
outfiles[mode].write(f'{sentence}\t{intents[i].strip()}\n')
slot = ' '.join(slots[i].strip().split()[1:-1])
outfiles[mode + '_slots'].write(slot + '\n')
shutil.copyfile(f'{infold}/atis.dict.intent.csv',
f'{outfold}/dict.intents.csv')
shutil.copyfile(f'{infold}/atis.dict.slots.csv',
f'{outfold}/dict.slots.csv')
for mode in modes:
outfiles[mode].close()
return outfold
def process_imdb(data_dir, uncased, modes=['train', 'test']):
if not os.path.exists(data_dir):
link = 'www.kaggle.com/iarunava/imdb-movie-reviews-dataset'
raise ValueError(f'Data not found at {data_dir}. '
f'Please download IMDB from {link}.')
outfold = f'{data_dir}/nemo-processed'
if uncased:
outfold = f'{outfold}_uncased'
if if_exist(outfold, [f'{mode}.tsv' for mode in modes]):
logging.info(DATABASE_EXISTS_TMP.format('IMDB', outfold))
return outfold
logging.info(f'Processing IMDB dataset and store at {outfold}')
os.makedirs(outfold, exist_ok=True)
outfiles = {}
for mode in modes:
outfiles[mode] = open(os.path.join(outfold, mode + '.tsv'), 'w')
outfiles[mode].write('sentence\tlabel\n')
for sent in ['neg', 'pos']:
if sent == 'neg':
label = 0
else:
label = 1
files = glob.glob(f'{data_dir}/{mode}/{sent}/*.txt')
for file in files:
with open(file, 'r') as f:
review = f.read().strip()
if uncased:
review = review.lower()
review = review.replace("<br />", "")
outfiles[mode].write(f'{review}\t{label}\n')
for mode in modes:
outfiles[mode].close()
return outfold
def process_mturk(infold, outfold, modes=['train', 'test']):
if not os.path.exists(infold):
link = 'www.mturk.com'
raise ValueError(
f'Data not found at {infold}. ' f'Export your mturk data from' f'{link} and unzip at {infold}.'
)
if if_exist(outfold, [f'{mode}.tsv' for mode in modes]):
logging.info(DATABASE_EXISTS_TMP.format('mturk', outfold))
return
logging.info(f'Processing dataset from mturk and storing at {outfold}')
os.makedirs(outfold, exist_ok=True)
classification_data_file = f'{infold}/classification.csv'
annotation_data_file = f'{infold}/annotation.manifest'
if not os.path.exists(classification_data_file):
raise FileNotFoundError(f'File not found ' f'at {classification_data_file}')
if not os.path.exists(annotation_data_file):
raise FileNotFoundError(f'File not found at {annotation_data_file}')
utterances = []
utterances = read_csv(classification_data_file)
# This function assumes that the intent classification data has been
# reviewed and cleaned and only one label per utterance is present.
agreed_all, intent_names = get_intents_mturk(utterances, outfold)
with open(annotation_data_file, 'r') as f:
slot_annotations = f.readlines()
# This function assumes that the preprocess step would have made
# the task_name of all the annotations generic
task_name = 'retail-combined'
# It is assumed that every utterances will have corresponding
# slot annotation information
if len(slot_annotations) < len(agreed_all):
raise ValueError(f'Every utterance must have corresponding' f'slot annotation information')
slot_labels, intent_queries, slot_tags = process_intent_slot_mturk(
slot_annotations, agreed_all, intent_names, task_name
)
assert len(slot_tags) == len(intent_queries)
dev_split = 0.1
train_queries, train_slots, test_queries, test_slots = partition_data(intent_queries, slot_tags, split=dev_split)
write_files(train_queries, f'{outfold}/train.tsv')
write_files(train_slots, f'{outfold}/train_slots.tsv')
write_files(test_queries, f'{outfold}/test.tsv')
write_files(test_slots, f'{outfold}/test_slots.tsv')
write_files(slot_labels, f'{outfold}/dict.slots.csv')
write_files(intent_names, f'{outfold}/dict.intents.csv')
def process_jarvis_datasets(infold, uncased, dataset_name, modes=['train', 'test', 'eval'], ignore_prev_intent=False):
""" process and convert Jarvis datasets into NeMo's BIO format
"""
outfold = f'{infold}/{dataset_name}-nemo-processed'
infold = f'{infold}/'
if uncased:
outfold = f'{outfold}-uncased'
if if_exist(outfold, ['dict.intents.csv', 'dict.slots.csv']):
logging.info(DATABASE_EXISTS_TMP.format(dataset_name, outfold))
return outfold
logging.info(f'Processing {dataset_name} dataset and store at {outfold}')
os.makedirs(outfold, exist_ok=True)
outfiles = {}
intents_list = {}
slots_list = {}
slots_list_all = {}
outfiles['dict_intents'] = open(f'{outfold}/dict.intents.csv', 'w')
outfiles['dict_slots'] = open(f'{outfold}/dict.slots.csv', 'w')
outfiles['dict_slots'].write('O\n')
slots_list["O"] = 0
slots_list_all["O"] = 0
for mode in modes:
if if_exist(outfold, [f'{mode}.tsv']):
logging.info(MODE_EXISTS_TMP.format(mode, dataset_name, outfold, mode))
continue
if not if_exist(infold, [f'{mode}.tsv']):
logging.info(f'{mode} mode of {dataset_name}' f' is skipped as it was not found.')
continue
outfiles[mode] = open(os.path.join(outfold, mode + '.tsv'), 'w')
outfiles[mode].write('sentence\tlabel\n')
outfiles[mode + '_slots'] = open(f'{outfold}/{mode}_slots.tsv', 'w')
queries = open(f'{infold}/{mode}.tsv', 'r').readlines()
for i, query in enumerate(queries):
line_splits = query.strip().split("\t")
if len(line_splits) == 3:
intent_str, slot_tags_str, sentence = line_splits
else:
intent_str, sentence = line_splits
slot_tags_str = ""
if intent_str not in intents_list:
intents_list[intent_str] = len(intents_list)
outfiles['dict_intents'].write(f'{intent_str}\n')
if ignore_prev_intent:
start_token = 2
else:
start_token = 1
sentence_cld = " ".join(sentence.strip().split()[start_token:-1])
outfiles[mode].write(f'{sentence_cld}\t' f'{str(intents_list[intent_str])}\n')
slot_tags_list = []
if slot_tags_str.strip():
slot_tags = slot_tags_str.strip().split(",")
for st in slot_tags:
if not st.strip():
continue
[start_i, end_i, slot_name] = st.strip().split(":")
slot_tags_list.append([int(start_i), int(end_i), slot_name])
if slot_name not in slots_list:
slots_list[slot_name] = len(slots_list)
slots_list_all[f'B-{slot_name}'] = len(slots_list_all)
slots_list_all[f'I-{slot_name}'] = len(slots_list_all)
outfiles['dict_slots'].write(f'B-{slot_name}\n')
outfiles['dict_slots'].write(f'I-{slot_name}\n')
slot_tags_list.sort(key=lambda x: x[0])
slots = []
processed_index = 0
for tag_start, tag_end, tag_str in slot_tags_list:
if tag_start > processed_index:
words_list = sentence[processed_index:tag_start].strip().split()
slots.extend([str(slots_list_all['O'])] * len(words_list))
words_list = sentence[tag_start:tag_end].strip().split()
slots.append(str(slots_list_all[f'B-{tag_str}']))
slots.extend([str(slots_list_all[f'I-{tag_str}'])] * (len(words_list) - 1))
processed_index = tag_end
if processed_index < len(sentence):
words_list = sentence[processed_index:].strip().split()
slots.extend([str(slots_list_all['O'])] * len(words_list))
slots = slots[1:-1]
slot = ' '.join(slots)
outfiles[mode + '_slots'].write(slot + '\n')
outfiles[mode + '_slots'].close()
outfiles[mode].close()
outfiles['dict_slots'].close()
outfiles['dict_intents'].close()
return outfold
def process_nlu(filename, uncased, modes=['train', 'test'], dataset_name='nlu-ubuntu'):
""" Dataset has to be of:
- ubuntu
- chat
- web
"""
if not os.path.exists(filename):
link = 'https://github.com/sebischair/NLU-Evaluation-Corpora'
raise ValueError(f'Data not found at {filename}. ' f'Please download IMDB from {link}.')
if dataset_name == 'nlu-ubuntu':
INTENT = {'makeupdate': 1, 'setupprinter': 2, 'shutdowncomputer': 3, 'softwarerecommendation': 4, 'none': 0}
elif dataset_name == 'nlu-chat':
INTENT = {'departuretime': 0, 'findconnection': 1}
elif dataset_name == 'nlu-web':
INTENT = {
'changepassword': 1,
'deleteaccount': 2,
'downloadvideo': 3,
'exportdata': 4,
'filterspam': 5,
'findalternative': 6,
'syncaccounts': 7,
'none': 0,
}
else:
raise ValueError(f'{dataset_name}: Invalid dataset name')
infold = filename[: filename.rfind('/')]
outfold = f'{infold}/{dataset_name}-nemo-processed'
if uncased:
outfold = f'{outfold}_uncased'
if if_exist(outfold, [f'{mode}.tsv' for mode in modes]):
logging.info(DATABASE_EXISTS_TMP.format(dataset_name.upper(), outfold))
return outfold
logging.info(f'Processing data and store at {outfold}')
os.makedirs(outfold, exist_ok=True)
outfiles = {}
for mode in modes:
outfiles[mode] = open(os.path.join(outfold, mode + '.tsv'), 'w')
outfiles[mode].write('sentence\tlabel\n')
with open(filename, 'r') as f:
data = json.load(f)
for obj in data['sentences']:
sentence = obj['text'].strip()
if uncased:
sentence = sentence.lower()
intent = obj['intent'].lower().replace(' ', '')
label = INTENT[intent]
txt = f'{sentence}\t{label}\n'
if obj['training']:
outfiles['train'].write(txt)
else:
outfiles['test'].write(txt)
for mode in modes:
outfiles[mode].close()
return outfold
def __init__(self, dataset_name, data_dir, do_lower_case):
if dataset_name == 'sst-2':
self.data_dir = process_sst_2(data_dir)
self.num_labels = 2
self.eval_file = self.data_dir + '/dev.tsv'
elif dataset_name == 'imdb':
self.num_labels = 2
self.data_dir = process_imdb(data_dir, do_lower_case)
self.eval_file = self.data_dir + '/test.tsv'
elif dataset_name == 'thucnews':
self.num_labels = 14
self.data_dir = process_thucnews(data_dir)
self.eval_file = self.data_dir + '/test.tsv'
elif dataset_name.startswith('nlu-'):
if dataset_name.endswith('chat'):
self.data_dir = f'{data_dir}/ChatbotCorpus.json'
self.num_labels = 2
elif dataset_name.endswith('ubuntu'):
self.data_dir = f'{data_dir}/AskUbuntuCorpus.json'
self.num_labels = 5
elif dataset_name.endswith('web'):
data_dir = f'{data_dir}/WebApplicationsCorpus.json'
self.num_labels = 8
self.data_dir = process_nlu(data_dir,
do_lower_case,
dataset_name=dataset_name)
self.eval_file = self.data_dir + '/test.tsv'
elif dataset_name.startswith('jarvis'):
self.data_dir = process_jarvis_datasets(
data_dir,
do_lower_case,
dataset_name,
modes=['train', 'test', 'eval'],
ignore_prev_intent=False)
intents = get_intent_labels(f'{self.data_dir}/dict.intents.csv')
self.num_labels = len(intents)
else:
raise ValueError("Looks like you passed a dataset name that isn't "
"already supported by NeMo. Please make sure "
"that you build the preprocessing method for it.")
self.train_file = self.data_dir + '/train.tsv'
for mode in ['train', 'test', 'eval']:
if not if_exist(self.data_dir, [f'{mode}.tsv']):
logging.info(f' Stats calculation for {mode} mode'
f' is skipped as {mode}.tsv was not found.')
continue
input_file = f'{self.data_dir}/{mode}.tsv'
with open(input_file, 'r') as f:
input_lines = f.readlines()[1:] # Skipping headers at index 0
queries, raw_sentences = [], []
for input_line in input_lines:
parts = input_line.strip().split()
raw_sentences.append(int(parts[-1]))
queries.append(' '.join(parts[:-1]))
infold = input_file[:input_file.rfind('/')]
logging.info(f'Three most popular classes during {mode}ing')
total_sents, sent_label_freq = get_label_stats(
raw_sentences, infold + f'/{mode}_sentence_stats.tsv')
if mode == 'train':
self.class_weights = calc_class_weights(sent_label_freq)
logging.info(f'Class weights are - {self.class_weights}')
logging.info(f'Total Sentences - {total_sents}')
logging.info(f'Sentence class frequencies - {sent_label_freq}')