def process_atis(infold, uncased, modes=['train', 'test'], dev_split=0):
""" MSFT's dataset, processed by Kaggle
https://www.kaggle.com/siddhadev/atis-dataset-from-ms-cntk
"""
outfold = f'{infold}/nemo-processed'
infold = f'{infold}/data/raw_data/ms-cntk-atis'
vocab = get_vocab(f'{infold}/atis.dict.vocab.csv')
if uncased:
outfold = f'{outfold}-uncased'
if if_exist(outfold, [f'{mode}.tsv' for mode in modes]):
logger.info(LOGGING_TMP.format('ATIS', outfold))
return outfold
logger.info(f'Processing ATIS dataset and store at {outfold}')
os.makedirs(outfold, exist_ok=True)
outfiles = {}
for mode in modes:
outfiles[mode] = open(os.path.join(outfold, mode + '.tsv'), 'w')
outfiles[mode].write('sentence\tlabel\n')
outfiles[mode + '_slots'] = open(f'{outfold}/{mode}_slots.tsv', 'w')
queries = open(f'{infold}/atis.{mode}.query.csv', 'r').readlines()
intents = open(f'{infold}/atis.{mode}.intent.csv', 'r').readlines()
slots = open(f'{infold}/atis.{mode}.slots.csv', 'r').readlines()
for i, query in enumerate(queries):
sentence = ids2text(query.strip().split()[1:-1], vocab)
outfiles[mode].write(f'{sentence}\t{intents[i].strip()}\n')
slot = ' '.join(slots[i].strip().split()[1:-1])
outfiles[mode + '_slots'].write(slot + '\n')
shutil.copyfile(f'{infold}/atis.dict.intent.csv',
f'{outfold}/dict.intents.csv')
shutil.copyfile(f'{infold}/atis.dict.slots.csv',
f'{outfold}/dict.slots.csv')
return outfold
def merge(data_dir, subdirs, dataset_name, modes=['train', 'test']):
outfold = f'{data_dir}/{dataset_name}'
if if_exist(outfold, [f'{mode}.tsv' for mode in modes]):
logger.info(LOGGING_TMP.format('SNIPS-ATIS', outfold))
slots = get_vocab(f'{outfold}/dict.slots.csv')
none_slot = 0
for key in slots:
if slots[key] == 'O':
none_slot = key
break
return outfold, int(none_slot)
os.makedirs(outfold, exist_ok=True)
data_files, slot_files = {}, {}
for mode in modes:
data_files[mode] = open(f'{outfold}/{mode}.tsv', 'w')
data_files[mode].write('sentence\tlabel\n')
slot_files[mode] = open(f'{outfold}/{mode}_slots.tsv', 'w')
intents, slots = {}, {}
intent_shift, slot_shift = 0, 0
none_intent, none_slot = -1, -1
for subdir in subdirs:
curr_intents = get_vocab(f'{data_dir}/{subdir}/dict.intents.csv')
curr_slots = get_vocab(f'{data_dir}/{subdir}/dict.slots.csv')
for key in curr_intents:
if intent_shift > 0 and curr_intents[key] == 'O':
continue
if curr_intents[key] == 'O' and intent_shift == 0:
none_intent = int(key)
intents[int(key) + intent_shift] = curr_intents[key]
for key in curr_slots:
if slot_shift > 0 and curr_slots[key] == 'O':
continue
if slot_shift == 0 and curr_slots[key] == 'O':
none_slot = int(key)
slots[int(key) + slot_shift] = curr_slots[key]
for mode in modes:
with open(f'{data_dir}/{subdir}/{mode}.tsv', 'r') as f:
for line in f.readlines()[1:]:
text, label = line.strip().split('\t')
label = int(label)
if curr_intents[label] == 'O':
label = none_intent
else:
label = label + intent_shift
data_files[mode].write(f'{text}\t{label}\n')
with open(f'{data_dir}/{subdir}/{mode}_slots.tsv', 'r') as f:
for line in f.readlines():
labels = [int(label) for label in line.strip().split()]
shifted_labels = []
for label in labels:
if curr_slots[label] == 'O':
shifted_labels.append(none_slot)
else:
shifted_labels.append(label + slot_shift)
slot_files[mode].write(list2str(shifted_labels) + '\n')
intent_shift += len(curr_intents)
slot_shift += len(curr_slots)
write_vocab_in_order(intents, f'{outfold}/dict.intents.csv')
write_vocab_in_order(slots, f'{outfold}/dict.slots.csv')
return outfold, none_slot