def __init__(self, mode, params): super().__init__(mode, params) # Load vocab and tag list self.char_to_idx = load_tkn_to_idx(os.path.join(self.get_processed_data_dir(), "vocab", "chars.txt")) self.word_to_idx = load_tkn_to_idx(os.path.join(self.get_processed_data_dir(), "vocab", "words.txt")) self.idx_to_word = load_idx_to_tkn(os.path.join(self.get_processed_data_dir(), "vocab", "words.txt")) self.tag_to_idx = load_tkn_to_idx(os.path.join( self.get_processed_data_dir(), "vocab", "seg_tags.txt" if self.params.dataset.tag_type == "seg" else "pos_tags.txt")) self.idx_to_tag = load_idx_to_tkn(os.path.join( self.get_processed_data_dir(), "vocab", "seg_tags.txt" if self.params.dataset.tag_type == "seg" else "pos_tags.txt")) def _add_tag(tag): self.tag_to_idx[tag] = len(self.tag_to_idx) self.idx_to_tag.append(tag) if '<pad>' not in self.tag_to_idx: _add_tag('<pad>') if '<sos>' not in self.tag_to_idx: _add_tag('<sos>') if '<eos>' not in self.tag_to_idx: _add_tag('<eos>') self.vocab_char_size = len(self.char_to_idx) self.vocab_word_size = len(self.idx_to_word) self.num_tags = len(self.tag_to_idx) # Load data self.data = [] if self.mode in ["test", "train"]: self.data = self.load_data_from_file(os.path.join(self.get_processed_data_dir(), self.mode + ".csv")) elif self.mode == "infer": self.data = []
def __init__(self, mode, params): super().__init__(mode, params) cfg = params.dataset self.lang_src = cfg.source self.lang_tgt = cfg.target self.lang = (self.lang_src, self.lang_tgt) self.dataset_name = (self.lang_src if self.lang_src != 'eng' else self.lang_tgt) + '-eng' # Load vocab self.word2index = {} self.index2word = {} for lang in self.lang: self.word2index[lang] = load_tkn_to_idx( os.path.join(self.get_processed_data_dir(), self.dataset_name, "vocab", lang + ".txt")) self.index2word[lang] = load_idx_to_tkn( os.path.join(self.get_processed_data_dir(), self.dataset_name, "vocab", lang + ".txt")) logger.info("%s vocab size: %d", lang, len(self.word2index[lang])) self.input_size = len(self.word2index[self.lang_src]) self.output_size = len(self.word2index[self.lang_tgt]) # Load data if self.mode in ["test", "train"]: data = [] fo = open(os.path.join(self.get_processed_data_dir(), self.dataset_name, self.mode + ".csv"), "r", encoding='utf-8') lang_pairs = fo.readline().strip().split('\t')[:2] for line in tqdm(fo): pairs = line.strip().split('\t')[:2] pairs = {lang: pairs[i] for i, lang in enumerate(lang_pairs)} data.append( dict(X=[self.word2index[self.lang_src]['<sos>']] + [int(i) for i in pairs[self.lang_src].split(' ')] + [self.word2index[self.lang_src]['<eos>']], Y=[self.word2index[self.lang_tgt]['<sos>']] + [int(i) for i in pairs[self.lang_tgt].split(' ')] + [self.word2index[self.lang_tgt]['<eos>']])) fo.close() self.data = data elif self.mode == "infer": self.data = []
def write_dataset(processed_dir, output_prefix, file_paths, transcripts, vocab_path, normalize_fn, tokenize_fn): """ :param str processed_dir: :param str output_prefix: Output file is saved to {output_prefix}_{train/test}.csv :param dict[str, list[str]] file_paths: :param dict[str, list[str]] transcripts: :param str vocab_path: :param (str) -> str normalize_fn: :param (str) -> list[str] tokenize_fn: """ outputs = {'train': [], 'test': []} vocab = load_tkn_to_idx(vocab_path) for mode in ["test", "train"]: os.makedirs(os.path.join(processed_dir, mode, "npy"), exist_ok=True) for file_path, transcript in tqdm(list(zip(file_paths[mode], transcripts[mode])), desc=mode): file_name = os.path.basename(file_path) file_name, _ = os.path.splitext(file_name) if file_name == "": continue npy_filename = os.path.join(processed_dir, mode, "npy", file_name + ".npy") if os.path.exists(npy_filename): outputs[mode].append(dict( filename=npy_filename, target=' '.join([str(get_token_id(vocab, tkn)) for tkn in tokenize_fn(normalize_fn(transcript))]), trans_words=transcript )) for mode in ["test", "train"]: # outputs[mode].sort(key=lambda item: len(item['target_word'])) fn = os.path.join(processed_dir, "%s_%s" % (output_prefix, mode) + '.csv') logger.info("Output to %s" % fn) with open(fn, 'w', encoding='utf-8') as f: f.write('\t'.join(['sound', 'target', 'trans']) + '\n') for o in outputs[mode]: f.write('\t'.join([ o['filename'], o['target'], o['trans_words'] ]) + '\n')
def __init__(self, mode, params): super().__init__(mode, params) cfg = params.dataset vocab = load_tkn_to_idx( os.path.join(self.get_processed_data_dir(), "vocab_%ss.txt" % cfg.unit)) self.id_to_token = load_idx_to_tkn( os.path.join(self.get_processed_data_dir(), "vocab_%ss.txt" % cfg.unit)) self.vocab = vocab vocab['<sos>'] = len(vocab) vocab['<eos>'] = len(vocab) self.output_size = len(vocab) is_debug = mode == "debug" if mode == "debug": mode = "train" with open(os.path.join(self.get_processed_data_dir(), "%s_%s" % (cfg.unit, mode) + '.csv'), 'r', encoding='utf-8') as f: lines = f.read().split('\n')[1:] lines = [l.split('\t') for l in lines if l != ""] self.data = [{ 'X_path': l[0], 'Y': [vocab['<sos>']] + [int(w) for w in l[1].split(' ')] + [vocab['<eos>']], 'Y_len': len(l[1].split(' ')) + 2 } for l in lines] if is_debug: self.data = self.data[:20]
def preprocess(raw_dir, processed_dir): logger.info("Write outputs to file") outputs = {'train': [], 'test': []} mean = np.load("mean.npy") var = np.load("var.npy") vocab_word = load_tkn_to_idx(os.path.join(processed_dir, "vocab_words.txt")) vocab_char = load_tkn_to_idx(os.path.join(processed_dir, "vocab_chars.txt")) for mode in ["test", "train"]: os.makedirs(os.path.join(processed_dir, mode, "npy"), exist_ok=True) with open(os.path.join(raw_dir, mode, "prompts.txt"), encoding="utf-8") as f: lines = f.read().split("\n") for i, s in tqdm(list(enumerate(lines)), desc=mode): filename = s.split(' ')[0] if filename == "": continue npy_filename = os.path.join(processed_dir, mode, "npy", filename + ".npy") if True: # (rate, sig) = wav.read(wav_filename) htk_filename = os.path.join(processed_dir, mode, "features", filename + ".htk") fh = open(htk_filename, "rb") spam = fh.read(12) nSamples, sampPeriod, sampSize, parmKind = unpack( ">IIHH", spam) veclen = int(sampSize / 4) fh.seek(12, 0) dat = np.fromfile(fh, dtype=np.float32) dat = dat.reshape(len(dat) // veclen, veclen) dat = dat.byteswap() fh.close() dat = (dat - mean) / np.sqrt(var) np.save(npy_filename, dat) trans = s.lower().split(' ', 1)[1].replace(':', '') outputs[mode].append( dict(filename=npy_filename, target_word=' '.join([ str(get_token_id(vocab_word, normalize_word(word))) for word in trans.split(' ') ]), target_char=' '.join([ str( get_token_id( vocab_char, normalize_char(c).replace(' ', '_'))) for c in trans ]), trans_words=' '.join(s.lower().split(' ')[1:]))) for mode in ["test", "train"]: #outputs[mode].sort(key=lambda item: len(item['target_word'])) for unit in ["word", "char"]: logger.info("Output to %s" % os.path.join(processed_dir, "%s_%s" % (unit, mode) + '.csv')) with open(os.path.join(processed_dir, "%s_%s" % (unit, mode) + '.csv'), 'w', encoding='utf-8') as f: f.write('\t'.join(['sound', 'target', 'trans']) + '\n') for o in outputs[mode]: f.write('\t'.join([ o['filename'], o['target_%s' % unit], o['trans_words'] ]) + '\n')
def maybe_preprocess(cls, force=False): super().maybe_preprocess(force) if os.path.exists(cls.get_processed_data_dir()): return for lang_pairs in DOWNLOAD_URLS: try: dataset_name = "-".join(lang_pairs) pairs = zip( read_lang( os.path.join( cls.get_working_dir(), dataset_name, "europarl-v7.%s.%s" % (dataset_name, lang_pairs[0]))), read_lang( os.path.join( cls.get_raw_data_dir(), dataset_name, "europarl-v7.%s.%s" % (dataset_name, lang_pairs[1])))) logger.info("Read %s sentence pairs", len(pairs)) pairs = filter_pairs(pairs) logger.info("Trimmed to %s sentence pairs", len(pairs)) os.makedirs(cls.get_processed_data_dir(), exist_ok=True) default_words = ['<pad>', '<sos>', '<eos>', '<oov>'] word_token_to_idx = {} for i in [0, 1]: write_vocab( os.path.join(cls.get_processed_data_dir(), dataset_name), [_p[i] for _p in pairs], lang_pairs[i], 0, default_words) word_token_to_idx[lang_pairs[i]] = load_tkn_to_idx( os.path.join(cls.get_processed_data_dir(), dataset_name, "vocab", lang_pairs[i] + ".txt")) data = {'train': pairs[10000:], 'test': pairs[:10000]} for mode in ['train', 'test']: with open( os.path.join(cls.get_processed_data_dir(), dataset_name, "%s.csv" % mode), 'w') as fo: fo.write('\t'.join( list(lang_pairs) + [l + '-original' for l in lang_pairs]) + '\n') for item in data[mode]: fo.write('\t'.join([ ' '.join([ str( get_token_id( word_token_to_idx[lang_pairs[0]], w)) for w in item[0] ]), ' '.join([ str( get_token_id( word_token_to_idx[lang_pairs[1]], w)) for w in item[1] ]), ' '.join([w for w in item[0]]), ' '.join( [w for w in item[1]]) ]) + "\n") except: logger.error("Failed to process %s" % '-'.join(lang_pairs))
def maybe_preprocess(path, working_dir): if os.path.exists(working_dir): return logger.info("Preprocess data") os.makedirs(working_dir) os.mkdir(os.path.join(working_dir, "vocab")) original_sentences = {'train': [], 'test': []} sentences = {'train': [], 'test': []} postags = {'train': [], 'test': []} for mode in ['train', 'test']: for file in glob.glob(os.path.join(path, "%s*" % mode)): for sent in open(file, encoding='utf-8'): sent = sent.strip() if sent == '': continue words = [normalize_word(s.split('//')[0]) for s in sent.split(' ')] tags = [] for word in sent.split(' '): word = word.lower() if 'a' <= word[0] <= 'z': if len(word.split('//')[1]) <= 2: tags.append(word.split('//')[1]) else: tags.append("<punc>") original_sentences[mode].append(sent) sentences[mode].append(words) postags[mode].append(tags) prepare_vocab_chars(working_dir, sentences['train']) write_vocab(working_dir, sentences['train'], min_freq=0) write_vocab(working_dir, postags['train'], name="pos_tags", min_freq=0, default_tags=['<w>', '<oov>']) prepare_tag_list(working_dir) word_token_to_idx = load_tkn_to_idx(os.path.join(working_dir, "vocab", "words.txt")) pos_tag_to_idx = load_tkn_to_idx(os.path.join(working_dir, "vocab", "pos_tags.txt")) char_token_to_idx = load_tkn_to_idx(os.path.join(working_dir, "vocab", "chars.txt")) for mode in ['train', 'test']: data = [] for p_tags, compound_words, sent in \ zip(postags[mode], sentences[mode], original_sentences[mode]): chars = tokenize(sent, 'char') words = [] seg_tags = [] pos_tags = [] for w, tag in zip(compound_words, p_tags): if '_' in w: w = w.split('_') words += w seg_tags += [1] + [0] * (len(w) - 1) pos_tags += [tag] + ['<w>'] * (len(w) - 1) else: words.append(w) seg_tags.append(1) pos_tags.append(tag) data.append((words, seg_tags, pos_tags, chars)) with open(os.path.join(working_dir, "%s.csv" % mode), "w", encoding='utf-8') as fo: fo.write('word_tokens,word_seg_tags,word_pos_tags,char_tokens\n') data.sort(key=lambda d: len(d[0]), reverse=True) for words, seg_tags, pos_tags, chars in data: fo.write(','.join([ ' '.join([str(get_token_id(word_token_to_idx, w)) for w in words]), ' '.join([str(t) for t in seg_tags]), ' '.join([str(get_token_id(pos_tag_to_idx, t)) for t in pos_tags]), ' '.join([str(get_token_id(char_token_to_idx, c)) for c in chars]), ]) + '\n')