def build_dataset(fields, data_type, src_data_iter=None, src_path=None, src_dir=None, tgt_data_iter=None, tgt_path=None, src_seq_length=0, tgt_seq_length=0, src_seq_length_trunc=0, tgt_seq_length_trunc=0, dynamic_dict=True, sample_rate=0, window_size=0, window_stride=0, window=None, normalize_audio=True, use_filter_pred=True): """ Build src/tgt examples iterator from corpus files, also extract number of features. """ def _make_examples_nfeats_tpl(data_type, src_data_iter, src_path, src_dir, src_seq_length_trunc, sample_rate, window_size, window_stride, window, normalize_audio): """ Process the corpus into (example_dict iterator, num_feats) tuple on source side for different 'data_type'. """ if data_type == 'text': src_examples_iter, num_src_feats = \ TextDataset.make_text_examples_nfeats_tpl( src_data_iter, src_path, src_seq_length_trunc, "src") elif data_type == 'img': src_examples_iter, num_src_feats = \ ImageDataset.make_image_examples_nfeats_tpl( src_data_iter, src_path, src_dir) elif data_type == 'audio': if src_data_iter: raise ValueError("""Data iterator for AudioDataset isn't implemented""") if src_path is None: raise ValueError("AudioDataset requires a non None path") src_examples_iter, num_src_feats = \ AudioDataset.make_audio_examples_nfeats_tpl( src_path, src_dir, sample_rate, window_size, window_stride, window, normalize_audio) return src_examples_iter, num_src_feats src_examples_iter, num_src_feats = \ _make_examples_nfeats_tpl(data_type, src_data_iter, src_path, src_dir, src_seq_length_trunc, sample_rate, window_size, window_stride, window, normalize_audio) # For all data types, the tgt side corpus is in form of text. tgt_examples_iter, num_tgt_feats = \ TextDataset.make_text_examples_nfeats_tpl( tgt_data_iter, tgt_path, tgt_seq_length_trunc, "tgt") if data_type == 'text': dataset = TextDataset(fields, src_examples_iter, tgt_examples_iter, num_src_feats, num_tgt_feats, src_seq_length=src_seq_length, tgt_seq_length=tgt_seq_length, dynamic_dict=dynamic_dict, use_filter_pred=use_filter_pred) elif data_type == 'img': dataset = ImageDataset(fields, src_examples_iter, tgt_examples_iter, num_src_feats, num_tgt_feats, tgt_seq_length=tgt_seq_length, use_filter_pred=use_filter_pred) elif data_type == 'audio': dataset = AudioDataset(fields, src_examples_iter, tgt_examples_iter, num_src_feats, num_tgt_feats, tgt_seq_length=tgt_seq_length, sample_rate=sample_rate, window_size=window_size, window_stride=window_stride, window=window, normalize_audio=normalize_audio, use_filter_pred=use_filter_pred) return dataset
def build_dataset(fields, data_type, src, knl, src_dir=None, tgt=None, knl_seq_len=800, src_seq_len=150, tgt_seq_len=50, knl_seq_length_trunc=200, src_seq_length_trunc=50, tgt_seq_length_trunc=0, dynamic_dict=False, sample_rate=0, window_size=0, window_stride=0, window=None, normalize_audio=True, use_filter_pred=True, image_channel_size=3, corpus_type='train', model_mode='default'): """ src: path to corpus file or iterator over source data tgt: path to corpus file, iterator over target data, or None """ dataset_classes = { 'text': TextDataset, 'img': ImageDataset, 'audio': AudioDataset } assert data_type in dataset_classes assert src is not None assert not dynamic_dict or data_type == 'text', \ 'it is not possible to use dynamic_dict with non-text input' if data_type == 'text': src_examples_iter = TextDataset.make_examples( src, src_seq_length_trunc, "src", corpus_type, model_mode ) knl_examples_iter = TextDataset.make_examples( knl, knl_seq_length_trunc, "knl", corpus_type, model_mode ) elif data_type == 'img': # there is a truncate argument as well, but it was never set to # anything besides None before src_examples_iter = ImageDataset.make_examples( src, src_dir, 'src', channel_size=image_channel_size ) else: src_examples_iter = AudioDataset.make_examples( src, src_dir, "src", sample_rate, window_size, window_stride, window, normalize_audio, None) if tgt is None: tgt_examples_iter = None else: tgt_examples_iter = TextDataset.make_examples( tgt, tgt_seq_length_trunc, "tgt", corpus_type, model_mode) # the second conjunct means nothing will be filtered at translation time # if there is no target data if use_filter_pred and tgt_examples_iter is not None: filter_pred = partial( filter_example, use_src_len=data_type == 'text', use_knl_len=data_type == 'text', max_src_len=src_seq_len, max_tgt_len=tgt_seq_len, max_knl_len=knl_seq_len ) else: filter_pred = None dataset_cls = dataset_classes[data_type] dataset = dataset_cls( fields, src_examples_iter, tgt_examples_iter, knl_examples_iter, dynamic_dict=dynamic_dict, filter_pred=filter_pred ) print("[onmt.inputters.inputter.py] dataset_cls:{}".format(dataset_cls)) print("[onmt.inputters.inputter.py] dataset:{}".format(dataset)) return dataset
def build_dataset(fields, data_type, src_data_iter=None, src_path=None, src_dir=None, tgt_data_iter=None, tgt_path=None, src_seq_len=0, tgt_seq_len=0, src_seq_length_trunc=0, tgt_seq_length_trunc=0, dynamic_dict=True, sample_rate=0, window_size=0, window_stride=0, window=None, normalize_audio=True, use_filter_pred=True, image_channel_size=3): """ Build src/tgt examples iterator from corpus files, also extract number of features. """ if data_type == 'text': src_examples_iter = TextDataset.make_text_examples( src_data_iter, src_path, src_seq_length_trunc, "src") elif data_type == 'img': src_examples_iter = ImageDataset.make_image_examples( src_data_iter, src_path, src_dir, image_channel_size) elif data_type == 'audio': if src_data_iter: raise ValueError("""Data iterator for AudioDataset isn't implemented""") if src_path is None: raise ValueError("AudioDataset requires a non None path") src_examples_iter = AudioDataset.make_audio_examples( src_path, src_dir, sample_rate, window_size, window_stride, window, normalize_audio) # For all data types, the tgt side corpus is in form of text. tgt_examples_iter = TextDataset.make_text_examples(tgt_data_iter, tgt_path, tgt_seq_length_trunc, "tgt") # I'm not certain about the practical utility of the second part if use_filter_pred and tgt_examples_iter is not None: filter_pred = partial(filter_example, use_src_len=data_type == 'text', max_src_len=src_seq_len, max_tgt_len=tgt_seq_len) else: filter_pred = None if data_type == 'text': dataset = TextDataset(fields, src_examples_iter, tgt_examples_iter, dynamic_dict=dynamic_dict, filter_pred=filter_pred) else: dataset_cls = ImageDataset if data_type == 'img' else AudioDataset dataset = dataset_cls(fields, src_examples_iter, tgt_examples_iter, filter_pred=filter_pred) return dataset