def get_test_stream(test_set=None, src_vocab=None, trg_vocab=None, src_vocab_size=120, trg_vocab_size=120, unk_id=1, bos_token='<S>', **kwargs): """Setup development set stream if necessary.""" test_stream = None if test_set is not None and src_vocab is not None and trg_vocab is not None: src_vocab = _ensure_special_tokens(src_vocab if isinstance( src_vocab, dict) else pickle.load(open(src_vocab, 'rb')), bos_idx=0, eos_idx=src_vocab_size - 1, unk_idx=unk_id) trg_vocab = _ensure_special_tokens(trg_vocab if isinstance( trg_vocab, dict) else pickle.load(open(trg_vocab, 'rb')), bos_idx=0, eos_idx=trg_vocab_size - 1, unk_idx=unk_id) test_src_dataset = TextFileWithSEOSS([test_set], src_vocab, None, level='character') test_stream = DataStream(test_src_dataset) test_stream.space_idx = { 'source': src_vocab[' '], 'target': trg_vocab[' '] } test_stream.trg_bos = trg_vocab[bos_token] test_stream.trg_vocab = trg_vocab test_stream.eos_token = '</S>' return test_stream