def test_sampling(): # Create Theano variables sampling_input = theano.tensor.lmatrix("input") # Construct model encoder = BidirectionalEncoder(vocab_size=10, embedding_dim=5, state_dim=8) decoder = Decoder(vocab_size=12, embedding_dim=6, state_dim=8, representation_dim=16, theano_seed=1234) sampling_representation = encoder.apply(sampling_input, theano.tensor.ones(sampling_input.shape)) generateds = decoder.generate(sampling_input, sampling_representation) model = Model(generateds[1]) # Initialize model encoder.weights_init = decoder.weights_init = IsotropicGaussian(0.01) encoder.biases_init = decoder.biases_init = Constant(0) encoder.push_initialization_config() decoder.push_initialization_config() encoder.bidir.prototype.weights_init = Orthogonal() decoder.transition.weights_init = Orthogonal() encoder.initialize() decoder.initialize() # Compile a function for the generated sampling_fn = model.get_theano_function() # Create literal variables numpy.random.seed(1234) x = numpy.random.randint(0, 10, size=(1, 2)) # Call function and check result generated_step = sampling_fn(x) assert len(generated_step[0].flatten()) == 4
def __init__(self, model_name, model, stream, **kwargs): super(RunOnTest, self).__init__(**kwargs) self.model_name = model_name cg = Model(model.predict(**stream.inputs())) self.inputs = cg.inputs self.outputs = model.predict.outputs req_vars_test = model.predict.inputs + ['trip_id'] self.test_stream = stream.test(req_vars_test) self.function = cg.get_theano_function() self.best_dvc = None self.best_tvc = None
def __init__(self, model_name, model, stream, **kwargs): super(RunOnTest, self).__init__(**kwargs) self.model_name = model_name cg = Model(model.predict(**stream.inputs())) self.inputs = cg.inputs self.outputs = model.predict.outputs req_vars_test = model.predict.inputs + ['trip_id'] self.test_stream = stream.test(req_vars_test) self.function = cg.get_theano_function()
def get_prediction_function(exp_config): # Create Theano variables logger.info('Creating theano variables') source_sentence = tensor.lmatrix('source') source_sentence_mask = tensor.matrix('source_mask') target_sentence = tensor.lmatrix('target_suffix') target_sentence_mask = tensor.matrix('target_suffix_mask') target_prefix = tensor.lmatrix('target_prefix') target_prefix_mask = tensor.matrix('target_prefix_mask') # build the model encoder = BidirectionalEncoder(exp_config['src_vocab_size'], exp_config['enc_embed'], exp_config['enc_nhids']) # Note: the 'min_risk' kwarg tells the decoder which sequence_generator and cost_function to use decoder = NMTPrefixDecoder(exp_config['trg_vocab_size'], exp_config['dec_embed'], exp_config['dec_nhids'], exp_config['enc_nhids'] * 2, loss_function='cross_entropy') # rename to match baseline NMT systems decoder.name = 'decoder' prediction_tags = decoder.prediction_tags( encoder.apply(source_sentence, source_sentence_mask), source_sentence_mask, target_sentence, target_sentence_mask, target_prefix, target_prefix_mask) logger.info('Creating computational graph') prediction_model = Model(prediction_tags) # Note that the parameters of this model must be pretrained, otherwise this doesn't make sense param_values = LoadNMT.load_parameter_values( exp_config['saved_parameters'], brick_delimiter=None) LoadNMT.set_model_parameters(prediction_model, param_values) prediction_function = prediction_model.get_theano_function() return prediction_function
print(cg.inputs) # Strangely, all the examples use : DataStreamMonitoring in MainLoop model = Model(labels) print("Model.dict_of_inputs():") print(model.dict_of_inputs()) print("Model list inputs:") print([v.name for v in model.inputs]) ## Model loading from saved file model.set_parameter_values(load_parameter_values(save_state_path)) examine_embedding(lookup.W.get_value()) label_ner = model.get_theano_function() print(model.inputs) print("printed label_ner.params") for test_data in data_stream.get_epoch_iterator(): ordered_batch = test_data[ 0:3] # Explicitly strip off the pre-defined labels #print(ordered_batch) results = label_ner(*ordered_batch) #print(results) # This is a pure array of labels inputs = _transpose(ordered_batch) for tokens, mask, labels in zip(inputs[0], inputs[1], np.transpose(results)): #print(labels)
print(cg.inputs) # Strangely, all the examples use : DataStreamMonitoring in MainLoop model = Model(labels) print("Model.dict_of_inputs():"); print(model.dict_of_inputs()) print("Model list inputs:"); print([ v.name for v in model.inputs]) ## Model loading from saved file model.set_parameter_values(load_parameter_values(save_state_path)) examine_embedding(lookup.W.get_value()) label_ner = model.get_theano_function() print(model.inputs) print("printed label_ner.params") for test_data in data_stream.get_epoch_iterator(): ordered_batch = test_data[0:3] # Explicitly strip off the pre-defined labels #print(ordered_batch) results = label_ner(*ordered_batch) #print(results) # This is a pure array of labels inputs = _transpose(ordered_batch) for tokens, mask, labels in zip(inputs[0], inputs[1], np.transpose(results)): #print(labels) for (token, m, label) in zip(tokens, mask, labels.tolist()): if m<0.1:
def get_confidence_function(exp_config): # Create Theano variables logger.info('Creating theano variables') source_sentence = tensor.lmatrix('source') source_sentence_mask = tensor.matrix('source_mask') target_sentence = tensor.lmatrix('target_suffix') target_sentence_mask = tensor.matrix('target_suffix_mask') target_prefix = tensor.lmatrix('target_prefix') target_prefix_mask = tensor.matrix('target_prefix_mask') logger.info('Creating computational graph') # build the model encoder = BidirectionalEncoder(exp_config['src_vocab_size'], exp_config['enc_embed'], exp_config['enc_nhids']) # Note: the 'min_risk' kwarg tells the decoder which sequence_generator and cost_function to use decoder = NMTPrefixDecoder(exp_config['trg_vocab_size'], exp_config['dec_embed'], exp_config['dec_nhids'], exp_config['enc_nhids'] * 2, loss_function='cross_entropy') # rename to match baseline NMT systems decoder.name = 'decoder' predictions, merged_states = decoder.prediction_tags( encoder.apply(source_sentence, source_sentence_mask), source_sentence_mask, target_sentence, target_sentence_mask, target_prefix, target_prefix_mask) # WORKING: also get the softmax prediction feature # WORKING: add features for source len, prefix len, position in suffix (position in suffix only makes sense if we're training on predictions) p_shape = predictions.shape predictions = predictions.reshape([p_shape[0] * p_shape[1], p_shape[2]]) prediction_softmax = tensor.nnet.nnet.softmax( predictions.reshape([p_shape[0] * p_shape[1], p_shape[2]])).reshape(p_shape) prediction_feature = prediction_softmax.max(axis=-1)[:, :, None] all_features = tensor.concatenate([merged_states, prediction_feature], axis=-1) confidence_output = decoder.sequence_generator.confidence_predictions( all_features) logger.info('Creating computational graph') confidence_model = Model(confidence_output) # Note that the parameters of this model must be pretrained, otherwise this doesn't make sense param_values = LoadNMT.load_parameter_values( exp_config['saved_parameters'], brick_delimiter=None) LoadNMT.set_model_parameters(confidence_model, param_values) confidence_param_values = LoadNMT.load_parameter_values( exp_config['confidence_saved_parameters'], brick_delimiter=None) LoadNMT.set_model_parameters(confidence_model, confidence_param_values) confidence_function = confidence_model.get_theano_function() return confidence_function
def main(name, model, epochs, batch_size, learning_rate, bokeh, layers, gamma, rectifier, predict, dropout, qlinear, sparse): runname = "vae%s-L%s%s%s%s-l%s-g%s-b%d" % (name, layers, 'r' if rectifier else '', 'd' if dropout else '', 'l' if qlinear else '', shnum(learning_rate), shnum(gamma), batch_size//100) if rectifier: activation = Rectifier() full_weights_init = Orthogonal() else: activation = Tanh() full_weights_init = Orthogonal() if sparse: runname += '-s%d'%sparse weights_init = Sparse(num_init=sparse, weights_init=full_weights_init) else: weights_init = full_weights_init layers = map(int,layers.split(',')) encoder_layers = layers[:-1] encoder_mlp = MLP([activation] * (len(encoder_layers)-1), encoder_layers, name="MLP_enc", biases_init=Constant(0.), weights_init=weights_init) enc_dim = encoder_layers[-1] z_dim = layers[-1] if qlinear: sampler = Qlinear(input_dim=enc_dim, output_dim=z_dim, biases_init=Constant(0.), weights_init=full_weights_init) else: sampler = Qsampler(input_dim=enc_dim, output_dim=z_dim, biases_init=Constant(0.), weights_init=full_weights_init) decoder_layers = layers[:] ## includes z_dim as first layer decoder_layers.reverse() decoder_mlp = MLP([activation] * (len(decoder_layers)-2) + [Logistic()], decoder_layers, name="MLP_dec", biases_init=Constant(0.), weights_init=weights_init) vae = VAEModel(encoder_mlp, sampler, decoder_mlp) vae.initialize() x = tensor.matrix('features')/256. x.tag.test_value = np.random.random((batch_size,layers[0])).astype(np.float32) if predict: mean_z, enc = vae.mean_z(x) # cg = ComputationGraph([mean_z, enc]) newmodel = Model([mean_z,enc]) else: x_recons, kl_terms = vae.reconstruct(x) recons_term = BinaryCrossEntropy().apply(x, x_recons) recons_term.name = "recons_term" cost = recons_term + kl_terms.mean() cg = ComputationGraph([cost]) if gamma > 0: weights = VariableFilter(roles=[WEIGHT])(cg.variables) cost += gamma * blocks.theano_expressions.l2_norm(weights) cost.name = "nll_bound" newmodel = Model(cost) if dropout: from blocks.roles import INPUT inputs = VariableFilter(roles=[INPUT])(cg.variables) # dropout_target = [v for k,v in newmodel.get_params().iteritems() # if k.find('MLP')>=0 and k.endswith('.W') and not k.endswith('MLP_enc/linear_0.W')] dropout_target = filter(lambda x: x.name.startswith('linear_'), inputs) cg = apply_dropout(cg, dropout_target, 0.5) target_cost = cg.outputs[0] else: target_cost = cost if name == 'mnist': if predict: train_ds = MNIST("train") else: train_ds = MNIST("train", sources=['features']) test_ds = MNIST("test") else: datasource_dir = os.path.join(fuel.config.data_path, name) datasource_fname = os.path.join(datasource_dir , name+'.hdf5') if predict: train_ds = H5PYDataset(datasource_fname, which_set='train') else: train_ds = H5PYDataset(datasource_fname, which_set='train', sources=['features']) test_ds = H5PYDataset(datasource_fname, which_set='test') train_s = Flatten(DataStream(train_ds, iteration_scheme=ShuffledScheme( train_ds.num_examples, batch_size))) test_s = Flatten(DataStream(test_ds, iteration_scheme=ShuffledScheme( test_ds.num_examples, batch_size))) if predict: from itertools import chain fprop = newmodel.get_theano_function() allpdata = None alledata = None f = train_s.sources.index('features') assert f == test_s.sources.index('features') sources = test_s.sources alllabels = dict((s,[]) for s in sources if s != 'features') for data in chain(train_s.get_epoch_iterator(), test_s.get_epoch_iterator()): for s,d in zip(sources,data): if s != 'features': alllabels[s].extend(list(d)) pdata, edata = fprop(data[f]) if allpdata is None: allpdata = pdata else: allpdata = np.vstack((allpdata, pdata)) if alledata is None: alledata = edata else: alledata = np.vstack((alledata, edata)) print 'Saving',allpdata.shape,'intermidiate layer, for all training and test examples, to',name+'_z.npy' np.save(name+'_z', allpdata) print 'Saving',alledata.shape,'last encoder layer to',name+'_e.npy' np.save(name+'_e', alledata) print 'Saving additional labels/targets:',','.join(alllabels.keys()), print ' of size',','.join(map(lambda x: str(len(x)),alllabels.values())), print 'to',name+'_labels.pkl' with open(name+'_labels.pkl','wb') as fp: pickle.dump(alllabels, fp, -1) else: cg = ComputationGraph([target_cost]) algorithm = GradientDescent( cost=target_cost, params=cg.parameters, step_rule=Adam(learning_rate) # Scale(learning_rate=learning_rate) ) extensions = [] if model: extensions.append(Load(model)) extensions += [Timing(), FinishAfter(after_n_epochs=epochs), DataStreamMonitoring( [cost, recons_term], test_s, prefix="test"), TrainingDataMonitoring( [cost, aggregation.mean(algorithm.total_gradient_norm)], prefix="train", after_epoch=True), Checkpoint(runname, every_n_epochs=10), Printing()] if bokeh: extensions.append(Plot( 'Auto', channels=[ ['test_recons_term','test_nll_bound','train_nll_bound' ], ['train_total_gradient_norm']])) main_loop = MainLoop( algorithm, train_s, model=newmodel, extensions=extensions) main_loop.run()
def main(mode, save_path, num_batches, data_path=None): # Experiment configuration dimension = 100 readout_dimension = len(char2code) # Build bricks encoder = Bidirectional(SimpleRecurrent(dim=dimension, activation=Tanh()), weights_init=Orthogonal()) fork = Fork( [name for name in encoder.prototype.apply.sequences if name != 'mask'], weights_init=IsotropicGaussian(0.1), biases_init=Constant(0)) fork.input_dim = dimension fork.output_dims = {name: dimension for name in fork.input_names} lookup = LookupTable(readout_dimension, dimension, weights_init=IsotropicGaussian(0.1)) transition = SimpleRecurrent(activation=Tanh(), dim=dimension, name="transition") attention = SequenceContentAttention(state_names=transition.apply.states, sequence_dim=2 * dimension, match_dim=dimension, name="attention") readout = LinearReadout(readout_dim=readout_dimension, source_names=["states"], emitter=SoftmaxEmitter(name="emitter"), feedbacker=LookupFeedback(readout_dimension, dimension), name="readout") generator = SequenceGenerator(readout=readout, transition=transition, attention=attention, weights_init=IsotropicGaussian(0.1), biases_init=Constant(0), name="generator") generator.push_initialization_config() transition.weights_init = Orthogonal() if mode == "train": # Data processing pipeline dataset_options = dict(dictionary=char2code, level="character", preprocess=_lower) if data_path: dataset = TextFile(data_path, **dataset_options) else: dataset = OneBillionWord("training", [99], **dataset_options) data_stream = DataStreamMapping( mapping=_transpose, data_stream=PaddingDataStream( BatchDataStream( iteration_scheme=ConstantScheme(10), data_stream=DataStreamMapping( mapping=reverse_words, add_sources=("targets", ), data_stream=DataStreamFilter( predicate=_filter_long, data_stream=dataset.get_default_stream()))))) # Build the cost computation graph chars = tensor.lmatrix("features") chars_mask = tensor.matrix("features_mask") targets = tensor.lmatrix("targets") targets_mask = tensor.matrix("targets_mask") batch_cost = generator.cost( targets, targets_mask, attended=encoder.apply(**dict_union(fork.apply( lookup.lookup(chars), return_dict=True), mask=chars_mask)), attended_mask=chars_mask).sum() batch_size = named_copy(chars.shape[1], "batch_size") cost = aggregation.mean(batch_cost, batch_size) cost.name = "sequence_log_likelihood" logger.info("Cost graph is built") # Give an idea of what's going on model = Model(cost) params = model.get_params() logger.info("Parameters:\n" + pprint.pformat([(key, value.get_value().shape) for key, value in params.items()], width=120)) # Initialize parameters for brick in model.get_top_bricks(): brick.initialize() # Fetch variables useful for debugging max_length = named_copy(chars.shape[0], "max_length") cost_per_character = named_copy( aggregation.mean(batch_cost, batch_size * max_length), "character_log_likelihood") cg = ComputationGraph(cost) (energies, ) = VariableFilter(application=readout.readout, name="output")(cg.variables) min_energy = named_copy(energies.min(), "min_energy") max_energy = named_copy(energies.max(), "max_energy") (activations, ) = VariableFilter( application=generator.transition.apply, name="states")(cg.variables) mean_activation = named_copy( abs(activations).mean(), "mean_activation") # Define the training algorithm. algorithm = GradientDescent(cost=cost, step_rule=CompositeRule( [StepClipping(10.0), Scale(0.01)])) # More variables for debugging observables = [ cost, min_energy, max_energy, mean_activation, batch_size, max_length, cost_per_character, algorithm.total_step_norm, algorithm.total_gradient_norm ] for name, param in params.items(): observables.append(named_copy(param.norm(2), name + "_norm")) observables.append( named_copy(algorithm.gradients[param].norm(2), name + "_grad_norm")) # Construct the main loop and start training! average_monitoring = TrainingDataMonitoring(observables, prefix="average", every_n_batches=10) main_loop = MainLoop( model=model, data_stream=data_stream, algorithm=algorithm, extensions=[ Timing(), TrainingDataMonitoring(observables, after_every_batch=True), average_monitoring, FinishAfter(after_n_batches=num_batches).add_condition( "after_batch", _is_nan), Plot(os.path.basename(save_path), [[average_monitoring.record_name(cost)], [average_monitoring.record_name(cost_per_character)]], every_n_batches=10), SerializeMainLoop(save_path, every_n_batches=500, save_separately=["model", "log"]), Printing(every_n_batches=1) ]) main_loop.run() elif mode == "test": logger.info("Model is loaded") chars = tensor.lmatrix("features") generated = generator.generate( n_steps=3 * chars.shape[0], batch_size=chars.shape[1], attended=encoder.apply(**dict_union( fork.apply(lookup.lookup(chars), return_dict=True))), attended_mask=tensor.ones(chars.shape)) model = Model(generated) model.set_param_values(load_parameter_values(save_path)) sample_function = model.get_theano_function() logging.info("Sampling function is compiled") while True: # Python 2-3 compatibility line = input("Enter a sentence\n") batch_size = int(input("Enter a number of samples\n")) encoded_input = [ char2code.get(char, char2code["<UNK>"]) for char in line.lower().strip() ] encoded_input = ([char2code['<S>']] + encoded_input + [char2code['</S>']]) print("Encoder input:", encoded_input) target = reverse_words((encoded_input, ))[0] print("Target: ", target) states, samples, glimpses, weights, costs = sample_function( numpy.repeat(numpy.array(encoded_input)[:, None], batch_size, axis=1)) messages = [] for i in range(samples.shape[1]): sample = list(samples[:, i]) try: true_length = sample.index(char2code['</S>']) + 1 except ValueError: true_length = len(sample) sample = sample[:true_length] cost = costs[:true_length, i].sum() message = "({})".format(cost) message += "".join(code2char[code] for code in sample) if sample == target: message += " CORRECT!" messages.append((cost, message)) messages.sort(key=operator.itemgetter(0), reverse=True) for _, message in messages: print(message)
def main(config, tr_stream, dev_stream, use_bokeh=False): print("~def main") # Create Theano variables logger.info('Creating theano variables') source_sentence = tensor.lmatrix('source') source_sentence_mask = tensor.matrix('source_mask') target_sentence = tensor.lmatrix('target') target_sentence_mask = tensor.matrix('target_mask') sampling_input = tensor.lmatrix('input') print("~sampling_input = tensor.lmatrix") # Construct model logger.info('Building RNN encoder-decoder') encoder = BidirectionalEncoder( config['src_vocab_size'], config['enc_embed'], config['enc_nhids']) decoder = Decoder( config['trg_vocab_size'], config['dec_embed'], config['dec_nhids'], config['enc_nhids'] * 2) cost = decoder.cost( encoder.apply(source_sentence, source_sentence_mask), source_sentence_mask, target_sentence, target_sentence_mask) print("~source_sentence_mask, target_sentence, target_sentence_mask") logger.info('Creating computational graph') cg = ComputationGraph(cost) print("~ComputationGraph") # Initialize model logger.info('Initializing model') encoder.weights_init = decoder.weights_init = IsotropicGaussian( config['weight_scale']) encoder.biases_init = decoder.biases_init = Constant(0) encoder.push_initialization_config() decoder.push_initialization_config() encoder.bidir.prototype.weights_init = Orthogonal() decoder.transition.weights_init = Orthogonal() encoder.initialize() decoder.initialize() print("~decoder.initialize()") # apply dropout for regularization if config['dropout'] < 1.0: # dropout is applied to the output of maxout in ghog logger.info('Applying dropout') dropout_inputs = [x for x in cg.intermediary_variables if x.name == 'maxout_apply_output'] cg = apply_dropout(cg, dropout_inputs, config['dropout']) print("~cg = apply_dropout") # Apply weight noise for regularization if config['weight_noise_ff'] > 0.0: logger.info('Applying weight noise to ff layers') enc_params = Selector(encoder.lookup).get_params().values() enc_params += Selector(encoder.fwd_fork).get_params().values() enc_params += Selector(encoder.back_fork).get_params().values() dec_params = Selector( decoder.sequence_generator.readout).get_params().values() dec_params += Selector( decoder.sequence_generator.fork).get_params().values() dec_params += Selector(decoder.state_init).get_params().values() cg = apply_noise(cg, enc_params+dec_params, config['weight_noise_ff']) print("~cg = apply_noise") # Print shapes shapes = [param.get_value().shape for param in cg.parameters] logger.info("Parameter shapes: ") for shape, count in Counter(shapes).most_common(): logger.info(' {:15}: {}'.format(shape, count)) logger.info("Total number of parameters: {}".format(len(shapes))) print("~logger.info") # Print parameter names enc_dec_param_dict = merge(Selector(encoder).get_parameters(), Selector(decoder).get_parameters()) logger.info("Parameter names: ") for name, value in enc_dec_param_dict.items(): logger.info(' {:15}: {}'.format(value.get_value().shape, name)) logger.info("Total number of parameters: {}" .format(len(enc_dec_param_dict))) # Set up training model logger.info("Building model") training_model = Model(cost) print("~training_model") # Set extensions logger.info("Initializing extensions") extensions = [ FinishAfter(after_n_batches=config['finish_after']), TrainingDataMonitoring([cost], after_batch=True), Printing(after_batch=True), CheckpointNMT(config['saveto'], every_n_batches=config['save_freq']) ] print("~every_n_batches=config") # Set up beam search and sampling computation graphs if necessary if config['hook_samples'] >= 1 or config['bleu_script'] is not None: logger.info("Building sampling model") sampling_representation = encoder.apply( sampling_input, tensor.ones(sampling_input.shape)) generated = decoder.generate(sampling_input, sampling_representation) search_model = Model(generated) _, samples = VariableFilter( bricks=[decoder.sequence_generator], name="outputs")( ComputationGraph(generated[1])) # generated[1] is next_outputs sample = Sampler(model=search_model, data_stream=tr_stream, hook_samples=config['hook_samples'], every_n_batches=config['sampling_freq'], src_vocab_size=config['src_vocab_size']) # Add sampling if config['hook_samples'] >= 1: logger.info("Building sampler") extensions.append( sample ) # Add early stopping based on bleu if config['bleu_script'] is not None: logger.info("Building bleu validator") extensions.append( BleuValidator(sampling_input, samples=samples, config=config, model=search_model, data_stream=dev_stream, normalize=config['normalized_bleu'], every_n_batches=config['bleu_val_freq'])) # Reload model if necessary if config['reload']: extensions.append(LoadNMT(config['saveto'])) # Plot cost in bokeh if necessary if use_bokeh and BOKEH_AVAILABLE: extensions.append( Plot('Cs-En', channels=[['decoder_cost_cost']], after_batch=True)) sampling_fn = search_model.get_theano_function() print(" - - - - - - - - - - - - - - " ) sort_k_batches = 12 batch_size = 80 seq_len = 50 trg_ivocab = None src_vocab_size = config['src_vocab_size'] trg_vocab_size = config['trg_vocab_size'] unk_id = config['unk_id'] src_vocab = config['src_vocab'] trg_vocab = config['trg_vocab'] src_vocab = ensure_special_tokens( src_vocab if isinstance(src_vocab, dict) else cPickle.load(open(src_vocab)), bos_idx=0, eos_idx=src_vocab_size - 1, unk_idx=unk_id) trg_vocab = ensure_special_tokens( trg_vocab if isinstance(trg_vocab, dict) else cPickle.load(open(trg_vocab)), bos_idx=0, eos_idx=trg_vocab_size - 1, unk_idx=unk_id) if not trg_ivocab: trg_ivocab = {v: k for k, v in trg_vocab.items()} src_data = config['src_data'] trg_data = config['trg_data'] src_dataset = TextFile([src_data], src_vocab, None) trg_dataset = TextFile([trg_data], trg_vocab, None) inputstringfile="inputstringfile.cs" input_dataset = TextFile([inputstringfile], src_vocab, None) stream = Merge([input_dataset.get_example_stream(), trg_dataset.get_example_stream()], ('source', 'target')) stream2 = Filter(stream, predicate=_too_long(seq_len=seq_len)) stream3 = Mapping(stream2, _oov_to_unk(src_vocab_size=src_vocab_size, trg_vocab_size=trg_vocab_size, unk_id=unk_id)) stream4 = Batch(stream3, iteration_scheme=ConstantScheme( batch_size*sort_k_batches)) stream5 = Mapping(stream4, SortMapping(_length)) stream6 = Unpack(stream5) stream7 = Batch( stream6, iteration_scheme=ConstantScheme(batch_size)) input_stream = DataStream(input_dataset) print("dev_stream : ", type( dev_stream ) ) print("input_stream : ", type( input_stream ) ) epochone = input_stream.get_epoch_iterator() vocab = input_stream.dataset.dictionary unk_sym = input_stream.dataset.unk_token eos_sym = input_stream.dataset.eos_token for i, line in enumerate(epochone): seq = oov_to_unk( line[0], config['src_vocab_size'], unk_id) input_ = numpy.tile(seq, ( 1 , 1)) print("seq : " , type( seq ) , seq ) print("input_ : ", type( input_ ) , input_ , inspect.getmembers( input_ ) ) _1, outputs, _2, _3, costs = ( sampling_fn( input_ ) ) outputs = outputs.flatten() costs = costs.T print(" outputs : " , outputs , type( outputs ) ) print("idx_to_word: ", idx_to_word(outputs , trg_ivocab)) print(" - - - - - - - - - - - - - - " )