def test_save_load_parameter_values(): param_values = [("/a/b", numpy.zeros(3)), ("/a/c", numpy.ones(4))] filename = tempfile.mkdtemp() + 'params.npz' save_parameter_values(dict(param_values), filename) loaded_values = sorted(list(load_parameter_values(filename).items()), key=lambda tuple_: tuple_[0]) assert len(loaded_values) == len(param_values) for old, new in zip(param_values, loaded_values): assert old[0] == new[0] assert numpy.all(old[1] == new[1])
def test_save_load_parameter_values(): param_values = [("/a/b", numpy.zeros(3)), ("/a/c", numpy.ones(4))] filename = tempfile.mkdtemp() + 'params.npz' save_parameter_values(dict(param_values), filename) loaded_values = sorted(list(load_parameter_values(filename).items()), key=lambda tuple_: tuple_[0]) assert len(loaded_values) == len(param_values) for old, new in equizip(param_values, loaded_values): assert old[0] == new[0] assert numpy.all(old[1] == new[1])
def train_model(cost, train_stream, valid_stream, valid_freq, valid_rare, load_location=None, save_location=None): cost.name = 'nll' perplexity = 2**(cost / tensor.log(2)) perplexity.name = 'ppl' # Define the model model = Model(cost) # Load the parameters from a dumped model if load_location is not None: logger.info('Loading parameters...') model.set_param_values(load_parameter_values(load_location)) cg = ComputationGraph(cost) algorithm = GradientDescent(cost=cost, step_rule=Scale(learning_rate=0.01), params=cg.parameters) main_loop = MainLoop( model=model, data_stream=train_stream, algorithm=algorithm, extensions=[ DataStreamMonitoring([cost, perplexity], valid_stream, prefix='valid_all', every_n_batches=5000), # Overfitting of rare words occurs between 3000 and 4000 iterations DataStreamMonitoring([cost, perplexity], valid_rare, prefix='valid_rare', every_n_batches=500), DataStreamMonitoring([cost, perplexity], valid_freq, prefix='valid_frequent', every_n_batches=5000), Printing(every_n_batches=500) ]) main_loop.run() # Save the main loop if save_location is not None: logger.info('Saving the main loop...') dump_manager = MainLoopDumpManager(save_location) dump_manager.dump(main_loop) logger.info('Saved')
def train_model(cost, error_rate, train_stream, load_location=None, save_location=None): cost.name = "Cross_entropy" error_rate.name = 'Error_rate' # Define the model model = Model(cost) # Load the parameters from a dumped model if load_location is not None: logger.info('Loading parameters...') model.set_param_values(load_parameter_values(load_location)) cg = ComputationGraph(cost) step_rule = Momentum(learning_rate=0.1, momentum=0.9) algorithm = GradientDescent(cost=cost, step_rule=step_rule, params=cg.parameters) main_loop = MainLoop( model=model, data_stream=train_stream, algorithm=algorithm, extensions=[ # DataStreamMonitoring([cost], test_stream, prefix='test', # after_epoch=False, every_n_epochs=10), DataStreamMonitoring([cost], train_stream, prefix='train', after_epoch=True), Printing(after_epoch=True) ]) main_loop.run() # Save the main loop if save_location is not None: logger.info('Saving the main loop...') dump_manager = MainLoopDumpManager(save_location) dump_manager.dump(main_loop) logger.info('Saved')
def train_model(cost, train_stream, valid_stream, valid_freq, valid_rare, load_location=None, save_location=None): cost.name = 'nll' perplexity = 2 ** (cost / tensor.log(2)) perplexity.name = 'ppl' # Define the model model = Model(cost) # Load the parameters from a dumped model if load_location is not None: logger.info('Loading parameters...') model.set_param_values(load_parameter_values(load_location)) cg = ComputationGraph(cost) algorithm = GradientDescent(cost=cost, step_rule=Scale(learning_rate=0.01), params=cg.parameters) main_loop = MainLoop( model=model, data_stream=train_stream, algorithm=algorithm, extensions=[ DataStreamMonitoring([cost, perplexity], valid_stream, prefix='valid_all', every_n_batches=5000), # Overfitting of rare words occurs between 3000 and 4000 iterations DataStreamMonitoring([cost, perplexity], valid_rare, prefix='valid_rare', every_n_batches=500), DataStreamMonitoring([cost, perplexity], valid_freq, prefix='valid_frequent', every_n_batches=5000), Printing(every_n_batches=500) ] ) main_loop.run() # Save the main loop if save_location is not None: logger.info('Saving the main loop...') dump_manager = MainLoopDumpManager(save_location) dump_manager.dump(main_loop) logger.info('Saved')
def train_model(cost, error_rate, train_stream, load_location=None, save_location=None): cost.name = "Cross_entropy" error_rate.name = 'Error_rate' # Define the model model = Model(cost) # Load the parameters from a dumped model if load_location is not None: logger.info('Loading parameters...') model.set_param_values(load_parameter_values(load_location)) cg = ComputationGraph(cost) step_rule = Momentum(learning_rate=0.1, momentum=0.9) algorithm = GradientDescent(cost=cost, step_rule=step_rule, params=cg.parameters) main_loop = MainLoop( model=model, data_stream=train_stream, algorithm=algorithm, extensions=[ # DataStreamMonitoring([cost], test_stream, prefix='test', # after_epoch=False, every_n_epochs=10), DataStreamMonitoring([cost], train_stream, prefix='train', after_epoch=True), Printing(after_epoch=True) ] ) main_loop.run() # Save the main loop if save_location is not None: logger.info('Saving the main loop...') dump_manager = MainLoopDumpManager(save_location) dump_manager.dump(main_loop) logger.info('Saved')
from fuel.streams import DataStream from fuel.schemes import ShuffledScheme import theano import theano.tensor as T import logging import numpy as np logging.basicConfig() m = VAModel() # load parameters model = Model(m.variational_cost) print "loading params" params = load_parameter_values(sys.argv[1]) model.set_param_values(params) test_dataset = CIFAR10('test', sources=['features']) test_scheme = ShuffledScheme(test_dataset.num_examples, 128) test_stream = DataStream(test_dataset, iteration_scheme=test_scheme) o = T.reshape(m.sampled, (m.sampled.shape[0], 3, 32, 32)) out = o.dimshuffle((0, 2, 3, 1)) _func_sample = theano.function([m.Z], out) #_func_noisy = theano.function([m.X], m.noisy) #_func_produced = theano.function([m.X], m.produced) #batch = test_stream.get_epoch_iterator().next()[0] #out_noise = _func_noisy(batch) #out_produced = _func_produced(batch)
from fuel.streams import DataStream from fuel.schemes import ShuffledScheme import theano import logging import numpy as np logging.basicConfig() m = VAModel() # load parameters model = Model(m.variational_cost) print "loading params" params = load_parameter_values(sys.argv[1]) model.set_param_values(params) test_dataset = MNIST("test", sources=["features"]) test_scheme = ShuffledScheme(test_dataset.num_examples, 128) test_stream = DataStream(test_dataset, iteration_scheme=test_scheme) _func_sample = theano.function([m.Z], m.sampled) # _func_noisy = theano.function([m.X], m.noisy) # _func_produced = theano.function([m.X], m.produced) # batch = test_stream.get_epoch_iterator().next()[0] # out_noise = _func_noisy(batch) # out_produced = _func_produced(batch) import cv2
def main(mode, save_path, num_batches, data_path=None): reverser = WordReverser(100, len(char2code), name="reverser") if mode == "train": # Data processing pipeline dataset_options = dict(dictionary=char2code, level="character", preprocess=_lower) if data_path: dataset = TextFile(data_path, **dataset_options) else: dataset = OneBillionWord("training", [99], **dataset_options) data_stream = dataset.get_example_stream() data_stream = Filter(data_stream, _filter_long) data_stream = Mapping(data_stream, reverse_words, add_sources=("targets",)) data_stream = Batch(data_stream, iteration_scheme=ConstantScheme(10)) data_stream = Padding(data_stream) data_stream = Mapping(data_stream, _transpose) # Initialization settings reverser.weights_init = IsotropicGaussian(0.1) reverser.biases_init = Constant(0.0) reverser.push_initialization_config() reverser.encoder.weghts_init = Orthogonal() reverser.generator.transition.weights_init = Orthogonal() # Build the cost computation graph chars = tensor.lmatrix("features") chars_mask = tensor.matrix("features_mask") targets = tensor.lmatrix("targets") targets_mask = tensor.matrix("targets_mask") batch_cost = reverser.cost( chars, chars_mask, targets, targets_mask).sum() batch_size = named_copy(chars.shape[1], "batch_size") cost = aggregation.mean(batch_cost, batch_size) cost.name = "sequence_log_likelihood" logger.info("Cost graph is built") # Give an idea of what's going on model = Model(cost) params = model.get_params() logger.info("Parameters:\n" + pprint.pformat( [(key, value.get_value().shape) for key, value in params.items()], width=120)) # Initialize parameters for brick in model.get_top_bricks(): brick.initialize() # Define the training algorithm. cg = ComputationGraph(cost) algorithm = GradientDescent( cost=cost, params=cg.parameters, step_rule=CompositeRule([StepClipping(10.0), Scale(0.01)])) # Fetch variables useful for debugging generator = reverser.generator (energies,) = VariableFilter( application=generator.readout.readout, name="output")(cg.variables) (activations,) = VariableFilter( application=generator.transition.apply, name=generator.transition.apply.states[0])(cg.variables) max_length = named_copy(chars.shape[0], "max_length") cost_per_character = named_copy( aggregation.mean(batch_cost, batch_size * max_length), "character_log_likelihood") min_energy = named_copy(energies.min(), "min_energy") max_energy = named_copy(energies.max(), "max_energy") mean_activation = named_copy(abs(activations).mean(), "mean_activation") observables = [ cost, min_energy, max_energy, mean_activation, batch_size, max_length, cost_per_character, algorithm.total_step_norm, algorithm.total_gradient_norm] for name, param in params.items(): observables.append(named_copy( param.norm(2), name + "_norm")) observables.append(named_copy( algorithm.gradients[param].norm(2), name + "_grad_norm")) # Construct the main loop and start training! average_monitoring = TrainingDataMonitoring( observables, prefix="average", every_n_batches=10) main_loop = MainLoop( model=model, data_stream=data_stream, algorithm=algorithm, extensions=[ Timing(), TrainingDataMonitoring(observables, after_batch=True), average_monitoring, FinishAfter(after_n_batches=num_batches) # This shows a way to handle NaN emerging during # training: simply finish it. .add_condition("after_batch", _is_nan), Plot(os.path.basename(save_path), [[average_monitoring.record_name(cost)], [average_monitoring.record_name(cost_per_character)]], every_n_batches=10), # Saving the model and the log separately is convenient, # because loading the whole pickle takes quite some time. Checkpoint(save_path, every_n_batches=500, save_separately=["model", "log"]), Printing(every_n_batches=1)]) main_loop.run() elif mode == "sample" or mode == "beam_search": chars = tensor.lmatrix("input") generated = reverser.generate(chars) model = Model(generated) logger.info("Loading the model..") model.set_param_values(load_parameter_values(save_path)) def generate(input_): """Generate output sequences for an input sequence. Incapsulates most of the difference between sampling and beam search. Returns ------- outputs : list of lists Trimmed output sequences. costs : list The negative log-likelihood of generating the respective sequences. """ if mode == "beam_search": samples, = VariableFilter( bricks=[reverser.generator], name="outputs")( ComputationGraph(generated[1])) # NOTE: this will recompile beam search functions # every time user presses Enter. Do not create # a new `BeamSearch` object every time if # speed is important for you. beam_search = BeamSearch(input_.shape[1], samples) outputs, costs = beam_search.search( {chars: input_}, char2code['</S>'], 3 * input_.shape[0]) else: _1, outputs, _2, _3, costs = ( model.get_theano_function()(input_)) outputs = list(outputs.T) costs = list(costs.T) for i in range(len(outputs)): outputs[i] = list(outputs[i]) try: true_length = outputs[i].index(char2code['</S>']) + 1 except ValueError: true_length = len(outputs[i]) outputs[i] = outputs[i][:true_length] costs[i] = costs[i][:true_length].sum() return outputs, costs while True: line = input("Enter a sentence\n") message = ("Enter the number of samples\n" if mode == "sample" else "Enter the beam size\n") batch_size = int(input(message)) encoded_input = [char2code.get(char, char2code["<UNK>"]) for char in line.lower().strip()] encoded_input = ([char2code['<S>']] + encoded_input + [char2code['</S>']]) print("Encoder input:", encoded_input) target = reverse_words((encoded_input,))[0] print("Target: ", target) samples, costs = generate( numpy.repeat(numpy.array(encoded_input)[:, None], batch_size, axis=1)) messages = [] for sample, cost in equizip(samples, costs): message = "({})".format(cost) message += "".join(code2char[code] for code in sample) if sample == target: message += " CORRECT!" messages.append((cost, message)) messages.sort(key=operator.itemgetter(0), reverse=True) for _, message in messages: print(message)
def main(mode, save_path, num_batches, data_path=None): reverser = WordReverser(100, len(char2code), name="reverser") if mode == "train": # Data processing pipeline dataset_options = dict(dictionary=char2code, level="character", preprocess=_lower) if data_path: dataset = TextFile(data_path, **dataset_options) else: dataset = OneBillionWord("training", [99], **dataset_options) data_stream = dataset.get_example_stream() data_stream = Filter(data_stream, _filter_long) data_stream = Mapping(data_stream, reverse_words, add_sources=("targets",)) data_stream = Batch(data_stream, iteration_scheme=ConstantScheme(10)) data_stream = Padding(data_stream) data_stream = Mapping(data_stream, _transpose) # Initialization settings reverser.weights_init = IsotropicGaussian(0.1) reverser.biases_init = Constant(0.0) reverser.push_initialization_config() reverser.encoder.weights_init = Orthogonal() reverser.generator.transition.weights_init = Orthogonal() # Build the cost computation graph chars = tensor.lmatrix("features") chars_mask = tensor.matrix("features_mask") targets = tensor.lmatrix("targets") targets_mask = tensor.matrix("targets_mask") batch_cost = reverser.cost( chars, chars_mask, targets, targets_mask).sum() batch_size = named_copy(chars.shape[1], "batch_size") cost = aggregation.mean(batch_cost, batch_size) cost.name = "sequence_log_likelihood" logger.info("Cost graph is built") # Give an idea of what's going on model = Model(cost) params = model.get_params() logger.info("Parameters:\n" + pprint.pformat( [(key, value.get_value().shape) for key, value in params.items()], width=120)) # Initialize parameters for brick in model.get_top_bricks(): brick.initialize() # Define the training algorithm. cg = ComputationGraph(cost) algorithm = GradientDescent( cost=cost, params=cg.parameters, step_rule=CompositeRule([StepClipping(10.0), Scale(0.01)])) # Fetch variables useful for debugging generator = reverser.generator (energies,) = VariableFilter( applications=[generator.readout.readout], name_regex="output")(cg.variables) (activations,) = VariableFilter( applications=[generator.transition.apply], name=generator.transition.apply.states[0])(cg.variables) max_length = named_copy(chars.shape[0], "max_length") cost_per_character = named_copy( aggregation.mean(batch_cost, batch_size * max_length), "character_log_likelihood") min_energy = named_copy(energies.min(), "min_energy") max_energy = named_copy(energies.max(), "max_energy") mean_activation = named_copy(abs(activations).mean(), "mean_activation") observables = [ cost, min_energy, max_energy, mean_activation, batch_size, max_length, cost_per_character, algorithm.total_step_norm, algorithm.total_gradient_norm] for name, param in params.items(): observables.append(named_copy( param.norm(2), name + "_norm")) observables.append(named_copy( algorithm.gradients[param].norm(2), name + "_grad_norm")) # Construct the main loop and start training! average_monitoring = TrainingDataMonitoring( observables, prefix="average", every_n_batches=10) main_loop = MainLoop( model=model, data_stream=data_stream, algorithm=algorithm, extensions=[ Timing(), TrainingDataMonitoring(observables, after_batch=True), average_monitoring, FinishAfter(after_n_batches=num_batches) # This shows a way to handle NaN emerging during # training: simply finish it. .add_condition("after_batch", _is_nan), Plot(os.path.basename(save_path), [[average_monitoring.record_name(cost)], [average_monitoring.record_name(cost_per_character)]], every_n_batches=10), # Saving the model and the log separately is convenient, # because loading the whole pickle takes quite some time. Checkpoint(save_path, every_n_batches=500, save_separately=["model", "log"]), Printing(every_n_batches=1)]) main_loop.run() elif mode == "sample" or mode == "beam_search": chars = tensor.lmatrix("input") generated = reverser.generate(chars) model = Model(generated) logger.info("Loading the model..") model.set_param_values(load_parameter_values(save_path)) def generate(input_): """Generate output sequences for an input sequence. Incapsulates most of the difference between sampling and beam search. Returns ------- outputs : list of lists Trimmed output sequences. costs : list The negative log-likelihood of generating the respective sequences. """ if mode == "beam_search": samples, = VariableFilter( bricks=[reverser.generator], name="outputs")( ComputationGraph(generated[1])) # NOTE: this will recompile beam search functions # every time user presses Enter. Do not create # a new `BeamSearch` object every time if # speed is important for you. beam_search = BeamSearch(input_.shape[1], samples) outputs, costs = beam_search.search( {chars: input_}, char2code['</S>'], 3 * input_.shape[0]) else: _1, outputs, _2, _3, costs = ( model.get_theano_function()(input_)) outputs = list(outputs.T) costs = list(costs.T) for i in range(len(outputs)): outputs[i] = list(outputs[i]) try: true_length = outputs[i].index(char2code['</S>']) + 1 except ValueError: true_length = len(outputs[i]) outputs[i] = outputs[i][:true_length] costs[i] = costs[i][:true_length].sum() return outputs, costs while True: line = input("Enter a sentence\n") message = ("Enter the number of samples\n" if mode == "sample" else "Enter the beam size\n") batch_size = int(input(message)) encoded_input = [char2code.get(char, char2code["<UNK>"]) for char in line.lower().strip()] encoded_input = ([char2code['<S>']] + encoded_input + [char2code['</S>']]) print("Encoder input:", encoded_input) target = reverse_words((encoded_input,))[0] print("Target: ", target) samples, costs = generate( numpy.repeat(numpy.array(encoded_input)[:, None], batch_size, axis=1)) messages = [] for sample, cost in equizip(samples, costs): message = "({})".format(cost) message += "".join(code2char[code] for code in sample) if sample == target: message += " CORRECT!" messages.append((cost, message)) messages.sort(key=operator.itemgetter(0), reverse=True) for _, message in messages: print(message)
parser.add_argument('--rnnrbm', action='store_true', help='Rnnrbm params') parser.add_argument('--train', action='store_true', help='Rnnrbm params') parser.add_argument('--bokeh', action='store_true', help='Rnnrbm params') parser.add_argument('--model', type=str, help='Rnnrbm params') parser.add_argument('--save', action='store_true', help='Rnnrbm params') rbm_epochs, rnn_epochs, rnnrbm_epochs = 1000, 600, 500 args = parser.parse_args() rnnrbm = Rnnrbm(88, 256, (350, 250), name='rnnrbm') rnnrbm.allocate() rnnrbm.initialize() params = OrderedDict() if args.model: params = load_parameter_values(args.model) newdir = datetime.now().isoformat().replace(':', '-') def run_main(main_loop, params=None): if bool(params): print "setting up params" main_loop.model.set_param_values(params) main_loop.run() params.update(main_loop.model.get_param_values()) for key, value in dict(params).iteritems(): if key in pre_training_params: new_key = pre_training_params[key] params[new_key] = params[key] return params datasets = ('midi', 'nottingham', 'muse', 'jsb')
def main(mode, save_path, num_batches, data_path=None): # Experiment configuration dimension = 100 readout_dimension = len(char2code) # Build bricks encoder = Bidirectional(SimpleRecurrent(dim=dimension, activation=Tanh()), weights_init=Orthogonal()) fork = Fork( [name for name in encoder.prototype.apply.sequences if name != 'mask'], weights_init=IsotropicGaussian(0.1), biases_init=Constant(0)) fork.input_dim = dimension fork.output_dims = {name: dimension for name in fork.input_names} lookup = LookupTable(readout_dimension, dimension, weights_init=IsotropicGaussian(0.1)) transition = SimpleRecurrent(activation=Tanh(), dim=dimension, name="transition") attention = SequenceContentAttention(state_names=transition.apply.states, sequence_dim=2 * dimension, match_dim=dimension, name="attention") readout = LinearReadout(readout_dim=readout_dimension, source_names=["states"], emitter=SoftmaxEmitter(name="emitter"), feedbacker=LookupFeedback(readout_dimension, dimension), name="readout") generator = SequenceGenerator(readout=readout, transition=transition, attention=attention, weights_init=IsotropicGaussian(0.1), biases_init=Constant(0), name="generator") generator.push_initialization_config() transition.weights_init = Orthogonal() if mode == "train": # Data processing pipeline dataset_options = dict(dictionary=char2code, level="character", preprocess=_lower) if data_path: dataset = TextFile(data_path, **dataset_options) else: dataset = OneBillionWord("training", [99], **dataset_options) data_stream = DataStreamMapping( mapping=_transpose, data_stream=PaddingDataStream( BatchDataStream( iteration_scheme=ConstantScheme(10), data_stream=DataStreamMapping( mapping=reverse_words, add_sources=("targets", ), data_stream=DataStreamFilter( predicate=_filter_long, data_stream=dataset.get_default_stream()))))) # Build the cost computation graph chars = tensor.lmatrix("features") chars_mask = tensor.matrix("features_mask") targets = tensor.lmatrix("targets") targets_mask = tensor.matrix("targets_mask") batch_cost = generator.cost( targets, targets_mask, attended=encoder.apply(**dict_union(fork.apply( lookup.lookup(chars), return_dict=True), mask=chars_mask)), attended_mask=chars_mask).sum() batch_size = named_copy(chars.shape[1], "batch_size") cost = aggregation.mean(batch_cost, batch_size) cost.name = "sequence_log_likelihood" logger.info("Cost graph is built") # Give an idea of what's going on model = Model(cost) params = model.get_params() logger.info("Parameters:\n" + pprint.pformat([(key, value.get_value().shape) for key, value in params.items()], width=120)) # Initialize parameters for brick in model.get_top_bricks(): brick.initialize() # Fetch variables useful for debugging max_length = named_copy(chars.shape[0], "max_length") cost_per_character = named_copy( aggregation.mean(batch_cost, batch_size * max_length), "character_log_likelihood") cg = ComputationGraph(cost) (energies, ) = VariableFilter(application=readout.readout, name="output")(cg.variables) min_energy = named_copy(energies.min(), "min_energy") max_energy = named_copy(energies.max(), "max_energy") (activations, ) = VariableFilter( application=generator.transition.apply, name="states")(cg.variables) mean_activation = named_copy( abs(activations).mean(), "mean_activation") # Define the training algorithm. algorithm = GradientDescent(cost=cost, step_rule=CompositeRule( [StepClipping(10.0), Scale(0.01)])) # More variables for debugging observables = [ cost, min_energy, max_energy, mean_activation, batch_size, max_length, cost_per_character, algorithm.total_step_norm, algorithm.total_gradient_norm ] for name, param in params.items(): observables.append(named_copy(param.norm(2), name + "_norm")) observables.append( named_copy(algorithm.gradients[param].norm(2), name + "_grad_norm")) # Construct the main loop and start training! average_monitoring = TrainingDataMonitoring(observables, prefix="average", every_n_batches=10) main_loop = MainLoop( model=model, data_stream=data_stream, algorithm=algorithm, extensions=[ Timing(), TrainingDataMonitoring(observables, after_every_batch=True), average_monitoring, FinishAfter(after_n_batches=num_batches).add_condition( "after_batch", _is_nan), Plot(os.path.basename(save_path), [[average_monitoring.record_name(cost)], [average_monitoring.record_name(cost_per_character)]], every_n_batches=10), SerializeMainLoop(save_path, every_n_batches=500, save_separately=["model", "log"]), Printing(every_n_batches=1) ]) main_loop.run() elif mode == "test": logger.info("Model is loaded") chars = tensor.lmatrix("features") generated = generator.generate( n_steps=3 * chars.shape[0], batch_size=chars.shape[1], attended=encoder.apply(**dict_union( fork.apply(lookup.lookup(chars), return_dict=True))), attended_mask=tensor.ones(chars.shape)) model = Model(generated) model.set_param_values(load_parameter_values(save_path)) sample_function = model.get_theano_function() logging.info("Sampling function is compiled") while True: # Python 2-3 compatibility line = input("Enter a sentence\n") batch_size = int(input("Enter a number of samples\n")) encoded_input = [ char2code.get(char, char2code["<UNK>"]) for char in line.lower().strip() ] encoded_input = ([char2code['<S>']] + encoded_input + [char2code['</S>']]) print("Encoder input:", encoded_input) target = reverse_words((encoded_input, ))[0] print("Target: ", target) states, samples, glimpses, weights, costs = sample_function( numpy.repeat(numpy.array(encoded_input)[:, None], batch_size, axis=1)) messages = [] for i in range(samples.shape[1]): sample = list(samples[:, i]) try: true_length = sample.index(char2code['</S>']) + 1 except ValueError: true_length = len(sample) sample = sample[:true_length] cost = costs[:true_length, i].sum() message = "({})".format(cost) message += "".join(code2char[code] for code in sample) if sample == target: message += " CORRECT!" messages.append((cost, message)) messages.sort(key=operator.itemgetter(0), reverse=True) for _, message in messages: print(message)
def main(save, load, sample, path, **kwargs): input_dim = 784 hidden_dim = 2 batch_size = 100 features = tensor.matrix('features') vae = VariationalAutoEncoder(input_dim, hidden_dim, weights_init=IsotropicGaussian(0.01), biases_init=Constant(0.)) vae.initialize() mu, logsigma, x_hat = vae.apply(features) cost = vae.cost(features) cost.name = 'cost' regularization_cost = vae.regularization_cost(mu, logsigma).mean() regularization_cost.name = 'regularization_cost' reconstruction_cost = vae.reconstruction_cost(features, x_hat).mean() reconstruction_cost.name = 'reconstruction_cost' cg = ComputationGraph([cost, reconstruction_cost, regularization_cost]) model = Model(cost) algorithm = GradientDescent(step_rule=RMSProp(1e-4), params=cg.parameters, cost=cost) extensions = [] if load: extensions.append(LoadFromDump(path)) if save: extensions.append(Dump(path, after_epoch=True)) extensions.append(FinishAfter(after_n_epochs=6001)) train_dataset = MNIST('train', binary=False, sources=('features',)) train_stream = DataStream(train_dataset, iteration_scheme=ShuffledScheme( examples=train_dataset.num_examples, batch_size=batch_size)) train_monitor = TrainingDataMonitoring( [cost, regularization_cost, reconstruction_cost], prefix='train', after_epoch=True) test_dataset = MNIST('test', binary=True, sources=('features',)) test_stream = DataStream(test_dataset, iteration_scheme=ShuffledScheme( examples=test_dataset.num_examples, batch_size=batch_size)) test_monitor = DataStreamMonitoring([cost], test_stream, prefix='test') extensions.extend([train_monitor, test_monitor]) extensions.extend([Timing(), Printing()]) main_loop = MainLoop(model=model, algorithm=algorithm, data_stream=train_stream, extensions=extensions) if not sample: main_loop.run() else: parameters = load_parameter_values(path + '/params.npz') model.set_param_values(parameters) num_samples = 10 samples = vae.sample(num_samples) samples = function([], samples)() z = tensor.matrix('z') decode_z = function([z], vae.decoder.apply(z)) from matplotlib import pyplot as plt sample = numpy.zeros((28, 0)) size = 40 z_val = numpy.zeros((size ** 2, 2)) for i in xrange(size): for j in xrange(size): z_val[i * size + j, :] = numpy.array( [i / float(0.3 * size) - .5 / .3, j / float(0.3 * size) - .5 / .3]) samples = decode_z(z_val) samples = samples.reshape((size, size, 28, 28)) samples = numpy.concatenate(samples, axis=1) samples = numpy.concatenate(samples, axis=1) plt.imshow(samples, cmap=plt.get_cmap('Greys')) plt.show() f = function([features], x_hat) for data in train_stream.get_epoch_iterator(): data_hat = f(data[0]) for image, image_hat in zip(data[0], data_hat): im = numpy.concatenate([image_hat.reshape((28, 28)), image.reshape((28, 28))]) plt.imshow(im, cmap=plt.get_cmap('Greys')) plt.show()
parser.add_argument('--train', action='store_true', help='Rnnrbm params') parser.add_argument('--bokeh', action='store_true', help='Rnnrbm params') parser.add_argument('--model', type=str, help='Rnnrbm params') parser.add_argument('--save', action='store_true', help='Rnnrbm params') rbm_epochs, rnn_epochs, rnnrbm_epochs = 1000, 600, 500 args = parser.parse_args() rnnrbm = Rnnrbm(88, 256, (350, 250), name='rnnrbm') rnnrbm.allocate() rnnrbm.initialize() params = OrderedDict() if args.model: params = load_parameter_values(args.model) newdir = datetime.now().isoformat().replace(':', '-') def run_main(main_loop, params=None): if bool(params): print "setting up params" main_loop.model.set_param_values(params) main_loop.run() params.update(main_loop.model.get_param_values()) for key, value in dict(params).iteritems(): if key in pre_training_params: new_key = pre_training_params[key] params[new_key] = params[key] return params datasets = ('midi', 'nottingham', 'muse', 'jsb')
import numpy as np from blocks.dump import load_parameter_values from blocks.graph import ComputationGraph from blocks.model import Model from rnn import construct_model cost, error = construct_model(101, 2) model = Model(cost) model.set_param_values(load_parameter_values("trained_rnn_classic/params.npz")) print(ComputationGraph(cost).parameters) # wh = ComputationGraph(cost).parameters[0].get_value() # w = ComputationGraph(cost).parameters[1].get_value() # w_lookup = ComputationGraph(cost).parameters[3].get_value() # eig = np.zeros((unit, 0)) # for i in range(module): # whi = wh[unit * i:unit * (i + 1), unit * i:unit * (i + 1)] # eigi = np.linalg.eig(whi)[0] # eigi = np.sort(np.absolute(eigi.reshape((unit, 1))), axis=0)[::-1] # eig = np.concatenate((eig, eigi), axis=1) # print(eig) # matshow(eig, cmap=cm.gray) # matshow(wh, cmap=cm.gray) # show() # return