class NoiseExtension(SimpleExtension, RNGMixin): def __init__(self, noise_parameters=None, **kwargs): kwargs.setdefault("before_training", True) kwargs.setdefault("after_training", True) self.noise_parameters = noise_parameters std = 1.0 self.noise_init = IsotropicGaussian(std=std) theano_seed = self.rng.randint(np.iinfo(np.int32).max) self.theano_generator = UnitNoiseGenerator(std=std, theano_seed=theano_seed) self.noise_updates = OrderedDict([(param, self.theano_generator.apply(param)) for param in self.noise_parameters]) super(NoiseExtension, self).__init__(**kwargs) def do(self, callback_name, *args): self.parse_args(callback_name, args) if callback_name == 'before_training': # Before training, intiaizlize noise for p in self.noise_parameters: self.noise_init.initialize(p, self.rng) # And set up update to change noise on every update self.main_loop.algorithm.add_updates(self.noise_updates) if callback_name == 'after_training': # After training, zero noise again. for p in self.noise_parameters: v = p.get_value() p.set_value(np.zeros(v.shape, dtype=v.dtype))
class NoiseExtension(SimpleExtension, RNGMixin): def __init__(self, noise_parameters=None, **kwargs): kwargs.setdefault("before_training", True) kwargs.setdefault("after_training", True) self.noise_parameters = noise_parameters std = 1.0 self.noise_init = IsotropicGaussian(std=std) theano_seed = self.rng.randint(np.iinfo(np.int32).max) self.theano_generator = UnitNoiseGenerator( std=std, theano_seed=theano_seed) self.noise_updates = OrderedDict( [(param, self.theano_generator.apply(param)) for param in self.noise_parameters]) super(NoiseExtension, self).__init__(**kwargs) def do(self, callback_name, *args): self.parse_args(callback_name, args) if callback_name == 'before_training': # Before training, intiaizlize noise for p in self.noise_parameters: self.noise_init.initialize(p, self.rng) # And set up update to change noise on every update self.main_loop.algorithm.add_updates(self.noise_updates) if callback_name == 'after_training': # After training, zero noise again. for p in self.noise_parameters: v = p.get_value() p.set_value(np.zeros(v.shape, dtype=v.dtype))
if __name__ == "__main__": theano.config.optimizer='None' import numpy as np from blocks.initialization import IsotropicGaussian, Constant x = tensor.tensor3("inp_variable") #x = tensor.tensor3("inp_variable") n_hid = 512 n_in = 512 np.random.seed(1) rng = np.random init = IsotropicGaussian(0.02) #init = Constant(0.00) inp_to_state = shared_floatx_zeros((n_in, n_hid)) init.initialize(inp_to_state, rng) inp_to_update = shared_floatx_zeros((n_in, n_hid)) init.initialize(inp_to_update, rng) inp_to_reset = shared_floatx_zeros((n_in, n_hid)) init.initialize(inp_to_reset, rng) inp_to_state_b = shared_floatx_zeros((n_hid,)) init.initialize(inp_to_state_b, rng) inp_to_update_b = shared_floatx_zeros((n_hid,)) init.initialize(inp_to_update_b, rng) inp_to_reset_b = shared_floatx_zeros((n_hid,)) init.initialize(inp_to_reset_b, rng) state_to_state = shared_floatx_zeros((n_hid, n_hid)) init.initialize(state_to_state, rng) state_to_update = shared_floatx_zeros((n_hid, n_hid))