class Embedder(Initializable): """ Linear Embedding Brick Parameters ---------- dim_in: :class:`int` Dimensionality of the input dim_out: :class:`int` Dimensionality of the output output_type: :class:`str` fc for fully connected. conv for convolutional """ def __init__(self, dim_in, dim_out, output_type='fc', **kwargs): self.dim_in = dim_in self.dim_out = dim_out self.output_type = output_type self.linear = Linear(dim_in, dim_out, name='embed_layer') children = [self.linear] kwargs.setdefault('children', []).extend(children) super(Embedder, self).__init__(**kwargs) @application(inputs=['y'], outputs=['outputs']) def apply(self, y): embedding = self.linear.apply(y) if self.output_type == 'fc': return embedding if self.output_type == 'conv': return embedding.reshape((-1, embedding.shape[-1], 1, 1)) def get_dim(self, name): if self.output_type == 'fc': return self.linear.get_dim(name) if self.output_type == 'conv': return (self.linear.get_dim(name), 1, 1)
class Qlinear(Initializable): """ brick to handle the intermediate layer of an Autoencoder. In this brick a simple linear mix is performed (a kind of PCA.) """ def __init__(self, input_dim, output_dim, **kwargs): super(Qlinear, self).__init__(**kwargs) self.mean_transform = Linear( name=self.name+'_mean', input_dim=input_dim, output_dim=output_dim, weights_init=self.weights_init, biases_init=self.biases_init, use_bias=True) self.children = [self.mean_transform] def get_dim(self, name): if name == 'input': return self.mean_transform.get_dim('input') elif name == 'output': return self.mean_transform.get_dim('output') else: raise ValueError @application(inputs=['x'], outputs=['z', 'kl_term']) def sample(self, x): """Sampling is trivial in this case """ mean = self.mean_transform.apply(x) z = mean # Calculate KL batch_size = x.shape[0] kl = T.zeros((batch_size,),dtype=floatX) return z, kl @application(inputs=['x'], outputs=['z']) def mean_z(self, x): return self.mean_transform.apply(x)
class Qsampler(Initializable, Random): def __init__(self, input_dim, output_dim, **kwargs): super(Qsampler, self).__init__(**kwargs) self.prior_mean = 0. self.prior_log_sigma = 0. self.mean_transform = Linear( name=self.name+'_mean', input_dim=input_dim, output_dim=output_dim, weights_init=self.weights_init, biases_init=self.biases_init, use_bias=True) self.log_sigma_transform = Linear( name=self.name+'_log_sigma', input_dim=input_dim, output_dim=output_dim, weights_init=self.weights_init, biases_init=self.biases_init, use_bias=True) self.children = [self.mean_transform, self.log_sigma_transform] def get_dim(self, name): if name == 'input': return self.mean_transform.get_dim('input') elif name == 'output': return self.mean_transform.get_dim('output') else: raise ValueError @application(inputs=['x', 'u'], outputs=['z', 'kl_term']) def sample(self, x, u): """Return a samples and the corresponding KL term Parameters ---------- x : Returns ------- z : tensor.matrix Samples drawn from Q(z|x) kl : tensor.vector KL(Q(z|x) || P_z) """ mean = self.mean_transform.apply(x) log_sigma = self.log_sigma_transform.apply(x) # Sample from mean-zeros std.-one Gaussian #u = self.theano_rng.normal( # size=mean.shape, # avg=0., std=1.) # ... and scale/translate samples z = mean + tensor.exp(log_sigma) * u # Calculate KL kl = ( self.prior_log_sigma - log_sigma + 0.5 * ( tensor.exp(2 * log_sigma) + (mean - self.prior_mean) ** 2 ) / tensor.exp(2 * self.prior_log_sigma) - 0.5 ).sum(axis=-1) return z, kl #@application(inputs=['n_samples']) @application(inputs=['u'], outputs=['z_prior']) def sample_from_prior(self, u): """Sample z from the prior distribution P_z. Parameters ---------- u : tensor.matrix gaussian random source Returns ------- z : tensor.matrix samples """ z_dim = self.mean_transform.get_dim('output') # Sample from mean-zeros std.-one Gaussian #u = self.theano_rng.normal( # size=(n_samples, z_dim), # avg=0., std=1.) # ... and scale/translate samples z = self.prior_mean + tensor.exp(self.prior_log_sigma) * u #z.name("z_prior") return z
class NoisyLinear(Initializable, Feedforward, Random): """Linear transformation sent through a learned noisy channel. Parameters ---------- input_dim : int The dimension of the input. Required by :meth:`~.Brick.allocate`. output_dim : int The dimension of the output. Required by :meth:`~.Brick.allocate`. num_pieces : int The number of linear functions. Required by :meth:`~.Brick.allocate`. """ @lazy(allocation=['input_dim', 'output_dim', 'noise_batch_size']) def __init__(self, input_dim, output_dim, noise_batch_size, prior_mean=0, prior_noise_level=0, **kwargs): self.linear = Linear() self.mask = Linear(name='mask') children = [self.linear, self.mask] kwargs.setdefault('children', []).extend(children) super(NoisyLinear, self).__init__(**kwargs) self.input_dim = input_dim self.output_dim = output_dim self.noise_batch_size = noise_batch_size self.prior_mean = prior_mean self.prior_noise_level = prior_noise_level def _push_allocation_config(self): self.linear.input_dim = self.input_dim self.linear.output_dim = self.output_dim self.mask.input_dim = self.output_dim self.mask.output_dim = self.output_dim def _allocate(self): N = shared_floatx_zeros((self.noise_batch_size, self.output_dim), name='N') add_role(N, NOISE) self.parameters.append(N) @application(inputs=['input_'], outputs=['output']) def apply(self, input_, application_call): """Apply the linear transformation followed by masking with noise. Parameters ---------- input_ : :class:`~tensor.TensorVariable` The input on which to apply the transformations Returns ------- output : :class:`~tensor.TensorVariable` The transformed input """ pre_noise = self.linear.apply(input_) noise_level = (self.prior_noise_level - tensor.clip(self.mask.apply(pre_noise), -16, 16)) noise_level = copy_and_tag_noise(noise_level, self, LOG_SIGMA, 'log_sigma') # Allow incomplete batches by just taking the noise that is needed # noise = Print('noise')(self.parameters[0][:noise_level.shape[0], :]) noise = self.parameters[0][:noise_level.shape[0], :] # noise = Print('noise')(self.theano_rng.normal(noise_level.shape)) kl = (self.prior_noise_level - noise_level + 0.5 * (tensor.exp(2 * noise_level) + (pre_noise - self.prior_mean)**2) / tensor.exp(2 * self.prior_noise_level) - 0.5) application_call.add_auxiliary_variable(kl, roles=[NITS], name='nits') return pre_noise + tensor.exp(noise_level) * noise def get_dim(self, name): if name == 'input_': return self.linear.get_dim(name) if name == 'output': return self.linear.get_dim(name) if name == 'nits': return self.linear.get_dim('output') return super(NoisyLinear, self).get_dim(name)
class Qsampler(Initializable, Random): def __init__(self, input_dim, output_dim, **kwargs): super(Qsampler, self).__init__(**kwargs) self.prior_mean = 0. self.prior_log_sigma = 0. self.mean_transform = Linear(name=self.name + '_mean', input_dim=input_dim, output_dim=output_dim, weights_init=self.weights_init, biases_init=self.biases_init, use_bias=True) self.log_sigma_transform = Linear(name=self.name + '_log_sigma', input_dim=input_dim, output_dim=output_dim, weights_init=self.weights_init, biases_init=self.biases_init, use_bias=True) self.children = [self.mean_transform, self.log_sigma_transform] def get_dim(self, name): if name == 'input': return self.mean_transform.get_dim('input') elif name == 'output': return self.mean_transform.get_dim('output') else: raise ValueError @application(inputs=['x', 'u'], outputs=['z', 'kl_term']) def sample(self, x, u): """Return a samples and the corresponding KL term Parameters ---------- x : Returns ------- z : tensor.matrix Samples drawn from Q(z|x) kl : tensor.vector KL(Q(z|x) || P_z) """ mean = self.mean_transform.apply(x) log_sigma = self.log_sigma_transform.apply(x) # Sample from mean-zeros std.-one Gaussian #u = self.theano_rng.normal( # size=mean.shape, # avg=0., std=1.) # ... and scale/translate samples z = mean + tensor.exp(log_sigma) * u # Calculate KL kl = (self.prior_log_sigma - log_sigma + 0.5 * (tensor.exp(2 * log_sigma) + (mean - self.prior_mean)**2) / tensor.exp(2 * self.prior_log_sigma) - 0.5).sum(axis=-1) return z, kl #@application(inputs=['n_samples']) @application(inputs=['u'], outputs=['z_prior']) def sample_from_prior(self, u): """Sample z from the prior distribution P_z. Parameters ---------- u : tensor.matrix gaussian random source Returns ------- z : tensor.matrix samples """ z_dim = self.mean_transform.get_dim('output') # Sample from mean-zeros std.-one Gaussian #u = self.theano_rng.normal( # size=(n_samples, z_dim), # avg=0., std=1.) # ... and scale/translate samples z = self.prior_mean + tensor.exp(self.prior_log_sigma) * u #z.name("z_prior") return z
class NoisyLinear(Initializable, Feedforward, Random): """Linear transformation sent through a learned noisy channel. Parameters ---------- input_dim : int The dimension of the input. Required by :meth:`~.Brick.allocate`. output_dim : int The dimension of the output. Required by :meth:`~.Brick.allocate`. num_pieces : int The number of linear functions. Required by :meth:`~.Brick.allocate`. """ @lazy(allocation=['input_dim', 'output_dim', 'noise_batch_size']) def __init__(self, input_dim, output_dim, noise_batch_size, prior_mean=0, prior_noise_level=0, **kwargs): self.linear = Linear() self.mask = Linear(name='mask') children = [self.linear, self.mask] kwargs.setdefault('children', []).extend(children) super(NoisyLinear, self).__init__(**kwargs) self.input_dim = input_dim self.output_dim = output_dim self.noise_batch_size = noise_batch_size self.prior_mean = prior_mean self.prior_noise_level = prior_noise_level def _push_allocation_config(self): self.linear.input_dim = self.input_dim self.linear.output_dim = self.output_dim self.mask.input_dim = self.output_dim self.mask.output_dim = self.output_dim def _allocate(self): N = shared_floatx_zeros( (self.noise_batch_size, self.output_dim), name='N') add_role(N, NOISE) self.parameters.append(N) @application(inputs=['input_'], outputs=['output']) def apply(self, input_, application_call): """Apply the linear transformation followed by masking with noise. Parameters ---------- input_ : :class:`~tensor.TensorVariable` The input on which to apply the transformations Returns ------- output : :class:`~tensor.TensorVariable` The transformed input """ pre_noise = self.linear.apply(input_) noise_level = (self.prior_noise_level - tensor.clip(self.mask.apply(pre_noise), -16, 16)) noise_level = copy_and_tag_noise( noise_level, self, LOG_SIGMA, 'log_sigma') # Allow incomplete batches by just taking the noise that is needed # noise = Print('noise')(self.parameters[0][:noise_level.shape[0], :]) noise = self.parameters[0][:noise_level.shape[0], :] # noise = Print('noise')(self.theano_rng.normal(noise_level.shape)) kl = ( self.prior_noise_level - noise_level + 0.5 * ( tensor.exp(2 * noise_level) + (pre_noise - self.prior_mean) ** 2 ) / tensor.exp(2 * self.prior_noise_level) - 0.5 ) application_call.add_auxiliary_variable(kl, roles=[NITS], name='nits') return pre_noise + tensor.exp(noise_level) * noise def get_dim(self, name): if name == 'input_': return self.linear.get_dim(name) if name == 'output': return self.linear.get_dim(name) if name == 'nits': return self.linear.get_dim('output') return super(NoisyLinear, self).get_dim(name)
class Qsampler(Initializable, Random): def __init__(self, input_dim, output_dim, **kwargs): super(Qsampler, self).__init__(**kwargs) self.mean_transform = Linear( name=self.name+'_mean', input_dim=input_dim, output_dim=output_dim, weights_init=self.weights_init, biases_init=self.biases_init, use_bias=True) self.logvar_transform = Linear( name=self.name+'_logvar', input_dim=input_dim, output_dim=output_dim, weights_init=self.weights_init, biases_init=self.biases_init, use_bias=True) self.children = [self.mean_transform, self.logvar_transform] return def get_dim(self, name): if name == 'input': return self.mean_transform.get_dim('input') elif name == 'output': return self.mean_transform.get_dim('output') else: raise ValueError return @application(inputs=['x', 'u'], outputs=['z', 'all_klds']) def sample(self, x, u): """ Return samples and the corresponding KL term Parameters ---------- x : input features for estimating mean and log-variance u : standard Normal samples to scale and shift Returns ------- z : tensor.matrix Samples drawn from Q(z|x) kl : tensor.vector KL(Q(z|x) || P_z) """ mean = self.mean_transform.apply(x) logvar = self.logvar_transform.apply(x) # ... and scale/translate samples z = mean + tensor.exp(0.5 * logvar) * u # Calculate KL all_klds = gaussian_kld(mean, logvar, 0.0, 0.0) return z, all_klds @application(inputs=['u'], outputs=['z_prior']) def sample_from_prior(self, u): """ Sample z from the prior distribution P_z. Parameters ---------- u : tensor.matrix gaussian random source Returns ------- z : tensor.matrix samples """ z_prior = u return z_prior