def __init__(self, generator, discriminator, inferer=None,

inference_monitoring_batch_size=128,

monitor_generator=True,

monitor_discriminator=True,

monitor_inference=True,

shrink_d = 0.):

Model.__init__(self)

self.__dict__.update(locals())

del self.self

def __setstate__(self, state):

self.__dict__.update(state)

if 'inferer' not in state:

self.inferer = None

if 'inference_monitoring_batch_size' not in state:

self.inference_monitoring_batch_size = 128 # TODO: HACK

if 'monitor_generator' not in state:

self.monitor_generator = True

if 'monitor_discriminator' not in state:

self.monitor_discriminator = True

if 'monitor_inference' not in state:

self.monitor_inference = True

def get_params(self):

p = self.generator.get_params() + self.discriminator.get_params()

if hasattr(self, 'inferer') and self.inferer is not None:

p += self.inferer.get_params()

return p

def get_input_space(self):

return self.discriminator.get_input_space()

def get_weights_topo(self):

return self.discriminator.get_weights_topo()

def get_weights(self):

return self.discriminator.get_weights()

def get_weights_format(self):

return self.discriminator.get_weights_format()

def get_weights_view_shape(self):

return self.discriminator.get_weights_view_shape()

def get_monitoring_channels(self, data):

rval = OrderedDict()

g_ch = self.generator.get_monitoring_channels(data)

d_ch = self.discriminator.get_monitoring_channels((data, None))

samples = self.generator.sample(100)

d_samp_ch = self.discriminator.get_monitoring_channels((samples, None))

i_ch = OrderedDict()

if self.inferer is not None:

batch_size = self.inference_monitoring_batch_size

sample, noise, _ = self.generator.sample_and_noise(batch_size)

i_ch.update(self.inferer.get_monitoring_channels((sample, noise)))

if self.monitor_generator:

for key in g_ch:

rval['gen_' + key] = g_ch[key]

if self.monitor_discriminator:

for key in d_ch:

rval['dis_on_data_' + key] = d_samp_ch[key]

for key in d_ch:

rval['dis_on_samp_' + key] = d_ch[key]

if self.monitor_inference:

for key in i_ch:

rval['inf_' + key] = i_ch[key]

return rval

def get_monitoring_data_specs(self):

space = self.discriminator.get_input_space()

source = self.discriminator.get_input_source()

return (space, source)

def _modify_updates(self, updates):

self.generator.modify_updates(updates)

self.discriminator.modify_updates(updates)

if self.shrink_d != 0.:

for param in self.discriminator.get_params():

if param in updates:

updates[param] = self.shrink_d * updates[param]

if self.inferer is not None:

self.inferer.modify_updates(updates)

def get_lr_scalers(self):

rval = self.generator.get_lr_scalers()

rval.update(self.discriminator.get_lr_scalers())

model = serial.load(pretrained)

pretrained_layers = model.generator.mlp.layers

assert pretrained_layers[start_layer].get_input_space() == mlp.layers[-1].get_output_space()

mlp.layers.extend(pretrained_layers[start_layer:])

def __init__(self, mlp, noise = "gaussian", monitor_ll = False, ll_n_samples = 100, ll_sigma = 0.2):

Model.__init__(self)

self.__dict__.update(locals())

del self.self

self.theano_rng = MRG_RandomStreams(2014 * 5 + 27)

def get_input_space(self):

return self.mlp.get_input_space()

def sample_and_noise(self, num_samples, default_input_include_prob=1., default_input_scale=1., all_g_layers=False):

n = self.mlp.get_input_space().get_total_dimension()

noise = self.get_noise((num_samples, n))

formatted_noise = VectorSpace(n).format_as(noise, self.mlp.get_input_space())

if all_g_layers:

rval = self.mlp.dropout_fprop(formatted_noise, default_input_include_prob=default_input_include_prob, default_input_scale=default_input_scale, return_all=all_g_layers)

other_layers, rval = rval[:-1], rval[-1]

else:

rval = self.mlp.dropout_fprop(formatted_noise, default_input_include_prob=default_input_include_prob, default_input_scale=default_input_scale)

other_layers = None

return rval, formatted_noise, other_layers

def sample(self, num_samples, default_input_include_prob=1., default_input_scale=1.):

sample, _, _ = self.sample_and_noise(num_samples, default_input_include_prob, default_input_scale)

return sample

def inpainting_sample_and_noise(self, X, default_input_include_prob=1., default_input_scale=1.):

# Very hacky! Specifically for inpainting right half of CIFAR-10 given left half

# assumes X is b01c

assert X.ndim == 4

input_space = self.mlp.get_input_space()

n = input_space.get_total_dimension()

image_size = input_space.shape[0]

half_image = int(image_size / 2)

data_shape = (X.shape[0], image_size, half_image, input_space.num_channels)

noise = self.theano_rng.normal(size=data_shape, dtype='float32')

Xg = T.set_subtensor(X[:,:,half_image:,:], noise)

sampled_part, noise = self.mlp.dropout_fprop(Xg, default_input_include_prob=default_input_include_prob, default_input_scale=default_input_scale), noise

sampled_part = sampled_part.reshape(data_shape)

rval = T.set_subtensor(X[:, :, half_image:, :], sampled_part)

return rval, noise

def get_monitoring_channels(self, data):

if data is None:

m = 100

else:

m = data.shape[0]

n = self.mlp.get_input_space().get_total_dimension()

noise = self.get_noise((m, n))

rval = OrderedDict()

try:

rval.update(self.mlp.get_monitoring_channels((noise, None)))

except Exception:

warnings.warn("something went wrong with generator.mlp's monitoring channels")

if self.monitor_ll:

rval['ll'] = T.cast(self.ll(data, self.ll_n_samples, self.ll_sigma),

theano.config.floatX).mean()

rval['nll'] = -rval['ll']

return rval

def get_noise(self, size):

# Allow just requesting batch size

if isinstance(size, int):

size = (size, self.get_input_space().get_total_dimension())

if not hasattr(self, 'noise'):

self.noise = "gaussian"

if self.noise == "uniform":

return self.theano_rng.uniform(low=-np.sqrt(3), high=np.sqrt(3), size=size, dtype='float32')

elif self.noise == "gaussian":

return self.theano_rng.normal(size=size, dtype='float32')

elif self.noise == "spherical":

noise = self.theano_rng.normal(size=size, dtype='float32')

noise = noise / T.maximum(1e-7, T.sqrt(T.sqr(noise).sum(axis=1))).dimshuffle(0, 'x')

return noise

else:

raise NotImplementedError(self.noise)

def get_params(self):

return self.mlp.get_params()

def get_output_space(self):

return self.mlp.get_output_space()

def ll(self, data, n_samples, sigma):

samples = self.sample(n_samples)

output_space = self.mlp.get_output_space()

if 'Conv2D' in str(output_space):

samples = output_space.convert(samples, output_space.axes, ('b', 0, 1, 'c'))

samples = samples.flatten(2)

data = output_space.convert(data, output_space.axes, ('b', 0, 1, 'c'))

data = data.flatten(2)

parzen = theano_parzen(data, samples, sigma)

return parzen

def _modify_updates(self, updates):

self.mlp.modify_updates(updates)

def get_lr_scalers(self):

return self.mlp.get_lr_scalers()

def __setstate__(self, state):

self.__dict__.update(state)

if 'monitor_ll' not in state:

def __init__(self, default_input_include_prob, default_input_scale,

input_include_probs=None, input_scales=None, **kwargs):

super(IntrinsicDropoutGenerator, self).__init__(**kwargs)

self.__dict__.update(locals())

del self.self

def sample_and_noise(self, num_samples, default_input_include_prob=1., default_input_scale=1., all_g_layers=False):

if all_g_layers:

raise NotImplementedError()

n = self.mlp.get_input_space().get_total_dimension()

noise = self.theano_rng.normal(size=(num_samples, n), dtype='float32')

formatted_noise = VectorSpace(n).format_as(noise, self.mlp.get_input_space())

# ignores dropout args

default_input_include_prob = self.default_input_include_prob

default_input_scale = self.default_input_scale

input_include_probs = self.input_include_probs

input_scales = self.input_scales

return self.mlp.dropout_fprop(formatted_noise,

default_input_include_prob=default_input_include_prob,

default_input_scale=default_input_scale,

input_include_probs=input_include_probs,

"""

"""

# Supplies own labels, don't get them from the dataset

supervised = False

def __init__(self, scale_grads=1, target_scale=.1,

discriminator_default_input_include_prob = 1.,

discriminator_input_include_probs=None,

discriminator_default_input_scale=1.,

discriminator_input_scales=None,

generator_default_input_include_prob = 1.,

generator_default_input_scale=1.,

inference_default_input_include_prob=None,

inference_input_include_probs=None,

inference_default_input_scale=1.,

inference_input_scales=None,

init_now_train_generator=True,

ever_train_discriminator=True,

ever_train_generator=True,

ever_train_inference=True,

no_drop_in_d_for_g=False,

alternate_g = False,

infer_layer=None,

noise_both = 0.,

blend_obj = False,

minimax_coeff = 1.,

zurich_coeff = 1.):

self.__dict__.update(locals())

del self.self

# These allow you to dynamically switch off training parts.

# If the corresponding ever_train_* is False, these have

# no effect.

self.now_train_generator = sharedX(init_now_train_generator)

self.now_train_discriminator = sharedX(numpy.array(1., dtype='float32'))

self.now_train_inference = sharedX(numpy.array(1., dtype='float32'))

def expr(self, model, data, **kwargs):

S, d_obj, g_obj, i_obj = self.get_samples_and_objectives(model, data)

l = []

# This stops stuff from ever getting computed if we're not training

# it.

if self.ever_train_discriminator:

l.append(d_obj)

if self.ever_train_generator:

l.append(g_obj)

if self.ever_train_inference:

l.append(i_obj)

return sum(l)

def get_samples_and_objectives(self, model, data):

space, sources = self.get_data_specs(model)

space.validate(data)

assert isinstance(model, AdversaryPair)

g = model.generator

d = model.discriminator

# Note: this assumes data is design matrix

X = data

m = data.shape[space.get_batch_axis()]

y1 = T.alloc(1, m, 1)

y0 = T.alloc(0, m, 1)

# NOTE: if this changes to optionally use dropout, change the inference

# code below to use a non-dropped-out version.

S, z, other_layers = g.sample_and_noise(m, default_input_include_prob=self.generator_default_input_include_prob, default_input_scale=self.generator_default_input_scale, all_g_layers=(self.infer_layer is not None))

if self.noise_both != 0.:

rng = MRG_RandomStreams(2014 / 6 + 2)

S = S + rng.normal(size=S.shape, dtype=S.dtype) * self.noise_both

X = X + rng.normal(size=X.shape, dtype=S.dtype) * self.noise_both

y_hat1 = d.dropout_fprop(X, self.discriminator_default_input_include_prob,

self.discriminator_input_include_probs,

self.discriminator_default_input_scale,

self.discriminator_input_scales)

y_hat0 = d.dropout_fprop(S, self.discriminator_default_input_include_prob,

self.discriminator_input_include_probs,

self.discriminator_default_input_scale,

self.discriminator_input_scales)

d_obj = 0.5 * (d.layers[-1].cost(y1, y_hat1) + d.layers[-1].cost(y0, y_hat0))

if self.no_drop_in_d_for_g:

y_hat0_no_drop = d.dropout_fprop(S)

g_obj = d.layers[-1].cost(y1, y_hat0_no_drop)

else:

g_obj = d.layers[-1].cost(y1, y_hat0)

if self.blend_obj:

g_obj = (self.zurich_coeff * g_obj - self.minimax_coeff * d_obj) / (self.zurich_coeff + self.minimax_coeff)

if model.inferer is not None:

# Change this if we ever switch to using dropout in the

# construction of S.

S_nograd = block_gradient(S) # Redundant as long as we have custom get_gradients

pred = model.inferer.dropout_fprop(S_nograd, self.inference_default_input_include_prob,

self.inference_input_include_probs,

self.inference_default_input_scale,

self.inference_input_scales)

if self.infer_layer is None:

target = z

else:

target = other_layers[self.infer_layer]

i_obj = model.inferer.layers[-1].cost(target, pred)

else:

i_obj = 0

return S, d_obj, g_obj, i_obj

def get_gradients(self, model, data, **kwargs):

space, sources = self.get_data_specs(model)

space.validate(data)

assert isinstance(model, AdversaryPair)

g = model.generator

d = model.discriminator

S, d_obj, g_obj, i_obj = self.get_samples_and_objectives(model, data)

g_params = g.get_params()

d_params = d.get_params()

for param in g_params:

assert param not in d_params

for param in d_params:

assert param not in g_params

d_grads = T.grad(d_obj, d_params)

g_grads = T.grad(g_obj, g_params)

if self.scale_grads:

S_grad = T.grad(g_obj, S)

scale = T.maximum(1., self.target_scale / T.sqrt(T.sqr(S_grad).sum()))

g_grads = [g_grad * scale for g_grad in g_grads]

rval = OrderedDict()

zeros = itertools.repeat(theano.tensor.constant(0., dtype='float32'))

if self.ever_train_discriminator:

rval.update(OrderedDict(safe_zip(d_params, [self.now_train_discriminator * dg for dg in d_grads])))

else:

rval.update(OrderedDict(zip(d_params, zeros)))

if self.ever_train_generator:

rval.update(OrderedDict(safe_zip(g_params, [self.now_train_generator * gg for gg in g_grads])))

else:

rval.update(OrderedDict(zip(g_params, zeros)))

if self.ever_train_inference and model.inferer is not None:

i_params = model.inferer.get_params()

i_grads = T.grad(i_obj, i_params)

rval.update(OrderedDict(safe_zip(i_params, [self.now_train_inference * ig for ig in i_grads])))

elif model.inferer is not None:

rval.update(OrderedDict(model.inferer.get_params(), zeros))

updates = OrderedDict()

# Two d steps for every g step

if self.alternate_g:

updates[self.now_train_generator] = 1. - self.now_train_generator

return rval, updates

def get_monitoring_channels(self, model, data, **kwargs):

rval = OrderedDict()

m = data.shape[0]

g = model.generator

d = model.discriminator

y_hat = d.fprop(data)

rval['false_negatives'] = T.cast((y_hat < 0.5).mean(), 'float32')

samples = g.sample(m)

y_hat = d.fprop(samples)

rval['false_positives'] = T.cast((y_hat > 0.5).mean(), 'float32')

# y = T.alloc(0., m, 1)

cost = d.cost_from_X((samples, y_hat))

sample_grad = T.grad(-cost, samples)

rval['sample_grad_norm'] = T.sqrt(T.sqr(sample_grad).sum())

_S, d_obj, g_obj, i_obj = self.get_samples_and_objectives(model, data)

if model.monitor_inference and i_obj != 0:

rval['objective_i'] = i_obj

if model.monitor_discriminator:

rval['objective_d'] = d_obj

if model.monitor_generator:

rval['objective_g'] = g_obj

rval['now_train_generator'] = self.now_train_generator

pair = serial.load(pair_path)

d = pair.discriminator

del d.layers[-1]

d.add_layers([new_head])

"""

Credit: Yann N. Dauphin

"""

x = data

a = ( x.dimshuffle(0, 'x', 1) - mu.dimshuffle('x', 0, 1) ) / sigma

E = log_mean_exp(-0.5*(a**2).sum(2))

Z = mu.shape[1] * T.log(sigma * numpy.sqrt(numpy.pi * 2))

#return theano.function([x], E - Z)

"""

Credit: Yann N. Dauphin

"""

max_ = a.max(1)

"""

Monitoring channels are hardcoded for C01B batches

"""

def __init__(self, layer_name):

Model.__init__(self)

self.__dict__.update(locals())

del self.self

self._params = []

def set_input_space(self, space):

self.input_space = space

assert isinstance(space, CompositeSpace)

self.output_space = space.components[0]

def fprop(self, state_below):

rval = state_below[0]

for i in xrange(1, len(state_below)):

rval = rval + state_below[i]

rval.came_from_sum = True

return rval

@functools.wraps(Layer.get_layer_monitoring_channels)

def get_layer_monitoring_channels(self, state_below=None,

state=None, targets=None):

rval = OrderedDict()

if state is None:

state = self.fprop(state_below)

vars_and_prefixes = [(state, '')]

for var, prefix in vars_and_prefixes:

if not hasattr(var, 'ndim') or var.ndim != 4:

print "expected 4D tensor, got "

print var

print type(var)

if isinstance(var, tuple):

print "tuple length: ", len(var)

assert False

v_max = var.max(axis=(1, 2, 3))

v_min = var.min(axis=(1, 2, 3))

v_mean = var.mean(axis=(1, 2, 3))

v_range = v_max - v_min

# max_x.mean_u is "the mean over *u*nits of the max over

# e*x*amples" The x and u are included in the name because

# otherwise its hard to remember which axis is which when reading

# the monitor I use inner.outer rather than outer_of_inner or

# something like that because I want mean_x.* to appear next to

# each other in the alphabetical list, as these are commonly

# plotted together

for key, val in [('max_x.max_u', v_max.max()),

('max_x.mean_u', v_max.mean()),

('max_x.min_u', v_max.min()),

('min_x.max_u', v_min.max()),

('min_x.mean_u', v_min.mean()),

('min_x.min_u', v_min.min()),

('range_x.max_u', v_range.max()),

('range_x.mean_u', v_range.mean()),

('range_x.min_u', v_range.min()),

('mean_x.max_u', v_mean.max()),

('mean_x.mean_u', v_mean.mean()),

('mean_x.min_u', v_mean.min())]:

rval[prefix+key] = val

def __init__(self, active_after, value=1.):

self.__dict__.update(locals())

del self.self

self.cur_epoch = 0

def on_monitor(self, model, dataset, algorithm):

if self.cur_epoch == self.active_after:

algorithm.cost.now_train_generator.set_value(np.array(self.value, dtype='float32'))

"""

"""

# Supplies own labels, don't get them from the dataset

supervised = False

def __init__(self, scale_grads=1, target_scale=.1,

discriminator_default_input_include_prob = 1.,

discriminator_input_include_probs=None,

discriminator_default_input_scale=1.,

discriminator_input_scales=None,

generator_default_input_include_prob = 1.,

generator_default_input_scale=1.,

inference_default_input_include_prob=None,

inference_input_include_probs=None,

inference_default_input_scale=1.,

inference_input_scales=None,

init_now_train_generator=True,

ever_train_discriminator=True,

ever_train_generator=True,

ever_train_inference=True,

no_drop_in_d_for_g=False,

alternate_g = False):

self.__dict__.update(locals())

del self.self

# These allow you to dynamically switch off training parts.

# If the corresponding ever_train_* is False, these have

# no effect.

self.now_train_generator = sharedX(init_now_train_generator)

self.now_train_discriminator = sharedX(numpy.array(1., dtype='float32'))

self.now_train_inference = sharedX(numpy.array(1., dtype='float32'))

def expr(self, model, data, **kwargs):

S, d_obj, g_obj, i_obj = self.get_samples_and_objectives(model, data)

return d_obj + g_obj + i_obj

def get_samples_and_objectives(self, model, data):

space, sources = self.get_data_specs(model)

space.validate(data)

assert isinstance(model, AdversaryPair)

g = model.generator

d = model.discriminator

# Note: this assumes data is b01c

X = data

assert X.ndim == 4

m = data.shape[space.get_batch_axis()]

y1 = T.alloc(1, m, 1)

y0 = T.alloc(0, m, 1)

# NOTE: if this changes to optionally use dropout, change the inference

# code below to use a non-dropped-out version.

S, z = g.inpainting_sample_and_noise(X, default_input_include_prob=self.generator_default_input_include_prob, default_input_scale=self.generator_default_input_scale)

y_hat1 = d.dropout_fprop(X, self.discriminator_default_input_include_prob,

self.discriminator_input_include_probs,

self.discriminator_default_input_scale,

self.discriminator_input_scales)

y_hat0 = d.dropout_fprop(S, self.discriminator_default_input_include_prob,

self.discriminator_input_include_probs,

self.discriminator_default_input_scale,

self.discriminator_input_scales)

d_obj = 0.5 * (d.layers[-1].cost(y1, y_hat1) + d.layers[-1].cost(y0, y_hat0))

if self.no_drop_in_d_for_g:

y_hat0_no_drop = d.dropout_fprop(S)

g_obj = d.layers[-1].cost(y1, y_hat0)

else:

g_obj = d.layers[-1].cost(y1, y_hat0)

if model.inferer is not None:

# Change this if we ever switch to using dropout in the

# construction of S.

S_nograd = block_gradient(S) # Redundant as long as we have custom get_gradients

z_hat = model.inferer.dropout_fprop(S_nograd, self.inference_default_input_include_prob,

self.inference_input_include_probs,

self.inference_default_input_scale,

self.inference_input_scales)

i_obj = model.inferer.layers[-1].cost(z, z_hat)

else:

i_obj = 0

return S, d_obj, g_obj, i_obj

def get_gradients(self, model, data, **kwargs):

space, sources = self.get_data_specs(model)

space.validate(data)

assert isinstance(model, AdversaryPair)

g = model.generator

d = model.discriminator

S, d_obj, g_obj, i_obj = self.get_samples_and_objectives(model, data)

g_params = g.get_params()

d_params = d.get_params()

for param in g_params:

assert param not in d_params

for param in d_params:

assert param not in g_params

d_grads = T.grad(d_obj, d_params)

g_grads = T.grad(g_obj, g_params)

if self.scale_grads:

S_grad = T.grad(g_obj, S)

scale = T.maximum(1., self.target_scale / T.sqrt(T.sqr(S_grad).sum()))

g_grads = [g_grad * scale for g_grad in g_grads]

rval = OrderedDict()

if self.ever_train_discriminator:

rval.update(OrderedDict(safe_zip(d_params, [self.now_train_discriminator * dg for dg in d_grads])))

else:

rval.update(OrderedDict(zip(d_params, itertools.repeat(theano.tensor.constant(0., dtype='float32')))))

if self.ever_train_generator:

rval.update(OrderedDict(safe_zip(g_params, [self.now_train_generator * gg for gg in g_grads])))

else:

rval.update(OrderedDict(zip(g_params, itertools.repeat(theano.tensor.constant(0., dtype='float32')))))

if self.ever_train_inference and model.inferer is not None:

i_params = model.inferer.get_params()

i_grads = T.grad(i_obj, i_params)

rval.update(OrderedDict(safe_zip(i_params, [self.now_train_inference * ig for ig in i_grads])))

updates = OrderedDict()

# Two d steps for every g step

if self.alternate_g:

updates[self.now_train_generator] = 1. - self.now_train_generator

return rval, updates

def get_monitoring_channels(self, model, data, **kwargs):

rval = OrderedDict()

m = data.shape[0]

g = model.generator

d = model.discriminator

y_hat = d.fprop(data)

rval['false_negatives'] = T.cast((y_hat < 0.5).mean(), 'float32')

samples, noise = g.inpainting_sample_and_noise(data)

y_hat = d.fprop(samples)

rval['false_positives'] = T.cast((y_hat > 0.5).mean(), 'float32')

# y = T.alloc(0., m, 1)

cost = d.cost_from_X((samples, y_hat))

sample_grad = T.grad(-cost, samples)

rval['sample_grad_norm'] = T.sqrt(T.sqr(sample_grad).sum())

_S, d_obj, g_obj, i_obj = self.get_samples_and_objectives(model, data)

if i_obj != 0:

rval['objective_i'] = i_obj

rval['objective_d'] = d_obj

rval['objective_g'] = g_obj

rval['now_train_generator'] = self.now_train_generator

def __init__(self, k):

self.__dict__.update(locals())

del self.self

self.i = 0

def __call__(self, sgd):

self.i = (self.i + 1) % self.k

def __init__(self, new_dim, std, layer_name):

Layer.__init__(self)

self.__dict__.update(locals())

del self.self

self._params = []

def set_input_space(self, space):

assert isinstance(space, VectorSpace)

self.input_space = space

self.output_space = VectorSpace(space.dim + self.new_dim)

self.theano_rng = MRG_RandomStreams(self.mlp.rng.randint(2 ** 16))

def fprop(self, state):

noise = self.theano_rng.normal(std=self.std, avg=0., size=(state.shape[0], self.new_dim),

dtype=state.dtype)

def __init__(self, layer_name, left_slope=0.0, **kwargs):

super(RectifiedLinear, self).__init__(**kwargs)

self.__dict__.update(locals())

del self.self

self._params = []

def set_input_space(self, space):

self.input_space = space

self.output_space = space

def fprop(self, state_below):

p = state_below

p = T.switch(p > 0., p, self.left_slope * p)

def __init__(self, layer_name, left_slope=0.0, **kwargs):

super(Sigmoid, self).__init__(**kwargs)

self.__dict__.update(locals())

del self.self

self._params = []

def set_input_space(self, space):

self.input_space = space

self.output_space = space

def fprop(self, state_below):

p = T.nnet.sigmoid(state_below)

def __init__(self, layer_name, left_slope=0.0, **kwargs):

super(SubtractHalf, self).__init__(**kwargs)

self.__dict__.update(locals())

del self.self

self._params = []

def set_input_space(self, space):

self.input_space = space

self.output_space = space

def fprop(self, state_below):

return state_below - 0.5

def get_weights(self):

return self.mlp.layers[1].get_weights()

def get_weights_format(self):

return self.mlp.layers[1].get_weights_format()

def get_weights_view_shape(self):

def __init__(self, layer_name, dataset, also_sd = False, **kwargs):

super(SubtractRealMean, self).__init__(**kwargs)

self.__dict__.update(locals())

del self.self

self._params = []

self.mean = sharedX(dataset.X.mean(axis=0))

if also_sd:

self.sd = sharedX(dataset.X.std(axis=0))

del self.dataset

def set_input_space(self, space):

self.input_space = space

self.output_space = space

def fprop(self, state_below):

return (state_below - self.mean) / self.sd

def get_weights(self):

return self.mlp.layers[1].get_weights()

def get_weights_format(self):

return self.mlp.layers[1].get_weights_format()

def get_weights_view_shape(self):

def __init__(self, scale, layer_name):

Layer.__init__(self)

self.__dict__.update(locals())

del self.self

self._params = []

def set_input_space(self, space):

assert isinstance(space, VectorSpace)

self.input_space = space

self.output_space = space

self.theano_rng = MRG_RandomStreams(self.mlp.rng.randint(2 ** 16))

def fprop(self, state):

noise = self.theano_rng.binomial(size=state.shape, p=0.5,

dtype=state.dtype) * 2. - 1.

"""

"""

# Supplies own labels, don't get them from the dataset

supervised = False

def __init__(self, scale_grads=1, target_scale=.1,

discriminator_default_input_include_prob = 1.,

discriminator_input_include_probs=None,

discriminator_default_input_scale=1.,

discriminator_input_scales=None,

generator_default_input_include_prob = 1.,

generator_default_input_scale=1.,

inference_default_input_include_prob=None,

inference_input_include_probs=None,

inference_default_input_scale=1.,

inference_input_scales=None,

init_now_train_generator=True,

ever_train_discriminator=True,

ever_train_generator=True,

ever_train_inference=True,

no_drop_in_d_for_g=False,

alternate_g = False,

infer_layer=None,

noise_both = 0.):

self.__dict__.update(locals())

del self.self

# These allow you to dynamically switch off training parts.

# If the corresponding ever_train_* is False, these have

# no effect.

self.now_train_generator = sharedX(init_now_train_generator)

self.now_train_discriminator = sharedX(numpy.array(1., dtype='float32'))

self.now_train_inference = sharedX(numpy.array(1., dtype='float32'))

def expr(self, model, data, **kwargs):

S, d_obj, g_obj, i_obj = self.get_samples_and_objectives(model, data)

l = []

# This stops stuff from ever getting computed if we're not training

# it.

if self.ever_train_discriminator:

l.append(d_obj)

if self.ever_train_generator:

l.append(g_obj)

if self.ever_train_inference:

l.append(i_obj)

return sum(l)

def get_samples_and_objectives(self, model, data):

space, sources = self.get_data_specs(model)

space.validate(data)

assert isinstance(model, AdversaryPair)

g = model.generator

d = model.discriminator

# Note: this assumes data is design matrix

X = data

m = data.shape[space.get_batch_axis()]

y1 = T.alloc(1, m, 1)

y0 = T.alloc(0, m, 1)

# NOTE: if this changes to optionally use dropout, change the inference

# code below to use a non-dropped-out version.

S, z, other_layers = g.sample_and_noise(m, default_input_include_prob=self.generator_default_input_include_prob, default_input_scale=self.generator_default_input_scale, all_g_layers=(self.infer_layer is not None))

if self.noise_both != 0.:

rng = MRG_RandomStreams(2014 / 6 + 2)

S = S + rng.normal(size=S.shape, dtype=S.dtype) * self.noise_both

X = X + rng.normal(size=X.shape, dtype=S.dtype) * self.noise_both

y_hat1 = d.dropout_fprop(X, self.discriminator_default_input_include_prob,

self.discriminator_input_include_probs,

self.discriminator_default_input_scale,

self.discriminator_input_scales)

y_hat0 = d.dropout_fprop(S, self.discriminator_default_input_include_prob,

self.discriminator_input_include_probs,

self.discriminator_default_input_scale,

self.discriminator_input_scales)

d_obj = 0.5 * (d.layers[-1].cost(y1, y_hat1) + d.layers[-1].cost(y0, y_hat0))

if self.no_drop_in_d_for_g:

y_hat0_no_drop = d.dropout_fprop(S)

g_cost_mat = d.layers[-1].cost_matrix(y1, y_hat0_no_drop)

else:

g_cost_mat = d.layers[-1].cost_matrix(y1, y_hat0)

assert g_cost_mat.ndim == 2

assert y_hat0.ndim == 2

mask = y_hat0 < 0.5

masked_cost = g_cost_mat * mask

g_obj = masked_cost.mean()

if model.inferer is not None:

# Change this if we ever switch to using dropout in the

# construction of S.

S_nograd = block_gradient(S) # Redundant as long as we have custom get_gradients

pred = model.inferer.dropout_fprop(S_nograd, self.inference_default_input_include_prob,

self.inference_input_include_probs,

self.inference_default_input_scale,

self.inference_input_scales)

if self.infer_layer is None:

target = z

else:

target = other_layers[self.infer_layer]

i_obj = model.inferer.layers[-1].cost(target, pred)

else:

i_obj = 0

return S, d_obj, g_obj, i_obj

def get_gradients(self, model, data, **kwargs):

space, sources = self.get_data_specs(model)

space.validate(data)

assert isinstance(model, AdversaryPair)

g = model.generator

d = model.discriminator

S, d_obj, g_obj, i_obj = self.get_samples_and_objectives(model, data)

g_params = g.get_params()

d_params = d.get_params()

for param in g_params:

assert param not in d_params

for param in d_params:

assert param not in g_params

d_grads = T.grad(d_obj, d_params)

g_grads = T.grad(g_obj, g_params)