def msconv2d(name, n_scales, n_in, n_out, size_kernel=(3, 3), zerodiagonal=True, flipmask=False, pad_channel=True, border_mode='valid', w={}): convs = [ conv2d(name + "_s" + str(i), n_in, n_out, size_kernel, zerodiagonal, flipmask, pad_channel, border_mode, w) for i in range(n_scales) ] def f(h, w): results = [] for i in range(n_scales - 1): results.append(convs[i](h, w)) h = N.conv.downsample2d_nearest_neighbour(h, scale=2) result = convs[-1](h, w) for i in range(n_scales - 1): result = N.conv.upsample2d_nearest_neighbour(result) result += results[-1 - i] return result def postup(updates, w): for conv in convs: updates = conv.postup(updates, w) return updates return G.Struct(__call__=f, w=w, postup=postup)
def gsm(name, k, w={}, logvar_minmax=16): w[name + '_weight'] = G.sharedf(np.zeros((k, ))) w[name + '_logvar'] = G.sharedf(np.random.randn(k) * .1) def logp(v, w): mixtureweights = T.exp(w[name + '_weight']) mixtureweights /= mixtureweights.sum() logvar = logvar_minmax * w[name + '_logvar'] var = T.exp(logvar) if k == 0: return 0. if k == 1: return -.5 * (v**2).sum() / var[0] - v.size.astype( G.floatX) * (.5 * T.log(2. * math.pi) + logvar[0]) p = 0. for i in range(k): p += mixtureweights[i] * T.exp(-.5 * v**2 / var[i]) / T.sqrt( 2. * math.pi * var[i]) logp = T.log(p).sum() return logp def postup(updates, w): updates[w[name + '_logvar']] = T.clip(updates[w[name + '_logvar']], -1., 1.) return updates return G.Struct(logp=logp, postup=postup, w=w)
def resnetv3_layer_b(name, n_feats, factor=4, nl='softplus', alpha=.1, w={}): f_nl1 = N.nonlinearity(name+"_nl1", nl) f_nl2 = N.nonlinearity(name+"_nl2", nl) f_nl3 = N.nonlinearity(name+"_nl3", nl) # either no change in shape, or subsampling conv1 = conv2d(name+'_conv1', n_feats, n_feats/factor, (1,1), w=w) conv2 = conv2d(name+'_conv2', n_feats/factor, n_feats/factor, (3,3), w=w) conv3 = conv2d(name+'_conv3', n_feats/factor, n_feats, (1,1), w=w) def f(_input, w): h = f_nl1(_input) h = f_nl2(conv1(h, w)) h = f_nl3(conv2(h, w)) h = conv3(h, w) return _input + alpha * h def postup(updates, w): updates = conv1.postup(updates, w) updates = conv2.postup(updates, w) updates = conv3.postup(updates, w) return updates return G.Struct(__call__=f, w=w, postup=postup)
def resnet_layer_a(name, n_feats, nl='elu', w={}): f_nl1 = N.nonlinearity(name + "_nl1", nl) f_nl2 = N.nonlinearity(name + "_nl2", nl) # either no change in shape, or subsampling conv1 = conv2d(name + '_conv1', n_feats, n_feats, (3, 3), zerodiagonal=False, w=w) conv2 = conv2d(name + '_conv2', n_feats, n_feats, (3, 3), zerodiagonal=False, w=w) def f(_input, w): h = f_nl1(_input) h = f_nl2(conv1(h, w)) h = conv2(h, w) return _input + .1 * h def postup(updates, w): updates = conv1.postup(updates, w) updates = conv2.postup(updates, w) return updates return G.Struct(__call__=f, w=w, postup=postup)
def resnetv1(name, n_layers, n_in, n_out, size_kernel=(3,3), downsample=1, upsample=1, nl='relu', w={}): layers = [] for i in range(n_layers): _n_in = n_in _n_out = n_out _downsample = downsample _upsample = upsample if _downsample > 1 and i > 0: _downsample = 1 _n_in = n_out if _upsample > 1 and i < n_layers-1: _upsample = 1 _n_out = n_in layer = resnetv1_layer(name+'_'+str(i), _n_in, _n_out, size_kernel, _downsample, _upsample, nl, w) layers.append(layer) def f(h, w): for i in range(n_layers): h = layers[i](h, w) return h def postup(updates, w): for i in range(n_layers): updates = layers[i].postup(updates, w) return updates return G.Struct(__call__=f, w=w, postup=postup)
def categorical(p, sample=None): if sample is None: sample = G.rng.multinomial(pvals=p, dtype='int32').argmax(axis=1) logp = -T.nnet.categorical_crossentropy(p, sample.flatten()) entr = -(p * T.log(p)).sum(axis=1) return G.Struct(**{'sample': sample, 'logp': logp, 'entr': entr, 'p': p}) return RandomVariable(sample, logp, entr, p=p)
def batchnorm_meanonly(name, n_h, w={}): w[name+'_b'] = G.sharedf(np.zeros((n_h,))) def f(h, w): h -= h.mean(axis=(0,2,3), keepdims=True) h += w[name+'_b'].dimshuffle('x',0,'x','x') return h return G.Struct(__call__=f, w=w)
def nonlinearity(name, which, shape=None, w={}): if which == 'prelu': w[name] = G.sharedf(np.zeros(shape)) if which == 'pelu': w[name] = G.sharedf(np.zeros(shape)) if which == 'softplus2': w[name] = G.sharedf(np.zeros(shape)) if which == 'softplus_shiftscale': w[name+'_in_s'] = G.sharedf(np.zeros(shape)) w[name+'_in_b'] = G.sharedf(np.zeros(shape)) if which == 'linearsigmoid': w[name+'_a'] = G.sharedf(.5*np.ones(shape)) w[name+'_b'] = G.sharedf(.5*np.ones(shape)) if which == 'meanonlybatchnorm_softplus': assert type(shape) == int w[name+'_b'] = G.sharedf(np.zeros(shape)) if which == 'meanonlybatchnorm_relu': assert type(shape) == int w[name+'_b'] = G.sharedf(np.zeros(shape)) def f(h, w=None): if which == None or which == 'None': return h elif which == 'tanh': return T.tanh(h) elif which == 'softmax': return T.nnet.softmax(h) elif which == 'prelu': return w[name]*h*(h<0.) + h*(h>=0.) elif which == 'relu': return h*(h>=0.) elif which == 'shiftedrelu': return T.switch(h < -1., -1., h) elif which == 'leakyrelu': return 0.01 * h*(h<0.) + h*(h>=0.) elif which == 'elu': return T.switch(h < 0., T.exp(h)-1, h) elif which == 'softplus': return T.nnet.softplus(h) elif which == 'softplus_shiftscale': return T.nnet.softplus(T.exp(w[name+'_in_s']) * h + w[name+'_in_b']) elif which == 'softplus2': return T.nnet.softplus(h) - w[name] * T.nnet.softplus(-h) elif which == 'linearsigmoid': return w[name+'_a'] * h + w[name+'_b'] * T.nnet.sigmoid(h) elif which == 'meanonlybatchnorm_softplus': h -= h.mean(axis=(0,2,3), keepdims=True) h += w[name+'_b'].dimshuffle('x',0,'x','x') return T.nnet.softplus(h) elif which == 'meanonlybatchnorm_relu': h -= h.mean(axis=(0,2,3), keepdims=True) h += w[name+'_b'].dimshuffle('x',0,'x','x') return T.nnet.relu(h) else: raise Exception("Unrecognized nonlinearity: "+which) return G.Struct(__call__=f, w=w)
def mlp_l2(name, n_in, n_h, n_out, nl_h, nl_out=None, nl_in=None, w={}): if not isinstance(n_out, list) and isinstance(n_out, int): n_out = [n_out] # parameters for input perturbation if nl_in != None: f_nl_in = N.nonlinearity(name + '_in_nl', nl_in, (n_in, ), w) # parameters for hidden units nh = [n_in] + n_h linear_h = [] f_nl_h = [] for i in range(len(n_h)): s = name + '_' + str(i) linear_h.append(N.linear_l2(s, nh[i], nh[i + 1], w)) f_nl_h.append(N.nonlinearity(s + '_nl', nl_h, (nh[i + 1], ), w)) # parameters for output f_nl_out = [] linear_out = [] for i in range(len(n_out)): s = name + '_out_' + str(i) linear_out.append(N.linear_l2(s, n_h[-1], n_out[i], w)) f_nl_out.append(N.nonlinearity(s + 'nl', nl_out, (n_out[i], ), w)) def f(h, w, return_hiddens=False): if nl_in != None: h = f_nl_in(h, w) hiddens = [] for i in range(len(n_h)): h = linear_h[i](h, w) h = f_nl_h[i](h, w) hiddens.append(h) out = [] for i in range(len(n_out)): _out = linear_out[i](h, w) _out = f_nl_out[i](_out, w) out.append(_out) if len(n_out) == 1: out = out[0] if return_hiddens: return hiddens, out return out def postup(updates, w): for l in linear_h: updates = l.postup(updates, w) for l in linear_out: updates = l.postup(updates, w) return updates return G.Struct(__call__=f, w=w, postup=postup)
def mlp(name, n_in, n_context, n_h, n_out, nl, w={}): if not isinstance(n_out, list) and isinstance(n_out, int): n_out = [n_out] if n_context > 0: # parameters for context input linear_context = N.linear_l2(name + '_context', n_context, n_h[0], w) # parameters for hidden units nh = [n_in] + n_h linear_h = [] f_nl_h = [] for i in range(len(n_h)): s = name + '_' + str(i) linear_h.append(linear(s, nh[i], nh[i + 1], False, True, w)) f_nl_h.append(N.nonlinearity(s + '_nl', nl, (nh[i + 1], ), w)) # parameters for output linear_out = [] for i in range(len(n_out)): s = name + '_out_' + str(i) linear_out.append(linear(s, n_h[-1], n_out[i], True, True, w)) def f(h, h_context, w, return_hiddens=False): # h_context can be None if n_context == 0 hiddens = [] for i in range(len(n_h)): h = linear_h[i](h, w) if i == 0 and n_context > 0: h += linear_context(h_context, w) h = f_nl_h[i](h, w) hiddens.append(h) out = [] for i in range(len(n_out)): _out = linear_out[i](h, w) out.append(_out) if len(n_out) == 1: out = out[0] if return_hiddens: return hiddens, out return out def postup(updates, w): if n_context > 0: updates = linear_context.postup(updates, w) for l in linear_h: updates = l.postup(updates, w) for l in linear_out: updates = l.postup(updates, w) return updates return G.Struct(__call__=f, w=w, postup=postup)
def zero_centered_laplace(name, w={}): w[name + '_logscale'] = G.sharedf(0.) def logp(v, w): return -abs(v).sum() / T.exp(w[name + '_logscale']) - v.size.astype( G.floatX) * (T.log(2.) + w[name + '_logscale']) postup = lambda updates, w: updates return G.Struct(logp=logp, postup=postup, w=w)
def zero_centered_gaussian(name, w={}): w[name + '_logvar'] = G.sharedf(0.) def logp(v, w): logvar = w[name + '_logvar'] * 10 return v.size.astype(G.floatX) * -.5 * ( T.log(2. * math.pi) + logvar) - .5 * (v**2).sum() / T.exp(logvar) postup = lambda updates, w: updates return G.Struct(logp=logp, postup=postup, w=w)
def resnetv2_layer_a(name, n_feats, nl='relu', w={}): f_nl = N.nonlinearity(name+"_nl", nl) # either no change in shape, or subsampling conv1 = conv2d(name+'_conv1', n_feats, n_feats, (3,3), w=w) conv2 = conv2d(name+'_conv2', n_feats, n_feats, (3,3), w=w) def f(_input, w): h = _input h = f_nl(conv1(h, w)) h = conv2(h, w) return T.nnet.relu(_input + .1 * h) def postup(updates, w): updates = conv1.postup(updates, w) updates = conv2.postup(updates, w) return updates return G.Struct(__call__=f, w=w, postup=postup)
def resnetv3(name, n_layers, n_feats, nl='softplus', layertype='a', factor=4, w={}): layers = [] for i in range(n_layers): if layertype == 'a': layers.append(resnetv3_layer_a(name+'_'+str(i), n_feats, nl, .1/n_layers, w)) if layertype == 'b': layers.append(resnetv3_layer_b(name+'_'+str(i), n_feats, factor, nl, .1/n_layers, w)) def f(h, w): for i in range(n_layers): h = layers[i](h, w) return h def postup(updates, w): for i in range(n_layers): updates = layers[i].postup(updates, w) return updates return G.Struct(__call__=f, w=w, postup=postup)
def resnetv1_layer(name, n_in, n_out, size_kernel=(3,3), downsample=1, upsample=1, nl='relu', w={}): #print 'resnet_layer', name, shape_in, shape_out, size_kernel, downsample, upsample f_nl = N.nonlinearity(name+"_nl", nl) border_mode = 'valid' if upsample == 1: # either no change in shape, or subsampling conv1 = conv2d(name+'_conv1', n_in, n_out, size_kernel, True, border_mode, downsample, upsample, w=w) conv2 = conv2d(name+'_conv2', n_out, n_out, size_kernel, True, border_mode, downsample=1, upsample=1, w=w) conv3 = None if downsample>1 or upsample>1 or n_out != n_in: conv3 = conv2d(name+'_conv3', n_in, n_out, (downsample, downsample), None, 'valid', downsample, upsample, w=w) else: # upsampling assert downsample == 1 conv1 = conv2d(name+'_conv1', n_in, n_in, size_kernel, True, border_mode, downsample=1, upsample=1, w=w) conv2 = conv2d(name+'_conv2', n_in, n_out, size_kernel, True, border_mode, downsample, upsample, w=w) conv3 = None if downsample>1 or upsample>1 or n_out != n_in: conv3 = conv2d(name+'_conv3', n_in, n_out, (downsample, downsample), None, 'valid', downsample, upsample, w=w) def f(_input, w): hidden = f_nl(conv1(_input, w)) _output = .1 * conv2(hidden, w) if conv3 != None: return T.nnet.relu(conv3(_input, w) + _output) return T.nnet.relu(_input + _output) def postup(updates, w): updates = conv1.postup(updates, w) updates = conv2.postup(updates, w) if conv3 != None: updates = conv3.postup(updates, w) return updates return G.Struct(__call__=f, w=w, postup=postup)
def cvae1(shape_x, depths, depth_ar, n_h1, n_h2, n_z, prior='diag', posterior='down_diag', px='logistic', nl='softplus', kernel_x=(5, 5), kernel_h=(3, 3), kl_min=0, optim='adamax', alpha=0.002, beta1=0.1, beta2=0.001, weightsharing=None, pad_x=0, data_init=None, downsample_type='nn'): _locals = locals() _locals.pop('data_init') print 'CVAE1 with ', _locals #assert posterior in ['diag1','diag2','iaf_linear','iaf_nonlinear'] assert px in ['logistic', 'bernoulli'] w = {} # model params if pad_x > 0: shape_x[1] += 2 * pad_x shape_x[2] += 2 * pad_x # Input whitening if px == 'logistic': w['logsd_x'] = G.sharedf(0.) # encoder x_enc = N.conv.conv2d('x_enc', shape_x[0], n_h1, kernel_x, downsample=2, w=w) x_dec = N.conv.conv2d('x_dec', n_h1, shape_x[0], kernel_x, upsample=2, w=w) x_dec_nl = N.nonlinearity('x_dec_nl', nl, n_h1, w) layers = [] for i in range(len(depths)): layers.append([]) for j in range(depths[i]): downsample = (i > 0 and j == 0) if weightsharing is None or not weightsharing: name = str(i) + '_' + str(j) elif weightsharing == 'all': name = '[sharedw]' + str(i) + '_' + str(j) + '[/sharedw]' elif weightsharing == 'acrosslevels': name = '[sharedw]' + str(i) + '[/sharedw]' + '_' + str(j) elif weightsharing == 'withinlevel': name = '[sharedw]' + str(i) + '[/sharedw]' + '_' + str(j) else: raise Exception() layers[i].append( cvae_layer(name, prior, posterior, n_h1, n_h2, n_z, depth_ar, downsample, nl, kernel_h, False, downsample_type, w)) # top-level value w['h_top'] = G.sharedf(np.zeros((n_h1, ))) # Initialize variables x = T.tensor4('x', dtype='uint8') x.tag.test_value = data_init['x'] n_batch_test = data_init['x'].shape[0] _x = T.clip((x + .5) / 256., 0, 1) #_x = T.clip(x / 255., 0, 1) if pad_x > 0: _x = N.conv.pad2d(_x, pad_x) # Objective function def f_encode_decode(w, train=True): results = {} h = x_enc(_x - .5, w) obj_kl = G.sharedf(0.) # bottom-up encoders for i in range(len(depths)): for j in range(depths[i]): h = layers[i][j].up(h, w) # top-level activations h = T.tile(w['h_top'].dimshuffle('x', 0, 'x', 'x'), (_x.shape[0], 1, shape_x[1] / 2**len(depths), shape_x[2] / 2**len(depths))) # top-down priors, posteriors and decoders for i in list(reversed(range(len(depths)))): for j in list(reversed(range(depths[i]))): h, kl = layers[i][j].down_q(h, train, w) kl_sum = kl.sum(axis=(1, 2, 3)) results['cost_z' + str(i).zfill(3) + '_' + str(j).zfill(3)] = kl_sum # Constraint: Minimum number of bits per featuremap, averaged across minibatch if kl_min > 0: if True: kl = kl.sum(axis=(2, 3)).mean(axis=0, dtype=G.floatX) obj_kl += T.maximum(np.asarray(kl_min, G.floatX), kl).sum(dtype=G.floatX) else: kl = T.maximum(np.asarray(kl_min, G.floatX), kl.sum(axis=(2, 3))).sum(axis=1, dtype=G.floatX) obj_kl += kl else: obj_kl += kl_sum output = .1 * x_dec(x_dec_nl(h, w), w) # empirical distribution if px == 'logistic': mean_x = T.clip(output + .5, 0 + 1 / 512., 1 - 1 / 512.) logsd_x = 0 * mean_x + w['logsd_x'] obj_logpx = N.rand.discretized_logistic(mean_x, logsd_x, 1 / 256., _x).logp #obj_z = T.printing.Print('obj_z')(obj_z) obj = obj_logpx - obj_kl # Compute the bits per pixel obj *= (1. / np.prod(shape_x) * 1. / np.log(2.)).astype('float32') #if not '__init' in w: # raise Exception() elif px == 'bernoulli': prob_x = T.nnet.sigmoid(output) prob_x = T.maximum(T.minimum(prob_x, 1 - 1e-7), 1e-7) #prob_x = T.printing.Print('prob_x')(prob_x) obj_logpx = N.rand.bernoulli(prob_x, _x).logp #obj_logqz = T.printing.Print('obj_logqz')(obj_logqz) #obj_logpz = T.printing.Print('obj_logpz')(obj_logpz) #obj_logpx = T.printing.Print('obj_logpx')(obj_logpx) obj = obj_logpx - obj_kl #obj = T.printing.Print('obj')(obj) results['cost_x'] = -obj_logpx results['cost'] = -obj return results # Turns Gaussian noise 'eps' into a sample def f_decoder(eps, w): # top-level activations h = T.tile(w['h_top'].dimshuffle('x', 0, 'x', 'x'), (eps['eps_0_0'].shape[0], 1, shape_x[1] / 2**len(depths), shape_x[2] / 2**len(depths))) # top-down priors, posteriors and decoders for i in list(reversed(range(len(depths)))): for j in list(reversed(range(depths[i]))): h = layers[i][j].down_p(h, eps['eps_' + str(i) + '_' + str(j)], w) output = .1 * x_dec(x_dec_nl(h, w), w) if px == 'logistic': mean_x = T.clip(output + .5, 0 + 1 / 512., 1 - 1 / 512.) elif px == 'bernoulli': mean_x = T.nnet.sigmoid(output) image = (256. * mean_x).astype('uint8') if pad_x > 0: image = image[:, :, pad_x:-pad_x, pad_x:-pad_x] return image def f_eps(n_batch, w): eps = {} for i in range(len(depths)): for j in range(depths[i]): eps['eps_' + str(i) + '_' + str(j)] = G.rng_curand.normal( (n_batch, n_z, shape_x[1] / 2**(i + 1), shape_x[2] / 2**(i + 1)), dtype=floatX) return eps def postup(updates, w): nodes = [x_enc, x_dec] for n in nodes: updates = n.postup(updates, w) for i in range(len(depths)): for j in range(depths[i]): updates = layers[i][j].postup(updates, w) return updates # Compile init function if data_init != None: w['__init'] = OrderedDict() f_encode_decode(w) w.pop('__init') #for i in w: print i, abs(w[i].get_value()).min(), abs(w[i].get_value()).max(), abs(w[i].get_value()).mean() # Compile training function #todo: replace postup with below #w['_updates'] = updates #f_cost(w) #updates = w.pop('_updates') w_avg = {i: G.sharedf(w[i].get_value()) for i in w} def lazy(f): def newf(*args, **kws): if not hasattr(f, 'cache'): f.cache = f() return f.cache(*args, **kws) return newf @lazy def f_train(): if optim == 'adamax': train_cost = f_encode_decode(w)['cost'] updates = G.misc.optim.AdaMaxAvg([w], [w_avg], train_cost, alpha=-alpha, beta1=beta1, beta2=beta2, disconnected_inputs='ignore') elif optim == 'eve': f = lambda w: f_encode_decode(w)['cost'] train_cost, updates = G.misc.optim.Eve( w, w_avg, f, alpha=-alpha, beta1=beta1, beta2=beta2, disconnected_inputs='ignore') updates = postup(updates, w) return G.function({'x': x}, train_cost, updates=updates, lazy=lazy) @lazy def f_train_q(): keys_q = [] for i in w: if '_q_' in i: keys_q.append(i) train_cost = f_encode_decode(w)['cost'] updates = G.misc.optim.AdaMaxAvg([w], None, train_cost, alpha=-alpha, beta1=beta1, beta2=beta2, update_keys=keys_q, disconnected_inputs='ignore') updates = postup(updates, w) return G.function({'x': x}, train_cost, updates=updates, lazy=lazy) # Compile evaluation function @lazy def f_eval(): results = f_encode_decode(w_avg, False) return G.function({'x': x}, results) # Compile epsilon generating function @lazy def f_eps_(): n_batch = T.lscalar() n_batch.tag.test_value = 16 eps = f_eps(n_batch, w) return G.function({'n_batch': n_batch}, eps, lazy=lazy) # Compile sampling function @lazy def f_decode(): eps = {} for i in range(len(depths)): for j in range(depths[i]): eps['eps_' + str(i) + '_' + str(j)] = T.tensor4('eps' + str(i)) eps['eps_' + str(i) + '_' + str(j)].tag.test_value = np.random.randn( n_batch_test, n_z, shape_x[1] / 2**(i + 1), shape_x[2] / 2**(i + 1)).astype(floatX) image = f_decoder(eps, w_avg) return G.function(eps, image, lazy=lazy) return G.Struct(train=f_train, eval=f_eval, decode=f_decode, eps=f_eps_, w=w, w_avg=w_avg)
def cvae_layer(name, prior, posterior, n_h1, n_h2, n_z, depth_ar, downsample, nl, kernel, weightsharing, downsample_type, w): if False: # New such that we can recognize variational params later name_q = name + '_q_' name_p = name + '_p_' else: name_q = name name_p = name n_conv_up1 = n_h2 + 2 * n_z n_conv_up2 = n_h2 + n_z n_conv_down_posterior = 0 n_conv_down_prior = n_h2 + 2 * n_z # Prior prior_conv1 = None if prior in ['diag', 'diag2']: n_conv_down_prior = n_h2 + 2 * n_z elif prior == 'made': prior_conv1 = N.ar.multiconv2d(name_p + '_prior_conv1', n_z, depth_ar * [n_h2], [n_z, n_z], kernel, False, nl=nl, w=w) n_conv_down_prior = n_h2 + n_h2 elif prior == 'bernoulli': n_conv_down_prior = n_h2 + n_z prior_conv1 = N.conv.conv2d(name_p + '_prior_conv1', n_z, n_z, kernel, w=w) else: raise Exception("Unknown prior") # Posterior posterior_conv1 = None posterior_conv2 = None posterior_conv3 = None posterior_conv4 = None if posterior == 'up_diag': pass elif posterior == 'up_iaf1': posterior_conv1 = N.ar.conv2d(name_q + '_posterior_conv1', n_z, n_z, kernel, w=w) elif posterior == 'up_iaf2': posterior_conv1 = N.ar.conv2d(name_q + '_posterior_conv1', n_z, 2 * n_z, kernel, w=w) elif posterior == 'up_iaf1_nl': n_conv_up1 = n_h2 + 2 * n_z + n_h2 posterior_conv1 = N.ar.multiconv2d(name_q + '_posterior_conv1', n_z, depth_ar * [n_h2], n_z, kernel, False, nl=nl, w=w) elif posterior == 'up_iaf2_nl': n_conv_up1 = n_h2 + 2 * n_z + n_h2 posterior_conv1 = N.ar.multiconv2d(name_q + '_posterior_conv1', n_z, depth_ar * [n_h2], [n_z, n_z], kernel, False, nl=nl, w=w) # elif posterior == 'down_diag': # n_conv_down1 = n_h2+4*n_z elif posterior == 'down_diag': n_conv_up2 = n_h2 n_conv_down_posterior = 2 * n_z elif posterior == 'down_bernoulli': n_conv_up2 = n_h2 n_conv_down_posterior = n_z elif posterior == 'down_tim': pass elif posterior == 'down_iaf1': n_conv_up2 = n_h2 n_conv_down_posterior = 2 * n_z posterior_conv1 = N.ar.conv2d(name_q + '_posterior_conv1', n_z, n_z, kernel, w=w) elif posterior == 'down_iaf2': n_conv_up2 = n_h2 n_conv_down_posterior = 2 * n_z posterior_conv1 = N.ar.conv2d(name_q + '_posterior_conv1', n_z, 2 * n_z, kernel, w=w) elif posterior == 'down_iaf1_nl': n_conv_up1 = n_h2 + 2 * n_z + n_h2 n_conv_up2 = n_h2 n_conv_down_posterior = 2 * n_z + n_h2 posterior_conv1 = N.ar.multiconv2d(name_q + '_posterior_conv1', n_z, depth_ar * [n_h2], n_z, kernel, False, nl=nl, w=w) elif posterior == 'down_iaf2_nl': n_conv_up1 = n_h2 + 2 * n_z + n_h2 n_conv_up2 = n_h2 n_conv_down_posterior = 2 * n_z + n_h2 posterior_conv1 = N.ar.multiconv2d(name_q + '_posterior_conv1', n_z, depth_ar * [n_h2], [n_z, n_z], kernel, False, nl=nl, w=w) elif posterior == 'down_iaf2_nl2': n_conv_up1 = n_h2 + 2 * n_z + n_h2 n_conv_up2 = n_h2 n_conv_down_posterior = 2 * n_z + n_h2 posterior_conv1 = N.ar.multiconv2d(name_q + '_posterior_conv1', n_z, depth_ar * [n_h2], [n_z, n_z], kernel, False, nl=nl, w=w) posterior_conv2 = N.ar.multiconv2d(name_q + '_posterior_conv2', n_z, depth_ar * [n_h2], [n_z, n_z], kernel, True, nl=nl, w=w) elif posterior == 'down_iaf1_deep': n_conv_up1 = n_h2 + 2 * n_z + n_h2 n_conv_up2 = n_h2 n_conv_down_posterior = 2 * n_z + n_h2 posterior_conv1 = N.ar.resnet(name_q + '_deepiaf', depth_ar, n_z, n_h2, n_z, kernel, False, nl=nl, weightsharing=weightsharing, w=w) elif posterior == 'down_iaf2_deep': n_conv_up1 = n_h2 + 2 * n_z + n_h2 n_conv_up2 = n_h2 n_conv_down_posterior = 2 * n_z + n_h2 posterior_conv1 = N.ar.resnet(name_q + '_deepiaf', depth_ar, n_z, n_h2, [n_z, n_z], kernel, False, nl=nl, weightsharing=weightsharing, w=w) #elif posterior == 'iaf_deep1': # extra1 = N.ar.resnet(name+'_posterior_2', depth_iaf, n_z, 2*n_h, n_h, n_z, (3,3), False, nl=nl, w=w) #elif posterior == 'iaf_deep2': # extra1 = N.ar.resnet(name+'_posterior_2', depth_iaf, n_z, 2*n_h, n_h, [n_z,n_z], (3,3), False, nl=nl, w=w) else: raise Exception("Unknown posterior " + posterior) ds = 1 if downsample: ds = 2 if downsample_type == 'conv': up_conv3 = N.conv.conv2d(name_q + '_up_conv3', n_h1, n_h1, kernel, downsample=ds, w=w) down_conv3 = N.conv.conv2d(name_q + '_down_conv3', n_h1, n_h1, kernel, upsample=ds, w=w) up_nl1 = N.nonlinearity(name_q + "_up_nl1", nl) up_conv1 = N.conv.conv2d(name_q + '_up_conv1_' + str(ds), n_h1, n_conv_up1, kernel, downsample=ds, w=w) up_nl2 = N.nonlinearity(name_q + "_nl_up2", nl) up_conv2 = N.conv.conv2d(name_q + '_up_conv2', n_conv_up2, n_h1, kernel, w=w) down_nl1 = N.nonlinearity(name_p + "_down_nl1", nl) down_conv1 = N.conv.conv2d(name_p + '_down_conv1', n_h1, n_conv_down_prior + n_conv_down_posterior, kernel, w=w) down_nl2 = N.nonlinearity(name_p + "_down_nl2", nl) down_conv2 = N.conv.conv2d(name_p + '_down_conv2_' + str(ds), n_h2 + n_z, n_h1, kernel, upsample=ds, w=w) up_output = [None] qz = [None] up_context = [None] def up(input, w): h = up_conv1(up_nl1(input, w), w) h_det = h[:, :n_h2, :, :] qz_mean = h[:, n_h2:n_h2 + n_z, :, :] qz_logsd = h[:, n_h2 + n_z:n_h2 + 2 * n_z, :, :] qz[0] = N.rand.gaussian_diag(qz_mean, 2 * qz_logsd) if posterior == 'up_diag': h = T.concatenate([h_det, qz[0].sample], axis=1) elif posterior == 'up_iaf1': arw_mean = posterior_conv1(qz[0].sample, w) arw_mean *= .1 qz[0].sample = (qz[0].sample - arw_mean) h = T.concatenate([h_det, qz[0].sample], axis=1) elif posterior == 'up_iaf2': arw_mean_logsd = posterior_conv1(qz[0].sample, w) arw_mean = arw_mean_logsd[:, ::2, :, :] arw_logsd = arw_mean_logsd[:, 1::2, :, :] arw_mean *= .1 arw_logsd *= .1 qz[0].sample = (qz[0].sample - arw_mean) / T.exp(arw_logsd) qz[0].logps += arw_logsd qz[0].logp += arw_logsd.flatten(2).sum(axis=1) h = T.concatenate([h_det, qz[0].sample], axis=1) elif posterior == 'up_iaf1_nl': context = h[:, n_h2 + 2 * n_z:n_h2 + 2 * n_z + n_h2] arw_mean = posterior_conv1(qz[0].sample, context, w) arw_mean *= .1 qz[0].sample = (qz[0].sample - arw_mean) h = T.concatenate([h_det, qz[0].sample], axis=1) elif posterior == 'up_iaf2_nl': context = h[:, n_h2 + 2 * n_z:n_h2 + 2 * n_z + n_h2] arw_mean, arw_logsd = posterior_conv1(qz[0].sample, context, w) arw_mean *= .1 arw_logsd *= .1 qz[0].sample = (qz[0].sample - arw_mean) / T.exp(arw_logsd) qz[0].logps += arw_logsd qz[0].logp += arw_logsd.flatten(2).sum(axis=1) h = T.concatenate([h_det, qz[0].sample], axis=1) elif posterior == 'down_tim': h = T.concatenate([h_det, qz[0].mean], axis=1) elif posterior in [ 'down_iaf1_nl', 'down_iaf2_nl', 'down_iaf2_nl2', 'down_iaf1_deep', 'down_iaf2_deep' ]: up_context[0] = h[:, n_h2 + 2 * n_z:n_h2 + 2 * n_z + n_h2] h = h_det elif posterior in [ 'down_diag', 'down_iaf1', 'down_iaf2', 'down_bernoulli' ]: h = h_det else: raise Exception() if downsample: if downsample_type == 'nn': input = N.conv.downsample2d_nearest_neighbour(input, 2) elif downsample_type == 'conv': input = up_conv3(input, w) output = input + .1 * up_conv2(up_nl2(h, w), w) up_output[0] = output return output def bernoulli_p(h): #p = T.clip(.5+.5*h, 1e-7, 1. - 1e-7) p = 1e-7 + (1 - 2e-7) * T.nnet.sigmoid(h) return p def down_q(input, train, w): #if name == '1': # print input.tag.test_value # prior h = down_nl1(input, w) #h = T.printing.Print('h1'+name)(h) h = down_conv1(h, w) #h = T.printing.Print('h2'+name)(h) logqs = 0 # posterior if posterior in [ 'up_diag', 'up_iaf1', 'up_iaf2', 'up_iaf1_nl', 'up_iaf2_nl' ]: z = qz[0].sample logqs = qz[0].logps elif posterior == 'down_diag': rz_mean = h[:, n_conv_down_prior:n_conv_down_prior + n_z, :, :] rz_logsd = h[:, n_conv_down_prior + n_z:n_conv_down_prior + 2 * n_z, :, :] _qz = N.rand.gaussian_diag(qz[0].mean + rz_mean, qz[0].logvar + 2 * rz_logsd) z = _qz.sample logqs = _qz.logps elif posterior == 'down_tim': assert prior == 'diag' pz_mean = h[:, n_h2:n_h2 + n_z, :, :] pz_logsd = h[:, n_h2 + n_z:n_h2 + 2 * n_z, :, :] qz_prec = 1. / T.exp(qz[0].logvar) pz_prec = 1. / T.exp(2 * pz_logsd) rz_prec = qz_prec + pz_prec rz_mean = (pz_prec / rz_prec) * pz_mean + (qz_prec / rz_prec) * qz[0].mean _qz = N.rand.gaussian_diag(rz_mean, -T.log(rz_prec)) z = _qz.sample logqs = _qz.logps elif posterior == 'down_iaf1': rz_mean = h[:, n_conv_down_prior:n_conv_down_prior + n_z, :, :] rz_logsd = h[:, n_conv_down_prior + n_z:n_conv_down_prior + 2 * n_z, :, :] _qz = N.rand.gaussian_diag(qz[0].mean + rz_mean, qz[0].logvar + 2 * rz_logsd) z = _qz.sample logqs = _qz.logps # ARW transform arw_mean = posterior_conv1(z, w) arw_mean *= .1 z = (z - arw_mean) elif posterior == 'down_iaf2': rz_mean = h[:, n_conv_down_prior:n_conv_down_prior + n_z, :, :] rz_logsd = h[:, n_conv_down_prior + n_z:n_conv_down_prior + 2 * n_z, :, :] _qz = N.rand.gaussian_diag(qz[0].mean + rz_mean, qz[0].logvar + 2 * rz_logsd) z = _qz.sample logqs = _qz.logps # ARW transform arw_mean_logsd = posterior_conv1(z, w) arw_mean = arw_mean_logsd[:, ::2, :, :] arw_logsd = arw_mean_logsd[:, 1::2, :, :] arw_mean *= .1 arw_logsd *= .1 z = (z - arw_mean) / T.exp(arw_logsd) logqs += arw_logsd elif posterior in ['down_iaf1_nl', 'down_iaf1_deep']: rz_mean = h[:, n_conv_down_prior:n_conv_down_prior + n_z, :, :] rz_logsd = h[:, n_conv_down_prior + n_z:n_conv_down_prior + 2 * n_z, :, :] _qz = N.rand.gaussian_diag(qz[0].mean + rz_mean, qz[0].logvar + 2 * rz_logsd) z = _qz.sample logqs = _qz.logps # ARW transform down_context = h[:, n_conv_down_prior + 2 * n_z:n_conv_down_prior + 2 * n_z + n_h2, :, :] context = up_context[0] + down_context arw_mean = posterior_conv1(z, context, w) arw_mean *= .1 z = (z - arw_mean) elif posterior in ['down_iaf2_nl', 'down_iaf2_nl2', 'down_iaf2_deep']: rz_mean = h[:, n_conv_down_prior:n_conv_down_prior + n_z, :, :] rz_logsd = h[:, n_conv_down_prior + n_z:n_conv_down_prior + 2 * n_z, :, :] _qz = N.rand.gaussian_diag(qz[0].mean + rz_mean, qz[0].logvar + 2 * rz_logsd) z = _qz.sample logqs = _qz.logps # ARW transform down_context = h[:, n_conv_down_prior + 2 * n_z:n_conv_down_prior + 2 * n_z + n_h2, :, :] context = up_context[0] + down_context arw_mean, arw_logsd = posterior_conv1(z, context, w) arw_mean *= .1 arw_logsd *= .1 z = (z - arw_mean) / T.exp(arw_logsd) logqs += arw_logsd if posterior == 'down_iaf2_nl2': arw_mean, arw_logsd = posterior_conv2(z, context, w) arw_mean *= .1 arw_logsd *= .1 z = (z - arw_mean) / T.exp(arw_logsd) logqs += arw_logsd # Prior if prior == 'diag': pz_mean = h[:, n_h2:n_h2 + n_z, :, :] pz_logsd = h[:, n_h2 + n_z:n_h2 + 2 * n_z, :, :] logps = N.rand.gaussian_diag(pz_mean, 2 * pz_logsd, z).logps elif prior == 'diag2': logps = N.rand.gaussian_diag(0 * z, 0 * z, z).logps pz_mean = h[:, n_h2:n_h2 + n_z, :, :] pz_logsd = h[:, n_h2 + n_z:n_h2 + 2 * n_z, :, :] z = pz_mean + z * T.exp(pz_logsd) elif prior == 'made': made_context = h[:, n_h2:2 * n_h2, :, :] made_mean, made_logsd = prior_conv1(z, made_context, w) made_mean *= .1 made_logsd *= .1 logps = N.rand.gaussian_diag(made_mean, 2 * made_logsd, z).logps elif prior == 'bernoulli': assert posterior == 'down_bernoulli' pz_p = bernoulli_p(h[:, n_h2:n_h2 + n_z, :, :]) logps = z01 * T.log(pz_p) + (1. - z01) * T.log(1. - pz_p) else: raise Exception() h_det = h[:, :n_h2, :, :] h = T.concatenate([h_det, z], axis=1) if downsample: if downsample_type == 'nn': input = N.conv.upsample2d_nearest_neighbour(input) elif downsample_type == 'conv': input = down_conv3(input, w) output = input + .1 * down_conv2(down_nl2(h, w), w) return output, logqs - logps def down_p(input, eps, w): # prior h = down_conv1(down_nl1(input, w), w) h_det = h[:, :n_h2, :, :] if prior in ['diag', 'diag2']: mean_prior = h[:, n_h2:n_h2 + n_z, :, :] logsd_prior = h[:, n_h2 + n_z:n_h2 + 2 * n_z, :, :] z = mean_prior + eps * T.exp(logsd_prior) elif prior == 'made': print "TODO: SAMPLES FROM MADE PRIOR" z = eps elif prior == 'bernoulli': assert posterior == 'down_bernoulli' pz_p = bernoulli_p(h[:, n_h2:n_h2 + n_z, :, :]) if False: z = N.rand.bernoulli(pz_p).sample else: print "Alert: Sampling using Gaussian approximation" z = pz_p + T.sqrt(pz_p * (1 - pz_p)) * eps z = prior_conv1(2 * z - 1, w) h = T.concatenate([h_det, z], axis=1) if downsample: if downsample_type == 'nn': input = N.conv.upsample2d_nearest_neighbour(input) elif downsample_type == 'conv': input = down_conv3(input, w) output = input + .1 * down_conv2(down_nl2(h, w), w) return output def postup(updates, w): modules = [up_conv1, up_conv2, down_conv1, down_conv2] if downsample and downsample_type == 'conv': modules += [up_conv3, down_conv3] if prior_conv1 != None: modules.append(prior_conv1) if posterior_conv1 != None: modules.append(posterior_conv1) if posterior_conv2 != None: modules.append(posterior_conv2) if posterior_conv3 != None: modules.append(posterior_conv3) if posterior_conv3 != None: modules.append(posterior_conv4) for m in modules: updates = m.postup(updates, w) return updates return G.Struct(up=up, down_q=down_q, down_p=down_p, postup=postup, w=w)
def resnet(name, depth, n_in, n_h, n_out, size_kernel=(3, 3), flipmask=False, nl='elu', layertype='a', factor=4, weightsharing=False, w={}): if not isinstance(n_out, list) and isinstance(n_out, int): n_out = [n_out] conv_input = conv2d(name + '_input', n_in, n_h, size_kernel, False, flipmask, w=w) # parameters for hidden units resnet = [] for i in range(depth): _name = name + '_' + str(i) if weightsharing: _name = name + '[sharedw]_' + str(i) + '[/sharedw]' if layertype == 'a': resnet.append(resnet_layer_a(_name, n_h, nl, w)) elif layertype == 'b': resnet.append(resnet_layer_b(_name, n_h, factor, nl, w)) else: raise Exception() # parameters for output conv_out = [ conv2d(name + '_out_' + str(i), n_h, n_out[i], size_kernel, True, flipmask, w=w) for i in range(len(n_out)) ] def f(h, h_context, w, return_hiddens=False): h = conv_input(h, w) if h_context != None: h += h_context hiddens = [] for i in range(len(resnet)): h = resnet[i](h, w) hiddens.append(h) out = [] for i in range(len(n_out)): _out = conv_out[i](h, w) out.append(_out) if len(n_out) == 1: out = out[0] if return_hiddens: return hiddens, out return out def postup(updates, w): for l in resnet: updates = l.postup(updates, w) for l in conv_out: updates = l.postup(updates, w) return updates return G.Struct(__call__=f, w=w, postup=postup)
def multiconv2d(name, n_in, n_h, n_out, size_kernel, flipmask, nl='relu', w={}): if not isinstance(n_out, list) and isinstance(n_out, int): n_out = [n_out] # parameters for hidden units sizes = [n_in] + n_h conv_h = [] f_nl_h = [] for i in range(len(n_h)): conv_h.append( conv2d(name + '_' + str(i), sizes[i], sizes[i + 1], size_kernel, False, flipmask, w=w)) f_nl_h.append( N.nonlinearity(name + '_' + str(i) + '_nl', nl, sizes[i + 1], w=w)) # parameters for output conv_out = [] for i in range(len(n_out)): conv_out.append( conv2d(name + '_out_' + str(i), sizes[-1], n_out[i], size_kernel, True, flipmask, w=w)) def f(h, context, w, return_hiddens=False): # h_context can be None if n_context == 0 hiddens = [] for i in range(len(n_h)): h = conv_h[i](h, w) # + context if i == 0: h += context h = f_nl_h[i](h, w) hiddens.append(h) out = [] for i in range(len(n_out)): _out = conv_out[i](h, w) out.append(_out) if len(n_out) == 1: out = out[0] if return_hiddens: return hiddens, out return out def postup(updates, w): for l in conv_h: updates = l.postup(updates, w) for l in conv_out: updates = l.postup(updates, w) return updates return G.Struct(__call__=f, w=w, postup=postup)
def conv2d(name, n_in, n_out, size_kernel=(3, 3), zerodiagonal=True, flipmask=False, pad_channel=True, border_mode='valid', zeroinit=False, l2norm=True, w={}): do_scale = False if zeroinit: l2norm = False do_scale = True if not pad_channel: border_mode = 'same' print 'No pad_channel, changing border_mode to same' #if 'whitener' not in name: # pad_channel = False # border_mode = 'same' if '[sharedw]' in name and '[/sharedw]' in name: name_w = name pre, b = name.split("[sharedw]") c, post = b.split("[/sharedw]") name_w = pre + "[s]" + post name = pre + c + post # Don't share the bias and scales #name = name_w # Also share the bias and scales else: name_w = name assert border_mode in ['valid', 'full', 'same'] _n_in = n_in if pad_channel: if size_kernel[0] > 1 or size_kernel[1] > 1: assert size_kernel[0] == size_kernel[1] assert border_mode == 'valid' _n_in += 1 else: pad_channel = False if border_mode == 'same': assert size_kernel[0] % 2 == 1 border_mode = ((size_kernel[0] - 1) / 2, (size_kernel[1] - 1) / 2) if True: # Build autoregressive mask l = (size_kernel[0] - 1) / 2 m = (size_kernel[1] - 1) / 2 mask = np.ones((n_out, _n_in, size_kernel[0], size_kernel[1]), dtype=G.floatX) mask[:, :, :l, :] = 0 mask[:, :, l, :m] = 0 if n_out >= n_in: assert n_out % n_in == 0 k = n_out / n_in for i in range(n_in): mask[i * k:(i + 1) * k, i + 1:, l, m] = 0 if zerodiagonal: mask[i * k:(i + 1) * k, i:i + 1, l, m] = 0 else: assert n_in % n_out == 0 k = n_in / n_out for i in range(n_out): mask[i:i + 1, (i + 1) * k:, l, m] = 0 if zerodiagonal: mask[i:i + 1, i * k:(i + 1) * k:, l, m] = 0 if flipmask: mask = mask[::-1, ::-1, ::-1, ::-1] def l2normalize(kerns): if zerodiagonal: # to prevent NaN gradients # TODO: smarter solution (also see below) l = (size_kernel[0] - 1) / 2 m = (size_kernel[1] - 1) / 2 if n_out >= n_in: kerns = T.set_subtensor(kerns[:n_out / n_in, :, l, m], 0.) else: kerns = T.set_subtensor(kerns[:1, :, l, m], 0.) targetnorm = 1. norm = T.sqrt((kerns**2).sum(axis=(1, 2, 3), keepdims=True)) norm += 1e-8 return kerns * (targetnorm / norm) def maxconstraint(kerns): return kerns * (maxweight / T.maximum( maxweight, abs(kerns).max(axis=(1, 2, 3), keepdims=True))) if zeroinit: w[name_w + '_w'] = G.sharedf( np.zeros((n_out, _n_in, size_kernel[0], size_kernel[1]))) else: w[name_w + '_w'] = G.sharedf( mask * 0.05 * np.random.randn(n_out, _n_in, size_kernel[0], size_kernel[1])) if maxweight > 0: w[name_w + '_w'].set_value( maxconstraint(w[name_w + '_w']).tag.test_value) w[name + '_b'] = G.sharedf(np.zeros((n_out, ))) if l2norm or do_scale: if logscale: w[name + '_s'] = G.sharedf(np.zeros((n_out, ))) else: w[name + '_s'] = G.sharedf(np.ones((n_out, ))) elif do_constant_rescale: print 'WARNING: constant rescale, these weights arent saved' constant_rescale = G.sharedf(np.ones((n_out, ))) def f(h, w): input_shape = h.tag.test_value.shape[1:] _input = h if pad_channel: h = N.conv.pad2dwithchannel(h, size_kernel) kerns = mask * w[name_w + '_w'] if l2norm: kerns = l2normalize(kerns) if l2norm or do_scale: if logscale: kerns *= T.exp(logscale_scale * w[name + '_s']).dimshuffle( 0, 'x', 'x', 'x') else: kerns *= w[name + '_s'].dimshuffle(0, 'x', 'x', 'x') elif do_constant_rescale: kerns *= constant_rescale.dimshuffle(0, 'x', 'x', 'x') h = N.conv.dnn_conv(h, kerns, border_mode=border_mode) # Center if bn: # mean-only batch norm h -= h.mean(axis=(0, 2, 3), keepdims=True) h += w[name + '_b'].dimshuffle('x', 0, 'x', 'x') if '__init' in w and not zeroinit: # Std data_std = h.std(axis=(0, 2, 3)) num_zeros = (data_std.tag.test_value == 0).sum() if num_zeros > 0: print "Warning: Stdev=0 for " + str( num_zeros ) + " features in " + name + ". Skipping data-dependent init." else: if name + '_s' in w: if logscale: w[name + '_s'].set_value( -T.log(data_std).tag.test_value / logscale_scale) else: w[name + '_s'].set_value( (1. / data_std).tag.test_value) elif do_constant_rescale: constant_rescale.set_value((1. / data_std).tag.test_value) #w[name+'_w'].set_value((kerns / std.dimshuffle(0,'x','x','x')).tag.test_value) h /= data_std.dimshuffle('x', 0, 'x', 'x') # Mean mean = h.mean(axis=(0, 2, 3)) w[name + '_b'].set_value(-mean.tag.test_value) h -= mean.dimshuffle('x', 0, 'x', 'x') #print name, w[name+'_w'].get_value().mean(), w[name+'_w'].get_value().std(), w[name+'_w'].get_value().max() if not '__init' in w: output_shape = h.tag.test_value.shape[1:] print 'ar.conv2d', name, input_shape, output_shape, size_kernel, zerodiagonal, flipmask, pad_channel, border_mode, zeroinit, l2norm #print name, abs(h).max().tag.test_value, abs(h).min().tag.test_value #h = T.printing.Print(name)(h) return h # Normalize weights to _norm L2 norm # TODO: check whether only_upper_bounds here really helps # (the effect is a higher learning rate in the beginning of training) def postup(updates, w): updates[w[name_w + '_w']] = mask * updates[w[name_w + '_w']] if l2norm and maxweight > 0.: updates[w[name_w + '_w']] = maxconstraint(updates[w[name_w + '_w']]) return updates return G.Struct(__call__=f, w=w, postup=postup)
def linear(name, n_in, n_out, diagonalzeros, l2norm=True, w={}): assert n_in % n_out == 0 or n_out % n_in == 0 mask = np.ones((n_in, n_out), dtype=G.floatX) if n_out >= n_in: k = n_out / n_in for i in range(n_in): mask[i + 1:, i * k:(i + 1) * k] = 0 if diagonalzeros: mask[i:i + 1, i * k:(i + 1) * k] = 0 else: k = n_in / n_out for i in range(n_out): mask[(i + 1) * k:, i:i + 1] = 0 if diagonalzeros: mask[i * k:(i + 1) * k:, i:i + 1] = 0 # L2 normalization of weights def l2normalize(_w, axis=0): if diagonalzeros: # to prevent NaN gradients # TODO: smarter solution (also see below) if n_out >= n_in: _w = T.set_subtensor(_w[:, :n_out / n_in], 0.) else: _w = T.set_subtensor(_w[:, :1], 0.) targetnorm = 1. norm = T.sqrt((_w**2).sum(axis=axis, keepdims=True)) norm += 1e-8 new_w = _w * (targetnorm / norm) return new_w def maxconstraint(_w): return _w * (maxweight / T.maximum(maxweight, abs(_w).max(axis=0, keepdims=True))) w[name + '_w'] = G.sharedf(mask * 0.05 * np.random.randn(n_in, n_out)) if maxweight > 0: w[name + '_w'].set_value(maxconstraint(w[name + '_w']).tag.test_value) w[name + '_b'] = G.sharedf(np.zeros((n_out, ))) if l2norm: if logscale: w[name + '_s'] = G.sharedf(np.zeros((n_out, ))) else: w[name + '_s'] = G.sharedf(np.ones((n_out, ))) elif do_constant_rescale: print 'WARNING: constant rescale, these weights arent saved' constant_rescale = G.sharedf(np.zeros((n_out, ))) def f(h, w): _input = h _w = mask * w[name + '_w'] if l2norm: _w = l2normalize(_w) h = T.dot(h, _w) if l2norm: if logscale: h *= T.exp(logscale_scale * w[name + '_s']) else: h *= abs(w[name + '_s']) elif do_constant_rescale: h *= T.exp(constant_rescale) h += w[name + '_b'] if '__init' in w: # Std std = (1. / init_stdev) * h.std(axis=0) std += (std <= 0) std += 1e-8 if name + '_s' in w: if logscale: w[name + '_s'].set_value(-T.log(std).tag.test_value / logscale_scale) else: w[name + '_s'].set_value((1. / std).tag.test_value) elif do_constant_rescale: constant_rescale.set_value(-T.log(std).tag.test_value) #w[name+'_w'].set_value((_w / std.dimshuffle('x',0)).tag.test_value) h /= std.dimshuffle('x', 0) # Mean mean = h.mean(axis=0) w[name + '_b'].set_value(-mean.tag.test_value) h -= mean.dimshuffle('x', 0) #print name, w[name+'_w'].get_value().mean(), w[name+'_w'].get_value().std(), w[name+'_w'].get_value().max() #print name, abs(h).max().tag.test_value, abs(h).min().tag.test_value #h = T.printing.Print(name)(h) return h # Post updates: normalize weights to unit L2 norm def postup(updates, w): updates[w[name + '_w']] = mask * updates[w[name + '_w']] if l2norm and maxweight > 0.: updates[w[name + '_w']] = maxconstraint(updates[w[name + '_w']]) return updates return G.Struct(__call__=f, postup=postup, w=w)
def linear_l2(name, n_in, n_out, w): # L2 normalization of weights def l2normalize(_w): targetnorm=1. norm = T.sqrt((_w**2).sum(axis=0, keepdims=True)) return _w * (targetnorm / norm) def maxconstraint(_w): return _w * (maxweight / T.maximum(maxweight, abs(_w).max(axis=0, keepdims=True))) w[name+'_w'] = G.sharedf(0.05*np.random.randn(n_in,n_out)) if maxweight > 0: w[name+'_w'].set_value(maxconstraint(w[name+'_w']).tag.test_value) w[name+'_b'] = G.sharedf(np.zeros((n_out,))) if l2norm: if logscale: w[name+'_s'] = G.sharedf(np.zeros((n_out,))) else: w[name+'_s'] = G.sharedf(np.ones((n_out,))) else: print 'WARNING: constant rescale, these weights arent saved' constant_rescale = G.sharedf(np.zeros((n_out,))) def f(h, w): _w = w[name+'_w'] if l2norm: _w = l2normalize(_w) h = T.dot(h, _w) if l2norm: if logscale: h *= T.exp(logscale_scale*w[name+'_s']) else: h *= abs(w[name+'_s']) else: h *= T.exp(constant_rescale) h += w[name+'_b'] if '__init' in w: # Std std = (1./init_stdev) * h.std(axis=0) + 1e-8 if name+'_s' in w: if logscale: w[name+'_s'].set_value(-T.log(std).tag.test_value/logscale_scale) else: w[name+'_s'].set_value((1./std).tag.test_value) else: constant_rescale.set_value(-T.log(std).tag.test_value) #w[name+'_w'].set_value((_w / std.dimshuffle('x',0)).tag.test_value) h /= std.dimshuffle('x',0) # Mean mean = h.mean(axis=0) w[name+'_b'].set_value(-mean.tag.test_value) h -= mean.dimshuffle('x',0) #print name, abs(w[name+'_w']).get_value().mean(), w[name+'_w'].get_value().std(), w[name+'_w'].get_value().max() #print name, abs(h).max().tag.test_value, abs(h).min().tag.test_value #h = T.printing.Print(name)(h) return h # Post updates: normalize weights to unit L2 norm def postup(updates, w): if l2norm and maxweight>0: updates[w[name+'_w']] = maxconstraint(updates[w[name+'_w']]) return updates return G.Struct(__call__=f, postup=postup, w=w)
def fcvae(shape_x, depth_model, depth_ar, n_h1, n_h2, n_z, posterior, px='logistic', nl='softplus', alpha=0.002, beta1=0.1, beta2=0.001, share_w=False, data_init=None): _locals = locals() _locals.pop('data_init') print 'CVAE9 with ', _locals #assert posterior in ['diag1','diag2','iaf_linear','iaf_nonlinear'] assert px in ['logistic', 'bernoulli'] w = {} # model params kernel_h = (1, 1) n_x = shape_x[0] * shape_x[1] * shape_x[2] # Input whitening if px == 'logistic': w['logsd_x'] = G.sharedf(0.) # encoder x_enc = N.conv.conv2d('x_enc', n_x, n_h1, (1, 1), w=w) x_dec = N.conv.conv2d('x_dec', n_h1, n_x, (1, 1), w=w) x_dec_nl = N.nonlinearity('x_dec_nl', nl, n_h1, w) layers = [] for i in range(depth_model): name = str(i) if share_w: name = '[sharedw]' + str(i) + '[/sharedw]' layers.append( cvae_layer(name, posterior, n_h1, n_h2, n_z, depth_ar, False, nl, kernel_h, share_w, w)) # top-level value #w['h_top'] = G.sharedf(np.zeros((n_h1,))) w['h_top'] = G.sharedf(np.random.normal(0, 0.01, size=(n_h1, ))) # Initialize variables x = T.tensor4('x') x.tag.test_value = data_init['x'] n_batch_test = data_init['x'].shape[0] _x = T.clip(x / 255., 0, 1) # Objective function def f_cost(w, train=True): results = {} h = x_enc(_x.reshape((-1, n_x, 1, 1)) - .5, w) obj_logpz = 0 obj_logqz = 0 # bottom-up encoders for i in range(depth_model): h = layers[i].up(h, w) # top-level activations h = T.tile(w['h_top'].dimshuffle('x', 0, 'x', 'x'), (_x.shape[0], 1, 1, 1)) # top-down priors, posteriors and decoders for i in list(reversed(range(depth_model))): h, _obj_logqz, _obj_logpz = layers[i].down_q(h, train, w) obj_logqz += _obj_logqz obj_logpz += _obj_logpz results['cost_z' + str(i).zfill(3)] = _obj_logqz - _obj_logpz output = .1 * x_dec(x_dec_nl(h, w), w).reshape( (-1, shape_x[0], shape_x[1], shape_x[2])) # empirical distribution if px == 'logistic': mean_x = T.clip(output, -.5, .5) logsd_x = 0 * mean_x + w['logsd_x'] obj_logpx = N.rand.discretized_logistic(mean_x, logsd_x, 1 / 255., _x - .5).logp obj = obj_logpz - obj_logqz + obj_logpx # Compute the bits per pixel obj *= (1. / np.prod(shape_x) * 1. / np.log(2.)).astype('float32') elif px == 'bernoulli': prob_x = T.nnet.sigmoid(output) prob_x = T.minimum(prob_x, 1 - 1e-7) prob_x = T.maximum(prob_x, 1e-7) #prob_x = T.printing.Print('prob_x')(prob_x) obj_logpx = N.rand.bernoulli(prob_x, _x).logp #obj_logqz = T.printing.Print('obj_logqz')(obj_logqz) #obj_logpz = T.printing.Print('obj_logpz')(obj_logpz) #obj_logpx = T.printing.Print('obj_logpx')(obj_logpx) obj = obj_logpz - obj_logqz + obj_logpx #obj = T.printing.Print('obj')(obj) results['cost_x'] = -obj_logpx results['cost'] = -obj return results #print 'obj_logpz', obj_logpz.tag.test_value #print 'obj_logqz', obj_logqz.tag.test_value #print 'obj_logpx', obj_x.tag.test_value #obj_logpz = T.printing.Print('obj_logpz')(obj_logpz) #obj_logqz = T.printing.Print('obj_logqz')(obj_logqz) #obj_x = T.printing.Print('obj_logpx')(obj_x) # Turns Gaussian noise 'eps' into a sample def f_decoder(eps, w): # top-level activations h = T.tile(w['h_top'].dimshuffle('x', 0, 'x', 'x'), (eps['eps_0'].shape[0], 1, 1, 1)) # top-down priors, posteriors and decoders for i in list(reversed(range(depth_model))): h = layers[i].down_p(h, eps['eps_' + str(i)], w) output = .1 * x_dec(x_dec_nl(h, w), w).reshape( (-1, shape_x[0], shape_x[1], shape_x[2])) if px == 'logistic': mean_x = T.clip(output[:, :, :, :] + .5, 0, 1) elif px == 'bernoulli': mean_x = T.nnet.sigmoid(output) image = (255. * T.clip(mean_x, 0, 1)).astype('uint8') return image def f_eps(n_batch, w): eps = {} for i in range(depth_model): eps['eps_' + str(i)] = G.rng_curand.normal((n_batch, n_z, 1, 1), dtype=floatX) return eps def postup(updates, w): nodes = [x_enc, x_dec] for n in nodes: updates = n.postup(updates, w) for i in range(depth_model): updates = layers[i].postup(updates, w) return updates # Compile init function if data_init != None: w['__init'] = OrderedDict() f_cost(w) w.pop('__init') #for i in w: print i, abs(w[i].get_value()).min(), abs(w[i].get_value()).max(), abs(w[i].get_value()).mean() # Compile training function results = f_cost(w) updates, (w_avg, ) = G.misc.optim.AdaMaxAvg([w], results['cost'], alpha=-alpha, beta1=beta1, beta2=beta2, disconnected_inputs='ignore') #todo: replace postup with below #w['_updates'] = updates #f_cost(w) #updates = w.pop('_updates') updates = postup(updates, w) f_train = G.function({'x': x}, results['cost'], updates=updates) # Compile evaluation function results = f_cost(w_avg, False) f_eval = G.function({'x': x}, results) # Compile epsilon generating function n_batch = T.lscalar() n_batch.tag.test_value = 16 eps = f_eps(n_batch, w) f_eps = G.function({'n_batch': n_batch}, eps) # Compile sampling function eps = {} for i in range(depth_model): eps['eps_' + str(i)] = T.tensor4('eps' + str(i)) eps['eps_' + str(i)].tag.test_value = np.random.randn( n_batch_test, n_z, 1, 1).astype(floatX) image = f_decoder(eps, w_avg) f_decode = G.function(eps, image) return G.Struct(train=f_train, eval=f_eval, decode=f_decode, eps=f_eps, w=w, w_avg=w_avg)
def RandomVariable(sample, logp, entr, **params): return G.Struct(sample=sample, logp=logp, entr=entr, **params)
def function(x, y, lazy=False, _debug=False, checknan='raise', **kwargs): # Default keyword arguments if not kwargs.has_key('on_unused_input'): kwargs['on_unused_input'] = 'warn' if not kwargs.has_key('mode'): kwargs['mode'] = default_function_mode # Order the input dict x = ndict.ordered(ndict.flatten(x)) # Check the output dict return_single_y = False if not isinstance(y, dict): return_single_y = True y = {str(y): y} y = ndict.ordered(y) # Lazily compiled function (saves a lot of time) f = [None] def _compile(verbose=True): t0 = time.time() print 'Compiling... ', #print '[graphy] Compiling function '+str(x.keys())+' => '+str(y.keys())+' ...' sys.stdout.flush() f[0] = theano.function(x.values(), y.values(), **kwargs) print "%.2f" % (time.time() - t0), 's' if not lazy: _compile() # The function to be called def func(data, n_batch=0, randomorder=True, data_global={}): data = ndict.ordered(ndict.flatten(data)) data_global = ndict.ordered(ndict.flatten(data_global)) # Check if keys of 'x' and 'inputs' match allkeys = (data.keys() + data_global.keys()) for i in range(len(data)): if x.keys()[i] not in allkeys: raise Exception('Non-matching keys:' + str(allkeys) + ' vs. ' + str(x.keys())) # Compile function if not already done if f[0] == None: _compile() if n_batch <= 0: # Get results _data = data.copy() _data.update(data_global) inputs_ordered = ndict.orderedvals((_data, )) _result = f[0](*inputs_ordered) # Put it in a dictionary with the corresponding keys result = {y.keys()[i]: _result[i] for i in range(len(y))} else: # Minibatch-based evaluation. # This assumes that input and output are tensors, and the first dimension iterates of datapoints n_tot = data.itervalues().next().shape[0] n_minibatches = int(math.ceil(1. * n_tot / n_batch)) n_tile = 1 if n_batch > n_tot: assert n_batch % n_tot == 0 n_tile = n_batch / n_tot indices = np.tile(np.arange(n_tot), n_tile) if randomorder: np.random.shuffle(indices) adict = dict(zip(np.tile(np.arange(n_tot), n_tile), indices)) indices_inverse = sorted(adict, key=adict.get) results = [] for i in range(n_minibatches): data_minibatch = ndict.getRowsFromIndices( data, indices[i * n_batch:(i + 1) * n_batch]) data_minibatch.update(data_global) inputs_ordered = ndict.orderedvals((data_minibatch, )) results.append(f[0](*inputs_ordered)) if _debug: print 'Function debug', i, results[-1] if checknan == 'raise': if np.isnan(np.sum(results[-1])): print results[-1] raise Exception("NaN detected") result = { y.keys()[i]: np.concatenate([results[j][i] for j in range(n_minibatches)]) for i in range(len(y)) } if randomorder: result = ndict.getRowsFromIndices(result, indices_inverse) result = OrderedDict(sorted(result.items())) # Return result #raise Exception() if return_single_y: return result[result.keys()[0]] return result # Return the func return G.Struct(__call__=func, f=f)
def conv2d(name, n_in, n_out, size_kernel=(3,3), pad_channel=True, border_mode='valid', downsample=1, upsample=1, datainit=True, zeroinit=False, l2norm=True, w={}): # TODO FIX: blows up parameters if all inputs are 0 if not pad_channel: border_mode = 'same' print 'No pad_channel, changing border_mode to same' if '[sharedw]' in name and '[/sharedw]' in name: name_w = name pre, b = name.split("[sharedw]") number, post = b.split("[/sharedw]") name_w = pre+"[s]"+post name = pre+number+post # Don't share the bias and scales #name = name_w # Also share the bias and scales else: name_w = name if type(downsample) == int: downsample = (downsample,downsample) assert type(downsample) == tuple assert border_mode in ['valid','full','same'] _n_in = n_in _n_out = n_out if upsample > 1: _n_out = n_out * upsample**2 if pad_channel: if size_kernel[0] > 1 or size_kernel[1] > 1: assert size_kernel[0] == size_kernel[1] assert border_mode == 'valid' _n_in += 1 else: pad_channel = False if border_mode == 'same': assert size_kernel[0]%2 == 1 border_mode = ((size_kernel[0]-1)/2,(size_kernel[1]-1)/2) def l2normalize(kerns): norm = T.sqrt((kerns**2).sum(axis=(1,2,3), keepdims=True)) return kerns / norm def maxconstraint(kerns): return kerns * (maxweight / T.maximum(maxweight, abs(kerns).max(axis=(1,2,3), keepdims=True))) if zeroinit: w[name_w+'_w'] = G.sharedf(np.zeros((_n_out, _n_in, size_kernel[0], size_kernel[1]))) datainit = False else: w[name_w+'_w'] = G.sharedf(0.05*np.random.randn(_n_out, _n_in, size_kernel[0], size_kernel[1])) if maxweight > 0: w[name_w+'_w'].set_value(maxconstraint(w[name_w+'_w']).tag.test_value) w[name+'_b'] = G.sharedf(np.zeros((_n_out,))) if bias_logscale: w[name+'_bs'] = G.sharedf(0.) if l2norm: if logscale: w[name+'_s'] = G.sharedf(np.zeros((_n_out,))) else: w[name+'_s'] = G.sharedf(np.ones((_n_out,))) elif do_constant_rescale: print 'WARNING: constant rescale, these weights arent saved' constant_rescale = G.sharedf(np.ones((_n_out,))) def f(h, w): input_shape = h.tag.test_value.shape[1:] _input = h if pad_channel: h = pad2dwithchannel(h, size_kernel) kerns = w[name_w+'_w'] #if name == '1_down_conv1': # kerns = T.printing.Print('kerns 1')(kerns) if l2norm: kerns = l2normalize(kerns) if logscale: kerns *= T.exp(logscale_scale*w[name+'_s']).dimshuffle(0,'x','x','x') else: kerns *= w[name+'_s'].dimshuffle(0,'x','x','x') elif do_constant_rescale: kerns *= constant_rescale.dimshuffle(0,'x','x','x') #if name == '1_down_conv1': # kerns = T.printing.Print('kerns 2')(kerns) h = dnn_conv(h, kerns, border_mode=border_mode, subsample=downsample) # Mean-only batch norm if bn: h -= h.mean(axis=(0,2,3), keepdims=True) _b = w[name+'_b'].dimshuffle('x',0,'x','x') if bias_logscale: _b *= T.exp(logscale_scale * w[name+'_bs']) h += _b if '__init' in w and datainit: # Std data_std = h.std(axis=(0,2,3)) num_zeros = (data_std.tag.test_value == 0).sum() if num_zeros > 0: print "Warning: Stdev=0 for "+str(num_zeros)+" features in "+name+". Skipping data-dependent init." else: std = (1./init_stdev) * data_std std += 1e-7 if name+'_s' in w: if logscale: w[name+'_s'].set_value(-T.log(std).tag.test_value/logscale_scale) else: w[name+'_s'].set_value((1./std).tag.test_value) elif do_constant_rescale: constant_rescale.set_value((1./std).tag.test_value) h /= std.dimshuffle('x',0,'x','x') # Mean mean = h.mean(axis=(0,2,3)) w[name+'_b'].set_value(-mean.tag.test_value) h -= mean.dimshuffle('x',0,'x','x') #print name, w[name+'_w'].get_value().mean(), w[name+'_w'].get_value().std(), w[name+'_w'].get_value().max() if upsample>1: h = depool2d_split(h, factor=upsample) if not '__init' in w: output_shape = h.tag.test_value.shape[1:] print 'conv2d', name, input_shape, output_shape, size_kernel, pad_channel, border_mode, downsample, upsample #print name, abs(h).max().tag.test_value, abs(h).min().tag.test_value #h = T.printing.Print(name)(h) return h # Normalize weights to _norm L2 norm # TODO: check whether only_upper_bounds here really helps # (the effect is a higher learning rate in the beginning of training) def postup(updates, w): if l2norm and maxweight>0.: updates[w[name_w+'_w']] = maxconstraint(updates[w[name_w+'_w']]) return updates return G.Struct(__call__=f, w=w, postup=postup)