def mean_field_from_activation(self, vmap):
return activation_functions.sigmoid(vmap[self] - vmap[self.flipped_units])
def success_probability_from_activation(self, vmap):
return activation_functions.sigmoid(vmap[self])
def mean_field_from_activation(self, vmap):
return activation_functions.sigmoid(vmap[self])
def sample_from_activation(self, vmap):
p = activation_functions.sigmoid(vmap[self] - vmap[self.flipped_units])
return samplers.bernoulli(p)
def apply_convolution(args, data_set_x, random_filters={}, normalisation={}):
############################################
# APPLY FIRST LAYER CONVO
############################################
if args.random_filters:
random_filters_rng = np.random.RandomState(args.random_filters_seed)
if args.convolution_type == "fullnoborder":
print " Removing neighbourhoods:"
print " before: ", data_set_x.shape
# cut borders
margin = (data_set_x.shape[2] - pic_w) / 2
data_set_x = data_set_x[:,:,(margin):(pic_w + margin),(margin):(pic_w + margin)]
print " after: ", data_set_x.shape
for prev_layer in args.previous_layer:
print ">> Processing layer: ", prev_layer
data_set_x_conv_collect = []
for fname in prev_layer.split(","):
print " >> Filter set: ", fname
with open(fname, "r") as f:
prev_layer_params = cPickle.load(f)
prev_W = prev_layer_params["W"]
prev_bh = prev_layer_params["bh"]
print " prev_W.shape: ", prev_W.shape
print " prev_bh.shape: ", prev_bh.shape
if args.random_filters:
print " Replacing filters by random values..."
if fname in random_filters:
prev_W = random_filters[fname]["prev_W"]
prev_bh = random_filters[fname]["prev_bh"]
else:
prev_W = np.array(random_filters_rng.uniform(low=-1, high=1, size=prev_W.shape), dtype=prev_W.dtype)
prev_bh = np.array(random_filters_rng.uniform(low=-1, high=1, size=prev_bh.shape), dtype=prev_bh.dtype)
random_filters[fname] = { "prev_W": prev_W, "prev_bh": prev_bh }
print " Compiling convolution..."
V = T.dtensor4()
W = T.dtensor4()
bh = T.dvector()
W_flipped = W[:, :, ::-1, ::-1]
subsample = (args.subsample, args.subsample)
reshaped_bh = bh.dimshuffle('x',0,'x','x')
if args.convolution_type == "fullnoborder":
c = conv.conv2d(V, W_flipped, border_mode="valid", subsample=subsample)
else:
c = conv.conv2d(V, W_flipped, border_mode="full", subsample=subsample)
c_act = activation_functions.sigmoid(c + reshaped_bh)
if args.skip_sigmoid:
conv_f = theano.function([V, W, bh], [ c ], on_unused_input="ignore")
else:
conv_f = theano.function([V, W, bh], [ c_act ])
print " Applying convolution..."
start_time = time.time()
data_set_x_conv = None
n_samples = data_set_x.shape[0]
batch_size = 5
for i in xrange(0, n_samples, batch_size):
cvf = conv_f(data_set_x[i:min(i+batch_size, n_samples), :,:,:], prev_W, prev_bh)[0]
if data_set_x_conv is None:
s = np.array(cvf.shape)
s[0] = data_set_x.shape[0]
data_set_x_conv = np.zeros(s, dtype=cvf.dtype)
data_set_x_conv[i:min(i+batch_size, n_samples), :,:,:] = cvf
if i % 10 == 0 and i > 0:
print " %d %0.2f/s" % (i, float(i) / (time.time() - start_time))
# release
prev_layer_params = None
prev_W = None
prev_bh = None
conv_f = None
cvf = None
print " After this layer:"
print " data_set_x_conv: ", data_set_x_conv.shape
data_set_x_conv_collect.append(data_set_x_conv)
# release
data_set_x_conv = None
# garbage collection
gc.collect()
if len(data_set_x_conv_collect) == 1:
data_set_x = data_set_x_conv_collect[0]
else:
data_set_x = np.concatenate(data_set_x_conv_collect, axis=1)
# release
data_set_x_conv_collect = None
# garbage collection
gc.collect()
print " After this layer:"
print " data_set_x: ", data_set_x.shape
if args.global_normalisation:
# normalise / whiten
if prev_layer in normalisation:
mu = normalisation[prev_layer]["mu"]
sigma = normalisation[prev_layer]["sigma"]
else:
print " Calculating normalisation parameters"
mu = np.mean(data_set_x)
sigma = np.std(data_set_x)
normalisation[prev_layer] = { "mu": mu, "sigma": sigma }
print " Normalising layer output..."
data_set_x -= mu
data_set_x /= (0.25 * sigma)
elif not args.skip_normalisation:
# normalise / whiten
print " Normalising layer output..."
n_samples = data_set_x.shape[0]
data_set_rows = data_set_x.reshape(n_samples, -1)
mu = np.mean(data_set_rows, axis=1).reshape(n_samples, 1, 1, 1)
sigma = memory_efficient_std(data_set_rows).reshape(n_samples, 1, 1, 1)
# sigma = np.std(data_set_rows, axis=1).reshape(n_samples, 1, 1, 1)
data_set_x -= mu
data_set_x /= (0.25 * sigma)
# release
data_set_rows = None
# garbage collection
gc.collect()
# cut borders
if args.convolution_type != "fullnoborder":
margin = (data_set_x.shape[2] - pic_w) / 2
data_set_x = data_set_x[:,:,(margin):(pic_w + margin),(margin):(pic_w + margin)]
print " After removing borders:"
print " data_set_x: ", data_set_x.shape
############################################
# END OF CONVOLUTION
############################################
return (data_set_x, random_filters, normalisation)
def success_probability_from_activation(self, vmap):
return activation_functions.sigmoid(vmap[self])
def mean_field_from_activation(self, vmap):
return activation_functions.sigmoid(vmap[self] -
vmap[self.flipped_units])
def sample_from_activation(self, vmap):
p = activation_functions.sigmoid(vmap[self] - vmap[self.flipped_units])
return samplers.bernoulli(p)
def mean_field_from_activation(self, vmap):
return activation_functions.sigmoid(vmap[self])
