def __init__(self, config, proc_type='standardize', **kwargs):
'''
config: dictionary with partition configuration information
proc_type: type of preprocessing (either standardize or pca_whiten)
if proc_type='standardize' no extra arguments required
if proc_type='pca_whiten' the following keyword arguments are required:
ncomponents = x where x is an integer
epsilon = y where y is a float (regularization parameter)
'''
recognized_types = ['standardize', 'pca_whiten']
assert proc_type in recognized_types
# load parition information
self.mean = config['mean']
self.mean = self.mean.reshape((np.prod(self.mean.shape),))
self.istd = np.reciprocal(np.sqrt(config['var']))
self.istd = self.istd.reshape((np.prod(self.istd.shape),))
self.tframes = config['tframes']
nvis = len(self.mean)
if proc_type == 'standardize':
dim = nvis
mask = (self.istd < 20) # in order to ignore near-zero variance inputs
self.biases = np.array(-self.mean * self.istd * mask, dtype=np.float32)
self.weights = np.array(np.diag(self.istd * mask), dtype=np.float32) #!!!gives memory error for convnet (because diag not treated as sparse mat)
if proc_type == 'pca_whiten':
raise NotImplementedError(
'''PCA whitening not yet implemented as a layer.
Use audio_dataset2d.AudioDataset2d to perform whitening from the dataset iterator''')
# dim = kwargs['ncomponents']
# S = config['S'][:dim] # eigenvalues
# U = config['U'][:,:dim] # eigenvectors
# self.pca = np.diag(1./(np.sqrt(S) + epsilon)).dot(U.T)
# self.biases = np.array(-self.mean.dot(self.pca.transpose()), dtype=np.float32)
# self.weights = np.array(self.pca.transpose(), dtype=np.float32)
# Autoencoder with linear units
pre_layer = Autoencoder(nvis=nvis, nhid=dim, act_enc=None, act_dec=None, irange=0)
# Set weights for pre-processing
params = pre_layer.get_param_values()
params[1] = self.biases
params[2] = self.weights
pre_layer.set_param_values(params)
super(PreprocLayer, self).__init__(layer_name='pre', layer_content=pre_layer, freeze_params=True)
def test_autoencoder_logistic_linear_tied():
data = np.random.randn(10, 5).astype(config.floatX)
ae = Autoencoder(5, 7, act_enc="sigmoid", act_dec="linear", tied_weights=True)
w = ae.weights.get_value()
ae.hidbias.set_value(np.random.randn(7).astype(config.floatX))
hb = ae.hidbias.get_value()
ae.visbias.set_value(np.random.randn(5).astype(config.floatX))
vb = ae.visbias.get_value()
d = tensor.matrix()
result = np.dot(1.0 / (1 + np.exp(-hb - np.dot(data, w))), w.T) + vb
ff = theano.function([d], ae.reconstruct(d))
assert _allclose(ff(data), result)
def test_autoencoder_logistic_linear_tied():
data = np.random.randn(10, 5).astype(config.floatX)
ae = Autoencoder(5, 7, act_enc='sigmoid', act_dec='linear',
tied_weights=True)
w = ae.weights.get_value()
ae.hidbias.set_value(np.random.randn(7).astype(config.floatX))
hb = ae.hidbias.get_value()
ae.visbias.set_value(np.random.randn(5).astype(config.floatX))
vb = ae.visbias.get_value()
d = tensor.matrix()
result = np.dot(1. / (1 + np.exp(-hb - np.dot(data, w))), w.T) + vb
ff = theano.function([d], ae.reconstruct(d))
assert _allclose(ff(data), result)
def test_autoencoder_tanh_cos_untied():
data = np.random.randn(10, 5).astype(config.floatX)
ae = Autoencoder(5, 7, act_enc='tanh', act_dec='cos', tied_weights=False)
w = ae.weights.get_value()
w_prime = ae.w_prime.get_value()
ae.hidbias.set_value(np.random.randn(7).astype(config.floatX))
hb = ae.hidbias.get_value()
ae.visbias.set_value(np.random.randn(5).astype(config.floatX))
vb = ae.visbias.get_value()
d = tensor.matrix()
result = np.cos(np.dot(np.tanh(hb + np.dot(data, w)), w_prime) + vb)
ff = theano.function([d], ae.reconstruct(d))
assert _allclose(ff(data), result)
def test_autoencoder_tanh_cos_untied():
data = np.random.randn(10, 5).astype(config.floatX)
ae = Autoencoder(5, 7, act_enc="tanh", act_dec="cos", tied_weights=False)
w = ae.weights.get_value()
w_prime = ae.w_prime.get_value()
ae.hidbias.set_value(np.random.randn(7).astype(config.floatX))
hb = ae.hidbias.get_value()
ae.visbias.set_value(np.random.randn(5).astype(config.floatX))
vb = ae.visbias.get_value()
d = tensor.matrix()
result = np.cos(np.dot(np.tanh(hb + np.dot(data, w)), w_prime) + vb)
ff = theano.function([d], ae.reconstruct(d))
assert _allclose(ff(data), result)
def test_stackedblocks_with_params():
"""
Test StackedBlocks when all layers have trainable params
"""
aes = [
Autoencoder(100, 50, 'tanh', 'tanh'),
Autoencoder(50, 10, 'tanh', 'tanh')
]
sb = StackedBlocks(aes)
_params = set([p for l in sb._layers for p in l._params])
assert sb._params == _params
def _make_aes(layer_sizes, activation_funcs, tied=True):
"""
Creates the Autoencoder objects needed by the GSN.
Parameters
----------
layer_sizes : WRITEME
activation_funcs : WRITEME
tied : WRITEME
"""
aes = []
assert len(activation_funcs) == len(layer_sizes)
for i in xrange(len(layer_sizes) - 1):
# activation for visible layer is aes[0].act_dec
act_enc = activation_funcs[i + 1]
act_dec = act_enc if i != 0 else activation_funcs[0]
aes.append(
Autoencoder(layer_sizes[i],
layer_sizes[i + 1],
act_enc,
act_dec,
tied_weights=tied))
return aes
def test_untied_ae():
"""
Tests that UntiedAutoencoder calls the Model superclass constructor
"""
ae = Autoencoder(5, 7, act_enc='tanh', act_dec='cos', tied_weights=True)
model = UntiedAutoencoder(ae)
model._ensure_extensions()
def test_dcae():
"""
Tests that DeepComposedAutoencoder calls the Model superclass constructor
"""
ae = Autoencoder(5, 7, act_enc='tanh', act_dec='cos',
tied_weights=True)
model = DeepComposedAutoencoder([ae])
model._ensure_extensions()
def get_autoencoder(self, structure, act_function='sigmoid'):
n_input, n_output = structure
config = {
'nvis': n_input,
'nhid': n_output,
'act_enc': act_function,
'act_dec': act_function,
"irange" : 0.05,
}
return Autoencoder(**config)
def test_dcae():
"""
Tests that DeepComposedAutoencoder works correctly
"""
ae = Autoencoder(5, 7, act_enc='tanh', act_dec='cos', tied_weights=True)
model = DeepComposedAutoencoder([ae])
model._ensure_extensions()
data = np.random.randn(10, 5).astype(config.floatX)
model.perform(data)
def get_autoencoder(structure):
n_input, n_output = structure
config = {
'nvis': n_input,
'nhid': n_output,
'act_enc': 'sigmoid',
'act_dec': None,
"irange": 0.05,
}
return Autoencoder(**config)
def get_autoencoder(structure):
n_input, n_output = structure
config = {
'nhid': n_output,
'nvis': n_input,
'tied_weights': True,
'act_enc': 'sigmoid',
'act_dec': 'sigmoid',
'irange': 0.001,
}
return Autoencoder(**config)
def test_sdae():
"""
Tests that StackedDenoisingAutoencoder works correctly
"""
data = np.random.randn(10, 5).astype(config.floatX) * 100
ae = Autoencoder(5, 7, act_enc='tanh', act_dec='cos', tied_weights=False)
corruptor = BinomialCorruptor(corruption_level=0.5)
model = StackedDenoisingAutoencoder([ae], corruptor)
model._ensure_extensions()
w = ae.weights.get_value()
w_prime = ae.w_prime.get_value()
ae.hidbias.set_value(np.random.randn(7).astype(config.floatX))
hb = ae.hidbias.get_value()
ae.visbias.set_value(np.random.randn(5).astype(config.floatX))
vb = ae.visbias.get_value()
d = tensor.matrix()
result = np.cos(np.dot(np.tanh(hb + np.dot(data, w)), w_prime) + vb)
ff = theano.function([d], model.reconstruct(d))
assert not _allclose(ff(data), result)
def test_autoencoder_properly_initialized():
ae = Autoencoder(1, 1, 'sigmoid', 'linear')
assert hasattr(ae, 'fn'), "Autoencoder didn't call Block.__init__"
assert hasattr(ae, 'extensions'), "Autoencoder didn't call Model.__init__"
