class DenoisingAutoencoder:
def __init__(
self,
n_input,
n_hidden,
tied=True,
noise=None,
ratio=None,
optimizer=optimizers.Adam(),
loss_function=F.sigmoid_cross_entropy,
activation_function=F.sigmoid,
):
self.model = FunctionSet(encoder=F.Linear(n_input, n_hidden), decoder=F.Linear(n_hidden, n_input))
if tied:
self.model.decoder.W = self.model.encoder.W.T
self.noise = noise
self.ratio = ratio
self.optimizer = optimizer
self.optimizer.setup(self.model.collect_parameters())
self.loss_function = loss_function
self.activation_function = activation_function
def train(self, x_data):
self.optimizer.zero_grads()
loss = self.autoencode(x_data, train=True)
loss.backward()
self.optimizer.update()
return loss
def test(self, x_data):
return self.autoencode(x_data, train=False)
def autoencode(self, x_data, train=True):
x = Variable(x_data)
if self.noise and train:
nx = Variable(self.noise.noise(x_data))
else:
nx = Variable(x_data)
if self.ratio:
h = F.dropout(self.encode(nx), ratio=self.ratio, train=train)
else:
h = self.encode(nx)
y = self.decode(h)
return self.loss_function(y, x)
def encode(self, x):
return self.activation_function(self.model.encoder(x))
def decode(self, x):
return self.activation_function(self.model.decoder(x))
class DA(object):
def __init__(
self,
rng,
data,
n_inputs=784,
n_hidden=784,
corruption_level=0.3,
optimizer=optimizers.AdaDelta,
gpu=-1
):
"""
Denoising AutoEncoder
data: data for train
n_inputs: a number of units of input layer and output layer
n_hidden: a number of units of hidden layer
corruption_level: a ratio of masking noise
"""
self.model = FunctionSet(
encoder=F.Linear(n_inputs, n_hidden),
decoder=F.Linear(n_hidden, n_inputs)
)
if gpu >= 0:
self.model.to_gpu()
self.gpu = gpu
self.x_train, self.x_test = data
self.n_train = len(self.x_train)
self.n_test = len(self.x_test)
self.n_inputs = n_inputs
self.n_hidden = n_hidden
self.optimizer = optimizer()
self.optimizer.setup(self.model)
self.corruption_level = corruption_level
self.rng = rng
self.train_losses = []
self.test_losses = []
@property
def xp(self):
return cuda.cupy if self.gpu >= 0 else numpy
def forward(self, x_data, train=True):
y_data = x_data
# add noise (masking noise)
x_data = self.get_corrupted_inputs(x_data, train=train)
x, t = Variable(x_data), Variable(y_data)
# encode
h = self.encode(x)
# decode
y = self.decode(h)
# compute loss
loss = F.mean_squared_error(y, t)
return loss
def compute_hidden(self, x_data):
# x_data = self.xp.asarray(x_data)
x = Variable(x_data)
h = self.encode(x)
# return cuda.to_cpu(h.data)
return h.data
def predict(self, x_data):
x = Variable(x_data)
# encode
h = self.encode(x)
# decode
y = self.decode(h)
return cuda.to_cpu(y.data)
def encode(self, x):
return F.relu(self.model.encoder(x))
def decode(self, h):
return F.relu(self.model.decoder(h))
def encoder(self):
initialW = self.model.encoder.W
initial_bias = self.model.encoder.b
return F.Linear(self.n_inputs,
self.n_hidden,
initialW=initialW,
initial_bias=initial_bias)
def decoder(self):
return self.model.decoder
def to_cpu(self):
self.model.to_cpu()
self.xp = np
def to_gpu(self):
if self.gpu < 0:
logging.error("something wrong")
raise
self.model.to_gpu()
self.xp = cuda.cupy
# masking noise
def get_corrupted_inputs(self, x_data, train=True):
if train and self.corruption_level != 0.0:
mask = self.rng.binomial(size=x_data.shape, n=1, p=1.0 - self.corruption_level)
mask = mask.astype(numpy.float32)
mask = self.xp.asarray(mask)
ret = mask * x_data
# return self.xp.asarray(ret.astype(numpy.float32))
return ret
else:
return x_data
def train_and_test(self, n_epoch=5, batchsize=100):
for epoch in xrange(1, n_epoch+1):
logging.info('epoch: {}'.format(epoch))
perm = self.rng.permutation(self.n_train)
sum_loss = 0
for i in xrange(0, self.n_train, batchsize):
x_batch = self.xp.asarray(self.x_train[perm[i:i+batchsize]])
real_batchsize = len(x_batch)
self.optimizer.zero_grads()
loss = self.forward(x_batch)
loss.backward()
self.optimizer.update()
sum_loss += float(loss.data) * real_batchsize
logging.info(
'train mean loss={}'.format(sum_loss / self.n_train)
)
# evaluation
sum_loss = 0
for i in xrange(0, self.n_test, batchsize):
x_batch = self.xp.asarray(self.x_test[i:i+batchsize])
real_batchsize = len(x_batch)
loss = self.forward(x_batch, train=False)
sum_loss += float(loss.data) * real_batchsize
logging.info(
'test mean loss={}'.format(sum_loss / self.n_test)
)
class DA(object):
def __init__(self,
rng,
data,
n_inputs=784,
n_hidden=784,
corruption_level=0.3,
gpu=-1,
sparse=False):
"""Denoising AutoEncoder
data: data for train
n_inputs: a number of units of input layer and output layer
n_hidden: a number of units of hidden layer
corruption_level: a ratio of masking noise
"""
self.model = FunctionSet(encoder=F.Linear(n_inputs, n_hidden),
decoder=F.Linear(n_hidden, n_inputs))
if gpu >= 0:
self.model.to_gpu()
self.gpu = gpu
self.x_train, self.x_test = data
self.n_train = len(self.x_train)
self.n_test = len(self.x_test)
self.optimizer = optimizers.Adam()
self.optimizer.setup(self.model.collect_parameters())
self.corruption_level = corruption_level
self.rng = rng
self.sparse = sparse
def forward(self, x_data, train=True):
y_data = x_data
# add noise (masking noise)
# x_data = self.get_corrupted_inputs(x_data, train=train)
if self.gpu >= 0:
x_data = cuda.to_gpu(x_data)
y_data = cuda.to_gpu(y_data)
x, t = Variable(x_data), Variable(y_data)
# encode
h = self.encode(x)
# decode
y = self.decode(h)
# compute loss
if self.sparse:
loss = F.mean_squared_error(y, t) + .001 * np.linalg.norm(h.data)
else:
loss = F.mean_squared_error(y, t)
return loss
def compute_hidden(self, x_data):
if self.gpu >= 0:
x_data = cuda.to_gpu(x_data)
x = Variable(x_data)
h = self.encode(x)
return cuda.to_cpu(h.data)
def predict(self, x_data):
if self.gpu >= 0:
x_data = cuda.to_gpu(x_data)
x = Variable(x_data)
# encode
h = self.encode(x)
# decode
y = self.decode(h)
return cuda.to_cpu(y.data)
def encode(self, x):
return F.relu(self.model.encoder(x))
def decode(self, h):
return F.relu(self.model.decoder(h))
def encoder(self):
return self.model.encoder
def decoder(self):
return self.model.decoder
# masking noise
def get_corrupted_inputs(self, x_data, train=True):
if train:
ret = self.rng.binomial(
size=x_data.shape, n=1, p=1.0 - self.corruption_level) * x_data
return ret.astype(np.float32)
else:
return x_data
def train_and_test(self, n_epoch=5, batchsize=100):
for epoch in xrange(1, n_epoch + 1):
print 'epoch', epoch
perm = self.rng.permutation(self.n_train)
sum_loss = 0
for i in xrange(0, self.n_train, batchsize):
x_batch = self.x_train[perm[i:i + batchsize]]
real_batchsize = len(x_batch)
self.optimizer.zero_grads()
loss = self.forward(x_batch)
loss.backward()
self.optimizer.update()
sum_loss += float(cuda.to_cpu(loss.data)) * real_batchsize
logging.info('train mean loss={}'.format(sum_loss / self.n_train))
# evaluation
sum_loss = 0
for i in xrange(0, self.n_test, batchsize):
x_batch = self.x_test[i:i + batchsize]
real_batchsize = len(x_batch)
loss = self.forward(x_batch, train=False)
sum_loss += float(cuda.to_cpu(loss.data)) * real_batchsize
logging.info('test mean loss={}'.format(sum_loss / self.n_test))
