def test_pylearn2_training():
# Construct the model
mlp = MLP(activations=[Sigmoid(), Sigmoid()],
dims=[784, 100, 784],
weights_init=IsotropicGaussian(),
biases_init=Constant(0.01))
mlp.initialize()
cost = SquaredError()
# Load the data
rng = numpy.random.RandomState(14)
train_dataset = random_dense_design_matrix(rng, 1024, 784, 10)
valid_dataset = random_dense_design_matrix(rng, 1024, 784, 10)
x = tensor.matrix('features')
block_cost = Pylearn2Cost(cost.apply(x, mlp.apply(x)))
block_model = Pylearn2Model(mlp)
# Silence Pylearn2's logger
logger = logging.getLogger(pylearn2.__name__)
logger.setLevel(logging.ERROR)
# Training algorithm
sgd = SGD(learning_rate=0.01,
cost=block_cost,
batch_size=128,
monitoring_dataset=valid_dataset)
train = Pylearn2Train(train_dataset, block_model, algorithm=sgd)
train.main_loop(time_budget=3)
def test_pylearn2_training():
# Construct the model
mlp = MLP(activations=[Sigmoid(), Sigmoid()], dims=[784, 100, 784],
weights_init=IsotropicGaussian(), biases_init=Constant(0.01))
mlp.initialize()
cost = SquaredError()
# Load the data
rng = numpy.random.RandomState(14)
train_dataset = random_dense_design_matrix(rng, 1024, 784, 10)
valid_dataset = random_dense_design_matrix(rng, 1024, 784, 10)
x = tensor.matrix('features')
block_cost = Pylearn2Cost(cost.apply(x, mlp.apply(x)))
block_model = Pylearn2Model(mlp)
# Silence Pylearn2's logger
logger = logging.getLogger(pylearn2.__name__)
logger.setLevel(logging.ERROR)
# Training algorithm
sgd = SGD(learning_rate=0.01, cost=block_cost, batch_size=128,
monitoring_dataset=valid_dataset)
train = Pylearn2Train(train_dataset, block_model, algorithm=sgd)
train.main_loop(time_budget=3)
def test_square():
from blocks.bricks.cost import SquaredError
x = tensor.tensor3()
y = tensor.tensor3()
c = SquaredError()
o = c.apply(x,y)
f = theano.function([x,y],o)
print(f(np.ones((3,3,3),dtype=theano.config.floatX),5*np.ones((3,3,3),dtype=theano.config.floatX)))
def test_square():
from blocks.bricks.cost import SquaredError
x = tensor.tensor3()
y = tensor.tensor3()
c = SquaredError()
o = c.apply(x, y)
f = theano.function([x, y], o)
print(
f(np.ones((3, 3, 3), dtype=theano.config.floatX), 5 * np.ones(
(3, 3, 3), dtype=theano.config.floatX)))
def decoder(self, clean, corr, batch_size):
get_unlabeled = lambda x: x[batch_size:] if x is not None else x
est = self.new_activation_dict()
costs = AttributeDict()
costs.denois = AttributeDict()
for i, ((_, spec), act_f) in self.layers[::-1]:
z_corr = get_unlabeled(corr.z[i])
z_clean = get_unlabeled(clean.z[i])
z_clean_s = get_unlabeled(clean.s.get(i))
z_clean_m = get_unlabeled(clean.m.get(i))
# It's the last layer
if i == len(self.layers) - 1:
fspec = (None, None)
ver = get_unlabeled(corr.h[i])
ver_dim = self.layer_dims[i]
top_g = True
else:
fspec = self.layers[i + 1][1][0]
ver = est.z.get(i + 1)
ver_dim = self.layer_dims.get(i + 1)
top_g = False
z_est = self.g(z_lat=z_corr,
z_ver=ver,
in_dims=ver_dim,
out_dims=self.layer_dims[i],
num=i,
fspec=fspec,
top_g=top_g)
# For semi-supervised version
if z_clean_s:
z_est_norm = (z_est - z_clean_m) / z_clean_s
else:
z_est_norm = z_est
z_est_norm = z_est
se = SquaredError('denois' + str(i))
costs.denois[i] = se.apply(z_est_norm.flatten(2),
z_clean.flatten(2)) \
/ np.prod(self.layer_dims[i], dtype=floatX)
costs.denois[i].name = 'denois' + str(i)
# Store references for later use
est.z[i] = z_est
est.h[i] = apply_act(z_est, act_f)
est.s[i] = None
est.m[i] = None
return est, costs
def decoder(self, clean, corr):
est = self.new_activation_dict()
costs = AttributeDict()
costs.denois = AttributeDict()
for i, ((_, spec), act_f) in self.layers[::-1]:
z_corr = corr.unlabeled.z[i]
z_clean = clean.unlabeled.z[i]
z_clean_s = clean.unlabeled.s.get(i)
z_clean_m = clean.unlabeled.m.get(i)
# It's the last layer
if i == len(self.layers) - 1:
fspec = (None, None)
ver = corr.unlabeled.h[i]
ver_dim = self.layer_dims[i]
top_g = True
else:
fspec = self.layers[i + 1][1][0]
ver = est.z.get(i + 1)
ver_dim = self.layer_dims.get(i + 1)
top_g = False
z_est = self.g(z_lat=z_corr,
z_ver=ver,
in_dims=ver_dim,
out_dims=self.layer_dims[i],
num=i,
fspec=fspec,
top_g=top_g)
# The first layer
if z_clean_s:
z_est_norm = (z_est - z_clean_m) / z_clean_s
else:
z_est_norm = z_est
se = SquaredError('denois' + str(i))
costs.denois[i] = se.apply(z_est_norm.flatten(2),
z_clean.flatten(2)) \
/ np.prod(self.layer_dims[i], dtype=floatX)
costs.denois[i].name = 'denois' + str(i)
# Store references for later use
est.z[i] = z_est
est.h[i] = apply_act(z_est, act_f)
est.s[i] = None
est.m[i] = None
return est, costs
def decoder(self, clean, corr, batch_size):
get_unlabeled = lambda x: x[batch_size:] if x is not None else x
est = self.new_activation_dict()
costs = AttributeDict()
costs.denois = AttributeDict()
for i, ((_, spec), act_f) in self.layers[::-1]:
z_corr = get_unlabeled(corr.z[i])
z_clean = get_unlabeled(clean.z[i])
z_clean_s = get_unlabeled(clean.s.get(i))
z_clean_m = get_unlabeled(clean.m.get(i))
# It's the last layer
if i == len(self.layers) - 1:
fspec = (None, None)
ver = get_unlabeled(corr.h[i])
ver_dim = self.layer_dims[i]
top_g = True
else:
fspec = self.layers[i + 1][1][0]
ver = est.z.get(i + 1)
ver_dim = self.layer_dims.get(i + 1)
top_g = False
z_est = self.g(
z_lat=z_corr, z_ver=ver, in_dims=ver_dim, out_dims=self.layer_dims[i], num=i, fspec=fspec, top_g=top_g
)
# For semi-supervised version
if z_clean_s:
z_est_norm = (z_est - z_clean_m) / z_clean_s
else:
z_est_norm = z_est
z_est_norm = z_est
se = SquaredError("denois" + str(i))
costs.denois[i] = se.apply(z_est_norm.flatten(2), z_clean.flatten(2)) / np.prod(
self.layer_dims[i], dtype=floatX
)
costs.denois[i].name = "denois" + str(i)
# Store references for later use
est.z[i] = z_est
est.h[i] = apply_act(z_est, act_f)
est.s[i] = None
est.m[i] = None
return est, costs
def apply(self, input_labeled, target_labeled, input_unlabeled):
self.layer_counter = 0
input_dim = self.p.encoder_layers[0]
# Store the dimension tuples in the same order as layers.
layers = self.layers
self.layer_dims = {0: input_dim}
self.lr = self.shared(self.default_lr, 'learning_rate', role=None)
self.costs = costs = AttributeDict()
self.costs.denois = AttributeDict()
self.act = AttributeDict()
self.error = AttributeDict()
self.oos = AttributeDict()
top = len(layers) - 1
N = input_labeled.shape[0]
self.join = lambda l, u: T.concatenate([l, u], axis=0)
self.labeled = lambda x: x[:N] if x is not None else x
self.unlabeled = lambda x: x[N:] if x is not None else x
self.split_lu = lambda x: (self.labeled(x), self.unlabeled(x))
input_concat = self.join(input_labeled, input_unlabeled)
def encoder(input_, path_name, input_noise_std=0, noise_std=[]):
h = input_
logger.info(' 0: noise %g' % input_noise_std)
if input_noise_std > 0.:
h = h + self.noise_like(h) * input_noise_std
d = AttributeDict()
d.unlabeled = self.new_activation_dict()
d.labeled = self.new_activation_dict()
d.labeled.z[0] = self.labeled(h)
d.unlabeled.z[0] = self.unlabeled(h)
prev_dim = input_dim
for i, (spec, _, act_f) in layers[1:]:
d.labeled.h[i - 1], d.unlabeled.h[i - 1] = self.split_lu(h)
noise = noise_std[i] if i < len(noise_std) else 0.
curr_dim, z, m, s, h = self.f(h, prev_dim, spec, i, act_f,
path_name=path_name,
noise_std=noise)
assert self.layer_dims.get(i) in (None, curr_dim)
self.layer_dims[i] = curr_dim
d.labeled.z[i], d.unlabeled.z[i] = self.split_lu(z)
d.unlabeled.s[i] = s
d.unlabeled.m[i] = m
prev_dim = curr_dim
d.labeled.h[i], d.unlabeled.h[i] = self.split_lu(h)
return d
# Clean, supervised
logger.info('Encoder: clean, labeled')
clean = self.act.clean = encoder(input_concat, 'clean')
# Corrupted, supervised
logger.info('Encoder: corr, labeled')
corr = self.act.corr = encoder(input_concat, 'corr',
input_noise_std=self.p.super_noise_std,
noise_std=self.p.f_local_noise_std)
est = self.act.est = self.new_activation_dict()
# Decoder path in opposite order
logger.info('Decoder: z_corr -> z_est')
for i, ((_, spec), l_type, act_f) in layers[::-1]:
z_corr = corr.unlabeled.z[i]
z_clean = clean.unlabeled.z[i]
z_clean_s = clean.unlabeled.s.get(i)
z_clean_m = clean.unlabeled.m.get(i)
fspec = layers[i+1][1][0] if len(layers) > i+1 else (None, None)
if i == top:
ver = corr.unlabeled.h[i]
ver_dim = self.layer_dims[i]
top_g = True
else:
ver = est.z.get(i + 1)
ver_dim = self.layer_dims.get(i + 1)
top_g = False
z_est = self.g(z_lat=z_corr,
z_ver=ver,
in_dims=ver_dim,
out_dims=self.layer_dims[i],
l_type=l_type,
num=i,
fspec=fspec,
top_g=top_g)
if z_est is not None:
# Denoising cost
if z_clean_s and self.p.zestbn == 'bugfix':
z_est_norm = (z_est - z_clean_m) / T.sqrt(z_clean_s + np.float32(1e-10))
elif z_clean_s is None or self.p.zestbn == 'no':
z_est_norm = z_est
else:
assert False, 'Not supported path'
se = SquaredError('denois' + str(i))
costs.denois[i] = se.apply(z_est_norm.flatten(2),
z_clean.flatten(2)) \
/ np.prod(self.layer_dims[i], dtype=floatX)
costs.denois[i].name = 'denois' + str(i)
denois_print = 'denois %.2f' % self.p.denoising_cost_x[i]
else:
denois_print = ''
# Store references for later use
est.h[i] = self.apply_act(z_est, act_f)
est.z[i] = z_est
est.s[i] = None
est.m[i] = None
logger.info(' g%d: %10s, %s, dim %s -> %s' % (
i, l_type,
denois_print,
self.layer_dims.get(i+1),
self.layer_dims.get(i)
))
# Costs
y = target_labeled.flatten()
Q = int(self.layer_dims[top][0]) - 1
logger.info('Q=%d'%Q)
costs.class_clean = CategoricalCrossEntropyIV(Q=Q,
alpha=self.p.alpha,
beta=self.p.beta,
dbeta=self.p.dbeta,
gamma=self.p.gamma,
gamma1=self.p.gamma1
).apply(y, clean.labeled.h[top])
costs.class_clean.name = 'cost_class_clean'
costs.class_corr = CategoricalCrossEntropyIV(Q=Q,
alpha=self.p.alpha,
beta=self.p.beta,
dbeta=self.p.dbeta,
gamma=self.p.gamma,
gamma1=self.p.gamma1,
).apply(y, corr.labeled.h[top])
costs.class_corr.name = 'cost_class_corr'
# This will be used for training
costs.total = costs.class_corr * 1.0
for i in range(top + 1):
if costs.denois.get(i) and self.p.denoising_cost_x[i] > 0:
costs.total += costs.denois[i] * self.p.denoising_cost_x[i]
if self.p.alpha_clean:
y_true = y
eps = np.float32(1e-6)
# scale preds so that the class probas of each sample sum to 1
y_pred = clean.labeled.h[top] + eps
y_pred /= y_pred.sum(axis=-1, keepdims=True)
y0 = T.or_(T.eq(y_true, 0), T.gt(y_true, Q)) # out-of-set or unlabeled
y0sum = y0.sum() + eps # number of oos
cost1 = T.nnet.categorical_crossentropy(y_pred, y_pred)
cost1 = T.dot(y0, cost1) / y0sum # average cost per labeled example
costs.total += self.p.alpha_clean * cost1
costs.total.name = 'cost_total'
# Classification error
mr = MisclassificationRateIV(oos_thr=self.p.oos_thr)
self.error.clean = mr.apply(y, clean.labeled.h[top]) * np.float32(100.)
self.error.clean.name = 'error_rate_clean'
oosr = OOSRateIV()
self.oos.clean = oosr.apply(y, clean.labeled.h[top]) * np.float32(100.)
self.oos.clean.name = 'oos_rate_clean'
def apply(self, input_labeled, target_labeled, input_unlabeled):
self.target_labeled = target_labeled
self.layer_counter = 0
input_dim = self.p.encoder_layers[0]
# Store the dimension tuples in the same order as layers.
layers = self.layers
self.layer_dims = {0: input_dim}
self.lr = self.default_lr
self.costs = costs = AttributeDict()
self.costs.denois = AttributeDict()
self.act = AttributeDict()
self.error = AttributeDict()
top = len(layers) - 1
N = input_labeled.shape[0]
self.join = lambda l, u: T.concatenate([l, u], axis=0)
self.labeled = lambda x: x[:N] if x is not None else x
self.unlabeled = lambda x: x[N:] if x is not None else x
self.split_lu = lambda x: (self.labeled(x), self.unlabeled(x))
input_concat = self.join(input_labeled, input_unlabeled)
def encoder(input_, path_name, input_noise_std=0, noise_std=[]):
h = input_
logger.info(' 0: noise %g' % input_noise_std)
if input_noise_std > 0.:
h = h + self.noise_like(h) * input_noise_std
d = AttributeDict()
d.unlabeled = self.new_activation_dict()
d.labeled = self.new_activation_dict()
d.labeled.z[0] = self.labeled(h)
d.unlabeled.z[0] = self.unlabeled(h)
prev_dim = input_dim
for i, (spec, _, act_f) in layers[1:]:
d.labeled.h[i - 1], d.unlabeled.h[i - 1] = self.split_lu(h)
noise = noise_std[i] if i < len(noise_std) else 0.
curr_dim, z, m, s, h = self.f(h, prev_dim, spec, i, act_f,
path_name=path_name,
noise_std=noise)
assert self.layer_dims.get(i) in (None, curr_dim)
self.layer_dims[i] = curr_dim
d.labeled.z[i], d.unlabeled.z[i] = self.split_lu(z)
d.unlabeled.s[i] = s
d.unlabeled.m[i] = m
prev_dim = curr_dim
d.labeled.h[i], d.unlabeled.h[i] = self.split_lu(h)
return d
# Clean, supervised
logger.info('Encoder: clean, labeled')
clean = self.act.clean = encoder(input_concat, 'clean')
# Corrupted, supervised
logger.info('Encoder: corr, labeled')
corr = self.act.corr = encoder(input_concat, 'corr',
input_noise_std=self.p.super_noise_std,
noise_std=self.p.f_local_noise_std)
est = self.act.est = self.new_activation_dict()
# Decoder path in opposite order
logger.info('Decoder: z_corr -> z_est')
for i, ((_, spec), l_type, act_f) in layers[::-1]:
z_corr = corr.unlabeled.z[i]
z_clean = clean.unlabeled.z[i]
z_clean_s = clean.unlabeled.s.get(i)
z_clean_m = clean.unlabeled.m.get(i)
fspec = layers[i+1][1][0] if len(layers) > i+1 else (None, None)
if i == top:
ver = corr.unlabeled.h[i]
ver_dim = self.layer_dims[i]
top_g = True
else:
ver = est.z.get(i + 1)
ver_dim = self.layer_dims.get(i + 1)
top_g = False
z_est = self.g(z_lat=z_corr,
z_ver=ver,
in_dims=ver_dim,
out_dims=self.layer_dims[i],
l_type=l_type,
num=i,
fspec=fspec,
top_g=top_g)
if z_est is not None:
# Denoising cost
if z_clean_s and self.p.zestbn == 'bugfix':
z_est_norm = (z_est - z_clean_m) / T.sqrt(z_clean_s + np.float32(1e-10))
elif z_clean_s is None or self.p.zestbn == 'no':
z_est_norm = z_est
else:
assert False, 'Not supported path'
se = SquaredError('denois' + str(i))
costs.denois[i] = se.apply(z_est_norm.flatten(2),
z_clean.flatten(2)) \
/ np.prod(self.layer_dims[i], dtype=floatX)
costs.denois[i].name = 'denois' + str(i)
denois_print = 'denois %.2f' % self.p.denoising_cost_x[i]
else:
denois_print = ''
# Store references for later use
est.h[i] = self.apply_act(z_est, act_f)
est.z[i] = z_est
est.s[i] = None
est.m[i] = None
logger.info(' g%d: %10s, %s, dim %s -> %s' % (
i, l_type,
denois_print,
self.layer_dims.get(i+1),
self.layer_dims.get(i)
))
# Costs
y = target_labeled.flatten()
costs.class_clean = CategoricalCrossEntropy().apply(y, clean.labeled.h[top])
costs.class_clean.name = 'cost_class_clean'
costs.class_corr = CategoricalCrossEntropy().apply(y, corr.labeled.h[top])
costs.class_corr.name = 'cost_class_corr'
# This will be used for training
costs.total = costs.class_corr * 1.0
for i in range(top + 1):
if costs.denois.get(i) and self.p.denoising_cost_x[i] > 0:
costs.total += costs.denois[i] * self.p.denoising_cost_x[i]
costs.total.name = 'cost_total'
# Classification error
mr = MisclassificationRate()
self.error.clean = mr.apply(y, clean.labeled.h[top]) * np.float32(100.)
self.error.clean.name = 'error_rate_clean'
def apply(self, input_labeled, target_labeled, input_unlabeled):
self.layer_counter = 0
input_dim = self.p.encoder_layers[0]
# Store the dimension tuples in the same order as layers.
layers = self.layers
self.layer_dims = {0: input_dim}
self.lr = self.shared(self.default_lr, 'learning_rate', role=None)
self.costs = costs = AttributeDict()
self.costs.denois = AttributeDict()
self.act = AttributeDict()
self.error = AttributeDict()
self.oos = AttributeDict()
top = len(layers) - 1
N = input_labeled.shape[0]
self.join = lambda l, u: T.concatenate([l, u], axis=0)
self.labeled = lambda x: x[:N] if x is not None else x
self.unlabeled = lambda x: x[N:] if x is not None else x
self.split_lu = lambda x: (self.labeled(x), self.unlabeled(x))
input_concat = self.join(input_labeled, input_unlabeled)
def encoder(input_, path_name, input_noise_std=0, noise_std=[]):
h = input_
logger.info(' 0: noise %g' % input_noise_std)
if input_noise_std > 0.:
h = h + self.noise_like(h) * input_noise_std
d = AttributeDict()
d.unlabeled = self.new_activation_dict()
d.labeled = self.new_activation_dict()
d.labeled.z[0] = self.labeled(h)
d.unlabeled.z[0] = self.unlabeled(h)
prev_dim = input_dim
for i, (spec, _, act_f) in layers[1:]:
d.labeled.h[i - 1], d.unlabeled.h[i - 1] = self.split_lu(h)
noise = noise_std[i] if i < len(noise_std) else 0.
curr_dim, z, m, s, h = self.f(h,
prev_dim,
spec,
i,
act_f,
path_name=path_name,
noise_std=noise)
assert self.layer_dims.get(i) in (None, curr_dim)
self.layer_dims[i] = curr_dim
d.labeled.z[i], d.unlabeled.z[i] = self.split_lu(z)
d.unlabeled.s[i] = s
d.unlabeled.m[i] = m
prev_dim = curr_dim
d.labeled.h[i], d.unlabeled.h[i] = self.split_lu(h)
return d
# Clean, supervised
logger.info('Encoder: clean, labeled')
clean = self.act.clean = encoder(input_concat, 'clean')
# Corrupted, supervised
logger.info('Encoder: corr, labeled')
corr = self.act.corr = encoder(input_concat,
'corr',
input_noise_std=self.p.super_noise_std,
noise_std=self.p.f_local_noise_std)
est = self.act.est = self.new_activation_dict()
# Decoder path in opposite order
logger.info('Decoder: z_corr -> z_est')
for i, ((_, spec), l_type, act_f) in layers[::-1]:
z_corr = corr.unlabeled.z[i]
z_clean = clean.unlabeled.z[i]
z_clean_s = clean.unlabeled.s.get(i)
z_clean_m = clean.unlabeled.m.get(i)
fspec = layers[i + 1][1][0] if len(layers) > i + 1 else (None,
None)
if i == top:
ver = corr.unlabeled.h[i]
ver_dim = self.layer_dims[i]
top_g = True
else:
ver = est.z.get(i + 1)
ver_dim = self.layer_dims.get(i + 1)
top_g = False
z_est = self.g(z_lat=z_corr,
z_ver=ver,
in_dims=ver_dim,
out_dims=self.layer_dims[i],
l_type=l_type,
num=i,
fspec=fspec,
top_g=top_g)
if z_est is not None:
# Denoising cost
if z_clean_s and self.p.zestbn == 'bugfix':
z_est_norm = (z_est - z_clean_m
) / T.sqrt(z_clean_s + np.float32(1e-10))
elif z_clean_s is None or self.p.zestbn == 'no':
z_est_norm = z_est
else:
assert False, 'Not supported path'
se = SquaredError('denois' + str(i))
costs.denois[i] = se.apply(z_est_norm.flatten(2),
z_clean.flatten(2)) \
/ np.prod(self.layer_dims[i], dtype=floatX)
costs.denois[i].name = 'denois' + str(i)
denois_print = 'denois %.2f' % self.p.denoising_cost_x[i]
else:
denois_print = ''
# Store references for later use
est.h[i] = self.apply_act(z_est, act_f)
est.z[i] = z_est
est.s[i] = None
est.m[i] = None
logger.info(' g%d: %10s, %s, dim %s -> %s' %
(i, l_type, denois_print, self.layer_dims.get(i + 1),
self.layer_dims.get(i)))
# Costs
y = target_labeled.flatten()
Q = int(self.layer_dims[top][0]) - 1
logger.info('Q=%d' % Q)
costs.class_clean = CategoricalCrossEntropyIV(
Q=Q,
alpha=self.p.alpha,
beta=self.p.beta,
dbeta=self.p.dbeta,
gamma=self.p.gamma,
gamma1=self.p.gamma1).apply(y, clean.labeled.h[top])
costs.class_clean.name = 'cost_class_clean'
costs.class_corr = CategoricalCrossEntropyIV(
Q=Q,
alpha=self.p.alpha,
beta=self.p.beta,
dbeta=self.p.dbeta,
gamma=self.p.gamma,
gamma1=self.p.gamma1,
).apply(y, corr.labeled.h[top])
costs.class_corr.name = 'cost_class_corr'
# This will be used for training
costs.total = costs.class_corr * 1.0
for i in range(top + 1):
if costs.denois.get(i) and self.p.denoising_cost_x[i] > 0:
costs.total += costs.denois[i] * self.p.denoising_cost_x[i]
if self.p.alpha_clean:
y_true = y
eps = np.float32(1e-6)
# scale preds so that the class probas of each sample sum to 1
y_pred = clean.labeled.h[top] + eps
y_pred /= y_pred.sum(axis=-1, keepdims=True)
y0 = T.or_(T.eq(y_true, 0), T.gt(y_true,
Q)) # out-of-set or unlabeled
y0sum = y0.sum() + eps # number of oos
cost1 = T.nnet.categorical_crossentropy(y_pred, y_pred)
cost1 = T.dot(y0,
cost1) / y0sum # average cost per labeled example
costs.total += self.p.alpha_clean * cost1
costs.total.name = 'cost_total'
# Classification error
mr = MisclassificationRateIV(oos_thr=self.p.oos_thr)
self.error.clean = mr.apply(y, clean.labeled.h[top]) * np.float32(100.)
self.error.clean.name = 'error_rate_clean'
oosr = OOSRateIV()
self.oos.clean = oosr.apply(y, clean.labeled.h[top]) * np.float32(100.)
self.oos.clean.name = 'oos_rate_clean'
def apply(self, input_labeled, target_labeled, input_unlabeled):
self.layer_counter = 0
input_dim = self.p.encoder_layers[0]
# Store the dimension tuples in the same order as layers.
layers = self.layers
self.layer_dims = {0: input_dim}
self.lr = self.shared(self.default_lr, 'learning_rate', role=None)
self.costs = costs = AttributeDict()
self.costs.denois = AttributeDict()
self.act = AttributeDict()
self.error = AttributeDict()
top = len(layers) - 1
N = input_labeled.shape[0]
self.join = lambda l, u: T.concatenate([l, u], axis=0)
self.labeled = lambda x: x[:N] if x is not None else x
self.unlabeled = lambda x: x[N:] if x is not None else x
self.split_lu = lambda x: (self.labeled(x), self.unlabeled(x))
input_concat = self.join(input_labeled, input_unlabeled)
def encoder(input_, path_name, input_noise_std=0, noise_std=[]):
h = input_
logger.info(' 0: noise %g' % input_noise_std)
if input_noise_std > 0.:
h = h + self.noise_like(h) * input_noise_std
d = AttributeDict()
d.unlabeled = self.new_activation_dict()
d.labeled = self.new_activation_dict()
d.labeled.z[0] = self.labeled(h)
d.unlabeled.z[0] = self.unlabeled(h)
prev_dim = input_dim
for i, (spec, _, act_f) in layers[1:]:
d.labeled.h[i - 1], d.unlabeled.h[i - 1] = self.split_lu(h)
noise = noise_std[i] if i < len(noise_std) else 0.
curr_dim, z, m, s, h = self.f(h,
prev_dim,
spec,
i,
act_f,
path_name=path_name,
noise_std=noise)
assert self.layer_dims.get(i) in (None, curr_dim)
self.layer_dims[i] = curr_dim
d.labeled.z[i], d.unlabeled.z[i] = self.split_lu(z)
d.unlabeled.s[i] = s
d.unlabeled.m[i] = m
prev_dim = curr_dim
d.labeled.h[i], d.unlabeled.h[i] = self.split_lu(h)
return d
# Clean, supervised
logger.info('Encoder: clean, labeled')
clean = self.act.clean = encoder(input_concat, 'clean')
# Corrupted, supervised
logger.info('Encoder: corr, labeled')
corr = self.act.corr = encoder(input_concat,
'corr',
input_noise_std=self.p.super_noise_std,
noise_std=self.p.f_local_noise_std)
est = self.act.est = self.new_activation_dict()
# Decoder path in opposite order
logger.info('Decoder: z_corr -> z_est')
for i, ((_, spec), l_type, act_f) in layers[::-1]:
z_corr = corr.unlabeled.z[i]
z_clean = clean.unlabeled.z[i]
z_clean_s = clean.unlabeled.s.get(i)
z_clean_m = clean.unlabeled.m.get(i)
fspec = layers[i + 1][1][0] if len(layers) > i + 1 else (None,
None)
if i == top:
ver = corr.unlabeled.h[i]
ver_dim = self.layer_dims[i]
top_g = True
else:
ver = est.z.get(i + 1)
ver_dim = self.layer_dims.get(i + 1)
top_g = False
z_est = self.g(z_lat=z_corr,
z_ver=ver,
in_dims=ver_dim,
out_dims=self.layer_dims[i],
l_type=l_type,
num=i,
fspec=fspec,
top_g=top_g)
if z_est is not None:
# Denoising cost
if z_clean_s:
z_est_norm = (z_est - z_clean_m) / z_clean_s
else:
z_est_norm = z_est
se = SquaredError('denois' + str(i))
costs.denois[i] = se.apply(z_est_norm.flatten(2),
z_clean.flatten(2)) \
/ np.prod(self.layer_dims[i], dtype=floatX)
costs.denois[i].name = 'denois' + str(i)
denois_print = 'denois %.2f' % self.p.denoising_cost_x[i]
else:
denois_print = ''
# Store references for later use
est.h[i] = self.apply_act(z_est, act_f)
est.z[i] = z_est
est.s[i] = None
est.m[i] = None
logger.info(' g%d: %10s, %s, dim %s -> %s' %
(i, l_type, denois_print, self.layer_dims.get(i + 1),
self.layer_dims.get(i)))
# Costs
y = target_labeled.flatten()
costs.class_clean = CategoricalCrossEntropy().apply(
y, clean.labeled.h[top])
costs.class_clean.name = 'cost_class_clean'
costs.class_corr = CategoricalCrossEntropy().apply(
y, corr.labeled.h[top])
costs.class_corr.name = 'cost_class_corr'
# This will be used for training
costs.total = costs.class_corr * 1.0
for i in range(top + 1):
if costs.denois.get(i) and self.p.denoising_cost_x[i] > 0:
costs.total += costs.denois[i] * self.p.denoising_cost_x[i]
costs.total.name = 'cost_total'
# Classification error
mr = MisclassificationRate()
self.error.clean = mr.apply(y, clean.labeled.h[top]) * np.float32(100.)
self.error.clean.name = 'error_rate_clean'
