def init_samples(self, num):
"""Generate exact samples from the model assuming the weights are all zero, i.e.
all units are independent."""
assert np.allclose(self.weights.as_numpy_array(), 0.)
vis = rbm_utils.sample_units(gnp.outer(gnp.ones(num), self.vbias))
hid = rbm_utils.sample_units(gnp.outer(gnp.ones(num), self.hbias))
return RBMState(vis, hid)
def init_samples(self, num):
"""Generate exact samples from the model assuming the weights are all zero, i.e.
all units are independent."""
assert np.allclose(self.weights.as_numpy_array(), 0.)
vis = rbm_utils.sample_units(gnp.outer(gnp.ones(num), self.vbias))
hid = rbm_utils.sample_units(gnp.outer(gnp.ones(num), self.hbias))
return RBMState(vis, hid)
def correlation_fraction(g, s, nvis, nhid):
with misc.gnumpy_conversion_check('allow'):
expect_vis = s[:nvis]
expect_hid = s[nvis:nvis+nhid]
da = g[:nvis]
db = g[nvis:nvis+nhid]
dW = g[nvis+nhid:].reshape((nvis, nhid))
first_order_expl = gnp.outer(da, expect_hid) + gnp.outer(expect_vis, db)
first_order_norm = gnp.sum(da**2) + gnp.sum(db**2) + gnp.sum(first_order_expl**2)
dcorr = dW - first_order_expl
dcorr_norm = gnp.sum(dcorr**2)
g_norm = gnp.sum(g**2)
#return first_order_norm, dcorr_norm, g_norm
return dcorr_norm / (dcorr_norm + first_order_norm)
def glog_l_grad_stoch(self, Wmat, coin, st):
grad = np.zeros((self.dim1, self.dim2), dtype=np.float)
ggrad = gnp.garray(grad)
gWmat = gnp.garray(Wmat)
for s in range(st):
n = coin[s]
dif = gnp.outer(gnp.garray(self.Xi[n][0].T), gnp.garray(self.Xi[n][2])) - gnp.outer(gnp.garray(self.Xi[n][1].T), gnp.garray(self.Xi[n][2]))
ggrad = ggrad + self.Y[n] * dif * logistic(-self.Y[n] * (gnp.dot(gnp.garray(self.Xi[n][0]),gnp.dot(gWmat, gnp.garray(self.Xi[n][2].T))) - gnp.dot(gnp.garray(self.Xi[n][1]),gnp.dot(gWmat, gnp.garray(self.Xi[n][2].T)))).as_numpy_array())[0,0]
return ggrad.as_numpy_array()
def correlation_fraction(g, s, nvis, nhid):
with misc.gnumpy_conversion_check('allow'):
expect_vis = s[:nvis]
expect_hid = s[nvis:nvis + nhid]
da = g[:nvis]
db = g[nvis:nvis + nhid]
dW = g[nvis + nhid:].reshape((nvis, nhid))
first_order_expl = gnp.outer(da, expect_hid) + gnp.outer(
expect_vis, db)
first_order_norm = gnp.sum(da**2) + gnp.sum(db**2) + gnp.sum(
first_order_expl**2)
dcorr = dW - first_order_expl
dcorr_norm = gnp.sum(dcorr**2)
g_norm = gnp.sum(g**2)
#return first_order_norm, dcorr_norm, g_norm
return dcorr_norm / (dcorr_norm + first_order_norm)
def from_moments(moments, weights_std=0.):
"""Initialize an RBM so the visible and hidden biases match the given moments and the weights are
set to small random values."""
assert isinstance(moments, Moments)
assert np.allclose(moments.expect_prod.as_numpy_array(),
gnp.outer(moments.expect_vis, moments.expect_hid).as_numpy_array())
vbias = gnp.log(moments.expect_vis) - gnp.log(1. - moments.expect_vis)
hbias = gnp.log(moments.expect_hid) - gnp.log(1. - moments.expect_hid)
assert np.all(np.isfinite(vbias.as_numpy_array())) and np.all(np.isfinite(hbias.as_numpy_array()))
if weights_std > 0.:
weights = gnp.garray(np.random.normal(0., weights_std, size=(vbias.size, hbias.size)))
else:
weights = gnp.zeros((vbias.size, hbias.size))
return RBM(vbias, hbias, weights)
def glog_l_grad_stoch(self, Wmat, coin, st):
grad = np.zeros((self.dim1, self.dim2), dtype=np.float)
ggrad = gnp.garray(grad)
gWmat = gnp.garray(Wmat)
for s in range(st):
n = coin[s]
# scores
#expS = np.exp((gnp.dot(self.Xi[n][0], gnp.dot(Wmat, self.Xi[n][1].T))).T[0].as_numpy_array())
expS = np.exp((gnp.dot(self.Xi[n][0], gnp.dot(Wmat, self.Xi[n][1].T))).as_numpy_array())
# normalizer
z = np.sum(expS)
# probability distribution
P = expS / z
num_heads = self.Xi[n][0].shape[0]
for j in xrange(num_heads):
coeff = 1 if j==self.Y[n] else 0
ggrad = ggrad + (coeff-P[j])*(gnp.outer(gnp.garray(self.Xi[n][0][j].T), gnp.garray(self.Xi[n][1]))).as_numpy_array()
return ggrad.as_numpy_array()
def from_moments(moments, weights_std=0.):
"""Initialize an RBM so the visible and hidden biases match the given moments and the weights are
set to small random values."""
assert isinstance(moments, Moments)
assert np.allclose(
moments.expect_prod.as_numpy_array(),
gnp.outer(moments.expect_vis, moments.expect_hid).as_numpy_array())
vbias = gnp.log(moments.expect_vis) - gnp.log(1. - moments.expect_vis)
hbias = gnp.log(moments.expect_hid) - gnp.log(1. - moments.expect_hid)
assert np.all(np.isfinite(vbias.as_numpy_array())) and np.all(
np.isfinite(hbias.as_numpy_array()))
if weights_std > 0.:
weights = gnp.garray(
np.random.normal(0.,
weights_std,
size=(vbias.size, hbias.size)))
else:
weights = gnp.zeros((vbias.size, hbias.size))
return RBM(vbias, hbias, weights)
def check_moments():
with misc.gnumpy_conversion_check('allow'):
rbm = random_rbm()
pfn = tractable.exact_partition_function(rbm, batch_units=BATCH_UNITS)
expect_vis = gnp.zeros(rbm.nvis)
expect_hid = gnp.zeros(rbm.nhid)
expect_prod = gnp.zeros((rbm.nvis, rbm.nhid))
for hid_ in itertools.product(*[[0, 1]] * NHID):
hid = gnp.garray(hid_)
cond_vis = rbm.vis_expectations(hid)
p = np.exp(rbm.free_energy_hid(hid[nax, :])[0] - pfn)
expect_vis += p * cond_vis
expect_hid += p * hid
expect_prod += p * gnp.outer(cond_vis, hid)
moments = tractable.exact_moments(rbm, batch_units=BATCH_UNITS)
assert np.allclose(expect_vis, moments.expect_vis)
assert np.allclose(expect_hid, moments.expect_hid)
assert np.allclose(expect_prod, moments.expect_prod)
def check_moments():
with misc.gnumpy_conversion_check('allow'):
rbm = random_rbm()
pfn = tractable.exact_partition_function(rbm, batch_units=BATCH_UNITS)
expect_vis = gnp.zeros(rbm.nvis)
expect_hid = gnp.zeros(rbm.nhid)
expect_prod = gnp.zeros((rbm.nvis, rbm.nhid))
for hid_ in itertools.product(*[[0, 1]] * NHID):
hid = gnp.garray(hid_)
cond_vis = rbm.vis_expectations(hid)
p = np.exp(rbm.free_energy_hid(hid[nax, :])[0] - pfn)
expect_vis += p * cond_vis
expect_hid += p * hid
expect_prod += p * gnp.outer(cond_vis, hid)
moments = tractable.exact_moments(rbm, batch_units=BATCH_UNITS)
assert np.allclose(expect_vis, moments.expect_vis)
assert np.allclose(expect_hid, moments.expect_hid)
assert np.allclose(expect_prod, moments.expect_prod)
def apply_update(self, pos_moments, neg_moments, rbm, weight_decay, lrates):
pos_prods = pos_moments.expect_prod + \
-gnp.outer(pos_moments.expect_vis, self.expect_hid) + \
-gnp.outer(self.expect_vis, pos_moments.expect_hid) + \
gnp.outer(self.expect_vis, self.expect_hid)
neg_prods = neg_moments.expect_prod + \
-gnp.outer(neg_moments.expect_vis, self.expect_hid) + \
-gnp.outer(self.expect_vis, neg_moments.expect_hid) + \
gnp.outer(self.expect_vis, self.expect_hid)
weights_update = lrates.weights * (pos_prods - neg_prods) + \
-lrates.weights * weight_decay * rbm.weights
vbias_update = lrates.vbias * (pos_moments.expect_vis - neg_moments.expect_vis) + \
-gnp.dot(weights_update, neg_moments.expect_hid)
hbias_update = lrates.hbias * (pos_moments.expect_hid - neg_moments.expect_hid) + \
-gnp.dot(neg_moments.expect_vis, weights_update)
rbm.vbias += vbias_update
rbm.hbias += hbias_update
rbm.weights += weights_update
def from_independent(cls, expect_vis, expect_hid):
return cls(expect_vis, expect_hid, gnp.outer(expect_vis, expect_hid))
def from_independent(cls, expect_vis, expect_hid):
return cls(expect_vis, expect_hid, gnp.outer(expect_vis, expect_hid))