def init_samples(self, num): """Generate exact samples from the model assuming the weights are all zero, i.e. all units are independent.""" assert np.allclose(self.weights.as_numpy_array(), 0.) vis = rbm_utils.sample_units(gnp.outer(gnp.ones(num), self.vbias)) hid = rbm_utils.sample_units(gnp.outer(gnp.ones(num), self.hbias)) return RBMState(vis, hid)
def correlation_fraction(g, s, nvis, nhid): with misc.gnumpy_conversion_check('allow'): expect_vis = s[:nvis] expect_hid = s[nvis:nvis+nhid] da = g[:nvis] db = g[nvis:nvis+nhid] dW = g[nvis+nhid:].reshape((nvis, nhid)) first_order_expl = gnp.outer(da, expect_hid) + gnp.outer(expect_vis, db) first_order_norm = gnp.sum(da**2) + gnp.sum(db**2) + gnp.sum(first_order_expl**2) dcorr = dW - first_order_expl dcorr_norm = gnp.sum(dcorr**2) g_norm = gnp.sum(g**2) #return first_order_norm, dcorr_norm, g_norm return dcorr_norm / (dcorr_norm + first_order_norm)
def glog_l_grad_stoch(self, Wmat, coin, st): grad = np.zeros((self.dim1, self.dim2), dtype=np.float) ggrad = gnp.garray(grad) gWmat = gnp.garray(Wmat) for s in range(st): n = coin[s] dif = gnp.outer(gnp.garray(self.Xi[n][0].T), gnp.garray(self.Xi[n][2])) - gnp.outer(gnp.garray(self.Xi[n][1].T), gnp.garray(self.Xi[n][2])) ggrad = ggrad + self.Y[n] * dif * logistic(-self.Y[n] * (gnp.dot(gnp.garray(self.Xi[n][0]),gnp.dot(gWmat, gnp.garray(self.Xi[n][2].T))) - gnp.dot(gnp.garray(self.Xi[n][1]),gnp.dot(gWmat, gnp.garray(self.Xi[n][2].T)))).as_numpy_array())[0,0] return ggrad.as_numpy_array()
def correlation_fraction(g, s, nvis, nhid): with misc.gnumpy_conversion_check('allow'): expect_vis = s[:nvis] expect_hid = s[nvis:nvis + nhid] da = g[:nvis] db = g[nvis:nvis + nhid] dW = g[nvis + nhid:].reshape((nvis, nhid)) first_order_expl = gnp.outer(da, expect_hid) + gnp.outer( expect_vis, db) first_order_norm = gnp.sum(da**2) + gnp.sum(db**2) + gnp.sum( first_order_expl**2) dcorr = dW - first_order_expl dcorr_norm = gnp.sum(dcorr**2) g_norm = gnp.sum(g**2) #return first_order_norm, dcorr_norm, g_norm return dcorr_norm / (dcorr_norm + first_order_norm)
def from_moments(moments, weights_std=0.): """Initialize an RBM so the visible and hidden biases match the given moments and the weights are set to small random values.""" assert isinstance(moments, Moments) assert np.allclose(moments.expect_prod.as_numpy_array(), gnp.outer(moments.expect_vis, moments.expect_hid).as_numpy_array()) vbias = gnp.log(moments.expect_vis) - gnp.log(1. - moments.expect_vis) hbias = gnp.log(moments.expect_hid) - gnp.log(1. - moments.expect_hid) assert np.all(np.isfinite(vbias.as_numpy_array())) and np.all(np.isfinite(hbias.as_numpy_array())) if weights_std > 0.: weights = gnp.garray(np.random.normal(0., weights_std, size=(vbias.size, hbias.size))) else: weights = gnp.zeros((vbias.size, hbias.size)) return RBM(vbias, hbias, weights)
def glog_l_grad_stoch(self, Wmat, coin, st): grad = np.zeros((self.dim1, self.dim2), dtype=np.float) ggrad = gnp.garray(grad) gWmat = gnp.garray(Wmat) for s in range(st): n = coin[s] # scores #expS = np.exp((gnp.dot(self.Xi[n][0], gnp.dot(Wmat, self.Xi[n][1].T))).T[0].as_numpy_array()) expS = np.exp((gnp.dot(self.Xi[n][0], gnp.dot(Wmat, self.Xi[n][1].T))).as_numpy_array()) # normalizer z = np.sum(expS) # probability distribution P = expS / z num_heads = self.Xi[n][0].shape[0] for j in xrange(num_heads): coeff = 1 if j==self.Y[n] else 0 ggrad = ggrad + (coeff-P[j])*(gnp.outer(gnp.garray(self.Xi[n][0][j].T), gnp.garray(self.Xi[n][1]))).as_numpy_array() return ggrad.as_numpy_array()
def from_moments(moments, weights_std=0.): """Initialize an RBM so the visible and hidden biases match the given moments and the weights are set to small random values.""" assert isinstance(moments, Moments) assert np.allclose( moments.expect_prod.as_numpy_array(), gnp.outer(moments.expect_vis, moments.expect_hid).as_numpy_array()) vbias = gnp.log(moments.expect_vis) - gnp.log(1. - moments.expect_vis) hbias = gnp.log(moments.expect_hid) - gnp.log(1. - moments.expect_hid) assert np.all(np.isfinite(vbias.as_numpy_array())) and np.all( np.isfinite(hbias.as_numpy_array())) if weights_std > 0.: weights = gnp.garray( np.random.normal(0., weights_std, size=(vbias.size, hbias.size))) else: weights = gnp.zeros((vbias.size, hbias.size)) return RBM(vbias, hbias, weights)
def check_moments(): with misc.gnumpy_conversion_check('allow'): rbm = random_rbm() pfn = tractable.exact_partition_function(rbm, batch_units=BATCH_UNITS) expect_vis = gnp.zeros(rbm.nvis) expect_hid = gnp.zeros(rbm.nhid) expect_prod = gnp.zeros((rbm.nvis, rbm.nhid)) for hid_ in itertools.product(*[[0, 1]] * NHID): hid = gnp.garray(hid_) cond_vis = rbm.vis_expectations(hid) p = np.exp(rbm.free_energy_hid(hid[nax, :])[0] - pfn) expect_vis += p * cond_vis expect_hid += p * hid expect_prod += p * gnp.outer(cond_vis, hid) moments = tractable.exact_moments(rbm, batch_units=BATCH_UNITS) assert np.allclose(expect_vis, moments.expect_vis) assert np.allclose(expect_hid, moments.expect_hid) assert np.allclose(expect_prod, moments.expect_prod)
def apply_update(self, pos_moments, neg_moments, rbm, weight_decay, lrates): pos_prods = pos_moments.expect_prod + \ -gnp.outer(pos_moments.expect_vis, self.expect_hid) + \ -gnp.outer(self.expect_vis, pos_moments.expect_hid) + \ gnp.outer(self.expect_vis, self.expect_hid) neg_prods = neg_moments.expect_prod + \ -gnp.outer(neg_moments.expect_vis, self.expect_hid) + \ -gnp.outer(self.expect_vis, neg_moments.expect_hid) + \ gnp.outer(self.expect_vis, self.expect_hid) weights_update = lrates.weights * (pos_prods - neg_prods) + \ -lrates.weights * weight_decay * rbm.weights vbias_update = lrates.vbias * (pos_moments.expect_vis - neg_moments.expect_vis) + \ -gnp.dot(weights_update, neg_moments.expect_hid) hbias_update = lrates.hbias * (pos_moments.expect_hid - neg_moments.expect_hid) + \ -gnp.dot(neg_moments.expect_vis, weights_update) rbm.vbias += vbias_update rbm.hbias += hbias_update rbm.weights += weights_update
def from_independent(cls, expect_vis, expect_hid): return cls(expect_vis, expect_hid, gnp.outer(expect_vis, expect_hid))