def compute_one_hot_reconstructions(rbm, fit, level, n_recon, num_to_avg=1): n = rbm.layers[level].len grid_size = int(sqrt(n_recon)) one_hotz = rbm.layers[level].onehot(n) v_model = be.zeros((n, rbm.layers[0].len)) for k in range(num_to_avg): # set up the initial state state = State.from_model(n, rbm) state.units[level] = one_hotz dropout_scale = State.dropout_rescale(rbm) # set up a sampler and update the state reconstructions = fit.SequentialMC(rbm, clamped=[level], updater='mean_field_iteration') reconstructions.set_state(state) reconstructions.update_state(10, dropout_scale) v_model += reconstructions.state.units[0] v_model /= num_to_avg # plot the resulting visible unit activations idx = numpy.random.choice(range(len(v_model)), n_recon, replace=False) recons = numpy.array([be.to_numpy_array(v_model[i]) for i in idx]) recons = recons.reshape(grid_size, grid_size, -1) return recons
def compute_reconstructions(rbm, v_data, fit, n_recon=10, vertical=False, num_to_avg=1): v_model = be.zeros_like(v_data) # Average over n reconstruction attempts for k in range(num_to_avg): data_state = State.from_visible(v_data, rbm) visible = data_state.units[0] reconstructions = fit.DrivenSequentialMC(rbm) reconstructions.set_state(data_state) dropout_scale = State.dropout_rescale(rbm) reconstructions.update_state(1, dropout_scale) v_model += rbm.deterministic_iteration(1, reconstructions.state, dropout_scale).units[0] v_model /= num_to_avg idx = numpy.random.choice(range(len(v_model)), n_recon, replace=False) grid = numpy.array( [[be.to_numpy_array(visible[i]), be.to_numpy_array(v_model[i])] for i in idx]) if vertical: return grid else: return grid.swapaxes(0, 1)
def test_state_for_grad_DrivenSequentialMC(): num_visible_units = 100 num_hidden_units = 50 batch_size = 25 # set a seed for the random number generator be.set_seed() # set up some layer and model objects vis_layer = layers.BernoulliLayer(num_visible_units) hid_layer = layers.BernoulliLayer(num_hidden_units) rbm = model.Model([vis_layer, hid_layer]) # randomly set the intrinsic model parameters a = be.randn((num_visible_units, )) b = be.randn((num_hidden_units, )) W = be.randn((num_visible_units, num_hidden_units)) rbm.layers[0].params.loc[:] = a rbm.layers[1].params.loc[:] = b rbm.weights[0].params.matrix[:] = W # generate a random batch of data vdata = rbm.layers[0].random((batch_size, num_visible_units)) data_state = State.from_visible(vdata, rbm) dropout_scale = State.dropout_rescale(rbm) # since we set no dropout, dropout_scale should be None assert dropout_scale is None for u in [ 'markov_chain', 'mean_field_iteration', 'deterministic_iteration' ]: # set up the sampler sampler = fit.DrivenSequentialMC(rbm, updater=u, clamped=[0]) sampler.set_state(data_state) # update the state of the hidden layer grad_state = sampler.state_for_grad(1, dropout_scale) assert be.allclose(data_state.units[0], grad_state.units[0]), \ "visible layer is clamped, and shouldn't get updated: {}".format(u) assert not be.allclose(data_state.units[1], grad_state.units[1]), \ "hidden layer is not clamped, and should get updated: {}".format(u) # compute the conditional mean with the layer function ave = rbm.layers[1].conditional_mean( rbm._connected_rescaled_units(1, data_state, dropout_scale), rbm._connected_weights(1)) assert be.allclose(ave, grad_state.units[1]), \ "hidden layer of grad_state should be conditional mean: {}".format(u)
def test_clamped_SequentialMC(): num_visible_units = 100 num_hidden_units = 50 batch_size = 25 steps = 1 # set a seed for the random number generator be.set_seed() # set up some layer and model objects vis_layer = layers.BernoulliLayer(num_visible_units) hid_layer = layers.BernoulliLayer(num_hidden_units) rbm = model.Model([vis_layer, hid_layer]) # randomly set the intrinsic model parameters a = be.randn((num_visible_units, )) b = be.randn((num_hidden_units, )) W = be.randn((num_visible_units, num_hidden_units)) rbm.layers[0].params.loc[:] = a rbm.layers[1].params.loc[:] = b rbm.weights[0].params.matrix[:] = W # generate a random batch of data vdata = rbm.layers[0].random((batch_size, num_visible_units)) data_state = State.from_visible(vdata, rbm) dropout_scale = State.dropout_rescale(rbm) # since we set no dropout, dropout_scale should be None assert dropout_scale is None for u in [ 'markov_chain', 'mean_field_iteration', 'deterministic_iteration' ]: # set up the sampler with the visible layer clamped sampler = fit.SequentialMC(rbm, updater=u, clamped=[0]) sampler.set_state(data_state) # update the sampler state and check the output sampler.update_state(steps, dropout_scale) assert be.allclose(data_state.units[0], sampler.state.units[0]), \ "visible layer is clamped, and shouldn't get updated: {}".format(u) assert not be.allclose(data_state.units[1], sampler.state.units[1]), \ "hidden layer is not clamped, and should get updated: {}".format(u)
def compute_fantasy_particles(rbm, v_data, fit, n_fantasy=25): grid_size = int(sqrt(n_fantasy)) assert grid_size == sqrt( n_fantasy), "n_fantasy must be the square of an integer" random_samples = rbm.random(v_data) model_state = State.from_visible(random_samples, rbm) schedule = schedules.PowerLawDecay(initial=1.0, coefficient=0.5) fantasy = fit.DrivenSequentialMC(rbm, schedule=schedule) dropout_scale = State.dropout_rescale(rbm) fantasy.set_state(model_state) fantasy.update_state(1000, dropout_scale) v_model = rbm.deterministic_iteration(1, fantasy.state, dropout_scale).units[0] idx = numpy.random.choice(range(len(v_model)), n_fantasy, replace=False) grid = numpy.array([be.to_numpy_array(v_model[i]) for i in idx]) return grid.reshape(grid_size, grid_size, -1)
def test_conditional_sampling(): """ Test sampling from one layer conditioned on the state of another layer. Note: This test compares values estimated by *sampling* to values computed analytically. It can fail for small batch_size, or strict tolerances, even if everything is working propery. """ num_visible_units = 20 num_hidden_units = 10 steps = 1000 mean_tol = 0.1 # set a seed for the random number generator be.set_seed() layer_types = [layers.BernoulliLayer, layers.GaussianLayer] for layer_type in layer_types: # set up some layer and model objects vis_layer = layer_type(num_visible_units) hid_layer = layer_type(num_hidden_units) rbm = model.Model([vis_layer, hid_layer]) # randomly set the intrinsic model parameters a = be.rand((num_visible_units, )) b = be.rand((num_hidden_units, )) W = 10 * be.rand((num_visible_units, num_hidden_units)) rbm.layers[0].params.loc[:] = a rbm.layers[1].params.loc[:] = b rbm.weights[0].params.matrix[:] = W if layer_type == layers.GaussianLayer: log_var_a = be.randn((num_visible_units, )) log_var_b = be.randn((num_hidden_units, )) rbm.layers[0].params.log_var[:] = log_var_a rbm.layers[1].params.log_var[:] = log_var_b # initialize a state state = State.from_model(1, rbm) dropout = State.dropout_rescale(rbm) # set up a calculator for the moments moments = mu.MeanVarianceArrayCalculator() for _ in range(steps): moments.update(rbm.layers[0].conditional_sample( rbm._connected_rescaled_units(0, state, dropout), rbm._connected_weights(0))) model_mean = rbm.layers[0].conditional_mean( rbm._connected_rescaled_units(0, state, dropout), rbm._connected_weights(0)) ave = moments.mean close = be.allclose(ave, model_mean[0], rtol=mean_tol, atol=mean_tol) assert close, "{} conditional mean".format(layer_type) if layer_type == layers.GaussianLayer: model_mean, model_var = rbm.layers[0]._conditional_params( rbm._connected_rescaled_units(0, state, dropout), rbm._connected_weights(0)) close = be.allclose(be.sqrt(moments.var), be.sqrt(model_var[0]), rtol=mean_tol, atol=mean_tol) assert close, "{} conditional standard deviation".format( layer_type)
def test_independent(): """ Test sampling from an rbm with two layers connected by a weight matrix that contains all zeros, so that the layers are independent. Note: This test compares values estimated by *sampling* to values computed analytically. It can fail for small batch_size, or strict tolerances, even if everything is working propery. """ num_visible_units = 20 num_hidden_units = 10 batch_size = 1000 steps = 100 mean_tol = 0.1 corr_tol = 0.2 # set a seed for the random number generator be.set_seed() layer_types = [layers.BernoulliLayer, layers.GaussianLayer] for layer_type in layer_types: # set up some layer and model objects vis_layer = layer_type(num_visible_units) hid_layer = layer_type(num_hidden_units) rbm = model.Model([vis_layer, hid_layer]) # randomly set the intrinsic model parameters a = be.rand((num_visible_units, )) b = be.rand((num_hidden_units, )) W = be.zeros((num_visible_units, num_hidden_units)) rbm.layers[0].params.loc[:] = a rbm.layers[1].params.loc[:] = b rbm.weights[0].params.matrix[:] = W if layer_type == layers.GaussianLayer: log_var_a = be.randn((num_visible_units, )) log_var_b = be.randn((num_hidden_units, )) rbm.layers[0].params.log_var[:] = log_var_a rbm.layers[1].params.log_var[:] = log_var_b # initialize a state state = State.from_model(batch_size, rbm) dropout = State.dropout_rescale(rbm) # run a markov chain to update the state state = rbm.markov_chain(steps, state, dropout) # compute the mean state_for_moments = State.from_model(1, rbm) sample_mean = [ be.mean(state.units[i], axis=0) for i in range(len(state.units)) ] model_mean = [ rbm.layers[i].conditional_mean( rbm._connected_rescaled_units(i, state_for_moments, dropout), rbm._connected_weights(i)) for i in range(rbm.num_layers) ] # check that the means are roughly equal for i in range(rbm.num_layers): ave = sample_mean[i] close = be.allclose(ave, model_mean[i][0], rtol=mean_tol, atol=mean_tol) assert close, "{0} {1}: sample mean does not match model mean".format( layer_type, i) # check the cross correlation between the layers crosscov = be.cov(state.units[0], state.units[1]) norm = be.outer(be.std(state.units[0], axis=0), be.std(state.units[1], axis=0)) crosscorr = be.divide(norm, crosscov) assert be.tmax( be.tabs(crosscorr) ) < corr_tol, "{} cross correlation too large".format(layer_type)