def test_counting(): BATCH_SIZE = 2 BATCHES = 3 NUM_FEATURES = 4 num_examples = BATCHES * BATCH_SIZE dataset = DummyDataset(num_examples=num_examples, num_features=NUM_FEATURES) algorithm = DefaultTrainingAlgorithm(batch_size=BATCH_SIZE, batches_per_iter=BATCHES) model = S3C(nvis=NUM_FEATURES, nhid=1, irange=.01, init_bias_hid=0., init_B=1., min_B=1., max_B=1., init_alpha=1., min_alpha=1., max_alpha=1., init_mu=0., m_step=Grad_M_Step(learning_rate=0.), e_step=E_Step(h_new_coeff_schedule=[1.])) algorithm.setup(model=model, dataset=dataset) algorithm.train(dataset=dataset) if not (model.monitor.get_batches_seen() == BATCHES): raise AssertionError('Should have seen '+str(BATCHES) + \ ' batches but saw '+str(model.monitor.get_batches_seen())) assert model.monitor.get_examples_seen() == num_examples assert isinstance(model.monitor.get_examples_seen(), py_integer_types) assert isinstance(model.monitor.get_batches_seen(), py_integer_types)
def test_multiple_monitoring_datasets(): # tests that DefaultTrainingAlgorithm can take multiple # monitoring datasets. BATCH_SIZE = 2 BATCHES = 3 NUM_FEATURES = 4 dim = 3 m = 10 rng = np.random.RandomState([2014, 02, 25]) X = rng.randn(m, dim) Y = rng.randn(m, dim) train = DenseDesignMatrix(X=X) test = DenseDesignMatrix(X=Y) algorithm = DefaultTrainingAlgorithm( batch_size=BATCH_SIZE, batches_per_iter=BATCHES, monitoring_dataset={'train': train, 'test': test}) model = S3C(nvis=NUM_FEATURES, nhid=1, irange=.01, init_bias_hid=0., init_B=1., min_B=1., max_B=1., init_alpha=1., min_alpha=1., max_alpha=1., init_mu=0., m_step=Grad_M_Step(learning_rate=0.), e_step=E_Step(h_new_coeff_schedule=[1.])) algorithm.setup(model=model, dataset=train) algorithm.train(dataset=train)
def __init__(self): """ gets a small batch of data sets up an S3C model and learns on the data creates an expression for the log likelihood of the data """ self.tol = 1e-5 #dataset = serial.load('${GOODFELI_TMP}/cifar10_preprocessed_train_1K.pkl') X = np.random.RandomState([1, 2, 3]).randn(1000, 108) #dataset.get_batch_design(1000) #X = X[:,0:2] #warnings.warn('hack') #X[0,0] = 1. #X[0,1] = -1. m, D = X.shape N = 300 self.model = S3C( nvis=D, #disable_W_update = 1, nhid=N, irange=.5, init_bias_hid=-.1, init_B=1., min_B=1e-8, max_B=1e8, tied_B=1, e_step=E_Step_Scan( #h_new_coeff_schedule = [ ], h_new_coeff_schedule=[.01]), init_alpha=1., min_alpha=1e-8, max_alpha=1e8, init_mu=1., m_step=Grad_M_Step(learning_rate=1.0), ) #warnings.warn('hack') #W = self.model.W.get_value() #W[0,0] = 1. #W[1,0] = 1. #self.model.W.set_value(W) self.orig_params = self.model.get_param_values() model = self.model self.mf_obs = model.e_step.infer(X) self.stats = SufficientStatistics.from_observations( needed_stats=model.m_step.needed_stats(), V=X, **self.mf_obs) self.prob = self.model.expected_log_prob_vhs( self.stats, H_hat=self.mf_obs['H_hat'], S_hat=self.mf_obs['S_hat']) self.X = X self.m = m self.D = D self.N = N
def __init__(self): """ gets a small batch of data sets up an S3C model """ # We also have to change the value of config.floatX in __init__. self.prev_floatX = config.floatX config.floatX = 'float64' try: self.tol = 1e-5 #dataset = serial.load('${PYLEARN2_DATA_PATH}/stl10/stl10_patches/data.pkl') #X = dataset.get_batch_design(1000) #X = X[:,0:5] X = np.random.RandomState([1, 2, 3]).randn(1000, 5) X -= X.mean() X /= X.std() m, D = X.shape N = 5 #don't give the model an e_step or learning rate so it won't spend years compiling a learn_func self.model = S3C( nvis=D, nhid=N, irange=.1, init_bias_hid=0., init_B=3., min_B=1e-8, max_B=1000., init_alpha=1., min_alpha=1e-8, max_alpha=1000., init_mu=1., e_step=None, m_step=Grad_M_Step(), min_bias_hid=-1e30, max_bias_hid=1e30, ) self.model.make_pseudoparams() self.h_new_coeff_schedule = [ .1, .2, .3, .4, .5, .6, .7, .8, .9, 1. ] self.e_step = E_Step_Scan( h_new_coeff_schedule=self.h_new_coeff_schedule) self.e_step.register_model(self.model) self.X = X self.N = N self.m = m finally: config.floatX = self.prev_floatX
def __init__(self): """ gets a small batch of data sets up an S3C model and learns on the data creates an expression for the log likelihood of the data """ # We also have to change the value of config.floatX in __init__. self.prev_floatX = config.floatX config.floatX = 'float64' try: self.tol = 1e-5 if config.mode in ["DebugMode", "DEBUG_MODE"]: X = np.random.RandomState([1, 2, 3]).randn(30, 108) m, D = X.shape N = 10 else: X = np.random.RandomState([1, 2, 3]).randn(1000, 108) m, D = X.shape N = 300 self.model = S3C(nvis = D, nhid = N, irange = .5, init_bias_hid = -.1, init_B = 1., min_B = 1e-8, max_B = 1e8, tied_B = 1, e_step = E_Step_Scan( h_new_coeff_schedule = [ .01 ] ), init_alpha = 1., min_alpha = 1e-8, max_alpha = 1e8, init_mu = 1., m_step = Grad_M_Step( learning_rate = 1.0 ), ) self.orig_params = self.model.get_param_values() model = self.model self.mf_obs = model.e_step.infer(X) self.stats = SufficientStatistics.from_observations(needed_stats = model.m_step.needed_stats(), V =X, ** self.mf_obs) self.prob = self.model.expected_log_prob_vhs( self.stats , H_hat = self.mf_obs['H_hat'], S_hat = self.mf_obs['S_hat']) self.X = X self.m = m self.D = D self.N = N finally: config.floatX = self.prev_floatX
def make_s3c(self): return S3C(nvis = self.D, nhid = self.N, irange = .1, init_bias_hid = -1.5, init_B = 3., min_B = 1e-8, max_B = 1000., init_alpha = 1., min_alpha = 1e-8, max_alpha = 1000., init_mu = 1., e_step = None, m_step = Grad_M_Step(), min_bias_hid = -1e30, max_bias_hid = 1e30, )
def __init__(self): """ gets a small batch of data sets up a PD-DBM model """ self.tol = 1e-5 X = np.random.RandomState([1,2,3]).randn(1000,5) X -= X.mean() X /= X.std() m, D = X.shape N = 6 N2 = 7 s3c = S3C(nvis = D, nhid = N, irange = .1, init_bias_hid = -1.5, init_B = 3., min_B = 1e-8, max_B = 1000., init_alpha = 1., min_alpha = 1e-8, max_alpha = 1000., init_mu = 1., e_step = None, m_step = Grad_M_Step(), min_bias_hid = -1e30, max_bias_hid = 1e30, ) rbm = RBM(nvis = N, nhid = N2, irange = .1, init_bias_vis = -1.5, init_bias_hid = 1.5) #don't give the model an inference procedure or learning rate so it won't spend years compiling a learn_func self.model = PDDBM( dbm = DBM( use_cd = 1, rbms = [ rbm ]), s3c = s3c ) self.model.make_pseudoparams() self.inference_procedure = InferenceProcedure( clip_reflections = True, rho = .5 ) self.inference_procedure.register_model(self.model) self.X = X self.N = N self.N2 = N2 self.m = m
local_rf_src=dataset, local_rf_shape=[6, 6], local_rf_max_shape=dataset.view_shape()[0:2], local_rf_draw_patches=True, tied_B=False, init_bias_hid=init_bias_hid, init_mu=init_mu, init_alpha=init_alpha, init_B=init_B, irange=None, min_B=.01, max_B=1e6, min_alpha=.01, max_alpha=1e6, e_step=None, m_step=Grad_M_Step()) model.make_pseudoparams() #this initial E step is just to help out the #BatchGradientInference object model.e_step = E_Step_Scan(clip_reflections=True, rho=0.5, h_new_coeff_schedule=[.1] * OVERKILL, s_new_coeff_schedule=[.3] * OVERKILL) model.e_step.register_model(model) import theano.tensor as T from theano import function def get_needed_steps(ip, X):
def __init__(self, model = None, X = None, tol = 1e-5, init_H = None, init_S = None, init_G = None): """ gets a small batch of data sets up a PD-DBM model """ self.tol = tol if X is None: X = np.random.RandomState([1,2,3]).randn(1000,5) X -= X.mean() X /= X.std() m, D = X.shape if model is None: N = 6 N2 = 7 s3c = S3C(nvis = D, nhid = N, irange = .1, init_bias_hid = -1.5, init_B = 3., min_B = 1e-8, max_B = 1000., init_alpha = 1., min_alpha = 1e-8, max_alpha = 1000., init_mu = 1., e_step = None, m_step = Grad_M_Step(), min_bias_hid = -1e30, max_bias_hid = 1e30, ) rbm = RBM(nvis = N, nhid = N2, irange = .5, init_bias_vis = -1.5, init_bias_hid = 1.5) #don't give the model an inference procedure or learning rate so it won't spend years compiling a learn_func self.model = PDDBM( dbm = DBM( use_cd = 1, rbms = [ rbm ]), s3c = s3c ) self.model.make_pseudoparams() self.inference_procedure = InferenceProcedure( clip_reflections = True, rho = .5 ) self.inference_procedure.register_model(self.model) else: self.model = model self.inference_procedure = model.inference_procedure N = model.s3c.nhid N2 = model.dbm.rbms[0].nhid self.X = X self.N = N self.N2 = N2 self.m = m if init_H is None: self.init_H = np.cast[config.floatX](self.model.rng.uniform(0.,1.,(self.m, self.N))) self.init_S = np.cast[config.floatX](self.model.rng.uniform(-5.,5.,(self.m, self.N))) self.init_G = np.cast[config.floatX](self.model.rng.uniform(0.,1.,(self.m,self.N2))) else: assert init_S is not None assert init_G is not None self.init_H = init_H self.init_S = init_S self.init_G = init_G