def test_counting():
BATCH_SIZE = 2
BATCHES = 3
NUM_FEATURES = 4
num_examples = BATCHES * BATCH_SIZE
dataset = DummyDataset(num_examples=num_examples,
num_features=NUM_FEATURES)
algorithm = DefaultTrainingAlgorithm(batch_size=BATCH_SIZE,
batches_per_iter=BATCHES)
model = S3C(nvis=NUM_FEATURES,
nhid=1,
irange=.01,
init_bias_hid=0.,
init_B=1.,
min_B=1.,
max_B=1.,
init_alpha=1.,
min_alpha=1.,
max_alpha=1.,
init_mu=0.,
m_step=Grad_M_Step(learning_rate=0.),
e_step=E_Step(h_new_coeff_schedule=[1.]))
algorithm.setup(model=model, dataset=dataset)
algorithm.train(dataset=dataset)
if not (model.monitor.get_batches_seen() == BATCHES):
raise AssertionError('Should have seen '+str(BATCHES) + \
' batches but saw '+str(model.monitor.get_batches_seen()))
assert model.monitor.get_examples_seen() == num_examples
assert isinstance(model.monitor.get_examples_seen(), py_integer_types)
assert isinstance(model.monitor.get_batches_seen(), py_integer_types)
def test_multiple_monitoring_datasets():
# tests that DefaultTrainingAlgorithm can take multiple
# monitoring datasets.
BATCH_SIZE = 2
BATCHES = 3
NUM_FEATURES = 4
dim = 3
m = 10
rng = np.random.RandomState([2014, 02, 25])
X = rng.randn(m, dim)
Y = rng.randn(m, dim)
train = DenseDesignMatrix(X=X)
test = DenseDesignMatrix(X=Y)
algorithm = DefaultTrainingAlgorithm(
batch_size=BATCH_SIZE,
batches_per_iter=BATCHES,
monitoring_dataset={'train': train, 'test': test})
model = S3C(nvis=NUM_FEATURES, nhid=1,
irange=.01, init_bias_hid=0., init_B=1.,
min_B=1., max_B=1., init_alpha=1.,
min_alpha=1., max_alpha=1., init_mu=0.,
m_step=Grad_M_Step(learning_rate=0.),
e_step=E_Step(h_new_coeff_schedule=[1.]))
algorithm.setup(model=model, dataset=train)
algorithm.train(dataset=train)
def __init__(self):
""" gets a small batch of data
sets up an S3C model and learns on the data
creates an expression for the log likelihood of the data
"""
self.tol = 1e-5
#dataset = serial.load('${GOODFELI_TMP}/cifar10_preprocessed_train_1K.pkl')
X = np.random.RandomState([1, 2, 3]).randn(1000, 108)
#dataset.get_batch_design(1000)
#X = X[:,0:2]
#warnings.warn('hack')
#X[0,0] = 1.
#X[0,1] = -1.
m, D = X.shape
N = 300
self.model = S3C(
nvis=D,
#disable_W_update = 1,
nhid=N,
irange=.5,
init_bias_hid=-.1,
init_B=1.,
min_B=1e-8,
max_B=1e8,
tied_B=1,
e_step=E_Step_Scan(
#h_new_coeff_schedule = [ ],
h_new_coeff_schedule=[.01]),
init_alpha=1.,
min_alpha=1e-8,
max_alpha=1e8,
init_mu=1.,
m_step=Grad_M_Step(learning_rate=1.0),
)
#warnings.warn('hack')
#W = self.model.W.get_value()
#W[0,0] = 1.
#W[1,0] = 1.
#self.model.W.set_value(W)
self.orig_params = self.model.get_param_values()
model = self.model
self.mf_obs = model.e_step.infer(X)
self.stats = SufficientStatistics.from_observations(
needed_stats=model.m_step.needed_stats(), V=X, **self.mf_obs)
self.prob = self.model.expected_log_prob_vhs(
self.stats, H_hat=self.mf_obs['H_hat'], S_hat=self.mf_obs['S_hat'])
self.X = X
self.m = m
self.D = D
self.N = N
def __init__(self):
""" gets a small batch of data
sets up an S3C model
"""
# We also have to change the value of config.floatX in __init__.
self.prev_floatX = config.floatX
config.floatX = 'float64'
try:
self.tol = 1e-5
#dataset = serial.load('${PYLEARN2_DATA_PATH}/stl10/stl10_patches/data.pkl')
#X = dataset.get_batch_design(1000)
#X = X[:,0:5]
X = np.random.RandomState([1, 2, 3]).randn(1000, 5)
X -= X.mean()
X /= X.std()
m, D = X.shape
N = 5
#don't give the model an e_step or learning rate so it won't spend years compiling a learn_func
self.model = S3C(
nvis=D,
nhid=N,
irange=.1,
init_bias_hid=0.,
init_B=3.,
min_B=1e-8,
max_B=1000.,
init_alpha=1.,
min_alpha=1e-8,
max_alpha=1000.,
init_mu=1.,
e_step=None,
m_step=Grad_M_Step(),
min_bias_hid=-1e30,
max_bias_hid=1e30,
)
self.model.make_pseudoparams()
self.h_new_coeff_schedule = [
.1, .2, .3, .4, .5, .6, .7, .8, .9, 1.
]
self.e_step = E_Step_Scan(
h_new_coeff_schedule=self.h_new_coeff_schedule)
self.e_step.register_model(self.model)
self.X = X
self.N = N
self.m = m
finally:
config.floatX = self.prev_floatX
def __init__(self):
""" gets a small batch of data
sets up an S3C model and learns on the data
creates an expression for the log likelihood of the data
"""
# We also have to change the value of config.floatX in __init__.
self.prev_floatX = config.floatX
config.floatX = 'float64'
try:
self.tol = 1e-5
if config.mode in ["DebugMode", "DEBUG_MODE"]:
X = np.random.RandomState([1, 2, 3]).randn(30, 108)
m, D = X.shape
N = 10
else:
X = np.random.RandomState([1, 2, 3]).randn(1000, 108)
m, D = X.shape
N = 300
self.model = S3C(nvis = D,
nhid = N,
irange = .5,
init_bias_hid = -.1,
init_B = 1.,
min_B = 1e-8,
max_B = 1e8,
tied_B = 1,
e_step = E_Step_Scan(
h_new_coeff_schedule = [ .01 ]
),
init_alpha = 1.,
min_alpha = 1e-8, max_alpha = 1e8,
init_mu = 1.,
m_step = Grad_M_Step( learning_rate = 1.0 ),
)
self.orig_params = self.model.get_param_values()
model = self.model
self.mf_obs = model.e_step.infer(X)
self.stats = SufficientStatistics.from_observations(needed_stats =
model.m_step.needed_stats(), V =X,
** self.mf_obs)
self.prob = self.model.expected_log_prob_vhs( self.stats , H_hat = self.mf_obs['H_hat'], S_hat = self.mf_obs['S_hat'])
self.X = X
self.m = m
self.D = D
self.N = N
finally:
config.floatX = self.prev_floatX
def make_s3c(self):
return S3C(nvis = self.D,
nhid = self.N,
irange = .1,
init_bias_hid = -1.5,
init_B = 3.,
min_B = 1e-8,
max_B = 1000.,
init_alpha = 1., min_alpha = 1e-8, max_alpha = 1000.,
init_mu = 1., e_step = None,
m_step = Grad_M_Step(),
min_bias_hid = -1e30, max_bias_hid = 1e30,
)
def __init__(self):
""" gets a small batch of data
sets up a PD-DBM model
"""
self.tol = 1e-5
X = np.random.RandomState([1,2,3]).randn(1000,5)
X -= X.mean()
X /= X.std()
m, D = X.shape
N = 6
N2 = 7
s3c = S3C(nvis = D,
nhid = N,
irange = .1,
init_bias_hid = -1.5,
init_B = 3.,
min_B = 1e-8,
max_B = 1000.,
init_alpha = 1., min_alpha = 1e-8, max_alpha = 1000.,
init_mu = 1., e_step = None,
m_step = Grad_M_Step(),
min_bias_hid = -1e30, max_bias_hid = 1e30,
)
rbm = RBM(nvis = N, nhid = N2, irange = .1, init_bias_vis = -1.5, init_bias_hid = 1.5)
#don't give the model an inference procedure or learning rate so it won't spend years compiling a learn_func
self.model = PDDBM(
dbm = DBM( use_cd = 1,
rbms = [ rbm ]),
s3c = s3c
)
self.model.make_pseudoparams()
self.inference_procedure = InferenceProcedure(
clip_reflections = True,
rho = .5 )
self.inference_procedure.register_model(self.model)
self.X = X
self.N = N
self.N2 = N2
self.m = m
local_rf_src=dataset,
local_rf_shape=[6, 6],
local_rf_max_shape=dataset.view_shape()[0:2],
local_rf_draw_patches=True,
tied_B=False,
init_bias_hid=init_bias_hid,
init_mu=init_mu,
init_alpha=init_alpha,
init_B=init_B,
irange=None,
min_B=.01,
max_B=1e6,
min_alpha=.01,
max_alpha=1e6,
e_step=None,
m_step=Grad_M_Step())
model.make_pseudoparams()
#this initial E step is just to help out the
#BatchGradientInference object
model.e_step = E_Step_Scan(clip_reflections=True,
rho=0.5,
h_new_coeff_schedule=[.1] * OVERKILL,
s_new_coeff_schedule=[.3] * OVERKILL)
model.e_step.register_model(model)
import theano.tensor as T
from theano import function
def get_needed_steps(ip, X):
def __init__(self, model = None, X = None, tol = 1e-5,
init_H = None, init_S = None, init_G = None):
""" gets a small batch of data
sets up a PD-DBM model
"""
self.tol = tol
if X is None:
X = np.random.RandomState([1,2,3]).randn(1000,5)
X -= X.mean()
X /= X.std()
m, D = X.shape
if model is None:
N = 6
N2 = 7
s3c = S3C(nvis = D,
nhid = N,
irange = .1,
init_bias_hid = -1.5,
init_B = 3.,
min_B = 1e-8,
max_B = 1000.,
init_alpha = 1., min_alpha = 1e-8, max_alpha = 1000.,
init_mu = 1., e_step = None,
m_step = Grad_M_Step(),
min_bias_hid = -1e30, max_bias_hid = 1e30,
)
rbm = RBM(nvis = N, nhid = N2, irange = .5, init_bias_vis = -1.5, init_bias_hid = 1.5)
#don't give the model an inference procedure or learning rate so it won't spend years compiling a learn_func
self.model = PDDBM(
dbm = DBM( use_cd = 1,
rbms = [ rbm ]),
s3c = s3c
)
self.model.make_pseudoparams()
self.inference_procedure = InferenceProcedure(
clip_reflections = True,
rho = .5 )
self.inference_procedure.register_model(self.model)
else:
self.model = model
self.inference_procedure = model.inference_procedure
N = model.s3c.nhid
N2 = model.dbm.rbms[0].nhid
self.X = X
self.N = N
self.N2 = N2
self.m = m
if init_H is None:
self.init_H = np.cast[config.floatX](self.model.rng.uniform(0.,1.,(self.m, self.N)))
self.init_S = np.cast[config.floatX](self.model.rng.uniform(-5.,5.,(self.m, self.N)))
self.init_G = np.cast[config.floatX](self.model.rng.uniform(0.,1.,(self.m,self.N2)))
else:
assert init_S is not None
assert init_G is not None
self.init_H = init_H
self.init_S = init_S
self.init_G = init_G
