def __init__(self, batch_size, learning_rate):
self.batch_size = batch_size
self.learning_rate = learning_rate
self.crbms = [
CRBM('layer1', 300, 3, 10, 32, 3, self.batch_size,
self.learning_rate, True),
CRBM('layer2', 100, 32, 10, 64, 2, self.batch_size,
self.learning_rate, False)
]
self._layer_iterator = self.__ops_generator()
def add_conv_layer(self, filter_shape, stride, padding, name,
use_gaussian=False, params={}):
assert self.last_conv is None, 'cannot add conv layer after fc layers'
if self.num_rbm == 0:
vis_shape = self.vis_shape
else:
vis_shape = self.rbm_list[-1].hid_shape
if use_gaussian:
rbm = GaussianCRBM(vis_shape, filter_shape, stride, padding, name, params)
else:
rbm = CRBM(vis_shape, filter_shape, stride, padding, name, params)
self.num_rbm += 1
self.rbm_list.append(rbm)
def __init__(self, state):
self.state = state
if TEST_CONFIG:
self.mnist = MNIST.first_1k()
print "Test config, so loaded MNIST first 1000"
else:
self.mnist = MNIST.full()#first_10k()
print "Loaded MNIST full"
self.cp = ConvolutionParams( \
num_filters=state.num_filters,
num_input_planes=1,
height_filters=state.filter_size,
width_filters=state.filter_size)
self.image_size = (28,28)
self.minibatch_size = state.minibatch_size
self.lr = state.learning_rate
self.sparsity_lambda = state.sparsity_lambda
# about 1/num_filters, so only one filter active at a time
# 40 * 0.05 = ~2 filters active for any given pixel
self.sparsity_p = state.sparsity_p
self.crbm = CRBM( \
minibatch_size=self.minibatch_size,
image_size=self.image_size,
conv_params=self.cp,
learning_rate=self.lr,
sparsity_lambda=self.sparsity_lambda,
sparsity_p=self.sparsity_p)
self.num_epochs = state.num_epochs
self.init_series()
from crbm import generate
#from rpy2.robjects.packages import importr
csvfile = '/home/alexeyche/my/git/alexeyche-junk/ml/dbn/th/ts_toy.csv'
#csvfile = '/home/alexeyche/prog/alexeyche-junk/ml/dbn/th/ts_toy.csv'
data = genfromtxt(csvfile, delimiter=',')
ncol = data.shape[1]
num_cases = data.shape[0]
num_dims = data.shape[1]
num_vis = num_dims
data_sh = theano.shared(np.asarray(data, dtype=theano.config.floatX), borrow=True)
n_delay = 3
crbm = CRBM(num_vis = num_dims, num_hid = 15, n_delay = n_delay)
batch_size = 100
batches = np.asarray(np.random.permutation(num_cases), dtype='int32')
# history calculations:
index = T.ivector()
hist = []
for h in xrange(0,n_delay):
hist.append(data_sh[index-1-h]) # magic of python: if we get negative index, python would give elements from the end
history = T.stack(hist)[0] # zero index is consenquence of our list trick for shape stack
get_hist = theano.function([index], history)
max_epoch = 1000
num_batches = len(batches)/batch_size
ep_inc = np.float32(1.0/(num_batches*max_epoch))
class MnistCrbm(object):
def __init__(self, state):
self.state = state
if TEST_CONFIG:
self.mnist = MNIST.first_1k()
print "Test config, so loaded MNIST first 1000"
else:
self.mnist = MNIST.full()#first_10k()
print "Loaded MNIST full"
self.cp = ConvolutionParams( \
num_filters=state.num_filters,
num_input_planes=1,
height_filters=state.filter_size,
width_filters=state.filter_size)
self.image_size = (28,28)
self.minibatch_size = state.minibatch_size
self.lr = state.learning_rate
self.sparsity_lambda = state.sparsity_lambda
# about 1/num_filters, so only one filter active at a time
# 40 * 0.05 = ~2 filters active for any given pixel
self.sparsity_p = state.sparsity_p
self.crbm = CRBM( \
minibatch_size=self.minibatch_size,
image_size=self.image_size,
conv_params=self.cp,
learning_rate=self.lr,
sparsity_lambda=self.sparsity_lambda,
sparsity_p=self.sparsity_p)
self.num_epochs = state.num_epochs
self.init_series()
def init_series(self):
series = {}
basedir = os.getcwd()
h5f = tables.openFile(os.path.join(basedir, "series.h5"), "w")
cd_series_names = self.crbm.cd_return_desc
series['cd'] = \
utils.get_accumulator_series_array( \
h5f, 'cd', cd_series_names,
REDUCE_EVERY,
stdout_too=SERIES_STDOUT_TOO)
sparsity_series_names = self.crbm.sparsity_return_desc
series['sparsity'] = \
utils.get_accumulator_series_array( \
h5f, 'sparsity', sparsity_series_names,
REDUCE_EVERY,
stdout_too=SERIES_STDOUT_TOO)
# so first we create the names for each table, based on
# position of each param in the array
params_stdout = []
if SERIES_STDOUT_TOO:
params_stdout = [StdoutAppendTarget()]
series['params'] = SharedParamsStatisticsWrapper(
new_group_name="params",
base_group="/",
arrays_names=['W','b_h','b_x'],
hdf5_file=h5f,
index_names=('epoch','minibatch'),
other_targets=params_stdout)
self.series = series
def train(self):
num_minibatches = len(self.mnist.train.x) / self.minibatch_size
for epoch in xrange(self.num_epochs):
for mb_index in xrange(num_minibatches):
mb_x = self.mnist.train.x \
[mb_index : mb_index+self.minibatch_size]
mb_x = mb_x.reshape((self.minibatch_size, 1, 28, 28))
#E_h = crbm.E_h_given_x_func(mb_x)
#print "Shape of E_h", E_h.shape
cd_return = self.crbm.CD_step(mb_x)
sp_return = self.crbm.sparsity_step(mb_x)
self.series['cd'].append( \
(epoch, mb_index), cd_return)
self.series['sparsity'].append( \
(epoch, mb_index), sp_return)
total_idx = epoch*num_minibatches + mb_index
if (total_idx+1) % REDUCE_EVERY == 0:
self.series['params'].append( \
(epoch, mb_index), self.crbm.params)
if total_idx % VISUALIZE_EVERY == 0:
self.visualize_gibbs_result(\
mb_x, GIBBS_STEPS_IN_VIZ_CHAIN,
"gibbs_chain_"+str(epoch)+"_"+str(mb_index))
self.visualize_gibbs_result(mb_x, 1,
"gibbs_1_"+str(epoch)+"_"+str(mb_index))
self.visualize_filters(
"filters_"+str(epoch)+"_"+str(mb_index))
if TEST_CONFIG:
# do a single epoch for cluster tests config
break
if SAVE_PARAMS:
utils.save_params(self.crbm.params, "params.pkl")
def visualize_gibbs_result(self, start_x, gibbs_steps, filename):
# Run minibatch_size chains for gibbs_steps
x_samples = None
if not start_x is None:
x_samples = self.crbm.gibbs_samples_from(start_x, gibbs_steps)
else:
x_samples = self.crbm.random_gibbs_samples(gibbs_steps)
x_samples = x_samples.reshape((self.minibatch_size, 28*28))
tile = tile_raster_images(x_samples, self.image_size,
(1, self.minibatch_size), output_pixel_vals=True)
filepath = os.path.join(IMAGE_OUTPUT_DIR, filename+".png")
img = Image.fromarray(tile)
img.save(filepath)
print "Result of running Gibbs", \
gibbs_steps, "times outputed to", filepath
def visualize_filters(self, filename):
cp = self.cp
# filter size
fsz = (cp.height_filters, cp.width_filters)
tile_shape = (cp.num_filters, cp.num_input_planes)
filters_flattened = self.crbm.W.value.reshape(
(tile_shape[0]*tile_shape[1],
fsz[0]*fsz[1]))
tile = tile_raster_images(filters_flattened, fsz,
tile_shape, output_pixel_vals=True)
filepath = os.path.join(IMAGE_OUTPUT_DIR, filename+".png")
img = Image.fromarray(tile)
img.save(filepath)
print "Filters (as images) outputed to", filepath
#from rpy2.robjects.packages import importr
csvfile = '/home/alexeyche/my/git/alexeyche-junk/ml/dbn/th/ts_toy.csv'
#csvfile = '/home/alexeyche/prog/alexeyche-junk/ml/dbn/th/ts_toy.csv'
data = genfromtxt(csvfile, delimiter=',')
ncol = data.shape[1]
num_cases = data.shape[0]
num_dims = data.shape[1]
num_vis = num_dims
data_sh = theano.shared(np.asarray(data, dtype=theano.config.floatX),
borrow=True)
n_delay = 3
crbm = CRBM(num_vis=num_dims, num_hid=15, n_delay=n_delay)
batch_size = 100
batches = np.asarray(np.random.permutation(num_cases), dtype='int32')
# history calculations:
index = T.ivector()
hist = []
for h in xrange(0, n_delay):
hist.append(
data_sh[index - 1 - h]
) # magic of python: if we get negative index, python would give elements from the end
history = T.stack(hist)[
0] # zero index is consenquence of our list trick for shape stack
get_hist = theano.function([index], history)
def __init__(self,
numpy_rng=None,
theano_rng=None,
n_input=150,
n_hidden=50,
n_label=3,
n_delay=6,
freq=3):
"""
:type numpy_rng: np.random.RandomState
:param numpy_rng: numpy random number generator used to draw initial weights
:type theano_rng: theano.tensor.shared_randomstreams.RandomStreams
:param theano_rng: Theano random generator; if None is given one is
generated based on a seed drawn from `rng`
:type n_input: int
:param n_input: dimension of the input to the DBN
:type n_hidden: int
:param n_hidden: intermediate layer size
:type n_label: int
:param n_label: dimension of the output of the network (label layers)
:type n_delay: int
:param n_delay: number of past visible layer in the CRBM
"""
self.params = []
self.n_input = n_input
self.n_hidden = n_hidden
self.n_label = n_label
self.delay = n_delay
self.freq = freq
if numpy_rng is None:
# create a number generator
numpy_rng = np.random.RandomState(1234)
if not theano_rng:
theano_rng = MRG_RandomStreams(numpy_rng.randint(2**30))
# allocate symbolic variables for the data
self.x = T.matrix('x') # the data is presented as rasterized images
self.x_history = T.matrix(
'x_history') #memory : past visible is a recopy of visible layer
self.y = T.lvector(
'y'
) # the labels are presented as 1D vector of [int] labels (digit)
# Construct an CRBM that shared weights with this layer
self.crbm_layer = CRBM(numpy_rng=numpy_rng,
theano_rng=theano_rng,
input=self.x,
input_history=self.x_history,
n_visible=n_input,
n_hidden=n_hidden,
delay=n_delay,
freq=freq)
self.params.append(self.crbm_layer.W)
self.params.append(self.crbm_layer.B)
self.params.append(self.crbm_layer.hbias)
# We now need to add a logistic layer on top of the MLP
input_logistic = T.nnet.sigmoid(
T.dot(self.x, self.crbm_layer.W) +
T.dot(self.x_history, self.crbm_layer.B) + self.crbm_layer.hbias)
self.logLayer = LogisticRegression(input=input_logistic,
n_in=n_hidden,
n_out=n_label)
self.params.extend(self.logLayer.params)
# compute the cost for second phase of training, defined as the
# negative log likelihood of the logistic regression (output) layer
self.finetune_cost = self.logLayer.negative_log_likelihood(self.y)
# compute the gradients with respect to the model parameters
# symbolic variable that points to the number of errors made on the
# minibatch given by self.x and self.y
self.errors = self.logLayer.errors(self.y)
