def __init__(self, batch_size, learning_rate): self.batch_size = batch_size self.learning_rate = learning_rate self.crbms = [ CRBM('layer1', 300, 3, 10, 32, 3, self.batch_size, self.learning_rate, True), CRBM('layer2', 100, 32, 10, 64, 2, self.batch_size, self.learning_rate, False) ] self._layer_iterator = self.__ops_generator()
def add_conv_layer(self, filter_shape, stride, padding, name, use_gaussian=False, params={}): assert self.last_conv is None, 'cannot add conv layer after fc layers' if self.num_rbm == 0: vis_shape = self.vis_shape else: vis_shape = self.rbm_list[-1].hid_shape if use_gaussian: rbm = GaussianCRBM(vis_shape, filter_shape, stride, padding, name, params) else: rbm = CRBM(vis_shape, filter_shape, stride, padding, name, params) self.num_rbm += 1 self.rbm_list.append(rbm)
def __init__(self, state): self.state = state if TEST_CONFIG: self.mnist = MNIST.first_1k() print "Test config, so loaded MNIST first 1000" else: self.mnist = MNIST.full()#first_10k() print "Loaded MNIST full" self.cp = ConvolutionParams( \ num_filters=state.num_filters, num_input_planes=1, height_filters=state.filter_size, width_filters=state.filter_size) self.image_size = (28,28) self.minibatch_size = state.minibatch_size self.lr = state.learning_rate self.sparsity_lambda = state.sparsity_lambda # about 1/num_filters, so only one filter active at a time # 40 * 0.05 = ~2 filters active for any given pixel self.sparsity_p = state.sparsity_p self.crbm = CRBM( \ minibatch_size=self.minibatch_size, image_size=self.image_size, conv_params=self.cp, learning_rate=self.lr, sparsity_lambda=self.sparsity_lambda, sparsity_p=self.sparsity_p) self.num_epochs = state.num_epochs self.init_series()
from crbm import generate #from rpy2.robjects.packages import importr csvfile = '/home/alexeyche/my/git/alexeyche-junk/ml/dbn/th/ts_toy.csv' #csvfile = '/home/alexeyche/prog/alexeyche-junk/ml/dbn/th/ts_toy.csv' data = genfromtxt(csvfile, delimiter=',') ncol = data.shape[1] num_cases = data.shape[0] num_dims = data.shape[1] num_vis = num_dims data_sh = theano.shared(np.asarray(data, dtype=theano.config.floatX), borrow=True) n_delay = 3 crbm = CRBM(num_vis = num_dims, num_hid = 15, n_delay = n_delay) batch_size = 100 batches = np.asarray(np.random.permutation(num_cases), dtype='int32') # history calculations: index = T.ivector() hist = [] for h in xrange(0,n_delay): hist.append(data_sh[index-1-h]) # magic of python: if we get negative index, python would give elements from the end history = T.stack(hist)[0] # zero index is consenquence of our list trick for shape stack get_hist = theano.function([index], history) max_epoch = 1000 num_batches = len(batches)/batch_size ep_inc = np.float32(1.0/(num_batches*max_epoch))
class MnistCrbm(object): def __init__(self, state): self.state = state if TEST_CONFIG: self.mnist = MNIST.first_1k() print "Test config, so loaded MNIST first 1000" else: self.mnist = MNIST.full()#first_10k() print "Loaded MNIST full" self.cp = ConvolutionParams( \ num_filters=state.num_filters, num_input_planes=1, height_filters=state.filter_size, width_filters=state.filter_size) self.image_size = (28,28) self.minibatch_size = state.minibatch_size self.lr = state.learning_rate self.sparsity_lambda = state.sparsity_lambda # about 1/num_filters, so only one filter active at a time # 40 * 0.05 = ~2 filters active for any given pixel self.sparsity_p = state.sparsity_p self.crbm = CRBM( \ minibatch_size=self.minibatch_size, image_size=self.image_size, conv_params=self.cp, learning_rate=self.lr, sparsity_lambda=self.sparsity_lambda, sparsity_p=self.sparsity_p) self.num_epochs = state.num_epochs self.init_series() def init_series(self): series = {} basedir = os.getcwd() h5f = tables.openFile(os.path.join(basedir, "series.h5"), "w") cd_series_names = self.crbm.cd_return_desc series['cd'] = \ utils.get_accumulator_series_array( \ h5f, 'cd', cd_series_names, REDUCE_EVERY, stdout_too=SERIES_STDOUT_TOO) sparsity_series_names = self.crbm.sparsity_return_desc series['sparsity'] = \ utils.get_accumulator_series_array( \ h5f, 'sparsity', sparsity_series_names, REDUCE_EVERY, stdout_too=SERIES_STDOUT_TOO) # so first we create the names for each table, based on # position of each param in the array params_stdout = [] if SERIES_STDOUT_TOO: params_stdout = [StdoutAppendTarget()] series['params'] = SharedParamsStatisticsWrapper( new_group_name="params", base_group="/", arrays_names=['W','b_h','b_x'], hdf5_file=h5f, index_names=('epoch','minibatch'), other_targets=params_stdout) self.series = series def train(self): num_minibatches = len(self.mnist.train.x) / self.minibatch_size for epoch in xrange(self.num_epochs): for mb_index in xrange(num_minibatches): mb_x = self.mnist.train.x \ [mb_index : mb_index+self.minibatch_size] mb_x = mb_x.reshape((self.minibatch_size, 1, 28, 28)) #E_h = crbm.E_h_given_x_func(mb_x) #print "Shape of E_h", E_h.shape cd_return = self.crbm.CD_step(mb_x) sp_return = self.crbm.sparsity_step(mb_x) self.series['cd'].append( \ (epoch, mb_index), cd_return) self.series['sparsity'].append( \ (epoch, mb_index), sp_return) total_idx = epoch*num_minibatches + mb_index if (total_idx+1) % REDUCE_EVERY == 0: self.series['params'].append( \ (epoch, mb_index), self.crbm.params) if total_idx % VISUALIZE_EVERY == 0: self.visualize_gibbs_result(\ mb_x, GIBBS_STEPS_IN_VIZ_CHAIN, "gibbs_chain_"+str(epoch)+"_"+str(mb_index)) self.visualize_gibbs_result(mb_x, 1, "gibbs_1_"+str(epoch)+"_"+str(mb_index)) self.visualize_filters( "filters_"+str(epoch)+"_"+str(mb_index)) if TEST_CONFIG: # do a single epoch for cluster tests config break if SAVE_PARAMS: utils.save_params(self.crbm.params, "params.pkl") def visualize_gibbs_result(self, start_x, gibbs_steps, filename): # Run minibatch_size chains for gibbs_steps x_samples = None if not start_x is None: x_samples = self.crbm.gibbs_samples_from(start_x, gibbs_steps) else: x_samples = self.crbm.random_gibbs_samples(gibbs_steps) x_samples = x_samples.reshape((self.minibatch_size, 28*28)) tile = tile_raster_images(x_samples, self.image_size, (1, self.minibatch_size), output_pixel_vals=True) filepath = os.path.join(IMAGE_OUTPUT_DIR, filename+".png") img = Image.fromarray(tile) img.save(filepath) print "Result of running Gibbs", \ gibbs_steps, "times outputed to", filepath def visualize_filters(self, filename): cp = self.cp # filter size fsz = (cp.height_filters, cp.width_filters) tile_shape = (cp.num_filters, cp.num_input_planes) filters_flattened = self.crbm.W.value.reshape( (tile_shape[0]*tile_shape[1], fsz[0]*fsz[1])) tile = tile_raster_images(filters_flattened, fsz, tile_shape, output_pixel_vals=True) filepath = os.path.join(IMAGE_OUTPUT_DIR, filename+".png") img = Image.fromarray(tile) img.save(filepath) print "Filters (as images) outputed to", filepath
#from rpy2.robjects.packages import importr csvfile = '/home/alexeyche/my/git/alexeyche-junk/ml/dbn/th/ts_toy.csv' #csvfile = '/home/alexeyche/prog/alexeyche-junk/ml/dbn/th/ts_toy.csv' data = genfromtxt(csvfile, delimiter=',') ncol = data.shape[1] num_cases = data.shape[0] num_dims = data.shape[1] num_vis = num_dims data_sh = theano.shared(np.asarray(data, dtype=theano.config.floatX), borrow=True) n_delay = 3 crbm = CRBM(num_vis=num_dims, num_hid=15, n_delay=n_delay) batch_size = 100 batches = np.asarray(np.random.permutation(num_cases), dtype='int32') # history calculations: index = T.ivector() hist = [] for h in xrange(0, n_delay): hist.append( data_sh[index - 1 - h] ) # magic of python: if we get negative index, python would give elements from the end history = T.stack(hist)[ 0] # zero index is consenquence of our list trick for shape stack get_hist = theano.function([index], history)
def __init__(self, numpy_rng=None, theano_rng=None, n_input=150, n_hidden=50, n_label=3, n_delay=6, freq=3): """ :type numpy_rng: np.random.RandomState :param numpy_rng: numpy random number generator used to draw initial weights :type theano_rng: theano.tensor.shared_randomstreams.RandomStreams :param theano_rng: Theano random generator; if None is given one is generated based on a seed drawn from `rng` :type n_input: int :param n_input: dimension of the input to the DBN :type n_hidden: int :param n_hidden: intermediate layer size :type n_label: int :param n_label: dimension of the output of the network (label layers) :type n_delay: int :param n_delay: number of past visible layer in the CRBM """ self.params = [] self.n_input = n_input self.n_hidden = n_hidden self.n_label = n_label self.delay = n_delay self.freq = freq if numpy_rng is None: # create a number generator numpy_rng = np.random.RandomState(1234) if not theano_rng: theano_rng = MRG_RandomStreams(numpy_rng.randint(2**30)) # allocate symbolic variables for the data self.x = T.matrix('x') # the data is presented as rasterized images self.x_history = T.matrix( 'x_history') #memory : past visible is a recopy of visible layer self.y = T.lvector( 'y' ) # the labels are presented as 1D vector of [int] labels (digit) # Construct an CRBM that shared weights with this layer self.crbm_layer = CRBM(numpy_rng=numpy_rng, theano_rng=theano_rng, input=self.x, input_history=self.x_history, n_visible=n_input, n_hidden=n_hidden, delay=n_delay, freq=freq) self.params.append(self.crbm_layer.W) self.params.append(self.crbm_layer.B) self.params.append(self.crbm_layer.hbias) # We now need to add a logistic layer on top of the MLP input_logistic = T.nnet.sigmoid( T.dot(self.x, self.crbm_layer.W) + T.dot(self.x_history, self.crbm_layer.B) + self.crbm_layer.hbias) self.logLayer = LogisticRegression(input=input_logistic, n_in=n_hidden, n_out=n_label) self.params.extend(self.logLayer.params) # compute the cost for second phase of training, defined as the # negative log likelihood of the logistic regression (output) layer self.finetune_cost = self.logLayer.negative_log_likelihood(self.y) # compute the gradients with respect to the model parameters # symbolic variable that points to the number of errors made on the # minibatch given by self.x and self.y self.errors = self.logLayer.errors(self.y)