def apply_cnn(self, l_emb1, l_size1, l_emb2, l_size2, r_emb1, r_size1, r_emb2, r_size2, embedding_size, mycnf): assert l_size1 == r_size1 assert l_size2 == r_size2 assert l_size1 == l_size1 max_len = l_size1 fv_len = 0 filter_sizes = mycnf['cnn_config']['filter_sizes'] num_filters = mycnf['cnn_config']['num_filters'] for i, fw in enumerate(filter_sizes): conv_left = ConvolutionalActivation( activation=Rectifier().apply, filter_size=(fw, embedding_size), num_filters=num_filters, num_channels=1, image_size=(max_len, embedding_size), name="conv" + str(fw) + l_emb1.name, seed=self.curSeed) conv_right = ConvolutionalActivation( activation=Rectifier().apply, filter_size=(fw, embedding_size), num_filters=num_filters, num_channels=1, image_size=(max_len, embedding_size), name="conv" + str(fw) + r_emb1.name, seed=self.curSeed) pooling = MaxPooling((max_len - fw + 1, 1), name="pool" + str(fw)) initialize([conv_left, conv_right]) l_convinp1 = l_emb1.flatten().reshape( (l_emb1.shape[0], 1, max_len, embedding_size)) l_convinp2 = l_emb2.flatten().reshape( (l_emb2.shape[0], 1, max_len, embedding_size)) l_pool1 = pooling.apply(conv_left.apply(l_convinp1)).flatten(2) l_pool2 = pooling.apply(conv_left.apply(l_convinp2)).flatten(2) r_convinp1 = r_emb1.flatten().reshape( (r_emb1.shape[0], 1, max_len, embedding_size)) r_convinp2 = r_emb2.flatten().reshape( (r_emb2.shape[0], 1, max_len, embedding_size)) r_pool1 = pooling.apply(conv_right.apply(r_convinp1)).flatten(2) r_pool2 = pooling.apply(conv_right.apply(r_convinp2)).flatten(2) onepools1 = T.concatenate([l_pool1, r_pool1], axis=1) onepools2 = T.concatenate([l_pool2, r_pool2], axis=1) fv_len += conv_left.num_filters * 2 if i == 0: outpools1 = onepools1 outpools2 = onepools2 else: outpools1 = T.concatenate([outpools1, onepools1], axis=1) outpools2 = T.concatenate([outpools2, onepools2], axis=1) return outpools1, outpools2, fv_len
def create_cnn_general(embedded_x, mycnf, max_len, embedding_size, inp_conv=False): fv_len = 0 filter_sizes = mycnf['cnn_config']['filter_sizes'] num_filters = mycnf['cnn_config']['num_filters'] for i, fw in enumerate(filter_sizes): conv = ConvolutionalActivation( activation=Rectifier().apply, filter_size=(fw, embedding_size), num_filters=num_filters, num_channels=1, image_size=(max_len, embedding_size), name="conv"+str(fw)+embedded_x.name) pooling = MaxPooling((max_len-fw+1, 1), name="pool"+str(fw)+embedded_x.name) initialize([conv]) if inp_conv: convinp = embedded_x else: convinp = embedded_x.flatten().reshape((embedded_x.shape[0], 1, max_len, embedding_size)) onepool = pooling.apply(conv.apply(convinp)).flatten(2) if i == 0: outpools = onepool else: outpools = T.concatenate([outpools, onepool], axis=1) fv_len += conv.num_filters return outpools, fv_len
def create_OLD_kim_cnn(layer0_input, embedding_size, input_len, config, pref): ''' One layer convolution with the same filtersize ''' filter_width_list = [ int(fw) for fw in config[pref + '_filterwidth'].split() ] print filter_width_list num_filters = int(config[pref + '_num_filters']) totfilters = 0 for i, fw in enumerate(filter_width_list): num_feature_map = input_len - fw + 1 #39 conv = Convolutional(filter_size=(fw, embedding_size), num_filters=num_filters, num_channels=1, image_size=(input_len, embedding_size), name="conv" + str(fw)) pooling = MaxPooling((num_feature_map, 1), name="pool" + str(fw)) initialize([conv]) totfilters += num_filters outpool = Flattener(name="flat" + str(fw)).apply( Rectifier(name=pref + 'act_' + str(fw)).apply( pooling.apply(conv.apply(layer0_input)))) if i == 0: outpools = outpool else: outpools = T.concatenate([outpools, outpool], axis=1) name_rep_len = totfilters return outpools, name_rep_len
def create_kim_cnn(layer0_input, embedding_size, input_len, config, pref): ''' One layer convolution with different filter-sizes and maxpooling ''' filter_width_list = [int(fw) for fw in config[pref + '_filterwidth'].split()] print filter_width_list num_filters = int(config[pref+'_num_filters']) #num_filters /= len(filter_width_list) totfilters = 0 for i, fw in enumerate(filter_width_list): num_feature_map = input_len - fw + 1 #39 conv = Convolutional( image_size=(input_len, embedding_size), filter_size=(fw, embedding_size), num_filters=min(int(config[pref + '_maxfilter']), num_filters * fw), num_channels=1 ) totfilters += conv.num_filters initialize2(conv, num_feature_map) conv.name = pref + 'conv_' + str(fw) convout = conv.apply(layer0_input) pool_layer = MaxPooling( pooling_size=(num_feature_map,1) ) pool_layer.name = pref + 'pool_' + str(fw) act = Rectifier() act.name = pref + 'act_' + str(fw) outpool = act.apply(pool_layer.apply(convout)).flatten(2) if i == 0: outpools = outpool else: outpools = T.concatenate([outpools, outpool], axis=1) name_rep_len = totfilters return outpools, name_rep_len
def create_kim_cnn(layer0_input, embedding_size, input_len, config, pref): ''' One layer convolution with different filter-sizes and maxpooling ''' filter_width_list = [ int(fw) for fw in config[pref + '_filterwidth'].split() ] print filter_width_list num_filters = int(config[pref + '_num_filters']) #num_filters /= len(filter_width_list) totfilters = 0 for i, fw in enumerate(filter_width_list): num_feature_map = input_len - fw + 1 #39 conv = Convolutional(image_size=(input_len, embedding_size), filter_size=(fw, embedding_size), num_filters=min(int(config[pref + '_maxfilter']), num_filters * fw), num_channels=1) totfilters += conv.num_filters initialize2(conv, num_feature_map) conv.name = pref + 'conv_' + str(fw) convout = conv.apply(layer0_input) pool_layer = MaxPooling(pooling_size=(num_feature_map, 1)) pool_layer.name = pref + 'pool_' + str(fw) act = Rectifier() act.name = pref + 'act_' + str(fw) outpool = act.apply(pool_layer.apply(convout)).flatten(2) if i == 0: outpools = outpool else: outpools = T.concatenate([outpools, outpool], axis=1) name_rep_len = totfilters return outpools, name_rep_len
def test_max_pooling(): x = tensor.tensor4("x") num_channels = 4 batch_size = 5 x_size = 17 y_size = 13 pool_size = 3 pool = MaxPooling((pool_size, pool_size)) y = pool.apply(x) func = function([x], y) x_val = numpy.ones((batch_size, num_channels, x_size, y_size), dtype=theano.config.floatX) assert_allclose(func(x_val), numpy.ones((batch_size, num_channels, x_size / pool_size, y_size / pool_size))) pool.input_dim = (x_size, y_size) pool.get_dim("output") == (num_channels, x_size / pool_size + 1, y_size / pool_size + 1)
def test_max_pooling(): x = tensor.tensor4('x') num_channels = 4 batch_size = 5 x_size = 17 y_size = 13 pool_size = 3 pool = MaxPooling((pool_size, pool_size)) y = pool.apply(x) func = function([x], y) x_val = numpy.ones((batch_size, num_channels, x_size, y_size), dtype=theano.config.floatX) assert_allclose( func(x_val), numpy.ones((batch_size, num_channels, x_size / pool_size + 1, y_size / pool_size + 1))) pool.input_dim = (x_size, y_size) pool.get_dim('output') == (num_channels, x_size / pool_size + 1, y_size / pool_size + 1)
def create_yy_cnn(numConvLayer, conv_input, embedding_size, input_len, config, pref): ''' CNN with several layers of convolution, each with specific filter size. Maxpooling at the end. ''' filter_width_list = [ int(fw) for fw in config[pref + '_filterwidth'].split() ] base_num_filters = int(config[pref + '_num_filters']) assert len(filter_width_list) == numConvLayer convs = [] fmlist = [] last_fm = input_len for i in range(numConvLayer): fw = filter_width_list[i] num_feature_map = last_fm - fw + 1 #39 conv = Convolutional(image_size=(last_fm, embedding_size), filter_size=(fw, embedding_size), num_filters=min(int(config[pref + '_maxfilter']), base_num_filters * fw), num_channels=1) fmlist.append(num_feature_map) last_fm = num_feature_map embedding_size = conv.num_filters convs.append(conv) initialize(convs) for i, conv in enumerate(convs): conv.name = pref + '_conv' + str(i) conv_input = conv.apply(conv_input) conv_input = conv_input.flatten().reshape( (conv_input.shape[0], 1, fmlist[i], conv.num_filters)) lastconv = conv lastconv_out = conv_input pool_layer = MaxPooling(pooling_size=(last_fm, 1)) pool_layer.name = pref + '_pool_' + str(fw) act = Rectifier() act.name = 'act_' + str(fw) outpool = act.apply(pool_layer.apply(lastconv_out).flatten(2)) return outpool, lastconv.num_filters
def create_yy_cnn(numConvLayer, conv_input, embedding_size, input_len, config, pref): ''' CNN with several layers of convolution, each with specific filter size. Maxpooling at the end. ''' filter_width_list = [int(fw) for fw in config[pref + '_filterwidth'].split()] base_num_filters = int(config[pref + '_num_filters']) assert len(filter_width_list) == numConvLayer convs = []; fmlist = [] last_fm = input_len for i in range(numConvLayer): fw = filter_width_list[i] num_feature_map = last_fm - fw + 1 #39 conv = Convolutional( image_size=(last_fm, embedding_size), filter_size=(fw, embedding_size), num_filters=min(int(config[pref + '_maxfilter']), base_num_filters * fw), num_channels=1 ) fmlist.append(num_feature_map) last_fm = num_feature_map embedding_size = conv.num_filters convs.append(conv) initialize(convs) for i, conv in enumerate(convs): conv.name = pref+'_conv' + str(i) conv_input = conv.apply(conv_input) conv_input = conv_input.flatten().reshape((conv_input.shape[0], 1, fmlist[i], conv.num_filters)) lastconv = conv lastconv_out = conv_input pool_layer = MaxPooling( pooling_size=(last_fm,1) ) pool_layer.name = pref+'_pool_' + str(fw) act = Rectifier(); act.name = 'act_' + str(fw) outpool = act.apply(pool_layer.apply(lastconv_out).flatten(2)) return outpool, lastconv.num_filters
def test_max_pooling_ignore_border_true(): x = tensor.tensor4("x") brick = MaxPooling((3, 4), ignore_border=True) y = brick.apply(x) out = y.eval({x: numpy.zeros((8, 3, 10, 13), dtype=theano.config.floatX)}) assert out.shape == (8, 3, 3, 3)
def train(self, X, Y, idx_folds, hyper_params, model_prefix, verbose=False): import os from collections import OrderedDict from fuel.datasets import IndexableDataset from blocks.model import Model from blocks.bricks import Linear, Softmax from blocks.bricks.conv import MaxPooling from blocks.initialization import Uniform from deepthought.bricks.cost import HingeLoss import numpy as np import theano from theano import tensor assert model_prefix is not None fold_weights_filename = '{}_weights.npy'.format(model_prefix) # convert Y to one-hot encoding n_classes = len(set(Y)) Y = np.eye(n_classes, dtype=int)[Y] features = tensor.matrix('features', dtype=theano.config.floatX) targets = tensor.lmatrix('targets') input_ = features dim = X.shape[-1] # optional additional layers if self.pipeline_factory is not None: # need to re-shape flattened input to restore bc01 format input_shape = (input_.shape[0],) + hyper_params['classifier_input_shape'] # tuple, uses actual batch size input_ = input_.reshape(input_shape) pipeline = self.pipeline_factory.build_pipeline(input_shape, hyper_params) input_ = pipeline.apply(input_) input_ = input_.flatten(ndim=2) # this is very hacky, but there seems to be no elegant way to obtain a value for dim dummy_fn = theano.function(inputs=[features], outputs=input_) dummy_out = dummy_fn(X[:1]) dim = dummy_out.shape[-1] if hyper_params['classifier_pool_width'] > 1: # FIXME: this is probably broken! # c = hyper_params['num_components'] # input_ = input_.reshape((input_.shape[0], c, input_.shape[-1] // c, 1)) # restore bc01 # need to re-shape flattened input to restore bc01 format input_shape = hyper_params['classifier_pool_input_shape'] # tuple input_ = input_.reshape(input_shape) pool = MaxPooling(name='pool', input_dim=input_shape[1:], # (c, X.shape[-1] // c, 1), pooling_size=(hyper_params['classifier_pool_width'], 1), step=(hyper_params['classifier_pool_stride'], 1)) input_ = pool.apply(input_) input_ = input_.reshape((input_.shape[0], tensor.prod(input_.shape[1:]))) dim = np.prod(pool.get_dim('output')) linear = Linear(name='linear', input_dim=dim, output_dim=n_classes, weights_init=Uniform(mean=0, std=0.01), use_bias=False) linear.initialize() softmax = Softmax('softmax') probs = softmax.apply(linear.apply(input_)) prediction = tensor.argmax(probs, axis=1) model = Model(probs) # classifier with raw probability outputs predict = theano.function([features], prediction) # ready-to-use predict function if os.path.isfile(fold_weights_filename): # load filter weights from existing file fold_weights = np.load(fold_weights_filename) print 'loaded filter weights from', fold_weights_filename else: # train model from blocks.bricks.cost import MisclassificationRate from blocks.filter import VariableFilter from blocks.graph import ComputationGraph from blocks.roles import WEIGHT from blocks.bricks import Softmax from blocks.model import Model from blocks.algorithms import GradientDescent, Adam from blocks.extensions import FinishAfter, Timing, Printing, ProgressBar from blocks.extensions.monitoring import DataStreamMonitoring, TrainingDataMonitoring from blocks.extensions.predicates import OnLogRecord from fuel.streams import DataStream from fuel.schemes import SequentialScheme, ShuffledScheme from blocks.monitoring import aggregation from blocks.main_loop import MainLoop from blocks.extensions.training import TrackTheBest from deepthought.extensions.parameters import BestParams # from deepthought.datasets.selection import DatasetMetaDB init_param_values = model.get_parameter_values() cost = HingeLoss().apply(targets, probs) # Note: this requires just the class labels, not in a one-hot encoding error_rate = MisclassificationRate().apply(targets.argmax(axis=1), probs) error_rate.name = 'error_rate' cg = ComputationGraph([cost]) # L1 regularization if hyper_params['classifier_l1wdecay'] > 0: weights = VariableFilter(roles=[WEIGHT])(cg.variables) cost = cost + hyper_params['classifier_l1wdecay'] * sum([abs(W).sum() for W in weights]) cost.name = 'cost' # iterate over trial folds fold_weights = [] fold_errors = [] # for ifi, ifold in fold_generator.get_inner_cv_folds(outer_fold): # # train_selectors = fold_generator.get_fold_selectors(outer_fold=outer_fold, inner_fold=ifold['train']) # valid_selectors = fold_generator.get_fold_selectors(outer_fold=outer_fold, inner_fold=ifold['valid']) # # metadb = DatasetMetaDB(meta, train_selectors.keys()) # # # get selected trial IDs # train_idx = metadb.select(train_selectors) # valid_idx = metadb.select(valid_selectors) for train_idx, valid_idx in idx_folds: # print train_idx # print valid_idx trainset = IndexableDataset(indexables=OrderedDict( [('features', X[train_idx]), ('targets', Y[train_idx])])) validset = IndexableDataset(indexables=OrderedDict( [('features', X[valid_idx]), ('targets', Y[valid_idx])])) model.set_parameter_values(init_param_values) best_params = BestParams() best_params.add_condition(['after_epoch'], predicate=OnLogRecord('error_rate_valid_best_so_far')) algorithm = GradientDescent(cost=cost, parameters=cg.parameters, step_rule=Adam()) extensions = [Timing(), FinishAfter(after_n_epochs=hyper_params['classifier_max_epochs']), DataStreamMonitoring( [cost, error_rate], DataStream.default_stream( validset, iteration_scheme=SequentialScheme( validset.num_examples, hyper_params['classifier_batch_size'])), suffix="valid"), TrainingDataMonitoring( [cost, error_rate, aggregation.mean(algorithm.total_gradient_norm)], suffix="train", after_epoch=True), TrackTheBest('error_rate_valid'), best_params # after TrackTheBest! ] if verbose: extensions.append(Printing()) # optional extensions.append(ProgressBar()) main_loop = MainLoop( algorithm, DataStream.default_stream( trainset, iteration_scheme=ShuffledScheme(trainset.num_examples, hyper_params['classifier_batch_size'])), model=model, extensions=extensions) main_loop.run() fold_weights.append(best_params.values['/linear.W']) fold_errors.append(main_loop.status['best_error_rate_valid']) # break # FIXME fold_errors = np.asarray(fold_errors).squeeze() print 'simple NN fold classification errors:', fold_errors fold_weights = np.asarray(fold_weights) # store filter weights for later analysis np.save(fold_weights_filename, fold_weights) weights = fold_weights.mean(axis=0) linear.parameters[0].set_value(weights) return model, predict
def test_max_pooling_padding(): x = tensor.tensor4('x') brick = MaxPooling((6, 2), padding=(3, 1), ignore_border=True) y = brick.apply(x) out = y.eval({x: numpy.zeros((2, 3, 6, 10), dtype=theano.config.floatX)}) assert out.shape == (2, 3, 2, 6)
def test_max_pooling_ignore_border_false(): x = tensor.tensor4('x') brick = MaxPooling((5, 7), ignore_border=False) y = brick.apply(x) out = y.eval({x: numpy.zeros((4, 6, 12, 15), dtype=theano.config.floatX)}) assert out.shape == (4, 6, 3, 3)
def test_max_pooling_ignore_border_true(): x = tensor.tensor4('x') brick = MaxPooling((3, 4), ignore_border=True) y = brick.apply(x) out = y.eval({x: numpy.zeros((8, 3, 10, 13), dtype=theano.config.floatX)}) assert out.shape == (8, 3, 3, 3)
for i, cp in enumerate(conv_eeg): bconv = Convolutional(filter_size=(cp['filter_size'], 1), num_filters=cp['num_filters'], num_channels=eeg_channels, border_mode='full', tied_biases=True, name="conv_eeg_%d" % i) bmaxpool = MaxPooling(pooling_size=(cp['pool_size'], 1), name='maxpool_eeg_%d' % i) # convolve eeg1 = bconv.apply(eeg) # cut borders d1 = (eeg1.shape[2] - eeg.shape[2]) / 2 eeg = eeg1[:, :, d1:d1 + eeg.shape[2], :] # subsample eeg = bmaxpool.apply(eeg) # normalize if cp['normalize']: eeg = normalize(eeg, axis=(0, 2)) # activation act = cp['activation'](name='act_eeg%d' % i) eeg = act.apply(eeg) # stuff bricks += [bconv, bmaxpool] eeg_channels = cp['num_filters'] if cp['dropout'] > 0: dropout_locs += [(VariableFilter(bricks=[act], name='output'), cp['dropout'])] # Now we can concatenate eeg and acc (dimensions should be right) data = tensor.concatenate([eeg, acc], axis=1)
def create_OLD_kim_cnn(layer0_input, embedding_size, input_len, config, pref): ''' One layer convolution with the same filtersize ''' filter_width_list = [int(fw) for fw in config[pref + '_filterwidth'].split()] print filter_width_list num_filters = int(config[pref+'_num_filters']) totfilters = 0 for i, fw in enumerate(filter_width_list): num_feature_map = input_len - fw + 1 #39 conv = Convolutional( filter_size=(fw, embedding_size), num_filters=num_filters, num_channels=1, image_size=(input_len, embedding_size), name="conv" + str(fw)) pooling = MaxPooling((num_feature_map,1), name="pool"+str(fw)) initialize([conv]) totfilters += num_filters outpool = Flattener(name="flat"+str(fw)).apply(Rectifier(name=pref+'act_'+str(fw)).apply(pooling.apply(conv.apply(layer0_input)))) if i == 0: outpools = outpool else: outpools = T.concatenate([outpools, outpool], axis=1) name_rep_len = totfilters return outpools, name_rep_len
def test_max_pooling_ignore_border_false(): x = tensor.tensor4("x") brick = MaxPooling((5, 7), ignore_border=False) y = brick.apply(x) out = y.eval({x: numpy.zeros((4, 6, 12, 15), dtype=theano.config.floatX)}) assert out.shape == (4, 6, 3, 3)
# first convolution only on eeg conv_eeg = Convolutional(filter_size=(300, 1), num_filters=20, num_channels=1, border_mode='full', tied_biases=True, name="conv_eeg") maxpool_eeg = MaxPooling(pooling_size=(5, 1), name='maxpool_eeg') # convolve eeg1 = conv_eeg.apply(eeg) # cut borders d1 = (eeg1.shape[2] - eeg.shape[2])/2 eeg1 = eeg1[:, :, d1:d1+eeg.shape[2], :] # subsample eeg1 = maxpool_eeg.apply(eeg1) # activation eeg1 = Tanh(name='act_eeg').apply(eeg1) # second convolution only on eeg conv_eeg2 = Convolutional(filter_size=(100, 1), num_filters=40, num_channels=20, border_mode='full', tied_biases=True, name="conv_eeg2") maxpool_eeg2 = MaxPooling(pooling_size=(5, 1), name='maxpool_eeg2') # convolve eeg2 = conv_eeg2.apply(eeg1) # cut borders d1 = (eeg2.shape[2] - eeg1.shape[2])/2
def test_max_pooling_padding(): x = tensor.tensor4("x") brick = MaxPooling((6, 2), padding=(3, 1), ignore_border=True) y = brick.apply(x) out = y.eval({x: numpy.zeros((2, 3, 6, 10), dtype=theano.config.floatX)}) assert out.shape == (2, 3, 2, 6)
# Convolution bricks conv = Convolutional( filter_size=(p["filter_size"], 1), # step=(p['stride'],1), num_filters=p["nfilter"], num_channels=conv_in_channels, batch_size=batch_size, border_mode="valid", tied_biases=True, name="conv%d" % i, ) cb.append(conv) maxpool = MaxPooling(pooling_size=(p["pool_stride"], 1), name="mp%d" % i) conv_out = conv.apply(conv_in)[:, :, :: p["stride"], :] conv_out = maxpool.apply(conv_out) if p["normalize"]: conv_out_mean = conv_out.mean(axis=2).mean(axis=0) conv_out_var = ((conv_out - conv_out_mean[None, :, None, :]) ** 2).mean(axis=2).mean(axis=0).sqrt() conv_out = (conv_out - conv_out_mean[None, :, None, :]) / conv_out_var[None, :, None, :] if p["activation"] is not None: conv_out = p["activation"].apply(conv_out) if p["dropout"] > 0: b = [p["activation"] if p["activation"] is not None else conv] dropout_locs.append((VariableFilter(bricks=b, name="output"), p["dropout"])) if p["skip"] is not None and len(p["skip"]) > 0: maxpooladd = MaxPooling(pooling_size=(p["stride"] * p["pool_stride"], 1), name="Mp%d" % i) skip = [] if "max" in p["skip"]: skip.append(maxpooladd.apply(conv_in)[:, :, : conv_out.shape[2], :]) if "min" in p["skip"]: