Beispiel #1
0
    def __init__(self, param):

        self.batch_size = param['batch_size']
        self.iterations = param['iterations']
        self.in_size = param['in_size']
        self.out_size = param['out_size']
        self.hid_sizes = param['hid_sizes']
        self.learning_rate = param['learning_rate']
        self.pre_epochs = param['pre_epochs']
        self.finetune_epochs = param['fine_epochs']
        self.lam = param['lam']
        self.act = param['act']

        self.sdae = SDAE(self.in_size, self.out_size, self.hid_sizes,
                         self.batch_size, self.learning_rate, self.lam,
                         self.act, self.iterations)
        self.sdae.process()

        self.theano_tr_ids, self.tr_pred, self.tr_act = [], [], []
    def __init__(self,param):

        self.batch_size = param['batch_size']
        self.iterations = param['iterations']
        self.in_size = param['in_size']
        self.out_size = param['out_size']
        self.hid_sizes = param['hid_sizes']
        self.learning_rate = param['learning_rate']
        self.pre_epochs = param['pre_epochs']
        self.finetune_epochs = param['fine_epochs']
        self.lam = param['lam']
        self.act = param['act']

        self.sdae = SDAE(self.in_size,self.out_size,self.hid_sizes,self.batch_size,self.learning_rate,self.lam,self.act,self.iterations)
        self.sdae.process()

        self.theano_tr_ids, self.tr_pred, self.tr_act = [],[],[]
Beispiel #3
0
class UseSDAE(AdaClassifier):
    def __init__(self, param):

        self.batch_size = param['batch_size']
        self.iterations = param['iterations']
        self.in_size = param['in_size']
        self.out_size = param['out_size']
        self.hid_sizes = param['hid_sizes']
        self.learning_rate = param['learning_rate']
        self.pre_epochs = param['pre_epochs']
        self.finetune_epochs = param['fine_epochs']
        self.lam = param['lam']
        self.act = param['act']

        self.sdae = SDAE(self.in_size, self.out_size, self.hid_sizes,
                         self.batch_size, self.learning_rate, self.lam,
                         self.act, self.iterations)
        self.sdae.process()

        self.theano_tr_ids, self.tr_pred, self.tr_act = [], [], []

    def train(self, tr_all, v_all, weights):
        from math import ceil
        tr_ids, tr_x, tr_y = tr_all
        v_ids, v_x, v_y = v_all
        weights_shr = shared(value=np.asarray(weights, dtype=config.floatX),
                             borrow=True)

        def get_shared_data(data_xy):
            data_x, data_y = data_xy
            shared_x = shared(value=np.asarray(data_x, dtype=config.floatX),
                              borrow=True)
            shared_y = shared(value=np.asarray(data_y, dtype=config.floatX),
                              borrow=True)

            return shared_x, T.cast(shared_y, 'int32')

        train = get_shared_data((tr_x, tr_y))
        valid = get_shared_data((v_x, v_y))

        n_train_batches = ceil(train[0].get_value(borrow=True).shape[0] /
                               self.batch_size)
        n_valid_batches = ceil(valid[0].get_value(borrow=True).shape[0] /
                               self.batch_size)

        pretrain_func = self.sdae.pre_train(train[0], train[1])
        finetune_func = self.sdae.fine_tune(train[0], train[1], weights_shr)

        my_valid_id_tensor = shared(value=np.asarray(v_ids,
                                                     dtype=config.floatX),
                                    borrow=True)
        my_valid_id_int_tensor = T.cast(my_valid_id_tensor, 'int32')
        validate_func = self.sdae.validate(valid[0], valid[1],
                                           my_valid_id_int_tensor)

        my_train_id_tensor = shared(value=np.asarray(tr_ids,
                                                     dtype=config.floatX),
                                    borrow=True)
        my_train_id_int_tensor = T.cast(my_train_id_tensor, 'int32')
        tr_validate_func = self.sdae.validate(train[0], train[1],
                                              my_train_id_int_tensor)

        for epoch in range(self.pre_epochs):
            pre_train_cost = []
            for b in range(n_train_batches):
                pre_train_cost.append(pretrain_func(b))
        print('Pretrain cost ', '(epoch ', epoch, '): ',
              np.mean(pre_train_cost))

        min_valid_err = np.inf
        for epoch in range(self.finetune_epochs):
            from random import shuffle
            finetune_cost = []

            b_idx = [i for i in range(0, n_train_batches)]
            shuffle(b_idx)
            for b in b_idx:
                cost = finetune_func(b)
                finetune_cost.append(cost)

            if epoch % 10 == 0:
                print('Finetune cost: ', '(epoch ', epoch, '): ',
                      np.mean(finetune_cost))
                valid_cost = []
                for b in range(n_valid_batches):
                    ids, errs, pred_y, act_y = validate_func(b)
                    valid_cost.append(errs)

                curr_valid_err = np.mean(valid_cost)
                print('Validation error: ', np.mean(valid_cost))
                if curr_valid_err * 0.99 > min_valid_err:
                    break
                elif curr_valid_err < min_valid_err:
                    min_valid_err = curr_valid_err

        for b in range(n_train_batches):
            t_ids, t_errs, t_pred_y, t_act_y = tr_validate_func(b)
            self.theano_tr_ids.extend(t_ids)
            self.tr_pred.extend([np.argmax(arr) for arr in t_pred_y])
            self.tr_act.extend([np.argmax(arr) for arr in t_act_y])

    def get_labels(self):
        return self.theano_tr_ids, self.tr_pred, self.tr_act

    def get_test_results(self, ts_data):
        ts_ids, test_x = ts_data

        test_x = shared(value=np.asarray(test_x, dtype=config.floatX),
                        borrow=True)

        test_func = self.sdae.test(test_x)
        n_test_batches = (test_x.get_value(borrow=True).shape[0])

        test_out_probs = []
        for b in range(n_test_batches):
            cls, probs = test_func(b)
            test_out_probs.append(probs[0])

        return ts_ids, test_out_probs
class UseSDAE(AdaClassifier):

    def __init__(self,param):

        self.batch_size = param['batch_size']
        self.iterations = param['iterations']
        self.in_size = param['in_size']
        self.out_size = param['out_size']
        self.hid_sizes = param['hid_sizes']
        self.learning_rate = param['learning_rate']
        self.pre_epochs = param['pre_epochs']
        self.finetune_epochs = param['fine_epochs']
        self.lam = param['lam']
        self.act = param['act']

        self.sdae = SDAE(self.in_size,self.out_size,self.hid_sizes,self.batch_size,self.learning_rate,self.lam,self.act,self.iterations)
        self.sdae.process()

        self.theano_tr_ids, self.tr_pred, self.tr_act = [],[],[]

    def train(self,tr_all,v_all,weights):
        from math import ceil
        tr_ids,tr_x,tr_y = tr_all
        v_ids,v_x,v_y = v_all
        weights_shr = shared(value=np.asarray(weights,dtype=config.floatX),borrow=True)

        def get_shared_data(data_xy):
            data_x,data_y = data_xy
            shared_x = shared(value=np.asarray(data_x,dtype=config.floatX),borrow=True)
            shared_y = shared(value=np.asarray(data_y,dtype=config.floatX),borrow=True)

            return shared_x,T.cast(shared_y,'int32')

        train = get_shared_data((tr_x,tr_y))
        valid = get_shared_data((v_x,v_y))

        n_train_batches = ceil(train[0].get_value(borrow=True).shape[0] / self.batch_size)
        n_valid_batches = ceil(valid[0].get_value(borrow=True).shape[0] / self.batch_size)


        pretrain_func = self.sdae.pre_train(train[0],train[1])
        finetune_func = self.sdae.fine_tune(train[0],train[1],weights_shr)


        my_valid_id_tensor = shared(value=np.asarray(v_ids,dtype=config.floatX),borrow=True)
        my_valid_id_int_tensor = T.cast(my_valid_id_tensor,'int32')
        validate_func = self.sdae.validate(valid[0],valid[1],my_valid_id_int_tensor)

        my_train_id_tensor = shared(value=np.asarray(tr_ids,dtype=config.floatX),borrow=True)
        my_train_id_int_tensor = T.cast(my_train_id_tensor,'int32')
        tr_validate_func = self.sdae.validate(train[0],train[1],my_train_id_int_tensor)

        for epoch in range(self.pre_epochs):
            pre_train_cost = []
            for b in range(n_train_batches):
                pre_train_cost.append(pretrain_func(b))
        print('Pretrain cost ','(epoch ', epoch,'): ',np.mean(pre_train_cost))

        min_valid_err = np.inf
        for epoch in range(self.finetune_epochs):
            from random import shuffle
            finetune_cost = []

            b_idx =[i for i in range(0,n_train_batches)]
            shuffle(b_idx)
            for b in b_idx:
                cost = finetune_func(b)
                finetune_cost.append(cost)

            if epoch%10==0:
                print('Finetune cost: ','(epoch ', epoch,'): ',np.mean(finetune_cost))
                valid_cost = []
                for b in range(n_valid_batches):
                    ids,errs,pred_y,act_y = validate_func(b)
                    valid_cost.append(errs)


                curr_valid_err = np.mean(valid_cost)
                print('Validation error: ',np.mean(valid_cost))
                if curr_valid_err*0.99>min_valid_err:
                    break
                elif  curr_valid_err<min_valid_err:
                    min_valid_err = curr_valid_err


        for b in range(n_train_batches):
            t_ids,t_errs,t_pred_y,t_act_y = tr_validate_func(b)
            self.theano_tr_ids.extend(t_ids)
            self.tr_pred.extend([np.argmax(arr) for arr in t_pred_y])
            self.tr_act.extend([np.argmax(arr) for arr in t_act_y])

    def get_labels(self):
        return self.theano_tr_ids,self.tr_pred,self.tr_act

    def get_test_results(self,ts_data):
        ts_ids, test_x = ts_data

        test_x = shared(value=np.asarray(test_x,dtype=config.floatX),borrow=True)

        test_func = self.sdae.test(test_x)
        n_test_batches = (test_x.get_value(borrow=True).shape[0])

        test_out_probs = []
        for b in range(n_test_batches):
            cls,probs = test_func(b)
            test_out_probs.append(probs[0])

        return ts_ids,test_out_probs
class MySDAE(object):
    def __init__(self, param):

        self.batch_size = param['batch_size']
        self.iterations = param['iterations']
        self.in_size = param['in_size']
        self.out_size = param['out_size']
        self.hid_sizes = param['hid_sizes']
        self.learning_rate = param['learning_rate']
        self.pre_epochs = param['pre_epochs']
        self.finetune_epochs = param['fine_epochs']
        self.lam = param['lam']
        self.act = param['act']
        self.denoise = param['denoise']
        self.corr_level = param['corr_level']
        self.sdae = SDAE(self.in_size, self.out_size, self.hid_sizes,
                         self.batch_size, self.learning_rate, self.lam,
                         self.act, self.iterations, self.denoise,
                         self.corr_level)
        self.sdae.process()

        self.theano_tr_ids, self.tr_pred, self.tr_act = [], [], []

    def pretrain(self, tr_all, v_all, ts_all, weights):

        tr_ids, tr_x, tr_y = tr_all
        v_ids, v_x, v_y = v_all
        ts_ids, ts_x = ts_all

        all_x = []
        all_x.extend(tr_x.get_value())
        all_x.extend(v_x.get_value())
        all_x.extend(ts_x.get_value())
        all_theano_x = shared(value=np.asarray(all_x, dtype=config.floatX),
                              borrow=True)

        if weights is not None:
            use_layer_wise_w = False
            if isinstance(weights[0], list):
                use_layer_wise_w = True
                th_all_weights = []
                for w in weights:
                    w_tmp = np.asarray(w, dtype=config.floatX).reshape(-1, 1)
                    w_tmp = np.append(
                        w_tmp,
                        np.asarray([
                            1 for _ in range(v_x.get_value().shape[0])
                        ]).reshape(-1, 1),
                        axis=0)
                    w_tmp = np.append(
                        w_tmp,
                        np.asarray([
                            1 for _ in range(ts_x.get_value().shape[0])
                        ]).reshape(-1, 1),
                        axis=0)
                    th_all_weights.append(shared(w_tmp))
            else:
                all_weights = np.asarray(tr_weights)
                all_weights = np.append(
                    all_weights,
                    np.asarray([1 for _ in range(v_x.get_value().shape[0])
                                ]).reshape(-1, 1),
                    axis=0)
                all_weights = np.append(
                    all_weights,
                    np.asarray([1 for _ in range(ts_x.get_value().shape[0])
                                ]).reshape(-1, 1),
                    axis=0)
                th_all_weights = shared(all_weights)
        else:
            th_all_weights = None
        n_pretrain_batches = ceil(
            all_theano_x.get_value(borrow=True).shape[0] / self.batch_size)

        pretrain_func = self.sdae.pre_train(all_theano_x, th_all_weights)

        for epoch in range(self.pre_epochs):
            pre_train_cost = []
            b_indices = [i for i in range(n_pretrain_batches)]
            np.random.shuffle(b_indices)
            for b in b_indices:
                pre_train_cost.append(pretrain_func(b))
            print('Pretrain cost (Layer-wise) ', '(epoch ', epoch, '): ',
                  np.mean(pre_train_cost))

    def full_pretrain(self, tr_all, v_all, ts_all, weights):

        tr_ids, tr_x, tr_y = tr_all
        v_ids, v_x, v_y = v_all
        ts_ids, ts_x = ts_all

        all_x = []
        all_x.extend(tr_x.get_value())
        all_x.extend(v_x.get_value())
        all_x.extend(ts_x.get_value())
        all_theano_x = shared(value=np.asarray(all_x, dtype=config.floatX),
                              borrow=True)

        if weights is not None:
            all_weights = np.asarray(tr_weights)
            all_weights = np.append(
                all_weights,
                np.asarray([1 for _ in range(v_x.get_value().shape[0])
                            ]).reshape(-1, 1),
                axis=0)
            all_weights = np.append(
                all_weights,
                np.asarray([1 for _ in range(ts_x.get_value().shape[0])
                            ]).reshape(-1, 1),
                axis=0)
            th_all_weights = shared(all_weights)
        else:
            th_all_weights = None
        n_pretrain_batches = ceil(
            all_theano_x.get_value(borrow=True).shape[0] / self.batch_size)

        full_pretrain_func = self.sdae.full_pretrain(all_theano_x,
                                                     th_all_weights)

        for epoch in range(self.pre_epochs // 2):
            full_pre_cost = []
            b_indices = [i for i in range(n_pretrain_batches)]
            np.random.shuffle(b_indices)
            for b in b_indices:
                full_pre_cost.append(full_pretrain_func(b))
            print('Pretrain cost (Full) ', '(epoch ', epoch, '): ',
                  np.mean(full_pre_cost))

    def finetune(self, tr_all, v_all, weights):
        tr_ids, tr_x, tr_y = tr_all
        v_ids, v_x, v_y = v_all
        if weights is not None:
            th_weights = shared(np.array(weights, dtype=config.floatX))
        else:
            th_weights = None

        finetune_func = self.sdae.fine_tune(tr_x, tr_y, th_weights)
        n_pretrain_batches = ceil(
            tr_x.get_value(borrow=True).shape[0] / self.batch_size)

        validate_func = self.sdae.validate(v_x, v_y, v_ids)
        n_valid_batches = ceil(
            v_x.get_value(borrow=True).shape[0] / self.batch_size)

        min_valid_err = np.inf
        for epoch in range(self.finetune_epochs):
            b_indices = [i for i in range(n_pretrain_batches)]
            np.random.shuffle(b_indices)
            finetune_cost = []
            for b in b_indices:
                finetune_cost.append(finetune_func(b))

            if epoch % 1 == 0:
                print('Finetune cost: ', '(epoch ', epoch, '): ',
                      np.mean(finetune_cost))
                valid_cost = []
                for b in range(n_valid_batches):
                    ids, errs, pred_y, act_y = validate_func(b)
                    valid_cost.append(errs)

                curr_valid_err = np.mean(valid_cost)
                print('Validation error: ', np.mean(valid_cost))
                if curr_valid_err * 0.99 > min_valid_err:
                    break
                elif curr_valid_err < min_valid_err:
                    min_valid_err = curr_valid_err
            print('Fintune with logreg (epoch ', epoch, '): ',
                  np.mean(finetune_cost))

    def get_features(self, tr_all, v_all, ts_all, layer_idx):

        tr_ids, tr_x, tr_y = tr_all
        v_ids, v_x, v_y = v_all
        ts_ids, ts_x = ts_all

        tr_feature_func = self.sdae.get_features(tr_x, tr_y, tr_ids, layer_idx,
                                                 False)
        v_features_func = self.sdae.get_features(v_x, v_y, v_ids, layer_idx,
                                                 False)
        ts_features_func = self.sdae.get_features(ts_x, None, ts_ids,
                                                  layer_idx, True)

        n_train_batches = ceil(
            tr_x.get_value(borrow=True).shape[0] / self.batch_size)
        n_valid_batches = ceil(
            v_x.get_value(borrow=True).shape[0] / self.batch_size)
        n_test_batches = ceil(
            ts_x.get_value(borrow=True).shape[0] / self.batch_size)

        all_tr_features = []
        all_tr_outputs = []
        all_ts_features = []
        for b in range(n_train_batches):
            features, y, ids = tr_feature_func(b)

            temp = np.concatenate((np.reshape(ids,
                                              (ids.shape[0], 1)), features),
                                  axis=1)

            all_tr_features.extend(temp.tolist())
            all_tr_outputs.extend(y)

        print('Size train features: ', len(all_tr_features), ' x ',
              len(all_tr_features[0]))

        for b in range(n_valid_batches):
            features, y, ids = v_features_func(b)
            temp = [ids]
            temp = np.concatenate((np.reshape(ids,
                                              (ids.shape[0], 1)), features),
                                  axis=1)

            all_tr_features.extend(temp.tolist())
            all_tr_outputs.extend(y)

        print('Size train+valid features: ', len(all_tr_features), ' x ',
              len(all_tr_features[0]))

        for b in range(n_test_batches):
            features, y, ids = ts_features_func(b)

            temp = np.concatenate((np.reshape(ids,
                                              (ids.shape[0], 1)), features),
                                  axis=1)
            all_ts_features.extend(temp.tolist())

        print('Size test features: ', len(all_ts_features), ' x ',
              len(all_ts_features[0]))

        return (all_tr_features, all_tr_outputs), all_ts_features

    def get_labels(self):
        return self.theano_tr_ids, self.tr_pred, self.tr_act

    def get_test_results(self, ts_data):
        ts_ids, test_x = ts_data

        test_x = shared(value=np.asarray(test_x, dtype=config.floatX),
                        borrow=True)

        test_func = self.sdae.test(test_x)
        n_test_batches = (test_x.get_value(borrow=True).shape[0])

        test_out_probs = []
        for b in range(n_test_batches):
            cls, probs = test_func(b)
            test_out_probs.append(probs[0])

        return ts_ids, test_out_probs
class MySDAE(object):

    def __init__(self,param):

        self.batch_size = param['batch_size']
        self.iterations = param['iterations']
        self.in_size = param['in_size']
        self.out_size = param['out_size']
        self.hid_sizes = param['hid_sizes']
        self.learning_rate = param['learning_rate']
        self.pre_epochs = param['pre_epochs']
        self.finetune_epochs = param['fine_epochs']
        self.lam = param['lam']
        self.act = param['act']
        self.denoise = param['denoise']
        self.corr_level = param['corr_level']
        self.sdae = SDAE(self.in_size,self.out_size,self.hid_sizes,self.batch_size,self.learning_rate,self.lam,self.act,self.iterations,self.denoise,self.corr_level)
        self.sdae.process()

        self.theano_tr_ids, self.tr_pred, self.tr_act = [],[],[]

    def pretrain(self,tr_all,v_all,ts_all,weights):

        tr_ids,tr_x,tr_y = tr_all
        v_ids,v_x,v_y = v_all
        ts_ids,ts_x = ts_all

        all_x = []
        all_x.extend(tr_x.get_value())
        all_x.extend(v_x.get_value())
        all_x.extend(ts_x.get_value())
        all_theano_x = shared(value=np.asarray(all_x,dtype=config.floatX),borrow=True)

        if weights is not None:
            use_layer_wise_w = False
            if isinstance(weights[0],list):
                use_layer_wise_w = True
                th_all_weights = []
                for w in weights:
                    w_tmp = np.asarray(w,dtype=config.floatX).reshape(-1,1)
                    w_tmp = np.append(w_tmp, np.asarray([1 for _ in range(v_x.get_value().shape[0])]).reshape(-1,1),axis=0)
                    w_tmp = np.append(w_tmp, np.asarray([1 for _ in range(ts_x.get_value().shape[0])]).reshape(-1,1),axis=0)
                    th_all_weights.append(shared(w_tmp))
            else:
                all_weights = np.asarray(tr_weights)
                all_weights = np.append(all_weights, np.asarray([1 for _ in range(v_x.get_value().shape[0])]).reshape(-1,1),axis=0)
                all_weights = np.append(all_weights, np.asarray([1 for _ in range(ts_x.get_value().shape[0])]).reshape(-1,1),axis=0)
                th_all_weights = shared(all_weights)
        else:
            th_all_weights = None
        n_pretrain_batches = ceil(all_theano_x.get_value(borrow=True).shape[0] / self.batch_size)

        pretrain_func = self.sdae.pre_train(all_theano_x,th_all_weights)

        for epoch in range(self.pre_epochs):
            pre_train_cost = []
            b_indices = [i for i in range(n_pretrain_batches)]
            np.random.shuffle(b_indices)
            for b in b_indices:
                pre_train_cost.append(pretrain_func(b))
            print('Pretrain cost (Layer-wise) ','(epoch ', epoch,'): ',np.mean(pre_train_cost))

    def full_pretrain(self,tr_all,v_all,ts_all,weights):

        tr_ids,tr_x,tr_y = tr_all
        v_ids,v_x,v_y = v_all
        ts_ids,ts_x = ts_all

        all_x = []
        all_x.extend(tr_x.get_value())
        all_x.extend(v_x.get_value())
        all_x.extend(ts_x.get_value())
        all_theano_x = shared(value=np.asarray(all_x,dtype=config.floatX),borrow=True)

        if weights is not None:
            all_weights = np.asarray(tr_weights)
            all_weights = np.append(all_weights,np.asarray([1 for _ in range(v_x.get_value().shape[0])]).reshape(-1,1),axis=0)
            all_weights = np.append(all_weights,np.asarray([1 for _ in range(ts_x.get_value().shape[0])]).reshape(-1,1),axis=0)
            th_all_weights = shared(all_weights)
        else:
            th_all_weights = None
        n_pretrain_batches = ceil(all_theano_x.get_value(borrow=True).shape[0] / self.batch_size)

        full_pretrain_func = self.sdae.full_pretrain(all_theano_x,th_all_weights)

        for epoch in range(self.pre_epochs//2):
            full_pre_cost = []
            b_indices = [i for i in range(n_pretrain_batches)]
            np.random.shuffle(b_indices)
            for b in b_indices:
                full_pre_cost.append(full_pretrain_func(b))
            print('Pretrain cost (Full) ','(epoch ', epoch,'): ',np.mean(full_pre_cost))

    def finetune(self,tr_all,v_all,weights):
        tr_ids,tr_x,tr_y = tr_all
        v_ids,v_x,v_y = v_all
        if weights is not None:
            th_weights = shared(np.array(weights,dtype=config.floatX))
        else:
            th_weights = None

        finetune_func = self.sdae.fine_tune(tr_x,tr_y,th_weights)
        n_pretrain_batches = ceil(tr_x.get_value(borrow=True).shape[0] / self.batch_size)

        validate_func = self.sdae.validate(v_x,v_y,v_ids)
        n_valid_batches = ceil(v_x.get_value(borrow=True).shape[0] / self.batch_size)

        min_valid_err = np.inf
        for epoch in range(self.finetune_epochs):
            b_indices = [i for i in range(n_pretrain_batches)]
            np.random.shuffle(b_indices)
            finetune_cost = []
            for b in b_indices:
                finetune_cost.append(finetune_func(b))

            if epoch%1==0:
                print('Finetune cost: ','(epoch ', epoch,'): ',np.mean(finetune_cost))
                valid_cost = []
                for b in range(n_valid_batches):
                    ids,errs,pred_y,act_y = validate_func(b)
                    valid_cost.append(errs)

                curr_valid_err = np.mean(valid_cost)
                print('Validation error: ',np.mean(valid_cost))
                if curr_valid_err*0.99>min_valid_err:
                    break
                elif curr_valid_err<min_valid_err:
                    min_valid_err = curr_valid_err
            print('Fintune with logreg (epoch ',epoch,'): ',np.mean(finetune_cost))

    def get_features(self,tr_all,v_all,ts_all,layer_idx):

        tr_ids,tr_x,tr_y = tr_all
        v_ids,v_x,v_y = v_all
        ts_ids,ts_x = ts_all

        tr_feature_func = self.sdae.get_features(tr_x,tr_y,tr_ids,layer_idx,False)
        v_features_func = self.sdae.get_features(v_x,v_y,v_ids,layer_idx,False)
        ts_features_func = self.sdae.get_features(ts_x,None,ts_ids,layer_idx,True)

        n_train_batches = ceil(tr_x.get_value(borrow=True).shape[0] / self.batch_size)
        n_valid_batches = ceil(v_x.get_value(borrow=True).shape[0] / self.batch_size)
        n_test_batches = ceil(ts_x.get_value(borrow=True).shape[0] / self.batch_size)

        all_tr_features = []
        all_tr_outputs = []
        all_ts_features = []
        for b in range(n_train_batches):
            features, y, ids = tr_feature_func(b)

            temp = np.concatenate((np.reshape(ids,(ids.shape[0],1)),features),axis=1)

            all_tr_features.extend(temp.tolist())
            all_tr_outputs.extend(y)

        print('Size train features: ',len(all_tr_features),' x ',len(all_tr_features[0]))

        for b in range(n_valid_batches):
            features, y, ids = v_features_func(b)
            temp = [ids]
            temp = np.concatenate((np.reshape(ids,(ids.shape[0],1)),features),axis=1)

            all_tr_features.extend(temp.tolist())
            all_tr_outputs.extend(y)

        print('Size train+valid features: ',len(all_tr_features),' x ',len(all_tr_features[0]))

        for b in range(n_test_batches):
            features, y, ids = ts_features_func(b)

            temp = np.concatenate((np.reshape(ids,(ids.shape[0],1)),features),axis=1)
            all_ts_features.extend(temp.tolist())

        print('Size test features: ',len(all_ts_features),' x ',len(all_ts_features[0]))

        return (all_tr_features,all_tr_outputs),all_ts_features

    def get_labels(self):
        return self.theano_tr_ids,self.tr_pred,self.tr_act

    def get_test_results(self,ts_data):
        ts_ids, test_x = ts_data

        test_x = shared(value=np.asarray(test_x,dtype=config.floatX),borrow=True)

        test_func = self.sdae.test(test_x)
        n_test_batches = (test_x.get_value(borrow=True).shape[0])

        test_out_probs = []
        for b in range(n_test_batches):
            cls,probs = test_func(b)
            test_out_probs.append(probs[0])

        return ts_ids,test_out_probs