Ejemplo n.º 1
0
    def __init__(self,
                 numpy_rng=None,
                 theano_rng=None,
                 n_ins=784,
                 hidden_layers_sizes=[500, 500],
                 n_outs=10):
        self.sigmoid_layers = []
        self.rbm_layers = []
        self.params = []
        self.n_layers = len(hidden_layers_sizes)

        assert self.n_layers > 0

        if not theano_rng:
            theano_rng = MRG_RandomStreams(numpy_rng.randint(2**30))
        self.x = T.matrix('x')
        self.y = T.ivector('y')

        for i in range(self.n_layers):
            if i == 0:
                input_size = n_ins
            else:
                input_size = hidden_layers_sizes[i - 1]

            if i == 0:
                layer_input = self.x
            else:
                layer_input = self.sigmoid_layers[-1].output
            sigmoid_layer = HiddenLayer(rng=numpy_rng,
                                        input=layer_input,
                                        n_in=input_size,
                                        n_out=hidden_layers_sizes[i],
                                        activation=T.nnet.sigmoid)
            self.sigmoid_layers.append(sigmoid_layer)
            self.params.extend(sigmoid_layer.params)
            rbm_layer = RBM(numpy_rng=numpy_rng,
                            theano_rng=theano_rng,
                            input=layer_input,
                            n_visible=input_size,
                            n_hidden=hidden_layers_sizes[i],
                            W=sigmoid_layer.W,
                            hbias=sigmoid_layer.b)
            self.rbm_layers.append(rbm_layer)
        self.logLayer = LogisticRegression(
            input=self.sigmoid_layers[-1].output,
            n_in=hidden_layers_sizes[-1],
            n_out=n_outs)
        self.params.extend(self.logLayer.params)
        self.finetune_cost = self.logLayer.negative_log_likelihood(self.y)
        self.errors = self.logLayer.error(self.y)
Ejemplo n.º 2
0
def infer(w):
    test = pd.read_csv('../data/census-income.test.clean',
                       header=None,
                       delim_whitespace=True)
    test_y = test.values[:, -1]
    X = test.values[:, :-1]
    X = np.array(X, dtype='float64')
    test_X = np.copy(X)
    normal_list = [0, 126, 210, 211, 212, 353, 499]
    for i in normal_list:
        test_X[:, i] = (X[:, i] - X[:, i].mean()) / (X[:, i].std())
    lr = LogisticRegression()
    print(lr.score(test_X, test_y, w))
    lr.F1(test_X, test_y, w)
Ejemplo n.º 3
0
def compare_lr():
    # Instantiate the Logistic Regression model
    model = LogisticRegression(l2penalty=0, verbose=True)

    # Evaluate and plot the model with different learning rates
    for exp in (-2, -4, -6):

        accuracies_tr = []
        accuracies_ts = []

        iterations = [i for i in range(1, ITERATIONS + 1)]

        # Fit the model. Note that while fitting we are predicting due to the presence of a callback.
        model.fit(x_tr,
                  y_tr,
                  lr=10**exp,
                  maxit=ITERATIONS,
                  callback=[accuracy_callback, accuracies_tr, accuracies_ts])

        # Plot the accuracies
        fig = plt.figure()

        plt.plot(iterations,
                 accuracies_tr.copy(),
                 label='Train',
                 color='blue',
                 linewidth=1)
        plt.plot(iterations,
                 accuracies_ts.copy(),
                 label='Test',
                 color='green',
                 linewidth=1)
        plt.title('Train vs Test with lr = 10^%s' % exp)
        plt.ylabel('Accuracy')
        plt.xlabel('Iteration')
        plt.grid(True, linestyle=':')
        plt.legend(loc="upper right")
        plt.show()

        fig.savefig('logistic_accuracy_lr10^%s.png' % exp)
Ejemplo n.º 4
0
def main():
    if args.modelIdx == '1':
        model = LogisticRegression()
    elif args.modelIdx == '2':
        model = SupportVectorMachine()
    elif args.modelIdx == '3':
        tssp = [0.01, 0.03, 0.05, 0.08, 0.1, 0.15]
        analysis_1(tssp)
        return
    elif args.modelIdx == '4':
        tssp = [0.01, 0.03, 0.05, 0.08, 0.1, 0.15]
        analysis_2(tssp)
        return
    else:
        return
    model.train_from_csv(args.trainingDataFilename)
    model.test_from_csv(args.testDataFilename)
 def calculator(self):
     print('enter calculator')
     self.X, self.y = self.read_data()
     self.lr = LogisticRegression(n_iter=70, eta=0.005, batch_size=10, gammar=0.5)
     while not self._begin_cal:
         time.sleep(1)
     start_time = time.time()
     st_clk = time.clock()
     print('start calculator')
     param_list = self.GetParams()
     param_list, loss = self.algorithm(param_list, 0)
     # index_epoch = (self._data_end - self._data_begin) // self.lr.batch_size
     index = 0
     while index < self._max_epoch:
         index += 1
         self.SetParams(param_list)
         param_list = self.GetParams()
         param_list, loss = self.algorithm(param_list, index)
         if index % 2000 == 0:
             print("Calculator.calculator::", index)
             print(loss)
     print('stop calculator', time.time() - start_time)
     print('CPU time', time.clock() - st_clk)
Ejemplo n.º 6
0
def cross_validate(csv_file_name,
                   tssp,
                   losses_file_name,
                   new_feature=False,
                   debug=False):
    '''
		Perform 10-fold incremental cross validation.
	'''
    total_num = 2000
    num_words = 4000
    lists_of_dict = []
    losses = zeros((3, len(tssp), 10))  # #models, #tss, #folds
    sklosses = zeros((2, len(tssp), 10))
    generate_train_and_test_files_cv(csv_file_name, 10)
    for i in range(10):
        lists_of_dict.append(csv_to_dict('cv%d.dat' % (i)))
    for i, proportion in enumerate(tssp):
        for j in range(10):
            # Contruct train set
            training_lists_of_dict = lists_of_dict[:j] + lists_of_dict[j + 1:]
            training_list_of_dict = [
                item for sublist in training_lists_of_dict for item in sublist
            ]
            testing_list_of_dict = lists_of_dict[j]
            # Randomly select samples
            random_indices = permutation(len(training_list_of_dict))
            random_indices = random_indices[:int(total_num * proportion)]
            training_list_of_dict = [
                training_list_of_dict[k] for k in random_indices
            ]
            # Find the word features
            feature_words = construct_word_feature(training_list_of_dict,
                                                   num_words)
            # Extract features and labels
            training_X, training_y = extract_word_feature_and_label(
                training_list_of_dict, feature_words, new_feature=new_feature)
            testing_X, testing_y = extract_word_feature_and_label(
                testing_list_of_dict, feature_words, new_feature=new_feature)
            # NBC
            M = nbc_train(feature_words,
                          training_X,
                          training_y,
                          new_feature=new_feature)
            losses[0, i, j], _ = nbc_test(M, testing_X, testing_y)
            # LR
            lr = LogisticRegression()
            lr.train(training_X, training_y)
            losses[1, i, j] = lr.test(testing_X, testing_y)
            # SVM
            svm = SupportVectorMachine()
            svm.train(training_X, training_y)
            losses[2, i, j] = svm.test(testing_X, testing_y)
            # Libary functions
            if debug:
                training_y[training_y == 0] = -1
                testing_y[testing_y == 0] = -1
                sklr = skLogisticRegresssion()
                sklr.fit(training_X.T, training_y)
                sklosses[0, i, j] = 1 - sklr.score(testing_X.T, testing_y)
                sksvm = skSupportVectorMachine()
                sksvm.fit(training_X.T, training_y)
                sklosses[1, i, j] = 1 - sksvm.score(testing_X.T, testing_y)
    save(losses_file_name, losses)
    save('debug_' + losses_file_name, sklosses)
Ejemplo n.º 7
0
# -*- coding: utf-8 -*-
"""
Created on Mon Sep 25 18:45:40 2016

@author: NISHIT
"""

import time
start_exec_time = time.time()

from sentiment_reader import SentimentCorpus
from lr import LogisticRegression

if __name__ == '__main__':
    dataset = SentimentCorpus()
    lr = LogisticRegression()
    lr.LR(dataset.train_X, dataset.train_y,dataset.test_X, dataset.test_y)
    
    print("Execution time: --- %s seconds ---" % (time.time() - start_exec_time))
    def __init__(self,
                 numpy_rng,
                 theano_rng=None,
                 n_ins=784,
                 hidden_layer_sizes=[500, 500],
                 n_out=10,
                 corruption_levels=[0.1, 0.1]):
        self.sigmoid_layers = []
        self.dA_layers = []
        self.params = []
        self.n_layers = len(hidden_layer_sizes)  #网络层数

        assert self.n_layers > 0
        if not theano_rng:
            theano_rng = RandomStreams(numpy_rng.randint(2**30))
        self.x = T.matrix("x")  #图像序列
        self.y = T.ivector("y")  #标签

        for i in range(self.n_layers):
            if i == 0:
                input_size = n_ins
            else:
                input_size = hidden_layer_sizes[i - 1]
            if i == 0:
                layer_input = self.x
            else:
                layer_input = self.sigmoid_layers[-1].output
            """HIddenLayer
            def __init__(self,rng,input,n_in,n_out,
                 W=None,b=None,activation=T.tanh):
            """
            sigmoid_layer = HiddenLayer(rng=numpy_rng,
                                        input=layer_input,
                                        n_in=input_size,
                                        n_out=hidden_layer_sizes[i],
                                        activation=T.nnet.sigmoid)
            self.sigmoid_layers.append(sigmoid_layer)
            self.params.extend(sigmoid_layer.params)
            #创建一个dA与HiddenLayer共享权值
            """dA
            def __init__(self,numpy_rng,theano_rng=None,input=None,
                 n_visible=784,n_hidden=500,
                 W=None,bhid=None,bvis=None):
            """
            dA_layer = dA(numpy_rng=numpy_rng,
                          theano_rng=theano_rng,
                          input=layer_input,
                          n_visible=input_size,
                          n_hidden=hidden_layer_sizes[i - 1],
                          W=sigmoid_layer.W,
                          bhid=sigmoid_layer.b)
            self.dA_layers.append(dA_layer)
        """LogisticRegression
        def __init__(self,input,n_in,n_out):
        """
        self.logLayer = LogisticRegression(
            input=self.sigmoid_layers[-1].output,
            n_in=hidden_layer_sizes[-1],
            n_out=n_out)
        self.params.extend(self.logLayer.params)
        # construct a function that implements one step of finetunining
        # compute the cost for second phase of training,
        # defined as the negative log likelihood
        self.finetune_cost = self.logLayer.negative_log_likelihood(self.y)
        # compute the gradients with respect to the model parameters
        # symbolic variable that points to the number of errors made on the
        # minibatch given by self.x and self.y
        self.errors = self.logLayer.error(self.y)
class sdA(object):
    """
    n_ins:输入层的维度(第一层为数据输入层)
    hidden_layer_sizes:中间层大小
    n_out:网络的输出大小
    corruption_levels:每层的corruption程度
    """
    def __init__(self,
                 numpy_rng,
                 theano_rng=None,
                 n_ins=784,
                 hidden_layer_sizes=[500, 500],
                 n_out=10,
                 corruption_levels=[0.1, 0.1]):
        self.sigmoid_layers = []
        self.dA_layers = []
        self.params = []
        self.n_layers = len(hidden_layer_sizes)  #网络层数

        assert self.n_layers > 0
        if not theano_rng:
            theano_rng = RandomStreams(numpy_rng.randint(2**30))
        self.x = T.matrix("x")  #图像序列
        self.y = T.ivector("y")  #标签

        for i in range(self.n_layers):
            if i == 0:
                input_size = n_ins
            else:
                input_size = hidden_layer_sizes[i - 1]
            if i == 0:
                layer_input = self.x
            else:
                layer_input = self.sigmoid_layers[-1].output
            """HIddenLayer
            def __init__(self,rng,input,n_in,n_out,
                 W=None,b=None,activation=T.tanh):
            """
            sigmoid_layer = HiddenLayer(rng=numpy_rng,
                                        input=layer_input,
                                        n_in=input_size,
                                        n_out=hidden_layer_sizes[i],
                                        activation=T.nnet.sigmoid)
            self.sigmoid_layers.append(sigmoid_layer)
            self.params.extend(sigmoid_layer.params)
            #创建一个dA与HiddenLayer共享权值
            """dA
            def __init__(self,numpy_rng,theano_rng=None,input=None,
                 n_visible=784,n_hidden=500,
                 W=None,bhid=None,bvis=None):
            """
            dA_layer = dA(numpy_rng=numpy_rng,
                          theano_rng=theano_rng,
                          input=layer_input,
                          n_visible=input_size,
                          n_hidden=hidden_layer_sizes[i - 1],
                          W=sigmoid_layer.W,
                          bhid=sigmoid_layer.b)
            self.dA_layers.append(dA_layer)
        """LogisticRegression
        def __init__(self,input,n_in,n_out):
        """
        self.logLayer = LogisticRegression(
            input=self.sigmoid_layers[-1].output,
            n_in=hidden_layer_sizes[-1],
            n_out=n_out)
        self.params.extend(self.logLayer.params)
        # construct a function that implements one step of finetunining
        # compute the cost for second phase of training,
        # defined as the negative log likelihood
        self.finetune_cost = self.logLayer.negative_log_likelihood(self.y)
        # compute the gradients with respect to the model parameters
        # symbolic variable that points to the number of errors made on the
        # minibatch given by self.x and self.y
        self.errors = self.logLayer.error(self.y)

    """
    下面这个函数是为!!!每i层!!!生成一个Theano function类型的预训练函数,
    返回值是一个list,list中就是每层的预训练函
    为了能在训练中更改 the corruption level or the learning rate,
    我们定义Theano variables。
    """

    def pretraining_functions(self, train_set_x, batch_size):
        index = T.lscalar('index')
        corruption_level = T.scalar('corruption')
        learning_rate = T.scalar('lr')
        batch_begin = index * batch_size
        batch_end = batch_begin + batch_size

        pretrain_fns = []
        for dA in self.dA_layers:
            cost, updates = dA.get_cost_update(corruption_level, learning_rate)
            fn = theano.function(
                inputs=[
                    index,
                    theano.In(corruption_level, value=0.2),
                    theano.In(learning_rate, value=0.1)
                ],
                outputs=cost,
                updates=updates,
                givens={self.x: train_set_x[batch_begin:batch_end]})
            pretrain_fns.append(fn)
        return pretrain_fns

    """
    创建微调函数
    Once all layers are pre-trained, 
    the network goes through a second stage of training called fine-tuning.
    降噪自编码完成之后,网络进行监督学习得到误差再微调参数
    """

    def build_finetune_functions(self, datasets, batch_size, learning_rate):
        (train_set_x, train_set_y) = datasets[0]
        (valid_set_x, valid_set_y) = datasets[1]
        (test_set_x, test_set_y) = datasets[2]
        n_valid_batches = valid_set_x.get_value(
            borrow=True).shape[0] // batch_size
        n_test_batches = test_set_x.get_value(
            borrow=True).shape[0] // batch_size
        index = T.lscalar('index')
        gparams = T.grad(self.finetune_cost, self.params)
        updates = [(param, param - gparam * learning_rate)
                   for param, gparam in zip(self.params, gparams)]

        train_fn = theano.function(
            inputs=[index],
            outputs=self.finetune_cost,
            updates=updates,
            givens={
                self.x:
                train_set_x[index * batch_size:(index + 1) * batch_size],
                self.y:
                train_set_y[index * batch_size:(index + 1) * batch_size]
            },
            name='train')

        test_score_i = theano.function(
            inputs=[index],
            outputs=self.errors,
            givens={
                self.x:
                test_set_x[index * batch_size:(index + 1) * batch_size],
                self.y: test_set_y[index * batch_size:(index + 1) * batch_size]
            },
            name='test')

        valid_score_i = theano.function(
            inputs=[index],
            outputs=self.errors,
            givens={
                self.x:
                valid_set_x[index * batch_size:(index + 1) * batch_size],
                self.y:
                valid_set_y[index * batch_size:(index + 1) * batch_size]
            },
            name='valid')

        def valid_score():
            return [valid_score_i(i) for i in range(n_valid_batches)]

        def test_score():
            return [test_score_i(i) for i in range(n_test_batches)]

        return train_fn, valid_score, test_score
Ejemplo n.º 10
0
import sys
from lr import LogisticRegression
#from sklearn.linear_model import LogisticRegression
from sklearn.model_selection import train_test_split

X, Y = [], []
for line in sys.stdin:
    p = line.strip().split(' ')
    X.append([float(x) for x in p[1:-1]])
    Y.append(int(p[-1]))

X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.33)

W = [0] * len(X[0])
m = LogisticRegression()
W = m.fit(X_train, Y_train, W, 1.0, 50000)
print '== test on kflearn:', m.score(X_test, Y_test, W)
print '== train on kflearn:', m.score(X_train, Y_train, W)
print '== all on kflearn:', m.score(X, Y, W)
'''
m = LogisticRegression()
m.fit(X_train, Y_train)
print '== test on sklearn:', m.score(X_test,Y_test)
'''
Ejemplo n.º 11
0
def main():

    # load training data
    print("####################################################")
    print("Data preprocessing...")
    filename = 'data/train.csv'
    Dict = transform(filename)

    X_train, X_test, y_train, y_test = cross_validation.train_test_split(
        Dict['data'],
        Dict['target'],
        test_size=test_size,
        random_state=random_state)

    train = pd.concat([X_train, y_train], axis=1)
    print("Cross validate on 10% of the whole dataset, training data shape: ",
          train.shape)
    thresh = nan_thresh * train.shape[1]
    train = train.dropna(thresh=thresh)
    #train = train.dropna(subset = ['Income','HouseholdStatus','EducationLevel'])

    train = fill_missing(train, strategy, isClassified)

    y = train['Happy']
    df = train.drop('Happy', 1)
    numeric_col = df[['YOB', 'votes']]
    #numeric_col = fill_missing(numeric_col,'mean',False)
    normalized = pd.DataFrame(normalize(numeric_col))
    categorical_col = df[[
        'Income', 'HouseholdStatus', 'EducationLevel', 'Party'
    ]]
    #categorical_col = fill_missing(categorical_col,'median',False)
    binary_col = df.drop(['UserID','YOB','Income','HouseholdStatus',\
                        'EducationLevel','Party','votes'],axis = 1)

    encoder = OneHotEncoder()
    #one_hot = pd.get_dummies(categorical_col)
    one_hot = pd.DataFrame(encoder.fit_transform(categorical_col).toarray())

    #X = pd.concat([normalized,one_hot,binary_col],axis = 1)

    X = np.hstack((normalized, one_hot, binary_col))
    #X = np.hstack((one_hot,binary_col))

    X_test = fill_missing(X_test, strategy, isClassified)
    UserID = X_test['UserID']

    categorical_col = X_test[[
        'Income', 'HouseholdStatus', 'EducationLevel', 'Party'
    ]]
    numeric_col = X_test[['YOB', 'votes']]
    binary_col = X_test.drop(['UserID','YOB','Income','HouseholdStatus',\
                        'EducationLevel','Party','votes'],axis = 1)
    one_hot = encoder.fit_transform(categorical_col).toarray()
    normalized = normalize(numeric_col)
    #X = np.hstack((one_hot,binary_col))

    X_test = np.hstack((normalized, one_hot, binary_col))
    X_a = X
    X_b = X_test
    y_a = y
    y_b = y_test
    print("Training data shape (After drop NaN and one hot encoding): ",
          X_a.shape)
    print("Cross validation data shape (after one hot encoding): ", X_b.shape)
    #X_a, X_b, y_a, y_b = cross_validation.train_test_split(X, y, test_size=0.1, random_state=0)
    print("####################################################")
    ### use the logistic regression
    print('Train the logistic regression classifier...')
    start = time.time()
    clf_lr = LogisticRegression()
    clf_lr = clf_lr.fit(X_a, y_a)
    clf_lr_predict = clf_lr.predict(X_b)
    end = time.time()
    print("Accuracy of logistic regression is ",
          (y_b == clf_lr_predict).sum() / y_b.shape[0])
    print("Time of of logistic regression is %.4fs." % (end - start))

    start = time.time()
    lr_model = linear_model.LogisticRegression()
    lr_model = lr_model.fit(X_a, y_a)
    lr_model_predict = lr_model.predict(X_b)
    print("Scikit learn n_iter_: ", lr_model.n_iter_)
    end = time.time()
    print("Accuracy of scikit-learn logistic Regression is ",
          (y_b == lr_model_predict).sum() / y_b.shape[0])
    print("Time of of scikit-learn logistic regression is %.4fs." %
          (end - start))
    print(" ")

    ### use the naive bayes
    print('Train the naive bayes classifier...')
    start = time.time()
    clf_nb = NaiveBayes()
    clf_nb = clf_nb.fit(X_a, y_a)
    clf_nb_predict = clf_nb.predict(X_b)
    end = time.time()
    print("Accuracy of naive bayes  is ",
          (y_b == clf_nb_predict).sum() / y_b.shape[0])
    print("Time of of naive bayes is %.4fs." % (end - start))

    start = time.time()
    nb_model = BernoulliNB()
    nb_model = nb_model.fit(X_a, y_a)
    nb_model_predict = nb_model.predict(X_b)
    end = time.time()
    print("Accuracy of scikit-learn Bernoulli NB is ",
          (y_b == nb_model_predict).sum() / y_b.shape[0])
    print("Time of of scikit-learn Bernoulli NB is %.4fs." % (end - start))
    print(" ")

    ## use the svm
    print('Train the SVM classifier...')
    start = time.time()
    svm_model = SVC(kernel=kernel)
    svm_model = svm_model.fit(X_a, y_a)
    svm_model_predict = svm_model.predict(X_b)
    end = time.time()

    print("Accuracy of scikit-learn SVM is ",
          (y_b == svm_model_predict).sum() / y_b.shape[0])
    print("Time of of scikit-learn SVM is %.4fs." % (end - start))
    print(" ")

    ## use the random forest
    print('Train the random forest classifier...')
    start = time.time()
    rf_model = RandomForestClassifier(n_estimators=n_estimators,
                                      random_state=random_state)
    rf_model = rf_model.fit(X_a, y_a)
    rf_model_predict = rf_model.predict(X_b)
    end = time.time()

    print("Accuracy of scikit-learn RF is ",
          (y_b == rf_model_predict).sum() / y_b.shape[0])
    print("Time of of scikit-learn RF is %.4fs." % (end - start))
    print(" ")

    print("####################################################")
    print("Start predicting test dataset...")
    filename_test = './data/test.csv'
    Dict = transform(filename_test)
    X_test_data = Dict['data']

    test = fill_missing(X_test_data, strategy, isClassified)
    UserID = test['UserID'].astype(int)

    categorical_col = test[[
        'Income', 'HouseholdStatus', 'EducationLevel', 'Party'
    ]]
    numeric_col = test[['YOB', 'votes']]
    binary_col = test.drop(['UserID','YOB','Income','HouseholdStatus',\
                        'EducationLevel','Party','votes'],axis = 1)
    one_hot = encoder.fit_transform(categorical_col).toarray()
    normalized = normalize(numeric_col)
    #X_test = np.hstack((one_hot,binary_col))

    X_test = np.hstack((normalized, one_hot, binary_col))
    print("Test data shape (after one hot encoding): ", X_test.shape)

    # clf_lr predictions
    lr_pred = clf_lr.predict(X_test)
    lr_Happy = pd.DataFrame({'Happy': lr_pred})
    predsLR = pd.concat([UserID, lr_Happy], axis=1)
    predsLR.to_csv('./predictions/lr_predictions.csv', index=False)

    # clf_nb predictions
    nb_pred = clf_nb.predict(X_test)
    nb_Happy = pd.DataFrame({'Happy': nb_pred})
    predsLR = pd.concat([UserID, nb_Happy], axis=1)
    predsLR.to_csv('./predictions/nb_predictions.csv', index=False)

    # rf predictions
    rf_pred = rf_model.predict(X_test)
    rf_Happy = pd.DataFrame({'Happy': rf_pred})
    predsLR = pd.concat([UserID, rf_Happy], axis=1)
    predsLR.to_csv('./predictions/rf_predictions.csv', index=False)

    # svm predictions
    svm_pred = svm_model.predict(X_test)
    svm_Happy = pd.DataFrame({'Happy': svm_pred})
    predsLR = pd.concat([UserID, svm_Happy], axis=1)
    predsLR.to_csv('./predictions/svm_predictions.csv', index=False)
    print("Prediction saved.")
Ejemplo n.º 12
0
class DBN(object):
    def __init__(self,
                 numpy_rng=None,
                 theano_rng=None,
                 n_ins=784,
                 hidden_layers_sizes=[500, 500],
                 n_outs=10):
        self.sigmoid_layers = []
        self.rbm_layers = []
        self.params = []
        self.n_layers = len(hidden_layers_sizes)

        assert self.n_layers > 0

        if not theano_rng:
            theano_rng = MRG_RandomStreams(numpy_rng.randint(2**30))
        self.x = T.matrix('x')
        self.y = T.ivector('y')

        for i in range(self.n_layers):
            if i == 0:
                input_size = n_ins
            else:
                input_size = hidden_layers_sizes[i - 1]

            if i == 0:
                layer_input = self.x
            else:
                layer_input = self.sigmoid_layers[-1].output
            sigmoid_layer = HiddenLayer(rng=numpy_rng,
                                        input=layer_input,
                                        n_in=input_size,
                                        n_out=hidden_layers_sizes[i],
                                        activation=T.nnet.sigmoid)
            self.sigmoid_layers.append(sigmoid_layer)
            self.params.extend(sigmoid_layer.params)
            rbm_layer = RBM(numpy_rng=numpy_rng,
                            theano_rng=theano_rng,
                            input=layer_input,
                            n_visible=input_size,
                            n_hidden=hidden_layers_sizes[i],
                            W=sigmoid_layer.W,
                            hbias=sigmoid_layer.b)
            self.rbm_layers.append(rbm_layer)
        self.logLayer = LogisticRegression(
            input=self.sigmoid_layers[-1].output,
            n_in=hidden_layers_sizes[-1],
            n_out=n_outs)
        self.params.extend(self.logLayer.params)
        self.finetune_cost = self.logLayer.negative_log_likelihood(self.y)
        self.errors = self.logLayer.error(self.y)

    def pretraining_functions(self, train_set_x, batch_size, k):
        index = T.lscalar('index')
        learning_rate = T.scalar('lr')
        batch_begin = index * batch_size
        batch_end = batch_begin + batch_size
        pretrain_fns = []
        for rbm in self.rbm_layers:
            cost, updates = rbm.get_cost_updates(learning_rate,
                                                 persistent=None,
                                                 k=k)
            fn = theano.function(
                inputs=[index, theano.In(learning_rate, value=0.1)],
                outputs=cost,
                updates=updates,
                givens={self.x: train_set_x[batch_begin:batch_end]})
            pretrain_fns.append(fn)
        return pretrain_fns

    def build_finetune_functions(self, datasets, batch_size, learning_rate):
        (train_set_x, train_set_y) = datasets[0]
        (valid_set_x, valid_set_y) = datasets[1]
        (test_set_x, test_set_y) = datasets[2]
        n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]
        n_valid_batches //= batch_size
        n_test_batches = test_set_x.get_value(borrow=True).shape[0]
        n_test_batches //= batch_size

        index = T.lscalar('index')

        gparams = T.grad(self.finetune_cost, self.params)

        updates = []
        for param, gparam in zip(self.params, gparams):
            updates.append((param, param - gparam * learning_rate))

        train_fn = theano.function(
            inputs=[index],
            outputs=self.finetune_cost,
            updates=updates,
            givens={
                self.x:
                train_set_x[index * batch_size:(index + 1) * batch_size],
                self.y:
                train_set_y[index * batch_size:(index + 1) * batch_size]
            })

        test_score_i = theano.function(
            [index],
            self.errors,
            givens={
                self.x:
                test_set_x[index * batch_size:(index + 1) * batch_size],
                self.y: test_set_y[index * batch_size:(index + 1) * batch_size]
            })

        valid_score_i = theano.function(
            [index],
            self.errors,
            givens={
                self.x:
                valid_set_x[index * batch_size:(index + 1) * batch_size],
                self.y:
                valid_set_y[index * batch_size:(index + 1) * batch_size]
            })

        def valid_score():
            return [valid_score_i(i) for i in range(n_valid_batches)]

        def test_score():
            return [test_score_i(i) for i in range(n_test_batches)]

        return train_fn, valid_score, test_score
Ejemplo n.º 13
0
def evaluate_lenet5(learning_rate=0.1,
                    n_epochs=200,
                    dataset='mnist.pkl.gz',
                    nkerns=[20, 50],
                    batch_size=500):
    rng = numpy.random.RandomState(23455)

    datasets = load_dataset(dataset)

    train_set_x, train_set_y = datasets[0]
    valid_set_x, valid_set_y = datasets[1]
    test_set_x, test_set_y = datasets[2]

    # compute number of minibatches for training, validation and testing
    n_train_batches = train_set_x.get_value(borrow=True).shape[0]
    n_valid_batches = valid_set_x.get_value(borrow=True).shape[0]
    n_test_batches = test_set_x.get_value(borrow=True).shape[0]
    n_train_batches //= batch_size
    n_valid_batches //= batch_size
    n_test_batches //= batch_size

    index = T.lscalar()
    x = T.matrix('x')
    y = T.ivector('y')

    print '...... building the model ......'
    layer0_input = x.reshape((batch_size, 1, 28, 28))
    layer0 = LeNetConvPoolLayer(rng=rng,
                                input=layer0_input,
                                image_shape=(batch_size, 1, 28, 28),
                                filter_shape=(nkerns[0], 1, 5, 5),
                                poolsize=(2, 2))
    layer1 = LeNetConvPoolLayer(rng,
                                input=layer0.output,
                                image_shape=(batch_size, nkerns[0], 12, 12),
                                filter_shape=(nkerns[1], nkerns[0], 5, 5),
                                poolsize=(2, 2))
    layer2_input = layer1.output.flatten(2)
    layer2 = HiddenLayer(rng=rng,
                         input=layer2_input,
                         n_in=nkerns[1] * 4 * 4,
                         n_out=500,
                         activation=T.tanh)
    layer3 = LogisticRegression(input=layer2.output, n_in=500, n_out=10)
    cost = layer3.negative_log_likelihood(y)

    test_model = theano.function(
        [index],
        layer3.error(y),
        givens={
            x: test_set_x[index * batch_size:(index + 1) * batch_size],
            y: test_set_y[index * batch_size:(index + 1) * batch_size]
        })
    validate_model = theano.function(
        [index],
        layer3.error(y),
        givens={
            x: valid_set_x[index * batch_size:(index + 1) * batch_size],
            y: valid_set_y[index * batch_size:(index + 1) * batch_size]
        })
    params = layer3.params + layer2.params + layer1.params + layer0.params
    grads = T.grad(cost, params)
    updates = [(param_i, param_i - learning_rate * grad_i)
               for param_i, grad_i in zip(params, grads)]
    train_model = theano.function(
        [index],
        cost,
        updates=updates,
        givens={
            x: train_set_x[index * batch_size:(index + 1) * batch_size],
            y: train_set_y[index * batch_size:(index + 1) * batch_size]
        })

    print '...... training ......'

    patience = 10000
    patience_increase = 2
    improvement_threshold = 0.995
    validation_frequency = min(n_train_batches, patience // 2)

    best_validation_loss = numpy.inf
    best_iter = 0
    test_score = 0.
    start_time = timeit.default_timer()

    epoch = 0
    done_looping = False

    while (epoch < n_epochs) and (not done_looping):
        epoch = epoch + 1
        for minibatch_index in range(n_train_batches):

            iter = (epoch - 1) * n_train_batches + minibatch_index

            if iter % 100 == 0:
                print('training @ iter = ', iter)
            cost_ij = train_model(minibatch_index)

            if (iter + 1) % validation_frequency == 0:
                validation_losses = [
                    validate_model(i) for i in range(n_valid_batches)
                ]
                this_validation_loss = numpy.mean(validation_losses)
                print('epoch %i, minibatch %i/%i, validation error %f %%' %
                      (epoch, minibatch_index + 1, n_train_batches,
                       this_validation_loss * 100.))
                if this_validation_loss < best_validation_loss:
                    if this_validation_loss < best_validation_loss *  \
                       improvement_threshold:
                        patience = max(patience, iter * patience_increase)
                    best_validation_loss = this_validation_loss
                    best_iter = iter
                    test_losses = [
                        test_model(i) for i in range(n_test_batches)
                    ]
                    test_score = numpy.mean(test_losses)
                    print(('     epoch %i, minibatch %i/%i, test error of '
                           'best model %f %%') %
                          (epoch, minibatch_index + 1, n_train_batches,
                           test_score * 100.))

            if patience <= iter:
                done_looping = True
                break

    end_time = timeit.default_timer()
    print('Optimization complete.')
    print(
        'Best validation score of %f %% obtained at iteration %i, '
        'with test performance %f %%' %
        (best_validation_loss * 100., best_iter + 1, test_score * 100.))
    print >> sys.stderr, ('The code for file ' + os.path.split(__file__)[1] +
                          ' ran for %.1fs' % ((end_time - start_time)))
Ejemplo n.º 14
0
best_label = None
for beta in initial_beta:
    tmp_rate = 0.
    for i, test_X in enumerate(k_X_set):
        test_y = k_y_set[i]
        #generate train_X
        train_X = []
        train_y = []
        for j in xrange(len(k_X_set)):
            if j == i:
                continue
            train_X.append(k_X_set[j])
            train_y.append(k_y_set[j])
        train_X = np.vstack(train_X)
        train_y = np.hstack(train_y)
        lr = LogisticRegression(beta=beta)
        lr.fit(train_X, train_y)
        exp_y = lr.predict(test_X)
        tmp_rate += 1.0 - (exp_y ^ test_y).sum() / float(len(test_y))
    tmp_rate /= cv_k
    if tmp_rate > best_rate:
        best_initial_beta = beta
        best_rate = tmp_rate

print best_initial_beta
print best_rate

lr = LogisticRegression(beta=best_initial_beta)
X = []
y = []
for i in xrange(cv_k - 1):
Ejemplo n.º 15
0
# -*- coding: utf-8 -*-
"""
LogisticRegression1 test runner

Chapter "Logistic Regression"
"""

# imports
import numpy as np
from lr import LogisticRegression

# load data
data = np.genfromtxt('iris2.tsv', dtype=[('X', float, 4), ('y', int)])

# learn model
clr = LogisticRegression()
clr.fit(data['X'], data['y'])

# test model
predict_y = clr.predict(data['X'])

# print results
for i in range(0, 100, 10):
    print(i, data['y'][i], predict_y[i])

# print accuracy
print("Accuracy =", end=' ')
print(np.sum(data['y'] == predict_y, dtype=float) / predict_y.shape[0])

# print parameters
print("coef =", clr.coef_)
Ejemplo n.º 16
0
def compare_penalty():

    penalties = (-8, -6, -4, -2, 0, 2)

    # Instantiate the Logistic Regression model with no regularization
    model_unreg = LogisticRegression(verbose=True)

    # Fit the model.
    model_unreg.fit(x_tr, y_tr, lr=10**(-4), maxit=ITERATIONS, tolerance=None)

    # Predict results
    pred_tr_unreg = model_unreg.predict(x_tr)
    pred_ts_unreg = model_unreg.predict(x_ts)

    # Evaluate
    train_scores = BinaryScorer(y_tr,
                                pred_tr_unreg,
                                description='Training',
                                positive_class=1,
                                negative_class=0)
    test_scores = BinaryScorer(y_ts,
                               pred_ts_unreg,
                               description='Testing',
                               positive_class=1,
                               negative_class=0)

    accuracy_tr_unreg = train_scores.accuracy()
    accuracy_ts_unreg = test_scores.accuracy()

    accuracies_tr_reg = []
    accuracies_ts_reg = []

    # Evaluate and plot the model with different learning rates
    for alpha in penalties:

        # Instantiate the Logistic Regression model with regularization
        model_reg = LogisticRegression(l2penalty=2**alpha, verbose=True)

        # Fit the model.
        model_reg.fit(x_tr,
                      y_tr,
                      lr=10**(-4),
                      maxit=ITERATIONS,
                      tolerance=None)

        # Predict results
        pred_tr_reg = model_reg.predict(x_tr)
        pred_ts_reg = model_reg.predict(x_ts)

        # Evaluate
        train_scores = BinaryScorer(y_tr,
                                    pred_tr_reg,
                                    description='Training',
                                    positive_class=1,
                                    negative_class=0)
        test_scores = BinaryScorer(y_ts,
                                   pred_ts_reg,
                                   description='Testing',
                                   positive_class=1,
                                   negative_class=0)

        accuracies_tr_reg.append(train_scores.accuracy())
        accuracies_ts_reg.append(test_scores.accuracy())

    fig = plt.figure()

    plt.plot(penalties,
             accuracies_tr_reg,
             'o-',
             label='Train w/ reg',
             color='blue')
    plt.plot(penalties,
             accuracies_ts_reg,
             'o-',
             label='Test w/ reg',
             color='purple')
    plt.plot([penalties[0], penalties[len(penalties) - 1]],
             [accuracy_tr_unreg, accuracy_tr_unreg],
             '-',
             label='Train w/o reg',
             color='red')
    plt.plot([penalties[0], penalties[len(penalties) - 1]],
             [accuracy_ts_unreg, accuracy_ts_unreg],
             '-',
             label='Test w/o reg',
             color='orange')

    plt.title('Regularization comparison, lr = 10^-4')
    plt.ylabel('Accuracy')
    plt.xlabel('Values of k, with alpha = 2^k')
    plt.grid(True, linestyle=':')
    plt.legend(loc="upper right")

    plt.show()
    fig.savefig('logistic_penalty_comp.png', dpi=300)
Ejemplo n.º 17
0
import numpy as np
from util import read_matlab, trans_label_to_onehot, calc_precision
from plot import show_img, show_100img
from lr import LogisticRegression, load_lr
from NeuralNetwork import NeuralNetwork

if __name__ == '__main__':
    # Read data from mat file.
    data = read_matlab('./ex3/ex3data1.mat')
    X = data['X']
    label = data['y']
    # Trans the label to onehot
    label_onehot = trans_label_to_onehot(label).transpose()
    # Show img
    show_100img(X)
    lr = LogisticRegression(400, 10)
    # This method uses the lib function, if you want to use self create train method, use function LogisticRegression.train
    lr.lib_train(X, label_onehot, 1)

    # You can use save and load function to save model file
    # lr.save_model('./model/lr.model')
    # lr = load_lr('./model/lr.model')
    ans = lr.predict_class(X.transpose())
    print(
        '\033[1;32mThe accuracy of Logistsic regression is {}%.\033[0m'.format(
            calc_precision(label, ans)))

    # Neural network
    nn_weight = read_matlab('./ex3/ex3weights.mat')
    theta1 = nn_weight['Theta1']
    theta2 = nn_weight['Theta2']