def run_models(settings = None):
    analysis_scr = []
    with_auc_score = settings['with_auc_score']
    f = gzip.open('mnist.pkl.gz', 'rb')
    train_set, valid_set, test_set = cPickle.load(f)
    X_train,y_train = train_set
    X_valid,y_valid = valid_set
    X_test,y_test = test_set
    n_outs = settings['n_outs']
    for subset_no in xrange(1,settings['number_iterations']+1):
                print("Subset:", subset_no)

                #(train_X_10fold, train_y_10fold),(train_X_reduced, train_y_reduced), (test_X, test_y) = (X_train[:1000],y_train[:1000]),(X_train[:1000],y_train[:1000]), (X_test[:1000],y_test[:1000])
                X_train,y_train = train_set
                X_valid,y_valid = valid_set
                X_test,y_test = test_set
                (train_X_10fold, train_y_10fold),(train_X_reduced, train_y_reduced), (test_X, test_y) = (X_train,y_train),(X_train,y_train), (X_test,y_test)
                standard_scaler = preprocessing.StandardScaler().fit(train_X_reduced)
                scaled_train_X = standard_scaler.transform(train_X_reduced)
                scaled_test_X = standard_scaler.transform(test_X)
                fisher_mode = settings['fisher_mode']
                
                if settings['SVM']:
                    print "SVM"                   
                    Linear_SVC = LinearSVC(C=1, penalty="l2")
                    Linear_SVC.fit(scaled_train_X, train_y_reduced)
                    predicted_test_y = Linear_SVC.predict(scaled_test_X)
                    isTest = True; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'SVM', isTest) + tuple(performance_score(test_y, predicted_test_y).values())) #new

                    predicted_train_y = Linear_SVC.predict(scaled_train_X)
                    isTest = False; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'SVM', isTest) + tuple(performance_score(train_y_reduced, predicted_train_y).values()))                    
                if settings['SVM_RBF']:
                    print "SVM_RBF"
                    L1_SVC_RBF_Selector = SVC(C=1, gamma=0.01, kernel='rbf').fit(scaled_train_X, train_y_reduced)

                    predicted_test_y = L1_SVC_RBF_Selector.predict(scaled_test_X)
                    isTest = True; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'SVM_RBF', isTest) + tuple(performance_score(test_y, predicted_test_y).values())) #new

                    predicted_train_y = L1_SVC_RBF_Selector.predict(scaled_train_X)
                    isTest = False; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'SVM_RBF', isTest) + tuple(performance_score(train_y_reduced, predicted_train_y).values()))
                if settings['SVM_POLY']:
                    print "SVM_POLY"
                    L1_SVC_POLY_Selector = SVC(C=1, kernel='poly').fit(scaled_train_X, train_y_reduced)

                    predicted_test_y = L1_SVC_POLY_Selector.predict(scaled_test_X)
                    isTest = True; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'SVM_POLY', isTest) + tuple(performance_score(test_y, predicted_test_y).values())) #new

                    predicted_train_y = L1_SVC_POLY_Selector.predict(scaled_train_X)
                    isTest = False; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'SVM_POLY', isTest) + tuple(performance_score(train_y_reduced, predicted_train_y).values()))
                
                min_max_scaler = Preprocessing_Scaler_with_mean_point5()
                X_train_pre_validation_minmax = min_max_scaler.fit(train_X_reduced)
                X_train_pre_validation_minmax = min_max_scaler.transform(train_X_reduced)
                x_test_minmax = min_max_scaler.transform(test_X)
                
                x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax = train_test_split(X_train_pre_validation_minmax, 
                                                                                                  train_y_reduced
                                                                    , test_size=0.4, random_state=42)
                finetune_lr = settings['finetune_lr']
                batch_size = settings['batch_size']
                pretraining_epochs = cal_epochs(settings['pretraining_interations'], x_train_minmax, batch_size = batch_size)
                #pretrain_lr=0.001
                pretrain_lr = settings['pretrain_lr']
                training_epochs = cal_epochs(settings['training_interations'], x_train_minmax, batch_size = batch_size)
                settings['lrate'] = settings['lrate_pre'] + str(training_epochs)
                hidden_layers_sizes= settings['hidden_layers_sizes']
                corruption_levels = settings['corruption_levels']
                settings['epoch_number'] = cal_epochs(settings['pretraining_interations'], x_train_minmax, batch_size = batch_size)
                # deep xy autoencoders
                settings['n_ins'] = x_train_minmax.shape[1]
                if settings['DL_xy']:
                    cfg = settings.copy()
                    cfg['weight_y'] = 100
                    print 'DL_xy'
                    train_x = x_train_minmax; train_y = y_train_minmax                    
                    sdaf = Sda_xy_factory(cfg)
                    sdaf.sda.pretraining(train_x, train_y) 
                    dnnf = DNN_factory(cfg) 
                    dnnf.dnn.load_pretrain_from_Sda(sdaf.sda)
                    dnnf.dnn.finetuning((x_train_minmax,  y_train_minmax),(x_validation_minmax, y_validation_minmax))
                    
                    training_predicted = dnnf.dnn.predict(x_train_minmax)
                    y_train = y_train_minmax
                    isTest = False; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'DL_xy', isTest) + tuple(performance_score(y_train, training_predicted).values()))

                    test_predicted = dnnf.dnn.predict(x_test_minmax)
                    y_test = test_y
                    isTest = True; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'DL_xy', isTest) + tuple(performance_score(y_test, test_predicted).values()))
                if settings['Sda_xy_with_first']: 
                    cfg = settings.copy()
                    cfg['weight_y'] = 10
                    cfg['firstlayer_xy'] = 1
                    print 'firstlayer_xy' 
                    train_x = x_train_minmax; train_y = y_train_minmax                    
                    sdaf = Sda_xy_factory(cfg)
                    sdaf.sda.pretraining(train_x, train_y) 
                    dnnf = DNN_factory(cfg) 
                    dnnf.dnn.load_pretrain_from_Sda(sdaf.sda)
                    dnnf.dnn.finetuning((x_train_minmax,  y_train_minmax),(x_validation_minmax, y_validation_minmax))
                    
                    training_predicted = dnnf.dnn.predict(x_train_minmax)
                    y_train = y_train_minmax
                    isTest = False; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'Sda_xy_with_first', isTest) + tuple(performance_score(y_train, training_predicted).values()))

                    test_predicted = dnnf.dnn.predict(x_test_minmax)
                    y_test = test_y
                    isTest = True; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'Sda_xy_with_first', isTest) + tuple(performance_score(y_test, test_predicted).values()))
                if settings['Sda_new']:
                    print 'Sda_new'
                    cfg = settings.copy()
                    train_x = x_train_minmax; train_y = y_train_minmax                    
                    cfg['n_ins'] = train_x.shape[1]
                    sdaf = Sda_factory(cfg)
                    sda = sdaf.sda.pretraining(train_x = train_x)
                    sdaf.dnn.finetuning((x_train_minmax,  y_train_minmax),(x_validation_minmax, y_validation_minmax))                    
                    training_predicted = sdaf.dnn.predict(x_train_minmax)
                    y_train = y_train_minmax
                    isTest = False; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'Sda_new', isTest) + tuple(performance_score(y_train, training_predicted).values()))

                    test_predicted = sdaf.dnn.predict(x_test_minmax)
                    y_test = test_y
                    isTest = True; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'Sda_new', isTest) + tuple(performance_score(y_test, test_predicted).values()))
                            
                #### new prepresentation
                x = X_train_pre_validation_minmax
                a_MAE_A = pretrain_a_Sda_with_estop(x, pretraining_epochs=pretraining_epochs, pretrain_lr=pretrain_lr, batch_size=batch_size, 
                                        hidden_layers_sizes =hidden_layers_sizes, corruption_levels=corruption_levels, n_outs = n_outs)
                new_x_train_minmax_A =  a_MAE_A.transform(X_train_pre_validation_minmax)
                new_x_test_minmax_A =  a_MAE_A.transform(x_test_minmax)
                standard_scaler = preprocessing.StandardScaler().fit(new_x_train_minmax_A)
                new_x_train_scaled = standard_scaler.transform(new_x_train_minmax_A)
                new_x_test_scaled = standard_scaler.transform(new_x_test_minmax_A)
                new_x_train_combo = np.hstack((scaled_train_X, new_x_train_scaled))
                new_x_test_combo = np.hstack((scaled_test_X, new_x_test_scaled))
                
                
                if settings['SAE_SVM']: 
                    print 'SAE followed by SVM'

                    Linear_SVC = LinearSVC(C=1, penalty="l2")
                    Linear_SVC.fit(new_x_train_scaled, train_y_reduced)
                    predicted_test_y = Linear_SVC.predict(new_x_test_scaled)
                    isTest = True; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'SAE_SVM', isTest) + tuple(performance_score(test_y, predicted_test_y).values())) #new
                    predicted_train_y = Linear_SVC.predict(new_x_train_scaled)
                    isTest = False; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'SAE_SVM', isTest) + tuple(performance_score(train_y_reduced, predicted_train_y).values()))
                if settings['SAE_SVM_RBF']: 
                    print 'SAE followed by SVM RBF'
                    x = X_train_pre_validation_minmax
                    L1_SVC_RBF_Selector = SVC(C=1, gamma=0.01, kernel='rbf').fit(new_x_train_scaled, train_y_reduced)
                    predicted_test_y = L1_SVC_RBF_Selector.predict(new_x_test_scaled)
                    isTest = True; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'SAE_SVM_RBF', isTest) + tuple(performance_score(test_y, predicted_test_y).values())) #new
                    predicted_train_y = L1_SVC_RBF_Selector.predict(new_x_train_scaled)
                    isTest = False; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'SAE_SVM_RBF', isTest) + tuple(performance_score(train_y_reduced, predicted_train_y).values()))
                if settings['SAE_SVM_COMBO']: 
                    print 'SAE followed by SVM with combo feature'
                    Linear_SVC = LinearSVC(C=1, penalty="l2")
                    Linear_SVC.fit(new_x_train_combo, train_y_reduced)
                    predicted_test_y = Linear_SVC.predict(new_x_test_combo)
                    isTest = True; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'SAE_SVM_COMBO', isTest) + tuple(performance_score(test_y, predicted_test_y).values())) #new
                    predicted_train_y = Linear_SVC.predict(new_x_train_combo)
                    isTest = False; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'SAE_SVM_COMBO', isTest) + tuple(performance_score(train_y_reduced, predicted_train_y).values()))                                
                if settings['SAE_SVM_RBF_COMBO']: 
                    print 'SAE followed by SVM RBF with combo feature'
                    L1_SVC_RBF_Selector = SVC(C=1, gamma=0.01, kernel='rbf').fit(new_x_train_combo, train_y_reduced)
                    predicted_test_y = L1_SVC_RBF_Selector.predict(new_x_test_combo)        
                    isTest = True; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'SAE_SVM_RBF_COMBO', isTest) + tuple(performance_score(test_y, predicted_test_y).values())) #new
                    predicted_train_y = L1_SVC_RBF_Selector.predict(new_x_train_combo)
                    isTest = False; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'SAE_SVM_RBF_COMBO', isTest) + tuple(performance_score(train_y_reduced, predicted_train_y).values()))                                                                  
                    
                if settings['DL']:
                    print "direct deep learning"
                    sda = train_a_Sda(x_train_minmax, pretrain_lr, finetune_lr,
                                      y_train_minmax,
                                 x_validation_minmax, y_validation_minmax , 
                                 x_test_minmax, test_y,
                                 hidden_layers_sizes = hidden_layers_sizes, corruption_levels = corruption_levels, batch_size = batch_size , \
                                 training_epochs = training_epochs, pretraining_epochs = pretraining_epochs, n_outs =n_outs
                                 
                     )
                    print 'hidden_layers_sizes:', hidden_layers_sizes
                    print 'corruption_levels:', corruption_levels
                    training_predicted = sda.predict(x_train_minmax)
                    y_train = y_train_minmax
                    isTest = False; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'DL', isTest) + tuple(performance_score(y_train, training_predicted).values()))

                    test_predicted = sda.predict(x_test_minmax)
                    y_test = test_y
                    isTest = True; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'DL', isTest) + tuple(performance_score(y_test, test_predicted).values()))
                
                if settings['DL_U']:
                # deep learning using unlabeled data for pretraining
                    print 'deep learning with unlabel data'
                    pretraining_X_minmax = min_max_scaler.transform(train_X_10fold)
                    pretraining_epochs = cal_epochs(settings['pretraining_interations'], x_train_minmax, batch_size = batch_size)
                    sda_unlabel = trainSda(x_train_minmax, y_train_minmax,
                                 x_validation_minmax, y_validation_minmax , 
                                 x_test_minmax, test_y, 
                                 pretraining_X_minmax = pretraining_X_minmax,
                                 hidden_layers_sizes = hidden_layers_sizes, corruption_levels = corruption_levels, batch_size = batch_size , \
                                 training_epochs = training_epochs, pretraining_epochs = pretraining_epochs, 
                                 pretrain_lr = pretrain_lr, finetune_lr=finetune_lr, n_outs =n_outs
                     )
                    print 'hidden_layers_sizes:', hidden_layers_sizes
                    print 'corruption_levels:', corruption_levels
                    training_predicted = sda_unlabel.predict(x_train_minmax)
                    y_train = y_train_minmax
                    isTest = False; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'DL_U', isTest) + tuple(performance_score(y_train, training_predicted, with_auc_score).values()))

                    test_predicted = sda_unlabel.predict(x_test_minmax)
                    y_test = test_y

                    isTest = True; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'DL_U', isTest) + tuple(performance_score(y_test, test_predicted, with_auc_score).values()))
                if settings['DL_S']:
                    # deep learning using split network
                    y_test = test_y
                    print 'deep learning using split network'
                    # get the new representation for A set. first 784-D
                    pretraining_epochs = cal_epochs(settings['pretraining_interations'], x_train_minmax, batch_size = batch_size)
                    
                    x = x_train_minmax[:, :x_train_minmax.shape[1]/2]
                    print "original shape for A", x.shape
                    a_MAE_A = pretrain_a_Sda_with_estop(x, pretraining_epochs=pretraining_epochs, pretrain_lr=pretrain_lr, batch_size=batch_size, 
                                            hidden_layers_sizes =hidden_layers_sizes, corruption_levels=corruption_levels, n_outs = n_outs)
                    new_x_train_minmax_A =  a_MAE_A.transform(x_train_minmax[:, :x_train_minmax.shape[1]/2])
                    x = x_train_minmax[:, x_train_minmax.shape[1]/2:]
                    
                    print "original shape for B", x.shape
                    a_MAE_B = pretrain_a_Sda_with_estop(x, pretraining_epochs=pretraining_epochs, pretrain_lr=pretrain_lr, batch_size=batch_size, 
                                            hidden_layers_sizes =hidden_layers_sizes, corruption_levels=corruption_levels, n_outs = n_outs)
                    new_x_train_minmax_B =  a_MAE_B.transform(x_train_minmax[:, x_train_minmax.shape[1]/2:])
                    
                    new_x_test_minmax_A = a_MAE_A.transform(x_test_minmax[:, :x_test_minmax.shape[1]/2])
                    new_x_test_minmax_B = a_MAE_B.transform(x_test_minmax[:, x_test_minmax.shape[1]/2:])
                    new_x_validation_minmax_A = a_MAE_A.transform(x_validation_minmax[:, :x_validation_minmax.shape[1]/2])
                    new_x_validation_minmax_B = a_MAE_B.transform(x_validation_minmax[:, x_validation_minmax.shape[1]/2:])
                    new_x_train_minmax_whole = np.hstack((new_x_train_minmax_A, new_x_train_minmax_B))
                    new_x_test_minmax_whole = np.hstack((new_x_test_minmax_A, new_x_test_minmax_B))
                    new_x_validationt_minmax_whole = np.hstack((new_x_validation_minmax_A, new_x_validation_minmax_B))

                    
                    sda_transformed = train_a_Sda(new_x_train_minmax_whole, pretrain_lr, finetune_lr,
                                                  y_train_minmax,
                         new_x_validationt_minmax_whole, y_validation_minmax , 
                         new_x_test_minmax_whole, y_test,
                         hidden_layers_sizes = hidden_layers_sizes, corruption_levels = corruption_levels, batch_size = batch_size , \
                         training_epochs = training_epochs, pretraining_epochs = pretraining_epochs, n_outs = n_outs
                         
                         )
                    
                    print 'hidden_layers_sizes:', hidden_layers_sizes
                    print 'corruption_levels:', corruption_levels
                    training_predicted = sda_transformed.predict(new_x_train_minmax_whole)
                    y_train = y_train_minmax
                    
                    isTest = False; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'DL_S', isTest) + tuple(performance_score(y_train, training_predicted, with_auc_score).values()))

                    test_predicted = sda_transformed.predict(new_x_test_minmax_whole)
                    y_test = test_y

                    isTest = True; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'DL_S', isTest) + tuple(performance_score(y_test, test_predicted, with_auc_score).values()))
                if settings['DL_S_new']:
                    # deep learning using split network
                    print 'new deep learning using split network'

                    cfg = settings.copy()
                    p_sda = Parellel_Sda_factory(cfg)                    
                    p_sda.supervised_training(x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax)
                    
                    isTest = False #new
                    training_predicted = p_sda.predict(x_train_minmax)
                    y_train = y_train_minmax                                       
                    analysis_scr.append(( subset_no, fisher_mode, 'DL_S_new', isTest) + tuple(performance_score(y_train, training_predicted, with_auc_score).values()))
                    
                    isTest = True #new
                    y_test = test_y
                    test_predicted = p_sda.predict(x_test_minmax)
                    analysis_scr.append(( subset_no, fisher_mode, 'DL_S_new', isTest) + tuple(performance_score(y_test, test_predicted, with_auc_score).values()))            
                if settings['DL_S_new_contraction']:
                    print 'DL_S_new_contraction'
                    cfg = settings.copy()
                    cfg['contraction_level'] = 0.1
                    p_sda = Parellel_Sda_factory(cfg)                    
                    p_sda.supervised_training(x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax)
                    
                    isTest = False #new
                    training_predicted = p_sda.predict(x_train_minmax)
                    y_train = y_train_minmax                                       
                    analysis_scr.append(( subset_no, fisher_mode, 'DL_S_new_contraction', isTest) + tuple(performance_score(y_train, training_predicted, with_auc_score).values()))
                    
                    isTest = True #new
                    y_test = test_y
                    test_predicted = p_sda.predict(x_test_minmax)
                    analysis_scr.append(( subset_no, fisher_mode, 'DL_S_new_contraction', isTest) + tuple(performance_score(y_test, test_predicted, with_auc_score).values()))            
               
                if settings['DL_S_new_sparsity'] == 1:
                    print 'DL_S_new_sparsity'
                    cfg = settings.copy()
                    cfg['sparsity'] = 0.01
                    cfg['sparsity_weight'] = 0.01
                    p_sda = Parellel_Sda_factory(cfg)                    
                    p_sda.supervised_training(x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax)
                    
                    isTest = False #new
                    training_predicted = p_sda.predict(x_train_minmax)
                    y_train = y_train_minmax                                       
                    analysis_scr.append(( subset_no, fisher_mode, 'DL_S_new_sparsity', isTest) + tuple(performance_score(y_train, training_predicted, with_auc_score).values()))
                    
                    isTest = True #new
                    y_test = test_y
                    test_predicted = p_sda.predict(x_test_minmax)
                    analysis_scr.append(( subset_no, fisher_mode, 'DL_S_new_sparsity', isTest) + tuple(performance_score(y_test, test_predicted, with_auc_score).values()))            
               
                if settings['DL_S_new_weight_decay'] == 2:
                    cfg = settings.copy()
                    cfg['l2_reg'] =0.01
                    print 'l2_reg'
                    p_sda = Parellel_Sda_factory(cfg)                    
                    p_sda.supervised_training(x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax)
                    
                    isTest = False #new
                    training_predicted = p_sda.predict(x_train_minmax)
                    y_train = y_train_minmax                                       
                    analysis_scr.append(( subset_no, fisher_mode, 'l2_reg', isTest) + tuple(performance_score(y_train, training_predicted, with_auc_score).values()))
                    
                    isTest = True #new
                    y_test = test_y
                    test_predicted = p_sda.predict(x_test_minmax)
                    analysis_scr.append(( subset_no, fisher_mode, 'l2_reg', isTest) + tuple(performance_score(y_test, test_predicted, with_auc_score).values()))            
                    
                if settings['DL_S_new_weight_decay'] == 1:
                    print 'l1_reg'
                    cfg = settings.copy()
                    cfg['l1_reg'] =0.01 
                    p_sda = Parellel_Sda_factory(cfg)                    
                    p_sda.supervised_training(x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax)
                    
                    isTest = False #new
                    training_predicted = p_sda.predict(x_train_minmax)
                    y_train = y_train_minmax                                       
                    analysis_scr.append(( subset_no, fisher_mode, 'l1_reg', isTest) + tuple(performance_score(y_train, training_predicted, with_auc_score).values()))
                    
                    isTest = True #new
                    y_test = test_y
                    test_predicted = p_sda.predict(x_test_minmax)
                    analysis_scr.append(( subset_no, fisher_mode, 'l1_reg', isTest) + tuple(performance_score(y_test, test_predicted, with_auc_score).values()))            
                                     
                if settings['DL_S_new_Drop_out'] == 1:
                    
                    cfg = settings.copy()
                    cfg['dropout_factor'] = 0.5
                    print 'DL_S_new_Drop_out'
                    p_sda = Parellel_Sda_factory(cfg)                    
                    p_sda.supervised_training(x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax)
                    
                    isTest = False #new
                    training_predicted = p_sda.predict(x_train_minmax)
                    y_train = y_train_minmax                                       
                    analysis_scr.append(( subset_no, fisher_mode, 'DL_S_new_Drop_out', isTest) + tuple(performance_score(y_train, training_predicted, with_auc_score).values()))
                    
                    isTest = True #new
                    y_test = test_y
                    test_predicted = p_sda.predict(x_test_minmax)
                    analysis_scr.append(( subset_no, fisher_mode, 'DL_S_new_Drop_out', isTest) + tuple(performance_score(y_test, test_predicted, with_auc_score).values()))            
                                     
                report_name = 'Hand_classification_' + '_'.join(map(str, hidden_layers_sizes)) + '_' + str(pretrain_lr) + '_' + str(finetune_lr) + '_' + str(settings['training_interations']) + '_' + current_date
                saveAsCsv(with_auc_score, report_name, performance_score(test_y, predicted_test_y, with_auc_score), analysis_scr)
    def get_ten_fold_crossvalid_perfermance(self, settings=None):
        fisher_mode = settings['fisher_mode']
        analysis_scr = []
        with_auc_score = settings['with_auc_score']
        reduce_ratio = settings['reduce_ratio']
        #for seq_no in range(1, self.ddi_obj.total_number_of_sequences+1):
        #subset_size = math.floor(self.ddi_obj.total_number_of_sequences / 10.0)
        kf = KFold(self.ddi_obj.total_number_of_sequences,
                   n_folds=10,
                   shuffle=True)
        #for subset_no in range(1, 11):
        for ((train_index, test_index), subset_no) in izip(kf, range(1, 11)):
            #for train_index, test_index in kf;
            print("Subset:", subset_no)
            print("Train index: ", train_index)
            print("Test index: ", test_index)
            #logger.info('subset number: ' + str(subset_no))
            (train_X_10fold,
             train_y_10fold), (train_X_reduced, train_y_reduced), (
                 test_X,
                 test_y) = self.ddi_obj.get_ten_fold_crossvalid_one_subset(
                     train_index,
                     test_index,
                     fisher_mode=fisher_mode,
                     reduce_ratio=reduce_ratio)
            standard_scaler = preprocessing.StandardScaler().fit(
                train_X_reduced)
            scaled_train_X = standard_scaler.transform(train_X_reduced)
            scaled_test_X = standard_scaler.transform(test_X)

            if settings['SVM']:
                print "SVM"
                Linear_SVC = LinearSVC(C=1, penalty="l2")
                Linear_SVC.fit(scaled_train_X, train_y_reduced)
                predicted_test_y = Linear_SVC.predict(scaled_test_X)
                isTest = True
                #new
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'SVM', isTest) + tuple(
                        performance_score(test_y,
                                          predicted_test_y).values()))  #new

                predicted_train_y = Linear_SVC.predict(scaled_train_X)
                isTest = False
                #new
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'SVM', isTest) + tuple(
                        performance_score(train_y_reduced,
                                          predicted_train_y).values()))
            if settings['SVM_RBF']:
                print "SVM_RBF"
                L1_SVC_RBF_Selector = SVC(C=1, gamma=0.01, kernel='rbf').fit(
                    scaled_train_X, train_y_reduced)

                predicted_test_y = L1_SVC_RBF_Selector.predict(scaled_test_X)
                isTest = True
                #new
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'SVM_RBF', isTest) +
                    tuple(
                        performance_score(test_y,
                                          predicted_test_y).values()))  #new

                predicted_train_y = L1_SVC_RBF_Selector.predict(scaled_train_X)
                isTest = False
                #new
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'SVM_RBF', isTest) +
                    tuple(
                        performance_score(train_y_reduced,
                                          predicted_train_y).values()))
            if settings['SVM_POLY']:
                print "SVM_POLY"
                L1_SVC_POLY_Selector = SVC(C=1, kernel='poly').fit(
                    scaled_train_X, train_y_reduced)

                predicted_test_y = L1_SVC_POLY_Selector.predict(scaled_test_X)
                isTest = True
                #new
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'SVM_POLY', isTest) +
                    tuple(
                        performance_score(test_y,
                                          predicted_test_y).values()))  #new

                predicted_train_y = L1_SVC_POLY_Selector.predict(
                    scaled_train_X)
                isTest = False
                #new
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'SVM_POLY', isTest) +
                    tuple(
                        performance_score(train_y_reduced,
                                          predicted_train_y).values()))

            min_max_scaler = Preprocessing_Scaler_with_mean_point5()
            X_train_pre_validation_minmax = min_max_scaler.fit(train_X_reduced)
            X_train_pre_validation_minmax = min_max_scaler.transform(
                train_X_reduced)
            x_test_minmax = min_max_scaler.transform(test_X)

            x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax = train_test_split(
                X_train_pre_validation_minmax,
                train_y_reduced,
                test_size=0.4,
                random_state=42)
            finetune_lr = settings['finetune_lr']
            batch_size = settings['batch_size']
            pretraining_epochs = cal_epochs(
                settings['pretraining_interations'],
                x_train_minmax,
                batch_size=batch_size)
            #pretrain_lr=0.001
            pretrain_lr = settings['pretrain_lr']
            training_epochs = cal_epochs(settings['training_interations'],
                                         x_train_minmax,
                                         batch_size=batch_size)
            hidden_layers_sizes = settings['hidden_layers_sizes']
            corruption_levels = settings['corruption_levels']
            settings['epoch_number'] = cal_epochs(
                settings['pretraining_interations'],
                x_train_minmax,
                batch_size=batch_size)
            # deep xy autoencoders
            settings['lrate'] = settings['lrate_pre'] + str(training_epochs)
            settings['n_ins'] = x_train_minmax.shape[1]
            if settings['DL_xy']:
                cfg = settings.copy()
                cfg['weight_y'] = 1
                print 'DL_xy'
                train_x = x_train_minmax
                train_y = y_train_minmax
                sdaf = Sda_xy_factory(cfg)
                sdaf.sda.pretraining(train_x, train_y)
                dnnf = DNN_factory(cfg)
                dnnf.dnn.load_pretrain_from_Sda(sdaf.sda)
                dnnf.dnn.finetuning((x_train_minmax, y_train_minmax),
                                    (x_validation_minmax, y_validation_minmax))

                training_predicted = dnnf.dnn.predict(x_train_minmax)
                y_train = y_train_minmax
                isTest = False
                #new
                analysis_scr.append((
                    self.ddi, subset_no, fisher_mode, 'DL_xy', isTest
                ) + tuple(
                    performance_score(y_train, training_predicted).values()))

                test_predicted = dnnf.dnn.predict(x_test_minmax)
                y_test = test_y
                isTest = True
                #new
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'DL_xy', isTest) +
                    tuple(performance_score(y_test, test_predicted).values()))
            if settings['Sda_xy_with_first']:
                cfg = settings.copy()
                cfg['weight_y'] = 10
                cfg['firstlayer_xy'] = 1
                print 'firstlayer_xy'
                train_x = x_train_minmax
                train_y = y_train_minmax
                sdaf = Sda_xy_factory(cfg)
                sdaf.sda.pretraining(train_x, train_y)
                dnnf = DNN_factory(cfg)
                dnnf.dnn.load_pretrain_from_Sda(sdaf.sda)
                dnnf.dnn.finetuning((x_train_minmax, y_train_minmax),
                                    (x_validation_minmax, y_validation_minmax))

                training_predicted = dnnf.dnn.predict(x_train_minmax)
                y_train = y_train_minmax
                isTest = False
                #new
                analysis_scr.append((
                    self.ddi, subset_no, fisher_mode, 'Sda_xy_with_first',
                    isTest
                ) + tuple(
                    performance_score(y_train, training_predicted).values()))

                test_predicted = dnnf.dnn.predict(x_test_minmax)
                y_test = test_y
                isTest = True
                #new
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'Sda_xy_with_first',
                     isTest) +
                    tuple(performance_score(y_test, test_predicted).values()))
            if settings['Sda_new']:
                print 'Sda_new'
                cfg = settings.copy()
                train_x = x_train_minmax
                train_y = y_train_minmax
                cfg['n_ins'] = train_x.shape[1]
                sdaf = Sda_factory(cfg)
                sda = sdaf.sda.pretraining(train_x=train_x)
                sdaf.dnn.finetuning((x_train_minmax, y_train_minmax),
                                    (x_validation_minmax, y_validation_minmax))
                training_predicted = sdaf.dnn.predict(x_train_minmax)
                y_train = y_train_minmax
                isTest = False
                #new
                analysis_scr.append((
                    self.ddi, subset_no, fisher_mode, 'Sda_new', isTest
                ) + tuple(
                    performance_score(y_train, training_predicted).values()))

                test_predicted = sdaf.dnn.predict(x_test_minmax)
                y_test = test_y
                isTest = True
                #new
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'Sda_new', isTest) +
                    tuple(performance_score(y_test, test_predicted).values()))

            #### new prepresentation
            x = X_train_pre_validation_minmax
            a_MAE_A = pretrain_a_Sda_with_estop(
                x,
                pretraining_epochs=pretraining_epochs,
                pretrain_lr=pretrain_lr,
                batch_size=batch_size,
                hidden_layers_sizes=hidden_layers_sizes,
                corruption_levels=corruption_levels)
            new_x_train_minmax_A = a_MAE_A.transform(
                X_train_pre_validation_minmax)
            new_x_test_minmax_A = a_MAE_A.transform(x_test_minmax)
            standard_scaler = preprocessing.StandardScaler().fit(
                new_x_train_minmax_A)
            new_x_train_scaled = standard_scaler.transform(
                new_x_train_minmax_A)
            new_x_test_scaled = standard_scaler.transform(new_x_test_minmax_A)
            new_x_train_combo = np.hstack((scaled_train_X, new_x_train_scaled))
            new_x_test_combo = np.hstack((scaled_test_X, new_x_test_scaled))

            if settings['SAE_SVM']:
                print 'SAE followed by SVM'

                Linear_SVC = LinearSVC(C=1, penalty="l2")
                Linear_SVC.fit(new_x_train_scaled, train_y_reduced)
                predicted_test_y = Linear_SVC.predict(new_x_test_scaled)
                isTest = True
                #new
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'SAE_SVM', isTest) +
                    tuple(
                        performance_score(test_y,
                                          predicted_test_y).values()))  #new
                predicted_train_y = Linear_SVC.predict(new_x_train_scaled)
                isTest = False
                #new
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'SAE_SVM', isTest) +
                    tuple(
                        performance_score(train_y_reduced,
                                          predicted_train_y).values()))
            if settings['SAE_SVM_RBF']:
                print 'SAE followed by SVM RBF'
                x = X_train_pre_validation_minmax
                L1_SVC_RBF_Selector = SVC(C=1, gamma=0.01, kernel='rbf').fit(
                    new_x_train_scaled, train_y_reduced)
                predicted_test_y = L1_SVC_RBF_Selector.predict(
                    new_x_test_scaled)
                isTest = True
                #new
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'SAE_SVM_RBF', isTest) +
                    tuple(
                        performance_score(test_y,
                                          predicted_test_y).values()))  #new
                predicted_train_y = L1_SVC_RBF_Selector.predict(
                    new_x_train_scaled)
                isTest = False
                #new
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'SAE_SVM_RBF', isTest) +
                    tuple(
                        performance_score(train_y_reduced,
                                          predicted_train_y).values()))
            if settings['SAE_SVM_COMBO']:
                print 'SAE followed by SVM with combo feature'
                Linear_SVC = LinearSVC(C=1, penalty="l2")
                Linear_SVC.fit(new_x_train_combo, train_y_reduced)
                predicted_test_y = Linear_SVC.predict(new_x_test_combo)
                isTest = True
                #new
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'SAE_SVM_COMBO', isTest)
                    + tuple(
                        performance_score(test_y,
                                          predicted_test_y).values()))  #new
                predicted_train_y = Linear_SVC.predict(new_x_train_combo)
                isTest = False
                #new
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'SAE_SVM_COMBO',
                     isTest) + tuple(
                         performance_score(train_y_reduced,
                                           predicted_train_y).values()))
            if settings['SAE_SVM_RBF_COMBO']:
                print 'SAE followed by SVM RBF with combo feature'
                L1_SVC_RBF_Selector = SVC(C=1, gamma=0.01, kernel='rbf').fit(
                    new_x_train_combo, train_y_reduced)
                predicted_test_y = L1_SVC_RBF_Selector.predict(
                    new_x_test_combo)
                isTest = True
                #new
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'SAE_SVM_RBF_COMBO',
                     isTest) + tuple(
                         performance_score(test_y,
                                           predicted_test_y).values()))  #new
                predicted_train_y = L1_SVC_RBF_Selector.predict(
                    new_x_train_combo)
                isTest = False
                #new
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'SAE_SVM_RBF_COMBO',
                     isTest) + tuple(
                         performance_score(train_y_reduced,
                                           predicted_train_y).values()))

            if settings['DL']:
                print "direct deep learning"
                sda = train_a_Sda(x_train_minmax, pretrain_lr, finetune_lr,
                                  y_train_minmax,
                             x_validation_minmax, y_validation_minmax ,
                             x_test_minmax, test_y,
                             hidden_layers_sizes = hidden_layers_sizes, corruption_levels = corruption_levels, batch_size = batch_size , \
                             training_epochs = training_epochs, pretraining_epochs = pretraining_epochs, n_outs = settings['n_outs']

                 )
                print 'hidden_layers_sizes:', hidden_layers_sizes
                print 'corruption_levels:', corruption_levels
                training_predicted = sda.predict(x_train_minmax)
                y_train = y_train_minmax
                isTest = False
                #new
                analysis_scr.append((
                    self.ddi, subset_no, fisher_mode, 'DL', isTest
                ) + tuple(
                    performance_score(y_train, training_predicted).values()))

                test_predicted = sda.predict(x_test_minmax)
                y_test = test_y
                isTest = True
                #new
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'DL', isTest) +
                    tuple(performance_score(y_test, test_predicted).values()))
            if settings['DL_old']:
                print "direct deep learning old without early stop"
                sda = trainSda(x_train_minmax, y_train,
                 x_validation_minmax, y_validation_minmax,
                 x_test_minmax, y_test,pretrain_lr, finetune_lr,
                 pretraining_X_minmax=None,
                             hidden_layers_sizes = hidden_layers_sizes, corruption_levels = corruption_levels, batch_size = batch_size , \
                             training_epochs = training_epochs, pretraining_epochs = pretraining_epochs, n_outs = settings['n_outs']

                 )

                print 'hidden_layers_sizes:', hidden_layers_sizes
                print 'corruption_levels:', corruption_levels
                training_predicted = sda.predict(x_train_minmax)
                y_train = y_train_minmax
                isTest = False
                #new
                analysis_scr.append((
                    self.ddi, subset_no, fisher_mode, 'DL_old', isTest
                ) + tuple(
                    performance_score(y_train, training_predicted).values()))

                test_predicted = sda.predict(x_test_minmax)
                y_test = test_y
                isTest = True
                #new
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'DL_old', isTest) +
                    tuple(performance_score(y_test, test_predicted).values()))
            if settings['DL_U']:
                # deep learning using unlabeled data for pretraining
                print 'deep learning with unlabel data'
                pretraining_X_minmax = min_max_scaler.transform(train_X_10fold)
                pretraining_epochs = cal_epochs(
                    settings['pretraining_interations'],
                    x_train_minmax,
                    batch_size=batch_size)
                sda_unlabel = trainSda(x_train_minmax, y_train_minmax,
                             x_validation_minmax, y_validation_minmax ,
                             x_test_minmax, test_y,
                             pretraining_X_minmax = pretraining_X_minmax,
                             hidden_layers_sizes = hidden_layers_sizes, corruption_levels = corruption_levels, batch_size = batch_size , \
                             training_epochs = training_epochs, pretraining_epochs = pretraining_epochs,
                             pretrain_lr = pretrain_lr, finetune_lr=finetune_lr, n_outs = settings['n_outs']
                 )
                print 'hidden_layers_sizes:', hidden_layers_sizes
                print 'corruption_levels:', corruption_levels
                training_predicted = sda_unlabel.predict(x_train_minmax)
                y_train = y_train_minmax
                isTest = False
                #new
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'DL_U', isTest) + tuple(
                        performance_score(y_train, training_predicted,
                                          with_auc_score).values()))

                test_predicted = sda_unlabel.predict(x_test_minmax)
                y_test = test_y

                isTest = True
                #new
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'DL_U', isTest) + tuple(
                        performance_score(y_test, test_predicted,
                                          with_auc_score).values()))
            if settings['DL_S']:
                # deep learning using split network
                y_test = test_y
                print 'deep learning using split network'
                # get the new representation for A set. first 784-D
                pretraining_epochs = cal_epochs(
                    settings['pretraining_interations'],
                    x_train_minmax,
                    batch_size=batch_size)

                x = x_train_minmax[:, :x_train_minmax.shape[1] / 2]
                print "original shape for A", x.shape
                a_MAE_A = pretrain_a_Sda_with_estop(
                    x,
                    pretraining_epochs=pretraining_epochs,
                    pretrain_lr=pretrain_lr,
                    batch_size=batch_size,
                    hidden_layers_sizes=hidden_layers_sizes,
                    corruption_levels=corruption_levels)
                new_x_train_minmax_A = a_MAE_A.transform(
                    x_train_minmax[:, :x_train_minmax.shape[1] / 2])
                x = x_train_minmax[:, x_train_minmax.shape[1] / 2:]

                print "original shape for B", x.shape
                a_MAE_B = pretrain_a_Sda_with_estop(
                    x,
                    pretraining_epochs=pretraining_epochs,
                    pretrain_lr=pretrain_lr,
                    batch_size=batch_size,
                    hidden_layers_sizes=hidden_layers_sizes,
                    corruption_levels=corruption_levels)
                new_x_train_minmax_B = a_MAE_B.transform(
                    x_train_minmax[:, x_train_minmax.shape[1] / 2:])

                new_x_test_minmax_A = a_MAE_A.transform(
                    x_test_minmax[:, :x_test_minmax.shape[1] / 2])
                new_x_test_minmax_B = a_MAE_B.transform(
                    x_test_minmax[:, x_test_minmax.shape[1] / 2:])
                new_x_validation_minmax_A = a_MAE_A.transform(
                    x_validation_minmax[:, :x_validation_minmax.shape[1] / 2])
                new_x_validation_minmax_B = a_MAE_B.transform(
                    x_validation_minmax[:, x_validation_minmax.shape[1] / 2:])
                new_x_train_minmax_whole = np.hstack(
                    (new_x_train_minmax_A, new_x_train_minmax_B))
                new_x_test_minmax_whole = np.hstack(
                    (new_x_test_minmax_A, new_x_test_minmax_B))
                new_x_validationt_minmax_whole = np.hstack(
                    (new_x_validation_minmax_A, new_x_validation_minmax_B))


                sda_transformed = train_a_Sda(new_x_train_minmax_whole, pretrain_lr, finetune_lr,
                                              y_train_minmax,
                     new_x_validationt_minmax_whole, y_validation_minmax ,
                     new_x_test_minmax_whole, y_test,
                     hidden_layers_sizes = hidden_layers_sizes, corruption_levels = corruption_levels, batch_size = batch_size , \
                     training_epochs = training_epochs, pretraining_epochs = pretraining_epochs, n_outs = settings['n_outs']

                     )

                print 'hidden_layers_sizes:', hidden_layers_sizes
                print 'corruption_levels:', corruption_levels
                training_predicted = sda_transformed.predict(
                    new_x_train_minmax_whole)
                y_train = y_train_minmax

                isTest = False
                #new
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'DL_S', isTest) + tuple(
                        performance_score(y_train, training_predicted,
                                          with_auc_score).values()))

                test_predicted = sda_transformed.predict(
                    new_x_test_minmax_whole)
                y_test = test_y

                isTest = True
                #new
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'DL_S', isTest) + tuple(
                        performance_score(y_test, test_predicted,
                                          with_auc_score).values()))
            if settings['DL_S_new']:
                # deep learning using split network
                print 'new deep learning using split network'

                cfg = settings.copy()
                p_sda = Parellel_Sda_factory(cfg)
                p_sda.supervised_training(x_train_minmax, x_validation_minmax,
                                          y_train_minmax, y_validation_minmax)

                isTest = False  #new
                training_predicted = p_sda.predict(x_train_minmax)
                y_train = y_train_minmax
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'DL_S_new', isTest) +
                    tuple(
                        performance_score(y_train, training_predicted,
                                          with_auc_score).values()))

                isTest = True  #new
                y_test = test_y
                test_predicted = p_sda.predict(x_test_minmax)
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'DL_S_new', isTest) +
                    tuple(
                        performance_score(y_test, test_predicted,
                                          with_auc_score).values()))
            if settings['DL_S_new_contraction']:
                print 'DL_S_new_contraction'
                cfg = settings.copy()
                cfg['contraction_level'] = 0.0001
                p_sda = Parellel_Sda_factory(cfg)
                p_sda.supervised_training(x_train_minmax, x_validation_minmax,
                                          y_train_minmax, y_validation_minmax)

                isTest = False  #new
                training_predicted = p_sda.predict(x_train_minmax)
                y_train = y_train_minmax
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'DL_S_new_contraction',
                     isTest) + tuple(
                         performance_score(y_train, training_predicted,
                                           with_auc_score).values()))

                isTest = True  #new
                y_test = test_y
                test_predicted = p_sda.predict(x_test_minmax)
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'DL_S_new_contraction',
                     isTest) + tuple(
                         performance_score(y_test, test_predicted,
                                           with_auc_score).values()))

            if settings['DL_S_new_sparsity'] == 1:
                print 'DL_S_new_sparsity'
                cfg = settings.copy()
                cfg['sparsity'] = 0.1
                cfg['sparsity_weight'] = 0.0001
                p_sda = Parellel_Sda_factory(cfg)
                p_sda.supervised_training(x_train_minmax, x_validation_minmax,
                                          y_train_minmax, y_validation_minmax)

                isTest = False  #new
                training_predicted = p_sda.predict(x_train_minmax)
                y_train = y_train_minmax
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'DL_S_new_sparsity',
                     isTest) + tuple(
                         performance_score(y_train, training_predicted,
                                           with_auc_score).values()))

                isTest = True  #new
                y_test = test_y
                test_predicted = p_sda.predict(x_test_minmax)
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'DL_S_new_sparsity',
                     isTest) + tuple(
                         performance_score(y_test, test_predicted,
                                           with_auc_score).values()))

            if settings['DL_S_new_weight_decay'] == 2:
                cfg = settings.copy()
                cfg['l2_reg'] = 0.0001
                print 'l2_reg'
                p_sda = Parellel_Sda_factory(cfg)
                p_sda.supervised_training(x_train_minmax, x_validation_minmax,
                                          y_train_minmax, y_validation_minmax)

                isTest = False  #new
                training_predicted = p_sda.predict(x_train_minmax)
                y_train = y_train_minmax
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'l2_reg', isTest) +
                    tuple(
                        performance_score(y_train, training_predicted,
                                          with_auc_score).values()))

                isTest = True  #new
                y_test = test_y
                test_predicted = p_sda.predict(x_test_minmax)
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'l2_reg', isTest) +
                    tuple(
                        performance_score(y_test, test_predicted,
                                          with_auc_score).values()))

            if settings['DL_S_new_weight_decay'] == 1:
                print 'l1_reg'
                cfg = settings.copy()
                cfg['l1_reg'] = 0.1
                p_sda = Parellel_Sda_factory(cfg)
                p_sda.supervised_training(x_train_minmax, x_validation_minmax,
                                          y_train_minmax, y_validation_minmax)

                isTest = False  #new
                training_predicted = p_sda.predict(x_train_minmax)
                y_train = y_train_minmax
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'l1_reg', isTest) +
                    tuple(
                        performance_score(y_train, training_predicted,
                                          with_auc_score).values()))

                isTest = True  #new
                y_test = test_y
                test_predicted = p_sda.predict(x_test_minmax)
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'l1_reg', isTest) +
                    tuple(
                        performance_score(y_test, test_predicted,
                                          with_auc_score).values()))

            if settings['DL_S_new_Drop_out'] == 1:

                cfg = settings.copy()
                cfg['dropout_factor'] = 0.5
                print 'DL_S_new_Drop_out'
                p_sda = Parellel_Sda_factory(cfg)
                p_sda.supervised_training(x_train_minmax, x_validation_minmax,
                                          y_train_minmax, y_validation_minmax)

                isTest = False  #new
                training_predicted = p_sda.predict(x_train_minmax)
                y_train = y_train_minmax
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'DL_S_new_Drop_out',
                     isTest) + tuple(
                         performance_score(y_train, training_predicted,
                                           with_auc_score).values()))

                isTest = True  #new
                y_test = test_y
                test_predicted = p_sda.predict(x_test_minmax)
                analysis_scr.append(
                    (self.ddi, subset_no, fisher_mode, 'DL_S_new_Drop_out',
                     isTest) + tuple(
                         performance_score(y_test, test_predicted,
                                           with_auc_score).values()))

        report_name = filename + '_' + '_newDL_'.join(
            map(str, hidden_layers_sizes)) + '_' + str(
                pretrain_lr) + '_' + str(finetune_lr) + '_' + str(
                    settings['training_interations']) + '_' + current_date
        saveAsCsv(with_auc_score, report_name,
                  performance_score(test_y, predicted_test_y, with_auc_score),
                  analysis_scr)
Exemple #3
0
def run_models(settings=None):
    analysis_scr = []
    with_auc_score = settings['with_auc_score']
    n_outs = settings['n_outs']
    for subset_no in xrange(1, settings['number_iterations'] + 1):
        print("Subset:", subset_no)

        ################## generate data ###################
        array_A = []
        array_B = []
        for i in range(100000):
            array_A.append(np.random.random_integers(0, 59999))
            array_B.append(np.random.random_integers(0, 59999))
        pos_index = []
        neg_index = []
        for index in xrange(100000):
            if y_total[array_A[index]] - y_total[array_B[index]] == 1:
                pos_index.append(index)
            else:
                neg_index.append(index)
        print 'number of positive examples', len(pos_index)
        selected_neg_index = neg_index[:len(pos_index)]

        array_A = np.array(array_A)
        array_B = np.array(array_B)
        index_for_positive_image_A = array_A[pos_index]
        index_for_positive_image_B = array_B[pos_index]
        index_for_neg_image_A = array_A[selected_neg_index]
        index_for_neg_image_B = array_B[selected_neg_index]

        X_pos_A = X_total[index_for_positive_image_A]
        X_pos_B = X_total[index_for_positive_image_B]
        X_pos_whole = np.hstack((X_pos_A, X_pos_B))
        X_neg_A = X_total[index_for_neg_image_A]
        X_neg_B = X_total[index_for_neg_image_B]
        X_neg_whole = np.hstack((X_neg_A, X_neg_B))
        print X_pos_A.shape, X_pos_B.shape, X_pos_whole.shape
        print X_neg_A.shape, X_neg_B.shape, X_neg_whole.shape

        X_whole = np.vstack((X_pos_whole, X_neg_whole))
        print X_whole.shape
        y_pos = np.ones(X_pos_whole.shape[0])
        y_neg = np.zeros(X_neg_whole.shape[0])
        y_whole = np.concatenate([y_pos, y_neg])
        print y_whole.shape

        x_train_pre_validation, x_test, y_train_pre_validation, y_test = train_test_split(
            X_whole, y_whole, test_size=0.2, random_state=211)
        for number_of_training in settings['number_of_training']:

            x_train, x_validation, y_train, y_validation = train_test_split(x_train_pre_validation[:number_of_training],
                                                                                                        y_train_pre_validation[:number_of_training],\
                                                                        test_size=0.2, random_state=21)
            '''
            x_train, x_validation, y_train, y_validation = train_test_split(x_train_pre_validation[:],
                                                                                                        y_train_pre_validation[:],\
                                                                        test_size=0.4, random_state=21)
            '''
            print x_train.shape, y_train.shape, x_validation.shape, \
            y_validation.shape, x_test.shape, y_test.shape
            x_train_minmax, x_validation_minmax, x_test_minmax = x_train, x_validation, x_test
            train_X_reduced = x_train
            train_y_reduced = y_train
            test_X = x_test
            test_y = y_test
            y_train_minmax = y_train
            y_validation_minmax = y_validation
            y_test_minmax = y_test
            ###original data###
            ################ end of data ####################
            standard_scaler = preprocessing.StandardScaler().fit(
                train_X_reduced)
            scaled_train_X = standard_scaler.transform(train_X_reduced)
            scaled_test_X = standard_scaler.transform(test_X)
            if settings['SVM']:
                print "SVM"
                Linear_SVC = LinearSVC(C=1, penalty="l2")
                Linear_SVC.fit(scaled_train_X, y_train)
                predicted_test_y = Linear_SVC.predict(scaled_test_X)
                isTest = True
                #new
                analysis_scr.append(
                    (subset_no, number_of_training, 'SVM', isTest) + tuple(
                        performance_score(test_y,
                                          predicted_test_y).values()))  #new

                predicted_train_y = Linear_SVC.predict(scaled_train_X)
                isTest = False
                #new
                analysis_scr.append(
                    (subset_no, number_of_training, 'SVM', isTest) + tuple(
                        performance_score(train_y_reduced,
                                          predicted_train_y).values()))

            if settings['SVM_RBF']:
                print "SVM_RBF"
                L1_SVC_RBF_Selector = SVC(C=1, gamma=0.01, kernel='rbf').fit(
                    scaled_train_X, y_train)
                predicted_test_y = L1_SVC_RBF_Selector.predict(scaled_test_X)
                isTest = True
                #new
                analysis_scr.append(
                    (subset_no, number_of_training, 'SVM_RBF', isTest) + tuple(
                        performance_score(test_y,
                                          predicted_test_y).values()))  #new
                predicted_train_y = L1_SVC_RBF_Selector.predict(scaled_train_X)
                isTest = False
                #new
                analysis_scr.append(
                    (subset_no, number_of_training, 'SVM_RBF', isTest) + tuple(
                        performance_score(train_y_reduced,
                                          predicted_train_y).values()))

            if settings['SVM_POLY']:
                print "SVM_POLY"
                L1_SVC_POLY_Selector = SVC(C=1, kernel='poly').fit(
                    scaled_train_X, train_y_reduced)

                predicted_test_y = L1_SVC_POLY_Selector.predict(scaled_test_X)
                isTest = True
                #new
                analysis_scr.append(
                    (subset_no, number_of_training, 'SVM_POLY', isTest) +
                    tuple(
                        performance_score(test_y,
                                          predicted_test_y).values()))  #new

                predicted_train_y = L1_SVC_POLY_Selector.predict(
                    scaled_train_X)
                isTest = False
                #new
                analysis_scr.append((
                    subset_no, number_of_training, 'SVM_POLY', isTest) + tuple(
                        performance_score(train_y_reduced,
                                          predicted_train_y).values()))

            if settings['Log']:
                print "Log"
                log_clf_l2 = sklearn.linear_model.LogisticRegression(
                    C=1, penalty='l2')
                log_clf_l2.fit(scaled_train_X, train_y_reduced)
                predicted_test_y = log_clf_l2.predict(scaled_test_X)
                isTest = True
                #new
                analysis_scr.append(
                    (subset_no, number_of_training, 'Log', isTest) + tuple(
                        performance_score(test_y,
                                          predicted_test_y).values()))  #new
                predicted_train_y = log_clf_l2.predict(scaled_train_X)
                isTest = False
                #new
                analysis_scr.append(
                    (subset_no, number_of_training, 'Log', isTest) + tuple(
                        performance_score(train_y_reduced,
                                          predicted_train_y).values()))

            # direct deep learning

            finetune_lr = settings['finetune_lr']
            batch_size = settings['batch_size']
            pretraining_epochs = cal_epochs(
                settings['pretraining_interations'],
                x_train_minmax,
                batch_size=batch_size)
            #pretrain_lr=0.001
            pretrain_lr = settings['pretrain_lr']
            training_epochs = cal_epochs(settings['training_interations'],
                                         x_train_minmax,
                                         batch_size=batch_size)
            hidden_layers_sizes = settings['hidden_layers_sizes']
            corruption_levels = settings['corruption_levels']
            settings['lrate'] = settings['lrate_pre'] + str(training_epochs)

            if settings['DL']:
                print "direct deep learning"
                sda = trainSda(x_train_minmax, y_train,
                             x_validation_minmax, y_validation,
                             x_test_minmax, test_y,
                             hidden_layers_sizes = hidden_layers_sizes, corruption_levels = corruption_levels, batch_size = batch_size , \
                             training_epochs = training_epochs, pretraining_epochs = pretraining_epochs,
                             pretrain_lr = pretrain_lr, finetune_lr=finetune_lr, n_outs = n_outs
                 )
                print 'hidden_layers_sizes:', hidden_layers_sizes
                print 'corruption_levels:', corruption_levels
                test_predicted = sda.predict(x_test_minmax)
                isTest = True
                #new
                analysis_scr.append(
                    (subset_no, number_of_training, 'DL', isTest) +
                    tuple(performance_score(y_test, test_predicted).values()))
                training_predicted = sda.predict(x_train_minmax)
                isTest = False
                #new
                analysis_scr.append((
                    subset_no, number_of_training, 'DL', isTest
                ) + tuple(
                    performance_score(y_train, training_predicted).values()))

            ####transformed original data####
            x = train_X_reduced
            a_MAE_original = train_a_MultipleAEs(
                x,
                pretraining_epochs=pretraining_epochs,
                pretrain_lr=pretrain_lr,
                batch_size=batch_size,
                hidden_layers_sizes=hidden_layers_sizes,
                corruption_levels=corruption_levels)
            new_x_train_minmax_A = a_MAE_original.transform(train_X_reduced)
            new_x_test_minmax_A = a_MAE_original.transform(x_test_minmax)
            standard_scaler = preprocessing.StandardScaler().fit(
                new_x_train_minmax_A)
            new_x_train_scaled = standard_scaler.transform(
                new_x_train_minmax_A)
            new_x_test_scaled = standard_scaler.transform(new_x_test_minmax_A)
            new_x_train_combo = np.hstack((scaled_train_X, new_x_train_scaled))
            new_x_test_combo = np.hstack((scaled_test_X, new_x_test_scaled))

            if settings['SAE_SVM']:
                # SAE_SVM
                print 'SAE followed by SVM'

                Linear_SVC = LinearSVC(C=1, penalty="l2")
                Linear_SVC.fit(new_x_train_scaled, train_y_reduced)
                predicted_test_y = Linear_SVC.predict(new_x_test_scaled)
                isTest = True
                #new
                analysis_scr.append(
                    (subset_no, number_of_training, 'SAE_SVM', isTest) + tuple(
                        performance_score(test_y,
                                          predicted_test_y).values()))  #new

                predicted_train_y = Linear_SVC.predict(new_x_train_scaled)
                isTest = False
                #new
                analysis_scr.append(
                    (subset_no, number_of_training, 'SAE_SVM', isTest) + tuple(
                        performance_score(train_y_reduced,
                                          predicted_train_y).values()))
            if settings['SAE_Log']:
                print 'SAE followed by Log'
                log_clf_l2 = sklearn.linear_model.LogisticRegression(
                    C=1, penalty='l2')
                log_clf_l2.fit(new_x_train_scaled, train_y_reduced)
                predicted_test_y = log_clf_l2.predict(new_x_test_scaled)
                isTest = True
                #new
                analysis_scr.append(
                    (subset_no, number_of_training, 'SAE_Log', isTest) + tuple(
                        performance_score(test_y,
                                          predicted_test_y).values()))  #new
                predicted_train_y = log_clf_l2.predict(new_x_train_scaled)
                isTest = False
                #new
                analysis_scr.append(
                    (subset_no, number_of_training, 'SAE_Log', isTest) + tuple(
                        performance_score(train_y_reduced,
                                          predicted_train_y).values()))

            if settings['SAE_SVM_RBF']:
                # SAE_SVM
                print 'SAE followed by SVM RBF'
                L1_SVC_RBF_Selector = SVC(C=1, gamma=0.01, kernel='rbf').fit(
                    new_x_train_scaled, train_y_reduced)

                predicted_test_y = L1_SVC_RBF_Selector.predict(
                    new_x_test_scaled)
                isTest = True
                #new
                analysis_scr.append(
                    (subset_no, number_of_training, 'SAE_SVM_RBF', isTest) +
                    tuple(
                        performance_score(test_y,
                                          predicted_test_y).values()))  #new

                predicted_train_y = L1_SVC_RBF_Selector.predict(
                    new_x_train_scaled)
                isTest = False
                #new
                analysis_scr.append(
                    (subset_no, number_of_training, 'SAE_SVM_RBF', isTest) +
                    tuple(
                        performance_score(train_y_reduced,
                                          predicted_train_y).values()))
            if settings['SAE_SVM_POLY']:
                # SAE_SVM
                print 'SAE followed by SVM POLY'
                L1_SVC_RBF_Selector = SVC(C=1, kernel='poly').fit(
                    new_x_train_scaled, train_y_reduced)

                predicted_test_y = L1_SVC_RBF_Selector.predict(
                    new_x_test_scaled)
                isTest = True
                #new
                analysis_scr.append(
                    (subset_no, number_of_training, 'SAE_SVM_POLY', isTest) +
                    tuple(
                        performance_score(test_y,
                                          predicted_test_y).values()))  #new

                predicted_train_y = L1_SVC_RBF_Selector.predict(
                    new_x_train_scaled)
                isTest = False
                #new
                analysis_scr.append(
                    (subset_no, number_of_training, 'SAE_SVM_POLY', isTest) +
                    tuple(
                        performance_score(train_y_reduced,
                                          predicted_train_y).values()))

            #### separated transformed data ####
            y_test = test_y
            print 'deep learning using split network'
            # get the new representation for A set. first 784-D
            pretraining_epochs = cal_epochs(
                settings['pretraining_interations'],
                x_train_minmax,
                batch_size=batch_size)

            x = x_train_minmax[:, :x_train_minmax.shape[1] / 2]
            print "original shape for A", x.shape
            a_MAE_A = train_a_MultipleAEs(
                x,
                pretraining_epochs=pretraining_epochs,
                pretrain_lr=pretrain_lr,
                batch_size=batch_size,
                hidden_layers_sizes=[x / 2 for x in hidden_layers_sizes],
                corruption_levels=corruption_levels)
            new_x_train_minmax_A = a_MAE_A.transform(
                x_train_minmax[:, :x_train_minmax.shape[1] / 2])
            x = x_train_minmax[:, x_train_minmax.shape[1] / 2:]

            print "original shape for B", x.shape
            a_MAE_B = train_a_MultipleAEs(
                x,
                pretraining_epochs=pretraining_epochs,
                pretrain_lr=pretrain_lr,
                batch_size=batch_size,
                hidden_layers_sizes=[x / 2 for x in hidden_layers_sizes],
                corruption_levels=corruption_levels)
            new_x_train_minmax_B = a_MAE_B.transform(
                x_train_minmax[:, x_train_minmax.shape[1] / 2:])

            new_x_test_minmax_A = a_MAE_A.transform(
                x_test_minmax[:, :x_test_minmax.shape[1] / 2])
            new_x_test_minmax_B = a_MAE_B.transform(
                x_test_minmax[:, x_test_minmax.shape[1] / 2:])
            new_x_validation_minmax_A = a_MAE_A.transform(
                x_validation_minmax[:, :x_validation_minmax.shape[1] / 2])
            new_x_validation_minmax_B = a_MAE_B.transform(
                x_validation_minmax[:, x_validation_minmax.shape[1] / 2:])
            new_x_train_minmax_whole = np.hstack(
                (new_x_train_minmax_A, new_x_train_minmax_B))
            new_x_test_minmax_whole = np.hstack(
                (new_x_test_minmax_A, new_x_test_minmax_B))
            new_x_validationt_minmax_whole = np.hstack(
                (new_x_validation_minmax_A, new_x_validation_minmax_B))
            standard_scaler = preprocessing.StandardScaler().fit(
                new_x_train_minmax_whole)
            new_x_train_minmax_whole_scaled = standard_scaler.transform(
                new_x_train_minmax_whole)
            new_x_test_minmax_whole_scaled = standard_scaler.transform(
                new_x_test_minmax_whole)
            if settings['DL_S']:
                # deep learning using split network
                sda_transformed = trainSda(new_x_train_minmax_whole, y_train,
                     new_x_validationt_minmax_whole, y_validation ,
                     new_x_test_minmax_whole, y_test,
                     hidden_layers_sizes = hidden_layers_sizes, corruption_levels = corruption_levels, batch_size = batch_size , \
                     training_epochs = training_epochs, pretraining_epochs = pretraining_epochs,
                     pretrain_lr = pretrain_lr, finetune_lr=finetune_lr
                     )
                print 'hidden_layers_sizes:', hidden_layers_sizes
                print 'corruption_levels:', corruption_levels

                predicted_test_y = sda_transformed.predict(
                    new_x_test_minmax_whole)
                y_test = test_y
                isTest = True
                #new
                analysis_scr.append(
                    (subset_no, number_of_training, 'DL_S', isTest) + tuple(
                        performance_score(y_test, predicted_test_y,
                                          with_auc_score).values()))

                training_predicted = sda_transformed.predict(
                    new_x_train_minmax_whole)
                isTest = False
                #new
                analysis_scr.append(
                    (subset_no, number_of_training, 'DL_S', isTest) + tuple(
                        performance_score(y_train, training_predicted,
                                          with_auc_score).values()))
            if settings['SAE_S_SVM']:
                print 'SAE_S followed by SVM'

                Linear_SVC = LinearSVC(C=1, penalty="l2")
                Linear_SVC.fit(new_x_train_minmax_whole_scaled,
                               train_y_reduced)
                predicted_test_y = Linear_SVC.predict(
                    new_x_test_minmax_whole_scaled)
                isTest = True
                #new
                analysis_scr.append(
                    (subset_no, number_of_training, 'SAE_S_SVM', isTest) +
                    tuple(
                        performance_score(test_y, predicted_test_y,
                                          with_auc_score).values()))  #new

                predicted_train_y = Linear_SVC.predict(
                    new_x_train_minmax_whole_scaled)
                isTest = False
                #new
                analysis_scr.append(
                    (subset_no, number_of_training, 'SAE_S_SVM', isTest) +
                    tuple(
                        performance_score(train_y_reduced, predicted_train_y,
                                          with_auc_score).values()))
            if settings['SAE_S_SVM_RBF']:
                print 'SAE S followed by SVM RBF'
                L1_SVC_RBF_Selector = SVC(C=1, gamma=0.01, kernel='rbf').fit(
                    new_x_train_minmax_whole_scaled, train_y_reduced)

                predicted_test_y = L1_SVC_RBF_Selector.predict(
                    new_x_test_minmax_whole_scaled)
                isTest = True
                #new
                analysis_scr.append(
                    (subset_no, number_of_training, 'SAE_S_SVM_RBF', isTest) +
                    tuple(
                        performance_score(test_y, predicted_test_y,
                                          with_auc_score).values()))  #new

                predicted_train_y = L1_SVC_RBF_Selector.predict(
                    new_x_train_minmax_whole_scaled)
                isTest = False
                #new
                analysis_scr.append(
                    (subset_no, number_of_training, 'SAE_S_SVM_RBF', isTest) +
                    tuple(
                        performance_score(train_y_reduced, predicted_train_y,
                                          with_auc_score).values()))
            if settings['SAE_S_SVM_POLY']:
                # SAE_SVM
                print 'SAE S followed by SVM POLY'
                L1_SVC_RBF_Selector = SVC(C=1, kernel='poly').fit(
                    new_x_train_minmax_whole_scaled, train_y_reduced)

                predicted_test_y = L1_SVC_RBF_Selector.predict(
                    new_x_test_minmax_whole_scaled)
                isTest = True
                #new
                analysis_scr.append(
                    (subset_no, number_of_training, 'SAE_S_SVM_POLY', isTest) +
                    tuple(
                        performance_score(test_y, predicted_test_y,
                                          with_auc_score).values()))  #new

                predicted_train_y = L1_SVC_RBF_Selector.predict(
                    new_x_train_minmax_whole_scaled)
                isTest = False
                #new
                analysis_scr.append(
                    (subset_no, number_of_training, 'SAE_S_SVM_POLY', isTest) +
                    tuple(
                        performance_score(train_y_reduced, predicted_train_y,
                                          with_auc_score).values()))

            settings['epoch_number'] = cal_epochs(
                settings['pretraining_interations'],
                x_train_minmax,
                batch_size=batch_size)
            # deep xy autoencoders
            settings['n_ins'] = x_train_minmax.shape[1]
            if settings['DL_xy']:
                cfg = settings.copy()
                cfg['weight_y'] = 0.1
                print 'DL_xy'
                train_x = x_train_minmax
                train_y = y_train_minmax
                sdaf = Sda_xy_factory(cfg)
                sdaf.sda.pretraining(train_x, train_y)
                dnnf = DNN_factory(cfg)
                dnnf.dnn.load_pretrain_from_Sda(sdaf.sda)
                dnnf.dnn.finetuning((x_train_minmax, y_train_minmax),
                                    (x_validation_minmax, y_validation_minmax))

                training_predicted = dnnf.dnn.predict(x_train_minmax)
                y_train = y_train_minmax
                isTest = False
                #new
                analysis_scr.append(
                    (subset_no, number_of_training, 'DL_xy', isTest) + tuple(
                        performance_score(train_y_reduced, training_predicted,
                                          with_auc_score).values()))

                test_predicted = dnnf.dnn.predict(x_test_minmax)
                y_test = test_y
                isTest = True
                #new
                analysis_scr.append(
                    (subset_no, number_of_training, 'DL_xy', isTest) + tuple(
                        performance_score(test_y, test_predicted,
                                          with_auc_score).values()))
            if settings['Sda_xy_with_first']:
                cfg = settings.copy()
                cfg['weight_y'] = 0.1
                cfg['firstlayer_xy'] = 1
                print 'firstlayer_xy'
                train_x = x_train_minmax
                train_y = y_train_minmax
                sdaf = Sda_xy_factory(cfg)
                sdaf.sda.pretraining(train_x, train_y)
                dnnf = DNN_factory(cfg)
                dnnf.dnn.load_pretrain_from_Sda(sdaf.sda)
                dnnf.dnn.finetuning((x_train_minmax, y_train_minmax),
                                    (x_validation_minmax, y_validation_minmax))

                training_predicted = dnnf.dnn.predict(x_train_minmax)
                y_train = y_train_minmax
                isTest = False
                #new
                analysis_scr.append(
                    (subset_no, number_of_training, 'Sda_xy_with_first',
                     isTest) + tuple(
                         performance_score(train_y_reduced, training_predicted,
                                           with_auc_score).values()))
                test_predicted = dnnf.dnn.predict(x_test_minmax)
                y_test = test_y
                isTest = True
                #new
                analysis_scr.append(
                    (subset_no, number_of_training, 'Sda_xy_with_first',
                     isTest) + tuple(
                         performance_score(test_y, test_predicted,
                                           with_auc_score).values()))
            if settings['Sda_new']:
                print 'Sda_new'
                cfg = settings.copy()
                train_x = x_train_minmax
                train_y = y_train_minmax
                cfg['n_ins'] = train_x.shape[1]
                sdaf = Sda_factory(cfg)
                sda = sdaf.sda.pretraining(train_x=train_x)
                sdaf.dnn.finetuning((x_train_minmax, y_train_minmax),
                                    (x_validation_minmax, y_validation_minmax))
                training_predicted = sdaf.dnn.predict(x_train_minmax)
                y_train = y_train_minmax
                isTest = False
                #new
                analysis_scr.append(
                    (subset_no, number_of_training, 'Sda_new', isTest) + tuple(
                        performance_score(train_y_reduced, training_predicted,
                                          with_auc_score).values()))

                test_predicted = sdaf.dnn.predict(x_test_minmax)
                y_test = test_y
                isTest = True
                #new
                analysis_scr.append(
                    (subset_no, number_of_training, 'Sda_new', isTest) + tuple(
                        performance_score(test_y, test_predicted,
                                          with_auc_score).values()))

            if settings['DL_S_new']:
                # deep learning using split network
                print 'new deep learning using split network'

                cfg = settings.copy()
                p_sda = Parellel_Sda_factory(cfg)
                p_sda.supervised_training(x_train_minmax, x_validation_minmax,
                                          y_train_minmax, y_validation_minmax)

                isTest = False  #new
                training_predicted = p_sda.predict(x_train_minmax)
                y_train = y_train_minmax

                analysis_scr.append((
                    subset_no, number_of_training, 'DL_S_new', isTest) + tuple(
                        performance_score(train_y_reduced, training_predicted,
                                          with_auc_score).values()))
                isTest = True  #new
                y_test = test_y
                test_predicted = p_sda.predict(x_test_minmax)
                analysis_scr.append((
                    subset_no, number_of_training, 'DL_S_new', isTest) + tuple(
                        performance_score(test_y, test_predicted,
                                          with_auc_score).values()))
            if settings['DL_S_new_contraction']:
                print 'DL_S_new_contraction'
                cfg = settings.copy()
                cfg['contraction_level'] = 0.01
                p_sda = Parellel_Sda_factory(cfg)
                p_sda.supervised_training(x_train_minmax, x_validation_minmax,
                                          y_train_minmax, y_validation_minmax)

                isTest = False  #new
                training_predicted = p_sda.predict(x_train_minmax)
                y_train = y_train_minmax
                analysis_scr.append(
                    (subset_no, number_of_training, 'DL_S_new_contraction',
                     isTest) + tuple(
                         performance_score(train_y_reduced, training_predicted,
                                           with_auc_score).values()))
                isTest = True  #new
                y_test = test_y
                test_predicted = p_sda.predict(x_test_minmax)
                analysis_scr.append(
                    (subset_no, number_of_training, 'DL_S_new_contraction',
                     isTest) + tuple(
                         performance_score(test_y, test_predicted,
                                           with_auc_score).values()))

            if settings['DL_S_new_sparsity'] == 1:
                print 'DL_S_new_sparsity'
                cfg = settings.copy()
                cfg['sparsity'] = 0.01
                cfg['sparsity_weight'] = 0.01
                p_sda = Parellel_Sda_factory(cfg)
                p_sda.supervised_training(x_train_minmax, x_validation_minmax,
                                          y_train_minmax, y_validation_minmax)

                isTest = False  #new
                training_predicted = p_sda.predict(x_train_minmax)
                y_train = y_train_minmax
                analysis_scr.append(
                    (subset_no, number_of_training, 'DL_S_new_sparsity',
                     isTest) + tuple(
                         performance_score(train_y_reduced, training_predicted,
                                           with_auc_score).values()))
                isTest = True  #new
                y_test = test_y
                test_predicted = p_sda.predict(x_test_minmax)
                analysis_scr.append(
                    (subset_no, number_of_training, 'DL_S_new_sparsity',
                     isTest) + tuple(
                         performance_score(test_y, test_predicted,
                                           with_auc_score).values()))
            if settings['DL_S_new_weight_decay'] == 2:
                cfg = settings.copy()
                cfg['l2_reg'] = 0.01
                print 'l2_reg'
                p_sda = Parellel_Sda_factory(cfg)
                p_sda.supervised_training(x_train_minmax, x_validation_minmax,
                                          y_train_minmax, y_validation_minmax)

                isTest = False  #new
                training_predicted = p_sda.predict(x_train_minmax)
                y_train = y_train_minmax
                analysis_scr.append(
                    (subset_no, number_of_training, 'l2_reg', isTest) + tuple(
                        performance_score(train_y_reduced, training_predicted,
                                          with_auc_score).values()))
                isTest = True  #new
                y_test = test_y
                test_predicted = p_sda.predict(x_test_minmax)
                analysis_scr.append(
                    (subset_no, number_of_training, 'l2_reg', isTest) + tuple(
                        performance_score(test_y, test_predicted,
                                          with_auc_score).values()))
            if settings['DL_S_new_weight_decay'] == 1:
                print 'l1_reg'
                cfg = settings.copy()
                cfg['l1_reg'] = 0.01
                p_sda = Parellel_Sda_factory(cfg)
                p_sda.supervised_training(x_train_minmax, x_validation_minmax,
                                          y_train_minmax, y_validation_minmax)

                isTest = False  #new
                training_predicted = p_sda.predict(x_train_minmax)
                y_train = y_train_minmax
                analysis_scr.append(
                    (subset_no, number_of_training, 'l1_reg', isTest) + tuple(
                        performance_score(train_y_reduced, training_predicted,
                                          with_auc_score).values()))
                isTest = True  #new
                y_test = test_y
                test_predicted = p_sda.predict(x_test_minmax)
                analysis_scr.append(
                    (subset_no, number_of_training, 'l1_reg', isTest) + tuple(
                        performance_score(test_y, test_predicted,
                                          with_auc_score).values()))

            if settings['DL_S_new_Drop_out'] == 1:

                cfg = settings.copy()
                cfg['dropout_factor'] = 0.5
                print 'DL_S_new_Drop_out'
                p_sda = Parellel_Sda_factory(cfg)
                p_sda.supervised_training(x_train_minmax, x_validation_minmax,
                                          y_train_minmax, y_validation_minmax)

                isTest = False  #new
                training_predicted = p_sda.predict(x_train_minmax)
                y_train = y_train_minmax
                analysis_scr.append(
                    (subset_no, number_of_training, 'DL_S_new_Drop_out',
                     isTest) + tuple(
                         performance_score(train_y_reduced, training_predicted,
                                           with_auc_score).values()))
                isTest = True  #new
                y_test = test_y
                test_predicted = p_sda.predict(x_test_minmax)
                analysis_scr.append(
                    (subset_no, number_of_training, 'DL_S_new_Drop_out',
                     isTest) + tuple(
                         performance_score(test_y, test_predicted,
                                           with_auc_score).values()))




        report_name = 'DL_handwritten_digits' + '_size_'.join(map(str, hidden_layers_sizes)) + \
                        '_' + str(pretrain_lr) + '_' + str(finetune_lr) + '_' + \
                '_' + str(settings['pretraining_interations']) + '_' + current_date
    saveAsCsv(with_auc_score, report_name,
              performance_score(test_y, predicted_test_y, with_auc_score),
              analysis_scr)
    return sda, a_MAE_original, a_MAE_A, a_MAE_B, analysis_scr
        def get_ten_fold_crossvalid_perfermance(self, settings = None):
            fisher_mode = settings['fisher_mode']
            analysis_scr = []
            with_auc_score = settings['with_auc_score']
            reduce_ratio = settings['reduce_ratio']
            #for seq_no in range(1, self.ddi_obj.total_number_of_sequences+1):
            #subset_size = math.floor(self.ddi_obj.total_number_of_sequences / 10.0)
            kf = KFold(self.ddi_obj.total_number_of_sequences, n_folds = 10, shuffle = True)
            #for subset_no in range(1, 11):
            for ((train_index, test_index),subset_no) in izip(kf,range(1,11)):
            #for train_index, test_index in kf;
                print("Subset:", subset_no)
                print("Train index: ", train_index)
                print("Test index: ", test_index)
                #logger.info('subset number: ' + str(subset_no))
                (train_X_10fold, train_y_10fold),(train_X_reduced, train_y_reduced), (test_X, test_y) = self.ddi_obj.get_ten_fold_crossvalid_one_subset(train_index, test_index, fisher_mode = fisher_mode, reduce_ratio = reduce_ratio)
                standard_scaler = preprocessing.StandardScaler().fit(train_X_reduced)
                scaled_train_X = standard_scaler.transform(train_X_reduced)
                scaled_test_X = standard_scaler.transform(test_X)
                
                if settings['SVM']:
                    print "SVM"                   
                    Linear_SVC = LinearSVC(C=1, penalty="l2")
                    Linear_SVC.fit(scaled_train_X, train_y_reduced)
                    predicted_test_y = Linear_SVC.predict(scaled_test_X)
                    isTest = True; #new
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'SVM', isTest) + tuple(performance_score(test_y, predicted_test_y).values())) #new

                    predicted_train_y = Linear_SVC.predict(scaled_train_X)
                    isTest = False; #new
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'SVM', isTest) + tuple(performance_score(train_y_reduced, predicted_train_y).values()))                    
                if settings['SVM_RBF']:
                    print "SVM_RBF"
                    L1_SVC_RBF_Selector = SVC(C=1, gamma=0.01, kernel='rbf').fit(scaled_train_X, train_y_reduced)

                    predicted_test_y = L1_SVC_RBF_Selector.predict(scaled_test_X)
                    isTest = True; #new
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'SVM_RBF', isTest) + tuple(performance_score(test_y, predicted_test_y).values())) #new

                    predicted_train_y = L1_SVC_RBF_Selector.predict(scaled_train_X)
                    isTest = False; #new
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'SVM_RBF', isTest) + tuple(performance_score(train_y_reduced, predicted_train_y).values()))
                if settings['SVM_POLY']:
                    print "SVM_POLY"
                    L1_SVC_POLY_Selector = SVC(C=1, kernel='poly').fit(scaled_train_X, train_y_reduced)

                    predicted_test_y = L1_SVC_POLY_Selector.predict(scaled_test_X)
                    isTest = True; #new
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'SVM_POLY', isTest) + tuple(performance_score(test_y, predicted_test_y).values())) #new

                    predicted_train_y = L1_SVC_POLY_Selector.predict(scaled_train_X)
                    isTest = False; #new
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'SVM_POLY', isTest) + tuple(performance_score(train_y_reduced, predicted_train_y).values()))
                
                min_max_scaler = Preprocessing_Scaler_with_mean_point5()
                X_train_pre_validation_minmax = min_max_scaler.fit(train_X_reduced)
                X_train_pre_validation_minmax = min_max_scaler.transform(train_X_reduced)
                x_test_minmax = min_max_scaler.transform(test_X)
                
                x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax = train_test_split(X_train_pre_validation_minmax, 
                                                                                                  train_y_reduced
                                                                    , test_size=0.4, random_state=42)
                finetune_lr = settings['finetune_lr']
                batch_size = settings['batch_size']
                pretraining_epochs = cal_epochs(settings['pretraining_interations'], x_train_minmax, batch_size = batch_size)
                #pretrain_lr=0.001
                pretrain_lr = settings['pretrain_lr']
                training_epochs = cal_epochs(settings['training_interations'], x_train_minmax, batch_size = batch_size)
                hidden_layers_sizes= settings['hidden_layers_sizes']
                corruption_levels = settings['corruption_levels']
                settings['epoch_number'] = cal_epochs(settings['pretraining_interations'], x_train_minmax, batch_size = batch_size)
                # deep xy autoencoders
                settings['lrate'] = settings['lrate_pre'] + str(training_epochs)
                settings['n_ins'] = x_train_minmax.shape[1]
                if settings['DL_xy']:
                    cfg = settings.copy()
                    cfg['weight_y'] = 0.1
                    print 'DL_xy'
                    train_x = x_train_minmax; train_y = y_train_minmax                    
                    sdaf = Sda_xy_factory(cfg)
                    sdaf.sda.pretraining(train_x, train_y) 
                    dnnf = DNN_factory(cfg) 
                    dnnf.dnn.load_pretrain_from_Sda(sdaf.sda)
                    dnnf.dnn.finetuning((x_train_minmax,  y_train_minmax),(x_validation_minmax, y_validation_minmax))
                    
                    training_predicted = dnnf.dnn.predict(x_train_minmax)
                    y_train = y_train_minmax
                    isTest = False; #new
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'DL_xy', isTest) + tuple(performance_score(y_train, training_predicted).values()))

                    test_predicted = dnnf.dnn.predict(x_test_minmax)
                    y_test = test_y
                    isTest = True; #new
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'DL_xy', isTest) + tuple(performance_score(y_test, test_predicted).values()))
                if settings['Sda_xy_with_first']: 
                    cfg = settings.copy()
                    cfg['weight_y'] = 0.1
                    cfg['firstlayer_xy'] = 1
                    print 'firstlayer_xy' 
                    train_x = x_train_minmax; train_y = y_train_minmax                    
                    sdaf = Sda_xy_factory(cfg)
                    sdaf.sda.pretraining(train_x, train_y) 
                    dnnf = DNN_factory(cfg) 
                    dnnf.dnn.load_pretrain_from_Sda(sdaf.sda)
                    dnnf.dnn.finetuning((x_train_minmax,  y_train_minmax),(x_validation_minmax, y_validation_minmax))
                    
                    training_predicted = dnnf.dnn.predict(x_train_minmax)
                    y_train = y_train_minmax
                    isTest = False; #new
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'Sda_xy_with_first', isTest) + tuple(performance_score(y_train, training_predicted).values()))

                    test_predicted = dnnf.dnn.predict(x_test_minmax)
                    y_test = test_y
                    isTest = True; #new
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'Sda_xy_with_first', isTest) + tuple(performance_score(y_test, test_predicted).values()))
                if settings['Sda_new']:
                    print 'Sda_new'
                    cfg = settings.copy()
                    train_x = x_train_minmax; train_y = y_train_minmax                    
                    cfg['n_ins'] = train_x.shape[1]
                    sdaf = Sda_factory(cfg)
                    sda = sdaf.sda.pretraining(train_x = train_x)
                    sdaf.dnn.finetuning((x_train_minmax,  y_train_minmax),(x_validation_minmax, y_validation_minmax))                    
                    training_predicted = sdaf.dnn.predict(x_train_minmax)
                    y_train = y_train_minmax
                    isTest = False; #new
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'Sda_new', isTest) + tuple(performance_score(y_train, training_predicted).values()))

                    test_predicted = sdaf.dnn.predict(x_test_minmax)
                    y_test = test_y
                    isTest = True; #new
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'Sda_new', isTest) + tuple(performance_score(y_test, test_predicted).values()))
                            
                #### new prepresentation
                x = X_train_pre_validation_minmax
                a_MAE_A = pretrain_a_Sda_with_estop(x, pretraining_epochs=pretraining_epochs, pretrain_lr=pretrain_lr, batch_size=batch_size, 
                                        hidden_layers_sizes =hidden_layers_sizes, corruption_levels=corruption_levels)
                new_x_train_minmax_A =  a_MAE_A.transform(X_train_pre_validation_minmax)
                new_x_test_minmax_A =  a_MAE_A.transform(x_test_minmax)
                standard_scaler = preprocessing.StandardScaler().fit(new_x_train_minmax_A)
                new_x_train_scaled = standard_scaler.transform(new_x_train_minmax_A)
                new_x_test_scaled = standard_scaler.transform(new_x_test_minmax_A)
                new_x_train_combo = np.hstack((scaled_train_X, new_x_train_scaled))
                new_x_test_combo = np.hstack((scaled_test_X, new_x_test_scaled))
                
                
                if settings['SAE_SVM']: 
                    print 'SAE followed by SVM'

                    Linear_SVC = LinearSVC(C=1, penalty="l2")
                    Linear_SVC.fit(new_x_train_scaled, train_y_reduced)
                    predicted_test_y = Linear_SVC.predict(new_x_test_scaled)
                    isTest = True; #new
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'SAE_SVM', isTest) + tuple(performance_score(test_y, predicted_test_y).values())) #new
                    predicted_train_y = Linear_SVC.predict(new_x_train_scaled)
                    isTest = False; #new
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'SAE_SVM', isTest) + tuple(performance_score(train_y_reduced, predicted_train_y).values()))
                if settings['SAE_SVM_RBF']: 
                    print 'SAE followed by SVM RBF'
                    x = X_train_pre_validation_minmax
                    L1_SVC_RBF_Selector = SVC(C=1, gamma=0.01, kernel='rbf').fit(new_x_train_scaled, train_y_reduced)
                    predicted_test_y = L1_SVC_RBF_Selector.predict(new_x_test_scaled)
                    isTest = True; #new
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'SAE_SVM_RBF', isTest) + tuple(performance_score(test_y, predicted_test_y).values())) #new
                    predicted_train_y = L1_SVC_RBF_Selector.predict(new_x_train_scaled)
                    isTest = False; #new
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'SAE_SVM_RBF', isTest) + tuple(performance_score(train_y_reduced, predicted_train_y).values()))
                if settings['SAE_SVM_COMBO']: 
                    print 'SAE followed by SVM with combo feature'
                    Linear_SVC = LinearSVC(C=1, penalty="l2")
                    Linear_SVC.fit(new_x_train_combo, train_y_reduced)
                    predicted_test_y = Linear_SVC.predict(new_x_test_combo)
                    isTest = True; #new
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'SAE_SVM_COMBO', isTest) + tuple(performance_score(test_y, predicted_test_y).values())) #new
                    predicted_train_y = Linear_SVC.predict(new_x_train_combo)
                    isTest = False; #new
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'SAE_SVM_COMBO', isTest) + tuple(performance_score(train_y_reduced, predicted_train_y).values()))                                
                if settings['SAE_SVM_RBF_COMBO']: 
                    print 'SAE followed by SVM RBF with combo feature'
                    L1_SVC_RBF_Selector = SVC(C=1, gamma=0.01, kernel='rbf').fit(new_x_train_combo, train_y_reduced)
                    predicted_test_y = L1_SVC_RBF_Selector.predict(new_x_test_combo)        
                    isTest = True; #new
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'SAE_SVM_RBF_COMBO', isTest) + tuple(performance_score(test_y, predicted_test_y).values())) #new
                    predicted_train_y = L1_SVC_RBF_Selector.predict(new_x_train_combo)
                    isTest = False; #new
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'SAE_SVM_RBF_COMBO', isTest) + tuple(performance_score(train_y_reduced, predicted_train_y).values()))                                                                  
                    
                if settings['DL']:
                    print "direct deep learning"
                    sda = train_a_Sda(x_train_minmax, pretrain_lr, finetune_lr,
                                      y_train_minmax,
                                 x_validation_minmax, y_validation_minmax , 
                                 x_test_minmax, test_y,
                                 hidden_layers_sizes = hidden_layers_sizes, corruption_levels = corruption_levels, batch_size = batch_size , \
                                 training_epochs = training_epochs, pretraining_epochs = pretraining_epochs, n_outs = settings['n_outs'] 
                                 
                     )
                    print 'hidden_layers_sizes:', hidden_layers_sizes
                    print 'corruption_levels:', corruption_levels
                    training_predicted = sda.predict(x_train_minmax)
                    y_train = y_train_minmax
                    isTest = False; #new
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'DL', isTest) + tuple(performance_score(y_train, training_predicted).values()))

                    test_predicted = sda.predict(x_test_minmax)
                    y_test = test_y
                    isTest = True; #new
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'DL', isTest) + tuple(performance_score(y_test, test_predicted).values()))
                if settings['DL_old']:
                    print "direct deep learning old without early stop"
                    sda = trainSda(x_train_minmax, y_train,
                     x_validation_minmax, y_validation_minmax,
                     x_test_minmax, y_test,pretrain_lr, finetune_lr,
                     pretraining_X_minmax=None,
                                 hidden_layers_sizes = hidden_layers_sizes, corruption_levels = corruption_levels, batch_size = batch_size , \
                                 training_epochs = training_epochs, pretraining_epochs = pretraining_epochs, n_outs = settings['n_outs'] 
                                 
                     )

                    print 'hidden_layers_sizes:', hidden_layers_sizes
                    print 'corruption_levels:', corruption_levels
                    training_predicted = sda.predict(x_train_minmax)
                    y_train = y_train_minmax
                    isTest = False; #new
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'DL_old', isTest) + tuple(performance_score(y_train, training_predicted).values()))

                    test_predicted = sda.predict(x_test_minmax)
                    y_test = test_y
                    isTest = True; #new
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'DL_old', isTest) + tuple(performance_score(y_test, test_predicted).values()))                
                if settings['DL_U']:
                # deep learning using unlabeled data for pretraining
                    print 'deep learning with unlabel data'
                    pretraining_X_minmax = min_max_scaler.transform(train_X_10fold)
                    pretraining_epochs = cal_epochs(settings['pretraining_interations'], x_train_minmax, batch_size = batch_size)
                    sda_unlabel = trainSda(x_train_minmax, y_train_minmax,
                                 x_validation_minmax, y_validation_minmax , 
                                 x_test_minmax, test_y, 
                                 pretraining_X_minmax = pretraining_X_minmax,
                                 hidden_layers_sizes = hidden_layers_sizes, corruption_levels = corruption_levels, batch_size = batch_size , \
                                 training_epochs = training_epochs, pretraining_epochs = pretraining_epochs, 
                                 pretrain_lr = pretrain_lr, finetune_lr=finetune_lr, n_outs = settings['n_outs']
                     )
                    print 'hidden_layers_sizes:', hidden_layers_sizes
                    print 'corruption_levels:', corruption_levels
                    training_predicted = sda_unlabel.predict(x_train_minmax)
                    y_train = y_train_minmax
                    isTest = False; #new
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'DL_U', isTest) + tuple(performance_score(y_train, training_predicted, with_auc_score).values()))

                    test_predicted = sda_unlabel.predict(x_test_minmax)
                    y_test = test_y

                    isTest = True; #new
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'DL_U', isTest) + tuple(performance_score(y_test, test_predicted, with_auc_score).values()))
                if settings['DL_S']:
                    # deep learning using split network
                    y_test = test_y
                    print 'deep learning using split network'
                    # get the new representation for A set. first 784-D
                    pretraining_epochs = cal_epochs(settings['pretraining_interations'], x_train_minmax, batch_size = batch_size)
                    
                    x = x_train_minmax[:, :x_train_minmax.shape[1]/2]
                    print "original shape for A", x.shape
                    a_MAE_A = pretrain_a_Sda_with_estop(x, pretraining_epochs=pretraining_epochs, pretrain_lr=pretrain_lr, batch_size=batch_size, 
                                            hidden_layers_sizes =hidden_layers_sizes, corruption_levels=corruption_levels)
                    new_x_train_minmax_A =  a_MAE_A.transform(x_train_minmax[:, :x_train_minmax.shape[1]/2])
                    x = x_train_minmax[:, x_train_minmax.shape[1]/2:]
                    
                    print "original shape for B", x.shape
                    a_MAE_B = pretrain_a_Sda_with_estop(x, pretraining_epochs=pretraining_epochs, pretrain_lr=pretrain_lr, batch_size=batch_size, 
                                            hidden_layers_sizes =hidden_layers_sizes, corruption_levels=corruption_levels)
                    new_x_train_minmax_B =  a_MAE_B.transform(x_train_minmax[:, x_train_minmax.shape[1]/2:])
                    
                    new_x_test_minmax_A = a_MAE_A.transform(x_test_minmax[:, :x_test_minmax.shape[1]/2])
                    new_x_test_minmax_B = a_MAE_B.transform(x_test_minmax[:, x_test_minmax.shape[1]/2:])
                    new_x_validation_minmax_A = a_MAE_A.transform(x_validation_minmax[:, :x_validation_minmax.shape[1]/2])
                    new_x_validation_minmax_B = a_MAE_B.transform(x_validation_minmax[:, x_validation_minmax.shape[1]/2:])
                    new_x_train_minmax_whole = np.hstack((new_x_train_minmax_A, new_x_train_minmax_B))
                    new_x_test_minmax_whole = np.hstack((new_x_test_minmax_A, new_x_test_minmax_B))
                    new_x_validationt_minmax_whole = np.hstack((new_x_validation_minmax_A, new_x_validation_minmax_B))

                    
                    sda_transformed = train_a_Sda(new_x_train_minmax_whole, pretrain_lr, finetune_lr,
                                                  y_train_minmax,
                         new_x_validationt_minmax_whole, y_validation_minmax , 
                         new_x_test_minmax_whole, y_test,
                         hidden_layers_sizes = hidden_layers_sizes, corruption_levels = corruption_levels, batch_size = batch_size , \
                         training_epochs = training_epochs, pretraining_epochs = pretraining_epochs, n_outs = settings['n_outs']
                         
                         )
                    
                    print 'hidden_layers_sizes:', hidden_layers_sizes
                    print 'corruption_levels:', corruption_levels
                    training_predicted = sda_transformed.predict(new_x_train_minmax_whole)
                    y_train = y_train_minmax
                    
                    isTest = False; #new
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'DL_S', isTest) + tuple(performance_score(y_train, training_predicted, with_auc_score).values()))

                    test_predicted = sda_transformed.predict(new_x_test_minmax_whole)
                    y_test = test_y

                    isTest = True; #new
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'DL_S', isTest) + tuple(performance_score(y_test, test_predicted, with_auc_score).values()))
                if settings['DL_S_new']:
                    # deep learning using split network
                    print 'new deep learning using split network'

                    cfg = settings.copy()
                    p_sda = Parellel_Sda_factory(cfg)                    
                    p_sda.supervised_training(x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax)
                    
                    isTest = False #new
                    training_predicted = p_sda.predict(x_train_minmax)
                    y_train = y_train_minmax                                       
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'DL_S_new', isTest) + tuple(performance_score(y_train, training_predicted, with_auc_score).values()))
                    
                    isTest = True #new
                    y_test = test_y
                    test_predicted = p_sda.predict(x_test_minmax)
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'DL_S_new', isTest) + tuple(performance_score(y_test, test_predicted, with_auc_score).values()))            
                if settings['DL_S_new_contraction']:
                    print 'DL_S_new_contraction'
                    cfg = settings.copy()
                    cfg['contraction_level'] = 0.1
                    p_sda = Parellel_Sda_factory(cfg)                    
                    p_sda.supervised_training(x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax)
                    
                    isTest = False #new
                    training_predicted = p_sda.predict(x_train_minmax)
                    y_train = y_train_minmax                                       
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'DL_S_new_contraction', isTest) + tuple(performance_score(y_train, training_predicted, with_auc_score).values()))
                    
                    isTest = True #new
                    y_test = test_y
                    test_predicted = p_sda.predict(x_test_minmax)
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'DL_S_new_contraction', isTest) + tuple(performance_score(y_test, test_predicted, with_auc_score).values()))            
               
                if settings['DL_S_new_sparsity'] == 1:
                    print 'DL_S_new_sparsity'
                    cfg = settings.copy()
                    cfg['sparsity'] = 0.1
                    cfg['sparsity_weight'] = 0.1
                    p_sda = Parellel_Sda_factory(cfg)                    
                    p_sda.supervised_training(x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax)
                    
                    isTest = False #new
                    training_predicted = p_sda.predict(x_train_minmax)
                    y_train = y_train_minmax                                       
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'DL_S_new_sparsity', isTest) + tuple(performance_score(y_train, training_predicted, with_auc_score).values()))
                    
                    isTest = True #new
                    y_test = test_y
                    test_predicted = p_sda.predict(x_test_minmax)
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'DL_S_new_sparsity', isTest) + tuple(performance_score(y_test, test_predicted, with_auc_score).values()))            
               
                if settings['DL_S_new_weight_decay'] == 2:
                    cfg = settings.copy()
                    cfg['l2_reg'] =0.1
                    print 'l2_reg'
                    p_sda = Parellel_Sda_factory(cfg)                    
                    p_sda.supervised_training(x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax)
                    
                    isTest = False #new
                    training_predicted = p_sda.predict(x_train_minmax)
                    y_train = y_train_minmax                                       
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'l2_reg', isTest) + tuple(performance_score(y_train, training_predicted, with_auc_score).values()))
                    
                    isTest = True #new
                    y_test = test_y
                    test_predicted = p_sda.predict(x_test_minmax)
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'l2_reg', isTest) + tuple(performance_score(y_test, test_predicted, with_auc_score).values()))            
                    
                if settings['DL_S_new_weight_decay'] == 1:
                    print 'l1_reg'
                    cfg = settings.copy()
                    cfg['l1_reg'] =0.1 
                    p_sda = Parellel_Sda_factory(cfg)                    
                    p_sda.supervised_training(x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax)
                    
                    isTest = False #new
                    training_predicted = p_sda.predict(x_train_minmax)
                    y_train = y_train_minmax                                       
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'l1_reg', isTest) + tuple(performance_score(y_train, training_predicted, with_auc_score).values()))
                    
                    isTest = True #new
                    y_test = test_y
                    test_predicted = p_sda.predict(x_test_minmax)
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'l1_reg', isTest) + tuple(performance_score(y_test, test_predicted, with_auc_score).values()))            
                                     
                if settings['DL_S_new_Drop_out'] == 1:
                    
                    cfg = settings.copy()
                    cfg['dropout_factor'] = 0.3
                    print 'DL_S_new_Drop_out'
                    p_sda = Parellel_Sda_factory(cfg)                    
                    p_sda.supervised_training(x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax)
                    
                    isTest = False #new
                    training_predicted = p_sda.predict(x_train_minmax)
                    y_train = y_train_minmax                                       
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'DL_S_new_Drop_out', isTest) + tuple(performance_score(y_train, training_predicted, with_auc_score).values()))
                    
                    isTest = True #new
                    y_test = test_y
                    test_predicted = p_sda.predict(x_test_minmax)
                    analysis_scr.append((self.ddi, subset_no, fisher_mode, 'DL_S_new_Drop_out', isTest) + tuple(performance_score(y_test, test_predicted, with_auc_score).values()))            
                                     
            report_name = filename + '_' + '_newDL_'.join(map(str, hidden_layers_sizes)) +                             '_' + str(pretrain_lr) + '_' + str(finetune_lr) + '_' + str(settings['training_interations']) + '_' + current_date
            saveAsCsv(with_auc_score, report_name, performance_score(test_y, predicted_test_y, with_auc_score), analysis_scr)
def run_models(settings = None):
    analysis_scr = []
    with_auc_score = settings['with_auc_score']
    n_outs = settings['settings']
    for subset_no in xrange(1,settings['number_iterations']+1):
        print("Subset:", subset_no)
        
        ################## generate data ###################
        array_A =[]
        array_B =[]
        for i in range(100000):
            array_A.append(np.random.random_integers(0, 59999))
            array_B.append(np.random.random_integers(0, 59999))
        pos_index = []
        neg_index = []
        for index in xrange(100000):
            if y_total[array_A[index]] - y_total[array_B[index]] == 1:
                pos_index.append(index)
            else:
                neg_index.append(index)
        print 'number of positive examples', len(pos_index)
        selected_neg_index= neg_index[ : len(pos_index)]
        
        array_A = np.array(array_A)
        array_B = np.array(array_B)
        index_for_positive_image_A = array_A[pos_index]
        index_for_positive_image_B = array_B[pos_index]
        index_for_neg_image_A = array_A[selected_neg_index]
        index_for_neg_image_B = array_B[selected_neg_index]

        X_pos_A = X_total[index_for_positive_image_A]
        X_pos_B = X_total[index_for_positive_image_B]
        X_pos_whole = np.hstack((X_pos_A,X_pos_B))
        X_neg_A = X_total[index_for_neg_image_A]
        X_neg_B = X_total[index_for_neg_image_B]
        X_neg_whole = np.hstack((X_neg_A, X_neg_B))
        print X_pos_A.shape,  X_pos_B.shape, X_pos_whole.shape
        print X_neg_A.shape,  X_neg_B.shape, X_neg_whole.shape

        X_whole = np.vstack((X_pos_whole, X_neg_whole))
        print X_whole.shape
        y_pos = np.ones(X_pos_whole.shape[0])
        y_neg = np.zeros(X_neg_whole.shape[0])
        y_whole = np.concatenate([y_pos,y_neg])
        print y_whole.shape
        
        x_train_pre_validation, x_test, y_train_pre_validation, y_test = train_test_split(X_whole,y_whole,                                                            test_size=0.2, random_state=211)
        for number_of_training in settings['number_of_training']:
            
            x_train, x_validation, y_train, y_validation = train_test_split(x_train_pre_validation[:number_of_training],
                                                                                                        y_train_pre_validation[:number_of_training],\
                                                                        test_size=0.2, random_state=21)
            '''
            x_train, x_validation, y_train, y_validation = train_test_split(x_train_pre_validation[:],
                                                                                                        y_train_pre_validation[:],\
                                                                        test_size=0.4, random_state=21)
            '''
            print x_train.shape, y_train.shape, x_validation.shape, \
            y_validation.shape, x_test.shape, y_test.shape
            x_train_minmax, x_validation_minmax, x_test_minmax = x_train, x_validation, x_test 
            train_X_reduced = x_train
            train_y_reduced = y_train
            test_X = x_test
            test_y = y_test
            y_train_minmax = y_train
            y_validation_minmax = y_validation
            y_test_minmax = y_test
            ###original data###
            ################ end of data ####################
            standard_scaler = preprocessing.StandardScaler().fit(train_X_reduced)
            scaled_train_X = standard_scaler.transform(train_X_reduced)
            scaled_test_X = standard_scaler.transform(test_X)
            if settings['SVM']:
                print "SVM"                   
                Linear_SVC = LinearSVC(C=1, penalty="l2")
                Linear_SVC.fit(scaled_train_X, y_train)
                predicted_test_y = Linear_SVC.predict(scaled_test_X)
                isTest = True; #new
                analysis_scr.append((subset_no, number_of_training, 'SVM', isTest) + tuple(performance_score(test_y, predicted_test_y).values())) #new

                predicted_train_y = Linear_SVC.predict(scaled_train_X)
                isTest = False; #new
                analysis_scr.append(( subset_no,number_of_training, 'SVM', isTest) + tuple(performance_score(train_y_reduced, predicted_train_y).values()))

            if settings['SVM_RBF']:
                print "SVM_RBF"
                L1_SVC_RBF_Selector = SVC(C=1, gamma=0.01, kernel='rbf').fit(scaled_train_X, y_train)
                predicted_test_y = L1_SVC_RBF_Selector.predict(scaled_test_X)
                isTest = True; #new
                analysis_scr.append((subset_no, number_of_training, 'SVM_RBF', isTest) + tuple(performance_score(test_y, predicted_test_y).values())) #new
                predicted_train_y = L1_SVC_RBF_Selector.predict(scaled_train_X)
                isTest = False; #new
                analysis_scr.append((subset_no,number_of_training,  'SVM_RBF', isTest) + tuple(performance_score(train_y_reduced, predicted_train_y).values()))
                
            if settings['SVM_POLY']:
                print "SVM_POLY"
                L1_SVC_POLY_Selector = SVC(C=1, kernel='poly').fit(scaled_train_X, train_y_reduced)

                predicted_test_y = L1_SVC_POLY_Selector.predict(scaled_test_X)
                isTest = True; #new
                analysis_scr.append(( subset_no, number_of_training,'SVM_POLY', isTest) + tuple(performance_score(test_y, predicted_test_y).values())) #new

                predicted_train_y = L1_SVC_POLY_Selector.predict(scaled_train_X)
                isTest = False; #new
                analysis_scr.append((subset_no, number_of_training,'SVM_POLY', isTest) + tuple(performance_score(train_y_reduced, predicted_train_y).values()))

            if settings['Log']:
                print "Log"
                log_clf_l2 = sklearn.linear_model.LogisticRegression(C=1, penalty='l2')
                log_clf_l2.fit(scaled_train_X, train_y_reduced)
                predicted_test_y = log_clf_l2.predict(scaled_test_X)
                isTest = True; #new
                analysis_scr.append((subset_no,number_of_training, 'Log', isTest) + tuple(performance_score(test_y, predicted_test_y).values())) #new
                predicted_train_y = log_clf_l2.predict(scaled_train_X)
                isTest = False; #new
                analysis_scr.append((subset_no, number_of_training,'Log', isTest) + tuple(performance_score(train_y_reduced, predicted_train_y).values()))        

            # direct deep learning 

            finetune_lr = settings['finetune_lr']
            batch_size = settings['batch_size']
            pretraining_epochs = cal_epochs(settings['pretraining_interations'], x_train_minmax, batch_size = batch_size)
            #pretrain_lr=0.001
            pretrain_lr = settings['pretrain_lr']
            training_epochs = cal_epochs(settings['training_interations'], x_train_minmax, batch_size = batch_size)
            hidden_layers_sizes = settings['hidden_layers_sizes']
            corruption_levels = settings['corruption_levels']
            settings['lrate'] = settings['lrate_pre'] + str(training_epochs)
            
            if settings['DL']:
                print "direct deep learning"
                sda = trainSda(x_train_minmax, y_train,
                             x_validation_minmax, y_validation, 
                             x_test_minmax, test_y,
                             hidden_layers_sizes = hidden_layers_sizes, corruption_levels = corruption_levels, batch_size = batch_size , \
                             training_epochs = training_epochs, pretraining_epochs = pretraining_epochs, 
                             pretrain_lr = pretrain_lr, finetune_lr=finetune_lr, n_outs = n_outs
                 )
                print 'hidden_layers_sizes:', hidden_layers_sizes
                print 'corruption_levels:', corruption_levels
                test_predicted = sda.predict(x_test_minmax)
                isTest = True; #new
                analysis_scr.append((subset_no,number_of_training, 'DL', isTest) + tuple(performance_score(y_test, test_predicted).values()))
                training_predicted = sda.predict(x_train_minmax)
                isTest = False; #new
                analysis_scr.append((subset_no,number_of_training, 'DL', isTest) + tuple(performance_score(y_train, training_predicted).values()))

            ####transformed original data####    
            x = train_X_reduced
            a_MAE_original = train_a_MultipleAEs(x, pretraining_epochs=pretraining_epochs, pretrain_lr=pretrain_lr, batch_size=batch_size, 
                                    hidden_layers_sizes =hidden_layers_sizes, corruption_levels=corruption_levels)
            new_x_train_minmax_A =  a_MAE_original.transform(train_X_reduced)
            new_x_test_minmax_A =  a_MAE_original.transform(x_test_minmax)  
            standard_scaler = preprocessing.StandardScaler().fit(new_x_train_minmax_A)
            new_x_train_scaled = standard_scaler.transform(new_x_train_minmax_A)
            new_x_test_scaled = standard_scaler.transform(new_x_test_minmax_A)
            new_x_train_combo = np.hstack((scaled_train_X, new_x_train_scaled))
            new_x_test_combo = np.hstack((scaled_test_X, new_x_test_scaled))

            if settings['SAE_SVM']: 
                # SAE_SVM
                print 'SAE followed by SVM'

                Linear_SVC = LinearSVC(C=1, penalty="l2")
                Linear_SVC.fit(new_x_train_scaled, train_y_reduced)
                predicted_test_y = Linear_SVC.predict(new_x_test_scaled)
                isTest = True; #new
                analysis_scr.append(( subset_no, number_of_training,'SAE_SVM', isTest) + tuple(performance_score(test_y, predicted_test_y).values())) #new

                predicted_train_y = Linear_SVC.predict(new_x_train_scaled)
                isTest = False; #new
                analysis_scr.append(( subset_no, number_of_training,'SAE_SVM', isTest) + tuple(performance_score(train_y_reduced, predicted_train_y).values()))
            if settings['SAE_Log']:
                print 'SAE followed by Log'
                log_clf_l2 = sklearn.linear_model.LogisticRegression(C=1, penalty='l2')
                log_clf_l2.fit(new_x_train_scaled, train_y_reduced)
                predicted_test_y = log_clf_l2.predict(new_x_test_scaled)
                isTest = True; #new
                analysis_scr.append((subset_no,number_of_training, 'SAE_Log', isTest) + tuple(performance_score(test_y, predicted_test_y).values())) #new
                predicted_train_y = log_clf_l2.predict(new_x_train_scaled)
                isTest = False; #new
                analysis_scr.append((subset_no, number_of_training,'SAE_Log', isTest) + tuple(performance_score(train_y_reduced, predicted_train_y).values()))        

            if settings['SAE_SVM_RBF']: 
                # SAE_SVM
                print 'SAE followed by SVM RBF'
                L1_SVC_RBF_Selector = SVC(C=1, gamma=0.01, kernel='rbf').fit(new_x_train_scaled, train_y_reduced)

                predicted_test_y = L1_SVC_RBF_Selector.predict(new_x_test_scaled)
                isTest = True; #new
                analysis_scr.append((subset_no, number_of_training, 'SAE_SVM_RBF', isTest) + tuple(performance_score(test_y, predicted_test_y).values())) #new

                predicted_train_y = L1_SVC_RBF_Selector.predict(new_x_train_scaled)
                isTest = False; #new
                analysis_scr.append((subset_no, number_of_training, 'SAE_SVM_RBF', isTest) + tuple(performance_score(train_y_reduced, predicted_train_y).values()))
            if settings['SAE_SVM_POLY']: 
                # SAE_SVM
                print 'SAE followed by SVM POLY'
                L1_SVC_RBF_Selector = SVC(C=1, kernel='poly').fit(new_x_train_scaled, train_y_reduced)

                predicted_test_y = L1_SVC_RBF_Selector.predict(new_x_test_scaled)
                isTest = True; #new
                analysis_scr.append((subset_no,  number_of_training,'SAE_SVM_POLY', isTest) + tuple(performance_score(test_y, predicted_test_y).values())) #new

                predicted_train_y = L1_SVC_RBF_Selector.predict(new_x_train_scaled)
                isTest = False; #new
                analysis_scr.append((subset_no, number_of_training, 'SAE_SVM_POLY', isTest) + tuple(performance_score(train_y_reduced, predicted_train_y).values()))

            #### separated transformed data ####
            y_test = test_y
            print 'deep learning using split network'
            # get the new representation for A set. first 784-D
            pretraining_epochs = cal_epochs(settings['pretraining_interations'], x_train_minmax, batch_size = batch_size)

            x = x_train_minmax[:, :x_train_minmax.shape[1]/2]
            print "original shape for A", x.shape
            a_MAE_A = train_a_MultipleAEs(x, pretraining_epochs=pretraining_epochs, pretrain_lr=pretrain_lr, batch_size=batch_size, 
                                    hidden_layers_sizes = [x/2 for x in hidden_layers_sizes], corruption_levels=corruption_levels)
            new_x_train_minmax_A =  a_MAE_A.transform(x_train_minmax[:, :x_train_minmax.shape[1]/2])
            x = x_train_minmax[:, x_train_minmax.shape[1]/2:]

            print "original shape for B", x.shape
            a_MAE_B = train_a_MultipleAEs(x, pretraining_epochs=pretraining_epochs, pretrain_lr=pretrain_lr, batch_size=batch_size, 
                                    hidden_layers_sizes = [x/2 for x in hidden_layers_sizes], corruption_levels=corruption_levels)
            new_x_train_minmax_B =  a_MAE_B.transform(x_train_minmax[:, x_train_minmax.shape[1]/2:])

            new_x_test_minmax_A = a_MAE_A.transform(x_test_minmax[:, :x_test_minmax.shape[1]/2])
            new_x_test_minmax_B = a_MAE_B.transform(x_test_minmax[:, x_test_minmax.shape[1]/2:])
            new_x_validation_minmax_A = a_MAE_A.transform(x_validation_minmax[:, :x_validation_minmax.shape[1]/2])
            new_x_validation_minmax_B = a_MAE_B.transform(x_validation_minmax[:, x_validation_minmax.shape[1]/2:])
            new_x_train_minmax_whole = np.hstack((new_x_train_minmax_A, new_x_train_minmax_B))
            new_x_test_minmax_whole = np.hstack((new_x_test_minmax_A, new_x_test_minmax_B))
            new_x_validationt_minmax_whole = np.hstack((new_x_validation_minmax_A, new_x_validation_minmax_B)) 
            standard_scaler = preprocessing.StandardScaler().fit(new_x_train_minmax_whole)
            new_x_train_minmax_whole_scaled = standard_scaler.transform(new_x_train_minmax_whole)
            new_x_test_minmax_whole_scaled = standard_scaler.transform(new_x_test_minmax_whole)            
            if settings['DL_S']:
                # deep learning using split network
                sda_transformed = trainSda(new_x_train_minmax_whole, y_train,
                     new_x_validationt_minmax_whole, y_validation , 
                     new_x_test_minmax_whole, y_test,
                     hidden_layers_sizes = hidden_layers_sizes, corruption_levels = corruption_levels, batch_size = batch_size , \
                     training_epochs = training_epochs, pretraining_epochs = pretraining_epochs, 
                     pretrain_lr = pretrain_lr, finetune_lr=finetune_lr
                     )
                print 'hidden_layers_sizes:', hidden_layers_sizes
                print 'corruption_levels:', corruption_levels

                predicted_test_y = sda_transformed.predict(new_x_test_minmax_whole)
                y_test = test_y
                isTest = True; #new
                analysis_scr.append((subset_no, number_of_training,'DL_S', isTest) + tuple(performance_score(y_test, predicted_test_y, with_auc_score).values()))

                training_predicted = sda_transformed.predict(new_x_train_minmax_whole)
                isTest = False; #new
                analysis_scr.append((subset_no,number_of_training, 'DL_S', isTest) + tuple(performance_score(y_train, training_predicted, with_auc_score).values()))
            if settings['SAE_S_SVM']:
                print 'SAE_S followed by SVM'

                Linear_SVC = LinearSVC(C=1, penalty="l2")
                Linear_SVC.fit(new_x_train_minmax_whole_scaled, train_y_reduced)
                predicted_test_y = Linear_SVC.predict(new_x_test_minmax_whole_scaled)
                isTest = True; #new
                analysis_scr.append(( subset_no, number_of_training,'SAE_S_SVM', isTest) + tuple(performance_score(test_y, predicted_test_y, with_auc_score).values())) #new

                predicted_train_y = Linear_SVC.predict(new_x_train_minmax_whole_scaled)
                isTest = False; #new
                analysis_scr.append(( subset_no,number_of_training, 'SAE_S_SVM', isTest) + tuple(performance_score(train_y_reduced, predicted_train_y, with_auc_score).values()))
            if settings['SAE_S_SVM_RBF']: 
                print 'SAE S followed by SVM RBF'
                L1_SVC_RBF_Selector = SVC(C=1, gamma=0.01, kernel='rbf').fit(new_x_train_minmax_whole_scaled, train_y_reduced)

                predicted_test_y = L1_SVC_RBF_Selector.predict(new_x_test_minmax_whole_scaled)
                isTest = True; #new
                analysis_scr.append((subset_no, number_of_training, 'SAE_S_SVM_RBF', isTest) + tuple(performance_score(test_y, predicted_test_y, with_auc_score).values())) #new

                predicted_train_y = L1_SVC_RBF_Selector.predict(new_x_train_minmax_whole_scaled)
                isTest = False; #new
                analysis_scr.append((subset_no,  number_of_training,'SAE_S_SVM_RBF', isTest) + tuple(performance_score(train_y_reduced, predicted_train_y, with_auc_score).values()))
            if settings['SAE_S_SVM_POLY']: 
                # SAE_SVM
                print 'SAE S followed by SVM POLY'
                L1_SVC_RBF_Selector = SVC(C=1, kernel='poly').fit(new_x_train_minmax_whole_scaled, train_y_reduced)

                predicted_test_y = L1_SVC_RBF_Selector.predict(new_x_test_minmax_whole_scaled)
                isTest = True; #new
                analysis_scr.append((subset_no,  number_of_training,'SAE_S_SVM_POLY', isTest) + tuple(performance_score(test_y, predicted_test_y, with_auc_score).values())) #new

                predicted_train_y = L1_SVC_RBF_Selector.predict(new_x_train_minmax_whole_scaled)
                isTest = False; #new
                analysis_scr.append((subset_no,  number_of_training,'SAE_S_SVM_POLY', isTest) + tuple(performance_score(train_y_reduced, predicted_train_y, with_auc_score).values()))

            settings['epoch_number'] = cal_epochs(settings['pretraining_interations'], x_train_minmax, batch_size = batch_size)
            # deep xy autoencoders
            settings['n_ins'] = x_train_minmax.shape[1]
            if settings['DL_xy']:
                cfg = settings.copy()
                cfg['weight_y'] = 0.1
                print 'DL_xy'
                train_x = x_train_minmax; train_y = y_train_minmax                    
                sdaf = Sda_xy_factory(cfg)
                sdaf.sda.pretraining(train_x, train_y) 
                dnnf = DNN_factory(cfg) 
                dnnf.dnn.load_pretrain_from_Sda(sdaf.sda)
                dnnf.dnn.finetuning((x_train_minmax,  y_train_minmax),(x_validation_minmax, y_validation_minmax))
                
                training_predicted = dnnf.dnn.predict(x_train_minmax)
                y_train = y_train_minmax
                isTest = False; #new
                analysis_scr.append((subset_no, number_of_training, 'DL_xy', isTest) + tuple(performance_score(train_y_reduced, training_predicted, with_auc_score).values()))

                test_predicted = dnnf.dnn.predict(x_test_minmax)
                y_test = test_y
                isTest = True; #new
                analysis_scr.append((subset_no, number_of_training, 'DL_xy', isTest) + tuple(performance_score(test_y, test_predicted, with_auc_score).values()))
            if settings['Sda_xy_with_first']: 
                cfg = settings.copy()
                cfg['weight_y'] = 0.1
                cfg['firstlayer_xy'] = 1
                print 'firstlayer_xy' 
                train_x = x_train_minmax; train_y = y_train_minmax                    
                sdaf = Sda_xy_factory(cfg)
                sdaf.sda.pretraining(train_x, train_y) 
                dnnf = DNN_factory(cfg) 
                dnnf.dnn.load_pretrain_from_Sda(sdaf.sda)
                dnnf.dnn.finetuning((x_train_minmax,  y_train_minmax),(x_validation_minmax, y_validation_minmax))
                
                training_predicted = dnnf.dnn.predict(x_train_minmax)
                y_train = y_train_minmax
                isTest = False; #new
                analysis_scr.append((subset_no, number_of_training, 'Sda_xy_with_first', isTest) + tuple(performance_score(train_y_reduced, training_predicted, with_auc_score).values()))
                test_predicted = dnnf.dnn.predict(x_test_minmax)
                y_test = test_y
                isTest = True; #new
                analysis_scr.append((subset_no, number_of_training, 'Sda_xy_with_first', isTest) + tuple(performance_score(test_y, test_predicted, with_auc_score).values()))
            if settings['Sda_new']:
                print 'Sda_new'
                cfg = settings.copy()
                train_x = x_train_minmax; train_y = y_train_minmax                    
                cfg['n_ins'] = train_x.shape[1]
                sdaf = Sda_factory(cfg)
                sda = sdaf.sda.pretraining(train_x = train_x)
                sdaf.dnn.finetuning((x_train_minmax,  y_train_minmax),(x_validation_minmax, y_validation_minmax))                    
                training_predicted = sdaf.dnn.predict(x_train_minmax)
                y_train = y_train_minmax
                isTest = False; #new
                analysis_scr.append((subset_no, number_of_training, 'Sda_new', isTest) + tuple(performance_score(train_y_reduced, training_predicted, with_auc_score).values()))

                test_predicted = sdaf.dnn.predict(x_test_minmax)
                y_test = test_y
                isTest = True; #new
                analysis_scr.append((subset_no, number_of_training, 'Sda_new', isTest) + tuple(performance_score(test_y, test_predicted, with_auc_score).values()))

            if settings['DL_S_new']:
                # deep learning using split network
                print 'new deep learning using split network'

                cfg = settings.copy()
                p_sda = Parellel_Sda_factory(cfg)                    
                p_sda.supervised_training(x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax)
                
                isTest = False #new
                training_predicted = p_sda.predict(x_train_minmax)
                y_train = y_train_minmax                                       
    
                analysis_scr.append((subset_no, number_of_training, 'DL_S_new', isTest) + tuple(performance_score(train_y_reduced, training_predicted, with_auc_score).values()))
                isTest = True #new
                y_test = test_y
                test_predicted = p_sda.predict(x_test_minmax)
                analysis_scr.append((subset_no, number_of_training, 'DL_S_new', isTest) + tuple(performance_score(test_y, test_predicted, with_auc_score).values()))            
            if settings['DL_S_new_contraction']:
                print 'DL_S_new_contraction'
                cfg = settings.copy()
                cfg['contraction_level'] = 0.01
                p_sda = Parellel_Sda_factory(cfg)                    
                p_sda.supervised_training(x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax)
                
                isTest = False #new
                training_predicted = p_sda.predict(x_train_minmax)
                y_train = y_train_minmax                                       
                analysis_scr.append((subset_no, number_of_training, 'DL_S_new_contraction', isTest) + tuple(performance_score(train_y_reduced, training_predicted, with_auc_score).values()))
                isTest = True #new
                y_test = test_y
                test_predicted = p_sda.predict(x_test_minmax)
                analysis_scr.append((subset_no, number_of_training, 'DL_S_new_contraction', isTest) + tuple(performance_score(test_y, test_predicted, with_auc_score).values()))   
            
            if settings['DL_S_new_sparsity'] == 1:
                print 'DL_S_new_sparsity'
                cfg = settings.copy()
                cfg['sparsity'] = 0.01
                cfg['sparsity_weight'] = 0.01
                p_sda = Parellel_Sda_factory(cfg)                    
                p_sda.supervised_training(x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax)
                
                isTest = False #new
                training_predicted = p_sda.predict(x_train_minmax)
                y_train = y_train_minmax                                       
                analysis_scr.append((subset_no, number_of_training, 'DL_S_new_sparsity', isTest) + tuple(performance_score(train_y_reduced, training_predicted, with_auc_score).values()))
                isTest = True #new
                y_test = test_y
                test_predicted = p_sda.predict(x_test_minmax)           
                analysis_scr.append((subset_no, number_of_training, 'DL_S_new_sparsity', isTest) + tuple(performance_score(test_y, test_predicted, with_auc_score).values()))            
            if settings['DL_S_new_weight_decay'] == 2:
                cfg = settings.copy()
                cfg['l2_reg'] =0.01
                print 'l2_reg'
                p_sda = Parellel_Sda_factory(cfg)                    
                p_sda.supervised_training(x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax)
                
                isTest = False #new
                training_predicted = p_sda.predict(x_train_minmax)
                y_train = y_train_minmax                                       
                analysis_scr.append((subset_no, number_of_training, 'l2_reg', isTest) + tuple(performance_score(train_y_reduced, training_predicted, with_auc_score).values()))                
                isTest = True #new
                y_test = test_y
                test_predicted = p_sda.predict(x_test_minmax)     
                analysis_scr.append((subset_no, number_of_training, 'l2_reg', isTest) + tuple(performance_score(test_y, test_predicted, with_auc_score).values()))     
            if settings['DL_S_new_weight_decay'] == 1:
                print 'l1_reg'
                cfg = settings.copy()
                cfg['l1_reg'] =0.01 
                p_sda = Parellel_Sda_factory(cfg)                    
                p_sda.supervised_training(x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax)
                
                isTest = False #new
                training_predicted = p_sda.predict(x_train_minmax)
                y_train = y_train_minmax                                       
                analysis_scr.append((subset_no, number_of_training, 'l1_reg', isTest) + tuple(performance_score(train_y_reduced, training_predicted, with_auc_score).values())) 
                isTest = True #new
                y_test = test_y
                test_predicted = p_sda.predict(x_test_minmax)
                analysis_scr.append((subset_no, number_of_training, 'l1_reg', isTest) + tuple(performance_score(test_y, test_predicted, with_auc_score).values()))            
                                 
            if settings['DL_S_new_Drop_out'] == 1:
                
                cfg = settings.copy()
                cfg['dropout_factor'] = 0.5
                print 'DL_S_new_Drop_out'
                p_sda = Parellel_Sda_factory(cfg)                    
                p_sda.supervised_training(x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax)
                
                isTest = False #new
                training_predicted = p_sda.predict(x_train_minmax)
                y_train = y_train_minmax                                       
                analysis_scr.append((subset_no, number_of_training, 'DL_S_new_Drop_out', isTest) + tuple(performance_score(train_y_reduced, training_predicted, with_auc_score).values())) 
                isTest = True #new
                y_test = test_y
                test_predicted = p_sda.predict(x_test_minmax)
                analysis_scr.append((subset_no, number_of_training, 'DL_S_new_Drop_out', isTest) + tuple(performance_score(test_y, test_predicted, with_auc_score).values()))            
                         



        report_name = 'DL_handwritten_digits' + '_size_'.join(map(str, hidden_layers_sizes)) + \
                        '_' + str(pretrain_lr) + '_' + str(finetune_lr) + '_' + \
                '_' + str(settings['pretraining_interations']) + '_' + current_date
    saveAsCsv(with_auc_score, report_name, performance_score(test_y, predicted_test_y, with_auc_score), analysis_scr)
    return sda, a_MAE_original, a_MAE_A, a_MAE_B, analysis_scr
def run_models(settings = None):
    analysis_scr = []
    with_auc_score = settings['with_auc_score']
    f = gzip.open('mnist.pkl.gz', 'rb')
    train_set, valid_set, test_set = cPickle.load(f)
    X_train,y_train = train_set
    X_valid,y_valid = valid_set
    X_test,y_test = test_set
    n_outs = settings['n_outs']
    for subset_no in xrange(1,settings['number_iterations']+1):
                print("Subset:", subset_no)

                #(train_X_10fold, train_y_10fold),(train_X_reduced, train_y_reduced), (test_X, test_y) = (X_train[:1000],y_train[:1000]),(X_train[:1000],y_train[:1000]), (X_test[:1000],y_test[:1000])
                (train_X_10fold, train_y_10fold),(train_X_reduced, train_y_reduced), (test_X, test_y) = (X_train,y_train),(X_train,y_train), (X_test,y_test)
                standard_scaler = preprocessing.StandardScaler().fit(train_X_reduced)
                scaled_train_X = standard_scaler.transform(train_X_reduced)
                scaled_test_X = standard_scaler.transform(test_X)
                fisher_mode = settings['fisher_mode']
                
                if settings['SVM']:
                    print "SVM"                   
                    Linear_SVC = LinearSVC(C=1, penalty="l2")
                    Linear_SVC.fit(scaled_train_X, train_y_reduced)
                    predicted_test_y = Linear_SVC.predict(scaled_test_X)
                    isTest = True; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'SVM', isTest) + tuple(performance_score(test_y, predicted_test_y).values())) #new

                    predicted_train_y = Linear_SVC.predict(scaled_train_X)
                    isTest = False; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'SVM', isTest) + tuple(performance_score(train_y_reduced, predicted_train_y).values()))                    
                if settings['SVM_RBF']:
                    print "SVM_RBF"
                    L1_SVC_RBF_Selector = SVC(C=1, gamma=0.01, kernel='rbf').fit(scaled_train_X, train_y_reduced)

                    predicted_test_y = L1_SVC_RBF_Selector.predict(scaled_test_X)
                    isTest = True; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'SVM_RBF', isTest) + tuple(performance_score(test_y, predicted_test_y).values())) #new

                    predicted_train_y = L1_SVC_RBF_Selector.predict(scaled_train_X)
                    isTest = False; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'SVM_RBF', isTest) + tuple(performance_score(train_y_reduced, predicted_train_y).values()))
                if settings['SVM_POLY']:
                    print "SVM_POLY"
                    L1_SVC_POLY_Selector = SVC(C=1, kernel='poly').fit(scaled_train_X, train_y_reduced)

                    predicted_test_y = L1_SVC_POLY_Selector.predict(scaled_test_X)
                    isTest = True; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'SVM_POLY', isTest) + tuple(performance_score(test_y, predicted_test_y).values())) #new

                    predicted_train_y = L1_SVC_POLY_Selector.predict(scaled_train_X)
                    isTest = False; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'SVM_POLY', isTest) + tuple(performance_score(train_y_reduced, predicted_train_y).values()))
                
                min_max_scaler = Preprocessing_Scaler_with_mean_point5()
                X_train_pre_validation_minmax = min_max_scaler.fit(train_X_reduced)
                X_train_pre_validation_minmax = min_max_scaler.transform(train_X_reduced)
                x_test_minmax = min_max_scaler.transform(test_X)
                
                x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax = train_test_split(X_train_pre_validation_minmax, 
                                                                                                  train_y_reduced
                                                                    , test_size=0.4, random_state=42)
                finetune_lr = settings['finetune_lr']
                batch_size = settings['batch_size']
                pretraining_epochs = cal_epochs(settings['pretraining_interations'], x_train_minmax, batch_size = batch_size)
                #pretrain_lr=0.001
                pretrain_lr = settings['pretrain_lr']
                training_epochs = cal_epochs(settings['training_interations'], x_train_minmax, batch_size = batch_size)
                settings['lrate'] = settings['lrate_pre'] + str(training_epochs)
                hidden_layers_sizes= settings['hidden_layers_sizes']
                corruption_levels = settings['corruption_levels']
                settings['epoch_number'] = cal_epochs(settings['pretraining_interations'], x_train_minmax, batch_size = batch_size)
                # deep xy autoencoders
                settings['n_ins'] = x_train_minmax.shape[1]
                if settings['DL_xy']:
                    cfg = settings.copy()
                    cfg['weight_y'] = 0.01
                    print 'DL_xy'
                    train_x = x_train_minmax; train_y = y_train_minmax                    
                    sdaf = Sda_xy_factory(cfg)
                    sdaf.sda.pretraining(train_x, train_y) 
                    dnnf = DNN_factory(cfg) 
                    dnnf.dnn.load_pretrain_from_Sda(sdaf.sda)
                    dnnf.dnn.finetuning((x_train_minmax,  y_train_minmax),(x_validation_minmax, y_validation_minmax))
                    
                    training_predicted = dnnf.dnn.predict(x_train_minmax)
                    y_train = y_train_minmax
                    isTest = False; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'DL_xy', isTest) + tuple(performance_score(y_train, training_predicted).values()))

                    test_predicted = dnnf.dnn.predict(x_test_minmax)
                    y_test = test_y
                    isTest = True; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'DL_xy', isTest) + tuple(performance_score(y_test, test_predicted).values()))
                if settings['Sda_xy_with_first']: 
                    cfg = settings.copy()
                    cfg['weight_y'] = 1
                    cfg['firstlayer_xy'] = 1
                    print 'firstlayer_xy' 
                    train_x = x_train_minmax; train_y = y_train_minmax                    
                    sdaf = Sda_xy_factory(cfg)
                    sdaf.sda.pretraining(train_x, train_y) 
                    dnnf = DNN_factory(cfg) 
                    dnnf.dnn.load_pretrain_from_Sda(sdaf.sda)
                    dnnf.dnn.finetuning((x_train_minmax,  y_train_minmax),(x_validation_minmax, y_validation_minmax))
                    
                    training_predicted = dnnf.dnn.predict(x_train_minmax)
                    y_train = y_train_minmax
                    isTest = False; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'Sda_xy_with_first', isTest) + tuple(performance_score(y_train, training_predicted).values()))

                    test_predicted = dnnf.dnn.predict(x_test_minmax)
                    y_test = test_y
                    isTest = True; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'Sda_xy_with_first', isTest) + tuple(performance_score(y_test, test_predicted).values()))
                if settings['Sda_new']:
                    print 'Sda_new'
                    cfg = settings.copy()
                    train_x = x_train_minmax; train_y = y_train_minmax                    
                    cfg['n_ins'] = train_x.shape[1]
                    sdaf = Sda_factory(cfg)
                    sda = sdaf.sda.pretraining(train_x = train_x)
                    sdaf.dnn.finetuning((x_train_minmax,  y_train_minmax),(x_validation_minmax, y_validation_minmax))                    
                    training_predicted = sdaf.dnn.predict(x_train_minmax)
                    y_train = y_train_minmax
                    isTest = False; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'Sda_new', isTest) + tuple(performance_score(y_train, training_predicted).values()))

                    test_predicted = sdaf.dnn.predict(x_test_minmax)
                    y_test = test_y
                    isTest = True; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'Sda_new', isTest) + tuple(performance_score(y_test, test_predicted).values()))
                            
                #### new prepresentation
                x = X_train_pre_validation_minmax
                a_MAE_A = pretrain_a_Sda_with_estop(x, pretraining_epochs=pretraining_epochs, pretrain_lr=pretrain_lr, batch_size=batch_size, 
                                        hidden_layers_sizes =hidden_layers_sizes, corruption_levels=corruption_levels, n_outs = n_outs)
                new_x_train_minmax_A =  a_MAE_A.transform(X_train_pre_validation_minmax)
                new_x_test_minmax_A =  a_MAE_A.transform(x_test_minmax)
                standard_scaler = preprocessing.StandardScaler().fit(new_x_train_minmax_A)
                new_x_train_scaled = standard_scaler.transform(new_x_train_minmax_A)
                new_x_test_scaled = standard_scaler.transform(new_x_test_minmax_A)
                new_x_train_combo = np.hstack((scaled_train_X, new_x_train_scaled))
                new_x_test_combo = np.hstack((scaled_test_X, new_x_test_scaled))
                
                
                if settings['SAE_SVM']: 
                    print 'SAE followed by SVM'

                    Linear_SVC = LinearSVC(C=1, penalty="l2")
                    Linear_SVC.fit(new_x_train_scaled, train_y_reduced)
                    predicted_test_y = Linear_SVC.predict(new_x_test_scaled)
                    isTest = True; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'SAE_SVM', isTest) + tuple(performance_score(test_y, predicted_test_y).values())) #new
                    predicted_train_y = Linear_SVC.predict(new_x_train_scaled)
                    isTest = False; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'SAE_SVM', isTest) + tuple(performance_score(train_y_reduced, predicted_train_y).values()))
                if settings['SAE_SVM_RBF']: 
                    print 'SAE followed by SVM RBF'
                    x = X_train_pre_validation_minmax
                    L1_SVC_RBF_Selector = SVC(C=1, gamma=0.01, kernel='rbf').fit(new_x_train_scaled, train_y_reduced)
                    predicted_test_y = L1_SVC_RBF_Selector.predict(new_x_test_scaled)
                    isTest = True; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'SAE_SVM_RBF', isTest) + tuple(performance_score(test_y, predicted_test_y).values())) #new
                    predicted_train_y = L1_SVC_RBF_Selector.predict(new_x_train_scaled)
                    isTest = False; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'SAE_SVM_RBF', isTest) + tuple(performance_score(train_y_reduced, predicted_train_y).values()))
                if settings['SAE_SVM_COMBO']: 
                    print 'SAE followed by SVM with combo feature'
                    Linear_SVC = LinearSVC(C=1, penalty="l2")
                    Linear_SVC.fit(new_x_train_combo, train_y_reduced)
                    predicted_test_y = Linear_SVC.predict(new_x_test_combo)
                    isTest = True; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'SAE_SVM_COMBO', isTest) + tuple(performance_score(test_y, predicted_test_y).values())) #new
                    predicted_train_y = Linear_SVC.predict(new_x_train_combo)
                    isTest = False; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'SAE_SVM_COMBO', isTest) + tuple(performance_score(train_y_reduced, predicted_train_y).values()))                                
                if settings['SAE_SVM_RBF_COMBO']: 
                    print 'SAE followed by SVM RBF with combo feature'
                    L1_SVC_RBF_Selector = SVC(C=1, gamma=0.01, kernel='rbf').fit(new_x_train_combo, train_y_reduced)
                    predicted_test_y = L1_SVC_RBF_Selector.predict(new_x_test_combo)        
                    isTest = True; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'SAE_SVM_RBF_COMBO', isTest) + tuple(performance_score(test_y, predicted_test_y).values())) #new
                    predicted_train_y = L1_SVC_RBF_Selector.predict(new_x_train_combo)
                    isTest = False; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'SAE_SVM_RBF_COMBO', isTest) + tuple(performance_score(train_y_reduced, predicted_train_y).values()))                                                                  
                    
                if settings['DL']:
                    print "direct deep learning"
                    sda = train_a_Sda(x_train_minmax, pretrain_lr, finetune_lr,
                                      y_train_minmax,
                                 x_validation_minmax, y_validation_minmax , 
                                 x_test_minmax, test_y,
                                 hidden_layers_sizes = hidden_layers_sizes, corruption_levels = corruption_levels, batch_size = batch_size , \
                                 training_epochs = training_epochs, pretraining_epochs = pretraining_epochs, n_outs =n_outs
                                 
                     )
                    print 'hidden_layers_sizes:', hidden_layers_sizes
                    print 'corruption_levels:', corruption_levels
                    training_predicted = sda.predict(x_train_minmax)
                    y_train = y_train_minmax
                    isTest = False; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'DL', isTest) + tuple(performance_score(y_train, training_predicted).values()))

                    test_predicted = sda.predict(x_test_minmax)
                    y_test = test_y
                    isTest = True; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'DL', isTest) + tuple(performance_score(y_test, test_predicted).values()))
                
                if settings['DL_U']:
                # deep learning using unlabeled data for pretraining
                    print 'deep learning with unlabel data'
                    pretraining_X_minmax = min_max_scaler.transform(train_X_10fold)
                    pretraining_epochs = cal_epochs(settings['pretraining_interations'], x_train_minmax, batch_size = batch_size)
                    sda_unlabel = trainSda(x_train_minmax, y_train_minmax,
                                 x_validation_minmax, y_validation_minmax , 
                                 x_test_minmax, test_y, 
                                 pretraining_X_minmax = pretraining_X_minmax,
                                 hidden_layers_sizes = hidden_layers_sizes, corruption_levels = corruption_levels, batch_size = batch_size , \
                                 training_epochs = training_epochs, pretraining_epochs = pretraining_epochs, 
                                 pretrain_lr = pretrain_lr, finetune_lr=finetune_lr, n_outs =n_outs
                     )
                    print 'hidden_layers_sizes:', hidden_layers_sizes
                    print 'corruption_levels:', corruption_levels
                    training_predicted = sda_unlabel.predict(x_train_minmax)
                    y_train = y_train_minmax
                    isTest = False; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'DL_U', isTest) + tuple(performance_score(y_train, training_predicted, with_auc_score).values()))

                    test_predicted = sda_unlabel.predict(x_test_minmax)
                    y_test = test_y

                    isTest = True; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'DL_U', isTest) + tuple(performance_score(y_test, test_predicted, with_auc_score).values()))
                if settings['DL_S']:
                    # deep learning using split network
                    y_test = test_y
                    print 'deep learning using split network'
                    # get the new representation for A set. first 784-D
                    pretraining_epochs = cal_epochs(settings['pretraining_interations'], x_train_minmax, batch_size = batch_size)
                    
                    x = x_train_minmax[:, :x_train_minmax.shape[1]/2]
                    print "original shape for A", x.shape
                    a_MAE_A = pretrain_a_Sda_with_estop(x, pretraining_epochs=pretraining_epochs, pretrain_lr=pretrain_lr, batch_size=batch_size, 
                                            hidden_layers_sizes =hidden_layers_sizes, corruption_levels=corruption_levels, n_outs = n_outs)
                    new_x_train_minmax_A =  a_MAE_A.transform(x_train_minmax[:, :x_train_minmax.shape[1]/2])
                    x = x_train_minmax[:, x_train_minmax.shape[1]/2:]
                    
                    print "original shape for B", x.shape
                    a_MAE_B = pretrain_a_Sda_with_estop(x, pretraining_epochs=pretraining_epochs, pretrain_lr=pretrain_lr, batch_size=batch_size, 
                                            hidden_layers_sizes =hidden_layers_sizes, corruption_levels=corruption_levels, n_outs = n_outs)
                    new_x_train_minmax_B =  a_MAE_B.transform(x_train_minmax[:, x_train_minmax.shape[1]/2:])
                    
                    new_x_test_minmax_A = a_MAE_A.transform(x_test_minmax[:, :x_test_minmax.shape[1]/2])
                    new_x_test_minmax_B = a_MAE_B.transform(x_test_minmax[:, x_test_minmax.shape[1]/2:])
                    new_x_validation_minmax_A = a_MAE_A.transform(x_validation_minmax[:, :x_validation_minmax.shape[1]/2])
                    new_x_validation_minmax_B = a_MAE_B.transform(x_validation_minmax[:, x_validation_minmax.shape[1]/2:])
                    new_x_train_minmax_whole = np.hstack((new_x_train_minmax_A, new_x_train_minmax_B))
                    new_x_test_minmax_whole = np.hstack((new_x_test_minmax_A, new_x_test_minmax_B))
                    new_x_validationt_minmax_whole = np.hstack((new_x_validation_minmax_A, new_x_validation_minmax_B))

                    
                    sda_transformed = train_a_Sda(new_x_train_minmax_whole, pretrain_lr, finetune_lr,
                                                  y_train_minmax,
                         new_x_validationt_minmax_whole, y_validation_minmax , 
                         new_x_test_minmax_whole, y_test,
                         hidden_layers_sizes = hidden_layers_sizes, corruption_levels = corruption_levels, batch_size = batch_size , \
                         training_epochs = training_epochs, pretraining_epochs = pretraining_epochs, n_outs = n_outs
                         
                         )
                    
                    print 'hidden_layers_sizes:', hidden_layers_sizes
                    print 'corruption_levels:', corruption_levels
                    training_predicted = sda_transformed.predict(new_x_train_minmax_whole)
                    y_train = y_train_minmax
                    
                    isTest = False; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'DL_S', isTest) + tuple(performance_score(y_train, training_predicted, with_auc_score).values()))

                    test_predicted = sda_transformed.predict(new_x_test_minmax_whole)
                    y_test = test_y

                    isTest = True; #new
                    analysis_scr.append(( subset_no, fisher_mode, 'DL_S', isTest) + tuple(performance_score(y_test, test_predicted, with_auc_score).values()))
                if settings['DL_S_new']:
                    # deep learning using split network
                    print 'new deep learning using split network'

                    cfg = settings.copy()
                    p_sda = Parellel_Sda_factory(cfg)                    
                    p_sda.supervised_training(x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax)
                    
                    isTest = False #new
                    training_predicted = p_sda.predict(x_train_minmax)
                    y_train = y_train_minmax                                       
                    analysis_scr.append(( subset_no, fisher_mode, 'DL_S_new', isTest) + tuple(performance_score(y_train, training_predicted, with_auc_score).values()))
                    
                    isTest = True #new
                    y_test = test_y
                    test_predicted = p_sda.predict(x_test_minmax)
                    analysis_scr.append(( subset_no, fisher_mode, 'DL_S_new', isTest) + tuple(performance_score(y_test, test_predicted, with_auc_score).values()))            
                if settings['DL_S_new_contraction']:
                    print 'DL_S_new_contraction'
                    cfg = settings.copy()
                    cfg['contraction_level'] = 0.1
                    p_sda = Parellel_Sda_factory(cfg)                    
                    p_sda.supervised_training(x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax)
                    
                    isTest = False #new
                    training_predicted = p_sda.predict(x_train_minmax)
                    y_train = y_train_minmax                                       
                    analysis_scr.append(( subset_no, fisher_mode, 'DL_S_new_contraction', isTest) + tuple(performance_score(y_train, training_predicted, with_auc_score).values()))
                    
                    isTest = True #new
                    y_test = test_y
                    test_predicted = p_sda.predict(x_test_minmax)
                    analysis_scr.append(( subset_no, fisher_mode, 'DL_S_new_contraction', isTest) + tuple(performance_score(y_test, test_predicted, with_auc_score).values()))            
               
                if settings['DL_S_new_sparsity'] == 1:
                    print 'DL_S_new_sparsity'
                    cfg = settings.copy()
                    cfg['sparsity'] = 0.01
                    cfg['sparsity_weight'] = 0.01
                    p_sda = Parellel_Sda_factory(cfg)                    
                    p_sda.supervised_training(x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax)
                    
                    isTest = False #new
                    training_predicted = p_sda.predict(x_train_minmax)
                    y_train = y_train_minmax                                       
                    analysis_scr.append(( subset_no, fisher_mode, 'DL_S_new_sparsity', isTest) + tuple(performance_score(y_train, training_predicted, with_auc_score).values()))
                    
                    isTest = True #new
                    y_test = test_y
                    test_predicted = p_sda.predict(x_test_minmax)
                    analysis_scr.append(( subset_no, fisher_mode, 'DL_S_new_sparsity', isTest) + tuple(performance_score(y_test, test_predicted, with_auc_score).values()))            
               
                if settings['DL_S_new_weight_decay'] == 2:
                    cfg = settings.copy()
                    cfg['l2_reg'] =0.01
                    print 'l2_reg'
                    p_sda = Parellel_Sda_factory(cfg)                    
                    p_sda.supervised_training(x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax)
                    
                    isTest = False #new
                    training_predicted = p_sda.predict(x_train_minmax)
                    y_train = y_train_minmax                                       
                    analysis_scr.append(( subset_no, fisher_mode, 'l2_reg', isTest) + tuple(performance_score(y_train, training_predicted, with_auc_score).values()))
                    
                    isTest = True #new
                    y_test = test_y
                    test_predicted = p_sda.predict(x_test_minmax)
                    analysis_scr.append(( subset_no, fisher_mode, 'l2_reg', isTest) + tuple(performance_score(y_test, test_predicted, with_auc_score).values()))            
                    
                if settings['DL_S_new_weight_decay'] == 1:
                    print 'l1_reg'
                    cfg = settings.copy()
                    cfg['l1_reg'] =0.01 
                    p_sda = Parellel_Sda_factory(cfg)                    
                    p_sda.supervised_training(x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax)
                    
                    isTest = False #new
                    training_predicted = p_sda.predict(x_train_minmax)
                    y_train = y_train_minmax                                       
                    analysis_scr.append(( subset_no, fisher_mode, 'l1_reg', isTest) + tuple(performance_score(y_train, training_predicted, with_auc_score).values()))
                    
                    isTest = True #new
                    y_test = test_y
                    test_predicted = p_sda.predict(x_test_minmax)
                    analysis_scr.append(( subset_no, fisher_mode, 'l1_reg', isTest) + tuple(performance_score(y_test, test_predicted, with_auc_score).values()))            
                                     
                if settings['DL_S_new_Drop_out'] == 1:
                    
                    cfg = settings.copy()
                    cfg['dropout_factor'] = 0.5
                    print 'DL_S_new_Drop_out'
                    p_sda = Parellel_Sda_factory(cfg)                    
                    p_sda.supervised_training(x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax)
                    
                    isTest = False #new
                    training_predicted = p_sda.predict(x_train_minmax)
                    y_train = y_train_minmax                                       
                    analysis_scr.append(( subset_no, fisher_mode, 'DL_S_new_Drop_out', isTest) + tuple(performance_score(y_train, training_predicted, with_auc_score).values()))
                    
                    isTest = True #new
                    y_test = test_y
                    test_predicted = p_sda.predict(x_test_minmax)
                    analysis_scr.append(( subset_no, fisher_mode, 'DL_S_new_Drop_out', isTest) + tuple(performance_score(y_test, test_predicted, with_auc_score).values()))            
                                     
                report_name = 'Hand_classification_' + '_'.join(map(str, hidden_layers_sizes)) + '_' + str(pretrain_lr) + '_' + str(finetune_lr) + '_' + str(settings['training_interations']) + '_' + current_date
                saveAsCsv(with_auc_score, report_name, performance_score(test_y, predicted_test_y, with_auc_score), analysis_scr)