from os import path import sys sys.path.append(path.dirname(path.dirname(path.abspath(__file__)))) from preprocessor import * from sklearn.cross_validation import KFold from seqlearn.perceptron import StructuredPerceptron from seqlearn.evaluation import bio_f_score import numpy as np # p = Preprocessor(inputdir="../.././../../brat/data/10k-wikipedia/00") p = Preprocessor() kf = KFold(p.L.shape[0], n_folds=2) for train_ids, test_ids in kf: L_train = p.L[train_ids] L_test = p.L[test_ids] X_train = np.zeros((0, p.X.shape[1])) y_train = np.zeros((0,)) X_test = np.zeros((0, p.X.shape[1])) y_test = np.zeros((0,)) for i, l in enumerate(L_train): start = sum(L_train[:i]) end = sum(L_train[:i+1]) X_train = np.vstack([X_train, p.X[start:end]]) y_train = np.append(y_train, p.Y[start:end])
# -*- coding: utf-8 -*- """ Created on Tue Sep 6 10:10:40 2016 """ import numpy as np from sklearn.cross_validation import KFold y = np.array([0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2]) print("kfold") kf = KFold(9, n_folds=3) print(len(kf)) for train, test in kf: print(train, test) print() for train, test in kf: print(y[train], y[test]) print() from sklearn.cross_validation import StratifiedKFold print("StratifiedKFold") kf = StratifiedKFold(y, n_folds=3) print(len(kf)) for train, test in kf: print(train, test) print() for train, test in kf:
process_mask[:, 30:] = 0 process_mask_img = nibabel.Nifti1Image(process_mask, mask_img.get_affine()) ### Searchlight computation ################################################### # Make processing parallel # /!\ As each thread will print its progress, n_jobs > 1 could mess up the # information output. n_jobs = 1 ### Define the cross-validation scheme used for validation. # Here we use a KFold cross-validation on the session, which corresponds to # splitting the samples in 4 folds and make 4 runs using each fold as a test # set once and the others as learning sets from sklearn.cross_validation import KFold cv = KFold(y.size, n_folds=4) import nilearn.decoding # The radius is the one of the Searchlight sphere that will scan the volume searchlight = nilearn.decoding.SearchLight(mask_img, process_mask_img=process_mask_img, radius=5.6, n_jobs=n_jobs, verbose=1, cv=cv) searchlight.fit(fmri_img, y) ### F-scores computation ###################################################### from nilearn.input_data import NiftiMasker # For decoding, standardizing is often very important
def main(): parser = argparse.ArgumentParser( description="Transform csv files into numpy array") parser.add_argument('-d', '--data', required=True, help="The data directory") args = parser.parse_args() learning_rate = 0.0001 L1_reg = 0.00 L2_reg = 0.0001 n_epochs = 1000 batch_size = 32 n_hidden = 1000 ds = pickle.load(open(os.path.join(args.data, 'ds.npy'))) trI = ds['trI'] trX = ds['trX'].toarray() trY = ds['trY'].astype(np.int32) teI = ds['teI'] teX = ds['teX'].toarray() allX = np.vstack((trX, teX)) means, stds = calculate_mean_and_std(allX) normailize_by_zvalue(means, stds, allX) #normailize_by_minmax(allX); trX = allX[0:trX.shape[0], :] teX = allX[trX.shape[0]:trX.shape[0] + teX.shape[0], :] kf = KFold(trX.shape[0], n_folds=5) trainIds = None testIds = None for train, test in kf: trainIds = train testIds = test cv_train_X = theano.shared(trX[trainIds, :], 'cv_train_X') cv_test_X = theano.shared(trX[testIds, :], 'cv_test_X') cv_train_Y = theano.shared(trY[trainIds], 'cv_train_Y') cv_test_Y = theano.shared(trY[testIds], 'cv_test_Y') ncvtr = len(trainIds) ncvte = len(testIds) nte = teX.shape[0] n_train_batches = int(np.ceil(len(trainIds) * 1.0 / batch_size)) n_valid_batches = int(np.ceil(len(testIds) * 1.0 / batch_size)) n_test_batches = int(np.ceil(teX.shape[0] * 1.0 / batch_size)) teX = theano.shared(teX) rng = np.random.RandomState(1234) print('... building the model') left = T.lscalar() right = T.lscalar() x = T.matrix('x') y = T.ivector('y') classifier = MLP(rng=rng, input=x, n_in=149, n_hidden=n_hidden, n_out=2) cost = (classifier.negative_log_likelihood(y) + L1_reg * classifier.L1 + L2_reg * classifier.L2_sqr) validate_model = theano.function(inputs=[left, right], outputs=classifier.errors(y), givens={ x: cv_test_X[left:right], y: cv_test_Y[left:right] }) gparams = [T.grad(cost, param) for param in classifier.params] updates = [(param, param - learning_rate * gparam) for param, gparam in zip(classifier.params, gparams)] train_model = theano.function(inputs=[left, right], outputs=cost, updates=updates, givens={ x: cv_train_X[left:right], y: cv_train_Y[left:right] }) test_model = theano.function(inputs=[left, right], outputs=classifier.output, givens={ x: teX[left:right], }) print('... training') # early-stopping parameters patience = 10000 # look as this many examples regardless patience_increase = 2 # wait this much longer when a new best is # found improvement_threshold = 0.995 # a relative improvement of this much is # considered significant validation_frequency = min(n_train_batches, patience // 2) # go through this many # minibatche before checking the network # on the validation set; in this case we # check every epoch best_validation_loss = np.inf best_iter = 0 test_score = 0. start_time = timeit.default_timer() epoch = 0 done_looping = False while (epoch < n_epochs) and (not done_looping): epoch = epoch + 1 for minibatch_index in range(n_train_batches): minibatch_avg_cost = train_model( minibatch_index * batch_size, min((minibatch_index + 1) * batch_size, ncvtr)) iter = (epoch - 1) * n_train_batches + minibatch_index if (iter + 1) % validation_frequency == 0: validation_losses = [ validate_model(i * batch_size, min((i + 1) * batch_size, ncvte)) for i in range(n_valid_batches) ] this_validation_loss = np.mean(validation_losses) print('epoch %i, minibatch %i/%i, validation error %f %%' % (epoch, minibatch_index + 1, n_train_batches, this_validation_loss * 100.)) if this_validation_loss < best_validation_loss: if (this_validation_loss < best_validation_loss * improvement_threshold): patience = max(patience, iter * patience_increase) best_validation_loss = this_validation_loss best_iter = iter if patience <= iter: done_looping = True break end_time = timeit.default_timer() print(('Optimization complete. Best validation score of %f %% ' 'obtained at iteration %i, with test performance %f %%') % (best_validation_loss * 100., best_iter + 1, test_score * 100.)) probs = np.vstack([ test_model(i * batch_size, min((i + 1) * batch_size, nte)) for i in range(n_test_batches) ]) fid = open('/local/db/uqdxingz/Santander/sub/mlp.csv', 'w') fid.write('ID,TARGET\n') for i in range(len(teI)): fid.write('%d,%.9f\n' % (int(teI[i]), probs[i][1])) fid.close()
def make_mf_sliced_classification(subset_tr, subset_te, clf, n_round=3, target_col='median_relevance'): print '\n [make_mf_slice]' print clf mf_tr = np.zeros(len(subset_tr)) mf_te = np.zeros(len(subset_te)) #query-slice for cur_query in subset_tr.query_stem.value_counts().index: mask_tr = subset_tr.query_stem == cur_query mask_te = subset_te.query_stem == cur_query # build Bow vect = CountVectorizer(min_df=1, ngram_range=(1, 2)) txts = (list((subset_tr[mask_tr]['title_ext']).values) + list( (subset_te[mask_te]['title_ext']).values)) vect.fit(txts) X_loc_base = vect.transform( list((subset_tr[mask_tr]['title_ext']).values)).todense() X_loc_hold = vect.transform( list((subset_te[mask_te]['title_ext']).values)).todense() y_loc_train = subset_tr[mask_tr][target_col].values # intersect terms feat_counts = np.array(np.sum(X_loc_base, axis=0))[0] * np.array( np.sum(X_loc_hold, axis=0))[0] feat_mask = np.where(feat_counts > 0)[0] # build final feats matrix X_loc_base = np.hstack( (X_loc_base[:, feat_mask], subset_tr[mask_tr][feat_list])) X_loc_hold = np.hstack( (X_loc_hold[:, feat_mask], subset_te[mask_te][feat_list])) # metafeatures iterators tmp_tr = np.zeros(sum(mask_tr)) tmp_te = np.zeros(sum(mask_te)) #print y_loc_train.shape, X_loc_base.shape for i in range(n_round): kf = KFold(len(y_loc_train), n_folds=2, shuffle=True, random_state=42 + i * 1000) for ind_tr, ind_te in kf: X_tr = X_loc_base[ind_tr] X_te = X_loc_base[ind_te] y_tr = y_loc_train[ind_tr] y_te = y_loc_train[ind_te] clf.fit(X_tr, y_tr) tmp_tr[ind_te] += clf.predict(X_te) tmp_te += clf.predict(X_loc_hold) * 0.5 mf_tr[mask_tr.values] = tmp_tr / n_round mf_te[mask_te.values] = tmp_te / n_round y_valid = subset_tr[target_col].values kappa = pykappa.quadratic_weighted_kappa(y_valid, np.round(mf_tr)) acc = np.mean(y_valid == np.round(mf_tr)) print '[{}] kappa:{}, acc:{}'.format(i, kappa, acc) return (mf_tr, mf_te)
def run_cross_validation_create_models(nfolds=10): # input image dimensions batch_size = 16 nb_epoch = 25 random_state = 51 restore_from_last_checkpoint = 1 train_data, train_target, train_id, driver_id, unique_drivers = read_and_normalize_train_data( ) yfull_train = dict() kf = KFold(len(unique_drivers), n_folds=nfolds, shuffle=True, random_state=random_state) num_fold = 0 sum_score = 0 for train_drivers, test_drivers in kf: model = VGG_16() unique_list_train = [unique_drivers[i] for i in train_drivers] X_train, Y_train, train_index = copy_selected_drivers( train_data, train_target, driver_id, unique_list_train) unique_list_valid = [unique_drivers[i] for i in test_drivers] X_valid, Y_valid, test_index = copy_selected_drivers( train_data, train_target, driver_id, unique_list_valid) num_fold += 1 print('Start KFold number {} from {}'.format(num_fold, nfolds)) print('Split train: ', len(X_train), len(Y_train)) print('Split valid: ', len(X_valid), len(Y_valid)) print('Train drivers: ', unique_list_train) print('Test drivers: ', unique_list_valid) kfold_weights_path = os.path.join( 'cache', 'weights_kfold_vgg16_' + str(num_fold) + '.h5') if not os.path.isfile( kfold_weights_path) or restore_from_last_checkpoint == 0: callbacks = [ EarlyStoppingByLossVal(monitor='val_loss', value=0.00001, verbose=1), EarlyStopping(monitor='val_loss', patience=5, verbose=0), ModelCheckpoint(kfold_weights_path, monitor='val_loss', save_best_only=True, verbose=0), ] model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch, shuffle=True, verbose=1, validation_data=(X_valid, Y_valid), callbacks=callbacks) if os.path.isfile(kfold_weights_path): model.load_weights(kfold_weights_path) # score = model.evaluate(X_valid, Y_valid, show_accuracy=True, verbose=0) # print('Score log_loss: ', score[0]) predictions_valid = model.predict(X_valid.astype('float32'), batch_size=batch_size, verbose=1) score = log_loss(Y_valid, predictions_valid) print('Score log_loss: ', score) sum_score += score * len(test_index) # Store valid predictions for i in range(len(test_index)): yfull_train[test_index[i]] = predictions_valid[i] score = sum_score / len(train_data) print("Log_loss train independent avg: ", score) predictions_valid = get_validation_predictions(train_data, yfull_train) print('Final log_loss: {}, nfolds: {} epoch: {}'.format( score, nfolds, nb_epoch)) info_string = 'loss_' + str(score) \ + '_folds_' + str(nfolds) \ + '_ep_' + str(nb_epoch) save_useful_data(predictions_valid, train_id, model, info_string) score1 = log_loss(train_target, predictions_valid) if abs(score1 - score) > 0.0001: print('Check error: {} != {}'.format(score, score1))
from sklearn.externals import joblib import time from sklearn.naive_bayes import MultinomialNB filename = '/Users/jzhy/Downloads/train.csv' data = pd.read_csv(filename) X = numpy.zeros((len(data.x), 4)) X[:, 0] = data.x X[:, 1] = data.y X[:, 2] = data.accuracy X[:, 3] = data.time Y = numpy.zeros((len(data.x), 1)) Y = data.place_id XX = preprocessing.scale(X) YY = numpy.unique(Y) kf = KFold(len(X), n_folds=len(Y) / 10000 + 1) clf = MultinomialNB() i = 0 for train, test in kf: clf.partial_fit(X[test, :], Y[test], YY.reshape((len(YY), -1))) i = i + 1 print i joblib.dump(clf, 'MultinomialNB.pkl') exit()
y = np.array(y) y_tr = y[train_index] x_te = x_train[test_index] clf.train(x_tr, y_tr) oof_train[test_index] = clf.predict(x_te) oof_test_skf[i, :] = clf.predict(x_test) oof_test[:] = oof_test_skf.mean(axis=0) return oof_train.reshape(-1, 1), oof_test.reshape(-1, 1) full_df = pd.concat([train_df, test_df]) sub_item_id = test_df["item_id"] len_train = len(train_df) len_test = len(test_df) kf = KFold(len_train, n_folds=NFOLDS, shuffle=True, random_state=SEED) del train_df, test_df gc.collect() feature_engineering(full_df) full_df, ready_full_df, tfvocab = data_vectorize(full_df) #'alpha':20.0 ridge_params = { 'alpha': 20.0, 'fit_intercept': True, 'normalize': False, 'copy_X': True, 'max_iter': None, 'tol': 0.001,
def __init__(self): # Read train data train_data = [] train_id = [] train_target = [] start_time = time.time() print('Read train images') folders = ['ALB', 'BET', 'DOL', 'LAG', 'NoF', 'OTHER', 'SHARK', 'YFT'] for fld in folders: index = folders.index(fld) print('Load folder {} (Index: {})'.format(fld, index)) path = os.path.join('/Users/Kevin/Desktop/fish_model/train', fld, '*.jpg') files = glob.glob(path) for fl in files: flbase = os.path.basename(fl) img = get_im_cv2(fl) train_data.append(img) train_id.append(flbase) train_target.append(index) print('Read train data time: {} seconds'.format( round(time.time() - start_time, 2))) # Normalize train data print('Convert to numpy...') train_data = np.array(train_data, dtype=np.uint8) train_target = np.array(train_target, dtype=np.uint8) print('Reshape...') train_data = train_data.transpose((0, 3, 1, 2)) print('Convert to float...') train_data = train_data.astype('float32') train_data = train_data / 255 train_target = np_utils.to_categorical(train_target, 8) print('Train shape:', train_data.shape) print(train_data.shape[0], 'train samples') # CNN Model Building model = Sequential() model.add( ZeroPadding2D((1, 1), input_shape=(3, 64, 64), dim_ordering='th')) model.add(Convolution2D(4, 3, 3, activation='relu', dim_ordering='th')) model.add(ZeroPadding2D((1, 1), dim_ordering='th')) model.add(Convolution2D(4, 3, 3, activation='relu', dim_ordering='th')) model.add( MaxPooling2D(pool_size=(2, 2), strides=(2, 2), dim_ordering='th')) model.add(ZeroPadding2D((1, 1), dim_ordering='th')) model.add(Convolution2D(8, 3, 3, activation='relu', dim_ordering='th')) model.add(ZeroPadding2D((1, 1), dim_ordering='th')) model.add(Convolution2D(8, 3, 3, activation='relu', dim_ordering='th')) model.add( MaxPooling2D(pool_size=(2, 2), strides=(2, 2), dim_ordering='th')) model.add(Flatten()) model.add(Dense(32, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(32, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(8, activation='softmax')) sgd = SGD(lr=1e-2, decay=1e-6, momentum=0.9, nesterov=True) model.compile(optimizer=sgd, loss='categorical_crossentropy') # CNN Training nfolds = 20 batch_size = 16 nb_epoch = 30 random_state = 51 yfull_train = dict() kf = KFold(len(train_id), n_folds=nfolds, shuffle=True, random_state=random_state) num_fold = 0 sum_score = 0 models = [] for train_index, test_index in kf: X_train = train_data[train_index] Y_train = train_target[train_index] X_valid = train_data[test_index] Y_valid = train_target[test_index] num_fold += 1 print('Start KFold number {} from {}'.format(num_fold, nfolds)) print('Split train: ', len(X_train), len(Y_train)) print('Split valid: ', len(X_valid), len(Y_valid)) callbacks = [ EarlyStopping(monitor='val_loss', patience=3, verbose=0), ] model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch, shuffle=True, verbose=2, validation_data=(X_valid, Y_valid), callbacks=callbacks) predictions_valid = model.predict(X_valid.astype('float32'), batch_size=batch_size, verbose=2) score = log_loss(Y_valid, predictions_valid) print('Score log_loss: ', score) sum_score += score * len(test_index) # Store valid predictions for i in range(len(test_index)): yfull_train[test_index[i]] = predictions_valid[i] models.append(model) score = sum_score / len(train_data) print("Log_loss train independent avg: ", score) info_string = 'loss_' + str(score) + '_folds_' + str( nfolds) + '_ep_' + str(nb_epoch) # Read test data path = os.path.join('/Users/Kevin/Desktop/fish_model/test', '*.jpg') files = sorted(glob.glob(path)) test_data = [] test_id = [] for fl in files: flbase = os.path.basename(fl) img = get_im_cv2(fl) test_data.append(img) test_id.append(flbase) # Nomilize test data start_time = time.time() test_data, test_id = load_test() test_data = np.array(test_data, dtype=np.uint8) test_data = test_data.transpose((0, 3, 1, 2)) test_data = test_data.astype('float32') test_data = test_data / 255 print('Test shape:', test_data.shape) print(test_data.shape[0], 'test samples') print('Read and process test data time: {} seconds'.format( round(time.time() - start_time, 2))) # CNN Prediction batch_size = 16 num_fold = 0 yfull_test = [] test_id = [] nfolds = len(models) for i in range(nfolds): model = models[i] num_fold += 1 print('Start KFold number {} from {}'.format(num_fold, nfolds)) test_prediction = model.predict(test_data, batch_size=batch_size, verbose=2) yfull_test.append(test_prediction) test_res = merge_several_folds_mean(yfull_test, nfolds) info_string = 'loss_' + info_string \ + '_folds_' + str(nfolds) create_submission(test_res, test_id, info_string)
elif pid==data[1]: no1=no1+1 a1.append(int(data[4])) a2.append(int(data[5])) index.append(no1+no2) else: min_err=100000000000000000 pid=data[1] a1.append(int(data[4])) a2.append(int(data[5])) index.append(no1+no2) no1=no1+1 x, y = genData(no1-1,no2,N,a1,a2,stDev) m, n = np.shape(x) theta = np.ones(n) kf = KFold(len(x), n_folds=4) for train, test in kf: x_train, x_test, y_train, y_test = x[train], x[test], y[train], y[test] #print("%s %s" % (train, test)) #print("%s %s" % (x_train,y_test)) stDev=np.std(y_train) theta = gradientDescent(x_train, y_train,stDev, theta, alpha, m, numIterations) #print(theta) error=0 err=0 StDev=np.std(y_test) for i in range(0, len(x_test)): if(a1[test[i]]>a2[test[i]]+theta[0]+theta[1]*StDev): error=error+1
features.extend(['manager_id_size', 'house_type_size']) # In[13]: features = list(set(features)) processMap(train_df) processMap(test_df) train_df = train_df.fillna(-1) test_df = test_df.fillna(-1) getCluster(train_df, test_df, 30) getCluster(train_df, test_df, 10) #K-FOLD evaluation for the statistic features skf = KFold(len(train_df['interest_level']), 5, shuffle=True, random_state=42) #dev set adding manager skill for train, test in skf: performance_eval(train_df.iloc[train, :], train_df.iloc[test, :], feature='manager_id', update_df=train_df, smoothing=False) temporalManagerPerf_f(train_df.iloc[train, :], train_df.iloc[test, :], update_df=train_df) performance_eval(train_df, test_df, feature='manager_id', smoothing=False) temporalManagerPerf_f(train_df, test_df) #statitstic
path = 'G:\BigDataEvent\血糖预测\\' # test = pd.read_csv(path + 'f_test_a_20180204.csv',encoding='gb2312') # train = pd.read_csv(path + 'f_train_20180204.csv', encoding='gb2312') #train_data, test_data = process_eigen.process_eigen(train, test) train_data = pd.read_csv(path + 'process_train_fb.csv', encoding='gb2312') test_data = pd.read_csv(path + 'process_test_fb.csv', encoding='gb2312') print(train_data.info()) print('开始训练。。。') predictors = [f for f in test_data.columns if f not in ['label', 'id']] train_preds = np.zeros(train_data.shape[0]) test_preds = np.zeros((test_data.shape[0], 5)) kf = KFold(len(train_data), n_folds=5, shuffle=True, random_state=50) for i, (train_index, test_index) in enumerate(kf): print('第{}次训练...'.format(i)) train_feat = train_data.iloc[train_index] valid_feat = train_data.iloc[test_index] estimator = RandomForestClassifier(n_estimators=500, min_samples_split=70, max_depth=12, min_samples_leaf=10, max_features=16, random_state=10, oob_score=True) estimator.fit(train_feat[predictors], train_feat['label']) train_preds[test_index] += estimator.predict(valid_feat[predictors]) print(estimator.oob_score_) test_preds[:, i] = estimator.predict(test_data[predictors])
def get_ten_fold_crossvalid_perfermance(self, settings=None): fisher_mode = settings['fisher_mode'] analysis_scr = [] with_auc_score = settings['with_auc_score'] reduce_ratio = settings['reduce_ratio'] #for seq_no in range(1, self.ddi_obj.total_number_of_sequences+1): #subset_size = math.floor(self.ddi_obj.total_number_of_sequences / 10.0) kf = KFold(self.ddi_obj.total_number_of_sequences, n_folds=10, shuffle=True) #for subset_no in range(1, 11): for ((train_index, test_index), subset_no) in izip(kf, range(1, 11)): #for train_index, test_index in kf; print("Subset:", subset_no) print("Train index: ", train_index) print("Test index: ", test_index) #logger.info('subset number: ' + str(subset_no)) (train_X_10fold, train_y_10fold), (train_X_reduced, train_y_reduced), ( test_X, test_y) = self.ddi_obj.get_ten_fold_crossvalid_one_subset( train_index, test_index, fisher_mode=fisher_mode, reduce_ratio=reduce_ratio) standard_scaler = preprocessing.StandardScaler().fit( train_X_reduced) scaled_train_X = standard_scaler.transform(train_X_reduced) scaled_test_X = standard_scaler.transform(test_X) if settings['SVM']: print "SVM" Linear_SVC = LinearSVC(C=1, penalty="l2") Linear_SVC.fit(scaled_train_X, train_y_reduced) predicted_test_y = Linear_SVC.predict(scaled_test_X) isTest = True #new analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'SVM', isTest) + tuple( performance_score(test_y, predicted_test_y).values())) #new predicted_train_y = Linear_SVC.predict(scaled_train_X) isTest = False #new analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'SVM', isTest) + tuple( performance_score(train_y_reduced, predicted_train_y).values())) if settings['SVM_RBF']: print "SVM_RBF" L1_SVC_RBF_Selector = SVC(C=1, gamma=0.01, kernel='rbf').fit( scaled_train_X, train_y_reduced) predicted_test_y = L1_SVC_RBF_Selector.predict(scaled_test_X) isTest = True #new analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'SVM_RBF', isTest) + tuple( performance_score(test_y, predicted_test_y).values())) #new predicted_train_y = L1_SVC_RBF_Selector.predict(scaled_train_X) isTest = False #new analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'SVM_RBF', isTest) + tuple( performance_score(train_y_reduced, predicted_train_y).values())) if settings['SVM_POLY']: print "SVM_POLY" L1_SVC_POLY_Selector = SVC(C=1, kernel='poly').fit( scaled_train_X, train_y_reduced) predicted_test_y = L1_SVC_POLY_Selector.predict(scaled_test_X) isTest = True #new analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'SVM_POLY', isTest) + tuple( performance_score(test_y, predicted_test_y).values())) #new predicted_train_y = L1_SVC_POLY_Selector.predict( scaled_train_X) isTest = False #new analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'SVM_POLY', isTest) + tuple( performance_score(train_y_reduced, predicted_train_y).values())) min_max_scaler = Preprocessing_Scaler_with_mean_point5() X_train_pre_validation_minmax = min_max_scaler.fit(train_X_reduced) X_train_pre_validation_minmax = min_max_scaler.transform( train_X_reduced) x_test_minmax = min_max_scaler.transform(test_X) x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax = train_test_split( X_train_pre_validation_minmax, train_y_reduced, test_size=0.4, random_state=42) finetune_lr = settings['finetune_lr'] batch_size = settings['batch_size'] pretraining_epochs = cal_epochs( settings['pretraining_interations'], x_train_minmax, batch_size=batch_size) #pretrain_lr=0.001 pretrain_lr = settings['pretrain_lr'] training_epochs = cal_epochs(settings['training_interations'], x_train_minmax, batch_size=batch_size) hidden_layers_sizes = settings['hidden_layers_sizes'] corruption_levels = settings['corruption_levels'] settings['epoch_number'] = cal_epochs( settings['pretraining_interations'], x_train_minmax, batch_size=batch_size) # deep xy autoencoders settings['lrate'] = settings['lrate_pre'] + str(training_epochs) settings['n_ins'] = x_train_minmax.shape[1] if settings['DL_xy']: cfg = settings.copy() cfg['weight_y'] = 0.1 print 'DL_xy' train_x = x_train_minmax train_y = y_train_minmax sdaf = Sda_xy_factory(cfg) sdaf.sda.pretraining(train_x, train_y) dnnf = DNN_factory(cfg) dnnf.dnn.load_pretrain_from_Sda(sdaf.sda) dnnf.dnn.finetuning((x_train_minmax, y_train_minmax), (x_validation_minmax, y_validation_minmax)) training_predicted = dnnf.dnn.predict(x_train_minmax) y_train = y_train_minmax isTest = False #new analysis_scr.append(( self.ddi, subset_no, fisher_mode, 'DL_xy', isTest ) + tuple( performance_score(y_train, training_predicted).values())) test_predicted = dnnf.dnn.predict(x_test_minmax) y_test = test_y isTest = True #new analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'DL_xy', isTest) + tuple(performance_score(y_test, test_predicted).values())) if settings['Sda_xy_with_first']: cfg = settings.copy() cfg['weight_y'] = 0.1 cfg['firstlayer_xy'] = 1 print 'firstlayer_xy' train_x = x_train_minmax train_y = y_train_minmax sdaf = Sda_xy_factory(cfg) sdaf.sda.pretraining(train_x, train_y) dnnf = DNN_factory(cfg) dnnf.dnn.load_pretrain_from_Sda(sdaf.sda) dnnf.dnn.finetuning((x_train_minmax, y_train_minmax), (x_validation_minmax, y_validation_minmax)) training_predicted = dnnf.dnn.predict(x_train_minmax) y_train = y_train_minmax isTest = False #new analysis_scr.append(( self.ddi, subset_no, fisher_mode, 'Sda_xy_with_first', isTest ) + tuple( performance_score(y_train, training_predicted).values())) test_predicted = dnnf.dnn.predict(x_test_minmax) y_test = test_y isTest = True #new analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'Sda_xy_with_first', isTest) + tuple(performance_score(y_test, test_predicted).values())) if settings['Sda_new']: print 'Sda_new' cfg = settings.copy() train_x = x_train_minmax train_y = y_train_minmax cfg['n_ins'] = train_x.shape[1] sdaf = Sda_factory(cfg) sda = sdaf.sda.pretraining(train_x=train_x) sdaf.dnn.finetuning((x_train_minmax, y_train_minmax), (x_validation_minmax, y_validation_minmax)) training_predicted = sdaf.dnn.predict(x_train_minmax) y_train = y_train_minmax isTest = False #new analysis_scr.append(( self.ddi, subset_no, fisher_mode, 'Sda_new', isTest ) + tuple( performance_score(y_train, training_predicted).values())) test_predicted = sdaf.dnn.predict(x_test_minmax) y_test = test_y isTest = True #new analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'Sda_new', isTest) + tuple(performance_score(y_test, test_predicted).values())) #### new prepresentation x = X_train_pre_validation_minmax a_MAE_A = pretrain_a_Sda_with_estop( x, pretraining_epochs=pretraining_epochs, pretrain_lr=pretrain_lr, batch_size=batch_size, hidden_layers_sizes=hidden_layers_sizes, corruption_levels=corruption_levels) new_x_train_minmax_A = a_MAE_A.transform( X_train_pre_validation_minmax) new_x_test_minmax_A = a_MAE_A.transform(x_test_minmax) standard_scaler = preprocessing.StandardScaler().fit( new_x_train_minmax_A) new_x_train_scaled = standard_scaler.transform( new_x_train_minmax_A) new_x_test_scaled = standard_scaler.transform(new_x_test_minmax_A) new_x_train_combo = np.hstack((scaled_train_X, new_x_train_scaled)) new_x_test_combo = np.hstack((scaled_test_X, new_x_test_scaled)) if settings['SAE_SVM']: print 'SAE followed by SVM' Linear_SVC = LinearSVC(C=1, penalty="l2") Linear_SVC.fit(new_x_train_scaled, train_y_reduced) predicted_test_y = Linear_SVC.predict(new_x_test_scaled) isTest = True #new analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'SAE_SVM', isTest) + tuple( performance_score(test_y, predicted_test_y).values())) #new predicted_train_y = Linear_SVC.predict(new_x_train_scaled) isTest = False #new analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'SAE_SVM', isTest) + tuple( performance_score(train_y_reduced, predicted_train_y).values())) if settings['SAE_SVM_RBF']: print 'SAE followed by SVM RBF' x = X_train_pre_validation_minmax L1_SVC_RBF_Selector = SVC(C=1, gamma=0.01, kernel='rbf').fit( new_x_train_scaled, train_y_reduced) predicted_test_y = L1_SVC_RBF_Selector.predict( new_x_test_scaled) isTest = True #new analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'SAE_SVM_RBF', isTest) + tuple( performance_score(test_y, predicted_test_y).values())) #new predicted_train_y = L1_SVC_RBF_Selector.predict( new_x_train_scaled) isTest = False #new analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'SAE_SVM_RBF', isTest) + tuple( performance_score(train_y_reduced, predicted_train_y).values())) if settings['SAE_SVM_COMBO']: print 'SAE followed by SVM with combo feature' Linear_SVC = LinearSVC(C=1, penalty="l2") Linear_SVC.fit(new_x_train_combo, train_y_reduced) predicted_test_y = Linear_SVC.predict(new_x_test_combo) isTest = True #new analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'SAE_SVM_COMBO', isTest) + tuple( performance_score(test_y, predicted_test_y).values())) #new predicted_train_y = Linear_SVC.predict(new_x_train_combo) isTest = False #new analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'SAE_SVM_COMBO', isTest) + tuple( performance_score(train_y_reduced, predicted_train_y).values())) if settings['SAE_SVM_RBF_COMBO']: print 'SAE followed by SVM RBF with combo feature' L1_SVC_RBF_Selector = SVC(C=1, gamma=0.01, kernel='rbf').fit( new_x_train_combo, train_y_reduced) predicted_test_y = L1_SVC_RBF_Selector.predict( new_x_test_combo) isTest = True #new analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'SAE_SVM_RBF_COMBO', isTest) + tuple( performance_score(test_y, predicted_test_y).values())) #new predicted_train_y = L1_SVC_RBF_Selector.predict( new_x_train_combo) isTest = False #new analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'SAE_SVM_RBF_COMBO', isTest) + tuple( performance_score(train_y_reduced, predicted_train_y).values())) if settings['DL']: print "direct deep learning" sda = train_a_Sda(x_train_minmax, pretrain_lr, finetune_lr, y_train_minmax, x_validation_minmax, y_validation_minmax , x_test_minmax, test_y, hidden_layers_sizes = hidden_layers_sizes, corruption_levels = corruption_levels, batch_size = batch_size , \ training_epochs = training_epochs, pretraining_epochs = pretraining_epochs, n_outs = settings['n_outs'] ) print 'hidden_layers_sizes:', hidden_layers_sizes print 'corruption_levels:', corruption_levels training_predicted = sda.predict(x_train_minmax) y_train = y_train_minmax isTest = False #new analysis_scr.append(( self.ddi, subset_no, fisher_mode, 'DL', isTest ) + tuple( performance_score(y_train, training_predicted).values())) test_predicted = sda.predict(x_test_minmax) y_test = test_y isTest = True #new analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'DL', isTest) + tuple(performance_score(y_test, test_predicted).values())) if settings['DL_old']: print "direct deep learning old without early stop" sda = trainSda(x_train_minmax, y_train, x_validation_minmax, y_validation_minmax, x_test_minmax, y_test,pretrain_lr, finetune_lr, pretraining_X_minmax=None, hidden_layers_sizes = hidden_layers_sizes, corruption_levels = corruption_levels, batch_size = batch_size , \ training_epochs = training_epochs, pretraining_epochs = pretraining_epochs, n_outs = settings['n_outs'] ) print 'hidden_layers_sizes:', hidden_layers_sizes print 'corruption_levels:', corruption_levels training_predicted = sda.predict(x_train_minmax) y_train = y_train_minmax isTest = False #new analysis_scr.append(( self.ddi, subset_no, fisher_mode, 'DL_old', isTest ) + tuple( performance_score(y_train, training_predicted).values())) test_predicted = sda.predict(x_test_minmax) y_test = test_y isTest = True #new analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'DL_old', isTest) + tuple(performance_score(y_test, test_predicted).values())) if settings['DL_U']: # deep learning using unlabeled data for pretraining print 'deep learning with unlabel data' pretraining_X_minmax = min_max_scaler.transform(train_X_10fold) pretraining_epochs = cal_epochs( settings['pretraining_interations'], x_train_minmax, batch_size=batch_size) sda_unlabel = trainSda(x_train_minmax, y_train_minmax, x_validation_minmax, y_validation_minmax , x_test_minmax, test_y, pretraining_X_minmax = pretraining_X_minmax, hidden_layers_sizes = hidden_layers_sizes, corruption_levels = corruption_levels, batch_size = batch_size , \ training_epochs = training_epochs, pretraining_epochs = pretraining_epochs, pretrain_lr = pretrain_lr, finetune_lr=finetune_lr, n_outs = settings['n_outs'] ) print 'hidden_layers_sizes:', hidden_layers_sizes print 'corruption_levels:', corruption_levels training_predicted = sda_unlabel.predict(x_train_minmax) y_train = y_train_minmax isTest = False #new analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'DL_U', isTest) + tuple( performance_score(y_train, training_predicted, with_auc_score).values())) test_predicted = sda_unlabel.predict(x_test_minmax) y_test = test_y isTest = True #new analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'DL_U', isTest) + tuple( performance_score(y_test, test_predicted, with_auc_score).values())) if settings['DL_S']: # deep learning using split network y_test = test_y print 'deep learning using split network' # get the new representation for A set. first 784-D pretraining_epochs = cal_epochs( settings['pretraining_interations'], x_train_minmax, batch_size=batch_size) x = x_train_minmax[:, :x_train_minmax.shape[1] / 2] print "original shape for A", x.shape a_MAE_A = pretrain_a_Sda_with_estop( x, pretraining_epochs=pretraining_epochs, pretrain_lr=pretrain_lr, batch_size=batch_size, hidden_layers_sizes=hidden_layers_sizes, corruption_levels=corruption_levels) new_x_train_minmax_A = a_MAE_A.transform( x_train_minmax[:, :x_train_minmax.shape[1] / 2]) x = x_train_minmax[:, x_train_minmax.shape[1] / 2:] print "original shape for B", x.shape a_MAE_B = pretrain_a_Sda_with_estop( x, pretraining_epochs=pretraining_epochs, pretrain_lr=pretrain_lr, batch_size=batch_size, hidden_layers_sizes=hidden_layers_sizes, corruption_levels=corruption_levels) new_x_train_minmax_B = a_MAE_B.transform( x_train_minmax[:, x_train_minmax.shape[1] / 2:]) new_x_test_minmax_A = a_MAE_A.transform( x_test_minmax[:, :x_test_minmax.shape[1] / 2]) new_x_test_minmax_B = a_MAE_B.transform( x_test_minmax[:, x_test_minmax.shape[1] / 2:]) new_x_validation_minmax_A = a_MAE_A.transform( x_validation_minmax[:, :x_validation_minmax.shape[1] / 2]) new_x_validation_minmax_B = a_MAE_B.transform( x_validation_minmax[:, x_validation_minmax.shape[1] / 2:]) new_x_train_minmax_whole = np.hstack( (new_x_train_minmax_A, new_x_train_minmax_B)) new_x_test_minmax_whole = np.hstack( (new_x_test_minmax_A, new_x_test_minmax_B)) new_x_validationt_minmax_whole = np.hstack( (new_x_validation_minmax_A, new_x_validation_minmax_B)) sda_transformed = train_a_Sda(new_x_train_minmax_whole, pretrain_lr, finetune_lr, y_train_minmax, new_x_validationt_minmax_whole, y_validation_minmax , new_x_test_minmax_whole, y_test, hidden_layers_sizes = hidden_layers_sizes, corruption_levels = corruption_levels, batch_size = batch_size , \ training_epochs = training_epochs, pretraining_epochs = pretraining_epochs, n_outs = settings['n_outs'] ) print 'hidden_layers_sizes:', hidden_layers_sizes print 'corruption_levels:', corruption_levels training_predicted = sda_transformed.predict( new_x_train_minmax_whole) y_train = y_train_minmax isTest = False #new analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'DL_S', isTest) + tuple( performance_score(y_train, training_predicted, with_auc_score).values())) test_predicted = sda_transformed.predict( new_x_test_minmax_whole) y_test = test_y isTest = True #new analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'DL_S', isTest) + tuple( performance_score(y_test, test_predicted, with_auc_score).values())) if settings['DL_S_new']: # deep learning using split network print 'new deep learning using split network' cfg = settings.copy() p_sda = Parellel_Sda_factory(cfg) p_sda.supervised_training(x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax) isTest = False #new training_predicted = p_sda.predict(x_train_minmax) y_train = y_train_minmax analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'DL_S_new', isTest) + tuple( performance_score(y_train, training_predicted, with_auc_score).values())) isTest = True #new y_test = test_y test_predicted = p_sda.predict(x_test_minmax) analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'DL_S_new', isTest) + tuple( performance_score(y_test, test_predicted, with_auc_score).values())) if settings['DL_S_new_contraction']: print 'DL_S_new_contraction' cfg = settings.copy() cfg['contraction_level'] = 0.1 p_sda = Parellel_Sda_factory(cfg) p_sda.supervised_training(x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax) isTest = False #new training_predicted = p_sda.predict(x_train_minmax) y_train = y_train_minmax analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'DL_S_new_contraction', isTest) + tuple( performance_score(y_train, training_predicted, with_auc_score).values())) isTest = True #new y_test = test_y test_predicted = p_sda.predict(x_test_minmax) analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'DL_S_new_contraction', isTest) + tuple( performance_score(y_test, test_predicted, with_auc_score).values())) if settings['DL_S_new_sparsity'] == 1: print 'DL_S_new_sparsity' cfg = settings.copy() cfg['sparsity'] = 0.1 cfg['sparsity_weight'] = 0.1 p_sda = Parellel_Sda_factory(cfg) p_sda.supervised_training(x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax) isTest = False #new training_predicted = p_sda.predict(x_train_minmax) y_train = y_train_minmax analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'DL_S_new_sparsity', isTest) + tuple( performance_score(y_train, training_predicted, with_auc_score).values())) isTest = True #new y_test = test_y test_predicted = p_sda.predict(x_test_minmax) analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'DL_S_new_sparsity', isTest) + tuple( performance_score(y_test, test_predicted, with_auc_score).values())) if settings['DL_S_new_weight_decay'] == 2: cfg = settings.copy() cfg['l2_reg'] = 0.1 print 'l2_reg' p_sda = Parellel_Sda_factory(cfg) p_sda.supervised_training(x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax) isTest = False #new training_predicted = p_sda.predict(x_train_minmax) y_train = y_train_minmax analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'l2_reg', isTest) + tuple( performance_score(y_train, training_predicted, with_auc_score).values())) isTest = True #new y_test = test_y test_predicted = p_sda.predict(x_test_minmax) analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'l2_reg', isTest) + tuple( performance_score(y_test, test_predicted, with_auc_score).values())) if settings['DL_S_new_weight_decay'] == 1: print 'l1_reg' cfg = settings.copy() cfg['l1_reg'] = 0.1 p_sda = Parellel_Sda_factory(cfg) p_sda.supervised_training(x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax) isTest = False #new training_predicted = p_sda.predict(x_train_minmax) y_train = y_train_minmax analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'l1_reg', isTest) + tuple( performance_score(y_train, training_predicted, with_auc_score).values())) isTest = True #new y_test = test_y test_predicted = p_sda.predict(x_test_minmax) analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'l1_reg', isTest) + tuple( performance_score(y_test, test_predicted, with_auc_score).values())) if settings['DL_S_new_Drop_out'] == 1: cfg = settings.copy() cfg['dropout_factor'] = 0.3 print 'DL_S_new_Drop_out' p_sda = Parellel_Sda_factory(cfg) p_sda.supervised_training(x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax) isTest = False #new training_predicted = p_sda.predict(x_train_minmax) y_train = y_train_minmax analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'DL_S_new_Drop_out', isTest) + tuple( performance_score(y_train, training_predicted, with_auc_score).values())) isTest = True #new y_test = test_y test_predicted = p_sda.predict(x_test_minmax) analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'DL_S_new_Drop_out', isTest) + tuple( performance_score(y_test, test_predicted, with_auc_score).values())) report_name = filename + '_' + '_newDL_'.join( map(str, hidden_layers_sizes)) + '_' + str( pretrain_lr) + '_' + str(finetune_lr) + '_' + str( settings['training_interations']) + '_' + current_date saveAsCsv(with_auc_score, report_name, performance_score(test_y, predicted_test_y, with_auc_score), analysis_scr)
GradientBoostingClassifier(random_state=1, n_estimators=25, max_depth=3), [ "Pclass", "Sex", "Age", "Fare", "Embarked", "FamilySize", "Title", "FamilyId" ] ], [ LogisticRegression(random_state=1), [ "Pclass", "Sex", "Fare", "FamilySize", "Title", "Age", "Embarked" ] ]] # Initialize the cross-validation folds kf = KFold(train.shape[0], n_folds=3, random_state=1) # In[ ]: predictions = [] for train_tmp, test_tmp in kf: train_target = train["Survived"].iloc[train_tmp] full_test_predictions = [] # Make predictions for each algorithm on each fold for alg, predictors in algorithms: # Fit the algorithm on the training data alg.fit(train[predictors].iloc[train_tmp, :], train_target) # Select and predict on the test fold # We need to use .astype(float) to convert the dataframe to all floats and avoid an sklearn error test_predictions = alg.predict_proba( train[predictors].iloc[test_tmp, :].astype(float))[:, 1]
9.others from shutil import copyfile copyfile(src,file) # make and write file script=open(os.path.join(output_file,'Dodge_'+str(idx)+'.txt'),'a') script.write('1'+'\n') script.close() 10. glob path = os.path.join('..','data','train',fld,'*jpg') files = glob.glob(path) 11. sclearn #K-Folds cross validation iterator from sklearn.cross_validation import KFold kf = KFold(len(X_train), n_folds=n_fold, shuffle=True, random_state=random_state) for train_idx, cv_idx in kf: 12. keras callbacks = [EarlyStopping(monitor='val_loss', patience=3, verbose=0)] model.fit(X_train, Y_train, batch_size=batch_size, nb_epoch=nb_epoch, shuffle=True, verbose=2, validation_data=(X_valid, Y_valid), callbacks=callbacks) model.add(Convolution2D(12,4,4, border_mode='same',trainable=False)) 13. pandas result1 = pd.DataFrame(predictions, columns=['ALB', 'BET', 'DOL', 'LAG', 'NoF', 'OTHER', 'SHARK', 'YFT']) result1.loc[:, 'image'] = pd.Series(test_id, index=result1.index)
max_depth=3), ["Pclass", "Sex", "Age", "Fare", "Embarked", "Title"] ], [ LogisticRegression(random_state=1), ["Pclass", "Sex", "Fare", "Age", "Embarked", "Title"] ], [ neighbors.KNeighborsClassifier(algorithm='kd_tree', n_neighbors=5), ["Pclass", "Sex", "Fare", "Age", "Embarked", "Title"] ], [clf1, ["Pclass", "Sex", "Fare", "Age", "Embarked", "Title"]], ] # 初始化交叉验证 kf = KFold(titanic.shape[0], n_folds=3, random_state=1) predictions = [] for train, test in kf: train_target = titanic["Survived"].iloc[train] full_test_predictions = [] # 对每个交叉验证分组,分别使用两种算法进行分类 for alg, predictors in algorithms: # 用训练集拟合算法 alg.fit(titanic[predictors].iloc[train, :], train_target) # 选择并预测测试集上的输出 # .astype(float) 可以把dataframe转换为浮点数类型 test_predictions = alg.predict_proba( titanic[predictors].iloc[test, :].astype(float))[:, 1] full_test_predictions.append(test_predictions) # 对两个预测结果取平均值
feats = df_train.drop("revenue", axis=1) X = feats.values #features y = df_train["revenue"].values #target for i in range(0, len(y) - 1): if y[i] > 10000000: print("sdfjsd") X.pop(i) y.pop(i) ### Linear Regression ### from sklearn.linear_model import LinearRegression from sklearn.preprocessing import StandardScaler kf = KFold(len(y), n_folds=15, shuffle=True) y_pred = np.zeros(len(y), dtype=y.dtype) # where we'll accumulate predictions lr = LinearRegression() # CV Loop for train_index, test_index in kf: # for each iteration of the for loop we'll do a test train split X_train, X_test = X[train_index], X[test_index] y_train, y_test = y[train_index], y[test_index] t = StandardScaler() X_train = t.fit_transform(X_train) lr.fit(X_train, y_train) # Train on the training data X_test = t.transform(X_test)
metricas = ("Metricas del modelo " + nombreClasificador).capitalize() imprimirTextoCentrado(metricas, tamanoConsola) mostrarMetricasGenerales(model, x_train, y_train, y_pred_train, "train") mostrarMetricasGenerales(model, x_test, y_test, y_pred_test, "test") imprimirTextoCentrado("", tamanoConsola, "*") imprimirTextoCentrado("Metricas importantes para clasificación", tamanoConsola, "*") confusion_matrix_train = mostrarMetricasClasificacion( model, x_train, y_train, y_pred_train, "train") imprimirTextoCentrado("", tamanoConsola, "#") confusion_matrix_test = mostrarMetricasClasificacion( model, x_test, y_test, y_pred_test, "test") #Crear un iterador de validación cruzada k-fold #Nota: Por defecto, la puntuación utilizada es la que se devuelve por el # método de puntuación del estimador (precisión) cv = KFold(n=len(y_train), n_folds=5, shuffle=True, random_state=0) scores = cross_val_score(model, x_train, y_train, cv=cv) print("Scores: ", (scores)) print("Mean score: {0:.3f} (+/-{1:.3f})".format(np.mean(scores), sem(scores))) print( "*******************************************************************") ########################################################################### if generarCompar: #Graficacion #En esta sección se grafican los resultados obtenidos #Se grafican la clasificación emocional con respecto # con respecto a las variables independiente que en este caso son los # pixeles de las imágenes de los rostros. mostrarGraficacionPrediVsReal(x_train, y_train, y_pred_train, "train", nombreClasificador)
def loadDataSet(filename): strArr = [line.strip().split('\t') for line in open(filename).readlines()] dataSet = [map(float, line) for line in strArr] dataMat = np.mat(dataSet) m, n = np.shape(dataMat) return dataMat[:, :n - 1], dataMat[:, -1] if __name__ == "__main__": x, y = loadDataSet('../2-knn/datingTestSet2.txt') #拆分数据集 m = np.shape(x)[0] kf = KFold(m, n_folds=5, shuffle=True) #1000分为5份 clf = neighbors.KNeighborsClassifier(n_neighbors=3) for iteration, data in enumerate(kf, start=1): clf.fit(x[data[0]], np.ravel(y[data[0]])) answer = clf.predict(x[data[1]]) print 'iteration', iteration print(classification_report(y[data[1]], answer)) #训练KNN分类器 # x_train,x_test,y_train,y_test=train_test_split(x,y,test_size=0.2) # clf = neighbors.KNeighborsClassifier(n_neighbors=3) # clf.fit(x_train,np.ravel(y_train)) # answer = clf.predict(x_test) # print(classification_report(y_test,answer)) # precision,recall,thresholds = precision_recall_curve(y_test,answer) 二分类问题
data = featureFormat(my_dataset, features_list) ### split into labels and features (this line assumes that the first ### feature in the array is the label, which is why "poi" must always ### be first in features_list labels, features = targetFeatureSplit(data) ### machine learning goes here! ### please name your classifier clf for easy export below ### deploying feature selection features_train, features_test, labels_train, labels_test = cross_validation.train_test_split( features, labels, test_size=0.1, random_state=42) ### use KFold for split and validate algorithm kf = KFold(len(labels), 3) for train_indices, test_indices in kf: #make training and testing sets features_train = [features[ii] for ii in train_indices] features_test = [features[ii] for ii in test_indices] labels_train = [labels[ii] for ii in train_indices] labels_test = [labels[ii] for ii in test_indices] t0 = time() clf = DecisionTreeClassifier() clf.fit(features_train, labels_train) score = clf.score(features_test, labels_test) print 'accuracy before tuning ', score print "Decision tree algorithm time:", round(time() - t0, 3), "s"
def main(): if len(sys.argv) == 1: print 'need filename' sys.exit(-1) else: infilename = sys.argv[-1] print infilename #npzfile = np.load('data/unigram_bigram_ner_senti_pos_lda_data.npz') npzfile = np.load(infilename) X = npzfile['X']; y = npzfile['y']; #split the data into 8:2 -> training:testing trainX, testX, trainy, testy = train_test_split(X, y, test_size=0.2, random_state=0) print 'feature size: '+str(np.shape(X)) feature_index = set() # store the best feature indices kf = KFold(trainX.shape[0], n_folds=5) # kfold on training set for feature selection for train_index, test_index in kf: trainX_train, trainX_test = trainX[train_index], trainX[test_index] trainy_train, trainy_test = y[train_index], y[test_index] auc_best_global = 0; # best auc in each cross validation xtrainBest = [] # store the best feture matrix for traing section of traingX xtestBest = [] #store the best feture matrix for testing section of traingX residual_col_indices = set() # residual column indices to check for each iteration when adding new features for i in range(0,X.shape[1]): #init the set with all col indices residual_col_indices.add(i) for i in range(0, X.shape[1]): colInd_best = -1; auc_best_local = 0 # init to 0 for colInd in residual_col_indices: if i == 0: # if it's the first feature to add xtrainCur = trainX_train[:,colInd].reshape(trainX_train.shape[0],-1) #convert to a column vector xtestCur = trainX_test[:,colInd].reshape(trainX_test.shape[0],-1) else: xtrainCur = np.hstack((xtrainBest, trainX_train[:,colInd].reshape(trainX_train.shape[0],-1) )) xtestCur = np.hstack((xtestBest, trainX_test[:,colInd].reshape(trainX_test.shape[0],-1) )) clf = LogisticRegression(); clf.fit(xtrainCur, trainy_train) y_true, y_pred = trainy_test, clf.predict(xtestCur) auc = roc_auc_score(y_true, y_pred) # auc score if auc_best_local < auc: auc_best_local = auc colInd_best = colInd print 'auc = ' + str(auc_best_local) + '\tcolInd_best = '+str(colInd_best) if auc_best_global < auc_best_local : # if auc is increasing by adding new features if i == 0: # if it's the first feature to add xtrainBest = trainX_train[:,colInd_best].reshape(trainX_train.shape[0],-1) xtestBest = trainX_test[:,colInd_best].reshape(trainX_test.shape[0],-1) else: xtrainBest = np.hstack((xtrainBest,trainX_train[:,colInd_best].reshape(trainX_train.shape[0],-1))) xtestBest = np.hstack((xtestBest,trainX_test[:,colInd_best].reshape(trainX_test.shape[0],-1))) print 'feature index to add: '+str(colInd_best) feature_index.add(colInd_best) # union of all features selected during each k-fold CV residual_col_indices.remove(colInd_best) auc_best_global = auc_best_local if auc_best_global == 1: break; else: break; print 'auc_best_global found on current trainX_test fold: '+str(auc_best_global) print '# features selected = '+str(len(feature_index)) feature_index = list(feature_index) print 'feature_index = ' + str(feature_index) # should NOT sort feature_index before test! outfilename = infilename[0:-8] +'selected.npz' np.savez(outfilename,X = X[:,feature_index], y = y) clf.fit(trainX[:,feature_index], trainy) testy_true, testy_pred = testy, clf.predict(testX[:,feature_index]) auc_test = roc_auc_score(testy_true, testy_pred) print 'auc test = '+str(auc_test) # ---------------------------------- tune params ---------------------------------- # Set the parameters by cross-validation tuned_parameters = [{}, {'penalty': ['l2'], 'C':np.logspace(-5, 4, 10), 'solver': ['sag'] ,'max_iter':[500] }, {'penalty': ['l2'], 'C':np.logspace(-5, 4, 10), 'solver': ['newton-cg'] ,'max_iter':[500] }, {'penalty': ['l2'], 'C':np.logspace(-5, 4, 10), 'solver': ['lbfgs'] ,'max_iter':[500] }, {'penalty': ['l2','l1'], 'C':np.logspace(-5, 4, 10), 'solver': ['liblinear'] ,'max_iter':[500] } ] clf = GridSearchCV(LogisticRegression(class_weight= 'balanced'), tuned_parameters, cv=5, scoring= None) clf.fit(trainX[:,feature_index], trainy) print("Best parameters set found on development set:") print(clf.best_params_) y_true, y_pred = testy, clf.predict(testX[:,feature_index]) auc = roc_auc_score(testy_true, testy_pred) print 'accuracy = ' + str(accuracy_score(y_true, y_pred)) print 'auc = ' + str(auc)
lasagne.layers.set_all_param_values(net['prob'], d['param values']) for i, (tr_ix, val_ix) in enumerate(kf): print('CV Fold', i) X_tr = X[tr_ix] y_tr = y[tr_ix] X_val = X[val_ix] y_val = y[val_ix] #net['new_output'] = DenseLayer(net['pool5/7x7_s1'], num_units=10, nonlinearity=softmax, W=lasagne.init.Normal(0.01)) lasagne.layers.set_all_param_values(net['prob'], d['param values']) learning_rate.set_value(0.0002) for epoch in range(2): kf2 = KFold(len(y_tr), n_folds=np.floor(len(y_tr) / BATCH_SIZE), shuffle=True, random_state=1) progbar = Progbar(np.floor(len(y_tr) / BATCH_SIZE)) for j, (_, ix) in enumerate(kf2): loss, acc = train_batch(ix) progbar.add(1) learning_rate.set_value(learning_rate.get_value() * learning_rate_decay) v_ix = range(len(y_val)) t_ix = range(len(y_tr)) np.random.shuffle(v_ix) np.random.shuffle(t_ix) tr_loss_tot = 0.
train_feat['id'] = train_feat['id'].apply(lambda x: 0 - int(x[1:]) if 'p' in x else int(x[1:])) test_feat['id'] = test_feat['id'].apply(lambda x: 0 - int(x[1:]) if 'p' in x else int(x[1:])) predictors = train_feat.columns.drop( ['label', 'enddate', 'hy_16.0', 'hy_91.0', 'hy_94.0']) print('开始CV 5折训练...') scores = [] t0 = time.time() mean_score = [] train_preds = np.zeros(len(train_feat)) test_preds = np.zeros(len(test_feat)) kf = KFold(len(train_feat), n_folds=5, shuffle=True, random_state=520) for i, (train_index, test_index) in enumerate(kf): lgb_train = lgb.Dataset(train_feat[predictors].iloc[train_index], train_feat['label'].iloc[train_index]) lgb_test = lgb.Dataset(train_feat[predictors].iloc[test_index], train_feat['label'].iloc[test_index]) params = { 'task': 'train', 'boosting_type': 'gbdt', 'objective': 'binary', 'metric': 'auc', 'max_depth': 20, 'num_leaves': 150, 'learning_rate': 0.01, 'subsample': 0.7,
def get_ten_fold_crossvalid_perfermance(self, fisher_mode, settings=None): analysis_scr = [] predicted_score = False reduce_ratio = 1 #for seq_no in range(1, self.ddi_obj.total_number_of_sequences+1): #subset_size = math.floor(self.ddi_obj.total_number_of_sequences / 10.0) kf = KFold(self.ddi_obj.total_number_of_sequences, n_folds=10) #for subset_no in range(1, 11): for ((train_index, test_index), subset_no) in izip(kf, range(1, 11)): #for train_index, test_index in kf; print("Subset:", subset_no) print("Train index: ", train_index) print("Test index: ", test_index) #logger.info('subset number: ' + str(subset_no)) if 1: print "SVM" #start_index = int((subset_no - 1) * subset_size + 1) #if subset_no == 10: # end_index = int(max(start_index + subset_size, self.ddi_obj.total_number_of_sequences)) #else: # end_index = int(start_index + subset_size) #print start_index, end_index #(train_X_10fold, train_y_10fold),(train_X_reduced, train_y_reduced), (test_X, test_y) = self.ddi_obj.get_ten_fold_crossvalid_one_subset(start_index, end_index, reduce_ratio = reduce_ratio) (train_X_10fold, train_y_10fold), (train_X_reduced, train_y_reduced), ( test_X, test_y) = self.ddi_obj.get_ten_fold_crossvalid_one_subset( train_index, test_index, reduce_ratio=reduce_ratio) standard_scaler = preprocessing.StandardScaler().fit( train_X_reduced) scaled_train_X = standard_scaler.transform(train_X_reduced) scaled_test_X = standard_scaler.transform(test_X) Linear_SVC = LinearSVC(C=1, penalty="l2") Linear_SVC.fit(scaled_train_X, train_y_reduced) predicted_test_y = Linear_SVC.predict(scaled_test_X) isTest = True #new analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'SVM', isTest) + tuple( performance_score(test_y, predicted_test_y).values())) #new predicted_train_y = Linear_SVC.predict(scaled_train_X) isTest = False #new analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'SVM', isTest) + tuple( performance_score(train_y_reduced, predicted_train_y).values())) # direct deep learning min_max_scaler = Precessing_Scaler_0_9() X_train_pre_validation_minmax = min_max_scaler.fit(train_X_reduced) X_train_pre_validation_minmax = min_max_scaler.transform( train_X_reduced) x_test_minmax = min_max_scaler.transform(test_X) pretraining_X_minmax = min_max_scaler.transform(train_X_10fold) x_train_minmax, x_validation_minmax, y_train_minmax, y_validation_minmax = train_test_split( X_train_pre_validation_minmax, train_y_reduced, test_size=0.4, random_state=42) finetune_lr = 1 batch_size = 100 pretraining_epochs = cal_epochs(5000, x_train_minmax, batch_size=batch_size) #pretrain_lr=0.001 pretrain_lr = 0.001 training_epochs = 1500 hidden_layers_sizes = [100, 100] corruption_levels = [0.1, 0.1] if 1: print "direct deep learning" sda = trainSda(x_train_minmax, y_train_minmax, x_validation_minmax, y_validation_minmax , x_test_minmax, test_y, hidden_layers_sizes = hidden_layers_sizes, corruption_levels = corruption_levels, batch_size = batch_size , \ training_epochs = training_epochs, pretraining_epochs = pretraining_epochs, pretrain_lr = pretrain_lr, finetune_lr=finetune_lr ) print 'hidden_layers_sizes:', hidden_layers_sizes print 'corruption_levels:', corruption_levels training_predicted = sda.predict(x_train_minmax) y_train = y_train_minmax isTest = False #new analysis_scr.append(( self.ddi, subset_no, fisher_mode, 'DL', isTest ) + tuple( performance_score(y_train, training_predicted).values())) test_predicted = sda.predict(x_test_minmax) y_test = test_y isTest = True #new analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'DL', isTest) + tuple(performance_score(y_test, test_predicted).values())) if 0: # deep learning using unlabeled data for pretraining print 'deep learning with unlabel data' pretraining_epochs = cal_epochs(5000, pretraining_X_minmax, batch_size=batch_size) sda_unlabel = trainSda(x_train_minmax, y_train_minmax, x_validation_minmax, y_validation_minmax , x_test_minmax, test_y, pretraining_X_minmax = pretraining_X_minmax, hidden_layers_sizes = hidden_layers_sizes, corruption_levels = corruption_levels, batch_size = batch_size , \ training_epochs = training_epochs, pretraining_epochs = pretraining_epochs, pretrain_lr = pretrain_lr, finetune_lr=finetune_lr ) print 'hidden_layers_sizes:', hidden_layers_sizes print 'corruption_levels:', corruption_levels training_predicted = sda_unlabel.predict(x_train_minmax) y_train = y_train_minmax isTest = False #new analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'DL_U', isTest) + tuple( performance_score(y_train, training_predicted, predicted_score).values())) test_predicted = sda_unlabel.predict(x_test_minmax) y_test = test_y isTest = True #new analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'DL_U', isTest) + tuple( performance_score(y_test, test_predicted, predicted_score).values())) if 0: # deep learning using split network print 'deep learning using split network' # get the new representation for A set. first 784-D pretraining_epochs = 5000 hidden_layers_sizes = [100, 100, 100] corruption_levels = [0, 0, 0] x = x_train_minmax[:, :x_train_minmax.shape[1] / 2] print "original shape for A", x.shape a_MAE_A = train_a_MultipleAEs( x, pretraining_epochs=pretraining_epochs, pretrain_lr=pretrain_lr, batch_size=batch_size, hidden_layers_sizes=hidden_layers_sizes, corruption_levels=corruption_levels) new_x_train_minmax_A = a_MAE_A.transform( x_train_minmax[:, :x_train_minmax.shape[1] / 2]) x = x_train_minmax[:, x_train_minmax.shape[1] / 2:] print "original shape for B", x.shape a_MAE_B = train_a_MultipleAEs( x, pretraining_epochs=pretraining_epochs, pretrain_lr=pretrain_lr, batch_size=batch_size, hidden_layers_sizes=hidden_layers_sizes, corruption_levels=corruption_levels) new_x_train_minmax_B = a_MAE_B.transform( x_train_minmax[:, x_train_minmax.shape[1] / 2:]) new_x_test_minmax_A = a_MAE_A.transform( x_test_minmax[:, :x_test_minmax.shape[1] / 2]) new_x_test_minmax_B = a_MAE_B.transform( x_test_minmax[:, x_test_minmax.shape[1] / 2:]) new_x_validation_minmax_A = a_MAE_A.transform( x_validation_minmax[:, :x_validation_minmax.shape[1] / 2]) new_x_validation_minmax_B = a_MAE_B.transform( x_validation_minmax[:, x_validation_minmax.shape[1] / 2:]) new_x_train_minmax_whole = np.hstack( (new_x_train_minmax_A, new_x_train_minmax_B)) new_x_test_minmax_whole = np.hstack( (new_x_test_minmax_A, new_x_test_minmax_B)) new_x_validationt_minmax_whole = np.hstack( (new_x_validation_minmax_A, new_x_validation_minmax_B)) finetune_lr = 1 batch_size = 100 pretraining_epochs = cal_epochs(5000, x_train_minmax, batch_size=batch_size) #pretrain_lr=0.001 pretrain_lr = 0.001 training_epochs = 1500 hidden_layers_sizes = [100, 100, 100] corruption_levels = [0, 0, 0] sda_transformed = trainSda(new_x_train_minmax_whole, y_train_minmax, new_x_validationt_minmax_whole, y_validation_minmax , new_x_test_minmax_whole, y_test, hidden_layers_sizes = hidden_layers_sizes, corruption_levels = corruption_levels, batch_size = batch_size , \ training_epochs = training_epochs, pretraining_epochs = pretraining_epochs, pretrain_lr = pretrain_lr, finetune_lr=finetune_lr ) print 'hidden_layers_sizes:', hidden_layers_sizes print 'corruption_levels:', corruption_levels training_predicted = sda_transformed.predict( new_x_train_minmax_whole) y_train = y_train_minmax isTest = False #new analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'DL_S', isTest) + tuple( performance_score(y_train, training_predicted, predicted_score).values())) test_predicted = sda_transformed.predict( new_x_test_minmax_whole) y_test = test_y isTest = True #new analysis_scr.append( (self.ddi, subset_no, fisher_mode, 'DL_S', isTest) + tuple( performance_score(y_test, test_predicted, predicted_score).values())) report_name = filename + '_' + '_test10fold_'.join( map(str, hidden_layers_sizes) ) + '_' + str(pretrain_lr) + '_' + str(finetune_lr) + '_' + str( reduce_ratio) + '_' + str(training_epochs) + '_' + current_date saveAsCsv(predicted_score, report_name, performance_score(y_test, test_predicted, predicted_score), analysis_scr)
parser.add_argument('-b','--minbin', help='Minimum categorical bin size', type=int, default=1) parser.add_argument('-cv','--cv', action='store_true') parser.add_argument('-codetest','--codetest', action='store_true') parser.add_argument('-getcached', '--getcached', action='store_true') parser.add_argument('-extra', '--extra', action='store_true') m_params = vars(parser.parse_args()) # Load data X, y, X_sub, ids = data.load(m_params) print("BNP Parabas: classification...\n") clf = ExtraTreesRegressor(n_estimators=700, max_features=60, min_samples_split= 4, max_depth=40, n_jobs=-1, min_samples_leaf=2) if m_params['cv']: # do cross validation scoring kf = KFold(X.shape[0], n_folds=4, shuffle=True, random_state=1) scr = np.zeros([len(kf)]) oob_pred = np.zeros(X.shape[0]) for i, (tr_ix, val_ix) in enumerate(kf): clf.fit(X[tr_ix], y[tr_ix]) pred = clf.predict(X[val_ix]) oob_pred[val_ix] = np.array(pred) scr[i] = log_loss(y[val_ix], np.array(pred)) print('Train score is:', scr[i]) print(log_loss(y, oob_pred)) print oob_pred[1:10] oob_filename = '../output/oob_pred_extrees_' + str(np.mean(scr)) + '.p' pkl.dump(oob_pred, open(oob_filename, 'wb')) else:
return et sql = '' Features_list, Target_list = SQLTrainData(sql) from sklearn import cross_validation from sklearn.cross_validation import KFold train, test, y_train, y_test = cross_validation.train_test_split( Features_list, Target_list, test_size=0.3, random_state=2017) ntrain = train.shape[0] ntest = test.shape[0] SEED = 0 # for reproducibility NFOLDS = 5 # set folds for out-of-fold prediction kf = KFold(ntrain, n_folds=NFOLDS, random_state=SEED) # Class to extend the Sklearn classifier class SklearnHelper(object): def __init__(self, clf, seed=0, params=None): params['random_state'] = seed self.clf = clf(**params) def train(self, x_train, y_train): self.clf.fit(x_train, y_train) def predict(self, x): return self.clf.predict(x) def fit(self, x, y):
# 2. Вычислите TF-IDF-признаки для всех текстов. Обратите внимание, что в этом задании мы предлагаем вам # вычислить TF-IDF по всем данным. При таком подходе получается, что признаки на обучающем множестве используют # информацию из тестовой выборки — но такая ситуация вполне законна, поскольку мы не используем значения целевой # переменной из теста. На практике нередко встречаются ситуации, когда признаки объектов тестовой выборки известны на # момент обучения, и поэтому можно ими пользоваться при обучении алгоритма. vectorizer = TfidfVectorizer() vectorizer.fit_transform(X) # 3. Подберите минимальный лучший параметр C из множества [10^-5, 10^-4, ... 10^4, 10^5] для SVM с # линейным ядром (kernel='linear') при помощи кросс-валидации по 5 блокам. Укажите параметр random_state=241 и для SVM, # и для KFold. В качестве меры качества используйте долю верных ответов (accuracy). grid = {'C': np.power(10.0, np.arange(-5, 6))} cv = KFold(y.size, n_folds=5, shuffle=True, random_state=241) model = SVC(kernel='linear', random_state=241) gs = grid_search.GridSearchCV(model, grid, scoring='accuracy', cv=cv) gs.fit(vectorizer.transform(X), y) score = 0 C = 0 for attempt in gs.grid_scores_: if attempt.mean_validation_score > score: score = attempt.mean_validation_score C = attempt.parameters['C'] # 4. Обучите SVM по всей выборке с оптимальным параметром C, найденным на предыдущем шаге. model = SVC(kernel='linear', random_state=241, C=C) model.fit(vectorizer.transform(X), y)
#%% # Lasso model = linear_model.Lasso(alpha = 0.001) model.fit(trainX, trainY) prediction = model.predict(testX) print("Lasso Accuracy: ", model.score(testX,testY)) #%% #Ridge model = linear_model.Ridge(alpha = 0.05, normalize=True) model.fit(trainX, trainY) prediction = model.predict(testX) print("Ridge Accuracy: ", model.score(testX,testY)) #%% kfold = KFold(n=10,random_state=10) #%% cvMean = [] results = [] classifiers = ['Linear Svm','Radial Svm','Logistic Regression','Decision Tree','KNN'] models = [svm.SVC(kernel='linear'),svm.SVC(kernel='rbf'),LogisticRegression(),DecisionTreeClassifier(),KNeighborsClassifier(n_neighbors=3)] for i in models: model = i result = cross_val_score(model, wine[wine.columns[:11]], wine['quality'],cv=kfold, scoring='accuracy') results.append(result) cvMean.append(result.mean()) new_models_df = pd.DataFrame(cvMean, index=classifiers) new_models_df.columns = ['CV Mean'] new_models_df
def stacking_classifier(folds, models): # Level 1 regression models regrs = models # 5-fold cross validation kf = list(KFold(len(target_train_bin), n_folds=folds, shuffle = True, random_state = 1991)) # Pre-allocate the data blend_train = np.zeros((regressors_train_pca.shape[0], len(regrs))) # Number of training data x Number of classifiers blend_test = np.zeros((regressors_validation_pca.shape[0], len(regrs))) # Number of testing data x Number of classifiers # For each classifier, we train the number of fold times (=len(kf)) for j, clf in enumerate(regrs): print('Training Regression Model [{}] - {}'.format(j, clf)) blend_test_j = np.zeros((regressors_validation_pca.shape[0], len(kf))) # Number of testing data x Number of folds , we will take the mean of the predictions later for i, (train_index, cv_index) in enumerate(kf): print('Fold [{}]'.format(i)) # This is the training and validation set X_train = regressors_train_pca[train_index] #Y_train = target_train_bin.iloc[train_index] Y_train = target_train_bin[train_index] X_cv = regressors_train_pca[cv_index] if(j == 0): # ANN Y_train = to_categorical(Y_train) clf.fit(X_train, Y_train, validation_split=0.2, epochs=5, batch_size=16, verbose=2) else: clf.fit(X_train, Y_train) # This output will be the basis for our blended classifier to train against, # which is also the output of level 1 Regressors if(j==0): blend_train[cv_index, j] = clf.predict_classes(X_cv).flatten() blend_test_j[:, i] = clf.predict_classes(regressors_validation_pca).flatten() else: blend_train[cv_index, j] = clf.predict(X_cv).flatten() blend_test_j[:, i] = clf.predict(regressors_validation_pca).flatten() # Take the mean of the predictions of the cross validation set blend_test[:, j] = blend_test_j.mean(1) # Blending (predict Level 2 based on predictions on the train set) # ridgecv = RidgeCV(alphas=alphas, scoring='neg_mean_squared_error', normalize=False) # ridgecv.fit(blend_train, target_train) # ridgecv.alpha_ # Fit Ridge model with best alpha # bclf = Ridge(alpha=ridgecv.alpha_, normalize=False, max_iter=10000) # bclf.fit(blend_train, target_train) bclf = LogisticRegression() bclf.fit(blend_train, target_train_bin) #bclf =NN_CLF_model(len(regrs)) #bclf.fit(blend_train, to_categorical(target_train_bin), validation_split=0.2, epochs=5, batch_size=16, verbose=2) # Predict now predicted_level2_bin = bclf.predict(blend_test) #predicted_level2_bin = bclf.predict_classes(blend_test) score = accuracy_score(target_validation_bin, predicted_level2_bin) return score, predicted_level2_bin
perturb[i] = 0 return grad d = Orange.data.Table('housing') d.X = np.hstack((d.X, np.ones((d.X.shape[0], 1)))) d.shuffle() # m = LinearRegressionLearner(lambda_=1.0) # print(m(d)(d)) # # gradient check # m = LinearRegressionLearner(lambda_=1.0) # theta = np.random.randn(d.X.shape[1]) # # ga = m.cost_grad(theta, d.X, d.Y.ravel())[1] # gm = numerical_grad(lambda t: m.cost_grad(t, d.X, d.Y.ravel())[0], theta) # # print(np.sum((ga - gm)**2)) for lambda_ in (0.01, 0.03, 0.1, 0.3, 1, 3): m = LinearRegressionLearner(lambda_=lambda_) scores = [] for tr_ind, te_ind in KFold(d.X.shape[0]): s = np.mean((m(d[tr_ind])(d[te_ind]) - d[te_ind].Y.ravel())**2) scores.append(s) print('{:5.2f} {}'.format(lambda_, np.mean(scores))) m = LinearRegressionLearner(lambda_=0) print('test data', np.mean((m(d)(d) - d.Y.ravel())**2)) print('majority', np.mean((np.mean(d.Y.ravel()) - d.Y.ravel())**2))