import sys

import numpy as np

import cPickle as pickle

import pandas as pd

from sklearn.preprocessing import LabelEncoder

from sklearn.preprocessing import StandardScaler

from sklearn.externals import joblib

from sklearn import metrics

from lasagne.layers import DenseLayer

from lasagne.layers import InputLayer

from lasagne.layers import DropoutLayer

from lasagne.nonlinearities import softmax

from lasagne.updates import nesterov_momentum

from nolearn.lasagne import NeuralNet

from sklearn import cross_validation, svm

from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier

from datetime import datetime as dt

from sklearn.metrics import log_loss

from scipy.optimize import minimize

from sklearn.cross_validation import StratifiedShuffleSplit

from sklearn.linear_model import LogisticRegression

from sklearn.externals import joblib

import os.path

def load_train_data(path, ordered=True, transform='log'):

return X, y, encoder, scaler

def load_test_data(path, scaler):

return X, ids

def make_submission(clf, X_test, ids, encoder, name='my_neural_net_submission.csv'):

print("Wrote submission to file {}.".format(name))

def fit(xTrain, yTrain, dense0_num=800, dropout_p=0.5, dense1_num=500, update_learning_rate=0.01,

#make_submission(clf, xTest, ids, encoder)

#clf_nn = fit()

### find the best weights for combining several classifiers

t0 = dt.now()

### read from files

def read_no_transform(trainf='train.csv', testf='test.csv', samplef='sampleSubmission.csv', train_ratio = 0.9):

return train_x, train_y, test_x, test_y, train_index, test_index, test, labels, sample

def do_RF(train_x, train_y, n_estimators=2000, max_depth=45, max_features=40, criterion='entropy',

return clf_rf

def do_gbdt(train_x, train_y, learning_rate=0.01, max_depth=5, max_features='auto', n_estimators=6000,

return clf_gbdt

def do_svc(train_x, train_y, C=3.467, gamma=0.05754, probability=True, load=False, save=True, outfile=None):

return clf_svc

def do_logreg(train_x, train_y):

logreg = LogisticRegression()

logreg.fit(train_x, train_y)

return logreg

def print_clf(clf, test_x, test_y, clf_name=None, outfname=None):

print('Classifier {name} has a LogLoss {score}'.format(name=clf_name, score=log_loss(test_y, clf.predict_proba(test_x))))

if outfname!=None:

outfname.write('Classifier {name} has a LogLoss {score} \n'.format(name=clf_name, score=log_loss(test_y, clf.predict_proba(test_x))))

###### end of functions

train_ratio = 0.9

train_x, train_y, test_x, test_y, train_index, test_index, test, labels, sample = read_no_transform(train_ratio=train_ratio)

foutname = 'otto_ensemble'+str(train_ratio*100)+'percent_training_set_results_with_nn_v8.6.txt'

### open file foutname for writing results

try:

print 'Opening file stream for writing results.\n'

outfile = open(foutname, 'a')

except IOError:

print 'There was an error opening file', foutname

sys.exit()

### log transformed features:

X, y, encoder, scaler = load_train_data('train.csv')

train_x2, train_y2 = X[train_index], y[train_index]

test_x2, test_y2 = X[test_index], y[test_index]

### BM25 transfomred features:

#X3, y3, encoder3, scaler3 = load_train_data('train.csv',transform='BM25')

#train_x3, train_y3 = X3[train_index], y3[train_index]

#test_x3, test_y3 = X3[test_index], y3[test_index]

#train_x3 = 2.0*train_x/(train_x+1.0)

#train_y3 = train_y

#test_x3 = 2.0*test_x/(test_x+1.0)

#test_y3 =

### building the classifiers, train them and predict for test set if desired

clfs = []

clf_rf = do_RF(train_x, train_y, outfile=outfile)

print_clf(clf_rf, test_x, test_y, clf_name="RF", outfname=outfile)

clfs.append(clf_rf)

clf_gbdt = do_gbdt(train_x, train_y, outfile=outfile)

print_clf(clf_gbdt, test_x, test_y, clf_name="GBDT", outfname=outfile)

clfs.append(clf_gbdt)

clf_nn = fit(train_x2, train_y2, dense0_num=500, dropout_p=0.5, dense1_num=1000, update_learning_rate=0.005, max_epochs=48)

print_clf(clf_nn, test_x2, test_y2, clf_name="Neural Network log transformed (cheating test)", outfname=outfile)

clfs.append(clf_nn)

clf_svc = do_svc(train_x, train_y, C=100., gamma=0.001, outfile=outfile)

print_clf(clf_svc, test_x, test_y, clf_name="SVC no transform", outfname=outfile)

clfs.append(clf_svc)

### finding the optimum weights

predictions = []

for i, clf in enumerate(clfs):

if i<2:

predictions.append(clf.predict_proba(test_x))

elif i==2:

predictions.append(clf.predict_proba(test_x2))

elif i==3:

predictions.append(clf.predict_proba(test_x))

def log_loss_func(weights):

return log_loss(test_y, final_prediction)

#the algorithms need a starting value, right not we chose 0.5 for all weights

#its better to choose many random starting points and run minimize a few times

starting_values = [0.5]*len(predictions)

#adding constraints and a different solver as suggested by user 16universe

#https://kaggle2.blob.core.windows.net/forum-message-attachments/75655/2393/otto%20model%20weights.pdf?sv=2012-02-12&se=2015-05-03T21%3A22%3A17Z&sr=b&sp=r&sig=rkeA7EJC%2BiQ%2FJ%2BcMpcA4lYQLFh6ubNqs2XAkGtFsAv0%3D

cons = ({'type':'eq','fun':lambda w: 1-sum(w)})

#our weights are bound between 0 and 1

bounds = [(0,1)]*len(predictions)

res = minimize(log_loss_func, starting_values, method='SLSQP', bounds=bounds, constraints=cons)

print('Ensemble Score: {best_score}'.format(best_score=res['fun']))

print('Best Weights: {weights}'.format(weights=res['x']))

outfile.write('Ensemble Score: {best_score} \n'.format(best_score=res['fun']))

outfile.write('Best Weights: {weights} \n'.format(weights=res['x']))

sub_file = 'otto_test_submit_benchmark_with_nn_v8.6.csv'

#pred_test=pd.DataFrame(np.zeros((test_rows, 9)), index=sample.id.values, columns=sample.columns[1:])

counter = 0

#pred_test = []

for clf, wt in zip(clfs, res['x']):

#test_outfile.write('Weight for this classifier: {} \n'.format(wt))

#pred_test_current_clf.to_csv(test_res_file, index_label='id')

if counter==0:

pred_test_current_clf = pd.DataFrame(clf.predict_proba(test)) * wt

pred_test = pred_test_current_clf

elif counter==1:

pred_test_current_clf = pd.DataFrame(clf.predict_proba(test)) * wt

pred_test = pred_test + pred_test_current_clf

elif counter==2:

#scaler = StandardScaler()

#test_X = scaler.fit_transform(np.log(test+1))

test_X, test_ids = load_test_data('test.csv', scaler)

pred_test_current_clf = pd.DataFrame(clf.predict_proba(test_X)) * wt

pred_test = pred_test + pred_test_current_clf

elif counter==3:

#scaler = StandardScaler()

#test_X = scaler.fit_transform(2.0*test/(test+1))

#test_X = 2.0*test/(test+1.0)

pred_test_current_clf = pd.DataFrame(clf.predict_proba(test)) * wt

pred_test = pred_test + pred_test_current_clf

counter += 1

pred_test.columns = sample.columns[1:]

pred_test.index += 1

pred_test.to_csv(sub_file, index_label='id')

print("\nJob done in {}\n".format(dt.now() - t0))

outfile.write("Job done in {}\n".format(dt.now() - t0))

outfile.close()