def keep_best_loss(X_train,y_train,X_valid,y_valid):
    PROB,LOSS,RF = [],[],[]
    for i in range(0,6):
        seed=randrange(1,10000)
        random.seed(seed)
        print("Seed: %s" % seed)
        rf = build_rf(X_train,y_train)
        prob = rf.predict_proba(X_valid)
        loss = of.logloss_mc(y_valid,prob)
        RF.append(rf)
        PROB.append(prob)
        LOSS.append(loss)
        print("Model %s Loss: %0.4f" % (i,loss))
    sum=0
    for p in PROB:
        sum+=p
    avg_prob = sum/len(PROB)
    avg_loss = of.logloss_mc(y_valid,avg_prob)
    print("Avg Loss of 5 models: %0.4f" % avg_loss)
    return RF,PROB
def build_XB1(X_train,y_train,X_cal,y_cal,X_test):
    cal_prob,test_prob = 0,0
    for i in range(5):
        print("--Building and Training model %s" % i)
        seed = randrange(1,10000)
        model = XGBoostClassifier(n_iter=500,
                                max_features=0.3,
                                max_depth=7,
                                min_child_weight=10,
                                gamma=0.0093,random_state=seed,
                                learning_rate=0.2,
                                l2_weight=0.1,
                                max_samples=0.9
                                )
        model = CalibratedClassifierCV(base_estimator=model,method='isotonic',cv=5).fit(X_train,y_train)
        print("Model %s training complete." % i)
        cal_prob += model.predict_proba(X_cal)
        test_prob += model.predict_proba(X_test)
    cal_prob = cal_prob/5.
    test_prob = test_prob/5.
    print("Average Model Loss: %0.4f" % of.logloss_mc(y_cal,cal_prob))
    return(cal_prob,test_prob)
def build_NN(X_train,y_train_nn,X_cal,y_cal,X_test):
    cal_prob,test_prob = 0,0
    nb_classes = y_train_nn.shape[1]
    dims = X_train.shape[1]

    for i in range(5):
        print("--Building and Training model %s" % i)
        seed = randrange(1,10000)
        np.random.seed(seed)

        model = Sequential()

        model.add(Dense(dims, 512, init='glorot_uniform'))
        model.add(PReLU((512,)))
        model.add(BatchNormalization((512,)))
        model.add(Dropout(0.5))

        model.add(Dense(512, 512, init='glorot_uniform'))
        model.add(PReLU((512,)))
        model.add(BatchNormalization((512,)))
        model.add(Dropout(0.5))

        model.add(Dense(512, 512, init='glorot_uniform'))
        model.add(PReLU((512,)))
        model.add(BatchNormalization((512,)))
        model.add(Dropout(0.5))

        model.add(Dense(512, nb_classes, init='glorot_uniform'))
        model.add(Activation('softmax'))

        model.compile(loss='categorical_crossentropy', optimizer="adam")

        model.fit(X_train, y_train_nn, nb_epoch=20, batch_size=16, validation_split=0.15)

        cal_prob += model.predict_proba(X_cal)
        test_prob += model.predict_proba(X_test)
    cal_prob,test_prob = cal_prob/10.,test_prob/10.
    print("Avg NN Model Loss: %0.4f" % of.logloss_mc(y_cal,cal_prob))
    return(cal_prob,test_prob)