# XGB
d_1 = xgb.DMatrix(X_tr_1, label = y_tr_1)
d_2 = xgb.DMatrix(X_tr_2, label = y_tr_2)
d_val = xgb.DMatrix(X_va)
d_test = xgb.DMatrix(X_te_s.drop(["image_name"],axis=1))
wlist = [(d_1, 'train'), (d_2, 'eval')]
model = xgb.train(xgb_params_s, dtrain = d_1, num_boost_round = 2000, evals = wlist, verbose_eval = 100, early_stopping_rounds = 100)
tr_pred = model.predict(d_val, ntree_limit=model.best_ntree_limit)
pred_xgb_s += model.predict(d_test, ntree_limit=model.best_ntree_limit) / (nsplits)
X_tr_ss_1["XGB"] = tr_pred
fimp_1 = pd.DataFrame()
fimp_1["feature"] = model.get_fscore().keys()
fimp_1["importance"] = model.get_fscore().values()
fimp = pd.concat([fimp, fimp_1], axis=0)
# Random Forest
model = RandomForestClassifier(criterion = rf_params_s["criterion"],
max_leaf_nodes = rf_params_s["max_leaf_nodes"],
min_samples_split = rf_params_s["min_samples_split"],
max_depth=rf_params_s["max_depth"],
n_estimators=rf_params_s["n_estimators"], class_weight='balanced', random_state=rand)
model.fit(X_tr, y_tr)
tr_pred = model.predict_proba(X_va)[:,1]
pred_rf_s += model.predict_proba(X_te_s.drop(["image_name"], axis = 1))[:,1] / (nsplits)
X_tr_ss_1["RF"] = tr_pred
del model
