def tlearn(self,dev,test,val,bad_ind,featureA,featureB,featureC,drop_rate):
"""
dev 训练集 源域
test 测试集 辅助域
val 验证集
bad_ind 标签
featureA 特征组A
featureB 特征组B
featureC 特征组C
"""
print(len(featureA),len(featureB),len(featureC))
result = pd.DataFrame()
temp_test = test
features = list(set(featureA+featureB+featureC))
turn = 1
while( turn <= self.max_turns):
new = pd.DataFrame()
"""
模型A对特征组featureA训练，
并预测得到dev和test和val的概率
以及test上的分类结果（分数分布在0.8*(min+max)两侧）
"""
self.clfA.fit(dev[featureA],dev[bad_ind])
predA= self.clfA.predict_proba(dev[featureA])[:,1]
probA = self.clfA.predict_proba(test[featureA])[:,1]
preA = (probA > (np.max(probA)+np.min(probA))*0.8)
valid_a = self.clfA.predict_proba(val[featureA])[:,1]
"""
模型B对特征组featureB训练，
并预测得到dev和test和val的概率
以及test上的分类结果（分数分布在0.8*(min+max)两侧）
"""
self.clfB.fit(dev[featureB],dev[bad_ind])
predB = self.clfB.predict_proba(dev[featureB])[:,1]
probB = self.clfB.predict_proba(test[featureB])[:,1]
preB = (probA > (np.max(probB)+np.min(probB))*0.8)
valid_b = self.clfB.predict_proba(val[featureB])[:,1]
"""
模型C对特征组featureC训练，
并预测得到dev和test和val的概率
以及test上的分类结果（分数分布在0.8*(min+max)两侧）
"""
self.clfC.fit(dev[featureC],dev[bad_ind])
predC= self.clfC.predict_proba(dev[featureC])[:,1]
probC = self.clfC.predict_proba(test[featureC])[:,1]
preC = (probC > (np.max(probC)+np.min(probC))*0.8)
valid_c = self.clfC.predict_proba(val[featureC])[:,1]
"""
分别计算三个模型在val上的AUC
模型加权融合的策略：以单模型的AUC作为权重
"""
valid_scoreA = AUC(val[bad_ind],valid_a)
valid_scoreB = AUC(val[bad_ind],valid_b)
valid_scoreC = AUC(val[bad_ind],valid_c)
valid_score = AUC(val[bad_ind], valid_a*valid_scoreA
+valid_b*valid_scoreB + valid_c*valid_scoreC)
"""
index1 三个模型在test上的预测概率相同的样本
sum_va 三个模型AUC之和为分母做归一化
prob 测试集分类结果融合，
index1（分类结果）*AUC（权重）/sum_va（归一化分母）
index2 分类结果升序排列，取出两端的test样本
new 筛选后样本集
"""
index1 = (preA==preB) & (preA==preC)
sum_va = valid_scoreA+valid_scoreB+valid_scoreC
prob = (probC[index1]*valid_scoreC+probA[index1]*valid_scoreA
+probB[index1]*valid_scoreB)/sum_va
Ap_low = np.sort(prob)[int(len(prob)*turn/2.0/self.max_turns)]-0.01
Ap_high= np.sort(prob)[int(len(prob)*
(1-turn/2.0/self.max_turns))]+0.01
index2 = ((prob>Ap_high) | (prob<Ap_low))
new['no'] = test['no'][index1][index2]
new['pred'] = prob[index2]
result = result.append(new)
"""
rightSamples 同时满足index1和index2条件的预测概率
score_sim 三个模型在test上的预测结果差异和
"""
rightSamples = test[index1][index2]
rightSamples[bad_ind] = preA[index1][index2]
score_sim = np.sum(abs(probA-probB)+
abs(probA-probC) +abs(probB-probC)+0.1)/len(probA)
"""
从数据集dev中取出step之后的部分样本并计算AUC
valid_score 前文三模型加权融合的AUC
得到drop
"""
true_y = dev.iloc[self.step:][bad_ind]
dev_prob = predA[self.step:]*valid_scoreA+ predB[self.step:]*valid_scoreB + predC[self.step:]*valid_scoreC
dev_score = AUC(true_y,dev_prob)
drop = self.max_turns/(1+ drop_rate*
np.exp(-self.max_turns)*valid_score)
"""
使用Traddaboost相同的权重调整方法，
挑选权重大于阈值的样本。
"""
loss_bias = 0
if(self.step>0):
true_y = dev.iloc[0:self.step][bad_ind]
temp = predA[0:self.step]*valid_scoreA \
+ predB[0:self.step]*valid_scoreB \
+ predC[0:self.step]*valid_scoreC
temp = (temp+0.1)/(max(temp)+0.2)#归一化
temp = (true_y-1)*np.log(1-temp)-true_y*np.log(temp)#样本权重
loc = int(min(self.step,len(rightSamples)*drop+2)
*np.random.rand())#去除样本的比例
loss_bias = np.sort(temp)[-loc]
temp = np.append(temp,np.zeros(len(dev)-self.step)-99)
remain_index = (temp <= loss_bias)
self.step = self.step-sum(1-remain_index)
else:
remain_index = []
"""
得到新的test
"""
dev = dev[remain_index].append(rightSamples[features+[bad_ind,'no']])
test = test[~test.index.isin(rightSamples.index)]
turn += 1
"""
计算原始test上的AUC
"""
probA = self.clfA.predict_proba(test[featureA])[:,1]
pA = self.clfA.predict_proba(temp_test[featureA])[:,1]
valid_a = self.clfA.predict_proba(val[featureA])[:,1]
probB = self.clfB.predict_proba(test[featureB])[:,1]
valid_b = self.clfB.predict_proba(val[featureB])[:,1]
pB = self.clfB.predict_proba(temp_test[featureB])[:,1]
probC = self.clfC.predict_proba(test[features])[:,1]
valid_c = self.clfC.predict_proba(val[features])[:,1]
pC = self.clfC.predict_proba(temp_test[features])[:,1]
self.scoreA = AUC(val[bad_ind],valid_a)
self.scoreB = AUC(val[bad_ind],valid_b)
self.scoreC = AUC(val[bad_ind],valid_c)
return pA,pB,pC
def run_all_classifiers_cv(cl_name, example_index, param_index):
''' get results for all the classifiers '''
data_dir = os.getcwd() + '/'
all_datasets = ['iris_bintarget', 'ilpd', 'adult_data_bintarget', 'credit_card_clients', \
'ionosphere', 'iris_bintarget', 'parkinsons', 'pima-indians-diabetes', \
'TomsHardware_bintarget', 'transfusion', 'Twitter_bintarget_small', 'wdbc']
datasets = []
if (example_index == -1):
datasets = all_datasets
else:
datasets.append(all_datasets[example_index])
num_exp = 10
for exp_name in datasets:
results = []
timeTakenData = []
### loop over the cv-fold (to be consistent with SAT results these are saved in files)
for e_num in range(num_exp):
fname_tr = data_dir + 'Train/' + '%s_%d_train.csv' % (exp_name,
e_num)
fname_tst = data_dir + 'Test/' + '%s_%d_test.csv' % (exp_name,
e_num)
X_tr, X_tst, y_tr, y_tst, feat_names, target_name = load_check_data(
fname_tr, fname_tst)
if cl_name == 'rf':
gen_cl = gen_rf
cv_param = 'min_samp'
elif cl_name == 'logreg':
gen_cl = gen_logreg
cv_param = 'C'
elif cl_name == 'nn':
gen_cl = lambda: gen_nn(max_nn=len(y_tr))
cv_param = 'num_nbs'
elif cl_name == 'svc':
gen_cl = gen_svc
cv_param = 'C'
else:
assert False, "No such classifier %s" % cl_name
print type(gen_cl())
#cl, min_samp= gen_cl()[0]
### generator over the classifier over a predefined range of paramters
#for i in range(0,1):
param_wait_index = 0
for cl, min_samp in gen_cl():
if (not (param_wait_index == param_index)):
param_wait_index += 1
continue
param_wait_index += 1
startTime = time.time()
cl.fit(X_tr, y_tr)
endTime = time.time()
timeTaken = endTime - startTime
y_hat = cl.predict(X_tst)
p = cl.predict_proba(X_tst)
acc = accuracy_score(y_tst, y_hat)
auc = AUC(y_tst, p[:, 1])
results.append([e_num, min_samp, acc, auc])
timeTakenData.append([e_num, min_samp, timeTaken])
print np.array(results)
results_pd = pd.DataFrame(results,
index=None,
columns=['exp_num', cv_param, 'acc', 'auc'])
timeTaken_pd = pd.DataFrame(timeTakenData,
index=None,
columns=['exp_num', cv_param, 'Time'])
#print results_pd
av_cv = results_pd.groupby(cv_param).mean()
av_time = timeTaken_pd.groupby(cv_param).mean()
del av_cv['exp_num']
del av_time['exp_num']
fname_out = 'Results/%s_%s_%d_results.csv' % (exp_name, cl_name,
param_index)
av_cv.to_csv(fname_out)
fname_out = 'Results/time_%s_%s_%d_results.csv' % (exp_name, cl_name,
param_index)
av_time.to_csv(fname_out)
print("{0}{1}".format(st, fraction_correct(svc_pred, te[1])))
#### Evaluate Classifier Performance
##### ROC
# In[148]:
# because we have a binary classification problem,
# we can use ROC to evaluate the quality of these models
#logistic regression
pred_prob_lr = lr.predict_proba(te[0])
false_pos_rate_lr, true_pos_rate_lr, thresholds_lr = ROC(
te[1], pred_prob_lr[:, 1])
roc_auc_lr = AUC(false_pos_rate_lr, true_pos_rate_lr)
print(
"Logisitc Regression, area under the curve: {0:>9.3f}".format(roc_auc_lr))
# svm
pred_prob_svm = svc.predict_proba(te[0])
false_pos_rate_svm, true_pos_rate_svm, thresholds_svm = ROC(
te[1], pred_prob_svm[:, 1])
roc_auc_svm = AUC(false_pos_rate_svm, true_pos_rate_svm)
print("SVM, area under the curve: {0:>25.3f}".format(roc_auc_svm))
# In[170]:
# plot the ROC curves for each classifier
fpr_lr, tpr_lr = false_pos_rate_lr, true_pos_rate_lr
# num_boost_round = 1500
# # watchlist = [(xg_train, 'train'), (xg_test, 'eval')]
# num_round=15
# bst = xgb.train(param, xg_train, num_round)
# preds = bst.predict(xg_test)
# auc = AUC(Y_test , preds)
# print(auc)
# 根据重要性筛选
# score = bst.get_score(importance_type='gain')
# print(score)
xgbc = XGBC(max_depth=200, seed=999, n_estimators=100, scale_pos_weight=13)
xgbc.fit(X_train, Y_train)
Y_test_proba = xgbc.predict_proba(X_test)
print("训练a的数据，得分auc：")
print(AUC(Y_test, Y_test_proba[:, 1]))
# 默认是信息增益 importance_type="gain"
feature_importances_ = xgbc.feature_importances_
feature_importances_series = pd.Series(feature_importances_)
feature_importances_series.index = effective_columns
effective_columns_before = effective_columns
#去除分类效果低的属性 反而效果更差
feature_importances_series = feature_importances_series[
feature_importances_series.values > 0]
print("根据属性重要性，提取属性大小")
print(feature_importances_series.shape)
effective_columns = feature_importances_series.index.values
# 展示列的重要性排序
sort_values = feature_importances_series.sort_values(ascending=False)
cv = CV.StratifiedKFold(y, n_folds=5, shuffle=True, random_state=20180512)
for f, (train_i, test_i) in enumerate(cv):
print("# fold {}, {}".format(f + 1, ctime()))
x_train = x.iloc[train_i]
x_test = x.iloc[test_i]
y_train = y.iloc[train_i]
y_test = y.iloc[test_i]
clf.fit(x_train, y_train)
p = clf.predict_proba(x_test)[:, 1]
auc = AUC(y_test, p)
print("# AUC: {:.2%}\n".format(auc))
predictions[test_i] = p
# fold 1, Mon Jun 25 19:39:24 2018
# AUC: 82.71%
# fold 2, Mon Jun 25 19:39:48 2018
# AUC: 82.99%
# fold 3, Mon Jun 25 19:40:10 2018
# AUC: 82.20%
# fold 4, Mon Jun 25 19:40:31 2018
# AUC: 83.54%
def fallback_auc(y_true, y_pred):
try:
return AUC(y_true, y_pred)
except:
return 0.5
def train_predict_lr_forward(train_file,
test_file,
predict_valid_file,
predict_test_file,
C,
n_fold=5):
feature_name = os.path.basename(train_file)[:-8]
algo_name = 'lr_forward_{}'.format(C)
model_name = '{}_{}'.format(algo_name, feature_name)
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
level=logging.DEBUG,
filename='{}.log'.format(model_name))
logging.info("Loading training and test data...")
X_trn, y_trn = load_data(train_file, dense=True)
X_tst, _ = load_data(test_file, dense=True)
logging.info('Normalizing data')
scaler = StandardScaler()
X_trn = scaler.fit_transform(X_trn)
X_tst = scaler.transform(X_tst)
cv = StratifiedKFold(y_trn,
n_folds=n_fold,
shuffle=True,
random_state=2015)
selected_features = []
features_to_test = [
x for x in range(X_trn.shape[1]) if x not in selected_features
]
auc_cv_old = .5
is_improving = True
while is_improving:
auc_cvs = []
for feature in features_to_test:
logging.info('{}'.format(selected_features + [feature]))
X = X_trn[:, selected_features + [feature]]
p_val = np.zeros_like(y_trn)
for i, (i_trn, i_val) in enumerate(cv, start=1):
clf = LR(C=C, class_weight='auto', random_state=2014)
clf.fit(X[i_trn], y_trn[i_trn])
p_val[i_val] = clf.predict_proba(X[i_val])[:, 1]
auc_cv = AUC(y_trn, p_val)
logging.info('AUC CV: {:.6f}'.format(auc_cv))
auc_cvs.append(auc_cv)
auc_cv_new = max(auc_cvs)
if auc_cv_new > auc_cv_old:
auc_cv_old = auc_cv_new
feature = features_to_test.pop(auc_cvs.index(auc_cv_new))
selected_features.append(feature)
logging.info('selected features: {}'.format(selected_features))
else:
is_improving = False
logging.info(
'final selected features: {}'.format(selected_features))
logging.info('saving selected features as a file')
with open('{}_selected.txt'.format(model_name), 'w') as f:
f.write('{}\n'.format(selected_features))
X = X_trn[:, selected_features]
logging.debug('feature matrix: {}x{}'.format(X.shape[0], X.shape[1]))
p_val = np.zeros_like(y_trn)
for i, (i_trn, i_val) in enumerate(cv, start=1):
logging.info('Training CV #{}'.format(i))
clf = LR(C=C, class_weight='auto', random_state=2015)
clf.fit(X[i_trn], y_trn[i_trn])
p_val[i_val] = clf.predict_proba(X[i_val])[:, 1]
auc_cv = AUC(y_trn, p_val)
logging.info('AUC CV: {:.6f}'.format(auc_cv))
logging.info("Writing test predictions to file")
np.savetxt(predict_valid_file, p_val, fmt='%.6f', delimiter=',')
logging.info('Retraining with 100% data...')
clf.fit(X, y_trn)
p_tst = clf.predict_proba(X_tst[:, selected_features])[:, 1]
np.savetxt(predict_test_file, p_tst, fmt='%.6f', delimiter=',')
print "loading y..."
y = np.loadtxt(y_file,
usecols=[0, 2],
converters={2: lambda x: x.split('|')[0]})
y_true = y[:, 0]
pos_ids = y[y[:, 0] == 1][:, 1].astype(int)
# calculate precision
print "calculating AP@{}...".format(k)
# this snippet is from Kaggle code
countRelevants = 0
listOfPrecisions = list()
for i, p_id in enumerate(p_ids):
currentk = i + 1.0
if p_id in pos_ids:
countRelevants += 1
precisionAtK = countRelevants / currentk
listOfPrecisions.append(precisionAtK)
print sum(listOfPrecisions) / k
print
print "confusion matrix:"
print confusion_matrix(y_true, y_predicted)
print
print "AUC:", AUC(y_true, p[:, 0])
#!/usr/bin/env python
from sklearn.metrics import roc_auc_score as AUC
import argparse
import json
import numpy as np
import os
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--target-file',
'-t',
required=True,
dest='target_file')
parser.add_argument('--predict-file',
'-p',
required=True,
dest='predict_file')
args = parser.parse_args()
# p contains the predictions across all K folds in cross validation
p = np.loadtxt(args.predict_file, delimiter=',')
y = np.loadtxt(args.target_file, delimiter=',')
model_name = os.path.splitext(
os.path.splitext(os.path.basename(args.predict_file))[0])[0]
print('{}\t{:.6f}'.format(model_name, AUC(y, p)))
y_file = sys.argv[1] p_file = sys.argv[2] print "loading p..." p = np.loadtxt( p_file ) y_predicted = np.ones(( p.shape[0] )) y_predicted[p < 0] = -1 print "loading y..." y = np.loadtxt( y_file, usecols= [0] ) print "accuracy:", accuracy( y, y_predicted ) print "AUC:", AUC( y, p ) print print "confusion matrix:" print confusion_matrix( y, y_predicted ) """ run score.py data/test_v.txt vw/p_v_logistic.txt accuracy: 0.994675826535 confusion matrix: [[27444 136] [ 236 42054]] AUC: 0.998418419401 """
for fold, (train_idx, test_idx) in enumerate(skf.split(x, y)):
X_train, X_test = x[train_idx], x[test_idx]
y_train, y_test = y[train_idx], y[test_idx]
m.fit(X_train, y_train)
probs = m.predict_proba(X_test)[:, 1]
predictions[test_idx] = probs
# ### Results
# We'll output the ROC-AUC metric for our classifier as an estimate how much covariate shift this data has. As we can see that value for AUC is very close to .5. It implies that our classifier is not able to distinguish the rows whether it is belonging to train or test. This implies that majority of the observations comes from a feature space which is not particular to test or train.
# In[25]:
print('ROC-AUC for X and Z distributions:', AUC(y, predictions))
# In the *predictions* array, we just computed the probability of a sample in the full dataset being sample taken from the training distribution ($train$). We'll call this $p(train|Data)$. Next we'll use the relationship that $p(train|Data) = 1 - p(test|Data)$ to estimate $\beta$ for our training samples,
#
# $\beta_i = \frac{p_i(test|Data)}{p_i(train|Data)} = \frac{1 - p_i(train|Data)}{p_i(train|Data)} = \frac{1}{p(train|Data)} - 1$.
#
# So we now have a method to convert the probability of each point belonging to the training distribution into our sample weights $\beta$. Let's see the distribution of these weights for the training samples
# In[27]:
plt.figure(figsize=(20, 10))
predictions_train = predictions[len(tst):] #filtering the actual training rows
weights = (1. / predictions_train) - 1.
weights /= np.mean(weights) # Normalizing the weights
plt.xlabel('Computed sample weight')
plt.ylabel('# Samples')
for i in tqdm(range(len(pIndexTrain))):
start, stop, length = pIndexTrain[i, :]
sampleWeights[start:stop] = 1 / length
##############################################################################################
from sklearn.linear_model import LogisticRegression as LR
from sklearn.metrics import roc_auc_score as AUC
from Helper.utilities import showCoef
model = LR(class_weight='balanced', C=1e-1)
model.fit(Xtrain, Ytrain, sample_weight=sampleWeights)
showCoef(model.coef_[0], baseVars + timeVarNames)
P = model.predict_proba(Xtest)[:, 1]
print(AUC(Ytest, P))
## performance of 0.781 for base + allVars (1week)
## performance of 0.775 for base + creatVars (1week) ## DB 0.640
## performance of 0.749 for base
## performance of 0.683 for AKI stage 1
##################################################################################################
import xgboost as xgb
dtrain = xgb.DMatrix(Xtrain,
label=Ytrain,
weight=sampleWeights,
feature_names=baseVars + timeVarNames)
dtest = xgb.DMatrix(Xtest, label=Ytest, feature_names=baseVars + timeVarNames)
param = {
def train_predict(train_file,
test_file,
predict_valid_file,
predict_test_file,
model_file,
n_iter=100,
dim=4,
lrate=.1,
n_tree=30,
depth=4,
eta=0.05,
n_fold=5):
# XGB parameters and configuration
param = {
'max_depth': depth,
'eta': eta,
'objective': 'binary:logistic',
'colsample_bytree': .5,
'subsample': .5,
'eval_metric': 'auc',
'seed': 2015
}
dir_feature = os.path.dirname(train_file)
dir_model = os.path.dirname(model_file)
dir_val = os.path.dirname(predict_valid_file)
feature_name = os.path.basename(train_file)[:-8]
xg_feature_name = 'xg_{}_{}_{}_{}'.format(n_tree, depth, eta, feature_name)
algo_name = 'ffm_{}_{}_{}'.format(n_iter, dim, lrate)
model_name = '{}_{}'.format(algo_name, feature_name)
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
level=logging.DEBUG,
filename='{}.log'.format(model_name))
logging.info('Loading training data')
X, y = load_svmlight_file(train_file)
cv = StratifiedKFold(y, n_folds=n_fold, shuffle=True, random_state=2015)
p = np.zeros_like(y)
for i, (i_trn, i_val) in enumerate(cv, 1):
logging.info('Training model #{}...'.format(i))
valid_train_file = os.path.join(
dir_feature, '{}.trn{}.ffm'.format(xg_feature_name, i))
valid_test_file = os.path.join(
dir_feature, '{}.val{}.ffm'.format(xg_feature_name, i))
valid_model_file = os.path.join(dir_model,
'{}.trn{}.mdl'.format(model_name, i))
valid_predict_file = os.path.join(dir_val,
'{}.val{}.yht'.format(model_name, i))
valid_train_svm_file = os.path.join(
dir_feature, '{}.trn{}.sps'.format(feature_name, i))
valid_test_svm_file = os.path.join(
dir_feature, '{}.val{}.sps'.format(feature_name, i))
# if there are no libsvm files, generate them first.
if (not os.path.isfile(valid_train_file)) or \
(not os.path.isfile(valid_test_file)):
# if there are no libsvm files, generate them first.
if not os.path.isfile(valid_train_svm_file):
dump_svmlight_file(X[i_trn],
y[i_trn],
valid_train_svm_file,
zero_based=False)
if not os.path.isfile(valid_test_svm_file):
dump_svmlight_file(X[i_val],
y[i_val],
valid_test_svm_file,
zero_based=False)
# generate XGB features
dtrain = xgb.DMatrix(valid_train_svm_file)
dtest = xgb.DMatrix(valid_test_svm_file)
watchlist = [(dtest, 'eval'), (dtrain, 'train')]
logging.info('Generating XGB features')
xg = xgb.train(param, dtrain, n_tree, watchlist)
xg_trn_feature = xg.predict(dtrain, pred_leaf=True)
xg_tst_feature = xg.predict(dtest, pred_leaf=True)
# save XGB features as the libffm format
np_to_ffm(xg_trn_feature, dtrain.get_label(), valid_train_file)
np_to_ffm(xg_tst_feature, dtest.get_label(), valid_test_file)
subprocess.call([
"ffm-train", '-k', '{}'.format(dim), '-r',
str(lrate), '-t',
str(n_iter), '-p', valid_test_file, valid_train_file,
valid_model_file
])
subprocess.call([
"ffm-predict", valid_test_file, valid_model_file,
valid_predict_file
])
p[i_val] = np.loadtxt(valid_predict_file)
logging.info('AUC = {:.6f}'.format(AUC(y, p)))
np.savetxt(predict_valid_file, p, fmt='%.6f')
ffm_train_file = os.path.join(dir_feature,
'{}.trn.ffm'.format(xg_feature_name))
ffm_test_file = os.path.join(dir_feature,
'{}.tst.ffm'.format(xg_feature_name))
if (not os.path.isfile(ffm_train_file)) or \
(not os.path.isfile(ffm_test_file)):
# generate XGB features
dtrain = xgb.DMatrix(train_file)
dtest = xgb.DMatrix(test_file)
watchlist = [(dtrain, 'train')]
logging.info('Generating XGB features')
xg = xgb.train(param, dtrain, n_tree, watchlist)
xg_trn_feature = xg.predict(dtrain, pred_leaf=True)
xg_tst_feature = xg.predict(dtest, pred_leaf=True)
# save XGB features as the libffm format
np_to_ffm(xg_trn_feature, dtrain.get_label(), ffm_train_file)
np_to_ffm(xg_tst_feature, dtest.get_label(), ffm_test_file)
logging.info('Retraining with 100% data...')
subprocess.call([
"ffm-train", '-k', '{}'.format(dim), '-r',
str(lrate), '-t',
str(n_iter), '-p', ffm_test_file, ffm_train_file, model_file
])
subprocess.call(
["ffm-predict", ffm_test_file, model_file, predict_test_file])
def train_predict(train_file,
test_file,
train_svm_file,
predict_valid_file,
predict_test_file,
model_file,
n_iter=100,
lrate=.1,
n_fold=5):
dir_feature = os.path.dirname(train_file)
dir_model = os.path.dirname(model_file)
dir_val = os.path.dirname(predict_valid_file)
feature_name = os.path.basename(train_file)[:-7]
algo_name = 'vw_{}_{}'.format(n_iter, lrate)
model_name = '{}_{}'.format(algo_name, feature_name)
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
level=logging.DEBUG,
filename='{}.log'.format(model_name))
logging.info('Loading training data')
X, y = load_data(train_svm_file)
cv = StratifiedKFold(y, n_folds=n_fold, shuffle=True, random_state=2015)
p_val = np.zeros_like(y)
for i, (i_trn, i_val) in enumerate(cv, 1):
logging.info('Training model #{}...'.format(i))
valid_train_file = os.path.join(dir_feature,
'{}.trn{}.vw'.format(feature_name, i))
valid_test_file = os.path.join(dir_feature,
'{}.val{}.vw'.format(feature_name, i))
valid_model_file = os.path.join(dir_model,
'{}.trn{}.mdl'.format(model_name, i))
valid_predict_file = os.path.join(dir_val,
'{}.val{}.yht'.format(model_name, i))
# if there is no CV training or validation file, then generate them
# first.
if (not os.path.isfile(valid_train_file)
or not os.path.isfile(valid_test_file)):
# generate libsvm files
valid_train_svm_file = os.path.join(
dir_feature, '{}.trn{}.sps'.format(feature_name, i))
valid_test_svm_file = os.path.join(
dir_feature, '{}.val{}.sps'.format(feature_name, i))
if not os.path.isfile(valid_train_svm_file):
dump_svmlight_file(X[i_trn],
y[i_trn],
valid_train_svm_file,
zero_based=False)
if not os.path.isfile(valid_test_svm_file):
dump_svmlight_file(X[i_val],
y[i_val],
valid_test_svm_file,
zero_based=False)
# then convert libsvm files into libffm formats
svm_to_vw(valid_train_svm_file, valid_train_file, feature_name)
svm_to_vw(valid_test_svm_file, valid_test_file, feature_name)
subprocess.call([
"vw", '--loss_function', 'logistic', '-d', valid_train_file, '-f',
valid_model_file, '-l',
str(lrate), '-b', '24', '-q', '::', '--passes',
str(n_iter), '--cache_file', valid_train_file + '.tmp'
])
subprocess.call([
"vw", '-t', '-d', valid_test_file, '-i', valid_model_file, '-p',
valid_predict_file
])
p_val[i_val] = np.loadtxt(valid_predict_file)
os.remove(valid_predict_file)
logging.info('AUC = {:.4f}'.format(AUC(y, p_val)))
np.savetxt(predict_valid_file, p_val, fmt='%.6f')
logging.info('Retraining with 100% data...')
subprocess.call([
"vw", '--loss_function', 'logistic', '-d', train_file, '-f',
model_file, '-l',
str(lrate), '-b', '24', '-q', '::', '--passes',
str(n_iter), '--cache_file', train_file + '.tmp'
])
subprocess.call([
"vw", '-t', '-d', test_file, '-i', model_file, '-p', predict_test_file
])
def train_and_eval_auc( model, train_x, train_y, test_x, test_y ):
model.fit( train_x, train_y )
p = model.predict_proba(test_x )
auc = AUC( test_y, p[:,1] )
return auc
data = pd.get_dummies(data=data, columns=cat_features)
# randomize before splitting them up into train and test sets
data = data.iloc[np.random.permutation(len(data))]
data.reset_index(drop = True, inplace = True)
x = data.drop(['Target'], axis = 1)
y = data.Target
train_examples = 100000
x_train = x[:train_examples]
x_test = x[train_examples:]
y_train = y[:train_examples]
y_test = y[train_examples:]
# Logistic Regression:
clf = LogisticRegression()
clf.fit(x_train, y_train)
pred = clf.predict_proba(x_test)[:,1]
auc = AUC(y_test, pred)
print("Logistic Regression AUC: ",auc)
# Random Forest, a simple model (100 trees) trained in parallel
clf = RandomForestClassifier(n_estimators=100, n_jobs=-1)
clf.fit(x_train, y_train)
pred = clf.predict_proba(x_test)[:,1]
auc = AUC(y_test, pred)
print ("Random Forest AUC: ",auc)
# Finally, CV our results (a very simple 2-fold CV):
scores = cross_val_score(LogisticRegression(), x, y, scoring='roc_auc', cv=2)
print ("Mean AUC: {:.2%}, std: {:.2%} \n",scores.mean(),scores.std())
def train_predict(train_file,
test_file,
predict_valid_file,
predict_test_file,
feature_map,
retrain=True):
model_name = os.path.splitext(
os.path.splitext(os.path.basename(predict_test_file))[0])[0]
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
level=logging.DEBUG,
filename='{}.log'.format(model_name))
logging.info('Loading training and test data...')
X_trn, y_trn = load_data(train_file)
X_tst, _ = load_data(test_file)
feature_map = pd.read_table(feature_map,
index_col=0,
header=None,
names=['feature_names', 'feature_type'])
features = feature_map['feature_names'].values
train_df = pd.DataFrame(data=X_trn.toarray(),
columns=feature_map['feature_names'])
test_df = pd.DataFrame(data=X_tst.toarray(),
columns=feature_map['feature_names'])
train_test = train_df.append(test_df)
test_data = [
test_df.loc[:, features].values[:, k]
for k in range(test_df.loc[:, features].values.shape[1])
]
logging.info('Loading CV Ids')
cv = StratifiedKFold(n_splits=50, shuffle=True, random_state=SEED)
vld_preds = np.zeros_like(y_trn)
tst_preds = np.zeros((X_tst.shape[0], ))
for cv_idx, (i_trn, i_vld) in enumerate(cv.split(X_trn, y_trn), 1):
X_trn_cv = train_df.iloc[i_trn, :].reset_index(drop=True)
X_vld_cv = train_df.iloc[i_vld, :].reset_index(drop=True)
y_trn_cv = y_trn[i_trn]
y_vld_cv = y_trn[i_vld]
logging.info('Training model #{}'.format(cv_idx))
clf = create_keras_embedding_model(train_test, features)
clf.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=[auc])
X_trn_cv = [
X_trn_cv.loc[:, features].values[:, k]
for k in range(X_trn_cv.loc[:, features].values.shape[1])
]
X_vld_cv = [
X_vld_cv.loc[:, features].values[:, k]
for k in range(X_vld_cv.loc[:, features].values.shape[1])
]
es = callbacks.EarlyStopping(monitor='val_auc',
min_delta=0.001,
patience=5,
verbose=1,
mode='max',
baseline=None,
restore_best_weights=True)
rlr = callbacks.ReduceLROnPlateau(monitor='val_auc',
factor=0.5,
patience=3,
min_lr=1e-6,
mode='max',
verbose=1)
clf.fit(X_trn_cv,
utils.to_categorical(y_trn_cv),
validation_data=(X_vld_cv, utils.to_categorical(y_vld_cv)),
verbose=0,
batch_size=1024,
callbacks=[es, rlr],
epochs=50)
vld_preds[i_vld] = clf.predict(X_vld_cv)[:, 1]
logging.info('CV #{}: {:.4f}'.format(
cv_idx, auc(y_trn[i_vld], vld_preds[i_vld])))
if not retrain:
tst_preds += (model.predict(test_data)[:, 1] / N_FOLDS).ravel()
logging.info('Saving validation predictions...')
logging.info('CV: {:.4f}'.format(AUC(y_trn, vld_preds)))
np.savetxt(predict_valid_file, vld_preds, fmt='%.6f', delimiter=',')
if retrain:
logging.info('Retraining with 100% training data')
clf = create_keras_embedding_model(train_test, features)
clf.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=[auc])
X_trn_all = [
train_df.loc[:, features].values[:, k]
for k in range(train_df.loc[:, features].values.shape[1])
]
clf.fit(X_trn_all,
utils.to_categorical(y_trn),
validation_data=(X_trn_all, utils.to_categorical(y_trn)),
verbose=0,
batch_size=1024,
callbacks=[es, rlr],
epochs=50)
tst_preds = (clf.predict(test_data)[:, 1]).ravel()
logging.info('Saving normalized test predictions...')
np.savetxt(predict_test_file, tst_preds, fmt='%.6f', delimiter=',')
def train_predict(train_file,
test_file,
train_svm_file,
predict_valid_file,
predict_test_file,
model_file,
n_iter=100,
dim=4,
lrate=.1,
n_fold=5):
dir_feature = os.path.dirname(train_file)
dir_model = os.path.dirname(model_file)
dir_val = os.path.dirname(predict_valid_file)
feature_name = os.path.basename(train_file)[:-8]
algo_name = 'ffm_{}_{}_{}'.format(n_iter, dim, lrate)
model_name = '{}_{}'.format(algo_name, feature_name)
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
level=logging.DEBUG,
filename='{}.log'.format(model_name))
logging.info('Loading training data')
X, y = load_svmlight_file(train_svm_file)
cv = StratifiedKFold(y, n_folds=n_fold, shuffle=True, random_state=2015)
p = np.zeros_like(y)
for i, (i_trn, i_val) in enumerate(cv, 1):
logging.info('Training model #{}...'.format(i))
valid_train_file = os.path.join(dir_feature,
'{}.trn{}.ffm'.format(feature_name, i))
valid_test_file = os.path.join(dir_feature,
'{}.val{}.ffm'.format(feature_name, i))
valid_model_file = os.path.join(dir_model,
'{}.trn{}.mdl'.format(model_name, i))
valid_predict_file = os.path.join(dir_val,
'{}.val{}.yht'.format(model_name, i))
# if there is no CV training or validation file, then generate them
# first.
if (not os.path.isfile(valid_train_file)
or not os.path.isfile(valid_test_file)):
# generate libsvm files
valid_train_svm_file = os.path.join(
dir_feature, '{}.trn{}.sps'.format(feature_name, i))
valid_test_svm_file = os.path.join(
dir_feature, '{}.val{}.sps'.format(feature_name, i))
if not os.path.isfile(valid_train_svm_file):
dump_svmlight_file(X[i_trn],
y[i_trn],
valid_train_svm_file,
zero_based=False)
if not os.path.isfile(valid_test_svm_file):
dump_svmlight_file(X[i_val],
y[i_val],
valid_test_svm_file,
zero_based=False)
# then convert libsvm files into libffm formats
svm_to_ffm(valid_train_svm_file, valid_train_file, feature_name)
svm_to_ffm(valid_test_svm_file, valid_test_file, feature_name)
subprocess.call([
"ffm-train", '-k', '{}'.format(dim), '-r',
str(lrate), '-t',
str(n_iter), '-p', valid_test_file, valid_train_file,
valid_model_file
])
subprocess.call([
"ffm-predict", valid_test_file, valid_model_file,
valid_predict_file
])
p[i_val] = np.loadtxt(valid_predict_file)
logging.info('AUC = {:.6f}'.format(AUC(y, p)))
np.savetxt(predict_valid_file, p, fmt='%.6f')
logging.info('Retraining with 100% data...')
subprocess.call([
"ffm-train", '-k', '{}'.format(dim), '-r',
str(lrate), '-t',
str(n_iter), '-p', test_file, train_file, model_file
])
subprocess.call(["ffm-predict", test_file, model_file, predict_test_file])
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.ensemble import RandomForestClassifier as RF
from sklearn.metrics import roc_auc_score as AUC
from sklearn.metrics import log_loss
import os
input_file = os.getenv('TRAINING')
#
d = pd.read_csv(input_file, header=0)
features = [f for f in list(d) if 'feature' in f]
train, val = train_test_split( d, test_size = 5000 )
# train, predict, evaluate
n_trees = 100
rf = RF( n_estimators = n_trees, verbose = True )
rf.fit(train[features], train.target)
p = rf.predict_proba(val[features])
ll = log_loss(val.target.values, p[:,1])
auc = AUC( val.target.values, p[:,1] )
print("AUC: {:.2%}, log loss: {:.2%}".format(auc, ll))
def train_predict(train_file,
test_file,
predict_valid_file,
predict_test_file,
n_est=100,
n_leaf=200,
lrate=.1,
n_min=8,
subcol=.3,
subrow=.8,
subrow_freq=100,
n_stop=100,
retrain=True):
model_name = os.path.splitext(
os.path.splitext(os.path.basename(predict_test_file))[0])[0]
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
level=logging.DEBUG,
filename=f'{model_name}.log')
logging.info('Loading training and test data...')
X, y = load_data(train_file)
X_tst, _ = load_data(test_file)
logging.info('Loading CV Ids')
cv = StratifiedKFold(n_splits=N_FOLD, shuffle=True, random_state=SEED)
params = {
'random_state': SEED,
'n_jobs': -1,
'objective': 'binary',
'boosting': 'gbdt',
'learning_rate': lrate,
'num_leaves': n_leaf,
'feature_fraction': subcol,
'bagging_fraction': subrow,
'bagging_freq': subrow_freq,
'verbosity': -1,
'min_child_samples': n_min,
'metric': 'auc'
}
p = np.zeros(X.shape[0])
p_tst = np.zeros(X_tst.shape[0])
n_bests = []
for i, (i_trn, i_val) in enumerate(cv.split(X, y), 1):
logging.info(f'Training model #{i}')
trn_lgb = lgb.Dataset(X[i_trn], label=y[i_trn])
val_lgb = lgb.Dataset(X[i_val], label=y[i_val])
logging.info('Training with early stopping')
clf = lgb.train(params,
trn_lgb,
n_est,
val_lgb,
early_stopping_rounds=n_stop,
verbose_eval=100)
n_best = clf.best_iteration
n_bests.append(n_best)
logging.info(f'best iteration={n_best}')
p[i_val] = clf.predict(X[i_val])
logging.info(f'CV #{i}: {AUC(y[i_val], p[i_val]):.4f}')
if not retrain:
p_tst += clf.predict(X_tst) / N_FOLD
logging.info('CV: {:.4f}'.format(AUC(y, p)))
logging.info('Saving validation predictions...')
np.savetxt(predict_valid_file, p, fmt='%.6f', delimiter=',')
if retrain:
logging.info('Retraining with 100% training data')
n_best = sum(n_bests) // N_FOLD
clf = lgb.LGBMRegressor(n_estimators=n_best,
num_leaves=n_leaf,
learning_rate=lrate,
min_child_samples=n_min,
subsample=subrow,
subsample_freq=subrow_freq,
colsample_bytree=subcol,
seed=SEED)
clf = clf.fit(X, y)
p_tst = clf.predict(X_tst)
logging.info('Saving test predictions...')
np.savetxt(predict_test_file, p_tst, fmt='%.6f', delimiter=',')
Y = data['future48_AKI2_overlap'].values
Ytrain, Ytest = Y[:cutoff], Y[cutoff:]
IDtrain = data['PAT_ENC_CSN_ID'].values[:cutoff]
selectTrain, selectTest = np.isfinite(Ytrain), np.isfinite(Ytest)
Xtrain, Ytrain, IDtrain = Xtrain[
selectTrain, :], Ytrain[selectTrain], IDtrain[selectTrain]
Xtest, Ytest = Xtest[selectTest, :], Ytest[selectTest]
pIndexSub = getPatientIndices(IDtrain)
sampleWeights = np.zeros(len(Xtrain))
for i in tqdm(range(len(pIndexSub))):
start, stop, length = pIndexSub[i, :]
sampleWeights[start:stop] = 1 / length
#X = np.concatenate((valueFeatures, timeFeatures, data2[['los']].values,
# data2[['creatinine']].values), axis = 1)
##############################################################################
from sklearn.linear_model import LogisticRegression as LR
from sklearn.metrics import roc_auc_score as AUC
from Helper.utilities import showCoef
model = LR(class_weight='balanced', C=1e-1)
model.fit(Xtrain, Ytrain) #,sample_weight = sampleWeights)
P = model.predict_proba(Xtest)[:, 1]
model.coef_
print(AUC(Ytest, P)) #, sample_weight = sampleWeights))
## performance is around 0.83 currently
def on_epoch_end(self, epoch, logs={}):
if epoch % self.interval == 0:
y_pred = self.model.predict_proba(self.X_val, verbose=0)
score = AUC(self.y_val, y_pred)
#logging.info("interval evaluation - epoch: {:d} - score: {:.6f}".format(epoch, score))
print( "interval evaluation - epoch: {:d} - score: {:.6f}".format(epoch, score))
p_train = p[0:train_end] p_test = p[test_start:] ### lr = LR() # default param values lr.fit(p_train.reshape(-1, 1), y_train) # LR needs X to be 2-dimensional p_calibrated = lr.predict_proba(p_test.reshape(-1, 1))[:, 1] ### acc = accuracy_score(y_test, np.round(p_test)) acc_calibrated = accuracy_score(y_test, np.round(p_calibrated)) auc = AUC(y_test, p_test) auc_calibrated = AUC(y_test, p_calibrated) ll = log_loss(y_test, p_test) ll_calibrated = log_loss(y_test, p_calibrated) print "accuracy - before/after:", acc, "/", acc_calibrated print "AUC - before/after: ", auc, "/", auc_calibrated print "log loss - before/after:", ll, "/", ll_calibrated """ accuracy - before/after: 0.847788697789 / 0.846805896806 AUC - before/after: 0.878139845077 / 0.878139845077 log loss - before/after: 0.630525772871 / 0.364873617584 """ ###
'data/output/temp/v17_stage1_ver2_all_fold.csv',
'data/output/temp/v18_stage1_ver2_all_fold.csv',
'data/output/temp/v19_stage1_ver2_all_fold.csv',
'data/output/temp/v20_stage1_ver2_all_fold.csv',
]:
x = pd.read_csv(i)
a = pd.concat([a, x], axis=1)
cv_index = {}
set_name = 'set{}'.format(set_idnex)
for i in xrange(5):
train_cv = set_data.loc[(set_data[set_name] != i).values,
set_name].index
test_cv = set_data.loc[(set_data[set_name] == i).values,
set_name].index
cv_index[i] = {}
cv_index[i]['train'] = train_cv.values
cv_index[i]['test'] = test_cv.values
skf = pd.DataFrame(cv_index).stack().T
auc = []
for i in xrange(5):
#print AUC(y.ix[skf['test'][i]].values, x.ix[skf['test'][i]].values)
auc.append(
AUC(y.ix[skf['test'][i]].values, x.ix[skf['test'][i]].values))
set_idnex += 1
print 'Per model, mean: {} std: {}'.format(np.mean(auc), np.std(auc))
print 'Averaging AUC:{}'.format(AUC(y.values, a.mean(1).values))
#AUC:0.842626778562
def train_predict(train_file, test_file, predict_valid_file, predict_test_file,
cid_train_file, cid_test_file):
feature_name = os.path.basename(train_file)[:-4]
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
level=logging.DEBUG,
filename='xg_cid_{}.log'.format(feature_name))
logging.info('Loading course IDs for training and test data')
cid_trn = np.loadtxt(cid_train_file, dtype=int)
cid_tst = np.loadtxt(cid_test_file, dtype=int)
logging.info('Loading training and test data...')
X, y = load_data(train_file)
X_tst, y_tst = load_data(test_file)
cv = StratifiedKFold(y, n_folds=5, shuffle=True, random_state=2015)
p = np.zeros_like(y)
for i, (i_trn, i_val) in enumerate(cv, 1):
logging.info('Training model #{}...'.format(i))
X_trn = X[i_trn]
y_trn = y[i_trn]
X_val = X[i_val]
y_val = y[i_val]
cid_valtrn = cid_trn[i_trn]
cid_valtst = cid_trn[i_val]
p_trn = np.zeros_like(y_trn)
p_val = np.zeros_like(y_val)
for j in range(39):
idx_trn = np.where(cid_valtrn == j)[0]
idx_val = np.where(cid_valtst == j)[0]
clf = xgb.XGBClassifier(max_depth=PARAM[j][2],
learning_rate=PARAM[j][1],
n_estimators=PARAM[j][0],
colsample_bytree=1,
subsample=.4,
nthread=6)
clf.fit(X_trn[idx_trn], y_trn[idx_trn])
p_trn[idx_trn] = clf.predict_proba(X_trn[idx_trn])[:, 1]
p_val[idx_val] = clf.predict_proba(X_val[idx_val])[:, 1]
logging.info('CID #{}: {:.4f}, {:.4f}'.format(
j, AUC(y_trn[idx_trn], p_trn[idx_trn]),
AUC(y_val[idx_val], p_val[idx_val])))
logging.info('AUC TRN = {:.6f}'.format(AUC(y_trn, p_trn)))
logging.info('AUC VAL = {:.6f}'.format(AUC(y_val, p_val)))
p[i_val] = p_val
logging.info('AUC = {:.6f}'.format(AUC(y, p)))
logging.info('Saving CV predictions...')
np.savetxt(predict_valid_file, p, fmt='%.6f')
logging.info('Retraining with 100% data...')
p_tst = np.zeros_like(y_tst)
n_tst = len(p_tst)
for j in range(39):
idx_trn = np.where(cid_trn == j)[0]
idx_tst = np.where(cid_tst == j)[0]
logging.info('CID #{}: {:.2f}%'.format(j, len(idx_tst) / n_tst * 100))
clf.fit(X[idx_trn], y[idx_trn])
p_tst[idx_tst] = clf.predict_proba(X_tst[idx_tst])[:, 1]
logging.info('Saving test predictions...')
np.savetxt(predict_test_file, p_tst, fmt='%.6f')
def train_predict(train_feature_file,
test_feature_file,
predict_valid_file,
predict_test_file,
C=1.0,
class_weight='balanced',
max_iter=1000,
solver='lbfgs',
retrain=True):
model_name = os.path.splitext(
os.path.splitext(os.path.basename(predict_test_file))[0])[0]
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
level=logging.DEBUG,
filename='{}.log'.format(model_name))
logging.info('Loading training and test data...')
X, y = load_data(train_feature_file)
X_tst, _ = load_data(test_feature_file)
logging.info('Loading CV Ids')
cv = StratifiedKFold(n_splits=N_FOLD, shuffle=True,
random_state=SEED).split(X, y)
p_val = np.zeros(X.shape[0])
p_tst = np.zeros(X_tst.shape[0])
for i, (i_trn, i_val) in enumerate(cv, 1):
logging.info('Training model #{}'.format(i))
logging.info('Training Logistic Regression')
clf = LogisticRegression(C=C,
class_weight=class_weight,
max_iter=max_iter,
solver=solver,
random_state=SEED)
clf = clf.fit(X[i_trn], y[i_trn])
p_val[i_val] = clf.predict_proba(X[i_val])[:, 1]
logging.info('CV #{}: {:.4f}'.format(i, AUC(y[i_val], p_val[i_val])))
if not retrain:
p_tst += clf.predict_proba(X_tst)[:, 1] / N_FOLD
logging.info('CV: {:.4f}'.format(AUC(y, p_val)))
logging.info('Saving validation predictions...')
np.savetxt(predict_valid_file, p_val, fmt='%.6f', delimiter=',')
if retrain:
logging.info('Retraining with 100% training data')
clf = LogisticRegression(C=C,
class_weight=class_weight,
max_iter=max_iter,
solver=solver,
random_state=SEED)
clf = clf.fit(X, y)
p_tst = clf.predict_proba(X_tst)[:, 1]
logging.info('Saving test predictions...')
np.savetxt(predict_test_file, p_tst, fmt='%.6f', delimiter=',')
def sort_training_set(self, classifier='RF'):
print "loading..."
train = pd.read_csv(resource_filename('numerai.data', self.train_file_name))
test = pd.read_csv(resource_filename('numerai.data', self.test_file_name))
test.drop('t_id', axis=1, inplace=True)
test['target'] = 0 # dummy for preserving column order when concatenating
train['is_test'] = 0
test['is_test'] = 1
orig_train = train.copy()
assert (np.all(orig_train.columns == test.columns))
train = pd.concat((orig_train, test))
train.reset_index(inplace=True, drop=True)
x = train.drop(['is_test', 'target'], axis=1)
y = train.is_test
print "cross-validating..."
n_estimators = 100
if classifier == 'RF':
clf = RF(bootstrap=True,
min_samples_leaf=3,
n_estimators=n_estimators,
max_features=20,
criterion='gini',
min_samples_split=20,
max_depth=None,
n_jobs=6)
else:
clf = LR(n_jobs=6)
predictions = np.zeros(y.shape)
cv = CV.StratifiedKFold(y, n_folds=5, shuffle=True, random_state=5678)
for f, (train_i, test_i) in enumerate(cv):
print "# fold {}, {}".format(f + 1, ctime())
x_train = x.iloc[train_i]
x_test = x.iloc[test_i]
y_train = y.iloc[train_i]
y_test = y.iloc[test_i]
clf.fit(x_train, y_train)
p = clf.predict_proba(x_test)[:, 1]
auc = AUC(y_test, p)
print "# AUC: {:.2%}\n".format(auc)
predictions[test_i] = p
train['p'] = predictions
i = predictions.argsort()
train_sorted = train.iloc[i]
train_sorted = train_sorted.loc[train_sorted.is_test == 0]
assert (train_sorted.target.sum() == orig_train.target.sum())
train_sorted.drop('is_test', axis=1, inplace=True)
train_sorted.to_csv(resource_filename('numerai.data', self.sorted_file_name), index=False)
x_test = np.hstack((x_num_test, vec_x_cat_test))
if __name__ == "__main__":
print "training..."
n_trees = 100
max_features = int(round(sqrt(x_train.shape[1]) *
2)) # try more features at each split
max_features = 'auto'
verbose = 1
n_jobs = 1
rf = RF(n_estimators=n_trees,
max_features=max_features,
verbose=verbose,
n_jobs=n_jobs)
rf.fit(x_train, y_train)
p = rf.predict_proba(x_test)
auc = AUC(y_test, p[:, 1])
print "AUC", auc
# AUC 0.701579086548
# AUC 0.676126704696
# max_features * 2
# AUC 0.710060065732
# AUC 0.706282346719
# Get a bag of words for the test set, and convert to a numpy array
test_data_features = vectorizer.transform(clean_test_reviews)
test_data_features = test_data_features.toarray()
###
print "Training the random forest (this may take a while)..."
forest = RandomForestClassifier(n_estimators=100, n_jobs=-1, verbose=1)
forest = forest.fit(train_data_features, train["sentiment"])
print "Predicting test labels...\n"
rf_p = forest.predict_proba(test_data_features)
auc = AUC(test['sentiment'].values, rf_p[:, 1])
print "random forest AUC:", auc
# a random score from a _random_ forest
# AUC: 0.919056767104
# let's define a helper function
def train_and_eval_auc(model, train_x, train_y, test_x, test_y):
model.fit(train_x, train_y)
p = model.predict_proba(test_x)
auc = AUC(test_y, p[:, 1])
return auc
def train_predict(train_file,
test_file,
predict_valid_file,
predict_test_file,
n_est=100,
batch_size=1024,
retrain=True):
model_name = os.path.splitext(
os.path.splitext(os.path.basename(predict_test_file))[0])[0]
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
level=logging.DEBUG,
filename='{}.log'.format(model_name))
logging.info('Loading training and test data...')
X, y = load_data(train_file)
X_tst, _ = load_data(test_file)
dims = X.shape[1]
logging.info('{} dims'.format(dims))
logging.info('Loading CV Ids')
cv = StratifiedKFold(n_splits=N_FOLD, shuffle=True, random_state=SEED)
p = np.zeros_like(y)
p_tst = np.zeros((X_tst.shape[0], ))
for i, (i_trn, i_val) in enumerate(cv.split(X, y), 1):
logging.info('Training model #{}'.format(i))
clf = nn_model(dims)
clf.fit_generator(generator=batch_generator(X[i_trn], y[i_trn],
batch_size, True),
nb_epoch=n_est,
samples_per_epoch=X[i_trn].shape[0],
verbose=1)
p[i_val] = clf.predict_generator(generator=batch_generatorp(
X[i_val], batch_size, False),
val_samples=X[i_val].shape[0])[:, 0]
logging.info('CV #{}: {:.4f}'.format(i, AUC(y[i_val], p[i_val])))
if not retrain:
p_tst += clf.predict_generator(
generator=batch_generatorp(X_tst, batch_size, False),
val_samples=X_tst.shape[0])[:, 0] / N_FOLD
logging.info('Saving validation predictions...')
logging.info('CV: {:.4f}'.format(AUC(y, p)))
np.savetxt(predict_valid_file, p, fmt='%.6f', delimiter=',')
if retrain:
logging.info('Retraining with 100% training data')
clf = nn_model(dims)
clf.fit_generator(generator=batch_generator(X, Y, batch_size, True),
nb_epoch=n_est)
p_tst = clf.predict_generator(generator=batch_generatorp(
X_tst, batch_size, False),
val_samples=X_tst.shape[0])[:, 0]
logging.info('Saving normalized test predictions...')
np.savetxt(predict_test_file, p_tst, fmt='%.6f', delimiter=',')