def gen_kappa_cv(self, bagging_iter, y_list_valid, cdf_list_valid,
num_valid_matrix, p_ens_list_valid_topk,
p_ens_list_valid):
"""
多次bagging 的结果平均值
:param bagging_iter:第几次bagging，有几次，权重是几
:param p_ens_list_valid: 多次执行有状态
:param p_ens_list_valid_topk:
:return:
"""
kappa_cv = np.zeros((config.n_runs, config.n_folds), dtype=float)
cutoff = np.zeros((3), dtype=float)
for run in range(config.n_runs):
for fold in range(config.n_folds):
numValid = num_valid_matrix[run][fold]
true_label = y_list_valid[run, fold, :numValid]
cdf = cdf_list_valid[run, fold, :]
# 每次bagging的结果平均
p_ens_list_valid[run, fold, :numValid] = (
bagging_iter * p_ens_list_valid[run, fold, :numValid] +
p_ens_list_valid_topk[run, fold, :numValid]) / (
bagging_iter + 1.)
score, cutoff_tmp = getScore(
p_ens_list_valid[run, fold, :numValid], cdf, "valid")
kappa_cv[run][fold] = quadratic_weighted_kappa(
score, true_label)
cutoff += cutoff_tmp
cutoff /= float(config.n_runs * config.n_folds)
# 没搞懂？
cutoff *= (22513 / ((2. / 3) * 10158))
print("Bag %d, kappa: %.6f (%.6f)" %
(bagging_iter + 1, np.mean(kappa_cv), np.std(kappa_cv)))
return kappa_cv, cutoff, p_ens_list_valid
def ensemble_selection_obj(self, param, p1_list, weight1, p2_list,
y_list_valid, cdf_list_valid, num_valid_matrix):
"""
优化param中的weight_current_model参数，使其平均kappa_cv_mean 最大
:param param:
:param p1_list: 集成前五个模型(也就是对前五个模型求平均值的结果)
:param weight1: 1
:param p2_list: 当前模型预测结果
:return:
"""
weight_current_model = param['weight_current_model']
kappa_cv = np.zeros((config.n_runs, config.n_folds), dtype=float)
for run in range(config.n_runs):
for fold in range(config.n_folds):
num_valid = num_valid_matrix[run][fold]
p1 = p1_list[run, fold, :num_valid]
p2 = p2_list[run, fold, :num_valid]
true_label = y_list_valid[run, fold, :num_valid]
cdf = cdf_list_valid[run, fold, :]
p_ens = (weight1 * p1 + weight_current_model * p2) / (
weight1 + weight_current_model)
p_ens_score = getScore(p_ens, cdf)
kappa_cv[run][fold] = quadratic_weighted_kappa(
p_ens_score, true_label)
kappa_cv_mean = np.mean(kappa_cv)
return {'loss': -kappa_cv_mean, 'status': STATUS_OK}
def evalerror_softmax_cdf(preds, dtrain, cdf):
## label are in [0,1,2,3]
labels = dtrain.get_label() + 1
preds = getClfScore(preds, cdf)
kappa = quadratic_weighted_kappa(labels, preds)
## we return -kappa for using early stopping
kappa *= -1.
return 'kappa', float(kappa)
def evalerror_cocr_cdf(preds, dtrain, cdf):
labels = dtrain.get_label() + 1
# print preds.shape
## get prediction
# preds = sigmoid(preds)
preds = applyCOCRRule(preds)
preds = getScore(preds, cdf)
kappa = quadratic_weighted_kappa(labels, preds)
## we return -kappa for using early stopping
kappa *= -1.
return 'kappa', float(kappa)
def init_topk_best_model(self, init_top_k, this_sorted_models,
pred_list_valid, y_list_valid, cdf_list_valid,
num_valid_matrix):
"""
选择前五个模型 返回整合后的预测值；前五个模型名字；前五个模型的权重(全是1,相当于取平均值)
读取实例变量：
pred_list_valid
num_valid_matrix
model2idx
cdf_list_valid
y_list_valid
:param init_top_k:
:param this_sorted_models:
:return:best_model_list, best_model_weight, p_ens_list_valid_topk, w_ens
"""
best_model_list = []
best_model_weight = []
p_ens_list_valid_topk = np.zeros(
(config.n_runs, config.n_folds, self.max_num_valid), dtype=float)
w_ens, this_w = 0, 1.0
cnt = 0
kappa_cv = np.zeros((config.n_runs, config.n_folds), dtype=float)
for model, kappa in this_sorted_models[0:init_top_k]:
print("add the following model to the ensembles ")
print("model: %s" % model)
print("kappa: %.6f" % kappa)
# 指定模型的预测结果
this_p_list_valid = pred_list_valid[self.model2idx[model]]
for run in range(config.n_runs):
for fold in range(config.n_folds):
num_valid = num_valid_matrix[run][fold]
# 多个模型预测值线性组合
p_ens_list_valid_topk[run, fold, :num_valid] = (
w_ens * p_ens_list_valid_topk[run, fold, :num_valid] +
this_w * this_p_list_valid[run, fold, :num_valid]) / (
w_ens + this_w)
# 在最后一个model，生成一些指标
if cnt == init_top_k - 1:
cdf = cdf_list_valid[run, fold, :]
true_label = y_list_valid[run, fold, :num_valid]
score = getScore(
p_ens_list_valid_topk[run, fold, :num_valid], cdf)
kappa_cv[run][fold] = quadratic_weighted_kappa(
score, true_label)
best_model_list.append(model)
best_model_weight.append(this_w)
w_ens += this_w
cnt += 1
print("Init kappa: %.6f (%.6f)" %
(np.mean(kappa_cv), np.std(kappa_cv)))
return best_model_list, best_model_weight, p_ens_list_valid_topk, w_ens
def evalerror_ebc_cdf(preds, dtrain, cdf, hard_threshold=False):
labels = dtrain.get_label()
## extended samples within the feature construction part
if np.min(labels) == -1 and np.max(labels) == 1:
labels = applyEBCRule(labels)
## extended samples within the objective value computation part
## See ebcobj function for detail
else:
## label are in [0,1,2,3]
labels += 1
# print preds.shape
## get prediction
# hard = False
if hard_threshold:
preds = applyEBCRule(preds, hard_threshold=hard_threshold)
else:
preds = sigmoid(preds)
preds = applyEBCRule(preds, hard_threshold=hard_threshold)
preds = getScore(preds, cdf)
kappa = quadratic_weighted_kappa(labels, preds)
## we return -kappa for using early stopping
kappa *= -1.
return 'kappa', float(kappa)
def find_best_model(self, this_sorted_models, pred_list_valid,
y_list_valid, cdf_list_valid, num_valid_matrix, w_ens,
w_min, w_max, best_kappa, hypteropt_max_evals,
p_ens_list_valid_topk):
"""
从模型集合中找出一个最佳模型，最佳系数，与topK集成结果进行线性组合
寻找最佳模型、权重、kappa值 从this_sorted_models找到一个最佳模型
:param this_sorted_models:
:param w_ens:
:param w_min:
:param w_max:
:param hypteropt_max_evals:
:param p_ens_list_valid_topk:
:return:
"""
best_model = None
best_weight = 0
for model, kappa in this_sorted_models:
# 当前模型预测值
this_p_list_valid = pred_list_valid[self.model2idx[model]]
# hyperopt 找当前模型最优权重
trials = Trials()
# 不同模型的权重
param_space = {
'weight_current_model':
hp.uniform('weight_current_model', w_min, w_max)
}
# topk权重是1 找另一个最佳权重
obj = lambda param: self.ensemble_selection_obj(
param, p_ens_list_valid_topk, 1., this_p_list_valid,
y_list_valid, cdf_list_valid, num_valid_matrix)
best_params = fmin(obj,
param_space,
algo=tpe.suggest,
trials=trials,
max_evals=hypteropt_max_evals)
# 返回当前模型权重
this_w = best_params['weight_current_model']
# 按比例缩放当前权重 1 this_w --- w_ens this_w * w_ens
this_w *= w_ens
# 当前kappa cv
kappa_cv = np.zeros((config.n_runs, config.n_folds), dtype=float)
for run in range(config.n_runs):
for fold in range(config.n_folds):
num_valid = num_valid_matrix[run][fold]
# topk预测
p1 = p_ens_list_valid_topk[run, fold, :num_valid]
# 当前预测
p2 = this_p_list_valid[run, fold, :num_valid]
# 真实label
true_label = y_list_valid[run, fold, :num_valid]
cdf = cdf_list_valid[run, fold, :]
# 集成后的结果
p_ens = (w_ens * p1 + this_w * p2) / (w_ens + this_w)
score = getScore(p_ens, cdf)
# 集成后kappa值
kappa_cv[run][fold] = quadratic_weighted_kappa(
score, true_label)
# 集成后平均kappa cv 由于现在
if np.mean(kappa_cv) > best_kappa:
best_kappa, best_model, best_weight = np.mean(
kappa_cv), model, this_w
return best_kappa, best_model, best_weight
def init_model_metrics_by_run_fold(self, feat_folder, cdf):
"""
为每个交叉验证数据按照 run-fold生成一系列指标
初始化实例变量,供后续方法使用
kappa_list ：每个模型的平均kappa值
num_valid_matrix:每个run-fold 的预测结果行数
y_list_valid :每个run-fold 的真实label
cdf_list_valid ：每个run-fold 的cdf
kappa_cv ：每个run-fold 的kappa cv
pred_list_valid :每个run-fold 的真实预测值
:param feat_folder:
:param cdf:
:return:
"""
kappa_list = dict()
# 模型-run-fold-行 交叉验证-valid数据集预测结果
pred_list_valid = np.zeros((len(self.model_list), config.n_runs,
config.n_folds, self.max_num_valid),
dtype=float)
# run-fold-行 交叉验证-valid数据集真实label
y_list_valid = np.zeros(
(config.n_runs, config.n_folds, self.max_num_valid), dtype=float)
# run-fold-4类别 交叉验证-valid数据集预测结果cdf
cdf_list_valid = np.zeros(
(config.n_runs, config.n_folds, config.num_of_class), dtype=float)
# run-fold valid 交叉验证-valid数据集预测结果行数
num_valid_matrix = np.zeros((config.n_runs, config.n_folds), dtype=int)
print("Load model...")
for i, model in enumerate(self.model_list):
print("model: %s" % model)
kappa_cv = np.zeros((config.n_runs, config.n_folds), dtype=float)
for run in range(config.n_runs):
for fold in range(config.n_folds):
path = "%s/Run%d/Fold%d/pred" % (self.model_folder,
run + 1, fold + 1)
pred_file = "%s/valid.pred.%s.csv" % (path, model)
cdf_file = "%s/Run%d/Fold%d/valid.cdf" % (
config.solution_info, run + 1, fold + 1)
this_p_valid = pd.read_csv(pred_file, dtype=float)
# 这些指标只需要执行一次就行了，每个模型都一样
if i == 0:
# 记录run-fold的行数
num_valid_matrix[run][fold] = this_p_valid.shape[0]
# 记录run-fold的真实值
y_list_valid[run, fold, :num_valid_matrix[run]
[fold]] = this_p_valid["target"].values
# load cdf
if cdf == None:
cdf_list_valid[run,
fold, :] = np.loadtxt(cdf_file,
dtype=float)
else:
cdf_list_valid[run, fold, :] = cdf
score = getScore(this_p_valid["prediction"].values,
cdf_list_valid[run, fold, :])
kappa_cv[run][fold] = quadratic_weighted_kappa(
score,
y_list_valid[run,
fold, :num_valid_matrix[run][fold]])
# 记录model-run-fold的预测值数组
pred_list_valid[
self.model2idx[model], run, fold, :this_p_valid.
shape[0]] = this_p_valid["prediction"].values
print("kappa: %.6f" % np.mean(kappa_cv))
# 算出每个模型的平均kappa_cv
kappa_list[model] = np.mean(kappa_cv)
return kappa_list, pred_list_valid, y_list_valid, cdf_list_valid, num_valid_matrix
