def main(): # Todo: # 1. Automatically pick best model for every question. # 4. Try new signals. # 2. Try kappa as loss function. # 3. Try better loss function. N_EST = 150 boost_params = dict(min_samples_leaf=5, min_samples_split=5, max_depth=1, n_estimators=N_EST, learn_rate=.3) list_of_signals = [G.num_words, G.is_crap, G.word_length, G.answer_length, G.num_sentences, G.choice] ml_params.PlotNumberOfEstimators(9, list_of_signals, mods_for_training=range(4), boost_params=boost_params, steps=N_EST) return vs = [] for q in range(10): if q == 9: list_of_signals.append(G.choice) dd = [] for msl in [3, 4, 5]: for n_estimators in range(30, 50, 5): for max_depth in range(1, 6): varz = dict(min_samples_leaf=msl, min_samples_split=msl, n_estimators=n_estimators, max_depth=max_depth) v = PredictForSignals(q, list_of_signals, boost_params=varz) dd.append((-v, varz)) # vs.append() #print kappa.mean_quadratic_weighted_kappa(vs) return # TODO: show difference between models! GenerateBasicFeatures() GenerateTrainingFeatures() return # TODO: try to integrate some other signal into "all zeroes" model # TODO: Find most popular words for each question, try to use # TODO: use different target function ''' for id, elem in enumerate(G.num_words): if elem < 3: print id, G.question[id], G.score[id], G.other_score[id], G.answer[id] ''' vars = {'min_word_cutoff': 3., 'ans_len': .5, 'num_sent': 10., } # TODO: compute scores for pub leaderboard. #scores = ScoresForModel(VerySimple2, vars) print 'Current' for k in range(5): CheckModel(models.Version0, util.FMod(k)) for k in range(5): CheckModel(models.Version1, util.FMod(k)) return data0 = [] data1 = [] data2 = [] data3 = [] ks = range(5, 100, 5) for k in ks: vars['min_word_cutoff'] = k scores0 = RawScoresForModel(VerySimple0, vars, extra_filter=util.FMod(1)) data0.append(metrics.EvalPerQuestion(scores0, extra_filter=util.FMod(1))) scores1 = RawScoresForModel(VerySimple1, vars, extra_filter=util.FMod(1)) data1.append(metrics.EvalPerQuestion(scores1, extra_filter=util.FMod(1))) scores2 = RawScoresForModel(VerySimple2, vars, extra_filter=util.FMod(1)) data2.append(metrics.EvalPerQuestion(scores2, extra_filter=util.FMod(1))) scores3 = RawScoresForModel(VerySimple3, vars, extra_filter=util.FMod(1)) data3.append(metrics.EvalPerQuestion(scores3, extra_filter=util.FMod(1))) # print k, '%.2f' % (sum(v2) / len(v2)), ['%.3f' % v for v in v2] cutoff = [] value = [] score = [] for q in range(10): s0 = [(-data0[i][q], (k, 0)) for i, k in enumerate(ks)] s1 = [(-data1[i][q], (k, 30)) for i, k in enumerate(ks)] s2 = [(-data2[i][q], (k, 50)) for i, k in enumerate(ks)] s3 = [(-data3[i][q], (k, 100)) for i, k in enumerate(ks)] x = sorted(s0 + s1 + s2 + s3) score.append(-x[0][0]) cutoff.append(x[0][1][0]) value.append(x[0][1][1]) print cutoff print value print kappa.mean_quadratic_weighted_kappa(score), util.PrintList(score)
def Eval(raw_scores, extra_filter=FTrue, only_questions=None): vals = EvalPerQuestion(raw_scores, extra_filter=extra_filter, only_questions=only_questions) known_vals = filter(lambda v: v != signal.UNKNOWN, vals) return kappa.mean_quadratic_weighted_kappa(known_vals)