def test_prediction_setitem(): "compare prediction._setitem__ with merge" data = nx.play_data() p1 = nx.production(nx.logistic(), data, 'bernie', verbosity=0) p2 = nx.production(nx.logistic(1e-5), data, 2, verbosity=0) p3 = nx.production(nx.logistic(1e-6), data, 3, verbosity=0) p4 = nx.backtest(nx.logistic(), data, 4, verbosity=0) p = nx.Prediction() p[('logistic', 1)] = p1 p[('logistic', 2)] = p2 p[('logistic', 3)] = p3 p[('logistic', 4)] = p4 pp = nx.Prediction() pp = pp.merge(p1) pp = pp.merge(p2) pp = pp.merge(p3) pp = pp.merge(p4) pd.testing.assert_frame_equal(p.df, pp.df) assert_raises(ValueError, p.__setitem__, ('logistic', 1), p1) assert_raises(ValueError, p.__setitem__, ('logistic', 1), p)
def test_prediction_setitem(): """compare prediction._setitem__ with merge""" data = nx.play_data() p1 = nx.production(nx.linear(), data, 'kazutsugi', verbosity=0) p2 = nx.production(nx.linear(), data, 8, verbosity=0) p3 = nx.production(nx.linear(), data, 8, verbosity=0) p4 = nx.backtest(nx.linear(), data, 8, verbosity=0) p = nx.Prediction() p[('linear', 1)] = p1 p[('linear', 2)] = p2 p[('linear', 3)] = p3 p[('linear', 4)] = p4 pp = nx.Prediction() pp = pp.merge(p1) pp = pp.merge(p2) pp = pp.merge(p3) pp = pp.merge(p4) pd.testing.assert_frame_equal(p.df, pp.df) assert_raises(ValueError, p.__setitem__, ('linear', 1), p1) assert_raises(ValueError, p.__setitem__, ('linear', 1), p)
def test_prediction_check(): "make sure prediction.check runs" d = nx.play_data() p = nx.production(nx.logistic(), d, verbosity=0) p += nx.production(nx.logisticPCA(), d, verbosity=0) df = p.check(['logistic'], d) ok_(isinstance(df, pd.DataFrame), 'expecting a dataframe')
def test_prediction_setitem(): "compare prediction._setitem__ with merge" data = nx.play_data() p1 = nx.production(nx.logistic(), data, 'model1', verbosity=0) p2 = nx.production(nx.logistic(1e-5), data, 'model2', verbosity=0) p3 = nx.production(nx.logistic(1e-6), data, 'model3', verbosity=0) p4 = nx.backtest(nx.logistic(), data, 'model1', verbosity=0) p = nx.Prediction() p['model1'] = p1 p['model2'] = p2 p['model3'] = p3 p['model1'] = p4 pp = nx.Prediction() pp = pp.merge(p1) pp = pp.merge(p2) pp = pp.merge(p3) pp = pp.merge(p4) pd.testing.assert_frame_equal(p.df, pp.df) assert_raises(ValueError, p.__setitem__, 'model1', p1) assert_raises(ValueError, p.__setitem__, 'model1', p)
def test_model_run(): "Make sure models run" d = nx.play_data() dfit = d['train'] dpre = d['tournament'] for model in get_models(): model.fit_predict(dfit, dpre, tournament=5)
def test_data_subsample(): "test data.subsample" d = nx.play_data() d2 = d.subsample(0.5, balance=True) for era, idx in d2.era_iter(): m = d2.y[idx].mean() if np.isfinite(m): ok_(d2.y[idx].mean() == 0.5, 'data is not balanced')
def test_cvsplitter_kfold(): """make sure cvsplitter runs k folds""" d = nx.play_data() for k in (2, 3): splitter = nx.CVSplitter(d, kfold=k) count = 0 for dfit, dpredict in splitter: count += 1 ok_(count == k, "CVSplitter iterated through wrong number of folds")
def test_prediction_check(): """make sure prediction.check runs""" d = nx.play_data() p1 = nx.production(nx.linear(), d, 'kazutsugi', verbosity=0) p2 = p1.copy() p2 = p2.rename('example_predictions') p = p1 + p2 with nx.testing.HiddenPrints(): df = p.check(d) ok_(isinstance(df, dict), 'expecting a dictionary')
def test_rollsplitter(): """make sure rollsplitter has no overlaps""" d = nx.play_data() splitter = nx.RollSplitter(d, fit_window=15, predict_window=10, step=15) for dfit, dpre in splitter: fera = dfit.unique_era() pera = dpre.unique_era() tera = np.unique(np.concatenate((fera, pera))) nfit = fera.size npre = pera.size ntot = tera.size ok_(nfit + npre == ntot, "RollSplitter has era overalp")
def test_prediction_regression(): "regression test of prediction performance evaluation" d = nx.play_data() p = nx.production(nx.logistic(), d, tournament=None, verbosity=0) for number, name in nx.tournament_iter(): p2 = nx.production(nx.logistic(), d, tournament=name, verbosity=0) df = p.performance_mean(d['validation'], mean_of='tournament') logloss1 = df.loc[name]['logloss'] logloss2 = p2.summary(d['validation']).loc['mean']['logloss'] diff = np.abs(logloss1 - logloss2) msg = 'failed on {}'.format(name) ok_(diff < 1e-6, msg)
def test_prediction_regression(): """regression test of prediction performance evaluation""" d = nx.play_data() p = nx.production(nx.linear(), d, tournament=None, verbosity=0) for number, name in nx.tournament_iter(): p2 = nx.production(nx.linear(), d, tournament=name, verbosity=0) df = p.performance_mean(d['validation'], mean_of='tournament') logloss1 = float('%.3f' % (df.loc[name]['mse'])) logloss2 = float('%.3f' % (p2.summary(d['validation']).loc['mean']['mse'])) diff = np.abs(logloss1 - logloss2) msg = f"failed on {name}" ok_(diff < 1e-6, msg)
def runner_example(): data = nx.play_data() splitter = nx.CVSplitter(data) # let's run 3 models m1 = {'model': nx.logistic(), 'prediction_file': None, 'csv_file': None} m2 = {'model': nx.logistic(1e-4)} m3 = {'model': nx.extratrees()} run_list = [m1, m2, m3] # we won't save anything, just display the results runner = nx.Runner(run_list, splitter, verbosity=1) runner.run()
def test_data_pca(): "test data.pca" d = nx.play_data() nfactors = (None, 3, 0.5) for nfactor in nfactors: d2 = d.pca(nfactor=nfactor) msg = "data.pca should return a copy" ok_(not shares_memory(d, d2), msg) if nfactor is None: ok_(d.shape == d2.shape, "shape should not change") corr = np.corrcoef(d2.x.T) corr.flat[::corr.shape[0] + 1] = 0 corr = np.abs(corr).max() ok_(corr < 1e-5, "features are not orthogonal")
def test_prediction_dominance(): "make sure prediction.dominance runs" d = nx.play_data() d = d['validation'] p = nx.Prediction() p = p.merge_arrays(d.ids, d.y, 'model1') p = p.merge_arrays(d.ids, d.y, 'model2') p = p.merge_arrays(d.ids, d.y, 'model3') df = p.dominance(d) ok_(isinstance(df, pd.DataFrame), 'expecting a dataframe') assert_raises(ValueError, p['model1'].dominance, d)
def test_loocvsplitter(): """test loocvsplitter""" d = nx.play_data()['train'] splitter = nx.LoocvSplitter(d) count = 0 for dfit, dpre in splitter: count += 1 eras = dfit.unique_era().tolist() era = dpre.unique_era().tolist() ok_(isinstance(eras, list), "expecting a list") ok_(isinstance(era, list), "expecting a list") ok_(len(era) == 1, "expecting a single era") ok_(era not in eras, "did not hold out era") k = d.unique_era().size ok_(count == k, "LoocvSplitter iterated through wrong number of folds")
def test_splitter_overlap(): "prediction data should not overlap" d = nx.play_data() splitters = [ nx.TournamentSplitter(d), nx.ValidationSplitter(d), nx.CheatSplitter(d), nx.CVSplitter(d), nx.IgnoreEraCVSplitter(d), nx.SplitSplitter(d, fit_fraction=0.5) ] for splitter in splitters: predict_ids = [] for dfit, dpredict in splitter: predict_ids.extend(dpredict.ids.tolist()) ok_(len(predict_ids) == len(set(predict_ids)), "ids overlap")
def test_prediction_dominance(): "make sure prediction.dominance runs" d = nx.play_data() d = d['validation'] p = nx.Prediction() p = p.merge_arrays(d.ids, d.y['bernie'], 'model1', 1) p = p.merge_arrays(d.ids, d.y['elizabeth'], 'model2', 2) p = p.merge_arrays(d.ids, d.y['jordan'], 'model3', 3) df = p.dominance(d, 3) df = p.dominance(d, 'jordan') ok_(isinstance(df, pd.DataFrame), 'expecting a dataframe') assert_raises(ValueError, p[('model1', 1)].dominance, d, 1)
def test_prediction_dominance(): """make sure prediction.dominance runs""" d = nx.play_data() d = d['validation'] p = nx.Prediction() p = p.merge_arrays(d.ids, d.y['kazutsugi'], 'model1', 8) p = p.merge_arrays(d.ids, d.y['kazutsugi'], 'model2', 8) p = p.merge_arrays(d.ids, d.y['kazutsugi'], 'model3', 8) df = p.dominance(d, 8) df = p.dominance(d, 'kazutsugi') ok_(isinstance(df, pd.DataFrame), 'expecting a dataframe') assert_raises(ValueError, p[('model1', 1)].dominance, d, 1)
def test_splitter_reset(): "splitter reset should not change results" d = nx.play_data() splitters = [ nx.TournamentSplitter(d), nx.ValidationSplitter(d), nx.CheatSplitter(d), nx.CVSplitter(d), nx.IgnoreEraCVSplitter(d), nx.SplitSplitter(d, fit_fraction=0.5) ] for splitter in splitters: ftups = [[], []] ptups = [[], []] for i in range(2): for dfit, dpredict in splitter: ftups[i].append(dfit) ptups[i].append(dpredict) splitter.reset() ok_(ftups[0] == ftups[1], "splitter reset changed fit split") ok_(ptups[0] == ptups[1], "splitter reset changed predict split")
def backtest_example(): data = nx.play_data() model = nx.logistic() prediction = nx.backtest(model, data) # noqa
def test_data_subsample(): "test data.subsample" d = nx.play_data() d2 = d.subsample(0.5) ok_(isinstance(d2, nx.Data), 'expecting a Data object') ok_(d2.shape[0] < 0.51 * d.shape[0], 'expecting smaller subsample')
def test_examples(): data = nx.play_data() with testing.HiddenPrints(): nx.examples.run_all_examples(data)
def test_data_hash(): "test data.hash" d = nx.play_data() ok_(d.hash() == d.hash(), "data.hash not reproduceable") d2 = nx.Data(d.df[::2]) ok_(d2.hash() == d2.hash(), "data.hash not reproduceable")
def concordance_example(): data = nx.play_data() model = nx.logistic() prediction = nx.production(model, data) concord = nx.concordance(data, prediction) print("concordance {:.4f} (less than 0.12 is passing)".format(concord))