def test_multiple_runs():
"test running multiple models through multiple tournaments"
d = testing.play_data()
models = [nx.logistic(), nx.fifty()]
with testing.HiddenPrints():
p = nx.production(models, d, 'bernie')
ok_(p.shape[1] == 2, 'wrong number of tournaments')
p = nx.backtest(models, d, 2)
ok_(p.shape[1] == 2, 'wrong number of tournaments')
p = nx.run(models, nx.ValidationSplitter(d), 'ken')
ok_(p.shape[1] == 2, 'wrong number of tournaments')
p = nx.production(models, d)
ok_(p.shape[1] == 10, 'wrong number of tournaments')
p = nx.backtest(models, d)
ok_(p.shape[1] == 10, 'wrong number of tournaments')
p = nx.run(models, nx.ValidationSplitter(d))
ok_(p.shape[1] == 10, 'wrong number of tournaments')
p = nx.production(models, d, [1, 5])
ok_(p.shape[1] == 4, 'wrong number of tournaments')
ok_(p.tournaments() == ['bernie', 'charles'], 'wrong tournaments')
p = nx.backtest(models, d, ['charles', 'bernie'])
ok_(p.shape[1] == 4, 'wrong number of tournaments')
ok_(p.tournaments() == ['bernie', 'charles'], 'wrong tournaments')
p = nx.run(models, nx.ValidationSplitter(d), ['ken'])
ok_(p.shape[1] == 2, 'wrong number of tournaments')
ok_(p.tournaments() == ['ken'], 'wrong tournaments')
def cv_warning(model, data, tournament='bernie', kfold=5, nsamples=100):
"Hold out a sample of eras not rows when doing cross validation."
data = data['train']
results_cve = pd.DataFrame()
results_cv = pd.DataFrame()
for i in range(nsamples):
# cv across eras
cve = nx.CVSplitter(data, kfold=kfold, seed=i)
prediction = nx.run(model, cve, tournament, verbosity=0)
df = prediction.performance(data, tournament)
results_cve = results_cve.append(df, ignore_index=True)
# cv ignoring eras but y balanced
cv = nx.IgnoreEraCVSplitter(data,
tournament=tournament,
kfold=kfold,
seed=i)
prediction = nx.run(model, cv, tournament, verbosity=0)
df = prediction.performance(data, tournament)
results_cv = results_cv.append(df, ignore_index=True)
# display results
rcve = results_cve.mean(axis=0)
rcv = results_cv.mean(axis=0)
rcve.name = 'cve'
rcv.name = 'cv'
r = pd.concat([rcve, rcv], axis=1)
print("\n{} runs".format(i + 1))
print(r)
def test_multiple_runs():
"""test running multiple models through multiple tournaments"""
d = testing.play_data()
models = [nx.linear(), nx.fifty()]
with testing.HiddenPrints():
p = nx.production(models, d, 'kazutsugi')
ok_(p.shape[1] == 2, 'wrong number of tournaments')
p = nx.backtest(models, d, 8)
ok_(p.shape[1] == 2, 'wrong number of tournaments')
p = nx.run(models, nx.ValidationSplitter(d), 'kazutsugi')
ok_(p.shape[1] == 2, 'wrong number of tournaments')
p = nx.production(models, d)
ok_(p.shape[1] == 2, 'wrong number of tournaments')
p = nx.backtest(models, d)
ok_(p.shape[1] == 2, 'wrong number of tournaments')
p = nx.run(models, nx.ValidationSplitter(d))
ok_(p.shape[1] == 2, 'wrong number of tournaments')
p = nx.production(models, d, [8])
ok_(p.shape[1] == 2, 'wrong number of tournaments')
ok_(p.tournaments() == ['kazutsugi'], 'wrong tournaments')
p = nx.backtest(models, d, ['kazutsugi'])
ok_(p.shape[1] == 2, 'wrong number of tournaments')
ok_(p.tournaments() == ['kazutsugi'], 'wrong tournaments')
p = nx.run(models, nx.ValidationSplitter(d), ['kazutsugi'])
ok_(p.shape[1] == 2, 'wrong number of tournaments')
ok_(p.tournaments() == ['kazutsugi'], 'wrong tournaments')
def test_run():
"Make sure run runs"
d = testing.play_data()
models = [nx.logistic(), fifty()]
splitters = [nx.TournamentSplitter(d),
nx.ValidationSplitter(d),
nx.CheatSplitter(d),
nx.CVSplitter(d, kfold=2),
nx.SplitSplitter(d, fit_fraction=0.5)]
for model in models:
for splitter in splitters:
nx.run(model, splitter, verbosity=0)
def cv_warning(data, nsamples=100):
model = nx.logistic()
for i in range(nsamples):
report = nx.Report()
# cv across eras
cve = nx.CVSplitter(data, seed=i)
prediction = nx.run(model, cve, verbosity=0)
report.append_prediction(prediction, 'cve')
# cv ignoring eras but y balanced
cv = nx.IgnoreEraCVSplitter(data, seed=i)
prediction = nx.run(model, cv, verbosity=0)
report.append_prediction(prediction, 'cv')
# save performance results
df = report.performance_df(data)
cols = df.columns.tolist()
cols[-1] = 'cv_type'
df.columns = cols
if i == 0:
results = df
else:
results = results.append(df, ignore_index=True)
# display results
rcve = results[results.cv_type == 'cve'].mean(axis=0)
rcv = results[results.cv_type == 'cv'].mean(axis=0)
rcve.name = 'cve'
rcv.name = 'cv'
r = pd.concat([rcve, rcv], axis=1)
print("\n{} runs".format(i+1))
print(r)
def test_run():
"Make sure run runs"
d = testing.play_data()
models = [nx.linear(), nx.fifty()]
splitters = [nx.TournamentSplitter(d),
nx.ValidationSplitter(d),
nx.CheatSplitter(d),
nx.CVSplitter(d, kfold=2),
nx.SplitSplitter(d, fit_fraction=0.5)]
for model in models:
for splitter in splitters:
p = nx.run(model, splitter, tournament=None, verbosity=0)
ok_(p.shape[1] == 1, 'wrong number of tournaments')
ok_(p.tournaments() == nx.tournament_all(), 'wrong tournaments')
assert_raises(ValueError, nx.run, None, nx.TournamentSplitter(d))
assert_raises(ValueError, nx.run, nx.fifty(), nx.TournamentSplitter(d), {})
