def test_multiple_runs():
"""test running multiple models through multiple tournaments"""
d = testing.play_data()
models = [nx.linear(), nx.fifty()]
with testing.HiddenPrints():
p = nx.production(models, d, 'kazutsugi')
ok_(p.shape[1] == 2, 'wrong number of tournaments')
p = nx.backtest(models, d, 8)
ok_(p.shape[1] == 2, 'wrong number of tournaments')
p = nx.run(models, nx.ValidationSplitter(d), 'kazutsugi')
ok_(p.shape[1] == 2, 'wrong number of tournaments')
p = nx.production(models, d)
ok_(p.shape[1] == 2, 'wrong number of tournaments')
p = nx.backtest(models, d)
ok_(p.shape[1] == 2, 'wrong number of tournaments')
p = nx.run(models, nx.ValidationSplitter(d))
ok_(p.shape[1] == 2, 'wrong number of tournaments')
p = nx.production(models, d, [8])
ok_(p.shape[1] == 2, 'wrong number of tournaments')
ok_(p.tournaments() == ['kazutsugi'], 'wrong tournaments')
p = nx.backtest(models, d, ['kazutsugi'])
ok_(p.shape[1] == 2, 'wrong number of tournaments')
ok_(p.tournaments() == ['kazutsugi'], 'wrong tournaments')
p = nx.run(models, nx.ValidationSplitter(d), ['kazutsugi'])
ok_(p.shape[1] == 2, 'wrong number of tournaments')
ok_(p.tournaments() == ['kazutsugi'], 'wrong tournaments')
def test_multiple_runs():
"test running multiple models through multiple tournaments"
d = testing.play_data()
models = [nx.logistic(), nx.fifty()]
with testing.HiddenPrints():
p = nx.production(models, d, 'bernie')
ok_(p.shape[1] == 2, 'wrong number of tournaments')
p = nx.backtest(models, d, 2)
ok_(p.shape[1] == 2, 'wrong number of tournaments')
p = nx.run(models, nx.ValidationSplitter(d), 'ken')
ok_(p.shape[1] == 2, 'wrong number of tournaments')
p = nx.production(models, d)
ok_(p.shape[1] == 10, 'wrong number of tournaments')
p = nx.backtest(models, d)
ok_(p.shape[1] == 10, 'wrong number of tournaments')
p = nx.run(models, nx.ValidationSplitter(d))
ok_(p.shape[1] == 10, 'wrong number of tournaments')
p = nx.production(models, d, [1, 5])
ok_(p.shape[1] == 4, 'wrong number of tournaments')
ok_(p.tournaments() == ['bernie', 'charles'], 'wrong tournaments')
p = nx.backtest(models, d, ['charles', 'bernie'])
ok_(p.shape[1] == 4, 'wrong number of tournaments')
ok_(p.tournaments() == ['bernie', 'charles'], 'wrong tournaments')
p = nx.run(models, nx.ValidationSplitter(d), ['ken'])
ok_(p.shape[1] == 2, 'wrong number of tournaments')
ok_(p.tournaments() == ['ken'], 'wrong tournaments')
def test_prediction_to_csv():
"make sure prediction.to_csv runs"
p = testing.micro_prediction()
with tempfile.NamedTemporaryFile() as temp:
p['model1'].to_csv(temp.name)
with testing.HiddenPrints():
p['model1'].to_csv(temp.name, verbose=True)
p2 = nx.load_prediction_csv(temp.name, 'model1')
ade(p2, p['model1'], "prediction corrupted during roundtrip")
assert_raises(ValueError, p.to_csv, 'unused')
def test_metrics_per_era():
"make sure metrics_per_era runs"
d = testing.micro_data()
p = testing.micro_prediction()
metrics_per_era(d, p, 1)
metrics_per_era(d, p, 2, join='yhat')
metrics_per_era(d, p, 3, join='inner')
assert_raises(ValueError, metrics_per_era, d, p, 4, 'outer')
with testing.HiddenPrints():
metrics_per_era(d, p, tournament=5, era_as_str=True)
def test_backtest_production():
"Make sure backtest and production run"
d = testing.micro_data()
model = fifty()
with testing.HiddenPrints():
for verbosity in (0, 1, 2, 3):
nx.backtest(model, d, kfold=2, verbosity=verbosity)
nx.production(model, d, verbosity=verbosity)
if verbosity == 3:
nx.production(model, d, name='test', verbosity=verbosity)
def test_load_zip():
"test nx.load_zip"
for i in (0, 1):
if i == 0:
d = nx.load_zip(TINY_DATASET_CSV)
else:
with testing.HiddenPrints():
d = nx.load_zip(TINY_DATASET_CSV, verbose=True)
ok_(len(d) == 14, "wrong number of rows")
ok_(d.shape == (14, 59), 'data has wrong shape')
ok_(d.x.shape == (14, 50), 'x has wrong shape')
ok_(d.df.iloc[2, 3] == 0.34143, 'wrong feature value')
def test_backtest_production():
"Make sure backtest and production run"
d = testing.micro_data()
model = nx.fifty()
with testing.HiddenPrints():
p = nx.production(model, d)
ok_(p.shape[1] == 5, 'wrong number of tournaments')
ok_(p.tournaments() == nx.tournament_all(), 'wrong tournaments')
p = nx.backtest(model, d, kfold=2)
ok_(p.shape[1] == 5, 'wrong number of tournaments')
ok_(p.tournaments() == nx.tournament_all(), 'wrong tournaments')
for verbosity in (0, 1, 2, 3):
nx.backtest(model, d, tournament=3, kfold=2, verbosity=verbosity)
nx.production(model, d, tournament='ken', verbosity=verbosity)
nx.production(model, d, tournament=4, verbosity=verbosity)
nx.production(model, d, tournament=None, verbosity=verbosity)
if verbosity == 3:
nx.production(model, d, tournament=5, verbosity=verbosity)
nx.production(model,
d,
tournament='charles',
verbosity=verbosity)
def test_prediction_correlation():
"make sure prediction.correlation runs"
p = testing.micro_prediction()
with testing.HiddenPrints():
p.correlation()
def test_examples():
data = nx.play_data()
with testing.HiddenPrints():
nx.examples.run_all_examples(data)
def test_reports():
"make sure low-level Report code runs"
# leaderboard
d = [[90, 'bot1', 0, 0, 0, 0, 0, 0.693, 0, nan, nan],
[90, 'bot2', 1, 1, 1, 1, 0, 0.692, 1, 0.1, 10],
[90, 'bot3', 0, 0, 0, 0, 3, 0.697, 3, 0.1, 30],
[91, 'bot1', 2, 3, 1, 1, 0, 0.691, 0, 0.1, 30],
[91, 'bot4', 0, 0, 0, 0, 0, 0.691, 0, nan, nan]]
cols = ['round', 'user', 'usd_main', 'usd_stake', 'nmr_main', 'nmr_stake',
'nmr_burn', 'live', 's', 'c', 'soc']
lb = pd.DataFrame(data=d, columns=cols)
# nmr price
dd = datetime.date
d = [[90, dd(2017, 12, 20), 52.12],
[91, dd(2017, 12, 27), 62.34]]
nmr_price = pd.DataFrame(data=d, columns=['round', 'date', 'usd'])
nmr_price = nmr_price.set_index('round')
# resolution dates
d = [[89, dd(2017, 12, 13)],
[90, dd(2017, 12, 20)],
[91, dd(2017, 12, 27)],
[92, dd(2018, 1, 4)]]
resolution_dates = pd.DataFrame(data=d, columns=['round', 'date'])
resolution_dates = resolution_dates.set_index('round')
with testing.HiddenPrints():
for verbose in (True, False):
df = report.consistency(lb, 0)
ok_(isinstance(df, pd.DataFrame), 'expecting a dataframe')
df = report.consistency(lb, 1)
ok_(isinstance(df, pd.DataFrame), 'expecting a dataframe')
df = report.reputation(lb, verbose=verbose)
ok_(isinstance(df, pd.DataFrame), 'expecting a dataframe')
df = report.group_consistency(lb, verbose=verbose)
ok_(isinstance(df, pd.DataFrame), 'expecting a dataframe')
df = report.group_confidence(lb, verbose=verbose)
ok_(isinstance(df, pd.DataFrame), 'expecting a dataframe')
df = report.group_burn(lb, verbose=verbose)
ok_(isinstance(df, pd.DataFrame), 'expecting a dataframe')
df = report.ten99(lb, 'bot1', nmr_price, resolution_dates)
ok_(isinstance(df, pd.DataFrame), 'expecting a dataframe')
df = report.stake(lb, 150.01, verbose=verbose)
ok_(isinstance(df, pd.DataFrame), 'expecting a dataframe')
df = report.earn(lb, 150.01, verbose=verbose)
ok_(isinstance(df, pd.DataFrame), 'expecting a dataframe')
df = report.burn(lb, verbose=verbose)
ok_(isinstance(df, pd.DataFrame), 'expecting a dataframe')
df = report.participation(lb, verbose=verbose)
ok_(isinstance(df, pd.DataFrame), 'expecting a dataframe')
df = report.big_staker(lb, verbose=verbose)
ok_(isinstance(df, pd.DataFrame), 'expecting a dataframe')
df = report.headcount(lb, verbose=verbose)
ok_(isinstance(df, pd.DataFrame), 'expecting a dataframe')
df = report.single_stake_payout(lb, 150.01, verbose=verbose)
ok_(isinstance(df, pd.DataFrame), 'expecting a dataframe')
df = report.user_summary(lb, ['bot1'], 89.23, verbose=verbose)
ok_(isinstance(df, pd.DataFrame), 'expecting a dataframe')
df = report.user_nmr(lb, ['bot1'], resolution_dates, verbose=True)
ok_(isinstance(df, pd.DataFrame), 'expecting a dataframe')
df = report.user_nmr(lb, ['bot4'], resolution_dates, verbose=True)
ok_(isinstance(df, pd.DataFrame), 'expecting a dataframe')
df = report.user_nmr_tax(lb, ['bot1', 'bot4'], nmr_price,
resolution_dates, price_zero_burns=True,
verbose=verbose)
ok_(isinstance(df, pd.DataFrame), 'expecting a dataframe')
df = report.user_nmr_tax(lb, ['bot1', 'bot4'], nmr_price,
resolution_dates, price_zero_burns=False,
verbose=verbose)
ok_(isinstance(df, pd.DataFrame), 'expecting a dataframe')
plist = report.user_participation(lb, 'bot3')
ok_(isinstance(plist, list), 'expecting a list')
