def test_prediction_setitem():
"compare prediction._setitem__ with merge"
data = nx.play_data()
p1 = nx.production(nx.logistic(), data, 'bernie', verbosity=0)
p2 = nx.production(nx.logistic(1e-5), data, 2, verbosity=0)
p3 = nx.production(nx.logistic(1e-6), data, 3, verbosity=0)
p4 = nx.backtest(nx.logistic(), data, 4, verbosity=0)
p = nx.Prediction()
p[('logistic', 1)] = p1
p[('logistic', 2)] = p2
p[('logistic', 3)] = p3
p[('logistic', 4)] = p4
pp = nx.Prediction()
pp = pp.merge(p1)
pp = pp.merge(p2)
pp = pp.merge(p3)
pp = pp.merge(p4)
pd.testing.assert_frame_equal(p.df, pp.df)
assert_raises(ValueError, p.__setitem__, ('logistic', 1), p1)
assert_raises(ValueError, p.__setitem__, ('logistic', 1), p)
def test_prediction_setitem():
"compare prediction._setitem__ with merge"
data = nx.play_data()
p1 = nx.production(nx.logistic(), data, 'model1', verbosity=0)
p2 = nx.production(nx.logistic(1e-5), data, 'model2', verbosity=0)
p3 = nx.production(nx.logistic(1e-6), data, 'model3', verbosity=0)
p4 = nx.backtest(nx.logistic(), data, 'model1', verbosity=0)
p = nx.Prediction()
p['model1'] = p1
p['model2'] = p2
p['model3'] = p3
p['model1'] = p4
pp = nx.Prediction()
pp = pp.merge(p1)
pp = pp.merge(p2)
pp = pp.merge(p3)
pp = pp.merge(p4)
pd.testing.assert_frame_equal(p.df, pp.df)
assert_raises(ValueError, p.__setitem__, 'model1', p1)
assert_raises(ValueError, p.__setitem__, 'model1', p)
def compare_change(data):
"""
Run multiple models: fit on training data, predict for tournament data.
Then change the data, rerun and compare performance with and without the
change.
"""
print('\nStandard dataset:\n')
# we'll look at 5 models
prediction = nx.production(nx.logistic(), data, verbosity=1)
prediction += nx.production(nx.extratrees(), data, verbosity=1)
prediction += nx.production(nx.randomforest(), data, verbosity=1)
prediction += nx.production(nx.mlpc(), data, verbosity=1)
prediction += nx.production(nx.logisticPCA(), data, verbosity=1)
# let's now make a change, could be anything; as an example let's add
# the square of each feature to the dataset
x = np.hstack((data.x, data.x * data.x))
data2 = data.xnew(x)
print('\nDataset expanded with squared features:\n')
# rerun all models with the new expanded data
prediction2 = nx.production(nx.logistic(), data2, verbosity=1)
prediction2 += nx.production(nx.extratrees(), data2, verbosity=1)
prediction2 += nx.production(nx.randomforest(), data2, verbosity=1)
prediction2 += nx.production(nx.mlpc(), data2, verbosity=1)
prediction2 += nx.production(nx.logisticPCA(), data2, verbosity=1)
# compare performance
print('\nCompare (1 is regular dataset; 2 expanded dataset):\n')
print(prediction.compare(data['validation'], prediction2))
def test_prediction_regression():
"regression test of prediction performance evaluation"
d = nx.play_data()
p = nx.production(nx.logistic(), d, tournament=None, verbosity=0)
for number, name in nx.tournament_iter():
p2 = nx.production(nx.logistic(), d, tournament=name, verbosity=0)
df = p.performance_mean(d['validation'], mean_of='tournament')
logloss1 = df.loc[name]['logloss']
logloss2 = p2.summary(d['validation']).loc['mean']['logloss']
diff = np.abs(logloss1 - logloss2)
msg = 'failed on {}'.format(name)
ok_(diff < 1e-6, msg)
def test_model_rename():
"Test renaming a model"
model = nx.logistic()
ok_(model.name == 'logistic', 'wrong name')
model.rename('LR')
ok_(model.name == 'LR', 'wrong name')
model = model.rename('logreg')
ok_(model.name == 'logreg', 'wrong name')
ok_(model.__repr__().startswith('logreg'), 'wrong name')
model = nx.logistic()
ok_(model.rename(None).name == 'logistic', 'wrong name')
assert_raises(ValueError, model.rename, 1)
def runner_example():
data = nx.play_data()
splitter = nx.CVSplitter(data)
# let's run 3 models
m1 = {'model': nx.logistic(), 'prediction_file': None, 'csv_file': None}
m2 = {'model': nx.logistic(1e-4)}
m3 = {'model': nx.extratrees()}
run_list = [m1, m2, m3]
# we won't save anything, just display the results
runner = nx.Runner(run_list, splitter, verbosity=1)
runner.run()
def test_multiple_runs():
"test running multiple models through multiple tournaments"
d = testing.play_data()
models = [nx.logistic(), nx.fifty()]
with testing.HiddenPrints():
p = nx.production(models, d, 'bernie')
ok_(p.shape[1] == 2, 'wrong number of tournaments')
p = nx.backtest(models, d, 2)
ok_(p.shape[1] == 2, 'wrong number of tournaments')
p = nx.run(models, nx.ValidationSplitter(d), 'ken')
ok_(p.shape[1] == 2, 'wrong number of tournaments')
p = nx.production(models, d)
ok_(p.shape[1] == 10, 'wrong number of tournaments')
p = nx.backtest(models, d)
ok_(p.shape[1] == 10, 'wrong number of tournaments')
p = nx.run(models, nx.ValidationSplitter(d))
ok_(p.shape[1] == 10, 'wrong number of tournaments')
p = nx.production(models, d, [1, 5])
ok_(p.shape[1] == 4, 'wrong number of tournaments')
ok_(p.tournaments() == ['bernie', 'charles'], 'wrong tournaments')
p = nx.backtest(models, d, ['charles', 'bernie'])
ok_(p.shape[1] == 4, 'wrong number of tournaments')
ok_(p.tournaments() == ['bernie', 'charles'], 'wrong tournaments')
p = nx.run(models, nx.ValidationSplitter(d), ['ken'])
ok_(p.shape[1] == 2, 'wrong number of tournaments')
ok_(p.tournaments() == ['ken'], 'wrong tournaments')
def cv_warning(data, nsamples=100):
"Hold out a sample of eras not rows when doing cross validation."
data = data['train']
model = nx.logistic()
results_cve = pd.DataFrame()
results_cv = pd.DataFrame()
for i in range(nsamples):
# cv across eras
cve = nx.CVSplitter(data, seed=i)
prediction = nx.run(model, cve, verbosity=0)
df = prediction.performance(data)
results_cve = results_cve.append(df, ignore_index=True)
# cv ignoring eras but y balanced
cv = nx.IgnoreEraCVSplitter(data, seed=i)
prediction = nx.run(model, cv, verbosity=0)
df = prediction.performance(data)
results_cv = results_cv.append(df, ignore_index=True)
# display results
rcve = results_cve.mean(axis=0)
rcv = results_cv.mean(axis=0)
rcve.name = 'cve'
rcv.name = 'cv'
r = pd.concat([rcve, rcv], axis=1)
print("\n{} runs".format(i + 1))
print(r)
def test_prediction_check():
"make sure prediction.check runs"
d = nx.play_data()
p = nx.production(nx.logistic(), d, verbosity=0)
p += nx.production(nx.logisticPCA(), d, verbosity=0)
df = p.check(['logistic'], d)
ok_(isinstance(df, pd.DataFrame), 'expecting a dataframe')
def test_prediction_roundtrip():
"save/load roundtrip shouldn't change prediction"
d = micro_data()
m = nx.logistic()
p = nx.production(m, d, verbosity=0)
with tempfile.NamedTemporaryFile() as temp:
p.save(temp.name)
p2 = nx.load_prediction(temp.name)
ade(p, p2, "prediction corrupted during roundtrip")
def concordance(data, tournament='bernie'):
"""
Example showing how to calculate concordance.
Concordance must be less than 0.12 to pass numerai's check.
For an accurate concordance calculation `data` must be the full dataset.
"""
models = [nx.logistic(), nx.extratrees(), nx.mlpc()]
p = nx.production(models, data, tournament)
print("\nA concordance less than 0.12 is passing")
print(p.concordance(data))
def test_prediction_check():
"make sure prediction.check runs"
d = nx.play_data()
p1 = nx.production(nx.logistic(), d, 'ken', verbosity=0)
p2 = p1.copy()
p2 = p2.rename('example_predictions')
p = p1 + p2
with nx.testing.HiddenPrints():
df = p.check(d)
ok_(isinstance(df, dict), 'expecting a dictionary')
def get_models():
models = [
nx.logistic(),
nx.extratrees(),
nx.randomforest(),
nx.mlpc(),
nx.logisticPCA(),
nx.example_predictions(),
fifty()
]
return models
def concordance_example(data):
"""
Example showing how to calculate concordance.
Concordance must be less than 0.12 to pass numerai's check.
For an accurate concordance calculation `data` must be the full dataset.
"""
prediction = nx.production(nx.logistic(), data)
prediction += nx.production(nx.extratrees(), data)
prediction += nx.production(nx.mlpc(), data)
print("\nA concordance less than 0.12 is passing")
print(prediction.concordance(data))
def test_run():
"Make sure run runs"
d = testing.play_data()
models = [nx.logistic(), fifty()]
splitters = [nx.TournamentSplitter(d),
nx.ValidationSplitter(d),
nx.CheatSplitter(d),
nx.CVSplitter(d, kfold=2),
nx.SplitSplitter(d, fit_fraction=0.5)]
for model in models:
for splitter in splitters:
nx.run(model, splitter, verbosity=0)
def main():
# download dataset from numerai
nx.download_dataset('numerai_dataset.zip', verbose=True)
# load numerai dataset
data = nx.load_zip('numerai_dataset.zip', verbose=True)
# we will use logistic regression; you will want to write your own model
model = nx.logistic()
# fit model with train data and make predictions for tournament data
prediction = nx.production(model, data)
# save predictions to csv file for later upload to numerai
prediction.to_csv('logistic.csv', verbose=True)
def numerox_example():
"""
Example of how to prepare a submission for the Numerai tournament.
It uses Numerox which you can install with: pip install numerox
For more information see: https://github.com/kwgoodman/numerox
"""
# download dataset from numerai, save it and then load it
data = nx.download('numerai_dataset.zip')
# we will use logistic regression; you will want to write your own model
model = nx.logistic()
# fit model with train data and make predictions for tournament data
prediction = nx.production(model, data, tournament='bernie')
# save predictions to csv file
prediction.to_csv('logistic.csv', verbose=True)
def test_run():
"Make sure run runs"
d = testing.play_data()
models = [nx.logistic(), nx.fifty()]
splitters = [
nx.TournamentSplitter(d),
nx.ValidationSplitter(d),
nx.CheatSplitter(d),
nx.CVSplitter(d, kfold=2),
nx.SplitSplitter(d, fit_fraction=0.5)
]
for model in models:
for splitter in splitters:
nx.run(model, splitter, tournament=2, verbosity=0)
nx.run(model, splitter, tournament='bernie', verbosity=0)
p = nx.run(model, splitter, tournament=None, verbosity=0)
ok_(p.shape[1] == 5, 'wrong number of tournaments')
ok_(p.tournaments() == nx.tournament_all(), 'wrong tournaments')
p = nx.run(model, splitter, verbosity=0)
ok_(p.shape[1] == 5, 'wrong number of tournaments')
ok_(p.tournaments() == nx.tournament_all(), 'wrong tournaments')
assert_raises(ValueError, nx.run, None, nx.TournamentSplitter(d))
assert_raises(ValueError, nx.run, nx.fifty(), nx.TournamentSplitter(d), {})
def compare_models(data):
"""
Run multiple models: fit on training data, predict for tournament data.
Then compare performance of the models
"""
# we'll look at 5 models
prediction = nx.production(nx.logistic(), data, verbosity=1)
prediction += nx.production(nx.extratrees(), data, verbosity=1)
prediction += nx.production(nx.randomforest(), data, verbosity=1)
prediction += nx.production(nx.mlpc(), data, verbosity=1)
prediction += nx.production(nx.logisticPCA(), data, verbosity=1)
# correlation of models with logistic regression
print('\nCorrelation:\n')
prediction.correlation('logistic')
# compare performance of models
print('\nPerformance comparison:\n')
print(prediction.performance(data['validation'], sort_by='logloss'))
# dominance of models
print('\nModel dominance:\n')
print(prediction.dominance(data['validation'], sort_by='logloss'))
# dominace between two models
print('\nModel dominance between two models:\n')
df = prediction[['logistic', 'logisticPCA']].dominance(data['validation'])
print(df)
# originality given that logistic model has already been submitted
print('\nModel originality (versus logistic):\n')
print(prediction.originality(['logistic']))
# concordance
print('\nConcordance:\n')
print(prediction.concordance(data))
def cv_warning(data, nsamples=100):
model = nx.logistic()
for i in range(nsamples):
report = nx.Report()
# cv across eras
cve = nx.CVSplitter(data, seed=i)
prediction = nx.run(model, cve, verbosity=0)
report.append_prediction(prediction, 'cve')
# cv ignoring eras but y balanced
cv = nx.IgnoreEraCVSplitter(data, seed=i)
prediction = nx.run(model, cv, verbosity=0)
report.append_prediction(prediction, 'cv')
# save performance results
df = report.performance_df(data)
cols = df.columns.tolist()
cols[-1] = 'cv_type'
df.columns = cols
if i == 0:
results = df
else:
results = results.append(df, ignore_index=True)
# display results
rcve = results[results.cv_type == 'cve'].mean(axis=0)
rcv = results[results.cv_type == 'cv'].mean(axis=0)
rcve.name = 'cve'
rcv.name = 'cv'
r = pd.concat([rcve, rcv], axis=1)
print("\n{} runs".format(i+1))
print(r)
def concordance_example():
data = nx.play_data()
model = nx.logistic()
prediction = nx.production(model, data)
concord = nx.concordance(data, prediction)
print("concordance {:.4f} (less than 0.12 is passing)".format(concord))
def backtest_example(data):
"Simple cross validation on training data using logistic regression"
model = nx.logistic()
prediction = nx.backtest(model, data) # noqa
def backtest_example():
data = nx.play_data()
model = nx.logistic()
prediction = nx.backtest(model, data) # noqa
def get_models():
models = [nx.logistic(), nx.extratrees(), nx.randomforest()]
if HAS_XGBOOST:
models.append(nx.xgboost())
return models
def get_model():
model = nx.logistic()
return model
def backtest(data, tournament='bernie'):
"Simple cross validation on training data using logistic regression"
model = nx.logistic()
prediction = nx.backtest(model, data, tournament) # noqa
def test_prediction_concordance():
"make sure prediction.concordance runs"
d = testing.play_data()
p = nx.production(nx.logistic(), d, 'model1', verbosity=0)
df = p.concordance(d)
ok_(isinstance(df, pd.DataFrame), 'expecting a dataframe')
