def test_multiple_runs():
"test running multiple models through multiple tournaments"
d = testing.play_data()
models = [nx.logistic(), nx.fifty()]
with testing.HiddenPrints():
p = nx.production(models, d, 'bernie')
ok_(p.shape[1] == 2, 'wrong number of tournaments')
p = nx.backtest(models, d, 2)
ok_(p.shape[1] == 2, 'wrong number of tournaments')
p = nx.run(models, nx.ValidationSplitter(d), 'ken')
ok_(p.shape[1] == 2, 'wrong number of tournaments')
p = nx.production(models, d)
ok_(p.shape[1] == 10, 'wrong number of tournaments')
p = nx.backtest(models, d)
ok_(p.shape[1] == 10, 'wrong number of tournaments')
p = nx.run(models, nx.ValidationSplitter(d))
ok_(p.shape[1] == 10, 'wrong number of tournaments')
p = nx.production(models, d, [1, 5])
ok_(p.shape[1] == 4, 'wrong number of tournaments')
ok_(p.tournaments() == ['bernie', 'charles'], 'wrong tournaments')
p = nx.backtest(models, d, ['charles', 'bernie'])
ok_(p.shape[1] == 4, 'wrong number of tournaments')
ok_(p.tournaments() == ['bernie', 'charles'], 'wrong tournaments')
p = nx.run(models, nx.ValidationSplitter(d), ['ken'])
ok_(p.shape[1] == 2, 'wrong number of tournaments')
ok_(p.tournaments() == ['ken'], 'wrong tournaments')
def test_prediction_setitem():
"compare prediction._setitem__ with merge"
data = nx.play_data()
p1 = nx.production(nx.logistic(), data, 'model1', verbosity=0)
p2 = nx.production(nx.logistic(1e-5), data, 'model2', verbosity=0)
p3 = nx.production(nx.logistic(1e-6), data, 'model3', verbosity=0)
p4 = nx.backtest(nx.logistic(), data, 'model1', verbosity=0)
p = nx.Prediction()
p['model1'] = p1
p['model2'] = p2
p['model3'] = p3
p['model1'] = p4
pp = nx.Prediction()
pp = pp.merge(p1)
pp = pp.merge(p2)
pp = pp.merge(p3)
pp = pp.merge(p4)
pd.testing.assert_frame_equal(p.df, pp.df)
assert_raises(ValueError, p.__setitem__, 'model1', p1)
assert_raises(ValueError, p.__setitem__, 'model1', p)
def test_multiple_runs():
"""test running multiple models through multiple tournaments"""
d = testing.play_data()
models = [nx.linear(), nx.fifty()]
with testing.HiddenPrints():
p = nx.production(models, d, 'kazutsugi')
ok_(p.shape[1] == 2, 'wrong number of tournaments')
p = nx.backtest(models, d, 8)
ok_(p.shape[1] == 2, 'wrong number of tournaments')
p = nx.run(models, nx.ValidationSplitter(d), 'kazutsugi')
ok_(p.shape[1] == 2, 'wrong number of tournaments')
p = nx.production(models, d)
ok_(p.shape[1] == 2, 'wrong number of tournaments')
p = nx.backtest(models, d)
ok_(p.shape[1] == 2, 'wrong number of tournaments')
p = nx.run(models, nx.ValidationSplitter(d))
ok_(p.shape[1] == 2, 'wrong number of tournaments')
p = nx.production(models, d, [8])
ok_(p.shape[1] == 2, 'wrong number of tournaments')
ok_(p.tournaments() == ['kazutsugi'], 'wrong tournaments')
p = nx.backtest(models, d, ['kazutsugi'])
ok_(p.shape[1] == 2, 'wrong number of tournaments')
ok_(p.tournaments() == ['kazutsugi'], 'wrong tournaments')
p = nx.run(models, nx.ValidationSplitter(d), ['kazutsugi'])
ok_(p.shape[1] == 2, 'wrong number of tournaments')
ok_(p.tournaments() == ['kazutsugi'], 'wrong tournaments')
def test_prediction_setitem():
"compare prediction._setitem__ with merge"
data = nx.play_data()
p1 = nx.production(nx.logistic(), data, 'bernie', verbosity=0)
p2 = nx.production(nx.logistic(1e-5), data, 2, verbosity=0)
p3 = nx.production(nx.logistic(1e-6), data, 3, verbosity=0)
p4 = nx.backtest(nx.logistic(), data, 4, verbosity=0)
p = nx.Prediction()
p[('logistic', 1)] = p1
p[('logistic', 2)] = p2
p[('logistic', 3)] = p3
p[('logistic', 4)] = p4
pp = nx.Prediction()
pp = pp.merge(p1)
pp = pp.merge(p2)
pp = pp.merge(p3)
pp = pp.merge(p4)
pd.testing.assert_frame_equal(p.df, pp.df)
assert_raises(ValueError, p.__setitem__, ('logistic', 1), p1)
assert_raises(ValueError, p.__setitem__, ('logistic', 1), p)
def test_prediction_check():
"make sure prediction.check runs"
d = nx.play_data()
p = nx.production(nx.logistic(), d, verbosity=0)
p += nx.production(nx.logisticPCA(), d, verbosity=0)
df = p.check(['logistic'], d)
ok_(isinstance(df, pd.DataFrame), 'expecting a dataframe')
def improve_model(data, tournament='bernie'):
"""
Run multiple models: fit on training data, predict for tournament data.
Then change the data, rerun and compare performance with and without the
change.
"""
# we'll look at 5 models
models = [
nx.logistic(),
nx.extratrees(),
nx.randomforest(),
nx.mlpc(),
nx.logisticPCA()
]
print('\nStandard dataset:\n')
# first run the base case
prediction = nx.production(models, data, tournament, verbosity=1)
# let's now make a change, could be anything; as an example let's add
# the square of each feature to the dataset
x = np.hstack((data.x, data.x * data.x))
data2 = data.xnew(x)
print('\nDataset expanded with squared features:\n')
# rerun all models with the new expanded data
prediction2 = nx.production(models, data2, tournament, verbosity=1)
# compare performance
print('\nCompare (1 is regular dataset; 2 expanded dataset):\n')
print(prediction.compare(data['validation'], prediction2, tournament))
def test_prediction_setitem():
"""compare prediction._setitem__ with merge"""
data = nx.play_data()
p1 = nx.production(nx.linear(), data, 'kazutsugi', verbosity=0)
p2 = nx.production(nx.linear(), data, 8, verbosity=0)
p3 = nx.production(nx.linear(), data, 8, verbosity=0)
p4 = nx.backtest(nx.linear(), data, 8, verbosity=0)
p = nx.Prediction()
p[('linear', 1)] = p1
p[('linear', 2)] = p2
p[('linear', 3)] = p3
p[('linear', 4)] = p4
pp = nx.Prediction()
pp = pp.merge(p1)
pp = pp.merge(p2)
pp = pp.merge(p3)
pp = pp.merge(p4)
pd.testing.assert_frame_equal(p.df, pp.df)
assert_raises(ValueError, p.__setitem__, ('linear', 1), p1)
assert_raises(ValueError, p.__setitem__, ('linear', 1), p)
def test_backtest_production():
"Make sure backtest and production run"
d = testing.micro_data()
model = fifty()
with testing.HiddenPrints():
for verbosity in (0, 1, 2, 3):
nx.backtest(model, d, kfold=2, verbosity=verbosity)
nx.production(model, d, verbosity=verbosity)
if verbosity == 3:
nx.production(model, d, name='test', verbosity=verbosity)
def concordance_example(data):
"""
Example showing how to calculate concordance.
Concordance must be less than 0.12 to pass numerai's check.
For an accurate concordance calculation `data` must be the full dataset.
"""
prediction = nx.production(nx.logistic(), data)
prediction += nx.production(nx.extratrees(), data)
prediction += nx.production(nx.mlpc(), data)
print("\nA concordance less than 0.12 is passing")
print(prediction.concordance(data))
def test_prediction_regression():
"regression test of prediction performance evaluation"
d = nx.play_data()
p = nx.production(nx.logistic(), d, tournament=None, verbosity=0)
for number, name in nx.tournament_iter():
p2 = nx.production(nx.logistic(), d, tournament=name, verbosity=0)
df = p.performance_mean(d['validation'], mean_of='tournament')
logloss1 = df.loc[name]['logloss']
logloss2 = p2.summary(d['validation']).loc['mean']['logloss']
diff = np.abs(logloss1 - logloss2)
msg = 'failed on {}'.format(name)
ok_(diff < 1e-6, msg)
def test_prediction_regression():
"""regression test of prediction performance evaluation"""
d = nx.play_data()
p = nx.production(nx.linear(), d, tournament=None, verbosity=0)
for number, name in nx.tournament_iter():
p2 = nx.production(nx.linear(), d, tournament=name, verbosity=0)
df = p.performance_mean(d['validation'], mean_of='tournament')
logloss1 = float('%.3f' % (df.loc[name]['mse']))
logloss2 = float('%.3f' %
(p2.summary(d['validation']).loc['mean']['mse']))
diff = np.abs(logloss1 - logloss2)
msg = f"failed on {name}"
ok_(diff < 1e-6, msg)
def predict_and_submit(tournaments, data, model_class, numerai_public,
numerai_secret):
model_name = model_class.__name__
for tournament_name in tournaments:
saved_model_name = f'model_trained_{model_name}_{tournament_name}'
if os.path.exists(saved_model_name):
print("using saved model for", tournament_name)
m = model_class.load(saved_model_name)
else:
print("saved model not found for", tournament_name)
m = model_class(verbose=True)
print("training model for", tournament_name)
m.fit(data['train'], tournament_name)
print(
f"running predictions for {model_name} on tournament {tournament_name}",
flush=True)
# fit model with train data and make predictions for tournament data
prediction = nx.production(m, data, tournament=tournament_name)
# save predictions to csv file
prediction_filename = f'/tmp/prediction_{model_name}_{tournament_name}.csv'
prediction.to_csv(prediction_filename, verbose=True)
# submit the prediction
for tournament_name in tournaments:
prediction_filename = f'/tmp/prediction_{model_name}_{tournament_name}.csv'
submission_id = nx.upload(prediction_filename,
tournament_name,
numerai_public,
numerai_secret,
block=False,
n_tries=3)
def test_prediction_roundtrip():
"save/load roundtrip shouldn't change prediction"
d = micro_data()
m = nx.logistic()
p = nx.production(m, d, verbosity=0)
with tempfile.NamedTemporaryFile() as temp:
p.save(temp.name)
p2 = nx.load_prediction(temp.name)
ade(p, p2, "prediction corrupted during roundtrip")
def concordance(data, tournament='bernie'):
"""
Example showing how to calculate concordance.
Concordance must be less than 0.12 to pass numerai's check.
For an accurate concordance calculation `data` must be the full dataset.
"""
models = [nx.logistic(), nx.extratrees(), nx.mlpc()]
p = nx.production(models, data, tournament)
print("\nA concordance less than 0.12 is passing")
print(p.concordance(data))
def test_prediction_check():
"""make sure prediction.check runs"""
d = nx.play_data()
p1 = nx.production(nx.linear(), d, 'kazutsugi', verbosity=0)
p2 = p1.copy()
p2 = p2.rename('example_predictions')
p = p1 + p2
with nx.testing.HiddenPrints():
df = p.check(d)
ok_(isinstance(df, dict), 'expecting a dictionary')
def predict_and_submit(data, model_class):
model_name = model_class.__name__
model_id = model_class.model_id
logger = logging.getLogger(model_name)
for tournament in nx.tournament_names():
logger.info(f"Predict and submit for {tournament} using {model_class}")
saved_model_name = f'model_trained_{model_name}_{tournament}'
if os.path.exists(saved_model_name):
logger.info(f'Using saved model {saved_model_name}')
m = model_class.load(saved_model_name)
else:
logger.info(f'Saved model {saved_model_name} not found')
m = model_class(verbose=True)
try:
logger.info(
f'Training model against {tournament} training data')
m.fit(data['train'], tournament)
except Exception as e:
logger.error(f'Failed to train {model_class} - {e}')
return
# fit model with train data and make predictions for tournament data
logger.info(f'Predicting with {model_class} on {tournament} data')
prediction = nx.production(m, data, tournament=tournament)
# save predictions to csv file
prediction_filename = f'/tmp/prediction_{model_name}_{tournament}.csv'
logger.info(f"Saving predictions to {prediction_filename}")
prediction.to_csv(prediction_filename, verbose=True)
try:
# submit the prediction
logger.info(
f"Submitting predictions from {prediction_filename} using {model_id}"
)
submission_id, status = nx.upload(prediction_filename,
tournament,
NUMERAI_PUBLIC_ID,
NUMERAI_SECRET_KEY,
block=False,
n_tries=N_TRIES,
sleep_seconds=SLEEP_SECONDS,
model_id=model_id)
logger.info(status)
logger.info(
f'Successfully submitted predictions using model_id {model_id}'
)
except Exception as e:
logger.error(f'Upload failed with - {e}')
def compare_models(data):
"""
Run multiple models: fit on training data, predict for tournament data.
Then compare performance of the models
"""
# we'll look at 5 models
prediction = nx.production(nx.logistic(), data, verbosity=1)
prediction += nx.production(nx.extratrees(), data, verbosity=1)
prediction += nx.production(nx.randomforest(), data, verbosity=1)
prediction += nx.production(nx.mlpc(), data, verbosity=1)
prediction += nx.production(nx.logisticPCA(), data, verbosity=1)
# correlation of models with logistic regression
print('\nCorrelation:\n')
prediction.correlation('logistic')
# compare performance of models
print('\nPerformance comparison:\n')
print(prediction.performance(data['validation'], sort_by='logloss'))
# dominance of models
print('\nModel dominance:\n')
print(prediction.dominance(data['validation'], sort_by='logloss'))
# dominace between two models
print('\nModel dominance between two models:\n')
df = prediction[['logistic', 'logisticPCA']].dominance(data['validation'])
print(df)
# originality given that logistic model has already been submitted
print('\nModel originality (versus logistic):\n')
print(prediction.originality(['logistic']))
# concordance
print('\nConcordance:\n')
print(prediction.concordance(data))
def main():
# download dataset from numerai
nx.download_dataset('numerai_dataset.zip', verbose=True)
# load numerai dataset
data = nx.load_zip('numerai_dataset.zip', verbose=True)
# we will use logistic regression; you will want to write your own model
model = nx.logistic()
# fit model with train data and make predictions for tournament data
prediction = nx.production(model, data)
# save predictions to csv file for later upload to numerai
prediction.to_csv('logistic.csv', verbose=True)
def predict():
tournaments = nx.tournament_names()
print(tournaments)
# download dataset from numerai
data = nx.download('numerai_dataset.zip', load=False)
print('data downloaded')
data = nx.load_zip('numerai_dataset.zip', single_precision=True)
print('data loaded')
for tournament_name in tournaments:
saved_model_name = 'model_trained_' + tournament_name
if os.path.exists(saved_model_name):
print("using saved model for", tournament_name)
m = model.LinearModel.load(saved_model_name)
else:
print("saved model not found for", tournament_name)
m = model.LinearModel(verbose=True)
print("training model for", tournament_name)
m.fit(data['train'], tournament_name)
print("running predictions for", tournament_name, flush=True)
# fit model with train data and make predictions for tournament data
prediction = nx.production(m, data, tournament=tournament_name)
# save predictions to csv file
prediction_filename = '/tmp/prediction_' + tournament_name + '.csv'
prediction.to_csv(prediction_filename, verbose=True)
# submit the prediction
# Numerai API key
# You will need to create an API key by going to https://numer.ai/account and clicking "Add" under the "Your API keys" section.
# Select the following permissions for the key: "Upload submissions", "Make stakes", "View historical submission info", "View user info"
public_id = os.environ["NUMERAI_PUBLIC_ID"]
secret_key = os.environ["NUMERAI_SECRET_KEY"]
for tournament_name in tournaments:
prediction_filename = '/tmp/prediction_' + tournament_name + '.csv'
api = NumerAPI(public_id=public_id, secret_key=secret_key)
model_id = api.get_models()
api.upload_predictions(prediction_filename,
model_id=model_id['akrimedes_2'])
def numerox_example():
"""
Example of how to prepare a submission for the Numerai tournament.
It uses Numerox which you can install with: pip install numerox
For more information see: https://github.com/kwgoodman/numerox
"""
# download dataset from numerai, save it and then load it
data = nx.download('numerai_dataset.zip')
# we will use logistic regression; you will want to write your own model
model = nx.logistic()
# fit model with train data and make predictions for tournament data
prediction = nx.production(model, data, tournament='bernie')
# save predictions to csv file
prediction.to_csv('logistic.csv', verbose=True)
def predict_and_submit():
# Numerai API key
# You will need to create an API key by going to https://numer.ai/account and clicking "Add" under the "Your API keys" section.
# Select the following permissions for the key: "Upload submissions", "Make stakes", "View historical submission info", "View user info"
public_id = os.environ["NUMERAI_PUBLIC_ID"]
secret_key = os.environ["NUMERAI_SECRET_KEY"]
tournaments = nx.tournament_names()
print(tournaments)
# download dataset from numerai
data = nx.download('numerai_dataset.zip')
for tournament_name in tournaments:
saved_model_name = 'model_trained_' + tournament_name
if os.path.exists(saved_model_name):
print("using saved model for", tournament_name)
m = model.LogisticModel.load(saved_model_name)
else:
print("saved model not found for", tournament_name)
m = model.LogisticModel(verbose=True)
print("training model for", tournament_name)
m.fit(data['train'], tournament_name)
print("running predictions for", tournament_name)
# fit model with train data and make predictions for tournament data
prediction = nx.production(m, data, tournament=tournament_name)
# save predictions to csv file
prediction_filename = '/tmp/prediction_' + tournament_name + '.csv'
prediction.to_csv(prediction_filename, verbose=True)
# submit the prediction
for tournament_name in tournaments:
prediction_filename = '/tmp/prediction_' + tournament_name + '.csv'
submission_id = nx.upload(prediction_filename,
tournament_name,
public_id,
secret_key,
block=False)
m.append(log_loss(y, yhat))
m.append(roc_auc_score(y, yhat))
yh = np.zeros(yhat.size)
yh[yhat >= 0.5] = 1
m.append(accuracy_score(y, yh))
m.append(yhat.std())
return m
if __name__ == '__main__':
# test prediction.performance()
import numerox as nx
data = nx.load_data('/data/nx/numerai_dataset_20171024.hdf')
model = nx.model.logistic()
prediction1 = nx.backtest(model, data, verbosity=1)
prediction2 = nx.production(model, data)
"""
prediction = prediction1 + prediction2
print prediction
prediction.performance(data)
prediction.save('/data/nx/pred/logistic_1e-5.pred')
"""
"""
for c in (1e-1, 1e-2, 1e-3, 1e-4, 1e-5):
print c
model = nx.model.logistic(c)
prediction1 = nx.backtest(model, data, verbosity=1)
prediction2 = nx.production(model, data)
prediction = prediction1 + prediction2
prediction.save('/data/nx/pred/logistic_{:.0e}.pred'.format(c))
def test_backtest_production():
"Make sure backtest and production run"
d = testing.micro_data()
model = nx.fifty()
with testing.HiddenPrints():
p = nx.production(model, d)
ok_(p.shape[1] == 5, 'wrong number of tournaments')
ok_(p.tournaments() == nx.tournament_all(), 'wrong tournaments')
p = nx.backtest(model, d, kfold=2)
ok_(p.shape[1] == 5, 'wrong number of tournaments')
ok_(p.tournaments() == nx.tournament_all(), 'wrong tournaments')
for verbosity in (0, 1, 2, 3):
nx.backtest(model, d, tournament=3, kfold=2, verbosity=verbosity)
nx.production(model, d, tournament='ken', verbosity=verbosity)
nx.production(model, d, tournament=4, verbosity=verbosity)
nx.production(model, d, tournament=None, verbosity=verbosity)
if verbosity == 3:
nx.production(model, d, tournament=5, verbosity=verbosity)
nx.production(model,
d,
tournament='charles',
verbosity=verbosity)
def test_prediction_concordance():
"""make sure prediction.concordance runs"""
d = nx.testing.play_data()
p = nx.production(nx.linear(), d, 8, verbosity=0)
df = p.concordance(d)
ok_(isinstance(df, pd.DataFrame), 'expecting a dataframe')
import numerox as nx
from model import get_model
data = nx.load_zip('numerai_dataset.zip')
model = get_model()
prediction = nx.production(model, data, 'bernie', verbosity=1)
prediction.to_csv('output.csv', tournament='bernie')
def test_prediction_concordance():
"make sure prediction.concordance runs"
d = testing.play_data()
p = nx.production(nx.logistic(), d, 'model1', verbosity=0)
df = p.concordance(d)
ok_(isinstance(df, pd.DataFrame), 'expecting a dataframe')
def concordance_example():
data = nx.play_data()
model = nx.logistic()
prediction = nx.production(model, data)
concord = nx.concordance(data, prediction)
print("concordance {:.4f} (less than 0.12 is passing)".format(concord))
def check(self, data, verbose=True):
"""
Run Numerai upload checks.
Parameters
----------
data : nx.Data
Data object of Numerai dataset.
verbose : bool
By default, True, output is printed to stdout.
Returns
-------
check : dict
A dictionary where the keys are the (name, tournament) pairs and
the values are Pandas DataFrames that contain the results of the
checks.
"""
# calc example predictions
example_y = {}
for tournament in self.tournaments(as_str=False):
ep = nx.production(nx.example_predictions(),
data,
tournament=tournament,
verbosity=0)
ep = ep.loc[self.ids]
example_y[tournament] = ep.y[:, 0]
df_dict = {}
columns = ['validation', 'test', 'live', 'all', 'pass']
data = data.loc[self.ids]
regions = data.region
pairs = list(self.pairs(as_str=False))
# check each model, tournament pair
for pair in pairs:
print('{}, {}'.format(pair[0], nx.tournament_str(pair[1])))
df = pd.DataFrame(columns=columns)
idx = pairs.index(pair)
y = self.y[:, idx]
for region in ('validation', 'test', 'live', 'all'):
yexi = example_y[pair[1]]
if region == 'all':
yi = y
else:
idx = regions == region
yi = y[idx]
yexi = yexi[idx]
df.loc['corr', region] = pearsonr(yi, yexi)[0]
df.loc['rcorr', region] = spearmanr(yi, yexi)[0]
df.loc['min', region] = yi.min()
df.loc['max', region] = yi.max()
maz = np.abs((yi - yi.mean()) / yi.std()).max()
df.loc['maz', region] = maz
df.loc['corr', 'pass'] = (df.loc['corr'][:-1] >= 0.2).all()
df.loc['rcorr', 'pass'] = (df.loc['rcorr'][:-1] >= 0.2).all()
df.loc['min', 'pass'] = (df.loc['min'][:-1] >= 0.3).all()
df.loc['max', 'pass'] = (df.loc['max'][:-1] <= 0.7).all()
df.loc['maz', 'pass'] = (df.loc['maz'][:-1] <= 15).all()
print(df)
df_dict[pair] = df
return df_dict
X_train = X[train_index][:,3:]
# y_train remains the same
y_train = y[train_index]
print ">> running split #", counter
print ">> finding best params"
clf = model_selection.GridSearchCV(linear_model.LogisticRegression(), parameters, scoring="neg_log_loss", cv=kfold_split, n_jobs=-1)
clf.fit(X_train, y_train)
best_params = clf.best_params_
print ">> best params: ", best_params
# create a new logistic regression model for the tournament
model = logistic(best_params)
print ">> training info:"
train = nx.backtest(model, data, tournament, verbosity=1)
print ">> validation info:"
validation = nx.production(model, data, tournament, verbosity=1)
print ">> saving validation info: "
validation.to_csv(MODEL_NAME + "-" + tournament + "-" + str(counter) + ".csv")
print ">> done saving validation info"
print "\n"
counter=counter+1
for tournament_name in tournaments:
saved_model_name = 'model_trained_' + tournament_name
if os.path.exists(saved_model_name):
print("using saved model for", tournament_name)
m = model.LinearModel.load(saved_model_name)
else:
print("saved model not found for", tournament_name)
m = model.LinearModel(verbose=True)
print("training model for", tournament_name)
m.fit(data['train'], tournament_name)
print("running predictions for", tournament_name, flush=True)
# fit model with train data and make predictions for tournament data
prediction = nx.production(m, data, tournament=tournament_name)
# save predictions to csv file
prediction_filename = '/tmp/prediction_' + tournament_name + '.csv'
prediction.to_csv(prediction_filename, verbose=True)
# submit the prediction
# Numerai API key
# You will need to create an API key by going to https://numer.ai/account and clicking "Add" under the "Your API keys" section.
# Select the following permissions for the key: "Upload submissions", "Make stakes", "View historical submission info", "View user info"
public_id = os.environ["NUMERAI_PUBLIC_ID"]
secret_key = os.environ["NUMERAI_SECRET_KEY"]
for tournament_name in tournaments:
prediction_filename = '/tmp/prediction_' + tournament_name + '.csv'
