def get_napi(self):
if self.config_dir is not None:
print('Authentification...')
public_id, secret_key = self.api_setup()
napi = numerapi.NumerAPI(public_id, secret_key)
else:
print('Using Numerapi without Authentification...')
napi = numerapi.NumerAPI()
return napi
def test_NumerAPI():
# passing only one of public_id and secret_key is not enough
api = numerapi.NumerAPI(public_id="foo", secret_key=None)
assert api.token is None
api = numerapi.NumerAPI(public_id=None, secret_key="bar")
assert api.token is None
# passing both works
api = numerapi.NumerAPI(public_id="foo", secret_key="bar")
assert api.token == ("foo", "bar")
# invalid log level should raise
with pytest.raises(AttributeError):
numerapi.NumerAPI(verbosity="FOO")
def init_numerapi():
public_key = os.environ.get("NUMERAI_PUBLIC_KEY")
secret_key = os.environ.get("NUMERAI_SECRET_KEY")
napi = numerapi.NumerAPI(verbosity="info",
public_id=public_key,
secret_key=secret_key)
return napi
def main():
# set example username and round
example_public_id = "somepublicid"
example_secret_key = "somesecretkey"
# some API calls do not require logging in
napi = numerapi.NumerAPI(verbosity="info")
# download current dataset
napi.download_current_dataset(unzip=True)
# get competitions
all_competitions = napi.get_competitions()
# get leaderboard for the current round
leaderboard = napi.get_leaderboard()
# leaderboard for a historic round
leaderboard_67 = napi.get_leaderboard(round_num=67)
# check if a new round has started
if napi.check_new_round():
print("new round has started wihtin the last 24hours!")
else:
print("no new round within the last 24 hours")
# provide api tokens
napi = NumerAPI(example_public_id, example_secret_key)
# upload predictions
submission_id = napi.upload_predictions("mypredictions.csv")
# check submission status
napi.submission_status()
def check_numerai_validity(key_id, secret):
try:
napi = numerapi.NumerAPI(key_id, secret)
napi.get_account()
except Exception:
raise exception_with_msg("Numerai keys seem to be invalid. "
"Make sure you've entered them correctly.")
def output(self):
try:
self.api = numerapi.NumerAPI(self.public_id, self.secret_key)
except ValueError:
print("Incorrect public_id or secret_key")
current_round = self.api.get_current_round()
dataset_name = "numerai_dataset_{0}.zip".format(current_round)
dataset_dir = "numerai_dataset_{0}".format(current_round)
# if false, assert throws an AssertionError
# download current dataset
assert self.api.download_current_dataset(dest_path=self.output_path,
dest_filename=dataset_name,
unzip=True,
tournament=1)
dataset_path = os.path.join(self.output_path, dataset_dir)
test_data_path = os.path.join(dataset_path, 'numerai_training_data.csv')
tournament_data_path = os.path.join(dataset_path,
'numerai_tournament_data.csv')
example_data_path = os.path.join(dataset_path,
'example_predictions.csv')
out = {
'zipfile': luigi.LocalTarget(os.path.join(self.output_path, dataset_name)),
'training_data.csv': luigi.LocalTarget(test_data_path),
'tournament_data.csv': luigi.LocalTarget(tournament_data_path),
'example_predictions.csv': luigi.LocalTarget(example_data_path)
}
print(out)
return out
def main():
"""
Download and unzip the latest dataset from Numerai.
This produces csv files in the data folder.
"""
print('Obtaining data...')
data_folder = common.PROJECT_PATH / 'data'
filename = 'numerai_dataset.zip'
file_path = data_folder / filename
# remove if exists
file_path.unlink()
# download dataset from numerai
napi = numerapi.NumerAPI(verbosity="info")
napi.download_current_dataset(dest_path=data_folder,
dest_filename=filename,
unzip=False)
# unzip content
with zipfile.ZipFile(file_path, 'r') as zip_ref:
zip_ref.extractall(data_folder)
print('...done!')
def info():
import numerapi
napi = numerapi.NumerAPI('email', 'password')
napi.download_current_dataset(dest_path='.', unzip=True)
napi.upload_prediction('path/to/prediction.csv')
napi.get_user('username')
napi.get_scores('username')
napi.get_earnings_per_round('username')
def main():
keys = credentials()
numapi = numerapi.NumerAPI(verbosity='INFO',
public_id=keys['PUBLIC_ID'],
secret_key=keys['PRIVATE_KEY'])
keys['LATEST_ROUND'] = download_data(numapi, keys)
update_env_file(keys)
# gbm_hpo.main()
nn_hpo.main()
def check_numerai_validity(key_id, secret):
try:
napi = numerapi.NumerAPI(key_id, secret)
napi.get_user()
return True
except Exception:
raise exception_with_msg(
'''Numerai keys seem to be invalid. Make sure you've entered them correctly.'''
)
def submit(df, model_name, filename='predictions.csv'):
df[['id', 'prediction']].to_csv(filename, index=False)
public_id = "xxx"
secret_key = "xxx"
napi = numerapi.NumerAPI(public_id=public_id, secret_key=secret_key)
models = napi.get_models()
model_id = models[model_name]
napi.upload_predictions(filename, model_id=model_id)
def setup_data(self):
if self.trainer_params['get_current_data']:
napi = numerapi.NumerAPI(verbosity="info")
if napi.check_new_round():
LOGGER.info('Loading current dataset from NumerAPI..')
self.data = self.get_tournament_data()
else:
if os.path.isfile(self.trainer_params['local_data']):
LOGGER.info(
f"Loading data locally from {self.trainer_params['local_data']}"
)
self.data = nx.load_zip(self.trainer_params['local_data'])
else:
return FileNotFoundError('local data not found')
def make_submission(num=1):
model, keys = get_model(num)
predictions = get_prediction(model)
# upload predictions
predictions.to_csv("predictions.csv", index=False)
napi = numerapi.NumerAPI(public_id=keys['public_id'], secret_key=keys['secret_key'])
# do not submit in test mode
if test:
logging('NUMERAI-INFO: test successful')
return
submission_id = napi.upload_predictions("predictions.csv", model_id=keys['model_id'])
logging('NUMERAI-INFO: submitted predictions from model {}'.format(num))
def __init__(self, dir_="./dataset"):
"""__init__ func
APIの準備
パラメータの設置
ディレクトリの作成
Args:
dir_ (str): データセットを保存するパス
"""
self.napi = numerapi.NumerAPI(verbosity="info")
self.dir_ = dir_
self.path_manager = PathManager(dir_)
os.makedirs(dir_, exist_ok=True)
def __init__(self,
public_id: Optional[str] = None,
secret_key: Optional[str] = None):
"""
Initializes the Numerai class to execute numerai things.
Can take the public id and secret key, else will load them from the env variables
"""
if public_id is None:
LOGGER.info("Loading Numerai credentials from environment.")
public_id = os.environ.get("NUMERAI_PUBLIC_ID", None)
if secret_key is None:
secret_key = os.environ.get("NUMERAI_SECRET_KEY", None)
assert public_id is not None and secret_key is not None, (
"You need to either provide the numerai public and and secret key, "
+ "or you need to put them in environment variables.")
self.napi: numerapi.NumerAPI = numerapi.NumerAPI(public_id=public_id,
secret_key=secret_key)
def main():
# login to numerai to get token
print 'Login into Numer.ai'
api = numerapi.NumerAPI(NUMERAI_USER, NUMERAI_PASS)
# get datasets
print 'Get dataset'
if not os.path.isfile(TRAIN_FNAME):
api.download_current_dataset(dest_path='.', unzip=True)
# read datasets
train = pd.read_csv(TRAIN_FNAME)
test = pd.read_csv(TEST_FNAME)
print 'Numer.ai data downloaded'
print 'Train shape', train.shape, 'test shape', test.shape
X_train = train[train.columns[:50]]
y_train = train['target']
X_test = test
print 'Create MLJAR project and experiment'
models = Mljar(
project='Auto-trading',
experiment="Raw data",
metric='logloss',
validation_kfolds=5, # we will use 5-fold CV with stratify and shuffle
validation_shuffle=True,
validation_stratify=True,
algorithms=['xgb', 'lgb',
'mlp'], # select Xgboost, LightGBM and Neural Network
tuning_mode=
'Normal', # number of models to be checked for each algorithm
single_algorithm_time_limit=5) # 5 minutes for training single model
print 'Train models:'
# fit models - that's all, only one line of code ;)
models.fit(X_train, y_train)
# get predictions on test data
predictions = models.predict(X_test)
# save predictions to file
predictions.to_csv(PREDICTIONS_FNAME, index=False)
result = api.upload_prediction(PREDICTIONS_FNAME)
print 'Your score:', result['submission']['accuracy_score']
def main(args):
api = numerapi.NumerAPI(verbosity='debug' if args.verbose else 'critical',
secret_key=credentials.key,
public_id=credentials.id)
current_round = api.get_current_round()
if args.round == 'latest':
args.round = current_round
elif int(args.round) > current_round:
raise ValueError('round {} does not exist (latest == {})'.format(
args.round, current_round))
round_dir = download(api, args.round, args.datadir)
targets = tournaments.keys() if args.target == 'all' else [args.target]
args.__dict__.pop('target')
for target in targets:
print(target)
results = predict(round_dir, target, **args.__dict__)
output_file = os.path.join(
round_dir,
'results_target-{}_model-{}.csv'.format(target, args.model))
np.savetxt(output_file, results, delimiter=',', fmt='%s')
if args.upload:
tournament = tournaments[target]
api.upload_predictions(output_file, tournament)
def submit(model_name, user):
"""Submits the results to the competition server.
Arguments:
model_name {str} -- Name of the model used for prediction.
user {str} -- User name, used for looking up submission credentials.
"""
print(f'Model is {model_name}')
print(f'User is {user}')
with open('authentication.json', "r") as credentials_file:
credentials = json.load(credentials_file)[user]
data_folder = common.PROJECT_PATH / 'data'
filename = f'{common.TOURNAMENT_NAME}_{model_name}_submission.csv'
print(f'Submitting {filename} for user {user}')
napi = numerapi.NumerAPI(public_id=credentials['id'],
secret_key=credentials['key'],
verbosity="info")
submission_id = napi.upload_predictions(data_folder / filename)
# check submission status
print(napi.submission_status())
print('...done!')
def __init__(self):
"""NumerAPI instance"""
self.napi = numerapi.NumerAPI(public_id=os.getenv("PUBLIC_ID"),
secret_key=os.getenv("SECRET_KEY"),
verbosity='info')
def test_NumerAPI():
# invalid log level should raise
with pytest.raises(AttributeError):
numerapi.NumerAPI(verbosity="FOO")
def api_fixture():
api = numerapi.NumerAPI(verbosity='DEBUG')
return api
import pprint
import click
import numerapi
napi = numerapi.NumerAPI()
def prettify(stuff):
pp = pprint.PrettyPrinter(indent=4)
return pp.pformat(stuff)
@click.group()
def cli():
"""Wrapper around the Numerai API"""
pass
@cli.command()
@click.option('--tournament',
default=1,
help='The ID of the tournament, defaults to 1')
@click.option('--unzip',
is_flag=True,
default=True,
help='indication of whether the data should be unzipped')
def download_dataset(tournament, unzip):
"""Download dataset for the current active round."""
click.echo(
napi.download_current_dataset(tournament=tournament, unzip=unzip))
confidence
}
return {
nmrAmount
}
stakeResolution {
destroyed
paid
successful
}
}
}
}
'''
napi = numerapi.NumerAPI(verbosity='warn')
logger = logging.getLogger(__name__)
# allow turning off rate limiting, for example during unit tests
noratelimit_flag = os.environ.get('NORATELIMIT', False)
period = 0 if noratelimit_flag else 30
# allow turning off caching, for example during unit tests
nocache_flag = os.environ.get('NOCACHE', False)
memory = Memory(None if nocache_flag else "./cache", verbose=0)
@utils.memoize
def api_fetch_rounds():
q = """
query {
tournaments {
import json
import os
import numerapi
import pandas as pd
example_public_id = os.getenv("NMR_PUBLIC_ID")
example_secret_key = os.getenv("NMR_SECRET_KEY")
NAPI = numerapi.NumerAPI(example_public_id, example_secret_key)
def download_current_data(directory: str):
"""
Downloads the data for the current round
:param directory: The path to the directory where the data needs to be saved
"""
current_round = NAPI.get_current_round()
if os.path.isdir(f"{directory}/numerai_dataset_{current_round}/"):
print(
f"You already have the newest data! Current round is: {current_round}"
)
else:
print(f"Downloading new data for round: {current_round}!")
NAPI.download_current_dataset(dest_path=directory)
def read_current(directory: str):
with open(os.path.dirname(__file__) + "/dtypes.json") as f:
dtypes = json.load(f)
full_path = f"{directory}/numerai_dataset_{NAPI.get_current_round()}/"
#%%
# Example way to upload my predicitons
# find only the feature columns
feature_cols = df.columns[df.columns.str.startswith('feature')]
# select those columns out of the training dataset
training_features = df[feature_cols]
# create a model and fit the training data (~30 sec to run)
model = sklearn.linear_model.LinearRegression()
model.fit(training_features, df.target)
# select the feature columns from the tournament data
live_features = tournament_data[feature_cols]
# predict the target on the live features
predictions = model.predict(live_features)
# predictions must have an `id` column and a `prediction_kazutsugi` column
predictions_df = tournament_data["id"].to_frame()
predictions_df["prediction_kazutsugi"] = predictions
predictions_df.head()
# Get your API keys and model_id from https://numer.ai/submit
public_id = "AAP7UFK3R2W2PS2GVU7MO5WTMPII2SER"
secret_key = "3AXY3QDUP2EKJJ4ZCDPJYSSVED5EJKBGEQJZ7KF4NMCINNXEZNQULZAPDM435G4N"
model_id = "265ea585-1af4-4e36-85fc-e6599054b7e8"
napi = numerapi.NumerAPI(public_id=public_id, secret_key=secret_key)
# Upload your predictions
predictions_df.to_csv("predictions.csv", index=False)
submission_id = napi.upload_predictions("predictions.csv", model_id=model_id)
#!/usr/bin/env python
# coding: utf-8
import os
import numpy as np
import random as rn
import pandas as pd
import seaborn as sns
import lightgbm as lgb
import matplotlib.pyplot as plt
from scipy.stats import spearmanr
from sklearn.metrics import mean_absolute_error
import numerapi
# Initialize Numerai's API
NAPI = numerapi.NumerAPI(verbosity="info")
# Weights and Biases requires you to add your WandB API key for logging in automatically. Because this is a secret key we will use [Kaggle User Secrets](https://www.kaggle.com/product-feedback/114053) to obfuscate the API key.
# Obfuscated WANDB API Key
# from kaggle_secrets import UserSecretsClient
# WANDB_KEY = '7d8786321b64e818153da23692d69d6ad4387b2e'#UserSecretsClient().get_secret("WANDB_API_KEY")
# wandb.login(key=WANDB_KEY)
# Data directory
DIR = "../working"
# Set seed for reproducability
seed = 1234
rn.seed(seed)
np.random.seed(seed)
os.environ['PYTHONHASHSEED'] = str(seed)
# Surpress Pandas warnings
pd.set_option('chained_assignment', None)
import pandas as pd
import pickle
from sklearn.model_selection import train_test_split
import xgboost
from numerai.helpers import DATA_DIRECTORY, numeric_era_converter, POSSIBLE_MARKET_NAMES
import numerapi
napi = numerapi.NumerAPI(verbosity="info")
current_round = napi.get_current_round()
PATH_TO_TOURNAMENT_DATA = f'{DATA_DIRECTORY}numerai_dataset_{current_round}/numerai_tournament_data.csv'
TARGET_NAME = 'target_bernie'
full_tournament_df = pd.read_csv(PATH_TO_TOURNAMENT_DATA, header=0)
full_tournament_df['era_numeric'] = full_tournament_df['era'].apply(
numeric_era_converter)
ids = full_tournament_df['id']
tournament_df_for_prediction = full_tournament_df.drop(
['id', 'era', 'data_type', *POSSIBLE_MARKET_NAMES], axis=1)
dtournament = xgboost.DMatrix(tournament_df_for_prediction)
bst = pickle.load(
open(
f'{DATA_DIRECTORY}numerai_dataset_{current_round}/xgboost_{TARGET_NAME}.pickle.dat',
'rb'))
predictions = bst.predict(dtournament)
try:
"""
Catboostを呼び出しnumeraiに予測を提出する
"""
# 必要なライブラリのinstall
#!pip install numerapi
#!pip install catboost
import numerapi
# 回帰を使うので、CatBoostRegressorを呼び出す
from catboost import CatBoostRegressor
# numerapiを使えばデータセットのダウンロードが簡単にできる
#インスタンス化（numerapiを使うための準備）
napi = numerapi.NumerAPI(verbosity="info")
# 現在のラウンドのデータセットをダウンロードして解凍する。
napi.download_current_dataset(unzip=True)
"""
準備：トーナメントの現在のラウンド数を取得
"""
# numerai_dataset_321/numerai_training_data.csv でトレーニングデータのファイル名
# numerai_dataset_321/numerai_tournament_data.csv でトーナメントデータのファイル名
# まずは現在のトーナメントのラウンド数を取得(int型)
current_ds = napi.get_current_round()
print(current_ds)
# ここはnumerai_dataset_321のようなパスを得るため
latest_round = os.path.join('numerai_dataset_' + str(current_ds))
import os
import numpy as np
import random as rn
import pandas as pd
import seaborn as sns
import lightgbm as lgb
import matplotlib.pyplot as plt
from scipy.stats import spearmanr
from sklearn.metrics import mean_absolute_error
import numerapi
import wandb
from wandb.lightgbm import wandb_callback
# Initialize Numerai's API
NAPI = numerapi.NumerAPI(verbosity="info")
# Weights and Biases requires you to add your WandB API key for logging in automatically. Because this is a secret key we will use [Kaggle User Secrets](https://www.kaggle.com/product-feedback/114053) to obfuscate the API key.
# Obfuscated WANDB API Key
# from kaggle_secrets import UserSecretsClient
# WANDB_KEY = '7d8786321b64e818153da23692d69d6ad4387b2e'#UserSecretsClient().get_secret("WANDB_API_KEY")
# wandb.login(key=WANDB_KEY)
# Data directory
DIR = "../working"
# Set seed for reproducability
seed = 1234
rn.seed(seed)
np.random.seed(seed)
os.environ['PYTHONHASHSEED'] = str(seed)
# Surpress Pandas warnings
pd.set_option('chained_assignment', None)
def download_new_round():
napi = numerapi.NumerAPI()
napi.download_current_dataset(unzip=True)
NUMBER = napi.get_competitions()[0]['number']
return NUMBER