def test_error_handling(api: NumerAPI):
# String instead of Int
with pytest.raises(ValueError):
api.get_leaderboard("foo")
# round that doesn't exist
with pytest.raises(ValueError):
api.get_leaderboard(-1)
# unauthendicated request
with pytest.raises(ValueError):
# set wrong token
api.token = ("foo", "bar")
api.submission_id = 1
api.submission_status()
def main():
# set example username and round
example_public_id = "somepublicid"
example_secret_key = "somesecretkey"
# some API calls do not require logging in
napi = NumerAPI(verbosity="info")
# download current dataset
napi.download_current_dataset(unzip=True)
# get competitions
all_competitions = napi.get_competitions()
# get leaderboard for the current round
leaderboard = napi.get_leaderboard()
# leaderboard for a historic round
leaderboard_67 = napi.get_leaderboard(round_num=67)
# provide api tokens
napi = NumerAPI(example_public_id, example_secret_key)
# upload predictions
submission_id = napi.upload_predictions("mypredictions.csv")
# check submission status
napi.submission_status()
class SubmissionTarget(luigi.target.Target):
"""
Implements a submission target to "output" predictions from luigi tasks on
the numer.ai servers.
"""
def __init__(self, path, public_id, secret):
"""
Creates a new SubmissionTarget.
:param: path (str):
local path to the predictions csv file
:param: public_id (str):
public_id as reported by the numer.ai website when creating API
credentials
:param: secret (str):
secret as reported by the numer.ai website when creating API
credentials
"""
self.path = path
self.fn = os.path.split(path)[1]
self.apc = NumerAPI(public_id, secret)
def exists(self):
"""
Checks if a submission for the file named :py:attr:`path` was uploaded.
NB: the filename as reported by the server is appended by a random
string (before the file extension), and we can just access the file
that was submitted last. This might result in double uploads for the
same file.
"""
qry = "query user { user { latestSubmission { filename } } }"
res = self.apc.raw_query(qry, authorization=True)
data = res['data']['user']['latestSubmission']
for d in data:
if d['filename'].startswith(self.fn.replace('.csv', '')):
return True
return False
def submit(self):
"""
Submits the predictions to the numer.ai servers and tries to report
back the status.
"""
ret = self.apc.upload_predictions(self.path)
print(self.apc.submission_status())
class Evaluator(threading.Thread):
def __init__(self):
super(Evaluator, self).__init__()
self.logger = logging.getLogger('evaluator')
self.logger.setLevel(logging.DEBUG)
self.login()
def login(self):
public_id = os.environ['PUBLIC_ID']
private_secret = os.environ['PRIVATE_SECRET']
while True:
try:
self.napi = NumerAPI(public_id, private_secret)
break
except Exception:
self.logger.exception('Login failed')
time.sleep(10)
def upload(self, prediction):
while True:
try:
self.logger.info('Uploading prediction: {}'.format(prediction))
self.napi.upload_predictions(file_path=prediction)
self.logger.info('Uploaded prediction: {}'.format(prediction))
break
except requests.exceptions.HTTPError as error:
if error.response.status_code == 429:
self.logger.info('Backing off')
time.sleep(30 * 60)
else:
self.logger.exception('Network failure')
time.sleep(60)
except Exception as exception:
self.logger.exception('Upload failure')
time.sleep(10)
def check(self, prediction):
while True:
try:
self.logger.info('Checking submission: {}'.format(prediction))
status = self.napi.submission_status()
self.logger.info('Got {}: {}'.format(prediction, str(status)))
logloss_ready = status['validation_logloss'] is not None
concordance_ready = not status['concordance']['pending']
originality_ready = not status['originality']['pending']
if logloss_ready and concordance_ready and originality_ready:
return status
else:
time.sleep(10)
except Exception:
self.logger.exception('Checking submission failed')
time.sleep(10)
def report(self, prediction, status):
result = {
'logloss': status['validation_logloss'],
'consistency': status['consistency'],
'concordance': status['concordance']['value'],
'originality': status['originality']['value']
}
with open(prediction + '.report.json', 'wb') as handle:
handle.write(json.dumps(result).encode('utf-8'))
def submit(self, prediction):
self.upload(prediction)
time.sleep(5)
self.report(prediction, self.check(prediction))
def run(self):
captor = Captor(os.getenv('STORING'), 'predictions*.csv')
while True:
while not captor.empty():
prediction = captor.grab()
if not os.path.isfile(prediction + '.report.json'):
self.submit(prediction)
time.sleep(1)