コード例 #1
0
class SubmissionTarget(luigi.target.Target):
    """
    Implements a submission target to "output" predictions from luigi tasks on
    the numer.ai servers.
    """
    def __init__(self, path, public_id, secret):
        """
        Creates a new SubmissionTarget.

        :param: path (str):
            local path to the predictions csv file
        :param: public_id (str):
            public_id as reported by the numer.ai website when creating API
            credentials
        :param: secret (str):
            secret as reported by the numer.ai website when creating API
            credentials
        """
        self.path = path
        self.fn = os.path.split(path)[1]
        self.apc = NumerAPI(public_id, secret)

    def exists(self):
        """
        Checks if a submission for the file named :py:attr:`path` was uploaded.

        NB: the filename as reported by the server is appended by a random
        string (before the file extension), and we can just access the file
        that was submitted last. This might result in double uploads for the
        same file.
        """
        qry = "query user { user { latestSubmission { filename } } }"
        res = self.apc.raw_query(qry, authorization=True)

        data = res['data']['user']['latestSubmission']

        for d in data:
            if d['filename'].startswith(self.fn.replace('.csv', '')):
                return True

        return False

    def submit(self):
        """
        Submits the predictions to the numer.ai servers and tries to report
        back the status.
        """
        ret = self.apc.upload_predictions(self.path)
        print(self.apc.submission_status())
コード例 #2
0
def main():
    # set example username and round
    example_public_id = "somepublicid"
    example_secret_key = "somesecretkey"

    # some API calls do not require logging in
    napi = NumerAPI(verbosity="info")
    # download current dataset
    napi.download_current_dataset(unzip=True)
    # get competitions
    all_competitions = napi.get_competitions()
    # get leaderboard for the current round
    leaderboard = napi.get_leaderboard()
    # leaderboard for a historic round
    leaderboard_67 = napi.get_leaderboard(round_num=67)

    # provide api tokens
    napi = NumerAPI(example_public_id, example_secret_key)

    # upload predictions
    submission_id = napi.upload_predictions("mypredictions.csv")
    # check submission status
    napi.submission_status()
コード例 #3
0
def main():
    # set example username and round
    example_username = "******"
    example_round = 51

    # set up paths for download of dataset and upload of predictions
    now = datetime.now().strftime("%Y%m%d")
    dataset_parent_folder = "./dataset"
    dataset_name = "numerai_dataset_{0}/example_predictions.csv".format(now)
    dataset_path = "{0}/{1}".format(dataset_parent_folder, dataset_name)

    # most API calls do not require logging in
    napi = NumerAPI(verbosity="info")

    # log in
    credentials = napi.login()
    print(json.dumps(credentials, indent=2))

    # download current dataset
    dl_succeeded = napi.download_current_dataset(
        dest_path=dataset_parent_folder, unzip=True)
    print("download succeeded: " + str(dl_succeeded))

    # get competitions (returned data is too long to print practically)
    # all_competitions = napi.get_all_competitions()
    # current_competition = napi.get_competition()
    # example_competition = napi.get_competition(round_id=example_round)

    # get user earnings per round
    user_earnings = napi.get_earnings_per_round()
    print("user earnings:")
    print(user_earnings)
    example_earnings = napi.get_earnings_per_round(username=example_username)
    print("example earnings:")
    print(example_earnings)

    # get scores for user
    personal_scores = napi.get_scores_for_user()
    print("personal scores:")
    print(personal_scores)
    other_scores = napi.get_scores_for_user(username=example_username)
    print("other scores:")
    print(other_scores)

    # get user information
    current_user = napi.get_user()
    print("current user:"******"example user:"******"submission:")
    print(json.dumps(submission, indent=2))

    # upload predictions
    ul_succeeded = napi.upload_predictions(dataset_path)
    print("upload succeeded: " + str(ul_succeeded))
コード例 #4
0
class Evaluator(threading.Thread):
    def __init__(self):
        super(Evaluator, self).__init__()
        self.logger = logging.getLogger('evaluator')
        self.logger.setLevel(logging.DEBUG)
        self.login()

    def login(self):
        public_id = os.environ['PUBLIC_ID']
        private_secret = os.environ['PRIVATE_SECRET']
        while True:
            try:
                self.napi = NumerAPI(public_id, private_secret)
                break
            except Exception:
                self.logger.exception('Login failed')
                time.sleep(10)

    def upload(self, prediction):
        while True:
            try:
                self.logger.info('Uploading prediction: {}'.format(prediction))
                self.napi.upload_predictions(file_path=prediction)
                self.logger.info('Uploaded prediction: {}'.format(prediction))
                break
            except requests.exceptions.HTTPError as error:
                if error.response.status_code == 429:
                    self.logger.info('Backing off')
                    time.sleep(30 * 60)
                else:
                    self.logger.exception('Network failure')
                    time.sleep(60)
            except Exception as exception:
                self.logger.exception('Upload failure')
                time.sleep(10)

    def check(self, prediction):
        while True:
            try:
                self.logger.info('Checking submission: {}'.format(prediction))
                status = self.napi.submission_status()
                self.logger.info('Got {}: {}'.format(prediction, str(status)))
                logloss_ready = status['validation_logloss'] is not None
                concordance_ready = not status['concordance']['pending']
                originality_ready = not status['originality']['pending']
                if logloss_ready and concordance_ready and originality_ready:
                    return status
                else:
                    time.sleep(10)
            except Exception:
                self.logger.exception('Checking submission failed')
                time.sleep(10)

    def report(self, prediction, status):
        result = {
            'logloss': status['validation_logloss'],
            'consistency': status['consistency'],
            'concordance': status['concordance']['value'],
            'originality': status['originality']['value']
        }
        with open(prediction + '.report.json', 'wb') as handle:
            handle.write(json.dumps(result).encode('utf-8'))

    def submit(self, prediction):
        self.upload(prediction)
        time.sleep(5)
        self.report(prediction, self.check(prediction))

    def run(self):
        captor = Captor(os.getenv('STORING'), 'predictions*.csv')
        while True:
            while not captor.empty():
                prediction = captor.grab()
                if not os.path.isfile(prediction + '.report.json'):
                    self.submit(prediction)
            time.sleep(1)