def __getitem__(self, index): "Index by round number, list (or tuple), or slice" if isinstance(index, slice): if index.step is not None: raise ValueError("slice step size must be 1") r1, r2 = self.rounds_to_ints(index.start, index.stop) rs = list(range(r1, r2 + 1)) ts = nx.tournament_all(as_str=False) elif nx.isint(index): rs = [index] ts = nx.tournament_all(as_str=False) elif isinstance(index, list): rs, ts = zip(*index) ts = [nx.tournament_int(i) for i in ts] elif isinstance(index, tuple): if len(index) != 2: raise IndexError("tuple index must have length 2") r, t = index if not nx.isint(r): raise IndexError("first element of tuple index must be int") if not (nx.isint(t) or nx.isstring(t)): msg = "second element of tuple index must be int or str" raise IndexError(msg) rs = [r] ts = [nx.tournament_int(t)] else: raise IndexError("indexing method not supported") self.gets(rs, ts) ridx = self.df['round'].isin(rs) tidx = self.df['tournament'].isin(ts) idx = ridx & tidx df = self.df[idx] return df
def test_tournament_all(): "test tournament_all" t = ['bernie', 'elizabeth', 'jordan', 'ken', 'charles', 'frank', 'hillary'] ok_(nx.tournament_all() == t, 'wrong tournaments') ok_(nx.tournament_all(True) == t, 'wrong tournaments') t = [1, 2, 3, 4, 5, 6, 7] ok_(nx.tournament_all(False) == t, 'wrong tournaments')
def pairs_df(self): """Bool dataframe with names as index and tournaments as columns""" names = self.names() tourns = nx.tournament_all(active_only=False) df = pd.DataFrame(index=names, columns=tourns) for name in names: for tourn in tourns: df.loc[name, tourn] = (name, tourn) in self return df
def metrics_per_era(data, prediction, tournament, join='data', columns=['logloss', 'auc', 'acc', 'ystd'], era_as_str=False, region_as_str=False, split_pairs=True): "Dataframe with columns era, model, and specified metrics. And region list" df = prediction.df # merge prediction with data (remove features x) if join == 'data': how = 'left' elif join == 'yhat': how = 'right' elif join == 'inner': how = 'inner' else: raise ValueError("`join` method not recognized") yhats_df = df.copy() cols = ['era', 'region'] + nx.tournament_all(as_str=True, active_only=False) data_df = data.df[cols] df = pd.merge(data_df, yhats_df, left_index=True, right_index=True, how=how) regions = df['region'].unique().tolist() if region_as_str: regions = [REGION_INT_TO_STR[r] for r in regions] # calc metrics for each era pairs = yhats_df.columns.values metrics = [] unique_eras = df.era.unique() for era in unique_eras: idx = df.era.isin([era]) df_era = df[idx] if era_as_str: era = ERA_INT_TO_STR[era] for pair in pairs: if tournament is None: # evaluate with targets that model trained on tourni = nx.tournament_str(pair[1]) else: # force evaluation targets to be from given tournament tourni = nx.tournament_str(tournament) y = df_era[tourni].values yhat = df_era[pair].values m = calc_metrics_arrays(y, yhat, columns) m = [era, pair] + m metrics.append(m) columns = ['era', 'pair'] + columns metrics = pd.DataFrame(metrics, columns=columns) if split_pairs: metrics = add_split_pairs(metrics) return metrics, regions
def cutoff(lb): "Independent calculation of confidence cutoff" cols = nx.tournament_all(as_str=True) df = pd.DataFrame(columns=cols) rounds = np.sort(lb['round'].unique()) for r in rounds: d = lb[lb['round'] == r] if r > 112: cut = [] for t in nx.tournament_all(as_str=False): dt = d[d.tournament == t] cutoff, ignore = calc_cutoff(dt) cut.append(cutoff) else: cut = [np.nan] * 5 df.loc[r] = cut df['mean'] = df.mean(axis=1) df.loc['mean'] = df.mean() return df
def logloss(lb, user): "Live logloss for `user`" cols = nx.tournament_all() df = pd.DataFrame(columns=cols) lb = lb[['user', 'round', 'tournament', 'live']] rounds = np.sort(lb['round'].unique()) for r in rounds: d = lb[lb['round'] == r] dom = [] for t in nx.tournament_all(as_str=False): dt = d[d.tournament == t] if user in dt.user.values: dm = dt[dt.user == user].live.iloc[0] dom.append(dm) else: dom.append(np.nan) df.loc[r] = dom df['mean'] = df.mean(axis=1) df.loc['mean'] = df.mean() return df
def dominance(lb, user): "Fraction of users that `user` beats in terms of live logloss." cols = nx.tournament_all() df = pd.DataFrame(columns=cols) lb = lb[['user', 'round', 'tournament', 'live']] rounds = np.sort(lb['round'].unique()) for r in rounds: d = lb[lb['round'] == r] dom = [] for t in nx.tournament_all(as_str=False): dt = d[d.tournament == t] dt = dt[dt.live.notna()] if user in dt.user.values: dm = (dt[dt.user == user].live.iloc[0] < dt.live).mean() dom.append(dm) else: dom.append(np.nan) df.loc[r] = dom df['mean'] = df.mean(axis=1) df.loc['mean'] = df.mean() return df
def test_backtest_production(): "Make sure backtest and production run" d = testing.micro_data() model = nx.fifty() with testing.HiddenPrints(): p = nx.production(model, d) ok_(p.shape[1] == 5, 'wrong number of tournaments') ok_(p.tournaments() == nx.tournament_all(), 'wrong tournaments') p = nx.backtest(model, d, kfold=2) ok_(p.shape[1] == 5, 'wrong number of tournaments') ok_(p.tournaments() == nx.tournament_all(), 'wrong tournaments') for verbosity in (0, 1, 2, 3): nx.backtest(model, d, tournament=3, kfold=2, verbosity=verbosity) nx.production(model, d, tournament='ken', verbosity=verbosity) nx.production(model, d, tournament=4, verbosity=verbosity) nx.production(model, d, tournament=None, verbosity=verbosity) if verbosity == 3: nx.production(model, d, tournament=5, verbosity=verbosity) nx.production(model, d, tournament='charles', verbosity=verbosity)
def test_run(): "Make sure run runs" d = testing.play_data() models = [nx.logistic(), nx.fifty()] splitters = [ nx.TournamentSplitter(d), nx.ValidationSplitter(d), nx.CheatSplitter(d), nx.CVSplitter(d, kfold=2), nx.SplitSplitter(d, fit_fraction=0.5) ] for model in models: for splitter in splitters: nx.run(model, splitter, tournament=2, verbosity=0) nx.run(model, splitter, tournament='bernie', verbosity=0) p = nx.run(model, splitter, tournament=None, verbosity=0) ok_(p.shape[1] == 5, 'wrong number of tournaments') ok_(p.tournaments() == nx.tournament_all(), 'wrong tournaments') p = nx.run(model, splitter, verbosity=0) ok_(p.shape[1] == 5, 'wrong number of tournaments') ok_(p.tournaments() == nx.tournament_all(), 'wrong tournaments') assert_raises(ValueError, nx.run, None, nx.TournamentSplitter(d)) assert_raises(ValueError, nx.run, nx.fifty(), nx.TournamentSplitter(d), {})
def whatif(lb, users, s, c): """ Profit if `users` had staked `s` and `c` in every tournament. Earnings are left in NMR instead of splitting the NMR earnings into NMR and USD. """ if isinstance(users, list): pass elif nx.isstring(users): users = [users] else: raise ValueError("`users` must be str or list (of str)") cols = ['nmr_staked', 'nmr_burn', 'nmr_earn', 'nmr_net'] df = pd.DataFrame(columns=cols) lb.insert(0, 'pass', lb['live'] < LOGLOSS_BENCHMARK) rounds = np.sort(lb['round'].unique()) for r in rounds: d = lb[lb['round'] == r] if r > 112: staked = 0 burn = 0 earn = 0 for t in nx.tournament_all(as_str=False): dt = d[d.tournament == t] if dt.shape[0] > 0: cutoff, ignore = calc_cutoff(dt) if c >= cutoff: idx = dt.user.isin(users) dti = dt[idx] idx = dti['pass'] nwin = idx.sum() nlos = (~idx & (dti['live'].notna())).sum() p = (1.0 - cutoff) / cutoff burn += nlos * s earn += nwin * s * p staked += idx.size * s net = earn - burn df.loc[r] = [staked, burn, earn, net] else: raise ValueError("`round1` must start at at least 113") df.loc['total'] = df.sum() return df
def __getitem__(self2, index): n = nx.tournament_count(active_only=False) if isinstance(index, str): if index in nx.tournament_all(as_str=True, active_only=True): return self2.df[index].values else: raise IndexError('string index not recognized') elif nx.isint(index): if index < 1 or index > n: txt = 'tournament number must be between 1 and {}' raise IndexError(txt.format(n)) return self2.df[nx.tournament_str(index)].values elif isinstance(index, slice): if (index.start is None and index.stop is None and index.step is None): # slicing below means a view is returned instead of a copy return self2.df.iloc[:, -n:].values else: raise IndexError('Start, stop, and step of slice must be None') else: raise IndexError('indexing type not recognized')
def load_zip(file_path, verbose=False): "Load numerai dataset from zip archive; return Data" # load zip zf = zipfile.ZipFile(file_path) train = pd.read_csv(zf.open(TRAIN_FILE), header=0, index_col=0) tourn = pd.read_csv(zf.open(TOURNAMENT_FILE), header=0, index_col=0) # turn into single dataframe and rename columns df = pd.concat([train, tourn], axis=0) rename_map = {'data_type': 'region'} for i in range(1, N_FEATURES + 1): rename_map['feature' + str(i)] = 'x' + str(i) for number, name in nx.tournament_iter(): rename_map['target_' + name] = name df.rename(columns=rename_map, inplace=True) # convert era, region, and labels to np.float64 df['era'] = df['era'].map(ERA_STR_TO_FLOAT) df['region'] = df['region'].map(REGION_STR_TO_FLOAT) n = len(nx.tournament_all()) df.iloc[:, -n:] = df.iloc[:, -n:].astype('float64') # no way we did something wrong, right? n = 2 + N_FEATURES + nx.tournament_count() if df.shape[1] != n: raise IOError("expecting {} columns; found {}".format(n, df.shape[1])) # make sure memory is contiguous so that, e.g., data.x is a view df = df.copy() # to avoid copies we need the dtype of each column to be the same if df.dtypes.unique().size != 1: raise TypeError("dtype of each column should be the same") data = Data(df) if verbose: print(data) return data
def pass_rate(lb): "Fraction of users who beat benchmark in each round" cols = ['all', 'stakers', 'nonstakers', 'above_cutoff', 'below_cutoff'] df = pd.DataFrame(columns=cols) rounds = np.sort(lb['round'].unique()) for r in rounds: d = lb[(lb['round'] == r) & (lb.live.notna())] d.insert(0, 'pass', d['live'] < LOGLOSS_BENCHMARK) pr_all = d['pass'].mean() pr_stakers = d[d['s'] > 0]['pass'].mean() pr_nonstakers = d[d['s'] == 0]['pass'].mean() if r > 112: nabove = 0 nbelow = 0 pabove = 0 pbelow = 0 for t in nx.tournament_all(as_str=False): dt = d[d.tournament == t] cutoff, ignore = calc_cutoff(dt) nabove += dt[dt.c > cutoff].shape[0] nbelow += dt[dt.c < cutoff].shape[0] pabove += dt[(dt.c > cutoff) & (dt['pass'])].shape[0] pbelow += dt[(dt.c < cutoff) & (dt['pass'])].shape[0] if nabove == 0: pr_above = np.nan else: pr_above = 1.0 * pabove / nabove if nbelow == 0: pr_below = np.nan else: pr_below = 1.0 * pbelow / nbelow else: pr_above = np.nan pr_below = np.nan df.loc[r] = [pr_all, pr_stakers, pr_nonstakers, pr_above, pr_below] df.loc['mean'] = df.mean() return df
def payout(lb): "NMR and USD payouts per round" cols = [ 'staked_nmr', 'staked_above_cutoff', 'burned_nmr', 'nmr_payout', 'usd_payout', 'total_payout_in_nmr' ] df = pd.DataFrame(columns=cols) rounds = np.sort(lb['round'].unique()) lb.insert(0, 'pass', lb['live'] < LOGLOSS_BENCHMARK) for r in rounds: d = lb[lb['round'] == r] if r > 112: nmr_cut = 0 nmr_cut_pass = 0 for t in nx.tournament_all(as_str=False): dt = d[d.tournament == t] cutoff, ignore = calc_cutoff(dt) nmr_cut += dt[dt.c >= cutoff].sum()['s'] nmr_cut_pass += dt[(dt.c >= cutoff) & (dt['pass'])].sum()['s'] else: nmr_cut = np.nan if cutoff == 0: total = np.nan else: total = nmr_cut_pass * (1.0 - cutoff) / cutoff ds = d.sum() pay = [ ds['s'], nmr_cut, ds['nmr_burn'], ds['nmr_stake'], ds['usd_stake'], total ] df.loc[r] = pay fraction = df['burned_nmr'] / df['staked_above_cutoff'] df.insert(3, 'fraction_burned', fraction) df.loc['mean'] = df.mean() df = df.round(2) return df
def test_tournament_all(): """test tournament_all""" # Testing all inactive tournaments t = [ 'bernie', 'elizabeth', 'jordan', 'ken', 'charles', 'frank', 'hillary', 'kazutsugi' ] ok_(nx.tournament_all(active_only=False) == t, 'wrong tournaments') ok_(nx.tournament_all(True, active_only=False) == t, 'wrong tournaments') t = [1, 2, 3, 4, 5, 6, 7, 8] ok_(nx.tournament_all(False, active_only=False) == t, 'wrong tournaments') # Testing all active tournaments t = ['kazutsugi'] ok_(nx.tournament_all() == t, 'wrong tournaments') ok_(nx.tournament_all(True) == t, 'wrong tournaments') t = [8] ok_(nx.tournament_all(False) == t, 'wrong tournaments')
def run(model, splitter, tournament=None, verbosity=2): """ Run a model/tournament pair (or pairs) through a data splitter. Parameters ---------- model : nx.Model, list, tuple Prediction model to run through the splitter. Can be a list or tuple of prediction models. Model names must be unique. splitter : nx.Splitter An iterator of fit/predict data pairs. tournament : {None, int, str, list, tuple}, optional The tournament(s) to run the model through. By default (None) the model is run through all active tournaments. If a list or tuple of tournaments is given then it must must not contain duplicate tournaments. verbosity : int, optional An integer that determines verbosity. Zero is silent. Returns ------- p : nx.Prediction A prediction object containing the predictions of the specified model/tournament pairs. """ # make list of models if isinstance(model, nx.Model): models = [model] elif isinstance(model, list) or isinstance(model, tuple): models = model else: raise ValueError('`model` must be a model, list, or tuple of models') names = [m.name for m in models] if len(names) != len(set(names)): raise ValueError('`model` cannot contain duplicate names') # make list of tournaments if tournament is None: tournaments = nx.tournament_all() elif nx.isint(tournament) or nx.isstring(tournament): tournaments = [tournament] elif isinstance(tournament, list) or isinstance(tournament, tuple): tournaments = tournament else: msg = '`tournament` must be an integer, string, list, tuple, or None.' raise ValueError(msg) tournaments = [nx.tournament_str(t) for t in tournaments] if len(tournaments) != len(set(tournaments)): raise ValueError('`tournament` cannot contain duplicates') # loop over all model/tournament pairs p = nx.Prediction() for m in models: for t in tournaments: p += run_one(m, splitter, t, verbosity=verbosity) splitter.reset() splitter.reset() return p