def performance_per_era(self, data, model_name): print(model_name) df = self.df[model_name].to_frame(model_name) df = metrics_per_era(data, Report(df))[model_name] df = df.round(decimals={'logloss': 6, 'auc': 4, 'acc': 4, 'ystd': 4}) with pd.option_context('display.colheader_justify', 'left'): print(df.to_string())
def dominance(self, data, tournament=None, sort_by='corr'): """Mean (across eras) of fraction of models bested per era""" columns = ['corr', 'mse'] mpe, regions = metrics_per_era(data, self, tournament, columns=columns) dfs = [] for i, col in enumerate(columns): pivot = mpe.pivot(index='era', columns='pair', values=col) pairs = pivot.columns.tolist() a = pivot.values n = a.shape[1] - 1.0 if n == 0: raise ValueError("Must have at least two pairs") m = [] for j in range(pivot.shape[1]): if col == 'corr': z = (a[:, j].reshape(-1, 1) < a).sum(axis=1) / n else: z = (a[:, j].reshape(-1, 1) > a).sum(axis=1) / n m.append(z.mean()) df = pd.DataFrame(data=m, index=pairs, columns=[col]) dfs.append(df) df = pd.concat(dfs, axis=1) df = add_split_pairs(df) df = df.sort_values([sort_by], ascending=[False]) return df
def metrics_per_era(self, data, metrics=['logloss', 'auc', 'acc', 'ystd'], era_as_str=True): "DataFrame containing given metrics versus era (as index)" metrics, regions = metrics_per_era(data, self, columns=metrics, era_as_str=era_as_str) metrics.index = metrics['era'] metrics = metrics.drop(['era'], axis=1) return metrics
def test_metrics_per_era(): "make sure metrics_per_era runs" d = testing.micro_data() p = testing.micro_prediction() metrics_per_era(d, p, 1) metrics_per_era(d, p, 2, join='yhat') metrics_per_era(d, p, 3, join='inner') assert_raises(ValueError, metrics_per_era, d, p, 4, 'outer') with testing.HiddenPrints(): metrics_per_era(d, p, tournament=5, era_as_str=True)
def test_metrics_per_era(): "make sure calc_metrics runs" d = micro_data() p = micro_prediction() metrics_per_era(d, p) metrics_per_era(d, p, 'yhat') metrics_per_era(d, p, 'inner') assert_raises(ValueError, metrics_per_era, d, p, 'outer')
def summary(self, data, tournament=None, round_output=True): """Performance summary of prediction object that contains a single pair""" if self.shape[1] != 1: raise ValueError("prediction must contain a single pair") # metrics metrics, regions = metrics_per_era(data, self, tournament, region_as_str=True, split_pairs=False) metrics = metrics.drop(['era', 'pair'], axis=1) # additional metrics region_str = ', '.join(regions) nera = metrics.shape[0] corr = metrics['corr'] consis = (corr > CORR_BENCHMARK).mean() # summary of metrics if tournament is None: t_str = self.tournaments(as_str=True)[0] else: t_str = nx.tournament_str(tournament) m1 = metrics.mean(axis=0).tolist() + ['tourn', t_str] m2 = metrics.std(axis=0).tolist() + ['region', region_str] m3 = metrics.min(axis=0).tolist() + ['eras', nera] m4 = metrics.max(axis=0).tolist() + ['consis', consis] data = [m1, m2, m3, m4] # make dataframe columns = metrics.columns.tolist() + ['stats', ''] df = pd.DataFrame(data=data, index=['mean', 'std', 'min', 'max'], columns=columns) # make output (optionally) pretty if round_output: round_dict = {'corr': 6, 'mse': 4, 'ystd': 4} df = df.round(decimals=round_dict) return df
def performance(self, data): metrics = metrics_per_era(data, self) metrics = metrics['yhat'] regions = data.unique_region().tolist() regions = ', '.join(regions) print(" logloss auc acc ystd") fmt = "{:<4} {:8.6f} {:6.4f} {:6.4f} {:6.4f}{extra}" extra = " | {:<7} {:<}".format('region', regions) print(fmt.format('mean', *metrics.mean(axis=0), extra=extra)) extra = " | {:<7} {:<}".format('eras', metrics.shape[0]) print(fmt.format('std', *metrics.std(axis=0), extra=extra)) consistency = (metrics['logloss'] < np.log(2)).mean() extra = " | {:<7} {:<.4f}".format('consis', consistency) print(fmt.format('min', *metrics.min(axis=0), extra=extra)) with warnings.catch_warnings(): warnings.filterwarnings('ignore', '', RuntimeWarning) prctile = np.percentile(metrics['logloss'], 75) extra = " | {:<7} {:<.4f}".format('75th', prctile) print(fmt.format('max', *metrics.max(axis=0), extra=extra))
def performance_df(self, data): # calc performance metrics = metrics_per_era(data, self) regions = data.unique_region().tolist() models = list(metrics.keys()) nera = metrics[models[0]].shape[0] regera = ', '.join(regions) + '; %d' % nera + ' eras' # create dataframe of performance cols = ['logloss', 'auc', 'acc', 'ystd', 'consis', '(%s)' % regera] df = pd.DataFrame(columns=cols) for i, model in enumerate(models): metric_df = metrics[model] metric = metric_df.mean(axis=0).tolist() consis = (metric_df['logloss'] < np.log(2)).mean() metric.extend([consis, model]) df.loc[i] = metric return df
def metrics_per_era(self, data, tournament=None, metrics=['corr', 'mse', 'ystd'], era_as_str=True, split_pairs=True): "DataFrame containing given metrics versus era (as index)" metrics, regions = metrics_per_era(data, self, tournament, columns=metrics, era_as_str=era_as_str) metrics.index = metrics['era'] metrics = metrics.drop(['era'], axis=1) if split_pairs: pair = metrics['pair'] metrics = metrics.drop('pair', axis=1) metrics.insert(0, 'pair', pair) else: metrics = metrics.drop('pair', axis=1) return metrics
def summary(self, data, round_output=True): "Performance summary of prediction object that contains a single name" if self.shape[1] != 1: raise ValueError("prediction must contain a single name") # metrics metrics, regions = metrics_per_era(data, self, region_as_str=True) metrics = metrics.drop(['era', 'name'], axis=1) # additional metrics region_str = ', '.join(regions) nera = metrics.shape[0] logloss = metrics['logloss'] consis = (logloss < LOGLOSS_BENCHMARK).mean() sharpe = (LOGLOSS_BENCHMARK - logloss).mean() / logloss.std() # summary of metrics m1 = metrics.mean(axis=0).tolist() + ['region', region_str] m2 = metrics.std(axis=0).tolist() + ['eras', nera] m3 = metrics.min(axis=0).tolist() + ['sharpe', sharpe] m4 = metrics.max(axis=0).tolist() + ['consis', consis] data = [m1, m2, m3, m4] # make dataframe columns = metrics.columns.tolist() + ['stats', ''] df = pd.DataFrame(data=data, index=['mean', 'std', 'min', 'max'], columns=columns) # make output (optionally) pretty if round_output: round_dict = {'logloss': 6, 'auc': 4, 'acc': 4, 'ystd': 4} df = df.round(decimals=round_dict) return df