def make_details(data, control): 'return a ColumnsTable' def append_feature_group_description(ct): ct.append_line(' ') ct.append_line('Features groups;') ct.append_line('s : only size features') ct.append_line('sw : only size and wealth features') ct.append_line('swp : only size, wealth, and property features') ct.append_line('swpn : all features: size, wealth, property, and neighborhood') ct = ColumnsTable(( ('month', 6, '%6s', ('', 'month'), 'training month'), ('features', 8, '%8s', ('features', 'group'), 'group of features'), ('model', 5, '%5s', ('best', 'model'), 'family of best model'), ('mae', 6, '%6.0f', ('', 'mae'), 'mae of best model in month using features'), ), verbose=True, ) for month in control.test_months: for features in control.feature_groups: mae_model = data[month][features] ct.append_detail( month=month, features=features, model=mae_model.model, mae=mae_model.mae, ) ct.append_detail() # blank line separates each month ct.append_legend() append_feature_group_description(ct) return ct
class ReportWithColumnsTable(object): def __init__(self, header_lines, column_defs, print_as_spaces, verbose=True): self._report = Report() self._header(header_lines) self._ct = ColumnsTable(column_defs, verbose) self._print_as_spaces = print_as_spaces def _header(self, header_lines): for line in header_lines: self._report.append(line) def append_detail(self, **kwds): # replace NaN with None with_spaces = { k: (None if self._print_as_spaces(k, v) else v) for k, v in kwds.iteritems() } self._ct.append_detail(**with_spaces) def write(self, path): self._t.append_legend() for line in self._t.iterlines(): self._report.append(line) self._report.write(path)
def make_report(title, ordered_cities): def make_detail_line(city): return { 'city': city, 'median_price': median_prices[city], 'median_price_index': median_prices_indices[city], 'n_trades': n_trades[city], 'n_trades_index': n_trades_indices[city], } c = ColumnsTable(( ('city', 30, '%30s', ('', '', '', '', '', 'City'), 'city name'), ('median_price', 7, '%7.0f', ('', '', '', '', 'median', 'price'), 'median price in city'), ('median_price_index', 7, '%7.2f', ('median', 'price', '/', 'overall', 'median', 'price'), 'median price as fraction of overall median price'), ('n_trades', 7, '%7.0f', ('', '', '', '', 'number', 'trades'), 'number of trades across all months'), ('n_trades_index', 7, '%7.2f', ('number', 'trades', '/ ', 'overall', 'median', 'trades'), 'median number trades as fraction of overall median number of trades' ), )) for city in ordered_cities: c.append_detail(**make_detail_line(city)) c.append_legend(40) r = Report() r.append(title) r.append(' ') for line in c.iterlines(): r.append(line) return r
def make_details(data, test_months): "return a ColumnTable" columns_table = ColumnsTable( ( ("mean_prob", 5, "%5.2f", ("mean", "prob"), "mean probability feature appears in a decision tree"), ("feature_name", 40, "%40s", (" ", "feature name"), "name of feature"), ), verbose=True, ) mean_importance = make_mean_importance_by_feature(test_months) for feature_name in sorted(mean_importance, key=mean_importance.get, reverse=True): columns_table.append_detail(mean_prob=mean_importance[feature_name] * 100.0, feature_name=feature_name) columns_table.append_legend() return columns_table
def make_column_table(df): ct = ColumnsTable(columns=( ('city', 30, '%30s', ('', 'city'), 'city in Los Angeles Country'), ('count', 6, '%6d', (' ', 'count'), 'number of transactions in 2007'), ('median_price', 7, '%7.0f', ('median', 'price'), 'median price'), ), ) for index, series in df.iterrows(): ct.append_detail( city=series['city'], count=series['count'], median_price=series['median_price'], ) ct.append_legend() return ct
class ChartCDReport(object): def __init__(self, column_definitions, test): self._column_definitions = column_definitions self._test = test self._report = Report() cd = self._column_definitions.defs_for_columns( 'validation_month', 'rank', 'median_absolute_error', 'median_price', 'model', 'n_months_back', 'max_depth', 'n_estimators', 'max_features', 'learning_rate', 'alpha', 'l1_ratio', 'units_X', 'units_y', ) self._ct = ColumnsTable(columns=cd, verbose=True) self._header() def append(self, line): self._report.append(line) def write(self, path): self._ct.append_legend() for line in self._ct.iterlines(): self._report.append(line) if self._test: self._report.append('** TESTING: DISCARD') self._report.write(path) def _header(self): self._report.append( 'Median Absolute Error (MAE) by month for best-performing models and their hyperparameters' ) self._report.append(' ') def append_detail(self, **kwds): with_spaces = { k: (None if self._column_definitions.replace_by_spaces(k, v) else v) for k, v in kwds.iteritems() } self._ct.append_detail(**with_spaces)
def make_table_stats(data, control, in_report_p): 'return Report with statistics for years and months that obey the filter' r = Report() r.append('Prices by Month') r.append('') ct = ColumnsTable(( ('year', 4, '%4d', (' ', ' ', 'year'), 'year of transaction'), ('month', 5, '%5d', (' ', ' ', 'month'), 'month of transaction'), ('mean_price', 6, '%6.0f', (' ', ' mean', 'price'), 'mean price in dollars'), ('median_price', 6, '%6.0f', (' ', 'median', 'price'), 'median price in dollars'), ('mean_price_ratio', 6, '%6.3f', (' mean', ' price', ' ratio'), 'ratio of price in current month to prior month'), ('median_price_ratio', 6, '%6.3f', ('median', ' price', ' ratio'), 'ratio of price in current month to prior month'), ('number_trades', 6, '%6d', ('number', 'of', 'trades'), 'number of trades in the month'), )) prior_mean_price = None prior_median_price = None for year in (2003, 2004, 2005, 2006, 2007, 2008, 2009): for month in (1, 2, 3) if year == 2009 else (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12): if in_report_p(year, month): selected = data.month == Month(year, month) prices = data[selected].price mean_price = prices.mean() median_price = prices.median() number_trades = len(prices) ct.append_detail( year=year, month=month, mean_price=mean_price, median_price=median_price, mean_price_ratio=None if prior_mean_price is None else mean_price / prior_mean_price, median_price_ratio=None if prior_median_price is None else median_price / prior_median_price, number_trades=number_trades, ) prior_mean_price = mean_price prior_median_price = median_price ct.append_legend() for line in ct.iterlines(): r.append(line) return r
class ChartEReport(object): def __init__(self, k, ensemble_weighting, column_definitions, test): self._column_definitions = column_definitions self._test = test self._report = Report() self._header(k, ensemble_weighting) cd = self._column_definitions.defs_for_columns( 'validation_month', 'model', 'n_months_back', 'n_estimators', 'max_features', 'max_depth', 'learning_rate', 'rank', 'weight', 'mae_validation', 'mae_query', 'mae_ensemble', ) self._ct = ColumnsTable(columns=cd, verbose=True) def write(self, path): self._ct.append_legend() for line in self._ct.iterlines(): self._report.append(line) if self._test: self._report.append('** TESTING: DISCARD') self._report.write(path) def detail_line(self, **kwds): with_spaces = { k: (None if self._column_definitions.replace_by_spaces(k, v) else v) for k, v in kwds.iteritems() } self._ct.append_detail(**with_spaces) def _header(self, k, ensemble_weighting): self._report.append( 'Performance of Best Models Separately and as an Ensemble') self._report.append(' ') self._report.append('Considering Best K = %d models' % k) self._report.append('Ensemble weighting: %s' % ensemble_weighting)
class ChartHReport(object): def __init__(self, k, validation_month, ensemble_weighting, column_definitions, test): self._column_definitions = column_definitions self._report = Report() self._test = test self._header(k, validation_month, ensemble_weighting) cd = self._column_definitions.defs_for_columns( 'description', 'mae_validation', 'mae_query', 'mare_validation', 'mare_query', ) self._ct = ColumnsTable(columns=cd, verbose=True) def write(self, path): self._ct.append_legend() for line in self._ct.iterlines(): self._report.append(line) if self._test: self._report.append('** TESTING: DISCARD') self._report.write(path) def detail_line(self, **kwds): with_spaces = { k: (None if self._column_definitions.replace_by_spaces(k, v) else v) for k, v in kwds.iteritems() } self._ct.append_detail(**with_spaces) def preformatted_line(self, line): print line self._ct.append_line(line) def _header(self, k, validation_month, ensemble_weighting): self._report.append( 'Performance of Best Models Separately and as an Ensemble') self._report.append(' ') self._report.append('Considering Best K = %d models' % k) self._report.append('For validation month %s' % validation_month) self._report.append('Ensemble weighting: %s' % ensemble_weighting)
def make_details(data, control): 'return a ColumnsTable' def append_feature_group_description(ct): ct.append_line(' ') ct.append_line('Features groups;') ct.append_line('s : only size features') ct.append_line('sw : only size and wealth features') ct.append_line('swp : only size, wealth, and property features') ct.append_line( 'swpn : all features: size, wealth, property, and neighborhood' ) ct = ColumnsTable( ( ('month', 6, '%6s', ('', 'month'), 'training month'), ('features', 8, '%8s', ('features', 'group'), 'group of features'), ('model', 5, '%5s', ('best', 'model'), 'family of best model'), ('mae', 6, '%6.0f', ('', 'mae'), 'mae of best model in month using features'), ), verbose=True, ) my_info = [] for month in control.test_months: for features in control.feature_groups: mae_model = data[month][features] ct.append_detail( month=month, features=features, model=mae_model.model, mae=mae_model.mae, ) my_info.append( [month, features, mae_model.model, mae_model.mae]) ct.append_detail() # blank line separates each month ct.append_legend() append_feature_group_description(ct) return ct, my_info
def make_details(data, test_months): 'return a ColumnTable' columns_table = ColumnsTable(( ('mean_prob', 5, '%5.2f', ('mean', 'prob'), 'mean probability feature appears in a decision tree'), ('feature_name', 40, '%40s', (' ', 'feature name'), 'name of feature'), ), verbose=True) my_prob = [] my_featname = [] mean_importance = make_mean_importance_by_feature(test_months) for feature_name in sorted(mean_importance, key=mean_importance.get, reverse=True): columns_table.append_detail( mean_prob=mean_importance[feature_name] * 100.0, feature_name=feature_name, ) if mean_importance[feature_name] * 100.0 >= 1: my_prob.append(mean_importance[feature_name] * 100.0) my_featname.append(feature_name) columns_table.append_legend() return columns_table, my_featname, my_prob
class ChartFReport(object): def __init__(self, k, ensemble_weighting, column_definitions, test): self._column_definitions = column_definitions self._test = test self._report = Report() self._header(k, ensemble_weighting) cd = self._column_definitions.defs_for_columns( 'validation_month', 'mae_index0', 'mae_ensemble', 'mae_best_next_month', 'median_price', 'fraction_median_price_next_month_index0', 'fraction_median_price_next_month_ensemble', 'fraction_median_price_next_month_best', ) self._ct = ColumnsTable(columns=cd, verbose=True) def write(self, path): self._ct.append_legend() for line in self._ct.iterlines(): self._report.append(line) if self._test: self._report.append('** TESTING: DISCARD') self._report.write(path) def detail_line(self, **kwds): with_spaces = { k: (None if self._column_definitions.replace_by_spaces(k, v) else v) for k, v in kwds.iteritems() } self._ct.append_detail(**with_spaces) def _header(self, k, ensemble_weighting): self._report.append( 'Comparison of Errors of Ensemble and Best Model That Know the Future' ) self._report.append(' ') self._report.append('Considering Best K = %d models' % k) self._report.append('Ensemble weighting: %s' % ensemble_weighting)
class ChartBReport(object): def __init__(self, validation_month, k, column_definitions, test): self._report = Report() self._header(validation_month, k) self._column_definitions = column_definitions self._test = test cd = self._column_definitions.defs_for_columns( 'median_absolute_error', 'model', 'n_months_back', 'max_depth', 'n_estimators', 'max_features', 'learning_rate', ) self._ct = ColumnsTable(columns=cd, verbose=True) def _header(self, validation_month, k): def a(line): self._report.append(line) a('MAE for %d best-performing models and their hyperparameters' % k) a('Validation month: %s' % validation_month) a(' ') def append_detail(self, **kwds): # replace NaN with None with_spaces = { k: (None if self._column_definitions.replace_by_spaces(k, v) else v) for k, v in kwds.iteritems() } self._ct.append_detail(**with_spaces) def write(self, path): self._ct.append_legend() for line in self._ct.iterlines(): self._report.append(line) if self._test: self._report.append('**TESTING: DISCARD') self._report.write(path)
def make_chart_stats(data, control, filter_f): 'return Report with statistics for years and months that obey the filter' r = Report() r.append('Prices by Month') r.append('') ct = ColumnsTable(( ('year', 4, '%4d', (' ', ' ', 'year'), 'year of transaction'), ('month', 5, '%5d', (' ', ' ', 'month'), 'month of transaction'), ('mean_price', 6, '%6.0f', (' ', ' mean', 'price'), 'mean price in dollars'), ('median_price', 6, '%6.0f', (' ', 'median', 'price'), 'median price in dollars'), ('mean_price_ratio', 6, '%6.3f', (' mean', ' price', ' ratio'), 'ratio of price in current month to prior month'), ('median_price_ratio', 6, '%6.3f', ('median', ' price', ' ratio'), 'ratio of price in current month to prior month'), ('number_trades', 6, '%6d', ('number', 'of', 'trades'), 'number of trades in the month'), )) prior_mean_price = None prior_median_price = None for year in xrange(2003, 2010): for month in xrange(1, 13): if filter_f(year, month): value = data[make_reduction_key(year, month)] mean_price = value['mean'] median_price = value['median'] number_trades = value['count'] ct.append_detail( year=year, month=month, mean_price=mean_price, median_price=median_price, mean_price_ratio=None if prior_mean_price is None else mean_price / prior_mean_price, median_price_ratio=None if prior_median_price is None else median_price / prior_median_price, number_trades=number_trades, ) prior_mean_price = mean_price prior_median_price = median_price ct.append_legend() for line in ct.iterlines(): r.append(line) return r
def make_details(data, test_months, n_best, n_worst): 'return a ColumnTable' extra_info = [] feature_names = Features().ege_names(control.arg.features) columns_table = ColumnsTable(( ('test_month', 6, '%6s', ('test', 'month'), 'test month'), ('nth', 2, '%2d', (' ', 'n'), 'rank of feature (1 ==> more frequently included)'), ('probability', 4, '%4.1f', (' ', 'prob'), 'probability feature appears in a decision tree'), ('feature_name', 40, '%40s', (' ', 'feature name'), 'name of feature'), ), verbose=True) for test_month in test_months: value = data[ReductionKey(test_month)] if 'feature_importances' not in value.importances: # one month has an ensemble model # skip that month print 'chart a sees an unexpected ensemble model' print 'test_month', test_month print 'value', value print 'value.importance', value.importances print 'skipping the test month' print 'entering debugger' pdb.set_trace() importances = value.importances['feature_importances'] assert value.importances['features_group'] == control.arg.features, value model = value.model assert type(model) == ResultKeyGbr or type(model) == ResultKeyRfr sorted_indices = importances.argsort() # sorted first lowest, last highest for nth_best in xrange(n_best): if nth_best == len(feature_names): break index = sorted_indices[len(importances) - nth_best - 1] columns_table.append_detail( test_month=test_month, nth=nth_best + 1, probability=importances[index] * 100.0, feature_name=feature_names[index] ) extra_info.append([test_month, nth_best+1, importances[index]*100.0, feature_names[index]]) for nth in xrange(n_worst): break # skip, for now if nth == len(feature_names): break nth_worst = n_worst - nth - 1 index = sorted_indices[nth_worst] columns_table.append_detail( test_month=test_month, nth=len(importances) - nth_worst, probability=importances[index] * 100.0, feature_name=feature_names[index] ) if n_best > 1 or n_worst > 1: # insert blank line between test_months if more than 1 row in a month columns_table.append_detail() columns_table.append_legend() return columns_table, extra_info
def make_details(data, test_months, n_best, n_worst): "return a ColumnTable" feature_names = Features().ege_names(control.arg.features) columns_table = ColumnsTable( ( ("test_month", 6, "%6s", ("test", "month"), "test month"), ("nth", 2, "%2d", (" ", "n"), "rank of feature (1 ==> more frequently included)"), ("probability", 4, "%4.1f", (" ", "prob"), "probability feature appears in a decision tree"), ("feature_name", 40, "%40s", (" ", "feature name"), "name of feature"), ), verbose=True, ) for test_month in test_months: value = data[ReductionKey(test_month)] importances = value.importances["feature_importances"] assert value.importances["features_group"] == control.arg.features, value model = value.model assert type(model) == ResultKeyGbr or type(model) == ResultKeyRfr sorted_indices = importances.argsort() # sorted first lowest, last highest for nth_best in xrange(n_best): if nth_best == len(feature_names): break index = sorted_indices[len(importances) - nth_best - 1] columns_table.append_detail( test_month=test_month, nth=nth_best + 1, probability=importances[index] * 100.0, feature_name=feature_names[index], ) for nth in xrange(n_worst): break # skip, for now if nth == len(feature_names): break nth_worst = n_worst - nth - 1 index = sorted_indices[nth_worst] columns_table.append_detail( test_month=test_month, nth=len(importances) - nth_worst, probability=importances[index] * 100.0, feature_name=feature_names[index], ) if n_best > 1 or n_worst > 1: # insert blank line between test_months if more than 1 row in a month columns_table.append_detail() columns_table.append_legend() return columns_table