def save_dataframe_interval(coverage, experiments, file, objs, resolution, save, sharpness, synthetic, times, q05, q25, q75, q95, steps, method): ret = [] if synthetic: for k in sorted(objs.keys()): mod = [] mfts = objs[k] mod.append(mfts.shortname) mod.append(mfts.order) l = len(mfts) if not mfts.benchmark_only: mod.append(mfts.partitioner.name) mod.append(mfts.partitioner.partitions) mod.append(l) else: mod.append('-') mod.append('-') mod.append('-') mod.append(steps[k]) mod.append(method[k]) mod.append(round(np.nanmean(sharpness[k]), 2)) mod.append(round(np.nanstd(sharpness[k]), 2)) mod.append(round(np.nanmean(resolution[k]), 2)) mod.append(round(np.nanstd(resolution[k]), 2)) mod.append(round(np.nanmean(coverage[k]), 2)) mod.append(round(np.nanstd(coverage[k]), 2)) mod.append(round(np.nanmean(times[k]), 2)) mod.append(round(np.nanstd(times[k]), 2)) mod.append(round(np.nanmean(q05[k]), 2)) mod.append(round(np.nanstd(q05[k]), 2)) mod.append(round(np.nanmean(q25[k]), 2)) mod.append(round(np.nanstd(q25[k]), 2)) mod.append(round(np.nanmean(q75[k]), 2)) mod.append(round(np.nanstd(q75[k]), 2)) mod.append(round(np.nanmean(q95[k]), 2)) mod.append(round(np.nanstd(q95[k]), 2)) mod.append(l) ret.append(mod) columns = interval_dataframe_synthetic_columns() else: for k in sorted(objs.keys()): try: mfts = objs[k] n = mfts.shortname o = mfts.order if not mfts.benchmark_only: s = mfts.partitioner.name p = mfts.partitioner.partitions l = len(mfts) else: s = '-' p = '-' l = '-' st = steps[k] mt = method[k] tmp = [n, o, s, p, l, st, mt, 'Sharpness'] tmp.extend(sharpness[k]) ret.append(deepcopy(tmp)) tmp = [n, o, s, p, l, st, mt, 'Resolution'] tmp.extend(resolution[k]) ret.append(deepcopy(tmp)) tmp = [n, o, s, p, l, st, mt, 'Coverage'] tmp.extend(coverage[k]) ret.append(deepcopy(tmp)) tmp = [n, o, s, p, l, st, mt, 'TIME'] tmp.extend(times[k]) ret.append(deepcopy(tmp)) tmp = [n, o, s, p, l, st, mt, 'Q05'] tmp.extend(q05[k]) ret.append(deepcopy(tmp)) tmp = [n, o, s, p, l, st, mt, 'Q25'] tmp.extend(q25[k]) ret.append(deepcopy(tmp)) tmp = [n, o, s, p, l, st, mt, 'Q75'] tmp.extend(q75[k]) ret.append(deepcopy(tmp)) tmp = [n, o, s, p, l, st, mt, 'Q95'] tmp.extend(q95[k]) ret.append(deepcopy(tmp)) except Exception as ex: print("Erro ao salvar ", k) print("Exceção ", ex) columns = interval_dataframe_analytic_columns(experiments) dat = pd.DataFrame(ret, columns=columns) if save: dat.to_csv(Util.uniquefilename(file), sep=";") return dat
def plot_dataframe_interval( file_synthetic, file_analytic, experiments, tam, save=False, file=None, sort_columns=['COVAVG', 'SHARPAVG', 'COVSTD', 'SHARPSTD'], sort_ascend=[True, False, True, True], save_best=False, ignore=None, replace=None): fig, axes = plt.subplots(nrows=3, ncols=1, figsize=tam) axes[0].set_title('Sharpness') axes[1].set_title('Resolution') axes[2].set_title('Coverage') dat_syn = pd.read_csv(file_synthetic, sep=";", usecols=interval_dataframe_synthetic_columns()) bests = find_best(dat_syn, sort_columns, sort_ascend) dat_ana = pd.read_csv( file_analytic, sep=";", usecols=interval_dataframe_analytic_columns(experiments)) data_columns = analytical_data_columns(experiments) if save_best: dat = pd.DataFrame.from_dict(bests, orient='index') dat.to_csv(Util.uniquefilename( file_synthetic.replace("synthetic", "best")), sep=";", index=False) sharpness = [] resolution = [] coverage = [] times = [] labels = [] bounds_shp = [] for b in sorted(bests.keys()): if check_ignore_list(b, ignore): continue best = bests[b] df = dat_ana[(dat_ana.Model == best["Model"]) & (dat_ana.Order == best["Order"]) & (dat_ana.Scheme == best["Scheme"]) & (dat_ana.Partitions == best["Partitions"])] sharpness.append(extract_measure(df, 'Sharpness', data_columns)) resolution.append(extract_measure(df, 'Resolution', data_columns)) coverage.append(extract_measure(df, 'Coverage', data_columns)) times.append(extract_measure(df, 'TIME', data_columns)) labels.append( check_replace_list(best["Model"] + " " + str(best["Order"]), replace)) axes[0].boxplot(sharpness, labels=labels, autorange=True, showmeans=True) axes[0].set_title("Sharpness") axes[1].boxplot(resolution, labels=labels, autorange=True, showmeans=True) axes[1].set_title("Resolution") axes[2].boxplot(coverage, labels=labels, autorange=True, showmeans=True) axes[2].set_title("Coverage") axes[2].set_ylim([0, 1.1]) plt.tight_layout() Util.show_and_save_image(fig, file, save)
def plot_dataframe_point(file_synthetic, file_analytic, experiments, tam, save=False, file=None, sort_columns=['UAVG', 'RMSEAVG', 'USTD', 'RMSESTD'], sort_ascend=[1, 1, 1, 1], save_best=False, ignore=None, replace=None): fig, axes = plt.subplots(nrows=3, ncols=1, figsize=tam) axes[0].set_title('RMSE') axes[1].set_title('SMAPE') axes[2].set_title('U Statistic') dat_syn = pd.read_csv(file_synthetic, sep=";", usecols=point_dataframe_synthetic_columns()) bests = find_best(dat_syn, sort_columns, sort_ascend) dat_ana = pd.read_csv( file_analytic, sep=";", usecols=point_dataframe_analytic_columns(experiments)) data_columns = analytical_data_columns(experiments) if save_best: dat = pd.DataFrame.from_dict(bests, orient='index') dat.to_csv(Util.uniquefilename( file_synthetic.replace("synthetic", "best")), sep=";", index=False) rmse = [] smape = [] u = [] times = [] labels = [] for b in sorted(bests.keys()): if check_ignore_list(b, ignore): continue best = bests[b] tmp = dat_ana[(dat_ana.Model == best["Model"]) & (dat_ana.Order == best["Order"]) & (dat_ana.Scheme == best["Scheme"]) & (dat_ana.Partitions == best["Partitions"])] rmse.append(extract_measure(tmp, 'RMSE', data_columns)) smape.append(extract_measure(tmp, 'SMAPE', data_columns)) u.append(extract_measure(tmp, 'U', data_columns)) times.append(extract_measure(tmp, 'TIME', data_columns)) labels.append( check_replace_list(best["Model"] + " " + str(best["Order"]), replace)) axes[0].boxplot(rmse, labels=labels, autorange=True, showmeans=True) axes[0].set_title("RMSE") axes[1].boxplot(smape, labels=labels, autorange=True, showmeans=True) axes[1].set_title("SMAPE") axes[2].boxplot(u, labels=labels, autorange=True, showmeans=True) axes[2].set_title("U Statistic") plt.tight_layout() Util.show_and_save_image(fig, file, save)
def save_dataframe_point(experiments, file, objs, rmse, save, synthetic, smape, times, u, steps, method): """ Create a dataframe to store the benchmark results :param experiments: dictionary with the execution results :param file: :param objs: :param rmse: :param save: :param synthetic: :param smape: :param times: :param u: :return: """ ret = [] if synthetic: for k in sorted(objs.keys()): try: mod = [] mfts = objs[k] mod.append(mfts.shortname) mod.append(mfts.order) if not mfts.benchmark_only: mod.append(mfts.partitioner.name) mod.append(mfts.partitioner.partitions) mod.append(len(mfts)) else: mod.append('-') mod.append('-') mod.append('-') mod.append(steps[k]) mod.append(method[k]) mod.append(np.round(np.nanmean(rmse[k]), 2)) mod.append(np.round(np.nanstd(rmse[k]), 2)) mod.append(np.round(np.nanmean(smape[k]), 2)) mod.append(np.round(np.nanstd(smape[k]), 2)) mod.append(np.round(np.nanmean(u[k]), 2)) mod.append(np.round(np.nanstd(u[k]), 2)) mod.append(np.round(np.nanmean(times[k]), 4)) mod.append(np.round(np.nanstd(times[k]), 4)) ret.append(mod) except Exception as ex: print("Erro ao salvar ", k) print("Exceção ", ex) columns = point_dataframe_synthetic_columns() else: for k in sorted(objs.keys()): try: mfts = objs[k] n = mfts.shortname o = mfts.order if not mfts.benchmark_only: s = mfts.partitioner.name p = mfts.partitioner.partitions l = len(mfts) else: s = '-' p = '-' l = '-' st = steps[k] mt = method[k] tmp = [n, o, s, p, l, st, mt, 'RMSE'] tmp.extend(rmse[k]) ret.append(deepcopy(tmp)) tmp = [n, o, s, p, l, st, mt, 'SMAPE'] tmp.extend(smape[k]) ret.append(deepcopy(tmp)) tmp = [n, o, s, p, l, st, mt, 'U'] tmp.extend(u[k]) ret.append(deepcopy(tmp)) tmp = [n, o, s, p, l, st, mt, 'TIME'] tmp.extend(times[k]) ret.append(deepcopy(tmp)) except Exception as ex: print("Erro ao salvar ", k) print("Exceção ", ex) columns = point_dataframe_analytic_columns(experiments) try: dat = pd.DataFrame(ret, columns=columns) if save: dat.to_csv(Util.uniquefilename(file), sep=";", index=False) return dat except Exception as ex: print(ex) print(experiments) print(columns) print(ret)
def plot_dataframe_probabilistic( file_synthetic, file_analytic, experiments, tam, save=False, file=None, sort_columns=['CRPS1AVG', 'CRPS2AVG', 'CRPS1STD', 'CRPS2STD'], sort_ascend=[True, True, True, True], save_best=False, ignore=None, replace=None): fig, axes = plt.subplots(nrows=2, ncols=1, figsize=tam) axes[0].set_title('CRPS') axes[1].set_title('CRPS') dat_syn = pd.read_csv(file_synthetic, sep=";", usecols=probabilistic_dataframe_synthetic_columns()) bests = find_best(dat_syn, sort_columns, sort_ascend) dat_ana = pd.read_csv( file_analytic, sep=";", usecols=probabilistic_dataframe_analytic_columns(experiments)) data_columns = analytical_data_columns(experiments) if save_best: dat = pd.DataFrame.from_dict(bests, orient='index') dat.to_csv(Util.uniquefilename( file_synthetic.replace("synthetic", "best")), sep=";", index=False) crps1 = [] crps2 = [] labels = [] for b in sorted(bests.keys()): if check_ignore_list(b, ignore): continue best = bests[b] df = dat_ana[(dat_ana.Model == best["Model"]) & (dat_ana.Order == best["Order"]) & (dat_ana.Scheme == best["Scheme"]) & (dat_ana.Partitions == best["Partitions"])] crps1.append(extract_measure(df, 'CRPS_Interval', data_columns)) crps2.append(extract_measure(df, 'CRPS_Distribution', data_columns)) labels.append( check_replace_list(best["Model"] + " " + str(best["Order"]), replace)) axes[0].boxplot(crps1, labels=labels, autorange=True, showmeans=True) axes[1].boxplot(crps2, labels=labels, autorange=True, showmeans=True) plt.tight_layout() Util.show_and_save_image(fig, file, save)
def save_dataframe_probabilistic(experiments, file, objs, crps, times, save, synthetic, steps, method): """ Save benchmark results for m-step ahead probabilistic forecasters :param experiments: :param file: :param objs: :param crps_interval: :param crps_distr: :param times: :param times2: :param save: :param synthetic: :return: """ ret = [] if synthetic: for k in sorted(objs.keys()): try: ret = [] for k in sorted(objs.keys()): try: mod = [] mfts = objs[k] mod.append(mfts.shortname) mod.append(mfts.order) if not mfts.benchmark_only: mod.append(mfts.partitioner.name) mod.append(mfts.partitioner.partitions) mod.append(len(mfts)) else: mod.append('-') mod.append('-') mod.append('-') mod.append(steps[k]) mod.append(method[k]) mod.append(np.round(np.nanmean(crps[k]), 2)) mod.append(np.round(np.nanstd(crps[k]), 2)) mod.append(np.round(np.nanmean(times[k]), 4)) mod.append(np.round(np.nanstd(times[k]), 4)) ret.append(mod) except Exception as e: print('Erro: %s' % e) except Exception as ex: print("Erro ao salvar ", k) print("Exceção ", ex) columns = probabilistic_dataframe_synthetic_columns() else: for k in sorted(objs.keys()): try: mfts = objs[k] n = mfts.shortname o = mfts.order if not mfts.benchmark_only: s = mfts.partitioner.name p = mfts.partitioner.partitions l = len(mfts) else: s = '-' p = '-' l = '-' st = steps[k] mt = method[k] tmp = [n, o, s, p, l, st, mt, 'CRPS'] tmp.extend(crps[k]) ret.append(deepcopy(tmp)) tmp = [n, o, s, p, l, st, mt, 'TIME'] tmp.extend(times[k]) ret.append(deepcopy(tmp)) except Exception as ex: print("Erro ao salvar ", k) print("Exceção ", ex) columns = probabilistic_dataframe_analytic_columns(experiments) dat = pd.DataFrame(ret, columns=columns) if save: dat.to_csv(Util.uniquefilename(file), sep=";") return dat
def plot_dataframe_interval_pinball( file_synthetic, file_analytic, experiments, tam, save=False, file=None, sort_columns=['COVAVG', 'SHARPAVG', 'COVSTD', 'SHARPSTD'], sort_ascend=[True, False, True, True], save_best=False, ignore=None, replace=None): fig, axes = plt.subplots(nrows=1, ncols=4, figsize=tam) axes[0].set_title(r'$\tau=0.05$') axes[1].set_title(r'$\tau=0.25$') axes[2].set_title(r'$\tau=0.75$') axes[3].set_title(r'$\tau=0.95$') dat_syn = pd.read_csv(file_synthetic, sep=";", usecols=interval_dataframe_synthetic_columns()) bests = find_best(dat_syn, sort_columns, sort_ascend) dat_ana = pd.read_csv( file_analytic, sep=";", usecols=interval_dataframe_analytic_columns(experiments)) data_columns = analytical_data_columns(experiments) if save_best: dat = pd.DataFrame.from_dict(bests, orient='index') dat.to_csv(Util.uniquefilename( file_synthetic.replace("synthetic", "best")), sep=";", index=False) q05 = [] q25 = [] q75 = [] q95 = [] labels = [] for b in sorted(bests.keys()): if check_ignore_list(b, ignore): continue best = bests[b] df = dat_ana[(dat_ana.Model == best["Model"]) & (dat_ana.Order == best["Order"]) & (dat_ana.Scheme == best["Scheme"]) & (dat_ana.Partitions == best["Partitions"])] q05.append(extract_measure(df, 'Q05', data_columns)) q25.append(extract_measure(df, 'Q25', data_columns)) q75.append(extract_measure(df, 'Q75', data_columns)) q95.append(extract_measure(df, 'Q95', data_columns)) labels.append( check_replace_list(best["Model"] + " " + str(best["Order"]), replace)) axes[0].boxplot(q05, labels=labels, vert=False, autorange=True, showmeans=True) axes[1].boxplot(q25, labels=labels, vert=False, autorange=True, showmeans=True) axes[2].boxplot(q75, labels=labels, vert=False, autorange=True, showmeans=True) axes[3].boxplot(q95, labels=labels, vert=False, autorange=True, showmeans=True) plt.tight_layout() Util.show_and_save_image(fig, file, save)