def get_experiment_stats(results, gpu, niter):
stats = {}
if os.path.isdir(results):
tracks = json_from_logs(results)
else:
tracks = read_results(results)
for experiment_name in tracks.iterkeys():
stats[experiment_name] = {}
experiment_tracks = tracks[experiment_name]
experiment_tracks = dict(
filter(lambda track: gpu == ('GPU' in track[0]),
experiment_tracks.items()))
for algorithm_name in experiment_tracks.iterkeys():
stats[experiment_name][algorithm_name] = {}
table_tracks = split_tracks(experiment_tracks[algorithm_name])
for params, cur_tracks in table_tracks.iteritems():
stat = calculate_statistics(cur_tracks, niter)
if stat == {}:
continue
stats[experiment_name][algorithm_name][params] = stat
stats = dict(
filter(lambda experiment_stat: len(experiment_stat[1]) > 0,
stats.items()))
return stats
def main():
plot_functions = {
'time-per-iter': plot_time_per_iter,
'best': plot_quality,
'quality-vs-time': plot_quality_vs_time,
'custom': plot_quality
}
parser = argparse.ArgumentParser()
parser.add_argument('--type', choices=plot_functions.keys(), required=True)
parser.add_argument('--only', nargs='+', choices=ONLY_TYPES.keys(), required=False)
parser.add_argument('-i', '--results-file', required=True)
parser.add_argument('-t', '--title')
parser.add_argument('-f', '--fig-size', nargs=2, type=int, default=FIGURE_SIZE)
parser.add_argument('-o', '--out-dir', default='plots')
parser.add_argument('--params-cases', help='draw plots only with those params (tracks filtering)'
' path to json file, each line corresponds to learner '
'parameter (e.g. max_depth) and list of its values')
parser.add_argument('--from-iter', type=int, default=0, help='only custom, best modes')
parser.add_argument('--to-iter', type=int, default=None, help='only custom, best modes')
parser.add_argument('--low-percent', type=float, default=0.9, help='only quality-vs-time mode')
parser.add_argument('--num-bins', type=int, default=200, help='only quality-vs-time mode')
parser.add_argument('--only-min', action='store_true', help='only quality-vs-time mode')
parser.add_argument('--top', type=int, default=3, help='only best mode')
args = parser.parse_args()
tracks = read_results(args.results_file)
for experiment_name in tracks.keys():
plot_experiment(tracks[experiment_name], experiment_name, args)
def print_n_experiment_duration(results, gpu, n, output):
niter = 5000
if os.path.isdir(results):
tracks = json_from_logs(results)
else:
tracks = read_results(results)
table = []
index = []
for experiment_name in tracks.iterkeys():
experiment_tracks = tracks[experiment_name]
experiment_tracks = dict(
filter(lambda track_: gpu == ('GPU' in track_[0]),
experiment_tracks.items()))
row = []
for algorithm_name in sorted(experiment_tracks.iterkeys()):
value = 0.
if len(experiment_tracks[algorithm_name]) < 2:
continue
for track in experiment_tracks[algorithm_name]:
value += (np.median(track.time_per_iter) *
niter) / 60. # minutes
row.append(value / 60. /
float(len(experiment_tracks[algorithm_name])) *
n) # hours
if len(row) != 0:
index.append(experiment_name)
table.append(row)
header = ['catboost', 'lightgbm', 'xgboost']
table = pd.DataFrame(table, index=index, columns=header)
with open(output, 'w') as f:
f.write('Optimization time, hours')
f.write('\n')
f.write(table.to_string())
f.write('\n')
def get_experiment_stats(results_file, gpu, niter):
stats = {}
tracks = read_results(results_file)
for experiment_name in tracks:
stats[experiment_name] = {}
experiment_tracks = tracks[experiment_name]
experiment_tracks = dict(
filter(lambda track: gpu == ('GPU' in track[0]),
experiment_tracks.items()))
for algorithm_name in experiment_tracks:
stats[experiment_name][algorithm_name] = {}
table_tracks = split_tracks(experiment_tracks[algorithm_name])
for params, cur_tracks in table_tracks.items():
stat = calculate_statistics(cur_tracks, niter)
if stat == {}:
continue
stats[experiment_name][algorithm_name][params] = stat
return stats
def get_experiment_stats(results_directory, gpu, niter):
EXPERIMENT_NAMES = EXPERIMENT_TYPE.keys()
stats = {}
for experiment_name in os.listdir(results_directory):
if experiment_name not in EXPERIMENT_NAMES:
continue
stats[experiment_name] = {}
tracks = read_results(os.path.join(results_directory, experiment_name))
tracks = dict(filter(lambda track: gpu == ('GPU' in track[0]), tracks.items()))
for algorithm_name in tracks.iterkeys():
stats[experiment_name][algorithm_name] = {}
table_tracks = split_tracks(tracks[algorithm_name])
for params, cur_tracks in table_tracks.iteritems():
stat = calculate_statistics(cur_tracks, niter)
if stat == {}:
continue
stats[experiment_name][algorithm_name][params] = stat
return stats
def main():
plot_functions = {
'time-per-iter': plot_time_per_iter,
'best': plot_quality,
'quality-vs-time': plot_quality_vs_time,
'custom': plot_quality
}
parser = argparse.ArgumentParser()
parser.add_argument('--type', choices=plot_functions.keys(), required=True)
parser.add_argument('--only', nargs='+', choices=ONLY_TYPES.keys(), required=False)
parser.add_argument('-i', '--results-dir', required=True)
parser.add_argument('-t', '--title')
parser.add_argument('-f', '--fig-size', nargs=2, type=int, default=FIGURE_SIZE)
parser.add_argument('-o', '--out-dir', default='plots')
parser.add_argument('-d', '--file-name', required=False)
parser.add_argument('--params-cases', help='draw plots only with those params (tracks filtering)'
' path to json file, each line corresponds to learner '
'parameter (e.g. max_depth) and list of its values')
parser.add_argument('--from-iter', type=int, default=0, help='only custom, best modes')
parser.add_argument('--to-iter', type=int, default=None, help='only custom, best modes')
parser.add_argument('--low-percent', type=float, default=0.9, help='only quality-vs-time mode')
parser.add_argument('--num-bins', type=int, default=200, help='only quality-vs-time mode')
parser.add_argument('--only-min', action='store_true', help='only quality-vs-time mode')
parser.add_argument('--top', type=int, default=3, help='only best mode')
args = parser.parse_args()
tracks = read_results(args.results_dir)
file_name = args.file_name if args.file_name else get_default_file_name(args.type, args.params_cases)
save_path = os.path.join(args.out_dir, file_name)
if args.only:
filtered_tracks = {}
for only_type in args.only:
for alg_name in ONLY_TYPES[only_type]:
filtered_tracks[alg_name] = tracks[alg_name]
tracks = filtered_tracks
if args.params_cases:
with open(args.params_cases) as f:
params_cases = json.load(f)
tracks = filter_tracks(tracks, params_cases)
if args.type == 'quality-vs-time':
best_tracks = get_best(tracks)
best_quality = min(map(lambda tracks: tracks[0].get_best_score(), best_tracks.values()))
print(best_quality)
if args.top:
tracks = get_best(tracks, top=args.top)
plot_quality_vs_time(tracks, best_quality=best_quality, low_percent=args.low_percent, only_min=args.only_min,
figsize=args.fig_size, num_bins=args.num_bins, save_path=save_path)
if args.type == 'best':
best_tracks = get_best(tracks, top=args.top)
for alg in best_tracks.keys():
for track in best_tracks[alg]:
print(track)
print(track.get_best_score())
plot_quality(best_tracks, args.from_iter, args.to_iter, figsize=args.fig_size,
title=args.title, save_path=save_path)
if args.type == 'custom':
plot_quality(tracks, args.from_iter, args.to_iter,
figsize=args.fig_size, title=args.title, save_path=save_path)
if args.type == 'time-per-iter':
plot_time_per_iter(tracks, figsize=args.fig_size, title=args.title, save_path=save_path)