def aggregate_scores(options, best_score_key):
runs = []
best_run_name = None
best_score = -float('inf')
for run_name in options.aggregate_run_names:
run_path = join(results_path, run_name)
scores_path = join(run_path, 'scores.json')
with open(scores_path) as scores_file:
scores_dict = json.load(scores_file)
score = scores_dict[best_score_key]
if score > best_score:
best_score = score
best_run_name = run_name
runs.append({
'run_name': run_name,
'scores': scores_dict
})
agg_scores = {
'best_run_name': best_run_name,
'best_score': best_score,
'runs': runs
}
scores_path = join(results_path, options.run_name, 'scores.json')
save_json(agg_scores, scores_path)
def setup_run(self):
"""Setup path for the results of a run.
Creates directory if doesn't exist, downloads results from cloud, and
write the options to <run_path>/options.json
"""
if not isdir(self.run_path):
self.sync_results(download=True)
_makedirs(self.run_path)
options_path = join(self.run_path, 'options.json')
save_json(self.options.__dict__, options_path)
def save_thresholds(run_path, thresholds):
save_json(thresholds.tolist(), join(run_path, 'thresholds.json'))
def write_channel_stats(self, datasets_path):
means, stds = self.compute_channel_stats(1000, False)
param_dict = {'means': means.tolist(), 'stds': stds.tolist()}
param_path = join(datasets_path, self.name + '_channel_stats.json')
save_json(param_dict, param_path)
def preprocess(datasets_path):
dataset_path = join(datasets_path, PLANET_KAGGLE)
tags_path = join(dataset_path, 'train_v2.csv')
tag_store = TagStore(tags_path=tags_path)
counts_path = join(dataset_path, 'tag_counts.json')
save_json(tag_store.get_tag_counts(), counts_path)