def collect_results(config, destination, experiment_name=None,
use_integration=False):
print(utils.colored("Collecting Results"))
if use_integration:
config = INT_CONFIG_PATH
if experiment_name is None:
experiments = hcnn.driver.Driver.available_experiments(config)
else:
experiments = [experiment_name]
results = []
for experiment_name in experiments:
if not use_integration and "integrationtest" in experiment_name:
continue
elif (use_integration and
"integrationtest" not in experiment_name):
continue
print("Collecting experiment", utils.colored(experiment_name, 'cyan'))
driver = hcnn.driver.Driver(config, experiment_name=experiment_name,
load_features=False,
skip_load_dataset=True)
results.append(driver.collect_results(destination))
return all(results)
def test_collect_results(config_with_workspace, pre_existing_experiment,
available_datasets, workspace):
driver = hcnn.driver.Driver(config_with_workspace,
experiment_name=pre_existing_experiment,
load_features=False,
skip_load_dataset=True)
destination_dir = os.path.join(workspace, "results")
result = driver.collect_results(destination_dir)
assert result is True
new_experiment_dir = os.path.join(destination_dir, pre_existing_experiment)
assert os.path.isdir(new_experiment_dir)
for dataset in available_datasets:
dataset_results = os.path.join(new_experiment_dir, dataset)
assert os.path.isdir(dataset_results)
training_loss_fp = os.path.join(dataset_results, "training_loss.pkl")
assert os.path.isfile(training_loss_fp)
training_loss_df = pd.read_pickle(training_loss_fp)
assert [x in training_loss_df.columns for x in ['iteration', 'loss']]
validation_loss_fp = os.path.join(dataset_results,
"validation_loss.pkl")
assert os.path.isfile(validation_loss_fp)
validation_loss_df = pd.read_pickle(validation_loss_fp)
assert [x in validation_loss_df.columns
for x in ['mean_acc', 'mean_loss', 'model_file',
'model_iteration']]
prediction_glob = os.path.join(dataset_results, "*predictions.pkl")
assert len(prediction_glob) > 0
prediction_file = glob.glob(prediction_glob)[0]
prediction_df = pd.read_pickle(prediction_file)
assert [x in prediction_df.columns for x in ['y_pred', 'y_true']]
# Finally, collect_results should create an overall analysis of the
# three-fold validation, and put it in
overall_results_fp = os.path.join(
new_experiment_dir, "experiment_results.json")
assert os.path.isfile(overall_results_fp)
with open(overall_results_fp, 'r') as fh:
result_data = json.load(fh)
for dataset in available_datasets:
assert dataset in result_data
assert 'mean_accuracy' in result_data[dataset]
assert 'mean_precision' in result_data[dataset]
assert 'mean_recall' in result_data[dataset]
assert 'mean_f1' in result_data[dataset]
assert 'class_precision' in result_data[dataset]
assert 'class_recall' in result_data[dataset]
assert 'class_f1' in result_data[dataset]
assert 'sample_weight' in result_data[dataset]
