def _make_selection_reports_for_item_list(hp_items: list,
base_path) -> list:
result = []
for split_index, hp_item in enumerate(hp_items):
result.append({
"split_index":
split_index + 1,
"has_encoding_train_reports":
len(hp_item.encoding_train_results) > 0,
"has_encoding_test_reports":
len(hp_item.encoding_test_results) > 0,
"has_ml_reports":
len(hp_item.model_report_results) > 0,
"encoding_train_reports":
Util.to_dict_recursive(hp_item.encoding_train_results,
base_path)
if len(hp_item.encoding_train_results) > 0 else None,
"encoding_test_reports":
Util.to_dict_recursive(hp_item.encoding_test_results,
base_path)
if len(hp_item.encoding_test_results) > 0 else None,
"ml_reports":
Util.to_dict_recursive(hp_item.model_report_results, base_path)
if len(hp_item.model_report_results) > 0 else None,
})
return result if len(result) > 0 else None
def _make_selection(state: TrainMLModelState, assessment_index: int, label: str, base_path):
selection_state = state.assessment_states[assessment_index].label_states[label].selection_state
hp_settings = []
optimal = selection_state.optimal_hp_setting.get_key()
for hp_setting, hp_items in selection_state.hp_items.items():
hp_splits = []
for hp_item in hp_items:
hp_splits.append(HPHTMLBuilder._print_metric(hp_item.performance, state.optimization_metric))
hp_settings.append({
"hp_setting": hp_setting,
"hp_splits": hp_splits,
"optimal": hp_setting == optimal
})
performances = [HPHTMLBuilder._print_metric(hp_item.performance, state.optimization_metric) for hp_item in hp_items]
if len(performances) > 1:
hp_settings[-1]["average"] = round(statistics.mean(perf for perf in performances if [isinstance(perf, float)]), HPHTMLBuilder.NUM_DIGITS)
hp_settings[-1]["show_average"] = True
else:
hp_settings[-1]["average"] = None
hp_settings[-1]["show_average"] = False
has_other_metrics = len([metric for metric in state.metrics if metric != state.optimization_metric]) > 0 and \
not (state.selection.split_strategy == SplitType.RANDOM and state.selection.training_percentage == 1)
return {
"css_style": Util.get_css_content(HPHTMLBuilder.CSS_PATH),
"label": label,
"assessment_split": assessment_index + 1,
"splits": [{"split_index": i} for i in range(1, state.selection.split_count + 1)],
"split_count": state.selection.split_count,
"optimization_metric": state.optimization_metric.name.lower(),
"has_other_metrics": has_other_metrics,
"metrics": [{"performance": HPHTMLBuilder._extract_selection_performance_per_metric(selection_state, metric, state.selection.split_count),
"metric": HPHTMLBuilder._get_heading_metric_name(metric.name.lower())}
for metric in state.metrics if metric != state.optimization_metric] if has_other_metrics else None,
"hp_settings": hp_settings,
"show_average": any(hps["show_average"] for hps in hp_settings),
"data_split_reports": [
{'split_index': index + 1,
'train': Util.to_dict_recursive(selection_state.train_data_reports[index], base_path)
if len(selection_state.train_data_reports) == state.selection.split_count else None,
'test': Util.to_dict_recursive(selection_state.val_data_reports[index], base_path)
if len(selection_state.train_data_reports) == state.selection.split_count else None}
for index in range(state.selection.split_count)] if len(state.selection.reports.data_split_reports) > 0 else None,
"has_data_split_reports": len(state.selection.reports.data_split_reports) > 0,
"has_reports_per_setting": len(state.selection.reports.encoding_reports) + len(state.selection.reports.model_reports) > 0,
"reports_per_setting": [{
"hp_setting": hp_setting,
"reports": HPHTMLBuilder._make_selection_reports_for_item_list(hp_items, base_path)
} for hp_setting, hp_items in selection_state.hp_items.items()]
}
def make_html_map(state: ExploratoryAnalysisState,
base_path: Path) -> dict:
html_map = {
"css_style":
Util.get_css_content(ExploratoryAnalysisHTMLBuilder.CSS_PATH),
"full_specs":
Util.get_full_specs_path(base_path),
'immuneML_version':
MLUtil.get_immuneML_version(),
"analyses": [{
"name":
name,
"dataset_name":
analysis.dataset.name if analysis.dataset.name is not None else
analysis.dataset.identifier,
"dataset_type":
StringHelper.camel_case_to_word_string(
type(analysis.dataset).__name__),
"example_count":
analysis.dataset.get_example_count(),
"dataset_size":
f"{analysis.dataset.get_example_count()} {type(analysis.dataset).__name__.replace('Dataset', 's').lower()}",
"show_labels":
analysis.label_config is not None
and len(analysis.label_config.get_labels_by_name()) > 0,
"labels": [{
"name": label.name,
"values": str(label.values)[1:-1]
} for label in analysis.label_config.get_label_objects()]
if analysis.label_config else None,
"encoding_key":
analysis.encoder.name
if analysis.encoder is not None else None,
"encoding_name":
StringHelper.camel_case_to_word_string(
type(analysis.encoder).__name__)
if analysis.encoder is not None else None,
"encoding_params": [{
"param_name": key,
"param_value": value
} for key, value in vars(analysis.encoder).items()]
if analysis.encoder is not None else None,
"show_encoding":
analysis.encoder is not None,
"report":
Util.to_dict_recursive(analysis.report_result, base_path)
} for name, analysis in state.exploratory_analysis_units.items()]
}
for analysis in html_map["analyses"]:
analysis["show_tables"] = len(
analysis["report"]["output_tables"]
) > 0 if "output_tables" in analysis["report"] else False
analysis["show_text"] = len(
analysis["report"]["output_text"]
) > 0 if "output_text" in analysis["report"] else False
return html_map
def _make_assessment_pages(state: TrainMLModelState, base_path: Path, label: str):
assessment_list = []
for i, assessment_state in enumerate(state.assessment_states):
assessment_item = {"css_style": Util.get_css_content(HPHTMLBuilder.CSS_PATH),
"optimization_metric": state.optimization_metric.name.lower(),
"split_index": assessment_state.split_index + 1,
"hp_settings": [],
"has_reports": len(state.assessment.reports.model_reports) + len(state.assessment.reports.encoding_reports) > 0,
"train_data_reports": Util.to_dict_recursive(assessment_state.train_val_data_reports, base_path),
"test_data_reports": Util.to_dict_recursive(assessment_state.test_data_reports, base_path),
"show_data_reports": len(assessment_state.train_val_data_reports) > 0 or len(assessment_state.test_data_reports) > 0}
if hasattr(assessment_state.train_val_dataset, "metadata_file") and assessment_state.train_val_dataset.metadata_file is not None:
assessment_item["train_metadata_path"] = os.path.relpath(str(assessment_state.train_val_dataset.metadata_file), str(base_path))
assessment_item["train_metadata"] = Util.get_table_string_from_csv(assessment_state.train_val_dataset.metadata_file)
else:
assessment_item["train_metadata_path"] = None
if hasattr(assessment_state.test_dataset, "metadata_file") and assessment_state.test_dataset.metadata_file is not None:
assessment_item['test_metadata_path'] = os.path.relpath(assessment_state.test_dataset.metadata_file, base_path)
assessment_item["test_metadata"] = Util.get_table_string_from_csv(assessment_state.test_dataset.metadata_file)
else:
assessment_item["test_metadata_path"] = None
assessment_item["label"] = label
for hp_setting, item in assessment_state.label_states[label].assessment_items.items():
optimal = str(assessment_state.label_states[label].optimal_hp_setting.get_key())
reports_path = HPHTMLBuilder._make_assessment_reports(state, i, hp_setting, assessment_state, label, base_path)
assessment_item["hp_settings"].append({
"optimal": str(hp_setting) == optimal,
"hp_setting": str(hp_setting),
"optimization_metric_val": HPHTMLBuilder._print_metric(item.performance, state.optimization_metric),
"reports_path": reports_path
})
assessment_item["show_non_optimal"] = len(assessment_item["hp_settings"]) > 1
assessment_item["selection_path"] = HPHTMLBuilder._make_selection_split_path(i, label, state.name)
assessment_item['performances_per_metric'] = HPHTMLBuilder._extract_assessment_performances_per_metric(state, assessment_state, label)
assessment_list.append(assessment_item)
return assessment_list
def _make_assessment_reports(state, i, hp_setting_key, assessment_state,
label, base_path: Path):
path = base_path / f"{state.name}_{label}_{hp_setting_key}_assessment_reports_split_{i + 1}.html"
hp_item = assessment_state.label_states[label].assessment_items[
hp_setting_key]
data = {
"split_index":
i + 1,
"hp_setting":
hp_setting_key,
"label":
label,
"css_style":
Util.get_css_content(HPHTMLBuilder.CSS_PATH),
"has_encoding_reports":
len(hp_item.encoding_train_results) > 0
or len(hp_item.encoding_test_results) > 0,
"has_ml_reports":
len(hp_item.model_report_results) > 0,
"encoding_train_reports":
Util.to_dict_recursive(hp_item.encoding_train_results, base_path)
if len(hp_item.encoding_train_results) > 0 else None,
"encoding_test_reports":
Util.to_dict_recursive(hp_item.encoding_test_results, base_path)
if len(hp_item.encoding_test_results) > 0 else None,
"ml_reports":
Util.to_dict_recursive(hp_item.model_report_results, base_path)
if len(hp_item.model_report_results) > 0 else None
}
if data["has_ml_reports"] or data["has_encoding_reports"]:
TemplateParser.parse(
template_path=EnvironmentSettings.html_templates_path /
"Reports.html",
template_map=data,
result_path=path)
return path.name
else:
return None
def _make_html_map(report_results: dict, result_path: Path, instruction_result_paths: dict) -> dict:
html_map = {
"css_style": Util.get_css_content(MultiDatasetBenchmarkHTMLBuilder.CSS_PATH),
"reports": Util.to_dict_recursive(report_results.values(), result_path),
'immuneML_version': MLUtil.get_immuneML_version(),
"show_reports": True,
"instruction_overviews": [{"name": name, "path": Path(os.path.relpath(path / "index.html", result_path))}
for name, path in instruction_result_paths.items()]
}
if len(html_map['reports']) == 0:
html_map['show_reports'] = False
return html_map
def _make_main_html_map(state: TrainMLModelState, base_path: Path) -> dict:
html_map = {
"css_style":
Util.get_css_content(HPHTMLBuilder.CSS_PATH),
"full_specs":
Util.get_full_specs_path(base_path),
"dataset_name":
state.dataset.name
if state.dataset.name is not None else state.dataset.identifier,
"dataset_type":
StringHelper.camel_case_to_word_string(
type(state.dataset).__name__),
"example_count":
state.dataset.get_example_count(),
"dataset_size":
f"{state.dataset.get_example_count()} {type(state.dataset).__name__.replace('Dataset', 's').lower()}",
"labels": [{
"name": label.name,
"values": str(label.values)[1:-1]
} for label in state.label_configuration.get_label_objects()],
"optimization_metric":
state.optimization_metric.name.lower(),
"other_metrics":
str([metric.name.lower()
for metric in state.metrics])[1:-1].replace("'", ""),
"metrics": [{
"name": metric.name.lower()
} for metric in state.metrics],
"assessment_desc":
state.assessment,
"selection_desc":
state.selection,
"show_hp_reports":
bool(state.report_results),
'hp_reports':
Util.to_dict_recursive(state.report_results, base_path)
if state.report_results else None,
"hp_per_label":
HPHTMLBuilder._make_hp_per_label(state),
'models_per_label':
HPHTMLBuilder._make_model_per_label(state, base_path),
'immuneML_version':
MLUtil.get_immuneML_version()
}
return html_map
def make_html_map(state: SimulationState, base_path: Path) -> dict:
html_map = {
"css_style":
Util.get_css_content(SimulationHTMLBuilder.CSS_PATH),
"name":
state.name,
'immuneML_version':
MLUtil.get_immuneML_version(),
"full_specs":
Util.get_full_specs_path(base_path),
"dataset_name":
state.resulting_dataset.name if state.resulting_dataset.name
is not None else state.resulting_dataset.identifier,
"dataset_type":
StringHelper.camel_case_to_word_string(
type(state.resulting_dataset).__name__),
"example_count":
state.resulting_dataset.get_example_count(),
"dataset_size":
f"{state.resulting_dataset.get_example_count()} {type(state.resulting_dataset).__name__.replace('Dataset', 's').lower()}",
"labels": [{
"label_name": label
} for label in state.resulting_dataset.get_label_names()],
"formats": [{
"format_name":
format_name,
"dataset_download_link":
os.path.relpath(path=Util.make_downloadable_zip(
state.result_path,
state.paths[state.resulting_dataset.name][format_name]),
start=base_path)
} for format_name in state.formats],
"implantings": [
Util.to_dict_recursive(implanting, base_path)
for implanting in state.simulation.implantings
]
}
return html_map