def _save_report(
self, models_specs: List[ClassificationModelSpec], tags: List[str],
output_directory: Union[str, Path],
n_true_bboxes_data: List[List[BboxData]], markdowns: List[str],
codes: List[str],
classifications_reports_datas: List[ClassificationReportData]):
output_directory = Path(output_directory)
output_directory.mkdir(exist_ok=True, parents=True)
for model_spec, tag in zip(models_specs, tags):
if model_spec is None:
continue
classification_model_spec_filepath = output_directory / f"{CLASSIFICATION_MODEL_SPEC_PREFIX}_{tag}.pkl"
with open(classification_model_spec_filepath, 'wb') as out:
pickle.dump(model_spec, out)
bboxes_data_filepath = output_directory / BBOXES_DATA_FILENAME
with open(bboxes_data_filepath, 'wb') as out:
pickle.dump(n_true_bboxes_data, out)
classification_reports_data_filepath = output_directory / CLASSIFICATIONS_REPORTS_DATAS_FILENAME
with open(classification_reports_data_filepath, "wb") as out:
pickle.dump(classifications_reports_datas, out)
nb = nbf.v4.new_notebook()
nb['cells'] = [
nbf.v4.new_markdown_cell(markdown) for markdown in markdowns
]
nb['cells'].extend([nbf.v4.new_code_cell(code) for code in codes])
nbf.write(nb, str(output_directory / 'report.ipynb'))
logger.info(f"Classification report saved to '{output_directory}'.")
def _save_report(
self,
models_specs: List[DetectionModelSpec],
tags: List[str],
output_directory: Union[str, Path],
true_images_data: List[ImageData],
markdowns: List[str],
codes: List[str],
):
output_directory = Path(output_directory)
output_directory.mkdir(exist_ok=True, parents=True)
for model_spec, tag in zip(models_specs, tags):
if model_spec is None:
continue
detection_model_spec_filepath = output_directory / f"{DETECTION_MODEL_SPEC_PREFIX}_{tag}.pkl"
with open(detection_model_spec_filepath, 'wb') as out:
pickle.dump(model_spec, out)
images_data_filepath = output_directory / IMAGES_DATA_FILENAME
with open(images_data_filepath, 'wb') as out:
pickle.dump(true_images_data, out)
nb = nbf.v4.new_notebook()
nb['cells'] = [
nbf.v4.new_markdown_cell(markdown) for markdown in markdowns
]
nb['cells'].extend([nbf.v4.new_code_cell(code) for code in codes])
nbf.write(nb, str(output_directory / 'report.ipynb'))
logger.info(f"Detection report saved to '{output_directory}'.")
def report(self,
models_specs: List[PipelineModelSpec],
tags: List[str],
detection_scores_thresholds: List[float],
extra_bbox_labels: List[str],
compare_tag: str,
output_directory: Union[str, Path],
true_images_data: List[ImageData],
minimum_iou: float,
pseudo_class_names: List[str],
batch_size: int = 16):
assert len(models_specs) == len(tags)
assert len(tags) == len(detection_scores_thresholds)
assert len(detection_scores_thresholds) == len(extra_bbox_labels)
assert compare_tag in tags
pipelines_reports_datas = []
for model_spec, tag, detection_score_threshold, extra_bbox_label in zip(
models_specs, tags, detection_scores_thresholds,
extra_bbox_labels):
logger.info(
f"Making inference and counting metrics for '{tag}'...")
tag_df_detection_metrics, tag_df_pipeline_metrics = self._inference_pipeline_and_get_metrics(
model_spec=model_spec,
true_images_data=true_images_data,
detection_score_threshold=detection_score_threshold,
minimum_iou=minimum_iou,
extra_bbox_label=extra_bbox_label,
pseudo_class_names=pseudo_class_names,
batch_size=batch_size,
)
pipelines_reports_datas.append(
PipelineReportData(
df_detection_metrics=tag_df_detection_metrics,
df_pipeline_metrics=tag_df_pipeline_metrics,
tag=tag))
pipeline_report_data = concat_pipelines_reports_datas(
pipelines_reports_datas=pipelines_reports_datas,
compare_tag=compare_tag)
markdowns = self._get_markdowns(
pipeline_report_data=pipeline_report_data, tags=tags)
codes = self._get_codes(
tags=tags,
detection_scores_thresholds=detection_scores_thresholds,
extra_bbox_labels=extra_bbox_labels,
minimum_iou=minimum_iou,
)
self._save_report(models_specs=models_specs,
tags=tags,
output_directory=output_directory,
true_images_data=true_images_data,
markdowns=markdowns,
codes=codes)
def get_label_to_base_label_image(
base_labels_images: Union[str, Path],
label_to_description: Union[str, Path, Dict[str, str]] = None,
add_label_to_image: bool = False,
make_labels_for_these_class_names_too: List[str] = [
] # add known description to classes without base images
) -> Dict[str, np.ndarray]:
if base_labels_images is None:
return None
base_labels_images_files = fsspec.open_files(str(base_labels_images))
ann_class_names_files = [
Pathy(base_label_image_file.path).stem
for base_label_image_file in base_labels_images_files
]
unique_ann_class_names = set(ann_class_names_files)
if 'unknown' not in unique_ann_class_names:
raise ValueError(
f'"{base_labels_images}" must have image with name "unknown.*"')
unknown_image_path = base_labels_images_files[ann_class_names_files.index(
'unknown')]
label_to_base_label_image = defaultdict(lambda: unknown_image_path)
label_to_base_label_image['unknown'] = unknown_image_path
logger.info(f"Loading base labels images from {base_labels_images}...")
for label in tqdm(
list(unique_ann_class_names) +
list(set(make_labels_for_these_class_names_too))):
if label in unique_ann_class_names:
base_label_image = base_labels_images_files[
ann_class_names_files.index(label)]
else:
base_label_image = label_to_base_label_image['unknown']
label_to_base_label_image[label] = base_label_image
def label_to_base_label_image_func(
label: str,
label_to_description: Union[str, Path,
Dict[str, str]] = label_to_description,
add_label_to_image: bool = add_label_to_image):
base_label_image = open_image(label_to_base_label_image[label])
if label_to_description is not None:
if isinstance(label_to_description, str) or isinstance(
label_to_description, Path):
label_to_description = get_label_to_description(
label_to_description_dict=label_to_description)
base_label_image = get_base_label_image_with_description(
base_label_image=base_label_image,
label=label,
description=label_to_description[label])
elif add_label_to_image:
base_label_image = get_base_label_image_with_description(
base_label_image=base_label_image, label=label, description='')
return base_label_image
return label_to_base_label_image_func
def report(self,
models_specs: List[ClassificationModelSpec],
tags: List[str],
compare_tag: str,
output_directory: Union[str, Path],
n_true_bboxes_data: List[List[BboxData]],
pseudo_class_names: List[str],
tops_n: List[int] = [1],
batch_size: int = 16,
step_penalty: int = 20) -> List[ClassificationReportData]:
for model_spec in models_specs:
if hasattr(model_spec, 'preprocess_input'):
assert (isinstance(model_spec.preprocess_input, str)
or isinstance(model_spec.preprocess_input, Path))
assert len(models_specs) == len(tags)
assert compare_tag in tags
classifications_reports_datas = []
df_classification_metrics_columns = None
for model_spec, tag in zip(models_specs, tags):
logger.info(
f"Making inference and counting metrics for '{tag}'...")
tag_df_classification_metrics = self._inference_classification_and_get_metrics(
model_spec=model_spec,
n_true_bboxes_data=n_true_bboxes_data,
pseudo_class_names=pseudo_class_names,
tops_n=tops_n,
batch_size=batch_size,
step_penalty=step_penalty)
if df_classification_metrics_columns is None:
df_classification_metrics_columns = tag_df_classification_metrics.columns
classifications_reports_datas.append(
ClassificationReportData(
df_classification_metrics=tag_df_classification_metrics,
tops_n=tops_n,
tag=tag))
classification_report_data = concat_classifications_reports_datas(
classifications_reports_datas=classifications_reports_datas,
df_classification_metrics_columns=df_classification_metrics_columns,
tops_n=tops_n,
compare_tag=compare_tag)
markdowns = self._get_markdowns(
classification_report_data=classification_report_data, )
codes = self._get_codes(tags=tags)
self._save_report(
models_specs=models_specs,
tags=tags,
output_directory=output_directory,
n_true_bboxes_data=n_true_bboxes_data,
markdowns=markdowns,
codes=codes,
classifications_reports_datas=classifications_reports_datas)
return classifications_reports_datas
def report(self,
models_specs: List[DetectionModelSpec],
tags: List[str],
scores_thresholds: List[float],
compare_tag: str,
output_directory: Union[str, Path],
true_images_data: List[ImageData],
minimum_iou: float,
batch_size: int = 16):
assert len(models_specs) == len(tags)
assert len(tags) == len(scores_thresholds)
assert compare_tag in tags
detections_reports_datas = []
for model_spec, tag, score_threshold in zip(models_specs, tags,
scores_thresholds):
logger.info(
f"Making inference and counting metrics for '{tag}'...")
tag_df_detection_metrics, tag_df_detection_recall_per_class = self._inference_detection_and_get_metrics(
model_spec=model_spec,
true_images_data=true_images_data,
score_threshold=score_threshold,
minimum_iou=minimum_iou,
batch_size=batch_size)
detections_reports_datas.append(
DetectionReportData(
df_detection_metrics=tag_df_detection_metrics,
df_detection_recall_per_class=
tag_df_detection_recall_per_class,
tag=tag))
detection_report_data = concat_detections_reports_datas(
detections_reports_datas=detections_reports_datas,
compare_tag=compare_tag)
markdowns = self._get_markdowns(
detection_report_data=detection_report_data, )
codes = self._get_codes(
tags=tags,
scores_thresholds=scores_thresholds,
minimum_iou=minimum_iou,
)
self._save_report(models_specs=models_specs,
tags=tags,
output_directory=output_directory,
true_images_data=true_images_data,
markdowns=markdowns,
codes=codes)
def report_on_predictions(
self,
n_true_bboxes_data: List[List[BboxData]],
n_pred_bboxes_data: List[List[BboxData]],
tag: str,
known_class_names: List[str],
compare_tag: str,
output_directory: Union[str, Path],
pseudo_class_names: List[str],
tops_n: List[int] = [1],
step_penalty: int = 20) -> List[ClassificationReportData]:
logger.info(f"Cunting metrics for '{tag}'...")
tag_df_classification_metrics = get_df_classification_metrics(
n_true_bboxes_data=n_true_bboxes_data,
n_pred_bboxes_data=n_pred_bboxes_data,
pseudo_class_names=pseudo_class_names,
known_class_names=known_class_names,
step_penalty=step_penalty)
df_classification_metrics_columns = tag_df_classification_metrics.columns
classifications_reports_datas = [
ClassificationReportData(
df_classification_metrics=tag_df_classification_metrics,
tops_n=tops_n,
tag=tag)
]
classification_report_data = concat_classifications_reports_datas(
classifications_reports_datas=classifications_reports_datas,
df_classification_metrics_columns=df_classification_metrics_columns,
tops_n=tops_n,
compare_tag=tag)
markdowns = self._get_markdowns(
classification_report_data=classification_report_data, )
self._save_report(
models_specs=[None],
tags=[tag],
output_directory=output_directory,
n_true_bboxes_data=n_true_bboxes_data,
markdowns=markdowns,
codes=[],
classifications_reports_datas=classifications_reports_datas)
return classifications_reports_datas
def copy_files_from_directory_to_temp_directory(
directory: str) -> tempfile.TemporaryDirectory:
directory_openfile = fsspec.open(directory)
directory = Pathy(directory)
temp_dir = tempfile.TemporaryDirectory()
temp_dir_path = Path(temp_dir.name)
for some_file in fsspec.open_files(str(directory / '**'), 'rb'):
some_file_path = Pathy(some_file.path)
relative_filepath = some_file_path.relative_to(directory_openfile.path)
filepath = temp_dir_path / relative_filepath
(filepath.parent).mkdir(exist_ok=True, parents=True)
logger.info(f'copy {some_file_path} => {filepath}')
with open(filepath, 'wb') as out:
with some_file as src:
out.write(src.read())
return temp_dir
def report_on_predictions(
self,
true_images_data: List[ImageData],
pred_images_data_detection: List[ImageData],
raw_pred_images_data_detection: List[ImageData],
pred_images_data_pipeline: List[ImageData],
tag: str,
known_class_names: List[str],
extra_bbox_label: List[str],
output_directory: Union[str, Path],
minimum_iou: float,
pseudo_class_names: List[str],
):
pipelines_reports_datas = []
logger.info(f"Counting metrics for '{tag}'...")
tag_df_detection_metrics = get_df_detection_metrics(
true_images_data=true_images_data,
pred_images_data=pred_images_data_detection,
minimum_iou=minimum_iou,
raw_pred_images_data=raw_pred_images_data_detection)
tag_df_pipeline_metrics = get_df_pipeline_metrics(
true_images_data=true_images_data,
pred_images_data=pred_images_data_pipeline,
minimum_iou=minimum_iou,
extra_bbox_label=extra_bbox_label,
pseudo_class_names=pseudo_class_names,
known_class_names=known_class_names)
pipelines_reports_datas = [
PipelineReportData(df_detection_metrics=tag_df_detection_metrics,
df_pipeline_metrics=tag_df_pipeline_metrics,
tag=tag)
]
pipeline_report_data = concat_pipelines_reports_datas(
pipelines_reports_datas=pipelines_reports_datas, compare_tag=tag)
markdowns = self._get_markdowns(
pipeline_report_data=pipeline_report_data, tags=[tag])
self._save_report(models_specs=[None],
tags=[tag],
output_directory=output_directory,
true_images_data=true_images_data,
markdowns=markdowns,
codes=[])
def report_on_predictions(
self,
true_images_data: List[ImageData],
pred_images_data: List[ImageData],
raw_pred_images_data: List[ImageData],
tag: str,
output_directory: Union[str, Path],
minimum_iou: float,
):
logger.info(f"Counting metrics for '{tag}'...")
tag_df_detection_metrics = get_df_detection_metrics(
true_images_data=true_images_data,
pred_images_data=pred_images_data,
minimum_iou=minimum_iou,
raw_pred_images_data=raw_pred_images_data)
tag_df_detection_recall_per_class = get_df_detection_recall_per_class(
true_images_data=true_images_data,
pred_images_data=pred_images_data,
minimum_iou=minimum_iou,
)
detections_reports_datas = [
DetectionReportData(
df_detection_metrics=tag_df_detection_metrics,
df_detection_recall_per_class=tag_df_detection_recall_per_class,
tag=tag)
]
detection_report_data = concat_detections_reports_datas(
detections_reports_datas=detections_reports_datas, compare_tag=tag)
markdowns = self._get_markdowns(
detection_report_data=detection_report_data, )
self._save_report(models_specs=[None],
tags=[tag],
output_directory=output_directory,
true_images_data=true_images_data,
markdowns=markdowns,
codes=[])