def eval_segment_with_annot(params, dict_annot, dict_segm, dict_label_hist=None,
name_csv=NAME_CSV_SEGM_STAT_SLIC_ANNOT, nb_jobs=1):
""" evaluate the segmentation results according given annotation
:param {str: ...} params:
:param {str: ndarray} dict_annot:
:param {str: ndarray} dict_segm:
:param {str: ndarray} dict_label_hist:
:param str name_csv:
:param int nb_jobs:
:return:
"""
if dict_label_hist is not None:
visu_histogram_labels(params, dict_label_hist)
assert sorted(dict_annot.keys()) == sorted(dict_segm.keys()), \
'mismatch in dictionary keys: \n%s \n%s' % (sorted(dict_annot.keys()),
sorted(dict_segm.keys()))
list_annot = [dict_annot[n] for n in dict_annot]
list_segm = [dict_segm[n] for n in dict_annot]
df_stat = seg_clf.compute_stat_per_image(list_segm, list_annot,
[n for n in dict_annot], nb_jobs)
path_csv = os.path.join(params['path_exp'], name_csv)
logging.info('STAT on seg_pipe and annot (%s):', name_csv)
df_stat.to_csv(path_csv)
logging.info(metrics.classification_report(
seg_label.convert_segms_2_list(list_segm),
seg_label.convert_segms_2_list(list_annot), digits=4))
logging.debug(repr(df_stat))
return df_stat
def main(dict_paths, nb_jobs=NB_THREADS, relabel=True):
""" main evaluation
:param {str: str} dict_paths:
:param int nb_jobs: number of thred running in parallel
:param bool relabel: whether relabel segmentation as sequential
"""
logging.info('running...')
if not os.path.isdir(dict_paths['output']):
assert os.path.isdir(os.path.dirname(dict_paths['output'])), \
'missing folder: %s' % dict_paths['output']
os.mkdir(dict_paths['output'])
name = os.path.basename(os.path.dirname(dict_paths['segm']))
list_dirs = [dict_paths['annot'], dict_paths['segm']]
if dict_paths['image'] != '':
list_dirs.append(dict_paths['image'])
df_paths = tl_data.find_files_match_names_across_dirs(list_dirs)
path_csv = os.path.join(dict_paths['output'], NAME_CVS_PER_IMAGE % name)
df_paths.to_csv(path_csv)
annots, _ = tl_data.load_images_list(df_paths['path_1'].values.tolist())
segms, names = tl_data.load_images_list(df_paths['path_2'].values.tolist())
logging.info('loaded %i annots and %i segms', len(annots), len(segms))
if relabel:
annots = [relabel_sequential(annot)[0] for annot in annots]
segms = list(map(wrapper_relabel_segm, zip(annots, segms)))
path_csv = os.path.join(dict_paths['output'], NAME_CVS_PER_IMAGE % name)
logging.debug('export to "%s"', path_csv)
df_stat = seg_clf.compute_stat_per_image(segms, annots, names, nb_jobs)
df_stat.to_csv(path_csv)
path_csv = os.path.join(dict_paths['output'], NAME_CVS_OVERALL % name)
logging.debug('export to "%s"', path_csv)
df_desc = df_stat.describe()
logging.info(df_desc.T[['count', 'mean', 'std']])
df_desc.to_csv(path_csv)
path_visu = os.path.join(dict_paths['output'], '%s__visual' % name)
if not os.path.isdir(path_visu):
os.mkdir(path_visu)
# for idx, row in df_paths.iterrows():
# export_visual(row, path_visu)
wrapper_visual = partial(export_visual, path_out=path_visu)
iterate = tl_expt.WrapExecuteSequence(
wrapper_visual, (row for idx, row in df_paths.iterrows()),
nb_jobs=nb_jobs)
list(iterate)
logging.info('DONE')
def eval_segment_with_annot(
params,
dict_annot,
dict_segm,
dict_label_hist=None,
name_csv='statistic___.csv',
drop_labels=None,
nb_workers=1,
):
""" evaluate the segmentation results according given annotation
:param dict params:
:param {str: ndarray} dict_annot:
:param {str: ndarray} dict_segm:
:param {str: ndarray} dict_label_hist:
:param str name_csv:
:param int nb_workers:
:return:
"""
if dict_label_hist is not None:
visu_histogram_labels(params, dict_label_hist)
if sorted(dict_annot) != sorted(dict_segm):
raise ValueError('mismatch in dictionary keys: \n%s \n%s' %
(sorted(dict_annot.keys()), sorted(dict_segm.keys())))
list_annot = [dict_annot[n] for n in dict_annot]
list_segm = [dict_segm[n] for n in dict_annot]
df_stat = seg_clf.compute_stat_per_image(list_segm,
list_annot,
list(dict_annot),
nb_workers,
drop_labels=drop_labels)
path_csv = os.path.join(params['path_exp'], name_csv)
logging.info('STATISTIC on segm and annot (%s):', name_csv)
df_stat.to_csv(path_csv)
logging.info(
metrics.classification_report(
seg_label.convert_segms_2_list(list_segm),
seg_label.convert_segms_2_list(list_annot),
digits=4))
logging.debug('%r', df_stat)
return df_stat
def main(dict_paths,
visual=True,
drop_labels=None,
relabel=True,
segm_alpha=1.,
nb_jobs=NB_THREADS):
""" main evaluation
:param {str: str} dict_paths:
:param int nb_jobs: number of thred running in parallel
:param bool relabel: whether relabel segmentation as sequential
"""
if not os.path.isdir(dict_paths['output']):
assert os.path.isdir(os.path.dirname(dict_paths['output'])), \
'missing folder: %s' % dict_paths['output']
os.mkdir(dict_paths['output'])
name = os.path.basename(os.path.dirname(dict_paths['segm']))
list_dirs = [dict_paths['annot'], dict_paths['segm']]
if dict_paths.get('image', '') != '':
list_dirs.append(dict_paths['image'])
df_paths = tl_data.find_files_match_names_across_dirs(list_dirs)
path_csv = os.path.join(dict_paths['output'], NAME_CVS_PER_IMAGE % name)
df_paths.to_csv(path_csv)
assert len(df_paths) > 0, 'nothing to compare'
annots, _ = tl_data.load_images_list(df_paths['path_1'].values.tolist())
segms, names = tl_data.load_images_list(df_paths['path_2'].values.tolist())
logging.info('loaded %i annots and %i segms', len(annots), len(segms))
if drop_labels is not None:
annots = [np.array(annot, dtype=float) for annot in annots]
for lb in drop_labels:
for i, annot in enumerate(annots):
annots[i][annot == lb] = np.nan
annots = [np.nan_to_num(annot + 1).astype(int) - 1 for annot in annots]
segms = [seg.astype(int) for seg in segms]
if relabel:
logging.info('relabel annotations and segmentations')
if drop_labels is None:
annots = [relabel_sequential(annot)[0] for annot in annots]
iterate = tl_expt.WrapExecuteSequence(wrapper_relabel_segm,
zip(annots, segms),
nb_jobs=nb_jobs,
ordered=True,
desc='relabeling')
segms = list(iterate)
logging.info('compute statistic per image')
path_csv = os.path.join(dict_paths['output'], NAME_CVS_PER_IMAGE % name)
logging.debug('export to "%s"', path_csv)
df_stat = seg_clf.compute_stat_per_image(segms,
annots,
names,
nb_jobs,
drop_labels=[-1])
df_stat.to_csv(path_csv)
logging.info('summarise statistic')
path_csv = os.path.join(dict_paths['output'], NAME_CVS_OVERALL % name)
logging.debug('export to "%s"', path_csv)
df_desc = df_stat.describe()
df_desc = df_desc.append(pd.Series(df_stat.median(), name='median'))
logging.info(df_desc.T[['count', 'mean', 'std', 'median']])
df_desc.to_csv(path_csv)
if visual:
images = [None] * len(annots)
if 'path_3' in df_paths:
images, _ = tl_data.load_images_list(df_paths['path_3'].values)
path_visu = os.path.join(dict_paths['output'],
'%s%s' % (name, SUFFIX_VISUAL))
if not os.path.isdir(path_visu):
os.mkdir(path_visu)
# for idx, row in df_paths.iterrows():
# export_visual(row, path_visu)
_wrapper_visual = partial(export_visual,
path_out=path_visu,
segm_alpha=segm_alpha)
it_values = zip(names, annots, segms, images)
iterate = tl_expt.WrapExecuteSequence(_wrapper_visual,
it_values,
desc='visualisations',
nb_jobs=nb_jobs)
list(iterate)