def main(dict_paths, padding=0, use_mask=False, bg_color=None, nb_jobs=NB_THREADS): """ the main executable :param dict_paths: :param int padding: :param int nb_jobs: """ logging.info('running...') if not os.path.isdir(dict_paths['output']): assert os.path.isdir(os.path.dirname(dict_paths['output'])), \ '"%s" should be folder' % dict_paths['output'] logging.debug('creating dir: %s', dict_paths['output']) os.mkdir(dict_paths['output']) list_dirs = [dict_paths['annot'], dict_paths['image']] df_paths = tl_data.find_files_match_names_across_dirs(list_dirs) logging.info('start cutting images') wrapper_cutting = partial(export_cut_objects, path_out=dict_paths['output'], padding=padding, use_mask=use_mask, bg_color=bg_color) iterate = tl_expt.WrapExecuteSequence(wrapper_cutting, (row for idx, row in df_paths.iterrows()), nb_jobs=nb_jobs) list(iterate) logging.info('DONE')
def main(params): """ compute the distance among segmented superpixels and given annotation :param {str: ...} params: """ logging.info('running...') if os.path.isdir(params['path_out']): logging.info('Missing output dir -> no visual export & results table.') list_paths = [params['path_images'], params['path_segms']] df_paths = tl_data.find_files_match_names_across_dirs(list_paths) df_paths.columns = ['path_image', 'path_segm'] df_dist = pd.DataFrame() wrapper_eval = partial(compute_boundary_distance, params=params, path_out=params['path_out']) iterate = tl_expt.WrapExecuteSequence(wrapper_eval, df_paths.iterrows(), nb_jobs=params['nb_jobs'], desc='evaluate SLIC') for name, dist in iterate: df_dist = df_dist.append({'name': name, 'mean boundary distance': dist}, ignore_index=True) df_dist.set_index('name', inplace=True) if os.path.isdir(params['path_out']): df_dist.to_csv(os.path.join(params['path_out'], NAME_CSV_DISTANCES)) logging.info('STATISTIC:') logging.info(df_dist.describe()) logging.info('DONE')
def main(params): """ PIPELINE candidate clustering :param {str: any} params: """ with open(os.path.join(params['path_expt'], 'config_clustering.json'), 'w') as fp: json.dump(params, fp) tl_expt.create_subfolders(params['path_expt'], LIST_SUBDIRS) list_paths = [params[k] for k in ['path_images', 'path_segms', 'path_centers']] df_paths = tl_data.find_files_match_names_across_dirs(list_paths) df_paths.columns = ['path_image', 'path_segm', 'path_points'] df_paths.index = range(1, len(df_paths) + 1) path_cover = os.path.join(params['path_expt'], run_train.NAME_CSV_TRIPLES) df_paths.to_csv(path_cover) logging.info('run clustering...') df_paths_new = pd.DataFrame() _wrapper_clustering = partial(cluster_points_draw_export, params=params, path_out=params['path_expt']) rows = (dict(row) for idx, row in df_paths.iterrows()) iterate = tl_expt.WrapExecuteSequence(_wrapper_clustering, rows, nb_jobs=params['nb_jobs']) for dict_center in iterate: df_paths_new = df_paths_new.append(dict_center, ignore_index=True) df_paths_new.set_index('image', inplace=True) df_paths_new.to_csv(path_cover)
def main(dict_paths, nb_jobs=NB_THREADS, relabel=True): """ main evaluation :param {str: str} dict_paths: :param int nb_jobs: number of thred running in parallel :param bool relabel: whether relabel segmentation as sequential """ logging.info('running...') if not os.path.isdir(dict_paths['output']): assert os.path.isdir(os.path.dirname(dict_paths['output'])), \ 'missing folder: %s' % dict_paths['output'] os.mkdir(dict_paths['output']) name = os.path.basename(os.path.dirname(dict_paths['segm'])) list_dirs = [dict_paths['annot'], dict_paths['segm']] if dict_paths['image'] != '': list_dirs.append(dict_paths['image']) df_paths = tl_data.find_files_match_names_across_dirs(list_dirs) path_csv = os.path.join(dict_paths['output'], NAME_CVS_PER_IMAGE % name) df_paths.to_csv(path_csv) annots, _ = tl_data.load_images_list(df_paths['path_1'].values.tolist()) segms, names = tl_data.load_images_list(df_paths['path_2'].values.tolist()) logging.info('loaded %i annots and %i segms', len(annots), len(segms)) if relabel: annots = [relabel_sequential(annot)[0] for annot in annots] segms = list(map(wrapper_relabel_segm, zip(annots, segms))) path_csv = os.path.join(dict_paths['output'], NAME_CVS_PER_IMAGE % name) logging.debug('export to "%s"', path_csv) df_stat = seg_clf.compute_stat_per_image(segms, annots, names, nb_jobs) df_stat.to_csv(path_csv) path_csv = os.path.join(dict_paths['output'], NAME_CVS_OVERALL % name) logging.debug('export to "%s"', path_csv) df_desc = df_stat.describe() logging.info(df_desc.T[['count', 'mean', 'std']]) df_desc.to_csv(path_csv) path_visu = os.path.join(dict_paths['output'], '%s__visual' % name) if not os.path.isdir(path_visu): os.mkdir(path_visu) # for idx, row in df_paths.iterrows(): # export_visual(row, path_visu) wrapper_visual = partial(export_visual, path_out=path_visu) iterate = tl_expt.WrapExecuteSequence( wrapper_visual, (row for idx, row in df_paths.iterrows()), nb_jobs=nb_jobs) list(iterate) logging.info('DONE')
def main(params): df_paths = tl_data.find_files_match_names_across_dirs([params['path_images'], params['path_segms'], params['path_centers']]) df_paths.columns = ['path_image', 'path_segm', 'path_centers'] df_paths.index = range(1, len(df_paths) + 1) if not os.path.exists(params['path_output']): assert os.path.exists(os.path.dirname(params['path_output'])), \ 'missing folder: "%s"' % os.path.dirname(params['path_output']) os.mkdir(params['path_output']) df_slices_info = seg_annot.load_info_group_by_slices(params['path_infofile'], params['stages']) _wrapper_export = partial(export_figure, df_slices_info=df_slices_info, path_out=params['path_output']) iterate = tl_expt.WrapExecuteSequence(_wrapper_export, df_paths.iterrows(), nb_jobs=params['nb_jobs']) list(iterate)
def find_match_images_segms_centers(path_pattern_imgs, path_pattern_segms, path_pattern_center=None): """ walk over dir with images and segmentation and pair those with the same name and if the folder with centers exists also add to each par a center NOTE: returns just paths :param str path_pattern_imgs: :param str path_pattern_segms: :param str path_pattern_center: :return DF: DF<path_img, path_segm, path_center> """ logging.info('find match images-segms-centres...') list_paths = [path_pattern_imgs, path_pattern_segms, path_pattern_center] df_paths = tl_data.find_files_match_names_across_dirs(list_paths) if path_pattern_center is None: df_paths.columns = ['path_image', 'path_segm'] df_paths['path_centers'] = '' else: df_paths.columns = ['path_image', 'path_segm', 'path_centers'] df_paths.index = range(1, len(df_paths) + 1) return df_paths
def evaluate_folder(path_dir, dict_paths, export_visual=EXPORT_VUSIALISATION): """ take a single folder with segmentation and compute statistic against annotation and export some visualisations, return computed stat. :param str path_dir: :param {str, str} dict_paths: :param bool export_visual: :return {str: float}: """ logging.info('evaluate folder: %s', path_dir) name = os.path.basename(path_dir) list_paths = [ dict_paths['images'], dict_paths['annots'], dict_paths['segments'], dict_paths['centers'], os.path.join(path_dir, '*.png') ] df_paths = tl_data.find_files_match_names_across_dirs(list_paths) if len(df_paths) == 0: return {'method': name, 'count': 0} if dict_paths['annots'] is not None: df_paths.columns = [ 'path_image', 'path_annot', 'path_in-segm', 'path_centers', 'path_egg-segm' ] else: df_paths.columns = [ 'path_image', 'path_in-segm', 'path_centers', 'path_egg-segm' ] df_paths.index = range(1, len(df_paths) + 1) df_paths.to_csv(os.path.join(dict_paths['results'], NAME_CSV_STAT % name)) if export_visual: for _, row in df_paths.iterrows(): expert_visual(row, name, path_out=dict_paths['results']) if dict_paths['annots'] is None: logging.info('no Annotation given') return {'method': name, 'count': 0} df_eval = pd.DataFrame() for _, row in df_paths.iterrows(): dict_seg = compute_metrics(row) df_eval = df_eval.append(dict_seg, ignore_index=True) df_eval.set_index(['name'], inplace=True) df_eval.to_csv(os.path.join(dict_paths['results'], NAME_CSV_STAT % name)) df_summary = df_eval.describe() cols = df_eval.columns.tolist() dict_eval = {'method': name, 'count': len(df_eval)} for n in ['mean', 'std']: names = ['%s (%s)' % (c, n) for c in cols] dict_eval.update(zip(names, df_summary.T[n].values.tolist())) dict_eval.update( zip(['%s (median)' % c for c in cols], df_eval.median(axis=0).values.tolist())) return dict_eval
def main(dict_paths, visual=True, drop_labels=None, relabel=True, segm_alpha=1., nb_jobs=NB_THREADS): """ main evaluation :param {str: str} dict_paths: :param int nb_jobs: number of thred running in parallel :param bool relabel: whether relabel segmentation as sequential """ if not os.path.isdir(dict_paths['output']): assert os.path.isdir(os.path.dirname(dict_paths['output'])), \ 'missing folder: %s' % dict_paths['output'] os.mkdir(dict_paths['output']) name = os.path.basename(os.path.dirname(dict_paths['segm'])) list_dirs = [dict_paths['annot'], dict_paths['segm']] if dict_paths.get('image', '') != '': list_dirs.append(dict_paths['image']) df_paths = tl_data.find_files_match_names_across_dirs(list_dirs) path_csv = os.path.join(dict_paths['output'], NAME_CVS_PER_IMAGE % name) df_paths.to_csv(path_csv) assert len(df_paths) > 0, 'nothing to compare' annots, _ = tl_data.load_images_list(df_paths['path_1'].values.tolist()) segms, names = tl_data.load_images_list(df_paths['path_2'].values.tolist()) logging.info('loaded %i annots and %i segms', len(annots), len(segms)) if drop_labels is not None: annots = [np.array(annot, dtype=float) for annot in annots] for lb in drop_labels: for i, annot in enumerate(annots): annots[i][annot == lb] = np.nan annots = [np.nan_to_num(annot + 1).astype(int) - 1 for annot in annots] segms = [seg.astype(int) for seg in segms] if relabel: logging.info('relabel annotations and segmentations') if drop_labels is None: annots = [relabel_sequential(annot)[0] for annot in annots] iterate = tl_expt.WrapExecuteSequence(wrapper_relabel_segm, zip(annots, segms), nb_jobs=nb_jobs, ordered=True, desc='relabeling') segms = list(iterate) logging.info('compute statistic per image') path_csv = os.path.join(dict_paths['output'], NAME_CVS_PER_IMAGE % name) logging.debug('export to "%s"', path_csv) df_stat = seg_clf.compute_stat_per_image(segms, annots, names, nb_jobs, drop_labels=[-1]) df_stat.to_csv(path_csv) logging.info('summarise statistic') path_csv = os.path.join(dict_paths['output'], NAME_CVS_OVERALL % name) logging.debug('export to "%s"', path_csv) df_desc = df_stat.describe() df_desc = df_desc.append(pd.Series(df_stat.median(), name='median')) logging.info(df_desc.T[['count', 'mean', 'std', 'median']]) df_desc.to_csv(path_csv) if visual: images = [None] * len(annots) if 'path_3' in df_paths: images, _ = tl_data.load_images_list(df_paths['path_3'].values) path_visu = os.path.join(dict_paths['output'], '%s%s' % (name, SUFFIX_VISUAL)) if not os.path.isdir(path_visu): os.mkdir(path_visu) # for idx, row in df_paths.iterrows(): # export_visual(row, path_visu) _wrapper_visual = partial(export_visual, path_out=path_visu, segm_alpha=segm_alpha) it_values = zip(names, annots, segms, images) iterate = tl_expt.WrapExecuteSequence(_wrapper_visual, it_values, desc='visualisations', nb_jobs=nb_jobs) list(iterate)