def dataset_load_images_segms_compute_features(params, df_paths, nb_workers=NB_THREADS):
""" create whole dataset composed from loading input data, computing features
and label points by label whether its positive or negative center candidate
:param {str: any} params: parameters
:param DF df_paths: DataFrame
:param int nb_workers: parallel
:return {str: ...}:
"""
dict_imgs, dict_segms, dict_center = dict(), dict(), dict()
logging.info('loading input data (images, segmentation and centers)')
path_show_in = os.path.join(params['path_expt'], FOLDER_INPUT)
_wrapper_load = partial(load_image_segm_center, path_out=path_show_in,
dict_relabel=params['dict_relabel'])
iterate = tl_expt.WrapExecuteSequence(_wrapper_load, df_paths.iterrows(),
nb_workers=nb_workers,
desc='loading input data')
for name, img, seg, center in iterate:
dict_imgs[name] = img
dict_segms[name] = seg
dict_center[name] = center
dict_slics, dict_points, dict_features = dict(), dict(), dict()
logging.info('estimate candidate points and compute features')
gene_name_img_seg = ((name, dict_imgs[name], dict_segms[name])
for name in dict_imgs)
_wrapper_pnt_features = partial(wrapper_estim_points_compute_features,
params=params)
feature_names = None
iterate = tl_expt.WrapExecuteSequence(_wrapper_pnt_features,
gene_name_img_seg, nb_workers=nb_workers,
desc='estimate candidates & features')
for name, slic, points, features, feature_names in iterate:
dict_slics[name] = slic
dict_points[name] = points
dict_features[name] = features
logging.debug('computed features:\n %r', feature_names)
dict_labels = dict()
logging.info('assign labels according close distance to center')
path_points_train = os.path.join(params['path_expt'], FOLDER_POINTS_TRAIN)
tqdm_bar = tqdm.tqdm(total=len(dict_center), desc='labels assignment')
for name in dict_center:
dict_labels[name] = label_close_points(dict_center[name],
dict_points[name], params)
points = np.asarray(dict_points[name])[np.asarray(dict_labels[name]) == 1]
path_csv = os.path.join(path_points_train, name + '.csv')
tl_data.save_landmarks_csv(path_csv, points)
tqdm_bar.update()
tqdm_bar.close()
return (dict_imgs, dict_segms, dict_slics, dict_points, dict_center,
dict_features, dict_labels, feature_names)
def main(paths, nb_workers=NB_WORKERS, segm_alpha=MIDDLE_ALPHA_OVERLAP):
logging.info('running...')
if paths['segms'] == paths['output']:
raise RuntimeError('overwriting segmentation dir')
if os.path.basename(paths['images']) == paths['output']:
raise RuntimeError('overwriting image dir')
logging.info(tl_expt.string_dict(paths, desc='PATHS'))
if not os.path.exists(paths['output']):
dir_out = os.path.dirname(paths['output'])
if not os.path.isdir(dir_out):
raise FileNotFoundError('missing folder: %s' % dir_out)
os.mkdir(paths['output'])
paths_imgs = glob.glob(paths['images'])
logging.info('found %i images in dir "%s"', len(paths_imgs),
paths['images'])
_warped_overlap = partial(perform_visu_overlap,
paths=paths,
segm_alpha=segm_alpha)
created = []
iterate = tl_expt.WrapExecuteSequence(_warped_overlap,
paths_imgs,
nb_workers=nb_workers,
desc='overlapping')
for r in iterate:
created.append(r)
logging.info('matched and created %i overlaps', np.sum(created))
logging.info('DONE')
def experiment_group_gmm(params, paths_img, path_out, path_visu,
show_debug_imgs=SHOW_DEBUG_IMAGES):
logging.info('load all images')
list_images = [load_image(path_img, params['img_type'])
for path_img in paths_img]
imgs_idx_path = list(zip([None] * len(paths_img), paths_img))
logging.info('Estimate image segmentation from whole sequence of images')
params['path_model'] = os.path.join(params['path_exp'], NAME_DUMP_MODEL)
if os.path.isfile(params['path_model']) and not FORCE_RECOMP_DATA:
model, _, _ = load_model(params['path_model'])
else:
model, _ = seg_pipe.estim_model_classes_group(
list_images, nb_classes=params['nb_classes'],
dict_features=params['features'], sp_size=params['slic_size'],
sp_regul=params['slic_regul'], pca_coef=params['pca_coef'],
model_type=params['estim_model'])
save_model(params['path_model'], model)
logging.info('Perform image segmentation from group model')
_wrapper_segment = partial(segment_image_model, params=params, model=model,
path_out=path_out, path_visu=path_visu,
show_debug_imgs=show_debug_imgs)
iterate = tl_expt.WrapExecuteSequence(_wrapper_segment, imgs_idx_path,
nb_workers=params['nb_workers'],
desc='experiment group GMM')
# dict_segms_group = {}
# for name, segm in iterate:
# dict_segms_group[name] = segm
dict_segms_group = dict(iterate)
gc.collect()
time.sleep(1)
return dict_segms_group
def experiment_single_gmm(params,
paths_img,
path_out,
path_visu,
show_debug_imgs=SHOW_DEBUG_IMAGES):
imgs_idx_path = list(zip([None] * len(paths_img), paths_img))
logging.info('Perform image segmentation as single image in each time')
_wrapper_segment = partial(
segment_image_independent,
params=params,
path_out=path_out,
path_visu=path_visu,
show_debug_imgs=show_debug_imgs,
)
iterate = tl_expt.WrapExecuteSequence(
_wrapper_segment,
imgs_idx_path,
nb_workers=params['nb_workers'],
desc='experiment single GMM',
)
# dict_segms_gmm = {}
# for name, segm in iterate:
# dict_segms_gmm[name] = segm
dict_segms_gmm = dict(iterate)
gc.collect()
time.sleep(1)
return dict_segms_gmm
def convert_folder_images(path_images, path_out, path_json=None, nb_workers=1):
""" perform single or multi thread image quantisation
:param list(str) path_images: list of input images
:param str path_out: output directory
:param str path_json: path to json file
:param int int nb_workers:
"""
if not os.path.isdir(os.path.dirname(path_images)):
raise FileNotFoundError('input folder does not exist')
path_imgs = sorted(glob.glob(path_images))
logging.info('found %i images', len(path_imgs))
if not os.path.exists(path_out):
dir_out = os.path.dirname(path_out)
if not os.path.isdir(dir_out):
raise FileNotFoundError('missing folder: %s' % dir_out)
os.mkdir(path_out)
dict_colors = load_dict_colours(path_json)
logging.debug('loaded dictionary %r', dict_colors)
_wrapper_img_convert = partial(perform_img_convert,
path_out=path_out,
dict_colors=dict_colors)
iterate = tl_expt.WrapExecuteSequence(_wrapper_img_convert,
path_imgs,
nb_workers=nb_workers,
desc='convert images')
list(iterate)
def main(dict_paths,
visual=True,
drop_labels=None,
relabel=True,
segm_alpha=1.,
nb_workers=NB_THREADS):
""" main evaluation
:param {str: str} dict_paths:
:param int nb_workers: number of thred running in parallel
:param bool relabel: whether relabel segmentation as sequential
"""
if not os.path.isdir(dict_paths['output']):
assert os.path.isdir(os.path.dirname(dict_paths['output'])), \
'missing folder: %s' % dict_paths['output']
os.mkdir(dict_paths['output'])
name = os.path.basename(os.path.dirname(dict_paths['segm']))
list_dirs = [dict_paths['annot'], dict_paths['segm']]
if dict_paths.get('image', '') != '':
list_dirs.append(dict_paths['image'])
df_paths = tl_data.find_files_match_names_across_dirs(list_dirs)
path_csv = os.path.join(dict_paths['output'], NAME_CVS_PER_IMAGE % name)
logging.info('found %i pairs', len(df_paths))
df_paths.to_csv(path_csv)
assert not df_paths.empty, 'nothing to compare'
name_seg_dir = os.path.basename(os.path.dirname(dict_paths['segm']))
path_visu = os.path.join(dict_paths['output'],
name_seg_dir + SUFFIX_VISUAL)
if visual and not os.path.isdir(path_visu):
os.mkdir(path_visu)
elif not visual:
path_visu = ''
logging.info('compute statistic per image')
_wrapper_stat = partial(stat_single_set,
drop_labels=drop_labels,
relabel=relabel,
path_visu=path_visu,
segm_alpha=segm_alpha)
iterate = tl_expt.WrapExecuteSequence(_wrapper_stat,
df_paths.iterrows(),
desc='compute statistic',
nb_workers=nb_workers)
list_stats = list(iterate)
df_stat = pd.DataFrame(list_stats)
path_csv = os.path.join(dict_paths['output'], NAME_CVS_PER_IMAGE % name)
logging.debug('export to "%s"', path_csv)
df_stat.to_csv(path_csv)
logging.info('summarise statistic')
path_csv = os.path.join(dict_paths['output'], NAME_CVS_OVERALL % name)
logging.debug('export to "%s"', path_csv)
df_desc = df_stat.describe()
df_desc = df_desc.append(pd.Series(df_stat.median(), name='median'))
logging.info(df_desc.T[['count', 'mean', 'std', 'median']])
df_desc.to_csv(path_csv)
def main(params):
""" PIPELINE for rotation
:param {str: str} params:
"""
logging.info(tl_expt.string_dict(params, desc='PARAMETERS'))
list_img_paths = sorted(
[p for p in glob.glob(params['path_images']) if os.path.isfile(p)])
logging.info('found images: %i' % len(list_img_paths))
if not os.path.isdir(params['path_output']):
os.mkdir(params['path_output'])
img_mean = compute_mean_image(list_img_paths)
_wrapper_object = partial(perform_orientation_swap,
path_out=params['path_output'],
img_template=img_mean)
dir_name = os.path.dirname(params['path_images'])
iterate = tl_expt.WrapExecuteSequence(_wrapper_object,
list_img_paths,
nb_jobs=params['nb_jobs'],
desc=dir_name)
list(iterate)
def perform_train_predictions(params,
paths_img,
classif,
show_debug_imgs=SHOW_DEBUG_IMAGES):
logging.info('run prediction on training images...')
imgs_idx_path = list(zip(range(1, len(paths_img) + 1), paths_img))
dict_segms, dict_segms_gc = {}, {}
path_out = os.path.join(params['path_exp'], FOLDER_TRAIN)
path_visu = os.path.join(params['path_exp'], FOLDER_TRAIN_VISU)
_wrapper_segment = partial(
segment_image,
params=params,
classif=classif,
path_out=path_out,
path_visu=path_visu,
show_debug_imgs=show_debug_imgs,
)
iterate = tl_expt.WrapExecuteSequence(
_wrapper_segment,
imgs_idx_path,
nb_workers=params['nb_workers'],
desc='image segm: prediction',
)
for name, segm, segm_gc in iterate:
dict_segms[name] = segm
dict_segms_gc[name] = segm_gc
return dict_segms, dict_segms_gc
def export_dataset_visual(path_output,
dict_imgs,
dict_segms,
dict_slics,
dict_points,
dict_labels,
nb_jobs=NB_THREADS):
""" visualise complete training dataset by marking labeld points
over image and input segmentation
:param {str: ndarray} dict_imgs:
:param {str: ndarray} dict_segms:
:param {str: ndarray} dict_slics:
:param {str: ndarray} dict_points:
:param {str: ndarray} dict_labels:
:param int nb_jobs: number processing in parallel
"""
logging.info('export training visualisations')
path_out = os.path.join(path_output, FOLDER_POINTS_TRAIN)
gener_args = ((path_out, name, dict_imgs[name], dict_segms[name],
dict_points[name], dict_labels[name], dict_slics[name],
None, '_train') for name in dict_imgs)
iterate = tl_expt.WrapExecuteSequence(wrapper_draw_export_slic_centers,
gener_args,
nb_jobs=nb_jobs,
desc='exporting visualisations')
list(iterate)
def experiment_lpo(params,
df_stat,
dict_annot,
idx_paths_img,
path_classif,
path_dump,
nb_holdout,
show_debug_imgs=SHOW_DEBUG_IMAGES):
""" experiment Leave-P-samples-Out
:param {str: ...} params:
:param DF df_stat:
:param {str: ndarray} dict_annot:
:param [str] paths_img:
:param str path_classif:
:param str path_dump:
:param int nb_holdout:
:param bool show_debug_imgs: whether show debug images
:return {}:
"""
logging.info('run prediction on training images as Leave-%i-Out...',
nb_holdout)
dict_segms, dict_segms_gc = dict(), dict()
cv = seg_clf.CrossValidatePOut(len(idx_paths_img), nb_hold_out=nb_holdout)
test_imgs_idx_path = [[idx_paths_img[i] for i in ids] for _, ids in cv]
path_out = os.path.join(params['path_exp'], FOLDER_LPO)
path_visu = os.path.join(params['path_exp'], FOLDER_LPO_VISU) \
if params.get('visual', False) else None
_wrapper_segment = partial(retrain_lpo_segment_image,
path_classif=path_classif,
path_dump=path_dump,
path_out=path_out,
path_visu=path_visu,
show_debug_imgs=show_debug_imgs)
iterate = tl_expt.WrapExecuteSequence(_wrapper_segment,
test_imgs_idx_path,
nb_jobs=params['nb_jobs'],
desc='experiment LPO')
for dict_seg, dict_seg_gc in iterate:
dict_segms.update(dict_seg)
dict_segms_gc.update(dict_seg_gc)
gc.collect()
time.sleep(1)
df = eval_segment_with_annot(params, dict_annot, dict_segms, None,
NAME_CSV_SEGM_STAT_RESULT_LPO % nb_holdout,
params.get('drop_labels',
None), params['nb_jobs'])
df_stat = df_stat.append(get_summary(df, 'segm (L-%i-O)' % nb_holdout),
ignore_index=True)
df = eval_segment_with_annot(params, dict_annot, dict_segms_gc, None,
NAME_CSV_SEGM_STAT_RESULT_LPO_GC % nb_holdout,
params.get('drop_labels',
None), params['nb_jobs'])
df_stat = df_stat.append(get_summary(df, 'segm GC (L-%i-O)' % nb_holdout),
ignore_index=True)
path_csv_stat = os.path.join(params['path_exp'],
NAME_CSV_SEGM_STAT_RESULTS)
df_stat.set_index(['name']).to_csv(path_csv_stat)
return df_stat
def relabel_folder_images(path_images, path_out, labels_old, labels_new,
nb_workers=1):
""" perform single or multi thread image quantisation
:param [int] labels_old:
:param [int] labels_new:
:param list(str) path_images: list of input images
:param path_out: output directory
:param [int] labels_old: list of labels to be replaced
:param [int] labels_new: list of new labels
:param int nb_workers:
"""
assert os.path.isdir(os.path.dirname(path_images)), \
'missing folder: %s' % path_images
assert os.path.isdir(path_out), 'missing ouput folder: %s' % path_out
path_imgs = sorted(glob.glob(path_images))
logging.info('found %i images', len(path_imgs))
_wrapper_img_relabel = partial(perform_image_relabel, path_out=path_out,
labels_old=labels_old, labels_new=labels_new)
iterate = tl_expt.WrapExecuteSequence(_wrapper_img_relabel, path_imgs,
nb_workers=nb_workers,
desc='relabel images')
list(iterate)
def main(params):
""" PIPELINE for matching
:param {str: str} params:
"""
logging.info(tl_expt.string_dict(params, desc='PARAMETERS'))
df_info = pd.read_csv(params['path_infofile'], sep='\t', index_col=0)
df_info = filter_table(df_info, params['path_ellipses'])
logging.info('filtered %i item in table' % len(df_info))
path_csv = os.path.join(params['path_output'], NAME_CSV_RESULTS)
list_evals = []
# get the folder
path_dir_csv = os.path.dirname(params['path_ellipses'])
_wrapper_match = partial(select_optimal_ellipse, path_dir_csv=path_dir_csv)
iterate = tl_expt.WrapExecuteSequence(_wrapper_match,
df_info.iterrows(),
nb_workers=params['nb_workers'])
for i, dict_row in enumerate(iterate):
list_evals.append(dict_row)
# every hundreds iteration do export
if i % 100 == 0:
pd.DataFrame(list_evals).to_csv(path_csv)
df_ellipses = pd.DataFrame(list_evals)
df_ellipses.to_csv(path_csv)
def main(path_segs, path_out, nb_workers):
""" the main for creating annotations
:param str path_segs: path with image pattern of images - obj segmentation
:param str path_out:
:param int nb_workers: number of processes in parallel
"""
assert os.path.dirname(path_segs) != path_out, \
'the output dir has to be different then the input object segmentation'
list_imgs = glob.glob(path_segs)
logging.info('found %i images', len(list_imgs))
if not os.path.exists(path_out):
assert os.path.isdir(os.path.dirname(path_out)), \
'missing: %s' % path_out
os.mkdir(path_out)
_wrapper_create_annot_centers = partial(
create_annot_centers,
path_out_seg=path_out,
path_out_csv=path_out,
)
iterate = tl_expt.WrapExecuteSequence(
_wrapper_create_annot_centers,
list_imgs,
nb_workers=nb_workers,
desc='annotating images',
)
list(iterate)
def convert_folder_images(path_images, path_out, path_json=None, nb_jobs=1):
""" perform single or multi thread image quantisation
:param [str] path_images: list of input images
:param path_out: output directory
:param path_json: path to json file
:param int nb_jobs: int
"""
assert os.path.isdir(os.path.dirname(path_images)), \
'input folder does not exist'
path_imgs = sorted(glob.glob(path_images))
logging.info('found %i images', len(path_imgs))
if not os.path.exists(path_out):
assert os.path.isdir(os.path.dirname(path_out)), \
'missing folder: %s' % os.path.dirname(path_out)
os.mkdir(path_out)
dict_colors = load_dict_colours(path_json)
logging.debug('loaded dictionary %s', repr(dict_colors))
_wrapper_img_convert = partial(perform_img_convert, path_out=path_out,
dict_colors=dict_colors)
iterate = tl_expt.WrapExecuteSequence(_wrapper_img_convert, path_imgs,
nb_jobs=nb_jobs,
desc='convert images')
list(iterate)
def main(paths, nb_jobs=NB_THREADS, segm_alpha=MIDDLE_ALPHA_OVERLAP):
logging.info('running...')
assert paths['segms'] != paths['output'], 'overwriting segmentation dir'
assert os.path.basename(paths['images']) != paths['output'], \
'overwriting image dir'
logging.info(tl_expt.string_dict(paths, desc='PATHS'))
if not os.path.exists(paths['output']):
assert os.path.isdir(os.path.dirname(paths['output'])), \
'missing folder: %s' % os.path.dirname(paths['output'])
os.mkdir(paths['output'])
paths_imgs = glob.glob(paths['images'])
logging.info('found %i images in dir "%s"', len(paths_imgs),
paths['images'])
_warped_overlap = partial(perform_visu_overlap,
paths=paths,
segm_alpha=segm_alpha)
created = []
iterate = tl_expt.WrapExecuteSequence(_warped_overlap,
paths_imgs,
nb_jobs=nb_jobs,
desc='overlapping')
for r in iterate:
created.append(r)
logging.info('matched and created %i overlaps', np.sum(created))
logging.info('DONE')
def main(params):
""" PIPELINE candidate clustering
:param {str: any} params:
"""
params['path_expt'] = os.path.join(params['path_output'], FOLDER_EXPERIMENT % params['name'])
tl_expt.save_config_yaml(os.path.join(params['path_expt'], NAME_YAML_PARAMS), params)
tl_expt.create_subfolders(params['path_expt'], LIST_SUBDIRS)
list_paths = [params[k] for k in ['path_images', 'path_segms', 'path_centers']]
df_paths = tl_data.find_files_match_names_across_dirs(list_paths)
df_paths.columns = ['path_image', 'path_segm', 'path_points']
df_paths.index = range(1, len(df_paths) + 1)
path_cover = os.path.join(params['path_expt'], run_train.NAME_CSV_TRIPLES)
df_paths.to_csv(path_cover)
logging.info('run clustering...')
df_paths_new = pd.DataFrame()
_wrapper_clustering = partial(cluster_points_draw_export, params=params, path_out=params['path_expt'])
rows = (dict(row) for idx, row in df_paths.iterrows())
iterate = tl_expt.WrapExecuteSequence(_wrapper_clustering, rows, nb_workers=params['nb_workers'])
for dict_center in iterate:
df_paths_new = df_paths_new.append(dict_center, ignore_index=True)
df_paths_new.set_index('image', inplace=True)
df_paths_new.to_csv(path_cover)
def quantize_folder_images(
path_images, colors=None, method='color', px_threshold=THRESHOLD_INVALID_PIXELS, nb_workers=1
):
""" perform single or multi thread image quantisation
:param str path_images:, input directory and image pattern for loading
:param colors: [(int, int, int)], list of possible colours
:param str method: interpolation method
:param float px_threshold: pixel threshold
:param int nb_workers: number of jobs
"""
path_imgs = sorted(glob.glob(path_images))
logging.info('found %i images', len(path_imgs))
if colors is None:
dict_colors = see_images_color_info(path_images, px_thr=px_threshold)
colors = list(dict_colors)
_wrapper_quantize_img = partial(perform_quantize_image, method=method, list_colors=colors)
iterate = tl_expt.WrapExecuteSequence(
_wrapper_quantize_img,
path_imgs,
nb_workers=nb_workers,
desc='quantize images',
)
list(iterate)
def main_predict(path_classif,
path_pattern_imgs,
path_out,
name='SEGMENT___',
params_local=None):
""" given trained classifier segment new images
:param str path_classif:
:param str path_pattern_imgs:
:param str path_out:
:param str name:
"""
logging.getLogger().setLevel(logging.INFO)
logging.info('running PREDICTION...')
assert path_pattern_imgs is not None
dict_classif = seg_clf.load_classifier(path_classif)
classif = dict_classif['clf_pipeline']
params = dict_classif['params']
if params_local is not None:
params.update({
k: params_local[k]
for k in params_local
if k.startswith('path_') or k.startswith('gc_')
})
path_out, path_visu = prepare_output_dir(path_pattern_imgs,
path_out,
name,
visual=params.get(
'visual', False))
tl_expt.set_experiment_logger(path_out)
logging.info(tl_expt.string_dict(params, desc='PARAMETERS'))
paths_img = sorted(glob.glob(path_pattern_imgs))
logging.info('found %i images on path "%s"', len(paths_img),
path_pattern_imgs)
logging.debug('run prediction...')
show_debug_imgs = params.get('visual', False)
_wrapper_segment = partial(
try_segment_image,
params=params,
classif=classif,
path_out=path_out,
path_visu=path_visu,
show_debug_imgs=show_debug_imgs,
)
list_img_path = list(zip([None] * len(paths_img), paths_img))
iterate = tl_expt.WrapExecuteSequence(
_wrapper_segment,
list_img_path,
nb_workers=params['nb_workers'],
desc='segmenting images',
)
for _ in iterate:
gc.collect()
time.sleep(1)
logging.info('prediction DONE')
def main(params):
""" compute the distance among segmented superpixels and given annotation
:param dict params:
"""
if os.path.isdir(params['path_out']):
logging.info('Missing output dir -> no visual export & results table.')
list_paths = [params['path_images'], params['path_segms']]
df_paths = tl_data.find_files_match_names_across_dirs(list_paths)
df_paths.columns = ['path_image', 'path_segm']
df_dist = pd.DataFrame()
_wrapper_eval = partial(compute_boundary_distance,
params=params,
path_out=params['path_out'])
iterate = tl_expt.WrapExecuteSequence(_wrapper_eval,
df_paths.iterrows(),
nb_workers=params['nb_workers'],
desc='evaluate SLIC')
for name, dist in iterate:
df_dist = df_dist.append({
'name': name,
'mean boundary distance': dist
},
ignore_index=True)
df_dist.set_index('name', inplace=True)
if os.path.isdir(params['path_out']):
csv_name = NAME_CSV_DISTANCES % (params['slic_size'],
params['slic_regul'], params['slico'])
df_dist.to_csv(os.path.join(params['path_out'], csv_name))
logging.info('STATISTIC:')
logging.info(df_dist.describe())
def perform_stage(df_group, stage, path_images, path_out):
""" perform cutting images for a particular development stage
and nom them into common image size
:param df_group: input dataframe with ellipse parameters
:param int stage: index of development stage
:param str path_images: path to the image folder
:param str path_out: path to the output folder
"""
logging.info('stage %i listing %i items' % (stage, len(df_group)))
stat_a = NORM_FUNC(df_group['ellipse_a'])
stat_b = NORM_FUNC(df_group['ellipse_b'])
norm_size = (int(stat_b), int(stat_a))
logging.info('normal dimension is %s' % repr(norm_size))
path_out_stage = os.path.join(path_out, str(stage))
if not os.path.isdir(path_out_stage):
os.mkdir(path_out_stage)
_wrapper_object = partial(extract_ellipse_object,
path_images=path_images,
path_out=path_out_stage,
norm_size=norm_size)
desc = 'stage %i - size %s' % (stage, norm_size)
iterate = tl_expt.WrapExecuteSequence(_wrapper_object,
df_group.iterrows(),
nb_jobs=params['nb_jobs'],
desc=desc)
list(iterate)
def main(dict_paths, export_visual=EXPORT_VUSIALISATION, nb_workers=NB_THREADS):
""" evaluate all segmentations in experiment folder
:param {str: str} paths: path to all required directories
:param bool export_visual: export visualisations
:param int nb_workers: number threads in parralel
"""
logging.info('running in %i jobs...', nb_workers)
logging.info(tl_expt.string_dict(dict_paths, desc='PATHS'))
list_results = sorted(glob.glob(os.path.join(dict_paths['results'], '*')))
_if_path = lambda p: all((os.path.isdir(p),
'___' not in os.path.basename(p),
os.path.basename(p) not in SKIP_DIRS))
list_results = sorted([p for p in list_results if _if_path(p)])
tl_expt.create_subfolders(dict_paths['results'],
[NAME_DIR_VISUAL_1, NAME_DIR_VISUAL_2, NAME_DIR_VISUAL_3])
df_all = pd.DataFrame()
_wrapper_eval = partial(evaluate_folder, dict_paths=dict_paths,
export_visual=export_visual)
iterate = tl_expt.WrapExecuteSequence(_wrapper_eval, list_results,
nb_workers=nb_workers)
for dict_eval in iterate:
df_all = df_all.append(dict_eval, ignore_index=True)
df_all.set_index(['method'], inplace=True)
df_all.sort_index(inplace=True)
logging.info('STATISTIC: \n %r', df_all)
df_all.to_csv(os.path.join(dict_paths['results'], NAME_CSV_STAT % 'OVERALL'))
def main(params, debug_export=DEBUG_EXPORT):
""" the main entry point
:param dict params: segmentation parameters
:param bool debug_export: whether export visualisations
"""
logging.getLogger().setLevel(logging.DEBUG)
params = tl_expt.create_experiment_folder(
params, dir_name=NAME_EXPERIMENT, stamp_unique=EACH_UNIQUE_EXPERIMENT)
tl_expt.set_experiment_logger(params['path_exp'])
logging.info(tl_expt.string_dict(params, desc='PARAMETERS'))
# tl_expt.create_subfolders(params['path_exp'], [FOLDER_IMAGE])
df_paths = pd.read_csv(params['path_list'], index_col=0)
logging.info('loaded %i items with columns: %r', len(df_paths),
df_paths.columns.tolist())
df_paths.dropna(how='any', inplace=True)
# create sub-folders if required
tl_expt.create_subfolders(params['path_exp'], ['input', 'simple'])
dict_segment = create_dict_segmentation(params, None, None, None, None)
dirs_center = [n + DIR_CENTRE_POSIX for n in dict_segment]
dirs_visu = [n + DIR_VISUAL_POSIX for n in dict_segment]
tl_expt.create_subfolders(params['path_exp'], [n for n in dict_segment] +
dirs_center + dirs_visu)
if debug_export:
list_dirs = [n + DIR_DEBUG_POSIX for n in dict_segment if 'rg2sp' in n]
tl_expt.create_subfolders(params['path_exp'], list_dirs)
_wrapper_segment = partial(image_segmentation, params=params)
iterate = tl_expt.WrapExecuteSequence(_wrapper_segment,
df_paths.iterrows(),
nb_workers=params['nb_workers'])
list(iterate)
def main(dict_paths,
padding=0,
use_mask=False,
bg_color=None,
nb_workers=NB_WORKERS):
""" the main executable
:param dict_paths:
:param int padding:
:param int nb_workers:
"""
if not os.path.isdir(dict_paths['output']):
if not os.path.isdir(os.path.dirname(dict_paths['output'])):
raise NotADirectoryError('"%s" should be folder' %
dict_paths['output'])
logging.debug('creating dir: %s', dict_paths['output'])
os.mkdir(dict_paths['output'])
list_dirs = [dict_paths['annot'], dict_paths['image']]
df_paths = tl_data.find_files_match_names_across_dirs(list_dirs)
logging.info('start cutting images')
_wrapper_cutting = partial(
export_cut_objects,
path_out=dict_paths['output'],
padding=padding,
use_mask=use_mask,
bg_color=bg_color,
)
iterate = tl_expt.WrapExecuteSequence(
_wrapper_cutting,
(row for idx, row in df_paths.iterrows()),
nb_workers=nb_workers,
)
list(iterate)
def compute_mean_image(list_img_paths):
iterate = tl_expt.WrapExecuteSequence(tl_data.load_image_2d, list_img_paths,
desc='compute mean image')
imgs = [im[:, :, IMAGE_CHANNEL] for im, _ in iterate]
min_size = np.min([img.shape for img in imgs], axis=0)
imgs = [img[:min_size[0], :min_size[1]] for img in imgs]
img_mean = np.median(imgs, axis=0)
return img_mean
def evaluate_detection_stage(df_paths,
stage,
path_info,
path_out,
nb_workers=1):
""" evaluate center detection for particular list of stages
:param df_paths:
:param [int] stage:
:param str path_info:
:param str path_out:
:param int nb_workers:
:return DF:
"""
logging.info('evaluate stages: %r', stage)
str_stage = '-'.join(map(str, stage))
path_csv = os.path.join(path_out, NAME_CSV_ANNOT_STAGE % str_stage)
if not os.path.exists(path_csv) or FORCE_RELOAD:
df_slices_info = seg_annot.load_info_group_by_slices(path_info, stage)
logging.debug('export slices_info to "%s"', path_csv)
df_slices_info.to_csv(path_csv)
else:
logging.debug('loading slices_info from "%s"', path_csv)
df_slices_info = pd.read_csv(path_csv, index_col=0)
if df_slices_info.empty:
return df_paths
# df_paths = pd.merge(df_paths, df_slices_info, how='inner',
# left_index=True, right_index=True)
df_eval = pd.DataFrame()
path_annot = os.path.join(path_out, FOLDER_ANNOT % str_stage)
path_visu = os.path.join(path_out, FOLDER_ANNOT_VISUAL % str_stage)
list_dirs = [os.path.basename(p) for p in [path_annot, path_visu]]
logging.debug('create sub-dirs: %r', list_dirs)
tl_expt.create_subfolders(path_out, list_dirs)
# perfom on new images
stage_prefix = '[stage-%s] ' % str_stage
logging.info('start section %s - load_center_evaluate ...', stage_prefix)
_wrapper_detection = partial(
load_center_evaluate,
df_annot=df_slices_info,
path_annot=path_annot,
path_visu=path_visu,
col_prefix=stage_prefix,
)
iterate = tl_expt.WrapExecuteSequence(_wrapper_detection,
df_paths.iterrows(),
nb_workers=nb_workers)
for dict_eval in iterate:
df_eval = df_eval.append(dict_eval, ignore_index=True)
df_eval.to_csv(os.path.join(path_out, NAME_CSV_TRIPLES_TEMP))
# gc.collect(), time.sleep(1)
return df_eval
def filter_train_with_purity(dict_imgs, dict_labels, dict_label_hist,
label_purity, dict_slics, drop_labels=None,
path_visu=None, nb_workers=NB_WORKERS):
_w_filter = partial(wrapper_filter_labels, label_purity=label_purity,
drop_labels=drop_labels, path_visu=path_visu)
iter_vals = ((n, dict_imgs[n], dict_labels[n], dict_slics[n],
dict_label_hist[n]) for n in dict_labels)
iterate = tl_expt.WrapExecuteSequence(_w_filter, iter_vals, nb_workers=nb_workers,
desc='filter labels (purity)')
for n, lbs in iterate:
dict_labels[n] = lbs
return dict_labels
def estim_model_classes_group(list_images,
nb_classes,
dict_features,
sp_size=30,
sp_regul=0.2,
use_scaler=True,
pca_coef=None,
model_type='GMM',
nb_jobs=NB_THREADS):
""" estimate a model from sequence of input images and return it as result
:param [ndarray] list_images:
:param int nb_classes: number of classes
:param int sp_size: initial size of a superpixel(meaning edge lenght)
:param float sp_regul: regularisation in range(0;1) where "0" gives elastic
and "1" nearly square slic
:param {str: [str]} dict_features: list of features to be extracted
:param float pca_coef: range (0, 1) or None
:param bool use_scaler: whether use a scaler
:param str model_type: model type
:param int nb_jobs: number of jobs running in parallel
:return:
"""
list_slic, list_features = list(), list()
_wrapper_compute = partial(compute_color2d_superpixels_features,
sp_size=sp_size,
sp_regul=sp_regul,
dict_features=dict_features)
iterate = tl_expt.WrapExecuteSequence(_wrapper_compute,
list_images,
desc='compute SLIC & features',
nb_jobs=nb_jobs)
for slic, features in iterate:
list_slic.append(slic)
list_features.append(features)
# for img in list_images:
# slic, features = compute_color2d_superpixels_features(img, sp_size,
# sp_regul, dict_features, fts_norm=False)
# list_slic.append(slic)
# list_features.append(features)
features = np.concatenate(tuple(list_features), axis=0)
features = np.nan_to_num(features)
model = seg_gc.estim_class_model(features, nb_classes, model_type,
pca_coef, use_scaler)
return model, list_features
def quantize_folder_images(path_images, label, nb_workers=1):
""" perform single or multi thread image quantisation
:param list(str) path_images: list of image paths
:param int nb_workers:
"""
assert os.path.isdir(os.path.dirname(path_images)), \
'input folder does not exist: %s' % os.path.dirname(path_images)
path_imgs = sorted(glob.glob(path_images))
logging.info('found %i images', len(path_imgs))
_wrapper_img_inpaint = partial(perform_img_inpaint, labels=label)
iterate = tl_expt.WrapExecuteSequence(
_wrapper_img_inpaint, path_imgs, nb_workers=nb_workers, desc='quantise images'
)
list(iterate)
def main(params):
""" PIPELINE for new detections
:param {str: str} params:
"""
params = run_train.prepare_experiment_folder(params, FOLDER_EXPERIMENT)
# run_train.check_pathes_patterns(paths)
tl_expt.set_experiment_logger(params['path_expt'])
logging.info('COMPUTER: \n%r', platform.uname())
logging.info(tl_expt.string_dict(params, desc='PARAMETERS'))
tl_expt.create_subfolders(params['path_expt'], LIST_SUBFOLDER)
path_csv = os.path.join(params['path_expt'], NAME_CSV_TRIPLES)
df_paths = get_csv_triplets(params['path_list'],
path_csv,
params['path_images'],
params['path_segms'],
force_reload=FORCE_RERUN)
dict_classif = seg_clf.load_classifier(params['path_classif'])
params_clf = dict_classif['params']
params_clf.update(params)
logging.info(tl_expt.string_dict(params, desc='UPDATED PARAMETERS'))
# perform on new images
df_stat = pd.DataFrame()
_wrapper_detection = partial(
load_compute_detect_centers,
params=params_clf,
path_classif=params['path_classif'],
path_output=params['path_expt'],
)
iterate = tl_expt.WrapExecuteSequence(_wrapper_detection,
df_paths.iterrows(),
nb_workers=params['nb_workers'])
for dict_center in iterate:
df_stat = df_stat.append(dict_center, ignore_index=True)
df_stat.to_csv(os.path.join(params['path_expt'],
NAME_CSV_TRIPLES_TEMP))
df_stat.set_index(['image'], inplace=True)
df_stat.to_csv(os.path.join(params['path_expt'], NAME_CSV_TRIPLES))
logging.info('STATISTIC: \n %r', df_stat.describe())
def main(params):
df_paths = tl_data.find_files_match_names_across_dirs([params['path_images'],
params['path_segms'],
params['path_centers']])
df_paths.columns = ['path_image', 'path_segm', 'path_centers']
df_paths.index = range(1, len(df_paths) + 1)
if not os.path.exists(params['path_output']):
assert os.path.exists(os.path.dirname(params['path_output'])), \
'missing folder: "%s"' % os.path.dirname(params['path_output'])
os.mkdir(params['path_output'])
df_slices_info = seg_annot.load_info_group_by_slices(params['path_infofile'],
params['stages'])
_wrapper_export = partial(export_figure, df_slices_info=df_slices_info,
path_out=params['path_output'])
iterate = tl_expt.WrapExecuteSequence(_wrapper_export, df_paths.iterrows(),
nb_workers=params['nb_workers'])
list(iterate)