def main(paths, nb_jobs=NB_THREADS, segm_alpha=MIDDLE_ALPHA_OVERLAP): logging.info('running...') assert paths['segms'] != paths['output'], 'overwriting segmentation dir' assert os.path.basename(paths['images']) != paths['output'], \ 'overwriting image dir' logging.info(tl_expt.string_dict(paths, desc='PATHS')) if not os.path.exists(paths['output']): assert os.path.isdir(os.path.dirname(paths['output'])), \ 'missing folder: %s' % os.path.dirname(paths['output']) os.mkdir(paths['output']) paths_imgs = glob.glob(paths['images']) logging.info('found %i images in dir "%s"', len(paths_imgs), paths['images']) _warped_overlap = partial(perform_visu_overlap, paths=paths, segm_alpha=segm_alpha) created = [] iterate = tl_expt.WrapExecuteSequence(_warped_overlap, paths_imgs, nb_jobs=nb_jobs, desc='overlapping') for r in iterate: created.append(r) logging.info('matched and created %i overlaps', np.sum(created)) logging.info('DONE')
def relabel_folder_images(path_images, path_out, labels_old, labels_new, nb_jobs=1): """ perform single or multi thread image quantisation :param [int] labels_old: :param [int] labels_new: :param [str] path_images: list of input images :param path_out: output directory :param [int] labels_old: list of labels to be replaced :param [int] labels_new: list of new labels :param int nb_jobs: """ assert os.path.isdir(os.path.dirname(path_images)), \ 'missing folder: %s' % path_images assert os.path.isdir(path_out), 'missing ouput folder: %s' % path_out path_imgs = sorted(glob.glob(path_images)) logging.info('found %i images', len(path_imgs)) _wrapper_img_relabel = partial(perform_image_relabel, path_out=path_out, labels_old=labels_old, labels_new=labels_new) iterate = tl_expt.WrapExecuteSequence(_wrapper_img_relabel, path_imgs, nb_jobs=nb_jobs, desc='relabel images') list(iterate)
def perform_stage(df_group, stage, path_images, path_out): """ perform cutting images for a particular development stage and nom them into common image size :param df_group: input dataframe with ellipse parameters :param int stage: index of development stage :param str path_images: path to the image folder :param str path_out: path to the output folder """ logging.info('stage %i listing %i items' % (stage, len(df_group))) stat_a = NORM_FUNC(df_group['ellipse_a']) stat_b = NORM_FUNC(df_group['ellipse_b']) norm_size = (int(stat_b), int(stat_a)) logging.info('normal dimension is %s' % repr(norm_size)) path_out_stage = os.path.join(path_out, str(stage)) if not os.path.isdir(path_out_stage): os.mkdir(path_out_stage) wrapper_object = partial(extract_ellipse_object, path_images=path_images, path_out=path_out_stage, norm_size=norm_size) desc = 'stage %i - size %s' % (stage, norm_size) iterate = tl_expt.WrapExecuteSequence(wrapper_object, df_group.iterrows(), nb_jobs=params['nb_jobs'], desc=desc) list(iterate)
def main(params): """ PIPELINE for matching :param {str: str} params: """ logging.info(tl_expt.string_dict(params, desc='PARAMETERS')) df_info = pd.read_csv(params['path_infofile'], sep='\t', index_col=0) df_info = filter_table(df_info, params['path_ellipses']) logging.info('filtered %i item in table' % len(df_info)) path_csv = os.path.join(params['path_output'], NAME_CSV_RESULTS) list_evals = [] # get the folder path_dir_csv = os.path.dirname(params['path_ellipses']) _wrapper_match = partial(select_optimal_ellipse, path_dir_csv=path_dir_csv) iterate = tl_expt.WrapExecuteSequence(_wrapper_match, df_info.iterrows(), nb_jobs=params['nb_jobs']) for i, dict_row in enumerate(iterate): list_evals.append(dict_row) # every hundreds iteration do export if i % 100 == 0: pd.DataFrame(list_evals).to_csv(path_csv) df_ellipses = pd.DataFrame(list_evals) df_ellipses.to_csv(path_csv)
def main(params): """ PIPELINE for rotation :param {str: str} params: """ logging.info('running...') logging.info(tl_expt.string_dict(params, desc='PARAMETERS')) list_imgs = sorted( [p for p in glob.glob(params['path_images']) if os.path.isfile(p)]) logging.info('found images: %i' % len(list_imgs)) if not os.path.isdir(params['path_output']): os.mkdir(params['path_output']) wrapper_object = partial(perform_orientation_swap, path_out=params['path_output']) dir_name = os.path.dirname(params['path_images']) iterate = tl_expt.WrapExecuteSequence(wrapper_object, list_imgs, nb_jobs=params['nb_jobs'], desc=dir_name) list(iterate) logging.info('DONE')
def main(path_segs, path_out, nb_jobs): """ the main for creating annotations :param str path_segs: path with image pattern of images - obj segmentation :param str path_out: :param int nb_jobs: number of processes in parallel :return ndarray: """ logging.info('running...') assert os.path.dirname(path_segs) != path_out, \ 'the output dir has to be different then the input object segmentation' list_imgs = glob.glob(path_segs) logging.info('found %i images', len(list_imgs)) if not os.path.exists(path_out): assert os.path.isdir(os.path.dirname(path_out)), \ 'missing: %s' % path_out os.mkdir(path_out) wrapper_create_annot_centers = partial(create_annot_centers, path_out_seg=path_out, path_out_csv=path_out) iterate = tl_expt.WrapExecuteSequence(wrapper_create_annot_centers, list_imgs, nb_jobs=nb_jobs, desc='annotating images') list(iterate) logging.info('DONE')
def main(params): """ PIPELINE candidate clustering :param {str: any} params: """ with open(os.path.join(params['path_expt'], 'config_clustering.json'), 'w') as fp: json.dump(params, fp) tl_expt.create_subfolders(params['path_expt'], LIST_SUBDIRS) list_paths = [params[k] for k in ['path_images', 'path_segms', 'path_centers']] df_paths = tl_data.find_files_match_names_across_dirs(list_paths) df_paths.columns = ['path_image', 'path_segm', 'path_points'] df_paths.index = range(1, len(df_paths) + 1) path_cover = os.path.join(params['path_expt'], run_train.NAME_CSV_TRIPLES) df_paths.to_csv(path_cover) logging.info('run clustering...') df_paths_new = pd.DataFrame() _wrapper_clustering = partial(cluster_points_draw_export, params=params, path_out=params['path_expt']) rows = (dict(row) for idx, row in df_paths.iterrows()) iterate = tl_expt.WrapExecuteSequence(_wrapper_clustering, rows, nb_jobs=params['nb_jobs']) for dict_center in iterate: df_paths_new = df_paths_new.append(dict_center, ignore_index=True) df_paths_new.set_index('image', inplace=True) df_paths_new.to_csv(path_cover)
def main(dict_paths, padding=0, use_mask=False, bg_color=None, nb_jobs=NB_THREADS): """ the main executable :param dict_paths: :param int padding: :param int nb_jobs: """ logging.info('running...') if not os.path.isdir(dict_paths['output']): assert os.path.isdir(os.path.dirname(dict_paths['output'])), \ '"%s" should be folder' % dict_paths['output'] logging.debug('creating dir: %s', dict_paths['output']) os.mkdir(dict_paths['output']) list_dirs = [dict_paths['annot'], dict_paths['image']] df_paths = tl_data.find_files_match_names_across_dirs(list_dirs) logging.info('start cutting images') wrapper_cutting = partial(export_cut_objects, path_out=dict_paths['output'], padding=padding, use_mask=use_mask, bg_color=bg_color) iterate = tl_expt.WrapExecuteSequence(wrapper_cutting, (row for idx, row in df_paths.iterrows()), nb_jobs=nb_jobs) list(iterate) logging.info('DONE')
def main(paths, nb_jobs=NB_THREADS): logging.info('running...') logging.info(tl_expt.string_dict(paths, desc='PATHS')) if not os.path.exists(paths['output']): assert os.path.isdir(os.path.dirname(paths['output'])), \ 'missing folder: %s' % os.path.dirname(paths['output']) os.mkdir(paths['output']) paths_imgs = glob.glob(paths['images']) logging.info('found %i images in dir "%s"', len(paths_imgs), paths['images']) warped_overlap = partial(perform_visu_overlap, paths=paths) created = [] iterate = tl_expt.WrapExecuteSequence(warped_overlap, paths_imgs, nb_jobs=nb_jobs, desc='overlapping') for r in iterate: created.append(r) logging.info('matched and created %i overlaps', np.sum(created)) logging.info('DONE')
def experiment_loo(params, df_stat, dict_annot, paths_img, path_classif, path_dump): imgs_idx_path = list(zip(range(1, len(paths_img) + 1), paths_img)) logging.info('run prediction on training images as Leave-One-Out...') dict_segms, dict_segms_gc = dict(), dict() path_out = os.path.join(params['path_exp'], FOLDER_LOO) path_visu = os.path.join(params['path_exp'], FOLDER_LOO_VISU) wrapper_segment = partial(retrain_loo_segment_image, path_classif=path_classif, path_dump=path_dump, path_out=path_out, path_visu=path_visu) iterate = tl_expt.WrapExecuteSequence(wrapper_segment, imgs_idx_path, nb_jobs=params['nb_jobs'], desc='experiment LOO') for name, segm, segm_gc in iterate: dict_segms[name] = segm dict_segms_gc[name] = segm_gc gc.collect() time.sleep(1) df = eval_segment_with_annot(params, dict_annot, dict_segms, None, NAME_CSV_SEGM_STAT_RESULT_LOO, params['nb_jobs']) df_stat = df_stat.append(get_summary(df, 'segm (LOO)'), ignore_index=True) df = eval_segment_with_annot(params, dict_annot, dict_segms_gc, None, NAME_CSV_SEGM_STAT_RESULT_LOO_GC, params['nb_jobs']) df_stat = df_stat.append(get_summary(df, 'segm GC (LOO)'), ignore_index=True) path_csv_stat = os.path.join(params['path_exp'], NAME_CSV_SEGM_STAT_RESULTS) df_stat.set_index(['name']).to_csv(path_csv_stat) return df_stat
def dataset_load_images_annot_compute_features(params, show_debug_imgs=SHOW_DEBUG_IMAGES): """ for all datasets perform the following steps: 1) load image and annotation 2) compute superpixel features and labels :param {str: ...} params: segmentation parameters :param bool show_debug_imgs: whether show debug images :return ({str: ndarray} * 6, [str]): """ dict_images, dict_annots = dict(), dict() dict_slics, dict_features, dict_labels, dict_label_hist = \ dict(), dict(), dict(), dict() feature_names = list() # compute features df_paths = pd.read_csv(params['path_train_list'], index_col=0) assert all(n in df_paths.columns for n in ['path_image', 'path_annot']), \ 'missing required columns in loaded csv file' wrapper_load_compute = partial(load_image_annot_compute_features_labels, params=params, show_debug_imgs=show_debug_imgs) iterate = tl_expt.WrapExecuteSequence(wrapper_load_compute, df_paths.iterrows(), nb_jobs=params['nb_jobs'], desc='extract training data') for name, img, annot, slic, features, labels, label_hist, feature_names in iterate: dict_images[name] = img dict_annots[name] = annot dict_slics[name] = slic dict_features[name] = features dict_labels[name] = labels dict_label_hist[name] = label_hist # gc.collect(), time.sleep(1) return dict_images, dict_annots, dict_slics, dict_features, dict_labels, \ dict_label_hist, feature_names
def convert_folder_images(path_images, path_out, path_json=None, nb_jobs=1): """ perform single or multi thread image quantisation :param [str] path_images: list of input images :param path_out: output directory :param path_json: path to json file :param int nb_jobs: int """ assert os.path.isdir(os.path.dirname(path_images)), \ 'input folder does not exist' path_imgs = sorted(glob.glob(path_images)) logging.info('found %i images', len(path_imgs)) if not os.path.exists(path_out): assert os.path.isdir(os.path.dirname(path_out)), \ 'missing folder: %s' % os.path.dirname(path_out) os.mkdir(path_out) dict_colors = load_dict_colours(path_json) logging.debug('loaded dictionary %s', repr(dict_colors)) wrapper_img_convert = partial(perform_img_convert, path_out=path_out, dict_colors=dict_colors) iterate = tl_expt.WrapExecuteSequence(wrapper_img_convert, path_imgs, nb_jobs=nb_jobs, desc='convert images') list(iterate)
def experiment_lpo(params, df_stat, dict_annot, idx_paths_img, path_classif, path_dump, nb_holdout, show_debug_imgs=SHOW_DEBUG_IMAGES): """ experiment Leave-P-samples-Out :param {str: ...} params: :param DF df_stat: :param {str: ndarray} dict_annot: :param [str] paths_img: :param str path_classif: :param str path_dump: :param int nb_holdout: :param bool show_debug_imgs: whether show debug images :return {}: """ logging.info('run prediction on training images as Leave-%i-Out...', nb_holdout) dict_segms, dict_segms_gc = dict(), dict() cv = seg_clf.CrossValidatePOut(len(idx_paths_img), nb_hold_out=nb_holdout) test_imgs_idx_path = [[idx_paths_img[i] for i in ids] for _, ids in cv] path_out = os.path.join(params['path_exp'], FOLDER_LPO) path_visu = os.path.join(params['path_exp'], FOLDER_LPO_VISU) \ if params.get('visual', False) else None _wrapper_segment = partial(retrain_lpo_segment_image, path_classif=path_classif, path_dump=path_dump, path_out=path_out, path_visu=path_visu, show_debug_imgs=show_debug_imgs) iterate = tl_expt.WrapExecuteSequence(_wrapper_segment, test_imgs_idx_path, nb_jobs=params['nb_jobs'], desc='experiment LPO') for dict_seg, dict_seg_gc in iterate: dict_segms.update(dict_seg) dict_segms_gc.update(dict_seg_gc) gc.collect() time.sleep(1) df = eval_segment_with_annot(params, dict_annot, dict_segms, None, NAME_CSV_SEGM_STAT_RESULT_LPO % nb_holdout, params.get('drop_labels', None), params['nb_jobs']) df_stat = df_stat.append(get_summary(df, 'segm (L-%i-O)' % nb_holdout), ignore_index=True) df = eval_segment_with_annot(params, dict_annot, dict_segms_gc, None, NAME_CSV_SEGM_STAT_RESULT_LPO_GC % nb_holdout, params.get('drop_labels', None), params['nb_jobs']) df_stat = df_stat.append(get_summary(df, 'segm GC (L-%i-O)' % nb_holdout), ignore_index=True) path_csv_stat = os.path.join(params['path_exp'], NAME_CSV_SEGM_STAT_RESULTS) df_stat.set_index(['name']).to_csv(path_csv_stat) return df_stat
def experiment_lpo(params, df_stat, dict_annot, paths_img, path_classif, path_dump, nb_holdout): imgs_idx_path = list(zip(range(1, len(paths_img) + 1), paths_img)) logging.info('run prediction on training images as Leave-%i-Out...', nb_holdout) dict_segms, dict_segms_gc = dict(), dict() cv = seg_clf.CrossValidatePOut(len(paths_img), nb_hold_out=nb_holdout) test_imgs_idx_path = [[imgs_idx_path[i] for i in ids] for _, ids in cv] path_out = os.path.join(params['path_exp'], FOLDER_LPO) path_visu = os.path.join(params['path_exp'], FOLDER_LPO_VISU) wrapper_segment = partial(retrain_lpo_segment_image, path_classif=path_classif, path_dump=path_dump, path_out=path_out, path_visu=path_visu) iterate = tl_expt.WrapExecuteSequence(wrapper_segment, test_imgs_idx_path, nb_jobs=params['nb_jobs'], desc='experiment LPO') for dict_seg, dict_seg_gc in iterate: dict_segms.update(dict_seg) dict_segms_gc.update(dict_seg_gc) gc.collect() time.sleep(1) df = eval_segment_with_annot(params, dict_annot, dict_segms, None, NAME_CSV_SEGM_STAT_RESULT_LPO % nb_holdout, params['nb_jobs']) df_stat = df_stat.append(get_summary(df, 'segm (L-%i-O)' % nb_holdout), ignore_index=True) df = eval_segment_with_annot(params, dict_annot, dict_segms_gc, None, NAME_CSV_SEGM_STAT_RESULT_LPO_GC % nb_holdout, params['nb_jobs']) df_stat = df_stat.append(get_summary(df, 'segm GC (L-%i-O)' % nb_holdout), ignore_index=True) path_csv_stat = os.path.join(params['path_exp'], NAME_CSV_SEGM_STAT_RESULTS) df_stat.set_index(['name']).to_csv(path_csv_stat) return df_stat
def main(params): """ compute the distance among segmented superpixels and given annotation :param {str: ...} params: """ logging.info('running...') if os.path.isdir(params['path_out']): logging.info('Missing output dir -> no visual export & results table.') list_paths = [params['path_images'], params['path_segms']] df_paths = tl_data.find_files_match_names_across_dirs(list_paths) df_paths.columns = ['path_image', 'path_segm'] df_dist = pd.DataFrame() wrapper_eval = partial(compute_boundary_distance, params=params, path_out=params['path_out']) iterate = tl_expt.WrapExecuteSequence(wrapper_eval, df_paths.iterrows(), nb_jobs=params['nb_jobs'], desc='evaluate SLIC') for name, dist in iterate: df_dist = df_dist.append({'name': name, 'mean boundary distance': dist}, ignore_index=True) df_dist.set_index('name', inplace=True) if os.path.isdir(params['path_out']): df_dist.to_csv(os.path.join(params['path_out'], NAME_CSV_DISTANCES)) logging.info('STATISTIC:') logging.info(df_dist.describe()) logging.info('DONE')
def export_dataset_visual(path_output, dict_imgs, dict_segms, dict_slics, dict_points, dict_labels, nb_jobs=NB_THREADS): """ visualise complete training dataset by marking labeld points over image and input segmentation :param {str: ndarray} dict_imgs: :param {str: ndarray} dict_segms: :param {str: ndarray} dict_slics: :param {str: ndarray} dict_points: :param {str: ndarray} dict_labels: :param int nb_jobs: number processing in parallel """ logging.info('export training visualisations') path_out = os.path.join(path_output, FOLDER_POINTS_TRAIN) gener_args = ((path_out, name, dict_imgs[name], dict_segms[name], dict_points[name], dict_labels[name], dict_slics[name], None, '_train') for name in dict_imgs) iterate = tl_expt.WrapExecuteSequence(wrapper_draw_export_slic_centers, gener_args, nb_jobs=nb_jobs, desc='exporting visualisations') list(iterate)
def main(params, debug_export=DEBUG_EXPORT): """ the main entry point :param {str: ...} params: segmentation parameters :param bool debug_export: whether export visualisations """ logging.getLogger().setLevel(logging.DEBUG) params = tl_expt.create_experiment_folder( params, dir_name=NAME_EXPERIMENT, stamp_unique=EACH_UNIQUE_EXPERIMENT) tl_expt.set_experiment_logger(params['path_exp']) logging.info(tl_expt.string_dict(params, desc='PARAMETERS')) # tl_expt.create_subfolders(params['path_exp'], [FOLDER_IMAGE]) df_paths = pd.read_csv(params['path_list'], index_col=0) logging.info('loaded %i items with columns: %s', len(df_paths), repr(df_paths.columns.tolist())) df_paths.dropna(how='any', inplace=True) # create sub-folders if required tl_expt.create_subfolders(params['path_exp'], ['input', 'simple']) dict_segment = create_dict_segmentation(params, None, None, None, None) tl_expt.create_subfolders(params['path_exp'], [n for n in dict_segment] + [n + DIR_CENTRE_POSIX for n in dict_segment] + [n + DIR_VISUAL_POSIX for n in dict_segment]) if debug_export: list_dirs = [n + DIR_DEBUG_POSIX for n in dict_segment if 'rg2sp' in n] tl_expt.create_subfolders(params['path_exp'], list_dirs) _wrapper_segment = partial(image_segmentation, params=params) iterate = tl_expt.WrapExecuteSequence(_wrapper_segment, df_paths.iterrows(), nb_jobs=params['nb_jobs']) list(iterate)
def quantize_folder_images(path_images, list_colors=None, method='color', px_threshold=THRESHOLD_INVALID_PIXELS, nb_jobs=1): """ perform single or multi thread image quantisation :param str method: :param float px_threshold: :param str path_images:, input directory and image pattern for loading :param list_colors: [(int, int, int)], list of possible colours :param str method: interpolation method :param float px_threshold: :param nb_jobs: int """ path_imgs = sorted(glob.glob(path_images)) logging.info('found %i images', len(path_imgs)) if list_colors is None: dict_colors = see_images_color_info(path_images, px_thr=px_threshold) list_colors = [c for c in dict_colors] _wrapper_quantize_img = partial(perform_quantize_image, method=method, list_colors=list_colors) iterate = tl_expt.WrapExecuteSequence(_wrapper_quantize_img, path_imgs, nb_jobs=nb_jobs, desc='quantize images') list(iterate)
def compute_mean_image(list_img_paths): iterate = tl_expt.WrapExecuteSequence(tl_data.load_image_2d, list_img_paths, desc='compute mean image') imgs = [im[:, :, IMAGE_CHANNEL] for im, _ in iterate] min_size = np.min([img.shape for img in imgs], axis=0) imgs = [img[:min_size[0], :min_size[1]] for img in imgs] img_mean = np.median(imgs, axis=0) return img_mean
def estim_model_classes_group(list_images, nb_classes=4, clr_space='rgb', sp_size=30, sp_regul=0.2, dict_features=FTS_SET_SIMPLE, pca_coef=None, proba_type='GMM', nb_jobs=NB_THREADS): """ estimate a model from sequence of input images and return it as result :param [ndarray] list_images: :param int nb_classes: number of clasees :param str clr_space: chose the color space :param int sp_size: initial size of a superpixel(meaning edge lenght) :param float sp_regul: regularisation in range(0;1) where "0" gives elastic and "1" nearly square slic :param {str: [str]} dict_features: list of features to be extracted :param float pca_coef: range (0, 1) or None :param str proba_type: model type :param int nb_jobs: number of jobs running in parallel :return: """ list_slic, list_features = list(), list() wrapper_compute = partial(compute_color2d_superpixels_features, sp_size=sp_size, sp_regul=sp_regul, dict_features=dict_features, clr_space=clr_space, fts_norm=False) iterate = tl_expt.WrapExecuteSequence(wrapper_compute, list_images, nb_jobs=nb_jobs) for slic, features in iterate: list_slic.append(slic) list_features.append(features) # for img in list_images: # slic, features = compute_color2d_superpixels_features(img, sp_size, # sp_regul, dict_features, clr_space, fts_norm=False) # list_slic.append(slic) # list_features.append(features) features = np.concatenate(tuple(list_features), axis=0) features = np.nan_to_num(features) # scaling scaler = preprocessing.StandardScaler() scaler.fit(features) features = scaler.transform(features) pca = None if pca_coef is not None: pca = decomposition.PCA(pca_coef) features = pca.fit_transform(features) model = seg_gc.estim_class_model(features, nb_classes, proba_type) return scaler, pca, model
def main_predict(path_classif, path_pattern_imgs, path_out, name='SEGMENT___', params_local=None): """ given trained classifier segment new images :param str path_classif: :param str path_pattern_imgs: :param str path_out: :param str name: """ logging.getLogger().setLevel(logging.INFO) logging.info('running PREDICTION...') assert path_pattern_imgs is not None dict_classif = seg_clf.load_classifier(path_classif) classif = dict_classif['clf_pipeline'] params = dict_classif['params'] if params_local is not None: params.update({ k: params_local[k] for k in params_local if k.startswith('path_') or k.startswith('gc_') }) path_out, path_visu = prepare_output_dir(path_pattern_imgs, path_out, name, visual=params.get( 'visual', False)) tl_expt.set_experiment_logger(path_out) logging.info(tl_expt.string_dict(params, desc='PARAMETERS')) paths_img = sorted(glob.glob(path_pattern_imgs)) logging.info('found %i images on path "%s"', len(paths_img), path_pattern_imgs) logging.debug('run prediction...') show_debug_imgs = params.get('visual', False) _wrapper_segment = partial(try_segment_image, params=params, classif=classif, path_out=path_out, path_visu=path_visu, show_debug_imgs=show_debug_imgs) list_img_path = list(zip([None] * len(paths_img), paths_img)) iterate = tl_expt.WrapExecuteSequence(_wrapper_segment, list_img_path, nb_jobs=params['nb_jobs'], desc='segmenting images') for _ in iterate: gc.collect() time.sleep(1) logging.info('prediction DONE')
def main(dict_paths, nb_jobs=NB_THREADS, relabel=True): """ main evaluation :param {str: str} dict_paths: :param int nb_jobs: number of thred running in parallel :param bool relabel: whether relabel segmentation as sequential """ logging.info('running...') if not os.path.isdir(dict_paths['output']): assert os.path.isdir(os.path.dirname(dict_paths['output'])), \ 'missing folder: %s' % dict_paths['output'] os.mkdir(dict_paths['output']) name = os.path.basename(os.path.dirname(dict_paths['segm'])) list_dirs = [dict_paths['annot'], dict_paths['segm']] if dict_paths['image'] != '': list_dirs.append(dict_paths['image']) df_paths = tl_data.find_files_match_names_across_dirs(list_dirs) path_csv = os.path.join(dict_paths['output'], NAME_CVS_PER_IMAGE % name) df_paths.to_csv(path_csv) annots, _ = tl_data.load_images_list(df_paths['path_1'].values.tolist()) segms, names = tl_data.load_images_list(df_paths['path_2'].values.tolist()) logging.info('loaded %i annots and %i segms', len(annots), len(segms)) if relabel: annots = [relabel_sequential(annot)[0] for annot in annots] segms = list(map(wrapper_relabel_segm, zip(annots, segms))) path_csv = os.path.join(dict_paths['output'], NAME_CVS_PER_IMAGE % name) logging.debug('export to "%s"', path_csv) df_stat = seg_clf.compute_stat_per_image(segms, annots, names, nb_jobs) df_stat.to_csv(path_csv) path_csv = os.path.join(dict_paths['output'], NAME_CVS_OVERALL % name) logging.debug('export to "%s"', path_csv) df_desc = df_stat.describe() logging.info(df_desc.T[['count', 'mean', 'std']]) df_desc.to_csv(path_csv) path_visu = os.path.join(dict_paths['output'], '%s__visual' % name) if not os.path.isdir(path_visu): os.mkdir(path_visu) # for idx, row in df_paths.iterrows(): # export_visual(row, path_visu) wrapper_visual = partial(export_visual, path_out=path_visu) iterate = tl_expt.WrapExecuteSequence( wrapper_visual, (row for idx, row in df_paths.iterrows()), nb_jobs=nb_jobs) list(iterate) logging.info('DONE')
def evaluate_detection_stage(df_paths, stage, path_info, path_out, nb_jobs=1): """ evaluate center detection for particular list of stages :param df_paths: :param [int] stage: :param str path_info: :param str path_out: :param int nb_jobs: :return DF: """ logging.info('evaluate stages: %s', repr(stage)) str_stage = '-'.join(map(str, stage)) path_csv = os.path.join(path_out, NAME_CSV_ANNOT_STAGE % str_stage) if not os.path.exists(path_csv) or FORCE_RELOAD: df_slices_info = seg_annot.load_info_group_by_slices(path_info, stage) logging.debug('export slices_info to "%s"', path_csv) df_slices_info.to_csv(path_csv) else: logging.debug('loading slices_info from "%s"', path_csv) df_slices_info = pd.read_csv(path_csv, index_col=0) if len(df_slices_info) == 0: return df_paths # df_paths = pd.merge(df_paths, df_slices_info, how='inner', # left_index=True, right_index=True) df_eval = pd.DataFrame() path_annot = os.path.join(path_out, FOLDER_ANNOT % str_stage) path_visu = os.path.join(path_out, FOLDER_ANNOT_VISUAL % str_stage) list_dirs = [os.path.basename(p) for p in [path_annot, path_visu]] logging.debug('create sub-dirs: %s', repr(list_dirs)) tl_expt.create_subfolders(path_out, list_dirs) # perfom on new images stage_prefix = '[stage-%s] ' % str_stage logging.info('start section %s - load_center_evaluate ...', stage_prefix) wrapper_detection = partial(load_center_evaluate, df_annot=df_slices_info, path_annot=path_annot, path_visu=path_visu, col_prefix=stage_prefix) iterate = tl_expt.WrapExecuteSequence(wrapper_detection, df_paths.iterrows(), nb_jobs=nb_jobs) for dict_eval in iterate: df_eval = df_eval.append(dict_eval, ignore_index=True) df_eval.to_csv(os.path.join(path_out, NAME_CSV_TRIPLES_TEMP)) # gc.collect(), time.sleep(1) return df_eval
def estim_model_classes_group(list_images, nb_classes, dict_features, sp_size=30, sp_regul=0.2, b_scaler=True, pca_coef=None, model_type='GMM', nb_jobs=NB_THREADS): """ estimate a model from sequence of input images and return it as result :param [ndarray] list_images: :param int nb_classes: number of classes :param int sp_size: initial size of a superpixel(meaning edge lenght) :param float sp_regul: regularisation in range(0;1) where "0" gives elastic and "1" nearly square slic :param {str: [str]} dict_features: list of features to be extracted :param float pca_coef: range (0, 1) or None :param bool b_scaler: whether use a scaler :param str model_type: model type :param int nb_jobs: number of jobs running in parallel :return: """ list_slic, list_features = list(), list() _wrapper_compute = partial(compute_color2d_superpixels_features, sp_size=sp_size, sp_regul=sp_regul, dict_features=dict_features, fts_norm=False) iterate = tl_expt.WrapExecuteSequence(_wrapper_compute, list_images, desc='compute SLIC & features', nb_jobs=nb_jobs) for slic, features in iterate: list_slic.append(slic) list_features.append(features) # for img in list_images: # slic, features = compute_color2d_superpixels_features(img, sp_size, # sp_regul, dict_features, fts_norm=False) # list_slic.append(slic) # list_features.append(features) features = np.concatenate(tuple(list_features), axis=0) features = np.nan_to_num(features) model = seg_gc.estim_class_model(features, nb_classes, model_type, pca_coef, b_scaler) return model, list_features
def experiment_single_gmm(params, paths_img, path_out, path_visu): imgs_idx_path = list(zip([None] * len(paths_img), paths_img)) logging.info('Perform image segmentation as single image in each time') dict_segms_gmm = {} wrapper_segment = partial(segment_image_independent, params=params, path_out=path_out, path_visu=path_visu) iterate = tl_expt.WrapExecuteSequence(wrapper_segment, imgs_idx_path, nb_jobs=params['nb_jobs'], desc='experiment single GMM') for name, segm in iterate: dict_segms_gmm[name] = segm return dict_segms_gmm
def perform_predictions(params, paths_img, classif): logging.info('run prediction on training images...') imgs_idx_path = list(zip(range(1, len(paths_img) + 1), paths_img)) dict_segms, dict_segms_gc = dict(), dict() path_out = os.path.join(params['path_exp'], FOLDER_SEGM) path_visu = os.path.join(params['path_exp'], FOLDER_SEGM_VISU) wrapper_segment = partial(segment_image, params=params, classif=classif, path_out=path_out, path_visu=path_visu) iterate = tl_expt.WrapExecuteSequence(wrapper_segment, imgs_idx_path, nb_jobs=params['nb_jobs'], desc='image segm: prediction') for name, segm, segm_gc in iterate: dict_segms[name] = segm dict_segms_gc[name] = segm_gc return dict_segms, dict_segms_gc
def main(params): """ PIPELINE for new detections :param {str: str} paths: """ logging.info('running...') params = run_train.prepare_experiment_folder(params, FOLDER_EXPERIMENT) # run_train.check_pathes_patterns(paths) tl_expt.set_experiment_logger(params['path_expt']) logging.info('COMPUTER: \n%s', repr(os.uname())) logging.info(tl_expt.string_dict(params, desc='PARAMETERS')) tl_expt.create_subfolders(params['path_expt'], LIST_SUBFOLDER) path_csv = os.path.join(params['path_expt'], NAME_CSV_TRIPLES) df_paths = get_csv_triplets(params['path_list'], path_csv, params['path_images'], params['path_segms'], force_reload=FORCE_RERUN) dict_classif = seg_clf.load_classifier(params['path_classif']) params_clf = dict_classif['params'] params_clf.update(params) logging.info(tl_expt.string_dict(params, desc='UPDATED PARAMETERS')) # perform on new images df_stat = pd.DataFrame() wrapper_detection = partial(load_compute_detect_centers, params=params_clf, path_classif=params['path_classif'], path_output=params['path_expt']) iterate = tl_expt.WrapExecuteSequence(wrapper_detection, df_paths.iterrows(), nb_jobs=params['nb_jobs']) for dict_center in iterate: df_stat = df_stat.append(dict_center, ignore_index=True) df_stat.to_csv(os.path.join(params['path_expt'], NAME_CSV_TRIPLES_TEMP)) df_stat.set_index(['image'], inplace=True) df_stat.to_csv(os.path.join(params['path_expt'], NAME_CSV_TRIPLES)) logging.info('STATISTIC: \n %s', repr(df_stat.describe())) logging.info('DONE')
def quantize_folder_images(path_images, label, nb_jobs=1): """ perform single or multi thread image quantisation :param [str] path_images: list of image paths :param int nb_jobs: """ assert os.path.isdir(os.path.dirname(path_images)), \ 'input folder does not exist: %s' % os.path.dirname(path_images) path_imgs = sorted(glob.glob(path_images)) logging.info('found %i images', len(path_imgs)) _wrapper_img_inpaint = partial(perform_img_inpaint, labels=label) iterate = tl_expt.WrapExecuteSequence(_wrapper_img_inpaint, path_imgs, nb_jobs=nb_jobs, desc='quantise images') list(iterate)
def experiment_group_gmm(params, paths_img, path_out, path_visu, show_debug_imgs=SHOW_DEBUG_IMAGES): logging.info('load all images') list_images = [ load_image(path_img, params['img_type']) for path_img in paths_img ] imgs_idx_path = list(zip([None] * len(paths_img), paths_img)) logging.info('Estimate image segmentation from whole sequence of images') params['path_model'] = os.path.join(params['path_exp'], NAME_DUMP_MODEL) if os.path.isfile(params['path_model']) and not FORCE_RECOMP_DATA: model, _, _ = load_model(params['path_model']) else: model, _ = seg_pipe.estim_model_classes_group( list_images, nb_classes=params['nb_classes'], dict_features=params['features'], sp_size=params['slic_size'], sp_regul=params['slic_regul'], pca_coef=params['pca_coef'], model_type=params['estim_model']) save_model(params['path_model'], model) logging.info('Perform image segmentation from group model') _wrapper_segment = partial(segment_image_model, params=params, model=model, path_out=path_out, path_visu=path_visu, show_debug_imgs=show_debug_imgs) iterate = tl_expt.WrapExecuteSequence(_wrapper_segment, imgs_idx_path, nb_jobs=params['nb_jobs'], desc='experiment group GMM') # dict_segms_group = {} # for name, segm in iterate: # dict_segms_group[name] = segm dict_segms_group = dict(iterate) gc.collect() time.sleep(1) return dict_segms_group
def main(params): df_paths = tl_data.find_files_match_names_across_dirs([params['path_images'], params['path_segms'], params['path_centers']]) df_paths.columns = ['path_image', 'path_segm', 'path_centers'] df_paths.index = range(1, len(df_paths) + 1) if not os.path.exists(params['path_output']): assert os.path.exists(os.path.dirname(params['path_output'])), \ 'missing folder: "%s"' % os.path.dirname(params['path_output']) os.mkdir(params['path_output']) df_slices_info = seg_annot.load_info_group_by_slices(params['path_infofile'], params['stages']) _wrapper_export = partial(export_figure, df_slices_info=df_slices_info, path_out=params['path_output']) iterate = tl_expt.WrapExecuteSequence(_wrapper_export, df_paths.iterrows(), nb_jobs=params['nb_jobs']) list(iterate)