def main(path_experiment, path_cover, path_dataset, path_output, path_reference=None,
path_comp_bm=None, nb_workers=NB_THREADS, min_landmarks=1., details=True):
""" main entry point
:param str path_experiment: path to experiment folder
:param str path_cover: path to assignment file (requested registration pairs)
:param str path_dataset: path to provided landmarkstext
:param str path_output: path to generated results
:param str|None path_reference: path to the complete landmark collection,
if None use dataset folder
:param str|None path_comp_bm: path to reference comp. benchmark
:param int nb_workers: number of parallel processes
:param float min_landmarks: required number of submitted landmarks in range (0, 1),
match values in COL_FOUND_LNDS
:param bool details: exporting case details
"""
path_results = os.path.join(path_experiment, NAME_CSV_REGISTRATION_PAIRS)
if not os.path.isfile(path_results):
raise AttributeError('Missing experiments results: %s' % path_results)
path_reference = path_dataset if not path_reference else path_reference
# drop time column from Cover which should be empty
df_cover = pd.read_csv(path_cover).drop([COL_TIME], axis=1, errors='ignore')
# drop Warp* column from Cover which should be empty
df_cover = df_cover.drop([col for col in df_cover.columns if 'warped' in col.lower()],
axis=1, errors='ignore')
df_results = pd.read_csv(path_results)
df_results = df_results[[col for col in list(COVER_COLUMNS_WRAP) + [COL_TIME]
if col in df_results.columns]]
df_experiments = pd.merge(df_cover, df_results, how='left', on=COVER_COLUMNS)
df_experiments.drop([COL_IMAGE_REF_WARP, COL_POINTS_REF_WARP],
axis=1, errors='ignore', inplace=True)
normalize_exec_time(df_experiments, path_experiment, path_comp_bm)
logging.info('Filter used landmarks.')
_filter_lnds = partial(filter_landmarks, path_output=path_output,
path_dataset=path_dataset, path_reference=path_reference)
for idx, ratio in wrap_execute_sequence(_filter_lnds, df_experiments.iterrows(),
desc='Filtering', nb_workers=nb_workers):
df_experiments.loc[idx, COL_FOUND_LNDS] = np.round(ratio, 2)
logging.info('Compute landmarks statistic.')
_compute_lnds_stat = partial(compute_registration_statistic, df_experiments=df_experiments,
path_dataset=path_output, path_experiment=path_experiment)
# NOTE: this has to run in SINGLE thread so there is SINGLE table instance
list(wrap_execute_sequence(_compute_lnds_stat, df_experiments.iterrows(),
desc='Statistic', nb_workers=1))
path_results = os.path.join(path_output, os.path.basename(path_results))
logging.debug('exporting CSV results: %s', path_results)
df_experiments.to_csv(path_results)
path_json = export_summary_json(df_experiments, path_experiment, path_output,
min_landmarks, details)
return path_json
def __execute_method(self,
method,
input_table,
path_csv=None,
desc='',
aggr_experiments=False,
nb_workers=None):
""" execute a method in sequence or parallel
:param func method: used method
:param DF input_table: iterate over table
:param str path_csv: path to the output temporal csv
:param str desc: name of the running process
:param bool aggr_experiments: append output to experiment DF
:param int|None nb_workers: number of jobs, by default using class setting
:return:
"""
# setting the temporal split
self._main_thread = False
# run the experiment in parallel of single thread
nb_workers = self.nb_workers if nb_workers is None else nb_workers
iter_table = ((idx, dict(row)) for idx, row, in input_table.iterrows())
for res in wrap_execute_sequence(method,
iter_table,
ordered=True,
nb_workers=nb_workers,
desc=desc):
if res is None or not aggr_experiments:
continue
self._df_experiments = self._df_experiments.append(
res, ignore_index=True)
self.__export_df_experiments(path_csv)
self._main_thread = True
def dataset_expand_landmarks(path_annots,
path_dataset,
nb_selected=None,
nb_total=None,
nb_workers=NB_THREADS):
""" select and expand over whole dataset
:param str path_annots: root path to original dataset
:param str path_dataset: root path to generated dataset
:param float|int|None nb_selected: portion of selected points
:param int|None nb_total: add extra points up to total number
:param int nb_workers: number of jobs running in parallel
:return [int]:
"""
list_sets = list_sub_folders(path_annots)
logging.info('Found sets: %i', len(list_sets))
_wrap_extend = partial(extend_landmarks,
path_dataset=path_dataset,
nb_selected=nb_selected,
nb_total=nb_total)
counts = list(
wrap_execute_sequence(_wrap_extend,
sorted(list_sets),
nb_workers=nb_workers,
desc='expand landmarks'))
return counts
def main(path_images, padding, nb_workers):
image_paths = sorted(glob.glob(path_images))
if not image_paths:
logging.info('No images found on "%s"', path_images)
return
_wrap_crop = partial(crop_image, padding=padding)
list(wrap_execute_sequence(_wrap_crop, image_paths,
desc='Crop image tissue', nb_workers=nb_workers))
def main(path_images, level=DEFAULT_LEVEL, overwrite=False, nb_workers=1):
paths_img = sorted(glob.glob(path_images))
_wrap_convert = partial(convert_image, level=level, overwrite=overwrite)
list(
wrap_execute_sequence(_wrap_convert,
paths_img,
desc='Converting images',
nb_workers=nb_workers))
def main(path_experiment, path_dataset, visual=False, nb_workers=NB_THREADS):
""" main entry points
:param str path_experiment: path to the experiment folder
:param str path_dataset: path to the dataset with all landmarks
:param bool visual: whether visualise the registration results
:param int nb_workers: number of parallel jobs
"""
path_results = os.path.join(path_experiment, NAME_CSV_REGISTRATION_PAIRS)
assert os.path.isfile(path_results)
df_experiments = pd.read_csv(path_results)
df_results = df_experiments.copy()
_compute_lnds_stat = partial(compute_registration_statistic,
df_experiments=df_results,
path_dataset=path_dataset,
path_experiment=path_experiment)
# NOTE: this has to run in SINGLE thread so there is SINGLE table instance
list(
wrap_execute_sequence(_compute_lnds_stat,
df_experiments.iterrows(),
desc='Statistic',
nb_workers=1))
path_csv = os.path.join(path_experiment, NAME_CSV_RESULTS)
logging.debug('exporting CSV results: %s', path_csv)
df_results.to_csv(path_csv, index=None)
export_summary_results(df_results,
path_experiment,
None,
name_csv=NAME_CSV_SUMMARY,
name_txt=NAME_TXT_SUMMARY)
if visual:
_visualise_regist = partial(visualise_registration,
path_dataset=path_dataset,
path_experiment=path_experiment)
list(
wrap_execute_sequence(_visualise_regist,
df_experiments.iterrows(),
desc='Visualisation',
nb_workers=nb_workers))
def main(path_images, dimension, overwrite, nb_workers):
image_paths = sorted(glob.glob(path_images))
if not image_paths:
logging.info('No images found on "%s"', path_images)
return
_wrap_split = partial(split_image, cut_dim=dimension, overwrite=overwrite)
list(
wrap_execute_sequence(_wrap_split,
image_paths,
desc='Cut image tissues',
nb_workers=nb_workers))
def export_summary_json(df_experiments,
path_experiments,
path_output,
min_landmarks=1.):
""" summarize results in particular JSON format
:param DF df_experiments: experiment DataFrame
:param str path_experiments: path to experiment folder
:param str path_output: path to generated results
:param float min_landmarks: required number of submitted landmarks in range (0, 1),
match values in COL_FOUND_LNDS
:return str: path to exported results
"""
# export partial results
cases = list(
wrap_execute_sequence(parse_landmarks,
df_experiments.iterrows(),
desc='Parsing landmarks',
nb_workers=1))
# copy the initial to final for missing
for col, col2 in zip(*_filter_measure_columns(df_experiments)):
mask = df_experiments[col].isnull()
df_experiments.loc[mask, col] = df_experiments.loc[mask, col2]
# parse final metrics
scores = compute_scores(df_experiments, min_landmarks)
path_comp_bm_expt = os.path.join(path_experiments, NAME_JSON_COMPUTER)
if os.path.isfile(path_comp_bm_expt):
with open(path_comp_bm_expt, 'r') as fp:
comp_exp = json.load(fp)
else:
comp_exp = None
results = {
'aggregates': scores,
'cases': dict(cases),
'computer': comp_exp,
'submission-time': time.strftime(FORMAT_DATE_TIME, time.gmtime()),
'required-landmarks': min_landmarks,
}
path_json = os.path.join(path_output, NAME_JSON_RESULTS)
logging.info('exporting JSON results: %s', path_json)
with open(path_json, 'w') as fp:
json.dump(results, fp)
return path_json
def main(path_images, scales, image_extension, overwrite, nb_workers):
image_paths = sorted(glob.glob(path_images))
image_path_scales = [(im_path, sc) for im_path in image_paths
for sc in scales]
if not image_paths:
logging.info('No images found on "%s"', path_images)
return
_wrap_scale = partial(wrap_scale_image,
image_ext=image_extension,
overwrite=overwrite)
list(
wrap_execute_sequence(_wrap_scale,
image_path_scales,
desc='Scaling images',
nb_workers=nb_workers))
def dataset_scale_landmarks(path_dataset,
scales=DEFAULT_SCALES,
nb_workers=NB_THREADS):
""" generate several scales within the same dataset
:param str path_dataset: path to the souorce/generated dataset
:param [inr] scales: created scales
:param int nb_workers: number of jobs running in parallel
:return:
"""
list_sets = list_sub_folders(path_dataset)
logging.info('Found sets: %i', len(list_sets))
_wrap_scale = partial(scale_set_landmarks, scales=scales)
counts = list(
wrap_execute_sequence(_wrap_scale,
sorted(list_sets),
nb_workers=nb_workers,
desc='scaling sets'))
return counts