def filter_export_landmarks(idx_row, path_output, path_dataset, path_reference): """ filter all relevant landmarks which were used and copy them to experiment The case is that in certain challenge stage users had provided just a subset of all image landmarks which could be laos shuffled. The idea is to filter identify all user used (provided in dataset) landmarks and filter them from temporary reference dataset. :param tuple(idx,dict|Series) idx_row: experiment DataFrame :param str path_output: path to output folder :param str path_dataset: path to provided landmarks :param str path_reference: path to the complete landmark collection :return tuple(idx,float): record index and match ratio """ idx, row = idx_row ratio_matches, lnds_filter_ref, lnds_filter_move = \ filter_paired_landmarks(row, path_dataset, path_reference, ImRegBenchmark.COL_POINTS_MOVE, ImRegBenchmark.COL_POINTS_REF) # moving and reference landmarks for col, lnds_flt in [(ImRegBenchmark.COL_POINTS_REF, lnds_filter_ref), (ImRegBenchmark.COL_POINTS_MOVE, lnds_filter_move)]: path_out = update_path(row[col], pre_path=path_output) create_folder(os.path.dirname(path_out), ok_existing=True) if os.path.isfile(path_out): assert np.array_equal(load_landmarks(path_out), lnds_flt), \ 'overwrite different set of landmarks' save_landmarks(path_out, lnds_flt) return idx, ratio_matches
def _perform_registration(self, df_row): """ run single registration experiment with all sub-stages :param (int, dict) df_row: tow from iterated table """ idx, row = df_row logging.debug('-> perform single registration #%d...', idx) # create folder for this particular experiment row['ID'] = idx row[COL_REG_DIR] = str(idx) path_dir_reg = self._get_path_reg_dir(row) # check whether the particular experiment already exists and have result if self.__check_exist_regist(idx, path_dir_reg): return None create_folder(path_dir_reg) row = self._prepare_img_registration(row) # measure execution time time_start = time.time() row = self._execute_img_registration(row) # if the experiment failed, return back None if not row: return None # compute the registration time in minutes row[COL_TIME] = (time.time() - time_start) / 60. row = self._parse_regist_results(row) row = self._clear_after_registration(row) return row
def scale_image(img_path, scale, image_ext=IMAGE_EXTENSION, overwrite=False): """ scaling images by given scale factor :param img_path: input image path :param int scale: selected scaling in percents :param str image_ext: image extension used on output :param bool overwrite: whether overwrite existing image on output """ base = os.path.dirname(os.path.dirname(img_path)) name = os.path.splitext(os.path.basename(img_path))[0] base_scale = parse_path_scale(os.path.dirname(img_path)) path_dir = os.path.join(base, FOLDER_TEMPLATE % scale) create_folder(path_dir) path_img_scale = os.path.join(path_dir, name + image_ext) if os.path.isfile(path_img_scale) and not overwrite: logging.debug('existing "%s"', path_img_scale) return img = load_large_image(img_path) sc = scale / float(base_scale) # for down-scaling use just linear if sc == 1.: img_sc = img else: interp = cv.INTER_CUBIC if sc > 1 else cv.INTER_LINEAR img_sc = cv.resize(img, None, fx=sc, fy=sc, interpolation=interp) del img gc.collect() time.sleep(1) logging.debug('creating >> %s', path_img_scale) save_large_image(path_img_scale, img_sc)
def scale_image(img_path, scale, image_ext=IMAGE_EXTENSION, overwrite=False): base = os.path.dirname(os.path.dirname(img_path)) name = os.path.splitext(os.path.basename(img_path))[0] base_scale = parse_path_scale(os.path.dirname(img_path)) path_dir = os.path.join(base, FOLDER_TEMPLATE % scale) create_folder(path_dir) path_img_scale = os.path.join(path_dir, name + image_ext) if os.path.isfile(path_img_scale) and not overwrite: logging.debug('existing "%s"', path_img_scale) return img = load_large_image(img_path) sc = scale / float(base_scale) # for down-scaling use just linear if sc == 1.: img_sc = img else: interp = cv.INTER_CUBIC if sc > 1 else cv.INTER_LINEAR img_sc = cv.resize(img, None, fx=sc, fy=sc, interpolation=interp) del img logging.debug('creating >> %s', path_img_scale) save_large_image(path_img_scale, img_sc) gc.collect() time.sleep(1)
def create_experiment_folder(path_out, dir_name, name='', stamp_unique=True): """ create the experiment folder and iterate while there is no available :param str path_out: path to the base experiment directory :param str name: special experiment name :param str dir_name: special folder name :param bool stamp_unique: whether add at the end of new folder unique tag >>> p_dir = create_experiment_folder('.', 'my_test', stamp_unique=False) >>> os.rmdir(p_dir) >>> p_dir = create_experiment_folder('.', 'my_test', stamp_unique=True) >>> os.rmdir(p_dir) """ assert os.path.exists(path_out), 'missing base folder "%s"' % path_out date = time.gmtime() if isinstance(name, str) and name: dir_name = '{}_{}'.format(dir_name, name) path_exp = os.path.join(path_out, dir_name) if stamp_unique: path_exp += '_' + time.strftime(FORMAT_DATE_TIME, date) path_created = None while not path_created: logging.warning('particular out folder already exists') if path_created is not None: path_exp += '-' + str(np.random.randint(0, 100)) path_created = create_folder(path_exp, ok_existing=False) else: path_created = create_folder(path_exp, ok_existing=False) logging.info('created experiment folder "%r"', path_created) return path_exp
def filter_landmarks(idx_row, path_output, path_dataset, path_reference): """ filter all relevant landmarks which were used and copy them to experiment :param (idx, {}|Series) idx_row: experiment DataFrame :param str path_output: path to output folder :param str path_dataset: path to provided landmarks :param str path_reference: path to the complete landmark collection :return (idx, float): record index and match ratio """ idx, row = idx_row path_ref = update_path_(row[COL_POINTS_MOVE], path_reference) path_load = update_path_(row[COL_POINTS_MOVE], path_dataset) pairs = common_landmarks(load_landmarks(path_ref), load_landmarks(path_load), threshold=1) pairs = sorted(pairs.tolist(), key=lambda p: p[1]) ind_ref = np.asarray(pairs)[:, 0] # moving and reference landmarks for col in [COL_POINTS_REF, COL_POINTS_MOVE]: path_in = update_path_(row[col], path_reference) path_out = update_path_(row[col], path_output) create_folder(os.path.dirname(path_out), ok_existing=True) save_landmarks(path_out, load_landmarks(path_in)[ind_ref]) # save ratio of found landmarks len_lnds_ref = len( load_landmarks(update_path_(row[COL_POINTS_REF], path_reference))) ratio_matches = len(pairs) / float(len_lnds_ref) return idx, ratio_matches
def _perform_registration(self, df_row): """ run single registration experiment with all sub-stages :param tuple(int,dict) df_row: row from iterated table """ idx, row = df_row logging.debug('-> perform single registration #%d...', idx) # create folder for this particular experiment row['ID'] = idx row[self.COL_REG_DIR] = str(idx) path_dir_reg = self._get_path_reg_dir(row) # check whether the particular experiment already exists and have result if self.__check_exist_regist(idx, path_dir_reg): return create_folder(path_dir_reg) time_start = time.time() # do some requested pre-processing if required row = self.__images_preprocessing(row) row[self.COL_TIME_PREPROC] = (time.time() - time_start) / 60. row = self._prepare_img_registration(row) # if the pre-processing failed, return back None if not row: return # measure execution time time_start = time.time() row = self._execute_img_registration(row) # if the experiment failed, return back None if not row: return # compute the registration time in minutes row[self.COL_TIME] = (time.time() - time_start) / 60. # remove some temporary images row = self.__remove_pproc_images(row) row = self._parse_regist_results(row) # if the post-processing failed, return back None if not row: return row = self._clear_after_registration(row) if self.params.get('visual', False): logging.debug('-> visualise results of experiment: %r', idx) self.visualise_registration( (idx, row), path_dataset=self.params.get('path_dataset'), path_experiment=self.params.get('path_exp'), ) return row
def extend_landmarks(path_set, path_dataset, nb_selected=None, nb_total=None): """ select and extend the original set of landmarks :param str path_set: path to the particular set if images/landmarks :param str path_dataset: root path to generated dataset :param float|int|None nb_selected: portion of selected points, if None use all original landmarks :param int|None nb_total: add extra points up to total number, if None, no adding extra points :return: """ logging.debug('> processing: %s', path_set) # search form mas scale in set and load all related landmarks names_lnds = load_largest_scale(path_set) if not names_lnds: logging.warning('no landmarks was loaded for "%s"', path_set) return # select subset of selected landmarks names_lnds_new = {} if nb_selected is not None: assert nb_selected >= 0, 'number of selected has to be positive' lens = list(map(len, names_lnds.values())) # transform the relative count to absolute number if nb_selected < 1: nb_selected = np.ceil(nb_selected * max(lens)).astype(int) # perform the selection indexes = list(range(min(lens))) np.random.shuffle(indexes) # just a required subset indexes = indexes[:nb_selected] for name in names_lnds: names_lnds_new[name] = names_lnds[name][indexes] else: names_lnds_new = names_lnds if nb_total is not None: names_lnds_new = expand_random_warped_landmarks( names_lnds, names_lnds_new, nb_total) # export the landmarks path_set_scale = os.path.join(path_dataset, os.path.basename(path_set), FOLDER_TEMPLATE % 100) create_folder(path_set_scale) for name in names_lnds_new: save_landmarks_csv(os.path.join(path_set_scale, name), names_lnds_new[name])
def scale_set_landmarks(path_set, scales=DEFAULT_SCALES): """ scale the updated (generated) landmarks the scales are created within the same path set :param str path_set: path to the image/landmark set :param list(int) scales: created scales :return dict: """ logging.debug('> processing: %s', path_set) path_scale100 = os.path.join(path_set, FOLDER_TEMPLATE % 100) if not os.path.isdir(path_scale100): logging.error('missing base scale 100pc in "%s"', path_scale100) return list_csv = glob.glob(os.path.join(path_scale100, '*.csv')) logging.debug('>> found landmarks: %i', len(list_csv)) dict_lnds = { os.path.basename(p): pd.read_csv(p, index_col=0) for p in list_csv } set_scales = {} for sc in (sc for sc in scales if sc not in [100]): # drop the base scale folder_name = FOLDER_TEMPLATE % sc path_scale = create_folder(os.path.join(path_set, folder_name)) for name in dict_lnds: df_scale = dict_lnds[name] * (sc / 100.) df_scale.to_csv(os.path.join(path_scale, name)) set_scales[sc] = len(dict_lnds) dict_lens = {os.path.basename(path_set): set_scales} return dict_lens
def create_experiment_folder(path_out, dir_name, name='', stamp_unique=True): """ create the experiment folder and iterate while there is no available :param str path_out: path to the base experiment directory :param str name: special experiment name :param str dir_name: special folder name :param bool stamp_unique: whether add at the end of new folder unique tag >>> p_dir = create_experiment_folder('.', 'my_test', stamp_unique=False) >>> os.rmdir(p_dir) >>> p_dir = create_experiment_folder('.', 'my_test', stamp_unique=True) >>> p_dir # doctest: +ELLIPSIS '...my_test_...-...' >>> os.rmdir(p_dir) """ if not os.path.isdir(path_out): raise FileNotFoundError('missing base folder "%s"' % path_out) date = time.gmtime() if isinstance(name, str) and name: dir_name = '%s_%s' % (dir_name, name) # if you require time stamp if stamp_unique: path_stamp = time.strftime(FORMAT_DATE_TIME, date) # prepare experiment path with initial timestamp - now path_exp = os.path.join(path_out, '%s_%s' % (dir_name, path_stamp)) if os.path.isdir(path_exp): logging.warning('particular out folder already exists') path_exp += '-' + str(uuid.uuid4().hex) else: path_exp = os.path.join(path_out, dir_name) path_created = create_folder(path_exp, ok_existing=False) logging.info('created experiment folder "%r"', path_created) return path_exp