def find_set_of_labels(image_list, field, output_key): label_set = set() for idx, image in enumerate(image_list): assert field in image, \ "label normalisation layer requires {} input, " \ "however it is not provided in the config file.\n" \ "Please consider setting " \ "label_normalisation to False.".format(field) print_progress_bar(idx, len(image_list), prefix='searching unique labels from training files', decimals=1, length=10, fill='*') unique_label = np.unique(image[field].get_data()) if len(unique_label) > 500 or len(unique_label) <= 1: tf.logging.warning( 'unusual discrete values: number of unique ' 'labels to normalise %s', len(unique_label)) label_set.update(set(unique_label)) label_set = list(label_set) label_set.sort() try: mapping_from_to = dict() mapping_from_to[output_key[0]] = tuple(label_set) mapping_from_to[output_key[1]] = tuple(range(0, len(label_set))) except (IndexError, ValueError): tf.logging.fatal("unable to create mappings keys: %s, image name %s", output_key, field) raise return mapping_from_to
def _filename_to_image_list(file_list, mod_dict, data_param): """ Converting a list of filenames to a list of image objects, Properties (e.g. interp_order) are added to each object """ volume_list = [] for idx in range(len(file_list)): print_progress_bar(idx, len(file_list), prefix='reading datasets headers', decimals=1, length=10, fill='*') # combine fieldnames and volumes as a dictionary _dict = {} for field, modalities in mod_dict.items(): image_instance = _create_image( file_list, idx, modalities, data_param) if image_instance is not None: _dict[field] = image_instance if _dict: volume_list.append(_dict) if not volume_list: tf.logging.fatal( "Empty filename lists, please check the csv " "files. (removing csv_file keyword if it is in the config file " "to automatically search folders and generate new csv " "files again)\n\n" "Please note in the matched file names, each subject id are " "created by removing all keywords listed `filename_contains` " "in the config.\n\n" "E.g., `filename_contains=foo, bar` will match file " "foo_subject42_bar.nii.gz, and the subject id is _subject42_.") raise IOError return volume_list
def find_set_of_labels(image_list, field, output_key): label_set = set() for idx, image in enumerate(image_list): assert field in image, \ "label normalisation layer requires {} input, " \ "however it is not provided in the config file.\n" \ "Please consider setting " \ "label_normalisation to False.".format(field) print_progress_bar( idx, len(image_list), prefix='searching unique labels from training files', decimals=1, length=10, fill='*') unique_label = np.unique(image[field].get_data()) if len(unique_label) > 500 or len(unique_label) <= 1: tf.logging.warning( 'unusual discrete values: number of unique ' 'labels to normalise %s', len(unique_label)) label_set.update(set(unique_label)) label_set = list(label_set) label_set.sort() try: mapping_from_to = dict() mapping_from_to[output_key[0]] = tuple(label_set) mapping_from_to[output_key[1]] = tuple(range(0, len(label_set))) except (IndexError, ValueError): tf.logging.fatal("unable to create mappings keys: %s, image name %s", output_key, field) raise return mapping_from_to
def _filename_to_image_list(file_list, mod_dict, data_param): """ Converting a list of filenames to a list of image objects, Properties (e.g. interp_order) are added to each object """ volume_list = [] for idx in range(len(file_list)): # create image instance for each subject print_progress_bar(idx, len(file_list), prefix='reading datasets headers', decimals=1, length=10, fill='*') # combine fieldnames and volumes as a dictionary _dict = {} for field, modalities in mod_dict.items(): _dict[field] = _create_image(file_list, idx, modalities, data_param) # skipping the subject if there're missing image components if _dict and None not in list(_dict.values()): volume_list.append(_dict) if not volume_list: tf.logging.fatal( "Empty filename lists, please check the csv " "files. (removing csv_file keyword if it is in the config file " "to automatically search folders and generate new csv " "files again)\n\n" "Please note in the matched file names, each subject id are " "created by removing all keywords listed `filename_contains` " "in the config.\n\n" "E.g., `filename_contains=foo, bar` will match file " "foo_subject42_bar.nii.gz, and the subject id is _subject42_.") raise IOError return volume_list
def progress_bar_wrapper(count, block_size, total_size): """ Uses the common progress bar in the urlretrieve hook format """ if block_size*5 >= total_size: # no progress bar for tiny files return print_progress_bar( iteration=count, total=math.ceil(float(total_size) / float(block_size)), prefix="Downloading (total: %.2f M): " % (total_size * 1.0 / 1e6))
def progress_bar_wrapper(count, block_size, total_size): """ Uses the common progress bar in the urlretrieve hook format """ if block_size * 5 >= total_size: # no progress bar for tiny files return print_progress_bar(iteration=count, total=math.ceil(float(total_size) / float(block_size)), prefix="Downloading (total: %.2f M): " % (total_size * 1.0 / 1e6))
def create_mapping_from_multimod_arrayfiles(array_files, field, modalities, mod_to_train, cutoff, masking_function): """ Performs the mapping creation based on a list of files. For each of the files (potentially multimodal), the landmarks are defined for each modality and stored in a database. The average of these landmarks is returned providing the landmarks to use for the linear mapping of any new incoming data :param array_files: List of image files to use :param modalities: Name of the modalities used for the standardisation and the corresponding order in the multimodal files :param cutoff: Minimum and maximum landmarks percentile values to use for the mapping :param masking_function: Describes how the mask is defined for each image. :return: """ perc_database = {} for (i, p) in enumerate(array_files): print_progress_bar(i, len(array_files), prefix='normalisation histogram training', decimals=1, length=10, fill='*') img_data = p[field].get_data() assert img_data.shape[4] == len(modalities), \ "number of modalities are not consistent in the input image" for mod_i, m in enumerate(modalities): if m not in mod_to_train: continue if m not in perc_database.keys(): perc_database[m] = [] for t in range(img_data.shape[3]): img_3d = img_data[..., t, mod_i] if masking_function is not None: mask_3d = masking_function(img_3d) else: mask_3d = np.ones_like(img_3d, dtype=np.bool) perc = __compute_percentiles(img_3d, mask_3d, cutoff) perc_database[m].append(perc) mapping = {} for m in list(perc_database): perc_database[m] = np.vstack(perc_database[m]) s1, s2 = create_standard_range() mapping[m] = tuple(__averaged_mapping(perc_database[m], s1, s2)) return mapping
def create_mapping_from_multimod_arrayfiles(array_files, field, modalities, mod_to_train, cutoff, masking_function): """ Performs the mapping creation based on a list of files. For each of the files (potentially multimodal), the landmarks are defined for each modality and stored in a database. The average of these landmarks is returned providing the landmarks to use for the linear mapping of any new incoming data :param array_files: List of image files to use :param modalities: Name of the modalities used for the standardisation and the corresponding order in the multimodal files :param cutoff: Minimum and maximum landmarks percentile values to use for the mapping :param masking_function: Describes how the mask is defined for each image. :return: """ perc_database = {} for (i, p) in enumerate(array_files): print_progress_bar(i, len(array_files), prefix='normalisation histogram training', decimals=1, length=10, fill='*') img_data = p[field].get_data() assert img_data.shape[4] == len(modalities), \ "number of modalities are not consistent in the input image" for mod_i, m in enumerate(modalities): if m not in mod_to_train: continue if m not in perc_database.keys(): perc_database[m] = [] for t in range(img_data.shape[3]): img_3d = img_data[..., t, mod_i] if masking_function is not None: mask_3d = masking_function(img_3d) else: mask_3d = np.ones_like(img_3d, dtype=np.bool) perc = __compute_percentiles(img_3d, mask_3d, cutoff) perc_database[m].append(perc) mapping = {} for m in list(perc_database): perc_database[m] = np.vstack(perc_database[m]) s1, s2 = create_standard_range() mapping[m] = tuple(__averaged_mapping(perc_database[m], s1, s2)) return mapping
def _filename_to_image_list(file_list, mod_dict, data_param): """ converting a list of filenames to a list of image objects useful properties (e.g. interp_order) are added to each object """ volume_list = [] for idx in range(len(file_list)): print_progress_bar(idx, len(file_list), prefix='reading datasets headers', decimals=1, length=10, fill='*') # combine fieldnames and volumes as a dictionary _dict = { field: _create_image(file_list, idx, modalities, data_param) for (field, modalities) in mod_dict.items() } volume_list.append(_dict) return volume_list