def load_and_merge_csv_files(data_param, default_folder=None): """ Converts a list of csv_files in data_param in to a joint list of file names (by matching the first column) This function returns a <pandas.core.frame.DataFrame> of the joint list """ if not data_param: tf.logging.fatal('nothing to load, please check reader.names') raise ValueError _file_list = None for modality_name in data_param: try: csv_file = data_param[modality_name].csv_file except AttributeError: tf.logging.fatal('unrecognised parameter format') raise if hasattr(data_param[modality_name], 'path_to_search') and \ len(data_param[modality_name].path_to_search): tf.logging.info('[%s] search file folders, writing csv file %s', modality_name, csv_file) section_tuple = data_param[modality_name].__dict__.items() matcher = KeywordsMatching.from_tuple(section_tuple, default_folder) match_and_write_filenames_to_csv([matcher], csv_file) else: tf.logging.info( '[%s] using existing csv file %s, skipped folder search', modality_name, csv_file) if not os.path.isfile(csv_file): tf.logging.fatal("[%s] csv file %s not found.", modality_name, csv_file) raise IOError csv_list = pandas.read_csv(csv_file, header=None, names=['subject_id', modality_name]) if _file_list is None: _file_list = csv_list continue # merge _file_list based on subject_ids (first column of each csv) n_rows = _file_list.shape[0] _file_list = pandas.merge(_file_list, csv_list, on='subject_id') if _file_list.shape[0] != n_rows: tf.logging.warning("rows not matched in %s", csv_file) if _file_list is None or _file_list.size == 0: tf.logging.fatal( "empty filename lists, please check the csv " "files. (remove csv_file keyword if it is in the config file " "to automatically search folders and generate new csv " "files again)\n\n" "Please note in the matched file names, each subject name are " "matched by removing any keywords listed `filename_contains` " "in the config.\n\n") raise IOError return _file_list
def grep_files_by_data_section(self, modality_name): """ list all files by a given input data section:: if the ``csv_file`` property of the section corresponds to a file, read the list from the file; otherwise write the list to ``csv_file``. :return: a table with two columns, the column names are ``(COLUMN_UNIQ_ID, modality_name)``. """ if modality_name not in self.data_param: tf.logging.fatal('unknown section name [%s], ' 'current input section names: %s.', modality_name, list(self.data_param)) raise ValueError # input data section must have a ``csv_file`` section for loading # or writing filename lists try: csv_file = self.data_param[modality_name].csv_file if not os.path.isfile(csv_file): # writing to the same folder as data_split_file csv_file = os.path.join(os.path.dirname(self.data_split_file), '{}.csv'.format(modality_name)) except (AttributeError, TypeError): tf.logging.fatal('Missing `csv_file` field in the config file, ' 'unknown configuration format.') raise if hasattr(self.data_param[modality_name], 'path_to_search') and \ self.data_param[modality_name].path_to_search: tf.logging.info('[%s] search file folders, writing csv file %s', modality_name, csv_file) section_properties = self.data_param[modality_name].__dict__.items() # grep files by section properties and write csv try: matcher = KeywordsMatching.from_tuple( section_properties, self.default_image_file_location) match_and_write_filenames_to_csv([matcher], csv_file) except (IOError, ValueError) as reading_error: tf.logging.warning('Ignoring input section: [%s], ' 'due to the following error:', modality_name) tf.logging.warning(repr(reading_error)) return pandas.DataFrame( columns=[COLUMN_UNIQ_ID, modality_name]) else: tf.logging.info( '[%s] using existing csv file %s, skipped filenames search', modality_name, csv_file) if not os.path.isfile(csv_file): tf.logging.fatal( '[%s] csv file %s not found.', modality_name, csv_file) raise IOError try: csv_list = pandas.read_csv( csv_file, header=None, dtype=(str, str), names=[COLUMN_UNIQ_ID, modality_name], skipinitialspace=True) except Exception as csv_error: tf.logging.fatal(repr(csv_error)) raise return csv_list
def grep_files_by_data_section(self, modality_name): """ list all files by a given input data section:: if the ``csv_file`` property of the section corresponds to a file, read the list from the file; otherwise write the list to ``csv_file``. :return: a table with two columns, the column names are ``(COLUMN_UNIQ_ID, modality_name)``. """ if modality_name not in self.data_param: tf.logging.fatal( 'unknown section name [%s], ' 'current input section names: %s.', modality_name, list(self.data_param)) raise ValueError # input data section must have a ``csv_file`` section for loading # or writing filename lists try: csv_file = self.data_param[modality_name].csv_file if not os.path.isfile(csv_file): # writing to the same folder as data_split_file csv_file = os.path.join(os.path.dirname(self.data_split_file), '{}.csv'.format(modality_name)) except (AttributeError, TypeError): tf.logging.fatal('Missing `csv_file` field in the config file, ' 'unknown configuration format.') raise if hasattr(self.data_param[modality_name], 'path_to_search') and \ self.data_param[modality_name].path_to_search: tf.logging.info('[%s] search file folders, writing csv file %s', modality_name, csv_file) section_properties = self.data_param[modality_name].__dict__.items( ) # grep files by section properties and write csv try: matcher = KeywordsMatching.from_tuple( section_properties, self.default_image_file_location) match_and_write_filenames_to_csv([matcher], csv_file) except (IOError, ValueError) as reading_error: tf.logging.warning( 'Ignoring input section: [%s], ' 'due to the following error:', modality_name) tf.logging.warning(repr(reading_error)) return pandas.DataFrame( columns=[COLUMN_UNIQ_ID, modality_name]) else: tf.logging.info( '[%s] using existing csv file %s, skipped filenames search', modality_name, csv_file) if not os.path.isfile(csv_file): tf.logging.fatal('[%s] csv file %s not found.', modality_name, csv_file) raise IOError try: csv_list = pandas.read_csv(csv_file, header=None, dtype=(str, str), names=[COLUMN_UNIQ_ID, modality_name], skipinitialspace=True) except Exception as csv_error: tf.logging.fatal(repr(csv_error)) raise return csv_list