def match_and_write_filenames_to_csv(list_constraints, csv_file): """ Combine all elements of file searching until finally writing the names :param list_constraints: list of constraints (defined by list of paths to search, list of elements the filename should contain and of those that are forbidden :param csv_file: file on which to write the final list of files. :return: """ name_tot = [] list_tot = [] if list_constraints is None or len(list_constraints) == 0: return for c in list_constraints: list_files, name_list = \ KeywordsMatching.matching_subjects_and_filenames(c) name_list = remove_duplicated_names(name_list) name_tot.append(name_list) list_tot.append(list_files) list_combined = join_subject_id_and_filename_list(name_tot, list_tot) touch_folder(os.path.dirname(csv_file)) # csv writer has different behaviour in python 2/3 if sys.version_info[0] >= 3: with open(csv_file, 'w', newline='', encoding='utf8') as csvfile: file_writer = csv.writer(csvfile) for list_temp in list_combined: file_writer.writerow(list_temp) else: with open(csv_file, 'wb') as csvfile: file_writer = csv.writer(csvfile, delimiter=',') for list_temp in list_combined: file_writer.writerow(list_temp) return
def match_and_write_filenames_to_csv(list_constraints, csv_file): """ Combine all elements of file searching until finally writing the names :param list_constraints: list of constraints (defined by list of paths to search, list of elements the filename should contain and of those that are forbidden :param csv_file: file on which to write the final list of files. :return: """ name_tot = [] list_tot = [] if list_constraints is None or len(list_constraints) == 0: return for c in list_constraints: list_files, name_list = \ KeywordsMatching.matching_subjects_and_filenames(c) name_list = remove_duplicated_names(name_list) name_tot.append(name_list) list_tot.append(list_files) list_combined = join_subject_id_and_filename_list(name_tot, list_tot) list_combined = filter(lambda names: '' not in names, list_combined) list_combined = list(list_combined) touch_folder(os.path.dirname(csv_file)) write_csv(csv_file, list_combined) return list_combined
def match_and_write_filenames_to_csv(list_constraints, csv_file): """ Combine all elements of file searching until finally writing the names :param list_constraints: list of constraints (defined by list of paths to search, list of elements the filename should contain and of those that are forbidden :param csv_file: file on which to write the final list of files. :return: """ name_tot = [] list_tot = [] if list_constraints is None or len(list_constraints) == 0: return for c in list_constraints: list_files, name_list = \ KeywordsMatching.matching_subjects_and_filenames(c) name_list = remove_duplicated_names(name_list) name_tot.append(name_list) list_tot.append(list_files) list_combined = join_subject_id_and_filename_list(name_tot, list_tot) list_combined = filter(lambda names: '' not in names, list_combined) list_combined = list(list_combined) if not list_combined: raise IOError('Nothing to write to {}'.format(csv_file)) touch_folder(os.path.dirname(csv_file)) write_csv(csv_file, list_combined) return list_combined
def main(): system_param, input_data_param = user_parameters_parser.run() if util.has_bad_inputs(system_param): return -1 # print all parameters to txt file for future reference all_param = {} all_param.update(system_param) all_param.update(input_data_param) txt_file = 'settings_{}.txt'.format(system_param['SYSTEM'].action) model_folder = touch_folder(system_param['SYSTEM'].model_dir) txt_file = os.path.join(model_folder, txt_file) util.print_save_input_parameters(all_param, txt_file) # keep all commandline outputs log_file_name = os.path.join( model_folder, '{}_{}'.format(all_param['SYSTEM'].action, 'niftynet_log')) set_logger(file_name=log_file_name) # start application app_driver = ApplicationDriver() app_driver.initialise_application(system_param, input_data_param) app_driver.run_application() return 0
def write_all_mod_mapping(hist_model_file, mapping): # backup existing file first if os.path.exists(hist_model_file): backup_name = '{}.backup'.format(hist_model_file) from shutil import copyfile try: copyfile(hist_model_file, backup_name) except OSError: tf.logging.warning('cannot backup file {}'.format(hist_model_file)) raise tf.logging.warning("moved existing histogram reference file\n" " from {} to {}".format(hist_model_file, backup_name)) touch_folder(os.path.dirname(hist_model_file)) __force_writing_new_mapping(hist_model_file, mapping)
def write_all_mod_mapping(hist_model_file, mapping): # backup existing file first if os.path.exists(hist_model_file): backup_name = '{}.backup'.format(hist_model_file) from shutil import copyfile try: copyfile(hist_model_file, backup_name) except OSError: tf.logging.warning('cannot backup file {}'.format(hist_model_file)) raise tf.logging.warning( "moved existing histogram reference file\n" " from {} to {}".format(hist_model_file, backup_name)) touch_folder(os.path.dirname(hist_model_file)) __force_writing_new_mapping(hist_model_file, mapping)
def initialise_application(self, workflow_param, data_param): """ This function receives all parameters from user config file, create an instance of application. :param workflow_param: a dictionary of user parameters, keys correspond to sections in the config file :param data_param: a dictionary of input image parameters, keys correspond to data properties to be used by image_reader :return: """ try: system_param = workflow_param.get('SYSTEM', None) net_param = workflow_param.get('NETWORK', None) infer_param = workflow_param.get('INFERENCE', None) eval_param = workflow_param.get('EVALUATION', None) app_param = workflow_param.get('CUSTOM', None) except AttributeError: tf.logging.fatal('parameters should be dictionaries') raise self.num_threads = 1 # self.num_threads = max(system_param.num_threads, 1) # self.num_gpus = system_param.num_gpus # set_cuda_device(system_param.cuda_devices) # set output TF model folders self.model_dir = touch_folder( os.path.join(system_param.model_dir, 'models')) self.session_prefix = os.path.join(self.model_dir, FILE_PREFIX) assert infer_param, 'inference parameters not specified' # create an application instance assert app_param, 'application specific param. not specified' self.app_param = app_param app_module = ApplicationFactory.create(app_param.name) self.app = app_module(net_param, infer_param, system_param.action) self.eval_param = eval_param data_param, self.app_param = \ self.app.add_inferred_output(data_param, self.app_param) # initialise data input data_partitioner = ImageSetsPartitioner() # clear the cached file lists data_partitioner.reset() if data_param: data_partitioner.initialise( data_param=data_param, new_partition=False, ratios=None, data_split_file=system_param.dataset_split_file) # initialise data input self.app.initialise_dataset_loader(data_param, self.app_param, data_partitioner) self.app.initialise_evaluator(eval_param)
def make_model_name(model_dir): """ Make the model checkpoint folder. the checkpoint file will be located at `model_dir/models/` folder, the filename will start with FILE_PREFIX. :param model_dir: niftynet model folder :return: a partial name of a checkpoint file `model_dir/model/FILE_PREFIX` """ _model_dir = touch_folder(os.path.join(model_dir, 'models')) return os.path.join(_model_dir, FILE_PREFIX)
def initialise_application(self, workflow_param, data_param): """ This function receives all parameters from user config file, create an instance of application. :param workflow_param: a dictionary of user parameters, keys correspond to sections in the config file :param data_param: a dictionary of input image parameters, keys correspond to data properties to be used by image_reader :return: """ try: system_param = workflow_param.get('SYSTEM', None) net_param = workflow_param.get('NETWORK', None) train_param = workflow_param.get('TRAINING', None) infer_param = workflow_param.get('INFERENCE', None) app_param = workflow_param.get('CUSTOM', None) except AttributeError: tf.logging.fatal('parameters should be dictionaries') raise assert os.path.exists(system_param.model_dir), \ 'Model folder not exists {}'.format(system_param.model_dir) self.is_training = (system_param.action == "train") # hardware-related parameters self.num_threads = max(system_param.num_threads, 1) \ if self.is_training else 1 self.num_gpus = system_param.num_gpus \ if self.is_training else min(system_param.num_gpus, 1) set_cuda_device(system_param.cuda_devices) # set output TF model folders self.model_dir = touch_folder( os.path.join(system_param.model_dir, 'models')) self.session_prefix = os.path.join(self.model_dir, FILE_PREFIX) if self.is_training: assert train_param, 'training parameters not specified' summary_root = os.path.join(system_param.model_dir, 'logs') self.summary_dir = get_latest_subfolder( summary_root, create_new=train_param.starting_iter == 0) # training iterations-related parameters self.initial_iter = train_param.starting_iter self.final_iter = train_param.max_iter self.save_every_n = train_param.save_every_n self.tensorboard_every_n = train_param.tensorboard_every_n self.max_checkpoints = train_param.max_checkpoints self.gradients_collector = GradientsCollector( n_devices=max(self.num_gpus, 1)) action_param = train_param else: assert infer_param, 'inference parameters not specified' self.initial_iter = infer_param.inference_iter action_param = infer_param self.outputs_collector = OutputsCollector( n_devices=max(self.num_gpus, 1)) # create an application instance assert app_param, 'application specific param. not specified' app_module = ApplicationDriver._create_app(app_param.name) self.app = app_module(net_param, action_param, self.is_training) # initialise data input self.app.initialise_dataset_loader(data_param, app_param) # pylint: disable=not-context-manager with self.graph.as_default(), tf.name_scope('Sampler'): self.app.initialise_sampler()
def initialise_application(self, workflow_param, data_param): """ This function receives all parameters from user config file, create an instance of application. :param workflow_param: a dictionary of user parameters, keys correspond to sections in the config file :param data_param: a dictionary of input image parameters, keys correspond to data properties to be used by image_reader :return: """ try: system_param = workflow_param.get('SYSTEM', None) net_param = workflow_param.get('NETWORK', None) train_param = workflow_param.get('TRAINING', None) infer_param = workflow_param.get('INFERENCE', None) app_param = workflow_param.get('CUSTOM', None) except AttributeError: tf.logging.fatal('parameters should be dictionaries') raise assert os.path.exists(system_param.model_dir), \ 'Model folder not exists {}'.format(system_param.model_dir) self.is_training = (system_param.action == "train") # hardware-related parameters self.num_threads = max(system_param.num_threads, 1) \ if self.is_training else 1 self.num_gpus = system_param.num_gpus \ if self.is_training else min(system_param.num_gpus, 1) set_cuda_device(system_param.cuda_devices) # set output TF model folders self.model_dir = touch_folder( os.path.join(system_param.model_dir, 'models')) self.session_prefix = os.path.join(self.model_dir, FILE_PREFIX) # set training params. if self.is_training: assert train_param, 'training parameters not specified' summary_root = os.path.join(system_param.model_dir, 'logs') self.summary_dir = get_latest_subfolder( summary_root, create_new=train_param.starting_iter == 0) self.initial_iter = train_param.starting_iter self.final_iter = max(train_param.max_iter, self.initial_iter) self.save_every_n = train_param.save_every_n self.tensorboard_every_n = train_param.tensorboard_every_n self.max_checkpoints = \ max(train_param.max_checkpoints, self.max_checkpoints) self.gradients_collector = GradientsCollector( n_devices=max(self.num_gpus, 1)) self.validation_every_n = train_param.validation_every_n if self.validation_every_n > 0: self.validation_max_iter = max(self.validation_max_iter, train_param.validation_max_iter) action_param = train_param else: # set inference params. assert infer_param, 'inference parameters not specified' self.initial_iter = infer_param.inference_iter action_param = infer_param self.outputs_collector = OutputsCollector( n_devices=max(self.num_gpus, 1)) # create an application instance assert app_param, 'application specific param. not specified' app_module = ApplicationDriver._create_app(app_param.name) self.app = app_module(net_param, action_param, system_param.action) # initialise data input data_partitioner = ImageSetsPartitioner() # clear the cached file lists data_partitioner.reset() do_new_partition = \ self.is_training and self.initial_iter == 0 and \ (not os.path.isfile(system_param.dataset_split_file)) and \ (train_param.exclude_fraction_for_validation > 0 or train_param.exclude_fraction_for_inference > 0) data_fractions = None if do_new_partition: assert train_param.exclude_fraction_for_validation > 0 or \ self.validation_every_n <= 0, \ 'validation_every_n is set to {}, ' \ 'but train/validation splitting not available,\nplease ' \ 'check "exclude_fraction_for_validation" in the config ' \ 'file (current config value: {}).'.format( self.validation_every_n, train_param.exclude_fraction_for_validation) data_fractions = (train_param.exclude_fraction_for_validation, train_param.exclude_fraction_for_inference) if data_param: data_partitioner.initialise( data_param=data_param, new_partition=do_new_partition, ratios=data_fractions, data_split_file=system_param.dataset_split_file) if data_param and self.is_training and self.validation_every_n > 0: assert data_partitioner.has_validation, \ 'validation_every_n is set to {}, ' \ 'but train/validation splitting not available.\nPlease ' \ 'check dataset partition list {} ' \ '(remove file to generate a new dataset partition). ' \ 'Or set validation_every_n to -1.'.format( self.validation_every_n, system_param.dataset_split_file) # initialise readers self.app.initialise_dataset_loader( data_param, app_param, data_partitioner) self._data_partitioner = data_partitioner # pylint: disable=not-context-manager with self.graph.as_default(), tf.name_scope('Sampler'): self.app.initialise_sampler()
def initialise_application(self, workflow_param, data_param): """ This function receives all parameters from user config file, create an instance of application. :param workflow_param: a dictionary of user parameters, keys correspond to sections in the config file :param data_param: a dictionary of input image parameters, keys correspond to data properties to be used by image_reader :return: """ try: system_param = workflow_param.get('SYSTEM', None) net_param = workflow_param.get('NETWORK', None) train_param = workflow_param.get('TRAINING', None) infer_param = workflow_param.get('INFERENCE', None) app_param = workflow_param.get('CUSTOM', None) except AttributeError: tf.logging.fatal('parameters should be dictionaries') raise assert os.path.exists(system_param.model_dir), \ 'Model folder not exists {}'.format(system_param.model_dir) self.is_training = (system_param.action == "train") # hardware-related parameters self.num_threads = max(system_param.num_threads, 1) \ if self.is_training else 1 self.num_gpus = system_param.num_gpus \ if self.is_training else min(system_param.num_gpus, 1) set_cuda_device(system_param.cuda_devices) # set output TF model folders self.model_dir = touch_folder( os.path.join(system_param.model_dir, 'models')) self.session_prefix = os.path.join(self.model_dir, FILE_PREFIX) if self.is_training: assert train_param, 'training parameters not specified' summary_root = os.path.join(system_param.model_dir, 'logs') self.summary_dir = get_latest_subfolder( summary_root, create_new=train_param.starting_iter == 0) self.initial_iter = train_param.starting_iter self.final_iter = max(train_param.max_iter, self.initial_iter) self.save_every_n = train_param.save_every_n self.tensorboard_every_n = train_param.tensorboard_every_n self.max_checkpoints = \ max(train_param.max_checkpoints, self.max_checkpoints) self.gradients_collector = GradientsCollector( n_devices=max(self.num_gpus, 1)) self.validation_every_n = train_param.validation_every_n if self.validation_every_n > 0: self.validation_max_iter = max(self.validation_max_iter, train_param.validation_max_iter) action_param = train_param else: assert infer_param, 'inference parameters not specified' self.initial_iter = infer_param.inference_iter action_param = infer_param self.outputs_collector = OutputsCollector( n_devices=max(self.num_gpus, 1)) # create an application instance assert app_param, 'application specific param. not specified' app_module = ApplicationDriver._create_app(app_param.name) self.app = app_module(net_param, action_param, self.is_training) # initialise data input data_partitioner = ImageSetsPartitioner() # clear the cached file lists data_partitioner.reset() do_new_partition = \ self.is_training and self.initial_iter == 0 and \ (not os.path.isfile(system_param.dataset_split_file)) and \ (train_param.exclude_fraction_for_validation > 0 or train_param.exclude_fraction_for_inference > 0) data_fractions = None if do_new_partition: assert train_param.exclude_fraction_for_validation > 0 or \ self.validation_every_n <= 0, \ 'validation_every_n is set to {}, ' \ 'but train/validation splitting not available,\nplease ' \ 'check "exclude_fraction_for_validation" in the config ' \ 'file (current config value: {}).'.format( self.validation_every_n, train_param.exclude_fraction_for_validation) data_fractions = (train_param.exclude_fraction_for_validation, train_param.exclude_fraction_for_inference) if data_param: data_partitioner.initialise( data_param=data_param, new_partition=do_new_partition, ratios=data_fractions, data_split_file=system_param.dataset_split_file) if data_param and self.is_training and self.validation_every_n > 0: assert data_partitioner.has_validation, \ 'validation_every_n is set to {}, ' \ 'but train/validation splitting not available.\nPlease ' \ 'check dataset partition list {} ' \ '(remove file to generate a new dataset partition). ' \ 'Or set validation_every_n to -1.'.format( self.validation_every_n, system_param.dataset_split_file) # initialise readers self.app.initialise_dataset_loader(data_param, app_param, data_partitioner) self._data_partitioner = data_partitioner # pylint: disable=not-context-manager with self.graph.as_default(), tf.name_scope('Sampler'): self.app.initialise_sampler()