config = json5.load(f) # build up the data generator with open(config['generator']['data']) as f: data_config = json5.load(f) data_list = data_config['list'] if args.test: data_list = data_list[:1] loader_config = data_config['loader'] loader_name = loader_config.pop('name') data_loader = DataLoader(loader_name, **loader_config) data_loader.set_data_list(data_list) data_gen = DataGenerator(data_loader, config['generator']['struct']) # build up the reverter reverter = Reverter(data_gen) DL = data_gen.struct['DL'] PG = data_gen.struct['PG'] BG = data_gen.struct['BG'] # ensure the order if PG.n_workers > 1: assert PG.ordered assert BG.n_workers == 1 if 'AG' in data_gen.struct: assert data_gen.struct['AG'].n_workers == 1 # - GPUs if 'gpus' in config: if isinstance(config['gpus'], list): gpus = ','.join([str(idx) for idx in config['gpus']]) else:
def run(self, stage): stage_config = self.config['stage'][stage] # build data flow from the given data generator # single data flow if isinstance(stage_config['generator'], str): data_gen = self.generators[stage_config['generator']] class_names = data_gen.struct['DL'].ROIs n_steps = len(data_gen) gen_tags = None # multiple data flows elif isinstance(stage_config['generator'], dict): gens = [ self.generators[cfg] for cfg in stage_config['generator'].values() ] data_gen = zip(*gens) class_names = gens[0].struct['DL'].ROIs n_steps = min([len(g) for g in gens]) gen_tags = list(stage_config['generator'].keys()) else: raise TypeError('generator of type %s is not supported.' % type(stage_config['generator'])) progress_bar = tqdm(data_gen, total=n_steps, ncols=get_tty_columns(), dynamic_ncols=True, desc='[%s] loss: %.5f, accu: %.5f' % (stage, 0.0, 0.0)) if stage not in self.step: self.step[stage] = 1 task_result_list = { task_name: [] for task_name in stage_config['task'] } need_revert = 'revert' in stage_config and stage_config['revert'] for batch in progress_bar: # format the batch into {tag: {key: data}} if gen_tags is None: assert isinstance(batch, dict) formatted_batch = {NO_TAG: dict()} for key in batch: if torch.cuda.device_count() >= 1: formatted_batch[NO_TAG][key] = batch[key].cuda() else: formatted_batch[NO_TAG][key] = batch[key] else: formatted_batch = dict() for (tag, tag_batch) in zip(gen_tags, batch): tag_data = dict() for key in tag_batch: if torch.cuda.device_count() >= 1: tag_data[key] = tag_batch[key].cuda() else: tag_data[key] = tag_batch[key] formatted_batch[tag] = tag_data # execute each task in this stage for task_name in stage_config['task']: task_config = self.tasks[task_name] # skip the task periodically if 'period' in task_config \ and self.step[stage] % task_config['period'] != 0: continue # modify the status of modules if the running task changed if self.running_task != task_name: # toggle trainable parameters of each module for (key, toggle) in task_config['toggle'].items(): self.handlers[key].model.train(toggle) for param in self.handlers[key].model.parameters(): param.requires_grad = toggle self.running_task = task_name task_result = self.run_task( task_config, formatted_batch, need_revert=need_revert, ) # detach all the results, move them to CPU, and convert them to numpy for key in task_result: task_result[key] = task_result[key].detach().cpu().numpy() # average accuracy if multi-dim assert 'accu' in task_result if task_result['accu'].ndim == 0: step_accu = math.nan if task_result[ 'accu'] == math.nan else task_result['accu'] else: assert task_result['accu'].ndim == 1 empty = True for acc in task_result['accu']: if not np.isnan(acc): empty = False break step_accu = math.nan if empty else np.nanmean( task_result['accu']) assert 'loss' in task_result progress_bar.set_description( '[%s][%s] loss: %.5f, accu: %.5f' % (stage, task_name, task_result['loss'], step_accu)) if self.logger is not None: self.logger.add_scalar( '%s/%s/step/loss' % (stage, task_name), task_result['loss'], self.step[stage]) self.logger.add_scalar( '%s/%s/step/accu' % (stage, task_name), -1 if math.isnan(step_accu) else step_accu, self.step[stage]) task_result_list[task_name].append(task_result) self.step[stage] += 1 # summarize the result list task_summary = dict() for (task_name, result_list) in task_result_list.items(): if len(result_list) == 0: continue summary = dict() if need_revert: reverter = Reverter(data_gen) result_collection_blacklist = reverter.revertible scores = dict() progress_bar = tqdm(reverter.on_batches(result_list), total=len(reverter.data_list), dynamic_ncols=True, ncols=get_tty_columns(), desc='[Data index]') for reverted in progress_bar: data_idx = reverted['idx'] scores[data_idx] = data_gen.struct['DL'].evaluate( data_idx, reverted['prediction']) info = '[%s] mean score: %.3f' % ( data_idx, np.mean(list(scores[data_idx].values()))) progress_bar.set_description(info) # summerize score of each class over data indices cls_scores = { cls: np.mean([scores[data_idx][cls] for data_idx in scores]) for cls in class_names } summary['scores'] = scores summary['cls_scores'] = cls_scores summary['cls_mean'] = np.mean(list(cls_scores.values())) else: result_collection_blacklist = [] # collect results except those revertible ones, e.g., collect accu, loss summary.update({ key: np.nanmean(np.vstack([result[key] for result in result_list]), axis=0) for key in result_list[0].keys() if key not in result_collection_blacklist }) # process 1D array accu to dictionary of each class score if len(summary['accu']) > 1: assert len(summary['accu']) == len(class_names), (len( summary['accu']), len(class_names)) summary['cls_accu'] = { cls: summary['accu'][i] for (i, cls) in enumerate(class_names) } summary['accu'] = summary['accu'].mean() # print summary info print('[%s][%s] Average: ' % (stage, task_name) + ', '.join([ '%s: %.3f' % (key, val) for (key, val) in summary.items() if not isinstance(val, dict) ])) if 'cls_scores' in summary: print('Class score: ' + ', '.join([ '%s: %.3f' % (key, val) for (key, val) in summary['cls_scores'].items() ])) print('Class mean: %.3f' % summary['cls_mean']) task_summary[task_name] = summary return task_summary
# - data pipeline data_gen = dict() loader_name = loader_config.pop('name') ROIs = None for stage in stages: data_loader = DataLoader(loader_name, **loader_config) if data_list[stage] is not None: data_loader.set_data_list(data_list[stage]) data_gen[stage] = DataGenerator(data_loader, generator_config[stage]) if ROIs is None: ROIs = data_loader.ROIs # FIXME reverter = Reverter(data_gen['valid']) # - GPUs os.environ['CUDA_VISIBLE_DEVICES'] = str(config['gpus']) torch.backends.cudnn.enabled = True # - model model_handlers = { 'seg': ModelHandler( config['models']['seg'], checkpoint=args.checkpoint, ), 'dis': ModelHandler( config['models']['dis'], checkpoint=args.checkpoint_dis, ),
timer = time.time() with open(args.loader_config) as f: loader_config = yaml.safe_load(f) loader_name = loader_config.pop('name') data_loader = DataLoader(loader_name, **loader_config) with open(args.generator_config) as f: generator_config = yaml.safe_load(f) data_gen = DataGenerator(data_loader, generator_config) os.makedirs(args.output_dir, exist_ok=True) DL = data_gen.struct['DL'] batch_list = list({'prediction': data['label']} for data in data_gen) reverter = Reverter(data_gen) progress_bar = tqdm(reverter.on_batches(batch_list), total=len(reverter.data_list), dynamic_ncols=True, desc='[Data index]') scores = dict() for result in progress_bar: data_idx = result['idx'] DL.save_prediction(data_idx, result['prediction'], args.output_dir) scores[data_idx] = DL.evaluate(data_idx, result['prediction']) info = '[%s] ' % data_idx info += ', '.join('%s: %.3f' % (key, val) for key, val in scores[data_idx].items()) progress_bar.set_description(info)
def run_invertible(self, stage): stage_config = self.config['stage'][stage] assert len(stage_config['task']) == 1 if stage not in self.step: self.step[stage] = 1 task_name = stage_config['task'][0] task_config = self.tasks[task_name] (data_gen, n_steps, gen_tags, class_names) = self.get_data_gen(stage) inv_gen = self.invertible_gen(stage, data_gen, n_steps, gen_tags, task_name, task_config) reverter = Reverter(data_gen) summary = dict() result_collection_blacklist = reverter.revertible # We use a mutable result list to collect the results during the validation result_list = [] scores = dict() progress_bar = tqdm(reverter.on_future_batches( inv_gen, mutable_results=result_list, ), total=len(reverter.data_list), dynamic_ncols=True, ncols=get_tty_columns(), desc='[Data index]') for reverted in progress_bar: data_idx = reverted['idx'] scores[data_idx] = data_gen.struct['DL'].evaluate( data_idx, reverted['prediction']) info = '[%s] mean score: %.3f' % ( data_idx, np.mean(list(scores[data_idx].values()))) progress_bar.set_description(info) # summerize score of each class over data indices cls_scores = { cls: np.mean([scores[data_idx][cls] for data_idx in scores]) for cls in class_names } summary['scores'] = scores summary['cls_scores'] = cls_scores summary['cls_mean'] = np.mean(list(cls_scores.values())) # collect results except those revertible ones, e.g., collect accu, loss summary.update({ key: np.nanmean(np.vstack([result[key] for result in result_list]), axis=0) for key in result_list[0].keys() if key not in result_collection_blacklist }) # process 1D array accu to dictionary of each class score if len(summary['accu']) > 1: assert len(summary['accu']) == len(class_names), (len( summary['accu']), len(class_names)) summary['cls_accu'] = { cls: summary['accu'][i] for (i, cls) in enumerate(class_names) } summary['accu'] = summary['accu'].mean() # print summary info print('[%s][%s] Average: ' % (stage, task_name) + ', '.join([ '%s: %.3f' % (key, val) for (key, val) in summary.items() if not isinstance(val, dict) ])) if 'cls_scores' in summary: print('Class score: ' + ', '.join([ '%s: %.3f' % (key, val) for (key, val) in summary['cls_scores'].items() ])) print('Class mean: %.3f' % summary['cls_mean']) return {task_name: summary}
def run(self, stage): stage_config = self.config['stage'][stage] # build data flow from the given data generator # single data flow if isinstance(stage_config['generator'], str): data_gen = self.generators[stage_config['generator']] class_names = data_gen.struct['DL'].ROIs n_steps = len(data_gen) gen_tags = None # multiple data flows elif isinstance(stage_config['generator'], dict): gens = [ self.generators[cfg] for cfg in stage_config['generator'].values() ] data_gen = zip(*gens) class_names = gens[0].struct['DL'].ROIs n_steps = min([len(g) for g in gens]) gen_tags = list(stage_config['generator'].keys()) # the forward config should match the multiple data flows assert isinstance(stage_config['forward'], dict) assert gen_tags == list(stage_config['forward'].keys()) else: raise TypeError('generator of type %s is not supported.' % type(stage_config['generator'])) progress_bar = tqdm(data_gen, total=n_steps, ncols=get_tty_columns(), dynamic_ncols=True, desc='[%s] loss: %.5f, accu: %.5f' % (stage, 0.0, 0.0)) if stage not in self.step: self.step[stage] = 1 # toggle trainable parameters of each module need_backward = False for key, toggle in stage_config['toggle'].items(): self.handlers[key].model.train(toggle) for param in self.handlers[key].model.parameters(): param.requires_grad = toggle if toggle: need_backward = True result_list = [] need_revert = 'revert' in stage_config and stage_config['revert'] for batch in progress_bar: self.step[stage] += 1 # single data flow if gen_tags is None: assert isinstance(batch, dict) # insert batch to data data = dict() for key in batch: if torch.cuda.device_count() >= 1: data[key] = batch[key].cuda() else: data[key] = batch[key] # forward for key in stage_config['forward']: data.update(self.handlers[key].model(data)) # multiple data flows else: assert isinstance(batch, tuple) data = dict() for (tag, tag_batch) in zip(gen_tags, batch): tag_data = dict() # insert batch to data for key in tag_batch: if torch.cuda.device_count() >= 1: tag_data[key] = tag_batch[key].cuda() else: tag_data[key] = tag_batch[key] # forward for key in stage_config['forward'][tag]: tag_data.update(self.handlers[key].model(tag_data)) # insert tag data back to the data data.update({ '%s_%s' % (key, tag): tag_data[key] for key in tag_data }) # compute loss and accuracy results = self.metrics[stage_config['metric']](data) # backpropagation if need_backward: results['loss'].backward() for key, toggle in stage_config['toggle'].items(): if toggle: self.optims[key].step() self.optims[key].zero_grad() # compute match for dice score of each case after reversion if need_revert: assert 'prediction' in data, list(data.keys()) assert 'label' in data, list(data.keys()) with torch.set_grad_enabled(False): match, total = match_up( data['prediction'], data['label'], needs_softmax=True, batch_wise=True, threshold=-1, ) results.update({'match': match, 'total': total}) # detach all results, move to CPU, and convert to numpy for key in results: results[key] = results[key].detach().cpu().numpy() # average accuracy if multi-dim assert 'accu' in results if results['accu'].ndim == 0: step_accu = math.nan if results[ 'accu'] == math.nan else results['accu'] else: assert results['accu'].ndim == 1 empty = True for acc in results['accu']: if not np.isnan(acc): empty = False break step_accu = math.nan if empty else np.nanmean(results['accu']) assert 'loss' in results progress_bar.set_description('[%s] loss: %.5f, accu: %.5f' % (stage, results['loss'], step_accu)) if self.logger is not None: self.logger.add_scalar('%s/step/loss' % stage, results['loss'], self.step[stage]) self.logger.add_scalar( '%s/step/accu' % stage, -1 if math.isnan(step_accu) else step_accu, self.step[stage]) result_list.append(results) summary = dict() if need_revert: reverter = Reverter(data_gen) result_collection_blacklist = reverter.revertible scores = dict() progress_bar = tqdm(reverter.on_batches(result_list), total=len(reverter.data_list), dynamic_ncols=True, ncols=get_tty_columns(), desc='[Data index]') for reverted in progress_bar: data_idx = reverted['idx'] scores[data_idx] = reverted['score'] info = '[%s] mean score: %.3f' % ( data_idx, np.mean(list(scores[data_idx].values()))) progress_bar.set_description(info) # summerize score of each class over data indices cls_scores = { cls: np.mean([scores[data_idx][cls] for data_idx in scores]) for cls in class_names } cls_scores.update( {'mean': np.mean([cls_scores[cls] for cls in class_names])}) summary['scores'] = scores summary['cls_scores'] = cls_scores else: result_collection_blacklist = [] # collect results except those revertible ones, e.g., accu, loss summary.update({ key: np.nanmean(np.vstack([result[key] for result in result_list]), axis=0) for key in result_list[0].keys() if key not in result_collection_blacklist }) # process 1D array accu to dictionary of each class score if len(summary['accu']) > 1: assert len(summary['accu']) == len(class_names), (len( summary['accu']), len(class_names)) summary['cls_accu'] = { cls: summary['accu'][i] for (i, cls) in enumerate(class_names) } summary['accu'] = summary['accu'].mean() # print summary info print('Average: ' + ', '.join([ '%s: %.3f' % (key, val) for (key, val) in summary.items() if not isinstance(val, dict) ])) if 'cls_scores' in summary: print('Class score: ' + ', '.join([ '%s: %.3f' % (key, val) for (key, val) in summary['cls_scores'].items() ])) return summary
# load config with open(args.config) as f: config = yaml.safe_load(f) generator_config = config['generator'] with open(config['data']) as f: data_config = yaml.safe_load(f) data_list = data_config['list'] loader_config = data_config['loader'] # - data pipeline loader_name = loader_config.pop('name') data_loader = DataLoader(loader_name, **loader_config) if data_list is not None: data_loader.set_data_list(data_list) data_gen = DataGenerator(data_loader, generator_config) reverter = Reverter(data_gen) ROIs = data_loader.ROIs DL = data_gen.struct['DL'] PG = data_gen.struct['PG'] BG = data_gen.struct['BG'] # ensure the order if PG.n_workers > 1: assert PG.ordered assert BG.n_workers == 1 if 'AG' in data_gen.struct: assert data_gen.struct['AG'].n_workers == 1 assert 'output_threshold' in config if args.prediction_dir is not None:
loader_name = loader_config.pop('name') ROIs = None for stage in stages: data_loader = DataLoader(loader_name, **loader_config) if stage == 'train_ssl' and stage not in data_list: data_loader.set_data_list(data_list['valid']) else: assert stage in data_list data_loader.set_data_list(data_list[stage]) data_gen[stage] = DataGenerator(data_loader, generator_config[stage]) if ROIs is None: ROIs = data_loader.ROIs # FIXME reverter = Reverter(data_gen['valid']) # - GPUs os.environ['CUDA_VISIBLE_DEVICES'] = str(config['gpus']) torch.backends.cudnn.enabled = True # - model model_handler = ModelHandler(**config['model']) # - checkpoint handler ckpt_handler = CheckpointHandler(model_handler, **config['ckpt_handler']) # - optimizer if 'optimizer' in config: optimizer = Optimizer(config['optimizer'])(model_handler.model) else:
# load config with open(args.config) as f: config = yaml.safe_load(f) generator_config = config['generator'] with open(config['data']) as f: data_config = yaml.safe_load(f) data_list = data_config['list'] loader_config = data_config['loader'] # - data pipeline loader_name = loader_config.pop('name') data_loader = DataLoader(loader_name, **loader_config) if data_list is not None: data_loader.set_data_list(data_list) data_gen = DataGenerator(data_loader, generator_config) reverter = Reverter(data_gen) ROIs = data_loader.ROIs DL = data_gen.struct['DL'] PG = data_gen.struct['PG'] BG = data_gen.struct['BG'] # ensure the order if PG.n_workers > 1: assert PG.ordered assert BG.n_workers == 1 if 'AG' in data_gen.struct: assert data_gen.struct['AG'].n_workers == 1 assert 'output_threshold' in config if args.prediction_dir is not None:
# load config with open(args.config) as f: config = yaml.safe_load(f) generator_config = config['generator'] with open(config['data']) as f: data_config = yaml.safe_load(f) data_list = data_config['list'] loader_config = data_config['loader'] # - data pipeline loader_name = loader_config.pop('name') data_loader = DataLoader(loader_name, **loader_config) if data_list is not None: data_loader.set_data_list(data_list) data_gen = DataGenerator(data_loader, generator_config) reverter = Reverter(data_gen) ROIs = data_loader.ROIs DL = data_gen.struct['DL'] PG = data_gen.struct['PG'] BG = data_gen.struct['BG'] # ensure the order if PG.n_workers > 1: assert PG.ordered assert BG.n_workers == 1 if 'AG' in data_gen.struct: assert data_gen.struct['AG'].n_workers == 1 assert 'output_threshold' in config if args.prediction_dir is not None: