def objective(trial): dataset_cls = EEGDataSet set_dataloader_func = DATALOADERS[TRAIN_CONF['dataloader_type']] expt_note = 'Test Patient\tAccuracy\tRecall\n' metrics = [ Metric('loss', direction='minimize', save_model=True), Metric('accuracy', direction='maximize'), Metric('recall_1', direction='maximize'), Metric('far', direction='minimize') ] TRAIN_CONF['class_names'] = list(set(LABELS.values())) TRAIN_CONF['iterations'] = trial.suggest_int('iterations', 10, 100) TRAIN_CONF['k_disc'] = trial.suggest_int('k_disc', 5, 30) TRAIN_CONF['k_clf'] = trial.suggest_int('k_clf', 5, 30) train_manager = TrainManager(TRAIN_CONF, load_func, label_func, dataset_cls, set_dataloader_func, metrics, expt_note) model, val_metrics, test_metrics = train_manager.train_test() for phase, metrics in zip(['val', 'test'], [val_metrics, test_metrics]): for metric, value in metrics.items(): trial.set_user_attr(f'{phase}_{metric}', value) return val_metrics['loss'].mean()
def experiment(train_conf) -> float: dataset_cls = EEGDataSet set_dataloader_func = set_dataloader metrics = [ Metric('loss', direction='minimize'), Metric('accuracy', direction='maximize', save_model=True), Metric('recall_1', direction='maximize'), Metric('recall_2', direction='maximize'), Metric('far', direction='minimize') ] train_conf['class_names'] = list(LABELS.values()) train_manager = TrainManager(train_conf, load_func, label_func, dataset_cls, set_dataloader_func, metrics) train_manager.train()
def experiment(train_conf) -> float: dataset_cls = EEGDataSet set_dataloader_func = set_dataloader expt_note = 'Test Patient\tAccuracy\tRecall\n' metrics = [ Metric('loss', direction='minimize'), Metric('accuracy', direction='maximize', save_model=True), Metric('recall_1', direction='maximize'), # Metric('far', direction='minimize') ] train_conf['class_names'] = [0, 1] train_conf['model_manager'] = 'keras' train_manager = TrainManager(train_conf, load_func, label_func, dataset_cls, set_dataloader_func, metrics, expt_note) if not train_conf['only_test']: model, val_metrics, test_metrics = train_manager.train_test() else: train_manager.test() now_time = datetime.today().strftime('%y%m%d%H%M') expt_name = f"{len(train_conf['class_names'])}-class_{train_conf['model_type']}_{train_conf['expt_id']}_{now_time}.txt" with open(Path(__file__).parent.parent / 'output' / expt_name, 'w') as f: f.write(f"experiment notes:\n{train_manager.expt_note}\n\n") f.write(f"{train_conf['k_fold']} fold results:\n") for phase in ['val', 'test']: if f'{phase}_metrics' not in locals().keys(): continue f.write(f"{phase} phase results:\n") metrics = locals()[f'{phase}_metrics'] for metric_name, meter in metrics.items(): f.write( f'{metric_name} score\t mean: {meter.mean() :.4f}\t std: {meter.std() :.4f}\n' ) f.write('\nParameters:\n') f.write(json.dumps(train_conf, indent=4))
def hss_experiment(train_conf) -> float: phases = ['train', 'val', 'test'] if train_conf['task_type'] == 'regress': train_conf['class_names'] = [0] else: train_conf['class_names'] = [0, 1, 2] train_conf['prev_classes'] = [0, 1] one_audio_sec = 10 sr = 4000 dataloaders = {} for phase in phases: process_func = Preprocessor(train_conf, phase, sr).preprocess load_func = set_load_func(sr, one_audio_sec) dataset = ManualDataSet(train_conf[f'{phase}_path'], train_conf, load_func, process_func, hss_label_func, phase) dataloaders[phase] = DATALOADERS[train_conf['dataloader_type']](dataset, phase, train_conf) metrics = [ Metric('loss', direction='minimize', save_model=True), Metric('uar', direction='maximize'), ] model_manager = BaseModelManager(train_conf['class_names'], train_conf, dataloaders, metrics) model_manager.train() _, _, metrics = model_manager.test(return_metrics=True) del model_manager.model uar = [metric for metric in metrics if metric.name == 'uar'][0] (Path(__file__).resolve().parent.parent / 'output' / 'params').mkdir(exist_ok=True) with open(Path(__file__).resolve().parent.parent / 'output' / 'params' / f"{train_conf['log_id']}.txt", 'w') as f: f.write('\nParameters:\n') f.write(json.dumps(train_conf, indent=4)) (Path(__file__).resolve().parent.parent / 'output' / 'metrics').mkdir(exist_ok=True) metrics2df(metrics, phase='test').to_csv( Path(__file__).resolve().parent.parent / 'output' / 'metrics' / f"{train_conf['log_id']}_test.csv", index=False) return uar.average_meter['test'].value
def experiment(train_conf) -> float: dataset_cls = EEGDataSet set_dataloader_func = set_dataloader metrics = [ Metric('loss', direction='minimize'), Metric('accuracy', direction='maximize', save_model=True), Metric('recall_1', direction='maximize'), Metric('recall_2', direction='maximize'), # Metric('far', direction='minimize') ] train_conf['class_names'] = list(range(len(LABELS))) train_manager = TrainManager(train_conf, load_func, label_func, dataset_cls, set_dataloader_func, metrics) train_manager.train() if train_conf['test']: pred_list, label_list = train_manager.test() voting(256, pred_list, label_list)
def test___init__(self): metric_pattern = ['loss', 'recall', 'far', 'accuracy', 'confusion_matrix'] test_pattern = [] for metric_name in metric_pattern: test_pattern.append({ 'description': f'{metric_name}の場合の初期化', 'metric': metric_name, 'expected': metric_name }) for test_case in test_pattern: actual = Metric(test_case['metric'], direction='minimize') with self.subTest(test_case['description']): self.assertEqual(test_case['expected'], actual.name)
def setUp(self): # TODO GPU/CPUの互換性テストをどうやるか self.base_classes = [0, 1, 2] self.input_size = 3 self.base_cfg = { 'model_type': 'rnn', 'gpu_id': 0, 'optimizer': 'adam', 'lr': 0.001, 'momentum': 0.9, 'weight_decay': 0.0, 'learning_anneal': 1.1, 'batch_size': 32, 'epoch_rate': 1.0, 'num_workers': 4, 'loss_weight': [1.0, 1.0], 'epochs': 2, 'model_path': f'{TEST_PATH}/outputs/models/sth.pth', 'seed': 0, 'silent': True, 'log_id': 'results', 'tensorboard': False, 'log_dir': 'visualize/', 'max_norm': 400, 'rnn_type': 'gru', 'rnn_hidden_size': 400, 'rnn_n_layers': 3, 'bidirectional': True, 'is_inference_softmax': True } self.base_cfg['cuda'] = torch.cuda.is_available() self.base_cfg['loss_weight'] = [1.0] * len(self.base_classes) self.base_cfg['input_size'] = self.input_size self.base_cfg['task_type'] = 'classify' # TODO regresのテストも実装 self.dataloaders = {} for phase in ['train', 'val']: dataset = MockDataSet(self.base_cfg['batch_size'], self.input_size) self.dataloaders[phase] = MockDataLoader(dataset, batch_size=self.base_cfg['batch_size']) self.metrics = [ Metric('loss', direction='minimize'), Metric('accuracy', direction='maximize', save_model=True), ]
def experiment(train_conf) -> float: dataset_cls = EEGDataSet labels = LABELS if train_conf['data_type'] == 'children' else ICTALS_3 label_func = set_label_func(labels) set_dataloader_func = DATALOADERS[train_conf['dataloader_type']] expt_note = 'Test Patient\tAccuracy\tRecall\n' metrics = [ Metric('loss', direction='minimize', save_model=True), Metric('accuracy', direction='maximize'), Metric('recall_1', direction='maximize'), Metric('far', direction='minimize') ] # train_conf['class_names'] = list(set(LABELS.values())) train_conf['class_names'] = [0, 1] train_manager = TrainManager(train_conf, load_func, label_func, dataset_cls, set_dataloader_func, metrics, expt_note) model, val_metrics, test_metrics = train_manager.train_test() now_time = datetime.today().strftime('%y%m%H%M') expt_name = f"{len(train_conf['class_names'])}-class_{train_conf['model_type']}_{train_conf['expt_id']}_{now_time}.txt" with open(Path(__file__).parent.parent / 'output' / expt_name, 'w') as f: f.write(f"experiment notes:\n{train_manager.expt_note}\n\n") f.write(f"{train_conf['k_fold']} fold results:\n") for phase in ['val', 'test']: f.write(f"{phase} phase results:\n") metrics = locals()[f'{phase}_metrics'] for metric_name, meter in metrics.items(): f.write( f'{metric_name} score\t mean: {meter.mean() :.4f}\t std: {meter.std() :.4f}\n' ) f.write('\nParameters:\n') f.write(json.dumps(train_conf, indent=4))
def cv_experiment(train_conf) -> float: phases = ['train', 'val', 'test'] one_audio_sec = 60 sr = 2000 if train_conf['task_type'] == 'regress': train_conf['class_names'] = [0] else: train_conf['class_names'] = [0, 1] train_conf['prev_classes'] = [0, 1] dataset_cls = ManualDataSet set_dataloader_func = set_dataloader process_func = Preprocessor(train_conf, phase='test', sr=sr).preprocess train_val_metrics = [ Metric('loss', direction='minimize', save_model=True), Metric('uar', direction='maximize'), ] test_metrics = [ Metric('loss', direction='minimize', save_model=True), Metric('uar', direction='maximize'), Metric('recall_1', direction='maximize'), Metric('specificity', direction='maximize'), Metric('f1', direction='maximize'), ] metrics = {'train': deepcopy(train_val_metrics), 'val': train_val_metrics, 'test': test_metrics} load_func = set_load_func(sr, one_audio_sec) train_manager = TrainManager(train_conf, load_func, cinc_label_func, dataset_cls, set_dataloader_func, metrics, process_func=process_func) model_manager, val_cv_metrics, test_cv_metrics = train_manager.train_test() (Path(__file__).resolve().parent.parent / 'output' / 'params').mkdir(exist_ok=True) with open(Path(__file__).resolve().parent.parent / 'output' / 'params' / f"{train_conf['log_id']}.txt", 'w') as f: f.write('\nParameters:\n') f.write(json.dumps(train_conf, indent=4)) (Path(__file__).resolve().parent.parent / 'output' / 'metrics').mkdir(exist_ok=True) metrics2df(test_cv_metrics, phase='test').to_csv( Path(__file__).resolve().parent.parent / 'output' / 'metrics' / f"{train_conf['log_id']}_test.csv", index=False) return val_cv_metrics['uar'].mean(), test_cv_metrics
def setUp(self): self.class_names = [0, 1, 2] self.metrics = [ Metric('loss', direction='minimize', save_model=True), Metric('accuracy', direction='maximize'), ]