def objective(trial):
dataset_cls = EEGDataSet
set_dataloader_func = DATALOADERS[TRAIN_CONF['dataloader_type']]
expt_note = 'Test Patient\tAccuracy\tRecall\n'
metrics = [
Metric('loss', direction='minimize', save_model=True),
Metric('accuracy', direction='maximize'),
Metric('recall_1', direction='maximize'),
Metric('far', direction='minimize')
]
TRAIN_CONF['class_names'] = list(set(LABELS.values()))
TRAIN_CONF['iterations'] = trial.suggest_int('iterations', 10, 100)
TRAIN_CONF['k_disc'] = trial.suggest_int('k_disc', 5, 30)
TRAIN_CONF['k_clf'] = trial.suggest_int('k_clf', 5, 30)
train_manager = TrainManager(TRAIN_CONF, load_func, label_func, dataset_cls, set_dataloader_func, metrics, expt_note)
model, val_metrics, test_metrics = train_manager.train_test()
for phase, metrics in zip(['val', 'test'], [val_metrics, test_metrics]):
for metric, value in metrics.items():
trial.set_user_attr(f'{phase}_{metric}', value)
return val_metrics['loss'].mean()
def experiment(train_conf) -> float:
dataset_cls = EEGDataSet
set_dataloader_func = set_dataloader
metrics = [
Metric('loss', direction='minimize'),
Metric('accuracy', direction='maximize', save_model=True),
Metric('recall_1', direction='maximize'),
Metric('recall_2', direction='maximize'),
Metric('far', direction='minimize')
]
train_conf['class_names'] = list(LABELS.values())
train_manager = TrainManager(train_conf, load_func, label_func,
dataset_cls, set_dataloader_func, metrics)
train_manager.train()
def experiment(train_conf) -> float:
dataset_cls = EEGDataSet
set_dataloader_func = set_dataloader
expt_note = 'Test Patient\tAccuracy\tRecall\n'
metrics = [
Metric('loss', direction='minimize'),
Metric('accuracy', direction='maximize', save_model=True),
Metric('recall_1', direction='maximize'),
# Metric('far', direction='minimize')
]
train_conf['class_names'] = [0, 1]
train_conf['model_manager'] = 'keras'
train_manager = TrainManager(train_conf, load_func, label_func,
dataset_cls, set_dataloader_func, metrics,
expt_note)
if not train_conf['only_test']:
model, val_metrics, test_metrics = train_manager.train_test()
else:
train_manager.test()
now_time = datetime.today().strftime('%y%m%d%H%M')
expt_name = f"{len(train_conf['class_names'])}-class_{train_conf['model_type']}_{train_conf['expt_id']}_{now_time}.txt"
with open(Path(__file__).parent.parent / 'output' / expt_name, 'w') as f:
f.write(f"experiment notes:\n{train_manager.expt_note}\n\n")
f.write(f"{train_conf['k_fold']} fold results:\n")
for phase in ['val', 'test']:
if f'{phase}_metrics' not in locals().keys():
continue
f.write(f"{phase} phase results:\n")
metrics = locals()[f'{phase}_metrics']
for metric_name, meter in metrics.items():
f.write(
f'{metric_name} score\t mean: {meter.mean() :.4f}\t std: {meter.std() :.4f}\n'
)
f.write('\nParameters:\n')
f.write(json.dumps(train_conf, indent=4))
def hss_experiment(train_conf) -> float:
phases = ['train', 'val', 'test']
if train_conf['task_type'] == 'regress':
train_conf['class_names'] = [0]
else:
train_conf['class_names'] = [0, 1, 2]
train_conf['prev_classes'] = [0, 1]
one_audio_sec = 10
sr = 4000
dataloaders = {}
for phase in phases:
process_func = Preprocessor(train_conf, phase, sr).preprocess
load_func = set_load_func(sr, one_audio_sec)
dataset = ManualDataSet(train_conf[f'{phase}_path'], train_conf, load_func, process_func, hss_label_func, phase)
dataloaders[phase] = DATALOADERS[train_conf['dataloader_type']](dataset, phase, train_conf)
metrics = [
Metric('loss', direction='minimize', save_model=True),
Metric('uar', direction='maximize'),
]
model_manager = BaseModelManager(train_conf['class_names'], train_conf, dataloaders, metrics)
model_manager.train()
_, _, metrics = model_manager.test(return_metrics=True)
del model_manager.model
uar = [metric for metric in metrics if metric.name == 'uar'][0]
(Path(__file__).resolve().parent.parent / 'output' / 'params').mkdir(exist_ok=True)
with open(Path(__file__).resolve().parent.parent / 'output' / 'params' / f"{train_conf['log_id']}.txt", 'w') as f:
f.write('\nParameters:\n')
f.write(json.dumps(train_conf, indent=4))
(Path(__file__).resolve().parent.parent / 'output' / 'metrics').mkdir(exist_ok=True)
metrics2df(metrics, phase='test').to_csv(
Path(__file__).resolve().parent.parent / 'output' / 'metrics' / f"{train_conf['log_id']}_test.csv", index=False)
return uar.average_meter['test'].value
def experiment(train_conf) -> float:
dataset_cls = EEGDataSet
set_dataloader_func = set_dataloader
metrics = [
Metric('loss', direction='minimize'),
Metric('accuracy', direction='maximize', save_model=True),
Metric('recall_1', direction='maximize'),
Metric('recall_2', direction='maximize'),
# Metric('far', direction='minimize')
]
train_conf['class_names'] = list(range(len(LABELS)))
train_manager = TrainManager(train_conf, load_func, label_func,
dataset_cls, set_dataloader_func, metrics)
train_manager.train()
if train_conf['test']:
pred_list, label_list = train_manager.test()
voting(256, pred_list, label_list)
def test___init__(self):
metric_pattern = ['loss', 'recall', 'far', 'accuracy', 'confusion_matrix']
test_pattern = []
for metric_name in metric_pattern:
test_pattern.append({
'description': f'{metric_name}の場合の初期化',
'metric': metric_name,
'expected': metric_name
})
for test_case in test_pattern:
actual = Metric(test_case['metric'], direction='minimize')
with self.subTest(test_case['description']):
self.assertEqual(test_case['expected'], actual.name)
def setUp(self):
# TODO GPU/CPUの互換性テストをどうやるか
self.base_classes = [0, 1, 2]
self.input_size = 3
self.base_cfg = {
'model_type': 'rnn', 'gpu_id': 0, 'optimizer': 'adam', 'lr': 0.001, 'momentum': 0.9, 'weight_decay': 0.0,
'learning_anneal': 1.1, 'batch_size': 32, 'epoch_rate': 1.0, 'num_workers': 4, 'loss_weight': [1.0, 1.0],
'epochs': 2, 'model_path': f'{TEST_PATH}/outputs/models/sth.pth', 'seed': 0, 'silent': True,
'log_id': 'results', 'tensorboard': False, 'log_dir': 'visualize/', 'max_norm': 400, 'rnn_type': 'gru',
'rnn_hidden_size': 400, 'rnn_n_layers': 3, 'bidirectional': True, 'is_inference_softmax': True
}
self.base_cfg['cuda'] = torch.cuda.is_available()
self.base_cfg['loss_weight'] = [1.0] * len(self.base_classes)
self.base_cfg['input_size'] = self.input_size
self.base_cfg['task_type'] = 'classify' # TODO regresのテストも実装
self.dataloaders = {}
for phase in ['train', 'val']:
dataset = MockDataSet(self.base_cfg['batch_size'], self.input_size)
self.dataloaders[phase] = MockDataLoader(dataset, batch_size=self.base_cfg['batch_size'])
self.metrics = [
Metric('loss', direction='minimize'),
Metric('accuracy', direction='maximize', save_model=True),
]
def experiment(train_conf) -> float:
dataset_cls = EEGDataSet
labels = LABELS if train_conf['data_type'] == 'children' else ICTALS_3
label_func = set_label_func(labels)
set_dataloader_func = DATALOADERS[train_conf['dataloader_type']]
expt_note = 'Test Patient\tAccuracy\tRecall\n'
metrics = [
Metric('loss', direction='minimize', save_model=True),
Metric('accuracy', direction='maximize'),
Metric('recall_1', direction='maximize'),
Metric('far', direction='minimize')
]
# train_conf['class_names'] = list(set(LABELS.values()))
train_conf['class_names'] = [0, 1]
train_manager = TrainManager(train_conf, load_func, label_func,
dataset_cls, set_dataloader_func, metrics,
expt_note)
model, val_metrics, test_metrics = train_manager.train_test()
now_time = datetime.today().strftime('%y%m%H%M')
expt_name = f"{len(train_conf['class_names'])}-class_{train_conf['model_type']}_{train_conf['expt_id']}_{now_time}.txt"
with open(Path(__file__).parent.parent / 'output' / expt_name, 'w') as f:
f.write(f"experiment notes:\n{train_manager.expt_note}\n\n")
f.write(f"{train_conf['k_fold']} fold results:\n")
for phase in ['val', 'test']:
f.write(f"{phase} phase results:\n")
metrics = locals()[f'{phase}_metrics']
for metric_name, meter in metrics.items():
f.write(
f'{metric_name} score\t mean: {meter.mean() :.4f}\t std: {meter.std() :.4f}\n'
)
f.write('\nParameters:\n')
f.write(json.dumps(train_conf, indent=4))
def cv_experiment(train_conf) -> float:
phases = ['train', 'val', 'test']
one_audio_sec = 60
sr = 2000
if train_conf['task_type'] == 'regress':
train_conf['class_names'] = [0]
else:
train_conf['class_names'] = [0, 1]
train_conf['prev_classes'] = [0, 1]
dataset_cls = ManualDataSet
set_dataloader_func = set_dataloader
process_func = Preprocessor(train_conf, phase='test', sr=sr).preprocess
train_val_metrics = [
Metric('loss', direction='minimize', save_model=True),
Metric('uar', direction='maximize'),
]
test_metrics = [
Metric('loss', direction='minimize', save_model=True),
Metric('uar', direction='maximize'),
Metric('recall_1', direction='maximize'),
Metric('specificity', direction='maximize'),
Metric('f1', direction='maximize'),
]
metrics = {'train': deepcopy(train_val_metrics), 'val': train_val_metrics, 'test': test_metrics}
load_func = set_load_func(sr, one_audio_sec)
train_manager = TrainManager(train_conf, load_func, cinc_label_func, dataset_cls, set_dataloader_func, metrics,
process_func=process_func)
model_manager, val_cv_metrics, test_cv_metrics = train_manager.train_test()
(Path(__file__).resolve().parent.parent / 'output' / 'params').mkdir(exist_ok=True)
with open(Path(__file__).resolve().parent.parent / 'output' / 'params' / f"{train_conf['log_id']}.txt", 'w') as f:
f.write('\nParameters:\n')
f.write(json.dumps(train_conf, indent=4))
(Path(__file__).resolve().parent.parent / 'output' / 'metrics').mkdir(exist_ok=True)
metrics2df(test_cv_metrics, phase='test').to_csv(
Path(__file__).resolve().parent.parent / 'output' / 'metrics' / f"{train_conf['log_id']}_test.csv", index=False)
return val_cv_metrics['uar'].mean(), test_cv_metrics
def setUp(self):
self.class_names = [0, 1, 2]
self.metrics = [
Metric('loss', direction='minimize', save_model=True),
Metric('accuracy', direction='maximize'),
]