def system_test(self):
nclass = len(self.data_loader.meta_data.label_names)
for fold_id in self.data_loader.meta_data.nfolds:
for train_mode in ['pretrain_set', 'finetune_set']:
print('/*========== Test %s model on FOLD %s ==========*/' %
(train_mode.upper(), fold_id))
model = self.inflate_model(self.config,
nclass,
train_mode=train_mode)
mfile = cfg.get_model_filename(path=path.join(
self.config['path'][train_mode], self.attrib_cls),
train_mode=train_mode,
fold=fold_id)
model.load(mfile)
print('Loss: %s' % model.model.loss)
trainer = OGITSModelTrainer(model.model,
self.data_loader,
self.config,
fold_id=fold_id,
train_mode=train_mode,
attrib_cls=self.attrib_cls,
verbose=self.verbose,
overwrite=self.overwrite)
trainer.test()
trainer.evaluate()
def system_test(self):
nclass = len(self.data_loader.meta_data.label_names)
for fold_id in self.data_loader.meta_data.nfolds:
for mode in ['pretrain_set', 'finetune_set']:
print('/*========== Test %s model on FOLD %s ==========*/' %
(mode, fold_id))
model = self.inflate_model(self.config,
nclass,
train_mode=mode)
mfile = cfg.get_model_filename(path=self.config['path'][mode],
train_mode=mode,
fold=fold_id)
model.load(mfile)
print('Loss: %s' % model.model.loss)
test_gen = DL.batch_handler(
batch_type='validation',
data_file=self.data_loader.feat_file,
fold_lst=self.data_loader.meta_data.fold_list(
fold_id, 'test'),
config=self.config['model'][mode],
meta_data=self.data_loader.meta_data)
history = ModelValidator.validate_model(
model=model.model,
batch_gen=test_gen,
metrics=self.config['model'][mode]['metrics'])
print(history)
def system_train(self):
"""
Pre-training and fine-tuning logic
"""
print('Model params:')
pprint.pprint(self.config['model'].toDict())
dirs.mkdirs(self.config['path']['finetune_set'],
self.config['path']['pretrain_set'])
nclass = len(self.data_loader.meta_data.label_names)
for fold_id in self.data_loader.meta_data.nfolds:
if self.config['model']['do_pretrain']:
print('/*========== Pre-training on FOLD %s ==========*/' %
fold_id)
model = self.inflate_model(self.config,
nclass,
train_mode='pretrain_set')
print('Pre-train with loss: %s' % model.model.loss)
trainer = DCASEModelTrainer(model.model,
self.data_loader,
self.config,
fold_id=fold_id,
train_mode='pretrain_set',
verbose=self.verbose,
overwrite=self.overwrite)
trainer.train()
if self.config['model']['do_finetune']:
print('/*========== Fine-tuning on FOLD ==========*/' %
fold_id)
mfile = cfg.get_model_filename(
path=self.config['path']['pretrain_set'],
train_mode='pretrain_set',
fold=fold_id)
pre_model = self.inflate_model(self.config,
nclass,
train_mode='pretrain_set')
pre_model.load(mfile)
pre_model.chage_optimizer(self.config['model']['finetune_set'])
print('Finetune with loss: %s' % pre_model.model.loss)
finetuner = DCASEModelTrainer(pre_model.model,
self.data_loader,
self.config,
fold_id=fold_id,
train_mode='finetune_set',
verbose=self.verbose,
overwrite=self.overwrite)
finetuner.train()
def system_train(self):
"""
Pre-training and fine-tuning logic
"""
print('Model params:')
pprint.pprint(self.config['model'].toDict())
dirs.mkdirs(
path.join(self.config['path']['finetune_set'], self.attrib_cls),
path.join(self.config['path']['pretrain_set'], self.attrib_cls))
nclass = len(self.data_loader.meta_data.label_names)
for fold_id in self.data_loader.meta_data.nfolds:
if self.config['model']['do_pretrain']:
print('/*========== Pre-training ==========*/')
model = self.inflate_model(config=self.config, nclass=nclass)
print('Pre-train with loss: %s' % model.model.loss)
trainer = OGITSModelTrainer(model.model,
self.data_loader,
self.config,
fold_id=fold_id,
train_mode='pretrain_set',
attrib_cls=self.attrib_cls,
verbose=self.verbose,
overwrite=self.overwrite)
trainer.train()
if self.config['model']['do_finetune']:
print('/*========== Fine-tuning ==========*/')
mfile = cfg.get_model_filename(path=path.join(
self.config['path']['pretrain_set'], self.attrib_cls),
train_mode='pretrain_set',
fold=fold_id)
pre_model = self.inflate_model(config=self.config,
nclass=nclass)
pre_model.load(mfile)
pre_model.chage_optimizer(self.config['model']['finetune_set'],
change_out_unit=True)
print('Finetune with loss: %s' % pre_model.model.loss)
finetuner = OGITSModelTrainer(pre_model.model,
self.data_loader,
self.config,
fold_id=fold_id,
train_mode='finetune_set',
attrib_cls=self.attrib_cls,
verbose=self.verbose,
overwrite=self.overwrite)
finetuner.train()
def __init__(self, model, data, config, **kwargs):
super(OGITSModelTrainer, self).__init__(model, data, config)
self.train_mode = kwargs.get('train_mode', 'pretrain_set')
self.fold_id = kwargs.get('fold_id', 1)
self.attrib_cls = kwargs.get('attrib_cls', 'manner')
self.verbose = kwargs.get('verbose', 1)
self.overwrite = kwargs.get('overwrite', False)
self.callbacks = []
mpath = path.join(self.config['path'][self.train_mode],
self.attrib_cls)
self.model_file = cfg.get_model_filename(path=mpath,
train_mode=self.train_mode,
fold=self.fold_id)
def __init__(self, model, data, config, **kwargs):
super(DCASEModelTrainer, self).__init__(model, data, config)
self.train_mode = kwargs.get('train_mode', 'pretrain_set')
self.fold_id = kwargs.get('fold_id', 1)
self.verbose = kwargs.get('verbose', 1)
self.overwrite = kwargs.get('overwrite', False)
self.callbacks = []
self.loss = []
self.val_loss = []
self.model_file = cfg.get_model_filename(
path=self.config['path'][self.train_mode],
train_mode=self.train_mode,
fold=self.fold_id)
def _experiment(self, params):
"""
Calculate model validation error on the current hyper parameters 'params'.
Hyper parameters are sampled from the search space.
NOTE: we have to use only validation data set here
NOTE2: hyperopt is minimizing, so use 100 - microF1, 100 - Acc, but EER!
"""
self.model_cfg.update(params)
print('=*' * 40)
print('\n Trying Model parameters: ')
pprint.pprint(self.model_cfg.toDict(), indent=2)
# rebuild model from scratch
self.model.rebuild(self.init_model_cfg)
# TODO load pre-trained weights to optimize those Bayes space
mfile = cfg.get_model_filename(path=path.join(
self.config['path']['pretrain_set'], self.attrib_cls),
train_mode=self.train_mode,
fold=self.fold_id)
self.model.load(mfile)
self.model.chage_optimizer(self.model_cfg, change_out_unit=True)
######################
# init batch generators
train_gen = DL.batch_handler(batch_type=self.model_cfg['batch_type'],
data_file=self.data.feat_file,
fold_lst=self.data.meta_data.fold_list(
self.fold_id, 'train'),
config=self.model_cfg,
meta_data=self.data.meta_data)
cv_gen = DL.batch_handler(batch_type='sed_validation',
data_file=self.data.feat_file,
fold_lst=self.data.meta_data.fold_list(
self.fold_id, 'validation'),
config=self.model_cfg,
meta_data=self.data.meta_data)
def mfom_batch_wrap(xy_gen):
for x, y in xy_gen.batch():
yield [y, x], y
wrap_gen = mfom_batch_wrap(train_gen) \
if self.is_mfom_objective(self.model.model) \
else train_gen.batch()
self._init_callbacks(cv_gen)
batch_sz = self.model_cfg['batch']
samp_sz = train_gen.samples_number()
print('Epoch size: %d observations' % samp_sz)
# train model
trial_result = {'params': self.model_cfg.toDict()}
try:
start = timer()
self.model.model.fit_generator(wrap_gen,
steps_per_epoch=samp_sz // batch_sz,
nb_epoch=self.MAX_EPOCHS,
verbose=self.verbose,
workers=1,
callbacks=self.callbacks)
run_time = timer() - start
# validate model
mvals = ModelValidator.validate_model(
model=self.model.model,
batch_gen=cv_gen,
metrics=self.model_cfg['metrics'])
# loss of the hyperopt
loss = mvals[self.config['callback']['monitor']]
loss = loss if self.config['callback'][
'mode'] == 'min' else 100. - loss
trial_result.update({
'validation': mvals,
'loss': loss,
'time': run_time,
'status': STATUS_OK
})
return trial_result
except Exception as e:
print('[error] Type: %s, %s' % (type(e), e.message))
print('-' * 30)
trial_result.update({
'loss': 999999,
'time': 0,
'status': STATUS_FAIL
})
return trial_result
finally:
train_gen.stop()
cv_gen.stop()