def train(self, callbacks=[]):
# Data setup.
data = h5py.File(self.cfg['hdf5_path_trn'])
tags = data.get('tags')[...]
idxs_trn, idxs_val = get_train_val_idxs(tags, self.cfg['trn_prop_data'], self.cfg['trn_prop_trn'])
steps_trn = math.ceil(len(idxs_trn) / self.cfg['trn_batch_size'])
steps_val = math.ceil(len(idxs_val) / self.cfg['trn_batch_size'])
gen_trn = self.batch_gen(data, idxs_trn, steps_trn, self.cfg['trn_augment_max_trn'], self.cfg['trn_balanced'])
gen_val = self.batch_gen(data, idxs_val, steps_val, self.cfg['trn_augment_max_val'])
opt = self.cfg['trn_optimizer'](**self.cfg['trn_optimizer_args'])
self.net.compile(optimizer=opt, loss=self.cfg['net_loss_func'], metrics=[F2, prec, reca] + tag_metrics())
self.net.summary()
pprint(self.cfg)
cb = [
FineTuneCB(**self.cfg['trn_finetune_args']),
ValidationCB(self.cfg['cpdir'], gen_val, self.cfg['trn_batch_size'], steps_val),
HistoryPlotCB('%s/history.png' % self.cpdir),
EarlyStopping(monitor='F2', min_delta=0.01, patience=20, verbose=1, mode='max'),
CSVLogger('%s/history.csv' % self.cpdir),
ModelCheckpoint('%s/model_{val_F2:.4f}_{epoch:02d}.hdf5' % self.cpdir,
monitor='val_F2', verbose=1, save_best_only=False, mode='max')
] + callbacks
if self.cfg['trn_monitor_val']:
cb.append(EarlyStopping(monitor='val_F2', min_delta=0.01, patience=20, verbose=1, mode='max'))
cb.append(ReduceLROnPlateau(monitor='val_F2', factor=0.5, patience=5, min_lr=1e-4,
epsilon=1e-2, verbose=1, mode='max'))
train = self.net.fit_generator(gen_trn, steps_per_epoch=steps_trn,
epochs=self.cfg['trn_epochs'], verbose=1, callbacks=cb)
return train.history
def train(self, weights_path=None, callbacks=[]):
# Metrics.
def prec(yt, yp):
yp = K.cast(yp > self.cfg['net_threshold'], 'float')
tp = K.sum(yt * yp)
fp = K.sum(K.clip(yp - yt, 0, 1))
return tp / (tp + fp + K.epsilon())
def reca(yt, yp):
yp = K.cast(yp > self.cfg['net_threshold'], 'float')
tp = K.sum(yt * yp)
fn = K.sum(K.clip(yt - yp, 0, 1))
return tp / (tp + fn + K.epsilon())
def F2(yt, yp):
p = prec(yt, yp)
r = reca(yt, yp)
b = 2.0
return (1 + b**2) * ((p * r) / (b**2 * p + r + K.epsilon()))
# Network setup. ResNet layers frozen at first.
net, self.cfg['preprocess_imgs_func'], self.cfg['preprocess_tags_func'] = \
self.cfg['net_builder_func'](self.cfg['input_shape'])
json.dump(net.to_json(),
open('%s/model.json' % self.cpdir, 'w'),
indent=2)
json.dump(serialize_config(self.cfg),
open('%s/config.json' % self.cpdir, 'w'))
# Data setup.
idxs_trn, idxs_val = get_train_val_idxs(self.cfg['hdf5_path_trn'],
self.cfg['trn_prop_data'],
self.cfg['trn_prop_trn'])
steps_trn = math.ceil(len(idxs_trn) / self.cfg['trn_batch_size'])
steps_val = math.ceil(len(idxs_val) / self.cfg['trn_batch_size'])
gen_trn = self.batch_gen(
idxs_trn,
steps_trn,
nb_augment_max=self.cfg['trn_augment_max_trn'])
gen_val = self.batch_gen(
idxs_val,
steps_val,
nb_augment_max=self.cfg['trn_augment_max_val'])
opt = self.cfg['trn_optimizer'](**self.cfg['trn_optimizer_args'])
net.compile(optimizer=opt,
loss=self.cfg['net_loss_func'],
metrics=[F2, prec, reca])
net.summary()
if weights_path is not None:
net.load_weights(weights_path)
pprint(self.cfg)
cb = [
# FineTuneCB(unfreeze_after=2, unfreeze_lr_mult=0.1),
HistoryPlotCB('%s/history.png' % self.cpdir),
EarlyStopping(monitor='F2',
min_delta=0.01,
patience=20,
verbose=1,
mode='max'),
CSVLogger('%s/history.csv' % self.cpdir),
ModelCheckpoint('%s/wvalf2.hdf5' % self.cpdir,
monitor='val_F2',
verbose=1,
save_best_only=True,
mode='max'),
TerminateOnNaN(),
] + callbacks
if self.cfg['trn_monitor_val']:
cb.append(
ReduceLROnPlateau(monitor='val_F2',
factor=0.5,
patience=5,
min_lr=1e-4,
epsilon=1e-2,
verbose=1,
mode='max'))
cb.append(
EarlyStopping(monitor='val_F2',
min_delta=0.01,
patience=20,
verbose=1,
mode='max'))
train = net.fit_generator(gen_trn,
steps_per_epoch=steps_trn,
epochs=self.cfg['trn_epochs'],
verbose=1,
callbacks=cb,
validation_data=gen_val,
validation_steps=steps_val,
workers=1)
return train.history
def train(self, callbacks=[]):
# Data setup.
data = h5py.File(self.cfg['hdf5_path_trn'])
tags = data.get('tags')[...]
idxs_trn, idxs_val = get_train_val_idxs(tags,
self.cfg['trn_prop_data'],
self.cfg['trn_prop_trn'])
steps_trn = math.ceil(
len(idxs_trn) / self.cfg['trn_batch_size'] /
self.cfg['trn_prop_data'])
steps_val = math.ceil(len(idxs_val) / self.cfg['trn_batch_size'])
gen_trn = self.batch_gen(
data,
idxs_trn,
steps_trn,
nb_augment_max=self.cfg['trn_augment_max_trn'])
gen_val = self.batch_gen(
data,
idxs_val,
steps_val,
nb_augment_max=self.cfg['trn_augment_max_val'])
opt = self.cfg['trn_optimizer'](**self.cfg['trn_optimizer_args'])
self.net.compile(optimizer=opt,
loss=self.cfg['net_loss_func'],
metrics=[F2, prec, reca])
# self.net.compile(optimizer=opt, loss=self.cfg['net_loss_func'], metrics=[F2, prec, reca] + tag_metrics())
def lrsched(epoch):
if epoch < 15:
return 1e-3
elif epoch < 23:
return 1e-4
else:
return 1e-5
cb = [
ValidationCB(self.cfg['cpdir'], gen_val,
self.cfg['trn_batch_size'], steps_val),
HistoryPlotCB('%s/history.png' % self.cpdir),
# EarlyStopping(monitor='val_loss', patience=3, verbose=1, mode='min'),
# EarlyStopping(monitor='F2', min_delta=0.01, patience=20, verbose=1, mode='max'),
CSVLogger('%s/history.csv' % self.cpdir),
ModelCheckpoint(
'%s/mvalf2_{epoch:02d}_{val_F2:.3f}.hdf5' % self.cpdir,
monitor='val_F2',
verbose=1,
save_best_only=True,
mode='max'),
LearningRateScheduler(lrsched)
# ReduceLROnPlateau(monitor='F2', factor=0.1, patience=5,
# min_lr=1e-4, epsilon=1e-2, verbose=1, mode='max')
] + callbacks
if self.cfg['trn_monitor_val']:
cb.append(
EarlyStopping(monitor='val_F2',
min_delta=0.01,
patience=20,
verbose=1,
mode='max'))
train = self.net.fit_generator(gen_trn,
steps_per_epoch=steps_trn,
epochs=self.cfg['trn_epochs'],
verbose=1,
callbacks=cb)
return train.history
def train(self, weights_path=None, callbacks=[]):
# Network setup.
net, inshp, ppimgs, pptags = self.cfg['net_builder_func'](self.cfg['net_out_func'], self.cfg['input_shape'])
self.cfg['input_shape'] = inshp
self.cfg['preprocess_imgs_func'] = ppimgs
self.cfg['preprocess_tags_func'] = pptags
json.dump(net.to_json(), open('%s/model.json' % self.cpdir, 'w'), indent=2)
json.dump(serialize_config(self.cfg), open('%s/config.json' % self.cpdir, 'w'))
# Data setup.
idxs_trn, idxs_val = get_train_val_idxs(self.cfg['hdf5_path_trn'], prop_data=self.cfg['trn_prop_data'],
prop_trn=self.cfg['trn_prop_trn'])
steps_trn = math.ceil(len(idxs_trn) / self.cfg['trn_batch_size'])
steps_val = math.ceil(len(idxs_val) / self.cfg['trn_batch_size'])
gen_trn = self.batch_gen(idxs_trn, steps_trn, nb_augment_max=self.cfg['trn_augment_max_trn'])
gen_val = self.batch_gen(idxs_val, steps_val, nb_augment_max=self.cfg['trn_augment_max_val'])
def prec(yt, yp):
yp = K.cast(yp > self.cfg['net_threshold'], 'float')
tp = K.sum(yt * yp)
fp = K.sum(K.clip(yp - yt, 0, 1))
return tp / (tp + fp + K.epsilon())
def reca(yt, yp):
yp = K.cast(yp > self.cfg['net_threshold'], 'float')
tp = K.sum(yt * yp)
fn = K.sum(K.clip(yt - yp, 0, 1))
return tp / (tp + fn + K.epsilon())
def F2(yt, yp):
p = prec(yt, yp)
r = reca(yt, yp)
b = 2.0
return (1 + b**2) * ((p * r) / (b**2 * p + r + K.epsilon()))
def acc(yt, yp):
fp = K.sum(K.clip(yp - yt, 0, 1))
fn = K.sum(K.clip(yt - yp, 0, 1))
sz = K.sum(K.ones_like(yt))
return (sz - (fp + fn)) / (sz + 1e-7)
net.compile(optimizer=Adam(**self.cfg['trn_adam_params']), loss=self.cfg['net_loss_func'],
metrics=[F2, prec, reca, acc])
net.summary()
if weights_path is not None:
net.load_weights(weights_path)
pprint(self.cfg)
cb = [
EarlyStopping(monitor='F2', min_delta=0.01, patience=20, verbose=1, mode='max'),
TensorBoardWrapper(gen_val, nb_steps=5, log_dir=self.cfg['cpdir'], histogram_freq=1,
batch_size=4, write_graph=False, write_grads=True),
CSVLogger('%s/history.csv' % self.cpdir),
ModelCheckpoint('%s/wvalf2.hdf5' % self.cpdir, monitor='val_F2', verbose=1,
save_best_only=True, mode='max'),
TerminateOnNaN()
] + callbacks
if self.cfg['trn_monitor_val']:
cb.append(ReduceLROnPlateau(monitor='val_F2', factor=0.1, patience=10,
min_lr=0.001 epsilon=1e-2, verbose=1, mode='max'))
cb.append(EarlyStopping(monitor='val_F2', min_delta=0.01, patience=20, verbose=1, mode='max'))