def train_classifier() -> tf.keras.Model:
model = get_classification_model()
model = compile_model(model)
train, val, test = create_datasets()
train = configure_dataset_for_performance(train, BATCH_SIZE)
val = configure_dataset_for_performance(val, BATCH_SIZE)
test = configure_dataset_for_performance(test, BATCH_SIZE)
log_dir = "logs/fit/" + datetime.now().strftime("%Y%m%d-%H%M%S")
callbacks = [
EarlyStopping(monitor="val_loss",
patience=5,
restore_best_weights=True),
TerminateOnNaN(),
TBCallback(log_dir=log_dir, histogram_freq=1),
PlotLossesKeras(),
]
fit_kwargs = dict(
epochs=20,
shuffle=True,
callbacks=callbacks,
)
history = model.fit(train, validation_data=val, **fit_kwargs)
plot_history(history.history)
scores = model.evaluate(test)
metric_dict = dict(zip(model.metrics_names, scores))
print("Test metrics:", metric_dict)
return model
def model_fit(self):
self.__model.fit(self.__x_train, self.__y_train, epochs=self.epochs,
batch_size= self.batch_size, validation_data=[self.__x_test, self.__y_test],verbose=0,
callbacks = [EarlyStopping(monitor='val_loss', patience=5, verbose=1),
ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=5, verbose=0),
TerminateOnNaN()])
def get_callbacks(model_name='tmp',
model_seed=666,
patience_LR=50,
patience_training=200):
nan_terminate = TerminateOnNaN()
ReduceLR = ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
patience=patience_LR,
verbose=1,
mode='auto',
min_delta=0.0001,
cooldown=0,
min_lr=0)
early_stop = EarlyStopping(monitor='val_loss',
min_delta=0,
patience=patience_training,
verbose=2,
mode='auto',
baseline=None,
restore_best_weights=True)
csv_logger = CSVLogger(f'models/{model_name}/log_{model_seed}.csv',
append=True)
checkpoint = ModelCheckpoint(
f'models/{model_name}/{model_name}_{model_seed}',
monitor='val_loss',
verbose=1,
save_best_only=True,
mode='min',
save_weights_only=True)
return [early_stop, checkpoint, csv_logger, nan_terminate, ReduceLR]
def get_callbacks(save_path: Union[os.PathLike, str],
model_name: str = "baseline"):
return [
EarlyStopping(
monitor=MONITOR_METRIC,
patience=10,
restore_best_weights=True,
),
CSVLogger(os.path.join(save_path, "training.log"),
separator=',',
append=False),
ReduceLROnPlateau(
monitor=MONITOR_METRIC,
factor=0.1,
patience=5,
verbose=1,
mode='min',
),
ModelCheckpoint(
filepath=os.path.join(save_path,
f"{model_name}" + ".{epoch:02d}.hdf5"),
save_best_only=True,
monitor=MONITOR_METRIC,
mode="min",
),
TensorBoard(log_dir=os.path.join(save_path, "logs")),
TerminateOnNaN()
]
def generate_clf(X_train,
y_train,
X_test,
y_test,
theme_base,
epochs,
batch,
seed,
dropout):
clf = pred_models.NewTownClassifier(theme_base=theme_base, seed=seed, dropout=dropout)
clf.compile(optimizer='adam',
loss=losses.BinaryCrossentropy(),
metrics=['binary_accuracy'])
# prepare path and check for previously trained model
dir_path = pathlib.Path(weights_path / f'predictive_weights/{clf.theme}')
if not dir_path.exists():
# prepare callbacks
callbacks = [
TensorBoard(
log_dir=str(
logs_path / f'{datetime.now().strftime("%Hh%Mm%Ss")}_{clf.theme}'),
histogram_freq=1,
write_graph=True,
write_images=True,
update_freq='epoch',
profile_batch=2,
embeddings_freq=0,
embeddings_metadata=None),
ReduceLROnPlateau(
monitor='val_loss',
factor=0.1,
patience=5,
verbose=1,
mode='auto',
min_delta=0.0001,
cooldown=0,
min_lr=0),
TerminateOnNaN(),
ModelCheckpoint(
str(dir_path / 'weights'),
monitor='val_loss',
verbose=1,
save_best_only=True,
save_weights_only=True,
mode='auto',
save_freq='epoch')
]
# train
clf.fit(x=X_train,
y=y_train,
batch_size=batch,
epochs=epochs,
verbose=1,
validation_data=(X_test, y_test),
shuffle=True,
callbacks=callbacks)
else:
clf.load_weights(str(dir_path / 'weights'))
#
return clf
def train(self):
LR = self.config['train']['learning_rate']
BATCH_SIZE = self.config['train']['batch_size']
NUM_EPOCHS = self.config['train']['num_epochs']
METRICS = self.config['train']['metrics']
_CALLBACKS = self.config['train']['callbacks']
self.model.compile(loss=self.jaccard_loss,
optimizer=optimizers.Adam(LR),
metrics=METRICS)
train_data, val_data = self._create_data()
CALLBACKS = [] if not _CALLBACKS \
else [EarlyStopping(patience=10),
CSVLogger('log.csv'),
TerminateOnNaN(),
ReduceLROnPlateau(),
ModelCheckpoint('chpts/w.{epoch:02d}.h5')]
self.model.fit(train_data,
validation_data=val_data,
epochs=NUM_EPOCHS,
steps_per_epoch=BATCH_SIZE,
validation_steps=4,
callbacks=CALLBACKS)
def fit(self,
x,
y,
ox=None,
oy=None,
batch_size=None,
n_shuffle=5,
epochs=None,
mi_eps=0.00001):
''' '''
n = x.shape[0]
batch_size = get_default_parameter(
'batch_size') if batch_size is None else batch_size
steps_per_epoch = n // batch_size
steps_per_epoch = min(max(steps_per_epoch, 100), 1000)
z_gen = PFSBatchGenerator(x, y, ox=ox, oy=oy, batch_size=batch_size, \
steps_per_epoch=steps_per_epoch, n_shuffle=n_shuffle)
epochs = get_default_parameter('epochs') if epochs is None else epochs
self.model.fit(z_gen, epochs=epochs, batch_size=batch_size, steps_per_epoch=steps_per_epoch, \
callbacks=[EarlyStopping(patience=3, monitor='loss'), TerminateOnNaN()])
self.mutual_information = -self.model.evaluate(z_gen)
w = self.model.w_layer.get_weights()[0]
if self.mutual_information < mi_eps:
# Retrain to avoid MI collapse to 0.
batch_size = 2 * batch_size
z_gen = PFSBatchGenerator(x, y, ox=ox, oy=oy, batch_size=batch_size, \
steps_per_epoch=steps_per_epoch, n_shuffle=n_shuffle)
self.model.fit(z_gen, epochs=epochs, batch_size=batch_size, steps_per_epoch=steps_per_epoch, \
callbacks=[EarlyStopping(patience=3, monitor='loss'), TerminateOnNaN()])
self.mutual_information = -self.model.evaluate(z_gen)
w = self.model.w_layer.get_weights()[0]
# The feature direction should be normal
w = w.flatten()
w = w / np.sqrt(np.dot(w, w))
# The principal feature should point in the same direction as the target (i.e. <y, w^Tx> = cov(y, w^Tx) > 0)
corr_sign = np.sign(np.corrcoef(y, np.dot(x, w))[0, 1])
w = corr_sign * w
self.feature_direction = w
self.fx = self.model.fx(tf.constant(z_gen.z)).numpy()
if self.expand_y:
self.gy = self.model.gy(tf.constant(z_gen.z)).numpy()
def fit(self, z, batch_size=10000, steps_per_epoch=1000, epochs=None):
''' '''
z_gen = CopulaBatchGenerator(z,
batch_size=batch_size,
steps_per_epoch=steps_per_epoch)
epochs = get_default_parameter('epochs') if epochs is None else epochs
self.model.fit(z_gen, epochs=epochs, batch_size=batch_size, steps_per_epoch=steps_per_epoch, \
callbacks=[EarlyStopping(patience=3, monitor='loss'), TerminateOnNaN()])
self.copula_entropy = self.model.evaluate(z_gen)
def objective(self, params):
"""
objective function to optimize
:param params: hyperparamters for optimizer
:return: maximum validation accuracy
:rtype: float
"""
# get instances
dataset = Datasets.get(self.dataset_name)
model = Models.get(self.model_name, dataset=dataset)
optimizer = Optimizers.get(self.optimizer_name, params=params)
# configure hyperdash experiment
hd_exp = HyperdashExperiment(
f'{self.dataset_name}',
api_key_getter=lambda: self.config['hyperdash']['api_key'])
hd_exp.param('dataset_name', self.dataset_name)
hd_exp.param('model_name', self.model_name)
hd_exp.param('optimizer_name', self.optimizer_name)
for k, v in params.items():
hd_exp.param(k, v)
# set callbacks
callbacks = [
Hyperdash(['accuracy', 'loss', 'val_accuracy', 'val_loss'],
hd_exp),
EarlyStopping('val_accuracy',
patience=10,
min_delta=0.01,
verbose=1),
TerminateOnNaN()
]
# get data
(x_train, y_train), *_ = dataset.get_batch()
# start learning
model.compile(loss=self.loss,
optimizer=optimizer,
metrics=['accuracy'])
history = model.fit(x_train,
y_train,
batch_size=self.batch_size,
epochs=self.epochs,
callbacks=callbacks,
validation_split=0.2,
verbose=2)
# stop hyperdash experiment
hd_exp.end()
# return maximum validation accuracy
val_accuracy = np.array(history.history['val_accuracy'])
return max(val_accuracy) * (-1)
def main(args):
w, h = args.model_input_size.split('x')
input_shape = np.asarray([h, w, 3], dtype=int)
checkpoint = ModelCheckpoint(str(WEIGHTS_OUTPUT_PATH),
monitor='val_loss',
verbose=1,
save_weights_only=True,
save_best_only=True)
logging = TensorBoard(log_dir=str(LOGS_DIR))
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
patience=25,
verbose=1,
cooldown=0,
min_lr=1e-7)
early_stopping = EarlyStopping(monitor='val_loss',
min_delta=0,
patience=300,
verbose=1)
terminate_on_nan = TerminateOnNaN()
callbacks = [
logging, checkpoint, reduce_lr, early_stopping, terminate_on_nan
]
optimizer = Adam(lr=args.learning_rate)
model = build_simple_conv_net(input_shape)
model.compile(loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
initial_epoch = args.init_epoch
epochs = args.total_epochs - initial_epoch
assert epochs >= 1
train_datagen = ImageDataGenerator(rescale=1 / 255)
test_datagen = ImageDataGenerator(rescale=1 / 255)
train_generator = train_datagen.flow_from_directory(
'./data/training/',
target_size=input_shape[:2],
batch_size=args.batch_size,
class_mode='binary')
validation_generator = test_datagen.flow_from_directory(
'./data/validation/',
target_size=input_shape[:2],
batch_size=args.batch_size,
class_mode='binary')
history = model.fit(train_generator,
epochs=epochs,
validation_data=validation_generator,
callbacks=callbacks)
model.save(MODEL_OUTPUT_PATH)
def callbacks(save_path: str, depth: int) -> List:
"""Keras callbacks which include ModelCheckpoint, CSVLogger, TensorBoard, LearningRateScheduler, TerminateOnNaN
Parameters
----------
save_path: str
local directory to save model weights
depth : int
Depth of ResNet model
Returns
-------
List
List all callbacks
"""
existsfolder(save_path)
model_checkpoint = ModelCheckpoint(
filepath=f"{save_path}/" + f"ResNet{depth}" +
"-epoch:{epoch:02d}-val_acc:{val_accuracy:.2f}.hdf5",
save_best_only=True,
save_weights_only=False,
verbose=1)
existsfolder('./assets/logs')
csv_logger = CSVLogger(filename=f"./assets/logs/logs-{now}.csv",
append=True)
def lr_schedule(epoch):
if epoch < 10:
return 0.003
elif epoch < 50:
return 0.0003
else:
return 0.00003
lr_reduce = ReduceLROnPlateau(monitor='val_loss',
factor=math.sqrt(0.1),
patience=5,
min_lr=3e-6,
verbose=1)
lr_scheduler = LearningRateScheduler(lr_schedule, verbose=1)
early = EarlyStopping(monitor='val_loss', min_delta=1e-4, patience=15)
terminate_on_nan = TerminateOnNaN()
callbacks_list = [
csv_logger, lr_scheduler, lr_reduce, early, model_checkpoint,
terminate_on_nan
]
return callbacks_list
def get_callbacks(output_folder, job_config, fold, val_loss, start_time,
fold_name, loss, metrics, optimizers):
log_folder = os.path.join(output_folder, "logs")
os.makedirs(log_folder, exist_ok=True)
lr_reducer = SavableReduceLROnPlateau(
os.path.join(output_folder, "lr_reducer.json"),
factor=job_config["LR_REDUCTION_FACTOR"],
cooldown=job_config["PATIENCE"],
patience=job_config["PATIENCE"],
min_lr=job_config["MIN_LR"],
monitor='val_loss',
mode='min',
min_delta=0,
verbose=2)
model_autosave = SubModelCheckpoint(filepath=os.path.join(
output_folder, "{epoch:04d}-{val_loss:.6f}.h5"),
submodel=fold_name,
save_best_only=False,
save_weights_only=True)
model_autosave.best = val_loss
early_stopping = SavableEarlyStopping(os.path.join(output_folder,
"early_stopping.json"),
patience=job_config["PATIENCE"] * 3,
verbose=2,
monitor='val_loss',
mode='min')
tensorboard = TensorBoard(log_dir=os.path.join(log_folder, "tenboard"),
profile_batch=0)
logger = CSVLogger(os.path.join(log_folder, 'train.csv'),
separator=',',
append=True)
time_limit = EarlyStoppingByTime(limit_seconds=int(
os.environ.get("LIMIT_SECONDS", -1)),
start_time=start_time,
verbose=0)
optimizer_saver = OptimizerSaver(os.path.join(output_folder,
"optimizer.pkl"),
loss=loss,
metrics=metrics)
return [
optimizer_saver, lr_reducer,
TerminateOnNaN(), early_stopping, logger, tensorboard, model_autosave,
time_limit
]
def get_callbacks():
callbacks = []
terminate_on_nan = TerminateOnNaN()
callbacks.append(terminate_on_nan)
early_stopping = EarlyStopping(monitor='loss',
patience=3,
verbose=1,
mode='auto')
callbacks.append(early_stopping)
return early_stopping
def __init__(self, output_dir, key):
# Variables to hold the description of the experiment
self.config_description = "This is the template config file."
# System dependent variable
self._workers = 1
self._multiprocessing = False
self._gpus = 1
self._displayer = MNISTDisplayer()
# Variables for comet.ml
self._project_name = "my_project"
self._workspace = "my_workspace"
self.output_dir = join(output_dir, "{}_{}_{}".format(self.workspace, self.project_name, key))
# Network variables
self.num_classes = 10
self.img_size = (28, 28)
self._weights = None
self._network = MNISTExample(self.num_classes)
# Training variables
self._epochs = 5
self._batch_size = 128
self._steps_per_epoch = 60000 // 128
self._optimizer = Adadelta()
self._loss = categorical_crossentropy
self._metrics = ['accuracy']
self._callbacks = []
self.early_stopping_params = {"monitor":'val_loss', "min_delta":0, "patience":7}
self.reduce_lr_on_plateau_params = {"monitor":'val_loss', "factor":0.1, "patience":5}
self.tensorboard = TensorBoard(join(self.output_dir, "checkpoints/logs"))
self.terminate_on_nan = TerminateOnNaN()
self.early_stopping = EarlyStopping(**self.early_stopping_params)
self.reduce_lr_on_plateau = ReduceLROnPlateau(**self.reduce_lr_on_plateau_params)
self.model_checkpoint = ModelCheckpoint(filepath=join(self.output_dir, "checkpoints", "cp-{epoch:04d}_loss-{loss:.4f}_val_loss-{val_loss:.4f}.ckpt"), verbose=1, save_best_only=True, save_weights_only=True)
self._callbacks = [self.tensorboard, self.terminate_on_nan, self.early_stopping, self.reduce_lr_on_plateau, self.model_checkpoint]
# Creating the training and validation generator (you may want to move these to the prepare functions)
train_data, validation_data = mnist.load_data()
self._train_generator = MNISTGenerator(train_data, self.batch_size)
self._validation_generator = MNISTGenerator(validation_data, self.batch_size)
# Dummy test for example
self._test_generator = MNISTGenerator(validation_data, self.batch_size)
self._evaluator = None
self._displayer = MNISTDisplayer()
def _fit_model(self, X, y, **kwargs):
self.model.fit(
x=X,
y=y,
epochs=self._epochs,
callbacks=[
EarlyStopping(patience=self._early_stop,
restore_best_weights=True),
TerminateOnNaN(),
],
validation_split=self._validation_split,
**kwargs,
)
def train_arcloss(self):
self.model.fit(self.dataset_train,
validation_data=self.dataset_eval,
epochs=cfg.train_epochs,
callbacks=[
self.tensorboard_callback, self.checkpoint_callback,
TerminateOnNaN(),
EarlyStopping(patience=5, restore_best_weights=True)
])
self.model.base_model.save_weights('./weights/{}_arcface'.format(
cfg.backbone))
self.model.base_model.save('./saved_model/{}_arcface'.format(
cfg.backbone))
def _fit_model(self, X, y, **kwargs):
self.model.fit(
self._generate_batches(X, y),
steps_per_epoch=len(X),
epochs=self._epochs,
callbacks=[
EarlyStopping(
patience=self._early_stop,
restore_best_weights=True,
monitor="accuracy",
),
TerminateOnNaN(),
],
**kwargs,
)
def model_fit(self):
self.__model.fit_generator(self.generator,
self.validation_data,
epochs=self.epochs,
verbose=0,
callbacks=[
EarlyStopping(monitor='val_loss',
patience=5,
verbose=1),
ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
patience=5,
verbose=0),
TerminateOnNaN()
])
def prepare_model(self, model, optimizer, loss):
"""
Called by `prepare_for_training` function.
Parameters
----------
model : Seg object.
optimizer : obj or None
Instance of a `Keras Optimizer <https://keras.io/optimizers/>`_ to be used for training.
If ``None`` (default), uses ``Adam`` with the learning rate specified in ``config``.
loss: `loss_seg`
computes Cross-Entropy between the class targets and predicted outputs
Returns
----------
`Keras Callbacks <https://keras.io/callbacks/>`_ to be used for training.
"""
from tensorflow.keras.optimizers import Optimizer
isinstance(optimizer, Optimizer) or _raise(ValueError())
if self.config.train_loss == 'seg':
loss_standard = eval('loss_seg(relative_weights=%s)' %
self.config.relative_weights)
_metrics = [loss_standard]
elif self.config.train_loss == 'denoiseg':
loss_standard = eval(
'loss_denoiseg(alpha={}, relative_weights={})'.format(
self.config.denoiseg_alpha, self.config.relative_weights))
seg_metric = eval(
'denoiseg_seg_loss(weight={}, relative_weights={})'.format(
1 - self.config.denoiseg_alpha,
self.config.relative_weights))
denoise_metric = eval('denoiseg_denoise_loss(weight={})'.format(
self.config.denoiseg_alpha))
_metrics = [loss_standard, seg_metric, denoise_metric]
else:
_raise('Unknown Loss!')
callbacks = [TerminateOnNaN()]
# compile model
model.compile(optimizer=optimizer,
loss=loss_standard,
metrics=_metrics)
return callbacks
def get_callbacks():
callbacks = []
callbacks.append(TerminateOnNaN())
callbacks.append(ResultKeeper("results.bin"))
callbacks.append(CustomModelCheckpoint('checkpoints'))
callbacks.append(CustomTensorBoard('tensorboard'))
callbacks.append(
CustomEarlyStopping(mini_targets={
5: 200,
10: 100
},
monitor="val_loss",
patience=3))
#lr_decay = lambda epoch, lr: lr / np.power(.1, epoch)
#callbacks.append(LearningRateScheduler(lr_decay, verbose= 1))
return callbacks
def setup_callbacks(cfg, callbacks, validation):
ckpt_cfg = cfg.ModelCheckpoint
es_cfg = cfg.EarlyStopping
tb_cfg = cfg.TensorBoard
if not validation:
if ckpt_cfg.enabled and ckpt_cfg.monitor.startswith("val_"):
ckpt_cfg.enabled = False
ckpt_cfg.monitor = ckpt_cfg.monitor[4:]
if es_cfg.enabled and es_cfg.monitor.startswith("val_"):
es_cfg.enabled = False
es_cfg.monitor = es_cfg.monitor[4:]
if es_cfg.enabled:
es_callback = EarlyStopping(
monitor=es_cfg.monitor,
patience=es_cfg.patience,
mode=es_cfg.mode,
verbose=es_cfg.verbose,
baseline=es_cfg.baseline,
restore_best_weights=es_cfg.restore_best_weights)
callbacks.append(es_callback)
if ckpt_cfg.enabled:
if not ckpt_cfg.path.endswith(gg.file_ext()):
ckpt_cfg.path += gg.file_ext()
makedirs_from_filepath(ckpt_cfg.path)
mc_callback = ModelCheckpoint(
ckpt_cfg.path,
monitor=ckpt_cfg.monitor,
save_best_only=ckpt_cfg.save_best_only,
save_weights_only=ckpt_cfg.save_weights_only,
verbose=ckpt_cfg.vervose)
callbacks.append(mc_callback)
if cfg.TerminateOnNaN.enabled:
callbacks.append(TerminateOnNaN())
if tb_cfg.enabled:
callbacks.append(
tf.keras.callbacks.TensorBoard(
tb_cfg.log_dir,
write_graph=tb_cfg.write_graph,
update_freq=tb_cfg.update_freq,
histogram_freq=tb_cfg.histogram_freq,
write_images=tb_cfg.write_images))
return cfg, callbacks
def prepare_model(self, model, optimizer, loss, metrics=('mse', 'mae')):
""" TODO """
from tensorflow.keras.optimizers import Optimizer
isinstance(optimizer, Optimizer) or _raise(ValueError())
if loss == 'mse':
loss_standard = eval('loss_mse()')
elif loss == 'mae':
loss_standard = eval('loss_mae()')
_metrics = [eval('loss_%s()' % m) for m in metrics]
callbacks = [TerminateOnNaN()]
# compile model
model.compile(optimizer=optimizer, loss=loss_standard, metrics=_metrics)
return callbacks
def train_model(self, model_type, load=True, epochs=0):
data_handler = self.harmonic_data_set.set_type(model_type)
if model_type == 0:
m_params = args.h_parser.parse_args()
model_loc = self.model_path + '/harmonic_model.h5'
else:
m_params = args.a_parser.parse_args()
model_loc = self.model_path + '/aperiodic_model.h5'
if epochs == 0:
epochs = m_params.epochs
if load and not os.path.isfile(model_loc):
print('Cannot find model :' + model_loc +
"\n Creating new model...")
load = False
if load:
model = load_model(model_loc)
print('Successfully loaded :' + model_loc +
"\n Continuing training...")
else:
model = self.build_model(data_handler, m_params)
adam_optimizer = Adam(learning_rate=m_params.learn_rate)
model.compile(optimizer=adam_optimizer, loss=network_loss)
if not os.path.isdir(self.model_path):
os.mkdir(self.model_path)
lr_schedule = LearningRateScheduler(self.lr_scheduler)
nan_terminator = TerminateOnNaN()
checkpoint = ModelCheckpoint(model_loc,
monitor='loss',
verbose=1,
save_best_only=True,
mode='min')
model.fit(data_handler,
epochs=epochs,
callbacks=[nan_terminator, lr_schedule, checkpoint])
def train(varBatchSize, varEpochs, model, PSIData, DSIData, PQIData, DQIData):
#Switch to correct directory
checkpoint = ModelCheckpoint(filepath='' + 'WEIGHTS-{epoch:02d}.hdf5', monitor='loss', verbose=0, save_best_only=True, save_weights_only=False, mode='auto', period=1)
#Switch to correct directory
saver = ModelCheckpoint(filepath='' + 'Latest.hdf5', monitor='val_loss', verbose=0, save_best_only=False, save_weights_only=False, mode='auto', period=1)
#Switch to correct directory
csvlogger = CSVLogger("")
terminator = TerminateOnNaN()
reduce_lr = ReduceLROnPlateau(monitor='loss', factor=0.1, patience=1, min_lr=0.0001, verbose=1)
model.fit(x={'Pickup_Spatial_Input': PSIData[0], 'Dropoff_Spatial_Input': DSIData[0], 'Pickup_Quantity_Input' : PQIData[0], 'Dropoff_Quantity_Input' : DQIData[0]},
y = {'Pickup_Spatial_Output': PSIData[1], 'Dropoff_Spatial_Output': DSIData[1], 'Pickup_Quantity_Output' : PQIData[1], 'Dropoff_Quantity_Output' : DQIData[1]},
batch_size = varBatchSize,
epochs=varEpochs,
verbose=1,
validation_data=({'Pickup_Spatial_Input': PSIData[2], 'Dropoff_Spatial_Input': DSIData[2], 'Pickup_Quantity_Input' : PQIData[2], 'Dropoff_Quantity_Input' : DQIData[2]},
{'Pickup_Spatial_Output': PSIData[3], 'Dropoff_Spatial_Output': DSIData[3], 'Pickup_Quantity_Output' : PQIData[3], 'Dropoff_Quantity_Output' : DQIData[3]}),
callbacks=[reduce_lr, terminator, csvlogger, checkpoint, saver], shuffle=False)
def fitModel(schema, n_cls, dgen_opt, datagen, shot, X_train, X_valid, y_train,
y_valid, aug_flag):
callbacks = [EarlyStopping(patience=PATIENCE), TerminateOnNaN()]
# TensorBoard(log_dir='./logs/tboard', histogram_freq=1, write_grads=True)]
if 'Original' == datagen:
if aug_flag:
datagen = ImageDataGenerator(**dgen_opt)
datagen.fit(X_train)
traingen = datagen.flow(X_train,
to_categorical(y_train, n_cls),
batch_size=BATCHSIZE)
validgen = (X_valid, to_categorical(y_valid, n_cls))
else:
history = schema.model.fit(
X_train,
to_categorical(y_train, n_cls),
**FITOPTS,
callbacks=callbacks,
validation_data=(X_valid, to_categorical(y_valid, n_cls)))
history.history.update({'params': history.params})
history.history.update({'epoch': history.epoch})
return history.history
else:
if aug_flag:
traingen = datagen[0](X_train, y_train, n_cls, dgen_opt, BATCHSIZE)
validgen = datagen[1](X_valid, y_valid, n_cls, BATCHSIZE)
else:
traingen = datagen(X_train, y_train, n_cls, BATCHSIZE)
validgen = datagen(X_valid, y_valid, n_cls, BATCHSIZE)
_STEPS_PER = dict(STEPS_PER)
if shot == None:
_STEPS_PER['steps_per_epoch'] = len(y_train) // BATCHSIZE
_STEPS_PER['validation_steps'] = len(y_valid) // BATCHSIZE
history = schema.model.fit_generator(traingen,
validation_data=validgen,
callbacks=callbacks,
**FITGENOPTS,
**_STEPS_PER)
history.history.update({'params': history.params})
history.history.update({'epoch': history.epoch})
return history.history
def _fit_model(self, X, y, **kwargs):
self.preprocessor.fit(X)
batch_size = 64
validation_size = min(int(0.25 * len(X)), 10 * batch_size)
Xtrain, Xvalid = X[:-validation_size], X[-validation_size:]
ytrain, yvalid = y[:-validation_size], y[-validation_size:]
self.model.fit(
self.preprocessor.flow(Xtrain, ytrain, batch_size=batch_size),
steps_per_epoch=len(Xtrain) // batch_size,
epochs=self._epochs,
callbacks=[
EarlyStopping(patience=self._early_stop,
restore_best_weights=True),
TerminateOnNaN(),
],
validation_data=(Xvalid, yvalid),
**kwargs,
)
def train(self,
epochs: int,
monitor_metric='val_acc',
patience: int = 5,
steps_per_epoch: Union[int, str] = 'auto',
validation_steps: Union[int, str] = 'auto',
log_dir: str = 'logs',
use_multiprocessing: bool = False):
pprint(locals())
log_dir = Path(log_dir).joinpath(
datetime.now().replace(microsecond=0).isoformat())
model_path = Path(log_dir).joinpath('checkpoints').joinpath(
'best-model.h5py')
model_path = str(model_path)
if steps_per_epoch == 'auto':
steps_per_epoch = self.nb_train_samples // self.batch_size
if validation_steps == 'auto':
validation_steps = self.nb_valid_samples // self.batch_size
self.model.compile(optimizer='adam',
loss='sparse_categorical_crossentropy',
metrics=['acc'])
history = self.model.fit_generator(
self.train_dataset.as_numpy_iterator(),
steps_per_epoch=steps_per_epoch,
validation_data=self.valid_dataset.as_numpy_iterator(),
validation_steps=validation_steps,
epochs=epochs,
use_multiprocessing=use_multiprocessing,
workers=os.cpu_count() - 1,
callbacks=[
TerminateOnNaN(),
TensorBoard(log_dir=log_dir),
ModelCheckpoint(model_path,
monitor=monitor_metric,
verbose=1,
save_best_only=True),
EarlyStopping(monitor=monitor_metric, patience=patience),
])
return history.history
def prepare_model(model, optimizer, loss, metrics=('mse','mae'),
loss_bg_thresh=0, loss_bg_decay=0.06, Y=None):
""" TODO """
from keras.optimizers import Optimizer
isinstance(optimizer,Optimizer) or _raise(ValueError())
loss_standard = eval('loss_%s()'%loss)
_metrics = [eval('loss_%s()'%m) for m in metrics]
callbacks = [TerminateOnNaN()]
# checks
assert 0 <= loss_bg_thresh <= 1
assert loss_bg_thresh == 0 or Y is not None
if loss == 'laplace':
assert K.image_data_format() == "channels_last", "TODO"
assert model.output.shape.as_list()[-1] >= 2 and model.output.shape.as_list()[-1] % 2 == 0
# loss
if loss_bg_thresh == 0:
_loss = loss_standard
else:
freq = np.mean(Y > loss_bg_thresh)
# print("class frequency:", freq)
alpha = K.variable(1.0)
loss_per_pixel = eval('loss_{loss}(mean=False)'.format(loss=loss))
_loss = loss_thresh_weighted_decay(loss_per_pixel, loss_bg_thresh,
0.5 / (0.1 + (1 - freq)),
0.5 / (0.1 + freq),
alpha)
callbacks.append(ParameterDecayCallback(alpha, loss_bg_decay, name='alpha'))
if not loss in metrics:
_metrics.append(loss_standard)
# compile model
model.compile(optimizer=optimizer, loss=_loss, metrics=_metrics)
return callbacks
def _setup_callbacks(
message: Text,
t0: int,
enable_tensorboard: bool = True,
tensorboard_logdir: Optional[Text] = None
) -> Tuple[List[Callback], Text]:
"""Setups the callbacks to monitor model estimation.
Args:
t0: specifies the number of days between the occurrence of the first
infected case (patient zero) and the first observed case.
message: optionally pass a prefix string in the filenames of training
weights (in the format of hdf5 file). We will generate a lot of such
files in the training process.
enable_tensorboard: whether or not use tensorboard to monitor training.
tensorboard_logdir: xxx.
Returns:
callbacks : TYPE DESCRIPTION.
min_loss_filepath: TYPE DESCRIPTION.
"""
min_loss_filepath = os.path.join(tempfile.tempdir,
message + str(t0) + "min_loss.hdf5")
callbacks = []
callbacks.append(
ModelCheckpoint(min_loss_filepath,
monitor="loss",
save_best_only=True,
save_weights_only=False))
callbacks.append(TerminateOnNaN())
if enable_tensorboard:
if not os.path.isdir(tensorboard_logdir):
os.makedirs(tensorboard_logdir)
logdir = os.path.join(
tensorboard_logdir,
f"job{t0}_" + datetime.now().strftime("%Y%m%d-%H%M%S"))
callbacks.append(TensorBoard(log_dir=logdir))
return callbacks, min_loss_filepath
def train_model():
optimizer = Adam(lr=LEARNING_RATE)
if OPTIMIZER == 'sgd':
optimizer = SGD(lr=LEARNING_RATE, momentum=MOMENTUM)
if LOSS_FUNC == 'triplet':
train_gen, val_gen = prepare_datagen_triplet()
model = triplet_model(IMAGE_SHAPE, NUM_FEATURES)
model.compile(optimizer=optimizer, loss=identity_loss)
else:
train_gen, val_gen = prepare_datagen_custom_loss()
model = construct_model(train_gen.num_classes, NUM_FEATURES)
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
if LOAD_MODEL:
load_pretrained(model)
model.summary()
callbacks = [
ModelCheckpoint(CHECKPOINT_FOLDER + 'model.hdf5',
verbose=1,
save_best_only=True),
CSVLogger(CHECKPOINT_FOLDER + 'log.csv'),
TerminateOnNaN()
]
if SCHEDULER == 'CosineAnnealing':
callbacks.append(
CosineAnnealingScheduler(T_max=EPOCHS,
eta_max=LEARNING_RATE,
eta_min=MIN_LEARNING_RATE,
verbose=1))
model.fit(train_gen,
validation_data=val_gen,
epochs=EPOCHS,
callbacks=callbacks,
verbose=1)
