def get_callbacks(self):
schedule_params = self.config.get('schedule', None)
cb_modules_names = ['tensorflow.keras.callbacks', 'dienen.callbacks']
cb_modules_names = cb_modules_names + self.modules
cb_modules = get_modules(cb_modules_names)
#Agregar para levantar modulos externos
available_cb = {}
for cbmod in cb_modules:
available_cb.update(
get_members_from_module(cbmod, filters=[inspect.isclass]))
#cb_list = [available_cb[k](**v) for k,v in schedule_params.items()]
#cb_list = [cbk[list(cbk.keys())[0]] for cbk in schedule_params]
#cb_list = [UseOptimizerIterationAsTrainStep()]
cb_list = []
if isinstance(schedule_params, list):
cb_keys = [list(cbk.keys())[0] for cbk in schedule_params]
cb_list += [
available_cb[k](**v[k])
for k, v in zip(cb_keys, schedule_params)
]
elif isinstance(schedule_params, dict):
cb_list += [
available_cb[k](**v) for k, v in schedule_params.items()
]
elif schedule_params is None:
cb_list += []
for cb in cb_list:
cb.model_path = self.model_path
cb.extra_data = self.extra_data
cb.cache = self.cache
return cb_list
def get_optimizer(self):
optimizer_params = self.config.get('optimizer', None)
optimizer_modules_names = [
'tensorflow.keras.optimizers',
'tensorflow.keras.optimizers.schedules'
]
optimizer_modules_names = optimizer_modules_names + self.modules
optimizer_modules = get_modules(optimizer_modules_names)
available_optimizers = {}
for optmod in optimizer_modules:
available_optimizers.update(
get_members_from_module(optmod, filters=[inspect.isclass]))
if isinstance(optimizer_params, str):
optimizer = available_optimizers.get(optimizer_params)
elif isinstance(optimizer_params, dict):
optimizer = available_optimizers[optimizer_params['type']]
optimizer_params.pop('type')
if 'dynamic_loss_scaling' in optimizer_params:
dynamic_loss_scaling = optimizer_params.pop(
'dynamic_loss_scaling')
else:
dynamic_loss_scaling = True
#Check if lr is a schedule:
if optimizer_params.get('learning_rate',
None) in available_optimizers.keys():
lr_schedule = available_optimizers[
optimizer_params['learning_rate']]
if optimizer_params.get('learning_rate_args', None):
optimizer_params['learning_rate'] = lr_schedule(
**optimizer_params['learning_rate_args'])
optimizer_params.pop('learning_rate_args')
else:
optimizer_params['learning_rate'] = lr_schedule()
optimizer = optimizer(**optimizer_params)
#if dynamic_loss_scaling: #No anda con tf keras 2.4
#self.logger.info('Dynamic loss scaling enabled')
# optimizer = tf.keras.mixed_precision.LossScaleOptimizer(optimizer)
return optimizer
def get_metrics(self):
metric_params = self.config.get('metrics', None)
metric_module_names = ['tensorflow.keras.metrics']
metric_module_names = metric_module_names + self.modules
metric_modules = get_modules(metric_module_names)
available_metrics = {}
for metricmod in metric_modules:
available_metrics.update(
dict(
get_members_from_module(
metricmod,
filters=[inspect.isclass, inspect.isfunction])))
if metric_params:
if not isinstance(metric_params, list):
metric_params = [metric_params]
metric_objs = []
for m in metric_params:
if isinstance(m, str):
if m in available_metrics:
metric_objs.append(available_metrics.get(m))
else:
raise Exception(
'{} is not an available metric. Available metrics are {}'
.format(m, list(available_metrics.keys())))
elif isinstance(m, dict):
if m['type'] in available_metrics:
metric_cls = available_metrics[m['type']]
m.pop('type')
metric_objs.append(metric_cls(**m))
else:
raise Exception(
'{} is not an available metric. Available metrics are {}'
.format(m, list(available_metrics.keys())))
return metric_objs
else:
return None
def get_loss_fn(self):
loss_params = self.config.get('loss', None)
loss_module_names = ['tensorflow.keras.losses']
#Agregar para levantar modulos externos
loss_module_names = loss_module_names + self.modules
loss_modules = get_modules(loss_module_names)
available_losses = {}
for lossmod in loss_modules:
available_losses.update(
dict(
get_members_from_module(
lossmod, filters=[inspect.isclass,
inspect.isfunction])))
if isinstance(loss_params, str):
loss_fn = available_losses.get(loss_params)
elif isinstance(loss_params, dict):
loss_fn = available_losses[loss_params['type']]
loss_params.pop('type')
loss_fn = loss_fn(**loss_params)
return loss_fn
def log_val_metrics(self, params, logs):
if 'custom_metrics' in params:
from dienen.utils import get_modules, get_members_from_module
import inspect
metrics_module = params.get('metrics_module')
if not isinstance(metrics_module, list):
metrics_module = [metrics_module]
metrics_module = get_modules(metrics_module)
available_metrics = {}
for metricmod in metrics_module:
available_metrics.update(
dict(
get_members_from_module(
metricmod,
filters=[inspect.isclass, inspect.isfunction])))
params_metrics = copy.deepcopy(params['custom_metrics'])
codebook_layers = list(
set([
m['codebook_layer'] for m in params_metrics
if 'Codebook' in m['type']
]))
codebook_layers = {
c: self.model.get_layer(c)
for c in codebook_layers
}
codebook_layers_weights = {
cl_name:
{v.name: w
for v, w in zip(cl.variables, cl.get_weights())}
for cl_name, cl in codebook_layers.items()
}
model_outs = [[self.model.predict(x), y]
for x, y in params['validation_data']]
y_pred = np.array([x[0] for x in model_outs])
y_true = np.array([x[1] for x in model_outs])
if y_pred.ndim > 1:
y_pred = np.concatenate(y_pred)
if y_true.ndim > 1:
y_true = np.concatenate(y_true)
y_pred = y_pred[:len(params['validation_data']._index)]
y_true = y_true[:len(params['validation_data']._index)]
corrected_logit = None
if 'train_priors' in params:
if isinstance(params['train_priors'], str):
train_priors = joblib.load(
Path(params['train_priors']).expanduser())
elif isinstance(params['train_priors'], float):
train_priors = [params['train_priors']] * y_pred.shape[-1]
corrected_logit = logit(y_pred) - logit(
np.array(train_priors))[np.newaxis, :]
if 'validation_priors' in params:
if isinstance(params['validation_priors'], str):
validation_priors = joblib.load(
Path(params['validation_priors']).expanduser())
elif isinstance(params['validation_priors'], float):
validation_priors = [params['validation_priors']
] * y_pred.shape[-1]
corrected_logit = corrected_logit + logit(
np.array(validation_priors))[np.newaxis, :]
if corrected_logit is not None:
y_pred = expit(corrected_logit)
metric_results = {}
for metric in params_metrics:
metric_type = metric.pop('type')
if inspect.isclass(available_metrics[metric_type]):
metric_cls = available_metrics[metric_type](**metric)
if 'Codebook' in metric_type:
metric_cls.codebook_layers = codebook_layers
metric_cls.validation_data = params['validation_data']
metric_cls.model = self.model
mres = metric_cls(y_true, y_pred)
elif inspect.isfunction(available_metrics[metric_type]):
mres = available_metrics[metric_type](y_true, y_pred)
else:
raise Exception('Unrecognized metric')
for k, v in mres.items():
if v.ndim == 0:
metric_results['val_{}'.format(k)] = v
elif v.ndim == 1:
labels = None
if 'labels' in params:
if isinstance(params['labels'], str):
labels = joblib.load(
Path(params['labels']).expanduser())
else:
labels = params['labels']
if len(v) > 100:
for i in range(len(v) // 100):
plt.figure(figsize=(20, 10))
sns.barplot(x=labels[i * 100:(i + 1) * 100],
y=v[i * 100:(i + 1) * 100])
plt.xticks(rotation=90)
wandb.log(
{
'val_{}_{}'.format(k, i):
wandb.Image(plt)
},
step=self.step)
if len(v) % 100 > 0:
plt.figure(figsize=(20, 10))
sns.barplot(x=labels[100 * (len(v) // 100):],
y=v[100 * (len(v) // 100):])
plt.xticks(rotation=90)
wandb.log(
{
'val_{}_{}'.format(k,
len(v) // 100):
wandb.Image(plt)
},
step=self.step)
else:
plt.figure(figsize=(20, 10))
if len(labels) == len(v):
sns.barplot(x=labels, y=v)
else:
sns.barplot(x=np.arange(len(v)), y=v)
plt.xticks(rotation=90)
wandb.log({'val_{}'.format(k): wandb.Image(plt)},
step=self.step)
elif v.ndim == 2:
from IPython import embed
embed()
elif v.ndim == 3:
for i, v_i in enumerate(v):
plt.figure(figsize=(10, 10))
plt.imshow(v_i, aspect='auto', origin='lower')
wandb.log(
{'val_{}_{}'.format(k, i): wandb.Image(plt)},
step=self.step)
wandb.log(metric_results, step=self.step)
else:
metric_results = self.model.evaluate(params['validation_data'],
return_dict=True)
metric_results = {
'val_{}'.format(k): v
for k, v in metric_results.items()
}
wandb.log(metric_results, step=self.step)
return metric_results
def fit(self,
keras_model,
data,
output_path,
from_epoch=0,
validation_data=None,
cache=True,
training_strategy=None,
from_step=0,
class_weights=None):
self.model_path = output_path
self.cache = cache
n_epochs = self.config.get('n_epochs', 10)
calculate_initial_loss = self.config.get('calculate_initial_loss',
False)
loss_fn = self.get_loss_fn()
optimizer = self.get_optimizer()
cb_list = self.get_callbacks()
for cb in cb_list:
cb.step = from_step
metrics = self.get_metrics()
for cb in cb_list:
cb.epoch = from_epoch
cb.data = data
with training_strategy.scope():
keras_model.compile(optimizer=optimizer,
loss=loss_fn,
metrics=metrics)
if self.weights:
load_weights(self.weights, keras_model)
if self.optimizer_weights:
keras_model.optimizer._create_all_weights(
keras_model.trainable_variables)
keras_model.optimizer.set_weights(self.optimizer_weights)
keras_model.initial_step = from_step
if calculate_initial_loss and validation_data:
if self.logger:
self.logger.info('Initial Loss: {}'.format(
keras_model.evaluate(validation_data)))
n_workers = self.config.get('workers', 1)
if n_workers > 1:
use_multiprocessing = True
else:
use_multiprocessing = False
custom_fit = self.config.get('custom_fit', None)
if hasattr(data, 'epoch'):
data.epoch = from_epoch
if hasattr(data, 'on_train_begin'):
data.on_train_begin()
if custom_fit:
modules = get_modules(self.modules)
custom_fit_fn = None
for module in modules:
available_fns = get_members_from_module(
module, filters=[inspect.isfunction])
if custom_fit in available_fns:
custom_fit_fn = available_fns[custom_fit]
break
if custom_fit_fn:
custom_fit_fn(data,
keras_model,
callbacks=cb_list,
initial_epoch=from_epoch,
epochs=n_epochs)
else:
raise Exception('{} not found'.format(custom_fit))
else:
if isinstance(data, list) or isinstance(data, tuple):
if validation_data is not None:
validation_data = tuple(validation_data)
keras_model.fit(x=data[0],
y=data[1],
initial_epoch=from_epoch,
epochs=n_epochs,
callbacks=cb_list,
validation_data=validation_data,
use_multiprocessing=use_multiprocessing,
workers=n_workers,
shuffle=True,
batch_size=self.config.get('batch_size', None),
steps_per_epoch=self.config.get(
'steps_per_epoch', None),
class_weight=class_weights)
else:
keras_model.fit(data,
initial_epoch=from_epoch,
epochs=n_epochs,
callbacks=cb_list,
validation_data=validation_data,
use_multiprocessing=use_multiprocessing,
workers=n_workers,
shuffle=False,
batch_size=self.config.get('batch_size', None),
steps_per_epoch=self.config.get(
'steps_per_epoch', None),
class_weight=class_weights)
def make_network(self, training_strategy):
dependency_order = self.find_dependency_order(self.processed_config,
self.inputs,
self.outputs)
self.tensors = {}
shared_layers_counts = {}
for layer in dependency_order:
if layer not in self.inputs:
layer_inputs = self.processed_config[layer]['input']
layer_inputs = layer_inputs if isinstance(
layer_inputs, list) else [layer_inputs]
layer_inputs = [self.tensors[inp] for inp in layer_inputs]
layer_mask = self.processed_config[layer].get('mask', None)
layer_training = self.processed_config[layer].get(
'training', None)
if len(layer_inputs) == 1:
layer_inputs = layer_inputs[0]
if layer in self.shared_layers:
target_layer = self.shared_layers[layer]
if target_layer in shared_layers_counts:
shared_layers_counts[target_layer] += 1
else:
shared_layers_counts[target_layer] = 0
if layer_mask is not None:
layer_mask = self.tensors[layer_mask]
self.tensors[layer] = self.layers[layer](
layer_inputs, mask=layer_mask)
else:
self.tensors[layer] = self.layers[layer](layer_inputs)
self.tensors[layer] = self.layers[layer].get_output_at(
shared_layers_counts[target_layer])
else:
try:
if layer_mask is not None:
layer_mask = self.tensors[layer_mask]
layer_mask = tf.cast(layer_mask, tf.bool)
self.tensors[layer] = self.layers[layer](
layer_inputs,
mask=layer_mask,
training=layer_training)
else:
self.tensors[layer] = self.layers[layer](
layer_inputs, training=layer_training)
except Exception as e:
raise Exception(
'Could not connect layer {} with its inputs {}. {}'
.format(layer,
self.processed_config[layer]['input'], e))
else:
self.tensors[layer] = self.layers[layer]
input_tensors = [self.tensors[in_tensor] for in_tensor in self.inputs]
output_tensors = [
self.tensors[out_tensor] for out_tensor in self.outputs
]
if self.custom_model is None:
model = tf.keras.Model(inputs=input_tensors,
outputs=output_tensors)
else:
available_modules = get_modules(self.modules)
available_cls = {}
for module in available_modules:
available_cls.update(
get_members_from_module(module, filters=[inspect.isclass]))
model_cls = self.custom_model.pop('type')
model = available_cls[model_cls](inputs=input_tensors,
outputs=output_tensors,
**self.custom_model)
model.distributed_strategy = training_strategy
for layer_name, weights in self.weights_to_assign.items():
model.get_layer(layer_name).set_weights(weights)
return model