def get_callbacks(self): schedule_params = self.config.get('schedule', None) cb_modules_names = ['tensorflow.keras.callbacks', 'dienen.callbacks'] cb_modules_names = cb_modules_names + self.modules cb_modules = get_modules(cb_modules_names) #Agregar para levantar modulos externos available_cb = {} for cbmod in cb_modules: available_cb.update( get_members_from_module(cbmod, filters=[inspect.isclass])) #cb_list = [available_cb[k](**v) for k,v in schedule_params.items()] #cb_list = [cbk[list(cbk.keys())[0]] for cbk in schedule_params] #cb_list = [UseOptimizerIterationAsTrainStep()] cb_list = [] if isinstance(schedule_params, list): cb_keys = [list(cbk.keys())[0] for cbk in schedule_params] cb_list += [ available_cb[k](**v[k]) for k, v in zip(cb_keys, schedule_params) ] elif isinstance(schedule_params, dict): cb_list += [ available_cb[k](**v) for k, v in schedule_params.items() ] elif schedule_params is None: cb_list += [] for cb in cb_list: cb.model_path = self.model_path cb.extra_data = self.extra_data cb.cache = self.cache return cb_list
def get_optimizer(self): optimizer_params = self.config.get('optimizer', None) optimizer_modules_names = [ 'tensorflow.keras.optimizers', 'tensorflow.keras.optimizers.schedules' ] optimizer_modules_names = optimizer_modules_names + self.modules optimizer_modules = get_modules(optimizer_modules_names) available_optimizers = {} for optmod in optimizer_modules: available_optimizers.update( get_members_from_module(optmod, filters=[inspect.isclass])) if isinstance(optimizer_params, str): optimizer = available_optimizers.get(optimizer_params) elif isinstance(optimizer_params, dict): optimizer = available_optimizers[optimizer_params['type']] optimizer_params.pop('type') if 'dynamic_loss_scaling' in optimizer_params: dynamic_loss_scaling = optimizer_params.pop( 'dynamic_loss_scaling') else: dynamic_loss_scaling = True #Check if lr is a schedule: if optimizer_params.get('learning_rate', None) in available_optimizers.keys(): lr_schedule = available_optimizers[ optimizer_params['learning_rate']] if optimizer_params.get('learning_rate_args', None): optimizer_params['learning_rate'] = lr_schedule( **optimizer_params['learning_rate_args']) optimizer_params.pop('learning_rate_args') else: optimizer_params['learning_rate'] = lr_schedule() optimizer = optimizer(**optimizer_params) #if dynamic_loss_scaling: #No anda con tf keras 2.4 #self.logger.info('Dynamic loss scaling enabled') # optimizer = tf.keras.mixed_precision.LossScaleOptimizer(optimizer) return optimizer
def get_metrics(self): metric_params = self.config.get('metrics', None) metric_module_names = ['tensorflow.keras.metrics'] metric_module_names = metric_module_names + self.modules metric_modules = get_modules(metric_module_names) available_metrics = {} for metricmod in metric_modules: available_metrics.update( dict( get_members_from_module( metricmod, filters=[inspect.isclass, inspect.isfunction]))) if metric_params: if not isinstance(metric_params, list): metric_params = [metric_params] metric_objs = [] for m in metric_params: if isinstance(m, str): if m in available_metrics: metric_objs.append(available_metrics.get(m)) else: raise Exception( '{} is not an available metric. Available metrics are {}' .format(m, list(available_metrics.keys()))) elif isinstance(m, dict): if m['type'] in available_metrics: metric_cls = available_metrics[m['type']] m.pop('type') metric_objs.append(metric_cls(**m)) else: raise Exception( '{} is not an available metric. Available metrics are {}' .format(m, list(available_metrics.keys()))) return metric_objs else: return None
def get_loss_fn(self): loss_params = self.config.get('loss', None) loss_module_names = ['tensorflow.keras.losses'] #Agregar para levantar modulos externos loss_module_names = loss_module_names + self.modules loss_modules = get_modules(loss_module_names) available_losses = {} for lossmod in loss_modules: available_losses.update( dict( get_members_from_module( lossmod, filters=[inspect.isclass, inspect.isfunction]))) if isinstance(loss_params, str): loss_fn = available_losses.get(loss_params) elif isinstance(loss_params, dict): loss_fn = available_losses[loss_params['type']] loss_params.pop('type') loss_fn = loss_fn(**loss_params) return loss_fn
def log_val_metrics(self, params, logs): if 'custom_metrics' in params: from dienen.utils import get_modules, get_members_from_module import inspect metrics_module = params.get('metrics_module') if not isinstance(metrics_module, list): metrics_module = [metrics_module] metrics_module = get_modules(metrics_module) available_metrics = {} for metricmod in metrics_module: available_metrics.update( dict( get_members_from_module( metricmod, filters=[inspect.isclass, inspect.isfunction]))) params_metrics = copy.deepcopy(params['custom_metrics']) codebook_layers = list( set([ m['codebook_layer'] for m in params_metrics if 'Codebook' in m['type'] ])) codebook_layers = { c: self.model.get_layer(c) for c in codebook_layers } codebook_layers_weights = { cl_name: {v.name: w for v, w in zip(cl.variables, cl.get_weights())} for cl_name, cl in codebook_layers.items() } model_outs = [[self.model.predict(x), y] for x, y in params['validation_data']] y_pred = np.array([x[0] for x in model_outs]) y_true = np.array([x[1] for x in model_outs]) if y_pred.ndim > 1: y_pred = np.concatenate(y_pred) if y_true.ndim > 1: y_true = np.concatenate(y_true) y_pred = y_pred[:len(params['validation_data']._index)] y_true = y_true[:len(params['validation_data']._index)] corrected_logit = None if 'train_priors' in params: if isinstance(params['train_priors'], str): train_priors = joblib.load( Path(params['train_priors']).expanduser()) elif isinstance(params['train_priors'], float): train_priors = [params['train_priors']] * y_pred.shape[-1] corrected_logit = logit(y_pred) - logit( np.array(train_priors))[np.newaxis, :] if 'validation_priors' in params: if isinstance(params['validation_priors'], str): validation_priors = joblib.load( Path(params['validation_priors']).expanduser()) elif isinstance(params['validation_priors'], float): validation_priors = [params['validation_priors'] ] * y_pred.shape[-1] corrected_logit = corrected_logit + logit( np.array(validation_priors))[np.newaxis, :] if corrected_logit is not None: y_pred = expit(corrected_logit) metric_results = {} for metric in params_metrics: metric_type = metric.pop('type') if inspect.isclass(available_metrics[metric_type]): metric_cls = available_metrics[metric_type](**metric) if 'Codebook' in metric_type: metric_cls.codebook_layers = codebook_layers metric_cls.validation_data = params['validation_data'] metric_cls.model = self.model mres = metric_cls(y_true, y_pred) elif inspect.isfunction(available_metrics[metric_type]): mres = available_metrics[metric_type](y_true, y_pred) else: raise Exception('Unrecognized metric') for k, v in mres.items(): if v.ndim == 0: metric_results['val_{}'.format(k)] = v elif v.ndim == 1: labels = None if 'labels' in params: if isinstance(params['labels'], str): labels = joblib.load( Path(params['labels']).expanduser()) else: labels = params['labels'] if len(v) > 100: for i in range(len(v) // 100): plt.figure(figsize=(20, 10)) sns.barplot(x=labels[i * 100:(i + 1) * 100], y=v[i * 100:(i + 1) * 100]) plt.xticks(rotation=90) wandb.log( { 'val_{}_{}'.format(k, i): wandb.Image(plt) }, step=self.step) if len(v) % 100 > 0: plt.figure(figsize=(20, 10)) sns.barplot(x=labels[100 * (len(v) // 100):], y=v[100 * (len(v) // 100):]) plt.xticks(rotation=90) wandb.log( { 'val_{}_{}'.format(k, len(v) // 100): wandb.Image(plt) }, step=self.step) else: plt.figure(figsize=(20, 10)) if len(labels) == len(v): sns.barplot(x=labels, y=v) else: sns.barplot(x=np.arange(len(v)), y=v) plt.xticks(rotation=90) wandb.log({'val_{}'.format(k): wandb.Image(plt)}, step=self.step) elif v.ndim == 2: from IPython import embed embed() elif v.ndim == 3: for i, v_i in enumerate(v): plt.figure(figsize=(10, 10)) plt.imshow(v_i, aspect='auto', origin='lower') wandb.log( {'val_{}_{}'.format(k, i): wandb.Image(plt)}, step=self.step) wandb.log(metric_results, step=self.step) else: metric_results = self.model.evaluate(params['validation_data'], return_dict=True) metric_results = { 'val_{}'.format(k): v for k, v in metric_results.items() } wandb.log(metric_results, step=self.step) return metric_results
def fit(self, keras_model, data, output_path, from_epoch=0, validation_data=None, cache=True, training_strategy=None, from_step=0, class_weights=None): self.model_path = output_path self.cache = cache n_epochs = self.config.get('n_epochs', 10) calculate_initial_loss = self.config.get('calculate_initial_loss', False) loss_fn = self.get_loss_fn() optimizer = self.get_optimizer() cb_list = self.get_callbacks() for cb in cb_list: cb.step = from_step metrics = self.get_metrics() for cb in cb_list: cb.epoch = from_epoch cb.data = data with training_strategy.scope(): keras_model.compile(optimizer=optimizer, loss=loss_fn, metrics=metrics) if self.weights: load_weights(self.weights, keras_model) if self.optimizer_weights: keras_model.optimizer._create_all_weights( keras_model.trainable_variables) keras_model.optimizer.set_weights(self.optimizer_weights) keras_model.initial_step = from_step if calculate_initial_loss and validation_data: if self.logger: self.logger.info('Initial Loss: {}'.format( keras_model.evaluate(validation_data))) n_workers = self.config.get('workers', 1) if n_workers > 1: use_multiprocessing = True else: use_multiprocessing = False custom_fit = self.config.get('custom_fit', None) if hasattr(data, 'epoch'): data.epoch = from_epoch if hasattr(data, 'on_train_begin'): data.on_train_begin() if custom_fit: modules = get_modules(self.modules) custom_fit_fn = None for module in modules: available_fns = get_members_from_module( module, filters=[inspect.isfunction]) if custom_fit in available_fns: custom_fit_fn = available_fns[custom_fit] break if custom_fit_fn: custom_fit_fn(data, keras_model, callbacks=cb_list, initial_epoch=from_epoch, epochs=n_epochs) else: raise Exception('{} not found'.format(custom_fit)) else: if isinstance(data, list) or isinstance(data, tuple): if validation_data is not None: validation_data = tuple(validation_data) keras_model.fit(x=data[0], y=data[1], initial_epoch=from_epoch, epochs=n_epochs, callbacks=cb_list, validation_data=validation_data, use_multiprocessing=use_multiprocessing, workers=n_workers, shuffle=True, batch_size=self.config.get('batch_size', None), steps_per_epoch=self.config.get( 'steps_per_epoch', None), class_weight=class_weights) else: keras_model.fit(data, initial_epoch=from_epoch, epochs=n_epochs, callbacks=cb_list, validation_data=validation_data, use_multiprocessing=use_multiprocessing, workers=n_workers, shuffle=False, batch_size=self.config.get('batch_size', None), steps_per_epoch=self.config.get( 'steps_per_epoch', None), class_weight=class_weights)
def make_network(self, training_strategy): dependency_order = self.find_dependency_order(self.processed_config, self.inputs, self.outputs) self.tensors = {} shared_layers_counts = {} for layer in dependency_order: if layer not in self.inputs: layer_inputs = self.processed_config[layer]['input'] layer_inputs = layer_inputs if isinstance( layer_inputs, list) else [layer_inputs] layer_inputs = [self.tensors[inp] for inp in layer_inputs] layer_mask = self.processed_config[layer].get('mask', None) layer_training = self.processed_config[layer].get( 'training', None) if len(layer_inputs) == 1: layer_inputs = layer_inputs[0] if layer in self.shared_layers: target_layer = self.shared_layers[layer] if target_layer in shared_layers_counts: shared_layers_counts[target_layer] += 1 else: shared_layers_counts[target_layer] = 0 if layer_mask is not None: layer_mask = self.tensors[layer_mask] self.tensors[layer] = self.layers[layer]( layer_inputs, mask=layer_mask) else: self.tensors[layer] = self.layers[layer](layer_inputs) self.tensors[layer] = self.layers[layer].get_output_at( shared_layers_counts[target_layer]) else: try: if layer_mask is not None: layer_mask = self.tensors[layer_mask] layer_mask = tf.cast(layer_mask, tf.bool) self.tensors[layer] = self.layers[layer]( layer_inputs, mask=layer_mask, training=layer_training) else: self.tensors[layer] = self.layers[layer]( layer_inputs, training=layer_training) except Exception as e: raise Exception( 'Could not connect layer {} with its inputs {}. {}' .format(layer, self.processed_config[layer]['input'], e)) else: self.tensors[layer] = self.layers[layer] input_tensors = [self.tensors[in_tensor] for in_tensor in self.inputs] output_tensors = [ self.tensors[out_tensor] for out_tensor in self.outputs ] if self.custom_model is None: model = tf.keras.Model(inputs=input_tensors, outputs=output_tensors) else: available_modules = get_modules(self.modules) available_cls = {} for module in available_modules: available_cls.update( get_members_from_module(module, filters=[inspect.isclass])) model_cls = self.custom_model.pop('type') model = available_cls[model_cls](inputs=input_tensors, outputs=output_tensors, **self.custom_model) model.distributed_strategy = training_strategy for layer_name, weights in self.weights_to_assign.items(): model.get_layer(layer_name).set_weights(weights) return model