def batch_predict_loop(model, inputs, batch_size, verbose=0): """Predict function for eager execution when input is arrays or tensors. Arguments: model: Instance of `Model`. inputs: List of input arrays. batch_size: Integer batch size. verbose: Verbosity mode. Returns: Array of predictions (if the model has a single output) or list of arrays of predictions (if the model has multiple outputs). """ outs = [] num_samples = training_utils.check_num_samples(inputs, batch_size) if verbose == 1: progbar = generic_utils.Progbar(target=num_samples) batches = generic_utils.make_batches(num_samples, batch_size) index_array = np.arange(num_samples) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] inputs_batch = slice_arrays(inputs, batch_ids) inputs_batch = [ ops.convert_to_tensor(val, dtype=backend.floatx()) for val in inputs_batch ] if len(inputs_batch) == 1: if model._expects_training_arg: batch_outs = model.call(inputs_batch[0], training=False) else: batch_outs = model.call(inputs_batch[0]) else: if model._expects_training_arg: batch_outs = model.call(inputs_batch, training=False) else: batch_outs = model.call(inputs_batch) if not isinstance(batch_outs, list): batch_outs = [batch_outs] if batch_index == 0: # Pre-allocate the results arrays. for batch_out in batch_outs: dims = batch_out.shape[1:].dims dims_list = [d.value for d in dims] shape = (num_samples, ) + tuple(dims_list) outs.append( np.zeros(shape, dtype=batch_out.dtype.as_numpy_dtype)) for i, batch_out in enumerate(batch_outs): outs[i][batch_start:batch_end] = batch_out if verbose == 1: progbar.update(batch_end) if len(outs) == 1: return outs[0] return outs
def batch_predict_loop(model, inputs, batch_size, verbose=0): """Predict function for eager execution when input is arrays or tensors. Arguments: model: Instance of `Model`. inputs: List of input arrays. batch_size: Integer batch size. verbose: Verbosity mode. Returns: Array of predictions (if the model has a single output) or list of arrays of predictions (if the model has multiple outputs). """ outs = [] num_samples = training_utils.check_num_samples(inputs, batch_size) if verbose == 1: progbar = generic_utils.Progbar(target=num_samples) batches = generic_utils.make_batches(num_samples, batch_size) index_array = np.arange(num_samples) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] inputs_batch = slice_arrays(inputs, batch_ids) inputs_batch = [ ops.convert_to_tensor(val, dtype=backend.floatx()) for val in inputs_batch ] if len(inputs_batch) == 1: if model._expects_training_arg: batch_outs = model.call(inputs_batch[0], training=False) else: batch_outs = model.call(inputs_batch[0]) else: if model._expects_training_arg: batch_outs = model.call(inputs_batch, training=False) else: batch_outs = model.call(inputs_batch) if not isinstance(batch_outs, list): batch_outs = [batch_outs] if batch_index == 0: # Pre-allocate the results arrays. for batch_out in batch_outs: dims = batch_out.shape[1:].dims dims_list = [d.value for d in dims] shape = (num_samples,) + tuple(dims_list) outs.append(np.zeros(shape, dtype=batch_out.dtype.as_numpy_dtype)) for i, batch_out in enumerate(batch_outs): outs[i][batch_start:batch_end] = batch_out if verbose == 1: progbar.update(batch_end) if len(outs) == 1: return outs[0] return outs
def fit_loop(model, inputs, targets, sample_weights=None, val_inputs=None, val_targets=None, val_sample_weights=None, batch_size=None, epochs=100, verbose=1, callbacks=None, shuffle=True, callback_metrics=None, initial_epoch=0, steps_per_epoch=None, validation_steps=None): """Abstract fit function for eager execution. Arguments: model: Instance of the model that is being executed in Eager mode. inputs: List of input arrays. targets: List of target arrays. sample_weights: Optional list of sample weight arrays. val_inputs: Input data for validation. val_targets: Target data for validation. val_sample_weights: Sample weight data for validation. batch_size: Integer batch size or None if unknown. epochs: Number of times to iterate over the data verbose: Verbosity mode, 0, 1 or 2 callbacks: List of callbacks to be called during training shuffle: Whether to shuffle the data at the beginning of each epoch callback_metrics: List of strings, the display names of the metrics passed to the callbacks. They should be the concatenation of list the display names of the outputs of `f` and the list of display names of the outputs of `f_val`. initial_epoch: Epoch at which to start training (useful for resuming a previous training run) steps_per_epoch: Total number of steps (batches of samples) before declaring one epoch finished and starting the next epoch. Ignored with the default value of `None`. validation_steps: Number of steps to run validation for (only if doing validation from data tensors). Ignored with default value of `None`. Returns: `History` object. Raises: ValueError: In case of invalid argument values. """ if not batch_size: raise ValueError( 'With eager execution, `batch_size` should be specified.') if steps_per_epoch or validation_steps: raise ValueError('With eager execution, `steps_per_epoch` and ' '`validation_steps` are not valid arguments ' '(set `batch_size` instead).') # Required for Eager mode with backend.learning_phase_scope(1): do_validation = False if val_inputs: do_validation = True if (verbose and inputs and hasattr(inputs[0], 'shape') and hasattr(val_inputs[0], 'shape')): print('Train on %d samples, validate on %d samples' % (inputs[0].shape[0], val_inputs[0].shape[0])) if validation_steps: if steps_per_epoch is None: raise ValueError( 'Can only use `validation_steps` when doing step-wise ' 'training, i.e. `steps_per_epoch` must be set.') do_validation = True out_labels = model.metrics_names if do_validation: callback_metrics = copy.copy(out_labels) + [ 'val_' + n for n in out_labels ] else: callback_metrics = copy.copy(out_labels) if sample_weights: feed_data = inputs + targets + sample_weights else: feed_data = inputs + targets num_train_samples = training_utils.check_num_samples( feed_data, batch_size=batch_size, steps=steps_per_epoch, steps_name='steps_per_epoch') if num_train_samples is not None: index_array = np.arange(num_train_samples) model.history = cbks.History() callbacks = [cbks.BaseLogger()] + (callbacks or []) + [model.history] if verbose: if steps_per_epoch is not None: count_mode = 'steps' else: count_mode = 'samples' callbacks += [cbks.ProgbarLogger(count_mode)] callbacks = cbks.CallbackList(callbacks) # it's possible to callback a different model than self # (used by Sequential models) if hasattr(model, 'callback_model') and model.callback_model: callback_model = model.callback_model else: callback_model = model callbacks.set_model(callback_model) callbacks.set_params({ 'batch_size': batch_size, 'epochs': epochs, 'steps': steps_per_epoch, 'samples': num_train_samples, 'verbose': verbose, 'do_validation': do_validation, 'metrics': callback_metrics or [], }) callbacks.on_train_begin() callback_model.stop_training = False for cbk in callbacks: if not val_inputs: cbk.validation_data = [] elif val_sample_weights: cbk.validation_data = val_inputs + val_targets + val_sample_weights else: cbk.validation_data = val_inputs + val_targets for epoch in range(initial_epoch, epochs): callbacks.on_epoch_begin(epoch) epoch_logs = {} if shuffle == 'batch': index_array = model._batch_shuffle(index_array, batch_size) elif shuffle: np.random.shuffle(index_array) batches = generic_utils.make_batches(num_train_samples, batch_size) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] try: inputs_batch = slice_arrays(inputs, batch_ids, contiguous=not shuffle) targets_batch = slice_arrays(targets, batch_ids, contiguous=not shuffle) if sample_weights: sample_weights_batch = slice_arrays( sample_weights, batch_ids, contiguous=not shuffle) else: sample_weights_batch = None except TypeError: raise TypeError('TypeError while preparing batch. ' 'If using HDF5 input data, ' 'pass shuffle="batch".') batch_logs = {} batch_logs['batch'] = batch_index batch_logs['size'] = len(batch_ids) callbacks.on_batch_begin(batch_index, batch_logs) inputs_batch = [ ops.convert_to_tensor(val, dtype=backend.floatx()) for val in inputs_batch ] targets_batch = [ ops.convert_to_tensor(val, dtype=backend.floatx()) for val in targets_batch ] if sample_weights: sample_weights_batch = [ ops.convert_to_tensor(val, dtype=backend.floatx()) if val is not None else None for val in sample_weights_batch ] outs, loss, loss_metrics = _process_single_batch( model, inputs_batch, targets_batch, sample_weights=sample_weights_batch, training=True) if not isinstance(outs, list): outs = [outs] for l, o in zip(out_labels, outs): batch_logs[l] = o # Required for Eager mode metrics_results = _eager_metrics_fn(model, outs, targets_batch) batch_logs['loss'] = tensor_util.constant_value( backend.mean(loss)) for k, v in zip(model.metrics_names, [backend.mean(loss)] + loss_metrics + metrics_results): batch_logs[k] = tensor_util.constant_value(v) callbacks.on_batch_end(batch_index, batch_logs) if callback_model.stop_training: break if batch_index == len(batches) - 1: # Last batch. if do_validation: val_outs = test_loop(model, val_inputs, val_targets, sample_weights=val_sample_weights, batch_size=batch_size, verbose=0) if not isinstance(val_outs, list): val_outs = [val_outs] # Same labels assumed. for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o callbacks.on_epoch_end(epoch, epoch_logs) if callback_model.stop_training: break callbacks.on_train_end() return model.history
def batch_test_loop(model, inputs, targets, batch_size, sample_weights=None, verbose=0): """Test function for eager execution when input is given as arrays or tensors. Arguments: model: Model instance that is being evaluated in Eager mode. inputs: List of input arrays. targets: List of target arrays. batch_size: Integer batch size. sample_weights: Optional list of sample weight arrays. verbose: Verbosity mode. Returns: Scalar loss (if the model has a single output and no metrics) or list of scalars (if the model has multiple outputs and/or metrics). The attribute `model.metrics_names` will give you the display labels for the scalar outputs. """ outs = [] feed_data = inputs + targets if sample_weights: feed_data += sample_weights num_samples = training_utils.check_num_samples( feed_data, batch_size=batch_size) if verbose == 1: progbar = generic_utils.Progbar(target=num_samples) batches = generic_utils.make_batches(num_samples, batch_size) index_array = np.arange(num_samples) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] inputs_batch = slice_arrays(inputs, batch_ids) targets_batch = slice_arrays(targets, batch_ids) if sample_weights: sample_weights_batch = slice_arrays(sample_weights, batch_ids) else: sample_weights_batch = None inputs_batch = [ ops.convert_to_tensor(val, dtype=backend.floatx()) for val in inputs_batch ] targets_batch = [ ops.convert_to_tensor(val, dtype=backend.floatx()) for val in targets_batch ] if sample_weights: sample_weights_batch = [ ops.convert_to_tensor(val, dtype=backend.floatx()) if val is not None else None for val in sample_weights_batch ] loss_outs, loss, loss_metrics = _model_loss( model, inputs_batch, targets_batch, sample_weights=sample_weights_batch, training=False) metrics_results = _eager_metrics_fn(model, loss_outs, targets_batch) batch_outs = [] for _, v in zip(model.metrics_names, [backend.mean(loss)] + loss_metrics + metrics_results): batch_outs.append(tensor_util.constant_value(v)) if isinstance(batch_outs, list): if batch_index == 0: for _ in enumerate(batch_outs): outs.append(0.) for i, batch_out in enumerate(batch_outs): outs[i] += batch_out * len(batch_ids) else: if batch_index == 0: outs.append(0.) outs[0] += batch_outs * len(batch_ids) if verbose == 1: progbar.update(batch_end) for i in range(len(outs)): outs[i] /= num_samples if len(outs) == 1: return outs[0] return outs
def test_loop(model, ins, batch_size=None, verbose=0, steps=None): """Abstract method to loop over some data in batches. Arguments: model: Model instance that is being evaluated in Eager mode. ins: list of tensors to be fed to `f`. batch_size: integer batch size or `None`. verbose: verbosity mode. steps: Total number of steps (batches of samples) before declaring predictions finished. Ignored with the default value of `None`. Returns: Scalar loss (if the model has a single output and no metrics) or list of scalars (if the model has multiple outputs and/or metrics). The attribute `model.metrics_names` will give you the display labels for the scalar outputs. """ K.set_learning_phase(False) num_samples = model._check_num_samples(ins, batch_size, steps, 'steps') outs = [] if verbose == 1: progbar = Progbar(target=num_samples) batches = make_batches(num_samples, batch_size) index_array = np.arange(num_samples) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] if isinstance(ins[-1], float): # Do not slice the training phase flag. ins_batch = slice_arrays(ins[:-1], batch_ids) + [ins[-1]] else: ins_batch = slice_arrays(ins, batch_ids) ins_batch_converted = [] for ib in ins_batch: ins_batch_converted.append( ops.convert_to_tensor(ib, dtype=K.floatx())) eager_model_inputs = [] eager_model_outputs = [] for i in range(len(model.inputs)): eager_model_inputs.append(ins_batch_converted[i]) for i in range(len(model.inputs), len(ins_batch_converted)): eager_model_outputs.append(ins_batch_converted[i]) loss_outs, loss, loss_metrics = _model_loss(model, eager_model_inputs, eager_model_outputs, training=False) _, metrics_results = _eager_metrics_fn(model, loss_outs, eager_model_outputs) batch_outs = [] for _, v in zip(model.metrics_names, [K.mean(loss)] + loss_metrics + metrics_results): batch_outs.append(tensor_util.constant_value(v)) if isinstance(batch_outs, list): if batch_index == 0: for batch_out in enumerate(batch_outs): outs.append(0.) for i, batch_out in enumerate(batch_outs): outs[i] += batch_out * len(batch_ids) else: if batch_index == 0: outs.append(0.) outs[0] += batch_outs * len(batch_ids) if verbose == 1: progbar.update(batch_end) for i in range(len(outs)): outs[i] /= num_samples if len(outs) == 1: return outs[0] return outs
def test_loop(model, inputs, targets, sample_weights=None, batch_size=None, verbose=0, steps=None): """Abstract method to loop over some data in batches. Arguments: model: Keras Model instance. inputs: List of input arrays. targets: List of target arrays. sample_weights: Optional list of sample weight arrays. batch_size: integer batch size or `None`. verbose: verbosity mode. steps: Total number of steps (batches of samples) before declaring predictions finished. Ignored with the default value of `None`. Returns: Scalar loss (if the model has a single output and no metrics) or list of scalars (if the model has multiple outputs and/or metrics). The attribute `model.metrics_names` will give you the display labels for the scalar outputs. """ model._make_test_function() f = model.test_function sample_weights = sample_weights or [] if model.uses_learning_phase and not isinstance(K.learning_phase(), int): ins = inputs + targets + sample_weights + [0] else: ins = inputs + targets + sample_weights if hasattr(model, 'metrics'): for m in model.metrics: if isinstance(m, Layer): m.reset_states() stateful_metric_indices = [ i for i, name in enumerate(model.metrics_names) if str(name) in model.stateful_metric_names ] else: stateful_metric_indices = [] num_samples = training_utils.check_num_samples(ins, batch_size, steps, 'steps') outs = [] if verbose == 1: if steps is not None: progbar = Progbar(target=steps) else: progbar = Progbar(target=num_samples) # To prevent a slowdown, we find beforehand the arrays that need conversion. feed = model._feed_inputs + model._feed_targets + model._feed_sample_weights indices_for_conversion_to_dense = [] for i in range(len(feed)): if issparse is not None and issparse( ins[i]) and not K.is_sparse(feed[i]): indices_for_conversion_to_dense.append(i) if steps is not None: for step in range(steps): batch_outs = f(ins) if isinstance(batch_outs, list): if step == 0: for _ in enumerate(batch_outs): outs.append(0.) for i, batch_out in enumerate(batch_outs): if i in stateful_metric_indices: outs[i] = batch_out else: outs[i] += batch_out else: if step == 0: outs.append(0.) outs[0] += batch_outs if verbose == 1: progbar.update(step + 1) for i in range(len(outs)): if i not in stateful_metric_indices: outs[i] /= steps else: batches = make_batches(num_samples, batch_size) index_array = np.arange(num_samples) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] if isinstance(ins[-1], int): # Do not slice the training phase flag. ins_batch = slice_arrays(ins[:-1], batch_ids) + [ins[-1]] else: ins_batch = slice_arrays(ins, batch_ids) for i in indices_for_conversion_to_dense: ins_batch[i] = ins_batch[i].toarray() batch_outs = f(ins_batch) if isinstance(batch_outs, list): if batch_index == 0: for batch_out in enumerate(batch_outs): outs.append(0.) for i, batch_out in enumerate(batch_outs): if i in stateful_metric_indices: outs[i] = batch_out else: outs[i] += batch_out * len(batch_ids) else: if batch_index == 0: outs.append(0.) outs[0] += batch_outs * len(batch_ids) if verbose == 1: progbar.update(batch_end) for i in range(len(outs)): if i not in stateful_metric_indices: outs[i] /= num_samples if len(outs) == 1: return outs[0] return outs
def predict_loop(model, inputs, batch_size=32, verbose=0, steps=None): """Abstract method to loop over some data in batches. Arguments: model: inputs: List of input arrays. batch_size: integer batch size. verbose: verbosity mode. steps: Total number of steps (batches of samples) before declaring `_predict_loop` finished. Ignored with the default value of `None`. Returns: Array of predictions (if the model has a single output) or list of arrays of predictions (if the model has multiple outputs). """ with backend.learning_phase_scope(0): num_samples = training_utils.check_num_samples( inputs, batch_size, steps, 'steps') if verbose == 1: if steps is not None: progbar = generic_utils.Progbar(target=steps) else: progbar = generic_utils.Progbar(target=num_samples) outs = [] batches = generic_utils.make_batches(num_samples, batch_size) index_array = np.arange(num_samples) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] inputs_batch = slice_arrays(inputs, batch_ids) inputs_batch = [ ops.convert_to_tensor(val, dtype=backend.floatx()) for val in inputs_batch] if len(inputs_batch) == 1: if model._expects_training_arg: batch_outs = model.call(inputs_batch[0], training=False) else: batch_outs = model.call(inputs_batch[0]) else: if model._expects_training_arg: batch_outs = model.call(inputs_batch, training=False) else: batch_outs = model.call(inputs_batch) if not isinstance(batch_outs, list): batch_outs = [batch_outs] if batch_index == 0: # Pre-allocate the results arrays. for batch_out in batch_outs: dims = batch_out.shape[1:].dims dims_list = [d.value for d in dims] shape = (num_samples,) + tuple(dims_list) outs.append(np.zeros(shape, dtype=batch_out.dtype.as_numpy_dtype)) for i, batch_out in enumerate(batch_outs): outs[i][batch_start:batch_end] = batch_out if verbose == 1: progbar.update(batch_end) if len(outs) == 1: return outs[0] return outs
def fit_loop( model, inputs, targets, sample_weights=None, val_inputs=None, val_targets=None, val_sample_weights=None, batch_size=None, epochs=100, verbose=1, callbacks=None, shuffle=True, callback_metrics=None, initial_epoch=0, steps_per_epoch=None, validation_steps=None): """Abstract fit function for eager execution. Arguments: model: Instance of the model that is being executed in Eager mode. inputs: List of input arrays. targets: List of target arrays. sample_weights: Optional list of sample weight arrays. val_inputs: Input data for validation. val_targets: Target data for validation. val_sample_weights: Sample weight data for validation. batch_size: Integer batch size or None if unknown. epochs: Number of times to iterate over the data verbose: Verbosity mode, 0, 1 or 2 callbacks: List of callbacks to be called during training shuffle: Whether to shuffle the data at the beginning of each epoch callback_metrics: List of strings, the display names of the metrics passed to the callbacks. They should be the concatenation of list the display names of the outputs of `f` and the list of display names of the outputs of `f_val`. initial_epoch: Epoch at which to start training (useful for resuming a previous training run) steps_per_epoch: Total number of steps (batches of samples) before declaring one epoch finished and starting the next epoch. Ignored with the default value of `None`. validation_steps: Number of steps to run validation for (only if doing validation from data tensors). Ignored with default value of `None`. Returns: `History` object. Raises: ValueError: In case of invalid argument values. """ if not batch_size: raise ValueError('With eager execution, `batch_size` should be specified.') if steps_per_epoch or validation_steps: raise ValueError('With eager execution, `steps_per_epoch` and ' '`validation_steps` are not valid arguments ' '(set `batch_size` instead).') # Required for Eager mode with backend.learning_phase_scope(1): do_validation = False if val_inputs: do_validation = True if (verbose and inputs and hasattr(inputs[0], 'shape') and hasattr(val_inputs[0], 'shape')): print('Train on %d samples, validate on %d samples' % (inputs[0].shape[0], val_inputs[0].shape[0])) if validation_steps: if steps_per_epoch is None: raise ValueError('Can only use `validation_steps` when doing step-wise ' 'training, i.e. `steps_per_epoch` must be set.') do_validation = True out_labels = model.metrics_names if do_validation: callback_metrics = copy.copy(out_labels) + [ 'val_' + n for n in out_labels ] else: callback_metrics = copy.copy(out_labels) if sample_weights: feed_data = inputs + targets + sample_weights else: feed_data = inputs + targets num_train_samples = training_utils.check_num_samples( feed_data, batch_size=batch_size, steps=steps_per_epoch, steps_name='steps_per_epoch') if num_train_samples is not None: index_array = np.arange(num_train_samples) model.history = cbks.History() callbacks = [cbks.BaseLogger()] + (callbacks or []) + [model.history] if verbose: if steps_per_epoch is not None: count_mode = 'steps' else: count_mode = 'samples' callbacks += [cbks.ProgbarLogger(count_mode)] callbacks = cbks.CallbackList(callbacks) # it's possible to callback a different model than self # (used by Sequential models) if hasattr(model, 'callback_model') and model.callback_model: callback_model = model.callback_model else: callback_model = model callbacks.set_model(callback_model) callbacks.set_params({ 'batch_size': batch_size, 'epochs': epochs, 'steps': steps_per_epoch, 'samples': num_train_samples, 'verbose': verbose, 'do_validation': do_validation, 'metrics': callback_metrics or [], }) callbacks.on_train_begin() callback_model.stop_training = False for cbk in callbacks: if not val_inputs: cbk.validation_data = [] elif val_sample_weights: cbk.validation_data = val_inputs + val_targets + val_sample_weights else: cbk.validation_data = val_inputs + val_targets for epoch in range(initial_epoch, epochs): callbacks.on_epoch_begin(epoch) epoch_logs = {} if shuffle == 'batch': index_array = model._batch_shuffle(index_array, batch_size) elif shuffle: np.random.shuffle(index_array) batches = generic_utils.make_batches(num_train_samples, batch_size) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] try: inputs_batch = slice_arrays(inputs, batch_ids, contiguous=not shuffle) targets_batch = slice_arrays(targets, batch_ids, contiguous=not shuffle) if sample_weights: sample_weights_batch = slice_arrays(sample_weights, batch_ids, contiguous=not shuffle) else: sample_weights_batch = None except TypeError: raise TypeError('TypeError while preparing batch. ' 'If using HDF5 input data, ' 'pass shuffle="batch".') batch_logs = {} batch_logs['batch'] = batch_index batch_logs['size'] = len(batch_ids) callbacks.on_batch_begin(batch_index, batch_logs) inputs_batch = [ ops.convert_to_tensor(val, dtype=backend.floatx()) for val in inputs_batch] targets_batch = [ ops.convert_to_tensor(val, dtype=backend.floatx()) for val in targets_batch] if sample_weights: sample_weights_batch = [ ops.convert_to_tensor(val, dtype=backend.floatx()) if val is not None else None for val in sample_weights_batch] outs, loss, loss_metrics = _process_single_batch( model, inputs_batch, targets_batch, sample_weights=sample_weights_batch, training=True) if not isinstance(outs, list): outs = [outs] for l, o in zip(out_labels, outs): batch_logs[l] = o # Required for Eager mode metrics_results = _eager_metrics_fn(model, outs, targets_batch) batch_logs['loss'] = tensor_util.constant_value(backend.mean(loss)) for k, v in zip(model.metrics_names, [backend.mean(loss)] + loss_metrics + metrics_results): batch_logs[k] = tensor_util.constant_value(v) callbacks.on_batch_end(batch_index, batch_logs) if callback_model.stop_training: break if batch_index == len(batches) - 1: # Last batch. if do_validation: val_outs = test_loop( model, val_inputs, val_targets, sample_weights=val_sample_weights, batch_size=batch_size, verbose=0) if not isinstance(val_outs, list): val_outs = [val_outs] # Same labels assumed. for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o callbacks.on_epoch_end(epoch, epoch_logs) if callback_model.stop_training: break callbacks.on_train_end() return model.history
def predict_loop(model, ins, batch_size=32, verbose=0, steps=None): """Abstract method to loop over some data in batches. Arguments: model: ins: list of tensors to be fed to `f`. batch_size: integer batch size. verbose: verbosity mode. steps: Total number of steps (batches of samples) before declaring `_predict_loop` finished. Ignored with the default value of `None`. Returns: Array of predictions (if the model has a single output) or list of arrays of predictions (if the model has multiple outputs). """ K.set_learning_phase(False) num_samples = model._check_num_samples(ins, batch_size, steps, 'steps') if verbose == 1: if steps is not None: progbar = Progbar(target=steps) else: progbar = Progbar(target=num_samples) outs = [] batches = make_batches(num_samples, batch_size) index_array = np.arange(num_samples) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] if ins and isinstance(ins[-1], float): # Do not slice the training phase flag. ins_batch = slice_arrays(ins[:-1], batch_ids) + [ins[-1]] else: ins_batch = slice_arrays(ins, batch_ids) ins_batch_converted = [] for ib in ins_batch: ins_batch_converted.append(ops.convert_to_tensor(ib, dtype=K.floatx())) eager_model_inputs = [] for i in range(len(model.inputs)): eager_model_inputs.append(ins_batch_converted[i]) if len(eager_model_inputs) == 1: batch_outs = model.call(eager_model_inputs[0]) else: batch_outs = model.call(eager_model_inputs) if not isinstance(batch_outs, list): batch_outs = [batch_outs] if batch_index == 0: # Pre-allocate the results arrays. for batch_out in batch_outs: dims = batch_out.shape[1:].dims dims_list = [d.value for d in dims] shape = (num_samples,) + tuple(dims_list) outs.append(np.zeros(shape, dtype=batch_out.dtype.as_numpy_dtype)) for i, batch_out in enumerate(batch_outs): outs[i][batch_start:batch_end] = batch_out if verbose == 1: progbar.update(batch_end) if len(outs) == 1: return outs[0] return outs
def test_loop(model, ins, batch_size=None, verbose=0, steps=None): """Abstract method to loop over some data in batches. Arguments: model: Model instance that is being evaluated in Eager mode. ins: list of tensors to be fed to `f`. batch_size: integer batch size or `None`. verbose: verbosity mode. steps: Total number of steps (batches of samples) before declaring predictions finished. Ignored with the default value of `None`. Returns: Scalar loss (if the model has a single output and no metrics) or list of scalars (if the model has multiple outputs and/or metrics). The attribute `model.metrics_names` will give you the display labels for the scalar outputs. """ K.set_learning_phase(False) num_samples = model._check_num_samples(ins, batch_size, steps, 'steps') outs = [] if verbose == 1: progbar = Progbar(target=num_samples) batches = make_batches(num_samples, batch_size) index_array = np.arange(num_samples) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] if isinstance(ins[-1], float): # Do not slice the training phase flag. ins_batch = slice_arrays(ins[:-1], batch_ids) + [ins[-1]] else: ins_batch = slice_arrays(ins, batch_ids) ins_batch_converted = [] for ib in ins_batch: ins_batch_converted.append(ops.convert_to_tensor(ib, dtype=K.floatx())) eager_model_inputs = [] eager_model_outputs = [] for i in range(len(model.inputs)): eager_model_inputs.append(ins_batch_converted[i]) for i in range(len(model.inputs), len(ins_batch_converted)): eager_model_outputs.append(ins_batch_converted[i]) loss_outs, loss, loss_metrics = _model_loss(model, eager_model_inputs, eager_model_outputs) _, metrics_results = _eager_metrics_fn(model, loss_outs, eager_model_outputs) batch_outs = [] for _, v in zip(model.metrics_names, [K.mean(loss)] + loss_metrics + metrics_results): batch_outs.append(tensor_util.constant_value(v)) if isinstance(batch_outs, list): if batch_index == 0: for batch_out in enumerate(batch_outs): outs.append(0.) for i, batch_out in enumerate(batch_outs): outs[i] += batch_out * len(batch_ids) else: if batch_index == 0: outs.append(0.) outs[0] += batch_outs * len(batch_ids) if verbose == 1: progbar.update(batch_end) for i in range(len(outs)): outs[i] /= num_samples if len(outs) == 1: return outs[0] return outs
def fit_loop( model, ins, out_labels=None, batch_size=None, epochs=100, verbose=1, callbacks=None, val_ins=None, shuffle=True, callback_metrics=None, initial_epoch=0, steps_per_epoch=None, validation_steps=None): """Abstract fit function for `f(ins)`. Assume that f returns a list, labeled by out_labels. Arguments: model: Instance of the model that is being executed in Eager mode. ins: List of tensors to be fed to `f` out_labels: List of strings, display names of the outputs of `f` batch_size: Integer batch size or None if unknown. epochs: Number of times to iterate over the data verbose: Verbosity mode, 0, 1 or 2 callbacks: List of callbacks to be called during training val_ins: List of tensors to be fed to `val_f` shuffle: Whether to shuffle the data at the beginning of each epoch callback_metrics: List of strings, the display names of the metrics passed to the callbacks. They should be the concatenation of list the display names of the outputs of `f` and the list of display names of the outputs of `f_val`. initial_epoch: Epoch at which to start training (useful for resuming a previous training run) steps_per_epoch: Total number of steps (batches of samples) before declaring one epoch finished and starting the next epoch. Ignored with the default value of `None`. validation_steps: Number of steps to run validation for (only if doing validation from data tensors). Ignored with default value of `None`. Returns: `History` object. Raises: ValueError: In case of invalid argument values. """ # Required for Eager mode K.set_learning_phase(True) do_validation = False if val_ins: do_validation = True if (verbose and ins and hasattr(ins[0], 'shape') and hasattr(val_ins[0], 'shape')): print('Train on %d samples, validate on %d samples' % (ins[0].shape[0], val_ins[0].shape[0])) if validation_steps: if steps_per_epoch is None: raise ValueError('Can only use `validation_steps` when doing step-wise ' 'training, i.e. `steps_per_epoch` must be set.') do_validation = True num_train_samples = model._check_num_samples( ins, batch_size, steps_per_epoch, 'steps_per_epoch') if num_train_samples is not None: index_array = np.arange(num_train_samples) model.history = cbks.History() callbacks = [cbks.BaseLogger()] + (callbacks or []) + [model.history] if verbose: if steps_per_epoch is not None: count_mode = 'steps' else: count_mode = 'samples' callbacks += [cbks.ProgbarLogger(count_mode)] callbacks = cbks.CallbackList(callbacks) out_labels = out_labels or [] # it's possible to callback a different model than self # (used by Sequential models) if hasattr(model, 'callback_model') and model.callback_model: callback_model = model.callback_model else: callback_model = model callbacks.set_model(callback_model) callbacks.set_params({ 'batch_size': batch_size, 'epochs': epochs, 'steps': steps_per_epoch, 'samples': num_train_samples, 'verbose': verbose, 'do_validation': do_validation, 'metrics': callback_metrics or [], }) callbacks.on_train_begin() callback_model.stop_training = False for cbk in callbacks: cbk.validation_data = val_ins for epoch in range(initial_epoch, epochs): callbacks.on_epoch_begin(epoch) epoch_logs = {} if shuffle == 'batch': index_array = model._batch_shuffle(index_array, batch_size) elif shuffle: np.random.shuffle(index_array) batches = make_batches(num_train_samples, batch_size) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] try: if isinstance(ins[-1], float): # Do not slice the training phase flag. ins_batch = slice_arrays(ins[:-1], batch_ids) + [ins[-1]] else: ins_batch = slice_arrays(ins, batch_ids) except TypeError: raise TypeError('TypeError while preparing batch. ' 'If using HDF5 input data, ' 'pass shuffle="batch".') batch_logs = {} batch_logs['batch'] = batch_index batch_logs['size'] = len(batch_ids) callbacks.on_batch_begin(batch_index, batch_logs) ins_batch_converted = [] for ib in ins_batch: ins_batch_converted.append(ops.convert_to_tensor(ib, dtype=K.floatx())) eager_model_inputs = [] eager_model_outputs = [] for i in range(len(model.inputs)): eager_model_inputs.append(ins_batch_converted[i]) for i in range(len(model.inputs), len(ins_batch_converted)): eager_model_outputs.append(ins_batch_converted[i]) outs, loss, loss_metrics = _process_single_batch(eager_model_inputs, eager_model_outputs, model) if not isinstance(outs, list): outs = [outs] for l, o in zip(out_labels, outs): batch_logs[l] = o # Required for Eager mode metrics_names, metrics_results = _eager_metrics_fn(model, outs, eager_model_outputs) batch_logs['loss'] = tensor_util.constant_value(K.mean(loss)) # TODO(anjalisridhar): Move this to compile to avoid duplicate code. # In graph mode we set the metric names in compile. However in # Eager mode we calculate the metrics for each batch in fit_loop. # We could calculate the metric names and functions in compile. # This would avoid setting the callback parameters separately. # We need to do this for the first iteration alone for m in metrics_names: if m not in callback_metrics: callback_metrics.append(m) callbacks.set_params({ 'batch_size': batch_size, 'epochs': epochs, 'steps': steps_per_epoch, 'samples': num_train_samples, 'verbose': verbose, 'do_validation': do_validation, 'metrics': callback_metrics or [], }) for k, v in zip(model.metrics_names, [K.mean(loss)] + loss_metrics + metrics_results): batch_logs[k] = tensor_util.constant_value(v) callbacks.on_batch_end(batch_index, batch_logs) if callback_model.stop_training: break if batch_index == len(batches) - 1: # Last batch. if do_validation: val_outs = test_loop( model, val_ins, batch_size=batch_size, verbose=0) if not isinstance(val_outs, list): val_outs = [val_outs] # Same labels assumed. for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o callbacks.on_epoch_end(epoch, epoch_logs) if callback_model.stop_training: break callbacks.on_train_end() return model.history
def batch_test_loop(model, inputs, targets, batch_size, sample_weights=None, verbose=0): """Test function for eager execution when input is given as arrays or tensors. Arguments: model: Model instance that is being evaluated in Eager mode. inputs: List of input arrays. targets: List of target arrays. batch_size: Integer batch size. sample_weights: Optional list of sample weight arrays. verbose: Verbosity mode. Returns: Scalar loss (if the model has a single output and no metrics) or list of scalars (if the model has multiple outputs and/or metrics). The attribute `model.metrics_names` will give you the display labels for the scalar outputs. """ outs = [] feed_data = inputs + targets if sample_weights: feed_data += sample_weights num_samples = training_utils.check_num_samples(feed_data, batch_size=batch_size) if verbose == 1: progbar = generic_utils.Progbar(target=num_samples) batches = generic_utils.make_batches(num_samples, batch_size) index_array = np.arange(num_samples) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] inputs_batch = slice_arrays(inputs, batch_ids) targets_batch = slice_arrays(targets, batch_ids) if sample_weights: sample_weights_batch = slice_arrays(sample_weights, batch_ids) else: sample_weights_batch = None inputs_batch = [ ops.convert_to_tensor(val, dtype=backend.floatx()) for val in inputs_batch ] targets_batch = [ ops.convert_to_tensor(val, dtype=backend.floatx()) for val in targets_batch ] if sample_weights: sample_weights_batch = [ ops.convert_to_tensor(val, dtype=backend.floatx()) if val is not None else None for val in sample_weights_batch ] loss_outs, loss, loss_metrics = _model_loss( model, inputs_batch, targets_batch, sample_weights=sample_weights_batch, training=False) metrics_results = _eager_metrics_fn(model, loss_outs, targets_batch) batch_outs = [] for _, v in zip(model.metrics_names, [backend.mean(loss)] + loss_metrics + metrics_results): batch_outs.append(tensor_util.constant_value(v)) if isinstance(batch_outs, list): if batch_index == 0: for _ in enumerate(batch_outs): outs.append(0.) for i, batch_out in enumerate(batch_outs): outs[i] += batch_out * len(batch_ids) else: if batch_index == 0: outs.append(0.) outs[0] += batch_outs * len(batch_ids) if verbose == 1: progbar.update(batch_end) for i in range(len(outs)): outs[i] /= num_samples if len(outs) == 1: return outs[0] return outs
def batch_fit_loop(model, inputs, targets, epoch_logs, index_array, out_labels, callback_model, batch_size, sample_weights=None, val_inputs=None, val_targets=None, val_sample_weights=None, callbacks=None, shuffle=True, num_train_samples=None, do_validation=False): """Fit function for eager execution when input is given as arrays or tensors. Updates the given epoch logs. Arguments: model: Instance of the `Model`. inputs: List of input arrays. targets: List of target arrays. epoch_logs: Dictionary of logs from every epoch. index_array: Index array generated from number of training samples. out_labels: Output labels generated from model metric names. callback_model: Instance of `Model` to callback. batch_size: Integer batch size or None if unknown. sample_weights: Optional list of sample weight arrays. val_inputs: Input data for validation. val_targets: Target data for validation. val_sample_weights: Sample weight data for validation. callbacks: List of callbacks to be called during training. shuffle: Whether to shuffle the data at the beginning of each epoch. num_train_samples: Integer number of training samples. do_validation: Boolean value indicating whether we should do validation. """ # TODO(psv): Create a dataset iterator instead of manually creating batches # here and in batch_test_loop, batch_predict_loop. if shuffle == 'batch': index_array = model._batch_shuffle(index_array, batch_size) elif shuffle: np.random.shuffle(index_array) batches = generic_utils.make_batches(num_train_samples, batch_size) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] inputs_batch = slice_arrays(inputs, batch_ids, contiguous=not shuffle) targets_batch = slice_arrays(targets, batch_ids, contiguous=not shuffle) if sample_weights: sample_weights_batch = slice_arrays(sample_weights, batch_ids, contiguous=not shuffle) else: sample_weights_batch = None batch_logs = {} batch_logs['batch'] = batch_index batch_logs['size'] = len(batch_ids) callbacks.on_batch_begin(batch_index, batch_logs) inputs_batch = [ ops.convert_to_tensor(val, dtype=backend.floatx()) for val in inputs_batch ] targets_batch = [ ops.convert_to_tensor(val, dtype=backend.floatx()) for val in targets_batch ] if sample_weights: sample_weights_batch = [ ops.convert_to_tensor(val, dtype=backend.floatx()) if val is not None else None for val in sample_weights_batch ] outs, loss, loss_metrics = _process_single_batch( model, inputs_batch, targets_batch, sample_weights=sample_weights_batch, training=True) if not isinstance(outs, list): outs = [outs] for l, o in zip(out_labels, outs): batch_logs[l] = o # Required for eager execution metrics_results = _eager_metrics_fn(model, outs, targets_batch) batch_logs['loss'] = tensor_util.constant_value(backend.mean(loss)) for k, v in zip(model.metrics_names, [backend.mean(loss)] + loss_metrics + metrics_results): batch_logs[k] = tensor_util.constant_value(v) callbacks.on_batch_end(batch_index, batch_logs) if callback_model.stop_training: break if batch_index == len(batches) - 1: # Last batch. if do_validation: val_outs = test_loop(model, val_inputs, val_targets, sample_weights=val_sample_weights, batch_size=batch_size, verbose=0) if not isinstance(val_outs, list): val_outs = [val_outs] # Same labels assumed. for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o
def predict_loop(model, ins, batch_size=32, verbose=0, steps=None): """Abstract method to loop over some data in batches. Arguments: model: ins: list of tensors to be fed to `f`. batch_size: integer batch size. verbose: verbosity mode. steps: Total number of steps (batches of samples) before declaring `_predict_loop` finished. Ignored with the default value of `None`. Returns: Array of predictions (if the model has a single output) or list of arrays of predictions (if the model has multiple outputs). """ K.set_learning_phase(False) num_samples = model._check_num_samples(ins, batch_size, steps, 'steps') if verbose == 1: if steps is not None: progbar = Progbar(target=steps) else: progbar = Progbar(target=num_samples) outs = [] batches = make_batches(num_samples, batch_size) index_array = np.arange(num_samples) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] if ins and isinstance(ins[-1], float): # Do not slice the training phase flag. ins_batch = slice_arrays(ins[:-1], batch_ids) + [ins[-1]] else: ins_batch = slice_arrays(ins, batch_ids) ins_batch_converted = [] for ib in ins_batch: ins_batch_converted.append( ops.convert_to_tensor(ib, dtype=K.floatx())) eager_model_inputs = [] for i in range(len(model.inputs)): eager_model_inputs.append(ins_batch_converted[i]) if len(eager_model_inputs) == 1: if model._expects_training_arg: batch_outs = model.call(eager_model_inputs[0], training=False) else: batch_outs = model.call(eager_model_inputs[0]) else: if model._expects_training_arg: batch_outs = model.call(eager_model_inputs, training=False) else: batch_outs = model.call(eager_model_inputs) if not isinstance(batch_outs, list): batch_outs = [batch_outs] if batch_index == 0: # Pre-allocate the results arrays. for batch_out in batch_outs: dims = batch_out.shape[1:].dims dims_list = [d.value for d in dims] shape = (num_samples, ) + tuple(dims_list) outs.append( np.zeros(shape, dtype=batch_out.dtype.as_numpy_dtype)) for i, batch_out in enumerate(batch_outs): outs[i][batch_start:batch_end] = batch_out if verbose == 1: progbar.update(batch_end) if len(outs) == 1: return outs[0] return outs
def test_loop(model, inputs, targets, sample_weights=None, batch_size=None, verbose=0, steps=None): """Abstract method to loop over some data in batches. Arguments: model: Model instance that is being evaluated in Eager mode. inputs: List of input arrays. targets: List of target arrays. sample_weights: Optional list of sample weight arrays. batch_size: integer batch size or `None`. verbose: verbosity mode. steps: Total number of steps (batches of samples) before declaring predictions finished. Ignored with the default value of `None`. Returns: Scalar loss (if the model has a single output and no metrics) or list of scalars (if the model has multiple outputs and/or metrics). The attribute `model.metrics_names` will give you the display labels for the scalar outputs. """ with backend.learning_phase_scope(0): feed_data = inputs + targets if sample_weights: feed_data += sample_weights num_samples = training_utils.check_num_samples(feed_data, batch_size=batch_size, steps=steps, steps_name='steps') outs = [] if verbose == 1: progbar = generic_utils.Progbar(target=num_samples) batches = generic_utils.make_batches(num_samples, batch_size) index_array = np.arange(num_samples) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] inputs_batch = slice_arrays(inputs, batch_ids) targets_batch = slice_arrays(targets, batch_ids) if sample_weights: sample_weights_batch = slice_arrays(sample_weights, batch_ids) else: sample_weights_batch = None inputs_batch = [ ops.convert_to_tensor(val, dtype=backend.floatx()) for val in inputs_batch ] targets_batch = [ ops.convert_to_tensor(val, dtype=backend.floatx()) for val in targets_batch ] if sample_weights: sample_weights_batch = [ ops.convert_to_tensor(val, dtype=backend.floatx()) if val is not None else None for val in sample_weights_batch ] loss_outs, loss, loss_metrics = _model_loss( model, inputs_batch, targets_batch, sample_weights=sample_weights_batch, training=False) metrics_results = _eager_metrics_fn(model, loss_outs, targets_batch) batch_outs = [] for _, v in zip(model.metrics_names, [backend.mean(loss)] + loss_metrics + metrics_results): batch_outs.append(tensor_util.constant_value(v)) if isinstance(batch_outs, list): if batch_index == 0: for batch_out in enumerate(batch_outs): outs.append(0.) for i, batch_out in enumerate(batch_outs): outs[i] += batch_out * len(batch_ids) else: if batch_index == 0: outs.append(0.) outs[0] += batch_outs * len(batch_ids) if verbose == 1: progbar.update(batch_end) for i in range(len(outs)): outs[i] /= num_samples if len(outs) == 1: return outs[0] return outs
def predict_loop(model, inputs, batch_size=32, verbose=0, steps=None): """Abstract method to loop over some data in batches. Arguments: model: Keras Model instance. inputs: list of tensors to be fed to `f`. batch_size: integer batch size. verbose: verbosity mode. steps: Total number of steps (batches of samples) before declaring `_predict_loop` finished. Ignored with the default value of `None`. Returns: Array of predictions (if the model has a single output) or list of arrays of predictions (if the model has multiple outputs). """ model._make_predict_function() f = model.predict_function if model.uses_learning_phase and not isinstance(K.learning_phase(), int): ins = inputs + [0] else: ins = inputs num_samples = training_utils.check_num_samples( inputs, batch_size, steps, 'steps') if verbose == 1: if steps is not None: progbar = Progbar(target=steps) else: progbar = Progbar(target=num_samples) indices_for_conversion_to_dense = [] for i in range(len(model._feed_inputs)): if (issparse is not None and issparse(inputs[i]) and not K.is_sparse(model._feed_inputs[i])): indices_for_conversion_to_dense.append(i) if steps is not None: # Step-based predictions. # Since we do not know how many samples # we will see, we cannot pre-allocate # the returned Numpy arrays. # Instead, we store one array per batch seen # and concatenate them upon returning. unconcatenated_outs = [] for step in range(steps): batch_outs = f(ins) if not isinstance(batch_outs, list): batch_outs = [batch_outs] if step == 0: for batch_out in batch_outs: unconcatenated_outs.append([]) for i, batch_out in enumerate(batch_outs): unconcatenated_outs[i].append(batch_out) if verbose == 1: progbar.update(step + 1) if len(unconcatenated_outs) == 1: return np.concatenate(unconcatenated_outs[0], axis=0) return [ np.concatenate(unconcatenated_outs[i], axis=0) for i in range(len(unconcatenated_outs)) ] else: # Sample-based predictions. outs = [] batches = make_batches(num_samples, batch_size) index_array = np.arange(num_samples) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] if ins and isinstance(ins[-1], int): # Do not slice the training phase flag. ins_batch = slice_arrays(ins[:-1], batch_ids) + [ins[-1]] else: ins_batch = slice_arrays(ins, batch_ids) for i in indices_for_conversion_to_dense: ins_batch[i] = ins_batch[i].toarray() batch_outs = f(ins_batch) if not isinstance(batch_outs, list): batch_outs = [batch_outs] if batch_index == 0: # Pre-allocate the results arrays. for batch_out in batch_outs: shape = (num_samples,) + batch_out.shape[1:] outs.append(np.zeros(shape, dtype=batch_out.dtype)) for i, batch_out in enumerate(batch_outs): outs[i][batch_start:batch_end] = batch_out if verbose == 1: progbar.update(batch_end) if len(outs) == 1: return outs[0] return outs
def predict_loop(model, inputs, batch_size=32, verbose=0, steps=None): """Abstract method to loop over some data in batches. Arguments: model: inputs: List of input arrays. batch_size: integer batch size. verbose: verbosity mode. steps: Total number of steps (batches of samples) before declaring `_predict_loop` finished. Ignored with the default value of `None`. Returns: Array of predictions (if the model has a single output) or list of arrays of predictions (if the model has multiple outputs). """ with backend.learning_phase_scope(0): num_samples = training_utils.check_num_samples(inputs, batch_size, steps, 'steps') if verbose == 1: if steps is not None: progbar = generic_utils.Progbar(target=steps) else: progbar = generic_utils.Progbar(target=num_samples) outs = [] batches = generic_utils.make_batches(num_samples, batch_size) index_array = np.arange(num_samples) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] inputs_batch = slice_arrays(inputs, batch_ids) inputs_batch = [ ops.convert_to_tensor(val, dtype=backend.floatx()) for val in inputs_batch ] if len(inputs_batch) == 1: if model._expects_training_arg: batch_outs = model.call(inputs_batch[0], training=False) else: batch_outs = model.call(inputs_batch[0]) else: if model._expects_training_arg: batch_outs = model.call(inputs_batch, training=False) else: batch_outs = model.call(inputs_batch) if not isinstance(batch_outs, list): batch_outs = [batch_outs] if batch_index == 0: # Pre-allocate the results arrays. for batch_out in batch_outs: dims = batch_out.shape[1:].dims dims_list = [d.value for d in dims] shape = (num_samples, ) + tuple(dims_list) outs.append( np.zeros(shape, dtype=batch_out.dtype.as_numpy_dtype)) for i, batch_out in enumerate(batch_outs): outs[i][batch_start:batch_end] = batch_out if verbose == 1: progbar.update(batch_end) if len(outs) == 1: return outs[0] return outs
def test_loop(model, inputs, targets, sample_weights=None, batch_size=None, verbose=0, steps=None): """Abstract method to loop over some data in batches. Arguments: model: Keras Model instance. inputs: List of input arrays. targets: List of target arrays. sample_weights: Optional list of sample weight arrays. batch_size: integer batch size or `None`. verbose: verbosity mode. steps: Total number of steps (batches of samples) before declaring predictions finished. Ignored with the default value of `None`. Returns: Scalar loss (if the model has a single output and no metrics) or list of scalars (if the model has multiple outputs and/or metrics). The attribute `model.metrics_names` will give you the display labels for the scalar outputs. """ model._make_test_function() f = model.test_function sample_weights = sample_weights or [] if model.uses_learning_phase and not isinstance(K.learning_phase(), int): ins = inputs + targets + sample_weights + [0] else: ins = inputs + targets + sample_weights if hasattr(model, 'metrics'): for m in model.metrics: if isinstance(m, Layer): m.reset_states() stateful_metric_indices = [ i for i, name in enumerate(model.metrics_names) if str(name) in model.stateful_metric_names ] else: stateful_metric_indices = [] num_samples = training_utils.check_num_samples( ins, batch_size, steps, 'steps') outs = [] if verbose == 1: if steps is not None: progbar = Progbar(target=steps) else: progbar = Progbar(target=num_samples) # To prevent a slowdown, we find beforehand the arrays that need conversion. feed = model._feed_inputs + model._feed_targets + model._feed_sample_weights indices_for_conversion_to_dense = [] for i in range(len(feed)): if issparse is not None and issparse(ins[i]) and not K.is_sparse(feed[i]): indices_for_conversion_to_dense.append(i) if steps is not None: for step in range(steps): batch_outs = f(ins) if isinstance(batch_outs, list): if step == 0: for _ in enumerate(batch_outs): outs.append(0.) for i, batch_out in enumerate(batch_outs): if i in stateful_metric_indices: outs[i] = batch_out else: outs[i] += batch_out else: if step == 0: outs.append(0.) outs[0] += batch_outs if verbose == 1: progbar.update(step + 1) for i in range(len(outs)): if i not in stateful_metric_indices: outs[i] /= steps else: batches = make_batches(num_samples, batch_size) index_array = np.arange(num_samples) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] if isinstance(ins[-1], int): # Do not slice the training phase flag. ins_batch = slice_arrays(ins[:-1], batch_ids) + [ins[-1]] else: ins_batch = slice_arrays(ins, batch_ids) for i in indices_for_conversion_to_dense: ins_batch[i] = ins_batch[i].toarray() batch_outs = f(ins_batch) if isinstance(batch_outs, list): if batch_index == 0: for batch_out in enumerate(batch_outs): outs.append(0.) for i, batch_out in enumerate(batch_outs): if i in stateful_metric_indices: outs[i] = batch_out else: outs[i] += batch_out * len(batch_ids) else: if batch_index == 0: outs.append(0.) outs[0] += batch_outs * len(batch_ids) if verbose == 1: progbar.update(batch_end) for i in range(len(outs)): if i not in stateful_metric_indices: outs[i] /= num_samples if len(outs) == 1: return outs[0] return outs
def test_loop(model, inputs, targets, sample_weights=None, batch_size=None, verbose=0, steps=None): """Abstract method to loop over some data in batches. Arguments: model: Model instance that is being evaluated in Eager mode. inputs: List of input arrays. targets: List of target arrays. sample_weights: Optional list of sample weight arrays. batch_size: integer batch size or `None`. verbose: verbosity mode. steps: Total number of steps (batches of samples) before declaring predictions finished. Ignored with the default value of `None`. Returns: Scalar loss (if the model has a single output and no metrics) or list of scalars (if the model has multiple outputs and/or metrics). The attribute `model.metrics_names` will give you the display labels for the scalar outputs. """ with backend.learning_phase_scope(0): feed_data = inputs + targets if sample_weights: feed_data += sample_weights num_samples = training_utils.check_num_samples( feed_data, batch_size=batch_size, steps=steps, steps_name='steps') outs = [] if verbose == 1: progbar = generic_utils.Progbar(target=num_samples) batches = generic_utils.make_batches(num_samples, batch_size) index_array = np.arange(num_samples) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] inputs_batch = slice_arrays(inputs, batch_ids) targets_batch = slice_arrays(targets, batch_ids) if sample_weights: sample_weights_batch = slice_arrays(sample_weights, batch_ids) else: sample_weights_batch = None inputs_batch = [ ops.convert_to_tensor(val, dtype=backend.floatx()) for val in inputs_batch] targets_batch = [ ops.convert_to_tensor(val, dtype=backend.floatx()) for val in targets_batch] if sample_weights: sample_weights_batch = [ ops.convert_to_tensor(val, dtype=backend.floatx()) if val is not None else None for val in sample_weights_batch] loss_outs, loss, loss_metrics = _model_loss( model, inputs_batch, targets_batch, sample_weights=sample_weights_batch, training=False) metrics_results = _eager_metrics_fn(model, loss_outs, targets_batch) batch_outs = [] for _, v in zip(model.metrics_names, [backend.mean(loss)] + loss_metrics + metrics_results): batch_outs.append(tensor_util.constant_value(v)) if isinstance(batch_outs, list): if batch_index == 0: for batch_out in enumerate(batch_outs): outs.append(0.) for i, batch_out in enumerate(batch_outs): outs[i] += batch_out * len(batch_ids) else: if batch_index == 0: outs.append(0.) outs[0] += batch_outs * len(batch_ids) if verbose == 1: progbar.update(batch_end) for i in range(len(outs)): outs[i] /= num_samples if len(outs) == 1: return outs[0] return outs
def fit_loop(model, inputs, targets, sample_weights=None, batch_size=None, epochs=100, verbose=1, callbacks=None, val_inputs=None, val_targets=None, val_sample_weights=None, shuffle=True, callback_metrics=None, initial_epoch=0, steps_per_epoch=None, validation_steps=None): """Abstract fit function for arrays of data. Arguments: model: Keras Model instance. inputs: List of input arrays. targets: List of target arrays. sample_weights: Optional list of sample weight arrays. batch_size: Integer batch size or None if unknown. epochs: Number of times to iterate over the data verbose: Verbosity mode, 0, 1 or 2 callbacks: List of callbacks to be called during training val_inputs: List of input arrays. val_targets: List of target arrays. val_sample_weights: Optional list of sample weight arrays. shuffle: Whether to shuffle the data at the beginning of each epoch callback_metrics: List of strings, the display names of the metrics passed to the callbacks. They should be the concatenation of list the display names of the outputs of `f` and the list of display names of the outputs of `f_val`. initial_epoch: Epoch at which to start training (useful for resuming a previous training run) steps_per_epoch: Total number of steps (batches of samples) before declaring one epoch finished and starting the next epoch. Ignored with the default value of `None`. validation_steps: Number of steps to run validation for (only if doing validation from data tensors). Ignored with the default value of `None`. Returns: `History` object. Raises: ValueError: in case of invalid arguments. """ model._make_train_function() f = model.train_function sample_weights = sample_weights or [] val_sample_weights = val_sample_weights or [] if model.uses_learning_phase and not isinstance(K.learning_phase(), int): ins = inputs + targets + sample_weights + [1] if val_inputs: val_ins = val_inputs + val_targets + val_sample_weights + [1] else: ins = inputs + targets + sample_weights if val_inputs: val_ins = val_inputs + val_targets + val_sample_weights if not val_inputs: val_ins = [] do_validation = False if val_inputs: do_validation = True if verbose and inputs and hasattr(inputs[0], 'shape') and hasattr( val_inputs[0], 'shape'): print('Train on %d samples, validate on %d samples' % (inputs[0].shape[0], val_inputs[0].shape[0])) if validation_steps: do_validation = True if steps_per_epoch is None: raise ValueError('Can only use `validation_steps` ' 'when doing step-wise ' 'training, i.e. `steps_per_epoch` ' 'must be set.') out_labels = model.metrics_names if do_validation: callback_metrics = copy.copy(out_labels) + [ 'val_' + n for n in out_labels ] else: callback_metrics = copy.copy(out_labels) num_train_samples = training_utils.check_num_samples( ins, batch_size, steps_per_epoch, 'steps_per_epoch') if num_train_samples is not None: index_array = np.arange(num_train_samples) model.history = cbks.History() all_callbacks = [cbks.BaseLogger( stateful_metrics=model.stateful_metric_names)] if verbose: if steps_per_epoch is not None: count_mode = 'steps' else: count_mode = 'samples' all_callbacks.append( cbks.ProgbarLogger( count_mode, stateful_metrics=model.stateful_metric_names)) all_callbacks += (callbacks or []) + [model.history] callbacks = cbks.CallbackList(all_callbacks) out_labels = out_labels or [] # it's possible to callback a different model than self # (used by Sequential models) if hasattr(model, 'callback_model') and model.callback_model: callback_model = model.callback_model else: callback_model = model callbacks.set_model(callback_model) callbacks.set_params({ 'batch_size': batch_size, 'epochs': epochs, 'steps': steps_per_epoch, 'samples': num_train_samples, 'verbose': verbose, 'do_validation': do_validation, 'metrics': callback_metrics or [], }) callbacks.on_train_begin() callback_model.stop_training = False for cbk in callbacks: cbk.validation_data = val_ins # To prevent a slowdown, we find beforehand the arrays that need conversion. feed = model._feed_inputs + model._feed_targets + model._feed_sample_weights indices_for_conversion_to_dense = [] for i in range(len(feed)): if issparse is not None and issparse(ins[i]) and not K.is_sparse(feed[i]): indices_for_conversion_to_dense.append(i) for epoch in range(initial_epoch, epochs): # Reset stateful metrics for m in model.metrics: if isinstance(m, Layer): m.reset_states() # Update callbacks callbacks.on_epoch_begin(epoch) epoch_logs = {} if steps_per_epoch is not None: for step_index in range(steps_per_epoch): batch_logs = {} batch_logs['batch'] = step_index batch_logs['size'] = 1 callbacks.on_batch_begin(step_index, batch_logs) outs = f(ins) if not isinstance(outs, list): outs = [outs] for l, o in zip(out_labels, outs): batch_logs[l] = o callbacks.on_batch_end(step_index, batch_logs) if callback_model.stop_training: break if do_validation: val_outs = test_loop( model, val_inputs, val_targets, sample_weights=val_sample_weights, batch_size=batch_size, steps=validation_steps, verbose=0) if not isinstance(val_outs, list): val_outs = [val_outs] # Same labels assumed. for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o else: if shuffle == 'batch': index_array = training_utils.batch_shuffle(index_array, batch_size) elif shuffle: np.random.shuffle(index_array) batches = make_batches(num_train_samples, batch_size) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] try: if isinstance(ins[-1], int): # Do not slice the training phase flag. ins_batch = slice_arrays(ins[:-1], batch_ids) + [ins[-1]] else: ins_batch = slice_arrays(ins, batch_ids) except TypeError: raise TypeError('TypeError while preparing batch. ' 'If using HDF5 input data, ' 'pass shuffle="batch".') batch_logs = {} batch_logs['batch'] = batch_index batch_logs['size'] = len(batch_ids) callbacks.on_batch_begin(batch_index, batch_logs) for i in indices_for_conversion_to_dense: ins_batch[i] = ins_batch[i].toarray() outs = f(ins_batch) if not isinstance(outs, list): outs = [outs] for l, o in zip(out_labels, outs): batch_logs[l] = o callbacks.on_batch_end(batch_index, batch_logs) if callback_model.stop_training: break if batch_index == len(batches) - 1: # Last batch. if do_validation: val_outs = test_loop( model, val_inputs, val_targets, sample_weights=val_sample_weights, batch_size=batch_size, verbose=0) if not isinstance(val_outs, list): val_outs = [val_outs] # Same labels assumed. for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o callbacks.on_epoch_end(epoch, epoch_logs) if callback_model.stop_training: break callbacks.on_train_end() return model.history
def predict_loop(model, inputs, batch_size=32, verbose=0, steps=None): """Abstract method to loop over some data in batches. Arguments: model: Keras Model instance. inputs: list of tensors to be fed to `f`. batch_size: integer batch size. verbose: verbosity mode. steps: Total number of steps (batches of samples) before declaring `_predict_loop` finished. Ignored with the default value of `None`. Returns: Array of predictions (if the model has a single output) or list of arrays of predictions (if the model has multiple outputs). """ model._make_predict_function() f = model.predict_function if model.uses_learning_phase and not isinstance(K.learning_phase(), int): ins = inputs + [0] else: ins = inputs num_samples = training_utils.check_num_samples(inputs, batch_size, steps, 'steps') if verbose == 1: if steps is not None: progbar = Progbar(target=steps) else: progbar = Progbar(target=num_samples) indices_for_conversion_to_dense = [] for i in range(len(model._feed_inputs)): if (issparse is not None and issparse(inputs[i]) and not K.is_sparse(model._feed_inputs[i])): indices_for_conversion_to_dense.append(i) if steps is not None: # Step-based predictions. # Since we do not know how many samples # we will see, we cannot pre-allocate # the returned Numpy arrays. # Instead, we store one array per batch seen # and concatenate them upon returning. unconcatenated_outs = [] for step in range(steps): batch_outs = f(ins) if not isinstance(batch_outs, list): batch_outs = [batch_outs] if step == 0: for batch_out in batch_outs: unconcatenated_outs.append([]) for i, batch_out in enumerate(batch_outs): unconcatenated_outs[i].append(batch_out) if verbose == 1: progbar.update(step + 1) if len(unconcatenated_outs) == 1: return np.concatenate(unconcatenated_outs[0], axis=0) return [ np.concatenate(unconcatenated_outs[i], axis=0) for i in range(len(unconcatenated_outs)) ] else: # Sample-based predictions. outs = [] batches = make_batches(num_samples, batch_size) index_array = np.arange(num_samples) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] if ins and isinstance(ins[-1], int): # Do not slice the training phase flag. ins_batch = slice_arrays(ins[:-1], batch_ids) + [ins[-1]] else: ins_batch = slice_arrays(ins, batch_ids) for i in indices_for_conversion_to_dense: ins_batch[i] = ins_batch[i].toarray() batch_outs = f(ins_batch) if not isinstance(batch_outs, list): batch_outs = [batch_outs] if batch_index == 0: # Pre-allocate the results arrays. for batch_out in batch_outs: shape = (num_samples, ) + batch_out.shape[1:] outs.append(np.zeros(shape, dtype=batch_out.dtype)) for i, batch_out in enumerate(batch_outs): outs[i][batch_start:batch_end] = batch_out if verbose == 1: progbar.update(batch_end) if len(outs) == 1: return outs[0] return outs
def batch_fit_loop(model, inputs, targets, epoch_logs, index_array, out_labels, callback_model, batch_size, sample_weights=None, val_inputs=None, val_targets=None, val_sample_weights=None, callbacks=None, shuffle=True, num_train_samples=None, do_validation=False): """Fit function for eager execution when input is given as arrays or tensors. Updates the given epoch logs. Arguments: model: Instance of the `Model`. inputs: List of input arrays. targets: List of target arrays. epoch_logs: Dictionary of logs from every epoch. index_array: Index array generated from number of training samples. out_labels: Output labels generated from model metric names. callback_model: Instance of `Model` to callback. batch_size: Integer batch size or None if unknown. sample_weights: Optional list of sample weight arrays. val_inputs: Input data for validation. val_targets: Target data for validation. val_sample_weights: Sample weight data for validation. callbacks: List of callbacks to be called during training. shuffle: Whether to shuffle the data at the beginning of each epoch. num_train_samples: Integer number of training samples. do_validation: Boolean value indicating whether we should do validation. """ # TODO(psv): Create a dataset iterator instead of manually creating batches # here and in batch_test_loop, batch_predict_loop. if shuffle == 'batch': index_array = model._batch_shuffle(index_array, batch_size) elif shuffle: np.random.shuffle(index_array) batches = generic_utils.make_batches(num_train_samples, batch_size) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] inputs_batch = slice_arrays(inputs, batch_ids, contiguous=not shuffle) targets_batch = slice_arrays(targets, batch_ids, contiguous=not shuffle) if sample_weights: sample_weights_batch = slice_arrays( sample_weights, batch_ids, contiguous=not shuffle) else: sample_weights_batch = None batch_logs = {} batch_logs['batch'] = batch_index batch_logs['size'] = len(batch_ids) callbacks.on_batch_begin(batch_index, batch_logs) inputs_batch = [ ops.convert_to_tensor(val, dtype=backend.floatx()) for val in inputs_batch ] targets_batch = [ ops.convert_to_tensor(val, dtype=backend.floatx()) for val in targets_batch ] if sample_weights: sample_weights_batch = [ ops.convert_to_tensor(val, dtype=backend.floatx()) if val is not None else None for val in sample_weights_batch ] outs, loss, loss_metrics = _process_single_batch( model, inputs_batch, targets_batch, sample_weights=sample_weights_batch, training=True) if not isinstance(outs, list): outs = [outs] for l, o in zip(out_labels, outs): batch_logs[l] = o # Required for eager execution metrics_results = _eager_metrics_fn(model, outs, targets_batch) batch_logs['loss'] = tensor_util.constant_value(backend.mean(loss)) for k, v in zip(model.metrics_names, [backend.mean(loss)] + loss_metrics + metrics_results): batch_logs[k] = tensor_util.constant_value(v) callbacks.on_batch_end(batch_index, batch_logs) if callback_model.stop_training: break if batch_index == len(batches) - 1: # Last batch. if do_validation: val_outs = test_loop( model, val_inputs, val_targets, sample_weights=val_sample_weights, batch_size=batch_size, verbose=0) if not isinstance(val_outs, list): val_outs = [val_outs] # Same labels assumed. for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o
def fit_loop(model, inputs, targets, sample_weights=None, batch_size=None, epochs=100, verbose=1, callbacks=None, val_inputs=None, val_targets=None, val_sample_weights=None, shuffle=True, callback_metrics=None, initial_epoch=0, steps_per_epoch=None, validation_steps=None): """Abstract fit function for arrays of data. Arguments: model: Keras Model instance. inputs: List of input arrays. targets: List of target arrays. sample_weights: Optional list of sample weight arrays. batch_size: Integer batch size or None if unknown. epochs: Number of times to iterate over the data verbose: Verbosity mode, 0, 1 or 2 callbacks: List of callbacks to be called during training val_inputs: List of input arrays. val_targets: List of target arrays. val_sample_weights: Optional list of sample weight arrays. shuffle: Whether to shuffle the data at the beginning of each epoch callback_metrics: List of strings, the display names of the metrics passed to the callbacks. They should be the concatenation of list the display names of the outputs of `f` and the list of display names of the outputs of `f_val`. initial_epoch: Epoch at which to start training (useful for resuming a previous training run) steps_per_epoch: Total number of steps (batches of samples) before declaring one epoch finished and starting the next epoch. Ignored with the default value of `None`. validation_steps: Number of steps to run validation for (only if doing validation from data tensors). Ignored with the default value of `None`. Returns: `History` object. Raises: ValueError: in case of invalid arguments. """ model._make_train_function() f = model.train_function sample_weights = sample_weights or [] val_sample_weights = val_sample_weights or [] if model.uses_learning_phase and not isinstance(K.learning_phase(), int): ins = inputs + targets + sample_weights + [1] if val_inputs: val_ins = val_inputs + val_targets + val_sample_weights + [1] else: ins = inputs + targets + sample_weights if val_inputs: val_ins = val_inputs + val_targets + val_sample_weights if not val_inputs: val_ins = [] do_validation = False if val_inputs: do_validation = True if verbose and inputs and hasattr(inputs[0], 'shape') and hasattr( val_inputs[0], 'shape'): print('Train on %d samples, validate on %d samples' % (inputs[0].shape[0], val_inputs[0].shape[0])) if validation_steps: do_validation = True if steps_per_epoch is None: raise ValueError('Can only use `validation_steps` ' 'when doing step-wise ' 'training, i.e. `steps_per_epoch` ' 'must be set.') out_labels = model.metrics_names if do_validation: callback_metrics = copy.copy(out_labels) + [ 'val_' + n for n in out_labels ] else: callback_metrics = copy.copy(out_labels) num_train_samples = training_utils.check_num_samples( ins, batch_size, steps_per_epoch, 'steps_per_epoch') if num_train_samples is not None: index_array = np.arange(num_train_samples) model.history = cbks.History() all_callbacks = [ cbks.BaseLogger(stateful_metrics=model.stateful_metric_names) ] if verbose: if steps_per_epoch is not None: count_mode = 'steps' else: count_mode = 'samples' all_callbacks.append( cbks.ProgbarLogger(count_mode, stateful_metrics=model.stateful_metric_names)) all_callbacks += (callbacks or []) + [model.history] callbacks = cbks.CallbackList(all_callbacks) out_labels = out_labels or [] # it's possible to callback a different model than self # (used by Sequential models) if hasattr(model, 'callback_model') and model.callback_model: callback_model = model.callback_model else: callback_model = model callbacks.set_model(callback_model) callbacks.set_params({ 'batch_size': batch_size, 'epochs': epochs, 'steps': steps_per_epoch, 'samples': num_train_samples, 'verbose': verbose, 'do_validation': do_validation, 'metrics': callback_metrics or [], }) callbacks.on_train_begin() callback_model.stop_training = False for cbk in callbacks: cbk.validation_data = val_ins # To prevent a slowdown, we find beforehand the arrays that need conversion. feed = model._feed_inputs + model._feed_targets + model._feed_sample_weights indices_for_conversion_to_dense = [] for i in range(len(feed)): if issparse is not None and issparse( ins[i]) and not K.is_sparse(feed[i]): indices_for_conversion_to_dense.append(i) for epoch in range(initial_epoch, epochs): # Reset stateful metrics for m in model.metrics: if isinstance(m, Layer): m.reset_states() # Update callbacks callbacks.on_epoch_begin(epoch) epoch_logs = {} if steps_per_epoch is not None: for step_index in range(steps_per_epoch): batch_logs = {} batch_logs['batch'] = step_index batch_logs['size'] = 1 callbacks.on_batch_begin(step_index, batch_logs) try: outs = f(ins) except errors.OutOfRangeError: logging.warning( 'Your dataset iterator ran out of data; ' 'interrupting training. Make sure that your dataset ' 'can generate at least `steps_per_epoch * epochs` ' 'batches (in this case, %d batches).' % steps_per_epoch * epochs) break if not isinstance(outs, list): outs = [outs] for l, o in zip(out_labels, outs): batch_logs[l] = o callbacks.on_batch_end(step_index, batch_logs) if callback_model.stop_training: break if do_validation: val_outs = test_loop(model, val_inputs, val_targets, sample_weights=val_sample_weights, batch_size=batch_size, steps=validation_steps, verbose=0) if not isinstance(val_outs, list): val_outs = [val_outs] # Same labels assumed. for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o else: if shuffle == 'batch': index_array = training_utils.batch_shuffle( index_array, batch_size) elif shuffle: np.random.shuffle(index_array) batches = make_batches(num_train_samples, batch_size) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] try: if isinstance(ins[-1], int): # Do not slice the training phase flag. ins_batch = slice_arrays(ins[:-1], batch_ids) + [ins[-1]] else: ins_batch = slice_arrays(ins, batch_ids) except TypeError: raise TypeError('TypeError while preparing batch. ' 'If using HDF5 input data, ' 'pass shuffle="batch".') batch_logs = {} batch_logs['batch'] = batch_index batch_logs['size'] = len(batch_ids) callbacks.on_batch_begin(batch_index, batch_logs) for i in indices_for_conversion_to_dense: ins_batch[i] = ins_batch[i].toarray() outs = f(ins_batch) if not isinstance(outs, list): outs = [outs] for l, o in zip(out_labels, outs): batch_logs[l] = o callbacks.on_batch_end(batch_index, batch_logs) if callback_model.stop_training: break if batch_index == len(batches) - 1: # Last batch. if do_validation: val_outs = test_loop(model, val_inputs, val_targets, sample_weights=val_sample_weights, batch_size=batch_size, verbose=0) if not isinstance(val_outs, list): val_outs = [val_outs] # Same labels assumed. for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o callbacks.on_epoch_end(epoch, epoch_logs) if callback_model.stop_training: break callbacks.on_train_end() return model.history
def fit_loop(model, ins, out_labels=None, batch_size=None, epochs=100, verbose=1, callbacks=None, val_ins=None, shuffle=True, callback_metrics=None, initial_epoch=0, steps_per_epoch=None, validation_steps=None): """Abstract fit function for `f(ins)`. Assume that f returns a list, labeled by out_labels. Arguments: model: Instance of the model that is being executed in Eager mode. ins: List of tensors to be fed to `f` out_labels: List of strings, display names of the outputs of `f` batch_size: Integer batch size or None if unknown. epochs: Number of times to iterate over the data verbose: Verbosity mode, 0, 1 or 2 callbacks: List of callbacks to be called during training val_ins: List of tensors to be fed to `val_f` shuffle: Whether to shuffle the data at the beginning of each epoch callback_metrics: List of strings, the display names of the metrics passed to the callbacks. They should be the concatenation of list the display names of the outputs of `f` and the list of display names of the outputs of `f_val`. initial_epoch: Epoch at which to start training (useful for resuming a previous training run) steps_per_epoch: Total number of steps (batches of samples) before declaring one epoch finished and starting the next epoch. Ignored with the default value of `None`. validation_steps: Number of steps to run validation for (only if doing validation from data tensors). Ignored with default value of `None`. Returns: `History` object. Raises: ValueError: In case of invalid argument values. """ # Required for Eager mode K.set_learning_phase(True) do_validation = False if val_ins: do_validation = True if (verbose and ins and hasattr(ins[0], 'shape') and hasattr(val_ins[0], 'shape')): print('Train on %d samples, validate on %d samples' % (ins[0].shape[0], val_ins[0].shape[0])) if validation_steps: if steps_per_epoch is None: raise ValueError( 'Can only use `validation_steps` when doing step-wise ' 'training, i.e. `steps_per_epoch` must be set.') do_validation = True num_train_samples = model._check_num_samples(ins, batch_size, steps_per_epoch, 'steps_per_epoch') if num_train_samples is not None: index_array = np.arange(num_train_samples) model.history = cbks.History() callbacks = [cbks.BaseLogger()] + (callbacks or []) + [model.history] if verbose: if steps_per_epoch is not None: count_mode = 'steps' else: count_mode = 'samples' callbacks += [cbks.ProgbarLogger(count_mode)] callbacks = cbks.CallbackList(callbacks) out_labels = out_labels or [] # it's possible to callback a different model than self # (used by Sequential models) if hasattr(model, 'callback_model') and model.callback_model: callback_model = model.callback_model else: callback_model = model callbacks.set_model(callback_model) callbacks.set_params({ 'batch_size': batch_size, 'epochs': epochs, 'steps': steps_per_epoch, 'samples': num_train_samples, 'verbose': verbose, 'do_validation': do_validation, 'metrics': callback_metrics or [], }) callbacks.on_train_begin() callback_model.stop_training = False for cbk in callbacks: cbk.validation_data = val_ins for epoch in range(initial_epoch, epochs): callbacks.on_epoch_begin(epoch) epoch_logs = {} if shuffle == 'batch': index_array = model._batch_shuffle(index_array, batch_size) elif shuffle: np.random.shuffle(index_array) batches = make_batches(num_train_samples, batch_size) for batch_index, (batch_start, batch_end) in enumerate(batches): batch_ids = index_array[batch_start:batch_end] try: if isinstance(ins[-1], float): # Do not slice the training phase flag. ins_batch = slice_arrays(ins[:-1], batch_ids) + [ins[-1]] else: ins_batch = slice_arrays(ins, batch_ids) except TypeError: raise TypeError('TypeError while preparing batch. ' 'If using HDF5 input data, ' 'pass shuffle="batch".') batch_logs = {} batch_logs['batch'] = batch_index batch_logs['size'] = len(batch_ids) callbacks.on_batch_begin(batch_index, batch_logs) ins_batch_converted = [] for ib in ins_batch: ins_batch_converted.append( ops.convert_to_tensor(ib, dtype=K.floatx())) eager_model_inputs = [] eager_model_outputs = [] for i in range(len(model.inputs)): eager_model_inputs.append(ins_batch_converted[i]) for i in range(len(model.inputs), len(ins_batch_converted)): eager_model_outputs.append(ins_batch_converted[i]) outs, loss, loss_metrics = _process_single_batch( eager_model_inputs, eager_model_outputs, model, training=True) if not isinstance(outs, list): outs = [outs] for l, o in zip(out_labels, outs): batch_logs[l] = o # Required for Eager mode metrics_names, metrics_results = _eager_metrics_fn( model, outs, eager_model_outputs) batch_logs['loss'] = tensor_util.constant_value(K.mean(loss)) # TODO(anjalisridhar): Move this to compile to avoid duplicate code. # In graph mode we set the metric names in compile. However in # Eager mode we calculate the metrics for each batch in fit_loop. # We could calculate the metric names and functions in compile. # This would avoid setting the callback parameters separately. # We need to do this for the first iteration alone for m in metrics_names: if m not in callback_metrics: callback_metrics.append(m) callbacks.set_params({ 'batch_size': batch_size, 'epochs': epochs, 'steps': steps_per_epoch, 'samples': num_train_samples, 'verbose': verbose, 'do_validation': do_validation, 'metrics': callback_metrics or [], }) for k, v in zip(model.metrics_names, [K.mean(loss)] + loss_metrics + metrics_results): batch_logs[k] = tensor_util.constant_value(v) callbacks.on_batch_end(batch_index, batch_logs) if callback_model.stop_training: break if batch_index == len(batches) - 1: # Last batch. if do_validation: val_outs = test_loop(model, val_ins, batch_size=batch_size, verbose=0) if not isinstance(val_outs, list): val_outs = [val_outs] # Same labels assumed. for l, o in zip(out_labels, val_outs): epoch_logs['val_' + l] = o callbacks.on_epoch_end(epoch, epoch_logs) if callback_model.stop_training: break callbacks.on_train_end() return model.history