示例#1
0
    def _make_train_function(self):
        if ops.executing_eagerly_outside_functions():
            return super(WideDeepModel, self)._make_train_function()

        # Only needed for graph mode and model_to_estimator.
        has_recompiled = self._recompile_weights_loss_and_weighted_metrics()
        self._check_trainable_weights_consistency()
        # If we have re-compiled the loss/weighted metric sub-graphs then create
        # train function even if one exists already. This is because
        # `_feed_sample_weights` list has been updated on re-compile.
        if getattr(self, 'train_function', None) is None or has_recompiled:
            # Restore the compiled trainable state.
            current_trainable_state = self._get_trainable_state()
            self._set_trainable_state(self._compiled_trainable_state)

            inputs = (self._feed_inputs + self._feed_targets +
                      self._feed_sample_weights)
            if not isinstance(K.symbolic_learning_phase(), int):
                inputs += [K.symbolic_learning_phase()]

            if isinstance(self.optimizer, (list, tuple)):
                linear_optimizer = self.optimizer[0]
                dnn_optimizer = self.optimizer[1]
            else:
                linear_optimizer = self.optimizer
                dnn_optimizer = self.optimizer

            with K.get_graph().as_default():
                with K.name_scope('training'):
                    # Training updates
                    updates = []
                    linear_updates = linear_optimizer.get_updates(
                        params=self.linear_model.trainable_weights,  # pylint: disable=protected-access
                        loss=self.total_loss)
                    updates += linear_updates
                    dnn_updates = dnn_optimizer.get_updates(
                        params=self.dnn_model.trainable_weights,  # pylint: disable=protected-access
                        loss=self.total_loss)
                    updates += dnn_updates
                    # Unconditional updates
                    updates += self.get_updates_for(None)
                    # Conditional updates relevant to this model
                    updates += self.get_updates_for(self.inputs)

                metrics = self._get_training_eval_metrics()
                metrics_tensors = [
                    m._call_result for m in metrics
                    if hasattr(m, '_call_result')  # pylint: disable=protected-access
                ]

            with K.name_scope('training'):
                # Gets loss and metrics. Updates weights at each call.
                fn = K.function(inputs, [self.total_loss] + metrics_tensors,
                                updates=updates,
                                name='train_function',
                                **self._function_kwargs)
                setattr(self, 'train_function', fn)

            # Restore the current trainable state
            self._set_trainable_state(current_trainable_state)
示例#2
0
  def _make_train_function(self):
    # TODO(tanzheny): This is a direct copy from super to make it work
    # refactor it so that common logic can be shared.
    has_recompiled = self._recompile_weights_loss_and_weighted_metrics()
    self._check_trainable_weights_consistency()
    # If we have re-compiled the loss/weighted metric sub-graphs then create
    # train function even if one exists already. This is because
    # `_feed_sample_weights` list has been updated on re-compile.
    if getattr(self, 'train_function', None) is None or has_recompiled:
      # Restore the compiled trainable state.
      current_trainable_state = self._get_trainable_state()
      self._set_trainable_state(self._compiled_trainable_state)

      inputs = (
          self._feed_inputs + self._feed_targets + self._feed_sample_weights)
      if not isinstance(K.symbolic_learning_phase(), int):
        inputs += [K.symbolic_learning_phase()]

      linear_optimizer, dnn_optimizer = self._get_optimizers()
      with K.get_graph().as_default():
        with K.name_scope('training'):
          # Training updates
          updates = []
          linear_updates = linear_optimizer.get_updates(
              params=self.linear_model.trainable_weights,  # pylint: disable=protected-access
              loss=self.total_loss)
          updates += linear_updates
          dnn_updates = dnn_optimizer.get_updates(
              params=self.dnn_model.trainable_weights,  # pylint: disable=protected-access
              loss=self.total_loss)
          updates += dnn_updates
          # Unconditional updates
          updates += self.get_updates_for(None)
          # Conditional updates relevant to this model
          updates += self.get_updates_for(self.inputs)

        metrics = self._get_training_eval_metrics()
        metrics_tensors = [
            m._call_result for m in metrics if hasattr(m, '_call_result')  # pylint: disable=protected-access
        ]

      with K.name_scope('training'):
        # Gets loss and metrics. Updates weights at each call.
        fn = K.function(
            inputs, [self.total_loss] + metrics_tensors,
            updates=updates,
            name='train_function',
            **self._function_kwargs)
        setattr(self, 'train_function', fn)

      # Restore the current trainable state
      self._set_trainable_state(current_trainable_state)
示例#3
0
 def __init__(self, model):
     # setup a function to predict the digit with dropout (default is to predict without dropout)
     self.f = K.function(
         [model.layers[0].input,
          K.symbolic_learning_phase()],
         [model.output],
     )
def _prepare_feed_values(model, inputs, targets, sample_weights, mode):
    """Prepare feed values to the model execution function.

  Arguments:
    model: Model to prepare feed values for.
    inputs: List or dict of model inputs.
    targets: Optional list of model targets.
    sample_weights: Optional list of sample weight arrays.
    mode: One of ModeKeys.TRAIN/ModeKeys.TEST/ModeKeys.PREDICT.

  Returns:
    Feed values for the model in the given mode.
  """
    strategy = model._distribution_strategy
    inputs, targets, sample_weights = _get_input_from_iterator(inputs, model)
    inputs = flatten_perdevice_values(strategy, inputs)
    targets = flatten_perdevice_values(strategy, targets)
    if mode == ModeKeys.PREDICT:
        sample_weights = []
        targets = []
    else:
        sample_weights = [
            None
            for _ in range(len(model.outputs) * strategy.num_replicas_in_sync)
        ]
    ins = inputs + targets + sample_weights
    if mode == ModeKeys.TRAIN and not isinstance(K.symbolic_learning_phase(),
                                                 int):
        ins += [True]
    return ins
def _prepare_feed_values(model, inputs, targets, sample_weights, mode):
  """Prepare feed values to the model execution function.

  Arguments:
    model: Model to prepare feed values for.
    inputs: List or dict of model inputs.
    targets: Optional list of model targets.
    sample_weights: Optional list of sample weight arrays.
    mode: One of 'train'/'test'/'predict'.

  Returns:
    Feed values for the model in the given mode.
  """
  strategy = model._distribution_strategy
  inputs, targets, sample_weights = _get_input_from_iterator(inputs, model)
  inputs = distributed_training_utils.flatten_perdevice_values(strategy, inputs)
  targets = distributed_training_utils.flatten_perdevice_values(
      strategy, targets)
  if mode == 'predict':
    sample_weights = []
    targets = []
  else:
    sample_weights = [
        None for _ in range(len(model.outputs) * strategy.num_replicas_in_sync)
    ]
  ins = inputs + targets + sample_weights
  if mode == 'train' and not isinstance(K.symbolic_learning_phase(), int):
    ins += [True]
  return ins
示例#6
0
def _prepare_feed_values(model, inputs, targets, sample_weights, mode):
  """Prepare feed values to the model execution function.

  Arguments:
    model: Model to prepare feed values for.
    inputs: List or dict of model inputs.
    targets: Optional list of model targets.
    sample_weights: Optional list of sample weight arrays.
    mode: One of ModeKeys.TRAIN/ModeKeys.TEST/ModeKeys.PREDICT.

  Returns:
    Feed values for the model in the given mode.
  """
  strategy = model._distribution_strategy
  inputs, targets, sample_weights = _get_input_from_iterator(inputs, model)
  inputs = flatten_perdevice_values(strategy, inputs)
  targets = flatten_perdevice_values(strategy, targets)
  # Expand 1-dimensional inputs.
  # TODO(b/124535720): Remove once this standarize data logic is shared with
  # main flow.
  inputs, targets = nest.map_structure(training_utils.standardize_single_array,
                                       (inputs, targets))
  if mode == ModeKeys.PREDICT:
    sample_weights = []
    targets = []
  else:
    sample_weights = [
        None for _ in range(len(model.outputs) * strategy.num_replicas_in_sync)
    ]
  ins = inputs + targets + sample_weights
  if mode == ModeKeys.TRAIN and not isinstance(K.symbolic_learning_phase(),
                                               int):
    ins += [True]
  return ins
示例#7
0
def _update_sample_weight_mode(model, mode, inputs):
    """Updates the sample_weight_mode of a given model."""
    # Add a quick return to prevent us from calling model._feed_targets that
    # accesses certain model properties that may not be set in the `PREDICT` mode.
    if mode == ModeKeys.PREDICT:
        return

    sample_weights = None
    # `inputs` is the model's inputs + targets + sample_weights +
    # learning phase placeholder if specified. To update the sample_weight_mode
    # we need to determine if the user has passed sample weights as part of the
    # input.
    if not callable(inputs):
        sample_weights = inputs[len(model._feed_inputs) +
                                len(model._feed_targets):]
        has_learning_phase_pl = (
            mode == ModeKeys.TRAIN
            and not isinstance(K.symbolic_learning_phase(), int))
        if has_learning_phase_pl:
            sample_weights = sample_weights[:-1]
        model._update_sample_weight_modes(sample_weights=sample_weights)

    # Call the DistributionStrategy specific function to update the
    # sample_weight_mode on the model.
    if model._distribution_strategy:
        distributed_training_utils_v1._update_sample_weight_modes(
            model, mode, sample_weights)
def _prepare_feed_values(model, inputs, targets, sample_weights, mode):
  """Prepare feed values to the model execution function.

  Arguments:
    model: Model to prepare feed values for.
    inputs: List or dict of model inputs.
    targets: Optional list of model targets.
    sample_weights: Optional list of sample weight arrays.
    mode: One of ModeKeys.TRAIN/ModeKeys.TEST/ModeKeys.PREDICT.

  Returns:
    Feed values for the model in the given mode.
  """
  strategy = model._distribution_strategy
  inputs, targets, sample_weights = _get_input_from_iterator(inputs, model)
  inputs = flatten_perdevice_values(strategy, inputs)
  targets = flatten_perdevice_values(strategy, targets)
  # Expand 1-dimensional inputs.
  # TODO(b/124535720): Remove once this standarize data logic is shared with
  # main flow.
  inputs, targets = nest.map_structure(training_utils.standardize_single_array,
                                       (inputs, targets))
  if mode == ModeKeys.PREDICT:
    sample_weights = []
    targets = []
  else:
    sample_weights = [
        None for _ in range(len(model.outputs) * strategy.num_replicas_in_sync)
    ]
  ins = inputs + targets + sample_weights
  if mode == ModeKeys.TRAIN and not isinstance(K.symbolic_learning_phase(),
                                               int):
    ins += [True]
  return ins
示例#9
0
def _prepare_feed_values(model, inputs, targets, sample_weights, mode):
    """Prepare feed values to the model execution function.

  Arguments:
    model: Model to prepare feed values for.
    inputs: List or dict of model inputs.
    targets: Optional list of model targets.
    sample_weights: Optional list of sample weight arrays.
    mode: One of ModeKeys.TRAIN/ModeKeys.TEST/ModeKeys.PREDICT.

  Returns:
    Feed values for the model in the given mode.
  """
    strategy = model._distribution_strategy
    inputs, targets, sample_weights = _get_input_from_iterator(inputs, model)

    # When the inputs are dict, then we want to flatten it in the same order as
    # the input layers, such that the data are fed into the input layers in the
    # correct order.
    if isinstance(inputs, dict):
        inputs = [inputs[key] for key in model._feed_input_names]
    if is_distributing_by_cloning(model):
        inputs = flatten_perdevice_values(strategy, inputs)
        targets = flatten_perdevice_values(strategy, targets)
    else:
        # TODO(b/129653859):  Simplify after PerReplica can be the input of
        # `def_function.function`.
        # Without cloning the `inputs` and `target` are the inputs to
        # `values.regroup`.  Instead of a flat list of `len(inputs) * num_replicas`
        # we need a list of `len(inputs)` lists, where each per-input list has
        # `len(num_replicas)` elements. Each element[i] in the per-input
        # list is the input to the i-th replica.  For example, if inputs are
        # `[[1, 2], [3, 4]]` and there are two replicas, then we want
        # `[[1, 3], [2, 4]]` (see `values_test.testWrapAListOfTwoTuples`) so that
        # we arrive at a `PerReplica(d0: 1, d1: 2)` and a `PerReplica(d0:3, d1:4)`.
        inputs = unwrap_perdevice_values(strategy, inputs)
        targets = unwrap_perdevice_values(strategy, targets)

    # Expand 1-dimensional inputs.
    # TODO(b/124535720): Remove once this standarize data logic is shared with
    # main flow.
    inputs, targets = nest.map_structure(
        training_utils.standardize_single_array, (inputs, targets))
    if mode == ModeKeys.PREDICT:
        sample_weights = []
        targets = []
    elif not is_distributing_by_cloning(model):
        sample_weights = None  # b/129503665
    else:
        sample_weights = [
            None
            for _ in range(len(model.outputs) * strategy.num_replicas_in_sync)
        ]
    ins = [inputs, targets, sample_weights]
    if mode == ModeKeys.TRAIN and not isinstance(K.symbolic_learning_phase(),
                                                 int):
        ins += [True]
    return tuple(ins)
示例#10
0
def _update_sample_weight_mode(model, mode, inputs):
    """Updates the sample_weight_mode of a given model."""
    if not model._distribution_strategy and mode != ModeKeys.PREDICT:
        # `inputs` is the model's inputs + targets + sample_weights +
        # learning phase placeholder if specified. To update the sample_weight_mode
        # we need to determine if the user has passed sample weights as part of the
        # input.
        sample_weights = inputs[len(model._feed_inputs) +
                                len(model._feed_targets):]
        has_learning_phase_pl = (
            mode == ModeKeys.TRAIN
            and not isinstance(K.symbolic_learning_phase(), int))
        if has_learning_phase_pl:
            sample_weights = sample_weights[:-1]
        model._update_sample_weight_modes(sample_weights=sample_weights)
示例#11
0
def _prepare_feed_values(model, inputs, targets, sample_weights, mode):
    """Prepare feed values to the model execution function.

  Arguments:
    model: Model to prepare feed values for.
    inputs: List or dict of model inputs.
    targets: Optional list of model targets.
    sample_weights: Optional list of sample weight arrays.
    mode: One of ModeKeys.TRAIN/ModeKeys.TEST/ModeKeys.PREDICT.

  Returns:
    Feed values for the model in the given mode.
  """
    if model._distribution_strategy:
        if isinstance(inputs, (dataset_ops.DatasetV1, dataset_ops.DatasetV2)):
            inputs = distributed_training_utils_v1.get_iterator(
                inputs, model._distribution_strategy)

        def get_distributed_inputs():
            return distributed_training_utils_v1._prepare_feed_values(
                model, inputs, targets, sample_weights, mode)

        # In the eager case, we want to call the input method per step, so return
        # a lambda from here that can be called. Note that this is applicable only
        # in Distribution Strategy case as it follows the same code path for both
        # eager and graph modes.
        # TODO(priyag,omalleyt): Either we should move the training DS with
        # IteratorBase to use training_generator code path, or figure out how to
        # set a symbolic Iterator out of a Dataset when in eager mode.
        if context.executing_eagerly():
            return get_distributed_inputs
        else:
            return get_distributed_inputs()

    if isinstance(
            inputs,
        (dataset_ops.DatasetV1, dataset_ops.DatasetV2, iterator_ops.Iterator)):
        inputs, targets, sample_weights = model._standardize_user_data(
            inputs, extract_tensors_from_dataset=True)

    inputs = training_utils_v1.ModelInputs(inputs).as_list()
    targets = list(targets or [])
    sample_weights = list(sample_weights or [])
    ins = inputs + targets + sample_weights
    if mode == ModeKeys.TRAIN and not isinstance(K.symbolic_learning_phase(),
                                                 int):
        ins += [True]  # Add learning phase value.
    return ins
示例#12
0
def _prepare_feed_values(model, inputs, targets, sample_weights, mode):
  """Prepare feed values to the model execution function.

  Arguments:
    model: Model to prepare feed values for.
    inputs: List or dict of model inputs.
    targets: Optional list of model targets.
    sample_weights: Optional list of sample weight arrays.
    mode: One of ModeKeys.TRAIN/ModeKeys.TEST/ModeKeys.PREDICT.

  Returns:
    Feed values for the model in the given mode.
  """
  if model._distribution_strategy:
    if isinstance(inputs, (dataset_ops.DatasetV1, dataset_ops.DatasetV2)):
      inputs = distributed_training_utils.get_iterator(
          inputs, model._distribution_strategy)

    def get_distributed_inputs():
      return distributed_training_utils._prepare_feed_values(
          model, inputs, targets, sample_weights, mode)

    # In the eager case, we want to call the input method per step, so return
    # a lambda from here that can be called. Note that this is applicable only
    # in Distribution Strategy case as it follows the same code path for both
    # eager and graph modes.
    # TODO(priyag,omalleyt): Either we should move the training DS with
    # EagerIterator to use training_generator code path, or figure out how to
    # set a symbolic Iterator out of a Dataset when in eager mode.
    if context.executing_eagerly():
      return get_distributed_inputs
    else:
      return get_distributed_inputs()

  if isinstance(inputs, (dataset_ops.DatasetV1, dataset_ops.DatasetV2,
                         iterator_ops.Iterator)):
    inputs, targets, sample_weights = model._standardize_user_data(
        inputs,
        extract_tensors_from_dataset=True)

  inputs = training_utils.ModelInputs(inputs).as_list()
  targets = targets or []
  sample_weights = sample_weights or []
  ins = inputs + targets + sample_weights
  if mode == ModeKeys.TRAIN and not isinstance(K.symbolic_learning_phase(),
                                               int):
    ins += [True]  # Add learning phase value.
  return ins
示例#13
0
    def get_layer_evaluation_generator(self, debug_input_tensor=None):

        """
        Todo
        :param debug_input_tensor:
        :return:
        """

        ''' All outputs we wanna have during the evaluation
            1. Input to the layers
            2. Layer output without activation
            3. Layer output with activation
        '''
        outputs = [[layer.input,  # Input to the layers
                    layer.output.op.inputs[0].op.inputs[0],  # Output without Activation
                    layer.output  # Output with activation
                    ] for layer in self._model.layers if layer.__class__.__name__ != 'InputLayer']

        ''' Keras function for getting the layer ouputs'''
        layer_outputs = K.function([self._model.input,
                                    K.symbolic_learning_phase()],
                                   outputs)  # evaluation function

        # predictions = K.function([self._model.layers.input,
        #                           K.symbolic_learning_phase()],
        #                           self._model.layers.output)

        # Create all the intemediate outputs with dropout disables. If Dropout is enables, we have MC-Dropout
        layer_outs = layer_outputs([debug_input_tensor, False])
        # preds = predictions([X_test, False])

        debug_input_tensor = self._debug_create_tensor(self._model.input.shape, debug_input_tensor)

        for i, o in enumerate(zip(layer_outs, self._model.layers[1:])):
            i += 1
            inputs = o[0][0]
            output_b_a = o[0][1]  # Output before activation
            output = o[0][2]  # Output after activation
            l = o[1]
            model_dict = {'layer_name': l.__class__.__name__,
                          'layer': l,
                          'inputs': inputs,
                          'output_b_a': output_b_a,
                          'output': output
                          }
            yield model_dict
示例#14
0
def _prepare_feed_values(model, inputs, targets, sample_weights, mode):
    """Prepare feed values to the model execution function.

  Arguments:
    model: Model to prepare feed values for.
    inputs: List or dict of model inputs.
    targets: Optional list of model targets.
    sample_weights: Optional list of sample weight arrays.
    mode: One of ModeKeys.TRAIN/ModeKeys.TEST/ModeKeys.PREDICT.

  Returns:
    Feed values for the model in the given mode.
  """
    strategy = model._distribution_strategy
    inputs, targets, sample_weights = _get_input_from_iterator(inputs, model)

    # When the inputs are dict, then we want to flatten it in the same order as
    # the input layers, such that the data are fed into the input layers in the
    # correct order.
    if isinstance(inputs, dict):
        inputs = [inputs[key] for key in model._feed_input_names]
    if is_distributing_by_cloning(model):
        inputs = flatten_per_replica_values(strategy, inputs)
        targets = flatten_per_replica_values(strategy, targets)
        # Expand 1-dimensional inputs.
        # TODO(b/124535720): Remove once this standarize data logic is shared with
        # main flow.
        inputs, targets = nest.map_structure(
            training_utils.standardize_single_array, (inputs, targets))

    if mode == ModeKeys.PREDICT:
        sample_weights = []
        targets = []
    elif not is_distributing_by_cloning(model):
        sample_weights = None  # b/129503665
    else:
        sample_weights = [
            None
            for _ in range(len(model.outputs) * strategy.num_replicas_in_sync)
        ]
    ins = [inputs, targets, sample_weights]
    if mode == ModeKeys.TRAIN and not isinstance(K.symbolic_learning_phase(),
                                                 int):
        ins += [True]
    return tuple(ins)
示例#15
0
def get_layer_outputs(model, patches):
    """
    Print internal layer outputs to files
    :param model: trained keras model
    :param patches: input samples
    :param layers: relevant layers, if None then all layer are relevant
    :return: layer_outs probability dist
    """
    inp = model.input  # input placeholder
    layers, _ = get_relevant_layers(model=model)
    if layers is None:
        outputs = [layer.output for layer in model.layers[1:]]
    else:
        outputs = [layer.output for layer in layers]
    functors = [K.function([inp, K.symbolic_learning_phase()], [out]) for out in outputs]  # evaluation functions
    layer_outs = [func([patches, False])[0] for func in functors]
    layer_outs_dist = list(map(lambda x: softmax(x, axis=-1), layer_outs))
    return layer_outs_dist
示例#16
0
def _update_sample_weight_mode(model, mode, adapter):
    """Updates the sample_weight_mode of a given model."""
    # Add a quick return to prevent us from calling model._feed_targets that
    # accesses certain model properties that may not be set in the `PREDICT` mode.
    if mode == ModeKeys.PREDICT:
        return

    sample_weights = None

    # Get some sample inputs from the data_adapter
    iterator = _create_dataset_iterator(model._distribution_strategy,
                                        adapter.get_dataset())
    inputs = create_batch_inputs(iterator, mode, model,
                                 model._distribution_strategy)
    # `inputs` is the model's inputs + targets + sample_weights +
    # learning phase placeholder if specified. To update the sample_weight_mode
    # we need to determine if the user has passed sample weights as part of the
    # input.
    if not callable(inputs):
        # if not isinstance(inputs, collections.Sequence):
        #   inputs = (inputs,)
        # Note that the batch inputs should be a tuple of 2, 3 or 4 items.
        # (input, target, {sample_weights}, {learning_phase})
        sample_weights_index = 0
        if model._feed_inputs:
            sample_weights_index += 1
        if model._feed_targets:
            sample_weights_index += 1

        sample_weights = inputs[sample_weights_index:]
        has_learning_phase_pl = (
            mode == ModeKeys.TRAIN
            and not isinstance(backend.symbolic_learning_phase(), int))
        if has_learning_phase_pl:
            sample_weights = sample_weights[:-1]
        model._update_sample_weight_modes(nest.flatten(sample_weights))

    # Call the DistributionStrategy specific function to update the
    # sample_weight_mode on the model.
    if model._distribution_strategy:
        dist_utils._update_sample_weight_modes(model, mode, sample_weights)

    # Force delete the iterator.
    del iterator
def _prepare_feed_values(model, inputs, targets, sample_weights, mode):
  """Prepare feed values to the model execution function.

  Arguments:
    model: Model to prepare feed values for.
    inputs: List or dict of model inputs.
    targets: Optional list of model targets.
    sample_weights: Optional list of sample weight arrays.
    mode: One of ModeKeys.TRAIN/ModeKeys.TEST/ModeKeys.PREDICT.

  Returns:
    Feed values for the model in the given mode.
  """
  strategy = model._distribution_strategy
  inputs, targets, sample_weights = _get_input_from_iterator(inputs, model)

  # When the inputs are dict, then we want to flatten it in the same order as
  # the input layers, such that the data are fed into the input layers in the
  # correct order.
  if isinstance(inputs, dict):
    inputs = [inputs[key] for key in model._feed_input_names]
  if is_distributing_by_cloning(model):
    inputs = flatten_per_replica_values(strategy, inputs)
    targets = flatten_per_replica_values(strategy, targets)
    # Expand 1-dimensional inputs.
    # TODO(b/124535720): Remove once this standarize data logic is shared with
    # main flow.
    inputs, targets = nest.map_structure(
        training_utils.standardize_single_array, (inputs, targets))

  if mode == ModeKeys.PREDICT:
    sample_weights = []
    targets = []
  elif not is_distributing_by_cloning(model):
    sample_weights = None  # b/129503665
  else:
    sample_weights = [
        None for _ in range(len(model.outputs) * strategy.num_replicas_in_sync)
    ]
  ins = [inputs, targets, sample_weights]
  if mode == ModeKeys.TRAIN and not isinstance(K.symbolic_learning_phase(),
                                               int):
    ins += [True]
  return tuple(ins)
示例#18
0
def create_batch_inputs(iterator, mode, model, strategy):
    """Create the input data from the iterator based on the model and strategy."""
    if strategy:
        # Note that the batch_ins is a function to avoid the tf.function
        # retrace.
        def distribute_batch_ins():
            return dist_utils._prepare_feed_values(model, iterator, None, None,
                                                   mode)

        batch_ins = distribute_batch_ins
    else:
        batch_ins = next(iterator)
        if (mode == ModeKeys.TRAIN and not model.run_eagerly
                and not isinstance(backend.symbolic_learning_phase(), int)):
            # Add learning phase value.
            if not isinstance(batch_ins, collections.Sequence):
                batch_ins = (batch_ins, True)
            else:
                batch_ins += (True, )
    return batch_ins
示例#19
0
    def _debug_output(self, output):

        ''' If you want to debug the values Eager Execution needs to be enabled'''
        if not tf.executing_eagerly():
            logging.warning('If eager execution is disabled not all debug values can be visualized!'
                            ' Enable Eager Execution for proper debugging!')
        
        # For debugging the layer we want a tensor only
        if self._debug[self.DEBUG.layer]:
            out = self._debug_create_tensor(output.shape)

        elif self._debug[self.DEBUG.model]:
            functor = K.function([self._model.input,
                                  K.symbolic_learning_phase()],
                                 output)  # evaluation function
    
            out = functor([self._debug[self.DEBUG.tensor], False])
            out = tf.convert_to_tensor(out, dtype=tf.float32)

        return out
示例#20
0
def _prepare_feed_values(model, inputs, targets, sample_weights, mode):
    """Prepare feed values to the model execution function.

  Arguments:
    model: Model to prepare feed values for.
    inputs: List or dict of model inputs.
    targets: Optional list of model targets.
    sample_weights: Optional list of sample weight arrays.
    mode: One of 'train'/'test'/'predict'.

  Returns:
    Feed values for the model in the given mode.
  """
    if model._distribution_strategy:

        def get_distributed_inputs():
            return training_distributed._prepare_feed_values(
                model, inputs, targets, sample_weights, mode)

        # In the eager case, we want to call the input method per step, so return
        # a lambda from here that can be called. Note that this is applicable only
        # in Distribution Strategy case as it follows the same code path for both
        # eager and graph modes.
        # TODO(priyag,omalleyt): Either we should move the training DS with
        # EagerIterator to use training_generator code path, or figure out how to
        # set a symbolic Iterator out of a Dataset when in eager mode.
        if context.executing_eagerly():
            return get_distributed_inputs
        else:
            return get_distributed_inputs()

    inputs = training_utils.ModelInputs(inputs).as_list()
    targets = targets or []
    sample_weights = sample_weights or []
    ins = inputs + targets + sample_weights
    if mode == 'train' and not isinstance(K.symbolic_learning_phase(), int):
        ins += [True]
    return ins
def _prepare_feed_values(model, inputs, targets, sample_weights, mode):
  """Prepare feed values to the model execution function.

  Arguments:
    model: Model to prepare feed values for.
    inputs: List or dict of model inputs.
    targets: Optional list of model targets.
    sample_weights: Optional list of sample weight arrays.
    mode: One of 'train'/'test'/'predict'.

  Returns:
    Feed values for the model in the given mode.
  """
  if model._distribution_strategy:
    return training_distributed._prepare_feed_values(model, inputs, targets,
                                                     sample_weights, mode)
  inputs = training_utils.ModelInputs(inputs).as_list()
  targets = targets or []
  sample_weights = sample_weights or []
  ins = inputs + targets + sample_weights
  if mode == 'train' and not isinstance(K.symbolic_learning_phase(), int):
    ins += [True]
  return ins
示例#22
0
def _prepare_feed_values(model, inputs, targets, sample_weights, mode):
    """Prepare feed values to the model execution function.

  Arguments:
    model: Model to prepare feed values for.
    inputs: List or dict of model inputs.
    targets: Optional list of model targets.
    sample_weights: Optional list of sample weight arrays.
    mode: One of 'train'/'test'/'predict'.

  Returns:
    Feed values for the model in the given mode.
  """
    if model._distribution_strategy:
        return training_distributed._prepare_feed_values(
            model, inputs, targets, sample_weights, mode)
    inputs = training_utils.ModelInputs(inputs).as_list()
    targets = targets or []
    sample_weights = sample_weights or []
    ins = inputs + targets + sample_weights
    if mode == 'train' and not isinstance(K.symbolic_learning_phase(), int):
        ins += [True]
    return ins
示例#23
0
def _prepare_feed_values(model, inputs, targets, sample_weights, mode):
  """Prepare feed values to the model execution function.

  Arguments:
    model: Model to prepare feed values for.
    inputs: List or dict of model inputs.
    targets: Optional list of model targets.
    sample_weights: Optional list of sample weight arrays.
    mode: One of 'train'/'test'/'predict'.

  Returns:
    Feed values for the model in the given mode.
  """
  if model._distribution_strategy:
    def get_distributed_inputs():
      return training_distributed._prepare_feed_values(
          model, inputs, targets, sample_weights, mode)

    # In the eager case, we want to call the input method per step, so return
    # a lambda from here that can be called. Note that this is applicable only
    # in Distribution Strategy case as it follows the same code path for both
    # eager and graph modes.
    # TODO(priyag,omalleyt): Either we should move the training DS with
    # EagerIterator to use training_generator code path, or figure out how to
    # set a symbolic Iterator out of a Dataset when in eager mode.
    if context.executing_eagerly():
      return get_distributed_inputs
    else:
      return get_distributed_inputs()

  inputs = training_utils.ModelInputs(inputs).as_list()
  targets = targets or []
  sample_weights = sample_weights or []
  ins = inputs + targets + sample_weights
  if mode == 'train' and not isinstance(K.symbolic_learning_phase(), int):
    ins += [True]
  return ins
def model_iteration(model,
                    inputs,
                    targets=None,
                    sample_weights=None,
                    batch_size=None,
                    epochs=1,
                    verbose=1,
                    callbacks=None,
                    val_inputs=None,
                    val_targets=None,
                    val_sample_weights=None,
                    shuffle=True,
                    initial_epoch=0,
                    steps_per_epoch=None,
                    validation_steps=None,
                    mode='train',
                    **kwargs):
  """Loop function for arrays of data with modes 'train'/'test'/'predict'.

  Arguments:
      model: Keras Model instance.
      inputs: Either a list of arrays or a dictionary.
      targets: List of target arrays.
      sample_weights: Optional list of sample weight arrays.
      batch_size: Integer batch size or None if unknown.
      epochs: Number of times to iterate over the data
      verbose: Verbosity mode, 0, 1 or 2
      callbacks: List of callbacks to be called during training
      val_inputs: List of input arrays.
      val_targets: List of target arrays.
      val_sample_weights: Optional list of sample weight arrays.
      shuffle: Whether to shuffle the data at the beginning of each epoch
        concatenation of list the display names of the outputs of `f` and the
        list of display names of the outputs of `f_val`.
      initial_epoch: Epoch at which to start training (useful for resuming a
        previous training run)
      steps_per_epoch: Total number of steps (batches of samples) before
        declaring one epoch finished and starting the next epoch. Ignored with
        the default value of `None`.
      validation_steps: Number of steps to run validation for (only if doing
        validation from data tensors). Ignored with the default value of `None`.
      mode: One of 'train'/'test'/'predict'.
      **kwargs: Additional arguments for backwards compatibility.

  Returns:
      - In 'train' mode: `History` object.
      - In 'test' mode: Evaluation metrics.
      - In 'predict' mode: Outputs of the Model called on inputs.

  Raises:
      ValueError: in case of invalid arguments.
  """
  # Backwards compatibility.
  if 'steps' in kwargs:
    steps_per_epoch = kwargs['steps']

  _validate_arguments(steps_per_epoch, validation_steps, kwargs)
  if mode == 'train':
    _print_train_info(inputs, val_inputs, steps_per_epoch, verbose)

  # Get step function and loop type.
  f = model._get_execution_function(mode)
  use_steps = steps_per_epoch is not None
  do_validation = val_inputs is not None

  # Prepare input data.
  inputs = training_utils.ModelInputs(inputs).as_list()
  targets = targets or []
  sample_weights = sample_weights or []
  learning_phase_input = []
  if not isinstance(K.symbolic_learning_phase(), int):
    learning_phase_input = [True] if mode == 'train' else [False]
  ins = inputs + targets + sample_weights + learning_phase_input
  num_samples_or_steps = _get_num_samples_or_steps(ins, batch_size,
                                                   steps_per_epoch)

  # Configure callbacks.
  count_mode = 'steps' if use_steps else 'samples'
  callbacks = cbks.configure_callbacks(
      callbacks,
      model,
      do_validation=do_validation,
      val_inputs=val_inputs,
      val_targets=val_targets,
      val_sample_weights=val_sample_weights,
      batch_size=batch_size,
      epochs=epochs,
      steps_per_epoch=steps_per_epoch,
      samples=num_samples_or_steps,
      validation_steps=validation_steps,
      verbose=0,  # Handle ProgBarLogger separately in this loop.
      count_mode=count_mode,
      mode=mode)
  # TODO(omalleyt): Handle ProgBar as part of Callbacks once hooks are ready.
  progbar = _get_progbar(model, count_mode)
  progbar.params = callbacks.params
  progbar.params['verbose'] = verbose

  # Find beforehand arrays that need sparse-to-dense conversion.
  if issparse is not None:
    indices_for_conversion_to_dense = []
    feed = _get_model_feed(model, mode)
    for i, (input_data, feed_tensor) in enumerate(zip(ins, feed)):
      if issparse(input_data) and not K.is_sparse(feed_tensor):
        indices_for_conversion_to_dense.append(i)

  # Select aggregation method.
  if mode == 'predict':
    aggregator = OutputsAggregator(use_steps, num_samples_or_steps)
  else:
    aggregator = MetricsAggregator(use_steps, num_samples_or_steps)

  callbacks.model.stop_training = False
  callbacks._call_begin_hook(mode)
  progbar.on_train_begin()
  for epoch in range(initial_epoch, epochs):
    if callbacks.model.stop_training:
      break

    # Setup work for each epoch
    results = []
    epoch_logs = {}
    if hasattr(model, 'stateful_metric_functions'):
      for m in model.stateful_metric_functions:
        m.reset_states()
    callbacks.on_epoch_begin(epoch, epoch_logs, mode=mode)
    progbar.on_epoch_begin(epoch, epoch_logs)

    if use_steps:
      # Step-wise loop.
      for step in range(steps_per_epoch):
        batch_logs = {'batch': step, 'size': 1}
        callbacks._call_batch_hook(mode, 'begin', step, batch_logs)
        progbar.on_batch_begin(step, batch_logs)

        # Get outputs.
        try:
          batch_outs = f(ins)
        except errors.OutOfRangeError:
          logging.warning('Your dataset iterator ran out of data; '
                          'interrupting training. Make sure that your dataset '
                          'can generate at least `steps_per_epoch * epochs` '
                          'batches (in this case, %d batches). You may need to'
                          'use the repeat() function when building your '
                          'dataset.' %
                          steps_per_epoch * epochs)
          break
        if not isinstance(batch_outs, list):
          batch_outs = [batch_outs]

        # Aggregate results.
        if step == 0:
          aggregator.create(batch_outs)
        aggregator.aggregate(batch_outs)

        # Callbacks batch end.
        batch_logs.update(_make_logs(model, batch_outs, mode))
        callbacks._call_batch_hook(mode, 'end', step, batch_logs)
        progbar.on_batch_end(step, batch_logs)

        if callbacks.model.stop_training:
          break
    else:
      # Sample-wise loop.
      index_array = np.arange(num_samples_or_steps)
      if shuffle == 'batch':
        index_array = training_utils.batch_shuffle(index_array, batch_size)
      elif shuffle:
        np.random.shuffle(index_array)
      batches = make_batches(num_samples_or_steps, batch_size)

      for batch_index, (batch_start, batch_end) in enumerate(batches):
        batch_ids = index_array[batch_start:batch_end]

        # Slice into a batch.
        try:
          if ins and isinstance(ins[-1], int):
            # Do not slice the training phase flag.
            ins_batch = slice_arrays(ins[:-1], batch_ids) + [ins[-1]]
          else:
            ins_batch = slice_arrays(ins, batch_ids)
        except TypeError:
          raise TypeError('TypeError while preparing batch. '
                          'If using HDF5 input data, '
                          'pass shuffle="batch".')

        # Sparse to dense conversion.
        for i in indices_for_conversion_to_dense:
          ins_batch[i] = ins_batch[i].toarray()

        # Callbacks batch_begin.
        batch_logs = {'batch': batch_index, 'size': len(batch_ids)}
        callbacks._call_batch_hook(mode, 'begin', batch_index, batch_logs)
        progbar.on_batch_begin(batch_index, batch_logs)

        # Get outputs.
        batch_outs = f(ins_batch)
        if not isinstance(batch_outs, list):
          batch_outs = [batch_outs]

        # Aggregate results.
        if batch_index == 0:
          aggregator.create(batch_outs)
        aggregator.aggregate(batch_outs, batch_start, batch_end)

        # Callbacks batch end.
        batch_logs.update(_make_logs(model, batch_outs, mode))
        callbacks._call_batch_hook(mode, 'end', batch_index, batch_logs)
        progbar.on_batch_end(batch_index, batch_logs)

        if callbacks.model.stop_training:
          break

    aggregator.finalize()
    results = aggregator.results
    epoch_logs.update(_make_logs(model, results, mode))
    if len(results) == 1:
      results = results[0]

    # Run the test loop every epoch during training.
    if do_validation and not callbacks.model.stop_training:
      val_results = model_iteration(
          model,
          val_inputs,
          targets=val_targets,
          sample_weights=val_sample_weights,
          batch_size=batch_size,
          steps_per_epoch=validation_steps,
          callbacks=callbacks,
          verbose=0,
          mode='test')
      if not isinstance(val_results, list):
        val_results = [val_results]
      epoch_logs.update(_make_logs(model, val_results, mode, prefix='val_'))

    callbacks.on_epoch_end(epoch, epoch_logs, mode=mode)
    progbar.on_epoch_end(epoch, epoch_logs)
  callbacks._call_end_hook(mode)

  if mode == 'train':
    return model.history
  return results
def model_iteration(model,
                    inputs,
                    targets=None,
                    sample_weights=None,
                    batch_size=None,
                    epochs=1,
                    verbose=1,
                    callbacks=None,
                    val_inputs=None,
                    val_targets=None,
                    val_sample_weights=None,
                    shuffle=True,
                    initial_epoch=0,
                    steps_per_epoch=None,
                    validation_steps=None,
                    mode='train',
                    **kwargs):
    """Loop function for arrays of data with modes 'train'/'test'/'predict'.

  Arguments:
      model: Keras Model instance.
      inputs: Either a list of arrays or a dictionary.
      targets: List of target arrays.
      sample_weights: Optional list of sample weight arrays.
      batch_size: Integer batch size or None if unknown.
      epochs: Number of times to iterate over the data
      verbose: Verbosity mode, 0, 1 or 2
      callbacks: List of callbacks to be called during training
      val_inputs: List of input arrays.
      val_targets: List of target arrays.
      val_sample_weights: Optional list of sample weight arrays.
      shuffle: Whether to shuffle the data at the beginning of each epoch
        concatenation of list the display names of the outputs of `f` and the
        list of display names of the outputs of `f_val`.
      initial_epoch: Epoch at which to start training (useful for resuming a
        previous training run)
      steps_per_epoch: Total number of steps (batches of samples) before
        declaring one epoch finished and starting the next epoch. Ignored with
        the default value of `None`.
      validation_steps: Number of steps to run validation for (only if doing
        validation from data tensors). Ignored with the default value of `None`.
      mode: One of 'train'/'test'/'predict'.
      **kwargs: Additional arguments for backwards compatibility.

  Returns:
      - In 'train' mode: `History` object.
      - In 'test' mode: Evaluation metrics.
      - In 'predict' mode: Outputs of the Model called on inputs.

  Raises:
      ValueError: in case of invalid arguments.
  """
    # Backwards compatibility.
    if 'steps' in kwargs:
        steps_per_epoch = kwargs['steps']

    _validate_arguments(steps_per_epoch, validation_steps, kwargs)
    if mode == 'train':
        _print_train_info(inputs, val_inputs, steps_per_epoch, verbose)

    # Get step function and loop type.
    f = model._get_execution_function(mode)
    use_steps = steps_per_epoch is not None
    do_validation = val_inputs is not None

    # Prepare input data.
    inputs = training_utils.ModelInputs(inputs).as_list()
    targets = targets or []
    sample_weights = sample_weights or []
    learning_phase_input = []
    if not isinstance(K.symbolic_learning_phase(), int):
        learning_phase_input = [True] if mode == 'train' else [False]
    ins = inputs + targets + sample_weights + learning_phase_input
    num_samples_or_steps = _get_num_samples_or_steps(ins, batch_size,
                                                     steps_per_epoch)

    # Configure callbacks.
    count_mode = 'steps' if use_steps else 'samples'
    callbacks = cbks.configure_callbacks(
        callbacks,
        model,
        do_validation=do_validation,
        val_inputs=val_inputs,
        val_targets=val_targets,
        val_sample_weights=val_sample_weights,
        batch_size=batch_size,
        epochs=epochs,
        steps_per_epoch=steps_per_epoch,
        samples=num_samples_or_steps,
        validation_steps=validation_steps,
        verbose=0,  # Handle ProgBarLogger separately in this loop.
        count_mode=count_mode,
        mode=mode)
    # TODO(omalleyt): Handle ProgBar as part of Callbacks once hooks are ready.
    progbar = _get_progbar(model, count_mode)
    progbar.params = callbacks.params
    progbar.params['verbose'] = verbose

    # Find beforehand arrays that need sparse-to-dense conversion.
    if issparse is not None:
        indices_for_conversion_to_dense = []
        feed = _get_model_feed(model, mode)
        for i, (input_data, feed_tensor) in enumerate(zip(ins, feed)):
            if issparse(input_data) and not K.is_sparse(feed_tensor):
                indices_for_conversion_to_dense.append(i)

    # Select aggregation method.
    if mode == 'predict':
        aggregator = OutputsAggregator(use_steps, num_samples_or_steps)
    else:
        aggregator = MetricsAggregator(use_steps, num_samples_or_steps)

    callbacks.model.stop_training = False
    callbacks._call_begin_hook(mode)
    progbar.on_train_begin()
    for epoch in range(initial_epoch, epochs):
        if callbacks.model.stop_training:
            break

        # Setup work for each epoch
        results = []
        epoch_logs = {}
        if hasattr(model, 'stateful_metric_functions'):
            for m in model.stateful_metric_functions:
                m.reset_states()
        callbacks.on_epoch_begin(epoch, epoch_logs, mode=mode)
        progbar.on_epoch_begin(epoch, epoch_logs)

        if use_steps:
            # Step-wise loop.
            for step in range(steps_per_epoch):
                batch_logs = {'batch': step, 'size': 1}
                callbacks._call_batch_hook(mode, 'begin', step, batch_logs)
                progbar.on_batch_begin(step, batch_logs)

                # Get outputs.
                try:
                    batch_outs = f(ins)
                except errors.OutOfRangeError:
                    logging.warning(
                        'Your dataset iterator ran out of data; '
                        'interrupting training. Make sure that your dataset '
                        'can generate at least `steps_per_epoch * epochs` '
                        'batches (in this case, %d batches). You may need to'
                        'use the repeat() function when building your '
                        'dataset.' % steps_per_epoch * epochs)
                    break
                if not isinstance(batch_outs, list):
                    batch_outs = [batch_outs]

                # Aggregate results.
                if step == 0:
                    aggregator.create(batch_outs)
                aggregator.aggregate(batch_outs)

                # Callbacks batch end.
                batch_logs.update(_make_logs(model, batch_outs, mode))
                callbacks._call_batch_hook(mode, 'end', step, batch_logs)
                progbar.on_batch_end(step, batch_logs)

                if callbacks.model.stop_training:
                    break
        else:
            # Sample-wise loop.
            index_array = np.arange(num_samples_or_steps)
            if shuffle == 'batch':
                index_array = training_utils.batch_shuffle(
                    index_array, batch_size)
            elif shuffle:
                np.random.shuffle(index_array)
            batches = make_batches(num_samples_or_steps, batch_size)

            for batch_index, (batch_start, batch_end) in enumerate(batches):
                batch_ids = index_array[batch_start:batch_end]

                # Slice into a batch.
                try:
                    if ins and isinstance(ins[-1], int):
                        # Do not slice the training phase flag.
                        ins_batch = slice_arrays(ins[:-1],
                                                 batch_ids) + [ins[-1]]
                    else:
                        ins_batch = slice_arrays(ins, batch_ids)
                except TypeError:
                    raise TypeError('TypeError while preparing batch. '
                                    'If using HDF5 input data, '
                                    'pass shuffle="batch".')

                # Sparse to dense conversion.
                if issparse is not None:
                    for i in indices_for_conversion_to_dense:
                        ins_batch[i] = ins_batch[i].toarray()

                # Callbacks batch_begin.
                batch_logs = {'batch': batch_index, 'size': len(batch_ids)}
                callbacks._call_batch_hook(mode, 'begin', batch_index,
                                           batch_logs)
                progbar.on_batch_begin(batch_index, batch_logs)

                # Get outputs.
                batch_outs = f(ins_batch)
                if not isinstance(batch_outs, list):
                    batch_outs = [batch_outs]

                # Aggregate results.
                if batch_index == 0:
                    aggregator.create(batch_outs)
                aggregator.aggregate(batch_outs, batch_start, batch_end)

                # Callbacks batch end.
                batch_logs.update(_make_logs(model, batch_outs, mode))
                callbacks._call_batch_hook(mode, 'end', batch_index,
                                           batch_logs)
                progbar.on_batch_end(batch_index, batch_logs)

                if callbacks.model.stop_training:
                    break

        aggregator.finalize()
        results = aggregator.results
        epoch_logs.update(_make_logs(model, results, mode))
        if len(results) == 1:
            results = results[0]

        # Run the test loop every epoch during training.
        if do_validation and not callbacks.model.stop_training:
            val_results = model_iteration(model,
                                          val_inputs,
                                          targets=val_targets,
                                          sample_weights=val_sample_weights,
                                          batch_size=batch_size,
                                          steps_per_epoch=validation_steps,
                                          callbacks=callbacks,
                                          verbose=0,
                                          mode='test')
            if not isinstance(val_results, list):
                val_results = [val_results]
            epoch_logs.update(
                _make_logs(model, val_results, mode, prefix='val_'))

        callbacks.on_epoch_end(epoch, epoch_logs, mode=mode)
        progbar.on_epoch_end(epoch, epoch_logs)
    callbacks._call_end_hook(mode)

    if mode == 'train':
        return model.history
    return results
示例#26
0
#%%
logger.info('Loading predictor')
bert_topic = ktrain.load_predictor('models/bert_20ng/trained_bert_4epochs_2e-2')
#%%
ARTICLE1 = "Google_2_en.yml"
ARTICLE2 = "Google_9_en.yml"

with open(f"articles/{ARTICLE1}") as f:
    article1 = yaml.load(f)
with open(f"articles/{ARTICLE2}") as f:
    article2 = yaml.load(f)
# %%
inp = bert_topic.model.input
outs = bert_topic.model.layers[-2].output

functor = K.function([inp, K.symbolic_learning_phase()], outs)

#%%
embeddings1 = []

for p in article1['paragraphs']:
    preproc = bert_topic.preproc.preprocess(p)
    out = functor([preproc, True])
    embeddings1.append(out.prod(axis=0))

embeddings2 = []

for p in article2['paragraphs']:
    preproc = bert_topic.preproc.preprocess(p)
    out = functor([preproc, True])
    embeddings2.append(out.prod(axis=0))