Пример #1
0
  def get(self, scope=None, name=None, full_name=None,
          roles=None, match_all=False, exact=False,
          beginning_scope=False):
    """ Return all variables and tensor with given roles

    Parameters
    ----------
    scope : {None, string}
      name of variable scope, any of the scope that match given name
      will be selected
    name : {None, string}
      the name of tensor without the output indexing ":0" and the scope
    full_name : {None, string}
      the full name includes both scope and tensor name without the
      output indexing ":0"
    roles : {None, odin.backend.role}
      specific roles of the tensor
    match_all : bool (default: False)
      If ``True``, checks if the variable has all given roles.
      If ``False``, any of the roles is sufficient.
    exact : bool (default: False)
      If ``True``, use ``==`` for comparison to get exactly same roles.
      If ``False``, use `issubclass` for comparison, hence, also match the
      descendant roles.
    beginning_scope : bool (default: True)
      if True, the provide scope must be the beginning scope,
      otherwise, it could be in the middle of multiple scopes
    """
    alltensors = self.tensors + self.variables
    # ====== by role ====== #
    if roles is not None:
      alltensors = [t for t in alltensors
                    if has_roles(t, roles=roles,
                                 match_all=match_all, exact=exact)]
    # ====== from general to detail ====== #
    if scope is not None:
      scope = str(scope)
      if len(scope) == 0:
        alltensors = [t for t in alltensors
                      if '/' not in t.name]
      else:
        scope_name_pattern = _TF_SCOPE_PATTERN(scope, beginning_scope)
        alltensors = [t for t in alltensors
                      if len(scope_name_pattern.findall(t.name))]
    # ====== filter by name ====== #
    if name is not None:
      name = as_tuple(name, t=string_types)
      alltensors = [t for t in alltensors
                    if any((n == t.name.split('/')[-1] or
                            n.split(':')[0] == t.name.split('/')[-1].split(':')[0])
                           for n in name)]
    # ====== full name ====== #
    if full_name is not None:
      full_name = as_tuple(full_name, t=string_types)
      alltensors = [t for t in alltensors
                    if any((n == t.name or
                            n.split(':')[0] == t.name.split(':')[0])
                           for n in full_name)]
    return alltensors
Пример #2
0
 def _initialize(self, X):
     # ====== check inputs dimensions ====== #
     if not hasattr(X, 'shape'):
         raise ValueError("`X` must have `shape` attribute.")
     feat_dim = np.prod(X.shape[1:])
     if self._feat_dim is None:
         self._feat_dim = feat_dim
     # validate input dimension
     if feat_dim != self._feat_dim:
         raise RuntimeError("Feature dimension mismatch %d and %d" %
                            (feat_dim, self.feat_dim))
     # check if tensorflow op initalized
     if hasattr(self, '_f_train'):
         return
     # ====== binary or multi-classes ====== #
     if self.nb_classes == 2:
         out_shape = (None, )
         fn_activation = tf.nn.sigmoid
         fn_loss = tf.losses.sigmoid_cross_entropy
         fn_acc = K.metrics.binary_accuracy
     else:
         out_shape = (None, self.nb_classes)
         fn_activation = tf.nn.softmax
         fn_loss = tf.losses.softmax_cross_entropy
         fn_acc = K.metrics.categorical_accuracy
     # ====== create model ====== #
     with tf.name_scope(self.name, 'logistic_regression'):
         # inputs
         self._X = K.placeholder(shape=(None, self.feat_dim),
                                 dtype=self.dtype,
                                 name='%s_input' % self.name)
         self._y = K.placeholder(shape=out_shape,
                                 dtype=self.dtype,
                                 name='%s_output' % self.name)
         # check the bias
         if is_number(self.fit_intercept):
             b_init = float(self.fit_intercept)
         elif self.fit_intercept is False or \
         self.fit_intercept is None:
             b_init = None
         else:
             b_init = self.fit_intercept
         # create the model and initialize
         with K.variable_dtype(dtype=self.dtype):
             self._model = N.Dense(
                 num_units=self.nb_classes,
                 W_init=init_ops.glorot_uniform_initializer(
                     seed=self._rand_state.randint()),
                 b_init=b_init,
                 activation=K.linear)
             y_logits = self._model(self._X)
         y_prob = fn_activation(y_logits)
         # applying class weights
         class_weights = tf.constant(value=self._class_weight,
                                     dtype=self.dtype,
                                     name="class_weights")
         weights = tf.gather(
             class_weights,
             tf.cast(self._y, 'int32')
             if self.nb_classes == 2 else tf.argmax(self._y, axis=-1))
         # optimizer
         params = [
             v for v in self._model.variables
             if has_roles(v, Weight) or has_roles(v, Bias)
         ]
         losses = fn_loss(self._y, y_logits, weights=weights)
         l1_norm = tf.norm(self._model.get('W'),
                           ord=1) if self.l1 > 0. else 0
         l2_norm = tf.norm(self._model.get('W'),
                           ord=2) if self.l2 > 0. else 0
         losses = losses + self.l1 * l1_norm + self.l2 * l2_norm
         acc = fn_acc(self._y, y_prob)
         updates = self._optimizer.get_updates(losses, params)
         # create function
         if self.confusion_matrix:
             cm = K.metrics.confusion_matrix(y_true=self._y,
                                             y_pred=y_prob,
                                             labels=self.nb_classes)
         metrics = [losses, acc, cm
                    ] if self.confusion_matrix else [losses, acc]
         self._f_train = K.function(inputs=(self._X, self._y),
                                    outputs=metrics,
                                    updates=updates,
                                    training=True)
         self._f_score = K.function(inputs=(self._X, self._y),
                                    outputs=metrics,
                                    training=False)
         self._f_pred_prob = K.function(inputs=self._X,
                                        outputs=y_prob,
                                        training=False)
         self._f_pred_logit = K.function(inputs=self._X,
                                         outputs=y_logits,
                                         training=False)
     return self
Пример #3
0
 def auxiliary_tensors(self):
   return [t for t in self._tensors if has_roles(t, Auxiliary)]
Пример #4
0
 def parameters(self):
   return [var for var in self._variables if has_roles(var, Parameter)]
Пример #5
0
  def minimize(self, loss, var_list=None,
               roles=[TrainableParameter], exclude_roles=[],
               reuse_scope=None, verbose=False):
    """
    Parameters
    ----------
    loss : {Tensor, list of Tensor}
      the loss for minimize
    var_list : {None, list of tensorflow.Variable}
      if None, automatically selecting the Variables, by getting
      all Variables associated with `loss` and using given
      `roles` and `exclude_roles`
    roles : {None, list of odin.backend.role}
      in case of `var_list=None`, all Variables with given role
      in this list will be selected for calculating gradients.
    exclude_roles : {None, list of odin.backend.role}
      all Variables have role in this list will be ignored.
    reuse_scope : {None, string}
      if provided, searching for all `OptimizerVariable` within
      given scope, and set the variables of this optimizer to
      the desire values
    verbose : bool (default: False)
      if True, print out all found variables and their roles
    """
    if exclude_roles is None:
      exclude_roles = []
    # ====== get all relevant variables ====== #
    if var_list is not None:
      all_variables = as_tuple(var_list, t=is_variable)
    else:
      all_variables = ComputationGraph(loss).variables
    # ====== filtering by Roles ====== #
    trainable = [v for v in all_variables
                 if has_roles(v, roles=roles)]
    if isinstance(exclude_roles, Role) or len(exclude_roles) > 0:
      trainable = [v for v in trainable
                   if not has_roles(v, roles=exclude_roles)]
    # ====== filtering by dtype ====== #
    # remove all boolean and string dtype
    trainable = [v
                 for v in trainable
                 if 'int' in str(v.dtype) or 'float' in str(v.dtype)]
    # ====== verbose ====== #
    if bool(verbose):
      # print loss first
      print("Loss:", ctext(str(loss), 'yellow'))
      # organize variable by role
      role_vars = defaultdict(list)
      for var in trainable:
        # remove all tensorflow collection name string
        roles = [r
                 for r in get_roles(var, return_string=False)
                 if not isinstance(r, string_types)]
        # only select highest role in the hierarchy
        high_roles = []
        for r1 in roles:
          found_ancester = False
          for r2 in roles:
            if r1 != r2 and issubclass(r2, r1):
              found_ancester = True
          if not found_ancester:
            high_roles.append(r1)
        # append to mapping role -> var_list
        for r in roles:
          role_vars[r.__name__].append(var)
      # print debug info (this is ugly but look nice and fun)
      if len(role_vars) > 0:
        max_name_length = str(max(
            max(len(i.name) for i in v)
            for v in role_vars.values()))
        max_shape_length = str(max(
            max(len(str(i.shape.as_list())) for i in v)
            for v in role_vars.values()))

        for role, var_list in sorted(role_vars.items(),
                                     key=lambda x: str(x[0])):
          print('Role:', ctext(role, 'yellow'))
          for var in sorted(var_list, key=lambda x: x.name):
            print(' ',
                  ('%-' + max_name_length + 's') % var.name.replace('/', ' '),
                  ctext(('%-' + max_shape_length + 's') % var.shape.as_list(), 'magenta'),
                  ctext(var.dtype.base_dtype.name, 'cyan'))
    # ====== get the updates ====== #
    updates = self.get_updates(loss_or_grads=loss, params=trainable)
    # with tf.variable_scope(self.name):
    #   updates = self.algorithm.minimize(loss=loss, global_step=self._step,
    #                                     var_list=trainable)
    # ====== re-use variable from previous scope ====== #
    if isinstance(reuse_scope, string_types):
      old_variables = [v for v in get_all_variables(scope=reuse_scope)
                       if has_roles(v, OptimizerVariable)]
      old_variables = sorted(old_variables,
                             key=lambda x: x.name.replace(reuse_scope, ''))
      old_varname = [v.name.replace(reuse_scope, '') for v in old_variables]
      # the hyper-parameters may not be included in `old_variables`
      # so they are needed to be excluded in new_variables
      new_variables = [v for v in get_all_variables(scope=self.name)
                       if has_roles(v, OptimizerVariable)]
      new_variables = sorted(new_variables,
                             key=lambda x: x.name.replace(self.name, ''))
      new_variables = [v for v in new_variables
                       if v.name.replace(self.name, '') in old_varname]
      assert len(old_variables) == len(new_variables), \
      "Number of variables in scope:'%s' is %d mismatch scope:'%s' with %d variables"\
      % (reuse_scope, len(old_variables), self.name, len(new_variables))
      for v_old, v_new in zip(old_variables, new_variables):
        assert v_new.shape == v_old.shape
        set_value(v_new, get_value(v_old), return_ops=False)
    return updates
Пример #6
0
 def _initialize(self, X):
   # ====== check inputs dimensions ====== #
   if not hasattr(X, 'shape'):
     raise ValueError("`X` must have `shape` attribute.")
   feat_dim = np.prod(X.shape[1:])
   if self._feat_dim is None:
     self._feat_dim = feat_dim
   # validate input dimension
   if feat_dim != self._feat_dim:
     raise RuntimeError("Feature dimension mismatch %d and %d" %
                        (feat_dim, self.feat_dim))
   # check if tensorflow op initalized
   if hasattr(self, '_f_train'):
     return
   # ====== binary or multi-classes ====== #
   if self.nb_classes == 2:
     out_shape = (None,)
     fn_activation = tf.nn.sigmoid
     fn_loss = tf.losses.sigmoid_cross_entropy
     fn_acc = K.metrics.binary_accuracy
   else:
     out_shape = (None, self.nb_classes)
     fn_activation = tf.nn.softmax
     fn_loss = tf.losses.softmax_cross_entropy
     fn_acc = K.metrics.categorical_accuracy
   # ====== create model ====== #
   with tf.name_scope(self.name, 'logistic_regression'):
     # inputs
     self._X = K.placeholder(shape=(None, self.feat_dim),
                             dtype=self.dtype,
                             name='%s_input' % self.name)
     self._y = K.placeholder(shape=out_shape,
                             dtype=self.dtype,
                             name='%s_output' % self.name)
     # check the bias
     if is_number(self.fit_intercept):
       b_init = float(self.fit_intercept)
     elif self.fit_intercept is False or \
     self.fit_intercept is None:
       b_init = None
     else:
       b_init = self.fit_intercept
     # create the model and initialize
     with K.variable_dtype(dtype=self.dtype):
       self._model = N.Dense(num_units=self.nb_classes,
                         W_init=init_ops.glorot_uniform_initializer(seed=self._rand_state.randint()),
                         b_init=b_init,
                         activation=K.linear)
       y_logits = self._model(self._X)
     y_prob = fn_activation(y_logits)
     # applying class weights
     class_weights = tf.constant(value=self._class_weight,
                                 dtype=self.dtype,
                                 name="class_weights")
     weights = tf.gather(class_weights,
                         tf.cast(self._y, 'int32') if self.nb_classes == 2 else
                         tf.argmax(self._y, axis=-1))
     # optimizer
     params = [v for v in self._model.variables
               if has_roles(v, Weight) or has_roles(v, Bias)]
     losses = fn_loss(self._y, y_logits, weights=weights)
     l1_norm = tf.norm(self._model.get('W'), ord=1) if self.l1 > 0. else 0
     l2_norm = tf.norm(self._model.get('W'), ord=2) if self.l2 > 0. else 0
     losses = losses + self.l1 * l1_norm + self.l2 * l2_norm
     acc = fn_acc(self._y, y_prob)
     updates = self._optimizer.get_updates(losses, params)
     # create function
     if self.confusion_matrix:
       cm = K.metrics.confusion_matrix(y_true=self._y, y_pred=y_prob,
                                       labels=self.nb_classes)
     metrics = [losses, acc, cm] if self.confusion_matrix else [losses, acc]
     self._f_train = K.function(inputs=(self._X, self._y),
                                outputs=metrics,
                                updates=updates,
                                training=True)
     self._f_score = K.function(inputs=(self._X, self._y),
                                outputs=metrics,
                                training=False)
     self._f_pred_prob = K.function(inputs=self._X,
                                    outputs=y_prob,
                                    training=False)
     self._f_pred_logit = K.function(inputs=self._X,
                                     outputs=y_logits,
                                     training=False)
   return self