def testTensorBoardDebuggerWrapperDisablingTracebackSourceSendingWorks(self):
    with session.Session(config=no_rewrite_session_config()) as sess:
      v_1 = variables.Variable(50.0, name="v_1")
      v_2 = variables.Variable(-50.0, name="v_2")
      delta_1 = constant_op.constant(5.0, name="delta_1")
      delta_2 = constant_op.constant(-5.0, name="delta_2")
      inc_v_1 = state_ops.assign_add(v_1, delta_1, name="inc_v_1")
      inc_v_2 = state_ops.assign_add(v_2, delta_2, name="inc_v_2")

      sess.run(variables.global_variables_initializer())

      # Disable the sending of traceback and source code.
      sess = grpc_wrapper.TensorBoardDebugWrapperSession(
          sess, self._debug_server_url_1, send_traceback_and_source_code=False)

      for i in xrange(4):
        self._server_1.clear_data()

        if i == 0:
          self._server_1.request_watch(
              "delta_1", 0, "DebugIdentity", breakpoint=True)

        output = sess.run([inc_v_1, inc_v_2])
        self.assertAllClose([50.0 + 5.0 * (i + 1), -50 - 5.0 * (i + 1)], output)

        # No op traceback or source code should have been received by the debug
        # server due to the disabling above.
        with self.assertRaisesRegexp(
            ValueError, r"Op .*delta_1.* does not exist"):
          self.assertTrue(self._server_1.query_op_traceback("delta_1"))
        with self.assertRaisesRegexp(
            ValueError, r".* has not received any source file"):
          self._server_1.query_source_file_line(__file__, 1)
示例#2
0
 def _dense_moving_average(self, x_tm1, b_t, name, beta=.9):
   """
   Creates a moving average for a dense variable.
   
   Inputs:
     x_tm1: the associated parameter (e.g. a weight matrix)
     b_t: the value to accumulate (e.g. the gradient)
     name: a string to use to retrieve it later (e.g. 'm')
     beta: the decay factor (defaults to .9)
   Outputs:
     a_t: the average after moving
     t: the internal timestep (used to correct initialization bias)
   """
   
   a_tm1 = self.get_slot(x_tm1, '%s' % name)
   tm1 = self.get_slot(x_tm1, '%s/tm1' % name)
   t = state_ops.assign_add(tm1, 1, use_locking = self._use_locking)
   if beta < 1:
     beta_t = ops.convert_to_tensor(beta, name='%s/decay' % name)
     beta_t = beta_t * (1-beta**tm1) / (1-beta**t)
   else:
     beta_t = tm1 / t
   a_t = state_ops.assign(a_tm1, beta_t*a_tm1, use_locking=self._use_locking)
   a_t = state_ops.assign_add(a_t, (1-beta_t)*b_t, use_locking=self._use_locking)
   return a_t, t
示例#3
0
 def adder(x, y):
   state_ops.assign_add(step, 1)
   summary_ops.generic('x', x)
   summary_ops.generic('y', y)
   sum_ = x + y
   summary_ops.generic('sum', sum_)
   return sum_
  def testClusterSpecPropagationThreeServers2Graphs(self):
    """Boots 3 servers, creates 2 sessions, ensures appropriate operations.

    We create 2 clusterspecs:
     1. server2 as the master, server1 as a worker
     2. server2 as the master, server3 as a worker

    We ensure that variables on the workers are independent.
    """
    server1 = server_lib.Server.create_local_server()
    server2 = server_lib.Server.create_local_server()
    server3 = server_lib.Server.create_local_server()
    cluster_def1 = cluster_pb2.ClusterDef()
    job1 = cluster_def1.job.add()
    job1.name = 'worker1'
    job1.tasks[0] = server2.target[len('grpc://'):]
    job1.tasks[1] = server1.target[len('grpc://'):]

    cluster_def2 = cluster_pb2.ClusterDef()
    job2 = cluster_def2.job.add()
    job2.name = 'worker2'
    job2.tasks[0] = server2.target[len('grpc://'):]
    job2.tasks[1] = server3.target[len('grpc://'):]

    config1 = config_pb2.ConfigProto(cluster_def=cluster_def1)
    config2 = config_pb2.ConfigProto(cluster_def=cluster_def2)

    with ops.Graph().as_default() as g1:
      with ops.device('/job:worker1/task:1'):
        var1 = variables.Variable(array_ops.zeros([2]), name='var1')
        update_op1 = state_ops.assign_add(
            var1, array_ops.ones([2]), name='var1_assign_add')
        init1 = variables.global_variables_initializer()

    with ops.Graph().as_default() as g2:
      with ops.device('/job:worker2/task:1'):
        var2 = variables.Variable(array_ops.zeros([2]), name='var2')
        update_op2 = state_ops.assign_add(
            var2, array_ops.ones([2]), name='var2_assign_add')
        init2 = variables.global_variables_initializer()

    sess1 = session.Session(server2.target, graph=g1, config=config1)
    sess2 = session.Session(server2.target, graph=g2, config=config2)

    init1.run(session=sess1)
    init2.run(session=sess2)

    expected_zeros = np.zeros([2])
    expected_ones = np.ones([2])

    self.assertAllEqual(expected_zeros, sess1.run(var1))
    self.assertAllEqual(expected_zeros, sess2.run(var2))

    self.assertAllEqual(expected_ones, sess1.run(update_op1))
    self.assertAllEqual(expected_ones, sess1.run(var1))
    self.assertAllEqual(expected_zeros, sess2.run(var2))
    self.assertAllEqual(expected_ones, sess2.run(update_op2))
    self.assertAllEqual(expected_ones + expected_ones, sess1.run(update_op1))
    self.assertAllEqual(expected_ones, sess2.run(var2))
    self.assertAllEqual(expected_ones + expected_ones, sess1.run(var1))
  def test_train_skip_train_if_max_step_already_saved(self):
    with ops.Graph().as_default() as g, self.test_session(g):
      with ops.control_dependencies(self._build_inference_graph()):
        train_op = state_ops.assign_add(variables_lib.get_global_step(), 1)
      learn.graph_actions._monitored_train(  # pylint: disable=protected-access
          g,
          output_dir=self._output_dir,
          train_op=train_op,
          loss_op=constant_op.constant(2.0),
          max_steps=10)
      step = checkpoint_utils.load_variable(
          self._output_dir, variables_lib.get_global_step().name)
      self.assertEqual(10, step)

    with ops.Graph().as_default() as g, self.test_session(g):
      with ops.control_dependencies(self._build_inference_graph()):
        train_op = state_ops.assign_add(variables_lib.get_global_step(), 1)
      learn.graph_actions._monitored_train(  # pylint: disable=protected-access
          g,
          output_dir=self._output_dir,
          train_op=train_op,
          loss_op=constant_op.constant(2.0),
          max_steps=10)
      step = checkpoint_utils.load_variable(
          self._output_dir, variables_lib.get_global_step().name)
      self.assertEqual(10, step)
  def test_train_max_steps_is_not_incremental(self):
    with ops.Graph().as_default() as g, self.test_session(g):
      with ops.control_dependencies(self._build_inference_graph()):
        train_op = state_ops.assign_add(variables_lib.get_global_step(), 1)
      learn.graph_actions.train(
          g,
          output_dir=self._output_dir,
          train_op=train_op,
          loss_op=constant_op.constant(2.0),
          max_steps=10)
      step = checkpoint_utils.load_variable(
          self._output_dir, variables_lib.get_global_step().name)
      self.assertEqual(10, step)

    with ops.Graph().as_default() as g, self.test_session(g):
      with ops.control_dependencies(self._build_inference_graph()):
        train_op = state_ops.assign_add(variables_lib.get_global_step(), 1)
      learn.graph_actions.train(
          g,
          output_dir=self._output_dir,
          train_op=train_op,
          loss_op=constant_op.constant(2.0),
          max_steps=15)
      step = checkpoint_utils.load_variable(
          self._output_dir, variables_lib.get_global_step().name)
      self.assertEqual(15, step)
def model_fn_diff_modes(features, labels, mode):
  _, _ = features, labels
  v = variables.Variable(21, name='some_var')
  train_op = None
  loss = constant_op.constant(104)
  if mode == model_fn_lib.ModeKeys.TRAIN:
    loss = constant_op.constant(105)
    predictions = constant_op.constant([501])
    train_op = control_flow_ops.group(
        state_ops.assign_add(training.get_global_step(), 1),
        state_ops.assign_add(v, 3))
  elif mode == model_fn_lib.ModeKeys.EVAL:
    loss = constant_op.constant(106)
    predictions = constant_op.constant([502])
  else:
    loss = constant_op.constant(107)
    predictions = constant_op.constant([503])
  return model_fn_lib.EstimatorSpec(
      mode,
      loss=loss,
      train_op=train_op,
      eval_metric_ops={
          'abs_err': metrics_lib.mean_absolute_error(
              constant_op.constant(0), predictions)},
      predictions=predictions)
示例#8
0
  def update_state(self, values, sample_weight=None):
    """Accumulates statistics for computing the mean.

    For example, if `values` is [1, 3, 5, 7] then the mean is 4. If
    the `sample_weight` is specified as [1, 1, 0, 0] then the mean would be 2.

    Args:
      values: Per-example value.
      sample_weight: Optional weighting of each example. Defaults to 1.
    """
    values = math_ops.cast(values, self._dtype)
    if sample_weight is None:
      num_values = math_ops.cast(array_ops.size(values), self._dtype)
    else:
      sample_weight = math_ops.cast(sample_weight, self._dtype)

      # Update dimensions of weights to match with values.
      values, _, sample_weight = _squeeze_or_expand_dimensions(
          values, None, sample_weight)
      sample_weight = weights_broadcast_ops.broadcast_weights(
          sample_weight, values)
      num_values = math_ops.reduce_sum(sample_weight)
      values = math_ops.multiply(values, sample_weight)
    values = math_ops.reduce_sum(values)

    # Update state variables
    state_ops.assign_add(self.total, values)
    state_ops.assign_add(self.count, num_values)
示例#9
0
  def testMultiEvalStepIncrements(self):
    checkpoint_dir = os.path.join(self.get_temp_dir(), 'eval_ops_and_final_ops')

    # Train a model for a single step to get a checkpoint.
    self._train_model(checkpoint_dir, num_steps=1)
    checkpoint_path = saver.latest_checkpoint(checkpoint_dir)

    # Create the model so we have something to restore.
    inputs = constant_op.constant(self._inputs, dtype=dtypes.float32)
    logistic_classifier(inputs)

    num_evals = 6

    my_var = local_variable(0.0, name='MyVar')
    # In eval ops, we also increase the eval step one more time.
    eval_ops = [state_ops.assign_add(my_var, 1.0),
                state_ops.assign_add(
                    evaluation._get_or_create_eval_step(), 1, use_locking=True)]
    expect_eval_update_counts = num_evals // 2

    final_ops = array_ops.identity(my_var)

    final_ops_values = evaluation._evaluate_once(
        checkpoint_path=checkpoint_path,
        eval_ops=eval_ops,
        final_ops={'value': final_ops},
        hooks=[evaluation._StopAfterNEvalsHook(num_evals),])
    self.assertEqual(final_ops_values['value'], expect_eval_update_counts)
 def _Update_global_variables():
   local_vars = [v for g, v in grads_and_vars if g is not None]
   global_center_vars = [self._global_map[var] for var in local_vars]
   local_center_vars = [self._local_map[var] for var in local_vars]
   local_center_vars_update = []
   for lvar, var in zip(local_center_vars, global_center_vars):
     local_center_vars_update.append(lvar.assign(var))
   update_ops = []
   differences = []
   with ops.control_dependencies(local_center_vars_update):
     for v, lv in zip(local_vars, local_center_vars):
       with ops.device(v.device):
         differences.append(math_ops.subtract(v, lv))
     for lvar, diff in zip(local_vars, differences):
       with ops.device(lvar.device):
         update_ops.append(
             state_ops.assign_sub(lvar,
                                  math_ops.multiply(self._moving_rate,
                                                    diff)))
     for var, diff in zip(global_center_vars, differences):
       with ops.device(var.device):
         update_ops.append(
             state_ops.assign_add(var,
                                  math_ops.multiply(self._moving_rate,
                                                    diff)))
     if global_step:
       with ops.colocate_with(global_step):
         update_ops.append(state_ops.assign_add(global_step, 1))
   variable_update = control_flow_ops.group(*(update_ops))
   return variable_update
  def testToggleBreakpointsWorks(self):
    with session.Session(
        config=session_debug_testlib.no_rewrite_session_config()) as sess:
      v_1 = variables.VariableV1(50.0, name="v_1")
      v_2 = variables.VariableV1(-50.0, name="v_2")
      delta_1 = constant_op.constant(5.0, name="delta_1")
      delta_2 = constant_op.constant(-5.0, name="delta_2")
      inc_v_1 = state_ops.assign_add(v_1, delta_1, name="inc_v_1")
      inc_v_2 = state_ops.assign_add(v_2, delta_2, name="inc_v_2")

      sess.run([v_1.initializer, v_2.initializer])

      run_metadata = config_pb2.RunMetadata()
      run_options = config_pb2.RunOptions(output_partition_graphs=True)
      debug_utils.watch_graph(
          run_options,
          sess.graph,
          debug_ops=["DebugIdentity(gated_grpc=true)"],
          debug_urls=[self._debug_server_url_1])

      for i in xrange(4):
        self._server_1.clear_data()

        if i in (0, 2):
          # Enable breakpoint at delta_[1,2]:0:DebugIdentity in runs 0 and 2.
          self._server_1.request_watch(
              "delta_1", 0, "DebugIdentity", breakpoint=True)
          self._server_1.request_watch(
              "delta_2", 0, "DebugIdentity", breakpoint=True)
        else:
          # Disable the breakpoint in runs 1 and 3.
          self._server_1.request_unwatch("delta_1", 0, "DebugIdentity")
          self._server_1.request_unwatch("delta_2", 0, "DebugIdentity")

        output = sess.run([inc_v_1, inc_v_2],
                          options=run_options, run_metadata=run_metadata)
        self.assertAllClose([50.0 + 5.0 * (i + 1), -50 - 5.0 * (i + 1)], output)

        if i in (0, 2):
          # During runs 0 and 2, the server should have received the published
          # debug tensor delta:0:DebugIdentity. The breakpoint should have been
          # unblocked by EventReply reponses from the server.
          self.assertAllClose(
              [5.0],
              self._server_1.debug_tensor_values["delta_1:0:DebugIdentity"])
          self.assertAllClose(
              [-5.0],
              self._server_1.debug_tensor_values["delta_2:0:DebugIdentity"])
          # After the runs, the server should have properly registered the
          # breakpoints due to the request_unwatch calls.
          self.assertSetEqual({("delta_1", 0, "DebugIdentity"),
                               ("delta_2", 0, "DebugIdentity")},
                              self._server_1.breakpoints)
        else:
          # After the end of runs 1 and 3, the server has received the requests
          # to disable the breakpoint at delta:0:DebugIdentity.
          self.assertSetEqual(set(), self._server_1.breakpoints)
示例#12
0
 def _model_fn_with_incremental_loss(features, labels, mode):
   _, _ = features, labels
   local_weight = variables.Variable(
       0., name='local_weight', collections=[ops.GraphKeys.LOCAL_VARIABLES])
   # Loss will be 2, 4, 6, ...
   loss = 2 * state_ops.assign_add(local_weight, 1.)
   return model_fn_lib.EstimatorSpec(
       mode,
       loss=loss,
       train_op=state_ops.assign_add(training.get_global_step(), 1))
 def model_fn(features, labels, mode):
   _, _ = features, labels
   v = variables.Variable(21, name='some_var')
   scaffold = monitored_session.Scaffold(
       local_init_op=state_ops.assign_add(v, -3).op
   )
   return model_fn_lib.EstimatorSpec(
       mode,
       scaffold=scaffold,
       train_op=state_ops.assign_add(training.get_global_step(), 1),
       loss=array_ops.identity(v))
示例#14
0
  def setUp(self):
    self.session_root = tempfile.mkdtemp()

    self.v = variables.Variable(10.0, dtype=dtypes.float32, name="v")
    self.delta = constant_op.constant(1.0, dtype=dtypes.float32, name="delta")
    self.eta = constant_op.constant(-1.4, dtype=dtypes.float32, name="eta")
    self.inc_v = state_ops.assign_add(self.v, self.delta, name="inc_v")
    self.dec_v = state_ops.assign_add(self.v, self.eta, name="dec_v")

    self.sess = session.Session()
    self.sess.run(self.v.initializer)
示例#15
0
  def test_get_updates_for(self):
    a = keras.layers.Input(shape=(1,))
    dense_layer = keras.layers.Dense(1)
    dense_layer.build((None, 1))
    update_1 = state_ops.assign_add(dense_layer.kernel, a)
    update_2 = state_ops.assign_add(dense_layer.kernel, [[1.]])
    dense_layer.add_update(update_1, inputs=a)
    dense_layer.add_update(update_2, inputs=None)

    self.assertListEqual(dense_layer.get_updates_for(a), [update_1])
    self.assertListEqual(dense_layer.get_updates_for(None), [update_2])
  def setUp(self):
    test.TestCase.setUp(self)

    self.log_dir = 'log/dir'
    self.summary_writer = fake_summary_writer.FakeSummaryWriter(self.log_dir)

    var = variable_scope.get_variable('var', initializer=0.0, use_resource=True)
    tensor = state_ops.assign_add(var, 1.0)
    self.summary_op = summary_lib.scalar('my_summary', tensor)

    with variable_scope.variable_scope('foo', use_resource=True):
      global_step = variables.get_or_create_global_step()
    self.train_op = state_ops.assign_add(global_step, 1)
  def setUp(self):
    test.TestCase.setUp(self)

    self.log_dir = 'log/dir'
    self.summary_writer = fake_summary_writer.FakeSummaryWriter(self.log_dir)

    var = variables_lib.Variable(0.0)
    tensor = state_ops.assign_add(var, 1.0)
    tensor2 = tensor * 2
    self.summary_op = summary_lib.scalar('my_summary', tensor)
    self.summary_op2 = summary_lib.scalar('my_summary2', tensor2)

    global_step = variables.get_or_create_global_step()
    self.train_op = state_ops.assign_add(global_step, 1)
  def setUp(self):
    self.session_root = tempfile.mkdtemp()

    self.v = variables.VariableV1(10.0, dtype=dtypes.float32, name="v")
    self.delta = constant_op.constant(1.0, dtype=dtypes.float32, name="delta")
    self.eta = constant_op.constant(-1.4, dtype=dtypes.float32, name="eta")
    self.inc_v = state_ops.assign_add(self.v, self.delta, name="inc_v")
    self.dec_v = state_ops.assign_add(self.v, self.eta, name="dec_v")

    self.ph = array_ops.placeholder(dtypes.float32, shape=(), name="ph")
    self.inc_w_ph = state_ops.assign_add(self.v, self.ph, name="inc_w_ph")

    self.sess = session.Session()
    self.sess.run(self.v.initializer)
  def testTensorBoardDebuggerWrapperToggleBreakpointsWorks(self):
    with session.Session(config=no_rewrite_session_config()) as sess:
      v_1 = variables.Variable(50.0, name="v_1")
      v_2 = variables.Variable(-50.0, name="v_2")
      delta_1 = constant_op.constant(5.0, name="delta_1")
      delta_2 = constant_op.constant(-5.0, name="delta_2")
      inc_v_1 = state_ops.assign_add(v_1, delta_1, name="inc_v_1")
      inc_v_2 = state_ops.assign_add(v_2, delta_2, name="inc_v_2")

      sess.run([v_1.initializer, v_2.initializer])

      # The TensorBoardDebugWrapperSession should add a DebugIdentity debug op
      # with attribute gated_grpc=True for every tensor in the graph.
      sess = grpc_wrapper.TensorBoardDebugWrapperSession(
          sess, self._debug_server_url_1)

      for i in xrange(4):
        self._server_1.clear_data()

        if i in (0, 2):
          # Enable breakpoint at delta_[1,2]:0:DebugIdentity in runs 0 and 2.
          self._server_1.request_watch(
              "delta_1", 0, "DebugIdentity", breakpoint=True)
          self._server_1.request_watch(
              "delta_2", 0, "DebugIdentity", breakpoint=True)
        else:
          # Disable the breakpoint in runs 1 and 3.
          self._server_1.request_unwatch("delta_1", 0, "DebugIdentity")
          self._server_1.request_unwatch("delta_2", 0, "DebugIdentity")

        output = sess.run([inc_v_1, inc_v_2])
        self.assertAllClose([50.0 + 5.0 * (i + 1), -50 - 5.0 * (i + 1)], output)

        if i in (0, 2):
          # During runs 0 and 2, the server should have received the published
          # debug tensor delta:0:DebugIdentity. The breakpoint should have been
          # unblocked by EventReply reponses from the server.
          self.assertAllClose(
              [5.0],
              self._server_1.debug_tensor_values["delta_1:0:DebugIdentity"])
          self.assertAllClose(
              [-5.0],
              self._server_1.debug_tensor_values["delta_2:0:DebugIdentity"])
          # After the runs, the server should have properly registered the
          # breakpoints.
        else:
          # After the end of runs 1 and 3, the server has received the requests
          # to disable the breakpoint at delta:0:DebugIdentity.
          self.assertSetEqual(set(), self._server_1.breakpoints)
  def _createGraph(self):
    """Create graph for testing.

    Returns:
      Python Graph object.
    """
    with ops.Graph().as_default() as graph:
      with ops.device("/job:worker/task:0/cpu:0"):
        self.a = variables.VariableV1(10.0, name="a")
        self.b = variables.VariableV1(100.0, name="b")
        self.inc_a = state_ops.assign_add(self.a, 2.0, name="inc_a")
        self.dec_b = state_ops.assign_add(self.b, -5.0, name="dec_b")
        self.p = math_ops.multiply(self.inc_a, self.dec_b, name="p")
        self.q = math_ops.negative(self.p, name="q")
    return graph
示例#21
0
  def _minimize(loss, global_step):
    trainable_vars = ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)
    testcase.assertItemsEqual(
        expected_var_names,
        [var.name for var in trainable_vars])

    # Verify loss. We can't check the value directly, so we add an assert op.
    testcase.assertEquals(0, loss.shape.ndims)
    if expected_loss is None:
      return state_ops.assign_add(global_step, 1).op
    assert_loss = _assert_close(
        math_ops.to_float(expected_loss, name='expected'), loss,
        name='assert_loss')
    with ops.control_dependencies((assert_loss,)):
      return state_ops.assign_add(global_step, 1).op
示例#22
0
 def _train_op_fn(loss):
   """Returns the op to optimize the loss."""
   update_op = gbdt_model_main.train(loss, predictions_dict, labels)
   with ops.control_dependencies(
       [update_op]), (ops.colocate_with(global_step)):
     update_op = state_ops.assign_add(global_step, 1).op
     return update_op
  def test_global_step_name(self):
    with ops.Graph().as_default() as g, session_lib.Session() as sess:
      with variable_scope.variable_scope('bar'):
        foo_step = variable_scope.get_variable(
            'foo',
            initializer=0,
            trainable=False,
            collections=[
                ops.GraphKeys.GLOBAL_STEP, ops.GraphKeys.GLOBAL_VARIABLES
            ])
      train_op = state_ops.assign_add(foo_step, 1)
      summary_writer = fake_summary_writer.FakeSummaryWriter(self.log_dir, g)
      hook = basic_session_run_hooks.StepCounterHook(
          summary_writer=summary_writer, every_n_steps=1, every_n_secs=None)

      hook.begin()
      sess.run(variables_lib.global_variables_initializer())
      mon_sess = monitored_session._HookedSession(sess, [hook])
      mon_sess.run(train_op)
      mon_sess.run(train_op)
      hook.end(sess)

      summary_writer.assert_summaries(
          test_case=self,
          expected_logdir=self.log_dir,
          expected_graph=g,
          expected_summaries={})
      self.assertTrue(summary_writer.summaries, 'No summaries were created.')
      self.assertItemsEqual([2], summary_writer.summaries.keys())
      summary_value = summary_writer.summaries[2][0].value[0]
      self.assertEqual('bar/foo/sec', summary_value.tag)
示例#24
0
  def _get_train_ops(self, features, targets):
    """Method that builds model graph and returns trainer ops.

    Args:
      features: `Tensor` or `dict` of `Tensor` objects.
      targets: `Tensor` or `dict` of `Tensor` objects.

    Returns:
      Tuple of train `Operation` and loss `Tensor`.
    """
    features, spec = data_ops.ParseDataTensorOrDict(features)
    labels = data_ops.ParseLabelTensorOrDict(targets)

    graph_builder = self.graph_builder_class(
        self.params, device_assigner=self.device_assigner,
        **self.construction_args)

    epoch = None
    if self.data_feeder:
      epoch = self.data_feeder.make_epoch_variable()

    train = control_flow_ops.group(
        graph_builder.training_graph(
            features, labels, data_spec=spec, epoch=epoch,
            **self.training_args),
        state_ops.assign_add(contrib_framework.get_global_step(), 1))

    self.training_loss = graph_builder.training_loss()

    return train, self.training_loss
 def setUp(self):
   self.model_dir = tempfile.mkdtemp()
   self.graph = ops.Graph()
   with self.graph.as_default():
     self.scaffold = monitored_session.Scaffold()
     self.global_step = variables.get_or_create_global_step()
     self.train_op = state_ops.assign_add(self.global_step, 1)
  def test_two_listeners_with_default_saver(self):
    with ops.Graph().as_default():
      global_step = variables.get_or_create_global_step()
      train_op = state_ops.assign_add(global_step, 1)
      listener1 = MockCheckpointSaverListener()
      listener2 = MockCheckpointSaverListener()
      hook = basic_session_run_hooks.CheckpointSaverHook(
          self.model_dir,
          save_steps=1,
          listeners=[listener1, listener2])
      with monitored_session.SingularMonitoredSession(
          hooks=[hook],
          checkpoint_dir=self.model_dir) as sess:
        sess.run(train_op)
        sess.run(train_op)
        global_step_val = sess.run(global_step)
      listener1_counts = listener1.get_counts()
      listener2_counts = listener2.get_counts()
    self.assertEqual(2, global_step_val)
    self.assertEqual({
        'begin': 1,
        'before_save': 2,
        'after_save': 2,
        'end': 1
    }, listener1_counts)
    self.assertEqual(listener1_counts, listener2_counts)

    with ops.Graph().as_default():
      global_step = variables.get_or_create_global_step()
      with monitored_session.SingularMonitoredSession(
          checkpoint_dir=self.model_dir) as sess2:
        global_step_saved_val = sess2.run(global_step)
    self.assertEqual(2, global_step_saved_val)
 def test_listener_with_monitored_session(self):
   with ops.Graph().as_default():
     scaffold = monitored_session.Scaffold()
     global_step = variables.get_or_create_global_step()
     train_op = state_ops.assign_add(global_step, 1)
     listener = MockCheckpointSaverListener()
     hook = basic_session_run_hooks.CheckpointSaverHook(
         self.model_dir,
         save_steps=1,
         scaffold=scaffold,
         listeners=[listener])
     with monitored_session.SingularMonitoredSession(
         hooks=[hook],
         scaffold=scaffold,
         checkpoint_dir=self.model_dir) as sess:
       sess.run(train_op)
       sess.run(train_op)
       global_step_val = sess.run(global_step)
     listener_counts = listener.get_counts()
   self.assertEqual(2, global_step_val)
   self.assertEqual({
       'begin': 1,
       'before_save': 2,
       'after_save': 2,
       'end': 1
   }, listener_counts)
示例#28
0
  def get_updates(self, loss, params):
    if distribute_lib.has_distribution_strategy():
      self.updates = []

      if not params:
        # After the model vars have been created, the second call to get_updates
        # is called with params as an empty list. This ensures that we call
        # compute_gradients with params=None.
        grads = self.optimizer.compute_gradients(loss)
      else:
        grads = self.optimizer.compute_gradients(loss, params)
      global_step = training_util.get_global_step()
      opt_update = self.optimizer.apply_gradients(grads, global_step)
    else:
      if not params:
        self.updates = [state_ops.assign_add(self.iterations, 1)]
        return self.updates

      # Updates list starts out empty because the iterations variable is
      # incremented in optimizer.apply_gradients()
      self.updates = []
      grads = self.optimizer.compute_gradients(loss, params)
      opt_update = self.optimizer.apply_gradients(
          grads, global_step=self.iterations)

    self.updates.append(opt_update)
    return self.updates
示例#29
0
  def get_updates(self, loss, params):
    grads = self.get_gradients(loss, params)
    shapes = [K.int_shape(p) for p in params]
    accumulators = [K.zeros(shape) for shape in shapes]
    delta_accumulators = [K.zeros(shape) for shape in shapes]
    self.weights = accumulators + delta_accumulators
    self.updates = [state_ops.assign_add(self.iterations, 1)]

    lr = self.lr
    if self.initial_decay > 0:
      lr = lr * (  # pylint: disable=g-no-augmented-assignment
          1. / (1. + self.decay * math_ops.cast(self.iterations,
                                                K.dtype(self.decay))))

    for p, g, a, d_a in zip(params, grads, accumulators, delta_accumulators):
      # update accumulator
      new_a = self.rho * a + (1. - self.rho) * math_ops.square(g)
      self.updates.append(state_ops.assign(a, new_a))

      # use the new accumulator and the *old* delta_accumulator
      update = g * K.sqrt(d_a + self.epsilon) / K.sqrt(new_a + self.epsilon)
      new_p = p - lr * update

      # Apply constraints.
      if getattr(p, 'constraint', None) is not None:
        new_p = p.constraint(new_p)

      self.updates.append(state_ops.assign(p, new_p))

      # update delta_accumulator
      new_d_a = self.rho * d_a + (1 - self.rho) * math_ops.square(update)
      self.updates.append(state_ops.assign(d_a, new_d_a))
    return self.updates
  def test_step_counter_every_n_secs(self):
    with ops.Graph().as_default() as g, session_lib.Session() as sess:
      global_step = variables.get_or_create_global_step()
      train_op = state_ops.assign_add(global_step, 1)
      summary_writer = fake_summary_writer.FakeSummaryWriter(self.log_dir, g)
      hook = basic_session_run_hooks.StepCounterHook(
          summary_writer=summary_writer, every_n_steps=None, every_n_secs=0.1)

      hook.begin()
      sess.run(variables_lib.global_variables_initializer())
      mon_sess = monitored_session._HookedSession(sess, [hook])
      mon_sess.run(train_op)
      time.sleep(0.2)
      mon_sess.run(train_op)
      time.sleep(0.2)
      mon_sess.run(train_op)
      hook.end(sess)

      summary_writer.assert_summaries(
          test_case=self,
          expected_logdir=self.log_dir,
          expected_graph=g,
          expected_summaries={})
      self.assertTrue(summary_writer.summaries, 'No summaries were created.')
      self.assertItemsEqual([2, 3], summary_writer.summaries.keys())
      for summary in summary_writer.summaries.values():
        summary_value = summary[0].value[0]
        self.assertEqual('global_step/sec', summary_value.tag)
        self.assertGreater(summary_value.simple_value, 0)
示例#31
0
 def testAssignUpdateNoValueShape(self):
     var = state_ops.variable_op([1, 2], dtypes.float32)
     added = state_ops.assign_add(var, self._NewShapelessTensor())
     self.assertEqual([1, 2], added.get_shape())
     subbed = state_ops.assign_sub(var, self._NewShapelessTensor())
     self.assertEqual([1, 2], subbed.get_shape())
示例#32
0
 def testAssignUpdateNoShape(self):
     var = state_ops.variable_op([1, 2], dtypes.float32, set_shape=False)
     added = state_ops.assign_add(var, self._NewShapelessTensor())
     self.assertEqual(tensor_shape.unknown_shape(), added.get_shape())
     subbed = state_ops.assign_sub(var, self._NewShapelessTensor())
     self.assertEqual(tensor_shape.unknown_shape(), subbed.get_shape())
    def apply_gradients(self, grads_and_vars, global_step=None, name=None):
        """Apply gradients to global variables.

    This is the second part of `minimize()`. It returns an `Operation` that
    applies gradients.

    Args:
      grads_and_vars: List of (gradient, variable) pairs as returned by
        `compute_gradients()`.
      global_step: Optional `Variable` to increment by one after the variables
        have been updated.
      name: Optional name for the returned operation.  Default to the name
        passed to the `Optimizer` constructor.

    Returns:
      An `Operation` that applies the specified gradients. If `global_step`
      was not None, that operation also increments `global_step`.

    Raises:
      TypeError: If `grads_and_vars` is malformed.
      ValueError: If none of the variables have gradients.
    """
        global_old = set(n.op.name for n in variables.global_variables())
        apply_updates = self._opt.apply_gradients(grads_and_vars)
        global_new = set(n.op.name for n in variables.global_variables())
        with ops.control_dependencies([apply_updates]):
            local_update = state_ops.assign_add(self._local_step,
                                                1,
                                                name='local_step_update').op

        # this is for place the variables created by optimizer to local collection
        # e.g., AdamOptimizer will create beta as global variables
        def _adjust_optimizer_variable_collection(opt_vars):
            g = ops.get_default_graph()
            idx = 0
            for _ in range(len(
                    g._collections[ops.GraphKeys.GLOBAL_VARIABLES])):
                var = g.get_collection_ref(ops.GraphKeys.GLOBAL_VARIABLES)[idx]
                name = var.op.name
                if name in opt_vars:
                    ops.add_to_collection(ops.GraphKeys.LOCAL_VARIABLES, var)
                    del g.get_collection_ref(
                        ops.GraphKeys.GLOBAL_VARIABLES)[idx]
                else:
                    idx += 1

        _adjust_optimizer_variable_collection(global_new - global_old)

        # update global variables.
        def _Update_global_variables():
            local_vars = [v for g, v in grads_and_vars if g is not None]
            global_center_vars = [self._global_map[var] for var in local_vars]
            local_center_vars = [self._local_map[var] for var in local_vars]
            local_center_vars_update = []
            for lvar, var in zip(local_center_vars, global_center_vars):
                local_center_vars_update.append(lvar.assign(var))
            update_ops = []
            differences = []
            with ops.control_dependencies(local_center_vars_update):
                for v, lv in zip(local_vars, local_center_vars):
                    with ops.device(v.device):
                        differences.append(math_ops.subtract(v, lv))
                for lvar, diff in zip(local_vars, differences):
                    with ops.device(lvar.device):
                        update_ops.append(
                            state_ops.assign_sub(
                                lvar,
                                math_ops.multiply(self._moving_rate, diff)))
                for var, diff in zip(global_center_vars, differences):
                    with ops.device(var.device):
                        update_ops.append(
                            state_ops.assign_add(
                                var, math_ops.multiply(self._moving_rate,
                                                       diff)))
                if global_step:
                    with ops.colocate_with(global_step):
                        update_ops.append(state_ops.assign_add(global_step, 1))
            variable_update = control_flow_ops.group(*(update_ops))
            return variable_update

        with ops.control_dependencies([local_update]):
            condition = math_ops.equal(
                math_ops.mod(self._local_step, self._period), 0)
            conditional_update = control_flow_ops.cond(
                condition, _Update_global_variables, control_flow_ops.no_op)
        return conditional_update
示例#34
0
def streaming_covariance(predictions,
                         labels,
                         weights=None,
                         metrics_collections=None,
                         updates_collections=None,
                         name=None):
    """Computes the unbiased sample covariance between `predictions` and `labels`.
  The `streaming_covariance` function creates four local variables,
  `comoment`, `mean_prediction`, `mean_label`, and `count`, which are used to
  compute the sample covariance between predictions and labels across multiple
  batches of data. The covariance is ultimately returned as an idempotent
  operation that simply divides `comoment` by `count` - 1. We use `count` - 1
  in order to get an unbiased estimate.
  The algorithm used for this online computation is described in
  https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance.
  Specifically, the formula used to combine two sample comoments is
  `C_AB = C_A + C_B + (E[x_A] - E[x_B]) * (E[y_A] - E[y_B]) * n_A * n_B / n_AB`
  The comoment for a single batch of data is simply
  `sum((x - E[x]) * (y - E[y]))`, optionally weighted.
  If `weights` is not None, then it is used to compute weighted comoments,
  means, and count. NOTE: these weights are treated as "frequency weights", as
  opposed to "reliability weights". See discussion of the difference on
  https://wikipedia.org/wiki/Weighted_arithmetic_mean#Weighted_sample_variance
  To facilitate the computation of covariance across multiple batches of data,
  the function creates an `update_op` operation, which updates underlying
  variables and returns the updated covariance.
  Args:
    predictions: A `Tensor` of arbitrary size.
    labels: A `Tensor` of the same size as `predictions`.
    weights: Optional `Tensor` indicating the frequency with which an example is
      sampled. Rank must be 0, or the same rank as `labels`, and must be
      broadcastable to `labels` (i.e., all dimensions must be either `1`, or
      the same as the corresponding `labels` dimension).
    metrics_collections: An optional list of collections that the metric
      value variable should be added to.
    updates_collections: An optional list of collections that the metric update
      ops should be added to.
    name: An optional variable_scope name.
  Returns:
    covariance: A `Tensor` representing the current unbiased sample covariance,
      `comoment` / (`count` - 1).
    update_op: An operation that updates the local variables appropriately.
  Raises:
    ValueError: If labels and predictions are of different sizes or if either
      `metrics_collections` or `updates_collections` are not a list or tuple.
  """
    with variable_scope.variable_scope(name, 'covariance',
                                       (predictions, labels, weights)):
        predictions, labels, weights = metrics_impl._remove_squeezable_dimensions(  # pylint: disable=protected-access
            predictions, labels, weights)
        predictions.get_shape().assert_is_compatible_with(labels.get_shape())
        count_ = metric_variable([], dtypes.float32, name='count')
        mean_prediction = metric_variable([],
                                          dtypes.float32,
                                          name='mean_prediction')
        mean_label = metric_variable([], dtypes.float32, name='mean_label')
        comoment = metric_variable(  # C_A in update equation
            [], dtypes.float32, name='comoment')

        if weights is None:
            batch_count = math_ops.to_float(
                array_ops.size(labels))  # n_B in eqn
            weighted_predictions = predictions
            weighted_labels = labels
        else:
            weights = weights_broadcast_ops.broadcast_weights(weights, labels)
            batch_count = math_ops.reduce_sum(weights)  # n_B in eqn
            weighted_predictions = math_ops.multiply(predictions, weights)
            weighted_labels = math_ops.multiply(labels, weights)

        update_count = state_ops.assign_add(count_, batch_count)  # n_AB in eqn
        prev_count = update_count - batch_count  # n_A in update equation

        # We update the means by Delta=Error*BatchCount/(BatchCount+PrevCount)
        # batch_mean_prediction is E[x_B] in the update equation
        batch_mean_prediction = _safe_div(
            math_ops.reduce_sum(weighted_predictions), batch_count,
            'batch_mean_prediction')
        delta_mean_prediction = _safe_div(
            (batch_mean_prediction - mean_prediction) * batch_count,
            update_count, 'delta_mean_prediction')
        update_mean_prediction = state_ops.assign_add(mean_prediction,
                                                      delta_mean_prediction)
        # prev_mean_prediction is E[x_A] in the update equation
        prev_mean_prediction = update_mean_prediction - delta_mean_prediction

        # batch_mean_label is E[y_B] in the update equation
        batch_mean_label = _safe_div(math_ops.reduce_sum(weighted_labels),
                                     batch_count, 'batch_mean_label')
        delta_mean_label = _safe_div(
            (batch_mean_label - mean_label) * batch_count, update_count,
            'delta_mean_label')
        update_mean_label = state_ops.assign_add(mean_label, delta_mean_label)
        # prev_mean_label is E[y_A] in the update equation
        prev_mean_label = update_mean_label - delta_mean_label

        unweighted_batch_coresiduals = ((predictions - batch_mean_prediction) *
                                        (labels - batch_mean_label))
        # batch_comoment is C_B in the update equation
        if weights is None:
            batch_comoment = math_ops.reduce_sum(unweighted_batch_coresiduals)
        else:
            batch_comoment = math_ops.reduce_sum(unweighted_batch_coresiduals *
                                                 weights)

        # View delta_comoment as = C_AB - C_A in the update equation above.
        # Since C_A is stored in a var, by how much do we need to increment that var
        # to make the var = C_AB?
        delta_comoment = (batch_comoment +
                          (prev_mean_prediction - batch_mean_prediction) *
                          (prev_mean_label - batch_mean_label) *
                          (prev_count * batch_count / update_count))
        update_comoment = state_ops.assign_add(comoment, delta_comoment)

        covariance = array_ops.where(math_ops.less_equal(count_, 1.),
                                     float('nan'),
                                     math_ops.truediv(comoment, count_ - 1),
                                     name='covariance')
        with ops.control_dependencies([update_comoment]):
            update_op = array_ops.where(math_ops.less_equal(count_, 1.),
                                        float('nan'),
                                        math_ops.truediv(comoment, count_ - 1),
                                        name='update_op')

    if metrics_collections:
        ops.add_to_collections(metrics_collections, covariance)

    if updates_collections:
        ops.add_to_collections(updates_collections, update_op)

    return covariance, update_op
示例#35
0
def streaming_mean(values, weights=None, metrics_collections=None,
                   updates_collections=None, name=None):
  """Computes the (weighted) mean of the given values.

  The `streaming_mean` function creates two local variables, `total` and `count`
  that are used to compute the average of `values`. This average is ultimately
  returned as `mean` which is an idempotent operation that simply divides
  `total` by `count`.

  For estimation of the metric  over a stream of data, the function creates an
  `update_op` operation that updates these variables and returns the `mean`.
  `update_op` increments `total` with the reduced sum of the product of `values`
  and `weights`, and it increments `count` with the reduced sum of `weights`.

  If `weights` is `None`, weights default to 1. Use weights of 0 to mask values.

  Args:
    values: A `Tensor` of arbitrary dimensions.
    weights: An optional `Tensor` whose shape is broadcastable to `values`.
    metrics_collections: An optional list of collections that `mean`
      should be added to.
    updates_collections: An optional list of collections that `update_op`
      should be added to.
    name: An optional variable_scope name.

  Returns:
    mean: A tensor representing the current mean, the value of `total` divided
      by `count`.
    update_op: An operation that increments the `total` and `count` variables
      appropriately and whose value matches `mean_value`.

  Raises:
    ValueError: If `weights` is not `None` and its shape doesn't match `values`,
      or if either `metrics_collections` or `updates_collections` are not a list
      or tuple.
  """
  with variable_scope.variable_scope(name, 'mean', [values, weights]):
    values = math_ops.to_float(values)

    total = _create_local('total', shape=[])
    count = _create_local('count', shape=[])

    if weights is not None:
      weights = math_ops.to_float(weights)
      values = math_ops.mul(values, weights)
      num_values = math_ops.reduce_sum(_broadcast_weights(weights, values))
    else:
      num_values = math_ops.to_float(array_ops.size(values))

    total_compute_op = state_ops.assign_add(total, math_ops.reduce_sum(values))
    count_compute_op = state_ops.assign_add(count, num_values)

    mean = _safe_div(total, count, 'value')
    with ops.control_dependencies([total_compute_op, count_compute_op]):
      update_op = _safe_div(total, count, 'update_op')

    if metrics_collections:
      ops.add_to_collections(metrics_collections, mean)

    if updates_collections:
      ops.add_to_collections(updates_collections, update_op)

    return mean, update_op
示例#36
0
    def _minimize_constrained(self,
                              minimization_problem,
                              global_step=None,
                              var_list=None,
                              gate_gradients=train_optimizer.Optimizer.GATE_OP,
                              aggregation_method=None,
                              colocate_gradients_with_ops=False,
                              name=None,
                              grad_loss=None):
        """Returns an `Operation` for minimizing the constrained problem.

    The `optimizer` constructor parameter will be used to update the model
    parameters, while the constraint/objective weight matrix (the analogue of
    Lagrange multipliers) will be updated using `constrained_optimizer` (if
    provided) or `optimizer` (if not). Whether the matrix updates are additive
    or multiplicative depends on the derived class.

    Args:
      minimization_problem: ConstrainedMinimizationProblem, the problem to
        optimize.
      global_step: as in `tf.train.Optimizer`'s `minimize` method.
      var_list: as in `tf.train.Optimizer`'s `minimize` method.
      gate_gradients: as in `tf.train.Optimizer`'s `minimize` method.
      aggregation_method: as in `tf.train.Optimizer`'s `minimize` method.
      colocate_gradients_with_ops: as in `tf.train.Optimizer`'s `minimize`
        method.
      name: as in `tf.train.Optimizer`'s `minimize` method.
      grad_loss: as in `tf.train.Optimizer`'s `minimize` method.

    Raises:
      ValueError: If the minimization_problem tensors have different dtypes.

    Returns:
      `Operation`, the train_op.
    """
        objective = minimization_problem.objective

        constraints = minimization_problem.constraints
        proxy_constraints = minimization_problem.proxy_constraints
        if proxy_constraints is None:
            proxy_constraints = constraints

        # Make sure that the objective, constraints and proxy constraints all have
        # the same dtype.
        if (objective.dtype.base_dtype != constraints.dtype.base_dtype
                or objective.dtype.base_dtype !=
                proxy_constraints.dtype.base_dtype):
            raise ValueError(
                "objective, constraints and proxy_constraints must "
                "have the same dtype")

        # Flatten both constraints tensors to 1d.
        num_constraints = minimization_problem.num_constraints
        constraints = standard_ops.reshape(constraints,
                                           shape=(num_constraints, ))
        proxy_constraints = standard_ops.reshape(proxy_constraints,
                                                 shape=(num_constraints, ))

        # We use a lambda to initialize the state so that, if this function call is
        # inside the scope of a tf.control_dependencies() block, the dependencies
        # will not be applied to the initializer.
        state = standard_ops.Variable(
            lambda: self._initial_state(num_constraints),
            trainable=False,
            name="swap_regret_optimizer_state")

        zero_and_constraints = standard_ops.concat((standard_ops.zeros(
            (1, ), dtype=constraints.dtype), constraints),
                                                   axis=0)
        objective_and_proxy_constraints = standard_ops.concat(
            (standard_ops.expand_dims(objective, 0), proxy_constraints),
            axis=0)

        distribution = self._distribution(state)
        loss = standard_ops.tensordot(
            standard_ops.cast(distribution,
                              objective_and_proxy_constraints.dtype),
            objective_and_proxy_constraints, 1)
        matrix_gradient = standard_ops.matmul(
            standard_ops.expand_dims(
                standard_ops.cast(zero_and_constraints, distribution.dtype),
                1), standard_ops.expand_dims(distribution, 0))

        update_ops = []
        if self.constraint_optimizer is None:
            # If we don't have a separate constraint_optimizer, then we use
            # self._optimizer for both the update of the model parameters, and that of
            # the internal state.
            grads_and_vars = self.optimizer.compute_gradients(
                loss,
                var_list=var_list,
                gate_gradients=gate_gradients,
                aggregation_method=aggregation_method,
                colocate_gradients_with_ops=colocate_gradients_with_ops,
                grad_loss=grad_loss)
            grads_and_vars.append(
                self._constraint_grad_and_var(state, matrix_gradient))
            update_ops.append(
                self.optimizer.apply_gradients(grads_and_vars, name="update"))
        else:
            # If we have a separate constraint_optimizer, then we use self._optimizer
            # for the update of the model parameters, and self._constraint_optimizer
            # for that of the internal state.
            grads_and_vars = self.optimizer.compute_gradients(
                loss,
                var_list=var_list,
                gate_gradients=gate_gradients,
                aggregation_method=aggregation_method,
                colocate_gradients_with_ops=colocate_gradients_with_ops,
                grad_loss=grad_loss)
            matrix_grads_and_vars = [
                self._constraint_grad_and_var(state, matrix_gradient)
            ]

            gradients = [
                gradient
                for gradient, _ in grads_and_vars + matrix_grads_and_vars
                if gradient is not None
            ]
            with ops.control_dependencies(gradients):
                update_ops.append(
                    self.optimizer.apply_gradients(grads_and_vars,
                                                   name="update"))
                update_ops.append(
                    self.constraint_optimizer.apply_gradients(
                        matrix_grads_and_vars, name="optimizer_state_update"))

        with ops.control_dependencies(update_ops):
            if global_step is None:
                # If we don't have a global step, just project, and we're done.
                return self._projection_op(state, name=name)
            else:
                # If we have a global step, then we need to increment it in addition to
                # projecting.
                projection_op = self._projection_op(state, name="project")
                with ops.colocate_with(global_step):
                    global_step_op = state_ops.assign_add(
                        global_step, 1, name="global_step_increment")
                return control_flow_ops.group(projection_op,
                                              global_step_op,
                                              name=name)
示例#37
0
    def apply_gradients(self, grads_and_vars, global_step=None, name=None):
        """Apply gradients to variables.

    This is the second part of `minimize()`. It returns an `Operation` that
    applies gradients.

    Args:
      grads_and_vars: List of (gradient, variable) pairs as returned by
        `compute_gradients()`.
      global_step: Optional `Variable` to increment by one after the
        variables have been updated.
      name: Optional name for the returned operation.  Default to the
        name passed to the `Optimizer` constructor.

    Returns:
      An `Operation` that applies the specified gradients. If `global_step`
      was not None, that operation also increments `global_step`.

    Raises:
      TypeError: If `grads_and_vars` is malformed.
      ValueError: If none of the variables have gradients.
    """
        # This is a default implementation of apply_gradients() that can be shared
        # by most optimizers.  It relies on the subclass implementing the following
        # methods: _create_slots(), _prepare(), _apply_dense(), and _apply_sparse().
        grads_and_vars = tuple(
            grads_and_vars)  # Make sure repeat iteration works
        for g, v in grads_and_vars:
            if not isinstance(g, (ops.Tensor, ops.IndexedSlices, type(None))):
                raise TypeError(
                    "Gradient must be a Tensor, IndexedSlices, or None: %s" %
                    g)
            if not isinstance(v, variables.Variable):
                raise TypeError("Variable must be a tf.Variable: %s" % v)
            if g is not None:
                self._assert_valid_dtypes([g, v])
        var_list = [v for g, v in grads_and_vars if g is not None]
        if not var_list:
            raise ValueError("No gradients provided for any variable: %s" %
                             (grads_and_vars, ))
        with ops.control_dependencies(None):
            self._create_slots(var_list)
        update_ops = []
        with ops.op_scope([], name, self._name) as name:
            self._prepare()
            for grad, var in grads_and_vars:
                if not grad:
                    continue
                with ops.name_scope("update_" + var.op.name), ops.device(
                        var.device):
                    if isinstance(grad, ops.Tensor):
                        update_ops.append(self._apply_dense(grad, var))
                    else:
                        update_ops.append(self._apply_sparse(grad, var))
            if global_step is None:
                return self._finish(update_ops, name)
            else:
                with ops.control_dependencies(
                    [self._finish(update_ops, "update")]):
                    with ops.device(global_step.device):
                        return state_ops.assign_add(global_step, 1,
                                                    name=name).op
示例#38
0
  def apply_gradients(self, grads_and_vars, global_step=None, name=None):
    """Apply gradients to variables.

    This is the second part of `minimize()`. It returns an `Operation` that
    applies gradients.

    Args:
      grads_and_vars: List of (gradient, variable) pairs as returned by
        `compute_gradients()`.
      global_step: Optional `Variable` to increment by one after the
        variables have been updated.
      name: Optional name for the returned operation.  Default to the
        name passed to the `Optimizer` constructor.

    Returns:
      An `Operation` that applies the specified gradients. If `global_step`
      was not None, that operation also increments `global_step`.

    Raises:
      TypeError: If `grads_and_vars` is malformed.
      ValueError: If none of the variables have gradients.
      RuntimeError: If you should use `_distributed_apply()` instead.
    """
    # This is a default implementation of apply_gradients() that can be shared
    # by most optimizers.  It relies on the subclass implementing the following
    # methods: _create_slots(), _prepare(), _apply_dense(), and _apply_sparse().

    # Handle DistributionStrategy case.
    if distribute_lib.get_cross_tower_context():
      raise RuntimeError("Use `_distributed_apply()` instead of "
                         "`apply_gradients()` in a cross-tower context.")
    # TODO(isaprykin): Get rid of `has_distribution_strategy()` check by
    # always calling _distributed_apply(), using the default distribution
    # as needed.
    if distribute_lib.has_distribution_strategy():
      grads_and_vars = get_filtered_grad_fn(lambda _: grads_and_vars)()
      return distribute_lib.get_tower_context().merge_call(
          self._distributed_apply, grads_and_vars, global_step, name)

    # No DistributionStrategy case.
    grads_and_vars = tuple(grads_and_vars)  # Make sure repeat iteration works.
    if not grads_and_vars:
      raise ValueError("No variables provided.")
    converted_grads_and_vars = []
    for g, v in grads_and_vars:
      if g is not None:
        try:
          # Convert the grad to Tensor or IndexedSlices if necessary.
          g = ops.convert_to_tensor_or_indexed_slices(g)
        except TypeError:
          raise TypeError(
              "Gradient must be convertible to a Tensor"
              " or IndexedSlices, or None: %s" % g)
        if not isinstance(g, (ops.Tensor, ops.IndexedSlices)):
          raise TypeError(
              "Gradient must be a Tensor, IndexedSlices, or None: %s" % g)
      p = _get_processor(v)
      converted_grads_and_vars.append((g, v, p))

    converted_grads_and_vars = tuple(converted_grads_and_vars)
    var_list = [v for g, v, _ in converted_grads_and_vars if g is not None]
    if not var_list:
      raise ValueError("No gradients provided for any variable: %s." %
                       ([str(v) for _, _, v in converted_grads_and_vars],))
    with ops.init_scope():
      self._create_slots(var_list)
    update_ops = []
    with ops.name_scope(name, self._name) as name:
      self._prepare()
      for grad, var, processor in converted_grads_and_vars:
        if grad is None:
          continue
        # We colocate all ops created in _apply_dense or _apply_sparse
        # on the same device as the variable.
        # TODO(apassos): figure out how to get the variable name here.
        if context.executing_eagerly() or isinstance(
            var,
            resource_variable_ops.ResourceVariable) and not var._in_graph_mode:  # pylint: disable=protected-access
          scope_name = ""
        else:
          scope_name = var.op.name
        with ops.name_scope("update_" + scope_name), ops.colocate_with(var):
          update_ops.append(processor.update_op(self, grad))
      if global_step is None:
        apply_updates = self._finish(update_ops, name)
      else:
        with ops.control_dependencies([self._finish(update_ops, "update")]):
          with ops.colocate_with(global_step):
            if isinstance(global_step, resource_variable_ops.ResourceVariable):
              # TODO(apassos): the implicit read in assign_add is slow; consider
              # making it less so.
              apply_updates = resource_variable_ops.assign_add_variable_op(
                  global_step.handle,
                  ops.convert_to_tensor(1, dtype=global_step.dtype),
                  name=name)
            else:
              apply_updates = state_ops.assign_add(global_step, 1, name=name)

      if not context.executing_eagerly():
        if isinstance(apply_updates, ops.Tensor):
          apply_updates = apply_updates.op
        train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP)
        if apply_updates not in train_op:
          train_op.append(apply_updates)

      return apply_updates
示例#39
0
def evaluate_repeatedly(checkpoint_dir,
                        master='',
                        scaffold=None,
                        eval_ops=None,
                        feed_dict=None,
                        final_ops=None,
                        final_ops_feed_dict=None,
                        eval_interval_secs=60,
                        hooks=None,
                        config=None,
                        max_number_of_evaluations=None,
                        timeout=None,
                        timeout_fn=None):
    """Repeatedly searches for a checkpoint in `checkpoint_dir` and evaluates it.

  During a single evaluation, the `eval_ops` is run until the session is
  interrupted or requested to finish. This is typically requested via a
  `tf.contrib.training.StopAfterNEvalsHook` which results in `eval_ops` running
  the requested number of times.

  Optionally, a user can pass in `final_ops`, a single `Tensor`, a list of
  `Tensors` or a dictionary from names to `Tensors`. The `final_ops` is
  evaluated a single time after `eval_ops` has finished running and the fetched
  values of `final_ops` are returned. If `final_ops` is left as `None`, then
  `None` is returned.

  One may also consider using a `tf.contrib.training.SummaryAtEndHook` to record
  summaries after the `eval_ops` have run. If `eval_ops` is `None`, the
  summaries run immediately after the model checkpoint has been restored.

  Note that `evaluate_once` creates a local variable used to track the number of
  evaluations run via `tf.contrib.training.get_or_create_eval_step`.
  Consequently, if a custom local init op is provided via a `scaffold`, the
  caller should ensure that the local init op also initializes the eval step.

  Args:
    checkpoint_dir: The directory where checkpoints are stored.
    master: The address of the TensorFlow master.
    scaffold: An tf.compat.v1.train.Scaffold instance for initializing variables
      and restoring variables. Note that `scaffold.init_fn` is used by the
      function to restore the checkpoint. If you supply a custom init_fn, then
      it must also take care of restoring the model from its checkpoint.
    eval_ops: A single `Tensor`, a list of `Tensors` or a dictionary of names to
      `Tensors`, which is run until the session is requested to stop, commonly
      done by a `tf.contrib.training.StopAfterNEvalsHook`.
    feed_dict: The feed dictionary to use when executing the `eval_ops`.
    final_ops: A single `Tensor`, a list of `Tensors` or a dictionary of names
      to `Tensors`.
    final_ops_feed_dict: A feed dictionary to use when evaluating `final_ops`.
    eval_interval_secs: The minimum number of seconds between evaluations.
    hooks: List of `tf.estimator.SessionRunHook` callbacks which are run inside
      the evaluation loop.
    config: An instance of `tf.compat.v1.ConfigProto` that will be used to
      configure the `Session`. If left as `None`, the default will be used.
    max_number_of_evaluations: The maximum times to run the evaluation. If left
      as `None`, then evaluation runs indefinitely.
    timeout: The maximum number of seconds to wait between checkpoints. If left
      as `None`, then the process will wait indefinitely.
    timeout_fn: Optional function to call after a timeout.  If the function
      returns True, then it means that no new checkpoints will be generated and
      the iterator will exit.  The function is called with no arguments.

  Returns:
    The fetched values of `final_ops` or `None` if `final_ops` is `None`.
  """
    eval_step = get_or_create_eval_step()

    # Prepare the run hooks.
    hooks = hooks or []

    if eval_ops is not None:
        update_eval_step = state_ops.assign_add(eval_step, 1)

        for h in hooks:
            if isinstance(h, StopAfterNEvalsHook):
                h._set_evals_completed_tensor(update_eval_step)  # pylint: disable=protected-access

        if isinstance(eval_ops, dict):
            eval_ops['update_eval_step'] = update_eval_step
        elif isinstance(eval_ops, (tuple, list)):
            eval_ops = list(eval_ops) + [update_eval_step]
        else:
            eval_ops = [eval_ops, update_eval_step]

    final_ops_hook = basic_session_run_hooks.FinalOpsHook(
        final_ops, final_ops_feed_dict)
    hooks.append(final_ops_hook)

    num_evaluations = 0

    ##########################################################
    ############# Get path of all checkpoint files ###########
    ##########################################################
    checkpoint_path_list = tf.train.get_checkpoint_state(
        checkpoint_dir).all_model_checkpoint_paths
    for checkpoint_path in checkpoint_path_list:
        checkpoint_name = os.path.basename(checkpoint_path)
        #  for checkpoint_path in checkpoints_iterator(
        #      checkpoint_dir,
        #      min_interval_secs=eval_interval_secs,
        #      timeout=timeout,
        #      timeout_fn=timeout_fn):

        session_creator = monitored_session.ChiefSessionCreator(
            scaffold=scaffold,
            checkpoint_filename_with_path=checkpoint_path,
            master=master,
            config=config)

        with monitored_session.MonitoredSession(
                session_creator=session_creator, hooks=hooks) as session:
            #      logging.info('Starting evaluation at ' +
            #                   time.strftime('%Y-%m-%d-%H:%M:%S', time.gmtime()))
            logging.info('Starting evaluation on checkpoint ' +
                         checkpoint_name)
            if eval_ops is not None:
                while not session.should_stop():
                    session.run(eval_ops, feed_dict)

            logging.info('Finished evaluation on checkpoint  ' +
                         checkpoint_name)
        num_evaluations += 1

        if (num_evaluations >= len(checkpoint_path_list)):
            return final_ops_hook.final_ops_values

    return final_ops_hook.final_ops_values
示例#40
0
def _evaluate_once(checkpoint_path,
                   master='',
                   scaffold=None,
                   eval_ops=None,
                   feed_dict=None,
                   final_ops=None,
                   final_ops_feed_dict=None,
                   hooks=None,
                   config=None):
  """Evaluates the model at the given checkpoint path.

  During a single evaluation, the `eval_ops` is run until the session is
  interrupted or requested to finish. This is typically requested via a
  `tf.contrib.training.StopAfterNEvalsHook` which results in `eval_ops` running
  the requested number of times.

  Optionally, a user can pass in `final_ops`, a single `Tensor`, a list of
  `Tensors` or a dictionary from names to `Tensors`. The `final_ops` is
  evaluated a single time after `eval_ops` has finished running and the fetched
  values of `final_ops` are returned. If `final_ops` is left as `None`, then
  `None` is returned.

  One may also consider using a `tf.contrib.training.SummaryAtEndHook` to record
  summaries after the `eval_ops` have run. If `eval_ops` is `None`, the
  summaries run immediately after the model checkpoint has been restored.

  Note that `evaluate_once` creates a local variable used to track the number of
  evaluations run via `tf.contrib.training.get_or_create_eval_step`.
  Consequently, if a custom local init op is provided via a `scaffold`, the
  caller should ensure that the local init op also initializes the eval step.

  Args:
    checkpoint_path: The path to a checkpoint to use for evaluation.
    master: The BNS address of the TensorFlow master.
    scaffold: An tf.train.Scaffold instance for initializing variables and
      restoring variables. Note that `scaffold.init_fn` is used by the function
      to restore the checkpoint. If you supply a custom init_fn, then it must
      also take care of restoring the model from its checkpoint.
    eval_ops: A single `Tensor`, a list of `Tensors` or a dictionary of names
      to `Tensors`, which is run until the session is requested to stop,
      commonly done by a `tf.contrib.training.StopAfterNEvalsHook`.
    feed_dict: The feed dictionary to use when executing the `eval_ops`.
    final_ops: A single `Tensor`, a list of `Tensors` or a dictionary of names
      to `Tensors`.
    final_ops_feed_dict: A feed dictionary to use when evaluating `final_ops`.
    hooks: List of `tf.train.SessionRunHook` callbacks which are run inside the
      evaluation loop.
    config: An instance of `tf.ConfigProto` that will be used to
      configure the `Session`. If left as `None`, the default will be used.

  Returns:
    The fetched values of `final_ops` or `None` if `final_ops` is `None`.
  """
  eval_step = _get_or_create_eval_step()

  # Prepare the run hooks.
  hooks = list(hooks or [])

  if eval_ops is not None:
    update_eval_step = state_ops.assign_add(eval_step, 1, use_locking=True)

    if isinstance(eval_ops, dict):
      eval_ops['update_eval_step'] = update_eval_step
    elif isinstance(eval_ops, (tuple, list)):
      eval_ops = list(eval_ops) + [update_eval_step]
    else:
      eval_ops = [eval_ops, update_eval_step]

    eval_step_value = _get_latest_eval_step_value(eval_ops)

    for h in hooks:
      if isinstance(h, _StopAfterNEvalsHook):
        h._set_evals_completed_tensor(eval_step_value)  # pylint: disable=protected-access

  logging.info('Starting evaluation at ' + time.strftime('%Y-%m-%d-%H:%M:%S',
                                                         time.gmtime()))

  # Prepare the session creator.
  session_creator = monitored_session.ChiefSessionCreator(
      scaffold=scaffold,
      checkpoint_filename_with_path=checkpoint_path,
      master=master,
      config=config)

  final_ops_hook = basic_session_run_hooks.FinalOpsHook(
      final_ops, final_ops_feed_dict)
  hooks.append(final_ops_hook)

  with monitored_session.MonitoredSession(
      session_creator=session_creator, hooks=hooks) as session:
    if eval_ops is not None:
      while not session.should_stop():
        session.run(eval_ops, feed_dict)

  logging.info('Finished evaluation at ' + time.strftime('%Y-%m-%d-%H:%M:%S',
                                                         time.gmtime()))
  return final_ops_hook.final_ops_values
示例#41
0
    def testToggleBreakpointsWorks(self):
        with session.Session(config=no_rewrite_session_config()) as sess:
            v_1 = variables.Variable(50.0, name="v_1")
            v_2 = variables.Variable(-50.0, name="v_2")
            delta_1 = constant_op.constant(5.0, name="delta_1")
            delta_2 = constant_op.constant(-5.0, name="delta_2")
            inc_v_1 = state_ops.assign_add(v_1, delta_1, name="inc_v_1")
            inc_v_2 = state_ops.assign_add(v_2, delta_2, name="inc_v_2")

            sess.run([v_1.initializer, v_2.initializer])

            run_metadata = config_pb2.RunMetadata()
            run_options = config_pb2.RunOptions(output_partition_graphs=True)
            debug_utils.watch_graph(
                run_options,
                sess.graph,
                debug_ops=["DebugIdentity(gated_grpc=true)"],
                debug_urls=[self._debug_server_url_1])

            for i in xrange(4):
                self._server_1.clear_data()

                if i in (0, 2):
                    # Enable breakpoint at delta_[1,2]:0:DebugIdentity in runs 0 and 2.
                    self._server_1.request_watch("delta_1",
                                                 0,
                                                 "DebugIdentity",
                                                 breakpoint=True)
                    self._server_1.request_watch("delta_2",
                                                 0,
                                                 "DebugIdentity",
                                                 breakpoint=True)
                else:
                    # Disable the breakpoint in runs 1 and 3.
                    self._server_1.request_unwatch("delta_1", 0,
                                                   "DebugIdentity")
                    self._server_1.request_unwatch("delta_2", 0,
                                                   "DebugIdentity")

                output = sess.run([inc_v_1, inc_v_2],
                                  options=run_options,
                                  run_metadata=run_metadata)
                self.assertAllClose(
                    [50.0 + 5.0 * (i + 1), -50 - 5.0 * (i + 1)], output)

                if i in (0, 2):
                    # During runs 0 and 2, the server should have received the published
                    # debug tensor delta:0:DebugIdentity. The breakpoint should have been
                    # unblocked by EventReply reponses from the server.
                    self.assertAllClose(
                        [5.0], self._server_1.
                        debug_tensor_values["delta_1:0:DebugIdentity"])
                    self.assertAllClose(
                        [-5.0], self._server_1.
                        debug_tensor_values["delta_2:0:DebugIdentity"])
                    # After the runs, the server should have properly registered the
                    # breakpoints due to the request_unwatch calls.
                    self.assertSetEqual(
                        {("delta_1", 0, "DebugIdentity"),
                         ("delta_2", 0, "DebugIdentity")},
                        self._server_1.breakpoints)
                else:
                    # After the end of runs 1 and 3, the server has received the requests
                    # to disable the breakpoint at delta:0:DebugIdentity.
                    self.assertSetEqual(set(), self._server_1.breakpoints)
示例#42
0
    def testTensorBoardDebuggerWrapperToggleBreakpointsWorks(self):
        with session.Session(config=no_rewrite_session_config()) as sess:
            v_1 = variables.Variable(50.0, name="v_1")
            v_2 = variables.Variable(-50.0, name="v_2")
            delta_1 = constant_op.constant(5.0, name="delta_1")
            delta_2 = constant_op.constant(-5.0, name="delta_2")
            inc_v_1 = state_ops.assign_add(v_1, delta_1, name="inc_v_1")
            inc_v_2 = state_ops.assign_add(v_2, delta_2, name="inc_v_2")

            sess.run([v_1.initializer, v_2.initializer])

            # The TensorBoardDebugWrapperSession should add a DebugIdentity debug op
            # with attribute gated_grpc=True for every tensor in the graph.
            sess = grpc_wrapper.TensorBoardDebugWrapperSession(
                sess, self._debug_server_url_1)

            for i in xrange(4):
                self._server_1.clear_data()

                if i in (0, 2):
                    # Enable breakpoint at delta_[1,2]:0:DebugIdentity in runs 0 and 2.
                    self._server_1.request_watch("delta_1",
                                                 0,
                                                 "DebugIdentity",
                                                 breakpoint=True)
                    self._server_1.request_watch("delta_2",
                                                 0,
                                                 "DebugIdentity",
                                                 breakpoint=True)
                else:
                    # Disable the breakpoint in runs 1 and 3.
                    self._server_1.request_unwatch("delta_1", 0,
                                                   "DebugIdentity")
                    self._server_1.request_unwatch("delta_2", 0,
                                                   "DebugIdentity")

                output = sess.run([inc_v_1, inc_v_2])
                self.assertAllClose(
                    [50.0 + 5.0 * (i + 1), -50 - 5.0 * (i + 1)], output)

                if i in (0, 2):
                    # During runs 0 and 2, the server should have received the published
                    # debug tensor delta:0:DebugIdentity. The breakpoint should have been
                    # unblocked by EventReply reponses from the server.
                    self.assertAllClose(
                        [5.0], self._server_1.
                        debug_tensor_values["delta_1:0:DebugIdentity"])
                    self.assertAllClose(
                        [-5.0], self._server_1.
                        debug_tensor_values["delta_2:0:DebugIdentity"])
                    # After the runs, the server should have properly registered the
                    # breakpoints.
                else:
                    # After the end of runs 1 and 3, the server has received the requests
                    # to disable the breakpoint at delta:0:DebugIdentity.
                    self.assertSetEqual(set(), self._server_1.breakpoints)

                if i == 0:
                    # Check that the server has received the stack trace.
                    self.assertTrue(
                        self._server_1.query_op_traceback("delta_1"))
                    self.assertTrue(
                        self._server_1.query_op_traceback("delta_2"))
                    self.assertTrue(
                        self._server_1.query_op_traceback("inc_v_1"))
                    self.assertTrue(
                        self._server_1.query_op_traceback("inc_v_2"))
                    # Check that the server has received the python file content.
                    # Query an arbitrary line to make sure that is the case.
                    with open(__file__, "rt") as this_source_file:
                        first_line = this_source_file.readline().strip()
                    self.assertEqual(
                        first_line,
                        self._server_1.query_source_file_line(__file__, 1))
                else:
                    # In later Session.run() calls, the traceback shouldn't have been sent
                    # because it is already sent in the 1st call. So calling
                    # query_op_traceback() should lead to an exception, because the test
                    # debug server clears the data at the beginning of every iteration.
                    with self.assertRaises(ValueError):
                        self._server_1.query_op_traceback("delta_1")
                    with self.assertRaises(ValueError):
                        self._server_1.query_source_file_line(__file__, 1)
    def apply_gradients(self, grads_and_vars, global_step=None, name=None):
        """Apply gradients to variables.

    This contains most of the synchronization implementation and also wraps the
    apply_gradients() from the real optimizer. The chief work updates global
    variables.

    Args:
      grads_and_vars: List of (gradient, variable) pairs as returned by
        compute_gradients().
      global_step: Optional Variable to increment by one after the
        variables have been updated.
      name: Optional name for the returned operation.  Default to the
        name passed to the Optimizer constructor.

    Returns:
      A conditional 'Operation' that update both local and global variables or
      just local variables

    Raises:
      ValueError: If the grads_and_vars is empty.
      ValueError: If global step is not provided, the staleness cannot be
        checked.
    """

        # update local variables
        if not grads_and_vars:
            raise ValueError("Must supply at least one variable")
        if global_step is None:
            raise ValueError("Global step is required")

        apply_updates = self._opt.apply_gradients(grads_and_vars)
        with ops.control_dependencies([apply_updates]):
            local_update = state_ops.assign_add(self._local_step,
                                                1,
                                                name="local_step_update").op

        # update global variables.
        def _update_global_variables():  # pylint: disable=missing-docstring
            local_vars = [v for g, v in grads_and_vars if g is not None]
            global_vars = [self._local_2_global[v] for v in local_vars]
            # sync queue
            with ops.colocate_with(global_step):
                sync_queue = data_flow_ops.FIFOQueue(-1, [dtypes.bool],
                                                     shapes=[[]],
                                                     shared_name="sync_queue")
            train_ops = []
            aggregated_vars = []
            with ops.name_scope(None, self._name + "/global"):
                for var, gvar in zip(local_vars, global_vars):
                    # pylint: disable=protected-access
                    with ops.device(gvar.device):
                        if isinstance(var._ref(), ops.Tensor):
                            var_accum = data_flow_ops.ConditionalAccumulator(
                                var.dtype,
                                shape=var.get_shape(),
                                shared_name=gvar.name + "/var_accum")
                            train_ops.append(
                                var_accum.apply_grad(var._ref(),
                                                     local_step=global_step))
                            aggregated_vars.append(
                                var_accum.take_grad(self._num_worker))
                        else:
                            raise ValueError("Unknown local variable type!")
                        self._accumulator_list.append((var_accum, gvar.device))
            # chief worker updates global vars and enqueues tokens to the sync queue
            if self._is_chief:
                update_ops = []
                with ops.control_dependencies(train_ops):
                    for avg_var, gvar in zip(aggregated_vars, global_vars):
                        with ops.device(gvar.device):
                            update_ops.append(state_ops.assign(gvar, avg_var))
                    with ops.device(global_step.device):
                        update_ops.append(state_ops.assign_add(global_step, 1))
                with ops.control_dependencies(update_ops), ops.device(
                        global_step.device):
                    tokens = array_ops.fill([self._num_worker - 1],
                                            constant_op.constant(False))
                    sync_op = sync_queue.enqueue_many(tokens)
            else:
                with ops.control_dependencies(train_ops), ops.device(
                        global_step.device):
                    sync_op = sync_queue.dequeue()

            with ops.control_dependencies([sync_op]):
                local_update_op = self._local_vars_update(local_vars)
            return local_update_op

        with ops.control_dependencies([local_update]):
            condition = math_ops.equal(
                math_ops.mod(self._local_step, self._interval_steps), 0)
            conditional_update = control_flow_ops.cond(
                condition, _update_global_variables, control_flow_ops.no_op)

        chief_init_ops = []
        for accum, dev in self._accumulator_list:
            with ops.device(dev):
                chief_init_ops.append(
                    accum.set_global_step(global_step, name="SetGlobalStep"))
        self._chief_init_op = control_flow_ops.group(*(chief_init_ops))

        return conditional_update
示例#44
0
 def testAssignUpdateNoVarShape(self):
     var = state_ops.variable_op([1, 2], dtypes.float32, set_shape=False)
     added = state_ops.assign_add(var, [[2.0, 3.0]])
     self.assertEqual([1, 2], added.get_shape())
     subbed = state_ops.assign_sub(var, [[12.0, 13.0]])
     self.assertEqual([1, 2], subbed.get_shape())
            def run_and_check():
                # Assign float32 values
                self.assertAllClose(3., self.evaluate(x.assign(v1)))
                self.assertAllClose(3. * 2, self.evaluate(x.assign_add(v1)))
                self.assertAllClose(3., self.evaluate(x.assign_sub(v1)))

                # Attempt to assign float16 values
                with self.assertRaisesRegex(
                        ValueError,
                        'conversion requested dtype float32 for Tensor with dtype float16'
                ):
                    self.evaluate(x.assign(v2))
                with self.assertRaisesRegex(
                        ValueError,
                        'conversion requested dtype float32 for Tensor with dtype float16'
                ):
                    self.evaluate(x.assign_add(v2))
                with self.assertRaisesRegex(
                        ValueError,
                        'conversion requested dtype float32 for Tensor with dtype float16'
                ):
                    self.evaluate(x.assign_sub(v2))

                # Assign Python floats
                self.assertAllClose(0., self.evaluate(x.assign(0.)))
                self.assertAllClose(3., self.evaluate(x.assign(3.)))
                self.assertAllClose(3. * 2, self.evaluate(x.assign_add(3.)))
                self.assertAllClose(3., self.evaluate(x.assign_sub(3.)))

                # Assign multiple times
                # This currently doesn't work in graph mode if a strategy is used
                if not ds_context.has_strategy() or context.executing_eagerly(
                ):
                    assign = x.assign(1.)
                    self.assertAllClose(1., self.evaluate(assign))
                    self.assertAllClose(0., self.evaluate(assign.assign(0.)))
                    assign_add = x.assign_add(3.)
                    self.assertAllClose(3., self.evaluate(assign_add))
                    self.assertAllClose(
                        3. * 3, self.evaluate(x.assign_add(3.).assign_add(3.)))
                    self.assertAllClose(3. * 3, x)
                    assign_sub = x.assign_sub(3.)
                    self.assertAllClose(3. * 2, self.evaluate(assign_sub))
                    self.assertAllClose(
                        0., self.evaluate(x.assign_sub(3.).assign_sub(3.)))

                # Assign with read_value=False
                self.assertIsNone(self.evaluate(x.assign(1.,
                                                         read_value=False)))
                self.assertAllClose(1., self.evaluate(x))
                self.assertIsNone(
                    self.evaluate(x.assign_add(2., read_value=False)))
                self.assertAllClose(3., self.evaluate(x))
                self.assertIsNone(
                    self.evaluate(x.assign_sub(3., read_value=False)))
                self.assertAllClose(0., self.evaluate(x))

                # Use the tf.assign functions instead of the var.assign methods.
                self.assertAllClose(0., self.evaluate(state_ops.assign(x, 0.)))
                self.assertAllClose(3., self.evaluate(state_ops.assign(x, 3.)))
                self.assertAllClose(3. * 2,
                                    self.evaluate(state_ops.assign_add(x, 3.)))
                self.assertAllClose(3.,
                                    self.evaluate(state_ops.assign_sub(x, 3.)))
 def _IncrementCounter(self, counter):
     return state_ops.assign_add(counter, 1, use_locking=True)
示例#47
0
    def testToggleEnableTwoDebugWatchesNoCrosstalkBetweenDebugNodes(self):
        with session.Session(config=session_debug_testlib.
                             no_rewrite_session_config()) as sess:
            v_1 = variables.VariableV1(50.0, name="v_1")
            v_2 = variables.VariableV1(-50.0, name="v_1")
            delta_1 = constant_op.constant(5.0, name="delta_1")
            delta_2 = constant_op.constant(-5.0, name="delta_2")
            inc_v_1 = state_ops.assign_add(v_1, delta_1, name="inc_v_1")
            inc_v_2 = state_ops.assign_add(v_2, delta_2, name="inc_v_2")

            sess.run([v_1.initializer, v_2.initializer])

            run_metadata = config_pb2.RunMetadata()
            run_options = config_pb2.RunOptions(output_partition_graphs=True)
            debug_utils.watch_graph(run_options,
                                    sess.graph,
                                    debug_ops=[
                                        "DebugIdentity(gated_grpc=true)",
                                        "DebugNumericSummary(gated_grpc=true)"
                                    ],
                                    debug_urls=[self._debug_server_url_1])

            for i in xrange(4):
                self._server_1.clear_data()

                if i % 2 == 0:
                    self._server_1.request_watch("delta_1", 0, "DebugIdentity")
                    self._server_1.request_watch("delta_2", 0, "DebugIdentity")
                    self._server_1.request_unwatch("delta_1", 0,
                                                   "DebugNumericSummary")
                    self._server_1.request_unwatch("delta_2", 0,
                                                   "DebugNumericSummary")
                else:
                    self._server_1.request_unwatch("delta_1", 0,
                                                   "DebugIdentity")
                    self._server_1.request_unwatch("delta_2", 0,
                                                   "DebugIdentity")
                    self._server_1.request_watch("delta_1", 0,
                                                 "DebugNumericSummary")
                    self._server_1.request_watch("delta_2", 0,
                                                 "DebugNumericSummary")

                sess.run([inc_v_1, inc_v_2],
                         options=run_options,
                         run_metadata=run_metadata)

                # Watched debug tensors are:
                #   Run 0: delta_[1,2]:0:DebugIdentity
                #   Run 1: delta_[1,2]:0:DebugNumericSummary
                #   Run 2: delta_[1,2]:0:DebugIdentity
                #   Run 3: delta_[1,2]:0:DebugNumericSummary
                self.assertEqual(2, len(self._server_1.debug_tensor_values))
                if i % 2 == 0:
                    self.assertAllClose(
                        [5.0], self._server_1.
                        debug_tensor_values["delta_1:0:DebugIdentity"])
                    self.assertAllClose(
                        [-5.0], self._server_1.
                        debug_tensor_values["delta_2:0:DebugIdentity"])
                else:
                    self.assertAllClose(
                        [[
                            1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 5.0, 5.0,
                            5.0, 0.0, 1.0, 0.0
                        ]], self._server_1.
                        debug_tensor_values["delta_1:0:DebugNumericSummary"])
                    self.assertAllClose(
                        [[
                            1.0, 1.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, -5.0, -5.0,
                            -5.0, 0.0, 1.0, 0.0
                        ]], self._server_1.
                        debug_tensor_values["delta_2:0:DebugNumericSummary"])
示例#48
0
    def apply_gradients(self, grads_and_vars, global_step=None, name=None):
        """Apply gradients to variables.

    This is the second part of `minimize()`. It returns an `Operation` that
    applies gradients.

    Args:
      grads_and_vars: List of (gradient, variable) pairs as returned by
        `compute_gradients()`.
      global_step: Optional `Variable` to increment by one after the
        variables have been updated.
      name: Optional name for the returned operation.  Default to the
        name passed to the `Optimizer` constructor.

    Returns:
      An `Operation` that applies the specified gradients. If `global_step`
      was not None, that operation also increments `global_step`.

    Raises:
      TypeError: If `grads_and_vars` is malformed.
      ValueError: If none of the variables have gradients.
    """
        # This is a default implementation of apply_gradients() that can be shared
        # by most optimizers.  It relies on the subclass implementing the following
        # methods: _create_slots(), _prepare(), _apply_dense(), and _apply_sparse().

        grads_and_vars = tuple(
            grads_and_vars)  # Make sure repeat iteration works.
        if not grads_and_vars:
            raise ValueError("No variables provided.")
        converted_grads_and_vars = []
        for g, v in grads_and_vars:
            if g is not None:
                try:
                    # Convert the grad to Tensor or IndexedSlices if necessary.
                    g = ops.convert_to_tensor_or_indexed_slices(g)
                except TypeError:
                    raise TypeError("Gradient must be convertible to a Tensor"
                                    " or IndexedSlices, or None: %s" % g)
                if not isinstance(g, (ops.Tensor, ops.IndexedSlices)):
                    raise TypeError(
                        "Gradient must be a Tensor, IndexedSlices, or None: %s"
                        % g)
            p = _get_processor(v)
            converted_grads_and_vars.append((g, v, p))

        converted_grads_and_vars = tuple(converted_grads_and_vars)
        var_list = [v for g, v, _ in converted_grads_and_vars if g is not None]
        if not var_list:
            raise ValueError("No gradients provided for any variable: %s." %
                             ([str(v)
                               for _, _, v in converted_grads_and_vars], ))
        with ops.control_dependencies(None):
            self._create_slots(var_list)
        update_ops = []
        with ops.name_scope(name, self._name) as name:
            self._prepare()
            for grad, var, processor in converted_grads_and_vars:
                if grad is None:
                    continue
                # We colocate all ops created in _apply_dense or _apply_sparse
                # on the same device as the variable.
                with ops.name_scope("update_" +
                                    var.op.name), ops.colocate_with(var):
                    update_ops.append(processor.update_op(self, grad))
            if global_step is None:
                apply_updates = self._finish(update_ops, name)
            else:
                with ops.control_dependencies(
                    [self._finish(update_ops, "update")]):
                    with ops.colocate_with(global_step):
                        apply_updates = state_ops.assign_add(global_step,
                                                             1,
                                                             name=name).op

            train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP)
            if apply_updates not in train_op:
                train_op.append(apply_updates)

            return apply_updates
 def update_state_ops_fn():
   update0 = state_ops.assign_add(v0, 11.0 * replica_id_plus_one())
   update1 = state_ops.assign_add(v1, 13.0 * replica_id_plus_one())
   return update0, update1
 def call(self, inputs):
   self.add_update(state_ops.assign_add(self.a, inputs, name='a_update'),
                   inputs=True)
   return inputs + 1
示例#51
0
def streaming_precision_recall_arrays(n_gbboxes,
                                      rclasses,
                                      rscores,
                                      tp_tensor,
                                      fp_tensor,
                                      remove_zero_labels=True,
                                      metrics_collections=None,
                                      updates_collections=None,
                                      name=None):
    """Streaming computation of precision / recall arrays. This metrics
    keeps tracks of boolean True positives and False positives arrays.
    """
    with variable_scope.variable_scope(
            name, 'stream_precision_recall',
        [n_gbboxes, rclasses, tp_tensor, fp_tensor]):
        n_gbboxes = math_ops.to_int64(n_gbboxes)
        rclasses = math_ops.to_int64(rclasses)
        rscores = math_ops.to_float(rscores)

        stype = tf.int32
        tp_tensor = tf.cast(tp_tensor, stype)
        fp_tensor = tf.cast(fp_tensor, stype)

        # Reshape TP and FP tensors and clean away 0 class values.
        rclasses = tf.reshape(rclasses, [-1])
        rscores = tf.reshape(rscores, [-1])
        tp_tensor = tf.reshape(tp_tensor, [-1])
        fp_tensor = tf.reshape(fp_tensor, [-1])
        if remove_zero_labels:
            mask = tf.greater(rclasses, 0)
            rclasses = tf.boolean_mask(rclasses, mask)
            rscores = tf.boolean_mask(rscores, mask)
            tp_tensor = tf.boolean_mask(tp_tensor, mask)
            fp_tensor = tf.boolean_mask(fp_tensor, mask)

        # Local variables accumlating information over batches.
        v_nobjects = _create_local('v_nobjects', shape=[], dtype=tf.int64)
        v_ndetections = _create_local('v_ndetections',
                                      shape=[],
                                      dtype=tf.int32)
        v_scores = _create_local('v_scores', shape=[
            0,
        ])
        v_tp = _create_local('v_tp', shape=[
            0,
        ], dtype=stype)
        v_fp = _create_local('v_fp', shape=[
            0,
        ], dtype=stype)

        # Update operations.
        nobjects_op = state_ops.assign_add(v_nobjects,
                                           tf.reduce_sum(n_gbboxes))
        ndetections_op = state_ops.assign_add(
            v_ndetections, tf.size(rscores, out_type=tf.int32))
        scores_op = state_ops.assign(v_scores,
                                     tf.concat([v_scores, rscores], axis=0),
                                     validate_shape=False)
        tp_op = state_ops.assign(v_tp,
                                 tf.concat([v_tp, tp_tensor], axis=0),
                                 validate_shape=False)
        fp_op = state_ops.assign(v_fp,
                                 tf.concat([v_fp, fp_tensor], axis=0),
                                 validate_shape=False)

        # Precision and recall computations.
        # r = _precision_recall(nobjects_op, scores_op, tp_op, fp_op, 'value')
        r = _precision_recall(v_nobjects, v_ndetections, v_scores, v_tp, v_fp,
                              'value')

        with ops.control_dependencies(
            [nobjects_op, ndetections_op, scores_op, tp_op, fp_op]):
            update_op = _precision_recall(nobjects_op, ndetections_op,
                                          scores_op, tp_op, fp_op, 'update_op')

            # update_op = tf.Print(update_op,
            #                      [tf.reduce_sum(tf.cast(mask, tf.int64)),
            #                       tf.reduce_sum(tf.cast(mask2, tf.int64)),
            #                       tf.reduce_min(rscores),
            #                       tf.reduce_sum(n_gbboxes)],
            #                      'Metric: ')
            # Some debugging stuff!
            # update_op = tf.Print(update_op,
            #                      [tf.shape(tp_op),
            #                       tf.reduce_sum(tf.cast(tp_op, tf.int64), axis=0)],
            #                      'TP and FP shape: ')
            # update_op[0] = tf.Print(update_op,
            #                      [nobjects_op],
            #                      '# Groundtruth bboxes: ')
            # update_op = tf.Print(update_op,
            #                      [update_op[0][0],
            #                       update_op[0][-1],
            #                       tf.reduce_min(update_op[0]),
            #                       tf.reduce_max(update_op[0]),
            #                       tf.reduce_min(update_op[1]),
            #                       tf.reduce_max(update_op[1])],
            #                      'Precision and recall :')

        if metrics_collections:
            ops.add_to_collections(metrics_collections, r)
        if updates_collections:
            ops.add_to_collections(updates_collections, update_op)
        return r, update_op
示例#52
0
def streaming_tp_fp_arrays(num_gbboxes,
                           tp,
                           fp,
                           scores,
                           remove_zero_scores=True,
                           metrics_collections=None,
                           updates_collections=None,
                           name=None):
    """Streaming computation of True and False Positive arrays. This metrics
    also keeps track of scores and number of grountruth objects.
    """
    # Input dictionaries: dict outputs as streaming metrics.
    if isinstance(scores, dict) or isinstance(fp, dict):
        d_values = {}
        d_update_ops = {}
        for c in num_gbboxes.keys():
            scope = 'streaming_tp_fp_%s' % c
            v, up = streaming_tp_fp_arrays(num_gbboxes[c],
                                           tp[c],
                                           fp[c],
                                           scores[c],
                                           remove_zero_scores,
                                           metrics_collections,
                                           updates_collections,
                                           name=scope)
            d_values[c] = v
            d_update_ops[c] = up
        return d_values, d_update_ops

    # Input Tensors...
    with variable_scope.variable_scope(name, 'streaming_tp_fp',
                                       [num_gbboxes, tp, fp, scores]):
        num_gbboxes = math_ops.to_int64(num_gbboxes)
        scores = math_ops.to_float(scores)
        stype = tf.bool
        tp = tf.cast(tp, stype)
        fp = tf.cast(fp, stype)
        # Reshape TP and FP tensors and clean away 0 class values.
        scores = tf.reshape(scores, [-1])
        tp = tf.reshape(tp, [-1])
        fp = tf.reshape(fp, [-1])
        # Remove TP and FP both false.
        mask = tf.logical_or(tp, fp)
        if remove_zero_scores:
            rm_threshold = 1e-4
            mask = tf.logical_and(mask, tf.greater(scores, rm_threshold))
            scores = tf.boolean_mask(scores, mask)
            tp = tf.boolean_mask(tp, mask)
            fp = tf.boolean_mask(fp, mask)

        # Local variables accumlating information over batches.
        v_nobjects = _create_local('v_num_gbboxes', shape=[], dtype=tf.int64)
        v_ndetections = _create_local('v_num_detections',
                                      shape=[],
                                      dtype=tf.int32)
        v_scores = _create_local('v_scores', shape=[
            0,
        ])
        v_tp = _create_local('v_tp', shape=[
            0,
        ], dtype=stype)
        v_fp = _create_local('v_fp', shape=[
            0,
        ], dtype=stype)

        # Update operations.
        nobjects_op = state_ops.assign_add(v_nobjects,
                                           tf.reduce_sum(num_gbboxes))
        ndetections_op = state_ops.assign_add(
            v_ndetections, tf.size(scores, out_type=tf.int32))
        scores_op = state_ops.assign(v_scores,
                                     tf.concat([v_scores, scores], axis=0),
                                     validate_shape=False)
        tp_op = state_ops.assign(v_tp,
                                 tf.concat([v_tp, tp], axis=0),
                                 validate_shape=False)
        fp_op = state_ops.assign(v_fp,
                                 tf.concat([v_fp, fp], axis=0),
                                 validate_shape=False)

        # Value and update ops.
        val = (v_nobjects, v_ndetections, v_tp, v_fp, v_scores)
        with ops.control_dependencies(
            [nobjects_op, ndetections_op, scores_op, tp_op, fp_op]):
            update_op = (nobjects_op, ndetections_op, tp_op, fp_op, scores_op)

        if metrics_collections:
            ops.add_to_collections(metrics_collections, val)
        if updates_collections:
            ops.add_to_collections(updates_collections, update_op)
        return val, update_op
示例#53
0
    def _model_fn(features, labels, mode):
        """Function that returns predictions, training loss, and training op."""
        if (isinstance(features, ops.Tensor)
                or isinstance(features, sparse_tensor.SparseTensor)):
            features = {'features': features}
        if feature_columns:
            features = features.copy()
            features.update(
                layers.transform_features(features, feature_columns))

        weights = None
        if weights_name and weights_name in features:
            weights = features.pop(weights_name)

        keys = None
        if keys_name and keys_name in features:
            keys = features.pop(keys_name)

        # If we're doing eval, optionally ignore device_assigner.
        # Also ignore device assigner if we're exporting (mode == INFER)
        dev_assn = device_assigner
        if (mode == model_fn_lib.ModeKeys.INFER
                or (local_eval and mode == model_fn_lib.ModeKeys.EVAL)):
            dev_assn = None

        graph_builder = graph_builder_class(params, device_assigner=dev_assn)

        logits, tree_paths, regression_variance = graph_builder.inference_graph(
            features)

        summary.scalar('average_tree_size', graph_builder.average_size())
        # For binary classification problems, convert probabilities to logits.
        # Includes hack to get around the fact that a probability might be 0 or 1.
        if not params.regression and params.num_classes == 2:
            class_1_probs = array_ops.slice(logits, [0, 1], [-1, 1])
            logits = math_ops.log(
                math_ops.maximum(
                    class_1_probs /
                    math_ops.maximum(1.0 - class_1_probs, EPSILON), EPSILON))

        # labels might be None if we're doing prediction (which brings up the
        # question of why we force everything to adhere to a single model_fn).
        training_graph = None
        training_hooks = []
        if labels is not None and mode == model_fn_lib.ModeKeys.TRAIN:
            with ops.control_dependencies([logits.op]):
                training_graph = control_flow_ops.group(
                    graph_builder.training_graph(features,
                                                 labels,
                                                 input_weights=weights,
                                                 num_trainers=num_trainers,
                                                 trainer_id=trainer_id),
                    state_ops.assign_add(training_util.get_global_step(), 1))

        # Put weights back in
        if weights is not None:
            features[weights_name] = weights

        # TensorForest's training graph isn't calculated directly from the loss
        # like many other models.
        def _train_fn(unused_loss):
            return training_graph

        model_ops = model_head.create_model_fn_ops(features=features,
                                                   labels=labels,
                                                   mode=mode,
                                                   train_op_fn=_train_fn,
                                                   logits=logits,
                                                   scope=head_scope)

        # Ops are run in lexigraphical order of their keys. Run the resource
        # clean-up op last.
        all_handles = graph_builder.get_all_resource_handles()
        ops_at_end = {
            '9: clean up resources':
            control_flow_ops.group(*[
                resource_variable_ops.destroy_resource_op(handle)
                for handle in all_handles
            ])
        }

        if report_feature_importances:
            ops_at_end['1: feature_importances'] = (
                graph_builder.feature_importances())

        training_hooks.append(TensorForestRunOpAtEndHook(ops_at_end))

        if early_stopping_rounds:
            training_hooks.append(
                TensorForestLossHook(
                    early_stopping_rounds,
                    early_stopping_loss_threshold=early_stopping_loss_threshold,
                    loss_op=model_ops.loss))

        model_ops.training_hooks.extend(training_hooks)

        if keys is not None:
            model_ops.predictions[keys_name] = keys

        if params.inference_tree_paths:
            model_ops.predictions[TREE_PATHS_PREDICTION_KEY] = tree_paths

        if params.regression:
            model_ops.predictions[
                VARIANCE_PREDICTION_KEY] = regression_variance

        return model_ops
示例#54
0
 def body(i):
     new_u = state_ops.assign_add(u, v)
     new_i = math_ops.add(i, 1)
     op = control_flow_ops.group(new_u)
     new_i = control_flow_ops.with_dependencies([op], new_i)
     return [new_i]
示例#55
0
    def training_graph(self,
                       input_data,
                       input_labels,
                       random_seed,
                       data_spec,
                       epoch=None,
                       input_weights=None):
        """Constructs a TF graph for training a random tree.

    Args:
      input_data: A tensor or SparseTensor or placeholder for input data.
      input_labels: A tensor or placeholder for labels associated with
        input_data.
      random_seed: The random number generator seed to use for this tree.  0
        means use the current time as the seed.
      data_spec: A list of tf.dtype values specifying the original types of
        each column.
      epoch: A tensor or placeholder for the epoch the training data comes from.
      input_weights: A float tensor or placeholder holding per-input weights,
        or None if all inputs are to be weighted equally.

    Returns:
      The last op in the random tree training graph.
    """
        epoch = [0] if epoch is None else epoch

        if input_weights is None:
            input_weights = []

        sparse_indices = []
        sparse_values = []
        sparse_shape = []
        if isinstance(input_data, ops.SparseTensor):
            sparse_indices = input_data.indices
            sparse_values = input_data.values
            sparse_shape = input_data.shape
            input_data = []

        # Count extremely random stats.
        (node_sums, node_squares, splits_indices, splits_sums, splits_squares,
         totals_indices, totals_sums, totals_squares,
         input_leaves) = (self.training_ops.count_extremely_random_stats(
             input_data,
             sparse_indices,
             sparse_values,
             sparse_shape,
             data_spec,
             input_labels,
             input_weights,
             self.variables.tree,
             self.variables.tree_thresholds,
             self.variables.node_to_accumulator_map,
             self.variables.candidate_split_features,
             self.variables.candidate_split_thresholds,
             self.variables.start_epoch,
             epoch,
             num_classes=self.params.num_output_columns,
             regression=self.params.regression))
        node_update_ops = []
        node_update_ops.append(
            state_ops.assign_add(self.variables.node_sums, node_sums))

        splits_update_ops = []
        splits_update_ops.append(
            self.training_ops.scatter_add_ndim(
                self.variables.candidate_split_sums, splits_indices,
                splits_sums))
        splits_update_ops.append(
            self.training_ops.scatter_add_ndim(self.variables.accumulator_sums,
                                               totals_indices, totals_sums))

        if self.params.regression:
            node_update_ops.append(
                state_ops.assign_add(self.variables.node_squares,
                                     node_squares))
            splits_update_ops.append(
                self.training_ops.scatter_add_ndim(
                    self.variables.candidate_split_squares, splits_indices,
                    splits_squares))
            splits_update_ops.append(
                self.training_ops.scatter_add_ndim(
                    self.variables.accumulator_squares, totals_indices,
                    totals_squares))

        # Sample inputs.
        update_indices, feature_updates, threshold_updates = (
            self.training_ops.sample_inputs(
                input_data,
                sparse_indices,
                sparse_values,
                sparse_shape,
                input_weights,
                self.variables.node_to_accumulator_map,
                input_leaves,
                self.variables.candidate_split_features,
                self.variables.candidate_split_thresholds,
                split_initializations_per_input=(
                    self.params.split_initializations_per_input),
                split_sampling_random_seed=random_seed))
        update_features_op = state_ops.scatter_update(
            self.variables.candidate_split_features, update_indices,
            feature_updates)
        update_thresholds_op = state_ops.scatter_update(
            self.variables.candidate_split_thresholds, update_indices,
            threshold_updates)

        # Calculate finished nodes.
        with ops.control_dependencies(splits_update_ops):
            finished, stale = self.training_ops.finished_nodes(
                self.variables.accumulator_to_node_map,
                self.variables.node_to_accumulator_map,
                self.variables.candidate_split_sums,
                self.variables.candidate_split_squares,
                self.variables.accumulator_sums,
                self.variables.accumulator_squares,
                self.variables.start_epoch,
                epoch,
                num_split_after_samples=self.params.split_after_samples,
                min_split_samples=self.params.min_split_samples)

        # Update leaf scores.
        # TODO(thomaswc): Store the leaf scores in a TopN and only update the
        # scores of the leaves that were touched by this batch of input.
        children = array_ops.squeeze(array_ops.slice(self.variables.tree,
                                                     [0, 0], [-1, 1]),
                                     squeeze_dims=[1])
        is_leaf = math_ops.equal(constants.LEAF_NODE, children)
        leaves = math_ops.to_int32(
            array_ops.squeeze(array_ops.where(is_leaf), squeeze_dims=[1]))
        non_fertile_leaves = array_ops.boolean_mask(
            leaves,
            math_ops.less(
                array_ops.gather(self.variables.node_to_accumulator_map,
                                 leaves), 0))

        # TODO(gilberth): It should be possible to limit the number of non
        # fertile leaves we calculate scores for, especially since we can only take
        # at most array_ops.shape(finished)[0] of them.
        with ops.control_dependencies(node_update_ops):
            sums = array_ops.gather(self.variables.node_sums,
                                    non_fertile_leaves)
            if self.params.regression:
                squares = array_ops.gather(self.variables.node_squares,
                                           non_fertile_leaves)
                non_fertile_leaf_scores = self._variance(sums, squares)
            else:
                non_fertile_leaf_scores = self._weighted_gini(sums)

        # Calculate best splits.
        with ops.control_dependencies(splits_update_ops):
            split_indices = self.training_ops.best_splits(
                finished,
                self.variables.node_to_accumulator_map,
                self.variables.candidate_split_sums,
                self.variables.candidate_split_squares,
                self.variables.accumulator_sums,
                self.variables.accumulator_squares,
                regression=self.params.regression)

        # Grow tree.
        with ops.control_dependencies(
            [update_features_op, update_thresholds_op]):
            (tree_update_indices, tree_children_updates,
             tree_threshold_updates, new_eot) = (self.training_ops.grow_tree(
                 self.variables.end_of_tree,
                 self.variables.node_to_accumulator_map, finished,
                 split_indices, self.variables.candidate_split_features,
                 self.variables.candidate_split_thresholds))
            tree_update_op = state_ops.scatter_update(self.variables.tree,
                                                      tree_update_indices,
                                                      tree_children_updates)
            thresholds_update_op = state_ops.scatter_update(
                self.variables.tree_thresholds, tree_update_indices,
                tree_threshold_updates)
            # TODO(thomaswc): Only update the epoch on the new leaves.
            new_epoch_updates = epoch * array_ops.ones_like(
                tree_threshold_updates, dtype=dtypes.int32)
            epoch_update_op = state_ops.scatter_update(
                self.variables.start_epoch, tree_update_indices,
                new_epoch_updates)

        # Update fertile slots.
        with ops.control_dependencies([tree_update_op]):
            (n2a_map_updates, a2n_map_updates, accumulators_cleared,
             accumulators_allocated) = (self.training_ops.update_fertile_slots(
                 finished,
                 non_fertile_leaves,
                 non_fertile_leaf_scores,
                 self.variables.end_of_tree,
                 self.variables.accumulator_sums,
                 self.variables.node_to_accumulator_map,
                 stale,
                 regression=self.params.regression))

        # Ensure end_of_tree doesn't get updated until UpdateFertileSlots has
        # used it to calculate new leaves.
        gated_new_eot, = control_flow_ops.tuple(
            [new_eot], control_inputs=[n2a_map_updates])
        eot_update_op = state_ops.assign(self.variables.end_of_tree,
                                         gated_new_eot)

        updates = []
        updates.append(eot_update_op)
        updates.append(tree_update_op)
        updates.append(thresholds_update_op)
        updates.append(epoch_update_op)

        updates.append(
            state_ops.scatter_update(self.variables.node_to_accumulator_map,
                                     n2a_map_updates[0], n2a_map_updates[1]))

        updates.append(
            state_ops.scatter_update(self.variables.accumulator_to_node_map,
                                     a2n_map_updates[0], a2n_map_updates[1]))

        cleared_and_allocated_accumulators = array_ops.concat(
            0, [accumulators_cleared, accumulators_allocated])

        # Calculate values to put into scatter update for candidate counts.
        # Candidate split counts are always reset back to 0 for both cleared
        # and allocated accumulators. This means some accumulators might be doubly
        # reset to 0 if the were released and not allocated, then later allocated.
        split_values = array_ops.tile(
            array_ops.expand_dims(
                array_ops.expand_dims(
                    array_ops.zeros_like(cleared_and_allocated_accumulators,
                                         dtype=dtypes.float32), 1), 2),
            [
                1, self.params.num_splits_to_consider,
                self.params.num_output_columns
            ])
        updates.append(
            state_ops.scatter_update(self.variables.candidate_split_sums,
                                     cleared_and_allocated_accumulators,
                                     split_values))
        if self.params.regression:
            updates.append(
                state_ops.scatter_update(
                    self.variables.candidate_split_squares,
                    cleared_and_allocated_accumulators, split_values))

        # Calculate values to put into scatter update for total counts.
        total_cleared = array_ops.tile(
            array_ops.expand_dims(
                math_ops.neg(
                    array_ops.ones_like(accumulators_cleared,
                                        dtype=dtypes.float32)), 1),
            [1, self.params.num_output_columns])
        total_reset = array_ops.tile(
            array_ops.expand_dims(
                array_ops.zeros_like(accumulators_allocated,
                                     dtype=dtypes.float32), 1),
            [1, self.params.num_output_columns])
        accumulator_updates = array_ops.concat(0, [total_cleared, total_reset])
        updates.append(
            state_ops.scatter_update(self.variables.accumulator_sums,
                                     cleared_and_allocated_accumulators,
                                     accumulator_updates))
        if self.params.regression:
            updates.append(
                state_ops.scatter_update(self.variables.accumulator_squares,
                                         cleared_and_allocated_accumulators,
                                         accumulator_updates))

        # Calculate values to put into scatter update for candidate splits.
        split_features_updates = array_ops.tile(
            array_ops.expand_dims(
                math_ops.neg(
                    array_ops.ones_like(cleared_and_allocated_accumulators)),
                1), [1, self.params.num_splits_to_consider])
        updates.append(
            state_ops.scatter_update(self.variables.candidate_split_features,
                                     cleared_and_allocated_accumulators,
                                     split_features_updates))

        updates += self.finish_iteration()

        return control_flow_ops.group(*updates)
示例#56
0
 def update(vu):
     if isinstance(vu, resource_variable_ops.ResourceVariable):
         return vu.assign_add(amount, read_value=False)
     else:
         return state_ops.assign_add(vu, amount)
示例#57
0
def assign_moving_mean_variance(mean_var,
                                variance_var,
                                value,
                                decay,
                                name=None):
    """Compute exponentially weighted moving {mean,variance} of a streaming value.

  The `value` updated exponentially weighted moving `mean_var` and
  `variance_var` are given by the following recurrence relations:

  ```python
  variance_var = decay * (variance_var + (1-decay) * (value - mean_var)**2)
  mean_var     = decay * mean_var + (1 - decay) * value
  ```

  Note: `mean_var` is updated *after* `variance_var`, i.e., `variance_var` uses
  the lag-1 mean.

  For derivation justification, see equation 143 of:
    T. Finch, Feb 2009. "Incremental calculation of weighted mean and variance".
    http://people.ds.cam.ac.uk/fanf2/hermes/doc/antiforgery/stats.pdf

  Args:
    mean_var: `float`-like `Variable` representing the exponentially weighted
      moving mean. Same shape as `variance_var` and `value`.
    variance_var: `float`-like `Variable` representing the
      exponentially weighted moving variance. Same shape as `mean_var` and
      `value`.
    value: `float`-like `Tensor`. Same shape as `mean_var` and `variance_var`.
    decay: A `float`-like `Tensor`. The moving mean decay. Typically close to
      `1.`, e.g., `0.999`.
    name: Optional name of the returned operation.

  Returns:
    mean_var: `Variable` representing the `value`-updated exponentially weighted
      moving mean.
    variance_var: `Variable` representing the `value`-updated
      exponentially weighted moving variance.

  Raises:
    TypeError: if `mean_var` does not have float type `dtype`.
    TypeError: if `mean_var`, `variance_var`, `value`, `decay` have different
      `base_dtype`.
  """
    with ops.name_scope(name, "assign_moving_mean_variance",
                        [variance_var, mean_var, value, decay]):
        with ops.colocate_with(variance_var):
            with ops.colocate_with(mean_var):
                base_dtype = mean_var.dtype.base_dtype
                if not base_dtype.is_floating:
                    raise TypeError(
                        "mean_var.base_dtype({}) does not have float type "
                        "`dtype`.".format(base_dtype.name))
                if base_dtype != variance_var.dtype.base_dtype:
                    raise TypeError(
                        "mean_var.base_dtype({}) != variance_var.base_dtype({})"
                        .format(base_dtype.name,
                                variance_var.dtype.base_dtype.name))
                value = ops.convert_to_tensor(value,
                                              dtype=base_dtype,
                                              name="value")
                decay = ops.convert_to_tensor(decay,
                                              dtype=base_dtype,
                                              name="decay")
                delta = value - mean_var
                with ops.control_dependencies([delta]):
                    mean_var = state_ops.assign_add(mean_var,
                                                    (1. - decay) * delta)
                    variance_var = state_ops.assign_sub(
                        variance_var, (1. - decay) *
                        (variance_var - decay * math_ops.square(delta)))
                return mean_var, variance_var
示例#58
0
 def _update_statistics_from_mini_batch(
     self, statistics, auxiliary_variables, times, values):
   """Given mini-batch input, update `statistics` and `auxiliary_variables`."""
   values = math_ops.cast(values, self._dtype)
   # The density (measured in times per observation) that we see in each part
   # of the mini-batch.
   batch_inter_observation_duration = (math_ops.cast(
       math_ops.reduce_max(times, axis=1) - math_ops.reduce_min(times, axis=1),
       self._dtype) / math_ops.cast(
           array_ops.shape(times)[1] - 1, self._dtype))
   # Co-locate updates with their variables to minimize race conditions when
   # updating statistics.
   with ops.device(auxiliary_variables.max_time_seen.device):
     # There is a race condition if this value is being updated from multiple
     # workers. However, it should eventually reach the correct value if the
     # last chunk is presented enough times.
     max_time_seen_assign = state_ops.assign(
         auxiliary_variables.max_time_seen,
         gen_math_ops.maximum(auxiliary_variables.max_time_seen,
                              math_ops.reduce_max(times)))
   with ops.device(auxiliary_variables.chunk_count.device):
     chunk_count_assign = state_ops.assign_add(auxiliary_variables.chunk_count,
                                               array_ops.shape(
                                                   times,
                                                   out_type=dtypes.int64)[0])
   with ops.device(auxiliary_variables.inter_observation_duration_sum.device):
     inter_observation_duration_assign = state_ops.assign_add(
         auxiliary_variables.inter_observation_duration_sum,
         math_ops.reduce_sum(batch_inter_observation_duration))
   with ops.device(auxiliary_variables.example_count.device):
     example_count_assign = state_ops.assign_add(
         auxiliary_variables.example_count,
         array_ops.size(times, out_type=dtypes.int64))
   # Note: These mean/variance updates assume that all points are equally
   # likely, which is not true if _chunks_ are sampled uniformly from the space
   # of all possible contiguous chunks, since points at the start and end of
   # the series are then members of fewer chunks. For series which are much
   # longer than the chunk size (the usual/expected case), this effect becomes
   # irrelevant.
   with ops.device(auxiliary_variables.overall_feature_sum.device):
     overall_feature_sum_assign = state_ops.assign_add(
         auxiliary_variables.overall_feature_sum,
         math_ops.reduce_sum(values, axis=[0, 1]))
   with ops.device(auxiliary_variables.overall_feature_sum_of_squares.device):
     overall_feature_sum_of_squares_assign = state_ops.assign_add(
         auxiliary_variables.overall_feature_sum_of_squares,
         math_ops.reduce_sum(values**2, axis=[0, 1]))
   per_chunk_aux_updates = control_flow_ops.group(
       max_time_seen_assign, chunk_count_assign,
       inter_observation_duration_assign, example_count_assign,
       overall_feature_sum_assign, overall_feature_sum_of_squares_assign)
   with ops.control_dependencies([per_chunk_aux_updates]):
     example_count_float = math_ops.cast(auxiliary_variables.example_count,
                                         self._dtype)
     new_feature_mean = (auxiliary_variables.overall_feature_sum /
                         example_count_float)
     overall_feature_mean_update = state_ops.assign(
         statistics.overall_feature_moments.mean, new_feature_mean)
     overall_feature_var_update = state_ops.assign(
         statistics.overall_feature_moments.variance,
         # De-biased n / (n - 1) variance correction
         example_count_float / (example_count_float - 1.) *
         (auxiliary_variables.overall_feature_sum_of_squares /
          example_count_float - new_feature_mean**2))
     # TODO(b/35675805): Remove this cast
     min_time_batch = math_ops.cast(math_ops.argmin(times[:, 0]), dtypes.int32)
     def series_start_updates():
       # If this is the lowest-time chunk that we have seen so far, update
       # series start moments to reflect that. Note that these statistics are
       # "best effort", as there are race conditions in the update (however,
       # they should eventually converge if the start of the series is
       # presented enough times).
       mean, variance = nn.moments(
           values[min_time_batch, :self._starting_variance_window_size],
           axes=[0])
       return control_flow_ops.group(
           state_ops.assign(statistics.series_start_moments.mean, mean),
           state_ops.assign(statistics.series_start_moments.variance,
                            variance))
     with ops.device(statistics.start_time.device):
       series_start_update = control_flow_ops.cond(
           # Update moments whenever we even match the lowest time seen so far,
           # to ensure that series start statistics are eventually updated to
           # their correct values, despite race conditions (i.e. eventually
           # statistics.start_time will reflect the global lowest time, and
           # given that we will eventually update the series start moments to
           # their correct values).
           math_ops.less_equal(times[min_time_batch, 0],
                               statistics.start_time),
           series_start_updates,
           control_flow_ops.no_op)
       with ops.control_dependencies([series_start_update]):
         # There is a race condition if this update is performed in parallel on
         # multiple workers. Since models may be sensitive to being presented
         # with times before the putative start time, the value of this
         # variable is post-processed above to guarantee that each worker is
         # presented with a start time which is at least as low as the lowest
         # time in its current mini-batch.
         start_time_update = state_ops.assign(statistics.start_time,
                                              gen_math_ops.minimum(
                                                  statistics.start_time,
                                                  math_ops.reduce_min(times)))
     inter_observation_duration_estimate = (
         auxiliary_variables.inter_observation_duration_sum / math_ops.cast(
             auxiliary_variables.chunk_count, self._dtype))
     # Estimate the total number of observations as:
     #   (end time - start time + 1) * average intra-chunk time density
     total_observation_count_update = state_ops.assign(
         statistics.total_observation_count,
         math_ops.cast(
             gen_math_ops.round(
                 math_ops.cast(max_time_seen_assign -
                               start_time_update + 1, self._dtype) /
                 inter_observation_duration_estimate), dtypes.int64))
     per_chunk_stat_updates = control_flow_ops.group(
         overall_feature_mean_update, overall_feature_var_update,
         series_start_update, start_time_update,
         total_observation_count_update)
   return per_chunk_stat_updates