Beispiel #1
0
  def testSequenceLoss(self):
    with self.test_session() as sess:
      output_classes = 5
      logits = [tf.constant(i + 0.5, shape=[2, 5]) for i in xrange(3)]
      targets = [tf.constant(i, tf.int32, shape=[2]) for i in xrange(3)]
      weights = [tf.constant(1.0, shape=[2]) for i in xrange(3)]

      average_loss_per_example = tf.nn.seq2seq.sequence_loss(
          logits, targets, weights, output_classes,
          average_across_timesteps=True,
          average_across_batch=True)
      res = sess.run(average_loss_per_example)
      self.assertAllClose(res, 1.60944)

      average_loss_per_sequence = tf.nn.seq2seq.sequence_loss(
          logits, targets, weights, output_classes,
          average_across_timesteps=False,
          average_across_batch=True)
      res = sess.run(average_loss_per_sequence)
      self.assertAllClose(res, 4.828314)

      total_loss = tf.nn.seq2seq.sequence_loss(
          logits, targets, weights, output_classes,
          average_across_timesteps=False,
          average_across_batch=False)
      res = sess.run(total_loss)
      self.assertAllClose(res, 9.656628)
Beispiel #2
0
def unzip(x, split_dim, current_length, num_splits=2, name=None):
  """Splits a tensor by unzipping along the split_dim.

  For example the following array split into 2 would be:
      [1, 2, 3, 4, 5, 6] -> [1, 3, 5], [2, 4, 6]
  and by 3:
      [1, 2, 3, 4] -> [1, 4], [2], [3]

  Args:
    x: The tensor to split.
    split_dim: The dimension to split along.
    current_length: Current length along the split_dim.
    num_splits: The number of splits.
    name: Optional name for this op.
  Returns:
    A length num_splits sequence.
  """
  with tf.op_scope([x], name, 'unzip') as scope:
    x = tf.convert_to_tensor(x, name='x')
    # There is probably a more efficient way to do this.
    all_splits = tf.split(split_dim, current_length, x, name=scope)
    splits = [[] for _ in xrange(num_splits)]
    for i in xrange(current_length):
      splits[i % num_splits].append(all_splits[i])
    return [tf.concat(split_dim, s) for s in splits]
Beispiel #3
0
    def test_maintenance_failure_recovery_with_capacity(self):
        backend = RedisBackend(capacity=10, truncation_chance=0.0)

        t = time.time()

        # Add 10 items to the timeline.
        for i in xrange(10):
            backend.add('timeline', Record('record:{}'.format(i), '{}'.format(i), t + i))

        try:
            with backend.digest('timeline', 0) as records:
                raise Exception('This causes the digest to not be closed.')
        except Exception:
            pass

        # The 10 existing items should now be in the digest set (the exception
        # prevented the close operation from occurring, so they were never
        # deleted from Redis or removed from the digest set.) If we add 10 more
        # items, they should be added to the timeline set (not the digest set.)
        for i in xrange(10, 20):
            backend.add('timeline', Record('record:{}'.format(i), '{}'.format(i), t + i))

        # Maintenance should move the timeline back to the waiting state, ...
        backend.maintenance(time.time())

        # The schedule should now contain the timeline.
        assert set(entry.key for entry in backend.schedule(time.time())) == set(['timeline'])

        # Only the new records should exist -- the older one should have been
        # trimmed to avoid the digest growing beyond the timeline capacity.
        with backend.digest('timeline', 0) as records:
            expected_keys = set('record:{}'.format(i) for i in xrange(10, 20))
            assert set(record.key for record in records) == expected_keys
Beispiel #4
0
    def next_batch(self):
        enc_batch = np.zeros((self._batch_size, self._enc_timesteps), dtype=np.int32)
        enc_input_lens = np.zeros(self._batch_size, dtype=np.int32)
        dec_batch = np.zeros((self._batch_size, self._dec_timesteps), dtype=np.int32)
        dec_output_lens = np.zeros(self._batch_size, dtype=np.int32)
        target_batch = np.zeros((self._batch_size, self._dec_timesteps), dtype=np.int32)
        loss_weights = np.zeros((self._batch_size, self._dec_timesteps), dtype=np.float32)
        origin_articles = ['None'] * self._batch_size
        origin_abstracts = ['None'] * self._batch_size

        buckets = self._bucket_input_queue.get()
        for i in xrange(self._batch_size):
            (enc_inputs, dec_inputs, targets, enc_input_len, dec_output_len, article, abstract) = buckets[i]

            origin_articles[i] = article
            origin_abstracts[i] = abstract
            enc_input_lens[i] = enc_input_len
            dec_output_lens[i] = dec_output_len
            enc_batch[i, :] = enc_inputs[:]
            dec_batch[i, :] = dec_inputs[:]
            target_batch[i, :] = targets[:]

            for j in xrange(dec_output_len):
                loss_weights[i][j] = 1

        return (
            enc_batch, dec_batch, target_batch, enc_input_lens, dec_output_lens, loss_weights,
            origin_articles, origin_abstracts
        )
Beispiel #5
0
def _shapes(tensor_list_list, shapes, enqueue_many):
  """Calculate and merge the shapes of incoming tensors.

  Args:
    tensor_list_list: List of tensor lists.
    shapes: List of shape tuples corresponding to tensors within the lists.
    enqueue_many: Boolean describing whether shapes will be enqueued as
      batches or individual entries.

  Returns:
    A list of shapes aggregating shape inference info from `tensor_list_list`,
    or returning `shapes` if it is not `None`.

  Raises:
    ValueError: If any of the inferred shapes in `tensor_list_list` lack a
      well defined rank.
  """
  if shapes is None:
    len0 = len(tensor_list_list[0])

    for tl in tensor_list_list:
      for i in xrange(len0):
        if tl[i].get_shape().ndims is None:
          raise ValueError("Cannot infer Tensor's rank: %s" % tl[i])

    shapes = [_merge_shapes(
        [tl[i].get_shape().as_list() for tl in tensor_list_list], enqueue_many)
              for i in xrange(len0)]
  return shapes
Beispiel #6
0
  def testError(self):
    for rank in VALID_FFT_RANKS:
      for dims in xrange(0, rank):
        x = np.zeros((1,) * dims).astype(np.complex64)
        with self.assertRaisesWithPredicateMatch(
            ValueError, "Shape must be .*rank {}.*".format(rank)):
          self._tfFFT(x, rank)
        with self.assertRaisesWithPredicateMatch(
            ValueError, "Shape must be .*rank {}.*".format(rank)):
          self._tfIFFT(x, rank)
      for dims in xrange(rank, rank + 2):
        x = np.zeros((1,) * rank)

        # Test non-rank-1 fft_length produces an error.
        fft_length = np.zeros((1, 1)).astype(np.int32)
        with self.assertRaisesWithPredicateMatch(ValueError,
                                                 "Shape must be .*rank 1"):
          self._tfFFT(x, rank, fft_length)
        with self.assertRaisesWithPredicateMatch(ValueError,
                                                 "Shape must be .*rank 1"):
          self._tfIFFT(x, rank, fft_length)

        # Test wrong fft_length length.
        fft_length = np.zeros((rank + 1,)).astype(np.int32)
        with self.assertRaisesWithPredicateMatch(
            ValueError, "Dimension must be .*but is {}.*".format(rank + 1)):
          self._tfFFT(x, rank, fft_length)
        with self.assertRaisesWithPredicateMatch(
            ValueError, "Dimension must be .*but is {}.*".format(rank + 1)):
          self._tfIFFT(x, rank, fft_length)
Beispiel #7
0
def get_run_op():
  # Create an optimizer that performs gradient descent.
  #opt = tf.train.GradientDescentOptimizer(learning_rate=0.01)
  slice_size = FLAGS.batch_size / FLAGS.num_cuts
  print('Slice size:{}'.format(slice_size))
  data = None
  label = None
  last_fc = [tf.no_op()]
  with tf.device('/gpu:0'):
    data = tf.get_variable(
        name = 'data',
        shape=[slice_size, FLAGS.hidden_size],
        trainable=False)
    '''
    label = tf.get_variable(
        name = 'label',
        shape = [slice_size, FLAGS.hidden_size],
        trainable=False))
    with tf.variable_scope('fc_in'):
      weight_in = tf.zeros([1000, FLAGS.hidden_size])
      for k in xrange(FLAGS.num_cuts):
        with tf.control_dependencies([last_fc[-1]]):
            last_fc.append(tf.matmul(data[k+1], weight_in))
    '''
  for i in xrange(FLAGS.num_cuts):
    last_fc.append(data)
  for i in xrange(FLAGS.num_layers):
    dev = '/gpu:%d' % (i * FLAGS.num_gpus / FLAGS.num_layers)
    with tf.device(dev), scopes.arg_scope([variables.variable], device=dev):
      tmp_fc = [tf.no_op()]
      with tf.variable_scope('fc%d' % i):
        w = tf.get_variable(
            name='w',
            shape=[FLAGS.hidden_size, FLAGS.hidden_size],
            trainable=True)
        for k in xrange(FLAGS.num_cuts):
          with tf.control_dependencies([tmp_fc[-1]]):
            tmp_fc.append(tf.matmul(last_fc[k+1], w))
      last_fc = tmp_fc
      if i == FLAGS.num_layers - 1:
        with tf.control_dependencies(last_fc):
          train_op = tf.no_op()
  '''
  with tf.device('/gpu:%d' % (FLAGS.num_gpus - 1)):
    tmp_fc = [tf.no_op()]
    with tf.variable_scope('fc_out'):
      weight_out = tf.zeros([FLAGS.hidden_size, 1000])
      for k in xrange(FLAGS.num_cuts):
        with tf.control_dependencies([tmp_fc[-1]]):
          tmp_fc.append(tf.matmul(last_fc[k+1], weight_out))
    last_fc = tmp_fc
  loss = tf.nn_softmax_cross_entropy_with_logits(last_fc, labels, name='xentropy')
  grads = opt.compute_gradients(loss)
  apply_gradient_op = opt.apply_gradients(grads)

  train_op = tf.group(apply_gradient_op)
  '''
  init_op = tf.initialize_all_variables()

  return init_op, train_op
Beispiel #8
0
    def conv2d(self, img, kern, mode="valid"):
        """
        Basic slow python implementatation for DebugMode
        """

        if not imported_scipy_signal:
            raise NotImplementedError(
                "AbstractConv perform requires the python package"
                " for scipy.signal to be installed.")
        if not (mode in ('valid', 'full')):
            raise ValueError(
                'invalid mode {}, which must be either '
                '"valid" or "full"'.format(mode))

        out_shape = get_conv_output_shape(img.shape, kern.shape, mode, [1, 1])
        out = numpy.zeros(out_shape, dtype=img.dtype)
        val = _valfrommode(mode)
        bval = _bvalfromboundary('fill')

        with warnings.catch_warnings():
            warnings.simplefilter('ignore', numpy.ComplexWarning)
            for b in xrange(img.shape[0]):
                for n in xrange(kern.shape[0]):
                    for im0 in xrange(img.shape[1]):
                        # some cast generates a warning here
                        out[b, n, ...] += _convolve2d(img[b, im0, ...],
                                                      kern[n, im0, ...],
                                                      1, val, bval, 0)
        return out
  def eval(self):
    """Evaluate analogy questions and reports accuracy."""

    # How many questions we get right at precision@1.
    correct = 0

    total = self._analogy_questions.shape[0]
    start = 0
    while start < total:
      limit = start + 2500
      sub = self._analogy_questions[start:limit, :]
      idx = self._predict(sub)
      start = limit
      for question in xrange(sub.shape[0]):
        for j in xrange(4):
          if idx[question, j] == sub[question, 3]:
            # Bingo! We predicted correctly. E.g., [italy, rome, france, paris].
            correct += 1
            break
          elif idx[question, j] in sub[question, :3]:
            # We need to skip words already in the question.
            continue
          else:
            # The correct label is not the precision@1
            break
    print()
    print("Eval %4d/%d accuracy = %4.1f%%" % (correct, total,
                                              correct * 100.0 / total))
Beispiel #10
0
def _lost_point_level3(modules, modules_count):
    modules_range_short = xrange(modules_count-6)

    lost_point = 0
    for row in xrange(modules_count):
        this_row = modules[row]
        for col in modules_range_short:
            if (this_row[col]
                    and not this_row[col + 1]
                    and this_row[col + 2]
                    and this_row[col + 3]
                    and this_row[col + 4]
                    and not this_row[col + 5]
                    and this_row[col + 6]):
                lost_point += 40

    for col in xrange(modules_count):
        for row in modules_range_short:
            if (modules[row][col]
                    and not modules[row + 1][col]
                    and modules[row + 2][col]
                    and modules[row + 3][col]
                    and modules[row + 4][col]
                    and not modules[row + 5][col]
                    and modules[row + 6][col]):
                lost_point += 40

    return lost_point
Beispiel #11
0
 def write_to_buffer(self, buffer, colors=None):
     if self.mode == MODE_NUMBER:
         for i in xrange(0, len(self.data), 3):
             chars = self.data[i:i + 3]
             bit_length = NUMBER_LENGTH[len(chars)]
             color = self._getColor(i, colors)
             buffer.put(int(chars), bit_length, color)
     elif self.mode == MODE_ALPHA_NUM:
         for i in xrange(0, len(self.data), 2):
             chars = self.data[i:i + 2]
             color = self._getColor(i, colors)
             if len(chars) > 1:
                 buffer.put(
                     ALPHA_NUM.find(chars[0]) * 45 +
                     ALPHA_NUM.find(chars[1]), 11, color)
             else:
                 buffer.put(ALPHA_NUM.find(chars), 6, color)
     else:
         if six.PY3:
             # Iterating a bytestring in Python 3 returns an integer,
             # no need to ord().
             data = self.data
         else:
             data = [ord(c) for c in self.data]
         for i, c in enumerate(data):
             color = self._getColor(i, colors)
             buffer.put(c, 8, color)
 def next_batch(self, batch_size, fake_data=False):
   """Return the next `batch_size` examples from this data set."""
   if fake_data:
     fake_image = [1] * 784
     if self.one_hot:
       fake_label = [1] + [0] * 9
     else:
       fake_label = 0
     return [fake_image for _ in xrange(batch_size)], [
         fake_label for _ in xrange(batch_size)]
   start = self._index_in_epoch
   self._index_in_epoch += batch_size
   if self._index_in_epoch > self._num_examples:
     # Finished epoch
     self._epochs_completed += 1
     # Shuffle the data
     perm = numpy.arange(self._num_examples)
     numpy.random.shuffle(perm)
     self._images = self._images[perm]
     self._labels = self._labels[perm]
     # Start next epoch
     start = 0
     self._index_in_epoch = batch_size
     assert batch_size <= self._num_examples
   end = self._index_in_epoch
   return self._images[start:end], self._labels[start:end]
  def testParallelDequeueUpToRandomPartition(self):
    with self.test_session() as sess:
      dequeue_sizes = [random.randint(50, 150) for _ in xrange(10)]
      total_elements = sum(dequeue_sizes)
      q = tf.RandomShuffleQueue(total_elements, 0, tf.float32, shapes=())

      elems = [10.0 * x for x in xrange(total_elements)]
      enqueue_op = q.enqueue_many((elems,))
      dequeue_ops = [q.dequeue_up_to(size) for size in dequeue_sizes]

      enqueue_op.run()

      # Dequeue random number of items in parallel on 10 threads.
      dequeued_elems = []

      def dequeue(dequeue_op):
        dequeued_elems.extend(sess.run(dequeue_op))
      threads = []
      for dequeue_op in dequeue_ops:
        threads.append(self.checkedThread(target=dequeue, args=(dequeue_op,)))
      for thread in threads:
        thread.start()
      for thread in threads:
        thread.join()
      self.assertItemsEqual(elems, dequeued_elems)
Beispiel #14
0
 def visit_node(n, levels):
     lbl = "{"
     if data is None:
         if self.k <= max_k_labeled:
             lbl = repr(n.label()).\
                   replace("{","\{").\
                   replace("}","\}").\
                   replace("|","\|").\
                   replace("<","\<").\
                   replace(">","\>")
         else:
             lbl = str(n)
     else:
         s = self.bucket_to_block(n.bucket)
         for i in xrange(self.blocks_per_bucket):
             lbl += "{%s}" % (data[s+i])
             if i + 1 != self.blocks_per_bucket:
                 lbl += "|"
     lbl += "}"
     f.write("  %s [penwidth=%s,label=\"%s\"];\n"
             % (n.bucket, 1, lbl))
     levels += 1
     if (max_levels is None) or (levels <= max_levels):
         for i in xrange(self.k):
             cn = n.child_node(i)
             if not self.is_nil_node(cn):
                 visit_node(cn, levels)
                 f.write("  %s -> %s ;\n" % (n.bucket, cn.bucket))
Beispiel #15
0
    def show_topics(self, topics=10, topn=10, log=False, formatted=True):
        shown = []
        if topics < 0:
            topics = len(self.data)

        topics = min(topics, len(self.data))

        for k in xrange(topics):
            lambdak = list(self.data[k, :])
            lambdak = lambdak / sum(lambdak)

            temp = zip(lambdak, xrange(len(lambdak)))
            temp = sorted(temp, key=lambda x: x[0], reverse=True)

            topic_terms = self.show_topic_terms(temp, topn)

            if formatted:
                topic = self.format_topic(k, topic_terms)

                # assuming we only output formatted topics
                if log:
                    logger.info(topic)
            else:
                topic = (k, topic_terms)
            shown.append(topic)

        return shown
Beispiel #16
0
  def testConv3DTransposeSame(self):
    with self.test_session():
      strides = [1, 2, 2, 2, 1]

      # Input, output: [batch, depth, height, width, depth]
      x_shape = [2, 5, 6, 4, 3]
      y_shape = [2, 10, 12, 8, 2]

      # Filter: [kernel_depth, kernel_height, kernel_width, out_depth, in_depth]
      f_shape = [3, 3, 3, 2, 3]

      x = tf.constant(1.0, shape=x_shape, name="x", dtype=tf.float32)
      f = tf.constant(1.0, shape=f_shape, name="filter", dtype=tf.float32)
      output = tf.nn.conv3d_transpose(x, f, y_shape, strides=strides,
                                      padding="SAME")
      value = output.eval()

      for n in xrange(x_shape[0]):
        for k in xrange(f_shape[3]):
          for w in xrange(y_shape[3]):
            for h in xrange(y_shape[2]):
              for d in xrange(y_shape[1]):
                # We add a case for locations divisible by the stride.
                d_in = d % strides[1] == 0 and 0 < d < y_shape[1] - 1
                h_in = h % strides[2] == 0 and 0 < h < y_shape[2] - 1
                w_in = w % strides[3] == 0 and 0 < w < y_shape[3] - 1
                if d_in + h_in + w_in == 3:
                  target = 8 * 3.0
                elif d_in + h_in + w_in == 2:
                  target = 4 * 3.0
                elif d_in or h_in or w_in:
                  target = 2 * 3.0
                else:
                  target = 3.0
                self.assertAllClose(target, value[n, d, h, w, k])
Beispiel #17
0
    def add_lines(self, levels, colors, linewidths, erase=True):
        '''
        Draw lines on the colorbar.

        *colors* and *linewidths* must be scalars or
        sequences the same length as *levels*.

        Set *erase* to False to add lines without first
        removing any previously added lines.
        '''
        y = self._locate(levels)
        igood = (y < 1.001) & (y > -0.001)
        y = y[igood]
        if cbook.iterable(colors):
            colors = np.asarray(colors)[igood]
        if cbook.iterable(linewidths):
            linewidths = np.asarray(linewidths)[igood]
        N = len(y)
        x = np.array([0.0, 1.0])
        X, Y = np.meshgrid(x, y)
        if self.orientation == 'vertical':
            xy = [list(zip(X[i], Y[i])) for i in xrange(N)]
        else:
            xy = [list(zip(Y[i], X[i])) for i in xrange(N)]
        col = collections.LineCollection(xy, linewidths=linewidths)

        if erase and self.lines:
            for lc in self.lines:
                lc.remove()
            self.lines = []
        self.lines.append(col)
        col.set_color(colors)
        self.ax.add_collection(col)
        self.stale = True
Beispiel #18
0
  def testConv2DTransposeSingleStride(self):
    with self.test_session():
      strides = [1, 1, 1, 1]

      # Input, output: [batch, height, width, depth]
      x_shape = [2, 6, 4, 3]
      y_shape = [2, 6, 4, 2]

      # Filter: [kernel_height, kernel_width, output_depth, input_depth]
      f_shape = [3, 3, 2, 3]

      x = tf.constant(1.0, shape=x_shape, name="x", dtype=tf.float32)
      f = tf.constant(1.0, shape=f_shape, name="filter", dtype=tf.float32)
      output = tf.nn.conv2d_transpose(x, f, y_shape, strides=strides,
                                      padding="SAME")
      value = output.eval()

      # We count the number of cells being added at the locations in the output.
      # At the center, #cells=kernel_height * kernel_width
      # At the corners, #cells=ceil(kernel_height/2) * ceil(kernel_width/2)
      # At the borders, #cells=ceil(kernel_height/2)*kernel_width or
      #                        kernel_height * ceil(kernel_width/2)

      for n in xrange(x_shape[0]):
        for k in xrange(f_shape[2]):
          for w in xrange(y_shape[2]):
            for h in xrange(y_shape[1]):
              target = 4 * 3.0
              h_in = h > 0 and h < y_shape[1] - 1
              w_in = w > 0 and w < y_shape[2] - 1
              if h_in and w_in:
                target += 5 * 3.0
              elif h_in or w_in:
                target += 2 * 3.0
              self.assertAllClose(target, value[n, h, w, k])
Beispiel #19
0
  def testConv2DTransposeSame(self):
    with self.test_session():
      strides = [1, 2, 2, 1]

      # Input, output: [batch, height, width, depth]
      x_shape = [2, 6, 4, 3]
      y_shape = [2, 12, 8, 2]

      # Filter: [kernel_height, kernel_width, output_depth, input_depth]
      f_shape = [3, 3, 2, 3]

      x = tf.constant(1.0, shape=x_shape, name="x", dtype=tf.float32)
      f = tf.constant(1.0, shape=f_shape, name="filter", dtype=tf.float32)
      output = tf.nn.conv2d_transpose(x, f, y_shape, strides=strides,
                                      padding="SAME")
      value = output.eval()

      for n in xrange(x_shape[0]):
        for k in xrange(f_shape[2]):
          for w in xrange(y_shape[2]):
            for h in xrange(y_shape[1]):
              target = 3.0
              # We add a case for locations divisible by the stride.
              h_in = h % strides[1] == 0 and h > 0 and h < y_shape[1] - 1
              w_in = w % strides[2] == 0 and w > 0 and w < y_shape[2] - 1
              if h_in and w_in:
                target += 9.0
              elif h_in or w_in:
                target += 3.0
              self.assertAllClose(target, value[n, h, w, k])
Beispiel #20
0
def populate(parent, howmany, max_children):
    if howmany > max_children:
        children = randint(2, max_children)
        distribution = []
        for _ in xrange(0, children - 1):
            distribution.append(int(howmany / children))
        distribution.append(howmany - sum(distribution, 0))
        for i in xrange(0, children):
            steal_target = randint(0, children - 1)
            while steal_target == i:
                steal_target = randint(0, children -1)
            steal_count = int(randint(-1 * distribution[i],
                    distribution[steal_target]) / 2)
            distribution[i] += steal_count
            distribution[steal_target] -= steal_count

        for i in xrange(0, children):
            make_dict = randint(0, 1)
            if make_dict:
                baby = {}
            else:
                baby = []
            populate(baby, distribution[i], max_children)
            if isinstance(parent, dict):
                key = os.urandom(8)
                key = "".join(chr(c) for c in hexlify(key)) \
                    if PY3 else key.encode("hex")
                parent[key] = baby
            else:
                parent.append(baby)
    else:
        populate_with_leaves(parent, howmany)
Beispiel #21
0
def test_simple():
    fig = plt.figure()
    # un-comment to debug
#    recursive_pickle(fig)
    pickle.dump(fig, BytesIO(), pickle.HIGHEST_PROTOCOL)

    ax = plt.subplot(121)
    pickle.dump(ax, BytesIO(), pickle.HIGHEST_PROTOCOL)

    ax = plt.axes(projection='polar')
    plt.plot(list(xrange(10)), label='foobar')
    plt.legend()

#    recursive_pickle(fig)
    pickle.dump(ax, BytesIO(), pickle.HIGHEST_PROTOCOL)

#    ax = plt.subplot(121, projection='hammer')
#    recursive_pickle(ax, 'figure')
#    pickle.dump(ax, BytesIO(), pickle.HIGHEST_PROTOCOL)

    plt.figure()
    plt.bar(left=list(xrange(10)), height=list(xrange(10)))
    pickle.dump(plt.gca(), BytesIO(), pickle.HIGHEST_PROTOCOL)

    fig = plt.figure()
    ax = plt.axes()
    plt.plot(list(xrange(10)))
    ax.set_yscale('log')
    pickle.dump(fig, BytesIO(), pickle.HIGHEST_PROTOCOL)
Beispiel #22
0
    def parse_atoms(self, tokens, command, min_size, max_size=None):
        """
        Parses a sequence of N atoms (min_size <= N <= max_size) consuming
        the tokens
        """
        if max_size is None:
            max_size = min_size

        res = []
        current = None
        for _ in xrange(min_size):
            current = next(tokens)
            if current == ")":
                raise SyntaxError("Expected at least %d arguments in %s command." % (min_size, command))
            if current == "(":
                raise SyntaxError("Unexpected token '(' in %s command." % command)
            res.append(current)

        for _ in xrange(min_size, max_size + 1):
            current = next(tokens)
            if current == ")":
                return res
            if current == "(":
                raise SyntaxError("Unexpected token '(' in %s command." % command)
            res.append(current)
        raise SyntaxError(
            "Unexpected token '%s' in %s command. Expected at " "most %d arguments." % (current, command, max_size)
        )
Beispiel #23
0
def xfun(n, d=None):
    """ Create a QTT-representation of 0:prod(n) _vector
        call examples:
        tt.xfun(2, 5)         # create 2 x 2 x 2 x 2 x 2 TT-vector
        tt.xfun(3)            # create [0, 1, 2] one-dimensional TT-vector
        tt.xfun([3, 5, 7], 2) # create 3 x 5 x 7 x 3 x 5 x 7 TT-vector
    """
    if isinstance(n, six.integer_types):
        n = [n]
    if d is None:
        n0 = _np.asanyarray(n, dtype=_np.int32)
    else:
        n0 = _np.array(n * d, dtype=_np.int32)
    d = n0.size
    if d == 1:
        return _vector.vector.from_list(
            [_np.reshape(_np.arange(n0[0]), (1, n0[0], 1))])
    cr = []
    cur_core = _np.ones((1, n0[0], 2))
    cur_core[0, :, 0] = _np.arange(n0[0])
    cr.append(cur_core)
    ni = float(n0[0])
    for i in xrange(1, d - 1):
        cur_core = _np.zeros((2, n0[i], 2))
        for j in xrange(n0[i]):
            cur_core[:, j, :] = _np.eye(2)
        cur_core[1, :, 0] = ni * _np.arange(n0[i])
        ni *= n0[i]
        cr.append(cur_core)
    cur_core = _np.ones((2, n0[d - 1], 1))
    cur_core[1, :, 0] = ni * _np.arange(n0[d - 1])
    cr.append(cur_core)
    return _vector.vector.from_list(cr)
Beispiel #24
0
    def test_neville2d(self):
        funcx = numpy.sin
        funcy = numpy.exp
        nrow = 10
        ncol = 10
        tol = 1.0e-4
        # TODO: As with test_neville; can this not be simplified with
        # vectorized code
        x = numpy.zeros((nrow, ))
        y = numpy.zeros((ncol, ))
        fval = numpy.empty((nrow, ncol))
        row_tmp = numpy.pi / nrow
        # col_tmp = 1.0 / float(ncol)
        for row in xrange(nrow):
            x[row] = (row + 1.0) * row_tmp
            for col in xrange(ncol):
                y[col] = (col + 1.0) / float(ncol)
                fval[row][col] = funcx(x[row]) * funcy(y[col])

        for row in xrange(ncol):
            xx = (-0.1 + (row + 1.0) / float(nrow)) * numpy.pi
            for col in xrange(4):
                yy = -0.1 + (col + 1.0) / float(ncol)
                answer = funcx(xx) * funcy(yy)
                val = utils.neville2d(xx, yy, x, y, fval)
                self.assertTrue(utils.Knuth_close(answer, val, tol))
Beispiel #25
0
    def filter(self, im):
        falloff = self.falloff
        extent = self.extent

        def length(start, end):
            start_x, start_y = start
            end_x, end_y = end
            dist_x = end_x - start_x
            dist_y = end_y - start_y
            return math.sqrt((dist_x ** 2) + (dist_y ** 2))

        def light_falloff(radius, outside):
            return ((radius / outside) ** falloff) * extent

        im = im.convert('RGBA')

        w, h = im.size
        center = w / 2, h / 2
        outside = length(center, (0, 0))

        data = []
        for y in xrange(h):
            for x in xrange(w):
                radius = length(center, (x, y))
                factor = light_falloff(radius, outside)
                data.append(factor)

        alpha_im = Image.new('L', im.size)
        alpha_im.putdata(data)
        overlay_im = Image.new('L', im.size, 'black')
        return Image.composite(overlay_im, im, alpha_im)
Beispiel #26
0
  def testParams(self):
    """Tests that the params work as intended."""
    num_classes = 2
    with self.test_session() as sess:
      # Experiment 1. Update weights only.
      data = constant_op.constant(self.data, dtype=dtypes.float32)
      gmm_tool = gmm_ops.GmmAlgorithm([data], num_classes,
                                      [[3.0, 3.0], [0.0, 0.0]], 'w')
      training_ops = gmm_tool.training_ops()
      variables.global_variables_initializer().run()
      sess.run(gmm_tool.init_ops())
      for _ in xrange(self.iterations):
        sess.run(training_ops)

      # Only the probability to each class is updated.
      alphas = sess.run(gmm_tool.alphas())
      self.assertGreater(alphas[1], 0.6)
      means = sess.run(gmm_tool.clusters())
      np.testing.assert_almost_equal(
          np.expand_dims([[3.0, 3.0], [0.0, 0.0]], 1), means)
      covs = sess.run(gmm_tool.covariances())
      np.testing.assert_almost_equal(covs[0], covs[1])

      # Experiment 2. Update means and covariances.
      gmm_tool = gmm_ops.GmmAlgorithm([data], num_classes,
                                      [[3.0, 3.0], [0.0, 0.0]], 'mc')
      training_ops = gmm_tool.training_ops()
      variables.global_variables_initializer().run()
      sess.run(gmm_tool.init_ops())
      for _ in xrange(self.iterations):
        sess.run(training_ops)
      alphas = sess.run(gmm_tool.alphas())
      self.assertAlmostEqual(alphas[0], alphas[1])
      means = sess.run(gmm_tool.clusters())
      np.testing.assert_almost_equal(
          np.expand_dims([[2.0, 2.0], [-1.0, -1.0]], 1), means, decimal=1)
      covs = sess.run(gmm_tool.covariances())
      np.testing.assert_almost_equal(
          [[0.371111, -0.0050774], [-0.0050774, 0.8651744]], covs[0], decimal=4)
      np.testing.assert_almost_equal(
          [[0.146976, 0.0259463], [0.0259463, 0.2543971]], covs[1], decimal=4)

      # Experiment 3. Update covariances only.
      gmm_tool = gmm_ops.GmmAlgorithm([data], num_classes,
                                      [[-1.0, -1.0], [1.0, 1.0]], 'c')
      training_ops = gmm_tool.training_ops()
      variables.global_variables_initializer().run()
      sess.run(gmm_tool.init_ops())
      for _ in xrange(self.iterations):
        sess.run(training_ops)
      alphas = sess.run(gmm_tool.alphas())
      self.assertAlmostEqual(alphas[0], alphas[1])
      means = sess.run(gmm_tool.clusters())
      np.testing.assert_almost_equal(
          np.expand_dims([[-1.0, -1.0], [1.0, 1.0]], 1), means)
      covs = sess.run(gmm_tool.covariances())
      np.testing.assert_almost_equal(
          [[0.1299582, 0.0435872], [0.0435872, 0.2558578]], covs[0], decimal=5)
      np.testing.assert_almost_equal(
          [[3.195385, 2.6989155], [2.6989155, 3.3881593]], covs[1], decimal=5)
Beispiel #27
0
  def SoftmaxEval(self, sess, model, num_steps):
    """Evaluate a model in softmax mode.

    Adds char, word recall and sequence error rate events to the sw summary
    writer, and returns them as well
    TODO(rays) Add LogisticEval.
    Args:
      sess:  A tensor flow Session.
      model: The model to run in the session. Requires a VGSLImageModel or any
        other class that has a using_ctc attribute and a RunAStep(sess) method
        that reurns a softmax result with corresponding labels.
      num_steps: Number of steps to evaluate for.
    Returns:
      ErrorRates named tuple.
    Raises:
      ValueError: If an unsupported number of dimensions is used.
    """
    coord = tf.train.Coordinator()
    threads = tf.train.start_queue_runners(sess=sess, coord=coord)
    # Run the requested number of evaluation steps, gathering the outputs of the
    # softmax and the true labels of the evaluation examples.
    total_label_counts = ec.ErrorCounts(0, 0, 0, 0)
    total_word_counts = ec.ErrorCounts(0, 0, 0, 0)
    sequence_errors = 0
    for _ in xrange(num_steps):
      softmax_result, labels = model.RunAStep(sess)
      # Collapse softmax to same shape as labels.
      predictions = softmax_result.argmax(axis=-1)
      # Exclude batch from num_dims.
      num_dims = len(predictions.shape) - 1
      batch_size = predictions.shape[0]
      null_label = softmax_result.shape[-1] - 1
      for b in xrange(batch_size):
        if num_dims == 2:
          # TODO(rays) Support 2-d data.
          raise ValueError('2-d label data not supported yet!')
        else:
          if num_dims == 1:
            pred_batch = predictions[b, :]
            labels_batch = labels[b, :]
          else:
            pred_batch = [predictions[b]]
            labels_batch = [labels[b]]
          text = self.StringFromCTC(pred_batch, model.using_ctc, null_label)
          truth = self.StringFromCTC(labels_batch, False, null_label)
          # Note that recall_errs is false negatives (fn) aka drops/deletions.
          # Actual recall would be 1-fn/truth_words.
          # Likewise precision_errs is false positives (fp) aka adds/insertions.
          # Actual precision would be 1-fp/ocr_words.
          total_word_counts = ec.AddErrors(total_word_counts,
                                           ec.CountWordErrors(text, truth))
          total_label_counts = ec.AddErrors(total_label_counts,
                                            ec.CountErrors(text, truth))
          if text != truth:
            sequence_errors += 1

    coord.request_stop()
    coord.join(threads)
    return ec.ComputeErrorRates(total_label_counts, total_word_counts,
                                sequence_errors, num_steps * batch_size)
Beispiel #28
0
def delta(n, d=None, center=0):
    """ Create TT-vector for delta-function :math:`\\delta(x - x_0)`. """
    if isinstance(n, six.integer_types):
        n = [n]
    if d is None:
        n0 = _np.asanyarray(n, dtype=_np.int32)
    else:
        n0 = _np.array(n * d, dtype=_np.int32)
    d = n0.size

    if center < 0:
        cind = [0] * d
    else:
        cind = []
        for i in xrange(d):
            cind.append(center % n0[i])
            center //= n0[i]
        if center > 0:
            cind = [0] * d
    cr = []
    for i in xrange(d):
        cur_core = _np.zeros((1, n0[i], 1))
        cur_core[0, cind[i], 0] = 1
        cr.append(cur_core)
    return _vector.vector.from_list(cr)
Beispiel #29
0
  def testTFSummaryScalar(self):
    """Verify processing of tf.summary.scalar."""
    event_sink = _EventGenerator(self, zero_out_timestamps=True)
    writer = SummaryToEventTransformer(event_sink)
    with self.test_session() as sess:
      ipt = array_ops.placeholder(dtypes.float32)
      summary_lib.scalar('scalar1', ipt)
      summary_lib.scalar('scalar2', ipt * ipt)
      merged = summary_lib.merge_all()
      writer.add_graph(sess.graph)
      for i in xrange(10):
        summ = sess.run(merged, feed_dict={ipt: i})
        writer.add_summary(summ, global_step=i)

    accumulator = ea.EventAccumulator(event_sink)
    accumulator.Reload()

    seq1 = [ea.ScalarEvent(wall_time=0, step=i, value=i) for i in xrange(10)]
    seq2 = [
        ea.ScalarEvent(
            wall_time=0, step=i, value=i * i) for i in xrange(10)
    ]

    self.assertTagsEqual(accumulator.Tags(), {
        ea.SCALARS: ['scalar1', 'scalar2'],
        ea.GRAPH: True,
        ea.META_GRAPH: False,
    })

    self.assertEqual(accumulator.Scalars('scalar1'), seq1)
    self.assertEqual(accumulator.Scalars('scalar2'), seq2)
    first_value = accumulator.Scalars('scalar1')[0].value
    self.assertTrue(isinstance(first_value, float))
Beispiel #30
0
    def back(self, expr, model=None):
        """Convert a Z3 expression back into a pySMT expression.

        This is done using the Z3 API. For very big expressions, it is
        sometimes faster to go through the SMT-LIB format. In those
        cases, consider using the method back_via_smtlib.
        """
        stack = [expr]
        while len(stack) > 0:
            current = stack.pop()
            key = (askey(current), model)
            if key not in self._back_memoization:
                self._back_memoization[key] = None
                stack.append(current)
                for i in xrange(current.num_args()):
                    stack.append(current.arg(i))
            elif self._back_memoization[key] is None:
                args = [self._back_memoization[(askey(current.arg(i)), model)]
                        for i in xrange(current.num_args())]
                res = self._back_single_term(current, args, model)
                self._back_memoization[key] = res
            else:
                # we already visited the node, nothing else to do
                pass
        return self._back_memoization[(askey(expr), model)]
 def _GenerateUnorderedInputs(self, size, n):
     inputs = [tf.random_uniform(shape=[size]) for _ in xrange(n)]
     random.shuffle(inputs)
     return inputs
Beispiel #32
0
  def get_data(self, how_many, offset, model_settings, background_frequency,
               background_volume_range, time_shift, mode, sess):
    """Gather samples from the data set, applying transformations as needed.

    When the mode is 'training', a random selection of samples will be returned,
    otherwise the first N clips in the partition will be used. This ensures that
    validation always uses the same samples, reducing noise in the metrics.

    Args:
      how_many: Desired number of samples to return. -1 means the entire
        contents of this partition.
      offset: Where to start when fetching deterministically.
      model_settings: Information about the current model being trained.
      background_frequency: How many clips will have background noise, 0.0 to
        1.0.
      background_volume_range: How loud the background noise will be.
      time_shift: How much to randomly shift the clips by in time.
      mode: Which partition to use, must be 'training', 'validation', or
        'testing'.
      sess: TensorFlow session that was active when processor was created.

    Returns:
      List of sample data for the transformed samples, and list of labels in
      one-hot form.
    """
    # Pick one of the partitions to choose samples from.
    candidates = self.data_index[mode]
    if how_many == -1:
      sample_count = len(candidates)
    else:
      sample_count = max(0, min(how_many, len(candidates) - offset))
    # Data and labels will be populated and returned.
    if model_settings['raw_data']:
      data = np.zeros((sample_count, model_settings['sample_rate']))
    else:
      data = np.zeros((sample_count, model_settings['fingerprint_size']))
    labels = np.zeros((sample_count, model_settings['label_count']))
    names = []
    desired_samples = model_settings['desired_samples']
    use_background = self.background_data and (mode == 'training')
    pick_deterministically = (mode != 'training')
    # Use the processing graph we created earlier to repeatedly to generate the
    # final output sample data we'll use in training.
    for i in xrange(offset, offset + sample_count):
      # Pick which audio sample to use.
      if how_many == -1 or pick_deterministically:
        sample_index = i
      else:
        sample_index = np.random.randint(len(candidates))
      sample = candidates[sample_index]
      # If we're time shifting, set up the offset for this sample.
      if time_shift > 0:
        time_shift_amount = np.random.randint(-time_shift, time_shift)
      else:
        time_shift_amount = 0
      if time_shift_amount > 0:
        time_shift_padding = [[time_shift_amount, 0], [0, 0]]
        time_shift_offset = [0, 0]
      else:
        time_shift_padding = [[0, -time_shift_amount], [0, 0]]
        time_shift_offset = [-time_shift_amount, 0]
      input_dict = {
          self.wav_filename_placeholder_: sample['file'],
          self.time_shift_padding_placeholder_: time_shift_padding,
          self.time_shift_offset_placeholder_: time_shift_offset,
      }
      # Choose a section of background noise to mix in.
      if use_background:
        background_index = np.random.randint(len(self.background_data))
        background_samples = self.background_data[background_index]
        background_offset = np.random.randint(
            0, len(background_samples) - model_settings['desired_samples'])
        background_clipped = background_samples[background_offset:(
            background_offset + desired_samples)]
        background_reshaped = background_clipped.reshape([desired_samples, 1])
        if np.random.uniform(0, 1) < background_frequency:
          background_volume = np.random.uniform(0, background_volume_range)
        else:
          background_volume = 0
      else:
        background_reshaped = np.zeros([desired_samples, 1])
        background_volume = 0
      input_dict[self.background_data_placeholder_] = background_reshaped
      input_dict[self.background_volume_placeholder_] = background_volume
      # If we want silence, mute out the main sample but leave the background.
      if sample['label'] == SILENCE_LABEL:
        input_dict[self.foreground_volume_placeholder_] = 0
      else:
        input_dict[self.foreground_volume_placeholder_] = 1
      # Run the graph to produce the output audio.
      data[i - offset, :] = sess.run(self.mfcc_, feed_dict=input_dict).flatten()
      names.append(sample['label'])
      label_index = self.word_to_index[sample['label']]
      labels[i - offset, label_index] = 1
    return data, labels, names
Beispiel #33
0
    def model_fn_body_sharded(self, sharded_features):
        train = self._hparams.mode == tf.contrib.learn.ModeKeys.TRAIN
        dp = self._data_parallelism
        hparams = self._hparams

        def flatten(inputs):
            return tf.expand_dims(common_layers.flatten4d3d(inputs), axis=2)

        inputs = dp(flatten, sharded_features["inputs"])
        inputs_pad = dp(slicenet.embedding_to_padding, inputs)
        inputs_mask = dp(lambda x: 1.0 - x, inputs_pad)
        inputs_encoded = dp(common_layers.add_timing_signal, inputs)
        expert_loss = 0.0
        for i in xrange(hparams.num_hidden_layers):
            with tf.variable_scope("enc_layer_%d" % i):
                inputs_encoded, moe_loss = conv_experts(
                    inputs_encoded, hparams, dp, self._ps_devices, "SAME",
                    inputs_mask, i)
                expert_loss += tf.reduce_mean(moe_loss) * hparams.moe_loss_coef

        # If we're just predicing a class, there is no use for a decoder, return.
        if isinstance(hparams.problems[self._problem_idx].target_modality,
                      modalities.ClassLabelModality):
            return inputs_encoded, tf.reduce_mean(expert_loss)

        # Decoder.
        inputs3d = dp(tf.squeeze, inputs, 2)
        inputs_encoded3d = dp(tf.squeeze, inputs_encoded, 2)
        encoder_padding = dp(common_attention.embedding_to_padding, inputs3d)
        encoder_attention_bias = dp(
            common_attention.attention_bias_ignore_padding, encoder_padding)
        targets = dp(common_layers.flatten4d3d, sharded_features["targets"])
        target_space_emb = dp(slicenet.embed_target_space,
                              sharded_features["target_space_id"],
                              hparams.hidden_size)

        (decoder_input,
         decoder_self_attention_bias) = dp(prepare_decoder, targets,
                                           target_space_emb)

        x = dp(tf.nn.dropout, decoder_input, 1.0 - hparams.dropout)
        for layer in xrange(hparams.num_hidden_layers):
            with tf.variable_scope("dec_layer_%d" % layer):
                with tf.variable_scope("attention"):
                    y = dp(common_attention.multihead_attention,
                           x,
                           None,
                           decoder_self_attention_bias,
                           hparams.hidden_size,
                           hparams.hidden_size,
                           hparams.hidden_size,
                           hparams.num_heads,
                           hparams.attention_dropout,
                           name="decoder_self_attention")
                    z = dp(common_attention.multihead_attention,
                           y,
                           inputs_encoded3d,
                           encoder_attention_bias,
                           hparams.hidden_size,
                           hparams.hidden_size,
                           hparams.hidden_size,
                           hparams.num_heads,
                           hparams.attention_dropout,
                           name="encdec_attention")
                    x = dp(residual_fn3, x, y, z, hparams)
                with tf.variable_scope("ffn"):
                    if str(layer) in hparams.moe_layers.split(","):
                        y, moe_loss = common_layers.moe_layer(
                            dp, self._ps_devices, x, train,
                            hparams.hidden_size, hparams.filter_size,
                            hparams.moe_n1, hparams.moe_n2,
                            hparams.moe_loss_coef)
                        expert_loss += tf.reduce_mean(moe_loss)
                    else:
                        y = dp(common_layers.conv_hidden_relu,
                               x,
                               hparams.filter_size,
                               hparams.hidden_size,
                               dropout=hparams.dropout)
                    x = dp(residual_fn2, x, y, hparams)

        x = dp(tf.expand_dims, x, 2)
        return x, tf.reduce_mean(expert_loss)
Beispiel #34
0
def embedding_lookup(params,
                     ids,
                     partition_strategy="mod",
                     name=None,
                     validate_indices=True):
    """Looks up `ids` in a list of embedding tensors.
  
    This function is used to perform parallel lookups on the list of
    tensors in `params`.  It is a generalization of
    [`tf.gather()`](../../api_docs/python/array_ops.md#gather), where `params` is
    interpreted as a partitioning of a large embedding tensor.  `params` may be
    a `PartitionedVariable` as returned by using `tf.get_variable()` with a
    partitioner.
  
    If `len(params) > 1`, each element `id` of `ids` is partitioned between
    the elements of `params` according to the `partition_strategy`.
    In all strategies, if the id space does not evenly divide the number of
    partitions, each of the first `(max_id + 1) % len(params)` partitions will
    be assigned one more id.
  
    If `partition_strategy` is `"mod"`, we assign each id to partition
    `p = id % len(params)`. For instance,
    13 ids are split across 5 partitions as:
    `[[0, 5, 10], [1, 6, 11], [2, 7, 12], [3, 8], [4, 9]]`
  
    If `partition_strategy` is `"div"`, we assign ids to partitions in a
    contiguous manner. In this case, 13 ids are split across 5 partitions as:
    `[[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10], [11, 12]]`
  
    The results of the lookup are concatenated into a dense
    tensor. The returned tensor has shape `shape(ids) + shape(params)[1:]`.
  
    Args:
      params: A list of tensors with the same type and which can be concatenated
        along dimension 0. Alternatively, a `PartitionedVariable`, created by
        partitioning along dimension 0.  Each element must be appropriately sized
        for the given `partition_strategy`.
      ids: A `Tensor` with type `int32` or `int64` containing the ids to be looked
        up in `params`.
      partition_strategy: A string specifying the partitioning strategy, relevant
        if `len(params) > 1`. Currently `"div"` and `"mod"` are supported. Default
        is `"mod"`.
      name: A name for the operation (optional).
      validate_indices: Whether or not to validate gather indices.
  
    Returns:
      A `Tensor` with the same type as the tensors in `params`.
  
    Raises:
      ValueError: If `params` is empty.
    """
    if params is None or params == []:  # pylint: disable=g-explicit-bool-comparison
        raise ValueError("Need at least one param")
    if isinstance(params, variables.PartitionedVariable):
        params = list(params)  # Iterate to get the underlying Variables.
    if not isinstance(params, list):
        params = [params]
    with ops.name_scope(name, "embedding_lookup", params + [ids]) as name:
        np = len(params)  # Number of partitions
        params = ops.convert_n_to_tensor_or_indexed_slices(params,
                                                           name="params")
        if np == 1:
            with ops.colocate_with(params[0]):
                return array_ops.gather(params[0],
                                        ids,
                                        name=name,
                                        validate_indices=validate_indices)
        else:
            ids = ops.convert_to_tensor(ids, name="ids")
            flat_ids = array_ops.reshape(ids, [-1])
            original_indices = math_ops.range(array_ops.size(flat_ids))

            # Create p_assignments and set new_ids depending on the strategy.
            if partition_strategy == "mod":
                p_assignments = flat_ids % np
                new_ids = flat_ids // np
            elif partition_strategy == "div":
                # Compute num_total_ids as the sum of dim-0 of params, then assign to
                # partitions based on a constant number of ids per partition. Optimize
                # if we already know the full shape statically.
                dim_0_size = params[0].get_shape()[0]
                for p in xrange(1, np):
                    dim_0_size += params[p].get_shape()[0]
                if dim_0_size.value:
                    num_total_ids = constant_op.constant(
                        dim_0_size.value, flat_ids.dtype)
                else:
                    dim_0_sizes = []
                    for p in xrange(np):
                        if params[p].get_shape()[0].value is not None:
                            dim_0_sizes.append(params[p].get_shape()[0].value)
                        else:
                            with ops.colocate_with(params[p]):
                                dim_0_sizes.append(
                                    array_ops.shape(params[p])[0])
                    num_total_ids = math_ops.reduce_sum(
                        math_ops.cast(array_ops.pack(dim_0_sizes),
                                      flat_ids.dtype))
                ids_per_partition = num_total_ids // np
                extras = num_total_ids % np

                p_assignments = math_ops.maximum(
                    flat_ids // (ids_per_partition + 1),
                    (flat_ids - extras) // ids_per_partition)

                # Emulate a conditional using a boolean indicator tensor
                is_in_first_extras_partitions = math_ops.cast(
                    p_assignments < extras, flat_ids.dtype)
                new_ids = (is_in_first_extras_partitions *
                           (flat_ids % (ids_per_partition + 1)) +
                           (1 - is_in_first_extras_partitions) *
                           ((flat_ids - extras) % ids_per_partition))
            else:
                raise ValueError("Unrecognized partition strategy: " +
                                 partition_strategy)

            # Cast partition assignments to int32 for use in dynamic_partition.
            # There really should not be more than 2^32 partitions.
            p_assignments = math_ops.cast(p_assignments, dtypes.int32)
            # Partition list of ids based on assignments into np separate lists
            gather_ids = data_flow_ops.dynamic_partition(
                new_ids, p_assignments, np)
            # Similarly, partition the original indices.
            pindices = data_flow_ops.dynamic_partition(original_indices,
                                                       p_assignments, np)
            # Do np separate lookups, finding embeddings for plist[p] in params[p]
            partitioned_result = []
            for p in xrange(np):
                with ops.colocate_with(params[p]):
                    partitioned_result.append(
                        array_ops.gather(params[p],
                                         gather_ids[p],
                                         validate_indices=validate_indices))
            # Stitch these back together
            ret = data_flow_ops.dynamic_stitch(pindices,
                                               partitioned_result,
                                               name=name)
            # Reshape to reverse the flattening of ids.
            element_shape = params[0].get_shape()[1:]
            for p in params[1:]:
                element_shape = element_shape.merge_with(p.get_shape()[1:])
            if element_shape.is_fully_defined():
                ret = array_ops.reshape(
                    ret,
                    array_ops.concat(0, [array_ops.shape(ids), element_shape]))
            else:
                # It's important that we compute params[0].shape on the right device
                # to avoid data motion.
                with ops.colocate_with(params[0]):
                    params_shape = array_ops.shape(params[0])
                ret = array_ops.reshape(
                    ret,
                    array_ops.concat(0, [
                        array_ops.shape(ids),
                        array_ops.slice(params_shape, [1], [-1])
                    ]))
            # output shape = ids.shape + params[*].shape[1:]
            # Normally the reshape is sufficient, but setting shape explicitly
            # teaches shape inference that params[1:].get_shape() matters.
            ret.set_shape(ids.get_shape().concatenate(element_shape))
            return ret
Beispiel #35
0
def main(argv=None):
    keep_probability = tf.placeholder(tf.float32, name="keep_probabilty")
    image = tf.placeholder(tf.float32,
                           shape=[None, IMAGE_SIZE, IMAGE_SIZE, 3],
                           name="input_image")
    annotation = tf.placeholder(tf.float32,
                                shape=[None, IMAGE_SIZE, IMAGE_SIZE, 3],
                                name="annotation")
    z = tf.placeholder(tf.float32, shape=[None, 7, 7, 3], name="z")

    pred_annotation, logits = inference(image, keep_probability, z)
    tf.summary.image("input_image", image, max_outputs=2)
    tf.summary.image("ground_truth",
                     tf.cast(annotation, tf.uint8),
                     max_outputs=2)
    tf.summary.image("pred_annotation",
                     tf.cast(pred_annotation, tf.uint8),
                     max_outputs=2)
    #    loss = tf.reduce_mean((tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits,
    #                                                                         labels=tf.squeeze(annotation, squeeze_dims=[3]),
    #                                                                    name="entropy")))

    loss = tf.reduce_mean(tf.squared_difference(logits, annotation))
    loss_summary = tf.summary.scalar("entropy", loss)

    grads = train_z(loss, z)

    trainable_var = tf.trainable_variables()
    if FLAGS.debug:
        for var in trainable_var:
            utils.add_to_regularization_and_summary(var)
    train_op = train(loss, trainable_var)

    print("Setting up summary op...")
    summary_op = tf.summary.merge_all()

    print("Setting up image reader...")
    train_records, valid_records = scene_parsing.read_dataset(FLAGS.data_dir)
    print(len(train_records))
    print(len(valid_records))

    print("Setting up dataset reader")
    image_options = {'resize': True, 'resize_size': IMAGE_SIZE}
    if FLAGS.mode == 'train':
        train_dataset_reader = dataset.BatchDatset(train_records,
                                                   image_options)
    validation_dataset_reader = dataset.BatchDatset(valid_records,
                                                    image_options)

    sess = tf.Session()

    print("Setting up Saver...")
    saver = tf.train.Saver()

    # create two summary writers to show training loss and validation loss in the same graph
    # need to create two folders 'train' and 'validation' inside FLAGS.logs_dir
    train_writer = tf.summary.FileWriter(FLAGS.logs_dir + '/train', sess.graph)
    validation_writer = tf.summary.FileWriter(FLAGS.logs_dir + '/validation')

    sess.run(tf.global_variables_initializer())
    ckpt = tf.train.get_checkpoint_state(FLAGS.logs_dir)
    if ckpt and ckpt.model_checkpoint_path:
        saver.restore(sess, ckpt.model_checkpoint_path)
        print("Model restored...")

    if FLAGS.mode == "train":
        for itr in xrange(MAX_ITERATION):

            train_images, train_annotations = train_dataset_reader.next_batch(
                FLAGS.batch_size)
            xx = random.randint(0, 100)
            xy = random.randint(0, 100)
            h = random.randint(50, 100)
            w = random.randint(50, 100)
            train_images[:, xx:xx + h, xy:xy + w, :] = 0

            z_ = np.random.uniform(low=-1.0,
                                   high=1.0,
                                   size=(FLAGS.batch_size, 7, 7, 3))

            feed_dict = {
                image: train_images,
                annotation: train_annotations,
                keep_probability: 0.85,
                z: z_
            }
            #train_images[:,50:100,50:100,:] =0
            v = 0

            for p in range(20):
                z_ol = np.copy(z_)
                # print("666666666666666666666666666666666666666")
                z_loss, summ = sess.run([loss, loss_summary],
                                        feed_dict=feed_dict)
                print("Step: %d, z_step: %d, Train_loss:%g" % (itr, p, z_loss))
                #                print(z_)
                g = sess.run([grads], feed_dict=feed_dict)
                v_prev = np.copy(v)
                # print(g[0][0].shape)
                v = 0.001 * v - 0.1 * g[0][0]
                z_ += 0.001 * v_prev + (1 + 0.001) * v
            # z_ = np.clip(z_, -1.0, 1.0)
            # print(v.shape)
            # print(z_.shape)
            sess.run(train_op, feed_dict=feed_dict)

            if itr % 10 == 0:
                train_loss, summary_str = sess.run([loss, loss_summary],
                                                   feed_dict=feed_dict)
                print("Step: %d, Train_loss:%g" % (itr, train_loss))

                train_writer.add_summary(summary_str, itr)

            if itr % 500 == 0:
                valid_images, valid_annotations = validation_dataset_reader.next_batch(
                    FLAGS.batch_size)
                xx = random.randint(50, 100)
                xy = random.randint(50, 100)
                h = random.randint(50, 100)
                w = random.randint(50, 100)
                valid_images[:, xx:xx + h, xy:xy + w, :] = 0
                valid_loss, summary_sva = sess.run(
                    [loss, loss_summary],
                    feed_dict={
                        image: valid_images,
                        annotation: valid_annotations,
                        keep_probability: 1.0,
                        z: z_
                    })
                print("%s ---> Validation_loss: %g" %
                      (datetime.datetime.now(), valid_loss))

                # add validation loss to TensorBoard
                validation_writer.add_summary(summary_sva, itr)
                saver.save(sess, FLAGS.logs_dir + "model_z.ckpt", 500)

    elif FLAGS.mode == "visualize":
        valid_images, valid_annotations = validation_dataset_reader.get_random_batch(
            2)
        xx = random.randint(50, 100)
        xy = random.randint(50, 100)
        h = random.randint(50, 100)
        w = random.randint(50, 100)
        valid_images[:, xx:xx + h, xy:xy + w, :] = 0
        z_ = np.random.uniform(low=-1.0,
                               high=1.0,
                               size=(FLAGS.batch_size, 7, 7, 3))
        feed_dict = {
            image: valid_images,
            annotation: valid_annotations,
            keep_probability: 0.85,
            z: z_
        }
        v = 0
        for p in range(20):
            z_ol = np.copy(z_)
            # print("666666666666666666666666666666666666666")
            z_loss, summ = sess.run([loss, loss_summary], feed_dict=feed_dict)
            print("z_step: %d, Train_loss:%g" % (p, z_loss))
            #                print(z_)
            g = sess.run([grads], feed_dict=feed_dict)
            v_prev = np.copy(v)
            # print(g[0][0].shape)
            v = 0.001 * v - 0.1 * g[0][0]
            z_ += 0.001 * v_prev + (1 + 0.001) * v
        # z_ = np.clip(z_, -1.0, 1.0)

        pred = sess.run(logits,
                        feed_dict={
                            image: valid_images,
                            annotation: valid_annotations,
                            z: z_,
                            keep_probability: 1.0
                        })

        # valid_annotations = np.squeeze(valid_annotations, axis=3)
        # pred = np.squeeze(pred, axis=3)
        print(valid_images.shape)
        print(valid_annotations.shape)
        print(pred.shape)

        for itr in range(FLAGS.batch_size):
            utils.save_image(valid_images[itr].astype(np.uint8),
                             FLAGS.logs_dir,
                             name="inp_" + str(5 + itr))
            utils.save_image(valid_annotations[itr].astype(np.uint8),
                             FLAGS.logs_dir,
                             name="gt_" + str(5 + itr))
            utils.save_image(pred[itr].astype(np.uint8),
                             FLAGS.logs_dir,
                             name="pred_" + str(5 + itr))
            print("Saved image: %d" % itr)
def run_training():
    # Get the sets of images and labels for training, validation, and
    # Tell TensorFlow that the model will be built into the default Graph.

    # Create model directory
    if not os.path.exists(model_save_dir):
        os.makedirs(model_save_dir)
    rgb_pre_model_save_dir = "/home/project/I3D/I3D/checkpoints/rgb_scratch"

    with tf.Graph().as_default():
        global_step = tf.get_variable('global_step', [],
                                      initializer=tf.constant_initializer(0),
                                      trainable=False)
        rgb_images_placeholder, flow_images_placeholder, labels_placeholder, is_training = placeholder_inputs(
            FLAGS.batch_size * gpu_num, FLAGS.num_frame_per_clib,
            FLAGS.crop_size, FLAGS.rgb_channels, FLAGS.flow_channels)

        learning_rate = tf.train.exponential_decay(FLAGS.learning_rate,
                                                   global_step,
                                                   decay_steps=3000,
                                                   decay_rate=0.1,
                                                   staircase=True)
        opt_rgb = tf.train.AdamOptimizer(learning_rate)
        #opt_stable = tf.train.MomentumOptimizer(learning_rate, 0.9)
        with tf.variable_scope('RGB'):
            rgb_logit, _ = InceptionI3d(
                num_classes=FLAGS.classics,
                spatial_squeeze=True,
                final_endpoint='Logits')(rgb_images_placeholder, is_training)
        rgb_loss = tower_loss(rgb_logit, labels_placeholder)
        accuracy = tower_acc(rgb_logit, labels_placeholder)
        update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
        with tf.control_dependencies(update_ops):
            rgb_grads = opt_rgb.compute_gradients(rgb_loss)
            apply_gradient_rgb = opt_rgb.apply_gradients(
                rgb_grads, global_step=global_step)
            train_op = tf.group(apply_gradient_rgb)
            null_op = tf.no_op()

        # Create a saver for loading trained checkpoints.
        rgb_variable_map = {}
        for variable in tf.global_variables():
            if variable.name.split(
                    '/')[0] == 'RGB' and 'Adam' not in variable.name.split(
                        '/')[-1] and variable.name.split('/')[2] != 'Logits':
                #rgb_variable_map[variable.name.replace(':0', '')[len('RGB/inception_i3d/'):]] = variable
                rgb_variable_map[variable.name.replace(':0', '')] = variable
        rgb_saver = tf.train.Saver(var_list=rgb_variable_map, reshape=True)

        # Create a saver for writing training checkpoints.
        saver = tf.train.Saver()
        init = tf.global_variables_initializer()

        # Create a session for running Ops on the Graph.
        sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
        sess.run(init)
        # Create summary writter
        tf.summary.scalar('accuracy', accuracy)
        tf.summary.scalar('rgb_loss', rgb_loss)
        tf.summary.scalar('learning_rate', learning_rate)
        merged = tf.summary.merge_all()
    # load pre_train models
    ckpt = tf.train.get_checkpoint_state(rgb_pre_model_save_dir)
    if ckpt and ckpt.model_checkpoint_path:
        print("loading checkpoint %s,waiting......" %
              ckpt.model_checkpoint_path)
        rgb_saver.restore(sess, ckpt.model_checkpoint_path)
        print("load complete!")

    train_writer = tf.summary.FileWriter(
        './visual_logs/train_rgb_scratch_10000_6_64_0.0001_decay', sess.graph)
    test_writer = tf.summary.FileWriter(
        './visual_logs/test_rgb_scratch_10000_6_64_0.0001_decay', sess.graph)
    for step in xrange(FLAGS.max_steps):
        start_time = time.time()
        rgb_train_images, flow_train_images, train_labels, _, _, _ = input_data.read_clip_and_label(
            filename='../../list/ucf_list/train_flow.list',
            batch_size=FLAGS.batch_size * gpu_num,
            num_frames_per_clip=FLAGS.num_frame_per_clib,
            crop_size=FLAGS.crop_size,
            shuffle=True)
        sess.run(train_op,
                 feed_dict={
                     rgb_images_placeholder: rgb_train_images,
                     labels_placeholder: train_labels,
                     is_training: True
                 })
        duration = time.time() - start_time
        print('Step %d: %.3f sec' % (step, duration))

        # Save a checkpoint and evaluate the model periodically.
        if step % 10 == 0 or (step + 1) == FLAGS.max_steps:
            print('Training Data Eval:')
            summary, acc, loss_rgb = sess.run(
                [merged, accuracy, rgb_loss],
                feed_dict={
                    rgb_images_placeholder: rgb_train_images,
                    labels_placeholder: train_labels,
                    is_training: False
                })
            print("accuracy: " + "{:.5f}".format(acc))
            print("rgb_loss: " + "{:.5f}".format(loss_rgb))
            train_writer.add_summary(summary, step)
            print('Validation Data Eval:')
            rgb_val_images, flow_val_images, val_labels, _, _, _ = input_data.read_clip_and_label(
                filename='../../list/ucf_list/test_flow.list',
                batch_size=FLAGS.batch_size * gpu_num,
                num_frames_per_clip=FLAGS.num_frame_per_clib,
                crop_size=FLAGS.crop_size,
                shuffle=True)
            summary, acc = sess.run(
                [merged, accuracy],
                feed_dict={
                    rgb_images_placeholder: rgb_val_images,
                    labels_placeholder: val_labels,
                    is_training: False
                })
            print("accuracy: " + "{:.5f}".format(acc))
            test_writer.add_summary(summary, step)
        if (step + 1) % 3000 == 0 or (step + 1) == FLAGS.max_steps:
            saver.save(sess,
                       os.path.join(model_save_dir, 'i3d_ucf_model'),
                       global_step=step)
    print("done")
Beispiel #37
0
def train():
  """Train CIFAR-10 for a number of steps."""
  with tf.Graph().as_default():
    global_step = tf.Variable(0, trainable=False)

    # Get images and labels for CIFAR-10.
    images, labels = cifar10.distorted_inputs()

    # Build a Graph that computes the logits predictions from the
    # inference model.
    logits = cifar10.inference(images)
    print(labels)
    print(logits)
    # Calculate loss.
    loss = cifar10.loss(logits, labels)

    # Build a Graph that trains the model with one batch of examples and
    # updates the model parameters.
    train_op = cifar10.train(loss, global_step)

    # Create a saver.
    saver = tf.train.Saver(tf.all_variables())

    # Build the summary operation based on the TF collection of Summaries.
    summary_op = tf.contrib.deprecated.merge_all_summaries()

    # Build an initialization operation to run below.
    init = tf.initialize_all_variables()

    # Start running operations on the Graph.
    sess = tf.Session(config=tf.ConfigProto(
        log_device_placement=FLAGS.log_device_placement))
    sess.run(init)

    # Start the queue runners.
    tf.train.start_queue_runners(sess=sess)

   # summary_writer = tf.train.SummaryWriter(FLAGS.train_dir,
  #                                          graph_def=sess.graph_def)


    for step in xrange(FLAGS.max_steps):
      start_time = time.time()
      _, loss_value = sess.run([train_op, loss])
      duration = time.time() - start_time

      assert not np.isnan(loss_value), 'Model diverged with loss = NaN'

      if step % 10 == 0:
        num_examples_per_step = FLAGS.batch_size
        examples_per_sec = num_examples_per_step / duration
        sec_per_batch = float(duration)

        format_str = ('%s: step %d, loss = %.2f (%.1f examples/sec; %.3f '
                      'sec/batch)')
        print (format_str % (datetime.now(), step, loss_value,
                             examples_per_sec, sec_per_batch))
        printfile(loss_value)

      # Save the model checkpoint periodically.
      if step % 1000 == 0 or (step + 1) == FLAGS.max_steps:
        checkpoint_path = os.path.join(FLAGS.train_dir, 'model.ckpt')
        saver.save(sess, checkpoint_path, global_step=step)

      if step % 1000 == 0:
        cifar10_eval.evaluate()
Beispiel #38
0
def cleanup(days, project, concurrency, silent, model, router, timed):
    """Delete a portion of trailing data based on creation date.

    All data that is older than `--days` will be deleted.  The default for
    this is 30 days.  In the default setting all projects will be truncated
    but if you have a specific project you want to limit this to this can be
    done with the `--project` flag which accepts a project ID or a string
    with the form `org/project` where both are slugs.
    """
    if concurrency < 1:
        click.echo('Error: Minimum concurrency is 1', err=True)
        raise click.Abort()

    os.environ['_SENTRY_CLEANUP'] = '1'

    # Make sure we fork off multiprocessing pool
    # before we import or configure the app
    from multiprocessing import Process, JoinableQueue as Queue

    pool = []
    task_queue = Queue(1000)
    for _ in xrange(concurrency):
        p = Process(target=multiprocess_worker, args=(task_queue, ))
        p.daemon = True
        p.start()
        pool.append(p)

    from sentry.runner import configure
    configure()

    from django.db import router as db_router
    from sentry.app import nodestore
    from sentry.db.deletion import BulkDeleteQuery
    from sentry import models

    if timed:
        import time
        from sentry.utils import metrics
        start_time = time.time()

    # list of models which this query is restricted to
    model_list = {m.lower() for m in model}

    def is_filtered(model):
        if router is not None and db_router.db_for_write(model) != router:
            return True
        if not model_list:
            return False
        return model.__name__.lower() not in model_list

    # Deletions that use `BulkDeleteQuery` (and don't need to worry about child relations)
    # (model, datetime_field, order_by)
    BULK_QUERY_DELETES = [
        (models.EventMapping, 'date_added', '-date_added'),
        (models.EventAttachment, 'date_added', None),
        (models.UserReport, 'date_added', None),
        (models.GroupEmailThread, 'date', None),
        (models.GroupRuleStatus, 'date_added', None),
    ] + EXTRA_BULK_QUERY_DELETES

    # Deletions that use the `deletions` code path (which handles their child relations)
    # (model, datetime_field, order_by)
    DELETES = (
        (models.Event, 'datetime', 'datetime'),
        (models.Group, 'last_seen', 'last_seen'),
    )

    if not silent:
        click.echo('Removing expired values for LostPasswordHash')

    if is_filtered(models.LostPasswordHash):
        if not silent:
            click.echo('>> Skipping LostPasswordHash')
    else:
        models.LostPasswordHash.objects.filter(date_added__lte=timezone.now() -
                                               timedelta(hours=48)).delete()

    if is_filtered(models.OrganizationMember) and not silent:
        click.echo('>> Skipping OrganizationMember')
    else:
        click.echo('Removing expired values for OrganizationMember')
        expired_threshold = timezone.now() - timedelta(days=days)
        models.OrganizationMember.delete_expired(expired_threshold)

    for model in [models.ApiGrant, models.ApiToken]:
        if not silent:
            click.echo(u'Removing expired values for {}'.format(
                model.__name__))

        if is_filtered(model):
            if not silent:
                click.echo(u'>> Skipping {}'.format(model.__name__))
        else:
            model.objects.filter(expires_at__lt=(
                timezone.now() -
                timedelta(days=API_TOKEN_TTL_IN_DAYS)), ).delete()

    project_id = None
    if project:
        click.echo(
            "Bulk NodeStore deletion not available for project selection",
            err=True)
        project_id = get_project(project)
        if project_id is None:
            click.echo('Error: Project not found', err=True)
            raise click.Abort()
    else:
        if not silent:
            click.echo("Removing old NodeStore values")

        cutoff = timezone.now() - timedelta(days=days)
        try:
            nodestore.cleanup(cutoff)
        except NotImplementedError:
            click.echo("NodeStore backend does not support cleanup operation",
                       err=True)

    for bqd in BULK_QUERY_DELETES:
        if len(bqd) == 4:
            model, dtfield, order_by, chunk_size = bqd
        else:
            chunk_size = 10000
            model, dtfield, order_by = bqd

        if not silent:
            click.echo(
                u"Removing {model} for days={days} project={project}".format(
                    model=model.__name__,
                    days=days,
                    project=project or '*',
                ))
        if is_filtered(model):
            if not silent:
                click.echo('>> Skipping %s' % model.__name__)
        else:
            BulkDeleteQuery(
                model=model,
                dtfield=dtfield,
                days=days,
                project_id=project_id,
                order_by=order_by,
            ).execute(chunk_size=chunk_size)

    for model, dtfield, order_by in DELETES:
        if not silent:
            click.echo(
                u"Removing {model} for days={days} project={project}".format(
                    model=model.__name__,
                    days=days,
                    project=project or '*',
                ))

        if is_filtered(model):
            if not silent:
                click.echo('>> Skipping %s' % model.__name__)
        else:
            imp = '.'.join((model.__module__, model.__name__))

            q = BulkDeleteQuery(
                model=model,
                dtfield=dtfield,
                days=days,
                project_id=project_id,
                order_by=order_by,
            )

            for chunk in q.iterator(chunk_size=100):
                task_queue.put((imp, chunk))

            task_queue.join()

    # Clean up FileBlob instances which are no longer used and aren't super
    # recent (as there could be a race between blob creation and reference)
    if not silent:
        click.echo("Cleaning up unused FileBlob references")
    if is_filtered(models.FileBlob):
        if not silent:
            click.echo('>> Skipping FileBlob')
    else:
        cleanup_unused_files(silent)

    # Shut down our pool
    for _ in pool:
        task_queue.put(_STOP_WORKER)

    # And wait for it to drain
    for p in pool:
        p.join()

    if timed:
        duration = int(time.time() - start_time)
        metrics.timing('cleanup.duration', duration, instance=router)
        click.echo("Clean up took %s second(s)." % duration)
Beispiel #39
0
def train():
    """Train SqueezeDet model"""
    assert FLAGS.dataset == 'KITTI', \
        'Currently only support KITTI dataset'

    os.environ['CUDA_VISIBLE_DEVICES'] = FLAGS.gpu

    with tf.Graph().as_default():

        assert FLAGS.net == 'vgg16' or FLAGS.net == 'resnet50' \
            or FLAGS.net == 'squeezeDet' or FLAGS.net == 'squeezeDet+', \
            'Selected neural net architecture not supported: {}'.format(FLAGS.net)
        if FLAGS.net == 'vgg16':
            mc = kitti_vgg16_config()
            mc.IS_TRAINING = True
            mc.PRETRAINED_MODEL_PATH = FLAGS.pretrained_model_path
            model = VGG16ConvDet(mc)
        elif FLAGS.net == 'resnet50':
            mc = kitti_res50_config()
            mc.IS_TRAINING = True
            mc.PRETRAINED_MODEL_PATH = FLAGS.pretrained_model_path
            model = ResNet50ConvDet(mc)
        elif FLAGS.net == 'squeezeDet':
            mc = kitti_squeezeDet_config()
            mc.IS_TRAINING = True
            mc.PRETRAINED_MODEL_PATH = FLAGS.pretrained_model_path
            model = SqueezeDet(mc)
        elif FLAGS.net == 'squeezeDet+':
            mc = kitti_squeezeDetPlus_config()
            mc.IS_TRAINING = True
            mc.PRETRAINED_MODEL_PATH = FLAGS.pretrained_model_path
            model = SqueezeDetPlus(mc)

        imdb = kitti(FLAGS.image_set, FLAGS.data_path, mc)

        # save model size, flops, activations by layers
        with open(os.path.join(FLAGS.train_dir, 'model_metrics.txt'),
                  'w') as f:
            f.write('Number of parameter by layer:\n')
            count = 0
            for c in model.model_size_counter:
                f.write('\t{}: {}\n'.format(c[0], c[1]))
                count += c[1]
            f.write('\ttotal: {}\n'.format(count))

            count = 0
            f.write('\nActivation size by layer:\n')
            for c in model.activation_counter:
                f.write('\t{}: {}\n'.format(c[0], c[1]))
                count += c[1]
            f.write('\ttotal: {}\n'.format(count))

            count = 0
            f.write('\nNumber of flops by layer:\n')
            for c in model.flop_counter:
                f.write('\t{}: {}\n'.format(c[0], c[1]))
                count += c[1]
            f.write('\ttotal: {}\n'.format(count))
        f.close()
        print('Model statistics saved to {}.'.format(
            os.path.join(FLAGS.train_dir, 'model_metrics.txt')))

        def _load_data(load_to_placeholder=True):
            # read batch input
            image_per_batch, label_per_batch, box_delta_per_batch, aidx_per_batch, \
                bbox_per_batch = imdb.read_batch()

            label_indices, bbox_indices, box_delta_values, mask_indices, box_values, \
                = [], [], [], [], []
            aidx_set = set()
            num_discarded_labels = 0
            num_labels = 0
            for i in range(len(label_per_batch)):  # batch_size
                for j in range(len(
                        label_per_batch[i])):  # number of annotations
                    num_labels += 1
                    if (i, aidx_per_batch[i][j]) not in aidx_set:
                        aidx_set.add((i, aidx_per_batch[i][j]))
                        label_indices.append(
                            [i, aidx_per_batch[i][j], label_per_batch[i][j]])
                        mask_indices.append([i, aidx_per_batch[i][j]])
                        bbox_indices.extend([[i, aidx_per_batch[i][j], k]
                                             for k in range(4)])
                        box_delta_values.extend(box_delta_per_batch[i][j])
                        box_values.extend(bbox_per_batch[i][j])
                    else:
                        num_discarded_labels += 1

            if mc.DEBUG_MODE:
                print(
                    'Warning: Discarded {}/({}) labels that are assigned to the same '
                    'anchor'.format(num_discarded_labels, num_labels))

            if load_to_placeholder:
                image_input = model.ph_image_input
                input_mask = model.ph_input_mask
                box_delta_input = model.ph_box_delta_input
                box_input = model.ph_box_input
                labels = model.ph_labels
            else:
                image_input = model.image_input
                input_mask = model.input_mask
                box_delta_input = model.box_delta_input
                box_input = model.box_input
                labels = model.labels

            feed_dict = {
                image_input:
                image_per_batch,
                input_mask:
                np.reshape(
                    sparse_to_dense(mask_indices, [mc.BATCH_SIZE, mc.ANCHORS],
                                    [1.0] * len(mask_indices)),
                    [mc.BATCH_SIZE, mc.ANCHORS, 1]),
                box_delta_input:
                sparse_to_dense(bbox_indices, [mc.BATCH_SIZE, mc.ANCHORS, 4],
                                box_delta_values),
                box_input:
                sparse_to_dense(bbox_indices, [mc.BATCH_SIZE, mc.ANCHORS, 4],
                                box_values),
                labels:
                sparse_to_dense(label_indices,
                                [mc.BATCH_SIZE, mc.ANCHORS, mc.CLASSES],
                                [1.0] * len(label_indices)),
            }

            return feed_dict, image_per_batch, label_per_batch, bbox_per_batch

        def _enqueue(sess, coord):
            try:
                while not coord.should_stop():
                    feed_dict, _, _, _ = _load_data()
                    sess.run(model.enqueue_op, feed_dict=feed_dict)
                    if mc.DEBUG_MODE:
                        print("added to the queue")
                if mc.DEBUG_MODE:
                    print("Finished enqueue")
            except Exception:
                # coord.request_stop(e)
                pass

        sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))

        saver = tf.train.Saver(tf.global_variables())
        summary_op = tf.summary.merge_all()

        ckpt = tf.train.get_checkpoint_state(FLAGS.train_dir)
        if ckpt and ckpt.model_checkpoint_path:
            saver.restore(sess, ckpt.model_checkpoint_path)

        summary_writer = tf.summary.FileWriter(FLAGS.train_dir, sess.graph)

        init = tf.global_variables_initializer()
        sess.run(init)

        coord = tf.train.Coordinator()

        if mc.NUM_THREAD > 0:
            enq_threads = []
            for _ in range(mc.NUM_THREAD):
                enq_thread = threading.Thread(target=_enqueue,
                                              args=[sess, coord])
                # enq_thread.isDaemon()
                enq_thread.start()
                enq_threads.append(enq_thread)

        threads = tf.train.start_queue_runners(coord=coord, sess=sess)
        run_options = tf.RunOptions(timeout_in_ms=60000)

        # try:
        for step in xrange(FLAGS.max_steps):
            if coord.should_stop():
                sess.run(model.FIFOQueue.close(cancel_pending_enqueues=True))
                coord.request_stop()
                coord.join(threads)
                break

            start_time = time.time()

            if step % FLAGS.summary_step == 0:
                feed_dict, image_per_batch, label_per_batch, bbox_per_batch = \
                    _load_data(load_to_placeholder=False)
                op_list = [
                    model.train_op, model.loss, summary_op, model.det_boxes,
                    model.det_probs, model.det_class, model.conf_loss,
                    model.bbox_loss, model.class_loss
                ]
                _, loss_value, summary_str, det_boxes, det_probs, det_class, \
                    conf_loss, bbox_loss, class_loss = sess.run(
                        op_list, feed_dict=feed_dict)

                _viz_prediction_result(model, image_per_batch, bbox_per_batch,
                                       label_per_batch, det_boxes, det_class,
                                       det_probs)
                image_per_batch = bgr_to_rgb(image_per_batch)
                viz_summary = sess.run(
                    model.viz_op,
                    feed_dict={model.image_to_show: image_per_batch})

                summary_writer.add_summary(summary_str, step)
                summary_writer.add_summary(viz_summary, step)
                summary_writer.flush()

                print('conf_loss: {}, bbox_loss: {}, class_loss: {}'.format(
                    conf_loss, bbox_loss, class_loss))
            else:
                if mc.NUM_THREAD > 0:
                    _, loss_value, conf_loss, bbox_loss, class_loss = sess.run(
                        [
                            model.train_op, model.loss, model.conf_loss,
                            model.bbox_loss, model.class_loss
                        ],
                        options=run_options)
                else:
                    feed_dict, _, _, _ = _load_data(load_to_placeholder=False)
                    _, loss_value, conf_loss, bbox_loss, class_loss = sess.run(
                        [
                            model.train_op, model.loss, model.conf_loss,
                            model.bbox_loss, model.class_loss
                        ],
                        feed_dict=feed_dict)

            duration = time.time() - start_time

            assert not np.isnan(loss_value), \
                'Model diverged. Total loss: {}, conf_loss: {}, bbox_loss: {}, ' \
                'class_loss: {}'.format(loss_value, conf_loss, bbox_loss, class_loss)

            if step % 10 == 0:
                num_images_per_step = mc.BATCH_SIZE
                images_per_sec = num_images_per_step / duration
                sec_per_batch = float(duration)
                format_str = (
                    '%s: step %d, loss = %.2f (%.1f images/sec; %.3f '
                    'sec/batch)')
                print(format_str % (datetime.now(), step, loss_value,
                                    images_per_sec, sec_per_batch))
                sys.stdout.flush()

            # Save the model checkpoint periodically.
            if step % FLAGS.checkpoint_step == 0 or (step +
                                                     1) == FLAGS.max_steps:
                checkpoint_path = os.path.join(FLAGS.train_dir, 'model.ckpt')
                saver.save(sess, checkpoint_path, global_step=step)
Beispiel #40
0
 def __del__(self):
     for i in xrange(len(self.files)):
         self.close(i, delete=True)
Beispiel #41
0
def read_annot(filepath, orig_ids=False):
    """Read in a Freesurfer annotation from a .annot file.

    Parameters
    ----------
    filepath : str
        Path to annotation file.
    orig_ids : bool
        Whether to return the vertex ids as stored in the annotation
        file or the positional colortable ids. With orig_ids=False
        vertices with no id have an id set to -1.

    Returns
    -------
    labels : ndarray, shape (n_vertices,)
        Annotation id at each vertex. If a vertex does not belong
        to any label and orig_ids=False, its id will be set to -1.
    ctab : ndarray, shape (n_labels, 5)
        RGBA + label id colortable array.
    names : list of str
        The names of the labels. The length of the list is n_labels.
    """
    with open(filepath, "rb") as fobj:
        dt = ">i4"
        vnum = np.fromfile(fobj, dt, 1)[0]
        data = np.fromfile(fobj, dt, vnum * 2).reshape(vnum, 2)
        labels = data[:, 1]

        ctab_exists = np.fromfile(fobj, dt, 1)[0]
        if not ctab_exists:
            raise Exception('Color table not found in annotation file')
        n_entries = np.fromfile(fobj, dt, 1)[0]
        if n_entries > 0:
            length = np.fromfile(fobj, dt, 1)[0]
            orig_tab = np.fromfile(fobj, '>c', length)
            orig_tab = orig_tab[:-1]

            names = list()
            ctab = np.zeros((n_entries, 5), np.int)
            for i in xrange(n_entries):
                name_length = np.fromfile(fobj, dt, 1)[0]
                name = np.fromfile(fobj, "|S%d" % name_length, 1)[0]
                names.append(name)
                ctab[i, :4] = np.fromfile(fobj, dt, 4)
                ctab[i, 4] = (ctab[i, 0] + ctab[i, 1] * (2 ** 8) +
                              ctab[i, 2] * (2 ** 16) +
                              ctab[i, 3] * (2 ** 24))
        else:
            ctab_version = -n_entries
            if ctab_version != 2:
                raise Exception('Color table version not supported')
            n_entries = np.fromfile(fobj, dt, 1)[0]
            ctab = np.zeros((n_entries, 5), np.int)
            length = np.fromfile(fobj, dt, 1)[0]
            np.fromfile(fobj, "|S%d" % length, 1)[0]  # Orig table path
            entries_to_read = np.fromfile(fobj, dt, 1)[0]
            names = list()
            for i in xrange(entries_to_read):
                np.fromfile(fobj, dt, 1)[0]  # Structure
                name_length = np.fromfile(fobj, dt, 1)[0]
                name = np.fromfile(fobj, "|S%d" % name_length, 1)[0]
                names.append(name)
                ctab[i, :4] = np.fromfile(fobj, dt, 4)
                ctab[i, 4] = (ctab[i, 0] + ctab[i, 1] * (2 ** 8) +
                              ctab[i, 2] * (2 ** 16))
        ctab[:, 3] = 255

    labels = labels.astype(np.int)

    if not orig_ids:
        ord = np.argsort(ctab[:, -1])
        mask = labels != 0
        labels[~mask] = -1
        labels[mask] = ord[np.searchsorted(ctab[ord, -1], labels[mask])]
    return labels, ctab, names
def timer(label, iters=30000):
  start = time.time()
  yield xrange(iters)
  end = time.time()
  t = (end - start) * 1e6 / iters
  print("%-40s took %.2fus (%d iterations)" % (label, t, iters))
Beispiel #43
0
    def forecast(self, y, horizon=0, u=None):
        """
        Calculate an one-step-ahead forecast.

        :param y: output time series
        :param horizon: number of predictions after y[T_max]
        :param u: external input time series
        :return: predicted time series as array
        """
        p = self.A.shape[1]
        a, b = self.A.shape[0], self.B.shape[0]
        c, m = self.C.shape[0], self.C.shape[2]
        u = u if u is not None else np.zeros((c, m))
        y = utils.atleast_2d(y)

        sampleT = y.shape[0]
        predictT = sampleT + horizon

        # diagonalize with respect to matrix of B's leading coefficients
        B0inv = linalg.inv(self.B[0, :, :])
        A = np.tensordot(self.A, B0inv, axes=1)
        B = np.tensordot(self.B, B0inv, axes=1)
        if c != 0:
            C = np.einsum('ijk,kl', self.C, B0inv)
        else:
            C = np.zeros((c, p, m))

        # calculate directly the residual ...
        res = -np.dot(self._prep_trend(sampleT, p)[:sampleT, ...], B0inv)
        # and perform prediction
        for t in xrange(sampleT):
            la, lb, lc = min(a - 1, t), min(b - 1, t), min(c - 1, t)
            ba, bb, bc = max(0, t - la), max(0, t - lb), max(0, t - lc)
            res[t, :] += np.einsum('ikj,ij', A[la::-1, ...], y[ba:t + 1, :])
            if b != 0:
                res[t, :] -= np.einsum('ikj,ij', B[lb:0:-1, ...], res[bb:t, :])
            if c != 0:
                res[t, :] -= np.einsum('ikj,ij', C[lc::-1, ...],
                                       u[bc:t + 1, :])

        pred = np.zeros((predictT, p))
        pred[:sampleT, :] = y[:sampleT, :] - np.dot(res, B[0, :, :])

        if predictT > sampleT:
            A0inv = linalg.inv(self.A[0, :, :])
            A = np.tensordot(self.A, A0inv, axes=1)
            B = np.tensordot(self.B, A0inv, axes=1)
            if c != 0:
                C = np.einsum('ijk,kl', self.C, A0inv)
            else:
                C = np.zeros((c, p, m))
            pred[sampleT:, :] = np.dot(self._prep_trend(horizon, p, sampleT),
                                       A0inv)
            # perform prediction for horizon period
            for t in xrange(sampleT, predictT):
                for l in xrange(1, a):
                    if t - l < sampleT:
                        pred[t, :] -= np.dot(A[l, :, :], y[t - l, :])
                    else:
                        pred[t, :] -= np.dot(A[l, :, :], pred[t - l, :])

                for l in xrange(b):
                    if t - l < sampleT:
                        pred[t, :] += np.dot(B[l, :, :], res[t - l, :])

                for l in xrange(c):
                    pred[t, :] += np.dot(C[l, :, :], u[t - l, :])

        return pred
Beispiel #44
0
def collect_general_canonical_repn(exp, idMap, compute_values):
#     global temp_const
#     global temp_var
#     global temp_nonl
    temp_const = { 0: {None:0.0} }
    temp_var = { 1: {GeneralCanonicalRepn({None:1}):1.0} }
    temp_nonl = { None: None }
    exp_type = type(exp)

    #
    # Constant
    #
    if exp.is_fixed():
        if compute_values:
            temp_const[0][None] = value(exp)
        else:
            temp_const[0][None] = exp
        return temp_const
    #
    # Expression
    #
    elif exp.is_expression():

        #
        # Sum
        #
        if exp_type is expr._SumExpression:
            if exp._const != 0.0:
                repn = { 0: {None:exp._const} }
            else:
                repn = {}
            for i in xrange(len(exp._args)):
                repn = repn_add(
                    repn,
                    collect_general_canonical_repn(exp._args[i],
                                                   idMap,
                                                   compute_values),
                    coef=exp._coef[i])
            return repn
        #
        # Product
        #
        elif exp_type is expr._ProductExpression:
            #
            # Iterate through the denominator.  If they aren't all
            # constants, then simply return this expression.
            #
            denom=1.0
            for e in exp._denominator:
                if e.is_fixed():
                    denom *= e()
                else:
                    temp_nonl[None] = exp
                    return temp_nonl
                if denom == 0.0:
                    print("Divide-by-zero error - offending sub-expression:")
                    e.pprint()
                    raise ZeroDivisionError
            #
            # OK, the denominator is a constant.
            #
            repn = { 0: {None:exp._coef / denom} }
            for e in exp._numerator:
                repn = repn_mult(
                    repn,
                    collect_general_canonical_repn(e,
                                                   idMap,
                                                   compute_values))
            return repn
        #
        # Power Expression
        #
        elif exp_type is expr._PowExpression:
            if exp.polynomial_degree() is None:
                raise TypeError("Unsupported general power expression: "
                                +str(exp._args))
                
            # If this is of the form EXPR**1, we can just get the
            # representation of EXPR
            if exp._args[1] == 1:
                return collect_general_canonical_repn(exp._args[0],
                                                      idMap,
                                                      compute_values)
            # The only other way to get a polynomial expression is if
            # exp=EXPR**p where p is fixed a nonnegative integer.  We
            # can expand this expression and generate a canonical
            # representation from there.  If p=0, this expression is
            # constant (and is processed by the is_fixed code above
            # NOTE: There is no check for 0**0
            return collect_general_canonical_repn(
                        reduce( lambda x,y: x*y, [exp._args[0]]*int(value(exp._args[1])), 1.0 ),
                        idMap,
                        compute_values)
        elif exp_type is expr.Expr_if:
            if exp._if.is_fixed():
                if exp._if():
                    return collect_general_canonical_repn(exp._then,
                                                          idMap,
                                                          compute_values)
                else:
                    return collect_general_canonical_repn(exp._else,
                                                          idMap,
                                                          compute_values)
            else:
                temp_nonl[None] = exp
                return temp_nonl
        #
        # Expression (the component)
        # (faster check)
        elif isinstance(exp, _ExpressionData):
            return collect_general_canonical_repn(exp.expr,
                                                  idMap,
                                                  compute_values)
        #
        # ERROR
        #
        else:
            raise ValueError("Unsupported expression type: "+str(exp))
    #
    # Variable
    #
    elif (exp.__class__ is _GeneralVarData) or isinstance(exp, _VarData):
        id_ = id(exp)
        if id_ in idMap[None]:
            key = idMap[None][id_]
        else:
            key = len(idMap) - 1
            idMap[None][id_] = key
            idMap[key] = exp
        temp_var = { -1: {key:exp}, 1: {GeneralCanonicalRepn({key:1}):1.0} }
        return temp_var
    #
    # Connector
    #
    elif exp_type is _ConnectorValue or exp.type() is Connector:
        # Silently omit constraint...  The ConnectorExpander should
        # expand this constraint into indvidual constraints that
        # reference "real" variables.
        return {}
    #
    # ERROR
    #
    else:
        raise ValueError("Unexpected expression (type %s): " %
                         ( type(exp).__name__, str(exp) ))
Beispiel #45
0
    def from_keypoint_image(image, if_not_found_coords={"x": -1, "y": -1},
                            threshold=1, nb_channels=None):
        """Convert ``to_keypoint_image()`` outputs to ``KeypointsOnImage``.

        This is the inverse of :func:`KeypointsOnImage.to_keypoint_image`.

        Parameters
        ----------
        image : (H,W,N) ndarray
            The keypoints image. N is the number of keypoints.

        if_not_found_coords : tuple or list or dict or None, optional
            Coordinates to use for keypoints that cannot be found in `image`.

            * If this is a ``list``/``tuple``, it must contain two ``int``
              values.
            * If it is a ``dict``, it must contain the keys ``x`` and
              ``y`` with each containing one ``int`` value.
            * If this is ``None``, then the keypoint will not be added to the
              final :class:`KeypointsOnImage` object.

        threshold : int, optional
            The search for keypoints works by searching for the argmax in
            each channel. This parameters contains the minimum value that
            the max must have in order to be viewed as a keypoint.

        nb_channels : None or int, optional
            Number of channels of the image on which the keypoints are placed.
            Some keypoint augmenters require that information.
            If set to ``None``, the keypoint's shape will be set
            to ``(height, width)``, otherwise ``(height, width, nb_channels)``.

        Returns
        -------
        imgaug.augmentables.kps.KeypointsOnImage
            The extracted keypoints.

        """
        # pylint: disable=dangerous-default-value
        assert image.ndim == 3, (
            "Expected 'image' to have three dimensions, "
            "got %d with shape %s instead." % (image.ndim, image.shape))
        height, width, nb_keypoints = image.shape

        drop_if_not_found = False
        if if_not_found_coords is None:
            drop_if_not_found = True
            if_not_found_x = -1
            if_not_found_y = -1
        elif isinstance(if_not_found_coords, (tuple, list)):
            assert len(if_not_found_coords) == 2, (
                "Expected tuple 'if_not_found_coords' to contain exactly two "
                "values, got %d values." % (len(if_not_found_coords),))
            if_not_found_x = if_not_found_coords[0]
            if_not_found_y = if_not_found_coords[1]
        elif isinstance(if_not_found_coords, dict):
            if_not_found_x = if_not_found_coords["x"]
            if_not_found_y = if_not_found_coords["y"]
        else:
            raise Exception(
                "Expected if_not_found_coords to be None or tuple or list "
                "or dict, got %s." % (type(if_not_found_coords),))

        keypoints = []
        for i in sm.xrange(nb_keypoints):
            maxidx_flat = np.argmax(image[..., i])
            maxidx_ndim = np.unravel_index(maxidx_flat, (height, width))

            found = (image[maxidx_ndim[0], maxidx_ndim[1], i] >= threshold)
            if found:
                x = maxidx_ndim[1] + 0.5
                y = maxidx_ndim[0] + 0.5
                keypoints.append(Keypoint(x=x, y=y))
            else:
                if drop_if_not_found:
                    # dont add the keypoint to the result list, i.e. drop it
                    pass
                else:
                    keypoints.append(Keypoint(x=if_not_found_x,
                                              y=if_not_found_y))

        out_shape = (height, width)
        if nb_channels is not None:
            out_shape += (nb_channels,)
        return KeypointsOnImage(keypoints, shape=out_shape)
Beispiel #46
0
                           normalized_embeddings,
                           transpose_b=True)

    # Add variable initializer.
    init = tf.global_variables_initializer()

# Step 5: Begin training.
num_steps = 20001

with tf.Session(graph=graph) as session:
    # We must initialize all variables before we use them.
    init.run()
    print('Initialized')

    average_loss = 0
    for step in xrange(num_steps):
        batch_inputs, batch_labels = generate_batch(batch_size, num_skips,
                                                    skip_window)
        feed_dict = {train_inputs: batch_inputs, train_labels: batch_labels}

        # We perform one update step by evaluating the optimizer op (including it
        # in the list of returned values for session.run()
        _, loss_val = session.run([optimizer, loss], feed_dict=feed_dict)
        average_loss += loss_val

        if step % 2000 == 0:
            if step > 0:
                average_loss /= 2000
            # The average loss is an estimate of the loss over the last 2000 batches.
            print('Average loss at step ', step, ': ', average_loss)
            average_loss = 0
Beispiel #47
0
def train(args):
  """Train different architectures for a number of epochs."""

  with tf.Graph().as_default(), tf.device('/cpu:0'):

    # Read data from disk
    images, labels = data_loader.read_inputs(True, args)
    
    #epoch number
    epoch_number = tf.get_variable('epoch_number', [], dtype= tf.int32, initializer= tf.constant_initializer(0), trainable= False)

    # Decay the learning rate
    lr = tf.train.piecewise_constant(epoch_number, args.LR_steps, args.LR_values, name= 'LearningRate')

    # Weight Decay policy
    wd = tf.train.piecewise_constant(epoch_number, args.WD_steps, args.WD_values, name= 'WeightDecay')
   
    # transfer mode 1 means using the model as a feature extractor, so we don't need batch norm updates and dropout 
    is_training= not args.transfer_mode[0]==1

    # Create an optimizer that performs gradient descent.
    opt = tf.train.MomentumOptimizer(lr, 0.9)

    # Calculate the gradients for each model tower.
    tower_grads = []
    # it is only needed for transfer mode 3 
    tower_auxgrads = []
    with tf.variable_scope(tf.get_variable_scope()):
      for i in xrange(args.num_gpus):
        with tf.device('/gpu:%d' % i):
          with tf.name_scope('Tower_%d' % i) as scope:
            # Calculate the loss for one tower. This function
            # constructs the entire model but shares the variables across
            # all towers.
            logits = arch.get_model(images, wd, is_training, args)
            
            # Top-1 accuracy
            top1acc = tf.reduce_mean(
                tf.cast(tf.nn.in_top_k(logits, labels, 1), tf.float32))
            # Top-n accuracy
            # topnacc = tf.reduce_mean(
            #     tf.cast(tf.nn.in_top_k(logits, labels, args.top_n), tf.float32))

            # Build the portion of the Graph calculating the losses. Note that we will
            # assemble the total_loss using a custom function below.
            cross_entropy_mean = loss(logits, labels)

            # Get all the regularization lesses and add them
            regularization_losses = tf.get_collection(
                tf.GraphKeys.REGULARIZATION_LOSSES)
            
            reg_loss = tf.add_n(regularization_losses)

            #Add a tensorboard summary
            tf.summary.scalar('Regularization Loss', reg_loss)

            # Compute the total loss (cross entropy loss + regularization loss)
            total_loss = tf.add(cross_entropy_mean, reg_loss)

            # Attach a scalar summary for the total loss and top-1 and top-5 accuracies
            tf.summary.scalar('Total Loss', total_loss)
            tf.summary.scalar('Top-1 Accuracy', top1acc)
            # tf.summary.scalar('Top-'+str(args.top_n)+' Accuracy', topnacc)

            # Reuse variables for the next tower.
            tf.get_variable_scope().reuse_variables()

            # Retain the summaries from the final tower.
            summaries = tf.get_collection(tf.GraphKeys.SUMMARIES, scope)

            # Gather batch normaliziation update operations
            batchnorm_updates = tf.get_collection(tf.GraphKeys.UPDATE_OPS, scope)

            # Calculate the gradients for the batch of data on this tower.
            if args.transfer_mode[0]== 3:
              # compute gradients for the last layer only
              grads = opt.compute_gradients(total_loss, var_list= tf.get_collection(tf.GraphKeys.VARIABLES, scope='output'))
              # compute gradients for all layers
              auxgrads = opt.compute_gradients(total_loss)
              tower_auxgrads.append(auxgrads)
            elif args.transfer_mode[0]==1:
              grads = opt.compute_gradients(total_loss,var_list= tf.get_collection(tf.GraphKeys.VARIABLES, scope='output'))
            else:
              grads = opt.compute_gradients(total_loss)

            # Keep track of the gradients across all towers.
            tower_grads.append(grads)

    # We must calculate the mean of each gradient. Note that this is the
    # synchronization point across all towers.
    grads = average_gradients(tower_grads)
    # average all gradients for transfer mode 3
    if args.transfer_mode[0]== 3:
      auxgrads = average_gradients(tower_auxgrads)

    # Add a summary to track the learning rate and weight decay
    summaries.append(tf.summary.scalar('learning_rate', lr))
    summaries.append(tf.summary.scalar('weight_decay', wd))

    # Setup the train operation
    if args.transfer_mode[0]==3:
      train_op = tf.cond(tf.less(epoch_number,args.transfer_mode[1]),
              lambda: tf.group(opt.apply_gradients(grads),*batchnorm_updates), lambda: tf.group(opt.apply_gradients(auxgrads),*batchnorm_updates))
    elif args.transfer_mode[0]==1:
      train_op = opt.apply_gradients(grads)
    else:
      batchnorm_updates_op = tf.group(*batchnorm_updates)
      train_op = tf.group(opt.apply_gradients(grads), batchnorm_updates_op)

    # a loader for loading the pretrained model (it does not load the last layer) 
    if args.retrain_from is not None:
      if args.transfer_mode[0]==0:
        pretrained_loader = tf.train.Saver()
      else:
        pretrained_loader = tf.train.Saver(var_list= exclude())

    # Create a saver.
    saver = tf.train.Saver(tf.global_variables(), max_to_keep= args.max_to_keep)

    # Build the summary operation from the last tower summaries.
    summary_op = tf.summary.merge_all()

    # Build an initialization operation to run below.
    init = tf.global_variables_initializer()

    # Logging the runtime information if requested
    if args.log_debug_info:
      run_options = tf.RunOptions(trace_level= tf.RunOptions.FULL_TRACE)
      run_metadata = tf.RunMetadata()
    else:
      run_options = None
      run_metadata = None

    # Creating a session to run the built graph
    tf_config = tf.ConfigProto()
    tf_config.gpu_options.per_process_gpu_memory_fraction = 0.5
    tf_config.log_device_placement = True
    tf_config.allow_soft_placement = True

    sess = tf.Session(config=tf_config)
    sess.run(init)

    # Continue training from a saved snapshot, load a pre-trained model
    if args.retrain_from is not None:
      ckpt = tf.train.get_checkpoint_state(args.retrain_from)
      if ckpt and ckpt.model_checkpoint_path:
        # Restores from checkpoint
        pretrained_loader.restore(sess, ckpt.model_checkpoint_path)
      else:
        return

    # Start the queue runners.
    tf.train.start_queue_runners(sess= sess)
    
    # Setup a summary writer
    summary_writer = tf.summary.FileWriter(args.log_dir, sess.graph)

    # Set the start epoch number
    start_epoch = sess.run(epoch_number + 1)

    # The main training loop
    for epoch in xrange(start_epoch, start_epoch + args.num_epochs):
      # update epoch_number
      sess.run(epoch_number.assign(epoch))

      # Trainig batches
      for step in xrange(args.num_batches):
    
        start_time = time.time()
        _, loss_value, top1_accuracy = sess.run(
            [train_op, cross_entropy_mean, top1acc], options= run_options, run_metadata= run_metadata)
        duration = time.time() - start_time

        # Check for errors
        assert not np.isnan(loss_value), 'Model diverged with loss = NaN'

        # Logging and writing tensorboard summaries
        if step % 10 == 0:
          num_examples_per_step = args.chunked_batch_size * args.num_gpus
          examples_per_sec = num_examples_per_step / duration
          sec_per_batch = duration / args.num_gpus

          format_str = ('%s: epoch %d, step %d, loss = %.2f, Top-1 = %.2f (%.1f examples/sec; %.3f '
                        'sec/batch)')
          print (format_str % (datetime.now(), epoch, step, loss_value, top1_accuracy,
                               examples_per_sec, sec_per_batch))
          sys.stdout.flush()
        if step % 2000 == 0:
          summary_str = sess.run(summary_op)
          summary_writer.add_summary(
              summary_str, args.num_batches * epoch + step)
          if args.log_debug_info:
            summary_writer.add_run_metadata(
                run_metadata, 'epoch%d step%d' % (epoch, step))

      # Save the model checkpoint periodically after each training epoch
      checkpoint_path = os.path.join(args.log_dir, args.snapshot_prefix)
      print("checkpoint_path-----------",checkpoint_path)
      saver.save(sess, checkpoint_path, global_step= epoch)
Beispiel #48
0
    def from_distance_maps(distance_maps, inverted=False,
                           if_not_found_coords={"x": -1, "y": -1},
                           threshold=None, nb_channels=None):
        """Convert outputs of ``to_distance_maps()`` to ``KeypointsOnImage``.

        This is the inverse of :func:`KeypointsOnImage.to_distance_maps`.

        Parameters
        ----------
        distance_maps : (H,W,N) ndarray
            The distance maps. ``N`` is the number of keypoints.

        inverted : bool, optional
            Whether the given distance maps were generated in inverted mode
            (i.e. :func:`KeypointsOnImage.to_distance_maps` was called with
            ``inverted=True``) or in non-inverted mode.

        if_not_found_coords : tuple or list or dict or None, optional
            Coordinates to use for keypoints that cannot be found
            in `distance_maps`.

            * If this is a ``list``/``tuple``, it must contain two ``int``
              values.
            * If it is a ``dict``, it must contain the keys ``x`` and
              ``y`` with each containing one ``int`` value.
            * If this is ``None``, then the keypoint will not be added to the
              final :class:`KeypointsOnImage` object.

        threshold : float, optional
            The search for keypoints works by searching for the
            argmin (non-inverted) or argmax (inverted) in each channel. This
            parameters contains the maximum (non-inverted) or
            minimum (inverted) value to accept in order to view a hit as a
            keypoint. Use ``None`` to use no min/max.

        nb_channels : None or int, optional
            Number of channels of the image on which the keypoints are placed.
            Some keypoint augmenters require that information.
            If set to ``None``, the keypoint's shape will be set
            to ``(height, width)``, otherwise ``(height, width, nb_channels)``.

        Returns
        -------
        imgaug.augmentables.kps.KeypointsOnImage
            The extracted keypoints.

        """
        # pylint: disable=dangerous-default-value
        assert distance_maps.ndim == 3, (
            "Expected three-dimensional input, got %d dimensions and "
            "shape %s." % (distance_maps.ndim, distance_maps.shape))
        height, width, nb_keypoints = distance_maps.shape

        drop_if_not_found = False
        if if_not_found_coords is None:
            drop_if_not_found = True
            if_not_found_x = -1
            if_not_found_y = -1
        elif isinstance(if_not_found_coords, (tuple, list)):
            assert len(if_not_found_coords) == 2, (
                "Expected tuple/list 'if_not_found_coords' to contain "
                "exactly two entries, got %d." % (len(if_not_found_coords),))
            if_not_found_x = if_not_found_coords[0]
            if_not_found_y = if_not_found_coords[1]
        elif isinstance(if_not_found_coords, dict):
            if_not_found_x = if_not_found_coords["x"]
            if_not_found_y = if_not_found_coords["y"]
        else:
            raise Exception(
                "Expected if_not_found_coords to be None or tuple or list or "
                "dict, got %s." % (type(if_not_found_coords),))

        keypoints = []
        for i in sm.xrange(nb_keypoints):
            # TODO introduce voting here among all distance values that have
            #      min/max values
            if inverted:
                hitidx_flat = np.argmax(distance_maps[..., i])
            else:
                hitidx_flat = np.argmin(distance_maps[..., i])
            hitidx_ndim = np.unravel_index(hitidx_flat, (height, width))
            if not inverted and threshold is not None:
                found = (distance_maps[hitidx_ndim[0], hitidx_ndim[1], i]
                         < threshold)
            elif inverted and threshold is not None:
                found = (distance_maps[hitidx_ndim[0], hitidx_ndim[1], i]
                         >= threshold)
            else:
                found = True
            if found:
                keypoints.append(Keypoint(x=hitidx_ndim[1], y=hitidx_ndim[0]))
            else:
                if drop_if_not_found:
                    # dont add the keypoint to the result list, i.e. drop it
                    pass
                else:
                    keypoints.append(Keypoint(x=if_not_found_x,
                                              y=if_not_found_y))

        out_shape = (height, width)
        if nb_channels is not None:
            out_shape += (nb_channels,)
        return KeypointsOnImage(keypoints, shape=out_shape)
 def ListsAreClose(self, result, expected, rtol, atol):
   """Tests closeness of two lists of floats."""
   self.assertEqual(len(result), len(expected))
   for i in xrange(len(result)):
     self.assertAllClose(result[i], expected[i], rtol, atol)
Beispiel #50
0
def main(_):
    # We want to see all the logging messages for this tutorial.
    tf.logging.set_verbosity(tf.logging.INFO)

    # Start a new TensorFlow session.
    sess = tf.InteractiveSession()

    # Begin by making sure we have the training data we need.
    model_settings = models.prepare_model_settings(
        len(input_data.prepare_words_list(FLAGS.wanted_words.split(','))),
        FLAGS.sample_rate, FLAGS.clip_duration_ms, FLAGS.window_size_ms,
        FLAGS.window_stride_ms, FLAGS.dct_coefficient_count)
    audio_processor = input_data.AudioProcessor(
        FLAGS.data_dir, FLAGS.silence_percentage, FLAGS.unknown_percentage,
        FLAGS.wanted_words.split(','), FLAGS.validation_percentage,
        FLAGS.testing_percentage, model_settings)
    fingerprint_size = model_settings['fingerprint_size']
    label_count = model_settings['label_count']
    time_shift_samples = int((FLAGS.time_shift_ms * FLAGS.sample_rate) / 1000)

    # Figure out the learning rates for each training phase.
    # Since it's often effective to have high learning rates at the start of training,
    # followed by lower levels towards the end, the number of steps and learning rates can be
    # specified as comma-separated lists to define the rate at each stage.
    # For example --how_many_training_steps=10000,3000 --learning_rate=0.001,0.0001
    # will run 13,000 training loops in total, with a rate of 0.001 for the first
    # 10,000, and 0.0001 for the final 3,000.
    training_steps_list = list(
        map(int, FLAGS.how_many_training_steps.split(',')))
    learning_rates_list = list(map(float, FLAGS.learning_rate.split(',')))

    if len(training_steps_list) != len(learning_rates_list):
        raise Exception(
            '--how_many_training_steps and --learning_rate must be equal length lists, '
            'but are %d and %d long instead' %
            (len(training_steps_list), len(learning_rates_list)))

    fingerprint_input = tf.placeholder(tf.float32, [None, fingerprint_size],
                                       name='fingerprint_input')

    logits, dropout_prob = models.create_model(fingerprint_input,
                                               model_settings,
                                               FLAGS.model_architecture,
                                               is_training=True)

    # Define loss and optimizer
    ground_truth_input = tf.placeholder(tf.int64, [None],
                                        name='groundtruth_input')

    # Optionally we can add runtime checks to spot when NaNs or other symptoms of
    # numerical errors start occurring during training.
    control_dependencies = []
    if FLAGS.check_nans:
        checks = tf.add_check_numerics_ops()
        control_dependencies = [checks]

    # Create the back propagation and training evaluation machinery in the graph.
    with tf.name_scope('cross_entropy'):
        cross_entropy_mean = tf.losses.sparse_softmax_cross_entropy(
            labels=ground_truth_input, logits=logits)
    tf.summary.scalar('cross_entropy', cross_entropy_mean)
    with tf.name_scope('train'), tf.control_dependencies(control_dependencies):
        learning_rate_input = tf.placeholder(tf.float32, [],
                                             name='learning_rate_input')
        train_step = tf.train.GradientDescentOptimizer(
            learning_rate_input).minimize(cross_entropy_mean)
    predicted_indices = tf.argmax(logits, 1)
    correct_prediction = tf.equal(predicted_indices, ground_truth_input)
    confusion_matrix = tf.confusion_matrix(ground_truth_input,
                                           predicted_indices,
                                           num_classes=label_count)
    evaluation_step = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
    tf.summary.scalar('accuracy', evaluation_step)

    global_step = tf.train.get_or_create_global_step()
    increment_global_step = tf.assign(global_step, global_step + 1)

    saver = tf.train.Saver(tf.global_variables())

    # Merge all the summaries and write them out to /tmp/retrain_logs (by default)
    merged_summaries = tf.summary.merge_all()
    train_writer = tf.summary.FileWriter(FLAGS.summaries_dir + '/train',
                                         sess.graph)
    validation_writer = tf.summary.FileWriter(FLAGS.summaries_dir +
                                              '/validation')

    tf.global_variables_initializer().run()

    start_step = 1

    if FLAGS.start_checkpoint:
        models.load_variables_from_checkpoint(sess, FLAGS.start_checkpoint)
        start_step = global_step.eval(session=sess)

    tf.logging.info('Training from step: %d ', start_step)

    # Save graph.pbtxt.
    tf.train.write_graph(sess.graph_def, FLAGS.train_dir,
                         FLAGS.model_architecture + '.pbtxt')

    # Save list of words.
    with gfile.GFile(
            os.path.join(FLAGS.train_dir,
                         FLAGS.model_architecture + '_labels.txt'), 'w') as f:
        f.write('\n'.join(audio_processor.words_list))

    # Training loop.
    training_steps_max = np.sum(training_steps_list)
    for training_step in xrange(start_step, training_steps_max + 1):
        # Figure out what the current learning rate is.
        training_steps_sum = 0
        for i in range(len(training_steps_list)):
            training_steps_sum += training_steps_list[i]
            if training_step <= training_steps_sum:
                learning_rate_value = learning_rates_list[i]
                break

        # Pull the audio samples we'll use for training.
        train_fingerprints, train_ground_truth = audio_processor.get_data(
            FLAGS.batch_size, 0, model_settings, FLAGS.background_frequency,
            FLAGS.background_volume, time_shift_samples, 'training', sess)

        # Run the graph with this batch of training data.
        train_summary, train_accuracy, cross_entropy_value, _, _ = sess.run(
            [
                merged_summaries, evaluation_step, cross_entropy_mean,
                train_step, increment_global_step
            ],
            feed_dict={
                fingerprint_input: train_fingerprints,
                ground_truth_input: train_ground_truth,
                learning_rate_input: learning_rate_value,
                dropout_prob: 0.5
            })
        train_writer.add_summary(train_summary, training_step)
        tf.logging.info(
            'Step #%d: rate %f, accuracy %.1f%%, cross entropy %f' %
            (training_step, learning_rate_value, train_accuracy * 100,
             cross_entropy_value))

        # Output the confusion matrix and validation accuracy periodically
        is_last_step = (training_step == training_steps_max)
        if (training_step % FLAGS.eval_step_interval) == 0 or is_last_step:
            set_size = audio_processor.set_size('validation')
            total_accuracy = 0
            total_conf_matrix = None
            for i in xrange(0, set_size, FLAGS.batch_size):
                validation_fingerprints, validation_ground_truth = audio_processor.get_data(
                    FLAGS.batch_size, i, model_settings, 0.0, 0.0, 0,
                    'validation', sess)

                # Run a validation step and capture training summaries for TensorBoard with the `merged` op.
                validation_summary, validation_accuracy, conf_matrix = sess.run(
                    [merged_summaries, evaluation_step, confusion_matrix],
                    feed_dict={
                        fingerprint_input: validation_fingerprints,
                        ground_truth_input: validation_ground_truth,
                        dropout_prob: 1.0
                    })
                validation_writer.add_summary(validation_summary,
                                              training_step)
                batch_size = min(FLAGS.batch_size, set_size - i)
                total_accuracy += (validation_accuracy * batch_size) / set_size
                if total_conf_matrix is None:
                    total_conf_matrix = conf_matrix
                else:
                    total_conf_matrix += conf_matrix
            tf.logging.info('Confusion Matrix:\n %s' % total_conf_matrix)
            tf.logging.info('Step %d: Validation accuracy = %.1f%% (N=%d)' %
                            (training_step, total_accuracy * 100, set_size))

        # Save the model checkpoint periodically.
        if training_step % FLAGS.save_step_interval == 0 or training_step == training_steps_max:
            checkpoint_path = os.path.join(FLAGS.train_dir,
                                           FLAGS.model_architecture + '.ckpt')
            tf.logging.info('Saving to "%s-%d"', checkpoint_path,
                            training_step)
            saver.save(sess, checkpoint_path, global_step=training_step)

    # Testing loop.
    set_size = audio_processor.set_size('testing')
    tf.logging.info('set_size=%d', set_size)
    total_accuracy = 0
    total_conf_matrix = None
    for i in xrange(0, set_size, FLAGS.batch_size):
        test_fingerprints, test_ground_truth = audio_processor.get_data(
            FLAGS.batch_size, i, model_settings, 0.0, 0.0, 0, 'testing', sess)
        test_accuracy, conf_matrix = sess.run(
            [evaluation_step, confusion_matrix],
            feed_dict={
                fingerprint_input: test_fingerprints,
                ground_truth_input: test_ground_truth,
                dropout_prob: 1.0
            })
        batch_size = min(FLAGS.batch_size, set_size - i)
        total_accuracy += (test_accuracy * batch_size) / set_size
        if total_conf_matrix is None:
            total_conf_matrix = conf_matrix
        else:
            total_conf_matrix += conf_matrix
    tf.logging.info('Confusion Matrix:\n %s' % (total_conf_matrix))
    tf.logging.info('Final test accuracy = %.1f%% (N=%d)' %
                    (total_accuracy * 100, set_size))
Beispiel #51
0
def refill_batches(batches, word2id, context_file, qn_file, ans_file,
                   batch_size, context_len, question_len, word_len,
                   discard_long):
    """
    Adds more batches into the "batches" list.

    Inputs:
      batches: list to add batches to
      word2id: dictionary mapping word (string) to word id (int)
      context_file, qn_file, ans_file: paths to {train/dev}.{context/question/answer} data files
      batch_size: int. how big to make the batches
      context_len, question_len: max length of context and question respectively
      discard_long: If True, discard any examples that are longer than context_len or question_len.
        If False, truncate those exmaples instead.
    """
    print "Refilling batches..."
    tic = time.time()
    examples = [
    ]  # list of (qn_ids, context_ids, ans_span, ans_tokens) triples
    context_line, qn_line, ans_line = context_file.readline(
    ), qn_file.readline(), ans_file.readline()  # read the next line from each

    while context_line and qn_line and ans_line:  # while you haven't reached the end

        # Convert tokens to word ids
        context_tokens, context_ids, context_char_ids = sentence_to_token_ids(
            context_line, word2id)
        qn_tokens, qn_ids, qn_char_ids = sentence_to_token_ids(
            qn_line, word2id)
        ans_span = intstr_to_intlist(ans_line)

        # read the next line from each file
        context_line, qn_line, ans_line = context_file.readline(
        ), qn_file.readline(), ans_file.readline()

        # get ans_tokens from ans_span
        assert len(ans_span) == 2
        if ans_span[1] < ans_span[0]:
            print "Found an ill-formed gold span: start=%i end=%i" % (
                ans_span[0], ans_span[1])
            continue
        ans_tokens = context_tokens[ans_span[0]:ans_span[1] +
                                    1]  # list of strings

        # always truncate too long words
        for context_char_id in context_char_ids:
            if len(context_char_id) > word_len:
                context_char_id = context_char_id[:word_len]
            if len(context_char_id) < word_len:
                context_char_id.extend([CHAR_PAD_ID] *
                                       (word_len - len(context_char_id)))
        context_char_ids_flat = [
            item for sublist in context_char_ids for item in sublist
        ]
        context_char_len = context_len * word_len
        if len(context_char_ids_flat) > context_char_len:
            if discard_long:
                continue
            else:
                context_char_ids_flat = context_char_ids_flat[:
                                                              context_char_len]

        # always truncate too long words
        for qn_char_id in qn_char_ids:
            if len(qn_char_id) > word_len:
                context_char_id = context_char_id[:word_len]
            if len(qn_char_id) < word_len:
                qn_char_id.extend([CHAR_PAD_ID] * (word_len - len(qn_char_id)))
        question_char_len = question_len * word_len
        qn_char_ids_flat = [
            item for sublist in qn_char_ids for item in sublist
        ]
        if len(qn_char_ids_flat) > question_char_len:
            if discard_long:
                continue
            else:
                qn_char_ids_flat = qn_char_ids_flat[:question_char_len]

        # discard or truncate too-long questions
        if len(qn_ids) > question_len:
            if discard_long:
                continue
            else:  # truncate
                qn_ids = qn_ids[:question_len]

        # discard or truncate too-long contexts
        if len(context_ids) > context_len:
            if discard_long:
                continue
            else:  # truncate
                context_ids = context_ids[:context_len]

        # add to examples
        examples.append(
            (context_ids, context_tokens, qn_ids, qn_tokens,
             context_char_ids_flat, qn_char_ids_flat, ans_span, ans_tokens))

        # stop refilling if you have 160 batches
        if len(examples) == batch_size * 160:
            break

    # Once you've either got 160 batches or you've reached end of file:

    # Sort by question length
    # Note: if you sort by context length, then you'll have batches which contain the same context many times (because each context appears several times, with different questions)
    examples = sorted(examples, key=lambda e: len(e[2]))

    # Make into batches and append to the list batches
    for batch_start in xrange(0, len(examples), batch_size):

        # Note: each of these is a list length batch_size of lists of ints (except on last iter when it might be less than batch_size)
        context_ids_batch, context_tokens_batch, qn_ids_batch, qn_tokens_batch, context_char_ids_batch, qn_char_ids_batch, ans_span_batch, ans_tokens_batch = zip(
            *examples[batch_start:batch_start + batch_size])

        batches.append((context_ids_batch, context_tokens_batch, qn_ids_batch,
                        qn_tokens_batch, context_char_ids_batch,
                        qn_char_ids_batch, ans_span_batch, ans_tokens_batch))

    # shuffle the batches
    random.shuffle(batches)

    toc = time.time()
    print "Refilling batches took %.2f seconds" % (toc - tic)
    return
Beispiel #52
0
def train():
  # prepare dataset
  print("Preparing data in %s" % gConfig['working_directory'])
  enc_train, dec_train, enc_dev, dec_dev, _, _ = data_utils.prepare_custom_data(gConfig['working_directory'],gConfig['train_enc'],gConfig['train_dec'],gConfig['test_enc'],gConfig['test_dec'],gConfig['enc_vocab_size'],gConfig['dec_vocab_size'])

  # Only allocate 2/3 of the gpu memory to allow for running gpu-based predictions while training:
  gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.666)
  config = tf.ConfigProto(gpu_options=gpu_options)
  config.gpu_options.allocator_type = 'BFC'

  with tf.Session(config=config) as sess:
    # Create model.
    print("Creating %d layers of %d units." % (gConfig['num_layers'], gConfig['layer_size']))
    model = create_model(sess, False)

    # Read data into buckets and compute their sizes.
    print ("Reading development and training data (limit: %d)."
           % gConfig['max_train_data_size'])
    dev_set = read_data(enc_dev, dec_dev)
    train_set = read_data(enc_train, dec_train, gConfig['max_train_data_size'])
    train_bucket_sizes = [len(train_set[b]) for b in xrange(len(_buckets))]
    train_total_size = float(sum(train_bucket_sizes))

    # A bucket scale is a list of increasing numbers from 0 to 1 that we'll use
    # to select a bucket. Length of [scale[i], scale[i+1]] is proportional to
    # the size if i-th training bucket, as used later.
    train_buckets_scale = [sum(train_bucket_sizes[:i + 1]) / train_total_size
                           for i in xrange(len(train_bucket_sizes))]

    # This is the training loop.
    step_time, loss = 0.0, 0.0
    current_step = 0
    previous_losses = []
    while True:
      # Choose a bucket according to data distribution. We pick a random number
      # in [0, 1] and use the corresponding interval in train_buckets_scale.
      random_number_01 = np.random.random_sample()
      bucket_id = min([i for i in xrange(len(train_buckets_scale))
                       if train_buckets_scale[i] > random_number_01])

      # Get a batch and make a step.
      start_time = time.time()
      encoder_inputs, decoder_inputs, target_weights = model.get_batch(
          train_set, bucket_id)
      _, step_loss, _ = model.step(sess, encoder_inputs, decoder_inputs,
                                   target_weights, bucket_id, False)
      step_time += (time.time() - start_time) / gConfig['steps_per_checkpoint']
      loss += step_loss / gConfig['steps_per_checkpoint']
      current_step += 1

      # Once in a while, we save checkpoint, print statistics, and run evals.
      if current_step % gConfig['steps_per_checkpoint'] == 0:
        # Print statistics for the previous epoch.
        perplexity = math.exp(loss) if loss < 300 else float('inf')
        print ("global step %d learning rate %.4f step-time %.2f perplexity "
               "%.2f" % (model.global_step.eval(), model.learning_rate.eval(),
                         step_time, perplexity))
        # Decrease learning rate if no improvement was seen over last 3 times.
        if len(previous_losses) > 2 and loss > max(previous_losses[-3:]):
          sess.run(model.learning_rate_decay_op)
        previous_losses.append(loss)
        # Save checkpoint and zero timer and loss.
        checkpoint_path = os.path.join(gConfig['working_directory'], "seq2seq.ckpt")
        model.saver.save(sess, checkpoint_path, global_step=model.global_step)
        step_time, loss = 0.0, 0.0
        # Run evals on development set and print their perplexity.
        for bucket_id in xrange(len(_buckets)):
          if len(dev_set[bucket_id]) == 0:
            print("  eval: empty bucket %d" % (bucket_id))
            continue
          encoder_inputs, decoder_inputs, target_weights = model.get_batch(
              dev_set, bucket_id)
          _, eval_loss, _ = model.step(sess, encoder_inputs, decoder_inputs,
                                       target_weights, bucket_id, True)
          eval_ppx = math.exp(eval_loss) if eval_loss < 300 else float('inf')
          print("  eval: bucket %d perplexity %.2f" % (bucket_id, eval_ppx))
        sys.stdout.flush()
Beispiel #53
0
def _GetGrads(grads, op):
    """Gets all gradients for op."""
    if op in grads:
        return grads[op]
    else:
        return [[] for _ in xrange(len(op.outputs))]
    def _build(self):
        """Builds a model."""
        self._blocks = []
        batch_norm_momentum = self._global_params.batch_norm_momentum
        batch_norm_epsilon = self._global_params.batch_norm_epsilon
        if self._global_params.data_format == 'channels_first':
            channel_axis = 1
            self._spatial_dims = [2, 3]
        else:
            channel_axis = -1
            self._spatial_dims = [1, 2]

        # Stem part.
        self._conv_stem = tf.layers.Conv2D(
            filters=round_filters(32, self._global_params),
            kernel_size=[3, 3],
            strides=[2, 2],
            kernel_initializer=conv_kernel_initializer,
            padding='same',
            data_format=self._global_params.data_format,
            use_bias=False)
        self._bn0 = self._batch_norm(axis=channel_axis,
                                     momentum=batch_norm_momentum,
                                     epsilon=batch_norm_epsilon)

        # Builds blocks.
        for block_args in self._blocks_args:
            assert block_args.num_repeat > 0
            assert block_args.super_pixel in [0, 1, 2]
            # Update block input and output filters based on depth multiplier.
            input_filters = round_filters(block_args.input_filters,
                                          self._global_params)
            output_filters = round_filters(block_args.output_filters,
                                           self._global_params)
            kernel_size = block_args.kernel_size
            block_args = block_args._replace(input_filters=input_filters,
                                             output_filters=output_filters,
                                             num_repeat=round_repeats(
                                                 block_args.num_repeat,
                                                 self._global_params))

            # The first block needs to take care of stride and filter size increase.
            conv_block = self._get_conv_block(block_args.conv_type)
            if not block_args.super_pixel:  # no super_pixel at all
                self._blocks.append(conv_block(block_args,
                                               self._global_params))
            else:
                # if superpixel, adjust filters, kernels, and strides.
                depth_factor = int(4 / block_args.strides[0] /
                                   block_args.strides[1])
                block_args = block_args._replace(
                    input_filters=block_args.input_filters * depth_factor,
                    output_filters=block_args.output_filters * depth_factor,
                    kernel_size=((block_args.kernel_size + 1) // 2 if
                                 depth_factor > 1 else block_args.kernel_size))
                # if the first block has stride-2 and super_pixel trandformation
                if (block_args.strides[0] == 2 and block_args.strides[1] == 2):
                    block_args = block_args._replace(strides=[1, 1])
                    self._blocks.append(
                        conv_block(block_args, self._global_params))
                    block_args = block_args._replace(  # sp stops at stride-2
                        super_pixel=0,
                        input_filters=input_filters,
                        output_filters=output_filters,
                        kernel_size=kernel_size)
                elif block_args.super_pixel == 1:
                    self._blocks.append(
                        conv_block(block_args, self._global_params))
                    block_args = block_args._replace(super_pixel=2)
                else:
                    self._blocks.append(
                        conv_block(block_args, self._global_params))
            if block_args.num_repeat > 1:  # rest of blocks with the same block_arg
                # pylint: disable=protected-access
                block_args = block_args._replace(
                    input_filters=block_args.output_filters, strides=[1, 1])
                # pylint: enable=protected-access
            for _ in xrange(block_args.num_repeat - 1):
                self._blocks.append(conv_block(block_args,
                                               self._global_params))

        # Head part.
        self._conv_head = tf.layers.Conv2D(
            filters=round_filters(1280, self._global_params),
            kernel_size=[1, 1],
            strides=[1, 1],
            kernel_initializer=conv_kernel_initializer,
            padding='same',
            use_bias=False)
        self._bn1 = self._batch_norm(axis=channel_axis,
                                     momentum=batch_norm_momentum,
                                     epsilon=batch_norm_epsilon)

        self._avg_pooling = tf.keras.layers.GlobalAveragePooling2D(
            data_format=self._global_params.data_format)
        if self._global_params.num_classes:
            self._fc = tf.layers.Dense(
                self._global_params.num_classes,
                kernel_initializer=dense_kernel_initializer)
        else:
            self._fc = None

        if self._global_params.dropout_rate > 0:
            self._dropout = tf.keras.layers.Dropout(
                self._global_params.dropout_rate)
        else:
            self._dropout = None
Beispiel #55
0
    def pad_pair(self, encoder_inputs_ori, decoder_inputs_ori, seg_num,
                 seg_len, bucket_id, BOS):
        """
        To pad the encoder_inputs and decoder_inputs to fit certain buckets in
        encode-decode process.

        Args:
            encoder_inputs_ori: A list of length batch_size. each element is an sentence
                in which each word is represented by its own id.
            decoder_inputs_ori: A list of length batch_size. each element is an sentence
                in which each word is represented by its own id.
            seg_num: The number(No.?) of the segment that is going through the decode process.
            seg_len: The length of the segment that is going through the decode process.
                Equals to FLAGS.segment_length.
            bucket_id: The bucket to use in the encode-decode process.
            BOS: Whether this is the first segment to decode.

        Returns:
            batch_encoder_inputs, batch_decoder_inputs: Padded inputs that are ready
                to be fed in nn.
            batch_weights: Implemented as vanilla Seq2Seq.
        """
        encoder_size, decoder_size = self.buckets[bucket_id]
        encoder_inputs = []
        decoder_inputs = []

        # Get a random batch of encoder and decoder inputs from data,
        # pad them if needed, reverse encoder inputs and add GO to decoder.
        for i in xrange(self.batch_size):
            dst_i_ori = decoder_inputs_ori[i]  # the full sentence
            src_i = encoder_inputs_ori[
                i] + dst_i_ori[:min(len(dst_i_ori), seg_num * seg_len)]
            if len(dst_i_ori) > seg_num * seg_len:
                dst_i = dst_i_ori[seg_num * seg_len:(seg_num + 1) *
                                  seg_len]  # a segment
            else:
                dst_i = [data_utils.EOS_ID]

            # Encoder inputs are padded and then reversed.
            encoder_pad = [data_utils.PAD_ID] * (encoder_size - len(src_i))
            encoder_inputs.append(list(reversed(src_i + encoder_pad)))

            # Decoder inputs get an extra "GO" symbol, and are padded then. GO可能和BOS是一样的?
            decoder_pad_size = decoder_size - len(dst_i) - 1
            if BOS:
                decoder_inputs.append([data_utils.GO_ID] + dst_i +
                                      [data_utils.PAD_ID] * decoder_pad_size)
            else:
                decoder_inputs.append(dst_i + [data_utils.PAD_ID] *
                                      (decoder_pad_size + 1))

        # Now we create batch-major vectors from the data selected above.
        batch_encoder_inputs, batch_decoder_inputs, batch_weights = [], [], []

        # Batch encoder inputs are just re-indexed encoder_inputs. 有点像转置了?
        for length_idx in xrange(encoder_size):
            batch_encoder_inputs.append(
                np.array([
                    encoder_inputs[batch_idx][length_idx]
                    for batch_idx in xrange(self.batch_size)
                ],
                         dtype=np.int32))

        # Batch decoder inputs are re-indexed decoder_inputs, we create weights.
        for length_idx in xrange(decoder_size):
            batch_decoder_inputs.append(
                np.array([
                    decoder_inputs[batch_idx][length_idx]
                    for batch_idx in xrange(self.batch_size)
                ],
                         dtype=np.int32))

            # Create target_weights to be 0 for targets that are padding.
            batch_weight = np.ones(self.batch_size, dtype=np.float32)
            for batch_idx in xrange(self.batch_size):
                # We set weight to 0 if the corresponding target is a PAD symbol.
                # The corresponding target is decoder_input shifted by 1 forward.
                if length_idx < decoder_size - 1:
                    target = decoder_inputs[batch_idx][length_idx + 1]
                if length_idx == decoder_size - 1 or target == data_utils.PAD_ID:
                    batch_weight[batch_idx] = 0.0
            batch_weights.append(batch_weight)
        return batch_encoder_inputs, batch_decoder_inputs, batch_weights
Beispiel #56
0
def run_test():
  test_list_file = 'test.list'
  num_test_videos = len(list(open(test_list_file,'r')))
  print("Number of test videos={}".format(num_test_videos))

  # Get the sets of images and labels for training, validation, and
  images_placeholder, labels_placeholder = placeholder_inputs(FLAGS.batch_size * gpu_num)

  logits = []
  for gpu_index in range(0, gpu_num):
    with tf.device('/gpu:%d' % gpu_index):
      logit = resnet.inference(
                        images_placeholder[gpu_index * FLAGS.batch_size:(gpu_index + 1) * FLAGS.batch_size,:,:,:,:],
                        5,
                        False
                        )
      logits.append(logit)
  logits = tf.concat(logits,0)
  norm_score = tf.nn.softmax(logits)
  saver = tf.train.Saver(tf.global_variables())

  config = tf.ConfigProto(allow_soft_placement = True)
  config.gpu_options.allow_growth = True
  sess = tf.Session(config=config)
  init = tf.global_variables_initializer()
  sess.run(init)
  # Create a saver for writing training checkpoints.
  #saver.restore(sess, model_name)
  saver.restore(sess,"./models/r3d_model-4200")
  # And then after everything is built, start the training loop.
  bufsize = 0
  write_file = open("predict_ret.txt", "w+")
  next_start_pos = 0
  all_steps = int((num_test_videos - 1) / (FLAGS.batch_size * gpu_num) + 1)
  accuracy = tower_acc(logits, labels_placeholder)

  acc_cnt,acc5_cnt,cnt = 0,0,1

  for step in xrange(all_steps):
    # Fill a feed dictionary with the actual set of images and labels
    # for this particular training step.
    start_time = time.time()
    test_images, test_labels, next_start_pos, _, valid_len = \
            input_data.read_clip_and_label(
                    test_list_file,
                    FLAGS.batch_size * gpu_num,
                    start_pos=next_start_pos,
                    shuffle=True
                    )
    predict_score = norm_score.eval(
            session=sess,
            feed_dict={images_placeholder: test_images}
            )
    acc5 = tf.nn.in_top_k(predict_score,test_labels,5)
    top5_score = acc5.eval(session=sess,
                feed_dict={images_placeholder: test_images}
                )
    '''
    acc = sess.run(accuracy,feed_dict={
                                      images_placeholder: test_images,
                                      labels_placeholder: test_labels
                                      })
    print(acc)
    '''
    for i in range(0, valid_len):
      true_label = test_labels[i],
      top1_predicted_label = np.argmax(predict_score[i])
      # Write results: true label, class prob for true label, predicted label, class prob for predicted label
      write_file.write('{}, {}, {}, {}\n'.format(
              true_label[0],
              predict_score[i][true_label],
              top1_predicted_label,
              predict_score[i][top1_predicted_label]))
      cnt += 1
      if top1_predicted_label == true_label[0]:
        acc_cnt += 1
      if top5_score[i]:
        acc5_cnt += 1
    
    



  print("Test Accuracy={}".format(float(acc_cnt)/float(cnt)))
  print("Top-5 Accuracy={}".format(float(acc5_cnt)/float(cnt)))

  write_file.close()
  print("done")
def _ps_replicas(all_workers=False):
    if all_workers:
        return list(range(FLAGS.ps_replicas))
    # Worker K will be using replicas {0,...n-1} + K*n if we have n replicas.
    num_replicas = FLAGS.ps_replicas // FLAGS.worker_replicas
    return [d + FLAGS.worker_id * num_replicas for d in xrange(num_replicas)]
Beispiel #58
0
    def __init__(self,
                 source_vocab_size,
                 target_vocab_size,
                 buckets,
                 size,
                 num_layers,
                 max_gradient_norm,
                 batch_size,
                 learning_rate,
                 learning_rate_decay_factor,
                 use_lstm=True,
                 num_samples=512,
                 forward_only=False,
                 dtype=tf.float32):
        """Create the model.

        Args:
          source_vocab_size: size of the source vocabulary.
          target_vocab_size: size of the target vocabulary.
          buckets: a list of pairs (I, O), where I specifies maximum input length
            that will be processed in that bucket, and O specifies maximum output
            length. Training instances that have inputs longer than I or outputs
            longer than O will be pushed to the next bucket and padded accordingly.
            We assume that the list is sorted, e.g., [(2, 4), (8, 16)].
          size: number of units in each layer of the model.
          num_layers: number of layers in the model.
          max_gradient_norm: gradients will be clipped to maximally this norm.
          batch_size: the size of the batches used during training;
            the model construction is independent of batch_size, so it can be
            changed after initialization if this is convenient, e.g., for decoding.
          learning_rate: learning rate to start with.
          learning_rate_decay_factor: decay learning rate by this much when needed.
          use_lstm: if true, we use LSTM cells instead of GRU cells.
          num_samples: number of samples for sampled softmax.
          forward_only: if set, we do not construct the backward pass in the model.
          dtype: the data type to use to store internal variables.
        """
        self.source_vocab_size = source_vocab_size
        self.target_vocab_size = target_vocab_size
        self.buckets = buckets
        self.batch_size = batch_size
        self.learning_rate = tf.Variable(float(learning_rate),
                                         trainable=False,
                                         dtype=dtype)
        self.learning_rate_decay_op = self.learning_rate.assign(
            self.learning_rate * learning_rate_decay_factor)
        self.global_step = tf.Variable(0, trainable=False)

        # If we use sampled softmax, we need an output projection.
        output_projection = None
        softmax_loss_function = None
        # Sampled softmax only makes sense if we sample less than vocabulary size.
        if num_samples > 0 and num_samples < self.target_vocab_size:
            w_t = tf.get_variable("proj_w", [self.target_vocab_size, size],
                                  dtype=dtype)
            w = tf.transpose(w_t)
            b = tf.get_variable("proj_b", [self.target_vocab_size],
                                dtype=dtype)
            output_projection = (w, b)

            def sampled_loss(inputs, labels):
                labels = tf.reshape(labels, [-1, 1])
                # We need to compute the sampled_softmax_loss using 32bit floats to
                # avoid numerical instabilities.
                local_w_t = tf.cast(w_t, tf.float32)
                local_b = tf.cast(b, tf.float32)
                local_inputs = tf.cast(inputs, tf.float32)
                return tf.cast(
                    tf.nn.sampled_softmax_loss(local_w_t, local_b,
                                               local_inputs, labels,
                                               num_samples,
                                               self.target_vocab_size), dtype)

            softmax_loss_function = sampled_loss

        # Create the internal multi-layer cell for our RNN.
        list_of_cell = []
        for layer in xrange(num_layers):
            if layer % 2 == 0:
                with tf.device('/gpu:0'):
                    single_cell = tf.nn.rnn_cell.LSTMCell(size)
                list_of_cell.append(single_cell)
            else:
                with tf.device('/gpu:1'):
                    single_cell = tf.nn.rnn_cell.LSTMCell(size)
                list_of_cell.append(single_cell)

        if num_layers > 1:
            cell = Stack_Residual_RNNCell.Stack_Residual_RNNCell(list_of_cell)

        # The seq2seq function: we use embedding for the input and attention.
        def seq2seq_f(encoder_inputs, decoder_inputs, do_decode):
            return seq2seq_for_MT.embedding_attention_seq2seq(
                encoder_inputs,
                decoder_inputs,
                cell,
                num_layers=num_layers,
                num_encoder_symbols=source_vocab_size,
                num_decoder_symbols=target_vocab_size,
                embedding_size=512,
                output_projection=output_projection,
                feed_previous=do_decode,
                dtype=dtype)

        # Feeds for inputs.
        self.encoder_inputs = []
        self.decoder_inputs = []
        self.target_weights = []
        for i in xrange(buckets[-1][0]):  # Last bucket is the biggest one.
            self.encoder_inputs.append(
                tf.placeholder(tf.int32,
                               shape=[None],
                               name="encoder{0}".format(i)))
        for i in xrange(buckets[-1][1] + 1):
            self.decoder_inputs.append(
                tf.placeholder(tf.int32,
                               shape=[None],
                               name="decoder{0}".format(i)))
            self.target_weights.append(
                tf.placeholder(dtype, shape=[None],
                               name="weight{0}".format(i)))

        # Our targets are decoder inputs shifted by one. 感觉这里有问题
        targets = [
            self.decoder_inputs[i + 1]
            for i in xrange(len(self.decoder_inputs) - 1)
        ]

        # Training outputs and losses.
        if forward_only:
            self.outputs, self.losses = tf.nn.seq2seq.model_with_buckets(
                self.encoder_inputs,
                self.decoder_inputs,
                targets,
                self.target_weights,
                buckets,
                lambda x, y: seq2seq_f(x, y, True),
                softmax_loss_function=softmax_loss_function)
            # If we use output projection, we need to project outputs for decoding.
            if output_projection is not None:
                for b in xrange(len(buckets)):
                    self.outputs[b] = [
                        tf.matmul(output, output_projection[0]) +
                        output_projection[1] for output in self.outputs[b]
                    ]
        else:
            self.outputs, self.losses = tf.nn.seq2seq.model_with_buckets(
                self.encoder_inputs,
                self.decoder_inputs,
                targets,
                self.target_weights,
                buckets,
                lambda x, y: seq2seq_f(x, y, False),
                softmax_loss_function=softmax_loss_function)

        # Gradients and SGD update operation for training the model.
        params = tf.trainable_variables()
        if not forward_only:
            self.gradient_norms = []
            self.updates = []
            opt = tf.train.GradientDescentOptimizer(self.learning_rate)
            for b in xrange(len(buckets)):
                gradients = tf.gradients(self.losses[b], params)
                clipped_gradients, norm = tf.clip_by_global_norm(
                    gradients, max_gradient_norm)
                self.gradient_norms.append(norm)
                self.updates.append(
                    opt.apply_gradients(zip(clipped_gradients, params),
                                        global_step=self.global_step))

        self.saver = tf.train.Saver(tf.all_variables())
Beispiel #59
0
 def _batches(iterable, size):
     for i in xrange(0, len(iterable), size):
         yield iterable[i:i + size]
Beispiel #60
0
    def step(self, session, encoder_inputs, decoder_inputs, target_weights,
             bucket_id, forward_only):
        """Run a step of the model feeding the given inputs.

        Args:
          session: tensorflow session to use.
          encoder_inputs: list of numpy int vectors to feed as encoder inputs.
          decoder_inputs: list of numpy int vectors to feed as decoder inputs.
          target_weights: list of numpy float vectors to feed as target weights.
          bucket_id: which bucket of the model to use.
          forward_only: whether to do the backward step or only forward.

        Returns:
          A triple consisting of gradient norm (or None if we did not do backward),
          average perplexity, and the outputs.

        Raises:
          ValueError: if length of encoder_inputs, decoder_inputs, or
            target_weights disagrees with bucket size for the specified bucket_id.
        """
        # Check if the sizes match.
        encoder_size, decoder_size = self.buckets[bucket_id]
        if len(encoder_inputs) != encoder_size:
            raise ValueError(
                "Encoder length must be equal to the one in bucket,"
                " %d != %d." % (len(encoder_inputs), encoder_size))
        if len(decoder_inputs) != decoder_size:
            raise ValueError(
                "Decoder length must be equal to the one in bucket,"
                " %d != %d." % (len(decoder_inputs), decoder_size))
        if len(target_weights) != decoder_size:
            raise ValueError(
                "Weights length must be equal to the one in bucket,"
                " %d != %d." % (len(target_weights), decoder_size))

        # Input feed: encoder inputs, decoder inputs, target_weights, as provided.
        input_feed = {}
        for l in xrange(encoder_size):
            input_feed[self.encoder_inputs[l].name] = encoder_inputs[l]
        for l in xrange(decoder_size):
            input_feed[self.decoder_inputs[l].name] = decoder_inputs[l]
            input_feed[self.target_weights[l].name] = target_weights[l]

        # Since our targets are decoder inputs shifted by one, we need one more.
        last_target = self.decoder_inputs[decoder_size].name
        input_feed[last_target] = np.zeros([self.batch_size], dtype=np.int32)

        # Output feed: depends on whether we do a backward step or not.
        if not forward_only:
            output_feed = [
                self.updates[bucket_id],  # Update Op that does SGD.
                self.gradient_norms[bucket_id],  # Gradient norm.
                self.losses[bucket_id]
            ]  # Loss for this batch.
        else:
            output_feed = [self.losses[bucket_id]]  # Loss for this batch.
            for l in xrange(decoder_size):  # Output logits.
                output_feed.append(self.outputs[bucket_id][l])

        outputs = session.run(output_feed, input_feed)
        if not forward_only:
            return outputs[1], outputs[
                2], None  # Gradient norm, loss, no outputs.
        else:
            return None, outputs[0], outputs[
                1:]  # No gradient norm, loss, outputs.