Example #1
0
    def build_model(self):
        # Placeholders for our input data, hidden layer, and y values
        self.X = tf.placeholder("float", [None, self.time_steps, self.input_size])
        hidden_state = tf.placeholder("float", [None, self.hidden_features], name="Hidden")
        self.Y = tf.placeholder("float", [None, self.output_classes], name="Output")

        # Weights adn Biases for hidden layer and output layer
        W_hidden = tf.Variable(tf.random_normal([self.input_size,self.hidden_features]))
        W_out = tf.Variable(tf.random_normal([self.hidden_features,self.output_classes]))
        b_hidden = tf.Variable(tf.random_normal([self.hidden_features]))
        b_out = tf.Variable(tf.random_normal([self.output_classes]))

        # The Formula for the Model
        input_ = tf.reshape(self.X, [-1, self.input_size])
        lstm_cell = tf.nn.rnn_cell.GRUCell(self.hidden_features)
        input_2 = tf.split(0, self.time_steps, input_)
        cells = tf.nn.rnn_cell.MultiRNNCell([lstm_cell]* self.num_layers, state_is_tuple=True)
        hidden_state = cells.zero_state(self.batch_size, tf.float32)
        outputs, state = seq2seq.basic_rnn_seq2seq(input_2, hidden_state, cells) # this is the black magic
        hypothesis = tf.matmul(outputs[-1], W_out) + b_out
        self.hypothesis_index = tf.argmax(hypothesis,1)

        # Define our cost and optimizer
        cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(hypothesis,self.Y))
        self.optimizer = tf.train.AdamOptimizer(learning_rate=self.lr).minimize(cost)

        # Define our model evaluator
        correct_prediction = tf.equal(tf.argmax(hypothesis, 1), tf.argmax(self.Y,1))
        self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
        self.acc_summary = tf.scalar_summary("Accuracy", self.accuracy)
def _compute_average_correct(input_layer, labels, per_example_weights, topk=1):
  """Returns the numerator and denominator of classifier accuracy."""
  dtype = tf.float32
  if topk == 1:
    true_labels = tf.argmax(input_layer, 1)
    predictions = tf.argmax(labels, 1)
    in_topk = tf.equal(true_labels, predictions)
  else:
    _, true_labels = tf.nn.top_k(labels, k=1)
    true_labels = tf.reshape(true_labels, [-1])
    in_topk = tf.nn.in_top_k(tf.cast(input_layer, dtype), true_labels, k=topk)
  correct_predictions = tf.cast(in_topk, dtype)

  # If individual examples are weighted, then we want to normalize by that.
  if per_example_weights:
    per_example_weights = tf.convert_to_tensor(per_example_weights,
                                               name='per_example_weights')
    if ((input_layer.get_shape() and not per_example_weights.get_shape(
    ).is_compatible_with([input_layer.get_shape().dims[0]])) or
        per_example_weights.get_shape().ndims != 1):
      raise ValueError(
          'per_example_weights must be a vector of the same length as '
          'labels: was %s but expected (%s,)' % (
              per_example_weights.get_shape()), input_layer[0])
    float_weights = tf.cast(per_example_weights, dtype)
    # TODO(eiderman): This should use an op that doesn't support broadcasting.
    correct_predictions *= float_weights
    num_examples = tf.reduce_sum(float_weights)
  else:
    # shape only holds ints, but we want to always return the same type
    # for num_examples to make everything compatible.
    num_examples = tf.cast(tf.gather(tf.shape(input_layer), 0), dtype)
  return tf.reduce_sum(correct_predictions), num_examples
def evaluate_pr(tf, x, y_, sentences_test, labels_test, y_conv, keep_prob):

    pred, ans = tf.argmax(y_conv,1), tf.argmax(y_, 1)
    pred = pred.eval(feed_dict={x: sentences_test, y_: labels_test, keep_prob: 1.0})
    ans = ans.eval(feed_dict={x: sentences_test, y_: labels_test, keep_prob: 1.0})

    tp = 0
    tn = 0
    fp = 0
    fn = 0
    for i in range(len(ans)):
        if pred[i] == 0:
            if pred[i] == ans[i]:
                tp += 1
            else:
                fp += 1
        else:
            if pred[i] == ans[i]:
                tn += 1
            else:
                fn += 1

    precision = 0
    recall = 0
    if (tp + fp) > 0:
        precision = float(tp) / (tp + fp)
    if (tp + fn) > 0:
        recall = float(tp) / (tp + fn)

    print 'tp: %d, tn: %d, fp: %d, fn: %d' % (tp, tn, fp, fn)
    print 'precision: ', precision
    print 'recall:    ', recall

    return pred, ans
Example #4
0
  def eval(self, data, label, lens):
    predictions = []
    vals = []
    for i in range(data.shape[0]/self.batch_size):
      D = data[range(self.batch_size*i,self.batch_size*(i+1))]
      L = label[range(self.batch_size*i,self.batch_size*(i+1))]
      if lens is not None:
        l = lens[range(self.batch_size*i,self.batch_size*(i+1))]
        feed_dict={self.dataset:D, self.labels:L, self.lengths:l}
      else:
        feed_dict={self.dataset:D, self.labels:L}
      predictions.extend(self.sess.run(self.correct_prediction, feed_dict))
      vals.extend(self.sess.run(tf.argmax(self.logits,1), feed_dict))

    ## DO THE EXTRA
    last_chunk = self.batch_size*(i+1)
    gap = self.batch_size - (data.shape[0] - last_chunk)
    D = np.pad(data[last_chunk:], ((0,gap),(0,0)), mode='constant', constant_values=0)
    L = np.pad(label[last_chunk:], ((0,gap),(0,0)), mode='constant', constant_values=0)
    if lens is not None:
      l = np.pad(lens[last_chunk:], (0,gap), mode='constant', constant_values=0)
      feed_dict={self.dataset:D, self.labels:L, self.lengths:l}
    else:
      feed_dict={self.dataset:D, self.labels:L}
    predictions.extend(self.sess.run(self.correct_prediction, feed_dict)[:self.batch_size - gap])
    vals.extend(self.sess.run(tf.argmax(self.logits,1), feed_dict)[:self.batch_size - gap])

    print vals

    ## PRINT THE PREDICTONS
    return 100.0*sum(predictions)/len(predictions)
Example #5
0
    def __init__(self):
        # Import data
        error = None
        for _ in range(10):
            try:
                self.mnist = input_data.read_data_sets(
                    "/tmp/tensorflow/mnist/input_data", one_hot=True)
                error = None
                break
            except Exception as e:
                error = e
                time.sleep(5)
        if error:
            raise ValueError("Failed to import data", error)

        # Set seed and build layers
        tf.set_random_seed(0)

        self.x = tf.placeholder(tf.float32, [None, 784], name="x")
        self.y_ = tf.placeholder(tf.float32, [None, 10], name="y_")
        y_conv, self.keep_prob = deepnn(self.x)

        # Need to define loss and optimizer attributes
        self.loss = tf.reduce_mean(
            tf.nn.softmax_cross_entropy_with_logits(
                labels=self.y_, logits=y_conv))
        self.optimizer = tf.train.AdamOptimizer(1e-4)
        self.variables = ray_tf_utils.TensorFlowVariables(
            self.loss, tf.get_default_session())

        # For evaluating test accuracy
        correct_prediction = tf.equal(
            tf.argmax(y_conv, 1), tf.argmax(self.y_, 1))
        self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
Example #6
0
def compute_accuracy(v_xs, v_ys):
    global prediction
    y_pre = sess.run(prediction, feed_dict={xs: v_xs})
    correct_prediction = tf.equal(tf.argmax(y_pre,1), tf.argmax(v_ys,1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
    result = sess.run(accuracy, feed_dict={xs: v_xs, ys: v_ys})
    return result
 def accuracy(self):
     if self._accuracy is None:
         with tf.variable_scope('accuracy'):
             correct_predictions = tf.equal(tf.argmax(self.inference, axis=1),
                                            tf.argmax(tf.one_hot(self.targets, depth=self.n_classes), axis=1))
             self._accuracy = tf.reduce_mean(tf.cast(correct_predictions, tf.float32))
     return self._accuracy
Example #8
0
    def test(self, images, labels):

        # 正解率
        correct_prediction = tf.equal(tf.argmax(self.y_conv, 1), tf.argmax(self.y_, 1))
        accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

        num_data = len(images)
        i = 0
        batch_size = 10
        test_accuracy = 0
        while i < num_data:

            test_images_batch = images[i:i + batch_size]
            test_labels_batch = labels[i:i + batch_size]

            acc = self.session.run(accuracy,
                                   feed_dict={self.x: test_images_batch,
                                              self.y_: test_labels_batch,
                                              self.keep_prob: 1.0})
            test_accuracy += acc * batch_size

            # ループカウンタ更新
            i += batch_size
            if (i + batch_size) > num_data:
                batch_size = num_data - i

        test_accuracy /= float(num_data)

        return(test_accuracy)
Example #9
0
def cnn_setup(x, y, keep_prob, lr, stddev):
    first_hidden = 32
    second_hidden = 64
    fc_hidden = 1024
    W_conv1 = weight([5, 5, 1, first_hidden], stddev)
    B_conv1 = bias([first_hidden])
    x_image = tf.reshape(x, [-1, 28, 28, 1])
    h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + B_conv1)
    h_pool1 = max_pool_2x2(h_conv1)
    W_conv2 = weight([5, 5, first_hidden, second_hidden], stddev)
    b_conv2 = bias([second_hidden])
    h_conv2 = tf.nn.relu(conv2d(h_pool1, W_conv2) + b_conv2)
    h_pool2 = max_pool_2x2(h_conv2)
    W_fc1 = weight([7 * 7 * second_hidden, fc_hidden], stddev)
    b_fc1 = bias([fc_hidden])
    h_pool2_flat = tf.reshape(h_pool2, [-1, 7 * 7 * second_hidden])
    h_fc1 = tf.nn.relu(tf.matmul(h_pool2_flat, W_fc1) + b_fc1)
    h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)
    W_fc2 = weight([fc_hidden, 10], stddev)
    b_fc2 = bias([10])
    y_conv = tf.nn.softmax(tf.matmul(h_fc1_drop, W_fc2) + b_fc2)
    cross_entropy = tf.reduce_mean(
        -tf.reduce_sum(y * tf.log(y_conv), reduction_indices=[1]))
    correct_pred = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y, 1))
    return (tf.train.AdamOptimizer(lr).minimize(cross_entropy),
            tf.reduce_mean(tf.cast(correct_pred, tf.float32)), cross_entropy)
def train(env, steps=30000, checkin_interval=5000):
    '''学習をsteps回おこなう。
    '''
    correct_prediction = tf.equal(
        tf.argmax(env.model, 1),
        tf.argmax(env.actual_classes, 1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))

    bestResult = None
    for i in range(1, 1 + steps):
        env.sess.run(
            env.training_step,
            feed_dict=feed_dict(env, test=False),
        )
        if i % checkin_interval == 0:
            """
            print(i, env.sess.run(
                accuracy,
                feed_dict=feed_dict(env, test=False),
            ))
            """
            result = tf_confusion_metrics(env.model, env.actual_classes, env.sess, feed_dict(env, True))
            if not bestResult:
                bestResult = result
            else:
                bast = math.fabs(bestResult['Accuracy'] - 0.5)
                now = math.fabs(result['Accuracy'] - 0.5)
                print(bast, now)
                if bast < now:
                    bestResult = result

    #bestResult = tf_confusion_metrics(env.model, env.actual_classes, env.sess, feed_dict(env, True))
    return bestResult
    def __init__(self, learning_rate=0.001, ):
        # 记录训练次数
        self.global_step = tf.Variable(0, trainable=False)
        # 学习速率
        self.learning_rate = learning_rate
        # 输入张量 28 * 28 = 784个像素的图片一维向量
        self.x = tf.placeholder(tf.float32, [None, 784])
        # 标签值,即图像对应的结果,如果对应数字是8,则对应label是 [0,0,0,0,0,0,0,0,1,0]
        # 这种方式称为 one-hot编码
        # 标签是一个长度为10的一维向量,值最大的下标即图片上写的数字
        self.label = tf.placeholder(tf.float32, [None, 10])

        # 权重,初始化 正态分布
        self.w = tf.Variable(tf.random_normal([784, 10]))
        # 偏置 bias, 初始化 正态分布
        self.b = tf.Variable(tf.random_normal([10]))
        # 输出 y = softmax(X * w + b)
        self.y = tf.nn.softmax(tf.matmul(self.x, self.w) + self.b)
        # 损失,即交叉熵,最常用的计算标签(label)与输出(y)之间差别的方法
        self.loss = - tf.reduce_sum(self.label * tf.log(self.y + 1e-10))
        # 反向传播,采用梯度下降的方法。调整w与b,使得损失(loss)最小
        # loss越小,那么计算出来的y值与 标签(label)值越接近,准确率越高
        # minimize 可传入参数 global_step, 每次训练 global_step的值会增加1
        # 因此,可以通过计算self.global_step这个张量的值,知道当前训练了多少步
        self.train = tf.train.GradientDescentOptimizer(self.learning_rate).minimize(self.loss, global_step=self.global_step)

        # 以下代码验证正确率时使用
        # argmax 返回最大值的下标,最大值的下标即答案
        # 例如 [0,0,0,0.9,0,0.1,0,0,0,0] 代表数字3
        predict = tf.equal(tf.argmax(self.label, 1), tf.argmax(self.y, 1))

        # predict -> [true, true, true, false, false, true]
        # reduce_mean即求predict的平均数 即 正确个数 / 总数,即正确率
        self.accuracy = tf.reduce_mean(tf.cast(predict, dtype=tf.float32))
    def __init__(self, nn_settings, mnistHandler):
        self.variables  = {}
        self.operations = {}
        self.vis        = {}
        self.mnistHandler   = mnistHandler
        self.nn_settings    = nn_settings
        self.conv_specs     = nn_settings['conv_specs']
        self.conv_layers    = nn_settings['conv_layers']
        self.hidden_layer   = nn_settings['hidden_layer']
        self.do_batch_norm  = nn_settings['do_batch_norm']
        self.layers     = []
        self.train_phase    =   tf.placeholder(tf.bool, name='train_phase')

        self.y = self.define_graph()
        # cost function
        cross_entropy = -tf.reduce_sum(self.y_*
                         tf.log(tf.clip_by_value(self.y,1e-10,1.0)))
        
        # optimisation function
        self.train_step = tf.train.AdamOptimizer(mnistHandler.settings['LEARNING_RATE']).minimize(cross_entropy)
        
        # evaluation
        correct_prediction = tf.equal(tf.argmax(self.y,1),
                                      tf.argmax(self.y_,1))
        
        self.accuracy = tf.reduce_mean(tf.to_double(correct_prediction))
        
        # prediction function
        # return the index with the highest probability
        self.predict = tf.argmax(self.y,1)
Example #13
0
    def train(self, eval_on_test=False):
        """ Train model and save it to file.

        Train model with given hidden layers. Training data is created
        by prepare_training_data(), which must be called before this function.
        """
        tf.reset_default_graph()
        with tf.Session() as sess:
            feature_data = tf.placeholder("float", [None, self.num_predictors])
            labels = tf.placeholder("float", [None, self.num_classes])

            layers = [self.num_predictors] + self.hidden_layers + [self.num_classes]
            model = self.inference(feature_data, layers)
            cost, cost_summary_op = self.loss(model, labels)
            training_op = self.training(cost, learning_rate=0.0001)

            correct_prediction = tf.equal(tf.argmax(model, 1), tf.argmax(labels, 1))
            accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))

            # Merge all variable summaries and save the results to log file
            # summary_op = tf.merge_all_summaries()
            accuracy_op_train = tf.scalar_summary("Accuracy on Train", accuracy)
            summary_op_train = tf.merge_summary([cost_summary_op, accuracy_op_train])
            if eval_on_test:
                accuracy_op_test = tf.scalar_summary("Accuracy on Test", accuracy)
                summary_op_test = tf.merge_summary([accuracy_op_test])

            summary_writer = tf.train.SummaryWriter(self.log_dir + self.model_name, sess.graph)

            train_dict = {
                feature_data: self.training_predictors_tf.values,
                labels: self.training_classes_tf.values.reshape(len(self.training_classes_tf.values), self.num_classes)}

            if eval_on_test:
                test_dict = {
                    feature_data: self.test_predictors_tf.values,
                    labels: self.test_classes_tf.values.reshape(len(self.test_classes_tf.values), self.num_classes)}

            init = tf.initialize_all_variables()
            sess.run(init)

            for i in range(1, self.max_iteration):
                sess.run(training_op, feed_dict=train_dict)

                # Write summary to log
                if i % 100 == 0:
                    summary_str = sess.run(summary_op_train, feed_dict=train_dict)
                    summary_writer.add_summary(summary_str, i)
                    if eval_on_test:
                        summary_str = sess.run(summary_op_test, feed_dict=test_dict)
                        summary_writer.add_summary(summary_str, i)
                    summary_writer.flush()

                # Print current accuracy to console
                if i%5000 == 0:
                    print (i, sess.run(accuracy, feed_dict=train_dict))

            # Save trained parameters
            saver = tf.train.Saver()
            saver.save(sess, self.model_filename)
Example #14
0
    def __init__(self,n_input,n_hidden,n_output,sigma=1.,h=tf.nn.relu):
        '''
        This tensor flow gives a base line accuracy for a simple ReLu model.
        See the script_base_line_relu.py for usage.

        :param n_input: number of inputs
        :param n_hidden: number of hidden units
        :param n_output: number of outputs
        :param sigma: standard deviation of the weights at initailization
        :param h: tensorflow activation function
        '''
        # Define the place holders, basically the data driven variables
        self.x = tf.placeholder(tf.float32, [None, n_input])
        self.out_ = tf.placeholder(tf.float32, [None, n_output])

        # Model parameters
        self.W_hid = tf.Variable(tf.random_normal([n_input, n_hidden],stddev=sigma * 1./ np.sqrt(n_input)))
        self.b_hid = tf.Variable(tf.zeros([n_hidden]))

        self.W_out = tf.Variable(tf.random_normal([n_hidden, n_output],stddev=sigma * 1./ np.sqrt(n_hidden)))
        self.b_out = tf.Variable(tf.zeros([n_output]))

        # build variables
        self.y = h(tf.matmul(self.x, self.W_hid) + self.b_hid)
        self.z = tf.matmul(self.y, self.W_out) + self.b_out

        # loss function
        self.out = tf.nn.softmax(self.z)
        self.loss = tf.reduce_mean(-tf.reduce_sum(self.out_ * tf.log(self.out), reduction_indices=[1]))

        self.correct_prediction = tf.equal(tf.argmax(self.out, 1), tf.argmax(self.out_, 1))
        self.accuracy = tf.reduce_mean(tf.cast(self.correct_prediction, tf.float32))
Example #15
0
    def build(self,  configuration):
        tf.reset_default_graph()

        # --- specify input data
        self.inputs = tf.placeholder(tf.float32, [None, 28, 28, 1], name='x')
        self.labels = tf.placeholder(tf.float32, [None, 10], name='labels')
        # tf.summary.image('input', inputs, 3)
        # TODO add name scopes and summaries

        # --- specify layers of network
        # TODO try another strides for conv layer
        # TODO try to get rid of pooling layer
        conv1 = tf.layers.conv2d(inputs=self.inputs, filters=configuration[0], kernel_size=[5, 5], padding="same",
                                 activation=tf.nn.relu, name='conv1')
        pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=2, name='pool1')
        conv2 = tf.layers.conv2d(inputs=pool1, filters=configuration[1], kernel_size=[5, 5], padding="same",
                                 activation=tf.nn.relu, name='conv2')
        pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2, 2], strides=2, name='pool2')
        flattened = tf.reshape(pool2, [-1, 7 * 7 * configuration[1]])
        dense = tf.layers.dense(inputs=flattened, units=1024, activation=tf.nn.relu, name='fc')
        logits = tf.layers.dense(inputs=dense, units=10, name='output')

        # --- specify cost function and how training is performed
        with tf.name_scope("train"):
            cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=self.labels, logits=logits)
            self.train_step = tf.train.AdamOptimizer(0.015).minimize(cross_entropy)

        # --- specify function to calculate accuracy
        with tf.name_scope("accuracy"):
            correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(self.labels, 1))
            self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
            tf.summary.scalar("accuracy", self.accuracy)

        self.summary = tf.summary.merge_all()
Example #16
0
  def infer(self, features, *args, **kwargs):
    """Produce predictions from the model by running it."""
    del args, kwargs
    if "targets" not in features:
      if "infer_targets" in features:
        targets_shape = common_layers.shape_list(features["infer_targets"])
      elif "inputs" in features:
        targets_shape = common_layers.shape_list(features["inputs"])
        targets_shape[1] = self.hparams.video_num_target_frames
      else:
        raise ValueError("no inputs are given.")
      features["targets"] = tf.zeros(targets_shape, dtype=tf.float32)

    output, _ = self(features)  # pylint: disable=not-callable

    if not isinstance(output, dict):
      output = {"targets": output}

    x = output["targets"]
    if self.is_per_pixel_softmax:
      x_shape = common_layers.shape_list(x)
      x = tf.reshape(x, [-1, x_shape[-1]])
      x = tf.argmax(x, axis=-1)
      x = tf.reshape(x, x_shape[:-1])
    else:
      x = tf.squeeze(x, axis=-1)
      x = tf.to_int64(tf.round(x))
    output["targets"] = x
    if self.hparams.reward_prediction:
      output["target_reward"] = tf.argmax(output["target_reward"], axis=-1)

    # only required for decoding.
    output["outputs"] = output["targets"]
    output["scores"] = output["targets"]
    return output
Example #17
0
    def fit(self):
        w = tf.Variable(tf.zeros([self.x_train.shape[1], self.y_train.shape[1]]))
        b = tf.Variable(tf.zeros([self.y_train.shape[1]]))

        activation = tf.nn.softmax(tf.matmul(self.x, w) + b)
        cost = -tf.reduce_sum(self.y * tf.log(activation))
        optimizer = tf.train.GradientDescentOptimizer(self.learning_rate).minimize(cost)
        self.init = tf.initialize_all_variables()
        with tf.Session() as sess:
            sess.run(self.init)
            for epoch in range(self.training_epochs):
                avg_cost = 0.
                if self.batch_size == -1:
                    self.batch_size = int(self.x_train.shape[0] / 10)
                total_batch = int(self.x_train.shape[0] / self.batch_size)
                for i in range(total_batch):
                    batch_xs = self.x_train[i * self.batch_size: (i + 1) * self.batch_size]
                    batch_ys = self.y_train[i * self.batch_size: (i + 1) * self.batch_size]
                    sess.run(optimizer, feed_dict={self.x: batch_xs, self.y: batch_ys})
                    avg_cost += sess.run(cost, feed_dict={self.x: batch_xs, self.y: batch_ys}) / total_batch
            ZLog.info("Optimization Finished!")

            self.pred = tf.argmax(activation, 1)
            if self.x_test is not None:
                correct_prediction = tf.equal(self.pred, tf.argmax(self.y, 1))
                accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
                ZLog.info("Accuracy:" + str(accuracy.eval({self.x: self.x_test, self.y: self.y_test})))
Example #18
0
 def learn(self, x=None, y= None):
     self.init_target_placeholder()
     out = tf.argmax(self.output,1)
     target = tf.argmax(self.target,1)
     correct_prediction = tf.equal(out, target ) # tf.argmax(y_,1))
     self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
     Model.learn(self,x,y)
Example #19
0
def train_neural_network(x):
    prediction = neural_network_model(x)
    # OLD VERSION:
    #cost = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(prediction,y) )
    # NEW:
    cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
        logits=prediction, labels=y))
    optimizer = tf.train.AdamOptimizer().minimize(cost)

    hm_epochs = 25
    with tf.Session() as sess:
        # OLD:
        # sess.run(tf.initialize_all_variables())
        # NEW:
        sess.run(tf.global_variables_initializer())

        for epoch in range(hm_epochs):
            epoch_loss = 0
            for _ in range(int(mnist.train.num_examples / batch_size)):
                epoch_x, epoch_y = mnist.train.next_batch(batch_size)
                _, c = sess.run([optimizer, cost], feed_dict={
                                x: epoch_x, y: epoch_y})
                epoch_loss += c

            print('Epoch', epoch, 'completed out of',
                  hm_epochs, 'loss:', epoch_loss)

        correct = tf.equal(tf.argmax(prediction, 1), tf.argmax(y, 1))

        accuracy = tf.reduce_mean(tf.cast(correct, 'float'))
        print('Accuracy:', accuracy.eval(
            {x: mnist.test.images, y: mnist.test.labels}))
Example #20
0
  def evaluate(self, batch_size, keep_prob):

    self.correct_prediction = tf.equal(tf.argmax(self.logits, 1), tf.argmax(self.Y, 1))
    self.accuracy = tf.reduce_mean(tf.cast(self.correct_prediction, tf.float32))

    N = self.dataset.get_test_set_size()
    print('test.size', N);
    correct_sample = 0
    for i in range(0, N, batch_size):
        batch_xs, batch_ys = self.dataset.next_batch_test(batch_size)

        N_batch = batch_xs.shape[0]

        feed_dict = {
            self.X: batch_xs.reshape([N_batch, 28, 28, 1]),
            self.Y: batch_ys,
            self.keep_prob: keep_prob
        }

        correct = self.sess.run(self.accuracy, feed_dict=feed_dict)
        correct_sample += correct * N_batch

    test_accuracy = correct_sample / N

    print("\nAccuracy Evaluates")
    print("-" * 30)
    print('Test Accuracy:', test_accuracy)
def main():
    print "Running {:s}".format(__file__)
    
    #tf.is_variable_initialized(ww)
    with tf.Session() as sess:
        convNN = MyConv2D(10)
        minimizer = convNN.minimizer()
        loss = convNN.loss()
        mnist = convNN.fetch()
        
        # Create a summary writer, add the 'graph' to the event file.
        writer = tf.train.SummaryWriter(".", sess.graph)
        
        # Init variables
        sess.run(tf.initialize_all_variables())
        
        for epoch in range(n_epoch):
            batch = mnist.train.next_batch(mini_batch_size)
            _, loss_val =sess.run([minimizer, loss], feed_dict={convNN.x: batch[0], convNN.yy: batch[1]})
            
            print "run epoch {:d}: loss value is {:f}".format(epoch, loss_val) 
            #print summaries
            #writer.add_summary(summaries,epoch)
        
        correct_prediction = tf.equal(tf.argmax(convNN.yy,1), tf.argmax(convNN.predicted_y,1))
        accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
        accuracy_val = accuracy.eval(feed_dict={convNN.x: mnist.test.images, convNN.yy: mnist.test.labels})
        print "\naccuracy is {:f}".format(accuracy_val*100)
def tf_logistic_reg(train_X, train_Y, test_X, test_Y):
	# Start time
	startTime = time.time();
	enc = preprocessing.OneHotEncoder(n_values='auto');
	train_Y_onehot = enc.fit_transform(np.int32((np.transpose(np.matrix(train_Y)) + 1)/2)).toarray();
	test_Y_onehot = enc.fit_transform(np.int32((np.transpose(np.matrix(test_Y)) + 1)/2)).toarray();

	x  = tf.placeholder("float", [None, train_X.shape[1]]);
	y_ = tf.placeholder("float", [None, 2]);

	W = tf.Variable(tf.zeros([train_X.shape[1], 2]));
	b = tf.Variable(tf.zeros([2]));
	y = tf.nn.softmax(tf.matmul(x,W) + b);
	cross_entropy = -tf.reduce_sum(y_*tf.log(y));
	train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy);

	init = tf.initialize_all_variables();
	sess = tf.Session();
	sess.run(init);

	cur_id = 0;
	for i in range(1000):
		batch_xs, batch_ys, cur_id = sample(train_X, train_Y_onehot, cur_id, 500);
    	sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys});
    
	correct_prediction = tf.equal(tf.argmax(y,1), tf.argmax(y_,1));
	accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"));
	accu_train = sess.run(accuracy, feed_dict={x: train_X, y_: train_Y_onehot});
	accu_test = sess.run(accuracy, feed_dict={x: test_X, y_: test_Y_onehot});
	print "Logistic Regression: train accuracy %f"%accu_train;
	print "Logistic Regression: test accuracy %f"%accu_test;
	# Stop time
	stopTime = time.time();
	print "Elapsed time (logistic regression): %f"%(stopTime - startTime);
def train_neural_network(X, Y):
    predict = neural_network(X)
    cost_func = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(predict, Y))
    optimizer = tf.train.AdamOptimizer().minimize(cost_func)  # learning rate 默认 0.001

    epochs = 13
    with tf.Session() as session:
        session.run(tf.global_variables_initializer())
        epoch_loss = 0

        i = 0
        random.shuffle(train_dataset)
        train_x = dataset[:, 0]
        train_y = dataset[:, 1]
        for epoch in range(epochs):
            while i < len(train_x):
                start = i
                end = i + batch_size

                batch_x = train_x[start:end]
                batch_y = train_y[start:end]

                _, c = session.run([optimizer, cost_func], feed_dict={X: list(batch_x), Y: list(batch_y)})
                epoch_loss += c
                i += batch_size

            print(epoch, ' : ', epoch_loss)

        text_x = test_dataset[:, 0]
        text_y = test_dataset[:, 1]
        correct = tf.equal(tf.argmax(predict, 1), tf.argmax(Y, 1))
        accuracy = tf.reduce_mean(tf.cast(correct, 'float'))
        print('准确率: ', accuracy.eval({X: list(text_x), Y: list(text_y)}))
def tf_format_mnist_images(X, Y, Y_, n=100, lines=10):
    correct_prediction = tf.equal(tf.argmax(Y,1), tf.argmax(Y_,1))
    correctly_recognised_indices = tf.squeeze(tf.where(correct_prediction), [1])  # indices of correctly recognised images
    incorrectly_recognised_indices = tf.squeeze(tf.where(tf.logical_not(correct_prediction)), [1]) # indices of incorrectly recognised images
    everything_incorrect_first = tf.concat([incorrectly_recognised_indices, correctly_recognised_indices], 0) # images reordered with indeces of unrecognised images first
    everything_incorrect_first = tf.slice(everything_incorrect_first, [0], [n]) # compute first 100 only - no space to display more anyway
    # compute n=100 digits to display only
    Xs = tf.gather(X, everything_incorrect_first)
    Ys = tf.gather(Y, everything_incorrect_first)
    Ys_ = tf.gather(Y_, everything_incorrect_first)
    correct_prediction_s = tf.gather(correct_prediction, everything_incorrect_first)

    digits_left = tf.image.grayscale_to_rgb(tensorflowvisu_digits.digits_left())
    correct_tags = tf.gather(digits_left, tf.argmax(Ys_, 1)) # correct digits to be printed on the images
    digits_right = tf.image.grayscale_to_rgb(tensorflowvisu_digits.digits_right())
    computed_tags = tf.gather(digits_right, tf.argmax(Ys, 1)) # computed digits to be printed on the images
    #superimposed_digits = correct_tags+computed_tags
    superimposed_digits = tf.where(correct_prediction_s, tf.zeros_like(correct_tags),correct_tags+computed_tags) # only pring the correct and computed digits on unrecognised images
    correct_bkg   = tf.reshape(tf.tile([1.3,1.3,1.3], [28*28]), [1, 28,28,3]) # white background
    incorrect_bkg = tf.reshape(tf.tile([1.3,1.0,1.0], [28*28]), [1, 28,28,3]) # red background
    recognised_bkg = tf.gather(tf.concat([incorrect_bkg, correct_bkg], 0), tf.cast(correct_prediction_s, tf.int32)) # pick either the red or the white background depending on recognised status

    I = tf.image.grayscale_to_rgb(Xs)
    I = ((1-(I+superimposed_digits))*recognised_bkg)/1.3 # stencil extra data on top of images and reorder them unrecognised first
    I = tf.image.convert_image_dtype(I, tf.uint8, saturate=True)
    Islices = [] # 100 images => 10x10 image block
    for imslice in range(lines):
        Islices.append(tf.concat(tf.unstack(tf.slice(I, [imslice*n//lines,0,0,0], [n//lines,28,28,3])), 1))
    I = tf.concat(Islices, 0)
    return I
Example #25
0
def train_a_teacher_network():
    x = tf.placeholder(tf.float32, shape=[None, 784])
    y_ = tf.placeholder(tf.float32, shape=[None, 10])
    x_image = tf.reshape(x, [-1,28,28,1])
    net = ops.conv2d(x_image, 32, [5, 5], scope='conv1', stddev=0.1, bias=0.1)
    net = ops.max_pool(net, [2, 2], scope='pool1')
    net = ops.conv2d(net, 64, [5, 5], scope='conv2', stddev=0.1, bias=0.1)
    net = ops.max_pool(net, [2, 2], scope='pool2')
    net = ops.flatten(net, scope='pool2_flat')
    net = ops.fc(net, 1024, scope='fc1', stddev=0.1, bias=0.1)
    net = ops.fc(net, 10, activation=None, scope='fc2', stddev=0.1, bias=0.1)
    y_conv = tf.nn.softmax(net)
    cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y_conv), axis=[1]))
    model = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
    correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
    tf.summary.scalar('loss', cross_entropy)
    tf.summary.scalar('acc', accuracy)
    merged = tf.summary.merge_all()
    saver = tf.train.Saver()
    with tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) as sess:
        writer = tf.summary.FileWriter('./logs', sess.graph)
        sess.run(tf.global_variables_initializer())
        print('Teacher Network...')
        for i in range(MAX_ITER):
            batch = mnist.train.next_batch(BATCH_SIZE)
            sess.run(model, feed_dict={x: batch[0], y_: batch[1]})
            # saver.save(sess, './my-model', global_step=TEST_ITER)
            if i % 100 == 0:
                summary_str, acc = sess.run([merged, accuracy], feed_dict={x: mnist.test.images, y_: mnist.test.labels})
                writer.add_summary(summary_str, i)
                print('[Iter: {}] Validation Accuracy : {:.4f}'.format(i,acc))
                saver.save(sess, './my-model', global_step=TEST_ITER)
Example #26
0
def train(args):

    device = args.device
    load_path = args.load_path
    # load data
    train_data = load_data('train')
    val_data = load_data('validation')

    # load model
    with tf.device('/gpu:%d' % device):
        model = get_model('policy')

    # trainer init
    optimizer = Config.optimizer
    train_step = optimizer.minimize(model.loss)

    # init session and server
    sess = tf.InteractiveSession()
    saver = tf.train.Saver()
    if load_path==None:
        sess.run(tf.initialize_all_variables())
    else:
        saver.restore(sess, load_path)
        print("Model restored from %s" % load_path)

    # accuracy
    pred = tf.reshape(model.pred, [-1, 9*10*16])
    label = tf.reshape(model.label, [-1, 9*10*16])
    correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(label,1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

    logging.basicConfig(filename='log.txt', level=logging.DEBUG)
    # train steps
    for i in range(Config.n_epoch):

        # training step
        batch_data, batch_label = train_data.next_batch(Config.minibatch_size)

        input_dict = {model.label:batch_label}
        for var, data in zip(model.inputs, batch_data):
            input_dict[var]=data

        #from IPython import embed;embed()
        sess.run(train_step, feed_dict=input_dict)

        # evalue step
        if (i+1)%Config.evalue_point == 0:
            batch_data, batch_label = val_data.next_batch(Config.minibatch_size)
            val_dict = {model.label:batch_label}
            for var, data in zip(model.inputs, batch_data):
                val_dict[var]=data
            score = accuracy.eval(feed_dict=val_dict)
            print("epoch %d, accuracy is %.2f" % (i,score))
            logging.info("epoch %d, accuracy is %.2f" % (i,score))

        # save step
        if (i+1)%Config.check_point == 0:
            save_path = saver.save(sess, "%s/epoch-%d" %(Config.save_path, i))
            print("Model saved in file: %s" % save_path)
            logging.info("Model saved in file: %s" % save_path)
Example #27
0
def main(_):
  # Import data
  mnist = input_data.read_data_sets(FLAGS.data_dir, one_hot=True)
  # Create the model
  x = tf.placeholder(tf.float32, [None, 784])
  W = tf.Variable(tf.zeros([784, 10]))
  b = tf.Variable(tf.zeros([10]))
  y = tf.matmul(x, W) + b
  # Define loss and optimizer
  y_ = tf.placeholder(tf.float32, [None, 10])
  # The raw formulation of cross-entropy,
  #
  #   tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),
  #                                 reduction_indices=[1]))
  #
  # can be numerically unstable.
  #
  # So here we use tf.nn.softmax_cross_entropy_with_logits on the raw
  # outputs of 'y', and then average across the batch.
  cross_entropy = tf.reduce_mean(
      tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))
  train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)
  sess = tf.InteractiveSession()
  tf.global_variables_initializer().run()
  # Train
  for _ in range(1000):
    batch_xs, batch_ys = mnist.train.next_batch(100)
    sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})
  # Test trained model
  correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
  accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
  print(sess.run(accuracy, feed_dict={x: mnist.test.images,
                                      y_: mnist.test.labels}))
Example #28
0
        def __graph__():
            """Building the inference graph"""

            with tf.name_scope('input'):
                # [BATCH_SIZE, NUM_FEATURES]
                x_input = tf.placeholder(dtype=tf.float32, shape=[None, self.num_features], name='x_input')

                # [BATCH_SIZE]
                y_input = tf.placeholder(dtype=tf.uint8, shape=[None], name='y_input')

                # [BATCH_SIZE, NUM_CLASSES]
                y_onehot = tf.one_hot(indices=y_input, depth=self.num_classes, on_value=1, off_value=-1,
                                      name='y_onehot')

            learning_rate = tf.placeholder(dtype=tf.float32, name='learning_rate')

            with tf.name_scope('training_ops'):
                with tf.name_scope('weights'):
                    weight = tf.get_variable(name='weights',
                                             initializer=tf.random_normal([self.num_features, self.num_classes],
                                                                          stddev=0.01))
                    self.variable_summaries(weight)
                with tf.name_scope('biases'):
                    bias = tf.get_variable(name='biases', initializer=tf.constant([0.1], shape=[self.num_classes]))
                    self.variable_summaries(bias)
                with tf.name_scope('Wx_plus_b'):
                    output = tf.matmul(x_input, weight) + bias
                    tf.summary.histogram('pre-activations', output)

            with tf.name_scope('svm'):
                regularization = tf.reduce_mean(tf.square(weight))
                hinge_loss = tf.reduce_mean(tf.square(tf.maximum(tf.zeros([self.batch_size, self.num_classes]),
                                                                 1 - tf.cast(y_onehot, tf.float32) * output)))
                with tf.name_scope('loss'):
                    loss = regularization + self.svm_c * hinge_loss
            tf.summary.scalar('loss', loss)

            optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss)

            with tf.name_scope('accuracy'):
                predicted_class = tf.sign(output)
                predicted_class = tf.identity(predicted_class, name='prediction')
                with tf.name_scope('correct_prediction'):
                    correct = tf.equal(tf.argmax(predicted_class, 1), tf.argmax(y_onehot, 1))
                with tf.name_scope('accuracy'):
                    accuracy = tf.reduce_mean(tf.cast(correct, 'float'))
            tf.summary.scalar('accuracy', accuracy)

            merged = tf.summary.merge_all()

            self.x_input = x_input
            self.y_input = y_input
            self.y_onehot = y_onehot
            self.learning_rate = learning_rate
            self.loss = loss
            self.optimizer = optimizer
            self.output = output
            self.predicted_class = predicted_class
            self.accuracy = accuracy
            self.merged = merged
Example #29
0
def train_neural_network(x):
    prediction = neural_network_model(x)

    #using cross entropy with logits as our cost function
    #calculates the difference between prediction and y(the labels on mnist data)
    cost = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(prediction,y) )

    #AdamOptimizer is synonymous with SGD, AdaGrad, so on.
    #learning rate = 0.001
    optimizer = tf.train.AdamOptimizer().minimize(cost)

    #epoch = (cycles of) feed forward + backprop
    hm_epochs = 10

    with tf.Session() as sess:
        sess.run(tf.initialize_all_variables())

        for epoch in range(hm_epochs):
            epoch_loss = 0

            #training the data
            for _ in range(int(mnist.train.num_examples/batch_size)):
                #next_batch helper func from tf
                epoch_x, epoch_y = mnist.train.next_batch(batch_size)

                #how this optimizes cost is unclear yet -rbdmtodo
                _, c = sess.run([optimizer, cost], feed_dict = {x: epoch_x, y: epoch_y})
                epoch_loss += c
            print('Epoch', epoch, 'completed out of', hm_epochs, 'loss:',epoch_loss)

        correct = tf.equal(tf.argmax(prediction,1), tf.argmax(y,1))

        accuracy = tf.reduce_mean(tf.cast(correct, 'float'))
        print('Accuracy:', accuracy.eval({x:mnist.test.images, y:mnist.test.labels}))
Example #30
0
def evaluate(mnist):
    with tf.Graph().as_default() as g:
        x = tf.placeholder(tf.float32, [None, mnist_inference.INPUT_NODE], name='x-input')
        y_ = tf.placeholder(tf.float32, [None, mnist_inference.OUTPUT_NODE], name='y-input')
        validate_feed = {x: mnist.validation.images, y_: mnist.validation.labels}

        y = mnist_inference.inference(x, None)
        # 分类,预测结果
        correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
        accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

        variable_averages = tf.train.ExponentialMovingAverage(mnist_train.MOVING_AVERAGE_DECAY)
        variables_to_restore = variable_averages.variables_to_restore()
        saver = tf.train.Saver(variables_to_restore)

        while True:
            with tf.Session() as sess:
                # 加载模型
                ckpt = tf.train.get_checkpoint_state(mnist_train.MODEL_SAVE_PATH)
                if ckpt and ckpt.model_checkpoint_path:
                    saver.restore(sess, ckpt.model_checkpoint_path)
                    # 通过文件名获得迭代轮数
                    global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1]
                    accuracy_score = sess.run(accuracy, feed_dict=validate_feed)
                    print("After %s training step(s), validation accuracy = %g" % (global_step, accuracy_score))
                else:
                    print('No checkpoint file found')
                    return
            time.sleep(EVAL_INTERVAL_SECS)
Example #31
0
def argmax_prediction(model, x):
    return tf.argmax(model(x), axis=1)
Example #32
0
b1 = tf.Variable(tf.zeros([10]))
b2 = tf.Variable(tf.zeros([3]))

L1 = tf.add(tf.matmul(X, W1), b1)
L1 = tf.nn.relu(L1)

model = tf.add(tf.matmul(L1, W2), b2)

cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(labels=Y, logits=model))

optimizer = tf.train.AdamOptimizer(learning_rate=0.01)
train_op = optimizer.minimize(cost)

sess = tf.Session()
sess.run(tf.global_variables_initializer())

for step in range(100):
    sess.run(train_op, feed_dict={X: x_data, Y: y_data})

    if (step + 1) % 10 == 0:
        print(step + 1, sess.run(cost, feed_dict={X: x_data, Y: y_data}))

prediction = tf.argmax(model, axis=1)
target = tf.argmax(Y, axis=1)
print('예측값:', sess.run(prediction, feed_dict={X: x_data}))
print('실제값:', sess.run(target, feed_dict={Y: y_data}))

is_correct = tf.equal(prediction, target)
accuracy = tf.reduce_mean(tf.cast(is_correct, tf.float32))
print('정확도: %.2f' % sess.run(accuracy * 100, feed_dict={X: x_data, Y: y_data}))
Example #33
0
def compute_accuracy(logits, labels):
    index = tf.argmax(logits, axis=1)
    values = tf.cast(tf.equal(index, labels), tf.float64)
    batch_size = int(logits.shape[0])

    return tf.reduce_sum(values) / batch_size
    # Get lstm cell output
    outputs, states = rnn.rnn(lstm_cell, x, dtype=tf.float32)

    # Linear activation, using rnn inner loop last output
    return tf.matmul(outputs[-1], weights["out"]) + biases["out"]


pred = RNN(x, weights, biases)

# Define loss and optimizer
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(pred, y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)

# Evaluate model
correct_pred = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))

# Initializing the variables
init = tf.initialize_all_variables()

# Launch the graph
with tf.Session() as sess:
    sess.run(init)
    step = 1
    # Keep training until reach max iterations
    while step * batch_size < training_iters:
        batch_x, batch_y = mnist.train.next_batch(batch_size)
        # Reshape data to get 28 seq of 28 elements
        batch_x = batch_x.reshape((batch_size, n_steps, n_input))
        # Run optimization op (backprop)
Example #35
0
     width_of_data = 1
     pick_band_array = np.where(np.array(list(imply_mask), dtype=int) == 0)
 else:
     print('Wrong error consideration.')
     exit()
 image_shape = (width_of_data, img_maj)
 kernal_shape = (width_of_data, 2)
 num_kernal_1 = 32
 num_kernal_2 = 64
 num_conn_neural = 100
 num_label = len(data.train.labels[0])
 #-----------------------------------
 # Construct an AI
 x = tf.placeholder(tf.float32, [None, width_of_data * img_maj], name='x')
 y_true = tf.placeholder(tf.float32, [None, 3], name='y_true')
 y_true_cls = tf.argmax(y_true, axis=1)
 x_image = tf.reshape(x, [-1, image_shape[0], image_shape[1], 1])
 # First layer( First kernal)
 W_conv1 = weight_variable(
     [kernal_shape[0], kernal_shape[1], 1, num_kernal_1])
 b_conv1 = bias_variable([num_kernal_1])
 h_conv1 = tf.nn.selu(
     tf.nn.conv2d(x_image, W_conv1, [1, 1, 1, 1], 'SAME') + b_conv1)
 # Second layer( Second kernal)
 W_conv2 = weight_variable(
     [kernal_shape[0], kernal_shape[1], num_kernal_1, num_kernal_2])
 b_conv2 = bias_variable([num_kernal_2])
 h_conv2 = tf.nn.selu(
     tf.nn.conv2d(h_conv1, W_conv2, [1, 1, 1, 1], 'SAME') + b_conv2)
 # Third layer ( Fully connected)
 W_fc1 = weight_variable(
Example #36
0
		def loop(prev, _):
			prev = tf.matmul(prev, softmax_w) + softmax_b
			prev_symbol = tf.stop_gradient(tf.argmax(prev, 1))
			return tf.nn.embedding_lookup(embedding, prev_symbol)
Example #37
0
with tf.name_scope('fc_softmax'):
    w_fc2 = weight_variable([1024,10])
    b_fc2 = bais_variable([10])
    y_conv=tf.nn.softmax(tf.matmul(h_fc1_drop,w_fc2)+b_fc2)

with tf.name_scope('cross_entropy'):
    cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y_conv), reduction_indices=[1]))
tf.summary.scalar('cross_entropy',cross_entropy)


with tf.name_scope('train'):
    train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)  # 优化函数可以更换

with tf.name_scope('accuracy'):
    with tf.name_scope('correct_prediction'):
        correct_prediction=tf.equal(tf.argmax(y_conv,1),tf.argmax(y_,1))
    with tf.name_scope('accuracy'):
        accuracy=tf.reduce_mean(tf.cast(correct_prediction,tf.float32))
tf.summary.scalar('accuracy',accuracy)

merged=tf.summary.merge_all()
train_writer=tf.summary.FileWriter(log_dir+'/train',sess.graph)
test_writer=tf.summary.FileWriter(log_dir+'/test')
tf.global_variables_initializer().run()


saver=tf.train.Saver()
for i in range(max_steps):
    batch = mnist.train.next_batch(50)
    if i % 100 ==0:
        # 这里通过trace_level参数配置运行时需要记录的信息, # tf.RunOptions.FULL_TRACE代表所有的信息
    def __init__(self, num_classes, vocab_size, embedding_size, max_length, vocab_proc, filter_sizes, num_filters,
      l2_reg_lambda=0.0, use_emb=True):

        # Placeholders for input, embedding matrix for unknown words and dropout
        self.input_q = tf.placeholder(tf.int32, [None, max_length], name="input_q")
        self.input_p = tf.placeholder(tf.int32, [None, max_length], name="input_p")
        self.input_y = tf.placeholder(tf.int32, [None, num_classes], name="input_y")
        self.W_emb = tf.placeholder(tf.float32,[vocab_size, embedding_size], name="emb_pretrained")
        self.dropout_keep_prob = tf.placeholder(tf.float32, name="dropout_keep_prob")

        # Embedding layer
        with tf.name_scope("embedding_text"):
          '''
          # Following block only useful if we want to train embeddings for words not in vocab
          if use_emb:
              self.train_W = tf.Variable(
                  tf.random_uniform([vocab_size, embedding_size], -1.0, 1.0),
                  name="W_train")
              # Build de matrix of embeddings of words in the vocab.
              # If word exists in pre-trained embeddings, take this embedding.
              # If not, create a random embedding
              self.W = tf.where(tf.equal(tf.reduce_sum(tf.abs(self.W_emb),1),0), self.train_W, self.W_emb)
          else:
              self.W = tf.Variable(
                  tf.random_uniform([vocab_size, embedding_size], -1.0, 1.0),
                  name="W")
          '''

          self.W = self.W_emb

          # Map word IDs to word embeddings
          # From dimension batch_size * max_length to batch_size * max_length * embedding_size
          self.input_q_emb = tf.nn.embedding_lookup(self.W, self.input_q)
          self.input_p_emb = tf.nn.embedding_lookup(self.W, self.input_p)

          # Create CBOW p and q embeddings by averaging over the word embeddings dimension
          # From dimension batch_size * max_length * embedding_size to batch_size * embedding_size
          self.input_q_CBOW, self.mask_input_q_nonzero = self.embed_CBOW(self.input_q, self.input_q_emb)
          self.input_p_CBOW, self.mask_input_p_nonzero = self.embed_CBOW(self.input_p, self.input_p_emb)

          # OPTIONAL : add dropout on the embeddings
          #self.input_q_CBOW_dropout = tf.nn.dropout(self.input_q_CBOW,self.dropout_keep_prob)
          #self.input_p_CBOW_dropout = tf.nn.dropout(self.input_p_CBOW,self.dropout_keep_prob)


        # OPTIONAL : Keeping track of l2 regularization loss (optional)
        #l2_loss = tf.constant(0.0)

        # Network Parameters
        n_hidden_1 = 256 # 1st layer number of features
        n_hidden_2 = 256 # 2nd layer number of features
        n_input = embedding_size*2 # we are going to concat paragraph and question
        n_classes = num_classes

        # Final (unnormalized) scores and predictions
        with tf.name_scope("output"):
            dif = tf.subtract(self.input_p_CBOW, self.input_q_CBOW,name="dif")
            dif_point_mul = tf.multiply(dif, dif, name ="dif_point_mul")
            pq_point_mul = tf.multiply(self.input_p_CBOW, self.input_q_CBOW,name="pq_point_mul")
            self.concatenated_input = tf.concat([dif_point_mul, pq_point_mul], 1,name="concatenated_input")
            self.scores = self.multilayer_perceptron(self.concatenated_input,
              n_input, n_hidden_1, n_hidden_2, n_classes, self.dropout_keep_prob)
            function_to_score = lambda x : x + (10.0**(-4))  # Where `f` instantiates myCustomOp.
            self.scores = tf.map_fn(function_to_score, self.scores)
            self.predictions = tf.argmax(self.scores, 1, name="predictions")
            self.y_true = tf.argmax(self.input_y, 1, name="y_true")

        # CalculateMean cross-entropy loss
        with tf.name_scope("loss"):
            losses = tf.nn.softmax_cross_entropy_with_logits(labels=self.input_y, logits=self.scores, name = "losses")
            self.loss = tf.reduce_mean(losses,0,name = 'loss_sub') #+ l2_reg_lambda * l2_loss

        # Accuracy
        with tf.name_scope("accuracy"):
            correct_predictions = tf.equal(self.predictions, tf.argmax(self.input_y, 1),name='correct_predictions')
            self.accuracy = tf.reduce_mean(tf.cast(correct_predictions, "float"), name="accuracy")


        # Random stuff to print and delete later
        self.input_q_01 = tf.gather(self.input_q , 1)
        self.input_p_06 = tf.gather(self.input_p , 6)
        self.input_p_07 = tf.gather(self.input_p , 7)
        self.input_p_08 = tf.gather(self.input_p , 8)
        self.input_q_CBOW_new_01 = tf.gather(self.input_q_CBOW, 1)
y__ = 200
totalEpisode = 0
numberSolutionFound =0
# atexit.register(plotter, x, y)
resume = False;
# model = neural_network_model(input_size)
avgReward = 0

tf.reset_default_graph
#These lines establish the feed-forward part of the network used to choose actions
inputs1 = tf.placeholder(shape=[1, input_size],dtype=tf.float32)
# indie vorm verdeeld tussen...
with tf.name_scope("fully_connected"):
    W = tf.Variable(tf.random_uniform([1500,5],0,0.02))
    Qout = tf.matmul(inputs1,W)
    predict = tf.argmax(Qout,1)
    tf.summary.histogram("weigths", W)


# loss = tf.reduce_sum(tf.square(nextQ - Qout))
# loss = tf.losses.softmax_cross_entropy(nextQ, Qout)

#Below we obtain the loss by taking the sum of squares difference between the target and prediction Q values.
nextQ = tf.placeholder(shape=[1,5],dtype=tf.float32)
with tf.name_scope("loss"):
    loss = tf.reduce_sum(tf.square(nextQ - Qout))
    tf.summary.scalar("loss", loss)
with tf.name_scope("train"):
    trainer = tf.train.GradientDescentOptimizer(learning_rate=0.0005)
    updateModel = trainer.minimize(loss)
def pullBandit(bandit):
    #Сгенерировать случайное число
    result = np.random.randn(1)
    # print(result)
    if result > bandit:
        #Выигрыш
        return 1
    else:
        #Проигрыш
        return -1

import tensorflow.compat.v1 as tf
tf.disable_v2_behavior()

weights = tf.Variable(tf.ones([num_bandits]))
chosen_action = tf.argmax(weights,0)

reward_holder = tf.placeholder(shape=[1],dtype=tf.float32)
action_holder = tf.placeholder(shape=[1],dtype=tf.int32)
responsible_weight = tf.slice(weights,action_holder,[1])
loss = -(tf.log(responsible_weight)*reward_holder)
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001)
update = optimizer.minimize(loss)

total_episodes = 1000 #Количество итераций обучения
total_reward = np.zeros(num_bandits) 
e = 0.1 #Вероятность случайного выбора
init = tf.global_variables_initializer()

with tf.Session() as sess:
    sess.run(init)
Example #41
0
        tr_logits = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "logits")
        up_conv5_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "up_conv5")

        # retrain vars
        rt_fc = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "fc_1")
        rt_up = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "up_conv1")
        rt_bn = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, "unpool1.1")

        # create a training step for vars that should be trained
        # train_op_2 = optimizer.minimize(loss, global_step=global_step, var_list=up_conv5_vars)
        train_op_2 = optimizer.minimize(loss, global_step=global_step, var_list=rt_fc + rt_up + rt_bn)

    train_op_1 = optimizer.minimize(loss, global_step=global_step)

    # if we reshape the predictions it won't work with images of other sizes
    predictions = tf.argmax(logits, axis=-1, output_type=tf.int32)

    # squash the predictions into a per image prediction - negative images will have a max of 0
    pred_sum = tf.reduce_sum(predictions, axis=[1, 2])
    image_predictions = tf.cast(tf.greater(pred_sum, (size * size // 750)), dtype=tf.uint8)
    image_truth = tf.reduce_max(y_adj, axis=[1, 2])

    # set a threshold on the predictions so we ignore images with only a few positive pixels
    pred_sum = tf.reduce_sum(predictions, axis=[1, 2])
    image_predictions = tf.cast(tf.greater(pred_sum, (size*size//750)),dtype=tf.uint8)

    # get the accuracy per pixel
    accuracy, acc_op = tf.metrics.accuracy(
        labels=y_adj,
        predictions=predictions,
        updates_collections=[tf.GraphKeys.UPDATE_OPS, 'metrics_ops'],
Example #42
0
def convertLabel(y):
    Y = np.zeros((y.shape[0], 10))
    Y[np.arange(y.shape[0]), y] = 1
    return Y


# yTrain = convertLabel(yTrain)
# yTest = convertLabel(yTest)
x = tf.placeholder(tf.float32, [None, numFeatures])
y = tf.placeholder(tf.int64, [None])

W = tf.Variable(tf.random_normal([numFeatures, 10], stddev=1 / sqrt(numTrain)))
z = tf.matmul(x, W)
predict = tf.nn.softmax(z)
predictLabel = tf.cast(tf.argmax(predict, 1), tf.int32)
# actualLabel = tf.cast(tf.argmax(y,1),tf.int32)
# actualLabel = y
# acc = tf.reduce_sum(tf.cast(tf.equal(tf.argmax(predict,1),tf.argmax(y,1)),tf.int32))
acc = tf.reduce_sum(
    tf.cast(tf.equal(tf.argmax(predict, 1), tf.cast(y, tf.int64)), tf.int32))
# accFloat = tf.cast(acc,tf.float32)
accuracy = (10000 * acc) / tf.shape(predict)[0]
# softmaxcrossEntropyWithLogits , first calculate softmax of logits
# then cross entropy using lables*log(softmax)
# cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = z ,labels = y))
cost = tf.reduce_mean(
    tf.nn.sparse_softmax_cross_entropy_with_logits(logits=z, labels=y))
train = tf.train.GradientDescentOptimizer(1e-3).minimize(cost)
# s = tf.nn.softmax(logits = l)
# a = np.array([[10,10,10,20,20]])
Example #43
0
b1=tf.Variable(tf.zeros([n_hidden]))

hidden=tf.sigmoid(tf.matmul(input, W1)+b1)

W2=tf.Variable(tf.zeros([n_hidden, 10]))
b2=tf.Variable(tf.zeros([10]))

output=tf.nn.softmax(tf.matmul(hidden,W2)+b2)

loss=-tf.reduce_sum(label*tf.log(output))

train_step=tf.train.AdamOptimizer().minimize(loss)



correct_prediction=tf.equal(tf.argmax(output,1), tf.argmax(label,1))
accuracy=tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
plt.title('Hidden layer size %d'%n_hidden)
plt.xlabel('Train step')
plt.ylabel('Accuracy')
for k in range(5):
    tf.global_variables_initializer().run()
    log=np.zeros([2,101])

    for i in range(10000):
        if i%100==0:
            accuracy_eval=accuracy.eval({input:mnist.test.images, label:mnist.test.labels})
            print('step %d, accuracy %s'%(i,accuracy_eval))
            log[0][i/100]=i
            log[1][i/100]=accuracy_eval
        x, y=mnist.train.next_batch(100)
Example #44
0
    fc3 = tf.layers.dense(fc1, NUM_H2, activation=tf.nn.relu, kernel_initializer=he_init, name='fc3') # Second hidden layer with relu

    print(x, conv1, pool1, conv2, pool2, conv3, conv4, conv5, pool3, fc1, fc2, fc3)

    logits = tf.layers.dense(fc2, NUM_OUTPUTS, name='logits')  # this tf.layers.dense is same as tf.matmul(x, W) + b
    prediction = tf.nn.softmax(logits)
    return logits, prediction

# Define loss and optimizer
logits, prediction = network()
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=Y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate)
trainer = optimizer.minimize(loss)

# Evaluate model
correct_pred = tf.equal(tf.argmax(prediction, 1), tf.argmax(Y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))

# Initalize varibles, and run network
init = tf.global_variables_initializer()
sess = tf.Session()
sess.run(init)

# Train network
_step = []
_acc = []
for step in range(num_steps):
    batch_xs, batch_ys = mnist.train.next_batch(batch_size)
    sess.run( trainer, feed_dict={X: batch_xs, Y: batch_ys, isTraining:True })

    if(step % display_step == 0):
Example #45
0
sess.run(tf.global_variables_initializer())

# parameters
training_epochs = 15
batch_size = 100
# Check out https://www.tensorflow.org/get_started/mnist/beginners for
# more information about the mnist dataset
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)

nb_classes = 10

# train my model
print('Learning stared. It takes sometime.')
for epoch in range(training_epochs):
   avg_cost = 0
   total_batch = int(mnist.train.num_examples / batch_size)
   for i in range(total_batch):
       batch_xs, batch_ys = mnist.train.next_batch(batch_size)
       feed_dict = {X: batch_xs, Y: batch_ys}
       c, _, = sess.run([cost, optimizer], feed_dict=feed_dict)
       avg_cost += c / total_batch
   print('Epoch:', '%04d' % (epoch + 1), 'cost =', '{:.9f}'.format(avg_cost))

print('Learning Finished!')

# Test model and check accuracy
correct_prediction = tf.equal(tf.argmax(hypothesis, 1), tf.argmax(Y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
print('Accuracy:', sess.run(accuracy, feed_dict={X: mnist.test.images, Y: mnist.test.labels}))

Example #46
0
                               img_size,
                               classes,
                               validation_size=validation_size)

print("Complete reading input data. Will Now print a snippet of it")
print("Number of files in Training-set:\t\t{}".format(len(data.train.labels)))
print("Number of files in Validation-set:\t{}".format(len(data.valid.labels)))

session = tf.Session()
x = tf.placeholder(tf.float32,
                   shape=[None, img_size, img_size, num_channels],
                   name='x')

## labels
y_true = tf.placeholder(tf.float32, shape=[None, num_classes], name='y_true')
y_true_cls = tf.argmax(y_true, dimension=1)

##Network graph params
filter_size_conv1 = 3
num_filters_conv1 = 32

filter_size_conv2 = 3
num_filters_conv2 = 32

filter_size_conv3 = 3
num_filters_conv3 = 64

fc_layer_size = 128


def create_weights(shape):
Example #47
0
def model_eval(sess,
               x,
               y,
               predictions,
               X_test=None,
               Y_test=None,
               feed=None,
               args=None):
    """
  Compute the accuracy of a TF model on some data
  :param sess: TF session to use
  :param x: input placeholder
  :param y: output placeholder (for labels)
  :param predictions: model output predictions
  :param X_test: numpy array with training inputs
  :param Y_test: numpy array with training outputs
  :param feed: An optional dictionary that is appended to the feeding
           dictionary before the session runs. Can be used to feed
           the learning phase of a Keras model for instance.
  :param args: dict or argparse `Namespace` object.
               Should contain `batch_size`
  :return: a float with the accuracy value
  """
    global _model_eval_cache
    args = _ArgsWrapper(args or {})

    assert args.batch_size, "Batch size was not given in args dict"
    if X_test is None or Y_test is None:
        raise ValueError("X_test argument and Y_test argument "
                         "must be supplied.")

    # Define accuracy symbolically
    key = (y, predictions)
    if key in _model_eval_cache:
        correct_preds = _model_eval_cache[key]
    else:
        correct_preds = tf.equal(tf.argmax(y, axis=-1),
                                 tf.argmax(predictions, axis=-1))
        _model_eval_cache[key] = correct_preds

    # Init result var
    accuracy = 0.0

    with sess.as_default():
        # Compute number of batches
        nb_batches = int(math.ceil(float(len(X_test)) / args.batch_size))
        assert nb_batches * args.batch_size >= len(X_test)

        X_cur = np.zeros((args.batch_size, ) + X_test.shape[1:],
                         dtype=X_test.dtype)
        Y_cur = np.zeros((args.batch_size, ) + Y_test.shape[1:],
                         dtype=Y_test.dtype)
        for batch in range(nb_batches):
            if batch % 100 == 0 and batch > 0:
                _logger.debug("Batch " + str(batch))

            # Must not use the `batch_indices` function here, because it
            # repeats some examples.
            # It's acceptable to repeat during training, but not eval.
            start = batch * args.batch_size
            end = min(len(X_test), start + args.batch_size)

            # The last batch may be smaller than all others. This should not
            # affect the accuarcy disproportionately.
            cur_batch_size = end - start
            X_cur[:cur_batch_size] = X_test[start:end]
            Y_cur[:cur_batch_size] = Y_test[start:end]
            feed_dict = {x: X_cur, y: Y_cur}
            if feed is not None:
                feed_dict.update(feed)
            cur_corr_preds = correct_preds.eval(feed_dict=feed_dict)

            accuracy += cur_corr_preds[:cur_batch_size].sum()

        assert end >= len(X_test)

        # Divide by number of examples to get final value
        accuracy /= len(X_test)

    return accuracy
Example #48
0
def contrastive_loss(hidden,
                     num_replicas,
                     normalize_hidden,
                     temperature,
                     model,
                     weight_decay):
  """Computes contrastive loss.

  Args:
    hidden: embedding of video clips after projection head.
    num_replicas: number of distributed replicas.
    normalize_hidden: whether or not to l2 normalize the hidden vector.
    temperature: temperature in the InfoNCE contrastive loss.
    model: keras model for calculating weight decay.
    weight_decay: weight decay parameter.

  Returns:
    A loss scalar.
    The logits for contrastive prediction task.
    The labels for contrastive prediction task.
  """
  large_num = 1e9

  hidden1, hidden2 = tf.split(hidden, num_or_size_splits=2, axis=0)
  if normalize_hidden:
    hidden1 = tf.math.l2_normalize(hidden1, -1)
    hidden2 = tf.math.l2_normalize(hidden2, -1)
  batch_size = tf.shape(hidden1)[0]

  if num_replicas == 1:
    # This is the local version
    hidden1_large = hidden1
    hidden2_large = hidden2
    labels = tf.one_hot(tf.range(batch_size), batch_size * 2)
    masks = tf.one_hot(tf.range(batch_size), batch_size)

  else:
    # This is the cross-tpu version.
    hidden1_large = tpu_cross_replica_concat(hidden1, num_replicas)
    hidden2_large = tpu_cross_replica_concat(hidden2, num_replicas)
    enlarged_batch_size = tf.shape(hidden1_large)[0]
    replica_id = tf.cast(tf.cast(xla.replica_id(), tf.uint32), tf.int32)
    labels_idx = tf.range(batch_size) + replica_id * batch_size
    labels = tf.one_hot(labels_idx, enlarged_batch_size * 2)
    masks = tf.one_hot(labels_idx, enlarged_batch_size)

  logits_aa = tf.matmul(hidden1, hidden1_large, transpose_b=True) / temperature
  logits_aa = logits_aa - tf.cast(masks, logits_aa.dtype) * large_num
  logits_bb = tf.matmul(hidden2, hidden2_large, transpose_b=True) / temperature
  logits_bb = logits_bb - tf.cast(masks, logits_bb.dtype) * large_num
  logits_ab = tf.matmul(hidden1, hidden2_large, transpose_b=True) / temperature
  logits_ba = tf.matmul(hidden2, hidden1_large, transpose_b=True) / temperature

  loss_a = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
      labels, tf.concat([logits_ab, logits_aa], 1)))
  loss_b = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(
      labels, tf.concat([logits_ba, logits_bb], 1)))
  loss = loss_a + loss_b

  l2_loss = weight_decay * tf.add_n([
      tf.nn.l2_loss(v)
      for v in model.trainable_variables
      if 'kernel' in v.name
    ])

  total_loss = loss +  tf.cast(l2_loss, loss.dtype)

  contrast_prob = tf.nn.softmax(logits_ab)
  contrast_entropy = - tf.reduce_mean(
      tf.reduce_sum(contrast_prob * tf.math.log(contrast_prob + 1e-8), -1))

  contrast_acc = tf.equal(tf.argmax(labels, 1), tf.argmax(logits_ab, axis=1))
  contrast_acc = tf.reduce_mean(tf.cast(contrast_acc, tf.float32))

  return {
      'total_loss': total_loss,
      'contrastive_loss': loss,
      'reg_loss': l2_loss,
      'contrast_acc': contrast_acc,
      'contrast_entropy': contrast_entropy,
  }
Example #49
0
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets('MNIST_data', one_hot=True)

# input and target output
x1 = tf.placeholder(tf.float32,shape=[None,784])
x2 = tf.placeholder(tf.float32,shape=[None,784])
x= tf.concat(1,[x1,x2])
#print x.get_shape()

y1_ = tf.placeholder(tf.float32,shape=[None,10])
y2_ = tf.placeholder(tf.float32,shape=[None,10])
y_ = tf.concat(1,[y1_,y2_])

# get the one-hot vectors of the sum
y1m = tf.argmax(y1_,1)
y2m = tf.argmax(y2_,1)
ym = tf.add(y1m,y2m)
y_ = tf.one_hot(ym,19)

def weight_variable(shape):
    initial = tf.truncated_normal(shape, stddev=0.1)
    return tf.Variable(initial)

def bias_variable(shape):
    initial = tf.constant(0.1, shape=shape)
    return tf.Variable(initial)

def conv2d(x,W):
    return tf.nn.conv2d(x,W,strides=[1,1,1,1],padding='SAME')
    def construct_network(self, frame_feat_input, batch_labels, weights, reuse, is_training):
        """
        :param frame_feat_input:[self.train_batch_size, SHOT_NUM, SHOT_FEAT_LENGTH]
        :param batch_labels:
        :param weights:
        :param reuse:
        :param is_training:
        :return:
        """
        with tf.variable_scope('video_level', reuse=reuse) as sc:
            nets = tf.layers.conv1d(frame_feat_input, 1024, kernel_size=3, name='conv_1')
            nets = slim.batch_norm(nets,
                                         decay=0.9997,
                                         epsilon=0.001,
                                         is_training=is_training)

            nets = tf.nn.relu(nets)
            nets = tf.layers.max_pooling1d(nets, pool_size=2, strides=2, name='pool1d_1')
            nets = tf.layers.conv1d(nets, filters=256, kernel_size=3, name='conv1d_2')
            nets = slim.batch_norm(nets,
                                         decay=0.9997,
                                         epsilon=0.001,
                                         is_training=is_training)
            nets = tf.nn.relu(nets)
            # layer 3
            nets = tf.layers.conv1d(nets, filters=256, kernel_size=3, name='conv1d_3')
            nets = slim.batch_norm(nets,
                                         decay=0.9997,
                                         epsilon=0.001,
                                         is_training=is_training)
            nets = tf.nn.relu(nets)
            # test flat
            nets = tf.layers.flatten(nets)
            fc_frame = tf.layers.dense(nets, 512, name='fc1')
            video_vector = tf.layers.dropout(fc_frame, FLAGS.dropout_rate, training=is_training)
            video_vector = tf.nn.relu(video_vector)
            video_vector = tf.layers.dense(video_vector, 512, name='dense_layer_1')
            video_vector = tf.layers.dropout(video_vector, FLAGS.dropout_rate, training=is_training)
            video_vector = tf.nn.relu(video_vector)
            video_vector = tf.layers.dense(video_vector, 1024, name='dense_layer_2')
            video_vector = tf.nn.relu(video_vector)
            logits = tf.layers.dense(video_vector, 2, name='dense_layer_3')
            predict_confidence = tf.nn.softmax(logits, name='confidence')  # [batch_size, 2]

        with tf.name_scope('loss'):
            cost = tf.reduce_mean(
                tf.losses.sparse_softmax_cross_entropy(logits=logits, labels=batch_labels,
                                                       weights=weights))

        L2_frame = 0
        for w in tl.layers.get_variables_with_name('video_level', True, True):
            L2_frame += tf.contrib.layers.l2_regularizer(1.0)(w)

        loss = cost + 0.001 * L2_frame
        with tf.name_scope('accuracy'):
            predict_index = tf.argmax(logits, 1)
            predicts = tf.equal(predict_index, batch_labels)
            accuracy = tf.reduce_mean(tf.cast(predicts, np.float32))
            tf.summary.scalar('accuracy', accuracy)

        end_point = {'L2_frame': L2_frame, 'loss': loss, 'cost': cost, 'accuracy': accuracy,
                     'logits': predict_confidence, 'predict': predict_index}
        return end_point
Example #51
0
# Loss function 
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels = y_true, logits = y_pred))

# Optimizer
optimizer = tf.train.AdamOptimizer(learning_rate = 0.001)
train = optimizer.minimize(cross_entropy)

init = tf.global_variables_initializer()

steps = 5000

with tf.Session() as sess:
	sess.run(init)

	for i in range(steps):
		batch_x, batch_y = mnist.train.next_batch(50)
		sess.run(train, feed_dict = {x:batch_x, y_true: batch_y, hold_prob:0.5})

		if i%100 == 0:
			print("ON Step: {}".format(i))
			print "Accuracy: "
			matches = tf.equal(tf.argmax(y_pred, 1), tf.argmax(y_true, 1))

			acc = tf.reduce_mean(tf.cast(matches, tf.float32))
			print(sess.run(acc, feed_dict = {x:mnist.test.images, y_true:mnist.test.labels, hold_prob: 1.0}))
			print '\n'



Example #52
0
    def _build(self,
               inputs,
               sequence_length=None,
               segment_ids=None,
               mode=None,
               **kwargs):
        """Feeds the inputs through the network and makes classification.

        The arguments are the same as in
        :class:`~texar.tf.modules.BertEncoder`.

        Args:
            inputs: A 2D Tensor of shape `[batch_size, max_time]`,
                containing the token ids of tokens in input sequences.
            sequence_length (optional): A 1D Tensor of shape `[batch_size]`.
                Input tokens beyond respective sequence lengths are masked
                out automatically.
            segment_ids (optional): A 2D Tensor of shape
                `[batch_size, max_time]`, containing the segment ids
                of tokens in input sequences. If `None` (default), a tensor
                with all elements set to zero is used.
            mode (optional): A tensor taking value in
                :tf_main:`tf.estimator.ModeKeys <estimator/ModeKeys>`,
                including `TRAIN`, `EVAL`, and `PREDICT`. Used to toggle
                dropout.
                If `None` (default), :func:`texar.tf.global_mode` is used.
            **kwargs: Keyword arguments.

        Returns:
            A tuple `(logits, pred)`, containing the logits over classes and
            the predictions, respectively.

            - If "clas_strategy"=="cls_time" or "all_time"

                - If "num_classes"==1, `logits` and `pred` are of both \
                shape `[batch_size]`
                - If "num_classes">1, `logits` is of shape \
                `[batch_size, num_classes]` and `pred` is of shape \
                `[batch_size]`.

            - If "clas_strategy"=="time_wise",

                - If "num_classes"==1, `logits` and `pred` are of both \
                shape `[batch_size, max_time]`
                - If "num_classes">1, `logits` is of shape \
                `[batch_size, max_time, num_classes]` and `pred` is of shape \
                `[batch_size, max_time]`.
        """

        enc_outputs, pooled_output = self._encoder(inputs, sequence_length,
                                                   segment_ids, mode)

        # Compute logits
        stra = self._hparams.clas_strategy
        if stra == 'time_wise':
            logits = enc_outputs
        elif stra == 'cls_time':
            logits = pooled_output
        elif stra == 'all_time':
            # Pad `enc_outputs` to have max_seq_length before flatten
            length_diff = self._hparams.max_seq_length - tf.shape(inputs)[1]
            length_diff = tf.reshape(length_diff, [1, 1])
            # Set `paddings = [[0, 0], [0, length_dif], [0, 0]]`
            paddings = tf.pad(length_diff, paddings=[[1, 1], [1, 0]])
            logit_input = tf.pad(enc_outputs, paddings=paddings)
            logit_input_dim = self._hparams.hidden_size * \
                              self._hparams.max_seq_length
            logits = tf.reshape(logit_input, [-1, logit_input_dim])
        else:
            raise ValueError(
                'Unknown classification strategy: {}'.format(stra))

        if self._logit_layer is not None:
            logits = self._dropout_layer(logits, training=mode)
            logits = self._logit_layer(logits)

        # Compute predications
        num_classes = self._hparams.num_classes
        is_binary = num_classes == 1
        is_binary = is_binary or (num_classes <= 0 and logits.shape[-1] == 1)

        if stra == 'time_wise':
            if is_binary:
                pred = tf.squeeze(tf.greater(logits, 0), -1)
                logits = tf.squeeze(logits, -1)
            else:
                pred = tf.argmax(logits, axis=-1)
        else:
            if is_binary:
                pred = tf.greater(logits, 0)
                logits = tf.reshape(logits, [-1])
            else:
                pred = tf.argmax(logits, axis=-1)
            pred = tf.reshape(pred, [-1])
        pred = tf.cast(pred, tf.int64)

        if not self._built:
            self._add_internal_trainable_variables()
            if self._logit_layer:
                self._add_trainable_variable(
                    self._logit_layer.trainable_variables)
            self._built = True

        return logits, pred
Example #53
0
out_flat = tf.nn.relu(out_flat)

out_final = dense_layer(out_flat,
                        shape_w=[1024, n_class],
                        shape_b=[n_class],
                        name='last')
print("out final: ", out_final.get_shape())

pred = tf.nn.softmax(out_final)
err1 = tf.nn.softmax_cross_entropy_with_logits_v2(labels=target,
                                                  logits=out_final)
err = tf.reduce_mean(err1) + tf.losses.get_regularization_loss()
err_val = tf.reduce_mean(err1)
print("error: ", err1.get_shape(), err.get_shape())

correct_pred = tf.cast(tf.equal(tf.argmax(pred, 1), tf.argmax(target, 1)),
                       tf.float32)
accuracy = tf.reduce_mean(correct_pred)
global_step = tf.train.get_or_create_global_step()
train_op = tf.train.AdamOptimizer(learning_rate=lr).minimize(
    err, global_step=global_step)

saver = tf.train.Saver(max_to_keep=10)
init = tf.global_variables_initializer()

with tf.Session() as sess:
    ckpt = tf.train.get_checkpoint_state(MDPATH)
    if ckpt and ckpt.model_checkpoint_path:
        print(ckpt, ckpt.model_checkpoint_path)
        sess.run(init)
        optimistic_restore(sess, MDPATH)
def dice_score_multiclass(predicted_labels, labels, num_classes, type_unet):

    #### Dice Score for at least 3 classes #####

    ### predicted_labels -- shape (num_batch, height, width, depth, num_classes)
    ### labels -- shape (num_batch, height, width, depth, num_classes)

    print('shape of predicted labels')
    print(predicted_labels)
    print('shape of actual labels')
    print(labels)

    shape_of_data = labels.get_shape().as_list()
    if type_unet == '3D':

        indices_predictions = tf.argmax(tf.reshape(predicted_labels,
                                                   [-1, shape_of_data[4]]),
                                        axis=-1)
        indices_predictions = tf.reshape(
            indices_predictions,
            [-1, shape_of_data[1] * shape_of_data[2] * shape_of_data[3] * 1])

        indices_labels = tf.argmax(tf.reshape(labels, [-1, shape_of_data[4]]),
                                   axis=-1)
        indices_labels = tf.reshape(
            indices_labels,
            [-1, shape_of_data[1] * shape_of_data[2] * shape_of_data[3] * 1])
    else:

        indices_predictions = tf.argmax(tf.reshape(predicted_labels,
                                                   [-1, shape_of_data[3]]),
                                        axis=-1)
        indices_predictions = tf.reshape(
            indices_predictions, [-1, shape_of_data[1] * shape_of_data[2] * 1])

        indices_labels = tf.argmax(tf.reshape(labels, [-1, shape_of_data[3]]),
                                   axis=-1)
        indices_labels = tf.reshape(
            indices_labels, [-1, shape_of_data[1] * shape_of_data[2] * 1])

    print('after transformation')
    print(indices_predictions)
    print(indices_labels)

    dice_score = defaultdict()
    for _ in range(num_classes):

        shared_bool = tf.logical_and(
            tf.equal(
                tf.cast(indices_predictions, tf.float32),
                tf.ones_like(indices_predictions, dtype=tf.float32) *
                tf.cast(_, tf.float32)),
            tf.equal(
                tf.cast(indices_labels, tf.float32),
                tf.ones_like(indices_predictions, dtype=tf.float32) *
                tf.cast(_, tf.float32)))
        area_shared = tf.reduce_sum(tf.cast(shared_bool, tf.float32), 1)

        predictions_bool = tf.equal(
            tf.cast(indices_predictions, tf.float32),
            tf.ones_like(indices_predictions, dtype=tf.float32) *
            tf.cast(_, tf.float32))
        area_predictions = tf.reduce_sum(tf.cast(predictions_bool, tf.float32),
                                         1)

        labels_bool = tf.equal(
            tf.cast(indices_labels, tf.float32),
            tf.ones_like(indices_predictions, dtype=tf.float32) *
            tf.cast(_, tf.float32))
        area_labels = tf.reduce_sum(tf.cast(labels_bool, tf.float32), 1)

        dice_score[_] = tf.reduce_mean((2.0 * area_shared + 1e-6) /
                                       (area_predictions + area_labels + 1e-6))

    return dice_score
Example #55
0
Ylogits = tf.matmul(Y4, W5) + B5

Y = tf.nn.softmax(Ylogits)


# 3. Define the loss function  

cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=Ylogits, labels=Y_)
cross_entropy = tf.reduce_mean(cross_entropy)*100

# 4. Define the accuracy 
sess = tf.InteractiveSession()
tf.global_variables_initializer().run()

correct_prediction = tf.equal(tf.argmax(Y,1), tf.argmax(Y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

print(sess.run(accuracy, feed_dict={X: fashion_mnist.test.images, Y_: fashion_mnist.test.labels}))

# 5. Define an optimizer
#train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)

train_step = tf.train.AdamOptimizer(0.003).minimize(cross_entropy)

    
# initialize
init = tf.initialize_all_variables()
sess = tf.Session()
sess.run(init)
Example #56
0
 def receive_output(self, likelihoods: tf.Tensor) -> tf.Operation:
     with tf.name_scope('post-processing'):
         prediction = tf.argmax(likelihoods, axis=1)
         op = tf.Print([], [prediction], summarize=self.BATCH_SIZE, message="ACTUAL ")
         return op
Example #57
0
                          layers.ReLU(),
                          layers.Conv2D(16,kernel_size=3,strides=1),
                          layers.MaxPooling2D(pool_size=2,strides=2),
                          layers.ReLU(),
                          layers.Flatten(),
                          layers.Dense(120,activation='relu'),
                          layers.Dense(84,activation='relu'),
                          layers.Dense(10)
                          ])
    network.build(input_shape=[None,28,28,1])
    network.summary()
    #模型训练
    optimizer = optimizers.SGD(lr=0.0001)
    acc_meter = metrics.Accuracy()

    for step,(x,y) in enumerate(train_dataset):
        with tf.GradientTape() as tape:
            x = tf.reshape(x,(-1,28,28,1))
            y_onehot = tf.one_hot(y,depth=10)
            out= network(x)
            loss = tf.square(y_onehot-out)
            loss = tf.reduce_sum(loss)/32
            grads = tape.gradient(loss,network.trainable_variables)
            optimizer.apply_gradients(zip(grads,network.trainable_variables))
            acc_meter.update_state(tf.argmax(out,axis=1),y)

        if step%20 == 0:
            print('step: ',step,'acc_meter:',acc_meter.result().numpy(),'loss:',float(loss))
            acc_meter.reset_states()

Example #58
0
def evaluate_pictures(n_epochs=200,batch_size=10,dataset='E:/bishe/cifar-100-python'):


    datasets = load_data(dataset)
    train_set_x, train_set_y = datasets[0]
    valid_set_x, valid_set_y = datasets[1]
    test_set_x, test_set_y = datasets[2]

    # 计算各数据集的batch个数
    n_train_batches = train_set_x.shape[0]
    n_train_batches = int(n_train_batches / batch_size)
    print("... building the model")

    x = tf.placeholder(tf.float32, shape=[None, 3072])
    y = tf.placeholder(tf.float32, shape=[None, 100])
    keep_prob = tf.placeholder(tf.float32)

    x_image = tf.transpose(tf.reshape(x, [-1, 3, 32, 32]),perm=[0,2,3,1])##
    W_conv1 = weight_variable([5, 5, 3, 64])
    b_conv1 = bias_variable([64])
    h_pool1 = max_pool_2x2(tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1))

    input1=tf.nn.local_response_normalization(h_pool1)
    W_conv2 = weight_variable([5, 5, 64, 128])
    b_conv2 = bias_variable([128])
    h_pool2 = mean_pool_2x2(tf.nn.relu(conv2d(input1, W_conv2) + b_conv2))

    input2 = tf.nn.local_response_normalization(h_pool2)
    W_conv3 = weight_variable([5, 5, 128, 500])
    b_conv3 = bias_variable([500])
    h_pool3 = mean_pool_2x2(tf.nn.relu(conv2d(input2, W_conv3) + b_conv3))

    input3 = tf.nn.local_response_normalization(h_pool3)
    h_fc1_drop = tf.nn.dropout(input3, keep_prob)
    W_fc1 = weight_variable([4 * 4 * 500, 100])
    b_fc1 = bias_variable([100])
    y_conv = tf.matmul(tf.reshape(h_fc1_drop, [-1, 4 * 4 * 500]), W_fc1) + b_fc1

    cross_entropy = tf.reduce_mean(
        tf.nn.softmax_cross_entropy_with_logits(labels=y, logits=y_conv))
    train_step = tf.train.AdadeltaOptimizer(0.1).minimize(cross_entropy)
    correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y, 1))
    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
    tf.summary.scalar('accuracy', accuracy)

    tf.summary.scalar('accuracy', accuracy)
    tf.summary.histogram('h_pool1', y_conv)
    sess=tf.InteractiveSession()
    sess.run(tf.global_variables_initializer())

    best_validation_acc = np.inf
    epoch = 0
    done_looping = False

    print("... training")
    summary_op=tf.summary.merge_all()
    summary_writer=tf.summary.FileWriter(FLAGS.summaries_dir,graph_def=sess.graph_def)
    while (epoch < n_epochs) and (not done_looping):
        epoch = epoch + 1
        for minibatch_index in range(n_train_batches):
            iter = (epoch - 1) * n_train_batches + minibatch_index
            _,acc=sess.run([train_step, accuracy],feed_dict={x: train_set_x[minibatch_index * batch_size: (minibatch_index + 1) * batch_size],
                y: train_set_y[minibatch_index * batch_size: (minibatch_index + 1) * batch_size], keep_prob: 0.2})
            print('epoch %i, step %d,minibatch %i/%i, train acc %g' %
                  (epoch, iter, minibatch_index + 1, n_train_batches,acc))
            if (iter + 1) % 1000 == 0:
                # compute zero-one loss on validation set
                validation_acc = accuracy.eval(feed_dict={x: valid_set_x, y: valid_set_y, keep_prob: 0.2})
                print('                         validation acc %g' %(validation_acc ))
                # test it on the test set
                summary_str, acc = sess.run([summary_op, accuracy],
                                            feed_dict={x: test_set_x, y: test_set_y, keep_prob: 0.2})
                summary_writer.add_summary(summary_str, iter)
                print('                         test acc %g' % (acc))
                # if we got the best validation score until now
                if validation_acc > best_validation_acc:
                    # save best validation score and iteration number
                    best_validation_acc = validation_acc
                    save_params(W_conv1, b_conv1, W_conv2, b_conv2,W_fc1,b_fc1)  # 保存参数


    print('Optimization complete.')
    print("test accuracy %g" % accuracy.eval(feed_dict={
        x: test_set_x, y: test_set_y, keep_prob: 1.0}))
Example #59
0
    if i % 100 == 0:
        # Save the variables to disk
        save_path = saver.save(sess, model_path + ckpt_file)
        print("Model saved in file: %s" % save_path)

    # Train step
    train_step.run(feed_dict={x: batch[0], y_true: batch[1]})

save_path = saver.save(sess, model_path + ckpt_file)
print("Model saved in file: %s" % save_path)

# Get data reader
data_reader = DataReader(test_dataset_dir, batch_size=1, file_names=True,
                         resize_to=(224, 224))

for i in range(30):
    # Get next batch of images and labels
    batch = data_reader.next()

    prediction = sess.run(y, feed_dict={x: batch[0], y_true: batch[1]})
    prediction = tf.argmax(prediction, 1).eval()
    print("Prediction for %s is %s (0 - fin, 1 - no fin):"
          % (batch[2], prediction))

    # Get and print accuracy
    test_accuracy = accuracy.eval(
        feed_dict={x: batch[0], y_true: batch[1]}
    )
    print("Testing step %d, testing accuracy %g" % (i, test_accuracy))
Example #60
0
        def model(X_train, Y_train, X_test, Y_test, learning_rate=0.009,
                  num_epochs=100, minibatch_size=64, print_cost=True):
            """
            Implements a three-layer ConvNet in Tensorflow:
            CONV2D -> RELU -> MAXPOOL -> CONV2D -> RELU -> MAXPOOL -> FLATTEN -> FULLYCONNECTED

            Arguments:
            X_train -- training set, of shape (None, 240, 240, 3)
            Y_train -- test set, of shape (None, n_y = 4)
            X_test -- training set, of shape (None, 240, 240, 3)
            Y_test -- test set, of shape (None, n_y = 4)
            learning_rate -- learning rate of the optimization
            num_epochs -- number of epochs of the optimization loop
            minibatch_size -- size of a minibatch
            print_cost -- True to print the cost every 100 epochs

            Returns:
            train_accuracy -- real number, accuracy on the train set (X_train)
            test_accuracy -- real number, testing accuracy on the test set (X_test)
            parameters -- parameters learnt by the model. They can then be used to predict.
            """

            ops.reset_default_graph()  # to be able to rerun the model without overwriting tf variables
            tf.set_random_seed(1)  # to keep results consistent (tensorflow seed)
            seed = 3  # to keep results consistent (numpy seed)
            (m, n_H0, n_W0, n_C0) = X_train.shape
            n_y = Y_train.shape[1]
            costs = []  # To keep track of the cost

            # Create Placeholders of the correct shape
            ### START CODE HERE ### (1 line)
            X, Y = create_placeholders(n_H0, n_W0, n_C0, n_y)
            ### END CODE HERE ###

            # Initialize parameters
            ### START CODE HERE ### (1 line)
            parameters = initialize_parameters()

            # W1 = tf.get_variable("W1", [m, n_H0, n_W0, n_C0], initializer =  tf.contrib.layers.xavier_initializer(seed = 0))
            # W2 = tf.get_variable("W2", [m, n_H0, n_W0, n_C0], initializer =  tf.contrib.layers.xavier_initializer(seed = 0))
            # parameters = {"W1": W1,
            #              "W2": W2}
            ### END CODE HERE ###

            # Forward propagation: Build the forward propagation in the tensorflow graph
            ### START CODE HERE ### (1 line)
            Z3 = forward_propagation(X, parameters)
            ### END CODE HERE ###

            # Cost function: Add cost function to tensorflow graph
            ### START CODE HERE ### (1 line)
            cost = compute_cost(Z3, Y)
            ### END CODE HERE ###

            # Backpropagation: Define the tensorflow optimizer. Use an AdamOptimizer that minimizes the cost.
            ### START CODE HERE ### (1 line)
            optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost)
            ### END CODE HERE ###

            # Initialize all the variables globally
            init = tf.global_variables_initializer()

            # Start the session to compute the tensorflow graph
            with tf.Session() as sess:

                # Run the initialization
                sess.run(init)

                # Do the training loop
                for epoch in range(num_epochs):

                    minibatch_cost = 0.
                    num_minibatches = int(
                        m / minibatch_size)  # number of minibatches of size minibatch_size in the train set
                    seed = seed + 1
                    minibatches = random_mini_batches(X_train, Y_train, minibatch_size, seed)

                    for minibatch in minibatches:
                        # Select a minibatch
                        (minibatch_X, minibatch_Y) = minibatch
                        """
                        # IMPORTANT: The line that runs the graph on a minibatch.
                        # Run the session to execute the optimizer and the cost.
                        # The feedict should contain a minibatch for (X,Y).
                        """
                        ### START CODE HERE ### (1 line)
                        _, temp_cost = sess.run(fetches=[optimizer, cost],
                                                feed_dict={X: minibatch_X, Y: minibatch_Y}
                                                )
                        ### END CODE HERE ###

                        minibatch_cost += temp_cost / num_minibatches

                    # Print the cost every epoch
                    if print_cost == True and epoch % 5 == 0:
                        print("Cost after epoch %i: %f" % (epoch, minibatch_cost))
                    if print_cost == True and epoch % 1 == 0:
                        costs.append(minibatch_cost)

                # plot the cost
                plt.plot(np.squeeze(costs))
                plt.ylabel('cost')
                plt.xlabel('iterations (per tens)')
                plt.title("Learning rate =" + str(learning_rate))
                plt.show()

                # Calculate the correct predictions
                predict_op = tf.argmax(Z3, 1)
                correct_prediction = tf.equal(predict_op, tf.argmax(Y, 1))

                # Calculate accuracy on the test set
                accuracy = tf.reduce_mean(tf.cast(correct_prediction, "float"))
                print(accuracy)
                train_accuracy = accuracy.eval({X: X_train, Y: Y_train})
                test_accuracy = accuracy.eval({X: X_test, Y: Y_test})
                print("Train Accuracy:", train_accuracy)
                print("Test Accuracy:", test_accuracy)

                return train_accuracy, test_accuracy, parameters