예제 #1
0
# neural network with 5 layers
#
# · · · · · · · · · ·       (input data, flattened pixels)       X [batch, 784]   # 784 = 28*28
# \x/x\x/x\x/x\x/x\x/ ✞  -- fully connected layer (relu+dropout) W1 [784, 200]      B1[200]
#  · · · · · · · · ·                                             Y1 [batch, 200]
#   \x/x\x/x\x/x\x/ ✞    -- fully connected layer (relu+dropout) W2 [200, 100]      B2[100]
#    · · · · · · ·                                               Y2 [batch, 100]
#    \x/x\x/x\x/ ✞       -- fully connected layer (relu+dropout) W3 [100, 60]       B3[60]
#     · · · · ·                                                  Y3 [batch, 60]
#     \x/x\x/ ✞          -- fully connected layer (relu+dropout) W4 [60, 30]        B4[30]
#      · · ·                                                     Y4 [batch, 30]
#      \x/               -- fully connected layer (softmax)      W5 [30, 10]        B5[10]
#       ·                                                        Y5 [batch, 10]

# Download images and labels
mnist = mnist_data.read_data_sets("data")

# input X: 28x28 grayscale images, the first dimension (None) will index the images in the mini-batch
X = tf.placeholder(tf.float32, [None, 28, 28, 1])
# correct answers will go here
Y_ = tf.placeholder(tf.float32, [None, 10])
# variable learning rate
lr = tf.placeholder(tf.float32)
# Probability of keeping a node during dropout = 1.0 at test time (no dropout) and 0.75 at training time
pkeep = tf.placeholder(tf.float32)

# five layers and their number of neurons (tha last layer has 10 softmax neurons)
L = 200
M = 100
N = 60
O = 30
 def LoadDataSet(self):
     self.mnist  = mnist_data.read_data_sets("mydata")
from triplet_clustering import TripletClustering
from clustering_util import run_loss_aug_clustering_on, evaluate_clustering, run_clustering_on
import mnist_data
import tensorflow as tf
import numpy as np
from datetime import datetime
from random import randrange

train_data = mnist_data.read_data_sets('./data/', one_hot=False)
test_im = train_data.test.images
test_l = train_data.test.labels


# HOS: shuffle but move every element. Creates copy.
def sattoloShuffle(x):
    items = np.copy(x)
    i = len(items)
    while i > 1:
        i = i - 1
        j = randrange(i)  # 0 <= j <= i-1
        items[j], items[i] = items[i], items[j]
    return items


def get_batch():
    im, l = train_data.train.next_batch(256)
    return im.astype(np.float32), l


def get_triplets(im, l):
    pos_idx = np.zeros((256, ), dtype=int)
import mnist_data
import tensorflow as tf

# based on tutorial at http://www.tensorflow.org/tutorials/mnist/pros/index.html

mnist = mnist_data.read_data_sets('MNIST_data', one_hot=True)


def weight_variable(shape):
    initial = tf.truncated_normal(shape, stddev=0.1)
    return tf.Variable(initial)


def bias_variable(shape):
    initial = tf.constant(0.1, shape=shape)
    return tf.Variable(initial)


def conv2d(x, W):
    return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')


# Helper method for maximum pooling. strides 2x2, 75% reduction in image size per pooling operation
def max_pool_2x2(x):
    return tf.nn.max_pool(x,
                          ksize=[1, 2, 2, 1],
                          strides=[1, 2, 2, 1],
                          padding='SAME')


x = tf.placeholder("float", shape=[None, 784],
예제 #5
0
        # predict test data
        predict(sess, x, model.keep_prob, model.pred, data.test.images, args.output)
        
        
        # origiinal test data from 'http://yann.lecun.com/exdb/mnist/'
        """
        acc = sess.run(model.accuracy, feed_dict={x: data.test.images, y: data.test.labels, model.keep_prob: 1.0})
        print("test accuracy %g"%acc)
        """

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument('-d', '--data_dir', type=str, default='mnist_data/', help='Directory for storing input data')
    parser.add_argument('-c', '--ckpt_dir', type=str, default='ckpts/', help='Directory for parameter checkpoints')
    parser.add_argument('-l', '--load_params', dest='load_params', action='store_true', help='Restore training from previous model checkpoint?')
    parser.add_argument("-o", "--output",  type=str, default='prediction.csv', help='Prediction filepath')
    parser.add_argument('-e', '--epochs', type=int, default=30, help='How many epochs to run in total?')
    parser.add_argument('-b', '--batch_size', type=int, default=50, help='Batch size during training per GPU')
    parser.add_argument('-v', '--val_size', type=int, default=5000)
    args = parser.parse_args()
  
    # pretty print args
    print('input args:\n', json.dumps(vars(args), indent=4, separators=(',',':'))) 

    data = mnist_data.read_data_sets(args.data_dir, one_hot=True, reshape=False, validation_size=args.val_size)
    #data = input_data.read_data_sets(args.data_dir, one_hot=True, reshape=False, validation_size=args.val_size)
    
    if not os.path.exists(args.ckpt_dir):
        os.makedirs(args.ckpt_dir)

    train(args, data)
예제 #6
0
def train(hold_out, dev_name, sample, go_on, semi):
    hold_out_s = int(hold_out)
    print hold_out_s
    if go_on:
        c = pickle.load(open('chosen_ball.bn'))
        ch = c['all']
        train_data = mnist_data.read_data_sets('./data/',
                                               one_hot=True,
                                               hold_out_size=hold_out_s,
                                               active=True,
                                               choices=ch)
    else:
        train_data = mnist_data.read_data_sets('./data/',
                                               one_hot=True,
                                               hold_out_size=hold_out_s)

    test_im = train_data.test.images[0:1000]
    test_l = train_data.test.labels[0:1000]

    params, str_v = read_params('settings.json')
    print train_data.train.images.shape, train_data.validation.images.shape
    config = tf.ConfigProto()
    config.gpu_options.allow_growth = True
    config.allow_soft_placement = True
    with tf.Session(config=config) as sesh:
        robust_cifar_train = AdverserialTrainer(params.learning_rate,
                                                0.1 * params.learning_rate,
                                                dev_name, 9)
        print 'Initial Variable List:'
        print[tt.name for tt in tf.trainable_variables()]

        init = tf.global_variables_initializer()
        sesh.run(init)
        saver = tf.train.Saver(max_to_keep=100)
        sum_writer = tf.summary.FileWriter(
            "./dumps_adverserial__semi_{}_go_on_{}__sample_{}__hold_out_{}__{}/"
            .format(semi, go_on, sample, hold_out_s, str_v), sesh.graph)

        im, l = train_data.train.next_batch(
            params.batch_size)  # Sample a boosted set
        for batch_id in range(200000):
            im, l = train_data.train.next_batch(
                params.batch_size)  # Sample a boosted set
            adv_im, _ = train_data.validation.next_batch(params.batch_size)

            flip_f = float(batch_id) / 200000.0

            robust_cifar_train.assign_batch({
                'images': im,
                'labels': l
            }, {'images': adv_im})
            # first epoch only observe then do staff

            create_summ = batch_id % 10 == 0
            if semi:
                robust_cifar_train.learning_step(sesh, flip_f, 0.0)
            else:
                robust_cifar_train.learning_step(sesh, flip_f, 1.0)

            if create_summ:
                loss, summ, a_l, a_s = robust_cifar_train.summary_step(sesh)
                sum_writer.add_summary(summ, batch_id)
                sum_writer.add_summary(a_s, batch_id)

            # save every 100th batch model
            if batch_id % 500 == 0:
                saver.save(
                    sesh,
                    'models/cifar10_go_on_{}_robust_{}_model_hold_out_{}__{}'.
                    format(go_on, sample, hold_out_s, str_v),
                    global_step=batch_id)
                acc, test_sum = robust_cifar_train.test_step(
                    test_im, test_l, sesh)
                print "{}: step{}, test acc {}".format(datetime.now(),
                                                       batch_id, acc)
                sum_writer.add_summary(test_sum, batch_id)

            if batch_id % 100 == 0:
                print "{}: step {}, loss {} adv_loss {}".format(
                    datetime.now(), batch_id, loss, a_l)
예제 #7
0
                     'If true, skip training the model.')
flags.DEFINE_boolean('restore', False,
                     'If true, restore the model from the latest checkpoint.')

# define artifact directories where results from the session can be saved
model_path = os.environ.get('MODEL_PATH', 'models/')
if not os.path.exists(model_path):
    os.mkdir(model_path)
checkpoint_path = os.environ.get('CHECKPOINT_PATH', 'checkpoints/')
if not os.path.exists(checkpoint_path):
    os.mkdir(checkpoint_path)
summary_path = os.environ.get('SUMMARY_PATH', 'logs/')
if not os.path.exists(summary_path):
    os.mkdir(summary_path)

mnist = mnist.read_data_sets('mnist', one_hot=True)


def weight_bias(W_shape, b_shape, bias_init=0.1):
    W = tf.Variable(tf.truncated_normal(shape=W_shape, stddev=0.1),
                    name='weight')
    b = tf.Variable(tf.constant(bias_init, shape=b_shape), name='bias')
    return W, b


def dense_layer(x, W_shape, b_shape, activation):
    W, b = weight_bias([W_shape, b_shape], [b_shape])
    return activation(tf.matmul(x, W) + b)


def highway_layer(x, size, activation, carry_bias=-1.0):
예제 #8
0
import tensorflow as tf
import numpy as np
import matplotlib.pyplot as plt

#MNIST Data
import mnist_data
mnist = mnist_data.read_data_sets("F:\ASU\pro_auto_encoder", one_hot=True)

#Hyperparameters
learning_rate = 0.01
num_steps = 100
batch_size = 256

display_step = 100
examples_to_show = 10

#Network

num_hidden = 256
latent_dim = 128
num_input = 784

#Graph

X = tf.placeholder("float", [None, num_input])

####weights
weights = {
    'encoder_h1': tf.Variable(tf.random_normal([num_input, num_hidden])),
    'encoder_h2': tf.Variable(tf.random_normal([num_hidden, latent_dim])),
    'decoder_h1': tf.Variable(tf.random_normal([latent_dim, num_hidden])),
import mnist_data
import tensorflow as tf


mnist = mnist_data.read_data_sets("MNIST_data", one_hot=True)

sess = tf.InteractiveSession()


x = tf.placeholder("float", shape=[None, 784])  # input images, 784 == 28**2 flattened
y_ = tf.placeholder("float", shape=[None, 10])  # target output class


W = tf.Variable(tf.zeros([784, 10]))
b = tf.Variable(tf.zeros([10]))


sess.run(tf.initialize_all_variables())


y = tf.nn.softmax(tf.matmul(x, W) + b)

cross_entropy = -tf.reduce_sum(y_ * tf.log(y))

train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy)

for i in range(1000):
    batch = mnist.train.next_batch(50)
    train_step.run(feed_dict={x: batch[0], y_: batch[1]})

    # Since evaluating the accuracy is expensive, only run it every 50 steps
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Import data
import mnist_data as input_data
import tensorflow as tf
import tensorflowvisu
import math
tf.set_random_seed(0)

mnist = input_data.read_data_sets("data")

# neural network structure for this sample:
#
# · · · · · · · · · ·      (input data, 1-deep)                 X [batch, 28, 28, 1]
# @ @ @ @ @ @ @ @ @ @   -- conv. layer 5x5x1=>24 stride 1       W1 [5, 5, 1, 24]       B1 [24]
# ∶∶∶∶∶∶∶∶∶∶∶∶∶∶∶∶∶∶∶                                           Y1 [batch, 28, 28, 24]
#   @ @ @ @ @ @ @ @     -- conv. layer 5x5x24=>48 stride 2      W2 [5, 5, 24, 48]      B2 [48]
#   ∶∶∶∶∶∶∶∶∶∶∶∶∶∶∶                                             Y2 [batch, 14, 14, 48]
#     @ @ @ @ @ @       -- conv. layer 5x5x48=>24 stride 1      W3 [5, 5, 48, 24]      B3 [24]
#     ∶∶∶∶∶∶∶∶∶∶∶                                               Y3 [batch, 14, 14, 24]
#       @ @ @ @         -- conv. layer 5x5x24=>48 stride 2      W4 [5, 5, 24, 48]      B4 [48]
#       ∶∶∶∶∶∶∶                                                 Y4 [batch, 7, 7, 48] reshaped to Z1 [batch, 7*7*48]
#       \x/x\x/ -       -- fully connected layer (relu+dropout) Wf1 [7*7*48, 1024]     Bf1 [1024]
#        · · ·                                                  T1 [batch, 1024]
#        \x/x/ -        -- fully connected layer (relu+dropout) Wf2 [1024, 512]        Bf2 [512]
예제 #11
0
import numpy as np
# Import MINST data
import mnist_data
#mnist = input_data.read_data_sets("/tmp/data/", one_hot=True)
mnist = mnist_data.read_data_sets("mnist_data/", one_hot=True, reshape=False)

# understand the size (1 , 28 , 28)
batch_xs, batch_ys = mnist.train.next_batch(1)
print(len(batch_xs[0]))
print(len(batch_xs[0][0]))
print(len(batch_xs[0][0][0]))
print(batch_ys)

import tensorflow as tf

# Parameters
learning_rate = 0.001
training_iters = 20000
batch_size = 128
display_step = 10

# Network Parameters
n_input = 784  # MNIST data input (img shape: 28*28)
n_classes = 10  # MNIST total classes (0-9 digits)
dropout = 0.75  # Dropout, probability to keep units


def weight_variable(shape):
    initializer = tf.truncated_normal_initializer(dtype=tf.float32,
                                                  stddev=1e-1)
    return tf.get_variable("weights",
예제 #12
0
import os
import models
import learning_rates
import mnist_data
import cifar_data

sess = tf.Session()
K.set_session(sess)

USE_CONV = True
MNIST = True  # True for MNIST, False for CIFAR-10

if (MNIST):
    print("Downloading MNIST if first run")
    dataset = mnist_data.read_data_sets('MNIST_data',
                                        one_hot=True,
                                        reshape=(not USE_CONV))
    mentor_model = models.build_mentor_model_conv(
    ) if USE_CONV else models.build_mentor_model()
    img_input = models.img_conv if USE_CONV else models.img_dense
else:
    dataset = cifar_data.read_data_sets('CIFAR_data')
    mentor_model = models.build_mentor_model_conv_cifar10()
    img_input = models.img_cifar

# run_name = "mentor" +
run_name = ("_conv" if USE_CONV else "") + ("_mnist" if MNIST else "_cifar10")
summary_name = run_name + "_accuracy"
model_save_name = "mentor" + run_name + ".h5"

mentor_preds = mentor_model.output
import mnist_data
import tensorflow as tf

# based on tutorial at http://www.tensorflow.org/tutorials/mnist/pros/index.html

mnist = mnist_data.read_data_sets('MNIST_data', one_hot=True)

def weight_variable(shape):
  initial = tf.truncated_normal(shape, stddev=0.1)
  return tf.Variable(initial)

def bias_variable(shape):
  initial = tf.constant(0.1, shape=shape)
  return tf.Variable(initial)

def conv2d(x, W):
  return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='SAME')

# Helper method for maximum pooling. strides 2x2, 75% reduction in image size per pooling operation
def max_pool_2x2(x):
  return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],
                        strides=[1, 2, 2, 1], padding='SAME')

x = tf.placeholder("float", shape=[None, 784], name="x") # None means the batch size can be any length
y_ = tf.placeholder("float", shape=[None, 10], name="y_") # One-hot 10-dimensional vector indicating which digit class the corresponding MNIST image belongs to




L2_SIZE = 64
L3_SIZE = 128
예제 #14
0
def distributed_model(config):

    tf.app.flags.DEFINE_string("job_name", "", "Enter 'ps' or 'worker' ")
    tf.app.flags.DEFINE_integer("task_index", 0,
                                "Index of the task within the jobs")
    tf.app.flags.DEFINE_bool("async", True, "Async or Sync Train")
    tf.app.flags.DEFINE_string("ps_hosts", "",
                               "Comma-separated list of hostname:port pairs")
    tf.app.flags.DEFINE_string("worker_hosts", "",
                               "Comma-separated list of hostname:port pairs")
    tf.app.flags.DEFINE_string("data_dir", "./data/", "Data directory")
    FLAGS = tf.app.flags.FLAGS

    ps_hosts = FLAGS.ps_hosts.split(",")
    worker_hosts = FLAGS.worker_hosts.split(",")

    mnist = mnist_data.read_data_sets(FLAGS.data_dir, one_hot=True)

    cluster = tf.train.ClusterSpec({"worker": worker_hosts, "ps": ps_hosts})

    server = tf.train.Server(cluster,
                             job_name=FLAGS.job_name,
                             task_index=FLAGS.task_index)

    if FLAGS.job_name == 'ps':
        server.join()
    else:
        with tf.device(
                tf.train.replica_device_setter(
                    worker_device='/job:worker/task:%d' % FLAGS.task_index,
                    cluster=cluster)):
            global_step = tf.Variable(0, name="global_step", trainable=False)
            x = tf.placeholder(tf.float32, [None, 784])
            label = tf.placeholder(tf.float32, [None, 10])

            pred_y = model(x)

            with tf.name_scope('loss'):
                cross_entropy = tf.nn.softmax_cross_entropy_with_logits(
                    labels=label, logits=pred_y)
                cross_entropy = tf.reduce_mean(cross_entropy)
                tf.summary.scalar('loss', cross_entropy)

            with tf.name_scope('adam_optimizer'):
                optimizer = tf.train.AdamOptimizer(config.lr)

            with tf.name_scope('accuracy'):
                correct_prediction = tf.equal(tf.argmax(pred_y, 1),
                                              tf.argmax(label, 1))
                correct_prediction = tf.cast(correct_prediction, tf.float32)
                accuracy = tf.reduce_mean(correct_prediction)
                tf.summary.scalar('accuracy', accuracy)

            with tf.name_scope('grads_and_vars'):

                grads_and_vars = optimizer.compute_gradients(cross_entropy)

            if FLAGS. async:

                # 异步模式
                train_op = optimizer.apply_gradients(grads_and_vars)

            else:

                rep_op = tf.train.SyncReplicasOptimizer(
                    optimizer,
                    replicas_to_aggregate=len(worker_hosts),
                    total_num_replicas=len(worker_hosts),
                    use_locking=True)

                train_op = rep_op.apply_gradients(grads_and_vars,
                                                  global_step=global_step)
                init_token_op = rep_op.get_init_tokens_op()
                chief_queue_runner = rep_op.get_chief_queue_runner()

            init_op = tf.global_variables_initializer()
            saver = tf.train.Saver()
            summary_op = tf.summary.merge_all()

            sv = tf.train.Supervisor(is_chief=(FLAGS.task_index == 0),
                                     logdir='./summary_log/',
                                     init_op=init_op,
                                     summary_op=None,
                                     saver=saver,
                                     global_step=global_step,
                                     save_model_secs=60)

            summary_writer = tf.summary.FileWriter('./summary_log/')

            with sv.prepare_or_wait_for_session(server.target) as sess:

                if FLAGS.task_index == 0 and not FLAGS. async:

                    sv.start_queue_runners(sess, [chief_queue_runner])
                    sess.run(init_token_op)

                for i in xrange(config.epoches):
                    loss = 0
                    for j in xrange(train_data_size / config.batch_size):

                        batch = mnist.train.next_batch(config.batch_size)
                        sess.run(train_op,
                                 feed_dict={
                                     x: batch[0],
                                     label: batch[1]
                                 })
                        loss += sess.run(cross_entropy,
                                         feed_dict={
                                             x: batch[0],
                                             label: batch[1]
                                         })
                        if j % 5 == 0:
                            train_accuracy = accuracy.eval(session=sess,
                                                           feed_dict={
                                                               x: batch[0],
                                                               label: batch[1]
                                                           })
                            print(
                                '[Epoch: %d Sample: %d] training accuracy: %g loss: %s'
                                % (i, (j + 1) * config.batch_size,
                                   train_accuracy, loss / (j + 1)))
                    summary = sess.run(summary_op)
                    summary_writer.add_summary(summary, global_step)
                    train_accuracy = accuracy.eval(session=sess,
                                                   feed_dict={
                                                       x: mnist.test.images,
                                                       label: mnist.test.labels
                                                   })
                    print("[Epoch: %s] Test Accuracy: %s" %
                          (i + 1, train_accuracy))
                sv.stop()
예제 #15
0
def train(config):

    mnist = mnist_data.read_data_sets(config.data_dir, one_hot=True)

    x = tf.placeholder(tf.float32, [None, 784])
    label = tf.placeholder(tf.float32, [None, 10])

    pred_y = model(x)

    with tf.name_scope('loss'):
        cross_entropy = tf.nn.softmax_cross_entropy_with_logits(labels=label,
                                                                logits=pred_y)
        cross_entropy = tf.reduce_mean(cross_entropy)
        tf.summary.scalar('loss', cross_entropy)

    with tf.name_scope('adam_optimizer'):
        train_step = tf.train.AdamOptimizer(config.lr).minimize(cross_entropy)

    with tf.name_scope('accuracy'):

        correct_prediction = tf.equal(tf.argmax(pred_y, 1),
                                      tf.argmax(label, 1))
        correct_prediction = tf.cast(correct_prediction, tf.float32)
        accuracy = tf.reduce_mean(correct_prediction)
        tf.summary.scalar('accuracy', accuracy)

    with tf.Session() as sess:

        sess.run(tf.global_variables_initializer())
        # merged = tf.summary.merge_all()
        # writer = tf.summary.FileWriter("logs/", sess.graph)
        #sess.run(tf.global_variables_initializer())

        for i in xrange(config.epoches):
            loss = 0
            for j in xrange(train_data_size / config.batch_size):

                batch = mnist.train.next_batch(config.batch_size)
                train_step.run(feed_dict={x: batch[0], label: batch[1]})
                loss += sess.run(
                    cross_entropy, feed_dict={
                        x: batch[0],
                        label: batch[1]
                    }
                )  #sess.run(tf.reduce_mean(sess.run(cross_entropy, feed_dict={x: batch[0], label: batch[1]})))
                if j % 5 == 0:
                    train_accuracy = accuracy.eval(feed_dict={
                        x: batch[0],
                        label: batch[1]
                    })
                    print(
                        '[Epoch: %d Sample: %d] training accuracy: %g loss: %s'
                        % (i,
                           (j + 1) * config.batch_size, train_accuracy, loss /
                           (j + 1)))

            train_accuracy = accuracy.eval(feed_dict={
                x: mnist.test.images,
                label: mnist.test.labels
            })
            print("[Epoch: %s] Test Accuracy: %s" % (i + 1, train_accuracy))