def main():
    tr_x, tr_y, te_x, te_y = load_cifar10_data()
    tr_x, te_x = global_mean_std(tr_x, te_x)
    mean_acc, max_acc, min_acc = eval_net_custom(
        tr_x,
        tr_y,
        te_x,
        te_y,
        net_func=partial(cifar10_sequential_c3d_selu_drop, drop_rate=0.5),
        n_epochs=50,
        batch_size=128,
        lr_decay_func=ExponentialDecay(start=0.001, stop=0.0001, max_steps=50),
        optimizer=tf.train.AdamOptimizer,
        weight_decay=None,
        augmentation=False)
    print("Mean accuracy: ", mean_acc)
    print("Max accuracy: ", max_acc)
    print("Min accuracy: ", min_acc)
def main():
    tr_x, tr_y, te_x, te_y = load_cifar10_data()
    tr_x, te_x = channel_mean_std(tr_x, te_x)
    mean_acc, max_acc, min_acc = eval_net_custom(
        tr_x,
        tr_y,
        te_x,
        te_y,
        net_func=cifar10_bn_inception_v1_wd,
        optimizer=tf.train.MomentumOptimizer,
        optimizer_args={'momentum': 0.9},
        n_epochs=250,
        batch_size=128,
        lr_decay_func=ExponentialDecay(start=0.01, stop=0.0001, max_steps=200),
        weight_decay=0.00004)
    print("Mean accuracy: ", mean_acc)
    print("Max accuracy: ", max_acc)
    print("Min accuracy: ", min_acc)
Example #3
0
def main():
    tr_x, tr_y, te_x, te_y = load_cifar10_data()
    tr_x, te_x = global_mean_std(tr_x, te_x)
    mean_acc, max_acc, min_acc = eval_net_custom(
        tr_x,
        tr_y,
        te_x,
        te_y,
        net_func=cifar10_inception_v3,
        optimizer=tf.train.AdamOptimizer,
        optimizer_args=None,
        n_epochs=50,
        batch_size=128,
        aux_loss_weight=0.3,
        label_smoothing=0.1,
        lr_decay_func=ExponentialDecay(start=0.01, stop=0.001, max_steps=50),
        weight_decay=None,
        augmentation=False)
    print("Mean accuracy: ", mean_acc)
    print("Max accuracy: ", max_acc)
    print("Min accuracy: ", min_acc)
def main():
    # input data is in NHWC format
    data_path = os.path.join(cifar10_data_folder, "data_nhwc.pkl")
    data = pickle.load(open(data_path, "rb"))
    tr = data['train']
    tr_x = tr[0]
    tr_y = tr[1]
    te = data['test']
    te_x = te[0]
    te_y = te[1]

    height = tr_x.shape[1]
    width = tr_x.shape[2]
    n_chans = tr_x.shape[3]
    n_classes = tr_y.shape[1]
    
    # initialization
    tf.reset_default_graph()
    np.random.seed(42)
    tf.set_random_seed(42)    
    
    # input variables, image data + ground truth labels
    x = tf.placeholder(tf.float32, [None, height, width, n_chans], name="input")
    gt = tf.placeholder(tf.float32, [None, n_classes], name="label")
    
    # create network
    layers, variables = cifar10_shufflenet(x)
    
    # training variable to control dropout
    training = tuple_list_find(variables, "training")[1]
    
    # logit output required for optimization
    logit = tuple_list_find(layers, "fc2")[1]
        
    n_epochs = 40
    
    # optimization is done one the cross-entropy between labels and predicted logit    
    cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=gt, logits=logit))
    
    # keeps track of the number of batches used for updating the network
    global_step = tf.Variable(0, trainable=False, name="global_step")
    
    # input variable for passing learning rates for the optimizer
    learning_rate = tf.placeholder(tf.float32, name='learning_rate')
    
    # some layers (e.g. batch normalization) require updates to internal variables
    update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
    with tf.control_dependencies(update_ops):
        train_step = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cross_entropy,
                                                                                  global_step = global_step)
        
    # correct classifications
    corr = tf.equal(tf.argmax(logit, 1), tf.argmax(gt, 1))
    
    # accuray = average of correct classifications
    accuracy = tf.reduce_mean(tf.cast(corr, tf.float32))
    
    # learning rate with exponential decay
    exp_decay = ExponentialDecay(start=0.01, stop=0.0001, max_steps=50)

    session = tf.Session()
    with session.as_default():
        # initialization of variables
        session.run(tf.global_variables_initializer())
        for i in range(n_epochs):
            lr = next(exp_decay)
            # training via random batches
            for (xb, yb) in random_batch_generator(256, tr_x, tr_y):
                session.run(train_step, feed_dict={x: xb,
                                                   gt: yb,
                                                   training: True,
                                                   learning_rate: lr})
    
    
            tr_acc = []
            # evaluations on test set
            for (xb, yb) in batch_generator(512, tr_x, tr_y, fixed_size=False):
                ac = session.run(accuracy, feed_dict={x: xb,
                                                      gt: yb,
                                                      training: False})
                tr_acc.append(ac)    
    
            acc = []
            # evaluations on test set
            for (xb, yb) in batch_generator(512, te_x, te_y, fixed_size=False):
                ac = session.run(accuracy, feed_dict={x: xb,
                                                      gt: yb,
                                                      training: False})
                acc.append(ac)
            print("Epoch: ", i)
            print("Learning rate: ", lr)
            print("Test accuracy: ", np.mean(acc))    
            print("Train accuracy: ", np.mean(tr_acc))
        net_path = os.path.join(cifar10_net_folder, "cifar10_shufflenet_expdecay.pkl")
        save_variables(session, net_path)
    session.close()
    session = None
Example #5
0
def main(seed=42):
    weight_decay = 0.0001
    margin = 0.2
    n_epochs = 50
    batch_size = 128

    _tr_x, _tr_y, _te_x, _te_y = load_mnist_data()
    _tr_x, _te_x = global_mean_std(_tr_x, _te_x)

    temp_path = os.path.join(temp_folder, "mnist_siamese_data.pkl")
    if not os.path.exists(temp_path):
        #        tr_x, tr_y = generate_pair_indices(_tr_x, _tr_y, max_pos = 9000, max_neg = 9000, seed = seed)
        tr_x, tr_y = generate_pair_indices2(_tr_x,
                                            _tr_y,
                                            max_pos=45000,
                                            max_neg=45000,
                                            seed=seed)
        #        te_x, te_y = generate_pair_indices(_te_x, _te_y, max_pos = 2250, max_neg = 2250, seed = seed)
        te_x, te_y = generate_pair_indices2(_te_x,
                                            _te_y,
                                            max_pos=9000,
                                            max_neg=9000,
                                            seed=seed)
        with open(temp_path, "wb") as f:
            pickle.dump((tr_x, tr_y, te_x, te_y), f)
    else:
        with open(temp_path, "rb") as f:
            tr_x, tr_y, te_x, te_y = pickle.load(f)

    height = _tr_x.shape[1]
    width = _tr_x.shape[2]
    n_chans = _tr_x.shape[3]
    n_classes = _tr_y.shape[1]

    tf.reset_default_graph()
    np.random.seed(seed)
    tf.set_random_seed(seed)

    x1 = tf.placeholder(tf.float32, [None, height, width, n_chans],
                        name="input1")
    x2 = tf.placeholder(tf.float32, [None, height, width, n_chans],
                        name="input2")
    label = tf.placeholder(tf.float32, [None, 1], name='label')

    layers, variables = mnist_siamese(x1,
                                      x2,
                                      weight_decay=weight_decay,
                                      seed=seed)
    training = tuple_list_find(variables, "training")[1]
    output = tuple_list_find(layers, "output")[1]
    output1 = tuple_list_find(output[0], "output")[1]
    output2 = tuple_list_find(output[1], "output")[1]

    _loss = contrastive_loss(output1, output2, label, margin=margin)
    if weight_decay is not None:
        reg_ws = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
        loss_fn = _loss + tf.reduce_sum(reg_ws)
    else:
        loss_fn = _loss

    global_step = tf.Variable(0, trainable=False, name="global_step")
    learning_rate = tf.placeholder(tf.float32, name='learning_rate')
    update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
    with tf.control_dependencies(update_ops):
        train_step = tf.train.MomentumOptimizer(learning_rate=learning_rate,
                                                momentum=0.99,
                                                use_nesterov=True).minimize(
                                                    loss_fn,
                                                    global_step=global_step)

    # learning rate with exponential decay
    lr_decay = ExponentialDecay(start=0.01, stop=0.001, max_steps=50)

    # data augmentation
    #    transformer = RandomizedTransformer(
    #            transformer_class = Affine,
    #            params = [('shape', (height, width, n_chans)),
    #                      ('scale', 1.0)],
    #            rand_params = [('r', [-3.0, 3.0]),
    #                           ('tx', [-3.0, 3.0]),
    #                           ('ty', [-3.0, 3.0]),
    #                           ('reflect_y', [False, True])],
    #            mode = 'each',
    #            random_seed = seed)
    transformer = None

    session = tf.Session()
    with session.as_default():
        session.run(tf.global_variables_initializer())
        for i in range(n_epochs):
            lr = next(lr_decay)
            # training via random batches
            for (xb, yb) in random_batch_generator(batch_size,
                                                   tr_x,
                                                   tr_y,
                                                   seed=seed + i):
                idx1 = xb[:, 0]
                idx2 = xb[:, 1]
                xb1 = _tr_x[idx1]
                xb2 = _tr_x[idx2]
                if transformer is not None:
                    xbtr1 = np.zeros_like(xb1)
                    for j in range(len(xb1)):
                        xbtr1[j] = transformer.transform(xb1[j])
                    xbtr2 = np.zeros_like(xb2)
                    for j in range(len(xb2)):
                        xbtr2[j] = transformer.transform(xb2[j])
                else:
                    xbtr1 = xb1
                    xbtr2 = xb2
                session.run(train_step,
                            feed_dict={
                                x1: xbtr1,
                                x2: xbtr2,
                                label: yb,
                                training: True,
                                learning_rate: lr
                            })

            tr_loss = []
            # evaluations on train set
            for (xb, yb) in batch_generator(256, tr_x, tr_y, fixed_size=False):
                idx1 = xb[:, 0]
                idx2 = xb[:, 1]
                xb1 = _tr_x[idx1]
                xb2 = _tr_x[idx2]
                loss = session.run(_loss,
                                   feed_dict={
                                       x1: xb1,
                                       x2: xb2,
                                       label: yb,
                                       training: False
                                   })
                tr_loss.append(loss)

            te_loss = []
            # evaluations on test set
            for (xb, yb) in batch_generator(256, te_x, te_y, fixed_size=False):
                idx1 = xb[:, 0]
                idx2 = xb[:, 1]
                xb1 = _te_x[idx1]
                xb2 = _te_x[idx2]
                loss = session.run(_loss,
                                   feed_dict={
                                       x1: xb1,
                                       x2: xb2,
                                       label: yb,
                                       training: False
                                   })
                te_loss.append(loss)

            print("Epoch: ", i)
            print("Learning rate: ", lr)
            print("Train loss: " + str(np.mean(tr_loss)))
            print("Test loss: " + str(np.mean(te_loss)))

        var_path = os.path.join(temp_folder, "mnist_siamese_net.h5")
        save_variables(session, var_path)

        embeddings = []
        labels = []
        for xb in batch_generator(256, tr_x, y=None, fixed_size=False):
            idx1 = xb[:, 0]
            idx2 = xb[:, 1]
            xb1 = _tr_x[idx1]
            xb2 = _tr_x[idx2]
            yb1 = _tr_y[idx1]
            yb2 = _tr_y[idx2]
            out1 = session.run(output1, feed_dict={x1: xb1, training: False})
            out2 = session.run(output1, feed_dict={x1: xb2, training: False})
            embeddings.append(out1)
            embeddings.append(out2)
            labels.append(yb1)
            labels.append(yb2)

    embeddings = np.concatenate(embeddings)
    labels = np.concatenate(labels)
    labels = np.argmax(labels, axis=1)

    # generated by glasbey
    palette = load_palette("palette50.txt", skip_first=True,
                           scale01=True)  # skip white
    palette = palette[0:n_classes]
    #palette = ['#ff0000', '#ffff00', '#00ff00', '#00ffff', '#0000ff', '#ff00ff', '#990000', '#999900', '#009900', '#009999']
    f = plt.figure()
    for i in range(n_classes):
        idx = labels == i
        coords = embeddings[idx, :]
        plt.plot(coords[:, 0], coords[:, 1], '.', color=palette[i])
    plt.legend(['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'])
    plt.show()