def main():
# input data is in NHWC format
data = pickle.load(
open("/home/ucu/Work/git/mnist/data/data_nhwc.pkl", "rb"))
te = data['test']
te_x = te[0]
te_y = te[1]
height = te_x.shape[1]
width = te_x.shape[2]
n_chans = te_x.shape[3]
n_classes = te_y.shape[1]
# initialization
tf.reset_default_graph()
np.random.seed(42)
tf.set_random_seed(42)
# input variables, image data + ground truth labels
x = tf.placeholder(tf.float32, [None, height, width, n_chans],
name="input")
gt = tf.placeholder(tf.float32, [None, n_classes], name="label")
# create network
layers, variables = mnist_sequential(x)
# training variable to control dropout
training = tuple_list_find(variables, "training")[1]
# logit output required for optimization
logit = tuple_list_find(layers, "fc2")[1]
# correct classifications
corr = tf.equal(tf.argmax(logit, 1), tf.argmax(gt, 1))
# accuray = average of correct classifications
accuracy = tf.reduce_mean(tf.cast(corr, tf.float32))
session = tf.Session()
with session.as_default():
# initialization of variables
session.run(tf.global_variables_initializer())
load_variables(session,
"/home/ucu/Work/git/mnist/network/mnist_expdecay.pkl")
acc = []
# evaluations on test set
for (xb, yb) in batch_generator(512, te_x, te_y, fixed_size=False):
ac = session.run(accuracy,
feed_dict={
x: xb,
gt: yb,
training: False
})
acc.append(ac)
print("Test accuracy: ", np.mean(acc))
session.close()
session = None
def main():
# input data is in NHWC format
data_path = os.path.join(cifar10_data_folder, "data_nhwc.pkl")
data = pickle.load(open(data_path, "rb"))
tr = data['train']
tr_x = tr[0]
tr_y = tr[1]
te = data['test']
te_x = te[0]
te_y = te[1]
height = tr_x.shape[1]
width = tr_x.shape[2]
n_chans = tr_x.shape[3]
n_classes = tr_y.shape[1]
# initialization
tf.reset_default_graph()
np.random.seed(42)
tf.set_random_seed(42)
# input variables, image data + ground truth labels
x = tf.placeholder(tf.float32, [None, height, width, n_chans], name="input")
gt = tf.placeholder(tf.float32, [None, n_classes], name="label")
# create network
layers, variables = cifar10_shufflenet(x)
# training variable to control dropout
training = tuple_list_find(variables, "training")[1]
# logit output required for optimization
logit = tuple_list_find(layers, "fc2")[1]
n_epochs = 40
# optimization is done one the cross-entropy between labels and predicted logit
cross_entropy = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=gt, logits=logit))
# keeps track of the number of batches used for updating the network
global_step = tf.Variable(0, trainable=False, name="global_step")
# input variable for passing learning rates for the optimizer
learning_rate = tf.placeholder(tf.float32, name='learning_rate')
# some layers (e.g. batch normalization) require updates to internal variables
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_step = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cross_entropy,
global_step = global_step)
# correct classifications
corr = tf.equal(tf.argmax(logit, 1), tf.argmax(gt, 1))
# accuray = average of correct classifications
accuracy = tf.reduce_mean(tf.cast(corr, tf.float32))
# learning rate with exponential decay
exp_decay = ExponentialDecay(start=0.01, stop=0.0001, max_steps=50)
session = tf.Session()
with session.as_default():
# initialization of variables
session.run(tf.global_variables_initializer())
for i in range(n_epochs):
lr = next(exp_decay)
# training via random batches
for (xb, yb) in random_batch_generator(256, tr_x, tr_y):
session.run(train_step, feed_dict={x: xb,
gt: yb,
training: True,
learning_rate: lr})
tr_acc = []
# evaluations on test set
for (xb, yb) in batch_generator(512, tr_x, tr_y, fixed_size=False):
ac = session.run(accuracy, feed_dict={x: xb,
gt: yb,
training: False})
tr_acc.append(ac)
acc = []
# evaluations on test set
for (xb, yb) in batch_generator(512, te_x, te_y, fixed_size=False):
ac = session.run(accuracy, feed_dict={x: xb,
gt: yb,
training: False})
acc.append(ac)
print("Epoch: ", i)
print("Learning rate: ", lr)
print("Test accuracy: ", np.mean(acc))
print("Train accuracy: ", np.mean(tr_acc))
net_path = os.path.join(cifar10_net_folder, "cifar10_shufflenet_expdecay.pkl")
save_variables(session, net_path)
session.close()
session = None
def main(seed=42):
weight_decay = 0.0001
margin = 0.2
n_epochs = 50
batch_size = 128
_tr_x, _tr_y, _te_x, _te_y = load_mnist_data()
_tr_x, _te_x = global_mean_std(_tr_x, _te_x)
temp_path = os.path.join(temp_folder, "mnist_siamese_data.pkl")
if not os.path.exists(temp_path):
# tr_x, tr_y = generate_pair_indices(_tr_x, _tr_y, max_pos = 9000, max_neg = 9000, seed = seed)
tr_x, tr_y = generate_pair_indices2(_tr_x,
_tr_y,
max_pos=45000,
max_neg=45000,
seed=seed)
# te_x, te_y = generate_pair_indices(_te_x, _te_y, max_pos = 2250, max_neg = 2250, seed = seed)
te_x, te_y = generate_pair_indices2(_te_x,
_te_y,
max_pos=9000,
max_neg=9000,
seed=seed)
with open(temp_path, "wb") as f:
pickle.dump((tr_x, tr_y, te_x, te_y), f)
else:
with open(temp_path, "rb") as f:
tr_x, tr_y, te_x, te_y = pickle.load(f)
height = _tr_x.shape[1]
width = _tr_x.shape[2]
n_chans = _tr_x.shape[3]
n_classes = _tr_y.shape[1]
tf.reset_default_graph()
np.random.seed(seed)
tf.set_random_seed(seed)
x1 = tf.placeholder(tf.float32, [None, height, width, n_chans],
name="input1")
x2 = tf.placeholder(tf.float32, [None, height, width, n_chans],
name="input2")
label = tf.placeholder(tf.float32, [None, 1], name='label')
layers, variables = mnist_siamese(x1,
x2,
weight_decay=weight_decay,
seed=seed)
training = tuple_list_find(variables, "training")[1]
output = tuple_list_find(layers, "output")[1]
output1 = tuple_list_find(output[0], "output")[1]
output2 = tuple_list_find(output[1], "output")[1]
_loss = contrastive_loss(output1, output2, label, margin=margin)
if weight_decay is not None:
reg_ws = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
loss_fn = _loss + tf.reduce_sum(reg_ws)
else:
loss_fn = _loss
global_step = tf.Variable(0, trainable=False, name="global_step")
learning_rate = tf.placeholder(tf.float32, name='learning_rate')
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_step = tf.train.MomentumOptimizer(learning_rate=learning_rate,
momentum=0.99,
use_nesterov=True).minimize(
loss_fn,
global_step=global_step)
# learning rate with exponential decay
lr_decay = ExponentialDecay(start=0.01, stop=0.001, max_steps=50)
# data augmentation
# transformer = RandomizedTransformer(
# transformer_class = Affine,
# params = [('shape', (height, width, n_chans)),
# ('scale', 1.0)],
# rand_params = [('r', [-3.0, 3.0]),
# ('tx', [-3.0, 3.0]),
# ('ty', [-3.0, 3.0]),
# ('reflect_y', [False, True])],
# mode = 'each',
# random_seed = seed)
transformer = None
session = tf.Session()
with session.as_default():
session.run(tf.global_variables_initializer())
for i in range(n_epochs):
lr = next(lr_decay)
# training via random batches
for (xb, yb) in random_batch_generator(batch_size,
tr_x,
tr_y,
seed=seed + i):
idx1 = xb[:, 0]
idx2 = xb[:, 1]
xb1 = _tr_x[idx1]
xb2 = _tr_x[idx2]
if transformer is not None:
xbtr1 = np.zeros_like(xb1)
for j in range(len(xb1)):
xbtr1[j] = transformer.transform(xb1[j])
xbtr2 = np.zeros_like(xb2)
for j in range(len(xb2)):
xbtr2[j] = transformer.transform(xb2[j])
else:
xbtr1 = xb1
xbtr2 = xb2
session.run(train_step,
feed_dict={
x1: xbtr1,
x2: xbtr2,
label: yb,
training: True,
learning_rate: lr
})
tr_loss = []
# evaluations on train set
for (xb, yb) in batch_generator(256, tr_x, tr_y, fixed_size=False):
idx1 = xb[:, 0]
idx2 = xb[:, 1]
xb1 = _tr_x[idx1]
xb2 = _tr_x[idx2]
loss = session.run(_loss,
feed_dict={
x1: xb1,
x2: xb2,
label: yb,
training: False
})
tr_loss.append(loss)
te_loss = []
# evaluations on test set
for (xb, yb) in batch_generator(256, te_x, te_y, fixed_size=False):
idx1 = xb[:, 0]
idx2 = xb[:, 1]
xb1 = _te_x[idx1]
xb2 = _te_x[idx2]
loss = session.run(_loss,
feed_dict={
x1: xb1,
x2: xb2,
label: yb,
training: False
})
te_loss.append(loss)
print("Epoch: ", i)
print("Learning rate: ", lr)
print("Train loss: " + str(np.mean(tr_loss)))
print("Test loss: " + str(np.mean(te_loss)))
var_path = os.path.join(temp_folder, "mnist_siamese_net.h5")
save_variables(session, var_path)
embeddings = []
labels = []
for xb in batch_generator(256, tr_x, y=None, fixed_size=False):
idx1 = xb[:, 0]
idx2 = xb[:, 1]
xb1 = _tr_x[idx1]
xb2 = _tr_x[idx2]
yb1 = _tr_y[idx1]
yb2 = _tr_y[idx2]
out1 = session.run(output1, feed_dict={x1: xb1, training: False})
out2 = session.run(output1, feed_dict={x1: xb2, training: False})
embeddings.append(out1)
embeddings.append(out2)
labels.append(yb1)
labels.append(yb2)
embeddings = np.concatenate(embeddings)
labels = np.concatenate(labels)
labels = np.argmax(labels, axis=1)
# generated by glasbey
palette = load_palette("palette50.txt", skip_first=True,
scale01=True) # skip white
palette = palette[0:n_classes]
#palette = ['#ff0000', '#ffff00', '#00ff00', '#00ffff', '#0000ff', '#ff00ff', '#990000', '#999900', '#009900', '#009999']
f = plt.figure()
for i in range(n_classes):
idx = labels == i
coords = embeddings[idx, :]
plt.plot(coords[:, 0], coords[:, 1], '.', color=palette[i])
plt.legend(['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'])
plt.show()
def _eval_net_custom(tr_x,
tr_y,
te_x,
te_y,
net_func,
n_epochs,
batch_size,
lr_decay_func,
optimizer,
optimizer_args=None,
weight_decay=0.0,
aux_loss_weight=None,
label_smoothing=None,
augmentation=True,
seed=42):
height = tr_x.shape[1]
width = tr_x.shape[2]
n_chans = tr_x.shape[3]
n_classes = tr_y.shape[1]
if optimizer_args is None:
optimizer_args = dict()
tf.reset_default_graph()
np.random.seed(seed)
tf.set_random_seed(seed)
# input variables, image data + ground truth labels
x = tf.placeholder(tf.float32, [None, height, width, n_chans],
name="input")
gt = tf.placeholder(tf.float32, [None, n_classes], name="label")
# create network
if weight_decay is None:
reg_weights = False
layers, variables = net_func(x, seed=seed)
else:
reg_weights = True
layers, variables = net_func(x, weight_decay=weight_decay, seed=seed)
# training variable to control dropout
training = tuple_list_find(variables, "training")[1]
# logit output required for optimization
logit = tuple_list_find(layers, "logit")[1]
# optimization is done one the cross-entropy between labels and predicted logit
if label_smoothing is None:
loss_fn = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=gt, logits=logit))
else:
loss_fn = tf.reduce_mean(
tf.losses.softmax_cross_entropy(onehot_labels=gt,
logits=logit,
label_smoothing=label_smoothing))
if aux_loss_weight is not None:
aux_logit = tuple_list_find(layers, "aux_logit")[1]
if label_smoothing is None:
aux_loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=gt,
logits=aux_logit))
else:
aux_loss = tf.reduce_mean(
tf.losses.softmax_cross_entropy(
onehot_labels=gt,
logits=aux_logit,
label_smoothing=label_smoothing))
loss_fn = loss_fn + aux_loss_weight * aux_loss
if reg_weights:
reg_ws = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
loss_fn = loss_fn + tf.reduce_sum(reg_ws)
# keeps track of the number of batches used for updating the network
global_step = tf.Variable(0, trainable=False, name="global_step")
# input variable for passing learning rates for the optimizer
learning_rate = tf.placeholder(tf.float32, name='learning_rate')
optimizer_args.update({'learning_rate': learning_rate})
# some layers (e.g. batch normalization) require updates to internal variables
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_step = optimizer(**optimizer_args).minimize(
loss_fn, global_step=global_step)
# correct classifications
corr = tf.equal(tf.argmax(logit, 1), tf.argmax(gt, 1))
# accuray = average of correct classifications
accuracy = tf.reduce_mean(tf.cast(corr, tf.float32))
# apply random affine transformations to training images
if augmentation:
transformer = RandomizedTransformer(transformer_class=Affine,
params=[('shape', (height, width,
n_chans)),
('scale', 1.0)],
rand_params=[('r', [-3.0, 3.0]),
('tx', [-3.0, 3.0]),
('ty', [-3.0, 3.0]),
('reflect_y',
[False, True])],
mode='each',
random_seed=seed)
else:
transformer = None
acc_final = None
session = tf.Session()
with session.as_default():
# initialization of variables
session.run(tf.global_variables_initializer())
for i in range(n_epochs):
lr = next(lr_decay_func)
# training via random batches
for (xb, yb) in random_batch_generator(batch_size,
tr_x,
tr_y,
seed=seed + i):
if augmentation:
xbtr = np.zeros_like(xb)
for j in range(len(xb)):
xbtr[j] = transformer.transform(xb[j])
else:
xbtr = xb
session.run(train_step,
feed_dict={
x: xbtr,
gt: yb,
training: True,
learning_rate: lr
})
tr_acc = []
# evaluations on train set
for (xb, yb) in batch_generator(256, tr_x, tr_y, fixed_size=False):
ac = session.run(accuracy,
feed_dict={
x: xb,
gt: yb,
training: False
})
tr_acc.append(ac)
acc = []
# evaluations on test set
for (xb, yb) in batch_generator(256, te_x, te_y, fixed_size=False):
ac = session.run(accuracy,
feed_dict={
x: xb,
gt: yb,
training: False
})
acc.append(ac)
print("Epoch: ", i)
print("Learning rate: ", lr)
print("Test accuracy: ", np.mean(acc))
print("Train accuracy: ", np.mean(tr_acc))
acc_final = np.mean(acc)
session.close()
session = None
return acc_final
def main():
# input data is in NHWC format
data_path = os.path.join(cifar10_data_folder, "data_nhwc.pkl")
data = pickle.load(open(data_path, "rb"))
tr = data['train']
tr_x = tr[0]
tr_y = tr[1]
te = data['test']
te_x = te[0]
te_y = te[1]
height = tr_x.shape[1]
width = tr_x.shape[2]
n_chans = tr_x.shape[3]
n_classes = tr_y.shape[1]
# data normalization
tr_x, te_x = channel_mean_std(tr_x, te_x)
# initialization
tf.reset_default_graph()
np.random.seed(42)
tf.set_random_seed(42)
# input variables, image data + ground truth labels
x = tf.placeholder(tf.float32, [None, height, width, n_chans],
name="input")
gt = tf.placeholder(tf.float32, [None, n_classes], name="label")
# create network
layers, variables = cifar10_nasnet_wd(x, drop_rate=0.5, weight_decay=1e-4)
# training variable to control dropout
training = tuple_list_find(variables, "training")[1]
# logit output required for optimization
logit = tuple_list_find(layers, "logit")[1]
aux_logit = tuple_list_find(layers, "aux_logit")[1]
n_epochs = 50
# optimization is done one the cross-entropy between labels and predicted logit
cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=gt, logits=logit))
aux_cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=gt, logits=aux_logit))
aux_weight = 0.4
cross_entropy = cross_entropy + aux_weight * aux_cross_entropy
reg_ws = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
loss_fn = cross_entropy + tf.reduce_sum(reg_ws)
# keeps track of the number of batches used for updating the network
global_step = tf.Variable(0, trainable=False, name="global_step")
learning_rate = 0.1
# some layers (e.g. batch normalization) require updates to internal variables
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_step = tf.train.MomentumOptimizer(learning_rate=learning_rate,
momentum=0.9,
use_nesterov=True).minimize(
loss_fn,
global_step=global_step)
# correct classifications
corr = tf.equal(tf.argmax(logit, 1), tf.argmax(gt, 1))
# accuray = average of correct classifications
accuracy = tf.reduce_mean(tf.cast(corr, tf.float32))
# apply random affine transformations to training images
transformer = RandomizedTransformer(transformer_class=Affine,
params=[('shape', (height, width,
n_chans)),
('scale', 1.0)],
rand_params=[('r', [-3.0, 3.0]),
('tx', [-3.0, 3.0]),
('ty', [-3.0, 3.0]),
('reflect_y',
[False, True])],
mode='each',
random_seed=42)
session = tf.Session()
with session.as_default():
# initialization of variables
session.run(tf.global_variables_initializer())
for i in range(n_epochs):
# training via random batches
for (xb, yb) in random_batch_generator(128,
tr_x,
tr_y,
seed=42 + i):
xbtr = np.zeros_like(xb)
for j in range(len(xb)):
xbtr[j] = transformer.transform(xb[j])
session.run(train_step,
feed_dict={
x: xbtr,
gt: yb,
training: True
})
tr_acc = []
# evaluations on train set
for (xb, yb) in batch_generator(256, tr_x, tr_y, fixed_size=False):
ac = session.run(accuracy,
feed_dict={
x: xb,
gt: yb,
training: False
})
tr_acc.append(ac)
acc = []
# evaluations on test set
for (xb, yb) in batch_generator(256, te_x, te_y, fixed_size=False):
ac = session.run(accuracy,
feed_dict={
x: xb,
gt: yb,
training: False
})
acc.append(ac)
print("Epoch: ", i)
print("Learning rate: ", learning_rate)
print("Test accuracy: ", np.mean(acc))
print("Train accuracy: ", np.mean(tr_acc))
net_path = os.path.join(cifar10_net_folder,
"cifar10_nasnet_wd_randtrans.pkl")
save_variables(session, net_path)
session.close()
session = None
