def __init__(self):
super().__init__()
im_arry = stl.read_all_images(stl.DATA_PATH)
im_list = [
hog(im, multichannel=True, cells_per_block=(1, 1))
for im in im_arry
]
self.images = torch.from_numpy(np.stack(im_list)).float()
self.labels = torch.from_numpy(stl.read_labels(stl.LABEL_PATH)) - 1
def load(data_dir, subset='train'):
maybe_download_and_extract(data_dir)
if subset == 'train':
#train_data = [unpickle(os.path.join(data_dir,'cifar-10-batches-py','data_batch_' + str(i))) for i in range(1,6)]
#trainx = np.concatenate([d['x'] for d in train_data],axis=0)
#trainy = np.concatenate([d['y'] for d in train_data],axis=0)
trainx = stl10_input.read_all_images(
os.path.join(data_dir, 'stl10_binary', 'train_X.bin'))
trainy = stl10_input.read_labels(
os.path.join(data_dir, 'stl10_binary', 'train_y.bin'))
return trainx, trainy
elif subset == 'test':
#test_data = unpickle(os.path.join(data_dir,'cifar-10-batches-py','test_batch'))
#testx = test_data['x']
#testy = test_data['y']
testx = stl10_input.read_all_images(
os.path.join(data_dir, 'stl10_binary', 'test_X.bin'))
testy = stl10_input.read_labels(
os.path.join(data_dir, 'stl10_binary', 'test_y.bin'))
return testx, testy
else:
raise NotImplementedError('subset should be either train or test')
def __init__(self): super().__init__() im_arry = stl.read_all_images(stl.DATA_PATH) im_list = [ cv2.resize(im, dsize=(224,224), interpolation=cv2.INTER_CUBIC) for im in im_arry ] self.images = torch.from_numpy( np.stack(im_list) ).float().permute(0,3,1,2) self.labels = torch.from_numpy( stl.read_labels(stl.LABEL_PATH) ) - 1
from tensorflow.keras.layers import Dense, Conv2D, Flatten, Dropout, MaxPooling2D
from tensorflow.keras.preprocessing.image import ImageDataGenerator
from data.stl10_input import read_labels, read_all_images
import matplotlib.pyplot as plt
import numpy as np
batch_size = 100
epochs = 15
IMG_HEIGHT = 96
IMG_WIDTH = 96
binary_path = "data/data/stl10_binary/"
train_images = read_all_images(binary_path + "train_x.bin")
train_labels = read_labels(binary_path + "train_Y.bin")
test_images = read_all_images(binary_path + "test_x.bin")
test_labels = read_labels(binary_path + "test_Y.bin")
def convert_labels(labels):
animal_labels = [2, 4, 5, 6, 7, 8]
machine_labels = [1, 3, 9, 10]
new_labels = []
for label in labels:
if label in animal_labels:
new_labels.append(1)
elif label in machine_labels:
new_labels.append(0)
def main(argv):
opts, args = getopt.getopt(argv, "hi:o:", ["test=", "experiment=", "subset=", "self_train=", "epochs=", "batch=", "threshold="])
self_train = False
subset_size = 1
epochs = 15
batch_size = 1000
threshold = .75
experiment = "default"
binary_classification = False
for opt, arg in opts:
if opt == '-h':
print('self-train.py --experiment=binary --subset=1 --self_train=True --epochs=20 --batch=1000 --threshold=.75')
sys.exit()
elif opt in ("-st", "--self_train"):
self_train= arg.lower() in ["true", "True"]
elif opt in ("-ss", "--subset"):
subset_size = float(arg)
elif opt in ("-exp", "--experiment"):
experiment = arg
if experiment=="binary":
binary_classification = True
elif opt in ("-e", "--epochs"):
epochs = int(arg)
elif opt in ("-b", "--batch"):
batch_size = int(arg)
elif opt in ("-t", "--threshold"):
threshold = float(arg)
print("self-train: ", self_train)
IMG_HEIGHT = 96
IMG_WIDTH = 96
binary_path = "data/stl10_binary/"
# if experiment != "disagreement":
og_train_images = read_all_images(binary_path + "train_X.bin")
og_train_labels = read_labels(binary_path + "train_y.bin")
og_test_images = read_all_images(binary_path + "test_X.bin")
og_test_labels = read_labels(binary_path + "test_y.bin")
experiments = []
if experiment == "disagreement":
experiments = ["disagreement_animal", "disagreement_machine"]
else:
experiments = [experiment]
models = []
all_train_labels = []
all_test_labels = []
histories = []
for sub_experiment in experiments:
checkpoint_path = "training/" + sub_experiment + "-cp-{epoch:04d}.ckpt"
checkpoint_dir = os.path.dirname(checkpoint_path)
# Create a callback that saves the model's weights every 5 epochs
cp_callback = tf.keras.callbacks.ModelCheckpoint(
filepath=checkpoint_path,
verbose=1,
save_weights_only=True,
period=5)
test_labels, test_images = convert_labels(og_test_labels, og_test_images, sub_experiment)
train_labels, train_images = convert_labels(og_train_labels, og_train_images, sub_experiment)
num_train = math.floor(len(train_labels)*subset_size)
num_test = math.floor(len(test_labels)*subset_size)
test_labels = test_labels[:num_test]
train_labels = train_labels[:num_train]
test_images = test_images[:num_test]
train_images = train_images[:num_train]
all_train_labels.append(train_labels)
all_test_labels.append(test_labels)
print(num_train, batch_size)
num_batches = math.ceil(num_train/batch_size)
batch_size = int(num_train/num_batches)
print("Adjusted batch size to " + str(batch_size))
if sub_experiment == "binary":
num_classes = 1
elif sub_experiment == "animal":
num_classes = 6
elif sub_experiment == "machine":
num_classes = 4
elif sub_experiment == "disagreement_animal":
num_classes = 7
elif sub_experiment == "disagreement_machine":
num_classes = 5
else:
num_classes = 10
# model = Sequential([
# Conv2D(16, 3, padding='same', activation='relu',
# input_shape=(IMG_HEIGHT, IMG_WIDTH ,3),
# kernel_regularizer=regularizers.l2(0.01),
# bias_regularizer = regularizers.l2(0.01),
# activity_regularizer = regularizers.l2(0.01)),
# MaxPooling2D(),
# Dropout(0.2),
# Conv2D(32, 3, padding='same', activation='relu'),
# MaxPooling2D(),
# Conv2D(64, 3, padding='same', activation='relu'),
# MaxPooling2D(),
# Dropout(0.2),
# Flatten(),
# Dense(512, activation='relu'),
# Dense(num_classes)
# ])
base_model = ResNet50(weights = None,
include_top = False,
input_shape = (IMG_HEIGHT, IMG_WIDTH, 3))
# Next 4 lines copied from:
#https://github.com/priya-dwivedi/Deep-Learning/blob/master/resnet_keras/Residual_Network_Keras.ipynb
x = base_model.output
x = GlobalAveragePooling2D()(x)
x = Dropout(0.2)(x)
predictions = Dense(num_classes, activation= 'softmax')(x)
model = Model(inputs = base_model.input, outputs = predictions)
if experiment=="binary":
model.compile(optimizer='adam',
loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
metrics=['accuracy'])
else:
model.compile(optimizer = Adam(lr = 0.0001),
loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
metrics=['accuracy'])
model.summary()
aug = ImageDataGenerator(
rotation_range=30,
zoom_range=0.15,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.15,
horizontal_flip=True,
fill_mode="nearest")
histories.append(model.fit(
# fixed
train_images, train_labels,
# broken
# aug.flow(train_images, train_labels, batch_size = batch_size),
steps_per_epoch=num_train // batch_size,
epochs=epochs,
callbacks=[cp_callback],
validation_data=(test_images, test_labels),
validation_steps=num_test // batch_size
))
if experiment == "disagreement":
model.save_weights((checkpoint_path).format(epoch=0))
models.append(model)
else:
model.save_weights(checkpoint_path.format(epoch=0))
models.append(model)
aug = ImageDataGenerator(
rotation_range=30,
zoom_range=0.15,
width_shift_range=0.2,
height_shift_range=0.2,
shear_range=0.15,
horizontal_flip=True,
fill_mode="nearest")
if self_train:
if experiment == "disagreement":
## Augment dataset with unlabeled images
animal_model = models[0]
machine_model = models[1]
animal_train_labels = all_train_labels[0]
machine_train_labels = all_train_labels[1]
animal_test_labels = all_test_labels[0]
machine_test_labels = all_test_labels[1]
# Reading only 1000 images for lack of computing power
#unlabeled_images = read_all_images(binary_path + "unlabeled_X.bin")
unlabeled_images = read_some_images(binary_path + "unlabeled_X.bin",
int(100000 * subset_size))
confident = True
first = True
# While there are still confident labelings
while confident:
# First train supervised model
print("Train images and Animal, machine Label shape: ",
train_images.shape, animal_train_labels.shape, machine_train_labels.shape)
print("test images and Animal, machine Label shape: ",
test_images.shape, animal_test_labels.shape, machine_test_labels.shape)
# history = model.fit(
# aug.flow(train_images, train_labels, batch_size = batch_size),
# steps_per_epoch=num_train // batch_size,
# epochs=epochs,
# callbacks=[cp_callback],
# validation_data=(test_images, test_labels),
# validation_steps=num_test // batch_size
# )
if not first:
histories[0] = animal_model.fit(
aug.flow(train_images, animal_train_labels, batch_size = batch_size),
steps_per_epoch=num_train // batch_size,
epochs=epochs,
callbacks=[cp_callback],
validation_data=(test_images, animal_test_labels),
validation_steps=num_test // batch_size
)
histories[1] = machine_model.fit(
aug.flow(train_images, machine_train_labels, batch_size = batch_size),
steps_per_epoch=num_train // batch_size,
epochs=epochs,
callbacks=[cp_callback],
validation_data=(test_images, machine_test_labels),
validation_steps=num_test // batch_size
)
first = False
# print(train_images[0])
# Predict unlabeled examples and add confident ones
count = 0
if (len(unlabeled_images) == 0): break
unlabeled_predictionsA = animal_model.predict(unlabeled_images)
unlabeled_predictionsM = machine_model.predict(unlabeled_images)
# Converting this to probabilities:
# print(unlabeled_predictions, unlabeled_predictions.shape)
probs_A = tf.nn.softmax(unlabeled_predictionsA)
probs_M = tf.nn.softmax(unlabeled_predictionsM)
temp_labA = np.argmax(probs_A, axis = 1)
temp_labM = np.argmax(probs_M, axis = 1)
# Combining animal and machine labels to one
# animal_labels = [2,4,5,6,7,8,0]
# machine_labels = [1,3,9,10,0]
# If both predicts other or both predict non-other,
# it should not get selected for ST so wrong labels wont matter
new_animal_labels = []
new_machine_labels = []
for i in range(len(temp_labA)):
new_animal_labels.append(temp_labA[i])
new_machine_labels.append(temp_labM[i])
# new_labels.append(max(animal_labels[temp_labA[i]],
# machine_labels[temp_labM[i]]))
class_predsA = np.amax(probs_A, axis = 1)
class_predsM = np.amax(probs_M, axis = 1)
class_predsA = tf.reshape(class_predsA, [len(class_predsA)])
class_predsM = tf.reshape(class_predsM, [len(class_predsM)])
class_preds = class_predsA * class_predsM
assert class_preds.shape == class_predsA.shape
print("pred shape", class_predsA.shape)
to_remove_thresh = (class_preds >= threshold)
to_remove_xor = (temp_labA == 6) != (temp_labM == 4)
# XOR: one and only one of the predictions is 'other'
to_remove = np.logical_and(to_remove_thresh, to_remove_xor)
train_images = np.append(train_images, unlabeled_images[to_remove])
# print(new_a / nimal_labels.dtype)
new_animal_labels = np.array(new_animal_labels)[to_remove]
new_machine_labels = np.array(new_machine_labels)[to_remove]
animal_train_labels = np.append(animal_train_labels, new_animal_labels)
machine_train_labels = np.append(machine_train_labels, new_machine_labels)
count = np.sum(to_remove)
unlabeled_images = unlabeled_images[np.logical_not(to_remove)]
print("New datapoints: ", count)
print("Remaining Datapoints:" ,unlabeled_images.shape)
train_images = train_images.reshape(-1, 96, 96, 3)
#print(train_images[0])
# Recalculating num_train and batch_size:
num_train = len(animal_train_labels)
num_batches = math.ceil(num_train/batch_size)
batch_size = int(num_train/num_batches)
# No confident labelings left
if count == 0: #< 50:
confident = False
print("Exiting loop")
animal_model.save_weights((checkpoint_path.replace("machine","animal")).format(epoch=0))
machine_model.save_weights((checkpoint_path).format(epoch=0))
else:
# Reading only 1000 images for lack of computing power
#unlabeled_images = read_all_images(binary_path + "unlabeled_X.bin")
unlabeled_images = read_some_images(binary_path + "unlabeled_X.bin",
int(100000 * subset_size))
confident = True
# While there are still confident labelings
while confident:
# First train supervised model
print("Train images and Label shape: ",
train_images.shape, train_labels.shape)
histories[0] = model.fit(
aug.flow(train_images, train_labels, batch_size = batch_size),
steps_per_epoch=num_train // batch_size,
epochs=epochs,
callbacks=[cp_callback],
validation_data=(test_images, test_labels),
validation_steps=num_test // batch_size
)
# print(train_images[0])
# Predict unlabeled examples and add confident ones
count = 0
if (len(unlabeled_images) == 0): break
unlabeled_predictions = model.predict(unlabeled_images)
# Converting this to probabilities:
temp_lab = -1
if experiment != "binary":
probss = tf.nn.softmax(unlabeled_predictions)
class_preds = np.amax(probss, axis = 1)
temp_lab = np.argmax(probss, axis = 1)
else:
# print(unlabeled_predictions, unlabeled_predictions.shape)
class_preds = tf.nn.sigmoid(unlabeled_predictions)
temp_lab = tf.round(class_preds)
class_preds = tf.reshape(class_preds, [len(class_preds)])
# Version 1: Not vectorized
# to_keep = list(range(len(class_preds)))
# for i in range(len(class_preds)):
# if i % 50 == 0:
# print("Augmenting dataset " + str(i) + "/" + str(len(unlabeled_images)) + " complete")
# pred = class_preds[i]
# image = unlabeled_images[i]
# if np.amax(pred) >= threshold:
# train_images = np.append(train_images, image)
# train_labels = np.append(train_labels, temp_lab[i])
# # decrease size of unlabeled images
# to_keep.remove(i)
# count += 1
#unlabeled_images = unlabeled_images[to_keep, :, :, :]
# Version 2: Vectorized?
to_remove = class_preds >= threshold
train_images = np.append(train_images, unlabeled_images[to_remove])
train_labels = np.append(train_labels, temp_lab[to_remove])
count = np.sum(to_remove)
unlabeled_images = unlabeled_images[np.logical_not(to_remove)]
print(count)
print(unlabeled_images.shape)
#train_images = train_images.reshape(-1, 3, 96, 96)
#train_images = np.transpose(train_images, (0, 3, 2, 1))
train_images = train_images.reshape(-1, 96, 96, 3)
#print(train_images[0])
# Recalculating num_train and batch_size:
num_train = len(train_labels)
num_batches = math.ceil(num_train/batch_size)
batch_size = int(num_train/num_batches)
# No confident labelings left
if count == 0: #< 50:
confident = False
print("Exiting loop")
model.save_weights(checkpoint_path.format(epoch=0))
# model.save(expe)
for history in histories:
acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss=history.history['loss']
val_loss=history.history['val_loss']
epochs_range = range(epochs)
plt.figure(figsize=(8, 8))
plt.subplot(1, 2, 1)
print("Training Accuracy:", acc)
print("Val Accuracy", val_acc)
plt.plot(epochs_range, acc, label='Training Accuracy')
plt.plot(epochs_range, val_acc, label='Validation Accuracy')
plt.legend(loc='lower right')
plt.title('Training and Validation Accuracy')
plt.subplot(1, 2, 2)
plt.plot(epochs_range, loss, label='Training Loss')
plt.plot(epochs_range, val_loss, label='Validation Loss')
plt.legend(loc='upper right')
plt.title('Training and Validation Loss')
plt.show()
if experiment == "disagreement":
animal_labels = [2,4,5,6,7,8]
machine_labels = [1,3,9,10]
animal_model = models[0]
machine_model = models[1]
animal_predictions = animal_model.predict(test_images)
animal_predictions = tf.nn.softmax(animal_predictions)
print("animal predictions shape: ", animal_predictions.shape)
animal_class_preds = np.amax(animal_predictions[:, 0:6], axis = 1)
animal_other_preds = animal_predictions[:, 6]
animal_class_labels = np.argmax(animal_predictions[:, 0:6], axis = 1)
machine_predictions = machine_model.predict(test_images)
print("machine predictions shape: ", machine_predictions.shape)
machine_class_preds = np.amax(machine_predictions[:, 0:4], axis = 1)
machine_other_preds = machine_predictions[:, 4]
machine_class_labels = np.argmax(machine_predictions[:, 0:4], axis = 1)
model_preds = []
for i in range(len(test_labels)):
if animal_class_preds[i] * machine_other_preds[i] > \
animal_other_preds[i] * machine_class_preds[i]:
model_preds.append(animal_labels[animal_class_labels[i]])
else:
model_preds.append(machine_labels[machine_class_labels[i]])
valid_accuracy = np.sum(model_preds == test_labels) / len(test_labels)
print("validation accuracy: ", valid_accuracy)
def main(argv):
opts, args = getopt.getopt(
argv, "hi:o:",
["a=", "b=", "m=", "animal_epoch=", "machine_epoch=", "binary_epoch="])
animal_path = ""
machine_path = ""
binary_path = ""
training_path = "training/"
for opt, arg in opts:
if opt in ("-a", "--animal_epoch"):
animal_path = training_path + "animal-cp-{:04d}.ckpt".format(
int(arg))
if opt in ("-m", "--machine_epoch"):
machine_path = training_path + "machine-cp-{:04d}.ckpt".format(
int(arg))
if opt in ("-b", "--binary_epoch"):
binary_path = training_path + "binary-cp-{:04d}.ckpt".format(
int(arg))
# machine = 0, animal = 1 for binary
animal_labels = [2, 4, 5, 6, 7, 8]
machine_labels = [1, 3, 9, 10]
print(binary_path, animal_path, machine_path)
# exit()
binary_model = get_model(binary_path, 1)
animal_model = get_model(animal_path, 6)
machine_model = get_model(machine_path, 4)
IMG_HEIGHT = 96
IMG_WIDTH = 96
binary_path = "data/stl10_binary/"
# train_images = read_all_images(binary_path + "train_X.bin")
# train_labels = read_labels(binary_path + "train_y.bin")
test_images = read_all_images(binary_path + "test_X.bin")
test_labels = read_labels(binary_path + "test_y.bin")
animal_labels = [2, 4, 5, 6, 7, 8]
machine_labels = [1, 3, 9, 10]
binary_labels = [0, 1]
# converted_labels = convert_labels(test_labels)
# for i in range(len(test_images)):
# return
test_binary = True
test_machine = True
test_animal = True
test_moe = True
# if test_binary:
# binary_logits = binary_model.predict(test_images)
# binary_preds = tf.round(tf.nn.sigmoid(binary_logits))
# correct = 0
# total = 0
# binary_labels, binary_images = convert_labels(test_labels, test_images, "binary")
# for i in range(len(binary_images)):
# prediction = int(binary_preds[i])
# if prediction == binary_labels[i]:
# correct += 1
# total += 1
# print("Binary reported accuracy: {:5.2f}%".format(100 * correct/total))
# loss, acc = binary_model.evaluate(test_images, binary_labels , verbose=2)
# print("Binary true accuracy: {:5.2f}%".format(100 * acc))
def test_model(name, model):
correct = 0
total = 0
converted_class_labels, class_images = convert_labels(
test_labels, test_images, name)
if name == "binary":
for i in range(len(converted_class_labels)):
if converted_class_labels[i] != int(
test_labels[i] in animal_labels):
print(
"Error in conversion for binary classification - exiting"
)
exit()
class_logits = model.predict(class_images)
if name != "binary":
class_preds = np.argmax(tf.nn.softmax(class_logits), axis=1)
else:
# print(class_logits, class_logits.shape)
class_preds = tf.round(tf.nn.sigmoid(class_logits))
class_preds = tf.reshape(class_preds, [len(class_preds)])
# class_preds = np.argmax(class_preds, axis=1)
for i in range(len(class_images)):
prediction = int(class_preds[i])
total += 1
if prediction == converted_class_labels[i]:
correct += 1
print(name +
" reported accuracy: {:5.2f}%".format(100 * correct / total))
loss, acc = model.evaluate(class_images,
converted_class_labels,
verbose=2)
print(name + " true accuracy: {:5.2f}%".format(100 * acc))
if test_binary:
test_model("binary", binary_model)
if test_machine:
test_model("machine", machine_model)
if test_animal:
test_model("animal", animal_model)
if test_moe:
binary_logits = binary_model.predict(test_images)
binary_preds = tf.round(tf.nn.sigmoid(binary_logits))
machine_logits = machine_model.predict(test_images)
machine_preds = tf.nn.softmax(machine_logits)
animal_logits = animal_model.predict(test_images)
animal_preds = tf.nn.softmax(animal_logits)
correct = 0
total = 0
for i in range(len(test_images)):
prediction = -1
binary_prediction = int(binary_preds[i])
if binary_prediction == 0:
prediction = machine_labels[np.argmax(machine_preds[i])]
else:
prediction = animal_labels[np.argmax(animal_preds[i])]
if prediction == test_labels[i]:
correct += 1
total += 1
print("MoE reported accuracy: {:5.2f}%".format(100 * correct / total))
def main(_):
print("\nParameters:")
for attr, value in tf.app.flags.FLAGS.flag_values_dict().items():
print("{}={}".format(attr, value))
print("")
# os.environ["CUDA_VISIBLE_DEVICES"] = str(FLAGS.gpu)
if not os.path.exists(FLAGS.logdir):
os.makedirs(FLAGS.logdir)
# Random seed
rng = np.random.RandomState(FLAGS.seed) # seed labels
rng_data = np.random.RandomState(rng.randint(0, 2**10)) # seed shuffling
# load CIFAR-10
# trainx, trainy = cifar10_input._get_dataset(FLAGS.data_dir, 'train') # float [-1 1] images
# testx, testy = cifar10_input._get_dataset(FLAGS.data_dir, 'test')
trainx = stl10_input.read_all_images(stl10_input.TRAINX_PATH)
testx = stl10_input.read_all_images(stl10_input.TESTX_PATH)
trainy = stl10_input.read_labels(stl10_input.TRAINY_PATH)
testy = stl10_input.read_labels(stl10_input.TESTY_PATH)
trainx_unl = stl10_input.read_all_images(stl10_input.UNL_PATH)[:50000]
trainx_unl2 = trainx_unl.copy()
if FLAGS.validation:
split = int(0.1 * trainx.shape[0])
print("validation enabled")
testx = trainx[:split]
testy = trainy[:split]
trainx = trainx[split:]
trainy = trainy[split:]
nr_batches_train = int(trainx.shape[0] / FLAGS.batch_size)
nr_batches_test = int(testx.shape[0] / FLAGS.batch_size)
# select labeled data
inds = rng_data.permutation(trainx.shape[0])
trainx = trainx[inds]
trainy = trainy[inds]
print('seed trainy:', trainy)
txs = []
tys = []
for j in range(10):
txs.append(trainx[trainy == j][:FLAGS.labeled])
tys.append(trainy[trainy == j][:FLAGS.labeled])
txs = np.concatenate(txs, axis=0)
tys = np.concatenate(tys, axis=0)
print('train examples %d, batch %d, test examples %d, batch %d' \
% (trainx.shape[0], nr_batches_train, testx.shape[0], nr_batches_test))
print('unl shape: ', trainx_unl.shape)
print('hist train', np.histogram(trainy, bins=10)[0])
print('hist test ', np.histogram(testy, bins=10)[0])
print("histlabeled", np.histogram(tys, bins=10)[0])
print("")
'''construct graph'''
unl = tf.placeholder(tf.float32, [FLAGS.batch_size, 96, 96, 3],
name='unlabeled_data_input_pl')
is_training_pl = tf.placeholder(tf.bool, [], name='is_training_pl')
inp = tf.placeholder(tf.float32, [FLAGS.batch_size, 96, 96, 3],
name='labeled_data_input_pl')
lbl = tf.placeholder(tf.int32, [FLAGS.batch_size], name='lbl_input_pl')
# scalar pl
lr_pl = tf.placeholder(tf.float32, [], name='learning_rate_pl')
acc_train_pl = tf.placeholder(tf.float32, [], 'acc_train_pl')
acc_test_pl = tf.placeholder(tf.float32, [], 'acc_test_pl')
acc_test_pl_ema = tf.placeholder(tf.float32, [], 'acc_test_pl')
random_z = tf.random_uniform([FLAGS.batch_size, 100], name='random_z')
generator(random_z, is_training_pl,
init=True) # init of weightnorm weights
gen_inp = generator(random_z, is_training_pl, init=False, reuse=True)
pert_n = tf.nn.l2_normalize(
tf.random_normal(shape=[FLAGS.batch_size, 100]), dim=[1])
random_z_pert = random_z + FLAGS.eta * pert_n
gen_inp_pert = generator(random_z_pert,
is_training=is_training_pl,
init=False,
reuse=True)
gen_adv = gen_inp + FLAGS.epsilon * tf.nn.l2_normalize(
gen_inp_pert - gen_inp, dim=[1, 2, 3])
if FLAGS.resize:
print('resizing')
inp_ = tf.image.resize_images(inp, (SHAPE, SHAPE))
unl_ = tf.image.resize_images(unl, (SHAPE, SHAPE))
if FLAGS.augmentation:
print('augmentation')
inp_aug = nn.flip_randomly(inp_, True, False, is_training_pl)
inp_aug = nn.random_translate(inp_aug, FLAGS.translate, is_training_pl)
unl_aug = nn.flip_randomly(unl_, True, False, is_training_pl)
unl_aug = nn.random_translate(unl_aug, FLAGS.translate, is_training_pl)
else:
unl_aug = unl
inp_aug = inp
discriminator(unl, is_training_pl, init=True)
logits_lab, _ = discriminator(inp_aug,
is_training_pl,
init=False,
reuse=True)
logits_gen, layer_fake = discriminator(gen_inp,
is_training_pl,
init=False,
reuse=True)
logits_unl, layer_real = discriminator(unl_aug,
is_training_pl,
init=False,
reuse=True)
logits_gen_adv, _ = discriminator(gen_adv,
is_training_pl,
init=False,
reuse=True)
with tf.name_scope('loss_functions'):
# discriminator
l_unl = tf.reduce_logsumexp(logits_unl, axis=1)
l_gen = tf.reduce_logsumexp(logits_gen, axis=1)
loss_lab = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(labels=lbl,
logits=logits_lab))
loss_unl = - 0.5 * tf.reduce_mean(l_unl) \
+ 0.5 * tf.reduce_mean(tf.nn.softplus(l_unl)) \
+ 0.5 * tf.reduce_mean(tf.nn.softplus(l_gen))
# generator
m1 = tf.reduce_mean(layer_real, axis=0)
m2 = tf.reduce_mean(layer_fake, axis=0)
manifold = tf.reduce_sum(
tf.sqrt(tf.square(logits_gen - logits_gen_adv) + 1e-8), axis=1)
j_loss = tf.reduce_mean(manifold)
if FLAGS.nabla == 1:
loss_dis = FLAGS.unl_weight * loss_unl + FLAGS.lbl_weight * loss_lab + FLAGS.gamma * j_loss
loss_gen = tf.reduce_mean(tf.abs(m1 - m2))
print('manifold reg enabled')
elif FLAGS.nabla == 2:
pz = tf.random_normal([FLAGS.batch_size, 32, 32, 3])
pert_n = FLAGS.epsilon * tf.nn.l2_normalize(pz, dim=[1, 2, 3])
logits_unl_pert, layer_real = discriminator(unl + pert_n,
is_training_pl,
init=False,
reuse=True)
ambient = tf.reduce_sum(
tf.sqrt(tf.square(logits_unl - logits_unl_pert) + 1e-8),
axis=1)
ambient_loss = tf.reduce_mean(ambient)
print('ambient enabled')
loss_dis = FLAGS.unl_weight * loss_unl + FLAGS.lbl_weight * loss_lab + FLAGS.gamma * ambient_loss
loss_gen = tf.reduce_mean(tf.abs(m1 - m2))
else:
loss_dis = FLAGS.unl_weight * loss_unl + FLAGS.lbl_weight * loss_lab
loss_gen = tf.reduce_mean(tf.abs(m1 - m2))
print('vanilla reg')
correct_pred = tf.equal(tf.cast(tf.argmax(logits_lab, 1), tf.int32),
tf.cast(lbl, tf.int32))
accuracy_classifier = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
with tf.name_scope('optimizers'):
# control op dependencies for batch norm and trainable variables
tvars = tf.trainable_variables()
dvars = [var for var in tvars if 'discriminator_model' in var.name]
gvars = [var for var in tvars if 'generator_model' in var.name]
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
update_ops_gen = [
x for x in update_ops if ('generator_model' in x.name)
]
update_ops_dis = [
x for x in update_ops if ('discriminator_model' in x.name)
]
optimizer_dis = tf.train.AdamOptimizer(learning_rate=lr_pl,
beta1=0.5,
name='dis_optimizer')
optimizer_gen = tf.train.AdamOptimizer(learning_rate=lr_pl,
beta1=0.5,
name='gen_optimizer')
with tf.control_dependencies(update_ops_gen):
train_gen_op = optimizer_gen.minimize(loss_gen, var_list=gvars)
dis_op = optimizer_dis.minimize(loss_dis, var_list=dvars)
ema = tf.train.ExponentialMovingAverage(decay=FLAGS.ma_decay)
maintain_averages_op = ema.apply(dvars)
with tf.control_dependencies([dis_op]):
train_dis_op = tf.group(maintain_averages_op)
logits_ema, _ = discriminator(inp_,
is_training_pl,
getter=get_getter(ema),
reuse=True)
correct_pred_ema = tf.equal(
tf.cast(tf.argmax(logits_ema, 1), tf.int32),
tf.cast(lbl, tf.int32))
accuracy_ema = tf.reduce_mean(tf.cast(correct_pred_ema, tf.float32))
with tf.name_scope('summary'):
with tf.name_scope('discriminator'):
tf.summary.scalar('loss_discriminator', loss_dis, ['dis'])
tf.summary.scalar('kl_loss', j_loss, ['dis'])
with tf.name_scope('generator'):
tf.summary.scalar('loss_generator', loss_gen, ['gen'])
with tf.name_scope('images'):
tf.summary.image('gen_images', gen_inp, 10, ['image'])
tf.summary.image('inp_images', inp_aug, 10, ['image'])
with tf.name_scope('epoch'):
tf.summary.scalar('accuracy_train', acc_train_pl, ['epoch'])
tf.summary.scalar('accuracy_test_moving_average', acc_test_pl_ema,
['epoch'])
tf.summary.scalar('accuracy_test_raw', acc_test_pl, ['epoch'])
tf.summary.scalar('learning_rate', lr_pl, ['epoch'])
sum_op_dis = tf.summary.merge_all('dis')
sum_op_gen = tf.summary.merge_all('gen')
sum_op_im = tf.summary.merge_all('image')
sum_op_epoch = tf.summary.merge_all('epoch')
# training global varialble
global_epoch = tf.Variable(0, trainable=False, name='global_epoch')
global_step = tf.Variable(0, trainable=False, name='global_step')
inc_global_step = tf.assign(global_step, global_step + 1)
inc_global_epoch = tf.assign(global_epoch, global_epoch + 1)
# op initializer for session manager
init_gen = [var.initializer for var in gvars][:-3]
with tf.control_dependencies(init_gen):
op = tf.global_variables_initializer()
init_feed_dict = {
inp: trainx_unl[:FLAGS.batch_size],
unl: trainx_unl[:FLAGS.batch_size],
is_training_pl: True
}
sv = tf.train.Supervisor(logdir=FLAGS.logdir,
global_step=global_epoch,
summary_op=None,
save_model_secs=0,
init_op=op,
init_feed_dict=init_feed_dict)
'''//////training //////'''
print('start training')
with sv.managed_session() as sess:
tf.set_random_seed(rng.randint(2**10))
print('\ninitialization done')
print('Starting training from epoch :%d, step:%d \n' %
(sess.run(global_epoch), sess.run(global_step)))
writer = tf.summary.FileWriter(FLAGS.logdir, sess.graph)
while not sv.should_stop():
epoch = sess.run(global_epoch)
train_batch = sess.run(global_step)
if (epoch >= FLAGS.epoch):
print("Training done")
sv.stop()
break
begin = time.time()
train_loss_lab = train_loss_unl = train_loss_gen = train_acc = test_acc = test_acc_ma = train_j_loss = 0
lr = FLAGS.learning_rate * linear_decay(FLAGS.decay_start,
FLAGS.epoch, epoch)
# construct randomly permuted batches
trainx = []
trainy = []
for t in range(
int(np.ceil(
trainx_unl.shape[0] /
float(txs.shape[0])))): # same size lbl and unlb
inds = rng.permutation(txs.shape[0])
trainx.append(txs[inds])
trainy.append(tys[inds])
trainx = np.concatenate(trainx, axis=0)
trainy = np.concatenate(trainy, axis=0)
trainx_unl = trainx_unl[rng.permutation(
trainx_unl.shape[0])] # shuffling unl dataset
trainx_unl2 = trainx_unl2[rng.permutation(trainx_unl2.shape[0])]
# training
for t in range(nr_batches_train):
# display_progression_epoch(t, nr_batches_train)
ran_from = t * FLAGS.batch_size
ran_to = (t + 1) * FLAGS.batch_size
# train discriminator
feed_dict = {
unl: trainx_unl[ran_from:ran_to],
is_training_pl: True,
inp: trainx[ran_from:ran_to],
lbl: trainy[ran_from:ran_to],
lr_pl: lr
}
_, acc, lu, lb, jl, sm = sess.run([
train_dis_op, accuracy_classifier, loss_lab, loss_unl,
j_loss, sum_op_dis
],
feed_dict=feed_dict)
train_loss_unl += lu
train_loss_lab += lb
train_acc += acc
train_j_loss += jl
if (train_batch % FLAGS.step_print) == 0:
writer.add_summary(sm, train_batch)
# train generator
_, lg, sm = sess.run(
[train_gen_op, loss_gen, sum_op_gen],
feed_dict={
unl: trainx_unl2[ran_from:ran_to],
is_training_pl: True,
lr_pl: lr
})
train_loss_gen += lg
if (train_batch % FLAGS.step_print) == 0:
writer.add_summary(sm, train_batch)
if (train_batch % FLAGS.freq_print == 0) & (train_batch != 0):
# ran_from = np.random.randint(0, trainx_unl.shape[0] - FLAGS.batch_size)
# ran_to = ran_from + FLAGS.batch_size
ran_from = 0
ran_to = FLAGS.batch_size
sm = sess.run(sum_op_im,
feed_dict={
is_training_pl: True,
inp: testx[ran_from:ran_to]
})
writer.add_summary(sm, train_batch)
train_batch += 1
sess.run(inc_global_step)
train_loss_lab /= nr_batches_train
train_loss_unl /= nr_batches_train
train_loss_gen /= nr_batches_train
train_acc /= nr_batches_train
train_j_loss /= nr_batches_train
# Testing moving averaged model and raw model
if (epoch % FLAGS.freq_test == 0) | (epoch == FLAGS.epoch - 1):
for t in range(nr_batches_test):
ran_from = t * FLAGS.batch_size
ran_to = (t + 1) * FLAGS.batch_size
feed_dict = {
inp: testx[ran_from:ran_to],
lbl: testy[ran_from:ran_to],
is_training_pl: False
}
acc, acc_ema = sess.run(
[accuracy_classifier, accuracy_ema],
feed_dict=feed_dict)
test_acc += acc
test_acc_ma += acc_ema
test_acc /= nr_batches_test
test_acc_ma /= nr_batches_test
sum = sess.run(sum_op_epoch,
feed_dict={
acc_train_pl: train_acc,
acc_test_pl: test_acc,
acc_test_pl_ema: test_acc_ma,
lr_pl: lr
})
writer.add_summary(sum, epoch)
print(
"Epoch %d | time = %ds | loss gen = %.4f | loss lab = %.4f | loss unl = %.4f "
"| train acc = %.4f| test acc = %.4f | test acc ema = %0.4f"
% (epoch, time.time() - begin, train_loss_gen,
train_loss_lab, train_loss_unl, train_acc, test_acc,
test_acc_ma))
sess.run(inc_global_epoch)
# save snapshots of model
if ((epoch % FLAGS.freq_save == 0) &
(epoch != 0)) | (epoch == FLAGS.epoch - 1):
string = 'model-' + str(epoch)
save_path = os.path.join(FLAGS.logdir, string)
sv.saver.save(sess, save_path)
print("Model saved in file: %s" % (save_path))
def main(argv):
opts, args = getopt.getopt(argv, "hi:o:", [
"test=", "experiment=", "subset=", "self_train=", "epochs=", "batch=",
"threshold="
])
self_train = False
subset_size = 1
epochs = 15
batch_size = 1000
threshold = .75
experiment = "default"
binary_classification = False
for opt, arg in opts:
if opt == '-h':
print(
'self-train.py --experiment=binary --subset=1 --self_train=True --epochs=20 --batch=1000 --threshold=.75'
)
sys.exit()
elif opt in ("-st", "--self_train"):
self_train = arg.lower in ["true", "True"]
elif opt in ("-ss", "--subset"):
subset_size = float(arg)
elif opt in ("-exp", "--experiment"):
experiment = arg
if experiment == "binary":
binary_classification = True
elif opt in ("-e", "--epochs"):
epochs = int(arg)
elif opt in ("-b", "--batch"):
batch_size = int(arg)
elif opt in ("-t", "--threshold"):
threshold = float(arg)
print(self_train)
IMG_HEIGHT = 96
IMG_WIDTH = 96
binary_path = "data/stl10_binary/"
train_images = read_all_images(binary_path + "train_X.bin")
train_labels = read_labels(binary_path + "train_y.bin")
test_images = read_all_images(binary_path + "test_X.bin")
test_labels = read_labels(binary_path + "test_y.bin")
checkpoint_path = "training/" + experiment + "-cp-{epoch:04d}.ckpt"
checkpoint_dir = os.path.dirname(checkpoint_path)
# Create a callback that saves the model's weights every 5 epochs
cp_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_path,
verbose=1,
save_weights_only=True,
period=5)
print(len(test_labels))
print(len(train_labels))
test_labels, test_images = convert_labels(test_labels, test_images,
experiment)
train_labels, train_images = convert_labels(train_labels, train_images,
experiment)
print(test_labels.shape)
print(train_labels.shape)
# exit()
num_train = math.floor(len(train_labels) * subset_size)
num_test = math.floor(len(test_labels) * subset_size)
test_labels = test_labels[:num_test]
train_labels = train_labels[:num_train]
test_images = test_images[:num_test]
train_images = train_images[:num_train]
num_batches = math.ceil(num_train / batch_size)
batch_size = int(num_train / num_batches)
print("Adjusted batch size to " + str(batch_size))
def plotImages(images_arr):
fig, axes = plt.subplots(1, 5, figsize=(20, 20))
axes = axes.flatten()
for img, ax in zip(images_arr, axes):
ax.imshow(img)
ax.axis('off')
plt.tight_layout()
plt.show()
#plotImages(train_images[:5])
if experiment == "binary":
num_classes = 1
elif experiment == "animal":
num_classes = 6
elif experiment == "machine":
num_classes = 4
else:
num_classes = 10
model = Sequential([
Conv2D(16,
3,
padding='same',
activation='relu',
input_shape=(IMG_HEIGHT, IMG_WIDTH, 3)),
MaxPooling2D(),
Dropout(0.2),
Conv2D(32, 3, padding='same', activation='relu'),
MaxPooling2D(),
Conv2D(64, 3, padding='same', activation='relu'),
MaxPooling2D(),
Dropout(0.2),
Flatten(),
Dense(512, activation='relu'),
Dense(num_classes)
])
if experiment == "binary":
model.compile(
optimizer='adam',
loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
metrics=['accuracy'])
else:
model.compile(optimizer='adam',
loss=tf.keras.losses.SparseCategoricalCrossentropy(
from_logits=True),
metrics=['accuracy'])
model.summary()
history = model.fit(train_images,
train_labels,
steps_per_epoch=num_train // batch_size,
epochs=epochs,
callbacks=[cp_callback],
validation_data=(test_images, test_labels),
validation_steps=num_test // batch_size)
model.save_weights(checkpoint_path.format(epoch=0))
## Augment dataset with unlabeled images
if self_train:
unlabeled_images = read_all_images(binary_path + "unlabeled_X.bin")
unlabeled_predictions = model.predict(unlabeled_images)
for i in range(len(unlabeled_predictions)):
if i % 50 == 0:
print("Augmenting dataset " + str(i) + "/" +
str(len(unlabeled_images)) + " complete")
pred = unlabeled_predictions[i]
image = unlabeled_images[i]
if np.amax(pred) >= .75:
train_images = np.append(train_images, image)
train_labels = np.append(train_labels, np.argmax(pred))
# train_images.append(image)
# converted_train_labels.append(np.argmax(pred))
history = model.fit(train_images,
train_labels,
steps_per_epoch=num_train // batch_size,
epochs=epochs,
validation_data=(test_images, test_labels),
validation_steps=num_test // batch_size)
# model.save(expe)
acc = history.history['accuracy']
val_acc = history.history['val_accuracy']
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs_range = range(epochs)
plt.figure(figsize=(8, 8))
plt.subplot(1, 2, 1)
print("Traing Accuracy:", acc)
print("Val Accuracy", val_acc)
plt.plot(epochs_range, acc, label='Training Accuracy')
plt.plot(epochs_range, val_acc, label='Validation Accuracy')
plt.legend(loc='lower right')
plt.title('Training and Validation Accuracy')
plt.subplot(1, 2, 2)
plt.plot(epochs_range, loss, label='Training Loss')
plt.plot(epochs_range, val_loss, label='Validation Loss')
plt.legend(loc='upper right')
plt.title('Training and Validation Loss')
plt.show()
def main(argv):
opts, args = getopt.getopt(argv, "hi:o:", ["a=", "b=", "m=", "animal_epoch=", "machine_epoch=", "binary_epoch="])
animal_path = ""
machine_path = ""
training_path = "training/"
for opt, arg in opts:
if opt in ("-a", "--animal_epoch"):
animal_path = training_path + "disagreement_animal-cp-{:04d}.ckpt".format(int(arg))
if opt in ("-m", "--machine_epoch"):
machine_path = training_path + "disagreement_machine-cp-{:04d}.ckpt".format(int(arg))
# if opt in ("-b", "--binary_epoch"):
# binary_path = training_path + "binary-cp-{:04d}.ckpt".format(int(arg))
# machine = 0, animal = 1 for binary
animal_labels = [2, 4, 5, 6, 7, 8]
machine_labels = [1, 3, 9, 10]
print("Path information:", animal_path, machine_path)
# exit()
animal_model = get_model(animal_path,7)
machine_model = get_model(machine_path,5)
IMG_HEIGHT = 96
IMG_WIDTH = 96
binary_path = "data/stl10_binary/"
# train_images = read_all_images(binary_path + "train_X.bin")
# train_labels = read_labels(binary_path + "train_y.bin")
test_images = read_all_images(binary_path + "test_X.bin")
test_labels = read_labels(binary_path + "test_y.bin")
animal_labels = [2, 4, 5, 6, 7, 8]
machine_labels = [1, 3, 9, 10]
test_machine=True
test_animal=True
test_disagreement=True
def test_model(name, model):
correct = 0
total = 0
converted_class_labels, class_images = convert_labels(test_labels, test_images, name)
class_logits = model.predict(class_images)
class_preds = np.argmax(tf.nn.softmax(class_logits), axis=1)
# class_preds = tf.round(tf.nn.sigmoid(class_logits))
class_preds = tf.reshape(class_preds, [len(class_preds)])
for i in range(len(class_images)):
prediction = int(class_preds[i])
total += 1
if prediction == converted_class_labels[i]:
correct += 1
print(name + " reported accuracy: {:5.2f}%".format(100 * correct / total))
loss, acc = model.evaluate(class_images, converted_class_labels, verbose=2)
print(name + " true accuracy: {:5.2f}%".format(100 * acc))
if test_machine:
test_model("disagreement_machine", machine_model)
if test_animal:
test_model("disagreement_animal", animal_model)
if test_disagreement:
animal_labels = [2, 4, 5, 6, 7, 8]
machine_labels = [1, 3, 9, 10]
animal_predictions = animal_model.predict(test_images)
animal_predictions = tf.nn.softmax(animal_predictions)
print("animal predictions shape: ", animal_predictions.shape)
animal_class_preds = np.amax(animal_predictions[:, 0:6], axis=1)
animal_other_preds = animal_predictions[:, 6]
animal_class_labels = np.argmax(animal_predictions[:, 0:6], axis=1)
machine_predictions = machine_model.predict(test_images)
print("machine predictions shape: ", machine_predictions.shape)
machine_class_preds = np.amax(machine_predictions[:, 0:4], axis=1)
machine_other_preds = machine_predictions[:, 4]
machine_class_labels = np.argmax(machine_predictions[:, 0:4], axis=1)
model_preds = []
for i in range(len(test_labels)):
if animal_class_preds[i] * machine_other_preds[i] > \
animal_other_preds[i] * machine_class_preds[i]:
model_preds.append(animal_labels[animal_class_labels[i]])
else:
model_preds.append(machine_labels[machine_class_labels[i]])
valid_accuracy = np.sum(model_preds == test_labels) / len(test_labels)
print("validation accuracy: ", valid_accuracy)
print("Disagreement reported accuracy: {:5.2f}%".format(100 * valid_accuracy))
