def gen_test_data(self, data_path):
print("Generate fc7 from testing data...")
data_path = self.data_dir + data_path
aug_data_path = self.data_dir + '/train_augment'
test_data = BaseDataSet(data_path, 500, 50)
test_data.image_list = sorted(test_data.image_list)
test_data_set = []
test_label_set = []
for image_path in test_data.image_list:
data, label = test_data.read_image(image_path)
if '0009.jpg' in image_path or '0010.jpg' in image_path:
print 'test:', image_path
test_data_set.append(data)
test_label_set.append(int(label) - 1)
train_data = BaseDataSet(aug_data_path, 500, 50)
train_data.image_list = sorted(train_data.image_list)
train_data_set = []
train_label_set = []
for image_path in train_data.image_list:
print 'Train:', image_path
data, label = train_data.read_image(image_path)
train_data_set.append(data)
train_label_set.append(int(label) - 1)
train_fc7 = []
test_fc7 = []
with tf.Session(config=tf.ConfigProto(
allow_soft_placement=True)) as sess:
inputs = tf.placeholder(tf.float32, [1, 227, 227, 3],
name="input_image")
alexnet = AlexNet(inputs,
keep_prob=1.0,
num_classes=1000,
skip_layer=[])
tf.global_variables_initializer().run()
alexnet.load_initial_weights(sess)
for i in range(len(train_data_set)):
fc7 = sess.run(
[alexnet.fc7],
feed_dict={
alexnet.X:
np.array(train_data_set[i]).reshape([1, 227, 227, 3])
})
train_fc7.append(fc7[0].reshape([4096]))
for i in range(len(test_data_set)):
fc7 = sess.run(
[alexnet.fc7],
feed_dict={
alexnet.X:
np.array(test_data_set[i]).reshape([1, 227, 227, 3])
})
test_fc7.append(fc7[0].reshape([4096]))
np.save(self.data_dir + '/train_fc7.npy', np.array(train_fc7))
np.save(self.data_dir + '/valid_fc7.npy', np.array(test_fc7))
np.save(self.data_dir + '/train_label.npy', np.array(train_label_set))
np.save(self.data_dir + '/valid_label.npy', np.array(test_label_set))
print('Finish generating!')
def get_encodings(self):
train_layers = []
batch_size = self.alexnet_batch_size
tf.reset_default_graph()
X = tf.placeholder(tf.float32, [batch_size, 227, 227, 3])
model = AlexNet(X, 1.0, 1000, train_layers)
encoding = model.fc7 # A 4096 x 1 encoding from AlexNet
# Initializing the data-generator.
img_utils = ImageDataGenerator(data_dir="tid2013/",
inp_size=[227, 227],
scale_size=[256, 256],
horizontal_flip=True)
img_encs = {}
mos = {}
for d in range(6):
img_encs['level' + str(d)] = np.ndarray(
[self.NUM_EXAMPLES, self.dim])
mos['level' + str(d)] = np.ndarray([
self.NUM_EXAMPLES,
])
with tf.Session() as sess:
# Initializing the variables.
tf.global_variables_initializer()
# Initializing the weights.
model.load_initial_weights(sess)
for _ in range(self.NUM_EXAMPLES // batch_size + 1):
# Sampling a batch
imgs_batch, mos_batch = img_utils.get_next_batch(batch_size)
first = img_utils.first
last = img_utils.last
for d in range(6):
img_enc = sess.run(
encoding, feed_dict={X: imgs_batch['level' + str(d)]})
img_encs['level' + str(d)][first:last] = img_enc
mos['level' + str(d)][first:last] = mos_batch['level' +
str(d)]
sess.close()
self.img_encs = img_encs
self.mos = mos
# Saving the data.
data = {'encodings': img_encs, 'scores': mos}
savemat('encoded_data.mat', data)
def train_model(image_path, learning_rate=0.001, num_epoch=100, batch_size=5):
X_train = np.load(image_path + ".npy")
Y_train = np.load(image_path + ".npy")
X_train = X_train / 255
#Creating one-hot for label
print(Y_train.shape)
print(Y_train[0][1])
Y_train = convert_to_one_hot(Y_train, 10).T
n_Y = Y_train.shape[1]
x = tf.placeholder(tf.float32, [None, 227, 227, 3])
Y = tf.placeholder(tf.float32, [None, n_Y])
keep_prob = tf.placeholder(tf.float32)
model = AlexNet(x, keep_prob, 10, [])
score = model.fc8
cost = tf.reduce_mean(
(tf.nn.softmax_cross_entropy_with_logits_v2(logits=score, labels=Y)))
softmax = tf.nn.softmax(score)
costs = []
optimizer = tf.train.AdamOptimizer(
learning_rate=learning_rate).minimize(cost)
init = tf.global_variables_initializer()
[m, nH, nW, nCH] = X_train.shape
seed = 0
#tf.reset_default_graph()
with tf.Session() as sess:
sess.run(init)
sess.run(model.load_initial_weights(sess))
for i in range(num_epoch):
minibatch_cost = 0
num_minibatch = int(m / batch_size) # number of batches
seed = seed + 1
minibatches = random_mini_batches(X_train, Y_train, batch_size,
seed)
for minibatch in minibatches:
X_batches, Y_batches = minibatch
#Use cost and optimizer to run
_, c = sess.run([optimizer, cost], {
x: X_batches,
Y: Y_batches,
keep_prob: 1
})
minibatch_cost += c
minibatch_cost = minibatch_cost / num_minibatch
if i % 5 == 0:
print("Cost===>" + str(c))
costs.append(c)
plt.plot(np.squeeze(costs))
plt.ylabel('cost')
plt.xlabel('epoch')
plt.title('learning_rate' + str(learning_rate))
plt.show()
def run(img_path=img_file_path):
#placeholder for input and dropout rate
x = tf.placeholder(tf.float32, [1, 227, 227, 3])
keep_prob = tf.placeholder(tf.float32)
#create model with default config ( == no skip_layer and 1000 units in the last layer)
model = AlexNet(x, keep_prob, 1000, [])
#define activation of last layer as score
score = model.fc8
#create op to calculate softmax
softmax = tf.nn.softmax(score)
sess = tf.Session()
# Load the pretrained weights into the model
sess.run(model.load_initial_weights(sess))
image = cv2.imread(img_path)
# Convert image to float32 and resize to (227x227)
img = cv2.resize(image.astype(np.float32), (227,227))
# Subtract the ImageNet mean
img -= imagenet_mean
# Reshape as needed to feed into model
img = img.reshape((1,227,227,3))
print(img.dtype)
# Run the session and calculate the class probability
probs = sess.run(softmax, feed_dict={x: img, keep_prob: 1})
# Get the class name of the class with the highest probability
class_name = class_names[np.argmax(probs)]
print("Class: " + class_name + ", probability: %.4f" %probs[0,np.argmax(probs)])
# Initialize an saver for store model checkpoints
saver = tf.train.Saver()
# Start Tensorflow session
with tf.Session() as sess:
# Add the model graph to TensorBoard
writer.add_graph(sess.graph)
# Load the Imagenet pretrained weights into the non-trainable layer
# Finetune the whole network
if newmodel:
# Initialize all variables
sess.run(tf.global_variables_initializer())
model.load_initial_weights(sess, trainablev=True, layersList=['fc8'])
else:
saver.restore(sess, pre_model)
print("{} Start training...".format(datetime.now()))
print("{} Open Tensorboard at --logdir {}".format(datetime.now(),
filewriter_path))
# Loop over number of epochs
for epoch in range(num_epochs):
print("{} Epoch number: {}".format(datetime.now(), epoch + 1))
# Initialize iterator with the training dataset
sess.run(training_init_op)
for step in range(train_batches_per_epoch):
class Hand(object):
"""docstring for Hand"""
def __init__(self):
super(Hand, self).__init__()
# Network params
learning_rate = 0.03
num_epochs = 10
batch_size = 1
display_step = 4
dropout = 0.5
num_classes = 7
train_layers = ['fc8']
print(batch_size)
self.x = tf.placeholder(tf.float32, [ 1,227, 227, 3])
y = tf.placeholder(tf.float32, [ batch_size,num_classes])
self.keep_prob = tf.placeholder(tf.float32)
self.model = AlexNet(self.x, self.keep_prob, num_classes, train_layers)
cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = self.model.fc8, labels = y))
optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost,var_list = tf.global_variables()[-2:])
correct_pred = tf.equal(tf.argmax(self.model.fc8,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
self.saver = tf.train.Saver(tf.global_variables())
self.sess = tf.InteractiveSession()
self.model.load_initial_weights(self.sess)
self.saver.restore(self.sess,"Model/model.ckpt")
def predict(self,img):
img_array = np.array(img)
x_test = np.zeros((1,227,227,3))
x_test[0] = img_array
# mean_px = x_test.mean().astype(np.float32)
# std_px = x_test.std().astype(np.float32)
# x_test =( x_test - mean_px)/std_px
pre = self.model.fc8.eval(feed_dict={self.x:x_test[0:1],self.keep_prob:1.})
rank = 0;
now = -1;
result = 6;
# print(np.shape(pre))
# print((pre))
for x in pre[0]:
rank += 1
if now < x:
now = x
result = rank
return result
class Model(object):
def __init__(self, mode='train'):
'''
Model class for you to modify for brick/ball/cylinder deep learning challenge.
mode = 'train' or 'test' - could be a useful flag if we want to use dataset augmentations.
'''
if mode == 'train' or mode == 'test':
self.mode = mode
else:
print('Mode must be train or test.')
# Learning params
self.learning_rate = 1e-3
self.minibatch_size = 32
self.num_iterations = 1500
self.train_test_split = 0.7
# Network params
self.keep_rate = 0.5 #Fraction of connections that we keep when performing dropout
# How many pretrained alexnet layers to we want to load up?
# This must be a number between 1 and 8.
self.num_layers_to_load = 5
# How often we want to write the tf.summary data to disk and measure performance
self.display_step = 5
self.save_step = 100
# Path for tf.summary.FileWriter and to store model checkpoints
self.save_dir = "tf_data/sample_model"
self.input_image_size = (227, 227)
self.label_indices = {'brick': 0, 'ball': 1, 'cylinder': 2}
self.labels_ordered = list(self.label_indices)
self.num_classes = len(self.label_indices)
self.channel_means = np.array([147.12697, 160.21092, 167.70029])
#Go ahead and build graph on initialization:
self.build_graph()
#Set to true with we create a tf session.
self.initialized = False
def build_graph(self):
'''
This is where you can experiment with various architectures.
'''
# TF placeholder for graph input and output
self.X = tf.placeholder(
tf.float32,
[None, self.input_image_size[0], self.input_image_size[1], 3])
self.y = tf.placeholder(tf.float32, [None, self.num_classes])
self.keep_prob = tf.placeholder(tf.float32)
# Build Alexnet portion of graph
self.AN = AlexNet(self.X, self.keep_prob, self.num_layers_to_load)
#### ------------ Setup Your Graph Here ------------------ ####
# We're taking the front of our graph from AlexNet, now let's
# build the rest - this is where we get to be creative!
# Grab the tensor from alexnet that we're going to build from
# Be sure to change this if you chnage num_layers_to_load
alexnet_out = self.AN.pool5
#To see the dimension of the output we're getting from alexnet, we can print our tensor:
print(alexnet_out)
# Your graph should produce an estimate our our labels, yhat.
# yhat should be of the same dimension as y.
# self.yhat = ?
# Make sure you inlude the names of all the variables you would like to train here.
# These may include variables from this part of the graph or the alexnet portion.
# You can give your variables whatever name you like by passing in a name into your layers
# for example if you setup a layer like this: tf.layers.dense(.... , name = 'my_fc6')
# be sure to add 'my_fc6' to the trainable_variable_names list here:
self.trainable_variable_names = ['conv5']
#### ---------------- End Graph Setup --------------------- ####
def train(self):
'''
Train model.
'''
# Start by building rest of graph that we needed for training.
#### ------------ Setup Cost Function Here ------------------ ####
# Your job here is to compute a cost to pass into our AdamOptimizer below.
# cost =
#### ------------ End Cost Function Setup ------------------ ####
# We only want to train some of the variables in our overall graph:
var_list = [v for v in tf.trainable_variables() if v.name.split('/')[0] \
in self.trainable_variable_names]
# Train op
with tf.name_scope("train"):
# Get gradients of all trainable variables
gradients = tf.gradients(cost, var_list)
gradients = list(zip(gradients, var_list))
# Create optimizer and apply gradient descent to the trainable variables
optimizer = tf.train.AdamOptimizer(self.learning_rate)
train_op = optimizer.apply_gradients(grads_and_vars=gradients)
# Add gradients to summary
for gradient, var in gradients:
tf.summary.histogram(var.name + '/gradient', gradient)
# Add the variables we train to the summary
for var in var_list:
tf.summary.histogram(var.name, var)
# Add the loss to summary
tf.summary.scalar('cross_entropy', cost)
# Evaluation op: Accuracy of the model
with tf.name_scope("accuracy"):
correct_pred = tf.equal(tf.argmax(self.yhat, 1),
tf.argmax(self.y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# Add the accuracy to the summary
tf.summary.scalar('accuracy', accuracy)
# Merge all summaries together
merged_summary = tf.summary.merge_all()
# Initialize the FileWriter
train_writer = tf.summary.FileWriter(
os.path.join(self.save_dir, 'train'))
test_writer = tf.summary.FileWriter(os.path.join(
self.save_dir, 'test'))
# Initialize an saver for store model checkpoints
# We'll save the pretrained alexnet weights and the new weight values that we train:
var_list_to_save = [v for v in tf.trainable_variables() if v.name.split('/')[0] in \
self.AN.all_variable_names + self.trainable_variable_names]
saver = tf.train.Saver(var_list=var_list_to_save)
#Load up data from image files.
data = data_loader(label_indices=self.label_indices,
channel_means=self.channel_means,
train_test_split=self.train_test_split,
input_image_size=self.input_image_size,
data_path='../data')
#Setup minibatch generators
G = Generator(data.train.X,
data.train.y,
minibatch_size=self.minibatch_size)
GT = Generator(data.test.X,
data.test.y,
minibatch_size=self.minibatch_size)
#Launch tf session
self.sess = tf.Session()
self.sess.run(tf.global_variables_initializer())
self.initialized = True
# Add the model graph to TensorBoard
train_writer.add_graph(self.sess.graph)
# Load the pretrained weights into the alexnet portion of our graph
self.AN.load_initial_weights(self.sess)
print("{} Start training...".format(datetime.now()))
print("{} Open Tensorboard at --logdir {}".format(
datetime.now(), self.save_dir))
#And Train
for i in range(self.num_iterations):
G.generate()
# And run the training op
self.sess.run(train_op,
feed_dict={
self.X: G.X,
self.y: G.y,
self.keep_prob: self.keep_rate
})
# Generate summary with the current batch of data and write to file
if i % self.display_step == 0:
s = self.sess.run(merged_summary,
feed_dict={
self.X: G.X,
self.y: G.y,
self.keep_prob: 1.0
})
train_writer.add_summary(s, i)
GT.generate()
s = self.sess.run(merged_summary,
feed_dict={
self.X: GT.X,
self.y: GT.y,
self.keep_prob: 1.0
})
test_writer.add_summary(s, i)
train_acc = self.sess.run(accuracy,
feed_dict={
self.X: G.X,
self.y: G.y,
self.keep_prob: 1.0
})
test_acc = self.sess.run(accuracy,
feed_dict={
self.X: GT.X,
self.y: GT.y,
self.keep_prob: 1.0
})
print(i, ' iterations,', str(G.num_epochs),
'epochs, train accuracy = ', train_acc,
', test accuracy = ', test_acc)
if i % self.save_step == 0 and i > 0:
print("{} Saving checkpoint of model...".format(
datetime.now()))
# save checkpoint of the model
checkpoint_name = os.path.join(
self.save_dir, 'model_epoch' + str(G.num_epochs) + '.ckpt')
save_path = saver.save(self.sess, checkpoint_name)
print("{} Model checkpoint saved at {}".format(
datetime.now(), checkpoint_name))
def predict(self, X, checkpoint_dir=None):
'''
X: Numpy array of dimension [n, 227, 227, 3].
Note that n here may not be the same as the minibatch_size defined above.
Returns:
yhat_numpy: A numpy array of dimension [n x 3] containing the predicted
one hot labels for each image passed in X.
'''
if not self.initialized:
self.restore_from_checkpoint(checkpoint_dir)
yhat_numpy = self.sess.run(self.yhat,
feed_dict={
self.X: X,
self.keep_prob: 1.0
})
return yhat_numpy
def restore_from_checkpoint(self, checkpoint_dir=None):
'''
Restore model from most recent checkpoint in save dir.
'''
saver = tf.train.Saver()
self.sess = tf.Session()
#Load latest checkpont, use savedir if we're not given a checkpoint dir:
if checkpoint_dir is None:
checkpoint_name = tf.train.latest_checkpoint(self.save_dir)
else:
checkpoint_name = tf.train.latest_checkpoint(checkpoint_dir)
print(checkpoint_name)
#Resore session
saver.restore(self.sess, checkpoint_name)
self.initialized = True
def main(_):
#convert jpg image(s) into iamge representations using alexnet:
filenames = [
os.path.join(image_dir, f) for f in [
'overly-attached-girlfriend.jpg',
'high-expectations-asian-father.jpg', 'foul-bachelor-frog.jpg',
'stoner-stanley.jpg', 'y-u-no.jpg', 'willy-wonka.jpg',
'futurama-fry.jpg', 'success-kid.jpg', 'one-does-not-simply.jpg',
'bad-luck-brian.jpg', 'first-world-problems.jpg',
'philosoraptor.jpg', 'what-if-i-told-you.jpg', 'TutorPP.jpg'
]
]
print(filenames)
tf.logging.info("Running caption generation on %d files matching %s",
len(filenames), FLAGS.input_files)
#mean of imagenet dataset in BGR
imagenet_mean = np.array([104., 117., 124.], dtype=np.float32)
#placeholder for input and dropout rate
x_Alex = tf.placeholder(tf.float32, [1, 227, 227, 3])
keep_prob_Alex = tf.placeholder(tf.float32)
#create model with default config ( == no skip_layer and 1000 units in the last layer)
modelAlex = AlexNet(x_Alex, keep_prob_Alex, 1000, [], ['fc7', 'fc8'],
512) #maybe need to put fc8 in skip_layers
#define activation of last layer as score
score = modelAlex.fc6
meme_embeddings = []
with tf.Session() as sess:
# Initialize all variables
sess.run(tf.global_variables_initializer())
# Load the pretrained weights into the model
modelAlex.load_initial_weights(sess)
for i, meme in enumerate(filenames):
img = Image.open(meme)
try:
img.thumbnail((227, 227), Image.ANTIALIAS)
#img = img.resize((227,227))
#use img.thumbnail for square images, img.resize for non square
assert np.shape(img) == (227, 227, 3)
except AssertionError:
img = img.resize((227, 227))
print('sizing error')
# Subtract the ImageNet mean
img = img - imagenet_mean #should probably change this
# Reshape as needed to feed into model
img = img.reshape((1, 227, 227, 3))
meme_vector = sess.run(score,
feed_dict={
x_Alex: img,
keep_prob_Alex: 1
}) #[1,4096]
meme_vector = np.reshape(meme_vector, [4096])
assert np.shape(meme_vector) == (4096, )
#now have np embeddings to feed for inference
meme_embeddings.append(meme_vector)
with open('Captions.txt', 'r') as f:
data_captions = f.readlines()
data_captions = [s.lower() for s in data_captions]
# Build the inference graph.
g = tf.Graph()
with g.as_default():
model = inference_wrapper.InferenceWrapper()
restore_fn = model.build_graph_from_config(configuration.ModelConfig(),
FLAGS.checkpoint_path)
g.finalize()
# Create the vocabulary.
vocab = vocabulary.Vocabulary(FLAGS.vocab_file)
#filenames = []
#for file_pattern in FLAGS.input_files.split(","):
#filenames.extend(tf.gfile.Glob(file_pattern))
#tf.logging.info("Running caption generation on %d files matching %s",
#len(filenames), FLAGS.input_files)
with tf.Session(graph=g) as sess:
# Load the model from checkpoint.
restore_fn(sess)
# Prepare the caption generator. Here we are implicitly using the default
# beam search parameters. See caption_generator.py for a description of the
# available beam search parameters.
generator = caption_generator.CaptionGenerator(model, vocab)
num_in_data_total = 0
num_captions = 0
for i, meme in enumerate(meme_embeddings):
#with tf.gfile.GFile(filename, "rb") as f:
#image = f.read()
captions = generator.beam_search(sess, meme)
print("Captions for image %s:" % os.path.basename(filenames[i]))
num_in_data = 0
for i, caption in enumerate(captions):
# Ignore begin and end words.
sentence = [
vocab.id_to_word(w) for w in caption.sentence[1:-1]
]
sentence = " ".join(sentence)
in_data = 0
if b_any(sentence in capt for capt in data_captions):
in_data = 1
num_in_data += 1
num_in_data_total += 1
num_captions += 1
else:
num_captions += 1
print(" %d) %s (p=%f) [in data = %d]" %
(i, sentence, math.exp(caption.logprob), in_data))
print("number of captions in data = %d" % (num_in_data))
print("(total number of captions in data = %d) percent in data = %f" %
(num_in_data_total, (num_in_data_total / num_captions)))
class FeatureExtractor(object):
def __init__(self, cfg):
self.cfg = cfg
self.dataset = {}
self.imagenet_mean = None
self.x_placeholder = None
self.keep_prob = None
self.model = None
self.score = None
self.softmax = None
self.load_dataset_indexes()
self.init_alexnet()
pass
@staticmethod
def add_sample_to_dict(d, s_id, sample_path):
sample_d = {'img': cv2.imread(sample_path), 'path': sample_path}
if s_id in d.keys():
d[s_id].append(sample_d)
else:
d[s_id] = [sample_d]
@staticmethod
def add_fmap_to_dict(d, s_id, p1, p5, fc2):
fmaps = {'p1': p1, 'p5': p5, 'fc2': fc2}
if s_id in d.keys():
d[s_id].append(fmaps)
else:
d[s_id] = [fmaps]
def load_dataset_indexes(self):
dataset_path = self.cfg['dataset']
for root, dirs, files in os.walk(dataset_path):
for name in files:
if not name.endswith('.bmp'):
continue
s_id = int(re.sub('[^0-9]', '', root.split('/')[2]))
#print "id:{} path:{}".format(s_id, root.split('/')[1])
full_img_path = os.path.join(root, name)
FeatureExtractor.add_sample_to_dict(self.dataset, s_id,
full_img_path)
# clean missing numbers
prev_num = 0
old_keys = sorted(self.dataset.keys())
for i in xrange(len(old_keys)):
old_k = old_keys[i]
self.dataset[prev_num] = self.dataset.pop(old_k)
prev_num += 1
print "[DEBUG]", "Number of classes:", len(
self.dataset.keys()), self.dataset.keys()
def init_alexnet(self):
"""
Start alexnet network and load weights
:return:
"""
self.imagenet_mean = np.array([104., 117., 124.], dtype=np.float32)
self.x_placeholder = tf.placeholder(tf.float32, [1, 227, 227, 3])
self.keep_prob = tf.placeholder(tf.float32)
# create model with default config ( == no skip_layer and 1000 units in the last layer)
self.model = AlexNet(self.x_placeholder, self.keep_prob, 1000, [])
# define activation of last layer as score
self.score = self.model.fc8
# create op to calculate softmax
self.softmax = tf.nn.softmax(self.score)
pass
def process_samples(self, samples):
"""
Process samples with the alexnet network and extract feature maps of C1 C5 and FC2 layers
:param samples:
:return:
"""
sample_r = {}
with tf.Session() as sess:
# Initialize all variables
sess.run(tf.global_variables_initializer())
# Load the pretrained weights into the model
self.model.load_initial_weights(sess)
s_keys = samples.keys()
for i in tqdm.trange(len(s_keys)):
s_id = s_keys[i]
v = samples[s_id]
for e in v:
# Convert image to float32 and resize to (227x227)
img = cv2.resize(e['img'].astype(np.float32), (227, 227))
# Subtract the ImageNet mean
img -= self.imagenet_mean
# Reshape as needed to feed into model
img = img.reshape((1, 227, 227, 3))
pool1_tensor = sess.graph.get_tensor_by_name('pool1:0')
pool5_tensor = sess.graph.get_tensor_by_name('pool5:0')
fc2_tensor = sess.graph.get_tensor_by_name('fc7/fc7:0')
p1, p5, fc2 = sess.run(
[pool1_tensor, pool5_tensor, fc2_tensor],
feed_dict={
self.x_placeholder: img,
self.keep_prob: 1
})
FeatureExtractor.add_fmap_to_dict(sample_r, s_id,
np.dstack(p1).flatten(),
np.dstack(p5).flatten(),
np.dstack(fc2).flatten())
# np.set_printoptions(threshold='nan')
# print("pool1 shape: {}".format(p1.shape))
# print("pool5 shape: {}".format(p5.shape))
# print("fc2 shape: {}".format(fc2.shape))
# Run the session and calculate the class probability
# probs = sess.run(self.softmax, feed_dict={self.x_placeholder: img, self.keep_prob: 1})
# Get the class name of the class with the highest probability
# class_name = class_names[np.argmax(probs)]
# prob = probs[0, np.argmax(probs)]
return sample_r
@staticmethod
def export_features_by_class(fmap, cutoff=0.3):
if 0 < cutoff > 1:
raise Exception("cutoff must be between 0 and 1")
t_train = []
p1_train = []
p5_train = []
fc2_train = []
t_test = []
p1_test = []
p5_test = []
fc2_test = []
for k, v in fmap.items():
total_e = len(v)
cut_p = int(math.ceil(total_e * (1 - cutoff)))
for i in xrange(len(v)):
e = v[i]
if i < cut_p:
t_train.append(k)
p1_train.append(e['p1'])
p5_train.append(e['p5'])
fc2_train.append(e['fc2'])
else:
t_test.append(k)
p1_test.append(e['p1'])
p5_test.append(e['p5'])
fc2_test.append(e['fc2'])
labels = [str(e) for e in sorted(list(set(t_train)))]
return [np.asarray(t_train), np.asarray(p1_train), np.asarray(p5_train), np.asarray(fc2_train)], \
[np.asarray(t_test), np.asarray(p1_test), np.asarray(p5_test), np.asarray(fc2_test)], \
labels
def compare_features(self, f):
"""
Compare the feature map of several layers
:param f:
:return:
"""
from scipy.spatial.distance import pdist, squareform
print '[INFO]', 'Compare features', f.shape
dist_condensed = pdist(f)
print '\tEuclidean mean:{} std:{}'.format(np.mean(dist_condensed),
np.std(dist_condensed))
dist_condensed = pdist(f, 'cityblock')
print '\tcityblock mean:{} std:{}'.format(np.mean(dist_condensed),
np.std(dist_condensed))
dist_condensed = pdist(f, 'seuclidean', V=None)
print '\tseuclidean mean:{} std:{}'.format(np.mean(dist_condensed),
np.std(dist_condensed))
dist_condensed = pdist(f, 'cosine', V=None)
print '\tcosine mean:{} std:{}'.format(np.mean(dist_condensed),
np.std(dist_condensed))
dist_condensed = pdist(f, 'correlation', V=None)
print '\tcorrelation mean:{} std:{}'.format(np.mean(dist_condensed),
np.std(dist_condensed))
dist_condensed = pdist(f, 'hamming', V=None)
print '\thamming mean:{} std:{}'.format(np.mean(dist_condensed),
np.std(dist_condensed))
# dist_condensed = pdist(f, 'mahalanobis', V=None)
# print 'mahalanobis mean:{} std:{}'.format(np.mean(dist_condensed), np.std(dist_condensed))
print 'End...'
pass
def start(self):
"""
Start clasification systems
:return:
"""
print "[INFO]", "Processing data..."
fmap = self.process_samples(self.dataset)
train, test, labels = self.export_features_by_class(fmap, cutoff=0)
target_train, p1_train, p5_train, fc2_train = train
target_test, p1_test, p5_test, fc2_test = test
print "[INFO]", "Data shapes:"
print '\tTarget:{}/{}'.format(target_train.shape[0],
target_test.shape[0])
print '\tC1:', p1_train.shape
print '\tC5:', p5_train.shape
print '\tFC2:', fc2_train.shape
print '\tLabels:', labels
prepend_date = datetime.datetime.now().strftime("%I-%M%p_%d-%b-%Y")
print "[INFO]", "Plots will be saved in: {}".format(
self.cfg["result_path"])
print "[INFO]", "Results will be saved in: {}".format(
self.cfg["plot_path"])
# Compare features
self.compare_features(p1_train)
self.compare_features(p5_train)
self.compare_features(fc2_train)
svn_fn = ClassificatorHelper(self.cfg)
res_p1_1 = svn_fn.svm_simple(p1_train,
target_train,
labels,
debug_level=self.cfg["verbose_level"])
ClassificatorHelper.save_results(res_p1_1, self.cfg["result_path"],
self.cfg["plot_path"],
"p1_svm_simple_" + prepend_date)
res_p5_1 = svn_fn.svm_simple(p5_train,
target_train,
labels,
debug_level=self.cfg["verbose_level"])
ClassificatorHelper.save_results(res_p5_1, self.cfg["result_path"],
self.cfg["plot_path"],
"p5_svm_simple_" + prepend_date)
res_fc2_1 = svn_fn.svm_simple(fc2_train,
target_train,
labels,
debug_level=self.cfg["verbose_level"])
ClassificatorHelper.save_results(res_fc2_1, self.cfg["result_path"],
self.cfg["plot_path"],
"fc2_svm_simple_" + prepend_date)
res_early = svn_fn.svm_early_fusion(
target_train,
p1_train,
p5_train,
fc2_train,
labels,
debug_level=self.cfg["verbose_level"])
for v in res_early:
ClassificatorHelper.save_results(
v, self.cfg["result_path"], self.cfg["plot_path"],
"early_" + str(v["combination"]) + prepend_date)
res_late = svn_fn.svm_late_fusion(
target_train,
p1_train,
p5_train,
fc2_train,
labels,
debug_level=self.cfg["verbose_level"])
ClassificatorHelper.save_results(res_late, self.cfg["result_path"],
self.cfg["plot_path"],
"late_" + prepend_date)
res_fc2_all = svn_fn.test_diversity_fc2(
target_train,
fc2_train,
labels,
debug_level=self.cfg["verbose_level"])
for k, v in res_fc2_all.items():
ClassificatorHelper.save_results(v, self.cfg["result_path"],
self.cfg["plot_path"],
k + "_" + prepend_date)
print "[INFO]", "Finished classifying"
pass
class KNN(object):
def __init__(self,
sess,
class_num,
train_data_path="../data/training",
test_data_path="../data/testing",
kneighbors=10):
self.sess = sess
self.train_data_path = train_data_path
self.test_data_path = test_data_path
self.class_num = class_num
self.training_data = [[], [], [], [], [], [], [], []]
self.label_set = []
self.knn_list = []
self.kneighbors = kneighbors
self.inputs = tf.placeholder(tf.float32, [1, 227, 227, 3],
name="input_image")
self.labels = tf.placeholder(tf.float32, [1, 50], name='label')
self.alexnet = AlexNet(self.inputs,
keep_prob=1.0,
num_classes=1000,
skip_layer=[])
tf.global_variables_initializer().run()
self.alexnet.load_initial_weights(sess)
for _i in range(7):
nn = KNeighborsClassifier(n_neighbors=kneighbors)
self.knn_list.append(nn)
pass
def load_training_data(self, img_num):
ds = DataSet(self.train_data_path, 1, self.class_num)
for i in range(img_num):
img, label = ds.next_batch()
out1, out2, out3, out4, out5, out6, out7 = self.sess.run(
[
self.alexnet.norm1, self.alexnet.norm2, self.alexnet.conv3,
self.alexnet.conv4, self.alexnet.pool5, self.alexnet.fc6,
self.alexnet.fc7
],
feed_dict={
self.alexnet.X: img,
self.labels: label
})
self.training_data[0].append(out1[0].reshape(1, 27 * 27 * 96)[0])
self.training_data[1].append(out2[0].reshape(1, 13 * 13 * 256)[0])
self.training_data[2].append(out3[0].reshape(1, 13 * 13 * 384)[0])
self.training_data[3].append(out4[0].reshape(1, 13 * 13 * 384)[0])
self.training_data[4].append(out5[0].reshape(1, 6 * 6 * 256)[0])
self.training_data[5].append(out6[0])
self.training_data[6].append(out7[0])
self.label_set.append(np.argmax(label, axis=1)[0])
self.calibration_size = 400
self.calibration_data = [
self.training_data[j][img_num - self.calibration_size:img_num]
for j in range(7)
]
self.calibration_label = self.label_set[img_num -
self.calibration_size:img_num]
self.training_data = [
self.training_data[j][:img_num - self.calibration_size]
for j in range(7)
]
self.label_set = self.label_set[:img_num - self.calibration_size]
self.build_model()
self.A = []
for i in range(self.calibration_size):
target_label = self.calibration_label[i]
tar = []
for k in range(7):
tp = [
self.label_set[int(j)] for j in
self.knn_list[k].kneighbors([self.calibration_data[k][i]],
n_neighbors=self.kneighbors,
return_distance=False)[0]
]
tar.extend(tp)
total = np.sum(np.not_equal(tar, target_label))
self.A.append(total)
def load_testing_data(self, img_num):
kneighbor = self.kneighbors
ds = DataSet(self.test_data_path, 1, self.class_num)
source = []
tar = []
count = 0
for i in range(img_num):
print(i)
img, label = ds.next_batch()
out1, out2, out3, out4, out5, out6, out7 = self.sess.run(
[
self.alexnet.norm1, self.alexnet.norm2, self.alexnet.conv3,
self.alexnet.conv4, self.alexnet.pool5, self.alexnet.fc6,
self.alexnet.fc7
],
feed_dict={
self.alexnet.X: img,
self.labels: label
})
source.clear()
source.append(out1[0].reshape(1, 27 * 27 * 96))
source.append(out2[0].reshape(1, 13 * 13 * 256))
source.append(out3[0].reshape(1, 13 * 13 * 384))
source.append(out4[0].reshape(1, 13 * 13 * 384))
source.append(out5[0].reshape(1, 6 * 6 * 256))
source.append(out6)
source.append(out7)
target_label = np.argmax(label, axis=1)[0]
tar.clear()
for k in range(7):
tp = [
self.label_set[int(j)] for j in
self.knn_list[k].kneighbors(source[k],
n_neighbors=kneighbor,
return_distance=False)[0]
]
tar.extend(tp)
max = 0
m_label = -1
for k_label in range(self.class_num):
alpha = np.sum(np.not_equal(tar, k_label))
pj = np.sum(np.greater_equal(self.A, alpha))
if pj >= max:
max = pj
m_label = k_label
if m_label == target_label:
count += 1
if (i + 1) % 20 == 0:
print("%d / %d, accuracy: %f" %
((i + 1), img_num, float(count / (i + 1))))
def build_model(self):
for i in range(7):
print("build ", i)
try:
self.knn_list[i].fit(self.training_data[i], self.label_set)
self.training_data[i].clear()
except:
print("something wrong")
def main():
"""
Configuration Part.
"""
# Path to the textfiles for the trainings and validation set
train_file = './train.txt'
val_file = './val.txt'
# Learning params
learning_rate = 0.01
num_epochs = 100
batch_size = 10
# Network params
dropout_rate = 0.5
num_classes = 2
train_layers = ['fc8', 'fc7', 'fc6']
# How often we want to write the tf.summary data to disk
display_step = 20
# Path for tf.summary.FileWriter and to store model checkpoints
filewriter_path = "./tmp/tensorboard"
checkpoint_path = "./tmp/checkpoints"
"""
Main Part of the finetuning Script.
"""
# Create parent path if it doesn't exist
if not os.path.isdir(checkpoint_path):
os.mkdir(checkpoint_path)
# Place data loading and preprocessing on the cpu
with tf.device('/cpu:0'):
tr_data = ImageDataGenerator(train_file,
mode='training',
batch_size=batch_size,
num_classes=num_classes,
shuffle=True)
val_data = ImageDataGenerator(val_file,
mode='inference',
batch_size=batch_size,
num_classes=num_classes,
shuffle=False)
# create an reinitializable iterator given the dataset structure
iterator = Iterator.from_structure(tr_data.data.output_types,
tr_data.data.output_shapes)
next_batch = iterator.get_next()
# Ops for initializing the two different iterators
training_init_op = iterator.make_initializer(tr_data.data)
validation_init_op = iterator.make_initializer(val_data.data)
# TF placeholder for graph input and output
x = tf.placeholder(tf.float32, [None, 227, 227, 3])
y = tf.placeholder(tf.float32, [None, num_classes])
keep_prob = tf.placeholder(tf.float32)
# Initialize model
model = AlexNet(x, keep_prob, num_classes, train_layers)
# Link variable to model output
score = model.fc8
pred = tf.nn.softmax(score)
# List of trainable variables of the layers we want to train
var_list = [
v for v in tf.trainable_variables()
if v.name.split('/')[0] in train_layers
]
# Op for calculating the loss
with tf.name_scope("cross_ent"):
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=score, labels=y))
# Train op
with tf.name_scope("train"):
# Get gradients of all trainable variables
gradients = tf.gradients(loss, var_list)
gradients = list(zip(gradients, var_list))
# Create optimizer and apply gradient descent to the trainable variables
# optimizer = tf.train.GradientDescentOptimizer(learning_rate)
optimizer = tf.train.AdamOptimizer(learning_rate)
train_op = optimizer.apply_gradients(grads_and_vars=gradients)
# Add gradients to summary
for gradient, var in gradients:
tf.summary.histogram(var.name + '/gradient', gradient)
# Add the variables we train to the summary
for var in var_list:
tf.summary.histogram(var.name, var)
# Add the loss to summary
tf.summary.scalar('cross_entropy', loss)
# Evaluation op: Accuracy of the model
with tf.name_scope("accuracy"):
correct_pred = tf.equal(tf.argmax(score, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# Add the accuracy to the summary
tf.summary.scalar('accuracy', accuracy)
# Merge all summaries together
merged_summary = tf.summary.merge_all()
# Initialize the FileWriter
writer = tf.summary.FileWriter(filewriter_path)
# Initialize an saver for store model checkpoints
saver = tf.train.Saver()
# Get the number of training/validation steps per epoch
train_batches_per_epoch = int(np.floor(tr_data.data_size / batch_size))
val_batches_per_epoch = int(np.floor(val_data.data_size / batch_size))
# Start Tensorflow session
with tf.Session() as sess:
print("# isTrain : ", args.istrain)
if not args.istrain:
ckpt = tf.train.get_checkpoint_state(checkpoint_path)
if ckpt and tf.train.checkpoint_exists(ckpt.model_checkpoint_path):
print("Reading model parameters from %s" %
ckpt.model_checkpoint_path)
saver.restore(sess, ckpt.model_checkpoint_path)
else:
print("checkpoint path isn't correct")
return
file = tf.read_file(args.input_path)
decoded_img = tf.image.decode_jpeg(file, channels=3)
resized_img = tf.image.resize_images(decoded_img, [227, 227])
resized_img = tf.reshape(resized_img, [1, 227, 227, 3])
print("# decoded img : ", decoded_img.eval().shape)
pred_ = sess.run(pred,
feed_dict={
x: resized_img.eval(),
y: [[1, 0]],
keep_prob: np.array(1.0)
})
print("P(man|data) : ", pred_[0][0])
print("P(woman|data) : ", pred_[0][1])
img = decoded_img.eval()
plt.imshow(img)
plt.show()
if args.visualize:
w1, w2, w3, w4, w5 = sess.run(model.weight,
feed_dict={
x: resized_img.eval(),
y: [[1, 0]],
keep_prob: np.array(1.0)
})
print("W1 : ", w1.shape)
visualize(w1[:, :, 0, :25])
print("W2 : ", w2.shape)
visualize(w2[:, :, 0, :25])
print("W3 : ", w3.shape)
visualize(w3[:, :, 0, :25])
print("W4 : ", w4.shape)
visualize(w4[:, :, 0, :25])
print("W5 : ", w5.shape)
visualize(w5[:, :, 0, :25])
f1, f2, f3, f4, f5 = sess.run(model.fm, {
x: resized_img.eval(),
y: [[1, 0]],
keep_prob: np.array(1.0)
})
print("F1 : ", f1.shape)
visualize(f1[0][:, :, :25])
print("F2 : ", f2.shape)
visualize(f2[0][:, :, :25])
print("F3 : ", f3.shape)
visualize(f3[0][:, :, :25])
print("F4 : ", f4.shape)
visualize(f4[0][:, :, :25])
print("F5 : ", f5.shape)
visualize(f5[0][:, :, :25])
return
else:
# Initialize all variables
sess.run(tf.global_variables_initializer())
# Add the model graph to TensorBoard
writer.add_graph(sess.graph)
# Load the pretrained weights into the non-trainable layer
model.load_initial_weights(sess)
print("{} Start training...".format(datetime.now()))
print("{} Open Tensorboard at --logdir {}".format(
datetime.now(), filewriter_path))
# Loop over number of epochs
for epoch in range(num_epochs):
print("{} Epoch number: {}".format(datetime.now(), epoch + 1))
# Initialize iterator with the training dataset
sess.run(training_init_op)
train_acc = 0.
train_count = 0.
for step in range(train_batches_per_epoch):
# get next batch of data
img_batch, label_batch = sess.run(next_batch)
# And run the training op
acc, _ = sess.run([accuracy, train_op],
feed_dict={
x: img_batch,
y: label_batch,
keep_prob: dropout_rate
})
train_acc += acc
train_count += 1
# Generate summary with the current batch of data and write to file
# if step % display_step == 0:
# s = sess.run(merged_summary, feed_dict={x: img_batch,
# y: label_batch,
# keep_prob: 1.})
#
# writer.add_summary(s, epoch*train_batches_per_epoch + step)
train_acc /= train_count
print("{} Train Accuracy = {:.4f}".format(datetime.now(),
train_acc))
# Validate the model on the entire validation set
print("{} Start validation".format(datetime.now()))
sess.run(validation_init_op)
test_acc = 0.
test_count = 0
for ind in range(val_batches_per_epoch):
img_batch, label_batch = sess.run(next_batch)
acc = sess.run(accuracy,
feed_dict={
x: img_batch,
y: label_batch,
keep_prob: 1.
})
test_acc += acc
test_count += 1
if epoch is 2 and ind is 0:
fm = sess.run(model.fm,
feed_dict={
x: img_batch,
y: label_batch,
keep_prob: 1.
})
weight = sess.run(model.weight,
feed_dict={
x: img_batch,
y: label_batch,
keep_prob: 1.
})
# print("fm0 : ", np.array(fm[0]).shape)
# print("fm1 : ", np.array(fm[1]).shape)
# print("fm2 : ", np.array(fm[2]).shape)
# print("fm3 : ", np.array(fm[3]).shape)
# print("fm4 : ", np.array(fm[4]).shape)
#
# print("weight0 : ", np.array(weight[0]).shape)
# print("weight1 : ", np.array(weight[1]).shape)
# print("weight2 : ", np.array(weight[2]).shape)
# print("weight3 : ", np.array(weight[3]).shape)
# print("weight4 : ", np.array(weight[4]).shape)
test_acc /= test_count
print("{} Validation Accuracy = {:.4f}".format(
datetime.now(), test_acc))
print("{} Saving checkpoint of model...".format(datetime.now()))
# save checkpoint of the model
checkpoint_name = os.path.join(
checkpoint_path, 'model_epoch' + str(epoch + 1) + '.ckpt')
save_path = saver.save(sess, checkpoint_name)
print("{} Model checkpoint saved at {}".format(
datetime.now(), checkpoint_name))
train_batches_per_epoch = int(np.floor(tr_data.data_size /
batch_size)) #获取多少个epoch
val_batches_per_epoch = int(np.floor(val_data.data_size /
batch_size)) #测试集epoch个数
# Start Tensorflow session
with tf.Session() as sess:
# Initialize all variables
sess.run(tf.global_variables_initializer()) #初始化所有变量
# Add the model graph to TensorBoard
writer.add_graph(sess.graph)
# Load the pretrained weights into the non-trainable layer
model.load_initial_weights(sess) #加载权重
print("{} Start training...".format(datetime.now()))
print("{} Open Tensorboard at --logdir {}".format(datetime.now(),
filewriter_path))
# Loop over number of epochs
for epoch in range(num_epochs): #多少个epoch，迭代多少次
print("{} Epoch number: {}".format(datetime.now(), epoch + 1))
# Initialize iterator with the training dataset
sess.run(training_init_op)
for step in range(train_batches_per_epoch):
def main(_):
if not FLAGS.train_paths:
raise ValueError("Must set --train_paths")
if not FLAGS.val_paths:
raise ValueError("Must set --val_paths")
if not FLAGS.test_paths:
raise ValueError("Must set --test_paths")
train_paths = FLAGS.train_paths
val_paths = FLAGS.val_paths
test_paths = FLAGS.test_paths
batch_size = FLAGS.batch_size
num_classes = FLAGS.num_classes
num_epochs = FLAGS.num_epochs
emb_dim = FLAGS.embedding_dim
keep_prob = FLAGS.keep_prob
exp = FLAGS.exp
loss_func = FLAGS.loss_func
learning_rate = FLAGS.learning_rate
checkpoint_path = FLAGS.checkpoint_path
filewriter_path = FLAGS.filewriter_path
display_step = FLAGS.display_step
max_threads = FLAGS.max_threads
# Create parent path if it doesn't exist
'''
if not os.path.isdir(checkpoint_path):
os.mkdir(checkpoint_path)
if not os.path.isdir(filewriter_path):
os.mkdir(filewriter_path)
'''
# TF placeholder for graph input and output
x = tf.placeholder(tf.float32, [batch_size, 6*6*256])
y = tf.placeholder(tf.float32, [batch_size, num_classes])
kp = tf.placeholder(tf.float32)
# Initialize model
layer_names = ['fc6', 'fc7', 'fc8']
train_layers = layer_names[-FLAGS.num_train_layers:]
model = AlexNet(x, kp, num_classes, emb_dim, train_layers)
# Link variable to model output
score = model.fc8
if exp != 1.0:
sign = tf.sign(score)
score = tf.multiply(sign, tf.pow(tf.abs(score), exp))
# List of trainable variables of the layers we want to train
var_list = [v for v in tf.trainable_variables() if v.name.split('/')[0] in train_layers]
# Op for calculating the loss
with tf.name_scope("loss_func"):
if loss_func == 'softmax':
loss = tf.nn.softmax_cross_entropy_with_logits(logits=score,
labels=y,
name='softmax_loss')
elif loss_func == 'logistic':
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y,
logits=score,
name='logistic_loss')
elif loss_func == 'mse':
loss = tf.losses.mean_squared_error(labels=y,
predictions=score)
elif loss_func == 'l2hinge':
loss = tf.losses.hinge_loss(labels=y,
logits=score,
reduction=tf.losses.Reduction.NONE)
loss = tf.square(loss)
loss = tf.reduce_mean(loss)
# Train op
with tf.name_scope("train"):
# Get gradients of all trainable variables
gradients = tf.gradients(loss, var_list)
gradients = list(zip(gradients, var_list))
# Create optimizer and apply gradient descent to the trainable variables
optimizer = tf.train.AdamOptimizer(learning_rate)
train_op = optimizer.apply_gradients(grads_and_vars=gradients)
# Add gradients to summary
for gradient, var in gradients:
tf.summary.histogram(var.name + '/gradient', gradient)
# Add the variables we train to the summary
for var in var_list:
tf.summary.histogram(var.name, var)
# Add the loss to summary
tf.summary.scalar('train_loss', loss)
# Initialize the datasets
start_time = time.time()
tr_data = Dataset(train_paths, batch_size, num_classes, True)
val_data = Dataset(val_paths, batch_size, num_classes, False)
te_data = Dataset(test_paths, batch_size, num_classes, False)
load_time = time.time() - start_time
log_buff = 'Data loading time: %.2f' % load_time + '\n'
# Ops for evaluation
acc_split_weights_all = tr_data.get_acc_split_weights(caffe_class_ids, 0)
acc_split_weights_all = tf.convert_to_tensor(acc_split_weights_all, tf.float32)
acc_split_weights_1000 = tr_data.get_acc_split_weights(caffe_class_ids, 1000)
acc_split_weights_1000 = tf.convert_to_tensor(acc_split_weights_1000, tf.float32)
with tf.name_scope('accuracy'):
# ops for top 1 accuracies and their splitting
top1_score, _ = tf.nn.top_k(score, 1)
top1_thresolds = tf.reduce_min(top1_score, axis = 1, keepdims = True)
top1_thresolds_bc = tf.broadcast_to(top1_thresolds, score.shape)
top1_y = tf.to_float(tf.math.greater_equal(score, top1_thresolds_bc))
top1_correct_pred = tf.multiply(y, top1_y)
top1_precision_all = tf.squeeze(tf.reduce_mean(tf.matmul(acc_split_weights_all, \
tf.transpose(top1_correct_pred)), axis = 1))
top1_precision_1000 = tf.squeeze(tf.reduce_mean(tf.matmul(acc_split_weights_1000, \
tf.transpose(top1_correct_pred)), axis = 1))
# ops for top 5 accuracies and their splitting
top5_score, _ = tf.nn.top_k(score, 5)
top5_thresolds = tf.reduce_min(top5_score, axis = 1, keepdims = True)
top5_thresolds_bc = tf.broadcast_to(top5_thresolds, score.shape)
top5_y = tf.to_float(tf.math.greater_equal(score, top5_thresolds_bc))
top5_correct_pred = tf.multiply(y, top5_y)
top5_precision_all = tf.squeeze(tf.reduce_mean(tf.matmul(acc_split_weights_all, \
tf.transpose(top5_correct_pred)), axis = 1))/5.0
top5_precision_1000 = tf.squeeze(tf.reduce_mean(tf.matmul(acc_split_weights_1000, \
tf.transpose(top5_correct_pred)), axis = 1))/5.0
# Merge all summaries together
merged_summary = tf.summary.merge_all()
# Initialize the FileWriter
#writer = tf.summary.FileWriter(filewriter_path)
# Initialize an saver for store model checkpoints
saver = tf.train.Saver()
# Start Tensorflow session
sess_conf = tf.ConfigProto(intra_op_parallelism_threads=max_threads,
inter_op_parallelism_threads=max_threads)
with tf.Session(config=sess_conf) as sess:
# Initialize all variables
sess.run(tf.global_variables_initializer())
# Add the model graph to TensorBoard
#writer.add_graph(sess.graph)
# Load the pretrained weights into the non-trainable layer
model.load_initial_weights(sess)
log_buff += "{} Start training...".format(datetime.now())+'\n'
#print("{} Open Tensorboard at --logdir {}".format(datetime.now(),
# filewriter_path))
print(log_buff)
log_buff = ''
# Loop over number of epochs
prev_top5_acc = 0
counter = 0
for epoch in range(num_epochs):
log_buff += "{} Epoch: {}".format(datetime.now(), epoch)+'\n'
tr_batches_per_epoch = tr_data.data_size // batch_size
tr_data.reset()
start_time = time.time()
tr_data.shuffle()
shuffle_time = time.time() - start_time
log_buff += 'Train data shuffling time: %.2f' % shuffle_time + '\n'
cost = 0.0
load_time = 0
train_time = 0
for step in range(tr_batches_per_epoch):
# get next batch of data
start_time = time.time()
img_batch, label_batch = tr_data.next_batch()
load_time += time.time() - start_time
# And run the training op
start_time = time.time()
_, lss = sess.run((train_op, loss), feed_dict={x: img_batch,
y: label_batch,
kp: keep_prob})
cost += lss
train_time += time.time() - start_time
# Generate summary with the current batch of data and write to file
'''
if step % display_step == 0:
s = sess.run(merged_summary, feed_dict={x: img_batch,
y: label_batch,
kp: 1.0})
writer.add_summary(s, epoch*tr_batches_per_epoch + step)
'''
elapsed_time = load_time + train_time
cost /= tr_batches_per_epoch
log_buff += 'Epoch: %d\tCost: %.6f\tElapsed Time: %.2f (%.2f / %.2f)' % \
(epoch+1, cost, elapsed_time, load_time, train_time) + '\n'
# Test the model on the sampled train set
tr_top1_all = ResultStruct()
tr_top1_1000 = ResultStruct()
tr_top5_all = ResultStruct()
tr_top5_1000 = ResultStruct()
# Evaluate on a for a smaller number of batches of trainset
tr_data.reset()
start_time = time.time()
num_batches = int(tr_batches_per_epoch/4);
for _ in range(num_batches):
img_batch, label_batch = tr_data.next_batch()
prec1_all, prec1_1000, prec5_all, prec5_1000 = \
sess.run((top1_precision_all, top1_precision_1000, \
top5_precision_all, top5_precision_1000), \
feed_dict={x: img_batch, \
y: label_batch, \
kp: 1.0});
tr_top1_all.add(prec1_all)
tr_top1_1000.add(prec1_1000)
tr_top5_all.add(prec5_all)
tr_top5_1000.add(prec5_1000)
tr_top1_all.scaler_div(num_batches)
tr_top1_1000.scaler_div(num_batches)
tr_top5_all.scaler_div(num_batches)
tr_top5_1000.scaler_div(num_batches)
log_buff += 'Epoch: ' + str(epoch+1) + '\tTrain Top 1 All Acc: ' + str(tr_top1_all) + '\n'
log_buff += 'Epoch: ' + str(epoch+1) + '\tTrain Top 1 1000 Acc: ' + str(tr_top1_1000) + '\n'
log_buff += 'Epoch: ' + str(epoch+1) + '\tTrain Top 5 All Acc: ' + str(tr_top5_all) + '\n'
log_buff += 'Epoch: ' + str(epoch+1) + '\tTrain Top 5 1000 Acc: ' + str(tr_top5_1000) + '\n'
tr_pred_time = time.time() - start_time
# Test the model on the entire validation set
val_top1_all = ResultStruct()
val_top1_1000 = ResultStruct()
val_top5_all = ResultStruct()
val_top5_1000 = ResultStruct()
val_data.reset()
val_batches_per_epoch = val_data.data_size // batch_size
start_time = time.time()
for _ in range(val_batches_per_epoch):
img_batch, label_batch = val_data.next_batch()
prec1_all, prec1_1000, prec5_all, prec5_1000 = \
sess.run((top1_precision_all, top1_precision_1000, \
top5_precision_all, top5_precision_1000), \
feed_dict={x: img_batch, \
y: label_batch, \
kp: 1.0})
val_top1_all.add(prec1_all)
val_top1_1000.add(prec1_1000)
val_top5_all.add(prec5_all)
val_top5_1000.add(prec5_1000)
val_top1_all.scaler_div(val_batches_per_epoch)
val_top1_1000.scaler_div(val_batches_per_epoch)
val_top5_all.scaler_div(val_batches_per_epoch)
val_top5_1000.scaler_div(val_batches_per_epoch)
log_buff += 'Epoch: ' + str(epoch+1) + '\tVal Top 1 All Acc: ' + str(val_top1_all) + '\n'
log_buff += 'Epoch: ' + str(epoch+1) + '\tVal Top 1 1000 ACC: ' + str(val_top1_1000) + '\n'
log_buff += 'Epoch: ' + str(epoch+1) + '\tVal Top 5 All Acc: ' + str(val_top5_all) + '\n'
log_buff += 'Epoch: ' + str(epoch+1) + '\tVal Top 5 1000 Acc: ' + str(val_top5_1000) + '\n'
val_pred_time = time.time() - start_time
# Test the model on the entire test set
te_top1_all = ResultStruct()
te_top1_1000 = ResultStruct()
te_top5_all = ResultStruct()
te_top5_1000 = ResultStruct()
te_data.reset()
te_batches_per_epoch = te_data.data_size // batch_size
start_time = time.time()
for _ in range(te_batches_per_epoch):
img_batch, label_batch = te_data.next_batch()
prec1_all, prec1_1000, prec5_all, prec5_1000 = \
sess.run((top1_precision_all, top1_precision_1000, \
top5_precision_all, top5_precision_1000), \
feed_dict={x: img_batch, \
y: label_batch, \
kp: 1.0});
te_top1_all.add(prec1_all)
te_top1_1000.add(prec1_1000)
te_top5_all.add(prec5_all)
te_top5_1000.add(prec5_1000)
te_top1_all.scaler_div(te_batches_per_epoch)
te_top1_1000.scaler_div(te_batches_per_epoch)
te_top5_all.scaler_div(te_batches_per_epoch)
te_top5_1000.scaler_div(te_batches_per_epoch)
log_buff += 'Epoch: ' + str(epoch+1) + '\tTest Top 1 All Acc: ' + str(te_top1_all) + '\n'
log_buff += 'Epoch: ' + str(epoch+1) + '\tTest Top 1 1000 Acc: ' + str(te_top1_1000) + '\n'
log_buff += 'Epoch: ' + str(epoch+1) + '\tTest Top 5 All Acc: ' + str(te_top5_all) + '\n'
log_buff += 'Epoch: ' + str(epoch+1) + '\tTest Top 5 1000 Acc: ' + str(te_top5_1000) + '\n'
te_pred_time = time.time() - start_time
elapsed_time = tr_pred_time + val_pred_time + te_pred_time
log_buff += 'Epoch %d Prediction: \tElapsed Time: %.2f (%.2f / %.2f / %.2f)' \
% (epoch+1, elapsed_time, tr_pred_time, val_pred_time, te_pred_time) + '\n'
if math.isnan(cost):
break
cur_top5_acc = val_top5_all.acc
if cur_top5_acc - prev_top5_acc > 0.003:
counter = 0
prev_top5_acc = cur_top5_acc
elif (cur_top5_acc - prev_top5_acc < -0.05) or (counter == 15):
break
else:
counter += 1
# save checkpoint of the model
'''
print("{} Saving checkpoint of model...".format(datetime.now()))
checkpoint_name = os.path.join(checkpoint_path,
'model_epoch'+str(epoch+1)+'.ckpt')
save_path = saver.save(sess, checkpoint_name)
print("{} Model checkpoint saved at {}".format(datetime.now(),
checkpoint_name))
'''
print(log_buff)
log_buff = ''
print(log_buff)
def readAndpredictloop():
## reset default graph
tf.reset_default_graph()
#placeholder for input and dropout rate
x = tf.placeholder(tf.float32, [1, 227, 227, 3])
keep_prob = tf.placeholder(tf.float32)
#create model with default config ( == no skip_layer and 1000 units in the last layer)
model = AlexNet(x, keep_prob, 1000, [])
#define activation of last layer as score
score = model.fc8
#create op to calculate softmax
softmax = tf.nn.softmax(score)
with tf.Session() as sess:
# Create the excel sheet workbook
path_to_excel = PATH_TO_EXCEL
rb = xlrd.open_workbook(path_to_excel, formatting_info=True)
wb = copy(rb)
sheet = wb.get_sheet(0)
style = XFStyle()
style.num_format_str = 'general'
# Get the ground truth list
sheet_r = rb.sheets()[0]
gt_label_list = sheet_r.col_values(1)
# Initialize all variables
sess.run(tf.global_variables_initializer())
# Load the pretrained weights into the model
model.load_initial_weights(sess)
# Create figure handle
#fig2 = plt.figure(figsize=(15,6))
# Loop over all images
# for i, name in enumerate(all_list[0:35000]):
for imgID in range(st, end):
original_img_ID = imgID
imgID = str(imgID).zfill(8)
shard_num = math.floor((original_img_ID - 1) / 10000)
folder_num = math.floor((original_img_ID - 1) / 1000) + 1
#'G:\\validation_generated_QF\\shard-%d\\%d\\ILSVRC2012_val_%d-QF-%d.JPEG'
start = time.time()
qp_list = [i for i in range(0, 52, 2)]
qp_list.append(51)
for qf in qp_list:
col_idx = int(2 * qf / 5) + 4
# print(col_idx)
row = original_img_ID
actual_idx = original_img_ID
style = XFStyle()
style.num_format_str = 'general'
# if (sheet_r.cell(row,col_idx+1).value==u''):
name = ((DATA_PATH) % (shard_num, folder_num, imgID, qf))
image = cv2.imread(name)
# Convert image to float32 and resize to (227x227)
try:
img = cv2.resize(image.astype(np.float32), (227, 227))
# Subtract the ImageNet mean
img -= imagenet_mean
# Reshape as needed to feed into model
img = img.reshape((1, 227, 227, 3))
# Run the session and calculate the class probability
probs = sess.run(softmax, feed_dict={x: img, keep_prob: 1})
rank_estimate(probs, gt_label_list, actual_idx, sheet,
col_idx, wb)
except:
writeFileContents(name, file_xls)
continue
print('image %s is done , time: %f' % (imgID, time.time() - start))
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--positive_data",
help=
"A number from 0 to 3 that defines which will be the positive class")
parser.add_argument("--fold", help="path to the dataset")
parser.add_argument("--version", help="version of the classifier")
parser.add_argument("--mode", help="Train or Test")
parser.add_argument("--batch_size",
default=40,
help="number of images in each iteration times 4")
parser.add_argument('--model_path', default='vgg_16.ckpt', type=str)
parser.add_argument('--num_workers', default=4, type=int)
parser.add_argument('--weight_decay', default=5e-4, type=float)
parser.add_argument('--dropout_keep_prob', default=0.5, type=float)
parser.add_argument('--num_epochs', default=10, type=int)
args = parser.parse_args()
positive_class = int(args.positive_data)
fold = args.fold
mode = args.mode
version = args.version
batch_size = int(args.batch_size)
model_path = args.model_path
num_workers = args.num_workers
weight_decay = args.weight_decay
dropout_keep_prob = args.dropout_keep_prob
num_epochs = args.num_epochs
# Read and processs the training images
print("Loading training set")
'''
data_path_pos_train, data_path_neg_train, data_path_pos_val, data_path_neg_val = DatasetSelection(fold, positive_class)
train_filenames = data_path_pos_train + data_path_neg_train
train_labels = [0]*len(data_path_pos_train) + [1]*len(data_path_neg_train)
val_filenames = data_path_pos_val + data_path_neg_val
val_labels = [0]*len(data_path_pos_val) + [1]*len(data_path_neg_val)
num_classes = len(set(train_labels))
'''
train_filenames, val_filenames, train_labels, val_labels = DatasetSelectionMult(
fold)
num_classes = len(set(train_labels))
# --------------------------------------------------------------------------
# In TensorFlow, you first want to define the computation graph with all the
# necessary operations: loss, training op, accuracy...
# Any tensor created in the `graph.as_default()` scope will be part of `graph`
#graph = tf.Graph()
#with graph.as_default():
# Standard preprocessing for VGG on ImageNet taken from here:
# https://github.com/tensorflow/models/blob/master/research/slim/preprocessing/vgg_preprocessing.py
# Also see the VGG paper for more details: https://arxiv.org/pdf/1409.1556.pdf
# ----------------------------------------------------------------------
# DATASET CREATION using tf.contrib.data.Dataset
# https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/data
# The tf.contrib.data.Dataset framework uses queues in the background to feed in
# data to the model.
# We initialize the dataset with a list of filenames and labels, and then apply
# the preprocessing functions described above.
# Behind the scenes, queues will load the filenames, preprocess them with multiple
# threads and apply the preprocessing in parallel, and then batch the data
# Training dataset
train_dataset = tf.data.Dataset.from_tensor_slices(
(train_filenames, train_labels))
train_dataset = train_dataset.map(_parse_function,
num_parallel_calls=args.num_workers)
train_dataset = train_dataset.map(training_preprocess,
num_parallel_calls=args.num_workers)
train_dataset = train_dataset.shuffle(
buffer_size=10000) # don't forget to shuffle
batched_train_dataset = train_dataset.batch(int(args.batch_size))
# Validation dataset
val_dataset = tf.data.Dataset.from_tensor_slices(
(val_filenames, val_labels))
val_dataset = val_dataset.map(_parse_function,
num_parallel_calls=args.num_workers)
val_dataset = val_dataset.map(val_preprocess,
num_parallel_calls=args.num_workers)
batched_val_dataset = val_dataset.batch(int(args.batch_size))
# Now we define an iterator that can operator on either dataset.
# The iterator can be reinitialized by calling:
# - sess.run(train_init_op) for 1 epoch on the training set
# - sess.run(val_init_op) for 1 epoch on the valiation set
# Once this is done, we don't need to feed any value for images and labels
# as they are automatically pulled out from the iterator queues.
# A reinitializable iterator is defined by its structure. We could use the
# `output_types` and `output_shapes` properties of either `train_dataset`
# or `validation_dataset` here, because they are compatible.
iterator = tf.data.Iterator.from_structure(
batched_train_dataset.output_types,
batched_train_dataset.output_shapes)
images, labels = iterator.get_next()
train_init_op = iterator.make_initializer(batched_train_dataset)
val_init_op = iterator.make_initializer(batched_val_dataset)
# Indicates whether we are in training or in test mode
#is_training = tf.placeholder(tf.bool)
# ---------------------------------------------------------------------
# Now that we have set up the data, it's time to set up the model.
# For this example, we'll use VGG-16 pretrained on ImageNet. We will remove the
# last fully connected layer (fc8) and replace it with our own, with an
# output size num_classes=4
# We will first train the last layer for a few epochs.
# Then we will train the entire model on our dataset for a few epochs.
# Get the pretrained model, specifying the num_classes argument to create a new
# fully connected replacing the last one, called "vgg_16/fc8"
# Each model has a different architecture, so "vgg_16/fc8" will change in another model.
# Here, logits gives us directly the predicted scores we wanted from the images.
# We pass a scope to initialize "vgg_16/fc8" weights with he_initializer
train_layers = ['fc8']
#num_classes = 2
keep_prob = 1.0
model = AlexNet(images, keep_prob, num_classes, train_layers)
logits = model.fc8
print(logits)
exit()
# Specify where the model checkpoint is (pretrained weights).
#model_path = args.model_path
#assert(os.path.isfile(model_path))
# ---------------------------------------------------------------------
# Using tf.losses, any loss is added to the tf.GraphKeys.LOSSES collection
# We can then call the total loss easily
loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)
#loss = tf.losses.get_total_loss()
# Then we want to finetune the entire model for a few epochs.
# We run minimize the loss only with respect to all the variables.
starter_learning_rate = 0.001
global_step = tf.Variable(0, trainable=False, name='gs')
gs = tf.contrib.framework.get_variables('gs')
gs_init = tf.variables_initializer(gs)
sess.run(gs_init)
learning_rate = tf.train.exponential_decay(starter_learning_rate,
global_step, 500, 0.9)
#full_optimizer = tf.train.GradientDescentOptimizer(learning_rate)
full_optimizer = tf.train.GradientDescentOptimizer(learning_rate)
full_train_op = full_optimizer.minimize(loss, global_step=global_step)
# Evaluation metrics
prediction = tf.to_int32(tf.argmax(logits, 1))
correct_prediction = tf.equal(prediction, labels)
conf_matrix = tf.confusion_matrix(labels,
prediction,
num_classes=num_classes)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
#tf.get_default_graph().finalize()
saver = tf.train.Saver()
# --------------------------------------------------------------------------
# Now that we have built the graph and finalized it, we define the session.
# The session is the interface to *run* the computational graph.
# We can call our training operations with `sess.run(train_op)` for instance
sess.run(tf.global_variables_initializer())
model.load_initial_weights(sess)
best_acc = 0.0
plot_acc_val = []
plot_acc_train = []
# Train the entire model for a few more epochs, continuing with the *same* weights.
for epoch in range(num_epochs):
print('Starting epoch %d / %d' % (epoch + 1, num_epochs))
sess.run(train_init_op)
while True:
try:
_ = sess.run(full_train_op)
except tf.errors.OutOfRangeError:
break
# Check accuracy on the train and val sets every epoch
train_acc, train_matrix = check_accuracy(sess, conf_matrix,
train_init_op)
val_acc, val_matrix = check_accuracy(sess, conf_matrix, val_init_op)
plot_acc_val.append(val_acc)
plot_acc_train.append(train_acc)
if (val_acc > best_acc):
best_acc = val_acc
saver.save(sess, '/work/gbertocco/TCC/model04/ER_' + version)
info_validation = [val_acc, val_matrix]
pickle.dump(info_validation, open("INFO_VAL_" + version, "wb"))
pickle.dump(plot_acc_train, open("plot_val_" + version, "wb"))
pickle.dump(plot_acc_val, open("plot_train_" + version, "wb"))
print('Train normalized accuracy: %f' % train_acc)
print('Val normalized accuracy: %f\n' % val_acc)
print('Best (Val) normalized accuracy so far: %f\n' % best_acc)
class transfer_model(object):
model_name = "transfer_model" # name for checkpoint
def __init__(self, sess, epoch, batch_size, checkpoint_dir, log_dir, learning_rate = 0.00001, beta1=0.5):
self.sess = sess
self.keep_prob = 1.0
#self.dataset_name = dataset_name
#self.result_dir = result_dir
self.log_dir = log_dir
self.checkpoint_dir = checkpoint_dir
self.epoch = epoch
self.batch_size = batch_size
self.beta1 = beta1
self.label_dim = 50
self.train_set = DataSet("../data/train_augment", self.batch_size, self.label_dim)
self.test_set = DataSet("../data/test_augment", self.batch_size, self.label_dim)
# parameters
self.input_height = 227
self.input_width = 227
#self.output_height = 224
#self.output_width = 224
self.c_dim = 3
# train
self.init_learning_rate = learning_rate
# get number of batches for a single epoch
self.num_batches = self.train_set.total_batches
self.test_num_batches = self.test_set.total_batches
def classifier(self, x, is_training=True, reuse=False):
with tf.variable_scope("classifier", reuse=reuse):
#net = tf.reshape(x, [-1, 6*6*256])
#net = tf.nn.relu(bn(linear(net, 4096, scope='fc1'), is_training=is_training, scope='bn1'))
#net = tf.nn.relu(bn(linear(net, 4096, scope='fc2'), is_training=is_training, scope='bn2'))
#out = linear(net, self.label_dim, scope='fc3')
out = linear(x, self.label_dim, scope='fc8')
return out
def build_model(self):
# some parameters
image_dims = [self.input_height, self.input_width, self.c_dim]
bs = self.batch_size
""" Graph Input """
# images
self.inputs = tf.placeholder(tf.float32, [self.batch_size] + image_dims, name='input_images')
# labels
self.labels = tf.placeholder(tf.float32, [self.batch_size, self.label_dim], name='label')
# keep prob
self.keep_prob = tf.placeholder(tf.float32, shape=[])
# learning rate
self.learning_rate = tf.placeholder(tf.float32, shape=[])
""" Loss Function """
# get fc output of alexnet
self.alexnet = AlexNet(self.inputs, keep_prob=self.keep_prob, num_classes=1000, skip_layer=[])
logits = self.classifier(self.alexnet.dropout7, is_training=True, reuse=False)
prob = tf.nn.softmax(logits)
self.prob = prob
#
self.loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=self.labels))
""" Training """
# divide trainable variables into a group for D and a group for G
vars = tf.trainable_variables()
# optimizer
# self.optim = tf.train.GradientDescentOptimizer(self.learning_rate) \
# .minimize(self.loss, var_list=vars)
self.optim = tf.train.AdamOptimizer(self.learning_rate) \
.minimize(self.loss, var_list=vars)
""" Testing """
# for test
test_logits = self.classifier(self.alexnet.dropout7, is_training=False, reuse=True)
self.test_prob = tf.nn.softmax(test_logits)
self.correct_pred = tf.equal(tf.argmax(self.test_prob, axis=1), tf.argmax(self.labels, axis=1))
self.acc = tf.reduce_mean(tf.cast(self.correct_pred, tf.float32))
""" Summary """
self.loss_sum = tf.summary.scalar("loss", self.loss)
self.acc_sum = tf.summary.scalar("acc", self.acc)
def train(self):
# initialize all variables
tf.global_variables_initializer().run()
self.alexnet.load_initial_weights(self.sess)
# saver to save model
self.saver = tf.train.Saver()
# summary writer
self.writer = tf.summary.FileWriter(self.log_dir + '/' + self.model_name, self.sess.graph)
# restore check-point if it exits
could_load, checkpoint_counter = self.load(self.checkpoint_dir)
if could_load:
start_epoch = (int)(checkpoint_counter / self.num_batches)
start_batch_id = checkpoint_counter - start_epoch * self.num_batches
counter = checkpoint_counter
print(" [*] Load SUCCESS")
else:
start_epoch = 0
start_batch_id = 0
counter = 1
print(" [!] Load failed...")
# loop for epoch
start_time = time.time()
lr = self.init_learning_rate
min_loss = 100
update_cnt = 0
for epoch in range(start_epoch, self.epoch):
cur_loss = 0
# get batch data
for idx in range(start_batch_id, self.num_batches):
''' TODO: add data'''
inputs, labels = self.train_set.next_batch()
# update network
_, loss_summary_str, acc_summary_str, loss, prob = self.sess.run([self.optim, self.loss_sum, self.acc_sum, self.loss, self.prob], feed_dict={
self.inputs: inputs,
self.labels: labels,
self.keep_prob: 0.5,
self.learning_rate: lr})
self.writer.add_summary(loss_summary_str, counter)
self.writer.add_summary(acc_summary_str, counter)
cur_loss += loss
#print("output", prob)
#print("answer", labels)
# display training status
counter += 1
if counter % 100 == 1:
print("Epoch: [%2d] [%4d/%4d] time: %4.4f, loss: %.8f, lr: %.6f" \
% (epoch, idx, self.num_batches, time.time() - start_time, loss, lr))
# adapt learning rate
if cur_loss / (self.num_batches - start_batch_id) > min_loss:
update_cnt += 1
else:
update_cnt = 0
if update_cnt == 5:
update_cnt = 0
lr = lr * 0.5
# After an epoch, start_batch_id is set to zero
# non-zero value is only for the first epoch after loading pre-trained model
start_batch_id = 0
# save model
self.save(self.checkpoint_dir, counter)
'''Test'''
acc = 0
for idx in range(0, self.test_num_batches):
inputs, labels = self.test_set.next_batch()
correct_pred, prob = self.sess.run([self.correct_pred, self.prob],
feed_dict={
self.inputs: inputs,
self.labels: labels,
self.keep_prob: 1.0})
#if np.sum(prob) == np.argmax(labels):
#print(prob[0])
#print(np.argmax(prob, axis=1), np.argmax(labels, axis=1))
acc += (np.sum(correct_pred) + 0.0)/ self.batch_size
#print np.sum(correct_pred), self.batch_size
#print(np.sum(correct_pred), self.batch_size)
#print(len(correct_pred), labels.shape)
#print(correct_pred)
print("Epoch: [%2d] acc: %.8f" \
% (epoch, (acc + 0.0) / self.test_num_batches))
# save model for final step
self.save(self.checkpoint_dir, counter)
def pred(self):
# saver to save model
self.saver = tf.train.Saver()
could_load, checkpoint_counter = self.load(self.checkpoint_dir)
if could_load:
print(" [*] Load SUCCESS")
else:
print(" [!] Load failed...")
for idx in range(0, self.predict_num_batches):
inputs, labels = self.predict_set.next_batch()
prob = self.sess.run([self.test_prob], feed_dict={self.inputs: inputs})
#print prob
#print labels
#print self.label_name[np.argmax(prob)], self.label_name[np.argmax(labels)]
#print "============"
@property
def model_dir(self):
return "{}_{}".format(
self.model_name,
self.batch_size)
def save(self, checkpoint_dir, step):
checkpoint_dir = os.path.join(checkpoint_dir, self.model_dir, self.model_name)
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
self.saver.save(self.sess,os.path.join(checkpoint_dir, self.model_name+'.model'), global_step=step)
def load(self, checkpoint_dir):
import re
print(" [*] Reading checkpoints...")
checkpoint_dir = os.path.join(checkpoint_dir, self.model_dir, self.model_name)
ckpt = tf.train.get_checkpoint_state(checkpoint_dir)
if ckpt and ckpt.model_checkpoint_path:
ckpt_name = os.path.basename(ckpt.model_checkpoint_path)
self.saver.restore(self.sess, os.path.join(checkpoint_dir, ckpt_name))
counter = int(next(re.finditer("(\d+)(?!.*\d)",ckpt_name)).group(0))
print(" [*] Success to read {}".format(ckpt_name))
return True, counter
else:
print(" [*] Failed to find a checkpoint")
return False, 0
def dataInputGen():
# File Path desc
train_file = "./TrainingFile_detail.txt"
Validate_file = "./ValidationFile_detail.txt"
class_file = "./Classmap.txt"
checkpoint_path = "./ckp/"
# Network params
learning_rate = 0.00001
dropout_rate = 0.5
num_classes = 1000
batch_size = 1
num_epochs = 100
display_step = 20 #check acc per epoach
train_layers = ['fc8', 'fc7', 'fc6']
#read all image path config
train_img_paths,train_labels = read_train_detail(train_file)
validate_img_paths,validate_labels = read_train_detail(Validate_file)
print("Total Dataset {}".format(int(len(train_labels)+len(validate_labels))))
print("Split to Training {} and to Validation {}".format(len(train_labels),len(validate_labels)))
with tf.device('/cpu:0'):
tr_data = ImageDataGenerator(mode='training',
batch_size=batch_size,
num_classes=num_classes,
class_file=class_file,
shuffle=False,
img_paths=train_img_paths,
labels=train_labels)
# create an reinitializable iterator given the dataset structure
iterator = Iterator.from_structure(tr_data.data.output_types,
tr_data.data.output_shapes)
next_batch = iterator.get_next()
# Ops for initializing the two different iterators
training_init_op = iterator.make_initializer(tr_data.data)
validation_init_op = iterator.make_initializer(val_data.data)
# Get the number of training/validation steps per epoch
train_batches_per_epoch = int(np.floor(tr_data.data_size/batch_size))
val_batches_per_epoch = int(np.floor(val_data.data_size/batch_size))
# TF placeholder for graph input and output
x = tf.placeholder(tf.float32, [None, 227, 227, 3])
y = tf.placeholder(tf.float32, [None, num_classes])
keep_prob = tf.placeholder(tf.float32)
# Initialize model
model = AlexNet(x, keep_prob, num_classes, train_layers,weights_path="./weights/bvlc_alexnet.npy")
# Link variable to model output
score = model.fc8
# List of trainable variables of the layers we want to train
var_list = [v for v in tf.trainable_variables() if v.name.split('/')[0] in train_layers]
# Op for calculating the loss
with tf.name_scope("cross_ent"):
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=score,labels=y))
# Train op
with tf.name_scope("train"):
optimizer = tf.train.AdamOptimizer(learning_rate).minimize(loss)
# Evaluation op: Accuracy of the model
with tf.name_scope("accuracy"):
correct_pred = tf.equal(tf.argmax(score, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32),name="ACC")
# Add the loss to summary
tf.summary.scalar('cross_entropy', loss)
tf.summary.scalar('Acc', accuracy)
# Merge all summaries together
merged_summary = tf.summary.merge_all()
# Initialize the FileWriter
writer = tf.summary.FileWriter("./Graph")
# Initialize an saver for store model checkpoints
saver = tf.train.Saver()
with tf.Session() as sess:
# Initialize all variables
sess.run(tf.global_variables_initializer())
# Add the model graph to TensorBoard
writer.add_graph(sess.graph)
# Load the pretrained weights into the non-trainable layer
print("Weight Loading...")
model.load_initial_weights(sess)
print("{} Start training...".format(datetime.now()))
best_acc = 0
# Loop over number of epochs
for epoch in range(num_epochs):
print("{} Epoch number: {}".format(datetime.now(), epoch+1))
# Initialize iterator with the training dataset
sess.run(training_init_op)
for step in tqdm(range(train_batches_per_epoch)):
# get next batch of data
img_batch, label_batch = sess.run(next_batch)
# And run the training op
sess.run(train_op, feed_dict={x: img_batch,
y: label_batch,
keep_prob: dropout_rate})
# Generate summary with the current batch of data and write to file
if step % display_step == 0:
s,acc_data = sess.run([merged_summary,accuracy], feed_dict={x: img_batch,
y: label_batch,
keep_prob: 1.})
writer.add_summary(s, epoch*train_batches_per_epoch + step)
#print("Accuracy at steps {} is {:f}".format((epoch*train_batches_per_epoch + step),acc_data))
# Validate the model on the entire validation set
print("{} Start validation".format(datetime.now()))
sess.run(validation_init_op)
test_acc = 0.
test_count = 0
for _ in tqdm(range(val_batches_per_epoch)):
img_batch, label_batch = sess.run(next_batch)
acc = sess.run(accuracy, feed_dict={x: img_batch,
y: label_batch,
keep_prob: 1.})
test_acc += acc
test_count += 1
test_acc /= test_count
print("{} Validation Accuracy = {:.4f}".format(datetime.now(), test_acc))
print("Checking improving Accuracy...")
if test_acc > best_acc:
print("Accuracy improve from {:.4f} to {:.4f}".format(best_acc,test_acc))
print("Saving best weights ...")
save_path = saver.save(sess, "./BestWeight/weight")
best_acc = test_acc
else:
print("Accuracy not improve")
print("{} Saving checkpoint of model...".format(datetime.now()))
# save checkpoint of the model
checkpoint_name = os.path.join(checkpoint_path,
'model_epoch'+str(epoch+1)+'.ckpt')
save_path = saver.save(sess, checkpoint_name)
print("{} Model checkpoint saved at {}".format(datetime.now(),
checkpoint_name))
class NearestNeighbor:
cluster_dict = {}
def __init__(self):
# The number of output classes of Alexnet
num_classes = 1000
# No train on the layers
train_layers = []
# TF placeholder for graph input and output
self.x = tf.placeholder(tf.float32, [1, 227, 227, 3])
self.keep_prob = tf.placeholder(tf.float32)
# Initialize model
self.model = AlexNet(self.x, self.keep_prob, num_classes, train_layers)
# Start the Tensorflow Session and initialize all variables
self.sess = tf.Session()
self.sess.run(tf.global_variables_initializer())
# Load the pre-trained weights into the non-trainable layer
self.model.load_initial_weights(self.sess)
# Declare the feature-vector-dictionary
self.load_cluster_dict()
# Init Audio
self.audio = Audio()
# Classifies the given picture and returns the label and if it was a new label
def classify(self, name, speech=False):
# Get the image
image_path = "classification/pictures/" + name + ".jpeg"
im1 = cv2.imread(image_path).astype(np.float32)
im1 = im1 - np.mean(im1)
# RGB to BGR
im1[:, :, 0], im1[:, :, 2] = im1[:, :, 2], im1[:, :, 0]
features = self.get_features(np.array([im1]))
best_match = ("", np.inf)
# Compute all distances to the clusters and search the nearest
for label in self.cluster_dict.keys():
cluster = self.cluster_dict[label]
distance = spatial.distance.euclidean(np.array(features[0]), cluster[0])
if distance < best_match[1]:
best_match = (label, distance)
self.audio.output("\nThe object is a " + best_match[0])
# check if the estimation was correct or not
while True:
correct_estimation = self.audio.input("Is the shown object a '" + best_match[0] + "' ? (yes or no)",
speech=speech)
# add the features to the neighbor dict if it was the correct estimation
if correct_estimation == "y" or correct_estimation == "Y" or correct_estimation == "yes":
self.calculate_cluster(np.array(features), best_match[0])
return best_match[0], False
# ask what the correct label is
elif correct_estimation == "n" or correct_estimation == "N" or correct_estimation == "no":
correct_label = self.audio.input("What is the correct label then? (correct spelling required!)",
speech=speech)
if correct_label in self.cluster_dict.keys():
self.calculate_cluster(np.array(features), correct_label)
return correct_label, False
# ask if it is a new label
else:
new_label = self.audio.input("Is this a complete new label? (yes or no)", speech=speech)
if new_label == "y" or new_label == "Y" or new_label == "yes":
self.calculate_cluster(np.array(features), correct_label)
return correct_label, True
else:
self.audio.output("Then may check the correct spelling.")
else:
self.audio.output("Wrong input, please check again.")
# Extracts the feature vector of the given picture and saves it in the dict
def learn(self, label, image_name="tmp_picture"):
image_path = "classification/pictures/" + image_name + ".jpeg"
# Get the Image
image = (cv2.imread(image_path)[:, :, :3]).astype(np.float32)
image = image - np.mean(image)
# RGB to BGR
image[:, :, 0], image[:, :, 2] = image[:, :, 2], image[:, :, 0]
features = self.get_features(np.array([image]))
self.calculate_cluster(np.array(features), label)
self.audio.output("\nCluster of the label is updated. \n")
# Saves all feature vectors of the pictures of the given folder with the given label
def batch_learn(self, label, folder_path="classification/pictures/"):
folder_path = folder_path + label
# Creates an array with all images as arrays
images = []
for image in os.listdir(folder_path):
if image.endswith(".jpg") or image.endswith(".jpeg"):
# Get the image
image_path = os.path.join(folder_path, image)
image = (cv2.imread(image_path)[:, :, :3]).astype(np.float32)
image = image - np.mean(image)
# RGB to BGR
image[:, :, 0], image[:, :, 2] = image[:, :, 2], image[:, :, 0]
images.append(image)
features = self.get_features(np.array(images))
self.calculate_cluster(np.array(features), label)
self.audio.output("Finished " + label)
# Get the feature vectors of all the images in the images array
def get_features(self, images):
features = []
# Link variable to model output
score = self.model.fc7
for im in images:
# Skip image if the shape is not the right one
if im.shape != (227, 227, 3):
print("Image has wrong resolution!")
continue
# Feed alexnet with the image
output = self.sess.run(score, feed_dict={self.x: [im], self.keep_prob: 1.})
features.append(output[0])
return features
# Calculates the new mean of the label regarding the feature array
def calculate_cluster(self, features, label):
mean_features = np.mean(features, axis=0)
if label not in self.cluster_dict.keys():
self.cluster_dict[label] = (mean_features, features.shape[0])
else:
cluster = self.cluster_dict[label]
new_cluster = (cluster[1] * cluster[0] + features.shape[0] * mean_features) / (
cluster[1] + features.shape[0])
self.cluster_dict[label] = (new_cluster, cluster[1]+features.shape[0])
# Load the neighbor list out of the file
def load_cluster_dict(self):
try:
print "open file"
self.cluster_dict = dict(pickle.load(open("classification/cluster_list.p", "rb")))
print "done"
except IOError:
print "no file found. start with empty dictionary"
# Shows all labels of the neighbor dictionary
def show_labels(self):
for label in self.cluster_dict.keys():
print(label + ": " + str(self.cluster_dict[label][1]) + " pictures included")
# Deletes a label from the neighbor dict and the feature vectors of it
def delete_label(self, label):
if label in self.cluster_dict.keys():
del self.cluster_dict[label]
else:
self.audio.output("The label is not included.")
def clear_cluster_dict(self):
self.cluster_dict.clear()
def save_cluster_dict(self):
pickle.dump(self.cluster_dict, open("classification/cluster_list.p", "wb"))
print("file saved")
# Saves the neighbor dictionary and closes the tensorflow session
def close(self):
self.save_cluster_dict()
self.sess.close()
# Get the number of training/validation steps per epoch
train_batches_per_epoch = np.floor(train_generator.data_size / batch_size).astype(np.int16)
val_batches_per_epoch = np.floor(val_generator.data_size / batch_size).astype(np.int16)
# Start Tensorflow session
with tf.Session() as sess:
# Initialize all variables
sess.run(tf.global_variables_initializer())
# Add the model graph to TensorBoard
writer.add_graph(sess.graph)
# Load the pretrained weights into the non-trainable layer
model.load_initial_weights(sess)
print("{} Start training...".format(datetime.now()))
print("{} Open Tensorboard at --logdir {}".format(datetime.now(),
filewriter_path))
# Loop over number of epochs
for epoch in range(num_epochs):
print("{} Epoch number: {}".format(datetime.now(), epoch+1))
step = 1
while step < train_batches_per_epoch:
# Get a batch of images and labels
def main(_):
if not FLAGS.image_paths:
raise ValueError("Must set --image_paths")
if not FLAGS.save_file:
raise ValueError("Must set --save_file")
image_paths_file = FLAGS.image_paths
save_file = FLAGS.save_file
batch_size = FLAGS.batch_size
num_classes = 1000
emb_dim = 4096
# TF placeholder for graph input and output
x = tf.placeholder(tf.float32, [None, 227, 227, 3])
kp = tf.placeholder(tf.float32)
# Initialize model
train_layers = []
model = AlexNet(x, kp, num_classes, emb_dim, train_layers)
lines = open(image_paths_file).readlines()
image_paths = [l.split()[0] for l in lines]
def path2id(path):
return path.split('/')[-1][:-4]
ids = [path2id(p.split()[0]) for p in image_paths]
labels = [int(l.split()[1]) for l in lines]
with tf.Session() as sess:
# Initialize all variables
sess.run(tf.global_variables_initializer())
# Load the pretrained weights into the non-trainable layer
model.load_initial_weights(sess)
pool5_tensor = sess.graph.get_tensor_by_name('pool5:0')
num_batches = m.ceil(len(image_paths) / float(batch_size))
features = []
for i in range(num_batches):
s_batch = i * batch_size
e_batch = min(s_batch + batch_size, len(image_paths))
# Read the image
img_pool = []
for b in range(s_batch, e_batch):
img = cv2.imread(image_paths[b])
# Convert image to float32 and resize to (227x227)
img = cv2.resize(img.astype(np.float32), (227, 227))
# Subtract the ImageNet mean
img -= imagenet_mean
# Reshape as needed to feed into model
img_pool.append(img.reshape((227, 227, 3)))
img_pool = np.stack(img_pool, axis=0)
feature = sess.run(pool5_tensor, feed_dict={x: img_pool, kp: 1.0})
feature = np.squeeze(np.reshape(feature, [-1, 6 * 6 * 256]))
features.append(feature)
if (i + 1) % 10000 == 0:
print('Processed ' + str(i + 1) + ' files')
features = np.vstack(features)
batch_size = 50000
array_list = []
split_idx = np.arange(batch_size, len(image_paths), batch_size)
features = np.split(features, split_idx, axis=0)
for i, feature in enumerate(features):
print('Seg: %d size: %d' % (i, feature.shape[0]))
pickle.dump((features, labels, ids), open(save_file, 'wb'))
def main():
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
# read data from tfrecord files
img, label = reader.read_and_decode(args.tfrecords,
epochs=args.num_epochs,
size=args.img_size)
img_batch, label_batch = tf.train.shuffle_batch([img, label],
batch_size=args.batch_size,
capacity=2000,
min_after_dequeue=1000)
global_step = tf.Variable(0, name="global_step", trainable=False)
# construct network model
model = AlexNet(img_batch, args.dropout_rate, args.num_bit, args.num_class,
skip_layers)
D = model.softsign
[d_loss, out] = hashing_loss(D, label_batch, args.alpha, args.belta,
args.gama, args.num_bit)
# List of trainable variables of the layers to finetune
var_list1 = [
v for v in tf.trainable_variables()
if v.name.split('/')[0] not in skip_layers
]
# List of trainable variables of the layers to train from scratch
var_list2 = [
v for v in tf.trainable_variables()
if v.name.split('/')[0] in skip_layers
]
# learning rate
learning_rate = tf.train.exponential_decay(args.lr,
global_step,
args.decay_step,
args.decay_rate,
staircase=True)
opt1 = tf.train.AdamOptimizer(learning_rate * 0.001)
opt2 = tf.train.AdamOptimizer(learning_rate)
# apply different grads for two type layers
grads = tf.gradients(d_loss, var_list1 + var_list2)
grads1 = grads[:len(var_list1)]
grads2 = grads[len(var_list1):]
train_op1 = opt1.apply_gradients(zip(grads1, var_list1))
train_op2 = opt2.apply_gradients(zip(grads2, var_list2),
global_step=global_step)
train_op = tf.group(train_op1, train_op2)
with tf.Session(config=config) as sess:
saver = tf.train.Saver(tf.global_variables())
sess.run([
tf.global_variables_initializer(),
tf.local_variables_initializer()
])
if args.checkpoint is not None:
checkpoint = tf.train.latest_checkpoint(args.checkpoint)
print('Restoring model from {}'.format(checkpoint))
saver.restore(sess, checkpoint)
else:
# Load the pretrained weights into the non-trainable layer
model.load_initial_weights(sess)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
start_time = time.time()
try:
while not coord.should_stop():
_, loss_t, dt, step1 = sess.run(
[train_op, d_loss, out, global_step])
elapsed_time = time.time() - start_time
start_time = time.time()
if step1 % 10 == 0:
print("iter: %4d, loss: %.8f, time: %.3f" %
(step1, loss_t, elapsed_time))
if step1 % args.save_freq == 0:
saver.save(sess,
args.output_dir + '/model.ckpt',
global_step=step1)
except tf.errors.OutOfRangeError:
saver.save(sess, args.output_dir + '/model-done.ckpt')
print('Done training -- epoch limit reached')
finally:
coord.request_stop()
coord.join(threads)
class Model(object):
def __init__(self, mode='train'):
if mode == 'train' or mode == 'test':
self.mode = mode
else:
print('Mode must be train or test.')
# Learning params
self.learning_rate = 1e-3
self.minibatch_size = 32
self.num_iterations = 1500
self.train_test_split = 0.7
self.keep_rate = 0.6
self.num_layers_to_load = 5
self.display_step = 5
self.save_step = 100
self.save_dir = "tf_data\sample_model"
self.input_image_size = (227, 227)
self.label_indices = {
'20': 0,
'30': 1,
'50': 2,
'60': 3,
'70': 4,
'80': 5,
'80x': 6,
'100': 7,
'120': 8
}
self.labels_ordered = list(self.label_indices)
self.num_classes = len(self.label_indices)
#Go ahead and build graph on initialization:
self.build_graph()
#Set to true with we create a tf session.
self.initialized = False
def build_graph(self):
self.X = tf.placeholder(
tf.float32,
[None, self.input_image_size[0], self.input_image_size[1], 3])
self.y = tf.placeholder(tf.float32, [None, self.num_classes])
self.keep_prob = tf.placeholder(tf.float32)
self.AN = AlexNet(self.X, self.keep_prob, self.num_layers_to_load)
alexnet_out = self.AN.pool5
print(alexnet_out)
self.flattened = tf.reshape(alexnet_out, [-1, 6 * 6 * 256])
#Dense layer followed by a dropout layer
self.dense1 = tf.layers.dense(inputs=self.flattened,
units=1024,
activation=tf.nn.relu,
name="my_dense1")
print("D1 Shape: ", self.dense1.get_shape())
self.dropout1 = tf.layers.dropout(
inputs=self.dense1,
rate=0.4,
training=self.mode == tf.estimator.ModeKeys.TRAIN,
name="my_drop1")
print("Drop1 Shape: ", self.dropout1.get_shape())
#Smaller dense layer followed by a dropout layer
self.dense2 = tf.layers.dense(inputs=self.dropout1,
units=128,
activation=tf.nn.relu,
name="my_dense2")
print("D2 Shape: ", self.dense2.get_shape())
self.dropout2 = tf.layers.dropout(
inputs=self.dense2,
rate=0.4,
training=self.mode == tf.estimator.ModeKeys.TRAIN,
name="my_drop2")
print("Ddrop2 Shape: ", self.dropout2.get_shape())
#Output of 3 for the three available classes
self.output = tf.layers.dense(inputs=self.dropout2,
units=9,
name="my_out")
print("Output Shape: ", self.output.get_shape())
self.yhat = self.output
print("yHat Shape: ", self.yhat.get_shape())
self.trainable_variable_names = [
'conv5', 'my_dense1', 'my_dense2', 'my_out'
]
def train(self):
cost = tf.losses.sparse_softmax_cross_entropy(
tf.argmax(self.y, axis=1), self.output)
var_list = [v for v in tf.trainable_variables() if v.name.split('/')[0] \
in self.trainable_variable_names]
# Train op
with tf.name_scope("train"):
# Get gradients of all trainable variables
gradients = tf.gradients(cost, var_list)
gradients = list(zip(gradients, var_list))
# Create optimizer and apply gradient descent to the trainable variables
optimizer = tf.train.AdamOptimizer(self.learning_rate)
train_op = optimizer.apply_gradients(grads_and_vars=gradients)
# Add gradients to summary
for gradient, var in gradients:
tf.summary.histogram(var.name + '/gradient', gradient)
# Add the variables we train to the summary
for var in var_list:
tf.summary.histogram(var.name, var)
# Add the loss to summary
tf.summary.scalar('cross_entropy', cost)
# Evaluation op: Accuracy of the model
with tf.name_scope("accuracy"):
correct_pred = tf.equal(tf.argmax(self.yhat, 1),
tf.argmax(self.y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# Add the accuracy to the summary
tf.summary.scalar('accuracy', accuracy)
# Merge all summaries together
merged_summary = tf.summary.merge_all()
# Initialize the FileWriter
train_writer = tf.summary.FileWriter(
os.path.join(self.save_dir, 'train'))
test_writer = tf.summary.FileWriter(os.path.join(
self.save_dir, 'test'))
# Initialize an saver for store model checkpoints
# We'll save the pretrained alexnet weights and the new weight values that we train:
var_list_to_save = [v for v in tf.trainable_variables() if v.name.split('/')[0] in \
self.AN.all_variable_names + self.trainable_variable_names]
saver = tf.train.Saver(var_list=var_list_to_save)
#Load up data from image files.
data = data_feeder(label_indices=self.label_indices,
train_test_split=self.train_test_split,
input_image_size=self.input_image_size,
data_path='GTSRB\Final_Training\SpeedLimitSigns')
#Setup minibatch generators
G = Generator(data.train.X,
data.train.y,
minibatch_size=self.minibatch_size)
GT = Generator(data.test.X,
data.test.y,
minibatch_size=self.minibatch_size)
#Launch tf session
self.sess = tf.Session()
self.sess.run(tf.global_variables_initializer())
self.initialized = True
# Add the model graph to TensorBoard
train_writer.add_graph(self.sess.graph)
# Load the pretrained weights into the alexnet portion of our graph
self.AN.load_initial_weights(self.sess)
print("{} Start training...".format(datetime.now()))
print("{} Open Tensorboard at --logdir {}".format(
datetime.now(), self.save_dir))
#And Train
for i in range(self.num_iterations):
G.generate()
# And run the training op
self.sess.run(train_op,
feed_dict={
self.X: G.X,
self.y: G.y,
self.keep_prob: self.keep_rate
})
# Generate summary with the current batch of data and write to file
if i % self.display_step == 0:
s = self.sess.run(merged_summary,
feed_dict={
self.X: G.X,
self.y: G.y,
self.keep_prob: 1.0
})
train_writer.add_summary(s, i)
GT.generate()
s = self.sess.run(merged_summary,
feed_dict={
self.X: GT.X,
self.y: GT.y,
self.keep_prob: 1.0
})
test_writer.add_summary(s, i)
train_acc = self.sess.run(accuracy,
feed_dict={
self.X: G.X,
self.y: G.y,
self.keep_prob: 1.0
})
test_acc = self.sess.run(accuracy,
feed_dict={
self.X: GT.X,
self.y: GT.y,
self.keep_prob: 1.0
})
print(i, ' iterations,', str(G.num_epochs),
'epochs, train accuracy = ', train_acc,
', test accuracy = ', test_acc)
if i % self.save_step == 0 and i > 0:
print("{} Saving checkpoint of model...".format(
datetime.now()))
# save checkpoint of the model
checkpoint_name = os.path.join(
self.save_dir, 'model_epoch' + str(G.num_epochs) + '.ckpt')
save_path = saver.save(self.sess, checkpoint_name)
print("{} Model checkpoint saved at {}".format(
datetime.now(), checkpoint_name))
def predict(self, X, checkpoint_dir=None):
'''
X: Numpy array of dimension [n, 227, 227, 3].
Note that n here may not be the same as the minibatch_size defined above.
Returns:
yhat_numpy: A numpy array of dimension [n x 3] containing the predicted
one hot labels for each image passed in X.
'''
if not self.initialized:
self.restore_from_checkpoint(checkpoint_dir)
yhat_numpy = self.sess.run(self.yhat,
feed_dict={
self.X: X,
self.keep_prob: 1.0
})
return yhat_numpy
def restore_from_checkpoint(self, checkpoint_dir=None):
'''
Restore model from most recent checkpoint in save dir.
'''
saver = tf.train.Saver()
self.sess = tf.Session()
#Load latest checkpont, use savedir if we're not given a checkpoint dir:
if checkpoint_dir is None:
checkpoint_name = tf.train.latest_checkpoint(self.save_dir)
else:
checkpoint_name = tf.train.latest_checkpoint(checkpoint_dir)
print(checkpoint_name)
#Resore session
saver.restore(self.sess, checkpoint_name)
self.initialized = True
loss = tf.reduce_mean(loss_v)
opt_gd = tf.train.GradientDescentOptimizer(learning_rate=1e-4)
train_step = opt_gd.minimize(loss)
##
# start TF session and init data loaders
sess = tf.Session()
db_train.init(sess)
db_test.init(sess)
# initialize network weights
sess.run(
tf.global_variables_initializer()) # this initialized the weights randomly
net.load_initial_weights(
sess) # this loads the pretrained alexnet weights from the npy file
# saver for the network weights
saver = tf.train.Saver(max_to_keep=1)
if not os.path.exists(SNAPSHOT_DIR):
os.mkdir(SNAPSHOT_DIR)
else:
shutil.rmtree(SNAPSHOT_DIR)
os.mkdir(SNAPSHOT_DIR)
# TRAINING LOOP
tr_loss = []
for step in range(TRAIN_ITER):
# work around to deal with cases when batch size doesnt divide the number of samples of the db set evenly
while True:
# load images and annotation
def main(argv=None):
if (argv is None):
argv = sys.argv
try:
try:
opts = argv
first_time_load = True
parent_dir = './'
keys = ['cov1', 'cov2', 'fc1', 'fc2', 'fc3']
prune_thresholds = {}
WITH_BIASES = False
save_for_next_iter = False
TEST = False
for key in keys:
prune_thresholds[key] = 0.
for item in opts:
print(item)
opt = item[0]
val = item[1]
if (opt == '-cRates'):
cRates = val
if (opt == '-first_time'):
first_time_load = val
if (opt == '-file_name'):
file_name = val
if (opt == '-train'):
TRAIN = val
if (opt == '-prune'):
PRUNE = val
if (opt == '-test'):
TEST = val
if (opt == '-parent_dir'):
parent_dir = val
if (opt == '-lr'):
lr = val
if (opt == '-with_biases'):
WITH_BIASES = val
if (opt == '-lambda1'):
lambda_1 = val
if (opt == '-lambda2'):
lambda_2 = val
if (opt == '-save'):
save_for_next_iter = val
if (opt == '-org_file_name'):
org_file_name = val
print('pruning thresholds are {}'.format(prune_thresholds))
except getopt.error, msg:
raise Usage(msg)
epochs = 100
dropout = 0.5
batch_size = 1
num_classes = 1000
NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN = 50000
INITIAL_LEARNING_RATE = 0.001
INITIAL_LEARNING_RATE = lr
LEARNING_RATE_DECAY_FACTOR = 0.1
NUM_EPOCHS_PER_DECAY = 350.0
DISPLAY_FREQ = 50
TEST = 1
TRAIN_OR_TEST = 0
NUM_CHANNELS = 3
first_time_load = 1
mask_dir = parent_dir
weights_dir = parent_dir
LOCAL_TEST = 1
file_name_part = compute_file_name(cRates)
if (save_for_next_iter):
(weights_mask, biases_mask) = initialize_weights_mask(
first_time_load, mask_dir, 'mask' + org_file_name + '.pkl')
else:
(weights_mask, biases_mask) = initialize_weights_mask(
first_time_load, mask_dir, 'mask' + file_name_part + '.pkl')
meta_data_dir = '/local/scratch/share/ImageNet/ILSVRC/Data/CLS-LOC'
index_file_dir = '/local/scratch/share/ImageNet/ILSVRC/ImageSets/CLS-LOC/'
index_file_dir = 'cpu_test_data/'
if (TRAIN):
train_file_txt = index_file_dir + 'train.txt'
val_file_txt = index_file_dir + 'val.txt'
test_file_txt = index_file_dir + 'test.txt'
else:
test_file_txt = index_file_dir + 'test.txt'
# if (first_time_load):
# PREV_MODEL_EXIST = 0
# weights, biases = initialize_variables(PREV_MODEL_EXIST, '')
# else:
# PREV_MODEL_EXIST = 1
# if (save_for_next_iter):
# file_name_part = org_file_name
# else:
# file_name_part = compute_file_name(cRates)
# weights, biases = initialize_variables( PREV_MODEL_EXIST,
# weights_dir + 'weights' + file_name_part + '.pkl')
#
x = tf.placeholder(tf.float32, [batch_size, 227, 227, 3])
y = tf.placeholder(tf.float32, [None, num_classes])
keep_prob = tf.placeholder(tf.float32)
# initilize the model from the class constructer
model = AlexNet(x, keep_prob, num_classes)
score = model.fc8
softmax = tf.nn.softmax(score)
with tf.name_scope("cross_ent"):
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=score,
labels=y))
with tf.name_scope("train"):
# l1_norm = lambda_1 * l1
# l2_norm = lambda_2 * l2
# regulization_loss = l1_norm + l2_norm
opt = tf.train.AdamOptimizer(lr)
# loss_value = tf.reduce_mean(cross_entropy) + regulization_loss
grads = opt.compute_gradients(loss)
org_grads = [(ClipIfNotNone(grad), var) for grad, var in grads]
train_step = opt.apply_gradients(org_grads)
with tf.name_scope("accuracy"):
correct_prediction = tf.equal(tf.argmax(score, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
# keys = ['cov1', 'cov2', 'fc1', 'fc2', 'fc3']
# weights_new[key] = weights[key] * tf.constant(weights_mask[key], dtype=tf.float32)
#
# l1, l2, pred = cov_network(images, weights_new, biases, keep_prob)
# _, _, test_pred = cov_network(test_images, weights_new, biases, keep_prob)
# cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits = pred, labels = y)
#
# new_grads = mask_gradients(weights, org_grads, weights_mask, biases, biases_mask, WITH_BIASES)
# Apply gradients.
init = tf.global_variables_initializer()
accuracy_list = np.zeros(20)
train_acc_list = []
# Launch the graph
print('Graph launching ..')
if (TRAIN):
train_generator = ImageDataGenerator(train_file_txt,
horizontal_flip=True,
shuffle=True)
val_generator = ImageDataGenerator(val_file_txt, shuffle=False)
# test_generator = ImageDataGenerator(test_file_txt, shuffle = False)
if (TRAIN):
# Get the number of training/validation steps per epoch
train_batches_per_epoch = np.floor(train_generator.data_size /
batch_size).astype(np.int16)
val_batches_per_epoch = np.floor(val_generator.data_size /
batch_size).astype(np.int16)
# test_batches_per_epoch = np.floor(test_generator.data_size / batch_size).astype(np.int16)
with tf.Session() as sess:
sess.run(init)
model.load_initial_weights(sess)
# print('pre train pruning info')
# prune_info(weights_new, 0)
# print(78*'-')
# start = time.time()
if TRAIN == 1:
print("{} Start training...".format(datetime.now()))
for i in range(0, epochs):
for step in range(train_batches_per_epoch):
(batch_x,
batch_y) = train_generator.next_batch(batch_size)
train_acc, cross_en = sess.run([accuracy, loss_value],
feed_dict={
x: batch_x,
y: batch_y,
keep_prob: 1.0
})
if (i % DISPLAY_FREQ == 0):
print('This is the {}th of {}pruning, time is {}'.
format(i, cRates, datetime.now()))
print("accuracy is {} and cross entropy is {}".
format(train_acc, cross_en))
accuracy_list = np.concatenate(
(np.array([train_acc]), accuracy_list[0:19]))
if (i % (DISPLAY_FREQ * 50) == 0 and i != 0):
train_acc_list.append(train_acc)
# file_name_part = compute_file_name(cRates)
# save_pkl_model(weights, biases, weights_dir, 'weights' + file_name_part + '.pkl')
# print("saved the network")
if (np.mean(accuracy_list) > 0.81
and train_acc >= 0.83):
accuracy_list = np.zeros(20)
test_acc = sess.run(accuracy,
feed_dict={
x: images_test,
y: labels_test,
keep_prob: 1.0
})
print('test accuracy is {}'.format(test_acc))
if (test_acc > 0.823):
print(
"training accuracy is large, show the list: {}"
.format(accuracy_list))
break
_ = sess.run(train_step,
feed_dict={
x: batch_x,
y: batch_y,
keep_prob: dropout
})
if (TEST):
test_acc_list = []
if (LOCAL_TEST == 1):
image_dir = "cpu_test_data/tmp_images/"
imagenet_mean = np.array([104., 117., 124.],
dtype=np.float32)
test_batches_per_epoch = 3
img_files = [
os.path.join(image_dir, f)
for f in os.listdir(image_dir) if f.endswith('.jpeg')
]
imgs_test = []
for i, f in enumerate(img_files):
tmp = cv2.imread(f)
tmp = cv2.resize(tmp.astype(np.float32), (227, 227))
tmp -= imagenet_mean[i]
tmp = tmp.reshape((1, 227, 227, 3))
imgs_test.append(tmp)
if (LOCAL_TEST):
names = []
probs = []
for step in range(test_batches_per_epoch):
prob = sess.run(
softmax,
feed_dict={
x: imgs_test[step],
# y: labels_test,
keep_prob: 1.0
})
name = class_names[np.argmax(prob)]
probs.append(np.max(prob))
names.append(name)
print("names are {}".format(names))
print("probs are {}".format(probs))
sys.exit()
train_batches_per_epoch = np.floor(train_generator.data_size /
batch_size).astype(np.int16)
val_batches_per_epoch = np.floor(val_generator.data_size / batch_size).astype(
np.int16)
# Start Tensorflow session
with tf.Session() as sess:
# Initialize all variables
sess.run(tf.global_variables_initializer())
# Add the model graph to TensorBoard
writer.add_graph(sess.graph)
# Load the pretrained weights into the non-trainable layer
model.load_initial_weights(sess)
print("{} Start training...".format(datetime.now()))
print("{} Open Tensorboard at --logdir {}".format(datetime.now(),
filewriter_path))
# Loop over number of epochs
for epoch in range(num_epochs):
print("{} Epoch number: {}".format(datetime.now(), epoch + 1))
step = 1
while step < train_batches_per_epoch:
# Get a batch of images and labels
def train(args):
batch_size = 1
num_epochs = 100
num_classes = 9
learning_rate = 1e-4
test_data_path = base_path + 'test/05/testImages/{:04d}/'
test_file = base_path + 'test/05/testImages/{:04d}/test{:04d}_2020.txt'
train_data_path = base_path + 'test/05/dummy/'
validation_data_path = train_data_path
if (args.mode == 'train'):
num_training_samples = count_text_lines(train_file)
num_validation_samples = count_text_lines(validate_file)
train_batches_per_epoch = np.ceil(num_training_samples /
batch_size).astype(np.int32)
val_batches_per_epoch = np.ceil(num_training_samples /
batch_size).astype(np.int32)
trainDataGen = ImageDataGenerator(train_file,
train_data_path,
num_classes,
'training',
batch_size,
num_preprocess_threads=5,
shuffle=True,
min_queue_examples=1000)
validationDataGen = ImageDataGenerator(validate_file,
validation_data_path,
num_classes,
'validation',
batch_size,
num_preprocess_threads=2,
shuffle=False,
min_queue_examples=100)
train_imgBatch = trainDataGen.img_batch
train_labelBatch = trainDataGen.label_batch
val_imgBatch = validationDataGen.img_batch
val_labelBatch = validationDataGen.label_batch
# TF placeholder for graph input and output
x = tf.placeholder(tf.float32, [batch_size, 227, 227, 4])
y = tf.placeholder(tf.float32, [batch_size, num_classes])
train_layers = ['fc8']
var_list = [
v for v in tf.trainable_variables()
if v.name.split('/')[0] in train_layers
]
# Initialize model
model = AlexNet(x, num_classes, train_layers)
score = model.fc8
with tf.name_scope("cross_ent"):
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=score,
labels=y))
# Op for calculating the loss
# Train op
with tf.name_scope("train"):
# Create optimizer and apply gradient descent to the trainable variables
gOpt = tf.train.GradientDescentOptimizer(learning_rate)
grads = gOpt.compute_gradients(loss)
train_op = gOpt.apply_gradients(grads)
with tf.name_scope("train"):
# Get gradients of all trainable variables
for grad, var in grads:
if grad is not None:
tf.summary.histogram(var.op.name + '/gradients', grad)
# Add the loss to summary
tf.summary.scalar('cross_entropy', loss)
# Evaluation op: Accuracy of the model
with tf.name_scope("accuracy"):
correct_pred = tf.equal(tf.argmax(score, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
# Add the accuracy to the summary
tf.summary.scalar('accuracy', accuracy)
# Merge all summaries together
merged_summary = tf.summary.merge_all()
elif (args.mode == 'test'):
if (args.testImg == None):
sys.exit(
'Please Input a valid image no with option --testImg NUMBER ')
testImageIdx = args.testImg
test_data_path = test_data_path.format(testImageIdx)
test_file = test_file.format(testImageIdx, testImageIdx)
num_test_samples = count_text_lines(test_file)
batch_size = 1000
test_batches_per_epoch = np.ceil(num_test_samples / batch_size).astype(
np.int32)
x = tf.placeholder(tf.float32, [batch_size, 227, 227, 4])
# keep_prob = tf.placeholder(tf.float32)
testDataGen = ImageDataGenerator(test_file,
test_data_path,
num_classes,
'test',
batch_size,
num_preprocess_threads=5,
shuffle=False,
min_queue_examples=10000)
test_imgBatch = testDataGen.img_batch
test_labelBatch = testDataGen.label_batch
# Initialize model
model = AlexNet(x, num_classes, [])
#
# Link variable to model output
score = model.fc8
softmax = tf.nn.softmax(score)
# testImg = getTestImage();
outImg = np.zeros([480, 640, 3])
saver = tf.train.Saver()
writer = tf.summary.FileWriter(filewriter_path)
with tf.Session() as sess:
if (args.mode == 'train'):
# Required to get the filename matching to run.
tf.local_variables_initializer().run()
tf.global_variables_initializer().run()
model.load_initial_weights(sess)
# saver = tf.train.import_meta_graph('../checkpoints/model_epoch100.ckpt.meta')
# saver.restore(sess, "../checkpoints/model_epoch100.ckpt")
# Coordinate the loading of image files.
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
# Add the model graph to TensorBoard
writer.add_graph(sess.graph)
print("{} Start training...".format(datetime.now()))
print("{} Open Tensorboard at --logdir {}".format(
datetime.now(), filewriter_path))
for epoch in range(0, num_epochs):
print("{} Epoch number: {}".format(datetime.now(), epoch + 1))
for step in range(0, train_batches_per_epoch):
print("{} Batch number: {}".format(datetime.now(),
step + 1))
# Get an image tensor and print its value.
imageB, labelB = sess.run(
[train_imgBatch, train_labelBatch])
# #For Debugging
# np.save('imgDump.npy',imageB)
# np.save('labelDump.npy',labelB)
one_hot_labels = np.zeros((batch_size, num_classes))
for i in range(len(labelB)):
one_hot_labels[i][int(labelB[i])] = 1
#
# image_summary_t = tf.summary.image('t', tf.reshape(train_img, [1, 227, 227, 3]), max_outputs=1)
# _,_, image_summary = sess.run([train_img, train_label, image_summary_t])
# writer.add_summary(image_summary)
sess.run([train_op],
feed_dict={
x: imageB,
y: one_hot_labels
})
# Generate summary with the current batch of data and write to file
if step % display_step == 0:
s = sess.run(merged_summary,
feed_dict={
x: imageB,
y: one_hot_labels
})
writer.add_summary(
s, epoch * train_batches_per_epoch + step)
# writer.add_summary(image_summary)
# Validate the model on the entire validation set
print("{} Start validation".format(datetime.now()))
test_acc = 0.
test_count = 0
for _ in range(val_batches_per_epoch):
imageB, labelB = sess.run([val_imgBatch, val_labelBatch])
one_hot_labels = np.zeros((batch_size, num_classes))
for i in range(len(labelB)):
one_hot_labels[i][int(labelB[i])] = 1
acc = sess.run(accuracy,
feed_dict={
x: imageB,
y: one_hot_labels,
})
test_acc += acc
test_count += 1
test_acc /= test_count
print("{} Validation Accuracy = {:.4f}".format(
datetime.now(), test_acc))
print("{} Saving checkpoint of model...".format(
datetime.now()))
# save checkpoint of the model
checkpoint_name = os.path.join(
checkpoint_path, 'model_epoch' + str(epoch + 1) + '.ckpt')
save_path = saver.save(sess, checkpoint_name)
print("{} Model checkpoint saved at {}".format(
datetime.now(), checkpoint_name))
# # image_summary_t = tf.summary.image('v'+label_v, tf.reshape(val_img, [1, 227, 227, 3]), max_outputs=1)
# _,_,image_summary = sess.run([val_img, val_label, image_summary_t])
# writer.add_summary(image_summary)
# Finish off the filename queue coordinator.
coord.request_stop()
coord.join(threads)
elif (args.mode == 'test'):
print 'inside Evaluation'
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
saver = tf.train.import_meta_graph(
'../checkpoints/model_epoch31.ckpt.meta')
saver.restore(sess, "../checkpoints/model_epoch31.ckpt")
print 'Loaded Model from Checkpoint'
# for step in range(0, test_batches_per_epoch):
for step in range(0, test_batches_per_epoch):
print("{} Batch number: {}".format(datetime.now(), step + 1))
imageB, labelB = sess.run([test_imgBatch, test_labelBatch])
pixelLabels = labelB
# print pixelLabels
pred = sess.run(softmax, feed_dict={x: imageB})
print pred
pixelPred = np.argmax(pred, 1)
print pixelPred
ct = 0
for pixel in pixelLabels:
px = int(pixel[0:3])
py = int(pixel[3:6])
if ((pixelPred[ct] == 0)):
outImg[px, py, :] = 0
else:
outImg[px, py,
0] = (colors[pixelPred[ct] - 1, 2]) * 255
outImg[px, py,
1] = (colors[pixelPred[ct] - 1, 1]) * 255
outImg[px, py,
2] = (colors[pixelPred[ct] - 1, 0]) * 255
# if(pixelPred[ct]==0): # Background
# outImg[px, py, :] = 255
# elif(pixelPred[ct]==1): # Foreground
# outImg[px,py,2]=255
# outImg[px, py, 1] = 255
ct = ct + 1
# img = Image.fromarray(outImg)
# img.show()
cv2.imwrite('output_2.png%s' % (args.testImg), outImg)
coord.request_stop()
coord.join(threads)
