def gen_test_data(self, data_path): print("Generate fc7 from testing data...") data_path = self.data_dir + data_path aug_data_path = self.data_dir + '/train_augment' test_data = BaseDataSet(data_path, 500, 50) test_data.image_list = sorted(test_data.image_list) test_data_set = [] test_label_set = [] for image_path in test_data.image_list: data, label = test_data.read_image(image_path) if '0009.jpg' in image_path or '0010.jpg' in image_path: print 'test:', image_path test_data_set.append(data) test_label_set.append(int(label) - 1) train_data = BaseDataSet(aug_data_path, 500, 50) train_data.image_list = sorted(train_data.image_list) train_data_set = [] train_label_set = [] for image_path in train_data.image_list: print 'Train:', image_path data, label = train_data.read_image(image_path) train_data_set.append(data) train_label_set.append(int(label) - 1) train_fc7 = [] test_fc7 = [] with tf.Session(config=tf.ConfigProto( allow_soft_placement=True)) as sess: inputs = tf.placeholder(tf.float32, [1, 227, 227, 3], name="input_image") alexnet = AlexNet(inputs, keep_prob=1.0, num_classes=1000, skip_layer=[]) tf.global_variables_initializer().run() alexnet.load_initial_weights(sess) for i in range(len(train_data_set)): fc7 = sess.run( [alexnet.fc7], feed_dict={ alexnet.X: np.array(train_data_set[i]).reshape([1, 227, 227, 3]) }) train_fc7.append(fc7[0].reshape([4096])) for i in range(len(test_data_set)): fc7 = sess.run( [alexnet.fc7], feed_dict={ alexnet.X: np.array(test_data_set[i]).reshape([1, 227, 227, 3]) }) test_fc7.append(fc7[0].reshape([4096])) np.save(self.data_dir + '/train_fc7.npy', np.array(train_fc7)) np.save(self.data_dir + '/valid_fc7.npy', np.array(test_fc7)) np.save(self.data_dir + '/train_label.npy', np.array(train_label_set)) np.save(self.data_dir + '/valid_label.npy', np.array(test_label_set)) print('Finish generating!')
def get_encodings(self): train_layers = [] batch_size = self.alexnet_batch_size tf.reset_default_graph() X = tf.placeholder(tf.float32, [batch_size, 227, 227, 3]) model = AlexNet(X, 1.0, 1000, train_layers) encoding = model.fc7 # A 4096 x 1 encoding from AlexNet # Initializing the data-generator. img_utils = ImageDataGenerator(data_dir="tid2013/", inp_size=[227, 227], scale_size=[256, 256], horizontal_flip=True) img_encs = {} mos = {} for d in range(6): img_encs['level' + str(d)] = np.ndarray( [self.NUM_EXAMPLES, self.dim]) mos['level' + str(d)] = np.ndarray([ self.NUM_EXAMPLES, ]) with tf.Session() as sess: # Initializing the variables. tf.global_variables_initializer() # Initializing the weights. model.load_initial_weights(sess) for _ in range(self.NUM_EXAMPLES // batch_size + 1): # Sampling a batch imgs_batch, mos_batch = img_utils.get_next_batch(batch_size) first = img_utils.first last = img_utils.last for d in range(6): img_enc = sess.run( encoding, feed_dict={X: imgs_batch['level' + str(d)]}) img_encs['level' + str(d)][first:last] = img_enc mos['level' + str(d)][first:last] = mos_batch['level' + str(d)] sess.close() self.img_encs = img_encs self.mos = mos # Saving the data. data = {'encodings': img_encs, 'scores': mos} savemat('encoded_data.mat', data)
def train_model(image_path, learning_rate=0.001, num_epoch=100, batch_size=5): X_train = np.load(image_path + ".npy") Y_train = np.load(image_path + ".npy") X_train = X_train / 255 #Creating one-hot for label print(Y_train.shape) print(Y_train[0][1]) Y_train = convert_to_one_hot(Y_train, 10).T n_Y = Y_train.shape[1] x = tf.placeholder(tf.float32, [None, 227, 227, 3]) Y = tf.placeholder(tf.float32, [None, n_Y]) keep_prob = tf.placeholder(tf.float32) model = AlexNet(x, keep_prob, 10, []) score = model.fc8 cost = tf.reduce_mean( (tf.nn.softmax_cross_entropy_with_logits_v2(logits=score, labels=Y))) softmax = tf.nn.softmax(score) costs = [] optimizer = tf.train.AdamOptimizer( learning_rate=learning_rate).minimize(cost) init = tf.global_variables_initializer() [m, nH, nW, nCH] = X_train.shape seed = 0 #tf.reset_default_graph() with tf.Session() as sess: sess.run(init) sess.run(model.load_initial_weights(sess)) for i in range(num_epoch): minibatch_cost = 0 num_minibatch = int(m / batch_size) # number of batches seed = seed + 1 minibatches = random_mini_batches(X_train, Y_train, batch_size, seed) for minibatch in minibatches: X_batches, Y_batches = minibatch #Use cost and optimizer to run _, c = sess.run([optimizer, cost], { x: X_batches, Y: Y_batches, keep_prob: 1 }) minibatch_cost += c minibatch_cost = minibatch_cost / num_minibatch if i % 5 == 0: print("Cost===>" + str(c)) costs.append(c) plt.plot(np.squeeze(costs)) plt.ylabel('cost') plt.xlabel('epoch') plt.title('learning_rate' + str(learning_rate)) plt.show()
def run(img_path=img_file_path): #placeholder for input and dropout rate x = tf.placeholder(tf.float32, [1, 227, 227, 3]) keep_prob = tf.placeholder(tf.float32) #create model with default config ( == no skip_layer and 1000 units in the last layer) model = AlexNet(x, keep_prob, 1000, []) #define activation of last layer as score score = model.fc8 #create op to calculate softmax softmax = tf.nn.softmax(score) sess = tf.Session() # Load the pretrained weights into the model sess.run(model.load_initial_weights(sess)) image = cv2.imread(img_path) # Convert image to float32 and resize to (227x227) img = cv2.resize(image.astype(np.float32), (227,227)) # Subtract the ImageNet mean img -= imagenet_mean # Reshape as needed to feed into model img = img.reshape((1,227,227,3)) print(img.dtype) # Run the session and calculate the class probability probs = sess.run(softmax, feed_dict={x: img, keep_prob: 1}) # Get the class name of the class with the highest probability class_name = class_names[np.argmax(probs)] print("Class: " + class_name + ", probability: %.4f" %probs[0,np.argmax(probs)])
# Initialize an saver for store model checkpoints saver = tf.train.Saver() # Start Tensorflow session with tf.Session() as sess: # Add the model graph to TensorBoard writer.add_graph(sess.graph) # Load the Imagenet pretrained weights into the non-trainable layer # Finetune the whole network if newmodel: # Initialize all variables sess.run(tf.global_variables_initializer()) model.load_initial_weights(sess, trainablev=True, layersList=['fc8']) else: saver.restore(sess, pre_model) print("{} Start training...".format(datetime.now())) print("{} Open Tensorboard at --logdir {}".format(datetime.now(), filewriter_path)) # Loop over number of epochs for epoch in range(num_epochs): print("{} Epoch number: {}".format(datetime.now(), epoch + 1)) # Initialize iterator with the training dataset sess.run(training_init_op) for step in range(train_batches_per_epoch):
class Hand(object): """docstring for Hand""" def __init__(self): super(Hand, self).__init__() # Network params learning_rate = 0.03 num_epochs = 10 batch_size = 1 display_step = 4 dropout = 0.5 num_classes = 7 train_layers = ['fc8'] print(batch_size) self.x = tf.placeholder(tf.float32, [ 1,227, 227, 3]) y = tf.placeholder(tf.float32, [ batch_size,num_classes]) self.keep_prob = tf.placeholder(tf.float32) self.model = AlexNet(self.x, self.keep_prob, num_classes, train_layers) cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits = self.model.fc8, labels = y)) optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost,var_list = tf.global_variables()[-2:]) correct_pred = tf.equal(tf.argmax(self.model.fc8,1), tf.argmax(y,1)) accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) self.saver = tf.train.Saver(tf.global_variables()) self.sess = tf.InteractiveSession() self.model.load_initial_weights(self.sess) self.saver.restore(self.sess,"Model/model.ckpt") def predict(self,img): img_array = np.array(img) x_test = np.zeros((1,227,227,3)) x_test[0] = img_array # mean_px = x_test.mean().astype(np.float32) # std_px = x_test.std().astype(np.float32) # x_test =( x_test - mean_px)/std_px pre = self.model.fc8.eval(feed_dict={self.x:x_test[0:1],self.keep_prob:1.}) rank = 0; now = -1; result = 6; # print(np.shape(pre)) # print((pre)) for x in pre[0]: rank += 1 if now < x: now = x result = rank return result
class Model(object): def __init__(self, mode='train'): ''' Model class for you to modify for brick/ball/cylinder deep learning challenge. mode = 'train' or 'test' - could be a useful flag if we want to use dataset augmentations. ''' if mode == 'train' or mode == 'test': self.mode = mode else: print('Mode must be train or test.') # Learning params self.learning_rate = 1e-3 self.minibatch_size = 32 self.num_iterations = 1500 self.train_test_split = 0.7 # Network params self.keep_rate = 0.5 #Fraction of connections that we keep when performing dropout # How many pretrained alexnet layers to we want to load up? # This must be a number between 1 and 8. self.num_layers_to_load = 5 # How often we want to write the tf.summary data to disk and measure performance self.display_step = 5 self.save_step = 100 # Path for tf.summary.FileWriter and to store model checkpoints self.save_dir = "tf_data/sample_model" self.input_image_size = (227, 227) self.label_indices = {'brick': 0, 'ball': 1, 'cylinder': 2} self.labels_ordered = list(self.label_indices) self.num_classes = len(self.label_indices) self.channel_means = np.array([147.12697, 160.21092, 167.70029]) #Go ahead and build graph on initialization: self.build_graph() #Set to true with we create a tf session. self.initialized = False def build_graph(self): ''' This is where you can experiment with various architectures. ''' # TF placeholder for graph input and output self.X = tf.placeholder( tf.float32, [None, self.input_image_size[0], self.input_image_size[1], 3]) self.y = tf.placeholder(tf.float32, [None, self.num_classes]) self.keep_prob = tf.placeholder(tf.float32) # Build Alexnet portion of graph self.AN = AlexNet(self.X, self.keep_prob, self.num_layers_to_load) #### ------------ Setup Your Graph Here ------------------ #### # We're taking the front of our graph from AlexNet, now let's # build the rest - this is where we get to be creative! # Grab the tensor from alexnet that we're going to build from # Be sure to change this if you chnage num_layers_to_load alexnet_out = self.AN.pool5 #To see the dimension of the output we're getting from alexnet, we can print our tensor: print(alexnet_out) # Your graph should produce an estimate our our labels, yhat. # yhat should be of the same dimension as y. # self.yhat = ? # Make sure you inlude the names of all the variables you would like to train here. # These may include variables from this part of the graph or the alexnet portion. # You can give your variables whatever name you like by passing in a name into your layers # for example if you setup a layer like this: tf.layers.dense(.... , name = 'my_fc6') # be sure to add 'my_fc6' to the trainable_variable_names list here: self.trainable_variable_names = ['conv5'] #### ---------------- End Graph Setup --------------------- #### def train(self): ''' Train model. ''' # Start by building rest of graph that we needed for training. #### ------------ Setup Cost Function Here ------------------ #### # Your job here is to compute a cost to pass into our AdamOptimizer below. # cost = #### ------------ End Cost Function Setup ------------------ #### # We only want to train some of the variables in our overall graph: var_list = [v for v in tf.trainable_variables() if v.name.split('/')[0] \ in self.trainable_variable_names] # Train op with tf.name_scope("train"): # Get gradients of all trainable variables gradients = tf.gradients(cost, var_list) gradients = list(zip(gradients, var_list)) # Create optimizer and apply gradient descent to the trainable variables optimizer = tf.train.AdamOptimizer(self.learning_rate) train_op = optimizer.apply_gradients(grads_and_vars=gradients) # Add gradients to summary for gradient, var in gradients: tf.summary.histogram(var.name + '/gradient', gradient) # Add the variables we train to the summary for var in var_list: tf.summary.histogram(var.name, var) # Add the loss to summary tf.summary.scalar('cross_entropy', cost) # Evaluation op: Accuracy of the model with tf.name_scope("accuracy"): correct_pred = tf.equal(tf.argmax(self.yhat, 1), tf.argmax(self.y, 1)) accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) # Add the accuracy to the summary tf.summary.scalar('accuracy', accuracy) # Merge all summaries together merged_summary = tf.summary.merge_all() # Initialize the FileWriter train_writer = tf.summary.FileWriter( os.path.join(self.save_dir, 'train')) test_writer = tf.summary.FileWriter(os.path.join( self.save_dir, 'test')) # Initialize an saver for store model checkpoints # We'll save the pretrained alexnet weights and the new weight values that we train: var_list_to_save = [v for v in tf.trainable_variables() if v.name.split('/')[0] in \ self.AN.all_variable_names + self.trainable_variable_names] saver = tf.train.Saver(var_list=var_list_to_save) #Load up data from image files. data = data_loader(label_indices=self.label_indices, channel_means=self.channel_means, train_test_split=self.train_test_split, input_image_size=self.input_image_size, data_path='../data') #Setup minibatch generators G = Generator(data.train.X, data.train.y, minibatch_size=self.minibatch_size) GT = Generator(data.test.X, data.test.y, minibatch_size=self.minibatch_size) #Launch tf session self.sess = tf.Session() self.sess.run(tf.global_variables_initializer()) self.initialized = True # Add the model graph to TensorBoard train_writer.add_graph(self.sess.graph) # Load the pretrained weights into the alexnet portion of our graph self.AN.load_initial_weights(self.sess) print("{} Start training...".format(datetime.now())) print("{} Open Tensorboard at --logdir {}".format( datetime.now(), self.save_dir)) #And Train for i in range(self.num_iterations): G.generate() # And run the training op self.sess.run(train_op, feed_dict={ self.X: G.X, self.y: G.y, self.keep_prob: self.keep_rate }) # Generate summary with the current batch of data and write to file if i % self.display_step == 0: s = self.sess.run(merged_summary, feed_dict={ self.X: G.X, self.y: G.y, self.keep_prob: 1.0 }) train_writer.add_summary(s, i) GT.generate() s = self.sess.run(merged_summary, feed_dict={ self.X: GT.X, self.y: GT.y, self.keep_prob: 1.0 }) test_writer.add_summary(s, i) train_acc = self.sess.run(accuracy, feed_dict={ self.X: G.X, self.y: G.y, self.keep_prob: 1.0 }) test_acc = self.sess.run(accuracy, feed_dict={ self.X: GT.X, self.y: GT.y, self.keep_prob: 1.0 }) print(i, ' iterations,', str(G.num_epochs), 'epochs, train accuracy = ', train_acc, ', test accuracy = ', test_acc) if i % self.save_step == 0 and i > 0: print("{} Saving checkpoint of model...".format( datetime.now())) # save checkpoint of the model checkpoint_name = os.path.join( self.save_dir, 'model_epoch' + str(G.num_epochs) + '.ckpt') save_path = saver.save(self.sess, checkpoint_name) print("{} Model checkpoint saved at {}".format( datetime.now(), checkpoint_name)) def predict(self, X, checkpoint_dir=None): ''' X: Numpy array of dimension [n, 227, 227, 3]. Note that n here may not be the same as the minibatch_size defined above. Returns: yhat_numpy: A numpy array of dimension [n x 3] containing the predicted one hot labels for each image passed in X. ''' if not self.initialized: self.restore_from_checkpoint(checkpoint_dir) yhat_numpy = self.sess.run(self.yhat, feed_dict={ self.X: X, self.keep_prob: 1.0 }) return yhat_numpy def restore_from_checkpoint(self, checkpoint_dir=None): ''' Restore model from most recent checkpoint in save dir. ''' saver = tf.train.Saver() self.sess = tf.Session() #Load latest checkpont, use savedir if we're not given a checkpoint dir: if checkpoint_dir is None: checkpoint_name = tf.train.latest_checkpoint(self.save_dir) else: checkpoint_name = tf.train.latest_checkpoint(checkpoint_dir) print(checkpoint_name) #Resore session saver.restore(self.sess, checkpoint_name) self.initialized = True
def main(_): #convert jpg image(s) into iamge representations using alexnet: filenames = [ os.path.join(image_dir, f) for f in [ 'overly-attached-girlfriend.jpg', 'high-expectations-asian-father.jpg', 'foul-bachelor-frog.jpg', 'stoner-stanley.jpg', 'y-u-no.jpg', 'willy-wonka.jpg', 'futurama-fry.jpg', 'success-kid.jpg', 'one-does-not-simply.jpg', 'bad-luck-brian.jpg', 'first-world-problems.jpg', 'philosoraptor.jpg', 'what-if-i-told-you.jpg', 'TutorPP.jpg' ] ] print(filenames) tf.logging.info("Running caption generation on %d files matching %s", len(filenames), FLAGS.input_files) #mean of imagenet dataset in BGR imagenet_mean = np.array([104., 117., 124.], dtype=np.float32) #placeholder for input and dropout rate x_Alex = tf.placeholder(tf.float32, [1, 227, 227, 3]) keep_prob_Alex = tf.placeholder(tf.float32) #create model with default config ( == no skip_layer and 1000 units in the last layer) modelAlex = AlexNet(x_Alex, keep_prob_Alex, 1000, [], ['fc7', 'fc8'], 512) #maybe need to put fc8 in skip_layers #define activation of last layer as score score = modelAlex.fc6 meme_embeddings = [] with tf.Session() as sess: # Initialize all variables sess.run(tf.global_variables_initializer()) # Load the pretrained weights into the model modelAlex.load_initial_weights(sess) for i, meme in enumerate(filenames): img = Image.open(meme) try: img.thumbnail((227, 227), Image.ANTIALIAS) #img = img.resize((227,227)) #use img.thumbnail for square images, img.resize for non square assert np.shape(img) == (227, 227, 3) except AssertionError: img = img.resize((227, 227)) print('sizing error') # Subtract the ImageNet mean img = img - imagenet_mean #should probably change this # Reshape as needed to feed into model img = img.reshape((1, 227, 227, 3)) meme_vector = sess.run(score, feed_dict={ x_Alex: img, keep_prob_Alex: 1 }) #[1,4096] meme_vector = np.reshape(meme_vector, [4096]) assert np.shape(meme_vector) == (4096, ) #now have np embeddings to feed for inference meme_embeddings.append(meme_vector) with open('Captions.txt', 'r') as f: data_captions = f.readlines() data_captions = [s.lower() for s in data_captions] # Build the inference graph. g = tf.Graph() with g.as_default(): model = inference_wrapper.InferenceWrapper() restore_fn = model.build_graph_from_config(configuration.ModelConfig(), FLAGS.checkpoint_path) g.finalize() # Create the vocabulary. vocab = vocabulary.Vocabulary(FLAGS.vocab_file) #filenames = [] #for file_pattern in FLAGS.input_files.split(","): #filenames.extend(tf.gfile.Glob(file_pattern)) #tf.logging.info("Running caption generation on %d files matching %s", #len(filenames), FLAGS.input_files) with tf.Session(graph=g) as sess: # Load the model from checkpoint. restore_fn(sess) # Prepare the caption generator. Here we are implicitly using the default # beam search parameters. See caption_generator.py for a description of the # available beam search parameters. generator = caption_generator.CaptionGenerator(model, vocab) num_in_data_total = 0 num_captions = 0 for i, meme in enumerate(meme_embeddings): #with tf.gfile.GFile(filename, "rb") as f: #image = f.read() captions = generator.beam_search(sess, meme) print("Captions for image %s:" % os.path.basename(filenames[i])) num_in_data = 0 for i, caption in enumerate(captions): # Ignore begin and end words. sentence = [ vocab.id_to_word(w) for w in caption.sentence[1:-1] ] sentence = " ".join(sentence) in_data = 0 if b_any(sentence in capt for capt in data_captions): in_data = 1 num_in_data += 1 num_in_data_total += 1 num_captions += 1 else: num_captions += 1 print(" %d) %s (p=%f) [in data = %d]" % (i, sentence, math.exp(caption.logprob), in_data)) print("number of captions in data = %d" % (num_in_data)) print("(total number of captions in data = %d) percent in data = %f" % (num_in_data_total, (num_in_data_total / num_captions)))
class FeatureExtractor(object): def __init__(self, cfg): self.cfg = cfg self.dataset = {} self.imagenet_mean = None self.x_placeholder = None self.keep_prob = None self.model = None self.score = None self.softmax = None self.load_dataset_indexes() self.init_alexnet() pass @staticmethod def add_sample_to_dict(d, s_id, sample_path): sample_d = {'img': cv2.imread(sample_path), 'path': sample_path} if s_id in d.keys(): d[s_id].append(sample_d) else: d[s_id] = [sample_d] @staticmethod def add_fmap_to_dict(d, s_id, p1, p5, fc2): fmaps = {'p1': p1, 'p5': p5, 'fc2': fc2} if s_id in d.keys(): d[s_id].append(fmaps) else: d[s_id] = [fmaps] def load_dataset_indexes(self): dataset_path = self.cfg['dataset'] for root, dirs, files in os.walk(dataset_path): for name in files: if not name.endswith('.bmp'): continue s_id = int(re.sub('[^0-9]', '', root.split('/')[2])) #print "id:{} path:{}".format(s_id, root.split('/')[1]) full_img_path = os.path.join(root, name) FeatureExtractor.add_sample_to_dict(self.dataset, s_id, full_img_path) # clean missing numbers prev_num = 0 old_keys = sorted(self.dataset.keys()) for i in xrange(len(old_keys)): old_k = old_keys[i] self.dataset[prev_num] = self.dataset.pop(old_k) prev_num += 1 print "[DEBUG]", "Number of classes:", len( self.dataset.keys()), self.dataset.keys() def init_alexnet(self): """ Start alexnet network and load weights :return: """ self.imagenet_mean = np.array([104., 117., 124.], dtype=np.float32) self.x_placeholder = tf.placeholder(tf.float32, [1, 227, 227, 3]) self.keep_prob = tf.placeholder(tf.float32) # create model with default config ( == no skip_layer and 1000 units in the last layer) self.model = AlexNet(self.x_placeholder, self.keep_prob, 1000, []) # define activation of last layer as score self.score = self.model.fc8 # create op to calculate softmax self.softmax = tf.nn.softmax(self.score) pass def process_samples(self, samples): """ Process samples with the alexnet network and extract feature maps of C1 C5 and FC2 layers :param samples: :return: """ sample_r = {} with tf.Session() as sess: # Initialize all variables sess.run(tf.global_variables_initializer()) # Load the pretrained weights into the model self.model.load_initial_weights(sess) s_keys = samples.keys() for i in tqdm.trange(len(s_keys)): s_id = s_keys[i] v = samples[s_id] for e in v: # Convert image to float32 and resize to (227x227) img = cv2.resize(e['img'].astype(np.float32), (227, 227)) # Subtract the ImageNet mean img -= self.imagenet_mean # Reshape as needed to feed into model img = img.reshape((1, 227, 227, 3)) pool1_tensor = sess.graph.get_tensor_by_name('pool1:0') pool5_tensor = sess.graph.get_tensor_by_name('pool5:0') fc2_tensor = sess.graph.get_tensor_by_name('fc7/fc7:0') p1, p5, fc2 = sess.run( [pool1_tensor, pool5_tensor, fc2_tensor], feed_dict={ self.x_placeholder: img, self.keep_prob: 1 }) FeatureExtractor.add_fmap_to_dict(sample_r, s_id, np.dstack(p1).flatten(), np.dstack(p5).flatten(), np.dstack(fc2).flatten()) # np.set_printoptions(threshold='nan') # print("pool1 shape: {}".format(p1.shape)) # print("pool5 shape: {}".format(p5.shape)) # print("fc2 shape: {}".format(fc2.shape)) # Run the session and calculate the class probability # probs = sess.run(self.softmax, feed_dict={self.x_placeholder: img, self.keep_prob: 1}) # Get the class name of the class with the highest probability # class_name = class_names[np.argmax(probs)] # prob = probs[0, np.argmax(probs)] return sample_r @staticmethod def export_features_by_class(fmap, cutoff=0.3): if 0 < cutoff > 1: raise Exception("cutoff must be between 0 and 1") t_train = [] p1_train = [] p5_train = [] fc2_train = [] t_test = [] p1_test = [] p5_test = [] fc2_test = [] for k, v in fmap.items(): total_e = len(v) cut_p = int(math.ceil(total_e * (1 - cutoff))) for i in xrange(len(v)): e = v[i] if i < cut_p: t_train.append(k) p1_train.append(e['p1']) p5_train.append(e['p5']) fc2_train.append(e['fc2']) else: t_test.append(k) p1_test.append(e['p1']) p5_test.append(e['p5']) fc2_test.append(e['fc2']) labels = [str(e) for e in sorted(list(set(t_train)))] return [np.asarray(t_train), np.asarray(p1_train), np.asarray(p5_train), np.asarray(fc2_train)], \ [np.asarray(t_test), np.asarray(p1_test), np.asarray(p5_test), np.asarray(fc2_test)], \ labels def compare_features(self, f): """ Compare the feature map of several layers :param f: :return: """ from scipy.spatial.distance import pdist, squareform print '[INFO]', 'Compare features', f.shape dist_condensed = pdist(f) print '\tEuclidean mean:{} std:{}'.format(np.mean(dist_condensed), np.std(dist_condensed)) dist_condensed = pdist(f, 'cityblock') print '\tcityblock mean:{} std:{}'.format(np.mean(dist_condensed), np.std(dist_condensed)) dist_condensed = pdist(f, 'seuclidean', V=None) print '\tseuclidean mean:{} std:{}'.format(np.mean(dist_condensed), np.std(dist_condensed)) dist_condensed = pdist(f, 'cosine', V=None) print '\tcosine mean:{} std:{}'.format(np.mean(dist_condensed), np.std(dist_condensed)) dist_condensed = pdist(f, 'correlation', V=None) print '\tcorrelation mean:{} std:{}'.format(np.mean(dist_condensed), np.std(dist_condensed)) dist_condensed = pdist(f, 'hamming', V=None) print '\thamming mean:{} std:{}'.format(np.mean(dist_condensed), np.std(dist_condensed)) # dist_condensed = pdist(f, 'mahalanobis', V=None) # print 'mahalanobis mean:{} std:{}'.format(np.mean(dist_condensed), np.std(dist_condensed)) print 'End...' pass def start(self): """ Start clasification systems :return: """ print "[INFO]", "Processing data..." fmap = self.process_samples(self.dataset) train, test, labels = self.export_features_by_class(fmap, cutoff=0) target_train, p1_train, p5_train, fc2_train = train target_test, p1_test, p5_test, fc2_test = test print "[INFO]", "Data shapes:" print '\tTarget:{}/{}'.format(target_train.shape[0], target_test.shape[0]) print '\tC1:', p1_train.shape print '\tC5:', p5_train.shape print '\tFC2:', fc2_train.shape print '\tLabels:', labels prepend_date = datetime.datetime.now().strftime("%I-%M%p_%d-%b-%Y") print "[INFO]", "Plots will be saved in: {}".format( self.cfg["result_path"]) print "[INFO]", "Results will be saved in: {}".format( self.cfg["plot_path"]) # Compare features self.compare_features(p1_train) self.compare_features(p5_train) self.compare_features(fc2_train) svn_fn = ClassificatorHelper(self.cfg) res_p1_1 = svn_fn.svm_simple(p1_train, target_train, labels, debug_level=self.cfg["verbose_level"]) ClassificatorHelper.save_results(res_p1_1, self.cfg["result_path"], self.cfg["plot_path"], "p1_svm_simple_" + prepend_date) res_p5_1 = svn_fn.svm_simple(p5_train, target_train, labels, debug_level=self.cfg["verbose_level"]) ClassificatorHelper.save_results(res_p5_1, self.cfg["result_path"], self.cfg["plot_path"], "p5_svm_simple_" + prepend_date) res_fc2_1 = svn_fn.svm_simple(fc2_train, target_train, labels, debug_level=self.cfg["verbose_level"]) ClassificatorHelper.save_results(res_fc2_1, self.cfg["result_path"], self.cfg["plot_path"], "fc2_svm_simple_" + prepend_date) res_early = svn_fn.svm_early_fusion( target_train, p1_train, p5_train, fc2_train, labels, debug_level=self.cfg["verbose_level"]) for v in res_early: ClassificatorHelper.save_results( v, self.cfg["result_path"], self.cfg["plot_path"], "early_" + str(v["combination"]) + prepend_date) res_late = svn_fn.svm_late_fusion( target_train, p1_train, p5_train, fc2_train, labels, debug_level=self.cfg["verbose_level"]) ClassificatorHelper.save_results(res_late, self.cfg["result_path"], self.cfg["plot_path"], "late_" + prepend_date) res_fc2_all = svn_fn.test_diversity_fc2( target_train, fc2_train, labels, debug_level=self.cfg["verbose_level"]) for k, v in res_fc2_all.items(): ClassificatorHelper.save_results(v, self.cfg["result_path"], self.cfg["plot_path"], k + "_" + prepend_date) print "[INFO]", "Finished classifying" pass
class KNN(object): def __init__(self, sess, class_num, train_data_path="../data/training", test_data_path="../data/testing", kneighbors=10): self.sess = sess self.train_data_path = train_data_path self.test_data_path = test_data_path self.class_num = class_num self.training_data = [[], [], [], [], [], [], [], []] self.label_set = [] self.knn_list = [] self.kneighbors = kneighbors self.inputs = tf.placeholder(tf.float32, [1, 227, 227, 3], name="input_image") self.labels = tf.placeholder(tf.float32, [1, 50], name='label') self.alexnet = AlexNet(self.inputs, keep_prob=1.0, num_classes=1000, skip_layer=[]) tf.global_variables_initializer().run() self.alexnet.load_initial_weights(sess) for _i in range(7): nn = KNeighborsClassifier(n_neighbors=kneighbors) self.knn_list.append(nn) pass def load_training_data(self, img_num): ds = DataSet(self.train_data_path, 1, self.class_num) for i in range(img_num): img, label = ds.next_batch() out1, out2, out3, out4, out5, out6, out7 = self.sess.run( [ self.alexnet.norm1, self.alexnet.norm2, self.alexnet.conv3, self.alexnet.conv4, self.alexnet.pool5, self.alexnet.fc6, self.alexnet.fc7 ], feed_dict={ self.alexnet.X: img, self.labels: label }) self.training_data[0].append(out1[0].reshape(1, 27 * 27 * 96)[0]) self.training_data[1].append(out2[0].reshape(1, 13 * 13 * 256)[0]) self.training_data[2].append(out3[0].reshape(1, 13 * 13 * 384)[0]) self.training_data[3].append(out4[0].reshape(1, 13 * 13 * 384)[0]) self.training_data[4].append(out5[0].reshape(1, 6 * 6 * 256)[0]) self.training_data[5].append(out6[0]) self.training_data[6].append(out7[0]) self.label_set.append(np.argmax(label, axis=1)[0]) self.calibration_size = 400 self.calibration_data = [ self.training_data[j][img_num - self.calibration_size:img_num] for j in range(7) ] self.calibration_label = self.label_set[img_num - self.calibration_size:img_num] self.training_data = [ self.training_data[j][:img_num - self.calibration_size] for j in range(7) ] self.label_set = self.label_set[:img_num - self.calibration_size] self.build_model() self.A = [] for i in range(self.calibration_size): target_label = self.calibration_label[i] tar = [] for k in range(7): tp = [ self.label_set[int(j)] for j in self.knn_list[k].kneighbors([self.calibration_data[k][i]], n_neighbors=self.kneighbors, return_distance=False)[0] ] tar.extend(tp) total = np.sum(np.not_equal(tar, target_label)) self.A.append(total) def load_testing_data(self, img_num): kneighbor = self.kneighbors ds = DataSet(self.test_data_path, 1, self.class_num) source = [] tar = [] count = 0 for i in range(img_num): print(i) img, label = ds.next_batch() out1, out2, out3, out4, out5, out6, out7 = self.sess.run( [ self.alexnet.norm1, self.alexnet.norm2, self.alexnet.conv3, self.alexnet.conv4, self.alexnet.pool5, self.alexnet.fc6, self.alexnet.fc7 ], feed_dict={ self.alexnet.X: img, self.labels: label }) source.clear() source.append(out1[0].reshape(1, 27 * 27 * 96)) source.append(out2[0].reshape(1, 13 * 13 * 256)) source.append(out3[0].reshape(1, 13 * 13 * 384)) source.append(out4[0].reshape(1, 13 * 13 * 384)) source.append(out5[0].reshape(1, 6 * 6 * 256)) source.append(out6) source.append(out7) target_label = np.argmax(label, axis=1)[0] tar.clear() for k in range(7): tp = [ self.label_set[int(j)] for j in self.knn_list[k].kneighbors(source[k], n_neighbors=kneighbor, return_distance=False)[0] ] tar.extend(tp) max = 0 m_label = -1 for k_label in range(self.class_num): alpha = np.sum(np.not_equal(tar, k_label)) pj = np.sum(np.greater_equal(self.A, alpha)) if pj >= max: max = pj m_label = k_label if m_label == target_label: count += 1 if (i + 1) % 20 == 0: print("%d / %d, accuracy: %f" % ((i + 1), img_num, float(count / (i + 1)))) def build_model(self): for i in range(7): print("build ", i) try: self.knn_list[i].fit(self.training_data[i], self.label_set) self.training_data[i].clear() except: print("something wrong")
def main(): """ Configuration Part. """ # Path to the textfiles for the trainings and validation set train_file = './train.txt' val_file = './val.txt' # Learning params learning_rate = 0.01 num_epochs = 100 batch_size = 10 # Network params dropout_rate = 0.5 num_classes = 2 train_layers = ['fc8', 'fc7', 'fc6'] # How often we want to write the tf.summary data to disk display_step = 20 # Path for tf.summary.FileWriter and to store model checkpoints filewriter_path = "./tmp/tensorboard" checkpoint_path = "./tmp/checkpoints" """ Main Part of the finetuning Script. """ # Create parent path if it doesn't exist if not os.path.isdir(checkpoint_path): os.mkdir(checkpoint_path) # Place data loading and preprocessing on the cpu with tf.device('/cpu:0'): tr_data = ImageDataGenerator(train_file, mode='training', batch_size=batch_size, num_classes=num_classes, shuffle=True) val_data = ImageDataGenerator(val_file, mode='inference', batch_size=batch_size, num_classes=num_classes, shuffle=False) # create an reinitializable iterator given the dataset structure iterator = Iterator.from_structure(tr_data.data.output_types, tr_data.data.output_shapes) next_batch = iterator.get_next() # Ops for initializing the two different iterators training_init_op = iterator.make_initializer(tr_data.data) validation_init_op = iterator.make_initializer(val_data.data) # TF placeholder for graph input and output x = tf.placeholder(tf.float32, [None, 227, 227, 3]) y = tf.placeholder(tf.float32, [None, num_classes]) keep_prob = tf.placeholder(tf.float32) # Initialize model model = AlexNet(x, keep_prob, num_classes, train_layers) # Link variable to model output score = model.fc8 pred = tf.nn.softmax(score) # List of trainable variables of the layers we want to train var_list = [ v for v in tf.trainable_variables() if v.name.split('/')[0] in train_layers ] # Op for calculating the loss with tf.name_scope("cross_ent"): loss = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits=score, labels=y)) # Train op with tf.name_scope("train"): # Get gradients of all trainable variables gradients = tf.gradients(loss, var_list) gradients = list(zip(gradients, var_list)) # Create optimizer and apply gradient descent to the trainable variables # optimizer = tf.train.GradientDescentOptimizer(learning_rate) optimizer = tf.train.AdamOptimizer(learning_rate) train_op = optimizer.apply_gradients(grads_and_vars=gradients) # Add gradients to summary for gradient, var in gradients: tf.summary.histogram(var.name + '/gradient', gradient) # Add the variables we train to the summary for var in var_list: tf.summary.histogram(var.name, var) # Add the loss to summary tf.summary.scalar('cross_entropy', loss) # Evaluation op: Accuracy of the model with tf.name_scope("accuracy"): correct_pred = tf.equal(tf.argmax(score, 1), tf.argmax(y, 1)) accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) # Add the accuracy to the summary tf.summary.scalar('accuracy', accuracy) # Merge all summaries together merged_summary = tf.summary.merge_all() # Initialize the FileWriter writer = tf.summary.FileWriter(filewriter_path) # Initialize an saver for store model checkpoints saver = tf.train.Saver() # Get the number of training/validation steps per epoch train_batches_per_epoch = int(np.floor(tr_data.data_size / batch_size)) val_batches_per_epoch = int(np.floor(val_data.data_size / batch_size)) # Start Tensorflow session with tf.Session() as sess: print("# isTrain : ", args.istrain) if not args.istrain: ckpt = tf.train.get_checkpoint_state(checkpoint_path) if ckpt and tf.train.checkpoint_exists(ckpt.model_checkpoint_path): print("Reading model parameters from %s" % ckpt.model_checkpoint_path) saver.restore(sess, ckpt.model_checkpoint_path) else: print("checkpoint path isn't correct") return file = tf.read_file(args.input_path) decoded_img = tf.image.decode_jpeg(file, channels=3) resized_img = tf.image.resize_images(decoded_img, [227, 227]) resized_img = tf.reshape(resized_img, [1, 227, 227, 3]) print("# decoded img : ", decoded_img.eval().shape) pred_ = sess.run(pred, feed_dict={ x: resized_img.eval(), y: [[1, 0]], keep_prob: np.array(1.0) }) print("P(man|data) : ", pred_[0][0]) print("P(woman|data) : ", pred_[0][1]) img = decoded_img.eval() plt.imshow(img) plt.show() if args.visualize: w1, w2, w3, w4, w5 = sess.run(model.weight, feed_dict={ x: resized_img.eval(), y: [[1, 0]], keep_prob: np.array(1.0) }) print("W1 : ", w1.shape) visualize(w1[:, :, 0, :25]) print("W2 : ", w2.shape) visualize(w2[:, :, 0, :25]) print("W3 : ", w3.shape) visualize(w3[:, :, 0, :25]) print("W4 : ", w4.shape) visualize(w4[:, :, 0, :25]) print("W5 : ", w5.shape) visualize(w5[:, :, 0, :25]) f1, f2, f3, f4, f5 = sess.run(model.fm, { x: resized_img.eval(), y: [[1, 0]], keep_prob: np.array(1.0) }) print("F1 : ", f1.shape) visualize(f1[0][:, :, :25]) print("F2 : ", f2.shape) visualize(f2[0][:, :, :25]) print("F3 : ", f3.shape) visualize(f3[0][:, :, :25]) print("F4 : ", f4.shape) visualize(f4[0][:, :, :25]) print("F5 : ", f5.shape) visualize(f5[0][:, :, :25]) return else: # Initialize all variables sess.run(tf.global_variables_initializer()) # Add the model graph to TensorBoard writer.add_graph(sess.graph) # Load the pretrained weights into the non-trainable layer model.load_initial_weights(sess) print("{} Start training...".format(datetime.now())) print("{} Open Tensorboard at --logdir {}".format( datetime.now(), filewriter_path)) # Loop over number of epochs for epoch in range(num_epochs): print("{} Epoch number: {}".format(datetime.now(), epoch + 1)) # Initialize iterator with the training dataset sess.run(training_init_op) train_acc = 0. train_count = 0. for step in range(train_batches_per_epoch): # get next batch of data img_batch, label_batch = sess.run(next_batch) # And run the training op acc, _ = sess.run([accuracy, train_op], feed_dict={ x: img_batch, y: label_batch, keep_prob: dropout_rate }) train_acc += acc train_count += 1 # Generate summary with the current batch of data and write to file # if step % display_step == 0: # s = sess.run(merged_summary, feed_dict={x: img_batch, # y: label_batch, # keep_prob: 1.}) # # writer.add_summary(s, epoch*train_batches_per_epoch + step) train_acc /= train_count print("{} Train Accuracy = {:.4f}".format(datetime.now(), train_acc)) # Validate the model on the entire validation set print("{} Start validation".format(datetime.now())) sess.run(validation_init_op) test_acc = 0. test_count = 0 for ind in range(val_batches_per_epoch): img_batch, label_batch = sess.run(next_batch) acc = sess.run(accuracy, feed_dict={ x: img_batch, y: label_batch, keep_prob: 1. }) test_acc += acc test_count += 1 if epoch is 2 and ind is 0: fm = sess.run(model.fm, feed_dict={ x: img_batch, y: label_batch, keep_prob: 1. }) weight = sess.run(model.weight, feed_dict={ x: img_batch, y: label_batch, keep_prob: 1. }) # print("fm0 : ", np.array(fm[0]).shape) # print("fm1 : ", np.array(fm[1]).shape) # print("fm2 : ", np.array(fm[2]).shape) # print("fm3 : ", np.array(fm[3]).shape) # print("fm4 : ", np.array(fm[4]).shape) # # print("weight0 : ", np.array(weight[0]).shape) # print("weight1 : ", np.array(weight[1]).shape) # print("weight2 : ", np.array(weight[2]).shape) # print("weight3 : ", np.array(weight[3]).shape) # print("weight4 : ", np.array(weight[4]).shape) test_acc /= test_count print("{} Validation Accuracy = {:.4f}".format( datetime.now(), test_acc)) print("{} Saving checkpoint of model...".format(datetime.now())) # save checkpoint of the model checkpoint_name = os.path.join( checkpoint_path, 'model_epoch' + str(epoch + 1) + '.ckpt') save_path = saver.save(sess, checkpoint_name) print("{} Model checkpoint saved at {}".format( datetime.now(), checkpoint_name))
train_batches_per_epoch = int(np.floor(tr_data.data_size / batch_size)) #获取多少个epoch val_batches_per_epoch = int(np.floor(val_data.data_size / batch_size)) #测试集epoch个数 # Start Tensorflow session with tf.Session() as sess: # Initialize all variables sess.run(tf.global_variables_initializer()) #初始化所有变量 # Add the model graph to TensorBoard writer.add_graph(sess.graph) # Load the pretrained weights into the non-trainable layer model.load_initial_weights(sess) #加载权重 print("{} Start training...".format(datetime.now())) print("{} Open Tensorboard at --logdir {}".format(datetime.now(), filewriter_path)) # Loop over number of epochs for epoch in range(num_epochs): #多少个epoch,迭代多少次 print("{} Epoch number: {}".format(datetime.now(), epoch + 1)) # Initialize iterator with the training dataset sess.run(training_init_op) for step in range(train_batches_per_epoch):
def main(_): if not FLAGS.train_paths: raise ValueError("Must set --train_paths") if not FLAGS.val_paths: raise ValueError("Must set --val_paths") if not FLAGS.test_paths: raise ValueError("Must set --test_paths") train_paths = FLAGS.train_paths val_paths = FLAGS.val_paths test_paths = FLAGS.test_paths batch_size = FLAGS.batch_size num_classes = FLAGS.num_classes num_epochs = FLAGS.num_epochs emb_dim = FLAGS.embedding_dim keep_prob = FLAGS.keep_prob exp = FLAGS.exp loss_func = FLAGS.loss_func learning_rate = FLAGS.learning_rate checkpoint_path = FLAGS.checkpoint_path filewriter_path = FLAGS.filewriter_path display_step = FLAGS.display_step max_threads = FLAGS.max_threads # Create parent path if it doesn't exist ''' if not os.path.isdir(checkpoint_path): os.mkdir(checkpoint_path) if not os.path.isdir(filewriter_path): os.mkdir(filewriter_path) ''' # TF placeholder for graph input and output x = tf.placeholder(tf.float32, [batch_size, 6*6*256]) y = tf.placeholder(tf.float32, [batch_size, num_classes]) kp = tf.placeholder(tf.float32) # Initialize model layer_names = ['fc6', 'fc7', 'fc8'] train_layers = layer_names[-FLAGS.num_train_layers:] model = AlexNet(x, kp, num_classes, emb_dim, train_layers) # Link variable to model output score = model.fc8 if exp != 1.0: sign = tf.sign(score) score = tf.multiply(sign, tf.pow(tf.abs(score), exp)) # List of trainable variables of the layers we want to train var_list = [v for v in tf.trainable_variables() if v.name.split('/')[0] in train_layers] # Op for calculating the loss with tf.name_scope("loss_func"): if loss_func == 'softmax': loss = tf.nn.softmax_cross_entropy_with_logits(logits=score, labels=y, name='softmax_loss') elif loss_func == 'logistic': loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=y, logits=score, name='logistic_loss') elif loss_func == 'mse': loss = tf.losses.mean_squared_error(labels=y, predictions=score) elif loss_func == 'l2hinge': loss = tf.losses.hinge_loss(labels=y, logits=score, reduction=tf.losses.Reduction.NONE) loss = tf.square(loss) loss = tf.reduce_mean(loss) # Train op with tf.name_scope("train"): # Get gradients of all trainable variables gradients = tf.gradients(loss, var_list) gradients = list(zip(gradients, var_list)) # Create optimizer and apply gradient descent to the trainable variables optimizer = tf.train.AdamOptimizer(learning_rate) train_op = optimizer.apply_gradients(grads_and_vars=gradients) # Add gradients to summary for gradient, var in gradients: tf.summary.histogram(var.name + '/gradient', gradient) # Add the variables we train to the summary for var in var_list: tf.summary.histogram(var.name, var) # Add the loss to summary tf.summary.scalar('train_loss', loss) # Initialize the datasets start_time = time.time() tr_data = Dataset(train_paths, batch_size, num_classes, True) val_data = Dataset(val_paths, batch_size, num_classes, False) te_data = Dataset(test_paths, batch_size, num_classes, False) load_time = time.time() - start_time log_buff = 'Data loading time: %.2f' % load_time + '\n' # Ops for evaluation acc_split_weights_all = tr_data.get_acc_split_weights(caffe_class_ids, 0) acc_split_weights_all = tf.convert_to_tensor(acc_split_weights_all, tf.float32) acc_split_weights_1000 = tr_data.get_acc_split_weights(caffe_class_ids, 1000) acc_split_weights_1000 = tf.convert_to_tensor(acc_split_weights_1000, tf.float32) with tf.name_scope('accuracy'): # ops for top 1 accuracies and their splitting top1_score, _ = tf.nn.top_k(score, 1) top1_thresolds = tf.reduce_min(top1_score, axis = 1, keepdims = True) top1_thresolds_bc = tf.broadcast_to(top1_thresolds, score.shape) top1_y = tf.to_float(tf.math.greater_equal(score, top1_thresolds_bc)) top1_correct_pred = tf.multiply(y, top1_y) top1_precision_all = tf.squeeze(tf.reduce_mean(tf.matmul(acc_split_weights_all, \ tf.transpose(top1_correct_pred)), axis = 1)) top1_precision_1000 = tf.squeeze(tf.reduce_mean(tf.matmul(acc_split_weights_1000, \ tf.transpose(top1_correct_pred)), axis = 1)) # ops for top 5 accuracies and their splitting top5_score, _ = tf.nn.top_k(score, 5) top5_thresolds = tf.reduce_min(top5_score, axis = 1, keepdims = True) top5_thresolds_bc = tf.broadcast_to(top5_thresolds, score.shape) top5_y = tf.to_float(tf.math.greater_equal(score, top5_thresolds_bc)) top5_correct_pred = tf.multiply(y, top5_y) top5_precision_all = tf.squeeze(tf.reduce_mean(tf.matmul(acc_split_weights_all, \ tf.transpose(top5_correct_pred)), axis = 1))/5.0 top5_precision_1000 = tf.squeeze(tf.reduce_mean(tf.matmul(acc_split_weights_1000, \ tf.transpose(top5_correct_pred)), axis = 1))/5.0 # Merge all summaries together merged_summary = tf.summary.merge_all() # Initialize the FileWriter #writer = tf.summary.FileWriter(filewriter_path) # Initialize an saver for store model checkpoints saver = tf.train.Saver() # Start Tensorflow session sess_conf = tf.ConfigProto(intra_op_parallelism_threads=max_threads, inter_op_parallelism_threads=max_threads) with tf.Session(config=sess_conf) as sess: # Initialize all variables sess.run(tf.global_variables_initializer()) # Add the model graph to TensorBoard #writer.add_graph(sess.graph) # Load the pretrained weights into the non-trainable layer model.load_initial_weights(sess) log_buff += "{} Start training...".format(datetime.now())+'\n' #print("{} Open Tensorboard at --logdir {}".format(datetime.now(), # filewriter_path)) print(log_buff) log_buff = '' # Loop over number of epochs prev_top5_acc = 0 counter = 0 for epoch in range(num_epochs): log_buff += "{} Epoch: {}".format(datetime.now(), epoch)+'\n' tr_batches_per_epoch = tr_data.data_size // batch_size tr_data.reset() start_time = time.time() tr_data.shuffle() shuffle_time = time.time() - start_time log_buff += 'Train data shuffling time: %.2f' % shuffle_time + '\n' cost = 0.0 load_time = 0 train_time = 0 for step in range(tr_batches_per_epoch): # get next batch of data start_time = time.time() img_batch, label_batch = tr_data.next_batch() load_time += time.time() - start_time # And run the training op start_time = time.time() _, lss = sess.run((train_op, loss), feed_dict={x: img_batch, y: label_batch, kp: keep_prob}) cost += lss train_time += time.time() - start_time # Generate summary with the current batch of data and write to file ''' if step % display_step == 0: s = sess.run(merged_summary, feed_dict={x: img_batch, y: label_batch, kp: 1.0}) writer.add_summary(s, epoch*tr_batches_per_epoch + step) ''' elapsed_time = load_time + train_time cost /= tr_batches_per_epoch log_buff += 'Epoch: %d\tCost: %.6f\tElapsed Time: %.2f (%.2f / %.2f)' % \ (epoch+1, cost, elapsed_time, load_time, train_time) + '\n' # Test the model on the sampled train set tr_top1_all = ResultStruct() tr_top1_1000 = ResultStruct() tr_top5_all = ResultStruct() tr_top5_1000 = ResultStruct() # Evaluate on a for a smaller number of batches of trainset tr_data.reset() start_time = time.time() num_batches = int(tr_batches_per_epoch/4); for _ in range(num_batches): img_batch, label_batch = tr_data.next_batch() prec1_all, prec1_1000, prec5_all, prec5_1000 = \ sess.run((top1_precision_all, top1_precision_1000, \ top5_precision_all, top5_precision_1000), \ feed_dict={x: img_batch, \ y: label_batch, \ kp: 1.0}); tr_top1_all.add(prec1_all) tr_top1_1000.add(prec1_1000) tr_top5_all.add(prec5_all) tr_top5_1000.add(prec5_1000) tr_top1_all.scaler_div(num_batches) tr_top1_1000.scaler_div(num_batches) tr_top5_all.scaler_div(num_batches) tr_top5_1000.scaler_div(num_batches) log_buff += 'Epoch: ' + str(epoch+1) + '\tTrain Top 1 All Acc: ' + str(tr_top1_all) + '\n' log_buff += 'Epoch: ' + str(epoch+1) + '\tTrain Top 1 1000 Acc: ' + str(tr_top1_1000) + '\n' log_buff += 'Epoch: ' + str(epoch+1) + '\tTrain Top 5 All Acc: ' + str(tr_top5_all) + '\n' log_buff += 'Epoch: ' + str(epoch+1) + '\tTrain Top 5 1000 Acc: ' + str(tr_top5_1000) + '\n' tr_pred_time = time.time() - start_time # Test the model on the entire validation set val_top1_all = ResultStruct() val_top1_1000 = ResultStruct() val_top5_all = ResultStruct() val_top5_1000 = ResultStruct() val_data.reset() val_batches_per_epoch = val_data.data_size // batch_size start_time = time.time() for _ in range(val_batches_per_epoch): img_batch, label_batch = val_data.next_batch() prec1_all, prec1_1000, prec5_all, prec5_1000 = \ sess.run((top1_precision_all, top1_precision_1000, \ top5_precision_all, top5_precision_1000), \ feed_dict={x: img_batch, \ y: label_batch, \ kp: 1.0}) val_top1_all.add(prec1_all) val_top1_1000.add(prec1_1000) val_top5_all.add(prec5_all) val_top5_1000.add(prec5_1000) val_top1_all.scaler_div(val_batches_per_epoch) val_top1_1000.scaler_div(val_batches_per_epoch) val_top5_all.scaler_div(val_batches_per_epoch) val_top5_1000.scaler_div(val_batches_per_epoch) log_buff += 'Epoch: ' + str(epoch+1) + '\tVal Top 1 All Acc: ' + str(val_top1_all) + '\n' log_buff += 'Epoch: ' + str(epoch+1) + '\tVal Top 1 1000 ACC: ' + str(val_top1_1000) + '\n' log_buff += 'Epoch: ' + str(epoch+1) + '\tVal Top 5 All Acc: ' + str(val_top5_all) + '\n' log_buff += 'Epoch: ' + str(epoch+1) + '\tVal Top 5 1000 Acc: ' + str(val_top5_1000) + '\n' val_pred_time = time.time() - start_time # Test the model on the entire test set te_top1_all = ResultStruct() te_top1_1000 = ResultStruct() te_top5_all = ResultStruct() te_top5_1000 = ResultStruct() te_data.reset() te_batches_per_epoch = te_data.data_size // batch_size start_time = time.time() for _ in range(te_batches_per_epoch): img_batch, label_batch = te_data.next_batch() prec1_all, prec1_1000, prec5_all, prec5_1000 = \ sess.run((top1_precision_all, top1_precision_1000, \ top5_precision_all, top5_precision_1000), \ feed_dict={x: img_batch, \ y: label_batch, \ kp: 1.0}); te_top1_all.add(prec1_all) te_top1_1000.add(prec1_1000) te_top5_all.add(prec5_all) te_top5_1000.add(prec5_1000) te_top1_all.scaler_div(te_batches_per_epoch) te_top1_1000.scaler_div(te_batches_per_epoch) te_top5_all.scaler_div(te_batches_per_epoch) te_top5_1000.scaler_div(te_batches_per_epoch) log_buff += 'Epoch: ' + str(epoch+1) + '\tTest Top 1 All Acc: ' + str(te_top1_all) + '\n' log_buff += 'Epoch: ' + str(epoch+1) + '\tTest Top 1 1000 Acc: ' + str(te_top1_1000) + '\n' log_buff += 'Epoch: ' + str(epoch+1) + '\tTest Top 5 All Acc: ' + str(te_top5_all) + '\n' log_buff += 'Epoch: ' + str(epoch+1) + '\tTest Top 5 1000 Acc: ' + str(te_top5_1000) + '\n' te_pred_time = time.time() - start_time elapsed_time = tr_pred_time + val_pred_time + te_pred_time log_buff += 'Epoch %d Prediction: \tElapsed Time: %.2f (%.2f / %.2f / %.2f)' \ % (epoch+1, elapsed_time, tr_pred_time, val_pred_time, te_pred_time) + '\n' if math.isnan(cost): break cur_top5_acc = val_top5_all.acc if cur_top5_acc - prev_top5_acc > 0.003: counter = 0 prev_top5_acc = cur_top5_acc elif (cur_top5_acc - prev_top5_acc < -0.05) or (counter == 15): break else: counter += 1 # save checkpoint of the model ''' print("{} Saving checkpoint of model...".format(datetime.now())) checkpoint_name = os.path.join(checkpoint_path, 'model_epoch'+str(epoch+1)+'.ckpt') save_path = saver.save(sess, checkpoint_name) print("{} Model checkpoint saved at {}".format(datetime.now(), checkpoint_name)) ''' print(log_buff) log_buff = '' print(log_buff)
def readAndpredictloop(): ## reset default graph tf.reset_default_graph() #placeholder for input and dropout rate x = tf.placeholder(tf.float32, [1, 227, 227, 3]) keep_prob = tf.placeholder(tf.float32) #create model with default config ( == no skip_layer and 1000 units in the last layer) model = AlexNet(x, keep_prob, 1000, []) #define activation of last layer as score score = model.fc8 #create op to calculate softmax softmax = tf.nn.softmax(score) with tf.Session() as sess: # Create the excel sheet workbook path_to_excel = PATH_TO_EXCEL rb = xlrd.open_workbook(path_to_excel, formatting_info=True) wb = copy(rb) sheet = wb.get_sheet(0) style = XFStyle() style.num_format_str = 'general' # Get the ground truth list sheet_r = rb.sheets()[0] gt_label_list = sheet_r.col_values(1) # Initialize all variables sess.run(tf.global_variables_initializer()) # Load the pretrained weights into the model model.load_initial_weights(sess) # Create figure handle #fig2 = plt.figure(figsize=(15,6)) # Loop over all images # for i, name in enumerate(all_list[0:35000]): for imgID in range(st, end): original_img_ID = imgID imgID = str(imgID).zfill(8) shard_num = math.floor((original_img_ID - 1) / 10000) folder_num = math.floor((original_img_ID - 1) / 1000) + 1 #'G:\\validation_generated_QF\\shard-%d\\%d\\ILSVRC2012_val_%d-QF-%d.JPEG' start = time.time() qp_list = [i for i in range(0, 52, 2)] qp_list.append(51) for qf in qp_list: col_idx = int(2 * qf / 5) + 4 # print(col_idx) row = original_img_ID actual_idx = original_img_ID style = XFStyle() style.num_format_str = 'general' # if (sheet_r.cell(row,col_idx+1).value==u''): name = ((DATA_PATH) % (shard_num, folder_num, imgID, qf)) image = cv2.imread(name) # Convert image to float32 and resize to (227x227) try: img = cv2.resize(image.astype(np.float32), (227, 227)) # Subtract the ImageNet mean img -= imagenet_mean # Reshape as needed to feed into model img = img.reshape((1, 227, 227, 3)) # Run the session and calculate the class probability probs = sess.run(softmax, feed_dict={x: img, keep_prob: 1}) rank_estimate(probs, gt_label_list, actual_idx, sheet, col_idx, wb) except: writeFileContents(name, file_xls) continue print('image %s is done , time: %f' % (imgID, time.time() - start))
def main(): parser = argparse.ArgumentParser() parser.add_argument( "--positive_data", help= "A number from 0 to 3 that defines which will be the positive class") parser.add_argument("--fold", help="path to the dataset") parser.add_argument("--version", help="version of the classifier") parser.add_argument("--mode", help="Train or Test") parser.add_argument("--batch_size", default=40, help="number of images in each iteration times 4") parser.add_argument('--model_path', default='vgg_16.ckpt', type=str) parser.add_argument('--num_workers', default=4, type=int) parser.add_argument('--weight_decay', default=5e-4, type=float) parser.add_argument('--dropout_keep_prob', default=0.5, type=float) parser.add_argument('--num_epochs', default=10, type=int) args = parser.parse_args() positive_class = int(args.positive_data) fold = args.fold mode = args.mode version = args.version batch_size = int(args.batch_size) model_path = args.model_path num_workers = args.num_workers weight_decay = args.weight_decay dropout_keep_prob = args.dropout_keep_prob num_epochs = args.num_epochs # Read and processs the training images print("Loading training set") ''' data_path_pos_train, data_path_neg_train, data_path_pos_val, data_path_neg_val = DatasetSelection(fold, positive_class) train_filenames = data_path_pos_train + data_path_neg_train train_labels = [0]*len(data_path_pos_train) + [1]*len(data_path_neg_train) val_filenames = data_path_pos_val + data_path_neg_val val_labels = [0]*len(data_path_pos_val) + [1]*len(data_path_neg_val) num_classes = len(set(train_labels)) ''' train_filenames, val_filenames, train_labels, val_labels = DatasetSelectionMult( fold) num_classes = len(set(train_labels)) # -------------------------------------------------------------------------- # In TensorFlow, you first want to define the computation graph with all the # necessary operations: loss, training op, accuracy... # Any tensor created in the `graph.as_default()` scope will be part of `graph` #graph = tf.Graph() #with graph.as_default(): # Standard preprocessing for VGG on ImageNet taken from here: # https://github.com/tensorflow/models/blob/master/research/slim/preprocessing/vgg_preprocessing.py # Also see the VGG paper for more details: https://arxiv.org/pdf/1409.1556.pdf # ---------------------------------------------------------------------- # DATASET CREATION using tf.contrib.data.Dataset # https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/data # The tf.contrib.data.Dataset framework uses queues in the background to feed in # data to the model. # We initialize the dataset with a list of filenames and labels, and then apply # the preprocessing functions described above. # Behind the scenes, queues will load the filenames, preprocess them with multiple # threads and apply the preprocessing in parallel, and then batch the data # Training dataset train_dataset = tf.data.Dataset.from_tensor_slices( (train_filenames, train_labels)) train_dataset = train_dataset.map(_parse_function, num_parallel_calls=args.num_workers) train_dataset = train_dataset.map(training_preprocess, num_parallel_calls=args.num_workers) train_dataset = train_dataset.shuffle( buffer_size=10000) # don't forget to shuffle batched_train_dataset = train_dataset.batch(int(args.batch_size)) # Validation dataset val_dataset = tf.data.Dataset.from_tensor_slices( (val_filenames, val_labels)) val_dataset = val_dataset.map(_parse_function, num_parallel_calls=args.num_workers) val_dataset = val_dataset.map(val_preprocess, num_parallel_calls=args.num_workers) batched_val_dataset = val_dataset.batch(int(args.batch_size)) # Now we define an iterator that can operator on either dataset. # The iterator can be reinitialized by calling: # - sess.run(train_init_op) for 1 epoch on the training set # - sess.run(val_init_op) for 1 epoch on the valiation set # Once this is done, we don't need to feed any value for images and labels # as they are automatically pulled out from the iterator queues. # A reinitializable iterator is defined by its structure. We could use the # `output_types` and `output_shapes` properties of either `train_dataset` # or `validation_dataset` here, because they are compatible. iterator = tf.data.Iterator.from_structure( batched_train_dataset.output_types, batched_train_dataset.output_shapes) images, labels = iterator.get_next() train_init_op = iterator.make_initializer(batched_train_dataset) val_init_op = iterator.make_initializer(batched_val_dataset) # Indicates whether we are in training or in test mode #is_training = tf.placeholder(tf.bool) # --------------------------------------------------------------------- # Now that we have set up the data, it's time to set up the model. # For this example, we'll use VGG-16 pretrained on ImageNet. We will remove the # last fully connected layer (fc8) and replace it with our own, with an # output size num_classes=4 # We will first train the last layer for a few epochs. # Then we will train the entire model on our dataset for a few epochs. # Get the pretrained model, specifying the num_classes argument to create a new # fully connected replacing the last one, called "vgg_16/fc8" # Each model has a different architecture, so "vgg_16/fc8" will change in another model. # Here, logits gives us directly the predicted scores we wanted from the images. # We pass a scope to initialize "vgg_16/fc8" weights with he_initializer train_layers = ['fc8'] #num_classes = 2 keep_prob = 1.0 model = AlexNet(images, keep_prob, num_classes, train_layers) logits = model.fc8 print(logits) exit() # Specify where the model checkpoint is (pretrained weights). #model_path = args.model_path #assert(os.path.isfile(model_path)) # --------------------------------------------------------------------- # Using tf.losses, any loss is added to the tf.GraphKeys.LOSSES collection # We can then call the total loss easily loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits) #loss = tf.losses.get_total_loss() # Then we want to finetune the entire model for a few epochs. # We run minimize the loss only with respect to all the variables. starter_learning_rate = 0.001 global_step = tf.Variable(0, trainable=False, name='gs') gs = tf.contrib.framework.get_variables('gs') gs_init = tf.variables_initializer(gs) sess.run(gs_init) learning_rate = tf.train.exponential_decay(starter_learning_rate, global_step, 500, 0.9) #full_optimizer = tf.train.GradientDescentOptimizer(learning_rate) full_optimizer = tf.train.GradientDescentOptimizer(learning_rate) full_train_op = full_optimizer.minimize(loss, global_step=global_step) # Evaluation metrics prediction = tf.to_int32(tf.argmax(logits, 1)) correct_prediction = tf.equal(prediction, labels) conf_matrix = tf.confusion_matrix(labels, prediction, num_classes=num_classes) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) #tf.get_default_graph().finalize() saver = tf.train.Saver() # -------------------------------------------------------------------------- # Now that we have built the graph and finalized it, we define the session. # The session is the interface to *run* the computational graph. # We can call our training operations with `sess.run(train_op)` for instance sess.run(tf.global_variables_initializer()) model.load_initial_weights(sess) best_acc = 0.0 plot_acc_val = [] plot_acc_train = [] # Train the entire model for a few more epochs, continuing with the *same* weights. for epoch in range(num_epochs): print('Starting epoch %d / %d' % (epoch + 1, num_epochs)) sess.run(train_init_op) while True: try: _ = sess.run(full_train_op) except tf.errors.OutOfRangeError: break # Check accuracy on the train and val sets every epoch train_acc, train_matrix = check_accuracy(sess, conf_matrix, train_init_op) val_acc, val_matrix = check_accuracy(sess, conf_matrix, val_init_op) plot_acc_val.append(val_acc) plot_acc_train.append(train_acc) if (val_acc > best_acc): best_acc = val_acc saver.save(sess, '/work/gbertocco/TCC/model04/ER_' + version) info_validation = [val_acc, val_matrix] pickle.dump(info_validation, open("INFO_VAL_" + version, "wb")) pickle.dump(plot_acc_train, open("plot_val_" + version, "wb")) pickle.dump(plot_acc_val, open("plot_train_" + version, "wb")) print('Train normalized accuracy: %f' % train_acc) print('Val normalized accuracy: %f\n' % val_acc) print('Best (Val) normalized accuracy so far: %f\n' % best_acc)
class transfer_model(object): model_name = "transfer_model" # name for checkpoint def __init__(self, sess, epoch, batch_size, checkpoint_dir, log_dir, learning_rate = 0.00001, beta1=0.5): self.sess = sess self.keep_prob = 1.0 #self.dataset_name = dataset_name #self.result_dir = result_dir self.log_dir = log_dir self.checkpoint_dir = checkpoint_dir self.epoch = epoch self.batch_size = batch_size self.beta1 = beta1 self.label_dim = 50 self.train_set = DataSet("../data/train_augment", self.batch_size, self.label_dim) self.test_set = DataSet("../data/test_augment", self.batch_size, self.label_dim) # parameters self.input_height = 227 self.input_width = 227 #self.output_height = 224 #self.output_width = 224 self.c_dim = 3 # train self.init_learning_rate = learning_rate # get number of batches for a single epoch self.num_batches = self.train_set.total_batches self.test_num_batches = self.test_set.total_batches def classifier(self, x, is_training=True, reuse=False): with tf.variable_scope("classifier", reuse=reuse): #net = tf.reshape(x, [-1, 6*6*256]) #net = tf.nn.relu(bn(linear(net, 4096, scope='fc1'), is_training=is_training, scope='bn1')) #net = tf.nn.relu(bn(linear(net, 4096, scope='fc2'), is_training=is_training, scope='bn2')) #out = linear(net, self.label_dim, scope='fc3') out = linear(x, self.label_dim, scope='fc8') return out def build_model(self): # some parameters image_dims = [self.input_height, self.input_width, self.c_dim] bs = self.batch_size """ Graph Input """ # images self.inputs = tf.placeholder(tf.float32, [self.batch_size] + image_dims, name='input_images') # labels self.labels = tf.placeholder(tf.float32, [self.batch_size, self.label_dim], name='label') # keep prob self.keep_prob = tf.placeholder(tf.float32, shape=[]) # learning rate self.learning_rate = tf.placeholder(tf.float32, shape=[]) """ Loss Function """ # get fc output of alexnet self.alexnet = AlexNet(self.inputs, keep_prob=self.keep_prob, num_classes=1000, skip_layer=[]) logits = self.classifier(self.alexnet.dropout7, is_training=True, reuse=False) prob = tf.nn.softmax(logits) self.prob = prob # self.loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=self.labels)) """ Training """ # divide trainable variables into a group for D and a group for G vars = tf.trainable_variables() # optimizer # self.optim = tf.train.GradientDescentOptimizer(self.learning_rate) \ # .minimize(self.loss, var_list=vars) self.optim = tf.train.AdamOptimizer(self.learning_rate) \ .minimize(self.loss, var_list=vars) """ Testing """ # for test test_logits = self.classifier(self.alexnet.dropout7, is_training=False, reuse=True) self.test_prob = tf.nn.softmax(test_logits) self.correct_pred = tf.equal(tf.argmax(self.test_prob, axis=1), tf.argmax(self.labels, axis=1)) self.acc = tf.reduce_mean(tf.cast(self.correct_pred, tf.float32)) """ Summary """ self.loss_sum = tf.summary.scalar("loss", self.loss) self.acc_sum = tf.summary.scalar("acc", self.acc) def train(self): # initialize all variables tf.global_variables_initializer().run() self.alexnet.load_initial_weights(self.sess) # saver to save model self.saver = tf.train.Saver() # summary writer self.writer = tf.summary.FileWriter(self.log_dir + '/' + self.model_name, self.sess.graph) # restore check-point if it exits could_load, checkpoint_counter = self.load(self.checkpoint_dir) if could_load: start_epoch = (int)(checkpoint_counter / self.num_batches) start_batch_id = checkpoint_counter - start_epoch * self.num_batches counter = checkpoint_counter print(" [*] Load SUCCESS") else: start_epoch = 0 start_batch_id = 0 counter = 1 print(" [!] Load failed...") # loop for epoch start_time = time.time() lr = self.init_learning_rate min_loss = 100 update_cnt = 0 for epoch in range(start_epoch, self.epoch): cur_loss = 0 # get batch data for idx in range(start_batch_id, self.num_batches): ''' TODO: add data''' inputs, labels = self.train_set.next_batch() # update network _, loss_summary_str, acc_summary_str, loss, prob = self.sess.run([self.optim, self.loss_sum, self.acc_sum, self.loss, self.prob], feed_dict={ self.inputs: inputs, self.labels: labels, self.keep_prob: 0.5, self.learning_rate: lr}) self.writer.add_summary(loss_summary_str, counter) self.writer.add_summary(acc_summary_str, counter) cur_loss += loss #print("output", prob) #print("answer", labels) # display training status counter += 1 if counter % 100 == 1: print("Epoch: [%2d] [%4d/%4d] time: %4.4f, loss: %.8f, lr: %.6f" \ % (epoch, idx, self.num_batches, time.time() - start_time, loss, lr)) # adapt learning rate if cur_loss / (self.num_batches - start_batch_id) > min_loss: update_cnt += 1 else: update_cnt = 0 if update_cnt == 5: update_cnt = 0 lr = lr * 0.5 # After an epoch, start_batch_id is set to zero # non-zero value is only for the first epoch after loading pre-trained model start_batch_id = 0 # save model self.save(self.checkpoint_dir, counter) '''Test''' acc = 0 for idx in range(0, self.test_num_batches): inputs, labels = self.test_set.next_batch() correct_pred, prob = self.sess.run([self.correct_pred, self.prob], feed_dict={ self.inputs: inputs, self.labels: labels, self.keep_prob: 1.0}) #if np.sum(prob) == np.argmax(labels): #print(prob[0]) #print(np.argmax(prob, axis=1), np.argmax(labels, axis=1)) acc += (np.sum(correct_pred) + 0.0)/ self.batch_size #print np.sum(correct_pred), self.batch_size #print(np.sum(correct_pred), self.batch_size) #print(len(correct_pred), labels.shape) #print(correct_pred) print("Epoch: [%2d] acc: %.8f" \ % (epoch, (acc + 0.0) / self.test_num_batches)) # save model for final step self.save(self.checkpoint_dir, counter) def pred(self): # saver to save model self.saver = tf.train.Saver() could_load, checkpoint_counter = self.load(self.checkpoint_dir) if could_load: print(" [*] Load SUCCESS") else: print(" [!] Load failed...") for idx in range(0, self.predict_num_batches): inputs, labels = self.predict_set.next_batch() prob = self.sess.run([self.test_prob], feed_dict={self.inputs: inputs}) #print prob #print labels #print self.label_name[np.argmax(prob)], self.label_name[np.argmax(labels)] #print "============" @property def model_dir(self): return "{}_{}".format( self.model_name, self.batch_size) def save(self, checkpoint_dir, step): checkpoint_dir = os.path.join(checkpoint_dir, self.model_dir, self.model_name) if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) self.saver.save(self.sess,os.path.join(checkpoint_dir, self.model_name+'.model'), global_step=step) def load(self, checkpoint_dir): import re print(" [*] Reading checkpoints...") checkpoint_dir = os.path.join(checkpoint_dir, self.model_dir, self.model_name) ckpt = tf.train.get_checkpoint_state(checkpoint_dir) if ckpt and ckpt.model_checkpoint_path: ckpt_name = os.path.basename(ckpt.model_checkpoint_path) self.saver.restore(self.sess, os.path.join(checkpoint_dir, ckpt_name)) counter = int(next(re.finditer("(\d+)(?!.*\d)",ckpt_name)).group(0)) print(" [*] Success to read {}".format(ckpt_name)) return True, counter else: print(" [*] Failed to find a checkpoint") return False, 0
def dataInputGen(): # File Path desc train_file = "./TrainingFile_detail.txt" Validate_file = "./ValidationFile_detail.txt" class_file = "./Classmap.txt" checkpoint_path = "./ckp/" # Network params learning_rate = 0.00001 dropout_rate = 0.5 num_classes = 1000 batch_size = 1 num_epochs = 100 display_step = 20 #check acc per epoach train_layers = ['fc8', 'fc7', 'fc6'] #read all image path config train_img_paths,train_labels = read_train_detail(train_file) validate_img_paths,validate_labels = read_train_detail(Validate_file) print("Total Dataset {}".format(int(len(train_labels)+len(validate_labels)))) print("Split to Training {} and to Validation {}".format(len(train_labels),len(validate_labels))) with tf.device('/cpu:0'): tr_data = ImageDataGenerator(mode='training', batch_size=batch_size, num_classes=num_classes, class_file=class_file, shuffle=False, img_paths=train_img_paths, labels=train_labels) # create an reinitializable iterator given the dataset structure iterator = Iterator.from_structure(tr_data.data.output_types, tr_data.data.output_shapes) next_batch = iterator.get_next() # Ops for initializing the two different iterators training_init_op = iterator.make_initializer(tr_data.data) validation_init_op = iterator.make_initializer(val_data.data) # Get the number of training/validation steps per epoch train_batches_per_epoch = int(np.floor(tr_data.data_size/batch_size)) val_batches_per_epoch = int(np.floor(val_data.data_size/batch_size)) # TF placeholder for graph input and output x = tf.placeholder(tf.float32, [None, 227, 227, 3]) y = tf.placeholder(tf.float32, [None, num_classes]) keep_prob = tf.placeholder(tf.float32) # Initialize model model = AlexNet(x, keep_prob, num_classes, train_layers,weights_path="./weights/bvlc_alexnet.npy") # Link variable to model output score = model.fc8 # List of trainable variables of the layers we want to train var_list = [v for v in tf.trainable_variables() if v.name.split('/')[0] in train_layers] # Op for calculating the loss with tf.name_scope("cross_ent"): loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=score,labels=y)) # Train op with tf.name_scope("train"): optimizer = tf.train.AdamOptimizer(learning_rate).minimize(loss) # Evaluation op: Accuracy of the model with tf.name_scope("accuracy"): correct_pred = tf.equal(tf.argmax(score, 1), tf.argmax(y, 1)) accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32),name="ACC") # Add the loss to summary tf.summary.scalar('cross_entropy', loss) tf.summary.scalar('Acc', accuracy) # Merge all summaries together merged_summary = tf.summary.merge_all() # Initialize the FileWriter writer = tf.summary.FileWriter("./Graph") # Initialize an saver for store model checkpoints saver = tf.train.Saver() with tf.Session() as sess: # Initialize all variables sess.run(tf.global_variables_initializer()) # Add the model graph to TensorBoard writer.add_graph(sess.graph) # Load the pretrained weights into the non-trainable layer print("Weight Loading...") model.load_initial_weights(sess) print("{} Start training...".format(datetime.now())) best_acc = 0 # Loop over number of epochs for epoch in range(num_epochs): print("{} Epoch number: {}".format(datetime.now(), epoch+1)) # Initialize iterator with the training dataset sess.run(training_init_op) for step in tqdm(range(train_batches_per_epoch)): # get next batch of data img_batch, label_batch = sess.run(next_batch) # And run the training op sess.run(train_op, feed_dict={x: img_batch, y: label_batch, keep_prob: dropout_rate}) # Generate summary with the current batch of data and write to file if step % display_step == 0: s,acc_data = sess.run([merged_summary,accuracy], feed_dict={x: img_batch, y: label_batch, keep_prob: 1.}) writer.add_summary(s, epoch*train_batches_per_epoch + step) #print("Accuracy at steps {} is {:f}".format((epoch*train_batches_per_epoch + step),acc_data)) # Validate the model on the entire validation set print("{} Start validation".format(datetime.now())) sess.run(validation_init_op) test_acc = 0. test_count = 0 for _ in tqdm(range(val_batches_per_epoch)): img_batch, label_batch = sess.run(next_batch) acc = sess.run(accuracy, feed_dict={x: img_batch, y: label_batch, keep_prob: 1.}) test_acc += acc test_count += 1 test_acc /= test_count print("{} Validation Accuracy = {:.4f}".format(datetime.now(), test_acc)) print("Checking improving Accuracy...") if test_acc > best_acc: print("Accuracy improve from {:.4f} to {:.4f}".format(best_acc,test_acc)) print("Saving best weights ...") save_path = saver.save(sess, "./BestWeight/weight") best_acc = test_acc else: print("Accuracy not improve") print("{} Saving checkpoint of model...".format(datetime.now())) # save checkpoint of the model checkpoint_name = os.path.join(checkpoint_path, 'model_epoch'+str(epoch+1)+'.ckpt') save_path = saver.save(sess, checkpoint_name) print("{} Model checkpoint saved at {}".format(datetime.now(), checkpoint_name))
class NearestNeighbor: cluster_dict = {} def __init__(self): # The number of output classes of Alexnet num_classes = 1000 # No train on the layers train_layers = [] # TF placeholder for graph input and output self.x = tf.placeholder(tf.float32, [1, 227, 227, 3]) self.keep_prob = tf.placeholder(tf.float32) # Initialize model self.model = AlexNet(self.x, self.keep_prob, num_classes, train_layers) # Start the Tensorflow Session and initialize all variables self.sess = tf.Session() self.sess.run(tf.global_variables_initializer()) # Load the pre-trained weights into the non-trainable layer self.model.load_initial_weights(self.sess) # Declare the feature-vector-dictionary self.load_cluster_dict() # Init Audio self.audio = Audio() # Classifies the given picture and returns the label and if it was a new label def classify(self, name, speech=False): # Get the image image_path = "classification/pictures/" + name + ".jpeg" im1 = cv2.imread(image_path).astype(np.float32) im1 = im1 - np.mean(im1) # RGB to BGR im1[:, :, 0], im1[:, :, 2] = im1[:, :, 2], im1[:, :, 0] features = self.get_features(np.array([im1])) best_match = ("", np.inf) # Compute all distances to the clusters and search the nearest for label in self.cluster_dict.keys(): cluster = self.cluster_dict[label] distance = spatial.distance.euclidean(np.array(features[0]), cluster[0]) if distance < best_match[1]: best_match = (label, distance) self.audio.output("\nThe object is a " + best_match[0]) # check if the estimation was correct or not while True: correct_estimation = self.audio.input("Is the shown object a '" + best_match[0] + "' ? (yes or no)", speech=speech) # add the features to the neighbor dict if it was the correct estimation if correct_estimation == "y" or correct_estimation == "Y" or correct_estimation == "yes": self.calculate_cluster(np.array(features), best_match[0]) return best_match[0], False # ask what the correct label is elif correct_estimation == "n" or correct_estimation == "N" or correct_estimation == "no": correct_label = self.audio.input("What is the correct label then? (correct spelling required!)", speech=speech) if correct_label in self.cluster_dict.keys(): self.calculate_cluster(np.array(features), correct_label) return correct_label, False # ask if it is a new label else: new_label = self.audio.input("Is this a complete new label? (yes or no)", speech=speech) if new_label == "y" or new_label == "Y" or new_label == "yes": self.calculate_cluster(np.array(features), correct_label) return correct_label, True else: self.audio.output("Then may check the correct spelling.") else: self.audio.output("Wrong input, please check again.") # Extracts the feature vector of the given picture and saves it in the dict def learn(self, label, image_name="tmp_picture"): image_path = "classification/pictures/" + image_name + ".jpeg" # Get the Image image = (cv2.imread(image_path)[:, :, :3]).astype(np.float32) image = image - np.mean(image) # RGB to BGR image[:, :, 0], image[:, :, 2] = image[:, :, 2], image[:, :, 0] features = self.get_features(np.array([image])) self.calculate_cluster(np.array(features), label) self.audio.output("\nCluster of the label is updated. \n") # Saves all feature vectors of the pictures of the given folder with the given label def batch_learn(self, label, folder_path="classification/pictures/"): folder_path = folder_path + label # Creates an array with all images as arrays images = [] for image in os.listdir(folder_path): if image.endswith(".jpg") or image.endswith(".jpeg"): # Get the image image_path = os.path.join(folder_path, image) image = (cv2.imread(image_path)[:, :, :3]).astype(np.float32) image = image - np.mean(image) # RGB to BGR image[:, :, 0], image[:, :, 2] = image[:, :, 2], image[:, :, 0] images.append(image) features = self.get_features(np.array(images)) self.calculate_cluster(np.array(features), label) self.audio.output("Finished " + label) # Get the feature vectors of all the images in the images array def get_features(self, images): features = [] # Link variable to model output score = self.model.fc7 for im in images: # Skip image if the shape is not the right one if im.shape != (227, 227, 3): print("Image has wrong resolution!") continue # Feed alexnet with the image output = self.sess.run(score, feed_dict={self.x: [im], self.keep_prob: 1.}) features.append(output[0]) return features # Calculates the new mean of the label regarding the feature array def calculate_cluster(self, features, label): mean_features = np.mean(features, axis=0) if label not in self.cluster_dict.keys(): self.cluster_dict[label] = (mean_features, features.shape[0]) else: cluster = self.cluster_dict[label] new_cluster = (cluster[1] * cluster[0] + features.shape[0] * mean_features) / ( cluster[1] + features.shape[0]) self.cluster_dict[label] = (new_cluster, cluster[1]+features.shape[0]) # Load the neighbor list out of the file def load_cluster_dict(self): try: print "open file" self.cluster_dict = dict(pickle.load(open("classification/cluster_list.p", "rb"))) print "done" except IOError: print "no file found. start with empty dictionary" # Shows all labels of the neighbor dictionary def show_labels(self): for label in self.cluster_dict.keys(): print(label + ": " + str(self.cluster_dict[label][1]) + " pictures included") # Deletes a label from the neighbor dict and the feature vectors of it def delete_label(self, label): if label in self.cluster_dict.keys(): del self.cluster_dict[label] else: self.audio.output("The label is not included.") def clear_cluster_dict(self): self.cluster_dict.clear() def save_cluster_dict(self): pickle.dump(self.cluster_dict, open("classification/cluster_list.p", "wb")) print("file saved") # Saves the neighbor dictionary and closes the tensorflow session def close(self): self.save_cluster_dict() self.sess.close()
# Get the number of training/validation steps per epoch train_batches_per_epoch = np.floor(train_generator.data_size / batch_size).astype(np.int16) val_batches_per_epoch = np.floor(val_generator.data_size / batch_size).astype(np.int16) # Start Tensorflow session with tf.Session() as sess: # Initialize all variables sess.run(tf.global_variables_initializer()) # Add the model graph to TensorBoard writer.add_graph(sess.graph) # Load the pretrained weights into the non-trainable layer model.load_initial_weights(sess) print("{} Start training...".format(datetime.now())) print("{} Open Tensorboard at --logdir {}".format(datetime.now(), filewriter_path)) # Loop over number of epochs for epoch in range(num_epochs): print("{} Epoch number: {}".format(datetime.now(), epoch+1)) step = 1 while step < train_batches_per_epoch: # Get a batch of images and labels
def main(_): if not FLAGS.image_paths: raise ValueError("Must set --image_paths") if not FLAGS.save_file: raise ValueError("Must set --save_file") image_paths_file = FLAGS.image_paths save_file = FLAGS.save_file batch_size = FLAGS.batch_size num_classes = 1000 emb_dim = 4096 # TF placeholder for graph input and output x = tf.placeholder(tf.float32, [None, 227, 227, 3]) kp = tf.placeholder(tf.float32) # Initialize model train_layers = [] model = AlexNet(x, kp, num_classes, emb_dim, train_layers) lines = open(image_paths_file).readlines() image_paths = [l.split()[0] for l in lines] def path2id(path): return path.split('/')[-1][:-4] ids = [path2id(p.split()[0]) for p in image_paths] labels = [int(l.split()[1]) for l in lines] with tf.Session() as sess: # Initialize all variables sess.run(tf.global_variables_initializer()) # Load the pretrained weights into the non-trainable layer model.load_initial_weights(sess) pool5_tensor = sess.graph.get_tensor_by_name('pool5:0') num_batches = m.ceil(len(image_paths) / float(batch_size)) features = [] for i in range(num_batches): s_batch = i * batch_size e_batch = min(s_batch + batch_size, len(image_paths)) # Read the image img_pool = [] for b in range(s_batch, e_batch): img = cv2.imread(image_paths[b]) # Convert image to float32 and resize to (227x227) img = cv2.resize(img.astype(np.float32), (227, 227)) # Subtract the ImageNet mean img -= imagenet_mean # Reshape as needed to feed into model img_pool.append(img.reshape((227, 227, 3))) img_pool = np.stack(img_pool, axis=0) feature = sess.run(pool5_tensor, feed_dict={x: img_pool, kp: 1.0}) feature = np.squeeze(np.reshape(feature, [-1, 6 * 6 * 256])) features.append(feature) if (i + 1) % 10000 == 0: print('Processed ' + str(i + 1) + ' files') features = np.vstack(features) batch_size = 50000 array_list = [] split_idx = np.arange(batch_size, len(image_paths), batch_size) features = np.split(features, split_idx, axis=0) for i, feature in enumerate(features): print('Seg: %d size: %d' % (i, feature.shape[0])) pickle.dump((features, labels, ids), open(save_file, 'wb'))
def main(): if not os.path.exists(args.output_dir): os.makedirs(args.output_dir) # read data from tfrecord files img, label = reader.read_and_decode(args.tfrecords, epochs=args.num_epochs, size=args.img_size) img_batch, label_batch = tf.train.shuffle_batch([img, label], batch_size=args.batch_size, capacity=2000, min_after_dequeue=1000) global_step = tf.Variable(0, name="global_step", trainable=False) # construct network model model = AlexNet(img_batch, args.dropout_rate, args.num_bit, args.num_class, skip_layers) D = model.softsign [d_loss, out] = hashing_loss(D, label_batch, args.alpha, args.belta, args.gama, args.num_bit) # List of trainable variables of the layers to finetune var_list1 = [ v for v in tf.trainable_variables() if v.name.split('/')[0] not in skip_layers ] # List of trainable variables of the layers to train from scratch var_list2 = [ v for v in tf.trainable_variables() if v.name.split('/')[0] in skip_layers ] # learning rate learning_rate = tf.train.exponential_decay(args.lr, global_step, args.decay_step, args.decay_rate, staircase=True) opt1 = tf.train.AdamOptimizer(learning_rate * 0.001) opt2 = tf.train.AdamOptimizer(learning_rate) # apply different grads for two type layers grads = tf.gradients(d_loss, var_list1 + var_list2) grads1 = grads[:len(var_list1)] grads2 = grads[len(var_list1):] train_op1 = opt1.apply_gradients(zip(grads1, var_list1)) train_op2 = opt2.apply_gradients(zip(grads2, var_list2), global_step=global_step) train_op = tf.group(train_op1, train_op2) with tf.Session(config=config) as sess: saver = tf.train.Saver(tf.global_variables()) sess.run([ tf.global_variables_initializer(), tf.local_variables_initializer() ]) if args.checkpoint is not None: checkpoint = tf.train.latest_checkpoint(args.checkpoint) print('Restoring model from {}'.format(checkpoint)) saver.restore(sess, checkpoint) else: # Load the pretrained weights into the non-trainable layer model.load_initial_weights(sess) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) start_time = time.time() try: while not coord.should_stop(): _, loss_t, dt, step1 = sess.run( [train_op, d_loss, out, global_step]) elapsed_time = time.time() - start_time start_time = time.time() if step1 % 10 == 0: print("iter: %4d, loss: %.8f, time: %.3f" % (step1, loss_t, elapsed_time)) if step1 % args.save_freq == 0: saver.save(sess, args.output_dir + '/model.ckpt', global_step=step1) except tf.errors.OutOfRangeError: saver.save(sess, args.output_dir + '/model-done.ckpt') print('Done training -- epoch limit reached') finally: coord.request_stop() coord.join(threads)
class Model(object): def __init__(self, mode='train'): if mode == 'train' or mode == 'test': self.mode = mode else: print('Mode must be train or test.') # Learning params self.learning_rate = 1e-3 self.minibatch_size = 32 self.num_iterations = 1500 self.train_test_split = 0.7 self.keep_rate = 0.6 self.num_layers_to_load = 5 self.display_step = 5 self.save_step = 100 self.save_dir = "tf_data\sample_model" self.input_image_size = (227, 227) self.label_indices = { '20': 0, '30': 1, '50': 2, '60': 3, '70': 4, '80': 5, '80x': 6, '100': 7, '120': 8 } self.labels_ordered = list(self.label_indices) self.num_classes = len(self.label_indices) #Go ahead and build graph on initialization: self.build_graph() #Set to true with we create a tf session. self.initialized = False def build_graph(self): self.X = tf.placeholder( tf.float32, [None, self.input_image_size[0], self.input_image_size[1], 3]) self.y = tf.placeholder(tf.float32, [None, self.num_classes]) self.keep_prob = tf.placeholder(tf.float32) self.AN = AlexNet(self.X, self.keep_prob, self.num_layers_to_load) alexnet_out = self.AN.pool5 print(alexnet_out) self.flattened = tf.reshape(alexnet_out, [-1, 6 * 6 * 256]) #Dense layer followed by a dropout layer self.dense1 = tf.layers.dense(inputs=self.flattened, units=1024, activation=tf.nn.relu, name="my_dense1") print("D1 Shape: ", self.dense1.get_shape()) self.dropout1 = tf.layers.dropout( inputs=self.dense1, rate=0.4, training=self.mode == tf.estimator.ModeKeys.TRAIN, name="my_drop1") print("Drop1 Shape: ", self.dropout1.get_shape()) #Smaller dense layer followed by a dropout layer self.dense2 = tf.layers.dense(inputs=self.dropout1, units=128, activation=tf.nn.relu, name="my_dense2") print("D2 Shape: ", self.dense2.get_shape()) self.dropout2 = tf.layers.dropout( inputs=self.dense2, rate=0.4, training=self.mode == tf.estimator.ModeKeys.TRAIN, name="my_drop2") print("Ddrop2 Shape: ", self.dropout2.get_shape()) #Output of 3 for the three available classes self.output = tf.layers.dense(inputs=self.dropout2, units=9, name="my_out") print("Output Shape: ", self.output.get_shape()) self.yhat = self.output print("yHat Shape: ", self.yhat.get_shape()) self.trainable_variable_names = [ 'conv5', 'my_dense1', 'my_dense2', 'my_out' ] def train(self): cost = tf.losses.sparse_softmax_cross_entropy( tf.argmax(self.y, axis=1), self.output) var_list = [v for v in tf.trainable_variables() if v.name.split('/')[0] \ in self.trainable_variable_names] # Train op with tf.name_scope("train"): # Get gradients of all trainable variables gradients = tf.gradients(cost, var_list) gradients = list(zip(gradients, var_list)) # Create optimizer and apply gradient descent to the trainable variables optimizer = tf.train.AdamOptimizer(self.learning_rate) train_op = optimizer.apply_gradients(grads_and_vars=gradients) # Add gradients to summary for gradient, var in gradients: tf.summary.histogram(var.name + '/gradient', gradient) # Add the variables we train to the summary for var in var_list: tf.summary.histogram(var.name, var) # Add the loss to summary tf.summary.scalar('cross_entropy', cost) # Evaluation op: Accuracy of the model with tf.name_scope("accuracy"): correct_pred = tf.equal(tf.argmax(self.yhat, 1), tf.argmax(self.y, 1)) accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) # Add the accuracy to the summary tf.summary.scalar('accuracy', accuracy) # Merge all summaries together merged_summary = tf.summary.merge_all() # Initialize the FileWriter train_writer = tf.summary.FileWriter( os.path.join(self.save_dir, 'train')) test_writer = tf.summary.FileWriter(os.path.join( self.save_dir, 'test')) # Initialize an saver for store model checkpoints # We'll save the pretrained alexnet weights and the new weight values that we train: var_list_to_save = [v for v in tf.trainable_variables() if v.name.split('/')[0] in \ self.AN.all_variable_names + self.trainable_variable_names] saver = tf.train.Saver(var_list=var_list_to_save) #Load up data from image files. data = data_feeder(label_indices=self.label_indices, train_test_split=self.train_test_split, input_image_size=self.input_image_size, data_path='GTSRB\Final_Training\SpeedLimitSigns') #Setup minibatch generators G = Generator(data.train.X, data.train.y, minibatch_size=self.minibatch_size) GT = Generator(data.test.X, data.test.y, minibatch_size=self.minibatch_size) #Launch tf session self.sess = tf.Session() self.sess.run(tf.global_variables_initializer()) self.initialized = True # Add the model graph to TensorBoard train_writer.add_graph(self.sess.graph) # Load the pretrained weights into the alexnet portion of our graph self.AN.load_initial_weights(self.sess) print("{} Start training...".format(datetime.now())) print("{} Open Tensorboard at --logdir {}".format( datetime.now(), self.save_dir)) #And Train for i in range(self.num_iterations): G.generate() # And run the training op self.sess.run(train_op, feed_dict={ self.X: G.X, self.y: G.y, self.keep_prob: self.keep_rate }) # Generate summary with the current batch of data and write to file if i % self.display_step == 0: s = self.sess.run(merged_summary, feed_dict={ self.X: G.X, self.y: G.y, self.keep_prob: 1.0 }) train_writer.add_summary(s, i) GT.generate() s = self.sess.run(merged_summary, feed_dict={ self.X: GT.X, self.y: GT.y, self.keep_prob: 1.0 }) test_writer.add_summary(s, i) train_acc = self.sess.run(accuracy, feed_dict={ self.X: G.X, self.y: G.y, self.keep_prob: 1.0 }) test_acc = self.sess.run(accuracy, feed_dict={ self.X: GT.X, self.y: GT.y, self.keep_prob: 1.0 }) print(i, ' iterations,', str(G.num_epochs), 'epochs, train accuracy = ', train_acc, ', test accuracy = ', test_acc) if i % self.save_step == 0 and i > 0: print("{} Saving checkpoint of model...".format( datetime.now())) # save checkpoint of the model checkpoint_name = os.path.join( self.save_dir, 'model_epoch' + str(G.num_epochs) + '.ckpt') save_path = saver.save(self.sess, checkpoint_name) print("{} Model checkpoint saved at {}".format( datetime.now(), checkpoint_name)) def predict(self, X, checkpoint_dir=None): ''' X: Numpy array of dimension [n, 227, 227, 3]. Note that n here may not be the same as the minibatch_size defined above. Returns: yhat_numpy: A numpy array of dimension [n x 3] containing the predicted one hot labels for each image passed in X. ''' if not self.initialized: self.restore_from_checkpoint(checkpoint_dir) yhat_numpy = self.sess.run(self.yhat, feed_dict={ self.X: X, self.keep_prob: 1.0 }) return yhat_numpy def restore_from_checkpoint(self, checkpoint_dir=None): ''' Restore model from most recent checkpoint in save dir. ''' saver = tf.train.Saver() self.sess = tf.Session() #Load latest checkpont, use savedir if we're not given a checkpoint dir: if checkpoint_dir is None: checkpoint_name = tf.train.latest_checkpoint(self.save_dir) else: checkpoint_name = tf.train.latest_checkpoint(checkpoint_dir) print(checkpoint_name) #Resore session saver.restore(self.sess, checkpoint_name) self.initialized = True
loss = tf.reduce_mean(loss_v) opt_gd = tf.train.GradientDescentOptimizer(learning_rate=1e-4) train_step = opt_gd.minimize(loss) ## # start TF session and init data loaders sess = tf.Session() db_train.init(sess) db_test.init(sess) # initialize network weights sess.run( tf.global_variables_initializer()) # this initialized the weights randomly net.load_initial_weights( sess) # this loads the pretrained alexnet weights from the npy file # saver for the network weights saver = tf.train.Saver(max_to_keep=1) if not os.path.exists(SNAPSHOT_DIR): os.mkdir(SNAPSHOT_DIR) else: shutil.rmtree(SNAPSHOT_DIR) os.mkdir(SNAPSHOT_DIR) # TRAINING LOOP tr_loss = [] for step in range(TRAIN_ITER): # work around to deal with cases when batch size doesnt divide the number of samples of the db set evenly while True: # load images and annotation
def main(argv=None): if (argv is None): argv = sys.argv try: try: opts = argv first_time_load = True parent_dir = './' keys = ['cov1', 'cov2', 'fc1', 'fc2', 'fc3'] prune_thresholds = {} WITH_BIASES = False save_for_next_iter = False TEST = False for key in keys: prune_thresholds[key] = 0. for item in opts: print(item) opt = item[0] val = item[1] if (opt == '-cRates'): cRates = val if (opt == '-first_time'): first_time_load = val if (opt == '-file_name'): file_name = val if (opt == '-train'): TRAIN = val if (opt == '-prune'): PRUNE = val if (opt == '-test'): TEST = val if (opt == '-parent_dir'): parent_dir = val if (opt == '-lr'): lr = val if (opt == '-with_biases'): WITH_BIASES = val if (opt == '-lambda1'): lambda_1 = val if (opt == '-lambda2'): lambda_2 = val if (opt == '-save'): save_for_next_iter = val if (opt == '-org_file_name'): org_file_name = val print('pruning thresholds are {}'.format(prune_thresholds)) except getopt.error, msg: raise Usage(msg) epochs = 100 dropout = 0.5 batch_size = 1 num_classes = 1000 NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN = 50000 INITIAL_LEARNING_RATE = 0.001 INITIAL_LEARNING_RATE = lr LEARNING_RATE_DECAY_FACTOR = 0.1 NUM_EPOCHS_PER_DECAY = 350.0 DISPLAY_FREQ = 50 TEST = 1 TRAIN_OR_TEST = 0 NUM_CHANNELS = 3 first_time_load = 1 mask_dir = parent_dir weights_dir = parent_dir LOCAL_TEST = 1 file_name_part = compute_file_name(cRates) if (save_for_next_iter): (weights_mask, biases_mask) = initialize_weights_mask( first_time_load, mask_dir, 'mask' + org_file_name + '.pkl') else: (weights_mask, biases_mask) = initialize_weights_mask( first_time_load, mask_dir, 'mask' + file_name_part + '.pkl') meta_data_dir = '/local/scratch/share/ImageNet/ILSVRC/Data/CLS-LOC' index_file_dir = '/local/scratch/share/ImageNet/ILSVRC/ImageSets/CLS-LOC/' index_file_dir = 'cpu_test_data/' if (TRAIN): train_file_txt = index_file_dir + 'train.txt' val_file_txt = index_file_dir + 'val.txt' test_file_txt = index_file_dir + 'test.txt' else: test_file_txt = index_file_dir + 'test.txt' # if (first_time_load): # PREV_MODEL_EXIST = 0 # weights, biases = initialize_variables(PREV_MODEL_EXIST, '') # else: # PREV_MODEL_EXIST = 1 # if (save_for_next_iter): # file_name_part = org_file_name # else: # file_name_part = compute_file_name(cRates) # weights, biases = initialize_variables( PREV_MODEL_EXIST, # weights_dir + 'weights' + file_name_part + '.pkl') # x = tf.placeholder(tf.float32, [batch_size, 227, 227, 3]) y = tf.placeholder(tf.float32, [None, num_classes]) keep_prob = tf.placeholder(tf.float32) # initilize the model from the class constructer model = AlexNet(x, keep_prob, num_classes) score = model.fc8 softmax = tf.nn.softmax(score) with tf.name_scope("cross_ent"): loss = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits=score, labels=y)) with tf.name_scope("train"): # l1_norm = lambda_1 * l1 # l2_norm = lambda_2 * l2 # regulization_loss = l1_norm + l2_norm opt = tf.train.AdamOptimizer(lr) # loss_value = tf.reduce_mean(cross_entropy) + regulization_loss grads = opt.compute_gradients(loss) org_grads = [(ClipIfNotNone(grad), var) for grad, var in grads] train_step = opt.apply_gradients(org_grads) with tf.name_scope("accuracy"): correct_prediction = tf.equal(tf.argmax(score, 1), tf.argmax(y, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) # keys = ['cov1', 'cov2', 'fc1', 'fc2', 'fc3'] # weights_new[key] = weights[key] * tf.constant(weights_mask[key], dtype=tf.float32) # # l1, l2, pred = cov_network(images, weights_new, biases, keep_prob) # _, _, test_pred = cov_network(test_images, weights_new, biases, keep_prob) # cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits = pred, labels = y) # # new_grads = mask_gradients(weights, org_grads, weights_mask, biases, biases_mask, WITH_BIASES) # Apply gradients. init = tf.global_variables_initializer() accuracy_list = np.zeros(20) train_acc_list = [] # Launch the graph print('Graph launching ..') if (TRAIN): train_generator = ImageDataGenerator(train_file_txt, horizontal_flip=True, shuffle=True) val_generator = ImageDataGenerator(val_file_txt, shuffle=False) # test_generator = ImageDataGenerator(test_file_txt, shuffle = False) if (TRAIN): # Get the number of training/validation steps per epoch train_batches_per_epoch = np.floor(train_generator.data_size / batch_size).astype(np.int16) val_batches_per_epoch = np.floor(val_generator.data_size / batch_size).astype(np.int16) # test_batches_per_epoch = np.floor(test_generator.data_size / batch_size).astype(np.int16) with tf.Session() as sess: sess.run(init) model.load_initial_weights(sess) # print('pre train pruning info') # prune_info(weights_new, 0) # print(78*'-') # start = time.time() if TRAIN == 1: print("{} Start training...".format(datetime.now())) for i in range(0, epochs): for step in range(train_batches_per_epoch): (batch_x, batch_y) = train_generator.next_batch(batch_size) train_acc, cross_en = sess.run([accuracy, loss_value], feed_dict={ x: batch_x, y: batch_y, keep_prob: 1.0 }) if (i % DISPLAY_FREQ == 0): print('This is the {}th of {}pruning, time is {}'. format(i, cRates, datetime.now())) print("accuracy is {} and cross entropy is {}". format(train_acc, cross_en)) accuracy_list = np.concatenate( (np.array([train_acc]), accuracy_list[0:19])) if (i % (DISPLAY_FREQ * 50) == 0 and i != 0): train_acc_list.append(train_acc) # file_name_part = compute_file_name(cRates) # save_pkl_model(weights, biases, weights_dir, 'weights' + file_name_part + '.pkl') # print("saved the network") if (np.mean(accuracy_list) > 0.81 and train_acc >= 0.83): accuracy_list = np.zeros(20) test_acc = sess.run(accuracy, feed_dict={ x: images_test, y: labels_test, keep_prob: 1.0 }) print('test accuracy is {}'.format(test_acc)) if (test_acc > 0.823): print( "training accuracy is large, show the list: {}" .format(accuracy_list)) break _ = sess.run(train_step, feed_dict={ x: batch_x, y: batch_y, keep_prob: dropout }) if (TEST): test_acc_list = [] if (LOCAL_TEST == 1): image_dir = "cpu_test_data/tmp_images/" imagenet_mean = np.array([104., 117., 124.], dtype=np.float32) test_batches_per_epoch = 3 img_files = [ os.path.join(image_dir, f) for f in os.listdir(image_dir) if f.endswith('.jpeg') ] imgs_test = [] for i, f in enumerate(img_files): tmp = cv2.imread(f) tmp = cv2.resize(tmp.astype(np.float32), (227, 227)) tmp -= imagenet_mean[i] tmp = tmp.reshape((1, 227, 227, 3)) imgs_test.append(tmp) if (LOCAL_TEST): names = [] probs = [] for step in range(test_batches_per_epoch): prob = sess.run( softmax, feed_dict={ x: imgs_test[step], # y: labels_test, keep_prob: 1.0 }) name = class_names[np.argmax(prob)] probs.append(np.max(prob)) names.append(name) print("names are {}".format(names)) print("probs are {}".format(probs)) sys.exit()
train_batches_per_epoch = np.floor(train_generator.data_size / batch_size).astype(np.int16) val_batches_per_epoch = np.floor(val_generator.data_size / batch_size).astype( np.int16) # Start Tensorflow session with tf.Session() as sess: # Initialize all variables sess.run(tf.global_variables_initializer()) # Add the model graph to TensorBoard writer.add_graph(sess.graph) # Load the pretrained weights into the non-trainable layer model.load_initial_weights(sess) print("{} Start training...".format(datetime.now())) print("{} Open Tensorboard at --logdir {}".format(datetime.now(), filewriter_path)) # Loop over number of epochs for epoch in range(num_epochs): print("{} Epoch number: {}".format(datetime.now(), epoch + 1)) step = 1 while step < train_batches_per_epoch: # Get a batch of images and labels
def train(args): batch_size = 1 num_epochs = 100 num_classes = 9 learning_rate = 1e-4 test_data_path = base_path + 'test/05/testImages/{:04d}/' test_file = base_path + 'test/05/testImages/{:04d}/test{:04d}_2020.txt' train_data_path = base_path + 'test/05/dummy/' validation_data_path = train_data_path if (args.mode == 'train'): num_training_samples = count_text_lines(train_file) num_validation_samples = count_text_lines(validate_file) train_batches_per_epoch = np.ceil(num_training_samples / batch_size).astype(np.int32) val_batches_per_epoch = np.ceil(num_training_samples / batch_size).astype(np.int32) trainDataGen = ImageDataGenerator(train_file, train_data_path, num_classes, 'training', batch_size, num_preprocess_threads=5, shuffle=True, min_queue_examples=1000) validationDataGen = ImageDataGenerator(validate_file, validation_data_path, num_classes, 'validation', batch_size, num_preprocess_threads=2, shuffle=False, min_queue_examples=100) train_imgBatch = trainDataGen.img_batch train_labelBatch = trainDataGen.label_batch val_imgBatch = validationDataGen.img_batch val_labelBatch = validationDataGen.label_batch # TF placeholder for graph input and output x = tf.placeholder(tf.float32, [batch_size, 227, 227, 4]) y = tf.placeholder(tf.float32, [batch_size, num_classes]) train_layers = ['fc8'] var_list = [ v for v in tf.trainable_variables() if v.name.split('/')[0] in train_layers ] # Initialize model model = AlexNet(x, num_classes, train_layers) score = model.fc8 with tf.name_scope("cross_ent"): loss = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits=score, labels=y)) # Op for calculating the loss # Train op with tf.name_scope("train"): # Create optimizer and apply gradient descent to the trainable variables gOpt = tf.train.GradientDescentOptimizer(learning_rate) grads = gOpt.compute_gradients(loss) train_op = gOpt.apply_gradients(grads) with tf.name_scope("train"): # Get gradients of all trainable variables for grad, var in grads: if grad is not None: tf.summary.histogram(var.op.name + '/gradients', grad) # Add the loss to summary tf.summary.scalar('cross_entropy', loss) # Evaluation op: Accuracy of the model with tf.name_scope("accuracy"): correct_pred = tf.equal(tf.argmax(score, 1), tf.argmax(y, 1)) accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) # Add the accuracy to the summary tf.summary.scalar('accuracy', accuracy) # Merge all summaries together merged_summary = tf.summary.merge_all() elif (args.mode == 'test'): if (args.testImg == None): sys.exit( 'Please Input a valid image no with option --testImg NUMBER ') testImageIdx = args.testImg test_data_path = test_data_path.format(testImageIdx) test_file = test_file.format(testImageIdx, testImageIdx) num_test_samples = count_text_lines(test_file) batch_size = 1000 test_batches_per_epoch = np.ceil(num_test_samples / batch_size).astype( np.int32) x = tf.placeholder(tf.float32, [batch_size, 227, 227, 4]) # keep_prob = tf.placeholder(tf.float32) testDataGen = ImageDataGenerator(test_file, test_data_path, num_classes, 'test', batch_size, num_preprocess_threads=5, shuffle=False, min_queue_examples=10000) test_imgBatch = testDataGen.img_batch test_labelBatch = testDataGen.label_batch # Initialize model model = AlexNet(x, num_classes, []) # # Link variable to model output score = model.fc8 softmax = tf.nn.softmax(score) # testImg = getTestImage(); outImg = np.zeros([480, 640, 3]) saver = tf.train.Saver() writer = tf.summary.FileWriter(filewriter_path) with tf.Session() as sess: if (args.mode == 'train'): # Required to get the filename matching to run. tf.local_variables_initializer().run() tf.global_variables_initializer().run() model.load_initial_weights(sess) # saver = tf.train.import_meta_graph('../checkpoints/model_epoch100.ckpt.meta') # saver.restore(sess, "../checkpoints/model_epoch100.ckpt") # Coordinate the loading of image files. coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) # Add the model graph to TensorBoard writer.add_graph(sess.graph) print("{} Start training...".format(datetime.now())) print("{} Open Tensorboard at --logdir {}".format( datetime.now(), filewriter_path)) for epoch in range(0, num_epochs): print("{} Epoch number: {}".format(datetime.now(), epoch + 1)) for step in range(0, train_batches_per_epoch): print("{} Batch number: {}".format(datetime.now(), step + 1)) # Get an image tensor and print its value. imageB, labelB = sess.run( [train_imgBatch, train_labelBatch]) # #For Debugging # np.save('imgDump.npy',imageB) # np.save('labelDump.npy',labelB) one_hot_labels = np.zeros((batch_size, num_classes)) for i in range(len(labelB)): one_hot_labels[i][int(labelB[i])] = 1 # # image_summary_t = tf.summary.image('t', tf.reshape(train_img, [1, 227, 227, 3]), max_outputs=1) # _,_, image_summary = sess.run([train_img, train_label, image_summary_t]) # writer.add_summary(image_summary) sess.run([train_op], feed_dict={ x: imageB, y: one_hot_labels }) # Generate summary with the current batch of data and write to file if step % display_step == 0: s = sess.run(merged_summary, feed_dict={ x: imageB, y: one_hot_labels }) writer.add_summary( s, epoch * train_batches_per_epoch + step) # writer.add_summary(image_summary) # Validate the model on the entire validation set print("{} Start validation".format(datetime.now())) test_acc = 0. test_count = 0 for _ in range(val_batches_per_epoch): imageB, labelB = sess.run([val_imgBatch, val_labelBatch]) one_hot_labels = np.zeros((batch_size, num_classes)) for i in range(len(labelB)): one_hot_labels[i][int(labelB[i])] = 1 acc = sess.run(accuracy, feed_dict={ x: imageB, y: one_hot_labels, }) test_acc += acc test_count += 1 test_acc /= test_count print("{} Validation Accuracy = {:.4f}".format( datetime.now(), test_acc)) print("{} Saving checkpoint of model...".format( datetime.now())) # save checkpoint of the model checkpoint_name = os.path.join( checkpoint_path, 'model_epoch' + str(epoch + 1) + '.ckpt') save_path = saver.save(sess, checkpoint_name) print("{} Model checkpoint saved at {}".format( datetime.now(), checkpoint_name)) # # image_summary_t = tf.summary.image('v'+label_v, tf.reshape(val_img, [1, 227, 227, 3]), max_outputs=1) # _,_,image_summary = sess.run([val_img, val_label, image_summary_t]) # writer.add_summary(image_summary) # Finish off the filename queue coordinator. coord.request_stop() coord.join(threads) elif (args.mode == 'test'): print 'inside Evaluation' sess.run(tf.global_variables_initializer()) sess.run(tf.local_variables_initializer()) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) saver = tf.train.import_meta_graph( '../checkpoints/model_epoch31.ckpt.meta') saver.restore(sess, "../checkpoints/model_epoch31.ckpt") print 'Loaded Model from Checkpoint' # for step in range(0, test_batches_per_epoch): for step in range(0, test_batches_per_epoch): print("{} Batch number: {}".format(datetime.now(), step + 1)) imageB, labelB = sess.run([test_imgBatch, test_labelBatch]) pixelLabels = labelB # print pixelLabels pred = sess.run(softmax, feed_dict={x: imageB}) print pred pixelPred = np.argmax(pred, 1) print pixelPred ct = 0 for pixel in pixelLabels: px = int(pixel[0:3]) py = int(pixel[3:6]) if ((pixelPred[ct] == 0)): outImg[px, py, :] = 0 else: outImg[px, py, 0] = (colors[pixelPred[ct] - 1, 2]) * 255 outImg[px, py, 1] = (colors[pixelPred[ct] - 1, 1]) * 255 outImg[px, py, 2] = (colors[pixelPred[ct] - 1, 0]) * 255 # if(pixelPred[ct]==0): # Background # outImg[px, py, :] = 255 # elif(pixelPred[ct]==1): # Foreground # outImg[px,py,2]=255 # outImg[px, py, 1] = 255 ct = ct + 1 # img = Image.fromarray(outImg) # img.show() cv2.imwrite('output_2.png%s' % (args.testImg), outImg) coord.request_stop() coord.join(threads)