def __init__(self, ckpt_path): checkpoints_dir = ckpt_path self.batch_size = 10 # classify initial inception_v4_arg_scope = inception_utils.inception_arg_scope() self.image_size = 299 arg_scope = inception_utils.inception_arg_scope() number_classes = 1001 self.input_batch = tf.placeholder( tf.float32, shape=[None, self.image_size, self.image_size, 3]) with slim.arg_scope(arg_scope): logits, end_points = inception_v4.inception_v4( self.input_batch, num_classes=number_classes, is_training=False) weights_restored_from_file = slim.get_variables_to_restore( exclude=['InceptionV4/Logits']) init_fn = slim.assign_from_checkpoint_fn( os.path.join(checkpoints_dir, 'inception_v4.ckpt'), weights_restored_from_file, ignore_missing_vars=True) # assign varible self.feature_global = end_points['global_pool'] self.sess = tf.Session() init_fn(self.sess) self.dist = DistanceBuilder()
def __call__(self, x_input): if (self.build): tf.get_variable_scope().reuse_variables() else: self.build = True inception_imags = (x_input / 255.0 - 0.5) * 2 resized_images_vgg = tf.image.resize_images( x_input, [224, 224]) - tf.constant([123.68, 116.78, 103.94]) with slim.arg_scope(vgg.vgg_arg_scope()): logits_vgg16, _ = self.network_fn_vgg16( resized_images_vgg, num_classes=self.num_classes, is_training=False) resized_images_res = ( tf.image.resize_images(x_input, [224, 224]) / 255.0 - 0.5) * 2 with slim.arg_scope(resnet_v2.resnet_arg_scope()): logits_res, _ = self.network_fn_res(resized_images_res, num_classes=self.num_classes + 1, is_training=False) logits_res = tf.reshape(logits_res, (-1, 1001)) logits_res = tf.slice(logits_res, [0, 1], [FLAGS.batch_size, self.num_classes]) with slim.arg_scope(inception_utils.inception_arg_scope()): logits_incepv3, _ = self.network_fn_incepv3( inception_imags, num_classes=self.num_classes + 1, is_training=False) logits_incepv3 = tf.slice(logits_incepv3, [0, 1], [FLAGS.batch_size, self.num_classes]) with slim.arg_scope(inception_utils.inception_arg_scope()): logits_incepv4, _ = self.network_fn_incepv4( inception_imags, num_classes=self.num_classes + 1, is_training=False) logits_incepv4 = tf.slice(logits_incepv4, [0, 1], [FLAGS.batch_size, self.num_classes]) with slim.arg_scope( inception_resnet_v2.inception_resnet_v2_arg_scope()): logits_incep_res, _ = self.network_fn_incep_res( inception_imags, num_classes=self.num_classes + 1, is_training=False) logits_incep_res = tf.slice(logits_incep_res, [0, 1], [FLAGS.batch_size, self.num_classes]) alex_images = tf.image.resize_images(x_input, [256, 256]) alex_images = tf.reverse(alex_images, axis=[-1]) alex_mean_npy = np.load('model/alex_mean.npy').swapaxes(0, 1).swapaxes( 1, 2).astype(np.float32) alex_mean_images = tf.constant(alex_mean_npy) alex_images = alex_images[:, ] - alex_mean_images alex_images = tf.slice(alex_images, [0, 14, 14, 0], [FLAGS.batch_size, 227, 227, 3]) _, logits_alex = self.network_fn_alex(alex_images) logits = [ logits_vgg16, logits_res, logits_incepv3, logits_incepv4, logits_incep_res, logits_alex ] ensemble_logits = tf.reduce_mean(tf.stack(logits), 0) return ensemble_logits
def init(): global g_tf_sess, probabilities, label_dict, input_images subprocess.run(["git", "clone", "https://github.com/tensorflow/models/"]) sys.path.append("./models/research/slim") parser = argparse.ArgumentParser( description="Start a tensorflow model serving") parser.add_argument('--model_name', dest="model_name", required=True) parser.add_argument('--labels_dir', dest="labels_dir", required=True) args, _ = parser.parse_known_args() from nets import inception_v3, inception_utils label_dict = get_class_label_dict(args.labels_dir) classes_num = len(label_dict) tf.disable_v2_behavior() with slim.arg_scope(inception_utils.inception_arg_scope()): input_images = tf.placeholder(tf.float32, [1, image_size, image_size, num_channel]) logits, _ = inception_v3.inception_v3(input_images, num_classes=classes_num, is_training=False) probabilities = tf.argmax(logits, 1) config = tf.ConfigProto() config.gpu_options.allow_growth = True g_tf_sess = tf.Session(config=config) g_tf_sess.run(tf.global_variables_initializer()) g_tf_sess.run(tf.local_variables_initializer()) model_path = Model.get_model_path(args.model_name) saver = tf.train.Saver() saver.restore(g_tf_sess, model_path)
def CreateFrozenGraph(checkpointFile, outputFile): graph = tf.Graph() with tf.Session(graph=graph) as sess: with slim.arg_scope(inception_arg_scope()): images_input = tf.placeholder(tf.float32, [None, None, None, 3], 'input_images') images_processed = tf.image.resize_images( images_input, (ImageNetEmbedder.EMBED_SIZE, ImageNetEmbedder.EMBED_SIZE)) images_processed = tf.subtract( images_processed, 0.5) #Inception net preprocessing images_processed = tf.multiply(images_processed, 2.0) images_processed = tf.identity(images_processed, name="processed_images") _, end_points = inception_resnet_v2.inception_resnet_v2( images_processed, is_training=False, create_aux_logits=False) predictions = tf.identity(end_points['Predictions'], name="predictions") embeddings = tf.identity(end_points['PreLogitsFlatten'], name="embeddings") restorer = tf.train.Saver() restorer.restore(sess, checkpointFile) output_graph_def = tf.graph_util.convert_variables_to_constants( sess, graph.as_graph_def(), ["processed_images", "predictions", "embeddings"]) with tf.gfile.GFile(outputFile, "wb") as f: f.write(output_graph_def.SerializeToString())
def model(images, weight_decay=1e-5, is_training=True): images = mean_image_subtraction(images) with slim.arg_scope(inception_arg_scope(weight_decay=weight_decay)): logits, end_points = inception_resnet_v2(images, num_classes=None, is_training=is_training) for key in end_points.keys(): print(key, end_points[key]) return logits, end_points
def GoogLeNet(num_classes, weight_decay=0.0, is_training=False): func = inception_v1.inception_v1 arg_scope = inception_utils.inception_arg_scope(weight_decay=weight_decay) def network_fn(images, **kwargs): with slim.arg_scope(arg_scope): return inception_v1.inception_v1(images, num_classes, is_training=is_training) if hasattr(func, 'default_image_size'): network_fn.default_image_size = func.default_image_size return network_fn
def classify(checkpoints_dir, images,logo_names=[""], reuse=False): image_size = inception.inception_v4.default_image_size probabilities_list = [] processed_image_list = [] images_list = [] output_probabilities = [] for image in images: image = tf.image.decode_jpeg(image, channels=3) #image = tf.image.resize_image_with_crop_or_pad(image, 299, 299) processed_image = inception_preprocessing.preprocess_image(image, image_size, image_size, is_training=False) #processed_images = tf.expand_dims(processed_image, 0) images_list.append(image) processed_image_list.append(processed_image) with slim.arg_scope(inception_utils.inception_arg_scope()): logits, _ = inception.inception_v4(processed_image_list, #num_classes=2, reuse=reuse, is_training=False, logo_names= logo_names) probabilities = [] output_probabilities = [] for logo_name in logo_names: probabilities.append(tf.nn.softmax(logits[logo_name])) if tf.gfile.IsDirectory(checkpoints_dir): checkpoints_dir = tf.train.latest_checkpoint(checkpoints_dir) init_fn = slim.assign_from_checkpoint_fn( checkpoints_dir, slim.get_model_variables('InceptionV4'), ignore_missing_vars=True) with tf.Session() as sess: init_fn(sess) output_probabilities = sess.run([images_list, processed_image_list] + probabilities)[2:] output_dict = {} print "range(len(output_probabilities): ", range(len(output_probabilities)) for index in range(len(output_probabilities)): print 'logo_names[index]:[{}]'.format(logo_names[index]) print('type logo_names[index]',type(logo_names[index])); if logo_names[index] == "": print "here???????????????????" output_dict [logo_names[index]] = output_probabilities[index] else: output_dict [logo_names[index]] = output_probabilities[index] print "output_dict [""]: ", output_dict [""].shape print "final 0,780 prob output_dict [""]: ", output_dict[""][0][780]; output_dict[""] = np.argsort(output_dict[""], axis=1)[:, ::-1][:, :5] print "final output_dict [""]: ", output_dict [""] return output_dict
def endpoints(image, is_training): if image.get_shape().ndims != 4: raise ValueError('Input must be of size [batch, height, width, 3]') image = image - tf.constant(_RGB_MEAN, dtype=tf.float32, shape=(1,1,1,3)) with tf.contrib.slim.arg_scope(inception_arg_scope(batch_norm_decay=0.9, weight_decay=0.0002)): _, endpoints = inception_v4(image, num_classes=None, is_training=is_training) endpoints['model_output'] = endpoints['global_pool'] = tf.reduce_mean( endpoints['Mixed_7d'], [1, 2], name='pool5', keep_dims=False) return endpoints, 'InceptionV4'
def __init__(self, num_classes): self.num_classes = num_classes with slim.arg_scope( inception_resnet_v2.inception_resnet_v2_arg_scope()): self.network_fn_incep_res = inception_resnet_v2.inception_resnet_v2 with slim.arg_scope(vgg.vgg_arg_scope()): self.network_fn_vgg16 = vgg.vgg_16 with slim.arg_scope(resnet_v2.resnet_arg_scope()): self.network_fn_res = resnet_v2.resnet_v2_152 with slim.arg_scope(inception_utils.inception_arg_scope()): self.network_fn_incepv3 = inception_v3.inception_v3 self.network_fn_incepv4 = inception_v4.inception_v4 self.network_fn_alex = AlexNet() self.build = False
im3 = im[-260:,0:260,:] im4 = im[-260:,-260:,:] im5 = im[19:279,19:279,:] imtemp = [cv2.resize(ims,(input_size[0:2])) for ims in (im1,im2,im3,im4,im5)] [img.append(ims) for ims in imtemp] except: print('Exception found: Image not read...') pass img = np.asarray(img,dtype=np.float32)/255.0 return img # ----- Construct network 1 ----- # with slim.arg_scope(inception_utils.inception_arg_scope()): logits,endpoints = inception_v4(data_in1, num_classes=numclasses1, is_training=is_training, scope='herbarium') herbarium_embs = endpoints['PreLogitsFlatten'] herbarium_bn = tf.layers.batch_normalization(herbarium_embs, training=is_train) herbarium_feat = tf.contrib.layers.fully_connected( inputs=herbarium_bn, num_outputs=500, activation_fn=None, normalizer_fn=None, trainable=True, scope='herbarium' ) herbarium_feat = tf.math.l2_normalize(
def eval(gpu, img_q, res_q, end_eval): ############### # Build graph # ############### with tf.Graph().as_default(): print('loading graph on /gpu:%d' % gpu) with tf.name_scope('tower-%d' % gpu): eval_input = tf.placeholder(tf.float32, [1, height, width, 3]) with slim.arg_scope(inception_arg_scope()): with counter_lock: global counter resue = None if counter == 0 else True counter += 1 logits, _ = inception_v4(eval_input, num_classes, reuse=resue, scope='InceptionV4', is_training=False) predictions = tf.argmax(logits, 1) probabilities = tf.nn.softmax(logits) config = tf.ConfigProto() config.gpu_options.visible_device_list = str(gpu) config.gpu_options.allow_growth = True # config.log_device_placement = True # config.allow_soft_placement = True sess = tf.Session(config=config) init_fn = slim.assign_from_checkpoint_fn( 'checkpoints/inception_v4.ckpt', slim.get_model_variables('InceptionV4')) init_fn(sess) ############## # Eval image # ############## st = time.time() img_cnt = 0 c1, c5 = 0, 0 while not end_eval.is_set(): try: processed_images, label = img_q.get(False) except Q.Empty: continue if fetch_test: img_cnt += 1 sys.stdout.write('\r{:07d}/{:07d}'.format(img_cnt, img_tlt)) sys.stdout.flush() continue pred, prob = sess.run([predictions, probabilities], feed_dict={ eval_input: processed_images }) prob = prob[0, 0:] sorted_inds = [i[0] for i in sorted(enumerate(-prob), key=lambda x:x[1])] top1 = sorted_inds[0] top5 = sorted_inds[0:5] if label == top1: c1 += 1 if label in top5: c5 += 1 idx = img_cnt print('gpu %d images: %d\ttop 1: %0.4f\ttop 5: %0.4f' % (gpu, idx + 1, c1/(idx + 1), c5/(idx + 1))) img_cnt += 1 res_q.put(EvalRes(c1, c5, img_cnt)) print('gpu %d finish in %.4f sec' % (gpu, time.time()-st))
imresize = lambda x, dim: scipy.misc.imresize(x, dim) from PIL import Image from nets.inception_v4 import inception_v4 from nets.inception_utils import inception_arg_scope slim = tf.contrib.slim height, width = 299, 299 num_classes = 1001 c1, c5 = 0, 0 lines = np.loadtxt('imagenet/val.txt', str, delimiter='\n') with tf.Graph().as_default(): eval_inputs = tf.placeholder(tf.float32, [1, height, width, 3]) with slim.arg_scope(inception_arg_scope()): logits, _ = inception_v4(eval_inputs, num_classes, is_training=False) predictions = tf.argmax(logits, 1) probabilities = tf.nn.softmax(logits) init_fn = slim.assign_from_checkpoint_fn( 'checkpoints/inception_v4.ckpt', slim.get_model_variables('InceptionV4')) config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) init_fn(sess) for idx, line in enumerate(lines):
def __init__(self, batch, iterbatch, numclasses, image_dir_parent_train, image_dir_parent_test, train_file, test_file, input_size, checkpoint_model, learning_rate, save_dir, max_iter, val_freq, val_iter): """ Wrapper module for multitask network using inceptionV4 """ self.batch = batch self.iterbatch = iterbatch self.image_dir_parent_train = image_dir_parent_train self.image_dir_parent_test = image_dir_parent_test self.train_file = train_file self.test_file = test_file self.input_size = input_size self.numclasses = numclasses self.checkpoint_model = checkpoint_model self.learning_rate = learning_rate self.save_dir = save_dir self.max_iter = max_iter self.val_freq = val_freq self.val_iter = val_iter logfile = holmeslog() logfile.logging('Initiating database...') # construct datalist and cursor self.train_database = database_module( image_source_dir = self.image_dir_parent_train, database_file = self.train_file, batch = self.batch, input_size = self.input_size, numclasses = self.numclasses, shuffle = True) self.test_database = database_module( image_source_dir = self.image_dir_parent_test, database_file = self.test_file, batch = self.batch, input_size = self.input_size, numclasses = self.numclasses, shuffle = True) logfile.logging('Initiating tensors...') x = tf.placeholder(tf.float32,(self.batch,) + self.input_size) y1 = tf.placeholder(tf.int32,(self.batch,)) y2 = tf.placeholder(tf.int32,(self.batch,)) y3 = tf.placeholder(tf.int32,(self.batch,)) # 10000 classes for species (classid) y_onehot1 = tf.one_hot(y1,self.numclasses) # 248 classes for family y_onehot2 = tf.one_hot(y2,248) # 1780 classes for genus y_onehot3 = tf.one_hot(y3,1780) self.is_training = tf.placeholder(tf.bool) train_preproc = lambda xi: inception_preprocessing.preprocess_image( xi,self.input_size[0],self.input_size[1],is_training=True) def data_in_train(): return tf.map_fn(fn = train_preproc,elems = x,dtype=np.float32) test_preproc = lambda xi: inception_preprocessing.preprocess_image( xi,self.input_size[0],self.input_size[1],is_training=False) def data_in_test(): return tf.map_fn(fn = test_preproc,elems = x,dtype=np.float32) data_in = tf.cond( self.is_training, true_fn = data_in_train, false_fn = data_in_test ) logfile.logging('Constructing network...') with slim.arg_scope(inception_utils.inception_arg_scope()): logits,endpoints = inception_v4(data_in, num_classes=self.numclasses, is_training=self.is_training) logits_family = slim.fully_connected(endpoints['PreLogitsFlatten'],248,activation_fn=None, scope='Family') logits_genus = slim.fully_connected(endpoints['PreLogitsFlatten'],1780,activation_fn=None, scope='Genus') with tf.name_scope("cross_entropy"): with tf.name_scope("auxloss"): self.auxloss = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits_v2( logits=endpoints['AuxLogits'], labels=y_onehot1)) with tf.name_scope("logits_loss"): self.loss = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits_v2( logits=logits, labels=y_onehot1)) with tf.name_scope("logits_loss_family"): self.loss_family = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits_v2( logits=logits_family, labels=y_onehot2)) with tf.name_scope("logits_loss_genus"): self.loss_genus = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits_v2( logits=logits_genus, labels=y_onehot3)) with tf.name_scope("L2_reg_loss"): self.regularization_loss = tf.add_n(tf.losses.get_regularization_losses(scope='InceptionV4')) with tf.name_scope("total_loss"): self.totalloss = self.loss + self.loss_family + self.loss_genus + self.auxloss + self.regularization_loss with tf.name_scope("accuracy"): with tf.name_scope('accuracy_species'): prediction = tf.argmax(logits,1) match = tf.equal(prediction,tf.argmax(y_onehot1,1)) self.accuracy = tf.reduce_mean(tf.cast(match,tf.float32)) with tf.name_scope('accuracy_family'): prediction2 = tf.argmax(logits_family,1) match = tf.equal(prediction2,tf.argmax(y_onehot2,1)) self.accuracy_family = tf.reduce_mean(tf.cast(match,tf.float32)) with tf.name_scope('accuracy_genus'): prediction3 = tf.argmax(logits_genus,1) match = tf.equal(prediction3,tf.argmax(y_onehot3,1)) self.accuracy_genus = tf.reduce_mean(tf.cast(match,tf.float32)) self.var_list = [v for v in tf.trainable_variables()] self.var_list_conv = self.var_list[0:-12] self.var_list_fc = self.var_list[-12:] self.var_list_train = self.var_list self.update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) # only load the conv layer but not the logits self.variables_to_restore = slim.get_variables_to_restore() restore_fn = slim.assign_from_checkpoint_fn( self.checkpoint_model, self.variables_to_restore[0:-12]) with tf.name_scope("train"): # the training involves iter_batch updating loss_accumulator = tf.Variable(0.0, trainable=False) acc_accumulator = tf.Variable(0.0, trainable=False) acc_accumulator_family = tf.Variable(0.0, trainable=False) acc_accumulator_genus = tf.Variable(0.0, trainable=False) self.collect_loss = loss_accumulator.assign_add(self.totalloss) self.collect_acc = acc_accumulator.assign_add(self.accuracy) self.collect_acc_family = acc_accumulator_family.assign_add(self.accuracy_family) self.collect_acc_genus = acc_accumulator_genus.assign_add(self.accuracy_genus) self.average_loss = tf.cond(self.is_training, lambda: loss_accumulator / self.iterbatch, lambda: loss_accumulator / self.val_iter) self.average_acc = tf.cond(self.is_training, lambda: acc_accumulator / self.iterbatch, lambda: acc_accumulator / self.val_iter) self.average_acc_family = tf.cond(self.is_training, lambda: acc_accumulator_family / self.iterbatch, lambda: acc_accumulator_family / self.val_iter) self.average_acc_genus = tf.cond(self.is_training, lambda: acc_accumulator_genus / self.iterbatch, lambda: acc_accumulator_genus / self.val_iter) self.zero_op_loss = tf.assign(loss_accumulator,0.0) self.zero_op_acc = tf.assign(acc_accumulator,0.0) self.zero_op_acc_family = tf.assign(acc_accumulator_family,0.0) self.zero_op_acc_genus = tf.assign(acc_accumulator_genus,0.0) self.accum_train = [tf.Variable(tf.zeros_like( tv.initialized_value()), trainable=False) for tv in self.var_list_train] self.zero_ops_train = [tv.assign(tf.zeros_like(tv)) for tv in self.accum_train] with tf.control_dependencies(self.update_ops): optimizer = tf.train.AdamOptimizer(self.learning_rate) gradient = optimizer.compute_gradients(self.totalloss,self.var_list_train) gradient_only = [gc[0] for gc in gradient] gradient_only,_ = tf.clip_by_global_norm(gradient_only,1.25) self.accum_train_ops = [self.accum_train[i].assign_add(gc) for i,gc in enumerate(gradient_only)] self.train_step = optimizer.apply_gradients( [(self.accum_train[i], gc[1]) for i, gc in enumerate(gradient)]) var_list = tf.trainable_variables() g_list = tf.global_variables() bn_moving_vars = [g for g in g_list if 'moving_mean' in g.name] bn_moving_vars += [g for g in g_list if 'moving_variance' in g.name] var_list += bn_moving_vars saver = tf.train.Saver(var_list=var_list, max_to_keep=0) tf.summary.scalar('loss',self.average_loss) tf.summary.scalar('accuracy',self.average_acc) tf.summary.scalar('accuracy_f',self.average_acc_family) tf.summary.scalar('accuracy_g',self.average_acc_genus) # self.merged = tf.summary.merge_all() self.merged = tf.summary.merge([tf.get_collection(tf.GraphKeys.SUMMARIES,'accuracy'), tf.get_collection(tf.GraphKeys.SUMMARIES,'loss')]) tensorboar_dir = '%s_tensorboard'%current_time() writer_train = tf.summary.FileWriter(tensorboar_dir+'/train') writer_test = tf.summary.FileWriter(tensorboar_dir+'/test') logfile.logging('Commencing training...') val_best = 0.0 with tf.Session() as sess: sess.run(tf.global_variables_initializer()) writer_train.add_graph(sess.graph) writer_test.add_graph(sess.graph) restore_fn(sess) #saver.restore(sess, r"checkpoints_clef3/best.ckpt") for i in range(self.max_iter+1): try: sess.run(self.zero_ops_train) sess.run([self.zero_op_acc_family,self.zero_op_acc_genus,self.zero_op_acc,self.zero_op_loss]) # validations if i % self.val_freq == 0: print('Start:%f'%sess.run(loss_accumulator)) for j in range(self.val_iter): img,lbl1,lbl2,lbl3 = self.test_database.read_batch() sess.run( [self.collect_loss,self.collect_acc,self.collect_acc_family,self.collect_acc_genus], feed_dict = {x : img, y1 : lbl1, y2 : lbl2, y3 : lbl3, self.is_training : False } ) print('[%i]:%f'%(j,sess.run(loss_accumulator))) print('End:%f'%sess.run(loss_accumulator)) s,self.netLoss,self.netAccuracy,self.netAccuracyFamily,self.netAccuracyGenus = sess.run( [self.merged,self.average_loss,self.average_acc,self.average_acc_family,self.average_acc_genus], feed_dict = { self.is_training : False } ) writer_test.add_summary(s, i) logfile.logging('[Valid] Epoch:%i Iter:%i Loss:%f, Accuracy:%f'%(self.train_database.epoch,i,self.netLoss,self.netAccuracy)) logfile.logging('[Valid] Epoch:%i Iter:%i Loss:%f, Accuracy Family:%f'%(self.train_database.epoch,i,self.netLoss,self.netAccuracyFamily)) logfile.logging('[Valid] Epoch:%i Iter:%i Loss:%f, Accuracy Genus:%f'%(self.train_database.epoch,i,self.netLoss,self.netAccuracyGenus)) #print('[Valid] Iter:%i Loss:%f, Accuracy:%f'%(i,self.netLoss,self.netAccuracy)) sess.run([self.zero_op_acc_family,self.zero_op_acc_genus,self.zero_op_acc,self.zero_op_loss]) if self.netAccuracy > val_best: val_best = self.netAccuracy saver.save(sess, os.path.join(self.save_dir,'best.ckpt')) logfile.logging('Model saved') # training for j in range(self.iterbatch): img,lbl1,lbl2,lbl3 = self.train_database.read_batch() sess.run( [self.collect_loss,self.collect_acc,self.collect_acc_family,self.collect_acc_genus,self.accum_train_ops], feed_dict = {x : img, y1 : lbl1, y2 : lbl2, y3 : lbl3, self.is_training : True } ) s,self.netLoss,self.netAccuracy,self.netAccuracyFamily,self.netAccuracyGenus = sess.run( [self.merged,self.average_loss,self.average_acc,self.average_acc_family,self.average_acc_genus], feed_dict = { self.is_training : True } ) writer_train.add_summary(s, i) sess.run( [self.train_step] ) logfile.logging('[Train] Epoch:%i Iter:%i Loss:%f, Accuracy:%f'%(self.train_database.epoch,i,self.netLoss,self.netAccuracy)) logfile.logging('[Train] Epoch:%i Iter:%i Loss:%f, Accuracy Family:%f'%(self.train_database.epoch,i,self.netLoss,self.netAccuracyFamily)) logfile.logging('[Train] Epoch:%i Iter:%i Loss:%f, Accuracy Genus:%f'%(self.train_database.epoch,i,self.netLoss,self.netAccuracyGenus)) #print('[Train] Iter:%i Loss:%f, Accuracy:%f'%(i,self.netLoss,self.netAccuracy)) if i % 5000 == 0: saver.save(sess, os.path.join(self.save_dir,'%06i.ckpt'%i)) except KeyboardInterrupt: logfile.logging('Interrupt detected. Ending...') break saver.save(sess, os.path.join(self.save_dir,'final.ckpt')) logfile.logging('Model saved')
def __init__(self, batch, iterbatch, numclasses1, numclasses2, image_dir_parent_train, image_dir_parent_test, train_file1, train_file2, test_file1, test_file2, input_size, checkpoint_model1, checkpoint_model2, learning_rate, save_dir, max_iter, val_freq, val_iter): self.batch = batch self.iterbatch = iterbatch self.image_dir_parent_train = image_dir_parent_train self.image_dir_parent_test = image_dir_parent_test self.train_file1 = train_file1 self.train_file2 = train_file2 self.test_file1 = test_file1 self.test_file2 = test_file2 self.input_size = input_size self.numclasses1 = numclasses1 self.numclasses2 = numclasses2 self.checkpoint_model1 = checkpoint_model1 self.checkpoint_model2 = checkpoint_model2 self.learning_rate = learning_rate self.save_dir = save_dir self.max_iter = max_iter self.val_freq = val_freq self.val_iter = val_iter # ----- Database module ----- # self.train_database = database_module( image_source_dir = self.image_dir_parent_train, database_file1 = self.train_file1, database_file2 = self.train_file2, batch = self.batch, input_size = self.input_size, numclasses1 = self.numclasses1, numclasses2 = self.numclasses2, shuffle = True) self.test_database = database_module( image_source_dir = self.image_dir_parent_test, database_file1 = self.test_file1, database_file2 = self.test_file2, batch = self.batch, input_size = self.input_size, numclasses1 = self.numclasses1, numclasses2 = self.numclasses2, shuffle = True) # ----- Tensors ------ # print('Initiating tensors...') x1 = tf.placeholder(tf.float32,(None,) + self.input_size) x2 = tf.placeholder(tf.float32,(None,) + self.input_size) herbarium_embs = tf.placeholder(tf.float32) field_embs = tf.placeholder(tf.float32) feat_concat = tf.placeholder(tf.float32, shape=[None, 500]) lbl_concat = tf.placeholder(tf.float32) y1 = tf.placeholder(tf.int32, (None,)) y2 = tf.placeholder(tf.int32, (None,)) self.is_training = tf.placeholder(tf.bool) is_train = tf.placeholder(tf.bool, name="is_training") # ----- Image pre-processing methods ----- # train_preproc = lambda xi: inception_preprocessing.preprocess_image( xi,self.input_size[0],self.input_size[1],is_training=True) test_preproc = lambda xi: inception_preprocessing.preprocess_image( xi,self.input_size[0],self.input_size[1],is_training=False) def data_in_train1(): return tf.map_fn(fn = train_preproc,elems = x1,dtype=np.float32) def data_in_test1(): return tf.map_fn(fn = test_preproc,elems = x1,dtype=np.float32) def data_in_train2(): return tf.map_fn(fn = train_preproc,elems = x2,dtype=np.float32) def data_in_test2(): return tf.map_fn(fn = test_preproc,elems = x2,dtype=np.float32) data_in1 = tf.cond( self.is_training, true_fn = data_in_train1, false_fn = data_in_test1 ) data_in2 = tf.cond( self.is_training, true_fn = data_in_train2, false_fn = data_in_test2 ) print('Constructing network...') # ----- Network 1 construction ----- # with slim.arg_scope(inception_utils.inception_arg_scope()): logits,endpoints = inception_v4(data_in1, num_classes=self.numclasses1, is_training=self.is_training, scope='herbarium' ) herbarium_embs = endpoints['PreLogitsFlatten'] herbarium_bn = tf.layers.batch_normalization(herbarium_embs, training=is_train) herbarium_feat = tf.contrib.layers.fully_connected( inputs=herbarium_bn, num_outputs=500, activation_fn=None, normalizer_fn=None, trainable=True, scope='herbarium' ) herbarium_feat = tf.math.l2_normalize( herbarium_feat, axis=1 ) # ----- Network 2 construction ----- # with slim.arg_scope(inception_utils.inception_arg_scope()): logits2,endpoints2 = inception_v4(data_in2, num_classes=self.numclasses2, is_training=self.is_training, scope='field' ) field_embs = endpoints2['PreLogitsFlatten'] field_bn = tf.layers.batch_normalization(field_embs, training=is_train) field_feat = tf.contrib.layers.fully_connected( inputs=field_bn, num_outputs=500, activation_fn=None, normalizer_fn=None, trainable=True, scope='field' ) field_feat = tf.math.l2_normalize( field_feat, axis=1 ) feat_concat = tf.concat([herbarium_feat, field_feat], 0) lbl_concat = tf.concat([y1, y2], 0) # ----- Get all variables ----- # self.variables_to_restore = tf.trainable_variables() self.variables_bn = [k for k in self.variables_to_restore if k.name.startswith('batch_normalization')] self.variables_herbarium = [k for k in self.variables_to_restore if k.name.startswith('herbarium')] self.variables_field = [k for k in self.variables_to_restore if k.name.startswith('field')] # ----- New variable list ----- # self.var_list_front = self.variables_herbarium[0:-10] + self.variables_field[0:-10] self.var_list_last = self.variables_herbarium[-10:] + self.variables_field[-10:] + self.variables_bn self.var_list_train = self.var_list_front + self.var_list_last # ----- Network losses ----- # with tf.name_scope("loss_calculation"): with tf.name_scope("triplets_loss"): self.triplets_loss = tf.reduce_mean( tf.contrib.losses.metric_learning.triplet_semihard_loss( labels=lbl_concat, embeddings=feat_concat, margin=1.0)) with tf.name_scope("L2_reg_loss"): self.regularization_loss = tf.add_n([ tf.nn.l2_loss(v) for v in self.var_list_train]) * 0.00004 with tf.name_scope("total_loss"): self.totalloss = self.triplets_loss + self.regularization_loss # ----- Create update operation ----- # self.update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) self.vars_ckpt = slim.get_variables_to_restore() vars_ckpt_herbarium = [k for k in self.vars_ckpt if k.name.startswith('herbarium')] vars_ckpt_field = [k for k in self.vars_ckpt if k.name.startswith('field')] # ----- Restore model 1 ----- # restore_fn1 = slim.assign_from_checkpoint_fn( self.checkpoint_model1, vars_ckpt_herbarium[:-2]) # ----- Restore model 2 ----- # restore_fn2 = slim.assign_from_checkpoint_fn( self.checkpoint_model2, vars_ckpt_field[:-2]) # ----- Training scope ----- # with tf.name_scope("train"): loss_accumulator = tf.Variable(0.0, trainable=False) self.collect_loss = loss_accumulator.assign_add(self.totalloss) self.average_loss = tf.cond(self.is_training, lambda: loss_accumulator / self.iterbatch, lambda: loss_accumulator / self.val_iter) self.zero_op_loss = tf.assign(loss_accumulator,0.0) self.accum_train = [tf.Variable(tf.zeros_like( tv.initialized_value()), trainable=False) for tv in self.var_list_train] self.zero_ops_train = [tv.assign(tf.zeros_like(tv)) for tv in self.accum_train] # ----- Set up optimizer / Compute gradients ----- # with tf.control_dependencies(self.update_ops): optimizer = tf.train.AdamOptimizer(self.learning_rate * 0.1) optimizer_lastlayers = tf.train.AdamOptimizer(self.learning_rate) gradient1 = optimizer.compute_gradients(self.totalloss,self.var_list_front) gradient2 = optimizer_lastlayers.compute_gradients(self.totalloss,self.var_list_last) gradient = gradient1 + gradient2 gradient_only = [gc[0] for gc in gradient] gradient_only,_ = tf.clip_by_global_norm(gradient_only,1.25) self.accum_train_ops = [self.accum_train[i].assign_add(gc) for i,gc in enumerate(gradient_only)] # ----- Apply gradients ----- # self.train_step = optimizer.apply_gradients( [(self.accum_train[i], gc[1]) for i, gc in enumerate(gradient)]) # ----- Global variables ----- # var_list = tf.trainable_variables() g_list = tf.global_variables() bn_moving_vars = [g for g in g_list if 'moving_mean' in g.name] bn_moving_vars += [g for g in g_list if 'moving_variance' in g.name] var_list += bn_moving_vars # ----- Create saver ----- # saver = tf.train.Saver(var_list=var_list, max_to_keep=0) tf.summary.scalar('loss',self.average_loss) self.merged = tf.summary.merge([tf.get_collection(tf.GraphKeys.SUMMARIES,'loss')]) # ----- Tensorboard writer--- # writer_train = tf.summary.FileWriter(tensorboard_dir+'/train') writer_test = tf.summary.FileWriter(tensorboard_dir+'/test') print('Commencing training...') # ----- Create session ----- # with tf.Session() as sess: sess.run(tf.global_variables_initializer()) writer_train.add_graph(sess.graph) writer_test.add_graph(sess.graph) restore_fn1(sess) restore_fn2(sess) for i in range(self.max_iter+1): try: sess.run(self.zero_ops_train) sess.run([self.zero_op_loss]) # ----- Validation ----- # if i % self.val_freq == 0: print('Start:%f'%sess.run(loss_accumulator)) for j in range(self.val_iter): img1,img2,lbl1, lbl2 = self.test_database.read_batch() sess.run( self.collect_loss, feed_dict = {x1 : img1, x2 : img2, y1 : lbl1, y2 : lbl2, self.is_training : False, is_train : False } ) print('[%i]:%f'%(j,sess.run(loss_accumulator))) print('End:%f'%sess.run(loss_accumulator)) s,self.netLoss = sess.run( [self.merged,self.average_loss], feed_dict = { self.is_training : False } ) writer_test.add_summary(s, i) print('[Valid] Epoch:%i Iter:%i Loss:%f'%(self.train_database.epoch,i,self.netLoss)) sess.run([self.zero_op_loss]) # ----- Train ----- # for j in range(self.iterbatch): img1,img2,lbl1,lbl2 = self.train_database.read_batch() sess.run( [self.collect_loss,self.accum_train_ops], feed_dict = {x1 : img1, x2 : img2, y1 : lbl1, y2 : lbl2, self.is_training : True, is_train : True } ) s,self.netLoss = sess.run( [self.merged,self.average_loss], feed_dict = { self.is_training : True } ) writer_train.add_summary(s, i) sess.run([self.train_step]) print('[Train] Epoch:%i Iter:%i Loss:%f'%(self.train_database.epoch,i,self.netLoss)) if i % 5000 == 0: saver.save(sess, os.path.join(self.save_dir,'%06i.ckpt'%i)) except KeyboardInterrupt: print('Interrupt detected. Ending...') break # ----- Save model --- # saver.save(sess, os.path.join(self.save_dir,'final.ckpt')) print('Model saved')