def inference(image, keep_prob): """ Semantic segmentation network definition :param image: input image. Should have values in range 0-255 :param keep_prob: :return: """ print("setting up inception_v3 initialized conv layers ...") with tf.variable_scope("inference"): net, end_points = inception_v3(image, NUM_OF_CLASSESS, True, keep_prob) # now to upscale to actual image size with tf.variable_scope('Upsampling'): with slim.arg_scope([slim.conv2d_transpose], stride=2, padding='SAME'): up_sampling = end_points[''] deconv_shape1 = image_net["pool4"].get_shape() W_t1 = utils.weight_variable( [4, 4, deconv_shape1[3].value, NUM_OF_CLASSESS], name="W_t1") b_t1 = utils.bias_variable([deconv_shape1[3].value], name="b_t1") conv_t1 = utils.conv2d_transpose_strided(conv8, W_t1, b_t1, output_shape=tf.shape( image_net["pool4"])) fuse_1 = tf.add(conv_t1, image_net["pool4"], name="fuse_1") deconv_shape2 = image_net["pool3"].get_shape() W_t2 = utils.weight_variable( [4, 4, deconv_shape2[3].value, deconv_shape1[3].value], name="W_t2") b_t2 = utils.bias_variable([deconv_shape2[3].value], name="b_t2") conv_t2 = utils.conv2d_transpose_strided(fuse_1, W_t2, b_t2, output_shape=tf.shape( image_net["pool3"])) fuse_2 = tf.add(conv_t2, image_net["pool3"], name="fuse_2") shape = tf.shape(image) deconv_shape3 = tf.stack( [shape[0], shape[1], shape[2], NUM_OF_CLASSESS]) W_t3 = utils.weight_variable( [16, 16, NUM_OF_CLASSESS, deconv_shape2[3].value], name="W_t3") b_t3 = utils.bias_variable([NUM_OF_CLASSESS], name="b_t3") conv_t3 = utils.conv2d_transpose_strided(fuse_2, W_t3, b_t3, output_shape=deconv_shape3, stride=8) annotation_pred = tf.argmax(conv_t3, dimension=3, name="prediction") return tf.expand_dims(annotation_pred, dim=3), conv_t3
def add_model(self, images, is_training): with tf.contrib.framework.arg_scope(inception_v3_arg_scope()): logits, end_points = inception_v3(images, num_classes=200, is_training=is_training) ''' feats = end_points['AvgPool_1a'] feats_flat = tf.squeeze(feats) fc_weights = tf.get_variable('fc-weights', [2048, 200], initializer=tf.random_normal_initializer()) logits = tf.matmul(feats_flat, fc_weights, name='fc') ''' return logits, end_points
def getCNNModle(self,typeCNN): model = None if typeCNN =="alexnet": model = models.alexnet() elif typeCNN =="densenet": model = models.densenet161() elif typeCNN =="inception": model = models.inception_v3() elif typeCNN =="resnet": model = models.resnet18() elif typeCNN =="vgg": model = models.vgg16() return model
def add_model(self, images, is_training): # get predicated theta values tf.summary.image("original", images, self.batch_size, collections=None) theta = self.localizer.localize(images, is_training) # penalize the difference between transforms and identity penalty = tf.reduce_sum(tf.square(theta), axis=1) theta1, theta2 = tf.split(theta, num_or_size_splits=2, axis=1) theta1, theta2 = tf.reshape(theta1, [-1, 2, 1]), tf.reshape(theta2, [-1, 2, 1]) tf.summary.histogram('histogram_theta1', theta1) tf.summary.histogram('histogram_theta2', theta2) tf.summary.scalar('max_theta1', tf.reduce_max(theta1)) tf.summary.scalar('max_theta2', tf.reduce_max(theta2)) tf.summary.scalar('min_theta1', tf.reduce_min(theta1)) tf.summary.scalar('min_theta2', tf.reduce_min(theta2)) # clamp between 0 and 1 #theta1 = tf.div(tf.subtract(theta1, tf.reduce_min(theta1)), tf.subtract(tf.reduce_max(theta1), tf.reduce_min(theta1))) #theta2 = tf.div(tf.subtract(theta2, tf.reduce_min(theta2)), tf.subtract(tf.reduce_max(theta2), tf.reduce_min(theta2))) # add the fixed size scale transform parameters theta_scale = tf.eye(2, batch_shape=[self.batch_size]) * 0.5 theta1 = tf.concat([theta_scale, theta1], axis=2) theta2 = tf.concat([theta_scale, theta2], axis=2) # flatten thetas for transform theta1, theta2 = tf.reshape(theta1, [self.batch_size, 6]), tf.reshape(theta2, [self.batch_size, 6]) transform1 = transform(images, theta1, out_size=(224, 224)) transform2 = transform(images, theta2, out_size=(224, 224)) tf.summary.image('transform1', transform1, self.batch_size, collections=None) tf.summary.image('transform2', transform2, self.batch_size, collections=None) # extract features with tf.contrib.framework.arg_scope(inception_v3_arg_scope()): _, end_points1 = inception_v3(transform1, num_classes=None, is_training=is_training) _, end_points2 = inception_v3(transform2, num_classes=None, is_training=is_training, reuse=True) # TODO: check if this should be changed features1 = tf.squeeze(end_points1['AvgPool_1a'], axis=[1,2], name='feats1') features2 = tf.squeeze(end_points2['AvgPool_1a'], axis=[1,2], name='feats2') features = tf.concat([features1, features2], axis=1) dropout = tf.nn.dropout(features, 0.7) with tf.variable_scope('final_out'): logits = tf.layers.dense(dropout, 200, name='feats2out') return logits, penalty
def add_model(self, images, is_training): # get predicated theta values tf.summary.image("original", images, self.batch_size, collections=None) # contains n*2 params for n transforms theta = self.localizer.localize(images, is_training) # print theta's over time self.theta = theta theta_list = tf.split(theta, num_or_size_splits=self.num_crops, axis=1) transform_list = [] # transform the images using theta for i in range(len(theta_list)): theta_i = tf.reshape(theta_list[i], [-1, 2, 1]) tf.summary.histogram('histogram_theta_' + str(i), theta_list[i]) # add the fixed size scale transform parameters theta_scale = tf.eye(2, batch_shape=[self.batch_size]) * 0.5 theta_i = tf.concat([theta_scale, theta_i], axis=2) # flatten thetas for transform theta_i = tf.reshape(theta_i, [self.batch_size, 6]) transform_i = transform(images, theta_i, out_size=(224, 224)) transform_list.append(transform_i) tf.summary.image('transform_' + str(i), transform_i, self.batch_size, collections=None) # extract features features_list = [] aux_logits_list = [] with tf.variable_scope('classifier'): with tf.contrib.framework.arg_scope(inception_v3_arg_scope()): for i in range(len(transform_list)): reuse = True if i > 0 else False transform_i = transform_list[i] _, end_points_i = inception_v3( transform_i, num_classes=self.num_classes, is_training=is_training, reuse=reuse) # TODO: check if this should be changed to something other than AbgPool_1a aux_logits_i = end_points_i['AuxLogits'] aux_logits_list.append(aux_logits_i) features_i = tf.squeeze(end_points_i['AvgPool_1a'], axis=[1, 2], name='feats' + str(i)) features_list.append(features_i) features = tf.concat(features_list, axis=1) dropout = tf.nn.dropout(features, 0.7) with tf.variable_scope('final_out'): logits = tf.layers.dense(dropout, self.num_classes, name='feats2out') return logits, aux_logits_list
def load_model(self): sess = tf.Session() input_tensor = tf.placeholder(tf.float32, [None, 299, 299, 3]) arg_scope = inception_v3_arg_scope() with slim.arg_scope(arg_scope): logits, end_points = inception_v3( input_tensor, is_training=False, num_classes=1001) saver = tf.train.Saver() # params_file = tf.train.latest_checkpoint(self.model_dir) saver.restore(sess, self.checkpoint_file) self.output['sess'] = sess self.output['input_tensor'] = input_tensor self.output['logits'] = logits self.output['end_points'] = end_points
def localize(self, x, is_training): """Creates an inception localization network with added FC final layer to estimate transform params Args: x: A tensor of shape (batch_size, n_features) Returns: h_fc_loc2: A tensor of shape (batch_size, theta_dimxnum_keys) """ # TODO: this is going to cause a problem when finding variables print('localize x:', x.get_shape()) with tf.variable_scope('localize', reuse=self.reuse): with tf.contrib.framework.arg_scope(inception_v3_arg_scope()): logits, end_points = inception_v3(x, num_classes=None, is_training=is_training) print('end points:', end_points.keys()) with tf.variable_scope('added_layers'): # feats with dim 8x8x2048b inception_features = end_points['Mixed_7c'] print('localize feats:', inception_features.get_shape()) # TODO: may not need to initalize all to 0 # 1 x 1 conv with 128 out channels conv = tf.layers.conv2d(inputs=inception_features, filters=128, kernel_size=[1, 1], padding="same", \ activation=tf.nn.relu, name='added_1x1conv') print('localize conv:', conv.get_shape()) # FC layer with 128 dim output (6x6x128 -> 128) conv_flat = tf.reshape(conv, [self.batch_size, 6 * 6 * 128]) print('conv flat:', conv_flat.get_shape()) fc = tf.layers.dense(inputs=conv_flat, units=128, activation=tf.nn.relu, name='added_fc') print('localize fc:', fc.get_shape()) # top left (-1, -1), top right (1, -1), bottom left (-1, -1), bottom right(1, 1) # set bias term to tile the image if self.num_keys == 2: value = [-0.5, 0.0, 0.5, 0.0] # middle of image elif self.num_keys == 4: value = [-0.5, -0.5, 0.5, -0.5, -0.5, 0.5, 0.5, 0.5] # four corners of image init = tf.constant_initializer(value) # FC layer that outputs theta params out = tf.layers.dense(inputs=fc, units=self.theta_dim*self.num_keys, bias_initializer=init, \ kernel_initializer=tf.zeros_initializer(), activation=tf.nn.tanh, name='added_out') print('localize out:', out.get_shape()) print(out) return out
def inference(image): checkpoint_dir = '../pretrain_model' checkpoint_file = '../pretrain_model/inception_v3.ckpt' input_tensor = tf.placeholder(tf.float32, [None, 299, 299, 3]) sess = tf.Session() arg_scope = inception_v3_arg_scope() with slim.arg_scope(arg_scope): logits, end_points = inception_v3( input_tensor, is_training=False, num_classes=1001) saver = tf.train.Saver() # ckpt = tf.train.get_checkpoint_state(checkpoint_dir) saver.restore(sess, checkpoint_file) im = Image.open(image).resize((299, 299)) im = np.array(im) / 255.0 im = im.reshape(-1, 299, 299, 3) start = time.time() predict_values, logit_values = sess.run( [end_points['Predictions'], logits], feed_dict={input_tensor: im}) print 'a image take time {0}'.format(time.time() - start) return image, predict_values
def inference(image): checkpoint_dir = './train_log' input_tensor = tf.placeholder(tf.float32, [None, 299, 299, 3]) sess = tf.Session() arg_scope = inception_v3_arg_scope() with slim.arg_scope(arg_scope): logits, end_points = inception_v3( input_tensor, is_training=False, num_classes=8) saver = tf.train.Saver() ckpt = tf.train.get_checkpoint_state(checkpoint_dir) if ckpt and ckpt.model_checkpoint_path: saver.restore(sess, ckpt.model_checkpoint_path) im = Image.open(image).resize((299, 299)) im = np.array(im) / 255.0 im = im.reshape(-1, 299, 299, 3) start = time.time() predict_values, logit_values = sess.run( [end_points['Predictions'], logits], feed_dict={input_tensor: im}) print 'a image take time {0}'.format(time.time() - start) return image, predict_values
def compute_feature_of_batch_ts_with_cnn(file_path_of_ts, file_path_of_feature, cnn_model_name, file_path_of_pretrained_model): r''' compute feature of somme time series with pretrained CNN :param file_path_of_ts: file path of time series :param file_path_of_feature: file path of saving feature :param cnn_model_name: name of CNN model :param file_path_of_pretrained_model: file path of pretrained CNN :return: '' ''' #tf.reset_default_graph() #read data data = pd.read_csv(file_path_of_ts) #data=data.sample(20) #change dataframe to list id_list = data.iloc[:, 0].tolist() data_list = change_dataframe_to_dict_(data) model = cnn_model_name checkpoint_file = file_path_of_pretrained_model # I only have these because I thought some take in size of (299,299), but maybe not if 'inception' in model: height, width, channels = 224, 224, 3 if 'resnet' in model: height, width, channels = 224, 224, 3 if 'vgg' in model: height, width, channels = 224, 224, 3 if model == 'inception_resnet_v2': height, width, channels = 299, 299, 3 x = tf.placeholder(tf.float32, shape=(1, height, width, channels)) # load up model specific stuff if model == 'inception_v1': #from inception_v1 import * from nets import inception_v1 arg_scope = inception_v1.inception_v1_arg_scope() with slim.arg_scope(arg_scope): logits, end_points = inception_v1.inception_v1(x, is_training=False, num_classes=None) features = end_points['AvgPool_0a_7x7'] # print('logits') # print(logits.shape) # print('features') # print(features.shape) elif model == 'inception_v2': #from inception_v2 import * from nets import inception_v2 arg_scope = inception_v2.inception_v2_arg_scope() with slim.arg_scope(arg_scope): logits, end_points = inception_v2(x, is_training=False, num_classes=None) features = end_points['AvgPool_1a'] elif model == 'inception_v3': #from inception_v3 import * from nets import inception_v3 arg_scope = inception_v3.inception_v3_arg_scope() with slim.arg_scope(arg_scope): logits, end_points = inception_v3(x, is_training=False, num_classes=None) features = end_points['AvgPool_1a'] elif model == 'inception_resnet_v2': #from inception_resnet_v2 import * from nets import inception_resnet_v2 arg_scope = inception_resnet_v2.inception_resnet_v2_arg_scope() with slim.arg_scope(arg_scope): logits, end_points = inception_resnet_v2(x, is_training=False, num_classes=1001) features = end_points['PreLogitsFlatten'] elif model == 'resnet_v1_50': #from resnet_v1 import * from nets import resnet_v1 arg_scope = resnet_v1.resnet_arg_scope() with slim.arg_scope(arg_scope): logits, end_points = resnet_v1.resnet_v1_50(x, is_training=False, num_classes=1000) features = end_points['global_pool'] elif model == 'resnet_v1_101': #from resnet_v1 import * from nets import resnet_v1 arg_scope = resnet_v1.resnet_arg_scope() with slim.arg_scope(arg_scope): logits, end_points = resnet_v1.resnet_v1_101(x, is_training=False, num_classes=1000) features = end_points['global_pool'] elif model == 'vgg_16': #from vgg import * from nets import vgg arg_scope = vgg.vgg_arg_scope() with slim.arg_scope(arg_scope): logits, end_points = vgg.vgg_16(x, is_training=False) features = end_points['vgg_16/fc8'] elif model == 'vgg_19': #from vgg import * from nets import vgg arg_scope = vgg.vgg_arg_scope() with slim.arg_scope(arg_scope): logits, end_points = vgg.vgg_19(x, is_training=False) features = end_points['vgg_19/fc8'] #cpu_config = tf.ConfigProto(intra_op_parallelism_threads = 8, inter_op_parallelism_threads = 8, device_count = {'CPU': 3}) #sess = tf.Session(config = cpu_config) sess = tf.Session() saver = tf.train.Saver() saver.restore(sess, checkpoint_file) feature_list = [] count_temp = 0 for i in range(len(data_list)): count_temp = count_temp + 1 #imaging ts ts_dict = data_list[i] ts = ts_dict['ts'] id = ts_dict['id'] new_ts = min_max_transform(ts) normalized = np.array(new_ts) fig, ax = plt.subplots() #plt.imshow(recurrence_plot.rec_plot(normalized), cmap=plt.cm.gray) plt.imshow(recurrence_plot.rec_plot(normalized)) ax.set_xticks([]) ax.set_yticks([]) #print(id) path = "inception-v1/" + id + ".jpg" plt.savefig(path) plt.close(fig) #compute feature # #begin to compute features image = misc.imread(path) #from matplotlib.pyplot import imread #image=imread(path) # print('image') # print(image.size) image = misc.imresize(image, (height, width)) image = np.expand_dims(image, 0) feature = np.squeeze(sess.run(features, feed_dict={x: image})) feature_list.append(feature) # print('feature-test') # print(feature) os.remove(path) if count_temp % 100 == 0: print(count_temp) #begin to process parellel result and write_to_csv feature_array = np.array(feature_list) feature_df = pd.DataFrame(feature_array) # print(feature_df.shape) # print(len(id_list)) #add id feature_df.insert(loc=0, column='id', value=id_list) # print(feature_final_df.shape) # print(feature_final_df.head()) feature_df.to_csv(file_path_of_feature, index=False) gc.collect()