def inference(image, keep_prob):
"""
Semantic segmentation network definition
:param image: input image. Should have values in range 0-255
:param keep_prob:
:return:
"""
print("setting up inception_v3 initialized conv layers ...")
with tf.variable_scope("inference"):
net, end_points = inception_v3(image, NUM_OF_CLASSESS, True, keep_prob)
# now to upscale to actual image size
with tf.variable_scope('Upsampling'):
with slim.arg_scope([slim.conv2d_transpose],
stride=2,
padding='SAME'):
up_sampling = end_points['']
deconv_shape1 = image_net["pool4"].get_shape()
W_t1 = utils.weight_variable(
[4, 4, deconv_shape1[3].value, NUM_OF_CLASSESS], name="W_t1")
b_t1 = utils.bias_variable([deconv_shape1[3].value], name="b_t1")
conv_t1 = utils.conv2d_transpose_strided(conv8,
W_t1,
b_t1,
output_shape=tf.shape(
image_net["pool4"]))
fuse_1 = tf.add(conv_t1, image_net["pool4"], name="fuse_1")
deconv_shape2 = image_net["pool3"].get_shape()
W_t2 = utils.weight_variable(
[4, 4, deconv_shape2[3].value, deconv_shape1[3].value],
name="W_t2")
b_t2 = utils.bias_variable([deconv_shape2[3].value], name="b_t2")
conv_t2 = utils.conv2d_transpose_strided(fuse_1,
W_t2,
b_t2,
output_shape=tf.shape(
image_net["pool3"]))
fuse_2 = tf.add(conv_t2, image_net["pool3"], name="fuse_2")
shape = tf.shape(image)
deconv_shape3 = tf.stack(
[shape[0], shape[1], shape[2], NUM_OF_CLASSESS])
W_t3 = utils.weight_variable(
[16, 16, NUM_OF_CLASSESS, deconv_shape2[3].value], name="W_t3")
b_t3 = utils.bias_variable([NUM_OF_CLASSESS], name="b_t3")
conv_t3 = utils.conv2d_transpose_strided(fuse_2,
W_t3,
b_t3,
output_shape=deconv_shape3,
stride=8)
annotation_pred = tf.argmax(conv_t3, dimension=3, name="prediction")
return tf.expand_dims(annotation_pred, dim=3), conv_t3
def add_model(self, images, is_training):
with tf.contrib.framework.arg_scope(inception_v3_arg_scope()):
logits, end_points = inception_v3(images,
num_classes=200,
is_training=is_training)
'''
feats = end_points['AvgPool_1a']
feats_flat = tf.squeeze(feats)
fc_weights = tf.get_variable('fc-weights', [2048, 200], initializer=tf.random_normal_initializer())
logits = tf.matmul(feats_flat, fc_weights, name='fc')
'''
return logits, end_points
def getCNNModle(self,typeCNN):
model = None
if typeCNN =="alexnet":
model = models.alexnet()
elif typeCNN =="densenet":
model = models.densenet161()
elif typeCNN =="inception":
model = models.inception_v3()
elif typeCNN =="resnet":
model = models.resnet18()
elif typeCNN =="vgg":
model = models.vgg16()
return model
def add_model(self, images, is_training):
# get predicated theta values
tf.summary.image("original", images, self.batch_size, collections=None)
theta = self.localizer.localize(images, is_training)
# penalize the difference between transforms and identity
penalty = tf.reduce_sum(tf.square(theta), axis=1)
theta1, theta2 = tf.split(theta, num_or_size_splits=2, axis=1)
theta1, theta2 = tf.reshape(theta1, [-1, 2, 1]), tf.reshape(theta2, [-1, 2, 1])
tf.summary.histogram('histogram_theta1', theta1)
tf.summary.histogram('histogram_theta2', theta2)
tf.summary.scalar('max_theta1', tf.reduce_max(theta1))
tf.summary.scalar('max_theta2', tf.reduce_max(theta2))
tf.summary.scalar('min_theta1', tf.reduce_min(theta1))
tf.summary.scalar('min_theta2', tf.reduce_min(theta2))
# clamp between 0 and 1
#theta1 = tf.div(tf.subtract(theta1, tf.reduce_min(theta1)), tf.subtract(tf.reduce_max(theta1), tf.reduce_min(theta1)))
#theta2 = tf.div(tf.subtract(theta2, tf.reduce_min(theta2)), tf.subtract(tf.reduce_max(theta2), tf.reduce_min(theta2)))
# add the fixed size scale transform parameters
theta_scale = tf.eye(2, batch_shape=[self.batch_size]) * 0.5
theta1 = tf.concat([theta_scale, theta1], axis=2)
theta2 = tf.concat([theta_scale, theta2], axis=2)
# flatten thetas for transform
theta1, theta2 = tf.reshape(theta1, [self.batch_size, 6]), tf.reshape(theta2, [self.batch_size, 6])
transform1 = transform(images, theta1, out_size=(224, 224))
transform2 = transform(images, theta2, out_size=(224, 224))
tf.summary.image('transform1', transform1, self.batch_size, collections=None)
tf.summary.image('transform2', transform2, self.batch_size, collections=None)
# extract features
with tf.contrib.framework.arg_scope(inception_v3_arg_scope()):
_, end_points1 = inception_v3(transform1, num_classes=None, is_training=is_training)
_, end_points2 = inception_v3(transform2, num_classes=None, is_training=is_training, reuse=True)
# TODO: check if this should be changed
features1 = tf.squeeze(end_points1['AvgPool_1a'], axis=[1,2], name='feats1')
features2 = tf.squeeze(end_points2['AvgPool_1a'], axis=[1,2], name='feats2')
features = tf.concat([features1, features2], axis=1)
dropout = tf.nn.dropout(features, 0.7)
with tf.variable_scope('final_out'):
logits = tf.layers.dense(dropout, 200, name='feats2out')
return logits, penalty
def add_model(self, images, is_training):
# get predicated theta values
tf.summary.image("original", images, self.batch_size, collections=None)
# contains n*2 params for n transforms
theta = self.localizer.localize(images, is_training)
# print theta's over time
self.theta = theta
theta_list = tf.split(theta, num_or_size_splits=self.num_crops, axis=1)
transform_list = []
# transform the images using theta
for i in range(len(theta_list)):
theta_i = tf.reshape(theta_list[i], [-1, 2, 1])
tf.summary.histogram('histogram_theta_' + str(i), theta_list[i])
# add the fixed size scale transform parameters
theta_scale = tf.eye(2, batch_shape=[self.batch_size]) * 0.5
theta_i = tf.concat([theta_scale, theta_i], axis=2)
# flatten thetas for transform
theta_i = tf.reshape(theta_i, [self.batch_size, 6])
transform_i = transform(images, theta_i, out_size=(224, 224))
transform_list.append(transform_i)
tf.summary.image('transform_' + str(i),
transform_i,
self.batch_size,
collections=None)
# extract features
features_list = []
aux_logits_list = []
with tf.variable_scope('classifier'):
with tf.contrib.framework.arg_scope(inception_v3_arg_scope()):
for i in range(len(transform_list)):
reuse = True if i > 0 else False
transform_i = transform_list[i]
_, end_points_i = inception_v3(
transform_i,
num_classes=self.num_classes,
is_training=is_training,
reuse=reuse)
# TODO: check if this should be changed to something other than AbgPool_1a
aux_logits_i = end_points_i['AuxLogits']
aux_logits_list.append(aux_logits_i)
features_i = tf.squeeze(end_points_i['AvgPool_1a'],
axis=[1, 2],
name='feats' + str(i))
features_list.append(features_i)
features = tf.concat(features_list, axis=1)
dropout = tf.nn.dropout(features, 0.7)
with tf.variable_scope('final_out'):
logits = tf.layers.dense(dropout,
self.num_classes,
name='feats2out')
return logits, aux_logits_list
def load_model(self):
sess = tf.Session()
input_tensor = tf.placeholder(tf.float32, [None, 299, 299, 3])
arg_scope = inception_v3_arg_scope()
with slim.arg_scope(arg_scope):
logits, end_points = inception_v3(
input_tensor, is_training=False, num_classes=1001)
saver = tf.train.Saver()
# params_file = tf.train.latest_checkpoint(self.model_dir)
saver.restore(sess, self.checkpoint_file)
self.output['sess'] = sess
self.output['input_tensor'] = input_tensor
self.output['logits'] = logits
self.output['end_points'] = end_points
def load_model(self):
sess = tf.Session()
input_tensor = tf.placeholder(tf.float32, [None, 299, 299, 3])
arg_scope = inception_v3_arg_scope()
with slim.arg_scope(arg_scope):
logits, end_points = inception_v3(
input_tensor, is_training=False, num_classes=1001)
saver = tf.train.Saver()
# params_file = tf.train.latest_checkpoint(self.model_dir)
saver.restore(sess, self.checkpoint_file)
self.output['sess'] = sess
self.output['input_tensor'] = input_tensor
self.output['logits'] = logits
self.output['end_points'] = end_points
def localize(self, x, is_training):
"""Creates an inception localization network with added FC final layer to estimate transform params
Args:
x: A tensor of shape (batch_size, n_features)
Returns:
h_fc_loc2: A tensor of shape (batch_size, theta_dimxnum_keys)
"""
# TODO: this is going to cause a problem when finding variables
print('localize x:', x.get_shape())
with tf.variable_scope('localize', reuse=self.reuse):
with tf.contrib.framework.arg_scope(inception_v3_arg_scope()):
logits, end_points = inception_v3(x,
num_classes=None,
is_training=is_training)
print('end points:', end_points.keys())
with tf.variable_scope('added_layers'):
# feats with dim 8x8x2048b
inception_features = end_points['Mixed_7c']
print('localize feats:', inception_features.get_shape())
# TODO: may not need to initalize all to 0
# 1 x 1 conv with 128 out channels
conv = tf.layers.conv2d(inputs=inception_features, filters=128, kernel_size=[1, 1], padding="same", \
activation=tf.nn.relu, name='added_1x1conv')
print('localize conv:', conv.get_shape())
# FC layer with 128 dim output (6x6x128 -> 128)
conv_flat = tf.reshape(conv, [self.batch_size, 6 * 6 * 128])
print('conv flat:', conv_flat.get_shape())
fc = tf.layers.dense(inputs=conv_flat,
units=128,
activation=tf.nn.relu,
name='added_fc')
print('localize fc:', fc.get_shape())
# top left (-1, -1), top right (1, -1), bottom left (-1, -1), bottom right(1, 1)
# set bias term to tile the image
if self.num_keys == 2:
value = [-0.5, 0.0, 0.5, 0.0] # middle of image
elif self.num_keys == 4:
value = [-0.5, -0.5, 0.5, -0.5, -0.5, 0.5, 0.5,
0.5] # four corners of image
init = tf.constant_initializer(value)
# FC layer that outputs theta params
out = tf.layers.dense(inputs=fc, units=self.theta_dim*self.num_keys, bias_initializer=init, \
kernel_initializer=tf.zeros_initializer(), activation=tf.nn.tanh, name='added_out')
print('localize out:', out.get_shape())
print(out)
return out
def inference(image):
checkpoint_dir = '../pretrain_model'
checkpoint_file = '../pretrain_model/inception_v3.ckpt'
input_tensor = tf.placeholder(tf.float32, [None, 299, 299, 3])
sess = tf.Session()
arg_scope = inception_v3_arg_scope()
with slim.arg_scope(arg_scope):
logits, end_points = inception_v3(
input_tensor, is_training=False, num_classes=1001)
saver = tf.train.Saver()
# ckpt = tf.train.get_checkpoint_state(checkpoint_dir)
saver.restore(sess, checkpoint_file)
im = Image.open(image).resize((299, 299))
im = np.array(im) / 255.0
im = im.reshape(-1, 299, 299, 3)
start = time.time()
predict_values, logit_values = sess.run(
[end_points['Predictions'], logits], feed_dict={input_tensor: im})
print 'a image take time {0}'.format(time.time() - start)
return image, predict_values
def inference(image):
checkpoint_dir = './train_log'
input_tensor = tf.placeholder(tf.float32, [None, 299, 299, 3])
sess = tf.Session()
arg_scope = inception_v3_arg_scope()
with slim.arg_scope(arg_scope):
logits, end_points = inception_v3(
input_tensor, is_training=False, num_classes=8)
saver = tf.train.Saver()
ckpt = tf.train.get_checkpoint_state(checkpoint_dir)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
im = Image.open(image).resize((299, 299))
im = np.array(im) / 255.0
im = im.reshape(-1, 299, 299, 3)
start = time.time()
predict_values, logit_values = sess.run(
[end_points['Predictions'], logits], feed_dict={input_tensor: im})
print 'a image take time {0}'.format(time.time() - start)
return image, predict_values
def compute_feature_of_batch_ts_with_cnn(file_path_of_ts, file_path_of_feature,
cnn_model_name,
file_path_of_pretrained_model):
r'''
compute feature of somme time series with pretrained CNN
:param file_path_of_ts: file path of time series
:param file_path_of_feature: file path of saving feature
:param cnn_model_name: name of CNN model
:param file_path_of_pretrained_model: file path of pretrained CNN
:return: ''
'''
#tf.reset_default_graph()
#read data
data = pd.read_csv(file_path_of_ts)
#data=data.sample(20)
#change dataframe to list
id_list = data.iloc[:, 0].tolist()
data_list = change_dataframe_to_dict_(data)
model = cnn_model_name
checkpoint_file = file_path_of_pretrained_model
# I only have these because I thought some take in size of (299,299), but maybe not
if 'inception' in model: height, width, channels = 224, 224, 3
if 'resnet' in model: height, width, channels = 224, 224, 3
if 'vgg' in model: height, width, channels = 224, 224, 3
if model == 'inception_resnet_v2': height, width, channels = 299, 299, 3
x = tf.placeholder(tf.float32, shape=(1, height, width, channels))
# load up model specific stuff
if model == 'inception_v1':
#from inception_v1 import *
from nets import inception_v1
arg_scope = inception_v1.inception_v1_arg_scope()
with slim.arg_scope(arg_scope):
logits, end_points = inception_v1.inception_v1(x,
is_training=False,
num_classes=None)
features = end_points['AvgPool_0a_7x7']
# print('logits')
# print(logits.shape)
# print('features')
# print(features.shape)
elif model == 'inception_v2':
#from inception_v2 import *
from nets import inception_v2
arg_scope = inception_v2.inception_v2_arg_scope()
with slim.arg_scope(arg_scope):
logits, end_points = inception_v2(x,
is_training=False,
num_classes=None)
features = end_points['AvgPool_1a']
elif model == 'inception_v3':
#from inception_v3 import *
from nets import inception_v3
arg_scope = inception_v3.inception_v3_arg_scope()
with slim.arg_scope(arg_scope):
logits, end_points = inception_v3(x,
is_training=False,
num_classes=None)
features = end_points['AvgPool_1a']
elif model == 'inception_resnet_v2':
#from inception_resnet_v2 import *
from nets import inception_resnet_v2
arg_scope = inception_resnet_v2.inception_resnet_v2_arg_scope()
with slim.arg_scope(arg_scope):
logits, end_points = inception_resnet_v2(x,
is_training=False,
num_classes=1001)
features = end_points['PreLogitsFlatten']
elif model == 'resnet_v1_50':
#from resnet_v1 import *
from nets import resnet_v1
arg_scope = resnet_v1.resnet_arg_scope()
with slim.arg_scope(arg_scope):
logits, end_points = resnet_v1.resnet_v1_50(x,
is_training=False,
num_classes=1000)
features = end_points['global_pool']
elif model == 'resnet_v1_101':
#from resnet_v1 import *
from nets import resnet_v1
arg_scope = resnet_v1.resnet_arg_scope()
with slim.arg_scope(arg_scope):
logits, end_points = resnet_v1.resnet_v1_101(x,
is_training=False,
num_classes=1000)
features = end_points['global_pool']
elif model == 'vgg_16':
#from vgg import *
from nets import vgg
arg_scope = vgg.vgg_arg_scope()
with slim.arg_scope(arg_scope):
logits, end_points = vgg.vgg_16(x, is_training=False)
features = end_points['vgg_16/fc8']
elif model == 'vgg_19':
#from vgg import *
from nets import vgg
arg_scope = vgg.vgg_arg_scope()
with slim.arg_scope(arg_scope):
logits, end_points = vgg.vgg_19(x, is_training=False)
features = end_points['vgg_19/fc8']
#cpu_config = tf.ConfigProto(intra_op_parallelism_threads = 8, inter_op_parallelism_threads = 8, device_count = {'CPU': 3})
#sess = tf.Session(config = cpu_config)
sess = tf.Session()
saver = tf.train.Saver()
saver.restore(sess, checkpoint_file)
feature_list = []
count_temp = 0
for i in range(len(data_list)):
count_temp = count_temp + 1
#imaging ts
ts_dict = data_list[i]
ts = ts_dict['ts']
id = ts_dict['id']
new_ts = min_max_transform(ts)
normalized = np.array(new_ts)
fig, ax = plt.subplots()
#plt.imshow(recurrence_plot.rec_plot(normalized), cmap=plt.cm.gray)
plt.imshow(recurrence_plot.rec_plot(normalized))
ax.set_xticks([])
ax.set_yticks([])
#print(id)
path = "inception-v1/" + id + ".jpg"
plt.savefig(path)
plt.close(fig)
#compute feature
# #begin to compute features
image = misc.imread(path)
#from matplotlib.pyplot import imread
#image=imread(path)
# print('image')
# print(image.size)
image = misc.imresize(image, (height, width))
image = np.expand_dims(image, 0)
feature = np.squeeze(sess.run(features, feed_dict={x: image}))
feature_list.append(feature)
# print('feature-test')
# print(feature)
os.remove(path)
if count_temp % 100 == 0:
print(count_temp)
#begin to process parellel result and write_to_csv
feature_array = np.array(feature_list)
feature_df = pd.DataFrame(feature_array)
# print(feature_df.shape)
# print(len(id_list))
#add id
feature_df.insert(loc=0, column='id', value=id_list)
# print(feature_final_df.shape)
# print(feature_final_df.head())
feature_df.to_csv(file_path_of_feature, index=False)
gc.collect()
