def inference(images,
train_mode,
pruned_vgg=False,
pooling_type="max",
activation_function="relu"):
"""Build the model up to where it may be used for inference.
Args:
images: Images placeholder, from inputs().
Returns:
softmax_linear: Output tensor with the computed logits.
"""
import vgg
v = vgg.Vgg19(int(images.shape[2]),
int(images.shape[1]),
activation_function=activation_function)
if pruned_vgg:
return v.build_pruned_vgg(images)
return v.build(images, train_mode, pooling_type=pooling_type)
def __init__(self, opt):
self.vocab_size = opt.vocab_size
self.input_encoding_size = opt.input_encoding_size
self.rnn_size = opt.rnn_size
self.num_layers = opt.num_layers
self.drop_prob_lm = opt.drop_prob_lm
self.seq_length = opt.seq_length
self.vocab_size = opt.vocab_size
self.seq_per_img = opt.seq_per_img
self.att_hid_size = opt.att_hid_size
self.opt = opt
# Variable indicating in training mode or evaluation mode
self.training = tf.Variable(True, trainable=False, name="training")
# Input variables
self.images = tf.placeholder(tf.float32, [None, 224, 224, 3],
name="images")
self.labels = tf.placeholder(tf.int32, [None, self.seq_length + 2],
name="labels")
self.masks = tf.placeholder(tf.float32, [None, self.seq_length + 2],
name="masks")
# Build CNN
if vars(self.opt).get('start_from', None):
cnn_weight = None
else:
cnn_weight = vars(self.opt).get('cnn_weight', None)
if self.opt.cnn_model == 'vgg16':
self.cnn = vgg.Vgg16(cnn_weight)
if self.opt.cnn_model == 'vgg19':
self.cnn = vgg.Vgg19(cnn_weight)
with tf.variable_scope("cnn"):
self.cnn.build(self.images)
if self.opt.cnn_model == 'vgg16':
self.context = self.cnn.conv5_3
if self.opt.cnn_model == 'vgg19':
self.context = self.cnn.conv5_4
self.fc7 = self.cnn.drop7
self.cnn_training = self.cnn.training
# Variable in language model
with tf.variable_scope("rnnlm"):
# Word Embedding table
self.Wemb = tf.Variable(tf.random_uniform(
[self.vocab_size + 1, self.input_encoding_size], -0.1, 0.1),
name='Wemb')
# RNN cell
if opt.rnn_type == 'rnn':
self.cell_fn = cell_fn = tf.contrib.rnn.BasicRNNCell
elif opt.rnn_type == 'gru':
self.cell_fn = cell_fn = tf.contrib.rnn.GRUCell
elif opt.rnn_type == 'lstm':
self.cell_fn = cell_fn = tf.contrib.rnn.LSTMCell
else:
raise Exception("RNN type not supported: {}".format(
opt.rnn_type))
# keep_prob is a function of training flag
self.keep_prob = tf.cond(
self.training,
lambda: tf.constant(1 - self.drop_prob_lm),
lambda: tf.constant(1.0),
name='keep_prob')
# basic cell has dropout wrapper
self.basic_cell = cell = tf.contrib.rnn.DropoutWrapper(
cell_fn(self.rnn_size), 1.0, self.keep_prob)
# cell is the final cell of each timestep
self.cell = tf.contrib.rnn.MultiRNNCell([cell] * opt.num_layers)
"""A simplified 2D convolution operation"""
x = tf.expand_dims(tf.expand_dims(x, 0), -1)
print x.shape
y = tf.nn.depthwise_conv2d(x[0, 0], k, [1, 1, 1, 1], padding='SAME')
return y[0, :, :, 0]
def laplace(x):
"""Compute the 2D laplacian of an array"""
laplace_k = make_kernel([[0.5, 1.0, 0.5], [1.0, -6., 1.0], [0.5, 1.0,
0.5]])
return simple_conv(x, laplace_k)
# initializing the VGG network from pre trained downloaded Data
vgg = vgg.Vgg19()
# getting input images and resizing the style image to content image
content_image = np.asarray(PIL.Image.open("./34.jpg"), dtype=float)
img_width = content_image.shape[0]
img_height = content_image.shape[1]
style_image = np.asarray(PIL.Image.open("./33.jpg"))
style_image = tf.image.resize_images(style_image, size=[img_width, img_height])
b = np.zeros(shape=[1, img_width, img_height, 3])
b[0] = content_image
input_var = tf.clip_by_value(tf.Variable(b, trainable=True, dtype=tf.float32),
0.0, 255.0)
# now building the pre trained vgg model graph for style transfer
vgg.build(input_var)
config = tf.ConfigProto()
GPUID = 0
os.environ["CUDA_VISIBLE_DEVICES"] = str(GPUID)
# Parameters
batch_size = 128
epochs = 80
lr = 1e-3
images = tf.placeholder(tf.float32, [batch_size, 224, 224, 3])
audios = tf.placeholder(tf.float32, [batch_size, 1024, 40])
true_out = tf.placeholder(tf.float32, [batch_size, 1000])
train_mode = tf.placeholder(tf.bool)
'''extract image and audio features'''
# VGG = vgg.Vgg19('./tensorflow-vgg/vgg19.npy')
VGG = vgg.Vgg19()
VGG.build(images, train_mode)
imageFeatures = VGG.conv6
audioFeatures = AudioFeature(audios)
''' design loss function (Rank Loss) '''
# S_p
Similarity_matrix = tf.einsum('eabc,ecd->eabd', imageFeatures, tf.transpose(audioFeatures,[0,2,1]))
Similarity_p = tf.reduce_mean(Similarity_matrix, [1,2,3])
#S_j and S_c
Similarity_j = tf.einsum('eabc,ecd->eabd', imageFeatures[::-1], tf.transpose(audioFeatures,[0,2,1]))
Similarity_j = tf.reduce_mean(Similarity_j, [1,2,3])
Similarity_c = tf.einsum('eabc,ecd->eabd', imageFeatures, tf.transpose(audioFeatures[::-1],[0,2,1]))
Similarity_c = tf.reduce_mean(Similarity_c, [1,2,3])
obj_loss = tf.reduce_mean(tf.maximum(Similarity_c - Similarity_p + 1, 0) \
GPUID = 0
os.environ["CUDA_VISIBLE_DEVICES"] = str(GPUID)
# Parameters
batch_size = 64
epochs = 80
lr = 1e-4
images = tf.placeholder(tf.float32, [batch_size, 224, 224, 3])
audios = tf.placeholder(tf.float32, [batch_size, 1024, 40])
true_out = tf.placeholder(tf.float32, [batch_size, 1000])
train_mode = tf.placeholder(tf.bool)
'''extract image and audio features'''
VGG = vgg.Vgg19('./tensorflow-vgg/vgg19.npy')
# VGG = vgg.Vgg19()
VGG.build(images, train_mode)
imageFeatures = VGG.conv6
audioFeatures = AudioFeature(audios)
''' design loss function (Rank Loss) '''
# S_p
Similarity_matrix = tf.einsum('eabc,ecd->eabd', imageFeatures, tf.transpose(audioFeatures,[0,2,1]))
Similarity_p = tf.reduce_mean(tf.reduce_max(Similarity_matrix, [1,2]), 1)
# S_j
Similarity_j = tf.einsum('eabc,ecd->eabd', imageFeatures[::-1], tf.transpose(audioFeatures,[0,2,1]))
Similarity_j = tf.reduce_mean(tf.reduce_max(Similarity_j, [1, 2]), 1)
# S_c
Similarity_c = tf.einsum('eabc,ecd->eabd', imageFeatures, tf.transpose(audioFeatures[::-1],[0,2,1]))
Similarity_c = tf.reduce_mean(tf.reduce_max(Similarity_c, [1, 2]), 1)
def __init__(self, opt):
self.vocab_size = opt.vocab_size
self.input_encoding_size = opt.input_encoding_size
self.rnn_size = opt.rnn_size
self.num_layers = opt.num_layers
self.drop_prob_lm = opt.drop_prob_lm
self.seq_length = opt.seq_length
self.vocab_size = opt.vocab_size
self.seq_per_img = opt.seq_per_img
#self.batch_size = opt.batch_size
self.opt = opt
# Variable indicating in training mode or evaluation mode
self.training = tf.Variable(True, trainable=False, name="training")
# Input varaibles
self.images = tf.placeholder(tf.float32, [None, 224, 224, 3],
name="images")
self.labels = tf.placeholder(tf.int32, [None, self.seq_length + 2],
name="labels")
self.masks = tf.placeholder(tf.float32, [None, self.seq_length + 2],
name="masks")
# VGG 16
if self.opt.start_from is not None:
cnn_weight = None
else:
cnn_weight = self.opt.cnn_weight
if self.opt.cnn_model == 'vgg16':
self.cnn = vgg.Vgg16(cnn_weight)
if self.opt.cnn_model == 'vgg19':
self.cnn = vgg.Vgg19(cnn_weight)
with tf.variable_scope("cnn"):
self.cnn.build(self.images)
self.fc7 = self.cnn.drop7
self.cnn_training = self.cnn.training
"""
# Old model loading
with open(self.opt.cnn_model) as f:
fileContent = f.read()
graph_def = tf.GraphDef()
graph_def.ParseFromString(fileContent)
tf.import_graph_def(graph_def, input_map={"images": self.images}, name='vgg16')
self.vgg16 = tf.get_default_graph()
self.fc7 = self.vgg16.get_tensor_by_name("vgg16/Relu_1:0")
"""
# Variable in language model
with tf.variable_scope("rnnlm"):
# Word Embedding table
#with tf.device("/cpu:0"):
self.Wemb = tf.Variable(tf.random_uniform(
[self.vocab_size + 1, self.input_encoding_size], -0.1, 0.1),
name='Wemb')
#
self.embed_word_W = tf.Variable(tf.random_uniform(
[self.rnn_size, self.vocab_size + 1], -0.1, 0.1),
name='embed_word_W')
self.embed_word_b = self.init_bias(self.vocab_size + 1,
name='embed_word_b')
# RNN cell
if opt.rnn_type == 'rnn':
self.cell_fn = cell_fn = tf.nn.rnn_cell.BasicRNNCell
elif opt.rnn_type == 'gru':
self.cell_fn = cell_fn = tf.nn.rnn_cell.GRUCell
elif opt.rnn_type == 'lstm':
self.cell_fn = cell_fn = tf.nn.rnn_cell.LSTMCell
else:
raise Exception("RNN type not supported: {}".format(
opt.rnn_type))
self.keep_prob = tf.cond(
self.training,
lambda: tf.constant(1 - self.drop_prob_lm),
lambda: tf.constant(1.0),
name='keep_prob')
self.basic_cell = cell = tf.nn.rnn_cell.DropoutWrapper(
cell_fn(self.rnn_size, state_is_tuple=True), 1.0,
self.keep_prob)
self.cell = tf.nn.rnn_cell.MultiRNNCell([cell] * opt.num_layers,
state_is_tuple=True)
def __init__(self, opt):
self.opt = opt
self.vocab_size = opt.vocab_size
self.input_encoding_size = opt.input_encoding_size
self.drop_prob_lm = opt.drop_prob_lm
self.seq_length = opt.seq_length
self.rnn_size = opt.rnn_size
self.seq_per_img = opt.seq_per_img
self.batch_size = opt.batch_size
self.seq_per_img = opt.seq_per_img
self.att_size = opt.att_size
self.num_boxes = opt.num_boxes
# Variable indicating in training mode or evaluation mode
self.training = tf.Variable(True, trainable=False, name="training")
self.num_layers = 1
self.cap_iter = opt.cap_iter
self.b = tf.placeholder(tf.float32, [None, self.num_boxes, 1, 1])
if self.opt.cnn_model == "frcnn":
print "using frcnn feature"
self.cnn_dim = 2048
self.images = tf.placeholder(tf.float32,
[None, self.num_boxes, self.cnn_dim],
name="features")
self.context = self.images
elif self.opt.cnn_model == 'vgg16' or self.opt.cnn_model == 'vgg19':
print "using cnn model"
self.images = tf.placeholder(tf.float32, [None, 224, 224, 3],
name="images")
cnn_weight = vars(self.opt).get('cnn_weight', None)
if self.opt.cnn_model == 'vgg16':
import vgg
self.cnn = vgg.Vgg16(cnn_weight)
self.cnn_dim = 512
elif self.opt.cnn_model == 'vgg19':
import vgg
self.cnn = vgg.Vgg19(cnn_weight)
self.cnn_dim = 512
with tf.variable_scope("cnn"):
self.cnn.build(self.images)
if self.opt.cnn_model == 'vgg16':
self.context = self.cnn.conv5_3
elif self.opt.cnn_model == 'vgg19':
self.context = self.cnn.conv5_4
self.context = tf.reshape(self.context,
[-1, self.num_boxes, self.cnn_dim])
elif self.opt.cnn_model == 'resnet':
print "using resnet feature"
self.cnn_dim = 2048
self.images = tf.placeholder(tf.float32,
[None, self.num_boxes, self.cnn_dim],
name="features")
self.context = self.images
self.labels = tf.placeholder(tf.int32, [None, self.seq_length + 2],
name="labels")
self.masks = tf.placeholder(tf.float32, [None, self.seq_length + 2],
name="masks")
with tf.variable_scope("rnnlm"):
#l2_norm
self.features = tf.nn.l2_normalize(self.context, axis=-1)
#self.att_feat = slim.fully_connected(self.features, self.att_size,activation_fn=None, scope='att_feature_proj')
self.avgFeat = tf.reduce_mean(self.features,
axis=1,
keep_dims=False)
# Word Embedding table
self.Wemb = tf.Variable(tf.random_uniform(
[self.vocab_size, self.input_encoding_size], -0.1, 0.1),
name='Wemb')
# RNN cell
if opt.rnn_type == 'rnn':
self.cell_fn = cell_fn = tf.contrib.rnn.BasicRNNCell
elif opt.rnn_type == 'gru':
self.cell_fn = cell_fn = tf.contrib.rnn.GRUCell
elif opt.rnn_type == 'lstm':
self.cell_fn = cell_fn = tf.contrib.rnn.LSTMCell
else:
raise Exception("RNN type not supported: {}".format(
opt.rnn_type))
self.keep_prob = tf.cond(
self.training,
lambda: tf.constant(1 - self.drop_prob_lm),
lambda: tf.constant(1.0),
name='keep_prob')
# basic cell has dropout wrapper
self.basic_cell1 = cell1 = tf.contrib.rnn.DropoutWrapper(
cell_fn(self.rnn_size), 1.0, self.keep_prob)
self.basic_cell2 = cell2 = tf.contrib.rnn.DropoutWrapper(
cell_fn(self.rnn_size), 1.0, self.keep_prob)
# cell is the final cell of each timestep
self.cell1 = tf.contrib.rnn.MultiRNNCell([cell1] * self.num_layers)
self.cell2 = tf.contrib.rnn.MultiRNNCell([cell2] * self.num_layers)
default=0.5)
args = parser.parse_args()
# prepare input images
content_image = load_image(args.content_image_path,
scale=float(args.content_scale))
WIDTH, HEIGHT = content_image.shape[1], content_image.shape[0]
content_image = content_image.reshape((1, HEIGHT, WIDTH, 3))
style_image = load_image(args.style_image_path, (WIDTH, HEIGHT))
style_image = style_image.reshape((1, HEIGHT, WIDTH, 3))
# prepare networks
images = np.concatenate((content_image, style_image), 0).astype(np.float32)
constants = tf.constant(images)
with tf.name_scope("constant"):
vgg_const = vgg.Vgg19()
vgg_const.build(constants)
# use noise as an initial image
#input_image = tf.Variable(tf.truncated_normal([1, HEIGHT, WIDTH, 3], 0.5, 0.1))
# use content image as an initial image
input_image = tf.Variable(np.expand_dims(images[0, :, :, :], 0))
with tf.name_scope("variable"):
vgg_var = vgg.Vgg19()
vgg_var.build(input_image)
# which layers we want to use?
style_layers_const = [
vgg_const.conv1_1, vgg_const.conv2_1, vgg_const.conv3_1, vgg_const.conv4_1,
vgg_const.conv5_1
]
# prepare input images
content_image = load_image(args.content_image_path,
scale=float(args.content_scale))
WIDTH, HEIGHT = content_image.shape[1], content_image.shape[0]
content_image = content_image.reshape((1, HEIGHT, WIDTH, 3))
style_image = load_image(args.style_image_path, (WIDTH, HEIGHT))
style_image = style_image.reshape((1, HEIGHT, WIDTH, 3))
print("step0")
# prepare networks
images = np.concatenate((content_image, style_image), 0).astype(np.float32)
constants = tf.constant(images)
print(constants)
with tf.name_scope("constant"):
print("step0.1")
vgg_const = vgg.Vgg19("./vgg19.npy")
print("step0.2")
vgg_const.build(constants)
print("step1")
# use noise as an initial image
#input_image = tf.Variable(tf.truncated_normal([1, HEIGHT, WIDTH, 3], 0.5, 0.1))
# use content image as an initial image
input_image = tf.Variable(np.expand_dims(images[0, :, :, :], 0))
with tf.name_scope("variable"):
vgg_var = vgg.Vgg19("./vgg19.npy")
vgg_var.build(input_image)
print("step2")