def call(self, inputs, **kwargs):
main_input, embedding_matrix = inputs
input_shape_tensor = tf.shape(main_input)
last_input_dim = tf.int_shape(main_input)[-1]
emb_input_dim, emb_output_dim = tf.int_shape(embedding_matrix)
projected = tf.dot(tf.reshape(main_input, (-1, last_input_dim)),
self.projection)
if self.add_biases:
projected = tf.bias_add(projected,
self.biases,
data_format='channels_last')
if 0 < self.projection_dropout < 1:
projected = tf.in_train_phase(
lambda: tf.dropout(projected, self.projection_dropout),
projected,
training=kwargs.get('training'))
attention = tf.dot(projected, tf.transpose(embedding_matrix))
if self.scaled_attention:
# scaled dot-product attention, described in
# "Attention is all you need" (https://arxiv.org/abs/1706.03762)
sqrt_d = tf.constant(math.sqrt(emb_output_dim), dtype=tf.floatx())
attention = attention / sqrt_d
result = tf.reshape(
self.activation(attention),
(input_shape_tensor[0], input_shape_tensor[1], emb_input_dim))
return result
def approximate_log_det(A, m, nv, sess):
# Inputs: A a linear operator, m degree, nv = number of samples
u = tfp.math.random_rademacher([A.shape[0]])
v = u/tf.linalg.norm(u)
T = lanczos_bidiag(A, m, starting_vector=v)
e, v = tf.eigh(T)
e = tf.reshape(e, [-1])
taus = tf.gather(v, [0], axis=1)
taus = tf.reshape(taus, [-1])
Gamma = tf.dot(taus**2, e, axes=[0,0])
return Gamma
def test_imagenet(self):
img = Image.open(test_sofa)
model = Img2Vec(cuda=True)
q_vec = model.extract_vec(img, True)
temp = Image.open(test_beach)
search_vec = model.extract_vec(temp, True)
scores = np.dot(search_vec.T, q_vec)
print(scores)
cos = cosine_similarity(np.reshape(q_vec, (1, -1)),
np.reshape(search_vec, (1, -1)))
print(cos)
def db_retrieval(query, db, db_encoded=None, encoder=None):
if db_encoded is None and encoder is None:
raise ValueError("db_encoded and encoder can't both be None.")
if db_encoded:
scores = tf.dot(tf.nn.l2_normalize(query),
tf.nn.l2_normalize(db_encoded))
best_idx = tf.argmax(scores)
best_score = tf.reduce_max(scores)
return db[best_idx], best_score
else:
db_encoded = encoder.model_fn({"inputs": db}, tf.ModeKeys.PREDICT,
encoder.params)
return db_retrieval_dense(query, db, db_encoded)
def bert_seq_aggerate(repres, input_mask, scope, reuse):
with tf.variable_scope(scope + "/seq_aggerate", reuse=reuse):
attn = tf.get_variable("seq_attention",
dtype=tf.float32,
shape=[
tf.shape(repres)[1],
],
initializer=tf.initializers.random_uniform(
0, 1))
out = tf.dot(repres, attn) # batch x seq
masked_out = attention_utils.mask_logits(out, input_mask)
weight = tf.exp(tf.nn.log_softmax(masked_out, axis=1)) # batch x seq
weight = tf.expand_dims(weight, -1) # batch x seq x 1
seq_repres = tf.reduce_sum(weight * out, axis=1)
return seq_repres
def reflection_matrix(point, normal):
"""Return matrix to mirror at plane defined by point and normal vector."""
dtype = point.dtype
if normal.dtype != point.dtype:
raise ValueError('point and normal must be same dtype')
if dtype not in [tf.float32, tf.float64]:
raise ValueError('dtype not in allowable list [float32, float64]')
normal = unit_vector(normal[..., :3])
dims = point.shape
ndims = dims.ndims
i3 = tf.identity_mnatrix(3)
z3 = tf.zeros((1,) * (ndims-1) + (3, 1), dtype=dtype)
one = tf.ones(dims[:-1].as_list() + [1], dtype=dtype)
M = tf.stack([
tf.stack([-2*outer(normal, normal) + i3, z3], axis=-1),
tf.stack([2 * tf.dot(point[..., :3], normal) * normal, one], axis=-1),
], axis=-1)
return M
def __init__(self, source, input_size, hidden_size, name, act_fn):
self.parent = source
self.source = get_source(source)
self.input_size = input_size
self.hidden_size = hidden_size
self.name = name
self.act_fn = act_fn
# Get weights and bias
self.W = get_weights(self.input_size, self.hidden_size, 'W_' + name)
self.b = get_bias(self.hidden_size, 'b_' + name)
# Calculate output
self.output_pre_activ = T.dot(self.source, self.W) + self.b.dimshuffle(
'x', 0)
# Activate it
self.output = get_activation(self.output_pre_activ,
act_fn=self.act_fn,
name=self.name)
self.params = [self.W, self.b]
def test_place365(self):
scene_model = scene_visual('resnet50', '../weights/places365/{}.pth',
'../weights/places365/categories.txt',
'cuda:0')
for i in ['test1.jpg', 'test2.jpg']:
temp = scene_model.detect(i)
print(temp)
temp = cv2.imread('test1.jpg')
temp = Image.fromarray(cv2.cvtColor(temp, cv2.COLOR_BGR2RGB))
temp = scene_model.detect(temp, True)
print(temp)
# Test vector extraction and cosine similarity
# TODO The accuracy is decreasing when transforming
temp = cv2.imread('test_sofa1.jpg')
q_tensor = Image.fromarray(cv2.cvtColor(temp, cv2.COLOR_BGR2RGB))
q_vec = scene_model.extract_vec(q_tensor, True)
print(type(q_vec))
temp = cv2.imread('test_beach.jpg')
search_tensor = Image.fromarray(cv2.cvtColor(temp, cv2.COLOR_BGR2RGB))
search_vec = scene_model.extract_vec(search_tensor, True)
scores = np.dot(search_vec.T, q_vec)
print(scores)
def igsp(source_positions, source_numbers, target_positions, target_numbers):
translation = tf.reduce_mean(target_positions, axis=0) - tf.reduce_mean(source_positions, axis=0)
source_positions_copy = tf.identity(source_positions)
n_source_atoms = tf.shape(source_positions)[0]
change_in_rotation = 1000
igsp_step = 0
while change_in_rotation > (10 * 3.14159/180) and igsp_step < 20:
euclidean_distance = get_distances(source_positions_copy, target_positions)
feature_distance = 1000 * get_distances(source_numbers, target_numbers)
feature_weight = tf.math.exp(-igsp_step / n_source_atoms)
euclidean_weight = 1 - feature_weight + 0.001
compound_distance = euclidean_weight * euclidean_distance + feature_weight * feature_distance
rows, columns = scipy.optimize.linear_sum_assignment(compound_distance)
ordered_source_positions = tf.gather(source_positions, columns) - tf.reduce_mean(ordered_source_positions, axis=0)
ordered_target_positions = tf.gather(target_positions, rows) - tf.reduce_mean(ordered_target_positions, axis=0)
covariance = tf.dot(ordered_source_positions, tf.transpose(ordered_target_positions))
s, u, v = tf.linalg.svd(covariance)
d = tf.det(v * tf.transpose(u))
temporary_rotation = v * tf.linalg.diag([1,1,d]) * tf.transpose(u)
source_positions_copy = temporary_rotation * source_positions
change_in_rotation = cos(rotation, temporary_rotation)
igsp_step += 1
return rows, columns, translation, rotation
def build_model(self):
"""Builds the model.
Inputs:
self.image_embeddings
self.seq_embeddings
self.target_seqs (training and eval only)
self.input_mask (training and eval only)
Outputs:
self.total_loss (training and eval only)
self.target_cross_entropy_losses (training and eval only)
self.target_cross_entropy_loss_weights (training and eval only)
"""
# This LSTM cell has biases and outputs tanh(new_c) * sigmoid(o), but the
# modified LSTM in the "Show and Tell" paper has no biases and outputs
# new_c * sigmoid(o).
lstm_cell = tf.contrib.rnn.BasicLSTMCell(
num_units=self.config.num_lstm_units, state_is_tuple=True)
if self.mode == "train":
lstm_cell = tf.contrib.rnn.DropoutWrapper(
lstm_cell,
input_keep_prob=self.config.lstm_dropout_keep_prob,
output_keep_prob=self.config.lstm_dropout_keep_prob)
with tf.variable_scope("lstm", initializer=self.initializer) as lstm_scope:
# Feed the image embeddings to set the initial LSTM state.
zero_state = lstm_cell.zero_state(
batch_size=self.image_embeddings.get_shape()[0], dtype=tf.float32)
_, initial_state = lstm_cell(self.image_embeddings, zero_state)
# Allow the LSTM variables to be reused.
lstm_scope.reuse_variables()
if self.mode == "inference":
# In inference mode, use concatenated states for convenient feeding and
# fetching.
# I think we canged that function, bring it back to the original one? TODO
tf.concat(axis=1, values=initial_state, name="initial_state")
# Placeholder for feeding a batch of concatenated states.
state_feed = tf.placeholder(dtype=tf.float32,
shape=[None, sum(lstm_cell.state_size)],
name="state_feed")
state_tuple = tf.split(value=state_feed, num_or_size_splits=2, axis=1)
print("ss shape",state_tuple.shape)
# Run a single LSTM step.
lstm_outputs, state_tuple = lstm_cell(
inputs=tf.squeeze(self.seq_embeddings, axis=[1]),
state=state_tuple)
#Now do the attention mechanism
if self.seq_run > 0 :
score = 0
context = 0
c_score = []
the_n = self.n
if self.allExamined :
the_n = 0
for j in range(self.n, i):
tScore += tf.dot(state_tuple,tf.matmul(self.wa,self.listState[j]))
c_score.append(tf.exp( tScore * listState[i-1]))
score+=tScore
c_score = np.asarray(c_score)
c_score /= score
context = np.sum(c_score)
attention_vector = tf.tanh(tf.matmul(self.wc,tf.concat([context,state_tuple],axis=2)))
lstm_outputs = attention_vector
self.seq_run+=1;
self.listOutput.append(lstm_outputs)
self.listState.append(state_tuple)
# Concatentate the resulting state.
tf.concat(axis=1, values=state_tuple, name="state")
else:
# Run the batch of sequence embeddings through the LSTM.
sequence_length = tf.reduce_sum(self.input_mask, 1)
print(sequence_length)
# Here we will need to call the self.attention_w variable that we created TODO
# We will need to update the attention lists, I think that state_list and output_list need to be replaced with the previous self var TODO
'''for i in range(0, sequence_length):
if i==0:
merged = self.seq_embeddings
LSTMState = initial_state
else:
merged = lstm_outputs
lstm_outputs, LSTMState= lstm_cell(merged, LSTMState) #lstm_outputs will be a sequence of logits? 1 logits for each generated word?
if i > 0:
score = 0
alpha_score = []
for j in range(0, i):
score += tf.dot(lstm_outputs,tf.matmul(wa,output_list[j]))
alpha_score.append(tf.exp(tf.dot(lstm_outputs,tf.matmul(wa,output_list[i-1])))/tf.exp(score))
context = 0
for j in range(0, i):
context += alphaa_score[j]*output_list[j]
attention_vector = tf.tanh(tf.matmul(wc,tf.concat([context,lstm_outputs],axis=2)))
lstm_outputs = attention_vector
state_list.append(LSTMState)
output_list.append(lstm_outputs)'''
print(self.seq_embeddings)
print(initial_state)
print(sequence_length)
sequence_length = tf.Print(sequence_length,["Seq length : ",sequence_length])
#first try to convert the normal one :
self.seq_embeddings = tf.Print(self.seq_embeddings, ["Seq embedding ",tf.shape(self.seq_embeddings)])
lstm_outputs, _ = tf.nn.dynamic_rnn(cell=lstm_cell,
inputs=self.seq_embeddings,
sequence_length=sequence_length,
initial_state=initial_state,
dtype=tf.float32,
scope=lstm_scope)
lstm_outputs = tf.Print(lstm_outputs, ["lstm output original ",tf.shape(lstm_outputs)])
'''i = tf.constant(0,dtype=tf.int32)
max_length = tf.reduce_max(sequence_length)
print(max_length,'testing')
lastStateC = initial_state.c
lastStateH = initial_state.h
lastOutput = tf.constant(0,shape=[32,512],dtype=tf.float32)
stateC_list = tf.TensorArray(size = 1,dynamic_size = True,dtype=tf.float32)
stateH_list = tf.TensorArray(size = 1,dynamic_size = True,dtype=tf.float32)
output_list = tf.TensorArray(size = 1,dynamic_size = True,dtype=tf.float32)
def condition(i,stateC_list,stateH_list,output_list,lastStateC,lastStateH,lastOutput):
return tf.less(i,max_length)
def body(i,stateC_list,stateH_list,output_list,lastStateC,lastStateH,lastOutput):
lastState = tf.nn.rnn_cell.LSTMStateTuple(lastStateC, lastStateH)
def f1():
return self.seq_embeddings[:,i]
def f2():
#return lastOutput
return self.seq_embeddings[:,i]
inputI = tf.cond(tf.less(i, 1), f1, f2)
lastOutput, lastState= lstm_cell(inputI, lastState) #lstm_outputs will be a sequence of logits? 1 logits for each generated word?
lastStateC = lastState.c
lastStateH = lastState.h
stateC_list=stateC_list.write(i,lastState.c)
stateH_list=stateH_list.write(i,lastState.h)
output_list=output_list.write(i,lastOutput)
return [tf.add(i,1),stateC_list,stateH_list,output_list,lastStateC,lastStateH,lastOutput]
lstm_outputs = tf.while_loop(
condition,body,
loop_vars = (i,stateC_list,stateH_list,output_list,lastStateC,lastStateH,lastOutput),
#lastout
)
lstm_outputs = lstm_outputs[3].stack()
lstm_outputs = tf.transpose(lstm_outputs, [1,0,2], name = "result_unwrapped_stats")'''
'''i = tf.constant(0,dtype=tf.int32)
max_length = tf.reduce_max(sequence_length)
print(max_length,'testing')
lastStateC = initial_state.c
lastStateH = initial_state.h
lastOutput = tf.constant(0,shape=[self.config.batch_size,512],dtype=tf.float32)
stateC_list = tf.TensorArray(size = 1,dynamic_size = True,dtype=tf.float32)
stateH_list = tf.TensorArray(size = 1,dynamic_size = True,dtype=tf.float32)
output_list = tf.TensorArray(size = 1,dynamic_size = True,dtype=tf.float32)
def condition(i,stateC_list,stateH_list,output_list,lastStateC,lastStateH,lastOutput):
return tf.less(i,max_length)
def body(i,stateC_list,stateH_list,output_list,lastStateC,lastStateH,lastOutput):
lastState = tf.nn.rnn_cell.LSTMStateTuple(lastStateC, lastStateH)
def f1():
return self.seq_embeddings[:,i]
def f2():
#return lastOutput
return self.seq_embeddings[:,i]
inputI = tf.cond(tf.less(i, 1), f1, f2)
lastOutput, lastState= lstm_cell(inputI, lastState) #lstm_outputs will be a sequence of logits? 1 logits for each generated word?
lastStateC = lastState.c
lastStateH = lastState.h
contextList = tf.TensorArray(size = 1,dynamic_size = True,dtype=tf.float32)
def f11(lastOutput):
return lastOutput
def f22(stateC_list,stateH_list):
#return lastOutput
score = tf.constant(0,dtype=tf.float32)
###this is loop of att weiights, and context
if self.allExamined :
j = tf.constant(0,dtype=tf.int32)
else :
j = tf.constant(tf.substract(i-self.n),dtype=tf.int32)
lim = tf.substract(i,1)
def condition2(j,stateH_list,lastStateH,score,contextList):
return tf.less(j,lim)
def body2(j,stateH_list,lastStateH,score,contextList):
ImScore = tf.exp(tf.dot(lastStateH,tf.matmul(self.wa,stateH_list.read(j))))
curContext = imScore * stateH_list.read(j)
contextList.write(j)
score += imScore
return (tf.add(j,1),stateH_list,lastStateH,score,contextList)
score = tf.while_loop(condition2,body2,loop_vars = (j,stateH_list,lastStateH,score))
#### end loop of score
contextList.stack()
contextList = tf.divide(contextList,score) #normalize by all score
contextList = tf.reduce_sum(contextList) #sum all
#### calcualte attention vector
attention_vector = tf.tanh(tf.matmul(self.wc,tf.concat([context,lstm_outputs],axis=2)))
#### this is ht end of loop of context
return attention_vector
lastOutput = tf.cond(tf.less(i, 1), f11(lastOutput), f22(stateC_list,stateH_list))
stateC_list=stateC_list.write(i,lastState.c)
stateH_list=stateH_list.write(i,lastState.h)
output_list=output_list.write(i,lastOutput)
return [tf.add(i,1),stateC_list,stateH_list,output_list,lastStateC,lastStateH,lastOutput]
lstm_outputs = tf.while_loop(
condition,body,
loop_vars = (i,stateC_list,stateH_list,output_list,lastStateC,lastStateH,lastOutput),
#lastout
)
lstm_outputs = lstm_outputs[3].stack()
lstm_outputs = tf.transpose(lstm_outputs, [1,0,2], name = "result_unwrapped_stats")'''
print("output",lstm_outputs)
lstm_outputs = tf.Print(lstm_outputs, ["lstm output alter ",tf.shape(lstm_outputs)])
# Stack batches vertically.
lstm_outputs = tf.reshape(lstm_outputs, [-1, lstm_cell.output_size])
#modify the inference result
with tf.variable_scope("logits") as logits_scope:
logits = tf.contrib.layers.fully_connected(
inputs=lstm_outputs,
num_outputs=self.config.vocab_size,
activation_fn=None,
weights_initializer=self.initializer,
scope=logits_scope)
# TODO do text simplification
if self.mode == "inference":
tf.nn.softmax(logits, name="softmax")
else:
targets = tf.reshape(self.target_seqs, [-1])
weights = tf.to_float(tf.reshape(self.input_mask, [-1]))
# Compute losses.
losses = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=targets,
logits=logits)
batch_loss = tf.div(tf.reduce_sum(tf.multiply(losses, weights)),
tf.reduce_sum(weights),
name="batch_loss")
tf.losses.add_loss(batch_loss)
total_loss = tf.losses.get_total_loss()
# Add summaries.
tf.summary.scalar("losses/batch_loss", batch_loss)
tf.summary.scalar("losses/total_loss", total_loss)
for var in tf.trainable_variables():
tf.summary.histogram("parameters/" + var.op.name, var)
self.total_loss = total_loss
self.target_cross_entropy_losses = losses # Used in evaluation.
self.target_cross_entropy_loss_weights = weights # Used in evaluation.
import tensorflow as tf
import numpy as np
x= np.random.randn(4,5,20)
input = tf.constant(x)
x = tf.cast(input,'float32')
con = tf.get_variable("weight",[20, 10])
z=tf.dot(x,con)
# z=tf.nn.conv2d(tf.cast(input,'float32'),con,strides=[1,1,1,1],padding="VALID")
sess=tf.Session()
sess.run(tf.global_variables_initializer())
output = sess.run(z)
print(output.shape)
def score(hDst, hSrc): return tf.reduce_sum(hDst * hSrc, 0) return tf.dot(hDst, hSrc)
def inner_cca_objective(y_true, y_pred):
"""
It is the loss function of CCA as introduced in the original paper. There can be other formulations.
It is implemented by Theano tensor operations, and does not work on Tensorflow backend
y_true is just ignored
"""
r1 = 1e-4
r2 = 1e-4
eps = 1e-12
o1 = o2 = y_pred.shape[1] // 2
# unpack (separate) the output of networks for view 1 and view 2
H1 = tf.transpose(y_pred[:, 0:o1])
H2 = tf.transpose(y_pred[:, o1:o1 + o2])
m = H1.shape[1]
H1bar = H1 - (tf.math.divide(1, m)) * tf.dot(H1, tf.ones([m, m]))
H2bar = H2 - (tf.math.divide(1, m)) * tf.dot(H2, tf.ones([m, m]))
SigmaHat12 = (tf.math.divide(1, m-1)) * \
tf.dot(H1bar, tf.transpose(H2bar))
SigmaHat11 = (tf.math.divide(1, m - 1)) * tf.dot(
H1bar, tf.transpose(H1bar)) + r1 * tf.eye(o1)
SigmaHat22 = (tf.math.divide(1, m - 1)) * tf.dot(
H2bar, tf.transpose(H2bar)) + r2 * tf.eye(o2)
# Calculating the root inverse of covariance matrices by using eigen decomposition
[D1, V1] = tf.nlinalg.eigh(SigmaHat11)
[D2, V2] = tf.nlinalg.eigh(SigmaHat22)
# Added to increase stability
posInd1 = tf.gt(D1, eps).nonzero()[0]
D1 = D1[posInd1]
V1 = V1[:, posInd1]
posInd2 = tf.gt(D2, eps).nonzero()[0]
D2 = D2[posInd2]
V2 = V2[:, posInd2]
SigmaHat11RootInv = tf.dot(tf.dot(V1, tf.nlinalg.diag(D1**-0.5)),
tf.transpose(V1))
SigmaHat22RootInv = tf.dot(tf.dot(V2, tf.nlinalg.diag(D2**-0.5)),
tf.transpose(V2))
Tval = tf.dot(tf.dot(SigmaHat11RootInv, SigmaHat12), SigmaHat22RootInv)
if use_all_singular_values:
# all singular values are used to calculate the correlation
corr = tf.sqrt(tf.nlinalg.trace(tf.dot(tf.transpose(Tval), Tval)))
else:
# just the top outdim_size singular values are used
[U, V] = tf.nlinalg.eigh(T.dot(tf.transpose(Tval), Tval))
U = U[tf.gt(U, eps).nonzero()[0]]
U = U.sort()
corr = tf.sum(tf.sqrt(U[0:outdim_size]))
return -corr