def gather_rows(self, needed_row_ids):
"""Gather needed rows from the sparsetensor"""
sp_trans = tf.sparse_transpose(self.sparse_tensor)
sp_trans_gather = gather_columns(sp_trans, needed_row_ids)
support = tf.sparse_transpose(sp_trans_gather)
return support
def _call(self, inputs):
if self.sparse_inputs:
num_users = inputs[0].dense_shape[0]
num_items = inputs[1].dense_shape[0]
users = tf.sparse_to_dense(inputs[0].indices,
inputs[0].dense_shape, inputs[0].values)
items = tf.sparse_to_dense(inputs[1].indices,
inputs[1].dense_shape, inputs[1].values)
else:
num_users = inputs[0].shape[0]
num_items = inputs[1].shape[0]
users = inputs[0]
items = inputs[1]
original_x = tf.concat(
[users, items], axis=0
) # CHECK THIS! need to combine users and items into one single array. becomes 6000 (users+items) x 6000 (input_dim)
original_x = tf.nn.dropout(original_x, 1 - self.dropout)
outputs = []
for i in range(len(self.E_start)):
# E_start, E_end : E x V
x = original_x
# conv1
Uix = dot(x, self.Ui1[i])
Ujx = dot(x, self.Uj1[i])
x2 = dot(self.E_start[i], Ujx, sparse=True)
x = tf.add(
Uix, dot(tf.sparse_transpose(self.E_end[i]), x2, sparse=True))
x = tf.nn.bias_add(x, self.bu1[i])
x = tf.layers.batch_normalization(x)
x = tf.nn.relu(x)
outputs.append(x)
output = tf.concat(axis=1, values=outputs)
outputs = []
for i in range(len(self.E_start)):
x = output
# conv2
Uix = dot(x, self.Ui2[i])
Ujx = dot(x, self.Uj2[i])
x2 = dot(self.E_start[i], Ujx, sparse=True)
x = tf.add(
Uix, dot(tf.sparse_transpose(self.E_end[i]), x2, sparse=True))
x = tf.nn.bias_add(x, self.bu1[i])
x = tf.layers.batch_normalization(x)
outputs.append(x)
output = tf.concat(axis=1, values=outputs)
output = tf.add(output, tf.matmul(original_x, self.R))
output = tf.nn.relu(output)
u = output[:tf.cast(num_users, tf.int32)]
v = output[tf.cast(num_users, tf.int32):]
return u, v
def _gmn(self, g1, g2):
if logging_enabled == True:
print("- Entered layers::GraphConvolution::_gmn Private Method")
g1x = g1[0]
g1_edge_index = self._to_dense(tf.sparse_transpose(g1[3]))
incidence_mat1 = tf.cast(g1[4], tf.float32)
g2x = g2[0]
g2_edge_index = self._to_dense(tf.sparse_transpose(g2[3]))
incidence_mat2 = tf.cast(g2[4], tf.float32)
if self.sparse_inputs:
g1x = self._to_dense(g1x)
g2x = self._to_dense(g2x)
# tf.concat([tf.gather(g1_edge_index, tf.constant(0)), tf.gather(g2_edge_index, tf.constant(0))], 1)
row1 = tf.gather(g1_edge_index, tf.constant(0))
# tf.concat([tf.gather(g1_edge_index, tf.constant(1)), tf.gather(g2_edge_index, tf.constant(1))], 1)
col1 = tf.gather(g1_edge_index, tf.constant(1))
row2 = tf.gather(g2_edge_index, tf.constant(0))
col2 = tf.gather(g2_edge_index, tf.constant(1))
h1_i = tf.gather(g1x, row1)
h1_j = tf.gather(g1x, col1)
h2_i = tf.gather(g2x, row2)
h2_j = tf.gather(g2x, col2)
m1 = tf.concat([h1_i, h1_j], 1)
m2 = tf.concat([h2_i, h2_j], 1)
for l in self.f_message:
m1 = l(m1)
m2 = l(m2)
m1_sum = dot(incidence_mat1, m1, sparse=True)
m2_sum = dot(incidence_mat2, m2, sparse=True)
u1_sum = self._gmn_f_match(g1x, g2x)
u2_sum = self._gmn_f_match(g2x, g1x)
h1_prime = tf.concat([g1x, m1_sum, u1_sum], axis=1)
h2_prime = tf.concat([g2x, m2_sum, u2_sum], axis=1)
for l in self.f_node:
h1_prime = l(h1_prime)
h2_prime = l(h2_prime)
return (h1_prime, h2_prime)
def __init__(self, n, k, A):
self.initw = tf.cast(tf.random_uniform([n, k], minval=0.0, maxval=1.0),
tf.float64)
self.initz = tf.cast(tf.random_uniform([k, n], minval=0.0, maxval=1.0),
tf.float64)
self.W = tf.Variable(self.initw, dtype=tf.float64)
self.Z = tf.Variable(self.initz, dtype=tf.float64)
indices = []
for i in range(A.nnz):
indices.append([A.row[i], A.col[i]])
self.A = tf.SparseTensor(indices=indices,
values=A.data,
dense_shape=[n, n])
self.up_w = tf.sparse_tensor_dense_matmul(self.A, tf.transpose(self.Z))
self.down_w = tf.matmul(
self.W, tf.matmul(self.Z, self.Z, transpose_b=True)
) + tf.sparse_tensor_dense_matmul(
self.A,
tf.sparse_tensor_dense_matmul(tf.sparse_transpose(self.A), self.W))
self.down_w_true = tf.maximum(self.down_w, 1e-80)
self.up_z = tf.transpose(
tf.sparse_tensor_dense_matmul(tf.sparse_transpose(self.A), self.W))
self.down_z = tf.matmul(tf.matmul(self.W, self.W, transpose_a=True),
self.Z) + self.Z
self.down_z_true = tf.maximum(self.down_z, 1e-80)
self.new_w = 2.0 * self.W * self.up_w / self.down_w_true
self.new_z = 2.0 * self.Z * self.up_z / self.down_z_true
self.epsilon = tf.placeholder(tf.float64)
self.New_w = tf.placeholder(tf.float64, shape=self.W.shape)
self.New_z = tf.placeholder(tf.float64, shape=self.Z.shape)
self.update_w = tf.assign(
self.W, self.epsilon * self.W + (1 - self.epsilon) * self.New_w)
self.update_z = tf.assign(
self.Z, self.epsilon * self.Z + (1 - self.epsilon) * self.New_z)
self.filter_w = tf.assign(
self.W,
tf.where(self.W > 1e-80, self.W,
tf.zeros(self.W.shape, self.W.dtype)))
self.filter_z = tf.assign(
self.Z,
tf.where(self.Z > 1e-80, self.Z,
tf.zeros(self.Z.shape, self.Z.dtype)))
self.sess = tf.Session()
self.sess.run(tf.global_variables_initializer())
self.pre = np.array(0 for _ in range(n))
def gconv(x, theta, Ks, c_in, c_out):
'''
Spectral-based graph convolution function.
:param x: tensor, [batch_size, n_route, c_in].
:param theta: tensor, [Ks*c_in, c_out], trainable kernel parameters.
:param Ks: int, kernel size of graph convolution.
:param c_in: int, size of input channel.
:param c_out: int, size of output channel.
:return: tensor, [batch_size, n_route, c_out].
'''
# graph kernel: tensor, [n_route, Ks*n_route]
kernel_indices = tf.get_collection('graph_kernel_indices')[0]
kernel_value = tf.get_collection('graph_kernel_value')[0]
kernel_shape = tf.get_collection('graph_kernel_shape')[0]
kernel = tf.SparseTensor(indices=kernel_indices, values=kernel_value, dense_shape=kernel_shape)
n = tf.shape(kernel)[0]
kernel = tf.sparse_transpose(kernel)
# x -> [batch_size, c_in, n_route] -> [batch_size*c_in, n_route]
x_tmp = tf.reshape(tf.transpose(x, [0, 2, 1]), [-1, n])
# x_mul = x_tmp * ker -> [batch_size*c_in, Ks*n_route] -> [batch_size, c_in, Ks, n_route]
x_tmp = tf.sparse_tensor_dense_matmul(kernel, tf.transpose(x_tmp))
x_mul = tf.reshape(tf.transpose(x_tmp), [-1, c_in, Ks, n])
# x_mul = tf.reshape(tf.matmul(x_tmp, kernel), [-1, c_in, Ks, n])
# x_ker -> [batch_size, n_route, c_in, K_s] -> [batch_size*n_route, c_in*Ks]
x_ker = tf.reshape(tf.transpose(x_mul, [0, 3, 1, 2]), [-1, c_in * Ks])
# x_gconv -> [batch_size*n_route, c_out] -> [batch_size, n_route, c_out]
x_gconv = tf.reshape(tf.matmul(x_ker, theta), [-1, n, c_out])
return x_gconv
def apply_neurosat(cfg, params, args):
n_vars, n_lits, n_clauses = args.n_vars, 2 * args.n_vars, args.n_clauses
CL = tf.sparse_reorder(tf.SparseTensor(indices=tf.cast(args.CL_idxs, tf.int64),
values=tf.ones(tf.shape(args.CL_idxs)[0]),
dense_shape=[tf.cast(n_clauses, tf.int64), tf.cast(n_lits, tf.int64)]))
L = tf.ones(shape=[2 * n_vars, cfg['d']], dtype=tf.float32) * params.L_init_scale
C = tf.ones(shape=[n_clauses, cfg['d']], dtype=tf.float32) * params.C_init_scale
LC = tf.sparse_transpose(CL)
def flip(lits): return tf.concat([lits[n_vars:, :], lits[0:n_vars, :]], axis=0)
for t in range(cfg['n_rounds']):
C_old, L_old = C, L
LC_msgs = tf.sparse_tensor_dense_matmul(CL, L, adjoint_a=False) * params.LC_scale
C = params.C_updates[t].forward(tf.concat([C, LC_msgs], axis=-1))
C = tf.check_numerics(C, message="C after update")
C = tfutil.normalize(C, axis=cfg['norm_axis'], eps=cfg['norm_eps'])
C = tf.check_numerics(C, message="C after norm")
if cfg['res_layers']: C = C + C_old
CL_msgs = tf.sparse_tensor_dense_matmul(LC, C, adjoint_a=False) * params.CL_scale
L = params.L_updates[t].forward(tf.concat([L, CL_msgs, flip(L)], axis=-1))
L = tf.check_numerics(L, message="L after update")
L = tfutil.normalize(L, axis=cfg['norm_axis'], eps=cfg['norm_eps'])
L = tf.check_numerics(L, message="L after norm")
if cfg['res_layers']: L = L + L_old
V = tf.concat([L[0:n_vars, :], L[n_vars:, :]], axis=1)
V_scores = params.V_score.forward(V) # (n_vars, 1)
return NeuroSATGuesses(pi_core_var_logits=tf.squeeze(V_scores))
def kernel(self, inputs):
# the default order of the image is z-dominant(z,y,x)
# for projection another two images are created.
img = inputs[self.KEYS.TENSOR.IMAGE].data
effmap = inputs[self.KEYS.TENSOR.EFFICIENCY_MAP].data
sinos = inputs[self.KEYS.TENSOR.SINOGRAM].data
matrixs = inputs[self.KEYS.TENSOR.SYSTEM_MATRIX].data
tran_matrixs = tf.sparse_transpose(matrixs)
"""
The following codes need rewrite
"""
proj = tf.sparse_tensor_dense_matmul(matrixs,img)
con = tf.ones(proj.shape)/100000000
proj = proj+con
temp_proj = sinos/proj
temp_bp = tf.sparse_tensor_dense_matmul(tran_matrixs,temp_proj)
result = img / effmap * temp_bp
#result = img * temp_bp
#result = result / tf.reduce_sum(result) * tf.reduce_sum(sinos)
###efficiencymap
#result = tf.sparse_tensor_dense_matmul(tran_matrixs,sinos)
return Tensor(result, None, self.graph_info.update(name=None))
def sparse_matmul(args, sparse_matrix, scope=None, use_sparse_mul=True):
if not isinstance(args, list):
args = [args]
# Now the computation.
if len(args) == 1:
# res = math_ops.matmul(args[0], sparse_matrix,b_is_sparse=True)
input = args[0]
else:
input = tf.concat(
args,
1,
)
output_shape = tf.shape(input)
input = tf.reshape(input, tf.concat([[-1], [output_shape[-1]]], axis=0))
input = tf.transpose(input, perm=[1, 0])
with tf.variable_scope(scope or "Linear"):
if use_sparse_mul:
sparse_matrix = tf.sparse_transpose(sparse_matrix, perm=[1, 0])
res = tf.sparse_tensor_dense_matmul(sparse_matrix, input)
res = tf.transpose(res, perm=[1, 0])
res = tf.reshape(res, tf.concat([output_shape[:-1], [-1]], axis=0))
return res
def get_init_grad(self, name, num_topics, num_words, num_docs, alpha):
U = tf.placeholder(dtype=tf.float32,
shape=[num_words, num_topics],
name="U-%s" % (name))
V = tf.placeholder(dtype=tf.float32,
shape=[num_topics, num_docs],
name="V-%s" % (name))
D = tf.sparse_reorder(self.D)
VVT = tf.matmul(V, V, transpose_b=True)
UTU = tf.matmul(U, U, transpose_a=True)
DVT = self.D_mul_U(D, tf.transpose(V), num_topics)
eye = tf.eye(num_topics, dtype=tf.float32)
u_grads = [
# 2.0 * tf.matmul(UV - D, V, transpose_b=True),
2.0 * (tf.matmul(U, VVT) - DVT),
4.0 * alpha * tf.matmul(U, UTU - eye)
]
UTD = tf.transpose(self.D_mul_U(tf.sparse_transpose(D), U, num_topics))
v_grads = [2.0 * (tf.matmul(UTU, V) - UTD)]
u_grads = tf.reduce_sum(tf.stack(u_grads, axis=0), axis=0)
v_grads = tf.reduce_sum(tf.stack(v_grads, axis=0), axis=0)
# loss = tf.reduce_sum([
# tf.reduce_sum(tf.norm(D - UV, ord=2)),
# tf.reduce_sum(tf.norm(tf.matmul(U, U, transpose_a=True) - eye, ord=2))
# ])
UV = tf.matmul(U, V[:, 0:100])
loss = tf.reduce_mean(tf.norm(self.Ds - UV, ord=2))
return u_grads, v_grads, loss, [U, V]
def _decode_conv(self, inputs, name='decode_conv'):
X, A, last_append_mask, mol_ids_rep, rep_ids_rep, hc = inputs
with tf.name_scope(name):
# process A
_to_sparse = lambda _i: tf.SparseTensor(
indices=tf.cast(_i, dtype=tf.int64),
dense_shape=tf.stack([
ops.get_len(X, out_type=tf.int64),
ops.get_len(X, out_type=tf.int64)
],
axis=0),
values=tf.ones_like(_i[:, 0], dtype=tf.float32))
_symmetric = lambda _s: tf.sparse_add(_s, tf.sparse_transpose(_s))
A_sparse = [_symmetric(_to_sparse(_A_i)) for _A_i in A]
# embedding
X = self.embedding_node(X) + self.embedding_mask(last_append_mask)
# preparing c
batch_size, iw_size = tf.reduce_max(
mol_ids_rep) + 1, tf.reduce_max(rep_ids_rep) + 1
hc_expand = tf.tile(hc[:, tf.newaxis, :], tf.stack([1, iw_size,
1]))
indices = tf.stack([mol_ids_rep, rep_ids_rep], axis=1)
# convolution
X_out = [X]
for conv in self.decoder_conv:
X_out_i = conv(X_out[-1], A_sparse, hc_expand, indices)
X_out.append(X_out_i)
X_out = tf.concat(X_out[1:], axis=1)
X_out = self.decoder_skip(X_out)
return X_out
def add_embedding_matrix(self):
self.user_item_embedding = tf.SparseTensor(
self.dataSet.train_list[:, 0:2],
values=self.dataSet.train_list[:, 2].astype(np.float64),
dense_shape=[self.nrow, self.ncol])
self.item_user_embedding = tf.sparse_transpose(
self.user_item_embedding)
def getData(self, train=None, test=None, ignore=False):
super(SVDPP, self).getData(train, test, ignore)
self.globalBias = self.trainData[2].mean()
N = tf.SparseTensor(self.trainData[:, :2],
values=[1] * self.trRow,
dense_shape=[self.users, self.items])
self.N = tf.sparse_transpose(N)
def fn_T(x):
T = tf.SparseTensor(indices=x[1],
values=x[2],
dense_shape=[self.max_len, self.max_len])
T = tf.sparse_reorder(T)
T_ = tf.sparse_transpose(T)
return tf.sparse_tensor_dense_matmul(T_, x[0])
def get_sim(x):
if isinstance(x, tf.SparseTensor):
x_T = tf.sparse_transpose(x, perm=[0, 2, 1])
return batch_matmul(x, x_T)
else:
x_T = tf.transpose(x, perm=[0, 2, 1])
return batch_matmul(x, x_T)
def _decode_conv(self, inputs, name='decode_conv'):
X, A, last_append_mask = inputs
with tf.name_scope(name):
# process A
_to_sparse = lambda _i: tf.SparseTensor(
indices=tf.cast(_i, dtype=tf.int64),
dense_shape=tf.stack([
ops.get_len(X, out_type=tf.int64),
ops.get_len(X, out_type=tf.int64)
],
axis=0),
values=tf.ones_like(_i[:, 0], dtype=tf.float32))
_symmetric = lambda _s: tf.sparse_add(_s, tf.sparse_transpose(_s))
A_sparse = [_symmetric(_to_sparse(_A_i)) for _A_i in A]
# embedding
X = self.embedding_node(X) + self.embedding_mask(last_append_mask)
# convolution
X_out = [X]
for conv in self.decoder_conv:
X_out_i = conv(X_out[-1], A_sparse)
X_out.append(X_out_i)
X_out = tf.concat(X_out[1:], axis=1)
X_out = self.decoder_skip(X_out)
return X_out
def _call(self, inputs):
if self.sparse_inputs:
num_users = inputs[0].dense_shape[0]
num_items = inputs[1].dense_shape[0]
users = tf.sparse_to_dense(inputs[0].indices,
inputs[0].dense_shape, inputs[0].values)
items = tf.sparse_to_dense(inputs[1].indices,
inputs[1].dense_shape, inputs[1].values)
else:
num_users = inputs[0].shape[0]
num_items = inputs[1].shape[0]
users = inputs[0]
items = inputs[1]
original_x = tf.concat(
[users, items], axis=0
) # CHECK THIS! need to combine users and items into one single array. becomes 6000 (users+items) x 6000 (input_dim)
original_x = tf.nn.dropout(original_x, 1 - self.dropout)
outputs = []
for i in range(len(self.E_start)):
# DROPOUT FOR EDGES
if self.dropout_edges:
self.E_start[i] = dropout_sparse(self.E_start[i],
1 - self.dropout,
self.E_start_nonzero_list[i])
self.E_end[i] = dropout_sparse(self.E_end[i], 1 - self.dropout,
self.E_end_nonzero_list[i])
# E_start, E_end : E x V
x = original_x
# conv1
Vix = dot(x, self.Vi1[i]) # Vij[i] is 6000x100
Vjx = dot(x, self.Vj1[i])
x1 = tf.add(dot(self.E_end[i], Vix, sparse=True),
dot(self.E_start[i], Vjx, sparse=True))
x1 = tf.nn.bias_add(x1, self.bv1[i])
x1 = tf.nn.sigmoid(x1)
Uix = dot(x, self.Ui1[i])
Ujx = dot(x, self.Uj1[i])
x2 = dot(self.E_start[i], Ujx, sparse=True)
x = tf.add(
Uix,
dot(tf.sparse_transpose(self.E_end[i]),
tf.multiply(x1, x2),
sparse=True))
x = tf.nn.bias_add(x, self.bu1[i])
x = tf.layers.batch_normalization(x)
x = tf.nn.relu(x)
outputs.append(x)
output = tf.concat(axis=1, values=outputs)
output = tf.add(output, tf.matmul(original_x, self.R))
output = tf.nn.relu(output)
u = output[:tf.cast(num_users, tf.int32)]
v = output[tf.cast(num_users, tf.int32):]
return u, v
def _attn_r_n_m_h(self):
h, r, n = self.heads, self.relations, self._nodes
attn_h_n_rm = self._attn_h_n_rm
attn_h_n_r_m = tf.sparse_reshape(attn_h_n_rm, [h, n, r, n])
attn_r_n_m_h = tf.sparse_transpose(attn_h_n_r_m, [2, 1, 3, 0])
return attn_r_n_m_h
def make_sparse_graph_pooling_layer(V, A, P, name=None):
with tf.variable_scope(name, default_name='Graph-Pooling') as scope:
#Generate shape array for BN
#oldVShapeNonTensor = V.get_shape()
#Generate shape tensor for vertex pooling function I know hacky af
oldVShape = tf.shape(V, out_type=tf.int64)
newVShape = tf.stack([oldVShape[0], P.dense_shape[3], oldVShape[2]])
Ptranspose = tf.sparse_transpose(P, perm=[0, 1, 3, 2])
#Areordered = tf.sparse_transpose(A, perm=[0, 2, 1, 3])
#print(P.get_shape())
#print(Ptranspose.get_shape())
#print(Areordered.get_shape())
Vout = _graphcnn_avg_vertex_pool_sparse_module.sparse_average_vertex_pool(
V, Ptranspose.indices, Ptranspose.values, newVShape)
Vout.set_shape([V.get_shape()[0].value,\
P.get_shape()[3].value,\
V.get_shape()[2].value])
#Change indices to pooled mapping
print(A.indices.get_shape())
print(A.values.get_shape())
Pcols = P.indices[:, 3]
newRowidx = tf.gather(Pcols, A.indices[:, 1])
newColidx = tf.gather(Pcols, A.indices[:, 3])
newIndices = tf.stack(
(A.indices[:, 0], newRowidx, A.indices[:, 2], newColidx), axis=1)
print('lol2')
print(newIndices.get_shape())
newShape = tf.stack((A.dense_shape[0], P.dense_shape[3],
A.dense_shape[2], P.dense_shape[3]),
axis=0)
Adupe = tf.sparse_reorder(
tf.SparseTensor(newIndices, A.values, newShape))
#Segment sum to merge duplicate indices
#Worried about this casting, but matmul doesn't support int64 for some dumb reason
linearized = tf.cast(
tf.matmul(
tf.cast(Adupe.indices, tf.float64),
tf.cast([[newShape[1] * newShape[2] * newShape[3]],
[newShape[2] * newShape[3]], [newShape[3]], [1]],
tf.float64)), tf.int64)
print(linearized.get_shape())
y, idx = tf.unique(tf.squeeze(linearized))
# Use the positions of the unique values as the segment ids to
# get the unique values
print(idx.get_shape())
print(Adupe.values.get_shape())
values = tf.segment_sum(Adupe.values, idx)
delinearized = tf.stack(
(y // (newShape[1] * newShape[2] * newShape[3]), y //
(newShape[2] * newShape[3]) % newShape[1],
y // newShape[3] % newShape[2], y % newShape[3]),
axis=1)
Aout = tf.SparseTensor(delinearized, values, newShape)
return Vout, Aout
def _sparse_tensordot_reshape(a, axes, flipped=False):
"""Helper method to perform transpose and reshape for contraction op."""
if a.get_shape().is_fully_defined() and isinstance(
axes, (list, tuple)):
shape_a = a.get_shape().as_list()
axes = [i if i >= 0 else i + len(shape_a) for i in axes]
free = [i for i in range(len(shape_a)) if i not in axes]
free_dims = [shape_a[i] for i in free]
prod_free = int(np.prod([shape_a[i] for i in free]))
prod_axes = int(np.prod([shape_a[i] for i in axes]))
perm = list(axes) + free if flipped else free + list(axes)
new_shape = [prod_axes, prod_free
] if flipped else [prod_free, prod_axes]
reshaped_a = tf.sparse_reshape(tf.sparse_transpose(a, perm),
new_shape)
return reshaped_a, free_dims, free_dims
else:
if a.get_shape().ndims is not None and isinstance(
axes, (list, tuple)):
shape_a = a.get_shape().as_list()
axes = [i if i >= 0 else i + len(shape_a) for i in axes]
free = [i for i in range(len(shape_a)) if i not in axes]
free_dims_static = [shape_a[i] for i in free]
else:
free_dims_static = None
shape_a = tf.shape(a)
rank_a = tf.rank(a)
axes = tf.convert_to_tensor(axes, dtype=tf.int32, name="axes")
axes = tf.cast(axes >= 0, tf.int32) * axes + tf.cast(
axes < 0, tf.int32) * (axes + rank_a)
free, _ = tf.setdiff1d(tf.range(rank_a), axes)
free_dims = tf.gather(shape_a, free)
axes_dims = tf.gather(shape_a, axes)
prod_free_dims = tf.reduce_prod(free_dims)
prod_axes_dims = tf.reduce_prod(axes_dims)
perm = tf.concat([axes_dims, free_dims], 0)
if flipped:
perm = tf.concat([axes, free], 0)
new_shape = tf.stack([prod_axes_dims, prod_free_dims])
else:
perm = tf.concat([free, axes], 0)
new_shape = tf.stack([prod_free_dims, prod_axes_dims])
reshaped_a = tf.sparse_reshape(tf.sparse_transpose(a, perm),
new_shape)
return reshaped_a, free_dims, free_dims_static
def moveaxis(a, axis_src, axis_dst):
"""Move an axis of a tensor to new position, similar to np.moveaxis
Other axes remain in the original order
Args:
a (Tensor): the tensor whose axes should be reordered
axis_src (int, Seq[int]): Original positions of the axes to
move. These must be unique.
axis_dst (int, Seq[int]): Destination position for each of the
origianl axes. These must also be unique.
Examples:
>>> a = np.zeros((3, 4, 5))
>>> moveaxis(a, 0, -1).get_shape().as_list()
[4, 5, 3]
>>> moveaxis(a, -1, 0).get_shape().as_list()
[5, 3, 4]
>>> moveaxis(a, [0, 1], [-1, -2]).get_shape().as_list()
[5, 4, 3]
>>> moveaxis(a, [0, 1, 2], [-1, -2, -3]).get_shape().as_list()
[5, 4, 3]
>>> moveaxis(a, [0, 1], [-1, -2, -3])
Traceback (most recent call last):
...
ValueError: ...
>>> sa = scipy.sparse.random(3, 4)
>>> moveaxis(sa, [0, 1], [-1, -2]).get_shape().as_list()
[4, 3]
"""
a = tf.convert_to_tensor_or_sparse_tensor(a)
# a = _tf_utils.as_tensor_or_sparse_tensor(a)
ndims = a.get_shape().ndims
# src = _utils.validate_axis(
# axis_src, ndims, 'axis_src', accept_none=False,
# scalar_to_seq=True)
# dst = _utils.validate_axis(
# axis_dst, ndims, 'axis_dst', accept_none=False,
# scalar_to_seq=True)
src = _utils.validate_axis2(axis_src, ndims)
dst = _utils.validate_axis2(axis_dst, ndims)
if len(src) != len(dst):
raise ValueError('`axis_src` and `axis_dst` arguments must have the '
'same number of elements')
order = [i for i in range(ndims) if i not in src]
for dst_1, src_1 in sorted(zip(dst, src)):
order.insert(dst_1, src_1)
if isinstance(a, tf.Tensor):
res = tf.transpose(a, order)
else:
res = tf.sparse_transpose(a, order)
return res
def map_func2(i):
adj_mat = tf.SparseTensor(
adj_ind[lbl, i], adj_data[lbl, i], [
tf.cast(max_nodes, tf.int64),
tf.cast(max_nodes, tf.int64)
])
adj_mat = tf.sparse_transpose(adj_mat)
return tf.sparse_tensor_dense_matmul(
adj_mat, inp_gin[i])
def calc_gradient(X, W, Y):
pred = tf.sparse_tensor_dense_matmul(X, W)
error = tf.sigmoid(tf.mul(Y, pred))
error_m1 = error - 1
error_Y = tf.mul(Y, error_m1)
X_T = tf.sparse_transpose(X)
gradient = tf.sparse_tensor_dense_matmul(X_T, error_Y)
return tf.reduce_sum(gradient, 1)
def calc_gradient(X, W, Y):
pred = tf.mul(tf.sparse_tensor_dense_matmul(X, W), -1.0)
error = tf.sigmoid(tf.mul(Y, pred))
error_m1 = error - 1
error_Y = tf.mul(Y, error_m1)
X_T = tf.sparse_transpose(X)
gradient = tf.sparse_tensor_dense_matmul(X_T, error_Y)
return tf.reshape(gradient, [num_features, 1])
def swapaxes(a, axis1, axis2):
"""Interchange two axes of a Tensor
Args:
a: input Tensor
axis1: int, first axis
axis2: int, second axis
Returns:
A tensor with axes swapped
Examples:
>>> import tensorflow as tf
>>> import awesomeml.utils as aml_utils
>>> sess = tf.Session()
>>> a = np.random.random((2,3,4,5))
>>> b = np.swapaxes(a, axis1=1, axis2=2)
>>> b2 = sess.run(swapaxes(a, axis1=1, axis2=2))
>>> np.testing.assert_array_almost_equal(b, b2)
>>> import sparse
>>> a = np.random.random((3,4))
>>> b = np.swapaxes(a, axis1=0, axis2=1)
>>> b2 = swapaxes(sparse.as_coo(a), axis1=0, axis2=1)
>>> isinstance(b2, tf.SparseTensor)
True
>>> b2 = sess.run(b2)
>>> b2 = aml_utils.as_numpy_array(b2)
>>> np.testing.assert_array_almost_equal(b, b2)
"""
print("USING SWAP AX")
a = tf.convert_tensor_or_sparse_tensor(a)
ndim = a.get_shape().ndims
if ndim is None: # pragma: no cover
raise ValueError('can not swapaxes for tensors with unknown shape')
axis1 = ndim + axis1 if axis1 < 0 else axis1
axis2 = ndim + axis2 if axis2 < 0 else axis2
if axis1 == axis2:
return a
perm = list(range(ndim))
perm[axis1] = axis2
perm[axis2] = axis1
if isinstance(a, tf.Tensor):
return tf.transpose(a, perm)
else:
return tf.sparse_transpose(a, perm)
def get_datapoint_iter(file_idx=[], batch_size=s_batch):
fileNames = map(lambda s: "/home/ubuntu/criteo-tfr-tiny/tfrecords" + s,
file_idx)
# We first define a filename queue comprising 5 files.
filename_queue = tf.train.string_input_producer(fileNames,
num_epochs=None)
# TFRecordReader creates an operator in the graph that reads data from queue
reader = tf.TFRecordReader()
# Include a read operator with the filenae queue to use. The output is a string
# Tensor called serialized_example
_, serialized_example = reader.read(filename_queue)
# The string tensors is essentially a Protobuf serialized string. With the
# following fields: label, index, value. We provide the protobuf fields we are
# interested in to parse the data. Note, feature here is a dict of tensors
features = tf.parse_single_example(
serialized_example,
features={
'label': tf.FixedLenFeature([1], dtype=tf.int64),
'index': tf.VarLenFeature(dtype=tf.int64),
'value': tf.VarLenFeature(dtype=tf.float32),
})
label = features['label']
index = features['index']
value = features['value']
# combine the two sparseTensors into one that will be used in gradient calcaulation
index_shaped = tf.reshape(index.values, [-1, 1])
value_shaped = tf.reshape(value.values, [-1])
combined_values = tf.sparse_transpose(
tf.SparseTensor(indices=index_shaped,
values=value_shaped,
shape=[33762578]))
label_flt = tf.cast(label, tf.float32)
# min_after_dequeue defines how big a buffer we will randomly sample
# from -- bigger means better shuffling but slower start up and more
# memory used.
# capacity must be larger than min_after_dequeue and the amount larger
# determines the maximum we will prefetch. Recommendation:
# min_after_dequeue + (num_threads + a small safety margin) * batch_size
min_after_dequeue = 10
capacity = min_after_dequeue + 3 * batch_size
value_batch, label_batch = tf.train.shuffle_batch(
[combined_values, label_flt],
batch_size=batch_size,
capacity=capacity,
min_after_dequeue=min_after_dequeue)
return value_batch, label_batch
def compute_coarse_matrix(model, n, A_stencil, A_matrices,grid_size,bb=True):
if bb == True:
P_stencil = model(A_stencil, True)
else:
P_stencil = model(A_stencil, phase="Test")
P_matrix = tf.to_double(compute_p2_sparse(P_stencil, P_stencil.shape.as_list()[0], grid_size))
P_matrix_t = tf.sparse_transpose(P_matrix, [0,2, 1])
A_matrices = tf.squeeze(A_matrices)
temp = tf.sparse_tensor_to_dense(P_matrix_t)
q = tf.matmul(temp, tf.to_double(A_matrices))
A_c = tf.transpose(tf.matmul(temp,tf.transpose(q,[0,2,1])),[0,2,1])
return A_c, compute_stencil(tf.squeeze(A_c),(grid_size//2)),P_matrix,P_matrix_t
def kernel(self, inputs):
KT = self.KEYS.TENSOR
image = inputs[KT.IMAGE]
efficiency_map = inputs[KT.EFFICIENCY_MAP]
projection_data = inputs[KT.PROJECTION_DATA]
psf_xy = inputs[KT.PSF_XY]
psf_z = inputs[KT.PSF_Z]
# here start the extra psf process,
# the varabiles create below are tensorflow-like type.
# TODO: rewrite by encapsulating in doufo and dxlearn.
import tensorflow as tf
grid = image.data.get_shape()
print("the type of image grid is ",grid)
image_vectorized = tf.reshape(image.data, [grid[0]*grid[1], grid[2]])
print('psf xy shape:', psf_xy.data.get_shape())
print('psf z shape:', psf_z.data.get_shape())
print('image_vectorized shape:', image_vectorized.shape)
# image_mul_psf_z = image_vectorized
image_mul_psf_z = [email protected](psf_z.data)
image_psf_processed = tf.sparse_tensor_dense_matmul(
psf_xy.data, image_mul_psf_z)
# image_psf_processed = image_mul_psf_z
image_psf_processed = tf.reshape(
image_psf_processed, shape=tf.shape(image.data))
image_psf = Image(image_psf_processed, image.center, image.size)
# ####################
# original ReconStep
proj = self.projection(image_psf, projection_data)
back_proj = self.backprojection(proj, image_psf)
# from dxl.learn.tensor import transpose
# here start the extra psf process,
# the varabiles create below are tensorflow-like type.
# TODO: rewrite by encapsulating in doufo and dxlearn.
back_proj_vectorized = tf.reshape(back_proj.data, [grid[0]*grid[1], grid[2]])
# back_proj_mul_psf_z = back_proj_vectorized
back_proj_mul_psf_z = back_proj_vectorized@psf_z.data
back_proj_psf_processed = tf.sparse_tensor_dense_matmul(
tf.sparse_transpose(psf_xy.data), back_proj_mul_psf_z)
# back_proj_psf_processed = back_proj_mul_psf_z
back_proj_psf = tf.reshape(
back_proj_psf_processed, shape=tf.shape(image.data))
back_proj = Image(back_proj_psf, image.center, image.size)
######################################
return self.update(image, back_proj, efficiency_map)
def _sparse_pool_trans_op(M,input,source_index,pooled_size):
#the other path of the sparse pooling
#pooled_size is different from the sparse_pool_op
#source_index is the img_index by default
#only support batch size 1
input_size = tf.shape(input) #(batch, height, width, depth)
bv_pooled_sequence = tf.sparse_tensor_dense_matmul(tf.sparse_transpose(M),tf.reshape(input,[-1,input_size[3]]))
# use scatter_nd_add
#bv_pooled = tf.Variable(tf.zeros(pooled_size))
#return tf.scatter_nd_add(bv_pooled,source_index,bv_pooled_sequence)
# use scatter_nd, however, the current verison of tf seems not to accumulate duplicate indexes
bv_pooled = tf.scatter_nd(source_index,bv_pooled_sequence,pooled_size)
return bv_pooled
def testTranspose(self):
with self.test_session(use_gpu=False) as sess:
np.random.seed(1618)
shapes = [np.random.randint(1, 10, size=rank) for rank in range(1, 6)]
for shape in shapes:
for dtype in [np.int32, np.int64, np.float32, np.float64]:
dn_input = np.random.randn(*shape).astype(dtype)
rank = tf.rank(dn_input).eval()
perm = np.random.choice(rank, rank, False)
sp_input, unused_a_nnz = _sparsify(dn_input)
sp_trans = tf.sparse_transpose(sp_input, perm=perm)
dn_trans = tf.sparse_tensor_to_dense(sp_trans).eval()
expected_trans = tf.transpose(dn_input, perm=perm).eval()
self.assertAllEqual(dn_trans, expected_trans)
def _create_bi_interaction_embed(self):
pi = 3.14159265358979323846
d1_embedding = tf.concat([self.weights['re_drug1_embed'],self.weights['im_drug1_embed']], axis=1)
d2_embedding = tf.concat([self.weights['re_drug2_embed'],self.weights['im_drug2_embed']], axis=1)
#使用R-GCN方式定义关系矩阵
relation_embedding = []
for i in range(self.n_relations):
weights = tf.reshape(self.weights['alpha'][i],[-1,1,1])
relation_matrix_temp = self.weights['relation_matrix'] * weights
relation_matrix_temp = tf.reduce_sum(relation_matrix_temp,axis=0)
relation_embedding.append(relation_matrix_temp)
d1_neigh,d2_neigh = [],[]
for i in range(self.n_relations):
# print(i)
r_d2_embedding = d2_embedding @ relation_embedding[i]
weight_d2_embedding = r_d2_embedding * self.weights['relation_2_att'][i]
weight_d2_embedding = weight_d2_embedding + d2_embedding
# weight_d2_embedding = d2_embedding
relation_1_neigh = tf.sparse_tensor_dense_matmul(self.sparse_adj_list[i],weight_d2_embedding)
r_d1_embedding = d1_embedding @ relation_embedding[i]
weight_d1_embedding = r_d1_embedding * self.weights['relation_1_att'][i]
weight_d1_embedding = weight_d1_embedding + d1_embedding
# weight_d1_embedding = d1_embedding
relation_2_neigh = tf.sparse_tensor_dense_matmul(tf.sparse_transpose(self.sparse_adj_list[i]),weight_d1_embedding)
d1_neigh.append(relation_1_neigh)
d2_neigh.append(relation_2_neigh)
d1_neigh = tf.reduce_sum(d1_neigh,0)
d2_neigh = tf.reduce_sum(d2_neigh,0)
neigh_embed = tf.concat([d1_neigh, d2_neigh], axis=0)
ego_embeddings = tf.concat([d1_embedding, d2_embedding], axis=0)
all_embeddings = []
# side_embeddings = tf.nn.l2_normalize(neigh_embed, dim=1)
side_embeddings = neigh_embed
side_embeddings = tf.concat([ego_embeddings, side_embeddings], 1)
pre_embeddings = tf.nn.relu(
tf.matmul(side_embeddings, self.weights['W_mlp_0']) + self.weights['b_mlp_0'])
pre_embeddings = tf.nn.dropout(pre_embeddings, 1 - self.mess_dropout[0])
all_embeddings += [pre_embeddings]
all_embeddings = tf.concat(all_embeddings, 1)
d1_embeddings, d2_embeddings = tf.split(all_embeddings, [self.n_drug1, self.n_drug2], 0)
return d1_embeddings, d2_embeddings
def testTranspose(self):
with self.test_session(use_gpu=False):
np.random.seed(1618)
shapes = [np.random.randint(1, 10, size=rank) for rank in range(1, 6)]
for shape in shapes:
for dtype in [np.int32, np.int64, np.float32, np.float64]:
dn_input = np.random.randn(*shape).astype(dtype)
rank = tf.rank(dn_input).eval()
perm = np.random.choice(rank, rank, False)
sp_input, unused_a_nnz = _sparsify(dn_input)
sp_trans = tf.sparse_transpose(sp_input, perm=perm)
dn_trans = tf.sparse_tensor_to_dense(sp_trans).eval()
expected_trans = tf.transpose(dn_input, perm=perm).eval()
self.assertAllEqual(dn_trans, expected_trans)
def decoded(self) -> tf.Tensor:
if self.beam_width == 1:
decoded, _ = tf.nn.ctc_greedy_decoder(
inputs=self.logits, sequence_length=self.encoder.lengths,
merge_repeated=self.merge_repeated_outputs)
else:
decoded, _ = tf.nn.ctc_beam_search_decoder(
inputs=self.logits, sequence_length=self.encoder.lengths,
beam_width=self.beam_width,
merge_repeated=self.merge_repeated_outputs)
return tf.sparse_tensor_to_dense(
tf.sparse_transpose(decoded[0]),
default_value=END_TOKEN_INDEX)
def __init__(self, config, ont, word_model):
#print("Creating the model graph")
tf.reset_default_graph()
self.ont = ont
self.word_model = word_model
##
config.concepts_size = len(self.ont.concepts) +1
##
self.config = config
### Inputs ###
self.label = tf.placeholder(tf.int32, shape=[None])
self.class_weights = tf.Variable(tf.ones([config.concepts_size]), False)
self.seq = tf.placeholder(tf.float32, shape=[None, config.max_sequence_length, word_model.dim])
self.seq_len = tf.placeholder(tf.int32, shape=[None])
self.lr = tf.Variable(config.lr, trainable=False)
self.is_training = tf.placeholder(tf.bool)
self.ancestry_sparse_tensor = tf.sparse_reorder(tf.SparseTensor(indices = ont.sparse_ancestrs, values = [1.0]*len(ont.sparse_ancestrs), dense_shape=[config.concepts_size, config.concepts_size]))
#######################
## Phrase embeddings ##
#######################
l2scale = 1.0
layer1 = tf.layers.conv1d(self.seq, self.config.cl1, 1, activation=tf.nn.elu,\
kernel_initializer=tf.random_normal_initializer(0.0,0.1),\
bias_initializer=tf.random_normal_initializer(stddev=0.01), use_bias=True)
layer2 = tf.layers.dense(tf.reduce_max(layer1, [1]), self.config.cl2, activation=tf.nn.relu,\
#layer2 = tf.layers.dense(tf.reduce_max(layer1, [1]), self.config.cl2,\
kernel_initializer=tf.random_normal_initializer(0.0,stddev=0.1),
bias_initializer=tf.random_normal_initializer(0.0,stddev=0.01), use_bias=True)
'''
layer2 = tf.layers.dense(layer2, self.config.cl2, activation=tf.nn.relu,\
#layer2 = tf.layers.dense(tf.reduce_max(layer1, [1]), self.config.cl2,\
kernel_initializer=tf.random_normal_initializer(0.0,stddev=0.1),
kernel_regularizer=tf.contrib.layers.l2_regularizer(l2scale,scope=None),
bias_initializer=tf.random_normal_initializer(0.0,stddev=0.01), use_bias=True)
layer2 = tf.reduce_max(layer1, [1])
'''
self.seq_embedding = tf.nn.l2_normalize(layer2, axis=1)
#self.seq_embedding = layer2
########################
## Concept embeddings ##
########################
self.embeddings = tf.get_variable("embeddings", shape = [self.config.concepts_size, self.config.cl2], initializer = tf.random_normal_initializer(stddev=0.1))
#self.embeddings = tf.nn.l2_normalize(self.embeddings, dim=1)
self.aggregated_embeddings = tf.sparse_tensor_dense_matmul(self.ancestry_sparse_tensor, self.embeddings)
if config.flat:
aggregated_w = self.embeddings
else:
aggregated_w = self.aggregated_embeddings
last_layer_b = tf.get_variable('last_layer_bias', shape = [self.config.concepts_size], initializer = tf.random_normal_initializer(stddev=0.001))
self.score_layer = tf.matmul(self.seq_embedding, tf.transpose(aggregated_w)) + last_layer_b
########################
########################
########################
label_one_hot = tf.one_hot(self.label, config.concepts_size)
#reg_losses = tf.get_collection(tf.GraphKeys.REGULARIZATION_LOSSES)
#reg_constant = 0.0002
self.loss = tf.reduce_mean(tf.losses.sparse_softmax_cross_entropy(self.label, self.score_layer)) # + reg_constant*tf.reduce_sum(reg_losses)
#self.loss = tf.reduce_mean(tf.losses.sigmoid_cross_entropy(tf.one_hot(self.label, config.concepts_size), self.score_layer))
#self.loss = -tf.reduce_mean(tf.reduce_sum(label_one_hot*tf.log(0.001+ tf.sigmoid(self.score_layer)) + (1-label_one_hot)*tf.log(0.001+1-tf.sigmoid(self.score_layer))/(config.concepts_size-1), axis=-1))
self.pred = tf.nn.softmax(self.score_layer)
#self.pred = tf.nn.sigmoid(self.score_layer)
self.agg_pred, _ = tf.nn.top_k(tf.transpose(tf.sparse_tensor_dense_matmul(tf.sparse_transpose(self.ancestry_sparse_tensor), tf.transpose(self.pred))), 2)
self.train_step = tf.train.AdamOptimizer(self.lr).minimize(self.loss)
self.sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
#print("initializing")
self.sess.run(tf.global_variables_initializer())
def sparse_message_pass(node_states,
adjacency_matrices,
num_edge_types,
hidden_size,
use_bias=True,
average_aggregation=False,
name="sparse_ggnn"):
"""One message-passing step for a GNN with a sparse adjacency matrix.
Implements equation 2 (the message passing step) in
[Li et al. 2015](https://arxiv.org/abs/1511.05493).
N = The number of nodes in each batch.
H = The size of the hidden states.
T = The number of edge types.
Args:
node_states: Initial states of each node in the graph. Shape is [N, H].
adjacency_matrices: Adjacency matrix of directed edges for each edge
type. Shape is [N, N, T] (sparse tensor).
num_edge_types: The number of edge types. T.
hidden_size: The size of the hidden state. H.
use_bias: Whether to use bias in the hidden layer.
average_aggregation: How to aggregate the incoming node messages. If
average_aggregation is true, the messages are averaged. If it is false,
they are summed.
name: (optional) The scope within which tf variables should be created.
Returns:
The result of one step of Gated Graph Neural Network (GGNN) message passing.
Shape: [N, H]
"""
n = tf.shape(node_states)[0]
t = num_edge_types
incoming_edges_per_type = tf.sparse_reduce_sum(adjacency_matrices, axis=1)
# Convert the adjacency matrix into shape [T, N, N] - one [N, N] adjacency
# matrix for each edge type. Since sparse tensor multiplication only supports
# two-dimensional tensors, we actually convert the adjacency matrix into a
# [T * N, N] tensor.
adjacency_matrices = tf.sparse_transpose(adjacency_matrices, [2, 0, 1])
adjacency_matrices = tf.sparse_reshape(adjacency_matrices, [t * n, n])
# Multiply the adjacency matrix by the node states, producing a [T * N, H]
# tensor. For each (edge type, node) pair, this tensor stores the sum of
# the hidden states of the node's neighbors over incoming edges of that type.
messages = tf.sparse_tensor_dense_matmul(adjacency_matrices, node_states)
# Rearrange this tensor to have shape [N, T * H]. The incoming states of each
# nodes neighbors are summed by edge type and then concatenated together into
# a single T * H vector.
messages = tf.reshape(messages, [t, n, hidden_size])
messages = tf.transpose(messages, [1, 0, 2])
messages = tf.reshape(messages, [n, t * hidden_size])
# Run each of those T * H vectors through a linear layer that produces
# a vector of size H. This process is equivalent to running each H-sized
# vector through a separate linear layer for each edge type and then adding
# the results together.
#
# Note that, earlier on, we added together all of the states of neighbors
# that were connected by edges of the same edge type. Since addition and
# multiplying by a linear layer are commutative, this process was equivalent
# to running each incoming edge through a linear layer separately and then
# adding everything at the end.
with tf.variable_scope(name, default_name="sparse_ggnn"):
final_node_states = common_layers.dense(
messages, hidden_size, use_bias=False)
# Multiply the bias by for each edge type by the number of incoming nodes
# of that edge type.
if use_bias:
bias = tf.get_variable("bias", initializer=tf.zeros([t, hidden_size]))
final_node_states += tf.matmul(incoming_edges_per_type, bias)
if average_aggregation:
incoming_edges = tf.reduce_sum(incoming_edges_per_type, -1, keepdims=True)
incoming_edges = tf.tile(incoming_edges, [1, hidden_size])
final_node_states /= incoming_edges + 1e-7
return final_node_states
