def __call__(self, x): size = int(np.sqrt(x.shape[1].value)) assert (size * size == x.shape[1].value) x = K.reshape(x, (-1, size, size)) xxt = K.batch_dot(x, x, axes=(2, 2)) regularization = 0.0 if self.l1: regularization += K.sum(self.l1 * K.abs(xxt - K.eye(size))) if self.l2: regularization += K.sum(self.l2 * K.square(xxt - K.eye(size))) return regularization
def ortho_reg(weight_matrix): # orthogonal regularization for aspect embedding matrix w_n = weight_matrix / K.cast(K.epsilon() + K.sqrt(K.sum(K.square(weight_matrix), axis=-1, keepdims=True)), K.floatx()) reg = K.sum(K.square(K.dot(w_n, K.transpose(w_n)) - K.eye(w_n.shape[0].value))) return args.ortho_reg * reg
def call(self, x): mat = x[0] #Matrix A val = x[1] eye = K.eye(self.gs) p1 = kron(eye, self.selfintact) p2 = kron_b1fx1(mat, self.neigintact) p = p1 + p2 v = K.reshape(val, (-1, self.gs * self.param)) p = K.reshape(p, (-1, self.gs * self.param, self.gs * self.param)) #print("p",p.shape,"v",v.shape) #exit() #does not yet work, how to define products with 2 batch dimensions #a bit inelegant if you ask me for i in range(self.iterations): v = K.batch_dot(p, v) if self.activate: v = self.advrelu(v, self.activation) #print("v",v.shape,"a",self.a,"gs",self.gs,"param",self.param) #exit() ret = K.reshape( v, (-1, self.a, self.gs, self.param)) #keine Ahnung ob richtig rum #doch scheint zu passen return ret
def convertTraining(training, numVertices, numInputs): """ Convert training data to categorical forms """ converted = [] for entry in training: inputs, labels = np.hsplit(entry, (numInputs, )) labels = np.matrix(labels) vertexTypes, denseGraph = np.hsplit(inputs, (numVertices, )) #Convert node types into categories vertexCategories = np.zeros((numVertices, len(mg.NODE_TYPES))) for i, t in enumerate(vertexTypes): vertexCategories[(i, t)] = 1 #Separate labels into circuit types and node inclusion labels classLabels, inclusionLabels = np.hsplit(labels, (1, )) inclusionLabels = inclusionLabels.reshape(inclusionLabels.shape[1], inclusionLabels.shape[0]) #Convert dense graph into adjacency matrix graph = buildGraph(denseGraph, denseGraph.shape[0]) #Add identity matrix to graph I = K.eye(graph.shape[0]) features = K.concatenate((I, vertexCategories)) spectral = SpectralLayer(graph, features.shape[1], activation=tf.nn.relu, input_shape=features.shape, useWeights=False) converted.append((graph, spectral, features, classLabels)) return converted
def call(self, x, **kwargs): assert isinstance(x, list) inp_a, inp_b = x outp_a = K.l2_normalize(inp_a, -1) outp_b = K.l2_normalize(inp_b, -1) alpha = K.batch_dot(outp_b, outp_a, axes=[2, 2]) alpha = K.l2_normalize(alpha, 1) alpha = K.one_hot(K.argmax(alpha, 1), K.int_shape(inp_a)[1]) hmax = K.batch_dot(alpha, outp_b, axes=[1, 1]) kcon = K.eye(K.int_shape(inp_a)[1], dtype='float32') m = [] for i in range(self.output_dim): outp_a = inp_a * self.W[i] outp_hmax = hmax * self.W[i] outp_a = K.l2_normalize(outp_a, -1) outp_hmax = K.l2_normalize(outp_hmax, -1) outp = K.batch_dot(outp_hmax, outp_a, axes=[2, 2]) outp = K.sum(outp * kcon, -1, keepdims=True) m.append(outp) if self.output_dim > 1: persp = K.concatenate(m, 2) else: persp = m[0] return [persp, persp]
def build(self, input_shape): if self.kernel is None: (p, l, u, u_diag_sign, u_diag_abs_log, l_mask, u_mask) = self.initializer(input_shape) self.kernel_p = self.add_weight(name='kernel_p', shape=p.shape, initializer=lambda _: p, trainable=False) self.kernel_l = self.add_weight(name='kernel_l', shape=l .shape, initializer=lambda _: l, trainable=True) self.kernel_u = self.add_weight(name='kernel_u', shape=u.shape, initializer=lambda _: u, trainable=True) self.kernel_u_diag_sign = self.add_weight(name='kernel_u_diag_sign', shape=u_diag_sign.shape, initializer=lambda _: u_diag_sign, trainable=False) self.kernel_u_diag_abs_log = self.add_weight(name='kernel_u_diag_abs_log', shape=u_diag_abs_log.shape, initializer=lambda _: u_diag_abs_log, trainable=True) self.kernel_l = self.kernel_l * l_mask + K.eye(input_shape[-1]) self.kernel_u = self.kernel_u * u_mask + K.tf.diag( self.kernel_u_diag_sign * K.exp(self.kernel_u_diag_abs_log)) self.kernel = K.dot(K.dot(self.kernel_p, self.kernel_l), self.kernel_u) super(InvDense, self).build(input_shape)
def call(self, x): mat = x[0] #Matrix A val = x[1] N = self.neigintact S = self.selfintact print("N", N.shape) print("S", S.shape) t1 = self.tp(S, K.eye(self.gs)) print("t1", t1.shape) exit() for i in range(self.iterations): weignei = K.batch_dot(mat, val) #Neighbours of the current nodes parta = K.dot(weignei, self.neigintact) #Neighbour part partb = K.dot(val, self.selfintact) #Self Interaction Part val = parta + partb if self.activate: val = self.advrelu(val, self.activation) return val
def build(self, input_shape): import tensorflow_probability as tfp dimensionality = input_shape[1] self.mvn = tfp.distributions.MultivariateNormalFullCovariance( loc=K.zeros(dimensionality), covariance_matrix=K.eye(dimensionality)) super().build(input_shape)
def __call__(self, R): """Returns a component dependence penalty for matrix R """ RRT = K.dot(R, K.transpose(R)) I = K.eye(RRT.shape.as_list()[0]) penalty = self.lamb * K.sqrt(K.sum(K.square(RRT - I))) return penalty
def uncorrelated_feature(self, x): if (self.encoding_dim <= 1): return 0.0 else: output = K.sum( K.square(self.covariance - tf.math.multiply( self.covariance, K.eye(self.encoding_dim)))) return output
def test_make_soft(_log, train_with_soft_target_stdev, _config): if train_with_soft_target_stdev is None: _config['train_with_soft_target_stdev'] = 1 y_true = K.reshape(K.eye(512)[:129, :256], (2, 129, 256)) y_soft = make_soft(y_true) f = K.function([], y_soft) _log.info('Output of soft:') f1 = f([]) _log.info(f1[0, 0]) _log.info(f1[-1, -1])
def regularization(w): #_, _, _, c = w.get_shape().as_list() _, _, _, c = K.int_shape(w) w = K.reshape(w, [-1, c]) w_t = K.transpose(w) identity = K.eye(c) w_t_w = K.dot(w_t, w) subtraction = w_t_w - identity loss = tf.nn.l2_loss(subtraction) return scale * loss
def build(self, input_shape): """ Initializes various network parameters. """ input_dim = input_shape[-1] # TODO: add regularizers self.encoders = self.add_weight( name="encoders", shape=(input_dim, self.units), initializer=self.encoder_initializer, trainable=self.trainable_encoders, ) self.dt = self.add_weight( name="dt", shape=(1, ), initializer=self.dt_initializer, trainable=self.trainable_dt, ) self.decoders = self.add_weight( name="decoders", shape=(self.units * self.order, self.output_size), initializer=self.decoder_initializer, trainable=self.trainable_decoders, ) self.AT = self.add_weight( name="AT", shape=(self.order, self.order), initializer=Constant(self._A.T), # note: transposed trainable=self.trainable_A, ) self.B = self.add_weight( name="B", shape=(1, 1, self.order), # system is SISO initializer=Constant(self._B[None, None, :]), trainable=self.trainable_B, ) self.I = K.eye(self.order) # noqa: E741 self.zero_padding = K.zeros((1, self.order + 1)) self.built = True
def build(self, input_shape): if type(input_shape) == list: self.input_spec = list( map(lambda x: keras.engine.InputSpec(shape=x), input_shape)) else: self.input_spec = keras.engine.InputSpec(shape=input_shape) if isinstance(self.layers, list) and len(self.layers) == 0: self.layer.build(input_shape) config = self.layer.get_config() name = config['name'] self.layers = [] for i in range(self.layer_num): copied = copy.copy(config) copied['name'] = name + '_{}'.format(i + 1) self.layers.append(self.layer.__class__.from_config(copied)) for layer in self.layers: layer.build(input_shape) if self.hidden_dim is not None: self.W = self.add_weight( shape=(int(input_shape[-1]), self.hidden_dim * self.layer_num), name='{}_W'.format(self.name), initializer=keras.initializers.get('uniform'), ) if self.use_bias: self.b = self.add_weight( shape=(self.hidden_dim * self.layer_num, ), name='{}_b'.format(self.name), initializer=keras.initializers.get('zeros'), ) if self.reg_index: for i, (index, interval, weight) in enumerate( zip(self.reg_index, self.reg_slice, self.reg_weight)): weights = [] if type(interval) is slice: interval = (interval, ) for layer in self.layers: if interval is None: weights.append(K.flatten(layer.get_weights()[index])) else: weights.append( K.flatten(layer.get_weights()[index][interval])) weights = K.stack(weights) self.add_loss(weight * K.sum( K.square( K.dot(weights, K.transpose(weights)) - K.eye(len(self.layers))))) super(MultiHead, self).build(input_shape)
def build(self, input_shape): if type(input_shape) == list: self.input_spec = list( map(lambda x: keras.engine.InputSpec(shape=x), input_shape)) else: self.input_spec = keras.engine.InputSpec(shape=input_shape) if not self.layers: self.layers = [ copy.deepcopy(self.layer) for _ in range(self.layer_num) ] if self.hidden_dim is not None: self.W = self.add_weight( shape=(input_shape[-1], self.hidden_dim * self.layer_num), name='{}_W'.format(self.name), initializer=keras.initializers.get('uniform'), ) if self.use_bias: self.b = self.add_weight( shape=(self.hidden_dim * self.layer_num, ), name='{}_b'.format(self.name), initializer=keras.initializers.get('zeros'), ) input_shape = input_shape[:-1] + (self.hidden_dim, ) for i, layer in enumerate(self.layers): if not layer.built: if self.rename: layer.name = layer.name + '_%d' % (i + 1) layer.build(input_shape) if self.reg_index: for i, (index, interval, weight) in enumerate( zip(self.reg_index, self.reg_slice, self.reg_weight)): weights = [] if type(interval) is slice: interval = (interval, ) for layer in self.layers: if interval is None: weights.append(K.flatten(layer.get_weights()[index])) else: weights.append( K.flatten(layer.get_weights()[index][interval])) weights = K.stack(weights) self.add_loss(weight * K.sum( K.square( K.dot(weights, K.transpose(weights)) - K.eye(len(self.layers))))) super(MultiHead, self).build(input_shape)
def build(self, input_shape): input_dim = input_shape[-1] # TODO: add regularizers self.encoders = self.add_weight( name='encoders', shape=(input_dim, self.units), initializer=self.encoder_initializer, trainable=self.trainable_encoders) self.dt = self.add_weight( name='dt', shape=(1,), initializer=self.dt_initializer, trainable=self.trainable_dt) self.decoders = self.add_weight( name='decoders', shape=(self.units*self.order, self.output_size), initializer=self.decoder_initializer, trainable=self.trainable_decoders) self.AT = self.add_weight( name='AT', shape=(self.order, self.order), initializer=Constant(self._A.T), # note: transposed trainable=self.trainable_A) self.B = self.add_weight( name='B', shape=(1, 1, self.order), # system is SISO initializer=Constant(self._B[None, None, :]), trainable=self.trainable_B) self.I = K.eye(self.order) self.zero_padding = K.zeros((1, self.order + 1)) self.built = True
def call(self,x): mat=x[0]#Matrix A val=x[1] eye=K.eye(self.gs) p1=kron(eye,self.selfintact) p2=kron_b1(mat,self.neigintact) p=p1+p2 v=K.reshape(val,(-1,self.gs*self.param)) for i in range(self.iterations): v=K.batch_dot(p,v) if self.activate: v=self.advrelu(v,self.activation) ret=K.reshape(v,(-1,self.gs,self.param))#keine Ahnung ob richtig rum #doch scheint zu passen return ret
def call(self,x): mat=x[0]#Matrix A val=x[1] eye=K.eye(self.gs) p1=kron(eye,self.selfintact) p2=kron_b1(mat,self.neigintact) p=p1+p2 p=t.linalg.inv(p)#invert the matrix p, is only truly the inverse of glm if activate=False v=K.reshape(val,(-1,self.gs*self.param)) for i in range(self.iterations): v=K.batch_dot(p,v) if self.activate: v=self.advrelu(v,self.activation) ret=K.reshape(v,(-1,self.gs,self.param))#keine Ahnung ob richtig rum #doch scheint zu passen return ret
# INITIALIZE EMBEDDING initial_user_embedding = tf.Variable( K.l2_normalize(K.random_uniform((1, args.embedding_dim))) ) # the initial user and item embeddings are learned during training as well initial_item_embedding = tf.Variable( K.l2_normalize(K.random_uniform((1, args.embedding_dim)))) user_embeddings = K.repeat_elements( initial_user_embedding, num_users, 0) # initialize all users to the same embedding item_embeddings = K.repeat_elements( initial_item_embedding, num_users, 0) # initialize all items to the same embedding item_embedding_static = tf.Variable( K.eye(num_items)) # one-hot vectors for static embeddings user_embedding_static = tf.Variable( K.eye(num_users)) # one-hot vectors for static embeddings # RUN THE JODIE MODEL ''' THE MODEL IS TRAINED FOR SEVERAL EPOCHS. IN EACH EPOCH, JODIES USES THE TRAINING SET OF INTERACTIONS TO UPDATE ITS PARAMETERS. ''' print("*** Training the JODIE model for %d epochs ***" % args.epochs) with trange(args.epochs) as progress_bar_1: for ep in progress_bar_1: progress_bar_1.set_description('Epoch %d of %d' % (ep, args.epochs)) # INITIALIZE EMBEDDING TRAJECTORY STORAGE user_embeddings_timeseries = tf.constant( tf.zeros([num_interactions, args.embedding_dim], tf.float32))
def compile_backprop(self, proj_type='l2'): self.proj_type = proj_type layers = [layer for layer in self.model.layers if layer.get_weights()] A = [Input(layer.output_shape[1:]) for layer in layers[:-1]] # Store the constraint gradients and biases for each layer. grads, biases = [], [] prev_grad, prev_bias = None, None for i, layer in enumerate(layers): W, b = layer.weights if isinstance(layer, Dense): if i > 0 and K.ndim(A[i - 1]) == 4 and (K.image_data_format() == 'channels_first'): # The `Flatten` layer doesn't respect the channels-first # dimension ordering, so it mixes up our dimensions. We need # to correct for that here. _, ch, h, w = K.int_shape(A[i - 1]) _, n_out = K.int_shape(W) W = K.reshape( K.permute_dimensions(K.reshape(W, (h, w, ch, n_out)), (2, 0, 1, 3)), (ch * h * w, n_out)) if len(grads) == 0: grad = K.transpose(W) bias = b # Expand to batch shape. grad = grad[None] * K.ones_like(A[i])[:, :, None] bias = bias[None] * K.ones_like(A[i]) else: A_i = K.reshape( A[i - 1], [-1, np.prod(K.int_shape(A[i - 1])[1:])]) grad = (K.transpose(W)[None] * A_i[:, None]) @ grads[-1] bias = (biases[-1] * A_i) @ W + b[None] grads.append(grad) biases.append(bias) else: if K.image_data_format() == 'channels_first': _, ch_in, h_in, w_in = layer.input_shape _, ch_out, h_out, w_out = layer.output_shape else: _, h_in, w_in, ch_in = layer.input_shape _, h_out, w_out, ch_out = layer.output_shape if len(grads) == 0: if K.image_data_format() == 'channels_first': grad = K.conv2d(K.reshape( K.eye(ch_in * h_in * w_in), [ch_in * h_in * w_in, ch_in, h_in, w_in]), W, padding=layer.padding, strides=layer.strides) bias = K.tile(b[:, None, None], [1, h_out, w_out]) else: grad = K.conv2d(K.reshape( K.eye(ch_in * h_in * w_in), [ch_in * h_in * w_in, h_in, w_in, ch_in]), W, padding=layer.padding, strides=layer.strides) bias = K.tile(b[None, None], [h_out, w_out, 1]) # Expand to batch shape. grad = grad[None] * K.ones_like(A[i])[:, None] bias = bias[None] * K.ones_like(A[i]) else: n = np.prod(self.input_shape) if K.image_data_format() == 'channels_first': grad = K.reshape( K.conv2d(K.reshape(grad * A[i - 1][:, None], (-1, ch_in, h_in, w_in)), W, padding=layer.padding, strides=layer.strides), (-1, n, ch_out, h_out, w_out)) bias = K.conv2d(bias * A[i - 1], W, padding=layer.padding, strides=layer.strides) + b[None, :, None, None] else: grad = K.reshape( K.conv2d(K.reshape(grad * A[i - 1][:, None], (-1, h_in, h_in, ch_in)), W, padding=layer.padding, strides=layer.strides), (-1, n, h_out, h_out, ch_out)) bias = K.conv2d(bias * A[i - 1], W, padding=layer.padding, strides=layer.strides) + b[None, None, None] grads.append( K.permute_dimensions( K.reshape(grad, (-1, n, ch_out * h_out * w_out)), (0, 2, 1))) biases.append(K.batch_flatten(bias)) # Handle the softmax constraints. c = K.placeholder((1, ), dtype='int32') softmax_grads = grads[-1] softmax_biases = biases[-1] c_grad = K.permute_dimensions( K.gather(K.permute_dimensions(softmax_grads, (1, 0, 2)), c), (1, 0, 2)) c_bias = K.transpose(K.gather(K.transpose(softmax_biases), c)) grads[-1] = softmax_grads - c_grad biases[-1] = softmax_biases - c_bias grads_no_first_layer = K.concatenate(grads[1:], axis=1) biases_no_first_layer = K.concatenate(biases[1:], axis=1) grads = K.concatenate(grads, axis=1) biases = K.concatenate(biases, axis=1) # Calculate distances. x = K.placeholder(self.input_shape) distances = proj_dist(proj_type, K.reshape(x, (1, -1)), grads, biases) distances_no_first_layer = proj_dist(proj_type, K.reshape(x, (1, -1)), grads_no_first_layer, biases_no_first_layer) self._grad_f = K.function(A + [c], [grads]) self._bias_f = K.function(A + [c], [biases]) self._dist_f = K.function(A + [c, x], [distances]) self._all_f = K.function(A + [c, x], [ distances, grads[:, -self.n_classes:], biases[:, -self.n_classes:] ]) self._all_except_first_f = K.function(A + [c, x], [ distances_no_first_layer, grads_no_first_layer[:, -self.n_classes:], biases_no_first_layer[:, -self.n_classes:] ]) self.compiled = True return self
def call(self, x): x = x[0] return x + K.eye(self.gs)
def constrained_l2_proj_distance(x, grads, biases): u, w, b = x, grads, -biases[:, :, None] ndim = K.int_shape(u)[1] neurons = K.int_shape(w)[1] w = K.reshape(w, (-1, ndim)) b = K.reshape(b, (-1, 1)) # Handy naming to make code a bit more readable. def is_(x): return K.cast(x, 'float32') def where(x): return K.cast(x, 'float32') # Check feasibility. # The problem is infeasible if `q` (defined below) can be brought to # infinity as `lam` -> +/-infinity. We see that as `lam` -> +infinity, # `x_star[i]` becomes 0 if `w[i]` is positive and if `w[i]` is negative. # Thus, dq/dlam = -b + sum_{i : w_i < 0}{w_i}. If dq/dlam is positive, then # `q` will go to infinity. The other case comes from the symmetric case for # when `lam` -> -infinity. infeasible = (is_(K.sum(w * where(w > 0), axis=1) < b[:, 0]) + is_(K.sum(w * where(w < 0), axis=1) > b[:, 0])) feasible = 1. - infeasible # Get the order (as lambda goes from -infinity to 0) in which each dimention # transitions to the I stage. I_in_order = tf.argsort(u / w * where(w < 0) + (u - 1) / w * where(w > 0)) # Get the order (as lambda goes from 0 to +infinity) in which each dimention # transitions out of the I stage. I_out_order = tf.argsort(u / w * where(w > 0) + (u - 1) / w * where(w < 0)) eye = K.eye(ndim) mask_I = K.cumsum(K.concatenate( (K.gather(eye, I_in_order), -K.gather(eye, I_out_order)), axis=1), axis=1) mask_I = tf.gather(mask_I, np.arange(2 * ndim - 1), axis=1) mask_1 = is_(w > 0)[:, None] + K.cumsum(K.concatenate( (-tf.gather( eye[None] * where(w > 0)[:, None], I_in_order, batch_dims=1), tf.gather( eye[None] * where(w < 0)[:, None], I_out_order, batch_dims=1)), axis=1), axis=1) mask_1 = tf.gather(mask_1, np.arange(2 * ndim - 1), axis=1) # Here we collect the dimensions in `w` and `u` that are in the I and 1 # phase in state `i`. An analysis of the cases shows that given these # dimension, the argmax of `q` can be found by the equation below. w_I = w[:, None] * mask_I u_I = u[:, None] * mask_I w_1 = w[:, None] * mask_1 argmaxes = ((K.sum(w_I * u_I, axis=2) + K.sum(w_1, axis=2) - b) / K.sum(w_I * w_I, axis=2)) # Find the inflection points in `q`. inflections = K.concatenate(((u - 1) / w, u / w)) lam = K.concatenate((argmaxes, inflections)) x_star_candidates = K.clip(u[:, None] - lam[:, :, None] * w[:, None], 0., 1.) # Evaluate `q` on all the candidate points for the max. Select the point # that is in fact the max (`opt_index`). max_q_candidates = (K.sum(.5 * (x_star_candidates - u[:, None])**2 + lam[:, :, None] * w[:, None] * x_star_candidates, axis=2) - lam * b) opt_index = K.argmax(max_q_candidates, axis=1) # `x_star` is the projection satisfying the constraints. x_star = K.sum(K.one_hot(opt_index, 4 * ndim - 1)[:, :, None] * x_star_candidates, axis=1) d = tf.norm(x_star - u, axis=1) / feasible return K.reshape(d, (-1, neurons))
def testBackEnd(): x = tf.zeros([3, 4], tf.int32) print('x=', x) x = tf.zeros((3, 4), tf.int32) print('x=', x) return a = tf.constant([1, 2, 3, 4, 5, 6, 7, 8], dtype=tf.float32) a = K.reshape(a, (4, 4)) print('a=', a) a = tf.constant([[1, 2], [3, 4]], dtype=tf.float32) #a = K.abs(-1) print('a=', type(a), a) #a= tf.Tensor(1, shape=(), dtype=int32) # a = a.numpy() # print('a=',a) # a = tf.zeros([0, 3]) # a = tf.concat([a, [[1, 2, 3], [5, 6, 8]]], axis=0) # print('a=',type(a),a) b = tf.constant([[1, 8], [2, 3]], dtype=tf.float32) c = K.square(a - b) print('c=', c) d = K.sum(c, axis=0) print('d=', d) d = K.sum(c, axis=1) print('d=', d) d = K.sum(c, axis=[0, 1]) print('d=', d) return a = K.abs([-1, 0, 9, -10]) print('a=', a) #a= tf.Tensor([ 1 0 9 10], shape=(4,), dtype=int32) a = K.abs(np.array([-1, 0, 9, -10])) print('a=', a) #a= tf.Tensor([ 1 0 9 10], shape=(4,), dtype=int32) a = K.all(np.array([-1, 0, 9, -10]), axis=0) print('a=', a) #a= tf.Tensor(False, shape=(), dtype=bool) a = K.all(np.array([[-1, -2, -1], [-1, 0, 9]]), axis=0) #x axis print('a=', a) #a= tf.Tensor([ True False True], shape=(3,), dtype=bool) a = K.all(np.array([[-1, -2, -1], [-1, 0, 9]]), axis=1) #y axis print('a=', a) #a= tf.Tensor([ True False], shape=(2,), dtype=bool) a = K.arange(1, 100, 10) print( 'a=', a ) #a= tf.Tensor([ 1 11 21 31 41 51 61 71 81 91], shape=(10,), dtype=int32) a = K.sum(np.array([-1, 0, 9, -10])) print('a=', a) #a= tf.Tensor(-2, shape=(), dtype=int32) a = K.square(np.array([-1, 0, 9, -10])) print('a=', a) #a= tf.Tensor([ 1 0 81 100], shape=(4,), dtype=int32) x = K.placeholder(shape=(2, 3)) y = K.placeholder(shape=(3, 4)) xy = K.dot(x, y) shape = K.int_shape(xy) print('xy=', xy) #xy= Tensor("MatMul:0", shape=(2, 4), dtype=float32) print('xy shape=', shape) #xy shape= (2, 4) kvar = K.eye(3) #K.eval(kvar) print('kvar=', kvar) ''' array([[1., 0., 0.], [0., 1., 0.], [0., 0., 1.]], dtype=float32)> ''' a = np.array([[1, 2], [3, 4]]) a = K.transpose(a) print('a=', a) ''' a= tf.Tensor( [[1 3] [2 4]], shape=(2, 2), dtype=int32) ''' a = K.clip(np.array([-1, 0, 1, 2, 3, 4, 5]), min_value=0, max_value=3) print('a=', a) #a= tf.Tensor([0 0 1 2 3 3 3], shape=(7,), dtype=int32)
def get_sub_mask(s): # todo check it latter len_s = K.shape(s)[1] mask = K.cumsum(K.eye(len_s), 1) return mask
def call(self, x): x = x[0] #print("!x",x.shape) gs = self.gs k = self.k param = self.param C = self.numericalC #print("gs",gs,"k",k,"param",param,"C",C) for i in range(10): t.print( "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" ) t.print("calling onetopk", self.metrik, output_stream=sys.stdout) #exit() mata = K.constant(self.getmata(gs)) matb = K.constant(self.getmatb(gs)) #print("mata",mata.shape) #print("matb",matb.shape) xp = K.permute_dimensions(x, (0, 2, 1)) #print("xp",xp.shape) xa = K.dot(xp, mata) xb = K.dot(xp, matb) #print("xa",xa.shape,"xb",xb.shape) #exit() isval = xa[:, self.flag, :] * xb[:, self.flag, :] #return isval,isval #print("isval",isval.shape) #exit() ds = xa - xb #print("ds",ds.shape) dsp = K.permute_dimensions(ds, (0, 2, 1)) #print("dsp",dsp.shape) dspsq = K.square(dsp) #print("dspsq",dspsq.shape) #print("self.metrik",self.metrik.shape) delt = K.dot(dspsq, self.metrik) #print("delt",delt.shape) delt = K.reshape( delt, (-1, self.gs * self.gs)) + (1 - isval) * self.emptyconst d = K.reshape(delt, (-1, gs, gs)) #print("d",d.shape) #return d,d #####no self interactions if self.self_interaction == False: one = K.eye(gs) #print("one",one.shape) d += self.self_interaction_const * one #####end no self interactions v, _ = t.math.top_k(-d, k=k) #print("v",v.shape) #return v,v vb = v[:, :, -1] #print("vb",vb.shape) vbs = K.reshape(vb, (-1, gs, 1)) #print("vbs",vbs.shape) su = d + vbs #plus since top_k(-d) #print("su",su.shape) #map anything above 0 to 0 and anything below to 1, also map 0 to 1 #p(-x)=C*d_C(-x) # =d(-C*x) # =1-r(Cx-1)+r(Cx) #experimentally: # r(1-Cx)-r(-Cx) rel = K.relu(1 - C * su) - K.relu(-C * su) #print("rel",rel.shape) rel = K.relu(rel) - K.relu(rel - 1) #return rel,rel dez1 = K.reshape(rel, (-1, self.gs * self.gs)) #print("dez1",dez1.shape) dez2 = dez1 * isval #print("dez2",dez2.shape) rel = K.reshape(dez2, (-1, self.gs, self.gs)) print("rel", rel.shape) numnei = K.sum(rel, axis=-1) print("numnei", numnei.shape) factor = self.k / (numnei + 0.00000000001) #return K.concatenate((numnei,factor),axis=-1),factor#,factor#numnei,numnei print("factor", factor.shape) refactor = K.repeat(factor, self.gs) print("refactor", refactor.shape) refactor = K.permute_dimensions(refactor, (0, 2, 1)) #return refactor,refactor rel = rel * refactor print("rel", rel.shape) #exit() if self.free == 0: return rel, x zero1 = K.zeros_like(x[:, :, 0]) zero1 = K.reshape(zero1, (-1, x.shape[1], 1)) #print("!",zero1.shape) zerolis = [] for i in range(self.free): zerolis.append(zero1) zeros = K.concatenate(zerolis, axis=-1) #print(zeros.shape) return rel, K.concatenate((x, zeros), axis=-1)
def call(self, inputs): """ Creates the layer as a Keras graph. Note that the inputs are tensors with a batch dimension of 1: Keras requires this batch dimension, and for full-batch methods we only have a single "batch". There are three inputs required, the node features, the output indices (the nodes that are to be selected in the final layer) and the graph adjacency matrix Notes: This does not add self loops to the adjacency matrix. The output indices are only used when ``final_layer=True`` Args: inputs (list): list of inputs with 3 items: node features (size 1 x N x F), output indices (size 1 x M), graph adjacency matrix (size N x N), where N is the number of nodes in the graph, F is the dimensionality of node features M is the number of output nodes """ X = inputs[0] # Node features (1 x N x F) out_indices = inputs[1] # output indices (1 x K) A = inputs[2] # Adjacency matrix (N x N) N = K.int_shape(A)[-1] batch_dim, n_nodes, _ = K.int_shape(X) if batch_dim != 1: raise ValueError( "Currently full-batch methods only support a batch dimension of one" ) else: # Remove singleton batch dimension X = K.squeeze(X, 0) out_indices = K.squeeze(out_indices, 0) outputs = [] for head in range(self.attn_heads): kernel = self.kernels[head] # W in the paper (F x F') attention_kernel = self.attn_kernels[ head] # Attention kernel a in the paper (2F' x 1) # Compute inputs to attention network features = K.dot(X, kernel) # (N x F') # Compute feature combinations # Note: [[a_1], [a_2]]^T [[Wh_i], [Wh_2]] = [a_1]^T [Wh_i] + [a_2]^T [Wh_j] attn_for_self = K.dot( features, attention_kernel[0]) # (N x 1), [a_1]^T [Wh_i] attn_for_neighs = K.dot( features, attention_kernel[1]) # (N x 1), [a_2]^T [Wh_j] # Attention head a(Wh_i, Wh_j) = a^T [[Wh_i], [Wh_j]] dense = attn_for_self + K.transpose( attn_for_neighs) # (N x N) via broadcasting # Add nonlinearity dense = LeakyReLU(alpha=0.2)(dense) # Mask values before activation (Vaswani et al., 2017) # YT: this only works for 'binary' A, not for 'weighted' A! # YT: if A does not have self-loops, the node itself will be masked, so A should have self-loops # YT: this is ensured by setting the diagonal elements of A tensor to 1 above if not self.saliency_map_support: mask = -10e9 * (1.0 - A) self.A = A dense += mask dense = K.softmax(dense) # (N x N), Eq. 3 of the paper else: # dense = dense - tf.reduce_max(dense) # GAT with support for saliency calculations W = (self.delta * A) * K.exp( dense - K.max(dense, axis=1, keepdims=True) ) * (1 - self.non_exist_edge) + self.non_exist_edge * ( A + self.delta * (K.ones(shape=[N, N], dtype="float") - A) + K.eye(N) ) * K.exp(dense - K.max(dense, axis=1, keepdims=True)) dense = W / K.sum(W, axis=1, keepdims=True) # Apply dropout to features and attention coefficients dropout_feat = Dropout(self.in_dropout_rate)(features) # (N x F') dropout_attn = Dropout(self.attn_dropout_rate)(dense) # (N x N) # Linear combination with neighbors' features [YT: see Eq. 4] node_features = K.dot(dropout_attn, dropout_feat) # (N x F') if self.use_bias: node_features = K.bias_add(node_features, self.biases[head]) # Add output of attention head to final output outputs.append(node_features) # Aggregate the heads' output according to the reduction method if self.attn_heads_reduction == "concat": output = K.concatenate(outputs) # (N x KF') else: output = K.mean(K.stack(outputs), axis=0) # N x F') # Nonlinear activation function output = self.activation(output) # On the final layer we gather the nodes referenced by the indices if self.final_layer: output = K.gather(output, out_indices) # Add batch dimension back if we removed it if batch_dim == 1: output = K.expand_dims(output, 0) return output