def scores(h, t, l): s = self._score(h, t, l) # [b,n] return mean(s, 1) # [b]
def critic_loss_function(real_prediction, fake_prediction, critic): real_loss = -mean(real_prediction) fake_loss = mean(fake_prediction) total_loss = real_loss + fake_loss return total_loss
fake_scores_out = D.get_output_for(fake_images_out, labels, is_training=True) fake_images_out, real_scores_out, fake_scores_out = tflib.run( [fake_images_out, real_scores_out, fake_scores_out]) car_label = labels[:, 0] car_real = labels[:, 0] car_fake = labels[:, 0] offset = 0 real_loss_disc = [] fake_loss_disc = [] fake_loss_gen = [] for i in range(len(offsets)): # labels = labels[:, offset:offset + offsets[i]] real = real_scores_out[:, offset:offset + offsets[i]] fake = fake_scores_out[:, offset:offset + offsets[i]] offset += offsets[i] fake_scores_out_sum = np.sum(real, axis=1, keepdims=True) real_scores_out_sum = np.sum(fake, axis=1, keepdims=True) fake_loss_disc.append(tf.mean(tf.nn.softplus(fake_scores_out_sum), axis=1)) # -log(1-sigmoid(fake_scores_out)) real_loss_disc.append(tf.mean(tf.nn.softplus(-real_scores_out_sum), axis=1)) # -log(sigmoid(real_scores_out)) plt.plot() # real_rotations_norm = real_rotations / tf.norm(real_rotations, ord='euclidean')
def scores(h, t, r): s = self._score(h, t, r) # [b,n] return mean(-s, 1) # [b]
def call(self, inputs, training=False): """ hidden_states: float Tensor in shape [bsz, seq_len, hidden_size], the hidden-states of the last layer. cls_index: [optional] position of the classification token if summary_type == 'cls_index', shape (bsz,) or more generally (bsz, ...) where ... are optional leading dimensions of hidden_states. if summary_type == 'cls_index' and cls_index is None: we take the last token of the sequence as classification token """ if not isinstance(inputs, (dict, tuple, list)): hidden_states = inputs cls_index = None elif isinstance(inputs, (tuple, list)): hidden_states = inputs[0] cls_index = inputs[1] if len(inputs) > 1 else None assert len(inputs) <= 2, "Too many inputs." else: input_ids = inputs.get('input_ids') cls_index = inputs.get('cls_index', None) if self.summary_type == 'last': output = hidden_states[:, -1] elif self.summary_type == 'first': output = hidden_states[:, 0] elif self.summary_type == 'mean': output = tf.mean(hidden_states, axis=1) elif self.summary_type == 'cls_index': hidden_shape = shape_list( hidden_states ) # e.g. [batch, num choices, seq length, hidden dims] if cls_index is None: cls_index = tf.fill( hidden_shape[:-2], hidden_shape[-2] - 1 ) # A tensor full of shape [batch] or [batch, num choices] full of sequence length cls_shape = shape_list(cls_index) if len(cls_shape) <= len(hidden_shape) - 2: cls_index = cls_index[..., tf.newaxis] # else: # cls_index = cls_index[..., tf.newaxis] # cls_index = cls_index.expand((-1,) * (cls_index.dim()-1) + (hidden_states.size(-1),)) # shape of cls_index: (bsz, XX, 1, hidden_size) where XX are optional leading dim of hidden_states output = tf.gather(hidden_states, cls_index, batch_dims=len(hidden_shape) - 2) output = tf.squeeze( output, axis=len(hidden_shape) - 2) # shape of output: (batch, num choices, hidden_size) elif self.summary_type == 'attn': raise NotImplementedError if self.has_first_dropout: output = self.first_dropout(output, training=training) if self.has_summary: output = self.summary(output) if self.has_activation: output = self.activation(output) if self.has_last_dropout: output = self.last_dropout(output, training=training) return output
def focal_loss_fixed(y_true, y_pred): pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred)) pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred)) return -tf.mean(alpha * tf.pow(1. - pt_1, gamma) * tf.log(pt_1)) - tf.mean((1 - alpha) * tf.pow(pt_0, gamma) * tf.log(1. - pt_0))
def psnr(y_true, y_pred): """ The cost function by computing the psnr. """ return 1 / (10.0 * tf.log(1.0 / (tf.mean(tf.square(y_pred - y_true)))) / tf.log(10.0))
def mae(y_pred, y_true): return tf.mean(tf.abs(y_pred, y_true))
def build_model(self): self.is_training = tf.placeholder(tf.bool) self.x = tf.placeholder(tf.float32, shape=[None] + self.config.state_size) self.y = tf.placeholder(tf.float32, shape=[None, 10]) # set initial feedforward and feedback weights p = self.config.state_size[0] #m = 512 #j = 200 m = 50 j = 20 n = 10 var_xi = self.config.var_xi #Scale weight initialization alpha0 = np.sqrt(2.0/p) alpha1 = np.sqrt(2.0/m) alpha2 = np.sqrt(2.0/j) alpha3 = 1 #Plus one for bias terms A = tf.Variable(rng.randn(p+1,m)*alpha0, name="hidden_weights", dtype=tf.float32) W1 = tf.Variable(rng.randn(m+1,j)*alpha1, name="hidden_weights2", dtype=tf.float32) W2 = tf.Variable(rng.randn(j+1,n)*alpha2, name="output_weights", dtype=tf.float32) B1 = tf.Variable(rng.randn(m+1,j)*alpha1, name="feedback_weights1", dtype=tf.float32) B2 = tf.Variable(rng.randn(j+1,n)*alpha2, name="feedback_weights2", dtype=tf.float32) # network architecture with ones added for bias terms e0 = tf.ones([self.config.batch_size, 1], tf.float32) e1 = tf.ones([self.config.batch_size, 1], tf.float32) x_aug = tf.concat([self.x, e0], 1) h1 = tf.sigmoid(tf.matmul(x_aug, A)) h1_aug = tf.concat([h1, e1], 1) #Compute unperturbed output h2_0 = tf.sigmoid(tf.matmul(h1_aug, W1)) h2_0_aug = tf.concat([h2_0, e1], 1) y_p_0 = tf.matmul(h2_0_aug, W2) self.trainable = [A, W1, W2, B1, B2] with tf.name_scope("loss"): #mean squared error self.loss = tf.reduce_sum(tf.pow(y_p_0-self.y, 2))/2 e = (y_p_0 - self.y) h1_prime_0 = tf.multiply(h1_aug, 1-h1_aug)[:,0:m] h2_prime_0 = tf.multiply(h2_0_aug, 1-h2_0_aug)[:,0:j] #Compute updates for W and A (based on B) grad_W2 = tf.gradients(xs=W2, ys=self.loss)[0] lmda2 = tf.matmul(e, tf.transpose(B2[0:j,:])) d2 = np.multiply(h2_prime_0, lmda2) grad_W1 = tf.matmul(tf.transpose(h1_aug), d2) lmda1 = tf.matmul(d2, tf.transpose(B1[0:m,:])) d1 = np.multiply(h1_prime_0, lmda1) grad_A = tf.matmul(tf.transpose(x_aug), d1) p1=tf.random_normal(shape=tf.shape(h1_aug), mean=0.0, stddev=var_xi, dtype=tf.float32) p2=tf.random_normal(shape=tf.shape(h2_aug), mean=0.0, stddev=var_xi, dtype=tf.float32) p12= tf.sigmoid(tf.matmul(p1, W1))#perturbation of layer 1 going to 2 p12_mean=p12-tf.mean(p12,axis=0) p2y=tf.sigmoid(tf.matmul(p2, W2))#perturbation of layer 2 going to y p2y_mean=p2y-tf.mean(p2y,axis=0) grad_B1 = tf.matmul(tf.transpose(p1),p12_mean)-self.config.lambda*B1 grad_B2 = tf.matmul(tf.transpose(p2),p2y_mean)-self.config.lambda*B2 new_W1 = W1.assign(W1 - self.config.learning_rate*grad_W1) new_W2 = W2.assign(W2 - self.config.learning_rate*grad_W2) new_A = A.assign(A - self.config.learning_rate*grad_A) #Train with SGD new_B1 = B1.assign(B1 - self.config.lmda_learning_rate*grad_B1) new_B2 = B2.assign(B2 - self.config.lmda_learning_rate*grad_B2) self.train_step = [new_W1, new_A, new_B1, new_W2, new_B2] self.train_step_mirror = [new_B1, new_B2] correct_prediction = tf.equal(tf.argmax(y_p, 1), tf.argmax(self.y, 1)) self.accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) #Save training metrics Bs = [B1, B2] Ws = [W1, W2] es = [d2, e] gradBs = [tf.norm(grad_B1), tf.norm(grad_B2)] self._set_training_metrics(Ws, Bs, es, gradBs)
def L2_loss(denseOut, dis_label): L2_dis = tf.square(dis_label - denseOut) return tf.mean(L2_dis)
def ambig_mean_absolute_error(y_true, y_pred): nonAmbig=tf.math.logical_not(tf.math.is_nan(y_true)) return tf.mean(tf.abs(tf.boolean_mask(y_pred,nonAmbig) - tf.boolean_mask(y_true,nonAmbig)), axis=-1)
def ambig_binary_crossentropy(y_true,y_pred): nonAmbig=tf.math.logical_not(tf.math.is_nan(y_true)) return tf.mean(tf.binary_crossentropy(tf.boolean_mask(y_true,nonAmbig), tf.boolean_mask(y_pred,nonAmbig)), axis=-1);
def weighted_binary_crossentropy(y_true,y_pred): weightsPerTaskRep = y_true*w1_weights[None,:] + (1-y_true)*w0_weights[None,:] nonAmbig=tf.math.logical_not(tf.math.is_nan(y_true)) nonAmbigTimesWeightsPerTask = tf.boolean_mask(weightsPerTaskRep,nonAmbig) return tf.mean(tf.binary_crossentropy(tf.boolean_mask(y_true,nonAmbig),tf.boolean_mask(y_pred,nonAmbig))*nonAmbigTimesWeightsPerTask, axis=-1);
def computeIoU(y_pred_batch, y_true_batch): return tf.mean( np.asarray([ pixelAccuracy(y_pred_batch[i], y_true_batch[i]) for i in range(len(y_true_batch)) ]))