def correlation(y_true, y_pred): '''Pearson correlation coefficient. (metric) The linear corrlation between y_true and y_pred is between -1.0 and 1.0, indicating positive correlation and negative correlation respectively. In particular, if the correlation is 0.0, it means y_true and y_pred are irrelevant linearly. This function is implemented by: corr = [mean(y_true * y_pred) - mean(y_true) * mean(y_pred)] / [ std(y_true) * std(m_y_pred) ] This function has been revised to prevent the division fail (0/0). When either y_true or y_pred is 0, the correlation would be set as 0.0. Input: y_true: label, tensor in any shape. y_pred: prediction, tensor in any shape. Output: scalar, the mean linear correlation between y_true and y_pred. ''' m_y_true = math_ops.reduce_mean(y_true, axis=0) m_y_pred = math_ops.reduce_mean(y_pred, axis=0) s_y_true = gen_math_ops.sqrt(math_ops.reduce_mean(gen_math_ops.square(y_true), axis=0) - gen_math_ops.square(m_y_true)) s_y_pred = gen_math_ops.sqrt(math_ops.reduce_mean(gen_math_ops.square(y_pred), axis=0) - gen_math_ops.square(m_y_pred)) s_denom = s_y_true * s_y_pred s_numer = math_ops.reduce_mean(y_true * y_pred, axis=0) - m_y_true * m_y_pred s_index = gen_math_ops.greater(s_denom, 0) f1 = lambda: constant_op.constant(0.0) f2 = lambda: math_ops.reduce_mean(array_ops.boolean_mask(s_numer,s_index)/array_ops.boolean_mask(s_denom,s_index)) return control_flow_ops.case([(math_ops.reduce_any(s_index), f2)], default=f1)
def GraphFn(self, x1, x2): x = x1 q = math_ops.abs(x) q = q + 1.0 q = gen_math_ops.exp(q) q = gen_math_ops.log(q) q = array_ops.squeeze(q, axis=-2) q = math_ops.abs(q) q = q + 2.2 q = gen_math_ops.sqrt(q) q = gen_math_ops.rsqrt(q) q = math_ops.negative(q) q = array_ops.squeeze(q, axis=3) q = math_ops.abs(q) q = q + 3.0 a = gen_math_ops.reciprocal(q) # this chain of operations has a batch size of 5, which is different from # the batch size for the other operations. x = constant_op.constant(np.random.randn(5, 8, 12), dtype=x.dtype) q = math_ops.abs(x) q = q + 2.0 q = gen_math_ops.exp(q) q = gen_math_ops.log(q) q = math_ops.abs(q) q = q + 2.1 q = gen_math_ops.sqrt(q) q = gen_math_ops.rsqrt(q) q = math_ops.negative(q) q = math_ops.abs(q) q = q + 4.0 b = gen_math_ops.reciprocal(q) # TODO(jie): this one will break, broadcasting on batch. x = x2 q = math_ops.abs(x) q = q + 5.0 q = gen_math_ops.exp(q) q = array_ops.squeeze(q, axis=[-1, -2, 3]) q = gen_math_ops.log(q) q = math_ops.abs(q) q = q + 5.1 q = gen_array_ops.reshape(q, [12, 5, 1, 1, 8, 1, 12]) q = array_ops.squeeze(q, axis=[5, 2, 3]) q = gen_math_ops.sqrt(q) q = math_ops.abs(q) q = q + 5.2 q = gen_math_ops.rsqrt(q) q = math_ops.negative(q) q = math_ops.abs(q) q = q + 5.3 c = gen_math_ops.reciprocal(q) q = a * b q = q / c return array_ops.squeeze(q, name="output_0")
def get_updates(self, loss, params): grads = self.get_gradients(loss, params) self.updates = [] lr = self.lr if self.initial_decay > 0: lr = lr * ( 1. / (1. + self.decay * math_ops.cast(self.iterations,K.dtype(self.decay))) ) with ops.control_dependencies([state_ops.assign_add(self.iterations, 1)]): t = math_ops.cast(self.iterations, K.floatx()) lr_t = gen_math_ops.sqrt(1. - math_ops.pow(self.beta_2, t)) / (1. - math_ops.pow(self.beta_1, t)) lower_bound = self.lr_boost * (1. - 1. / (self.gamma * t + 1.)) upper_bound = self.lr_boost * (1. + 1. / (self.gamma * t)) if self.sgdcorr: m_rate = 1. - self.beta_1 / (self.gamma * t + 1.) else: m_rate = 1. - self.beta_1 ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params] vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params] if self.amsgrad: vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params] else: vhats = [K.zeros(1) for _ in params] self.weights = [self.iterations] + ms + vs + vhats for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats): m_t = (self.beta_1 * m) + m_rate * g v_t = (self.beta_2 * v) + (1. - self.beta_2) * math_ops.square(g) if self.amsgrad: vhat_t = math_ops.maximum(vhat, v_t) lr_v = gen_math_ops.reciprocal(gen_math_ops.sqrt(vhat_t) + self.epsilon) self.updates.append(state_ops.assign(vhat, vhat_t)) else: lr_v = gen_math_ops.reciprocal(gen_math_ops.sqrt(v_t) + self.epsilon) lr_bound = gen_math_ops.minimum(gen_math_ops.maximum(lr_t * lr_v, lower_bound), upper_bound) p_t = p - lr * lr_bound * m_t self.updates.append(state_ops.assign(m, m_t)) self.updates.append(state_ops.assign(v, v_t)) new_p = p_t # Apply constraints. if getattr(p, 'constraint', None) is not None: new_p = p.constraint(new_p) self.updates.append(state_ops.assign(p, new_p)) return self.updates
def get_updates(self, loss, params): grads = self.get_gradients(loss, params) self.updates = [] lr = self.lr if self.initial_decay > 0: lr = lr * (1. / (1. + self.decay * math_ops.cast(self.iterations, K.dtype(self.decay)))) with ops.control_dependencies( [state_ops.assign_add(self.iterations, 1)]): t = math_ops.cast(self.iterations, K.floatx()) lr_t = lr * (gen_math_ops.sqrt(1. - math_ops.pow(self.beta_2, t)) / (1. - math_ops.pow(self.beta_1, t))) ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params] vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params] if self.amsgrad: vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params] else: vhats = [K.zeros(1) for _ in params] self.weights = [self.iterations] + ms + vs + vhats for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats): m_t = (self.beta_1 * m) + self.beta_g * g v_t = (self.beta_2 * v) + (1. - self.beta_2) * math_ops.square(g) if self.amsgrad: vhat_t = math_ops.maximum(vhat, v_t) p_t_ada = p - lr_t * m_t / (gen_math_ops.sqrt(vhat_t) + self.epsilon) self.updates.append(state_ops.assign(vhat, vhat_t)) else: p_t_ada = p - lr_t * m_t / (gen_math_ops.sqrt(v_t) + self.epsilon) p_t_sgd = p - self.lr_boost * lr * m_t self.updates.append(state_ops.assign(m, m_t)) self.updates.append(state_ops.assign(v, v_t)) new_p = m_switch(self.switch_flag, p_t_sgd, p_t_ada) # Apply constraints. if getattr(p, 'constraint', None) is not None: new_p = p.constraint(new_p) self.updates.append(state_ops.assign(p, new_p)) return self.updates
def get_updates(self, loss, params): grads = self.get_gradients(loss, params) self.updates = [state_ops.assign_add(self.iterations, 1)] lr = self.lr if self.initial_decay > 0: lr = lr * ( 1. / (1. + self.decay * math_ops.cast(self.iterations,K.dtype(self.decay))) ) t = math_ops.cast(self.iterations, K.floatx()) + 1 # Due to the recommendations in [2], i.e. warming momentum schedule momentum_cache_t = self.beta_1 * ( 1. - 0.5 * (math_ops.pow(K.cast_to_floatx(0.96), t * self.schedule_decay))) momentum_cache_t_1 = self.beta_1 * ( 1. - 0.5 * (math_ops.pow(K.cast_to_floatx(0.96), (t + 1) * self.schedule_decay))) m_schedule_new = self.m_schedule * momentum_cache_t m_schedule_next = self.m_schedule * momentum_cache_t * momentum_cache_t_1 self.updates.append((self.m_schedule, m_schedule_new)) ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params] vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params] if self.amsgrad: vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params] else: vhats = [K.zeros(1) for _ in params] self.weights = [self.iterations] + ms + vs + vhats for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats): # the following equations given in [1] g_prime = g / (1. - m_schedule_new) m_t = self.beta_1 * m + (1. - self.beta_1) * g m_t_prime = m_t / (1. - m_schedule_next) v_t = self.beta_2 * v + (1. - self.beta_2) * math_ops.square(g) if self.amsgrad: vhat_t = math_ops.maximum(vhat, v_t) self.updates.append(state_ops.assign(vhat, vhat_t)) v_t_prime = vhat_t / (1. - math_ops.pow(self.beta_2, t)) else: v_t_prime = v_t / (1. - math_ops.pow(self.beta_2, t)) m_t_bar = (1. - momentum_cache_t) * g_prime + momentum_cache_t_1 * m_t_prime self.updates.append(state_ops.assign(m, m_t)) self.updates.append(state_ops.assign(v, v_t)) p_t = p - lr * m_t_bar / (gen_math_ops.sqrt(v_t_prime) + self.epsilon) new_p = p_t # Apply constraints. if getattr(p, 'constraint', None) is not None: new_p = p.constraint(new_p) self.updates.append(state_ops.assign(p, new_p)) return self.updates
def call(self, inputs): inputs = ops.convert_to_tensor(inputs, dtype=self.dtype) if inputs.shape.rank != 2: raise ValueError( '`StressIntensityRange` only takes "rank 2" inputs.') output = gen_math_ops.mul(self.kernel * inputs[:, 1], gen_math_ops.sqrt(np.pi * inputs[:, 0])) output = array_ops.reshape(output, (array_ops.shape(output)[0], 1)) # outputs should be (None, 1), so it is still rank = 2 return output
def get_updates(self, loss, params): grads = self.get_gradients(loss, params) self.updates = [] lr = self.lr if self.initial_decay > 0: lr = lr * (1. / (1. + self.decay * math_ops.cast(self.iterations, K.dtype(self.decay)))) with ops.control_dependencies( [state_ops.assign_add(self.iterations, 1)]): t = math_ops.cast(self.iterations, K.floatx()) lr_bc = gen_math_ops.sqrt(1. - math_ops.pow(self.beta_2, t)) / ( 1. - math_ops.pow(self.beta_1, t)) ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params] vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params] lams = [K.zeros(1, dtype=K.dtype(p)) for p in params] conds = [K.variable(False, dtype='bool') for p in params] if self.amsgrad: vhats = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params] else: vhats = [K.zeros(1) for _ in params] self.weights = [self.iterations] + ms + vs + vhats + lams + conds for p, g, m, v, vhat, lam, cond in zip(params, grads, ms, vs, vhats, lams, conds): beta_g = m_switch(cond, 1.0, 1.0 - self.beta_1) m_t = (self.beta_1 * m) + beta_g * g v_t = (self.beta_2 * v) + (1. - self.beta_2) * math_ops.square(g) if self.amsgrad: vhat_t = math_ops.maximum(vhat, v_t) p_t_ada = lr_bc * m_t / (gen_math_ops.sqrt(vhat_t) + self.epsilon) self.updates.append(state_ops.assign(vhat, vhat_t)) else: p_t_ada = lr_bc * m_t / (gen_math_ops.sqrt(v_t) + self.epsilon) gamma_den = math_ops.reduce_sum(p_t_ada * g) gamma = math_ops.reduce_sum(gen_math_ops.square(p_t_ada)) / ( math_ops.abs(gamma_den) + self.epsilon) * (gen_math_ops.sign(gamma_den) + self.epsilon) lam_t = (self.beta_2 * lam) + (1. - self.beta_2) * gamma lam_prime = lam / (1. - math_ops.pow(self.beta_2, t)) lam_t_prime = lam_t / (1. - math_ops.pow(self.beta_2, t)) lg_err = math_ops.abs(lam_t_prime - gamma) cond_update = gen_math_ops.logical_or( gen_math_ops.logical_and( gen_math_ops.logical_and(self.iterations > 1, lg_err < 1e-5), lam_t > 0), cond)[0] lam_update = m_switch(cond_update, lam, lam_t) self.updates.append(state_ops.assign(lam, lam_update)) self.updates.append(state_ops.assign(cond, cond_update)) p_t_sgd = (1. - self.beta_1) * lam_prime * m_t self.updates.append(state_ops.assign(m, m_t)) self.updates.append(state_ops.assign(v, v_t)) new_p = m_switch(cond, p - lr * p_t_sgd, p - lr * p_t_ada) # Apply constraints. if getattr(p, 'constraint', None) is not None: new_p = p.constraint(new_p) self.updates.append(state_ops.assign(p, new_p)) return self.updates
def GetParams(self): """Test for unary operations in TF-TRT.""" dtype = dtypes.float32 input_name = "input" input_dims = [12, 5, 8, 1, 1, 12] output_name = "output" input2_name = "input_2" input2_dims = [12, 5, 8, 1, 12, 1, 1] g = ops.Graph() with g.as_default(): x = array_ops.placeholder(dtype=dtype, shape=input_dims, name=input_name) q = math_ops.abs(x) q = q + 1.0 q = gen_math_ops.exp(q) q = gen_math_ops.log(q) q = array_ops.squeeze(q, axis=-2) q = math_ops.abs(q) q = q + 2.2 q = gen_math_ops.sqrt(q) q = gen_math_ops.rsqrt(q) q = math_ops.negative(q) q = array_ops.squeeze(q, axis=3) q = math_ops.abs(q) q = q + 3.0 a = gen_math_ops.reciprocal(q) x = constant_op.constant(np.random.randn(5, 8, 12), dtype=dtype) q = math_ops.abs(x) q = q + 2.0 q = gen_math_ops.exp(q) q = gen_math_ops.log(q) q = math_ops.abs(q) q = q + 2.1 q = gen_math_ops.sqrt(q) q = gen_math_ops.rsqrt(q) q = math_ops.negative(q) q = math_ops.abs(q) q = q + 4.0 b = gen_math_ops.reciprocal(q) # TODO(jie): this one will break, broadcasting on batch. x = array_ops.placeholder( dtype=dtype, shape=input2_dims, name=input2_name) q = math_ops.abs(x) q = q + 5.0 q = gen_math_ops.exp(q) q = array_ops.squeeze(q, axis=[-1, -2, 3]) q = gen_math_ops.log(q) q = math_ops.abs(q) q = q + 5.1 q = gen_array_ops.reshape(q, [12, 5, 1, 1, 8, 1, 12]) q = array_ops.squeeze(q, axis=[5, 2, 3]) q = gen_math_ops.sqrt(q) q = math_ops.abs(q) q = q + 5.2 q = gen_math_ops.rsqrt(q) q = math_ops.negative(q) q = math_ops.abs(q) q = q + 5.3 c = gen_math_ops.reciprocal(q) q = a * b q = q / c array_ops.squeeze(q, name=output_name) return trt_test.TfTrtIntegrationTestParams( gdef=g.as_graph_def(), input_names=[input_name, input2_name], input_dims=[input_dims, input2_dims], output_names=[output_name], expected_output_dims=[(12, 5, 8, 12)])
def GetParams(self): """Test for unary operations in TF-TRT.""" dtype = dtypes.float32 input_name = "input" input_dims = [12, 5, 8, 1, 1, 12] input2_name = "input_2" input2_dims = [12, 5, 8, 1, 12, 1, 1] g = ops.Graph() with g.as_default(): x = array_ops.placeholder(dtype=dtype, shape=input_dims, name=input_name) q = math_ops.abs(x) q = q + 1.0 q = gen_math_ops.exp(q) q = gen_math_ops.log(q) q = array_ops.squeeze(q, axis=-2) q = math_ops.abs(q) q = q + 2.2 q = gen_math_ops.sqrt(q) q = gen_math_ops.rsqrt(q) q = math_ops.negative(q) q = array_ops.squeeze(q, axis=3) q = math_ops.abs(q) q = q + 3.0 a = gen_math_ops.reciprocal(q) x = constant_op.constant(np.random.randn(5, 8, 12), dtype=dtype) q = math_ops.abs(x) q = q + 2.0 q = gen_math_ops.exp(q) q = gen_math_ops.log(q) q = math_ops.abs(q) q = q + 2.1 q = gen_math_ops.sqrt(q) q = gen_math_ops.rsqrt(q) q = math_ops.negative(q) q = math_ops.abs(q) q = q + 4.0 b = gen_math_ops.reciprocal(q) # TODO(jie): this one will break, broadcasting on batch. x = array_ops.placeholder( dtype=dtype, shape=input2_dims, name=input2_name) q = math_ops.abs(x) q = q + 5.0 q = gen_math_ops.exp(q) q = array_ops.squeeze(q, axis=[-1, -2, 3]) q = gen_math_ops.log(q) q = math_ops.abs(q) q = q + 5.1 q = gen_array_ops.reshape(q, [12, 5, 1, 1, 8, 1, 12]) q = array_ops.squeeze(q, axis=[5, 2, 3]) q = gen_math_ops.sqrt(q) q = math_ops.abs(q) q = q + 5.2 q = gen_math_ops.rsqrt(q) q = math_ops.negative(q) q = math_ops.abs(q) q = q + 5.3 c = gen_math_ops.reciprocal(q) q = a * b q = q / c array_ops.squeeze(q, name=self.output_name) return trt_test.TfTrtIntegrationTestParams( gdef=g.as_graph_def(), input_names=[input_name, input2_name], input_dims=[input_dims, input2_dims], num_expected_engines=5, expected_output_dims=(12, 5, 8, 12), allclose_atol=1.e-03, allclose_rtol=1.e-03)