def simple_bn(x, gamma, beta, moving_mean, moving_var, axis=1, epsilon=1e-5, shape=None): # expect = (x - moving_mean) / sqrt(moving_var + eps) * gamma + beta scale = rly.multiply(rly.const(1, 'float32') / rly.sqrt(moving_var + rly.const(epsilon, 'float32')), gamma) shift = rly.add( rly.multiply(rly.negative(moving_mean), scale), beta) num_newaxis = len(shape) - (axis + 1) if num_newaxis: scale = rly.expand_dims(scale, axis=1, num_newaxis=num_newaxis) shift = rly.expand_dims(shift, axis=1, num_newaxis=num_newaxis) return x * scale + shift
def simple_bn(x, gamma, beta, moving_mean, moving_var, axis=1, epsilon=1e-5, shape=None): # expect = (x - moving_mean) / sqrt(moving_var + eps) * gamma + beta scale = rly.multiply(rly.const(1, dtype) / rly.sqrt(moving_var + rly.const(epsilon, dtype)), gamma) shift = rly.add( rly.multiply(rly.negative(moving_mean), scale), beta) num_newaxis = len(shape) - (axis + 1) if num_newaxis: scale = rly.expand_dims(scale, axis=1, num_newaxis=num_newaxis) shift = rly.expand_dims(shift, axis=1, num_newaxis=num_newaxis) return x * scale + shift
def test_negative_grad(): data = relay.var("data", relay.TensorType((10, 4), "float32")) fwd_func = relay.Function([data], relay.negative(data)) check_grad(fwd_func)