def hard_sigmoid(x): out_dtype = scalar.upgrade_to_float(scalar.Scalar(dtype=x.dtype))[0].dtype slope = T.constant(0.2, dtype=out_dtype) shift = T.constant(0.5, dtype=out_dtype) x = (x * slope) + shift x = T.clip(x, 0, 1) return x
def PRelu(x): out_dtype = scalar.upgrade_to_float( scalar.Scalar(dtype=x.dtype))[0].dtype a = T.constant(0.625, dtype=out_dtype) b = T.constant(0.375, dtype=out_dtype) # x = (x * slope) + shift y = x * a + abs(x) * b r = T.clip(y, 0, 1) return r
def Relu(x): out_dtype = scalar.upgrade_to_float( scalar.Scalar(dtype=x.dtype))[0].dtype a = T.constant(0.5, dtype=out_dtype) # ab = T.constant(abs(x), dtype=out_dtype) # x = (x * slope) + shift y = (x + abs(x)) * a r = T.clip(y, 0, 1) return r
def hard_sigmoid(x): """An approximation of sigmoid. More approximate and faster than ultra_fast_sigmoid. Approx in 3 parts: 0, scaled linear, 1 Removing the slope and shift does not make it faster. """ # Use the same dtype as determined by "upgrade_to_float", # and perform computation in that dtype. out_dtype = scalar.upgrade_to_float(scalar.Scalar(dtype=x.dtype))[0].dtype slope = tensor.constant(0.2, dtype=out_dtype) shift = tensor.constant(0.5, dtype=out_dtype) x = (x * slope) + shift x = tensor.clip(x, 0, 1) return x