def learning_rate(step): # pylint: disable=invalid-name """Step to learning rate function.""" ret = 1.0 for name in factors: if name == "constant": ret *= constant elif name == "linear_warmup": ret *= np.minimum(1.0, step / warmup_steps) elif name == "rsqrt_decay": ret /= np.sqrt(np.maximum(step, warmup_steps)) else: raise ValueError("Unknown factor %s." % name) return ret
def learning_rate(step): # pylint: disable=invalid-name """Step to learning rate function.""" ret = 1.0 for name in factors: if name == "constant": ret *= constant elif name == "linear_warmup": ret *= np.minimum(1.0, step / warmup_steps) elif name == "rsqrt_decay": ret /= np.sqrt(np.maximum(step, warmup_steps)) elif name == "decay_every": ret *= (decay_factor**(step // steps_per_decay)) else: raise ValueError("Unknown factor %s." % name) ret = np.asarray(ret, dtype=np.float32) return {"learning_rate": ret}
def SaturationCost(x, limit=0.9): return np.minimum(0, np.abs(x) - limit)
def _minimum(self, tensor_list): minimum = tensor_list[0] for i in range(1, len(tensor_list)): minimum = np.minimum(minimum, tensor_list[i]) return minimum
def HardSigmoid(x, **unused_kwargs): """Linear approximation to sigmoid.""" return np.maximum(0, np.minimum(1, (1 + x)))
def HardTanh(x, **unused_kwargs): """Linear approximation to tanh.""" return np.maximum(-1, np.minimum(1, x))