def learning_rate(step): # pylint: disable=invalid-name
"""Step to learning rate function."""
ret = 1.0
for name in factors:
if name == "constant":
ret *= constant
elif name == "linear_warmup":
ret *= np.minimum(1.0, step / warmup_steps)
elif name == "rsqrt_decay":
ret /= np.sqrt(np.maximum(step, warmup_steps))
else:
raise ValueError("Unknown factor %s." % name)
return ret
def learning_rate(step): # pylint: disable=invalid-name
"""Step to learning rate function."""
ret = 1.0
for name in factors:
if name == "constant":
ret *= constant
elif name == "linear_warmup":
ret *= np.minimum(1.0, step / warmup_steps)
elif name == "rsqrt_decay":
ret /= np.sqrt(np.maximum(step, warmup_steps))
elif name == "decay_every":
ret *= (decay_factor**(step // steps_per_decay))
else:
raise ValueError("Unknown factor %s." % name)
ret = np.asarray(ret, dtype=np.float32)
return {"learning_rate": ret}
def SaturationCost(x, limit=0.9):
return np.minimum(0, np.abs(x) - limit)
def _minimum(self, tensor_list):
minimum = tensor_list[0]
for i in range(1, len(tensor_list)):
minimum = np.minimum(minimum, tensor_list[i])
return minimum
def HardSigmoid(x, **unused_kwargs):
"""Linear approximation to sigmoid."""
return np.maximum(0, np.minimum(1, (1 + x)))
def HardTanh(x, **unused_kwargs):
"""Linear approximation to tanh."""
return np.maximum(-1, np.minimum(1, x))
