def __init__(self, units: int, dtype: Union[str, type] = 'float32'):
"""Use a geometric sequence of timescales.
Parameters
----------
units
The number of units for positional embedding
dtype
The dtype of the inner positional embeddings
"""
super().__init__()
def _init_sinusodial_base(units):
half_units = units // 2
val = np.log(10000) / (half_units - 1)
val = np.exp(np.arange(half_units, dtype=np.float32) * -val)
return val
self._units = units
self._dtype = dtype
self.base_mult = Constant(_init_sinusodial_base(units))
def __init__(self):
super().__init__()
self.weight = Constant(np.ones((10, 10)))