def forward(self, positions): """ Parameters ---------- positions : NDArray Shape (..., ) Returns ------- ret : Shape (..., units) """ emb = np.expand_dims(positions.astype(self._dtype), axis=-1) * self.base_mult.data() sin_emb = np.sin(emb) cos_emb = np.cos(emb) if self._units % 2 == 0: return np.concatenate([sin_emb, cos_emb], axis=-1) else: return np.concatenate([ sin_emb, cos_emb, np.expand_dims(np.zeros_like(positions).astype(self._dtype), axis=-1) ], axis=-1)
def __init__(self, num_hiddens, dropout, max_len=1000): super(PositionalEncoding, self).__init__() self.dropout = nn.Dropout(dropout) # Create a long enough `P` self.P = np.zeros((1, max_len, num_hiddens)) X = np.arange(0, max_len).reshape(-1, 1) / np.power( 10000, np.arange(0, num_hiddens, 2) / num_hiddens) self.P[:, :, 0::2] = np.sin(X) self.P[:, :, 1::2] = np.cos(X)
def get_positional_embeddings(length, depth) -> np.ndarray: utils.check_condition( depth % 2 == 0, "Positional embeddings require an even embedding size it " "is however %d." % depth) # (1, depth) channels = np.arange(depth // 2).reshape((1, -1)) # (length, 1) positions = np.arange(0, length).reshape((-1, 1)) scaled_positions = positions / np.power(10000, (2 * channels) / depth) # sinusoids: sin = np.sin(scaled_positions) # cosines: cos = np.cos(scaled_positions) # interleave: (length, num_embed) encodings = np.hstack([sin, cos]) return encodings