def forward(self, positions): """ Parameters ---------- positions : NDArray Shape (..., ) Returns ------- ret : Shape (..., units) """ emb = np.expand_dims(positions.astype(self._dtype), axis=-1) * self.base_mult.data() sin_emb = np.sin(emb) cos_emb = np.cos(emb) if self._units % 2 == 0: return np.concatenate([sin_emb, cos_emb], axis=-1) else: return np.concatenate([ sin_emb, cos_emb, np.expand_dims(np.zeros_like(positions).astype(self._dtype), axis=-1) ], axis=-1)
def __init__(self, num_hiddens, dropout, max_len=1000): super(PositionalEncoding, self).__init__() self.dropout = nn.Dropout(dropout) # Create a long enough `P` self.P = np.zeros((1, max_len, num_hiddens)) X = np.arange(0, max_len).reshape(-1, 1) / np.power( 10000, np.arange(0, num_hiddens, 2) / num_hiddens) self.P[:, :, 0::2] = np.sin(X) self.P[:, :, 1::2] = np.cos(X)
def get_positional_embeddings(length, depth) -> np.ndarray: utils.check_condition( depth % 2 == 0, "Positional embeddings require an even embedding size it " "is however %d." % depth) # (1, depth) channels = np.arange(depth // 2).reshape((1, -1)) # (length, 1) positions = np.arange(0, length).reshape((-1, 1)) scaled_positions = positions / np.power(10000, (2 * channels) / depth) # sinusoids: sin = np.sin(scaled_positions) # cosines: cos = np.cos(scaled_positions) # interleave: (length, num_embed) encodings = np.hstack([sin, cos]) return encodings
def n_dim_array_operations(): x = np.array([1, 2, 4, 8]) y = np.array([2, 2, 2, 2]) print(x + y, x - y, x * y, x / y, x**y) # The ** operator is exponentiation print("e^x of {} = {}".format(x, np.exp(x))) print("sin(x) of {} = {}".format(x, np.sin(x))) x = np.arange(12).reshape(3, 4) y = np.array([[2, 1, 4, 3], [1, 2, 3, 4], [4, 3, 2, 1]]) axis0 = np.concatenate([x, y], axis=0) print("concat axis 0 : {}, shape {}".format(axis0, axis0.shape)) axis1 = np.concatenate([x, y], axis=1) print("concat axis 1 : {}, shape {}".format(axis1, axis1.shape)) equal = x == y greater = x > y print("equal x = y: {} == {} = {}".format(x, y, equal)) print("greater x > y: {} > {} = {}".format(x, y, greater))
def memory(): x = np.arange(12).reshape(3, 4) y = np.array([[2, 1, 4, 3], [1, 2, 3, 4], [4, 3, 2, 1]]) before = id(x) x += y equals = id(x) == before print("id(x) after {} : id(x) before {} : equals {}".format( id(x), before, equals)) z = np.sin(y) print('id(z):', id(z)) z[:] = x + y print('id(z):', id(z)) a = x.asnumpy() b = np.array(a) print("type a {}, type b {}, id(a) {}, id(b) {}".format( type(a), type(b), id(a), id(b))) a = np.array([3.5]) a, a.item(), float(a), int(a)
import d2l_dx from mxnet import autograd, np, npx, gluon, init from mxnet.gluon import nn import matplotlib.pyplot as plt npx.set_np() T = 1000 time = np.arange(0, T) x = np.sin(0.01 * time) + 0.2 * np.random.normal(size=T) # plt.plot(x) # plt.show() tau = 4 features = np.zeros((T - tau, tau)) for i in range(tau): features[:, i] = x[i: T - tau + i] labels = x[tau:]
def f(x): return np.sin(x)
def f(p): return np.sin(p)
if b.sum() > 0: c = b else: c = 100 * b return c a = np.random.normal() a.attach_grad() with autograd.record(): d = f(a) d.backward() a.grad def f(p): return np.sin(p) p = np.arange(0, 100, 0.01) p.attach_grad() with autograd.record(): r = f(p) r.backward() p.grad d2l.plot(p, [np.sin(p), 2 * p - 3], 'p', 'f(p)', legend=['f(p)', 'Tangent line (p = 1)'])