def forward(self, positions):
"""
Parameters
----------
positions : NDArray
Shape (..., )
Returns
-------
ret :
Shape (..., units)
"""
emb = np.expand_dims(positions.astype(self._dtype),
axis=-1) * self.base_mult.data()
sin_emb = np.sin(emb)
cos_emb = np.cos(emb)
if self._units % 2 == 0:
return np.concatenate([sin_emb, cos_emb], axis=-1)
else:
return np.concatenate([
sin_emb, cos_emb,
np.expand_dims(np.zeros_like(positions).astype(self._dtype),
axis=-1)
],
axis=-1)
def __init__(self, num_hiddens, dropout, max_len=1000):
super(PositionalEncoding, self).__init__()
self.dropout = nn.Dropout(dropout)
# Create a long enough `P`
self.P = np.zeros((1, max_len, num_hiddens))
X = np.arange(0, max_len).reshape(-1, 1) / np.power(
10000, np.arange(0, num_hiddens, 2) / num_hiddens)
self.P[:, :, 0::2] = np.sin(X)
self.P[:, :, 1::2] = np.cos(X)
def get_positional_embeddings(length, depth) -> np.ndarray:
utils.check_condition(
depth % 2 == 0,
"Positional embeddings require an even embedding size it "
"is however %d." % depth)
# (1, depth)
channels = np.arange(depth // 2).reshape((1, -1))
# (length, 1)
positions = np.arange(0, length).reshape((-1, 1))
scaled_positions = positions / np.power(10000, (2 * channels) / depth)
# sinusoids:
sin = np.sin(scaled_positions)
# cosines:
cos = np.cos(scaled_positions)
# interleave: (length, num_embed)
encodings = np.hstack([sin, cos])
return encodings
def n_dim_array_operations():
x = np.array([1, 2, 4, 8])
y = np.array([2, 2, 2, 2])
print(x + y, x - y, x * y, x / y,
x**y) # The ** operator is exponentiation
print("e^x of {} = {}".format(x, np.exp(x)))
print("sin(x) of {} = {}".format(x, np.sin(x)))
x = np.arange(12).reshape(3, 4)
y = np.array([[2, 1, 4, 3], [1, 2, 3, 4], [4, 3, 2, 1]])
axis0 = np.concatenate([x, y], axis=0)
print("concat axis 0 : {}, shape {}".format(axis0, axis0.shape))
axis1 = np.concatenate([x, y], axis=1)
print("concat axis 1 : {}, shape {}".format(axis1, axis1.shape))
equal = x == y
greater = x > y
print("equal x = y: {} == {} = {}".format(x, y, equal))
print("greater x > y: {} > {} = {}".format(x, y, greater))
def memory():
x = np.arange(12).reshape(3, 4)
y = np.array([[2, 1, 4, 3], [1, 2, 3, 4], [4, 3, 2, 1]])
before = id(x)
x += y
equals = id(x) == before
print("id(x) after {} : id(x) before {} : equals {}".format(
id(x), before, equals))
z = np.sin(y)
print('id(z):', id(z))
z[:] = x + y
print('id(z):', id(z))
a = x.asnumpy()
b = np.array(a)
print("type a {}, type b {}, id(a) {}, id(b) {}".format(
type(a), type(b), id(a), id(b)))
a = np.array([3.5])
a, a.item(), float(a), int(a)
import d2l_dx
from mxnet import autograd, np, npx, gluon, init
from mxnet.gluon import nn
import matplotlib.pyplot as plt
npx.set_np()
T = 1000
time = np.arange(0, T)
x = np.sin(0.01 * time) + 0.2 * np.random.normal(size=T)
# plt.plot(x)
# plt.show()
tau = 4
features = np.zeros((T - tau, tau))
for i in range(tau):
features[:, i] = x[i: T - tau + i]
labels = x[tau:]
def f(x):
return np.sin(x)
def f(p):
return np.sin(p)
if b.sum() > 0:
c = b
else:
c = 100 * b
return c
a = np.random.normal()
a.attach_grad()
with autograd.record():
d = f(a)
d.backward()
a.grad
def f(p):
return np.sin(p)
p = np.arange(0, 100, 0.01)
p.attach_grad()
with autograd.record():
r = f(p)
r.backward()
p.grad
d2l.plot(p, [np.sin(p), 2 * p - 3],
'p',
'f(p)',
legend=['f(p)', 'Tangent line (p = 1)'])
