def test_cos(test_case):
input = flow.Tensor(np.random.randn(1, 3, 6), dtype=flow.float32)
of_out = flow.cos(input)
np_out = np.cos(input.numpy())
test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-5, 1e-5))
test_case.assertTrue(
np.allclose(input.cos().numpy(), np_out, 1e-5, 1e-5))
arr = np.array([1.4309, 1.2706, -0.8562, 0.9796])
input2 = flow.Tensor(arr, dtype=flow.float32)
np_out2 = np.array([0.13944048, 0.29570782, 0.6553126, 0.5573547])
of_out2 = flow.cos(input2)
test_case.assertTrue(np.allclose(of_out2.numpy(), np_out2))
def test_cos(test_case):
input = flow.Tensor(np.random.randn(1, 3, 6), dtype=flow.float32)
of_out = flow.cos(input)
np_out = np.cos(input.numpy())
test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-5, 1e-5))
test_case.assertTrue(
np.allclose(input.cos().numpy(), np_out, 1e-5, 1e-5))
def _test_cos(test_case, shape, device):
input = flow.tensor(np.random.randn(*shape),
dtype=flow.float32,
device=flow.device(device))
of_out = flow.cos(input)
np_out = np.cos(input.numpy())
test_case.assertTrue(np.allclose(of_out.numpy(), np_out, 1e-05, 1e-05))
def _test_cos_backward(test_case, shape, device):
x = flow.tensor(
np.random.randn(*shape),
dtype=flow.float32,
device=flow.device(device),
requires_grad=True,
)
y = flow.cos(x)
z = y.sum()
z.backward()
np_grad = -np.sin(x.numpy())
test_case.assertTrue(np.allclose(x.grad.numpy(), np_grad, 1e-05, 1e-05))
def __init__(self, d_model, dropout=0.1, max_len=5000):
super(PositionalEncoding, self).__init__()
self.dropout = nn.Dropout(p=dropout)
pe = flow.zeros((max_len, d_model))
position = flow.arange(0, max_len, dtype=flow.float).unsqueeze(1)
div_term = flow.exp(
flow.arange(0, d_model, 2).to(flow.float) * (-math.log(10000.0) / d_model)
).unsqueeze(0)
pe[:, 0::2] = flow.sin(position * div_term)
pe[:, 1::2] = flow.cos(position * div_term)
pe = pe.unsqueeze(0).transpose(0, 1)
self.pe = flow.nn.Parameter(pe, requires_grad=False)
def __init__(self, d_model, max_len=5000):
super(PositionalEncoding, self).__init__()
# Compute the positional encodings once in log space.
pe = flow.zeros(max_len, d_model, requires_grad=False)
position = flow.arange(0, max_len).unsqueeze(1).to(dtype=flow.float32)
div_term = flow.exp(
flow.arange(0, d_model, 2).to(dtype=flow.float32)
* -(math.log(10000.0) / d_model)
)
pe[:, 0::2] = flow.sin(position * div_term)
pe[:, 1::2] = flow.cos(position * div_term)
pe = pe.unsqueeze(0)
self.register_buffer("pe", pe)
def _embedding_from_positions(self, position):
"""get absolute pos embedding based position.
Args:
position (torch.Tensor): Input. Its shape is (b, t)
Returns:
posemb (torch.Tensor): Encoded tensor. Its shape is (b, time, emb_dim)
"""
batch_size, time_step = position.size()
posemb = flow.zeros(batch_size,
time_step,
self.emb_dim,
device=position.device)
div_term = flow.exp(
flow.arange(
0, self.emb_dim, 2, device=position.device, dtype=flow.float32)
* -(math.log(10000.0) / self.emb_dim))
posemb[:, :,
0::2] = flow.sin(position.float().unsqueeze(-1) * div_term)
posemb[:, :,
1::2] = flow.cos(position.float().unsqueeze(-1) * div_term)
return posemb
def _cos(self):
return flow.cos(self)
def __init__(
self,
out_channels,
kernel_size,
sample_rate=16000,
in_channels=1,
stride=1,
padding=0,
dilation=1,
bias=False,
groups=1,
min_low_hz=50,
min_band_hz=50,
):
super(SincConv_fast, self).__init__()
if in_channels != 1:
msg = (
"SincConv only support one input channel (here, in_channels = {%i})"
% (in_channels)
)
raise ValueError(msg)
self.out_channels = out_channels
self.kernel_size = kernel_size
# Forcing the filters to be odd (i.e, perfectly symmetrics)
if kernel_size % 2 == 0:
self.kernel_size = self.kernel_size + 1
self.stride = stride
self.padding = padding
self.dilation = dilation
if bias:
raise ValueError("SincConv does not support bias.")
if groups > 1:
raise ValueError("SincConv does not support groups.")
self.sample_rate = sample_rate
self.min_low_hz = min_low_hz
self.min_band_hz = min_band_hz
# initialize filterbanks such that they are equally spaced in Mel scale
low_hz = 30
high_hz = self.sample_rate / 2 - (self.min_low_hz + self.min_band_hz)
mel = np.linspace(
self.to_mel(low_hz), self.to_mel(high_hz), self.out_channels + 1
)
hz = self.to_hz(mel)
# filter lower frequency (out_channels, 1)
self.low_hz_ = nn.Parameter(flow.Tensor(hz[:-1]).reshape(-1, 1))
# filter frequency band (out_channels, 1)
self.band_hz_ = nn.Parameter(flow.Tensor(np.diff(hz)).reshape(-1, 1))
# Hamming window
n_lin = flow.Tensor(
np.linspace(0, (self.kernel_size / 2) - 1, int((self.kernel_size / 2)))
)
self.window_ = 0.54 - 0.46 * flow.cos(2 * math.pi * n_lin / self.kernel_size)
# (1, kernel_size/2)
n = (self.kernel_size - 1) / 2.0
self.n_ = (
2
* math.pi
* flow.Tensor(np.arange(-n, 0).reshape(1, -1) / self.sample_rate)
)