def create_batches_rnd(batch_size, data_folder, wav_lst, N_snt, wlen, lab_dict,
fact_amp):
# Initialization of the minibatch (batch_size,[0=>x_t,1=>x_t+N,1=>random_samp])
sig_batch = np.zeros([batch_size, wlen])
lab_batch = np.zeros(batch_size)
snt_id_arr = np.random.randint(N_snt, size=batch_size)
rand_amp_arr = np.random.uniform(1.0 - fact_amp, 1 + fact_amp, batch_size)
for i in range(batch_size):
# select a random sentence from the list
[signal, fs] = sf.read(data_folder + wav_lst[snt_id_arr[i]])
# accesing to a random chunk
snt_len = signal.shape[0]
snt_beg = np.random.randint(snt_len - wlen - 1)
snt_end = snt_beg + wlen
channels = len(signal.shape)
if channels == 2:
print("WARNING: stereo to mono: " + data_folder +
wav_lst[snt_id_arr[i]])
signal = signal[:, 0]
sig_batch[i, :] = signal[snt_beg:snt_end] * rand_amp_arr[i]
lab_batch[i] = lab_dict[wav_lst[snt_id_arr[i]].lower()]
inp = flow.Tensor(sig_batch).to("cuda")
lab = flow.Tensor(lab_batch).to("cuda")
return inp, lab
def test_module_cpu_cuda(test_case):
class CustomModule(flow.nn.Module):
def __init__(self, param1, param2):
super().__init__()
self.param1 = param1
self.param2 = param2
tensor0 = flow.nn.Parameter(
flow.Tensor(2, 3, device=flow.device("cpu")))
tensor1 = flow.nn.Parameter(
flow.Tensor(2, 3, device=flow.device("cpu")))
sub_module = CustomModule(tensor0, tensor1)
m = CustomModule(tensor1, sub_module)
m.cuda()
state_dict = m.state_dict()
test_case.assertEqual(state_dict["param2.param1"].device,
flow.device("cuda:0"))
test_case.assertEqual(state_dict["param2.param2"].device,
flow.device("cuda:0"))
m.cpu()
state_dict = m.state_dict()
test_case.assertEqual(state_dict["param2.param1"].device,
flow.device("cpu"))
test_case.assertEqual(state_dict["param2.param2"].device,
flow.device("cpu"))
def _test_conv1d_bias_true(test_case, device):
np_arr = np.array(
[
[
[0.90499806, -1.11683071, 0.71605605, -0.56754625, 0.61944169],
[-0.31317389, -0.26271924, 0.95579433, 0.52468461, 1.48926127],
]
]
)
input = flow.tensor(
np_arr, dtype=flow.float32, device=flow.device(device), requires_grad=True
)
weight = np.array(
[
[
[0.01997352, 0.23834395, 0.00526353],
[-0.04861857, -0.22751901, -0.06725175],
],
[
[0.13344523, -0.35202524, 0.15168799],
[-0.25714493, -0.17459838, 0.28768948],
],
[
[0.10671382, -0.28205597, -0.39752254],
[0.36393702, 0.07843742, -0.33898622],
],
[
[0.20485674, 0.04222689, -0.1898618],
[0.22519711, -0.15910202, -0.35057363],
],
]
)
bias = np.array([0.01012857, 0.38912651, -0.01600273, -0.3883304])
m = nn.Conv1d(2, 4, 3, stride=1, bias=True)
m.weight = flow.nn.Parameter(flow.Tensor(weight))
m.bias = flow.nn.Parameter(flow.Tensor(bias))
m = m.to(device)
np_out = np.array(
[
[
[-0.22349545, -0.08447243, -0.37358052],
[1.4130373, -0.04644597, 0.86949122],
[-0.34765026, -0.31004351, -0.14158708],
[-0.74985039, -0.87430149, -0.77354753],
]
]
)
output = m(input)
test_case.assertTrue(np.allclose(output.numpy(), np_out, 1e-06, 1e-06))
output = output.sum()
output.backward()
np_grad = np.array(
[
[
[0.4649893, 0.11147892, -0.3189539, -0.78394318, -0.43043283],
[0.28337064, -0.19941133, -0.66853344, -0.95190406, -0.46912211],
]
]
)
test_case.assertTrue(np.allclose(input.grad.numpy(), np_grad, 1e-06, 1e-06))
def __init__(
self,
in_channels: int,
out_channels: int,
kernel_size: _size_1_t,
stride: _size_1_t = 1,
padding: Union[str, _size_1_t] = 0,
dilation: _size_1_t = 1,
groups: int = 1,
bias: bool = True,
padding_mode: str = "zeros",
):
super().__init__()
assert padding_mode == "zeros"
self.padding_mode = padding_mode
self.kernel_size = _single(kernel_size)
self.stride = _single(stride)
self.dilation = _single(dilation)
self.padding = (get_padding(padding, self.kernel_size,
self.dilation, self.stride) if isinstance(
padding, str) else _single(padding))
self.groups = groups
self.channel_pos = "channels_first"
assert in_channels % groups == 0
assert out_channels % groups == 0
self.in_channels = in_channels
self.out_channels = out_channels
self.weight = flow.nn.Parameter(
flow.Tensor(out_channels, in_channels // groups,
*self.kernel_size))
self.out_channel_groups = out_channels // groups
self.bias = None
if bias:
self.bias = flow.nn.Parameter(flow.Tensor(out_channels))
self.reset_parameters()
def __init__(
self,
in_channels: int,
out_channels: int,
kernel_size: _size_2_t,
stride: _size_2_t = 1,
padding: _size_2_t = 0,
output_padding: _size_2_t = 0,
groups: int = 1,
bias: bool = True,
dilation: int = 1,
padding_mode: str = "zeros",
) -> None:
super().__init__()
assert padding_mode == "zeros"
self.kernel_size = _pair(kernel_size)
self.stride = _pair(stride)
self.padding = _pair(padding)
self.output_padding = _pair(output_padding)
self.dilation = _pair(dilation)
self.groups = groups
assert in_channels % groups == 0
assert out_channels % groups == 0
self.weight = flow.nn.Parameter(
flow.Tensor(in_channels, out_channels // groups,
*self.kernel_size))
self.in_channel_groups = in_channels // groups
self.filters = out_channels
self.bias = None
self._bias_add_op = None
if bias:
self.bias = flow.nn.Parameter(flow.Tensor(out_channels))
self.reset_parameters()
def test_concat_with_axis_one(test_case):
input1 = flow.Tensor(np.random.randn(2, 6, 5, 3), dtype=flow.float32)
input2 = flow.Tensor(np.random.randn(2, 6, 5, 3), dtype=flow.float32)
of_out = flow.cat([input1, input2], dim=1)
np_out = np.concatenate((input1.numpy(), input2.numpy()), axis=1)
test_case.assertTrue(np.array_equal(of_out.numpy(), np_out))
def _test_fused_tril_softmax_mask_scale(test_case, seq_length, channel, p,
diagonal, tril_scale_value):
x = np.random.randn(4, seq_length, channel)
# fused version only support in GPU
fused_x_tensor = flow.Tensor(x).to("cuda")
fused_x_tensor.requires_grad = True
fused_out = flow._C.fused_scale_tril_softmax_mask_scale(
fused_x_tensor,
p=p,
diagonal=diagonal,
tril_scale_value=tril_scale_value)[0] # The second output is softmax_y
origin_x_tensor = flow.Tensor(x).to("cuda")
origin_x_tensor.requires_grad = True
origin_out = flow.tril(origin_x_tensor, diagonal)
origin_out = origin_out * tril_scale_value
origin_out = flow.softmax(origin_out, dim=-1)
origin_out = flow._C.dropout(origin_out, p=p)
total_out = fused_out.sum() + origin_out.sum()
total_out.backward()
test_case.assertTrue(
np.allclose(fused_out.numpy(),
origin_out.numpy(),
atol=1e-4,
rtol=1e-4))
test_case.assertTrue(
np.allclose(
fused_x_tensor.grad.numpy(),
origin_x_tensor.grad.numpy(),
atol=1e-4,
rtol=1e-4,
))
def test_tensor_register_post_grad_accumulation_hook(test_case):
shape = (2, 3)
x = flow.Tensor(*shape)
x.requires_grad = True
x._register_post_grad_accumulation_hook(lambda grad: grad * 2 + 1)
y = x.sum() + (x * 2).sum()
y.backward()
test_case.assertTrue(
np.allclose(x.grad.numpy(),
np.ones(shape) * 7,
atol=1e-4,
rtol=1e-4))
x = flow.Tensor(*shape)
x.requires_grad = True
def inplace_add_and_return_none(x):
x.add_(1)
return None
x._register_post_grad_accumulation_hook(inplace_add_and_return_none)
y = x.sum() + (x * 2).sum()
y.backward()
test_case.assertTrue(
np.allclose(x.grad.numpy(),
np.ones(shape) * 4,
atol=1e-4,
rtol=1e-4))
def test_tensor_register_hook(test_case):
shape = (2, 3)
x = flow.Tensor(*shape)
x.requires_grad = True
x.register_hook(lambda grad: grad * 2 + 1)
y = x.sum() + (x * 2).sum()
y.backward()
test_case.assertTrue(
np.allclose(x.grad.numpy(),
np.ones(shape) * 7,
atol=1e-4,
rtol=1e-4))
x = flow.Tensor(*shape)
x.requires_grad = True
new_grad = flow.Tensor([[1, 2, 3], [4, 5, 6]])
x.register_hook(lambda _: new_grad)
y = x.sum() + (x * 2).sum()
y.backward()
test_case.assertTrue(np.allclose(x.grad.numpy(), new_grad.numpy()))
grad_nonlocal = None
def assign_nonlocal_variable_and_return_none(grad):
nonlocal grad_nonlocal
grad_nonlocal = grad
x = flow.Tensor(*shape)
x.requires_grad = True
new_grad = flow.tensor([[1, 2, 3], [4, 5, 6]], dtype=flow.float32)
x.register_hook(assign_nonlocal_variable_and_return_none)
y = x.sum() + (x * 2).sum()
y.backward()
test_case.assertTrue(
np.allclose(grad_nonlocal.numpy(),
np.ones(shape) * 3))
def test_module_setattr(test_case):
class CustomModule(flow.nn.Module):
def __init__(self, param1, param2):
super().__init__()
self.param1 = param1
self.param2 = param2
param0 = flow.nn.Parameter(flow.Tensor(2, 3))
param1 = flow.nn.Parameter(flow.Tensor(2, 3))
param2 = CustomModule(param0, param1)
m = CustomModule(param1, param2)
# m.parameters() contains param0 + param1 in submodule param2
# and param1 in m
params = list(m.parameters())
test_case.assertEqual(len(params), 2)
test_case.assertEqual(params[0], param1)
test_case.assertEqual(params[1], param0)
children = list(m.children())
test_case.assertEqual(len(children), 1)
child = children[0]
test_case.assertEqual(child, param2)
child_params = list(child.parameters())
test_case.assertEqual(len(child_params), 2)
test_case.assertEqual(child_params[0], param0)
test_case.assertEqual(child_params[1], param1)
def test_tensor_autograd_related_methods(test_case):
shape = (2, 3, 4, 5)
x = flow.Tensor(*shape)
y = flow.Tensor(*shape)
y.requires_grad = True
x.fill_(1.0)
y.fill_(2.0)
z = x + y
test_case.assertFalse(x.requires_grad)
test_case.assertTrue(x.is_leaf)
test_case.assertTrue(y.requires_grad)
test_case.assertTrue(y.is_leaf)
test_case.assertTrue(z.requires_grad)
test_case.assertFalse(z.is_leaf)
with flow.no_grad():
m = x + y
test_case.assertTrue(m.is_leaf)
test_case.assertFalse(m.requires_grad)
m.requires_grad = True
v = flow.Tensor(*shape)
v.requires_grad = True
z.retain_grad()
w = v + z
grad = flow.Tensor(*shape)
grad.fill_(1.0)
w.backward(gradient=grad, retain_graph=True)
test_case.assertTrue(
np.allclose(v.grad.numpy(), np.ones(shape), atol=1e-4, rtol=1e-4))
test_case.assertTrue(
np.allclose(y.grad.numpy(), np.ones(shape), atol=1e-4, rtol=1e-4))
test_case.assertTrue(
np.allclose(z.grad.numpy(), np.ones(shape), atol=1e-4, rtol=1e-4))
test_case.assertIsNone(x.grad)
w.backward(gradient=grad, retain_graph=True)
def __init__(self, features, eps=1e-6):
super(LayerNorm, self).__init__()
self.eps = eps
self.weight = nn.Parameter(
flow.Tensor(flow.ones(features, dtype=flow.float32)))
self.bias = nn.Parameter(
flow.Tensor(flow.zeros(features, dtype=flow.float32)))
def _test_roi_align(test_case, device):
input = flow.Tensor(np.random.randn(2, 3, 64, 64),
dtype=flow.float32,
device=flow.device(device))
random_img_idx = np.random.randint(low=0, high=2, size=(200, 1))
random_box_idx = np.random.uniform(low=0, high=64 * 64,
size=(200, 2)).astype(np.float32)
def get_h_w(idx1, idx2):
if idx1 > idx2:
idx1, idx2 = idx2, idx1
h1 = idx1 // 64
w1 = idx1 % 64
h2 = idx2 // 64
w2 = idx2 % 64
return [x / 2 for x in [h1, w1, h2, w2]]
zipped = zip(random_box_idx[:, 0], random_box_idx[:, 1])
concated = [get_h_w(idx1, idx2) for (idx1, idx2) in zipped]
concated = np.array(concated)
rois = flow.Tensor(
np.hstack((random_img_idx, concated)),
dtype=flow.float32,
device=flow.device(device),
)
roi_align_module = RoIAlign((14, 14), 2.0, 2, True)
of_out = roi_align_module(input, rois)
np_out = roi_align_np(input.numpy(), rois.numpy(), 14, 14, 2.0, 2, True)
test_case.assertTrue(
np.allclose(of_out.numpy(), np_out, rtol=1e-4, atol=1e-4))
def __init__(
self,
in_channels: int,
out_channels: int,
kernel_size: _size_2_t,
stride: _size_2_t = 1,
padding: _size_2_t = 0,
dilation: _size_2_t = 1,
groups: int = 1,
bias: bool = True,
padding_mode: str = "zeros", # TODO: refine this type
):
super().__init__()
assert padding_mode == "zeros"
kernel_size = _pair(kernel_size)
stride = _pair(stride)
padding = _pair(padding)
dilation = _pair(dilation)
self.groups = groups
self.weight = flow.nn.Parameter(
flow.Tensor(out_channels, in_channels // groups, *kernel_size))
self.bias = None
self._bias_add_op = None
if bias:
self.bias = flow.nn.Parameter(flow.Tensor(out_channels))
self._bias_add_op = (flow.builtin_op("bias_add").Input("a").Input(
"b").Output("out").Attr("axis", 1).Build())
self._op = (flow.builtin_op("conv2d").Input("in").Input("weight").Attr(
"filters", out_channels).Attr("padding_before", padding).Attr(
"strides", stride).Attr("kernel_size", kernel_size).Attr(
"dilation_rate", dilation).Attr("groups", groups).Attr(
"data_format", "channels_first").Output("out").Build())
self.reset_parameters()
def _test_ddp_basic(test_case, dev_type):
class Mul(flow.nn.Module):
def __init__(self):
super().__init__()
self.w = flow.nn.Parameter(flow.Tensor([1, 1]))
def forward(self, x):
return x * self.w
rank = flow.env.get_rank()
if rank == 0:
x = flow.Tensor([1, 1])
elif rank == 1:
x = flow.Tensor([2, 2])
else:
raise ValueError()
x = x.to(dev_type)
m = Mul().to(dev_type)
m = ddp(m)
y = m(x)
y.sum().backward()
test_case.assertTrue(
np_allclose_with_shape(m.w.grad.numpy(), np.array([1.5, 1.5]))
)
def test_indexing(test_case):
class SliceExtracter:
def __getitem__(self, key):
return key
se = SliceExtracter()
def compare_getitem_with_numpy(tensor, slices):
np_arr = tensor.numpy()
test_case.assertTrue(
np.allclose(np_arr[slices], tensor[slices].numpy()))
def compare_setitem_with_numpy(tensor, slices, value):
np_arr = tensor.numpy()
if isinstance(value, flow.Tensor):
np_value = value.numpy()
else:
np_value = value
np_arr[slices] = np_value
tensor[slices] = value
test_case.assertTrue(np.allclose(np_arr, tensor.numpy()))
x = flow.randn(5, 5)
v = flow.Tensor([[0, 1, 2, 3, 4]])
compare_getitem_with_numpy(x, se[-4:-1:2])
compare_getitem_with_numpy(x, se[-1:])
compare_setitem_with_numpy(x, se[-1:], v)
compare_setitem_with_numpy(x, se[2::2], 2)
x = flow.Tensor(2, 3, 4)
v = flow.Tensor(3)
compare_setitem_with_numpy(x, se[:, :, 2], v)
x = flow.Tensor(2, 3, 4)
compare_setitem_with_numpy(x, se[1, :, 2], v)
def _test_ddp_with_unused_param(test_case, dev_type):
class Model(flow.nn.Module):
def __init__(self):
super().__init__()
self.w = flow.nn.Parameter(flow.Tensor([1]))
self.used_only_in_rank0 = flow.nn.Parameter(flow.Tensor([2]))
self.unused_in_all_ranks = flow.nn.Parameter(flow.Tensor([3]))
def forward(self, x):
x = x * self.w
if flow.env.get_rank() == 0:
x = x * self.used_only_in_rank0
return x
rank = flow.env.get_rank()
if rank == 0:
x = flow.Tensor([1])
elif rank == 1:
x = flow.Tensor([2])
else:
raise ValueError()
x = x.to(dev_type)
m = Model().to(dev_type)
m = ddp(m)
y = m(x)
y.backward()
test_case.assertTrue(np_allclose_with_shape(m.w.grad.numpy(), np.array([2])))
test_case.assertTrue(
np_allclose_with_shape(m.used_only_in_rank0.grad.numpy(), np.array([0.5]))
)
test_case.assertTrue(
np_allclose_with_shape(m.unused_in_all_ranks.grad.numpy(), np.array([0]))
)
def __init__(
self,
n_heads,
d_model,
dropout_rate=0.0,
skip_term_b=False,
share_qvk_proj=False,
):
super(MultiHeadedSelfAttentionWithRelPos, self).__init__(
n_heads, d_model, dropout_rate, share_qvk_proj
)
self.d_model = d_model
self.share_qvk_proj = share_qvk_proj
self.skip_term_b = skip_term_b
self.nheads = n_heads
self.d_k = d_model // n_heads
self.qvk_proj = nn.Linear(
d_model, d_model if self.share_qvk_proj else d_model * 3
)
self.pos_proj = nn.Linear(d_model, d_model, bias=False)
self.posu = nn.Parameter(flow.Tensor(1, 1, n_heads, self.d_k))
self.posv = nn.Parameter(flow.Tensor(1, 1, n_heads, self.d_k))
def __init__(
self,
num_features: int,
eps: float = 1e-5,
momentum: float = 0.1,
affine: bool = True,
track_running_stats: bool = True,
) -> None:
super().__init__()
self.num_features = num_features
self.eps = eps
self.momentum = momentum
self.affine = affine
self.track_running_stats = track_running_stats
if self.affine:
self.weight = flow.nn.Parameter(flow.Tensor(num_features))
self.bias = flow.nn.Parameter(flow.Tensor(num_features))
else:
self.register_parameter("weight", None)
self.register_parameter("bias", None)
if self.track_running_stats:
self.register_buffer(
"running_mean",
flow.Tensor(num_features),
)
self.register_buffer(
"running_var",
flow.Tensor(num_features),
)
else:
self.register_parameter("running_mean", None)
self.register_parameter("running_var", None)
self.reset_parameters()
def _test_ddp_multiple_buckets(test_case, dev_type):
class Mul(flow.nn.Module):
def __init__(self):
super().__init__()
for i in range(10):
self.register_parameter(
f"w{i}",
flow.nn.Parameter(flow.Tensor([i % 2 + 1, i % 2 + 1])))
def forward(self, x):
for i in range(10):
x = x * getattr(self, f"w{i}")
return x
rank = flow.env.get_rank()
if rank == 0:
x = flow.Tensor([1, 1])
elif rank == 1:
x = flow.Tensor([2, 2])
else:
raise ValueError()
x = x.to(dev_type)
m = Mul().to(dev_type)
m = ddp(m, bucket_size=3)
y = m(x)
y.sum().backward()
for i in range(10):
test_case.assertTrue(
np_allclose_with_shape(
getattr(m, f"w{i}").grad.numpy(),
np.array([48, 48]) if i % 2 == 0 else np.array([24, 24]),
))
def __init__(
self,
normalized_shape: _shape_t,
eps: float = 1e-5,
elementwise_affine: bool = True,
) -> None:
super(LayerNorm, self).__init__()
if isinstance(normalized_shape, int):
# mypy error: incompatible types in assignment
normalized_shape = (normalized_shape, ) # type: ignore[assignment]
self.normalized_shape = tuple(
normalized_shape) # type: ignore[arg-type]
self.epsilon = eps
self.elementwise_affine = elementwise_affine
if self.elementwise_affine:
self.weight = flow.nn.Parameter(
flow.Tensor(*self.normalized_shape))
self.bias = flow.nn.Parameter(flow.Tensor(*self.normalized_shape))
else:
self.register_parameter("weight", None)
self.register_parameter("bias", None)
self.reset_parameters()
# An integer specifies which axis to normalize at first, defaults to 1.
self.begin_norm_axis = 1
# An integer specifies which axis params at, defaults to 1 in 'NCHW' format
self.begin_params_axis = 1
self._op = (flow.builtin_op("layer_norm").Input("x").Input(
"gamma").Input("beta").Output("y").Output("mean").Output(
"inv_variance").Output("normalized").Build())
self._op2 = (flow.builtin_op("layer_norm").Input("x").Output(
"y").Output("mean").Output("inv_variance").Build())
def test_module_setattr(test_case):
class CustomModule(flow.nn.Module):
def __init__(self, param1, param2):
super().__init__()
self.param1 = param1
self.param2 = param2
param0 = flow.nn.Parameter(flow.Tensor(2, 3))
param1 = flow.nn.Parameter(flow.Tensor(2, 3))
param2 = CustomModule(param0, param1)
m = CustomModule(param1, param2)
params = list(m.parameters())
test_case.assertEqual(len(params), 2)
test_case.assertTrue(
np.allclose(params[0].numpy(),
param1.numpy(),
atol=1e-4,
rtol=1e-4))
test_case.assertTrue(
np.allclose(params[1].numpy(),
param0.numpy(),
atol=1e-4,
rtol=1e-4))
children = list(m.children())
test_case.assertEqual(len(children), 1)
child = children[0]
test_case.assertEqual(child, param2)
child_params = list(child.parameters())
test_case.assertEqual(len(child_params), 2)
test_case.assertTrue(
np.allclose(child_params[0].numpy(), param0.numpy()))
test_case.assertTrue(
np.allclose(child_params[1].numpy(), param1.numpy()))
def __init__(
self,
in_channels: int,
out_channels: int,
kernel_size: _size_1_t,
stride: _size_1_t = 1,
padding: _size_1_t = 0,
output_padding: _size_1_t = 0,
groups: int = 1,
bias: bool = True,
dilation: _size_1_t = 1,
padding_mode: str = "zeros",
) -> None:
super().__init__()
assert (padding_mode == "zeros"
), "Only `zeros` padding mode is supported for ConvTranspose1d"
self.kernel_size = _single(kernel_size)
self.stride = _single(stride)
self.padding = _single(padding)
self.dilation = _single(dilation)
self.output_padding = _single(output_padding)
self.groups = groups
assert in_channels % groups == 0
assert out_channels % groups == 0
self.weight = flow.nn.Parameter(
flow.Tensor(in_channels, out_channels // groups,
*self.kernel_size))
self.filters = out_channels
self.bias = None
self._bias_add_op = None
if bias:
self.bias = flow.nn.Parameter(flow.Tensor(out_channels))
self.reset_parameters()
def __init__(self, in_features: int, out_features: int, bias: bool = True) -> None:
super().__init__()
self.use_bias = bias
self.weight = flow.nn.Parameter(flow.Tensor(out_features, in_features))
self.bias = None
if bias:
self.bias = flow.nn.Parameter(flow.Tensor(out_features))
self._matmul_op = (
flow.builtin_op("matmul")
.Input("a")
.Input("b")
.Output("out")
.Attr("transpose_a", False)
.Attr("transpose_b", True)
.Attr("alpha", 1.0)
.Build()
)
self._broadcast_matmul_op = (
flow.builtin_op("broadcast_matmul")
.Input("a")
.Input("b")
.Output("out")
.Attr("transpose_a", False)
.Attr("transpose_b", True)
.Attr("alpha", 1.0)
.Build()
)
self.reset_parameters()
def example_precess(wav, lab, wlen=3200, fact_amp=0.2):
np.random.seed(10)
sig_batch = np.zeros([1, wlen])
lab_batch = np.zeros(1)
rand_amp_arr = np.random.uniform(1.0 - fact_amp, 1 + fact_amp, 1)
[signal, fs] = sf.read(wav)
snt_len = signal.shape[0]
snt_beg = np.random.randint(snt_len - wlen - 1)
snt_end = snt_beg + wlen
channels = len(signal.shape)
if channels == 2:
print("WARNING: stereo to mono")
signal = signal[:, 0]
sig_batch[0, :] = signal[snt_beg:snt_end] * rand_amp_arr[0]
lab_batch[0] = int(lab)
inp = flow.Tensor(sig_batch, dtype=flow.float32).to("cuda")
lab = flow.Tensor(lab_batch, dtype=flow.float32).to("cuda")
return inp, lab
def _test_conv1d_compilcate(test_case, device):
np_arr = np.array([[
[-1.00674784, 0.51784992, 0.39896572, 0.11018554, 0.91136694],
[1.95886874, 0.89779067, 0.4748213, 0.33313531, -0.49350029],
[-0.19280219, 0.04023677, 1.66438103, -0.83563608, 0.15925731],
[1.49166429, 1.45189261, -1.86512125, 0.34329697, 0.20413807],
]])
input = flow.tensor(np_arr,
dtype=flow.float32,
device=flow.device(device),
requires_grad=True)
weight = np.array([
[
[-0.36045218, 0.37349278, 0.04565236],
[0.0242328, -0.09459515, -0.30684742],
],
[
[-0.30345008, -0.1196513, -0.26765293],
[0.09876197, 0.03346226, 0.2748405],
],
[
[-0.37798449, 0.00242459, -0.34125558],
[-0.05174343, -0.10443231, 0.09526101],
],
[
[0.34196907, -0.32667893, 0.40264183],
[0.38025281, 0.26807079, -0.09074812],
],
])
bias = np.array([-0.03499984, -0.21616256, 0.13312563, -0.24104381])
m = nn.Conv1d(4,
4,
3,
groups=2,
stride=2,
padding=2,
dilation=2,
bias=True)
m.weight = flow.nn.Parameter(flow.Tensor(weight))
m.bias = flow.nn.Parameter(flow.Tensor(bias))
m = m.to(device)
np_out = np.array([[
[-0.72379637, 0.67248386, 0.21977007],
[-0.00643994, -0.1286152, -0.41589433],
[-0.76877236, 0.29273134, -0.42040929],
[1.0612179, -0.73787093, -0.37839717],
]])
output = m(input)
test_case.assertTrue(np.allclose(output.numpy(), np_out, 1e-06, 1e-06))
output = output.sum()
output.backward()
np_grad = np.array([[
[-0.41006082, 0.0, -0.63206136, 0.0, 0.03184089],
[0.06186188, 0.0, 0.02985496, 0.0, -0.09313981],
[-0.36026976, 0.0, -0.2988835, 0.0, -0.26286808],
[0.49214786, 0.0, 0.49666074, 0.0, 0.16815135],
]])
test_case.assertTrue(np.allclose(input.grad.numpy(), np_grad, 1e-06,
1e-06))
def __init__(
self,
input_size: int,
hidden_size: int,
num_layers: int = 1,
bias: bool = True,
batch_first: bool = False,
dropout: float = 0,
bidirectional: bool = False,
):
super().__init__()
self.input_size = input_size
self.hidden_size = hidden_size
self.num_layers = num_layers
self.bias = bias
self.batch_first = batch_first
self.dropout = dropout
self.bidirectional = bidirectional
num_directions = 2 if bidirectional else 1
gate_size = 3 * hidden_size
self.drop = nn.Dropout(self.dropout)
for layer in range(num_layers):
for direction in range(num_directions):
real_hidden_size = hidden_size
layer_input_size = (
input_size if layer == 0 else real_hidden_size * num_directions
)
# TODO: Modify after adding the stride attribute
# w_ih = flow.nn.Parameter(flow.Tensor(gate_size, layer_input_size))
# w_hh = flow.nn.Parameter(flow.Tensor(gate_size, real_hidden_size))
# b_ih = flow.nn.Parameter(flow.Tensor(gate_size))
# b_hh = flow.nn.Parameter(flow.Tensor(gate_size))
w_ih = flow.nn.Parameter(flow.Tensor(layer_input_size, gate_size))
w_hh = flow.nn.Parameter(flow.Tensor(real_hidden_size, gate_size))
b_ih = flow.nn.Parameter(flow.Tensor(gate_size))
b_hh = flow.nn.Parameter(flow.Tensor(gate_size))
layer_params = ()
if bias:
layer_params = (w_ih, w_hh, b_ih, b_hh)
else:
layer_params = (w_ih, w_hh)
suffix = "_reverse" if direction == 1 else ""
param_names = ["weight_ih_l{}{}", "weight_hh_l{}{}"]
if bias:
param_names += ["bias_ih_l{}{}", "bias_hh_l{}{}"]
param_names = [x.format(layer, suffix) for x in param_names]
for name, param in zip(param_names, layer_params):
setattr(self, name, param)
self.reset_parameters()
def test_tensor_greater(test_case):
input1 = flow.Tensor(np.array([1, 1, 4]).astype(np.float32),
dtype=flow.float32)
input2 = flow.Tensor(np.array([1, 2, 3]).astype(np.float32),
dtype=flow.float32)
of_out = input1.gt(input2)
np_out = np.greater(input1.numpy(), input2.numpy())
test_case.assertTrue(np.array_equal(of_out.numpy(), np_out))
def _test_stack_tuple_input(test_case, device, shape):
x = np.random.rand(*shape)
y = np.random.rand(*shape)
x_tensor = flow.Tensor(x, dtype=flow.float32, device=flow.device(device))
y_tensor = flow.Tensor(y, dtype=flow.float32, device=flow.device(device))
out_np = np.stack([x, y], axis=0)
out_of = flow.experimental.stack((x_tensor, y_tensor), dim=0).numpy()
test_case.assertTrue(np.allclose(out_np, out_of, 1e-5, 1e-5))
def test_tensor_using_tensor(test_case):
tensor = flow.Tensor(np.random.randn(2, 3, 4, 5),
device="cuda",
dtype=flow.int)
input = flow.Tensor(np.random.randn(2, 3))
output = input.to(tensor)
test_case.assertEqual(output.dtype, flow.int)
test_case.assertEqual(output.device, flow.device("cuda"))
