def transpose_test():
for _ in range(10):
for shape_len in range(2, 5):
try:
shape = np.random.randint( 8, size=(shape_len,) )+1
axes_order = np.array([*range(shape_len)])
np.random.shuffle(axes_order)
val_n = np.random.randint( 2**8, size=shape ).astype(np.float32)
transposed_n = np.transpose(val_n, axes_order)
val_t = nn.Tensor_from_value(val_n)
transposed_t = nn.transpose (val_t, axes_order )
if transposed_n.shape != transposed_t.shape:
raise Exception('shape is not equal')
if not all ( np.ndarray.flatten( transposed_t.np() == transposed_n ) ):
raise Exception(f'data is not equal {shape} {axes_order}')
transposed_n_grad = np.random.randint( 2**8, size=transposed_n.shape ).astype(np.float32)
val_t.get_grad().fill(1.0) # Check addition to gradient
nn.backward( {transposed_t:transposed_n_grad} , grad_for_non_trainables=True )
if not all ( np.ndarray.flatten(np.transpose(val_t.get_grad().np()-1.0, axes_order) == transposed_n_grad) ):
raise Exception(f'grad is not equal')
except:
raise Exception(f"""
shape : {shape}
axes_order : {axes_order}
transposed_n_shape : {transposed_n.shape}
transposed_t_shape : {transposed_t.shape}
exception : {traceback.format_exc()}
""")
for size in [2,3,4]:
try:
N = 1+np.random.randint(8)
C = (1+np.random.randint(8))*size*size
H = W = (1+np.random.randint(8))*size
shape = (N,C,H,W)
val_n = np.random.randint( 2**8, size=shape ).astype(np.float32)
val_t = nn.Tensor_from_value(val_n)
d2s_val_t = depth_to_space(val_t, size)
s2d_val_t = space_to_depth(d2s_val_t, size)
if not all ( np.ndarray.flatten( val_t.np() == s2d_val_t.np() ) ):
raise Exception(f'data is not equal')
except:
raise Exception(f"""
depth_to_space/space_to_depth
shape : {shape}
size : {size}
exception : {traceback.format_exc()}
""")
def concat_test():
for _ in range(10):
for shape_len in range(2, 5):
try:
shape = (np.random.randint(8, size=(shape_len, )) + 1).tolist()
axis = np.random.randint(shape_len)
count = np.random.randint(4) + 1
shapes = tuple(
tuple(dim if i != axis else np.random.randint(8) + 1
for i, dim in enumerate(shape))
for shape in ([shape] * count))
vals_n = [
np.random.randint(2**8, size=shape).astype(np.float32)
for shape in shapes
]
concat_n = np.concatenate(vals_n, axis)
vals_t = [
nn.Tensor_from_value(vals_n[i]) for i in range(count)
]
concat_t = nn.concat(vals_t, axis)
concat_n_grad = np.random.randint(
2**8, size=concat_n.shape).astype(np.float32)
for t in vals_t:
t.get_grad().fill(1.0) # Check addition to gradient
nn.backward({concat_t: concat_n_grad},
grad_for_non_trainables=True)
if not all(np.ndarray.flatten(concat_t.np() == concat_n)):
raise Exception(f'data is not equal')
axis_offset = 0
for n in range(count):
axis_size = vals_n[n].shape[axis]
axis_slice = slice(axis_offset, axis_offset + axis_size, 1)
slices = (slice(None, None, None), ) * axis + (
axis_slice, ) + (slice(None, None, None), ) * (
len(concat_n.shape) - axis - 1)
axis_offset += axis_size
if not all(
np.ndarray.flatten(vals_t[n].get_grad().np() -
1.0 == concat_n_grad[slices])):
raise Exception(f'grad is not equal')
except:
raise Exception(f"""
shape : {shape}
axis : {axis}
count : {count}
exception : {traceback.format_exc()}
""")
def resize2D_nearest_test():
for n in [1, 4]:
for ic in [1, 2, 4]:
for iw, ih in zip(*[[4, 8, 16]] * 2):
for size in [2, 3, 4]:
try:
input_shape = (n, ic, ih, iw)
input_n = np.random.randint(
2**4, size=input_shape).astype(np.float32)
input_t = nn.Tensor_from_value(input_n)
upsampled_t = nn.resize2D_nearest(input_t, size=size)
upsampled_n = input_n.reshape((n, ic, ih, 1, iw, 1))
upsampled_n = np.tile(upsampled_n,
(1, 1, 1, size, 1, size))
upsampled_n = upsampled_n.reshape(
(n, ic, ih * size, iw * size))
if upsampled_n.shape != upsampled_t.shape:
raise Exception(f'shape is not equal')
if not all(
np.ndarray.flatten(
upsampled_t.np() == upsampled_n)):
raise Exception(f'data is not equal')
upsampled_n_grad = np.random.randint(
2**8, size=upsampled_n.shape).astype(np.float32)
input_t.get_grad().fill(1.0)
nn.backward({upsampled_t: upsampled_n_grad},
grad_for_non_trainables=True)
input_n_grad = upsampled_n_grad.reshape(
(n, ic, ih, size, iw, size))
input_n_grad = input_n_grad.sum((-3, -1))
if not all(
np.ndarray.flatten(input_t.get_grad().np() -
1.0) == np.ndarray.flatten(
input_n_grad)):
raise Exception('grad is not equal')
except:
raise Exception(f"""
input_shape : {input_shape}
size : {size}
upsampled_n.shape : {upsampled_n.shape}
upsampled_t.shape : {upsampled_t.shape}
{traceback.format_exc()}
""")
def depthwise_conv2d_test():
for padding in ['same','valid',0,1,2]:
for dilation in [1,2]:
for stride in [1,2,3]:
for ks in [1,3,5,7]:
for n in [1,4]:
for ic in [1,2,4]:
for ih,iw in zip(*[[4,8,16]]*2):
if padding == 'valid' and iw < ks:
continue
try:
input_shape = (n, ic, ih, iw)
kernel_shape = (ic, ks, ks)
input_n = np.random.randint( 2**4, size=input_shape ).astype(np.float32)
kernel_n = np.random.randint( 2**4, size=kernel_shape ).astype(np.float32)
input_t = nn.Tensor_from_value(input_n)
kernel_t = nn.Tensor_from_value(kernel_n)
conved_t = nn.depthwise_conv2D(input_t, kernel_t, stride=stride, dilation=dilation, padding=padding)
conved_n_grad = np.random.randint( 2**4, size=conved_t.shape).astype(np.float32)
conved_n, dI_val, dK_val = _numpy_depthwise_conv2d(input_n, kernel_n, conved_n_grad, STRIDE=stride, DILATION=dilation, padding=padding)
if conved_n.shape != conved_t.shape:
raise Exception(f'shape is not equal')
if not all ( np.ndarray.flatten( conved_t.np() == conved_n) ):
raise Exception(f'data is not equal')
input_t.get_grad().fill(1.0)
kernel_t.get_grad().fill(1.0)
nn.backward( {conved_t:conved_n_grad}, grad_for_non_trainables=True )
if not all ( np.ndarray.flatten( (input_t.get_grad().np()-1.0) == dI_val )):
raise Exception(f'dI not equal')
if not all ( np.ndarray.flatten( (kernel_t.get_grad().np()-1.0) == dK_val )):
raise Exception(f'dK not equal')
except:
raise Exception(f"""
input_shape : {input_shape}
kernel_shape : {kernel_shape}
padding : {padding}
stride : {stride}
dilation : {dilation}
conved_n.shape : {conved_n.shape}
conved_t.shape : {conved_t.shape}
{traceback.format_exc()}
""")
def backward(self, stop_grad=None, grad_for_non_trainables=False):
"""
Perform backward computation.
stop_grad (None)
None or Tensor or list of Tensors where backprop should stop.
grad_for_non_trainables (False)
If False, the backprop will stop at those branches that have no trainable tensors (not attached to any Optimizer),
also gradient for intermediate non trainable tensors will be freed in order to reduce memory consumption.
"""
nn.backward(self,
stop_grad=stop_grad,
grad_for_non_trainables=grad_for_non_trainables)
def stack_test():
for _ in range(10):
for shape_len in range(1, 4):
try:
shape = tuple(np.random.randint(8, size=(shape_len, )) + 1)
axis = np.random.randint(shape_len + 1)
stack_count = np.random.randint(4) + 1
vals_n = [
np.random.randint(2**8, size=shape).astype(np.float32)
for i in range(stack_count)
]
vals_t = [
nn.Tensor_from_value(vals_n[i]) for i in range(stack_count)
]
stack_n = np.stack(vals_n, axis)
stack_t = nn.stack(vals_t, axis)
if not all(np.ndarray.flatten(stack_t.np() == stack_n)):
raise Exception(f'data is not equal')
stack_n_grad = np.random.randint(
2**8, size=stack_n.shape).astype(np.float32)
for val_t in vals_t:
val_t.get_grad().fill(1.0)
nn.backward({stack_t: stack_n_grad},
grad_for_non_trainables=True)
for n in range(stack_count):
slices = (slice(None, None, None), ) * axis + (
n, ) + (slice(None, None, None), ) * (
len(stack_n.shape) - axis - 1)
if not all(
np.ndarray.flatten(vals_t[n].get_grad().np() -
1.0 == stack_n_grad[slices])):
raise Exception(f'grad is not equal')
except:
raise Exception(f"""
shape : {shape}
axis : {axis}
stack_count : {stack_count}
stack_n_shape : {stack_n.shape}
stack_t_shape : {stack_t.shape}
exception : {traceback.format_exc()}
""")
def resize2D_bilinear_test():
for n in [1, 4]:
for ic in [1, 2, 4]:
for iw, ih in zip(*[[4, 8, 16]] * 2):
for size in [0.6, 1.0, 2.0]:
try:
input_shape = (n, ic, ih, iw)
input_n = np.random.randint(
2**4, size=input_shape).astype(np.float32)
input_t = nn.Tensor_from_value(input_n)
resized_t = nn.resize2D_bilinear(input_t, size)
nn.backward([resized_t], grad_for_non_trainables=True)
except:
raise Exception(f"""
input_shape : {input_shape}
size : {size}
resized_t.shape : {resized_t.shape}
{traceback.format_exc()}
""")
def tile_test():
for _ in range(10):
for shape_len in range(3, 5):
try:
shape = tuple(np.random.randint( 8, size=(shape_len,) )+1)
tiles = tuple(np.random.randint( 4, size=(shape_len,) )+1)
val_n = np.random.randint( 2**8, size=shape ).astype(np.float32)
tiled_n = np.tile(val_n, tiles)
val_t = nn.Tensor_from_value(val_n)
tiled_t = nn.tile(val_t, tiles)
if tiled_n.shape != tiled_t.shape:
raise Exception(f'shape is not equal')
if not all ( np.ndarray.flatten( tiled_t.np() == tiled_n ) ):
raise Exception(f'data is not equal')
tiled_n_grad = np.random.randint( 2**8, size=tiled_n.shape ).astype(np.float32)
val_t.get_grad().fill(1.0)
nn.backward( {tiled_t:tiled_n_grad} , grad_for_non_trainables=True )
info = nc.info.InfoTile( nc.TensorShape(shape), tiles)
val_n_grad = sum([ tiled_n_grad[axes_slice] for axes_slice in info.axes_slices ])
if not all ( np.ndarray.flatten(val_t.get_grad().np()-1.0 == val_n_grad) ):
raise Exception(f'grad is not equal')
except:
raise Exception(f"""
shape : {shape}
tiles : {tiles}
tiled_n_shape : {tiled_n.shape}
tiled_t_shape : {tiled_t.shape}
exception : {traceback.format_exc()}
""")
def matmul_test():
for _ in range(10):
try:
BATCH = np.random.randint(8) + 1
M = np.random.randint(8) + 1
N = np.random.randint(32768) + 1
K = np.random.randint(32768) + 1
while K * N > (8000000 // BATCH):
K = max(1, K // 2)
N = max(1, N // 2)
if np.random.randint(2) == 0:
size = [2, 4, 8, 16][np.random.randint(4)]
M = max(1, M // size) * size
N = max(1, N // size) * size
K = max(1, K // size) * size
if BATCH == 1:
A_shape = (M, K)
B_shape = (K, N)
else:
A_shape = (BATCH, M, K)
B_shape = (BATCH, K, N)
A_n = np.random.randint(2**4, size=A_shape).astype(np.float32)
B_n = np.random.randint(2**4, size=B_shape).astype(np.float32)
O_n = np.matmul(A_n, B_n)
A_t = nn.Tensor_from_value(A_n)
B_t = nn.Tensor_from_value(B_n)
O_t = nn.matmul(A_t, B_t)
if O_n.shape != O_t.shape:
raise Exception('shape is not equal')
if not all(np.ndarray.flatten(O_t.np() == O_n)):
raise Exception(f'data is not equal')
O_n_grad = np.random.randint(2**3,
size=O_n.shape).astype(np.float32)
b_n_axes = tuple(np.arange(len(B_n.shape)))
B_n_T = B_n.transpose(b_n_axes[:-2] + b_n_axes[-1:] +
b_n_axes[-2:-1])
a_n_axes = tuple(np.arange(len(A_n.shape)))
A_n_T = A_n.transpose(a_n_axes[:-2] + a_n_axes[-1:] +
a_n_axes[-2:-1])
A_n_grad = np.matmul(O_n_grad, B_n_T)
B_n_grad = np.matmul(A_n_T, O_n_grad)
A_t.get_grad().fill(1.0)
B_t.get_grad().fill(1.0)
nn.backward({O_t: O_n_grad}, grad_for_non_trainables=True)
if not all(
np.ndarray.flatten(
(A_t.get_grad().np() - 1.0) == A_n_grad)):
raise Exception(f'dA is not equal')
if not all(
np.ndarray.flatten(
(B_t.get_grad().np() - 1.0) == B_n_grad)):
raise Exception(f'dB is not equal')
except:
raise Exception(f"""
M : {M}
N : {N}
K : {K}
O_n.shape : {O_n.shape}
O_t.shape : {O_t.shape}
{traceback.format_exc()}
""")
def reduce_test():
for _ in range(10):
for op_type in ['sum', 'mean', 'min', 'max']:
for shape_len in range(2, 5):
try:
shape = np.random.randint( 8, size=(shape_len,) )+1
reduction_axes = np.array([*range(shape_len)])
np.random.shuffle(reduction_axes)
# Cut random amount of reduction_axes
reduction_axes = tuple(reduction_axes [:np.random.randint(shape_len+1)])
if len(reduction_axes) == 0:
reduction_axes = None
keepdims = np.random.randint(2) == 0
value_n = np.random.randint( 2**8, size=shape ).astype(np.float32)
value_t = nn.Tensor_from_value(value_n)
if op_type == 'sum':
reducted_t = value_t.sum(reduction_axes, keepdims=keepdims)
reducted_n = value_n.sum(reduction_axes, keepdims=keepdims)
reducted_n_keepdims_shape = value_n.sum(reduction_axes, keepdims=True).shape
elif op_type == 'mean':
reducted_t = value_t.mean(reduction_axes, keepdims=keepdims)
reducted_n = value_n.mean(reduction_axes, keepdims=keepdims)
reducted_n_keepdims_shape = value_n.mean(reduction_axes, keepdims=True).shape
elif op_type == 'max':
reducted_t = value_t.max(reduction_axes, keepdims=keepdims)
reducted_n = value_n.max(reduction_axes, keepdims=keepdims)
reducted_n_keepdims_shape = value_n.max(reduction_axes, keepdims=True).shape
elif op_type == 'min':
reducted_t = value_t.min(reduction_axes, keepdims=keepdims)
reducted_n = value_n.min(reduction_axes, keepdims=keepdims)
reducted_n_keepdims_shape = value_n.min(reduction_axes, keepdims=True).shape
if sum (np.ndarray.flatten( reducted_t.np() - reducted_n)) >= 1.0:
raise Exception(f'data is not equal')
value_t.get_grad().fill(1.0)
reducted_n_grad = np.random.randint( 2**8, size=reducted_n.shape ).astype(np.float32)
nn.backward( {reducted_t:reducted_n_grad}, grad_for_non_trainables=True )
if op_type == 'sum':
value_n_grad = np.broadcast_to( np.reshape(reducted_n_grad, reducted_n_keepdims_shape), value_n.shape )
if sum (np.ndarray.flatten( value_t.get_grad().np()-1.0 - value_n_grad )) >= 1.0:
raise Exception(f'dI is not equal')
except:
raise Exception(f"""
op_type : {op_type}
shape : {shape}
reduction_axes : {reduction_axes}
keepdims : {keepdims}
reducted_n.shape : {reducted_n.shape}
reducted_t.shape : {reducted_t.shape}
exception : {traceback.format_exc() }
""")
def pool2d_test():
for batch in [1, 4]:
for in_ch in [1, 2, 4]:
for w, h in zip(*[[4, 8, 16]] * 2):
for pool_size in [2, 3]:
for stride in [1, 2, 3]:
for padding in ['same', 'valid', 0, 1, 2]:
for op_type in ['avg', 'max', 'min']:
try:
input_shape = (batch, in_ch, h, w)
if op_type == 'avg':
input_n = np.random.randint(
2**4, size=input_shape).astype(
np.float32)
else:
# for minmax make unique values in order not to test 'same-hit'
input_shape_prod = int(
np.prod(input_shape))
input_n = np.arange(
input_shape_prod,
dtype=np.float32).reshape(
input_shape) / input_shape_prod
input_t = nn.Tensor_from_value(input_n)
pooled_n, input_n_grad = _numpy_pool2d(
op_type,
input_n,
pool_size,
STRIDE=stride,
padding=padding)
if op_type == 'avg':
pooled_t = nn.avg_pool2D(
input_t, pool_size, stride,
padding)
elif op_type == 'max':
pooled_t = nn.max_pool2D(
input_t, pool_size, stride,
padding)
elif op_type == 'min':
pooled_t = nn.min_pool2D(
input_t, pool_size, stride,
padding)
if pooled_n.shape != pooled_t.shape:
raise Exception(f'shape is not equal')
if np.sum(np.abs(pooled_t.np() -
pooled_n)) > 0.1:
raise Exception(f'data is not equal')
input_t.get_grad().fill(1.0)
nn.backward(pooled_t,
grad_for_non_trainables=True)
if sum(
np.ndarray.flatten(
input_t.get_grad().np() - 1.0 -
input_n_grad)) >= 1.0:
raise Exception('grad is not equal')
except:
raise Exception(f"""
op_type : {op_type}
input_shape : {input_shape}
pool_size : {pool_size}
stride : {stride}
padding : {padding}
pooled_n.shape : {pooled_n.shape}
pooled_t.shape : {pooled_t.shape}
{traceback.format_exc()}
""")
def slice_test():
for iteration in range(10):
for shape_len in range(5, 1, -1):
try:
while True:
shape = np.random.randint(1, 8, size=(shape_len, ))
if iteration == 0:
slices = [
slice_cls(None, None, None),
] * shape_len
axis = np.random.randint(shape_len)
shape[axis] = 1
slices[axis] = 0
else:
slices = []
for i in range(shape_len):
axis_size = shape[i]
if np.random.randint(2) == 0:
v = axis_size - np.random.randint(
axis_size * 2) - 1
slices.append(v)
else:
b = None if np.random.randint(
2) == 0 else axis_size - np.random.randint(
axis_size * 2)
e = None if np.random.randint(
2) == 0 else axis_size - np.random.randint(
axis_size * 2)
s = 1 if np.random.randint(2) == 0 else -1
slices.append(slice_cls(b, e, s))
if np.random.randint(2) == 0:
axis = np.random.randint(shape_len)
slices[axis] = Ellipsis
shape = tuple(shape)
slices = tuple(slices)
val_n = np.random.randint(2**8,
size=shape).astype(np.float32)
sliced_n = val_n[slices]
val_t = nn.Tensor_from_value(val_n)
sliced_t = val_t[slices]
if 0 in sliced_n.shape:
# some cases like 0:1:-1 will produce zero shape and invalid array on numpy
# but nn.slice has no such behaviour, thus we have to generate new slice again
continue
if np.prod(sliced_n.shape) != sliced_t.shape.size:
raise Exception(f'shape is not equal')
if not all(
np.ndarray.flatten(np.array(sliced_t.np())) ==
np.ndarray.flatten(np.array(sliced_n))):
raise Exception(f'data is not equal')
sliced_n_grad = np.random.randint(
2**8, size=sliced_n.shape).astype(np.float32)
val_t.get_grad().fill(1.0)
nn.backward({sliced_t: sliced_n_grad},
grad_for_non_trainables=True)
sliced_t_grad = np.array(val_t.get_grad().np()[slices])
if not all(
np.ndarray.flatten(np.array([sliced_t_grad - 1.0]))
== np.ndarray.flatten(np.array([sliced_n_grad]))):
raise Exception(f'grad is not equal')
break
except:
raise Exception(f"""
shape : {shape}
slices : {slices}
sliced_n_shape : {sliced_n.shape}
sliced_t_shape : {sliced_t.shape}
exception : {traceback.format_exc()}
""")
def dual_wise_op_test():
for op in [
add, binary_crossentropy, categorical_crossentropy, sub, max, min,
mul, div
]:
print(f'{op.__name__}()')
for _ in range(10):
if op == categorical_crossentropy:
shape_gen = [2]
else:
shape_gen = range(1, 5)
for shape_len in shape_gen:
try:
a_shape = tuple(
np.random.randint(8, size=(shape_len, )) + 1)
if op == categorical_crossentropy:
b_shape = a_shape
else:
if np.random.randint(2) == 0:
b_shape = tuple(
a_shape[np.random.randint(len(a_shape)):])
b_shape = (1, ) if len(b_shape) == 0 else b_shape
else:
b_shape = list(a_shape)
b_shape[np.random.randint(len(b_shape))] = 1
b_shape = tuple(b_shape)
shapes = [a_shape, b_shape]
if np.random.randint(2) == 0:
shapes = shapes[::-1]
a_shape, b_shape = shapes
a_n = np.random.randint(1, 2**8,
size=a_shape).astype(np.float32)
b_n = np.random.randint(1, 2**8,
size=b_shape).astype(np.float32)
a_t = nn.Tensor_from_value(a_n)
b_t = nn.Tensor_from_value(b_n)
r_t = op(a_t, b_t)
r_n_grad = np.random.randint(2**8, size=r_t.shape).astype(
np.float32)
a_t.get_grad().fill(1.0)
b_t.get_grad().fill(1.0)
nn.backward({r_t: r_n_grad}, grad_for_non_trainables=True)
if op == div:
# Test validness and gradient only for div
r_n = a_n / b_n
if r_n.shape != r_t.shape:
raise Exception(f'shapes are not equal')
if np.abs(np.sum(
(np.ndarray.flatten(r_t.np() - r_n)))) > 1:
raise Exception(f'data is not equal')
info = nc.info.InfoBroadcast(nc.TensorShape(a_shape),
nc.TensorShape(b_shape))
a_n_grad = r_n_grad / b_n
axes = info.a_shape_reduction_axes
if axes.rank == 0:
a_n_grad = a_n_grad.reshape(a_n.shape)
else:
a_n_grad = a_n_grad.sum(tuple(axes), keepdims=True)
b_n_grad = r_n_grad * (-a_n / (b_n**2))
axes = info.b_shape_reduction_axes
if axes.rank == 0:
b_n_grad = b_n_grad.reshape(b_n.shape)
else:
b_n_grad = b_n_grad.sum(tuple(axes), keepdims=True)
if np.abs(
np.sum(
(np.ndarray.flatten(a_t.get_grad().np() -
1.0 - a_n_grad)))) > 1:
raise Exception(f'grad A is not equal')
if np.abs(
np.sum(
(np.ndarray.flatten(b_t.get_grad().np() -
1.0 - b_n_grad)))) > 1:
raise Exception(f'grad B is not equal')
else:
if not a_t.has_grad():
raise Exception(f'a_t has no grad')
if not b_t.has_grad():
raise Exception(f'b_t has no grad')
except:
raise Exception(f"""
op : {op}
a_shape : {a_shape}
b_shape : {b_shape}
r_n_shape : {r_n.shape}
exception : {traceback.format_exc() }
""")
