def test_conv2d_with_bias_fma():
(c_in, h_in, w_in), (c_out, h_out, w_out) = (3, 10, 10), (12, 10, 10)
k = 3
x = np.random.randn(1, c_in, h_in, w_in).astype(np.float32)
W = np.random.randn(c_out, c_in, k, k).astype(np.float32)
b = np.random.randn(c_out).astype(np.float32)
f = Convolution2DFunction(pad=(np.int64(1), np.int64(1)))
flops, mr, mw, params = calculate_cost(f, [x, W, b], fma_1flop=True)
assert f.apply([x, W, b])[0].shape == (1, c_out, h_out, w_out)
assert flops == (c_in * c_out * k * k * h_out * w_out)
assert mr == c_in * h_in * w_in + c_out * c_in * k * k + c_out
assert mw == c_out * h_out * w_out
assert params == {
'k': k,
's': 1,
'p': 1,
'd': 1,
'groups': 1,
'nobias': False
}
assert type(params['k']) is int
assert type(params['s']) is int
assert type(params['p']) is int
assert type(params['d']) is int
assert type(params['groups']) is int
def test_shift():
x = np.random.randn(1, 32, 10, 10).astype(np.float32)
f = F.connection.shift.Shift(ksize=3, dilate=1)
flops, mr, mw, params = calculate_cost(f, [x])
assert flops == 0 # exclude index calculation
assert mr == x.size
assert mw == x.size
assert params == {'k': 3, 'd': 1}
def test_activation_leaky_relu():
x = np.random.randn(1, 3, 100, 100).astype(np.float32)
f = A.leaky_relu.LeakyReLU()
flops, mread, mwrite, params = calculate_cost(f, [x])
assert flops == 2 * x.size
assert mread == x.size
assert mwrite == x.size
assert params == dict()
def test_resize_expand():
x = np.random.randn(1, 3, 10, 10).astype(np.float32)
f = A.resize_images.ResizeImages((15, 15))
flops, mread, mwrite, params = calculate_cost(f, [x])
assert flops == 3 * 15 * 15 * 9
assert mread == 3 * 10 * 10
assert mwrite == 3 * 15 * 15
assert params == {'size': (15, 15)}
def test_activation_sigmoid():
x = np.random.randn(1, 3, 100, 100).astype(np.float32)
f = A.sigmoid.Sigmoid()
flops, mread, mwrite, params = calculate_cost(f, [x])
assert flops == x.size
assert mread == x.size
assert mwrite == x.size
assert params == dict()
def test_sub_constant():
x = np.random.randn(1, 3, 10, 10).astype(np.float32)
f = F.math.basic_math.SubFromConstant(x)
flops, mread, mwrite, params = calculate_cost(f, [x])
assert flops == 3 * 10 * 10
assert mread == (3 * 10 * 10) * 2
assert mwrite == 3 * 10 * 10
assert params == dict()
def test_max_noaxis():
x = np.random.randn(1, 3, 10, 10).astype(np.float32)
f = F.math.minmax.Max()
flops, mread, mwrite, params = calculate_cost(f, [x])
assert flops == 3 * 10 * 10 - 1
assert mread == 3 * 10 * 10
assert mwrite == 1
assert params == {'axis': None}
def test_resize_shrink():
x = np.random.randn(1, 3, 10, 10).astype(np.float32)
f = A.resize_images.ResizeImages((4, 4))
flops, mread, mwrite, params = calculate_cost(f, [x])
assert flops == 3 * 4 * 4 * 9
assert mread == 4 * 3 * 4 * 4 # 4(neighbors)*3(channel)*out_w*out_h
assert mwrite == 3 * 4 * 4
assert params == {'size': (4, 4)}
def test_add():
x = np.random.randn(1, 3, 10, 10).astype(np.float32)
f = F.math.basic_math.Add()
flops, mread, mwrite, params = calculate_cost(f, [x, x])
assert flops == 3 * 10 * 10
assert mread == (3 * 10 * 10) * 2
assert mwrite == 3 * 10 * 10
assert params == dict()
def test_broadcast_to():
x = np.random.randn(1, 3, 1, 1).astype(np.float32)
f = A.broadcast.BroadcastTo((1, 3, 10, 10))
flops, mread, mwrite, params = calculate_cost(f, [x])
assert flops == 0
assert mread == 3
assert mwrite == 3 * 10 * 10
assert params == {'shape': (1, 3, 10, 10)}
def test_sum_axes():
x = np.random.randn(1, 3, 10, 10).astype(np.float32)
f = F.math.sum.Sum(axis=(1, 2))
flops, mread, mwrite, params = calculate_cost(f, [x])
assert flops == (3 - 1) * 10 * 10 + (10 - 1) * 10
assert mread == 3 * 10 * 10
assert mwrite == 10
assert params == {'axis': (1, 2)}
def test_transpose():
x = np.random.randn(1, 3, 10, 10).astype(np.float32)
f = A.transpose.Transpose((0, 3, 1, 2)) # typical HWC2CHW transpose
flops, mread, mwrite, params = calculate_cost(f, [x])
assert flops == 0
assert mread == 3 * 10 * 10
assert mwrite == 3 * 10 * 10
assert params == {'axes': (0, 3, 1, 2)}
def test_get_item_one():
x = np.random.randn(1, 3, 10, 10).astype(np.float32)
f = A.get_item.GetItem((0, 0, 0, 0))
flops, mread, mwrite, params = calculate_cost(f, [x])
assert flops == 0
assert mread == 1
assert mwrite == 1
assert params == {'slices': [0, 0, 0, 0]}
def test_reshape():
x = np.random.randn(1, 3, 100, 100).astype(np.float32)
f = A.reshape.Reshape((1, -1))
flops, mread, mwrite, params = calculate_cost(f, [x])
assert flops == 0
assert mread == 0
assert mwrite == 0
assert params == {'in_shape': (1, 3, 100, 100), 'out_shape': (1, -1)}
def test_concat_more():
x = np.random.randn(1, 3, 10, 10).astype(np.float32)
f = A.concat.Concat(axis=2)
flops, mread, mwrite, params = calculate_cost(f, [x, x, x, x])
assert flops == 0
assert mread == 4 * (3 * 10 * 10)
assert mwrite == 4 * (3 * 10 * 10)
assert params == {'axis': 2}
def test_activation_prelu():
x = np.random.randn(1, 3, 100, 100).astype(np.float32)
W = np.random.randn(3).astype(np.float32)
f = A.prelu.PReLUFunction()
flops, mread, mwrite, params = calculate_cost(f, [x, W])
assert flops == x.size
assert mread == x.size + W.size
assert mwrite == x.size
assert params == {'w_shape': W.shape}
def test_add_multiple():
x = np.random.randn(1, 3, 10, 10).astype(np.float32)
f = F.math.basic_math.Add()
n_array = 10
flops, mread, mwrite, params = calculate_cost(f, [x] * n_array)
assert flops == (n_array - 1) * 3 * 10 * 10
assert mread == n_array * (3 * 10 * 10)
assert mwrite == 3 * 10 * 10
assert params == dict()
def test_lrn_fma():
c, h, w = 8, 10, 10
x = np.random.randn(1, c, h, w).astype(np.float32)
f = N.local_response_normalization.LocalResponseNormalization()
flops, mread, mwrite, params = calculate_cost(f, [x], fma_1flop=True)
assert flops == (5 * c - 1) * h * w
assert mread == x.size
assert mwrite == x.size
assert params == {'n': 5, 'k': 2, 'alpha': 0.0001, 'beta': 0.75}
def test_linear_nobias_fma():
x = np.random.randn(1, 10).astype(np.float32)
w = np.random.randn(20, 10).astype(np.float32)
f = LinearFunction()
flops, mr, mw, params = calculate_cost(f, [x, w], fma_1flop=True)
assert flops == 10 * 20
assert mr == 10 + 10 * 20 # input data, and weight matrix
assert mw == 20
assert params == {'nobias': True}
def test_normalize_fma():
c, h, w = 3, 10, 10
x = np.random.randn(1, c, h, w).astype(np.float32)
f = N.l2_normalization.NormalizeL2(axis=2)
flops, mread, mwrite, params = calculate_cost(f, [x], fma_1flop=True)
assert flops == (2 * h + 2) * c * w
assert mread == x.size
assert mwrite == x.size
assert params == {'axis': 2}
def test_get_item_slice2():
x = np.random.randn(1, 3, 10, 10).astype(np.float32)
# x[0, 0, :]
slices = (0, 0, slice(None, None, None))
f = A.get_item.GetItem(slices)
flops, mread, mwrite, params = calculate_cost(f, [x])
assert flops == 0
assert mread == 10 * 10
assert mwrite == 10 * 10
assert params == {'slices': [0, 0, (None, None, None)]}
def test_max_axes_rev():
x = np.random.randn(1, 3, 10, 10).astype(np.float32)
f = F.math.minmax.Max(axis=(2, 1))
flops, mread, mwrite, params = calculate_cost(f, [x])
# Completely equivament to axis=(1, 2) case
assert flops == (3 - 1) * 10 * 10 + (10 - 1) * 10
assert mread == 3 * 10 * 10
assert mwrite == 10
assert params == {'axis': (2, 1)}
def test_max_all_axes():
x = np.random.randn(1, 3, 10, 10).astype(np.float32)
f = F.math.minmax.Max(axis=(0, 1, 2, 3))
flops, mread, mwrite, params = calculate_cost(f, [x])
# Completely equivament to axis=None case
assert flops == 3 * 10 * 10 - 1
assert mread == 3 * 10 * 10
assert mwrite == 1
assert params == {'axis': (0, 1, 2, 3)}
def test_argmin():
x = np.random.randn(1, 3, 10, 10).astype(np.float32)
f = F.math.minmax.ArgMin(axis=1)
flops, mread, mwrite, params = calculate_cost(f, [x])
# exactly same as min/max
assert flops == (3 - 1) * 10 * 10
assert mread == 3 * 10 * 10
assert mwrite == 10 * 10
assert params == {'axis': 1}
def test_sum_noaxis():
x = np.random.randn(1, 3, 10, 10).astype(np.float32)
f = F.math.sum.Sum()
flops, mread, mwrite, params = calculate_cost(f, [x])
# exactly same as min/max
assert flops == 3 * 10 * 10 - 1
assert mread == 3 * 10 * 10
assert mwrite == 1
assert params == {'axis': None}
def test_softmax():
x = np.random.randn(1, 100).astype(np.float32)
f = A.softmax.Softmax()
flops, mread, mwrite, params = calculate_cost(f, [x])
# flops: exp term, sum term, div term
assert flops == x.size + (x.size - 1) + x.size
assert mread == x.size
assert mwrite == x.size
assert params == {'axis': 1}
assert type(params['axis']) is int
def test_softmax_axis():
x = np.random.randn(1, 32, 128).astype(np.float32)
f = A.softmax.Softmax(axis=np.int64(2))
flops, mread, mwrite, params = calculate_cost(f, [x])
# flops: exp term, sum term, div term
assert flops == x.size + 32 * (128 - 1) + x.size
assert mread == x.size
assert mwrite == x.size
assert params == {'axis': 2}
assert type(params['axis']) is int
def test_linear_nobias_no_fma():
x = np.random.randn(1, 10).astype(np.float32)
w = np.random.randn(20, 10).astype(np.float32)
f = LinearFunction()
flops, mr, mw, params = calculate_cost(f, [x, w], fma_1flop=False)
# for each output neuron, weight multiplication is applied 10 times and
# addition (10-1) times.
assert flops == (10 + 10 - 1) * 20
assert mr == 10 + 10 * 20
assert mw == 20
assert params == {'nobias': True}
def test_max_pooling():
x = np.random.randn(1, 3, 100, 100).astype(np.float32)
f = P.max_pooling_2d.MaxPooling2D(np.int64(2), np.int64(2),
np.int64(0), cover_all=True)
flops, mread, mwrite, params = calculate_cost(f, [x])
# flops is (output size) * (inside window operation)
# when window size is 2x2, max operation is applied 2x2-1 times.
assert flops == (3 * 50 * 50) * (2 * 2 - 1)
assert mread == x.size
assert mwrite == (3 * 50 * 50)
assert params == {'k': 2, 's': 2, 'p': 0}
assert type(params['k']) is int
assert type(params['s']) is int
assert type(params['p']) is int
def test_upsampling_2d_no_outsize():
x = np.random.randn(1, 3, 10, 10).astype(np.float32)
indices = np.random.randint(0, 9, (1, 3, 10, 10)).astype(np.int32)
f = P.upsampling_2d.Upsampling2D(indices, ksize=np.int64(3),
stride=np.int64(3))
flops, mread, mwrite, params = calculate_cost(f, [x])
assert flops == 0
assert mread == 2 * 3 * 10 * 10
assert mwrite == 3 * 28 * 28
assert params == {
'k': 3, 's': 3, 'p': 0, 'outsize': (28, 28), 'cover_all': True
}
assert type(params['k']) is int
assert type(params['s']) is int
assert type(params['p']) is int