def assertGradientChecks(
self,
device_option,
op,
inputs,
outputs_to_check,
outputs_with_grads,
grad_ops=None,
threshold=0.005,
stepsize=0.05,
input_device_options=None,
):
"""
Implements a standard numerical gradient checker for the operator
in question.
Useful for checking the consistency of the forward and
backward implementations of operators.
Usage example:
@given(inputs=hu.tensors(n=2), in_place=st.booleans(), **hu.gcs)
def test_sum(self, inputs, in_place, gc, dc):
op = core.CreateOperator("Sum", ["X1", "X2"],
["Y" if not in_place else "X1"])
X1, X2 = inputs
self.assertGradientChecks(gc, op, [X1, X2], 0, [0])
"""
gc = gradient_checker.GradientChecker(
stepsize=stepsize,
threshold=threshold,
device_option=device_option,
workspace_name=str(device_option),
input_device_options=input_device_options,
)
res, grad, grad_estimated = gc.CheckSimple(
op, inputs, outputs_to_check, outputs_with_grads,
grad_ops=grad_ops,
input_device_options=input_device_options
)
self.assertEqual(grad.shape, grad_estimated.shape)
self.assertTrue(
res,
"Gradient check failed for input " + str(op.input[outputs_to_check])
)
def _run_op_test(self, X, I, check_grad=False):
with core.DeviceScope(core.DeviceOption(caffe2_pb2.CUDA, 0)):
op = core.CreateOperator('BatchPermutation', ['X', 'I'], ['Y'])
workspace.FeedBlob('X', X)
workspace.FeedBlob('I', I)
workspace.RunOperatorOnce(op)
Y = workspace.FetchBlob('Y')
if check_grad:
gc = gradient_checker.GradientChecker(
stepsize=0.1,
threshold=0.001,
device_option=core.DeviceOption(caffe2_pb2.CUDA, 0))
res, grad, grad_estimated = gc.CheckSimple(op, [X, I], 0, [0])
self.assertTrue(res, 'Grad check failed')
Y_ref = X[I]
np.testing.assert_allclose(Y, Y_ref, rtol=1e-5, atol=1e-08)
def _run_test(self, A, B, check_grad=False):
with core.DeviceScope(core.DeviceOption(caffe2_pb2.CUDA, 0)):
op = core.CreateOperator('SpatialNarrowAs', ['A', 'B'], ['C'])
workspace.FeedBlob('A', A)
workspace.FeedBlob('B', B)
workspace.RunOperatorOnce(op)
C = workspace.FetchBlob('C')
if check_grad:
gc = gradient_checker.GradientChecker(
stepsize=0.005,
threshold=0.005,
device_option=core.DeviceOption(caffe2_pb2.CUDA, 0))
res, grad, grad_estimated = gc.CheckSimple(op, [A, B], 0, [0])
self.assertTrue(res, 'Grad check failed')
dims = C.shape
C_ref = A[:dims[0], :dims[1], :dims[2], :dims[3]]
np.testing.assert_allclose(C, C_ref, rtol=1e-5, atol=1e-08)
def _run_test(self, scores, labels, rois, bbox_pred, gt_centers, im_info,
regression_ws, num_classes, check_grad=False):
with core.DeviceScope(core.DeviceOption(caffe2_pb2.CUDA, 0)):
# op = core.CreateOperator('SsDistanceLoss',
# ['scores', 'labels', 'rois', 'bbox_pred', 'gt_centers', 'im_info'],
# ['loss', 'p', 'pred_class'],
# regression_ws=regression_ws, num_classes=num_classes)
op = core.CreateOperator('SsDistanceLoss',
['scores', 'labels', 'rois', 'bbox_pred', 'gt_centers', 'im_info'],
['loss', 'prob', 'pred_class'],
num_classes=num_classes)
workspace.FeedBlob('scores', scores)
workspace.FeedBlob('labels', labels)
workspace.FeedBlob('rois', rois)
workspace.FeedBlob('bbox_pred', bbox_pred)
workspace.FeedBlob('gt_centers', gt_centers)
workspace.FeedBlob('im_info', im_info)
workspace.RunOperatorOnce(op)
pred_class = workspace.FetchBlob('pred_class')
prob = workspace.FetchBlob('prob')
loss = workspace.FetchBlob('loss')
print('pred_class:', pred_class)
print('loss:', loss)
if check_grad:
gc = gradient_checker.GradientChecker(
stepsize=0.005,
threshold=0.005,
device_option=core.DeviceOption(caffe2_pb2.CUDA, 0)
)
res, grad, grad_estimated = gc.CheckSimple(op, [scores, labels, rois, bbox_pred, gt_centers, im_info],
0, [0])
print('res:', res)
self.assertTrue(res, 'Grad check failed')
self.assertTrue(
grad.shape == grad_estimated.shape,
'Fail check: grad.shape != grad_estimated.shape'
)
def test_forward_and_gradient(self):
Y = np.random.randn(128, 4 * 21).astype(np.float32)
Y_hat = np.random.randn(128, 4 * 21).astype(np.float32)
inside_weights = np.random.randn(128, 4 * 21).astype(np.float32)
inside_weights[inside_weights < 0] = 0
outside_weights = np.random.randn(128, 4 * 21).astype(np.float32)
outside_weights[outside_weights < 0] = 0
scale = np.random.random()
beta = np.random.random()
op = core.CreateOperator(
'SmoothL1Loss', ['Y_hat', 'Y', 'inside_weights', 'outside_weights'],
['loss'],
scale=scale,
beta=beta
)
gc = gradient_checker.GradientChecker(
stepsize=0.005,
threshold=0.005,
device_option=core.DeviceOption(caffe2_pb2.CUDA, 0)
)
res, grad, grad_estimated = gc.CheckSimple(
op, [Y_hat, Y, inside_weights, outside_weights], 0, [0]
)
self.assertTrue(
grad.shape == grad_estimated.shape,
'Fail check: grad.shape != grad_estimated.shape'
)
# To inspect the gradient and estimated gradient:
# np.set_printoptions(precision=3, suppress=True)
# print('grad:')
# print(grad)
# print('grad_estimated:')
# print(grad_estimated)
self.assertTrue(res)
def test_gradient_ref(self):
N, C, H, W = 2, 6, 5, 7
G = 3
scale = np.random.randn(C, ).astype(np.float32)
bias = np.random.randn(C, ).astype(np.float32)
X = np.random.randn(N, C, H, W).astype(np.float32) + 2
op = core.CreateOperator('GroupNorm', ['X', 'scale', 'bias'],
['Y', 'mu', 'sig'],
num_groups=G)
gc = gradient_checker.GradientChecker(stepsize=0.05,
threshold=0.005,
device_option=core.DeviceOption(
caffe2_pb2.CUDA, 0))
for i in range(3): # 3 inputs
res, grad, grad_estimated = gc.CheckSimple(op, [X, scale, bias], i,
[0])
self.assertTrue(grad.shape == grad_estimated.shape,
'grad.shape not matching.')
self.assertTrue(res, 'gradient check fail for input[{}]'.format(i))
import unittest
if workspace.has_gpu_support and workspace.NumGpuDevices() > 0:
gpu_device_option = caffe2_pb2.DeviceOption()
gpu_device_option.device_type = workspace.GpuDeviceType
cpu_device_option = caffe2_pb2.DeviceOption()
gpu_device_checker = device_checker.DeviceChecker(
0.01, [gpu_device_option]
)
device_checker = device_checker.DeviceChecker(
0.01, [gpu_device_option, cpu_device_option]
)
gpu_gradient_checkers = [
gradient_checker.GradientChecker(
0.005, 0.05, gpu_device_option, "gpu_checker_ws"
),
]
gradient_checkers = [
gradient_checker.GradientChecker(
0.005, 0.05, gpu_device_option, "gpu_checker_ws"
),
gradient_checker.GradientChecker(
0.01, 0.05, cpu_device_option, "cpu_checker_ws"
),
]
else:
cpu_device_option = caffe2_pb2.DeviceOption()
gpu_device_option = None
gpu_device_checker = device_checker.DeviceChecker(
0.01, []
def test_forward_and_gradient(self):
N = 64
Ncls = 2
Y = np.random.randn(N, 4 * Ncls).astype(np.float32)
Y_hat = np.random.randn(N, 4 * Ncls).astype(np.float32)
U = np.random.randn(N, 10 * Ncls).astype(np.float32)
print(np.prod(Y.shape))
print(np.prod(U.shape))
inside_weights = np.random.randn(N, 4 * Ncls).astype(np.float32)
inside_weights[inside_weights < 0] = 0
#outside_weights = np.ones((N, 4 * Ncls)).astype(np.float32)
outside_weights_ind = np.random.randint(0, Ncls, N)
outside_weights = np.zeros((N, 4 * Ncls)).astype(np.float32)
for i in range(N):
ind = 4 * outside_weights_ind[i]
outside_weights[i, ind:ind + 4] = 1
scale = np.random.random()
beta = np.random.random()
bbox_inw = outside_weights * (Y - Y_hat)
#op = CreatePythonOperator(
# CovLossOp().forward, ["x1", "x2", "x3"], ["y1"],
# grad_f=CovLossOp().backward,
# grad_input_indices=[0, 1]
# )
#op = CreatePythonOperator(
# UpperTriangularOp().forward, ["x1"], ["y1"],
# grad_f=UpperTriangularOp().backward,
# #grad_input_indices=[0, 1]
# )
op = CreatePythonOperator(
LogDetOp().forward,
["x1"],
["y1"],
grad_f=LogDetOp().backward,
#grad_input_indices=[0, 1]
)
#op = CreatePythonOperator(
# SimpleOp().forward, ["x1"], ["y1"],
# grad_f=SimpleOp().backward,
# #grad_input_indices=[0, 1]
# )
gc = gradient_checker.GradientChecker(
stepsize=0.005,
threshold=0.005,
#device_option=core.DeviceOption(caffe2_pb2.CUDA, 0)
)
#res, grad, grad_estimated = gc.CheckSimple(
# op, [bbox_inw, U, outside_weights], 0, [0]
#)
U = U.reshape((N, Ncls, 10))
U_reshape = np.zeros((N, Ncls, 4, 4), dtype=U.dtype)
U_reshape[:, :, 0, 0] = np.exp(U[:, :, 0])
U_reshape[:, :, 1, 1] = np.exp(U[:, :, 4])
U_reshape[:, :, 2, 2] = np.exp(U[:, :, 7])
U_reshape[:, :, 3, 3] = np.exp(U[:, :, 9])
U_reshape[:, :, 0, 1:] = U[:, :, 1:4]
U_reshape[:, :, 1, 2:] = U[:, :, 5:7]
U_reshape[:, :, 2, 3] = U[:, :, 8]
res, grad, grad_estimated = gc.CheckSimple(op, [U_reshape], 0, [0])
#res, grad, grad_estimated = gc.CheckSimple(
# op, [U], 0, [0]
#)
self.assertTrue(grad.shape == grad_estimated.shape,
'Fail check: grad.shape != grad_estimated.shape')
## To inspect the gradient and estimated gradient:
#np.set_printoptions(precision=3, suppress=True)
#print('grad:')
#print(grad)
#print('grad_estimated:')
#print(grad_estimated)
self.assertTrue(res)
