def assertGradientChecks( self, device_option, op, inputs, outputs_to_check, outputs_with_grads, grad_ops=None, threshold=0.005, stepsize=0.05, input_device_options=None, ): """ Implements a standard numerical gradient checker for the operator in question. Useful for checking the consistency of the forward and backward implementations of operators. Usage example: @given(inputs=hu.tensors(n=2), in_place=st.booleans(), **hu.gcs) def test_sum(self, inputs, in_place, gc, dc): op = core.CreateOperator("Sum", ["X1", "X2"], ["Y" if not in_place else "X1"]) X1, X2 = inputs self.assertGradientChecks(gc, op, [X1, X2], 0, [0]) """ gc = gradient_checker.GradientChecker( stepsize=stepsize, threshold=threshold, device_option=device_option, workspace_name=str(device_option), input_device_options=input_device_options, ) res, grad, grad_estimated = gc.CheckSimple( op, inputs, outputs_to_check, outputs_with_grads, grad_ops=grad_ops, input_device_options=input_device_options ) self.assertEqual(grad.shape, grad_estimated.shape) self.assertTrue( res, "Gradient check failed for input " + str(op.input[outputs_to_check]) )
def _run_op_test(self, X, I, check_grad=False): with core.DeviceScope(core.DeviceOption(caffe2_pb2.CUDA, 0)): op = core.CreateOperator('BatchPermutation', ['X', 'I'], ['Y']) workspace.FeedBlob('X', X) workspace.FeedBlob('I', I) workspace.RunOperatorOnce(op) Y = workspace.FetchBlob('Y') if check_grad: gc = gradient_checker.GradientChecker( stepsize=0.1, threshold=0.001, device_option=core.DeviceOption(caffe2_pb2.CUDA, 0)) res, grad, grad_estimated = gc.CheckSimple(op, [X, I], 0, [0]) self.assertTrue(res, 'Grad check failed') Y_ref = X[I] np.testing.assert_allclose(Y, Y_ref, rtol=1e-5, atol=1e-08)
def _run_test(self, A, B, check_grad=False): with core.DeviceScope(core.DeviceOption(caffe2_pb2.CUDA, 0)): op = core.CreateOperator('SpatialNarrowAs', ['A', 'B'], ['C']) workspace.FeedBlob('A', A) workspace.FeedBlob('B', B) workspace.RunOperatorOnce(op) C = workspace.FetchBlob('C') if check_grad: gc = gradient_checker.GradientChecker( stepsize=0.005, threshold=0.005, device_option=core.DeviceOption(caffe2_pb2.CUDA, 0)) res, grad, grad_estimated = gc.CheckSimple(op, [A, B], 0, [0]) self.assertTrue(res, 'Grad check failed') dims = C.shape C_ref = A[:dims[0], :dims[1], :dims[2], :dims[3]] np.testing.assert_allclose(C, C_ref, rtol=1e-5, atol=1e-08)
def _run_test(self, scores, labels, rois, bbox_pred, gt_centers, im_info, regression_ws, num_classes, check_grad=False): with core.DeviceScope(core.DeviceOption(caffe2_pb2.CUDA, 0)): # op = core.CreateOperator('SsDistanceLoss', # ['scores', 'labels', 'rois', 'bbox_pred', 'gt_centers', 'im_info'], # ['loss', 'p', 'pred_class'], # regression_ws=regression_ws, num_classes=num_classes) op = core.CreateOperator('SsDistanceLoss', ['scores', 'labels', 'rois', 'bbox_pred', 'gt_centers', 'im_info'], ['loss', 'prob', 'pred_class'], num_classes=num_classes) workspace.FeedBlob('scores', scores) workspace.FeedBlob('labels', labels) workspace.FeedBlob('rois', rois) workspace.FeedBlob('bbox_pred', bbox_pred) workspace.FeedBlob('gt_centers', gt_centers) workspace.FeedBlob('im_info', im_info) workspace.RunOperatorOnce(op) pred_class = workspace.FetchBlob('pred_class') prob = workspace.FetchBlob('prob') loss = workspace.FetchBlob('loss') print('pred_class:', pred_class) print('loss:', loss) if check_grad: gc = gradient_checker.GradientChecker( stepsize=0.005, threshold=0.005, device_option=core.DeviceOption(caffe2_pb2.CUDA, 0) ) res, grad, grad_estimated = gc.CheckSimple(op, [scores, labels, rois, bbox_pred, gt_centers, im_info], 0, [0]) print('res:', res) self.assertTrue(res, 'Grad check failed') self.assertTrue( grad.shape == grad_estimated.shape, 'Fail check: grad.shape != grad_estimated.shape' )
def test_forward_and_gradient(self): Y = np.random.randn(128, 4 * 21).astype(np.float32) Y_hat = np.random.randn(128, 4 * 21).astype(np.float32) inside_weights = np.random.randn(128, 4 * 21).astype(np.float32) inside_weights[inside_weights < 0] = 0 outside_weights = np.random.randn(128, 4 * 21).astype(np.float32) outside_weights[outside_weights < 0] = 0 scale = np.random.random() beta = np.random.random() op = core.CreateOperator( 'SmoothL1Loss', ['Y_hat', 'Y', 'inside_weights', 'outside_weights'], ['loss'], scale=scale, beta=beta ) gc = gradient_checker.GradientChecker( stepsize=0.005, threshold=0.005, device_option=core.DeviceOption(caffe2_pb2.CUDA, 0) ) res, grad, grad_estimated = gc.CheckSimple( op, [Y_hat, Y, inside_weights, outside_weights], 0, [0] ) self.assertTrue( grad.shape == grad_estimated.shape, 'Fail check: grad.shape != grad_estimated.shape' ) # To inspect the gradient and estimated gradient: # np.set_printoptions(precision=3, suppress=True) # print('grad:') # print(grad) # print('grad_estimated:') # print(grad_estimated) self.assertTrue(res)
def test_gradient_ref(self): N, C, H, W = 2, 6, 5, 7 G = 3 scale = np.random.randn(C, ).astype(np.float32) bias = np.random.randn(C, ).astype(np.float32) X = np.random.randn(N, C, H, W).astype(np.float32) + 2 op = core.CreateOperator('GroupNorm', ['X', 'scale', 'bias'], ['Y', 'mu', 'sig'], num_groups=G) gc = gradient_checker.GradientChecker(stepsize=0.05, threshold=0.005, device_option=core.DeviceOption( caffe2_pb2.CUDA, 0)) for i in range(3): # 3 inputs res, grad, grad_estimated = gc.CheckSimple(op, [X, scale, bias], i, [0]) self.assertTrue(grad.shape == grad_estimated.shape, 'grad.shape not matching.') self.assertTrue(res, 'gradient check fail for input[{}]'.format(i))
import unittest if workspace.has_gpu_support and workspace.NumGpuDevices() > 0: gpu_device_option = caffe2_pb2.DeviceOption() gpu_device_option.device_type = workspace.GpuDeviceType cpu_device_option = caffe2_pb2.DeviceOption() gpu_device_checker = device_checker.DeviceChecker( 0.01, [gpu_device_option] ) device_checker = device_checker.DeviceChecker( 0.01, [gpu_device_option, cpu_device_option] ) gpu_gradient_checkers = [ gradient_checker.GradientChecker( 0.005, 0.05, gpu_device_option, "gpu_checker_ws" ), ] gradient_checkers = [ gradient_checker.GradientChecker( 0.005, 0.05, gpu_device_option, "gpu_checker_ws" ), gradient_checker.GradientChecker( 0.01, 0.05, cpu_device_option, "cpu_checker_ws" ), ] else: cpu_device_option = caffe2_pb2.DeviceOption() gpu_device_option = None gpu_device_checker = device_checker.DeviceChecker( 0.01, []
def test_forward_and_gradient(self): N = 64 Ncls = 2 Y = np.random.randn(N, 4 * Ncls).astype(np.float32) Y_hat = np.random.randn(N, 4 * Ncls).astype(np.float32) U = np.random.randn(N, 10 * Ncls).astype(np.float32) print(np.prod(Y.shape)) print(np.prod(U.shape)) inside_weights = np.random.randn(N, 4 * Ncls).astype(np.float32) inside_weights[inside_weights < 0] = 0 #outside_weights = np.ones((N, 4 * Ncls)).astype(np.float32) outside_weights_ind = np.random.randint(0, Ncls, N) outside_weights = np.zeros((N, 4 * Ncls)).astype(np.float32) for i in range(N): ind = 4 * outside_weights_ind[i] outside_weights[i, ind:ind + 4] = 1 scale = np.random.random() beta = np.random.random() bbox_inw = outside_weights * (Y - Y_hat) #op = CreatePythonOperator( # CovLossOp().forward, ["x1", "x2", "x3"], ["y1"], # grad_f=CovLossOp().backward, # grad_input_indices=[0, 1] # ) #op = CreatePythonOperator( # UpperTriangularOp().forward, ["x1"], ["y1"], # grad_f=UpperTriangularOp().backward, # #grad_input_indices=[0, 1] # ) op = CreatePythonOperator( LogDetOp().forward, ["x1"], ["y1"], grad_f=LogDetOp().backward, #grad_input_indices=[0, 1] ) #op = CreatePythonOperator( # SimpleOp().forward, ["x1"], ["y1"], # grad_f=SimpleOp().backward, # #grad_input_indices=[0, 1] # ) gc = gradient_checker.GradientChecker( stepsize=0.005, threshold=0.005, #device_option=core.DeviceOption(caffe2_pb2.CUDA, 0) ) #res, grad, grad_estimated = gc.CheckSimple( # op, [bbox_inw, U, outside_weights], 0, [0] #) U = U.reshape((N, Ncls, 10)) U_reshape = np.zeros((N, Ncls, 4, 4), dtype=U.dtype) U_reshape[:, :, 0, 0] = np.exp(U[:, :, 0]) U_reshape[:, :, 1, 1] = np.exp(U[:, :, 4]) U_reshape[:, :, 2, 2] = np.exp(U[:, :, 7]) U_reshape[:, :, 3, 3] = np.exp(U[:, :, 9]) U_reshape[:, :, 0, 1:] = U[:, :, 1:4] U_reshape[:, :, 1, 2:] = U[:, :, 5:7] U_reshape[:, :, 2, 3] = U[:, :, 8] res, grad, grad_estimated = gc.CheckSimple(op, [U_reshape], 0, [0]) #res, grad, grad_estimated = gc.CheckSimple( # op, [U], 0, [0] #) self.assertTrue(grad.shape == grad_estimated.shape, 'Fail check: grad.shape != grad_estimated.shape') ## To inspect the gradient and estimated gradient: #np.set_printoptions(precision=3, suppress=True) #print('grad:') #print(grad) #print('grad_estimated:') #print(grad_estimated) self.assertTrue(res)