def RunningAllreduceWithGPUs(self, gpu_ids, allreduce_function): """A base function to test different scenarios.""" workspace.ResetWorkspace() net = core.Net("mujitest") for id in gpu_ids: net.ConstantFill([], "testblob_gpu_" + str(id), shape=[1, 2, 3, 4], value=float(id + 1), device_option=muji.OnGPU(id)) allreduce_function(net, ["testblob_gpu_" + str(i) for i in gpu_ids], "_reduced", gpu_ids) workspace.RunNetOnce(net) target_value = sum(gpu_ids) + len(gpu_ids) all_blobs = workspace.Blobs() all_blobs.sort() for blob in all_blobs: print blob, workspace.FetchBlob(blob) for id in gpu_ids: blob = workspace.FetchBlob("testblob_gpu_" + str(i) + "_reduced") np.testing.assert_array_equal(blob, target_value, err_msg="gpu id %d of %s" % (id, str(gpu_ids)))
def _testMiniAlexNet(self, order): # First, we get all the random initialization of parameters. model = self._MiniAlexNetNoDropout(order) workspace.ResetWorkspace() workspace.RunNetOnce(model.param_init_net) inputs = dict([(str(name), workspace.FetchBlob(str(name))) for name in model.params]) if order == "NCHW": inputs["data"] = np.random.rand(4, 3, 227, 227).astype(np.float32) else: inputs["data"] = np.random.rand(4, 227, 227, 3).astype(np.float32) inputs["label"] = np.array([1, 2, 3, 4]).astype(np.int32) cpu_device = caffe2_pb2.DeviceOption() cpu_device.device_type = caffe2_pb2.CPU gpu_device = caffe2_pb2.DeviceOption() gpu_device.device_type = caffe2_pb2.CUDA checker = device_checker.DeviceChecker(1e-5, [cpu_device, gpu_device]) ret = checker.CheckNet( model.net.Proto(), inputs, # The indices sometimes may be sensitive to small numerical differences # in the input, so we ignore checking them. ignore=['_pool1_idx', '_pool2_idx', '_pool5_idx']) self.assertEqual(ret, True)
def testMNISTNetworks(self): # First, we get all the random initialization of parameters. init_net, train_net = self._MNISTNetworks() workspace.ResetWorkspace() workspace.RunNetOnce(init_net) inputs = dict([(str(name), workspace.FetchBlob(str(name))) for name in workspace.Blobs()]) cpu_device = caffe2_pb2.DeviceOption() cpu_device.device_type = caffe2_pb2.CPU gpu_device = caffe2_pb2.DeviceOption() gpu_device.device_type = caffe2_pb2.CUDA checker = device_checker.DeviceChecker(1e-2, [cpu_device, gpu_device]) ret = checker.CheckNet(train_net.Proto(), inputs) self.assertEqual(ret, True)
def CheckSimple(self, op, inputs, outputs_to_check): """Checks the operator in a very simple fashion by stacking a sum of squares on the top. Inputs: op: the operator to be checked. inputs: the input data in numpy arrays. input_to_check: an index specifying which input blob we should check. outputs_with_grads: indices specifying which output blobs will we need to check gradients with. For these outputs, we will collect a squared sum and also feed in their gradients. grad_operator: the gradient operator. If not given, we will get the gradient operator from the gradient registry. Outputs: boolean: True if it passes, False if it does not pass. """ # Entering the checker workspace old_ws_name = workspace.CurrentWorkspace() results = [] workspace.SwitchWorkspace("_device_check_", True) for i, device_option in enumerate(self._device_options): for i, arr in enumerate(inputs): workspace.FeedBlob(op.inputs[i], arr, device_option) op.device_option.CopyFrom(device_option) workspace.RunOperatorOnce(op) results.append([ workspace.FetchBlob(op.outputs[idx]) for idx in outputs_to_check ]) # Everything is done, reset the workspace. workspace.ResetWorkspace() # After running on all devices, check correctness success = True for i in range(1, len(self._device_options)): for j in range(len(outputs_to_check)): x = results[i][j] y = results[0][j] if np.any(np.abs(x - y) > self._threshold): print 'Failure in checking device option', i, 'and output ', print op.outputs[j], '. The outputs are:' print x.flatten() print y.flatten() success = False continue workspace.SwitchWorkspace(old_ws_name) return success
def CheckSimple(self, op, inputs, outputs_to_check): """Checks the operator with different device implementations. Inputs: op: the operator to be checked. inputs: the input data in numpy arrays. outputs_to_check: the outputs to check between devices. Outputs: boolean: True if it passes, False if it does not pass. """ # Entering the checker workspace old_ws_name = workspace.CurrentWorkspace() results = [] workspace.SwitchWorkspace("_device_check_", True) for i, device_option in enumerate(self._device_options): for i, arr in enumerate(inputs): workspace.FeedBlob(op.input[i], arr, device_option) op.device_option.CopyFrom(device_option) workspace.RunOperatorOnce(op) results.append([ workspace.FetchBlob(op.output[idx]) for idx in outputs_to_check ]) # Everything is done, reset the workspace. workspace.ResetWorkspace() # After running on all devices, check correctness success = True for i in range(1, len(self._device_options)): for j in range(len(outputs_to_check)): x = results[i][j] y = results[0][j] if np.any(np.abs(x - y) > self._threshold): print 'Failure in checking device option', i, 'and output ', print op.output[j], '. The outputs are:' print x.flatten() print y.flatten() success = False #else: # print ('Passed device pair (0, %d), %s %s' % # (i, outputs_to_check[j], y.shape)) workspace.SwitchWorkspace(old_ws_name) return success
def setUp(self): workspace.SwitchWorkspace("default") workspace.ResetWorkspace()
def setUp(self): self.net = core.Net("test-net") self.net.ConstantFill([], "testblob", shape=[1, 2, 3, 4], value=1.0) workspace.ResetWorkspace()
def testResetWorkspace(self): self.assertEqual(workspace.RunNetOnce(self.net.Proto().SerializeToString()), True) self.assertEqual(workspace.HasBlob("testblob"), True) self.assertEqual(workspace.ResetWorkspace(), True) self.assertEqual(workspace.HasBlob("testblob"), False)
def testRootFolder(self): self.assertEqual(workspace.ResetWorkspace(), True) self.assertEqual(workspace.RootFolder(), ".") self.assertEqual(workspace.ResetWorkspace("/home/test"), True) self.assertEqual(workspace.RootFolder(), "/home/test")
def CheckSimple(self, op, inputs, input_to_check, outputs_with_grads, grad_ops=None): """Checks the operator in a very simple fashion by stacking a sum of squares on the top. Inputs: op: the operator to be checked. inputs: the input data in numpy arrays. input_to_check: an index specifying which input blob we should check. outputs_with_grads: indices specifying which output blobs will we need to check gradients with. For these outputs, we will collect a squared sum and also feed in their gradients. grad_operator: the gradient operator. If not given, we will get the gradient operator from the gradient registry. Outputs: boolean: True if it passes, False if it does not pass. """ # Entering the checker workspace old_ws_name = workspace.CurrentWorkspace() if self._workspace_name != old_ws_name: workspace.SwitchWorkspace(self._workspace_name, True) op.device_option.CopyFrom(self._device_option) if grad_ops is None: grad_ops = core.GradientRegistry.GetGradient(op) dims_to_check = inputs[input_to_check].size # First, feed in the input. for i, arr in enumerate(inputs): workspace.FeedBlob(op.inputs[i], arr, self._device_option) # Get the loss and gradient for the original. input_name = op.inputs[input_to_check] loss, grad = self.GetLossAndGrad(op, grad_ops, inputs[input_to_check], input_name, outputs_with_grads) grad_estimate = np.zeros_like(inputs[input_to_check]) for current_dim in range(dims_to_check): # Positive gradient inputs[input_to_check].flat[current_dim] += self._stepsize pos_loss, _ = self.GetLossAndGrad(op, grad_ops, inputs[input_to_check], input_name, outputs_with_grads) # Negative gradient inputs[input_to_check].flat[current_dim] -= self._stepsize * 2 neg_loss, _ = self.GetLossAndGrad(op, grad_ops, inputs[input_to_check], input_name, outputs_with_grads) # Recover the value inputs[input_to_check].flat[current_dim] += self._stepsize grad_estimate.flat[current_dim] = (pos_loss - neg_loss) / self._stepsize / 2 # Now, check correctness scale = np.maximum(np.maximum(np.abs(grad), np.abs(grad_estimate)), 1) fail_mat = (np.abs(grad - grad_estimate) > scale * self._threshold) if np.any(fail_mat): idx = np.flatnonzero(fail_mat) #print 'Failed. [idx, grad, grad_estimate] are:' #print np.vstack([idx, grad.flat[idx], grad_estimate.flat[idx]]).T ret = False else: ret = True # After finishing, cleaning up things. if self._workspace_name != old_ws_name: # We reset the workspace to make sure everything intermediate is cleaned # up. Note that there is no need to delete a workspace - when empty it # takes a very limited amount of memory. workspace.ResetWorkspace() workspace.SwitchWorkspace(old_ws_name) return ret, grad, grad_estimate
def testRootFolder(self): self.assertEqual(workspace.ResetWorkspace(), True) self.assertEqual(workspace.RootFolder(), ".") self.assertEqual(workspace.ResetWorkspace("/tmp/caffe-workspace-test"), True) self.assertEqual(workspace.RootFolder(), "/tmp/caffe-workspace-test")