def check_grad_with_place(self, place, inputs_to_check, output_names, no_grad_set=None, numeric_grad_delta=0.005, in_place=False, max_relative_error=0.005, user_defined_grads=None, check_dygraph=True, transpose_input_list=[]): self.scope = core.Scope() op_inputs = self.inputs if hasattr(self, "inputs") else dict() op_outputs = self.outputs if hasattr(self, "outputs") else dict() op_attrs = self.attrs if hasattr(self, "attrs") else dict() self._check_grad_helper() cache_list = None if hasattr(self, "cache_name_list"): cache_list = self.cache_name_list self.op = create_op(self.scope, self.op_type, op_inputs, op_outputs, op_attrs, cache_list=cache_list) if no_grad_set is None: no_grad_set = set() for input_to_check in inputs_to_check: set_input(self.scope, self.op, self.inputs, place) tensor_to_check = self.scope.find_var(input_to_check).get_tensor() tensor_size = six.moves.reduce(lambda a, b: a * b, tensor_to_check.shape(), 1) if tensor_size < 100: self.__class__.input_shape_is_large = False if not type(output_names) is list: output_names = [output_names] numeric_grads = user_defined_grads or [ self.get_numeric_gradient( place, self.scope, self.op, self.inputs, input_to_check, output_names, delta=numeric_grad_delta, in_place=in_place, transpose_input_list=transpose_input_list) for input_to_check in inputs_to_check ] analytic_grads = self._get_gradient(inputs_to_check, place, output_names, no_grad_set) self._assert_is_close(numeric_grads, analytic_grads, inputs_to_check, max_relative_error, "Gradient Check On %s" % str(place))
def get_numeric_gradient(place, scope, op, inputs, input_to_check, output_names, delta=0.005, in_place=False): # FIXME: change this method by compile time concepts set_input(scope, op, inputs, place) def product(dim): return six.moves.reduce(lambda a, b: a * b, dim, 1) def get_output(): sum = [] op.run(scope, place) for output_name in output_names: sum.append( np.array(scope.find_var(output_name).get_tensor()).mean()) return np.array(sum).sum() / len(output_names) tensor_to_check = scope.find_var(input_to_check).get_tensor() tensor_size = product(tensor_to_check.shape()) tensor_to_check_dtype = tensor_to_check._dtype() if tensor_to_check_dtype == core.VarDesc.VarType.FP32: tensor_to_check_dtype = np.float32 elif tensor_to_check_dtype == core.VarDesc.VarType.FP64: tensor_to_check_dtype = np.float64 elif tensor_to_check_dtype == core.VarDesc.VarType.FP16: tensor_to_check_dtype = np.float16 # set delta as np.float16, will automatic convert to float32, float64 delta = np.array(delta).astype(np.float16) else: raise ValueError("Not supported data type " + str(tensor_to_check_dtype)) gradient_flat = np.zeros(shape=(tensor_size, ), dtype=tensor_to_check_dtype) def __get_elem__(tensor, i): if tensor_to_check_dtype == np.float16: numpy_tensor = np.array(tensor).astype(np.float16) numpy_tensor = numpy_tensor.flatten() return numpy_tensor[i] elif tensor_to_check_dtype == np.float32: return tensor._get_float_element(i) else: return tensor._get_double_element(i) def __set_elem__(tensor, i, e): if tensor_to_check_dtype == np.float16: numpy_tensor = np.array(tensor).astype(np.float16) shape = numpy_tensor.shape numpy_tensor = numpy_tensor.flatten() numpy_tensor[i] = e numpy_tensor = numpy_tensor.reshape(shape).view(np.uint16) tensor.set(numpy_tensor, place) elif tensor_to_check_dtype == np.float32: tensor._set_float_element(i, e) else: tensor._set_double_element(i, e) # we only compute gradient of one element each time. # we use a for loop to compute the gradient of every element. for i in six.moves.xrange(tensor_size): if in_place: set_input(scope, op, inputs, place) # get one input element throw it's index i. origin = __get_elem__(tensor_to_check, i) # add delta to it, run op and then get the sum of the result tensor. x_pos = origin + delta __set_elem__(tensor_to_check, i, x_pos) y_pos = get_output() if in_place: set_input(scope, op, inputs, place) x_neg = origin - delta __set_elem__(tensor_to_check, i, x_neg) y_neg = get_output() __set_elem__(tensor_to_check, i, origin) gradient_flat[i] = (y_pos - y_neg) / delta / 2 return gradient_flat.reshape(tensor_to_check.shape())
def get_grad_with_place(self, place, inputs_to_check, output_names, no_grad_set=None, numeric_grad_delta=0.005, in_place=False, max_relative_error=0.005, user_defined_grads=None, check_dygraph=True): self.scope = core.Scope() op_inputs = self.inputs if hasattr(self, "inputs") else dict() op_outputs = self.outputs if hasattr(self, "outputs") else dict() op_attrs = self.attrs if hasattr(self, "attrs") else dict() self._check_grad_helper() if self.dtype == np.float64 and \ self.op_type not in op_threshold_white_list.NEED_FIX_FP64_CHECK_GRAD_THRESHOLD_OP_LIST: numeric_grad_delta = 1e-5 max_relative_error = 1e-7 cache_list = None if hasattr(self, "cache_name_list"): cache_list = self.cache_name_list # oneDNN numeric gradient should use CPU kernel use_onednn = False if "use_mkldnn" in op_attrs and op_attrs["use_mkldnn"] == True: op_attrs["use_mkldnn"] = False use_onednn = True self.op = create_op( self.scope, self.op_type, op_inputs, op_outputs, op_attrs, cache_list=cache_list) if use_onednn: op_attrs["use_mkldnn"] = True if no_grad_set is None: no_grad_set = set() else: if (self.op_type not in no_grad_set_white_list.NEED_TO_FIX_OP_LIST ) and ( self.op_type not in no_grad_set_white_list.NOT_CHECK_OP_LIST ) and (not self.is_bfloat16_op()): raise AssertionError("no_grad_set must be None, op_type is " + self.op_type + " Op.") for input_to_check in inputs_to_check: set_input(self.scope, self.op, self.inputs, place) tensor_to_check = self.scope.find_var(input_to_check).get_tensor() tensor_size = six.moves.reduce(lambda a, b: a * b, tensor_to_check.shape(), 1) if tensor_size < 100: self.__class__.input_shape_is_large = False if not type(output_names) is list: output_names = [output_names] analytic_grads = self._get_gradient(inputs_to_check, place, output_names, no_grad_set) return analytic_grads
def get_numeric_gradient(self, place, scope, op, inputs, input_to_check, output_names, delta=0.005, in_place=False, transpose_input_list=[]): # FIXME: change this method by compile time concepts set_input(scope, op, inputs, place) def product(dim): return six.moves.reduce(lambda a, b: a * b, dim, 1) delta_origin = delta tensor_to_check = scope.find_var(input_to_check).get_tensor() input_shape = tensor_to_check.shape() # some input with lod need to transpose at begin # it is be reconstructed here first if input_to_check in transpose_input_list: tensor_to_check = np.transpose(tensor_to_check, [1, 0]) tensor_to_check_ = fluid.LoDTensor() tensor_to_check_.set(tensor_to_check, fluid.CPUPlace()) tensor_to_check = tensor_to_check_ tensor_size = int(product(input_shape)) tensor_to_check_dtype = tensor_to_check._dtype() input_plain_shape = tensor_to_check.shape()[:] if tensor_to_check_dtype == core.VarDesc.VarType.FP32: tensor_to_check_dtype = np.float32 elif tensor_to_check_dtype == core.VarDesc.VarType.FP64: tensor_to_check_dtype = np.float64 elif tensor_to_check_dtype == core.VarDesc.VarType.FP16: tensor_to_check_dtype = np.float16 # set delta as np.float16, will automatic convert to float32, float64 delta = np.array(delta).astype(np.float16) elif tensor_to_check_dtype == core.VarDesc.VarType.INT64: tensor_to_check_dtype = np.int64 delta = self.lazy_share(delta) else: raise ValueError("Not supported data type " + str(tensor_to_check_dtype)) def get_output(): sum = [] return_results = dict() for name in (output_names): return_results[name] = Manager().list() def closure(**kwargs): role = kwargs['role'] pfl_mpc.init("privc", role, "localhost", self.server, int(self.port)) executor = Executor(place) executor.run() op.run(scope, place) for name in output_names: out = np.array(scope.find_var(name).get_tensor()) return_results[name].append(out) ret = self.multi_party_run(target=closure) self.assertEqual(ret[0], True) for output_name in output_names: plain = self.reconstruct(np.array(return_results[output_name])) sum.append(plain.mean()) return (np.array(sum).sum() / len(output_names)).astype(np.float) gradient_flat = np.zeros(shape=(tensor_size, ), dtype=np.float) def __get_elem__(tensor, i): if tensor_to_check_dtype == np.float16: numpy_tensor = np.array(tensor).astype(np.float16) numpy_tensor = numpy_tensor.flatten() return numpy_tensor[i] elif tensor_to_check_dtype == np.int64: numpy_tensor = np.array(tensor).astype(np.int64) numpy_tensor = numpy_tensor.flatten() return numpy_tensor[i] elif tensor_to_check_dtype == np.float32: return tensor._get_float_element(i) else: return tensor._get_double_element(i) def __set_elem__(tensor, i, e): if tensor_to_check_dtype == np.float16: numpy_tensor = np.array(tensor).astype(np.float16) shape = numpy_tensor.shape numpy_tensor = numpy_tensor.flatten() numpy_tensor[i] = e numpy_tensor = numpy_tensor.reshape(shape) tensor.set(numpy_tensor, place) elif tensor_to_check_dtype == np.int64: numpy_tensor = np.array(tensor).astype(np.int64) shape = numpy_tensor.shape numpy_tensor = numpy_tensor.flatten() numpy_tensor[i] = e numpy_tensor = numpy_tensor.reshape(shape) if input_to_check in transpose_input_list: numpy_tensor = np.transpose(numpy_tensor, [1, 0]) numpy_tensor = np.array(numpy_tensor).reshape(input_shape) tensor_input_origin = scope.find_var( input_to_check).get_tensor() tensor_input_origin.set(numpy_tensor, place) elif tensor_to_check_dtype == np.float32: tensor._set_float_element(i, e) else: tensor._set_double_element(i, e) # we only compute gradient of one element each time. # we use a for loop to compute the gradient of every element. for i in six.moves.xrange(tensor_size): if in_place: set_input(scope, op, inputs, place) # get one input element throw it's index i. origin = __get_elem__(tensor_to_check, i) x_pos = origin + delta __set_elem__(tensor_to_check, i, x_pos) y_pos = get_output() if in_place: set_input(scope, op, inputs, place) x_neg = origin - delta __set_elem__(tensor_to_check, i, x_neg) y_neg = get_output() __set_elem__(tensor_to_check, i, origin) gradient_flat[i] = (y_pos - y_neg) / delta_origin / 2 return gradient_flat.reshape(input_plain_shape)
def check_grad_with_place(self, place, inputs_to_check, output_names, no_grad_set=None, numeric_grad_delta=0.005, in_place=False, max_relative_error=0.005, user_defined_grads=None, check_dygraph=True): self.scope = core.Scope() op_inputs = self.inputs if hasattr(self, "inputs") else dict() op_outputs = self.outputs if hasattr(self, "outputs") else dict() op_attrs = self.attrs if hasattr(self, "attrs") else dict() prog = Program() block = prog.global_block() op_init = block.append_op( type="mpc_init", #inputs=inputs, #outputs=outputs, attrs={"protocol_name": "privc"}) op_init.desc.infer_shape(block.desc) mpc_protocol_index = MpcProtocols["PRIVC"].value fluid.global_scope().var("mpc_protocol_index").get_tensor().set( np.array((mpc_protocol_index)), fluid.CPUPlace()) self._check_grad_helper() cache_list = None if hasattr(self, "cache_name_list"): cache_list = self.cache_name_list self.op = create_op(self.scope, self.op_type, op_inputs, op_outputs, op_attrs, cache_list=cache_list) if no_grad_set is None: no_grad_set = set() for input_to_check in inputs_to_check: set_input(self.scope, self.op, self.inputs, place) tensor_to_check = self.scope.find_var(input_to_check).get_tensor() tensor_size = six.moves.reduce(lambda a, b: a * b, tensor_to_check.shape(), 1) if tensor_size < 100: self.__class__.input_shape_is_large = False if not type(output_names) is list: output_names = [output_names] numeric_grads = user_defined_grads or [ self.get_numeric_gradient(place, self.scope, self.op, self.inputs, input_to_check, output_names, delta=numeric_grad_delta, in_place=in_place) for input_to_check in inputs_to_check ] analytic_grads = self._get_gradient(inputs_to_check, place, output_names, no_grad_set) self._assert_is_close(numeric_grads, analytic_grads, inputs_to_check, max_relative_error, "Gradient Check On %s" % str(place))