def check_grad_with_place(self,
place,
inputs_to_check,
output_names,
no_grad_set=None,
numeric_grad_delta=0.005,
in_place=False,
max_relative_error=0.005,
user_defined_grads=None,
check_dygraph=True,
transpose_input_list=[]):
self.scope = core.Scope()
op_inputs = self.inputs if hasattr(self, "inputs") else dict()
op_outputs = self.outputs if hasattr(self, "outputs") else dict()
op_attrs = self.attrs if hasattr(self, "attrs") else dict()
self._check_grad_helper()
cache_list = None
if hasattr(self, "cache_name_list"):
cache_list = self.cache_name_list
self.op = create_op(self.scope,
self.op_type,
op_inputs,
op_outputs,
op_attrs,
cache_list=cache_list)
if no_grad_set is None:
no_grad_set = set()
for input_to_check in inputs_to_check:
set_input(self.scope, self.op, self.inputs, place)
tensor_to_check = self.scope.find_var(input_to_check).get_tensor()
tensor_size = six.moves.reduce(lambda a, b: a * b,
tensor_to_check.shape(), 1)
if tensor_size < 100:
self.__class__.input_shape_is_large = False
if not type(output_names) is list:
output_names = [output_names]
numeric_grads = user_defined_grads or [
self.get_numeric_gradient(
place,
self.scope,
self.op,
self.inputs,
input_to_check,
output_names,
delta=numeric_grad_delta,
in_place=in_place,
transpose_input_list=transpose_input_list)
for input_to_check in inputs_to_check
]
analytic_grads = self._get_gradient(inputs_to_check, place,
output_names, no_grad_set)
self._assert_is_close(numeric_grads, analytic_grads, inputs_to_check,
max_relative_error,
"Gradient Check On %s" % str(place))
def get_numeric_gradient(place,
scope,
op,
inputs,
input_to_check,
output_names,
delta=0.005,
in_place=False):
# FIXME: change this method by compile time concepts
set_input(scope, op, inputs, place)
def product(dim):
return six.moves.reduce(lambda a, b: a * b, dim, 1)
def get_output():
sum = []
op.run(scope, place)
for output_name in output_names:
sum.append(
np.array(scope.find_var(output_name).get_tensor()).mean())
return np.array(sum).sum() / len(output_names)
tensor_to_check = scope.find_var(input_to_check).get_tensor()
tensor_size = product(tensor_to_check.shape())
tensor_to_check_dtype = tensor_to_check._dtype()
if tensor_to_check_dtype == core.VarDesc.VarType.FP32:
tensor_to_check_dtype = np.float32
elif tensor_to_check_dtype == core.VarDesc.VarType.FP64:
tensor_to_check_dtype = np.float64
elif tensor_to_check_dtype == core.VarDesc.VarType.FP16:
tensor_to_check_dtype = np.float16
# set delta as np.float16, will automatic convert to float32, float64
delta = np.array(delta).astype(np.float16)
else:
raise ValueError("Not supported data type " +
str(tensor_to_check_dtype))
gradient_flat = np.zeros(shape=(tensor_size, ),
dtype=tensor_to_check_dtype)
def __get_elem__(tensor, i):
if tensor_to_check_dtype == np.float16:
numpy_tensor = np.array(tensor).astype(np.float16)
numpy_tensor = numpy_tensor.flatten()
return numpy_tensor[i]
elif tensor_to_check_dtype == np.float32:
return tensor._get_float_element(i)
else:
return tensor._get_double_element(i)
def __set_elem__(tensor, i, e):
if tensor_to_check_dtype == np.float16:
numpy_tensor = np.array(tensor).astype(np.float16)
shape = numpy_tensor.shape
numpy_tensor = numpy_tensor.flatten()
numpy_tensor[i] = e
numpy_tensor = numpy_tensor.reshape(shape).view(np.uint16)
tensor.set(numpy_tensor, place)
elif tensor_to_check_dtype == np.float32:
tensor._set_float_element(i, e)
else:
tensor._set_double_element(i, e)
# we only compute gradient of one element each time.
# we use a for loop to compute the gradient of every element.
for i in six.moves.xrange(tensor_size):
if in_place:
set_input(scope, op, inputs, place)
# get one input element throw it's index i.
origin = __get_elem__(tensor_to_check, i)
# add delta to it, run op and then get the sum of the result tensor.
x_pos = origin + delta
__set_elem__(tensor_to_check, i, x_pos)
y_pos = get_output()
if in_place:
set_input(scope, op, inputs, place)
x_neg = origin - delta
__set_elem__(tensor_to_check, i, x_neg)
y_neg = get_output()
__set_elem__(tensor_to_check, i, origin)
gradient_flat[i] = (y_pos - y_neg) / delta / 2
return gradient_flat.reshape(tensor_to_check.shape())
def get_grad_with_place(self,
place,
inputs_to_check,
output_names,
no_grad_set=None,
numeric_grad_delta=0.005,
in_place=False,
max_relative_error=0.005,
user_defined_grads=None,
check_dygraph=True):
self.scope = core.Scope()
op_inputs = self.inputs if hasattr(self, "inputs") else dict()
op_outputs = self.outputs if hasattr(self, "outputs") else dict()
op_attrs = self.attrs if hasattr(self, "attrs") else dict()
self._check_grad_helper()
if self.dtype == np.float64 and \
self.op_type not in op_threshold_white_list.NEED_FIX_FP64_CHECK_GRAD_THRESHOLD_OP_LIST:
numeric_grad_delta = 1e-5
max_relative_error = 1e-7
cache_list = None
if hasattr(self, "cache_name_list"):
cache_list = self.cache_name_list
# oneDNN numeric gradient should use CPU kernel
use_onednn = False
if "use_mkldnn" in op_attrs and op_attrs["use_mkldnn"] == True:
op_attrs["use_mkldnn"] = False
use_onednn = True
self.op = create_op(
self.scope,
self.op_type,
op_inputs,
op_outputs,
op_attrs,
cache_list=cache_list)
if use_onednn:
op_attrs["use_mkldnn"] = True
if no_grad_set is None:
no_grad_set = set()
else:
if (self.op_type not in no_grad_set_white_list.NEED_TO_FIX_OP_LIST
) and (
self.op_type not in no_grad_set_white_list.NOT_CHECK_OP_LIST
) and (not self.is_bfloat16_op()):
raise AssertionError("no_grad_set must be None, op_type is " +
self.op_type + " Op.")
for input_to_check in inputs_to_check:
set_input(self.scope, self.op, self.inputs, place)
tensor_to_check = self.scope.find_var(input_to_check).get_tensor()
tensor_size = six.moves.reduce(lambda a, b: a * b,
tensor_to_check.shape(), 1)
if tensor_size < 100:
self.__class__.input_shape_is_large = False
if not type(output_names) is list:
output_names = [output_names]
analytic_grads = self._get_gradient(inputs_to_check, place,
output_names, no_grad_set)
return analytic_grads
def get_numeric_gradient(self,
place,
scope,
op,
inputs,
input_to_check,
output_names,
delta=0.005,
in_place=False,
transpose_input_list=[]):
# FIXME: change this method by compile time concepts
set_input(scope, op, inputs, place)
def product(dim):
return six.moves.reduce(lambda a, b: a * b, dim, 1)
delta_origin = delta
tensor_to_check = scope.find_var(input_to_check).get_tensor()
input_shape = tensor_to_check.shape()
# some input with lod need to transpose at begin
# it is be reconstructed here first
if input_to_check in transpose_input_list:
tensor_to_check = np.transpose(tensor_to_check, [1, 0])
tensor_to_check_ = fluid.LoDTensor()
tensor_to_check_.set(tensor_to_check, fluid.CPUPlace())
tensor_to_check = tensor_to_check_
tensor_size = int(product(input_shape))
tensor_to_check_dtype = tensor_to_check._dtype()
input_plain_shape = tensor_to_check.shape()[:]
if tensor_to_check_dtype == core.VarDesc.VarType.FP32:
tensor_to_check_dtype = np.float32
elif tensor_to_check_dtype == core.VarDesc.VarType.FP64:
tensor_to_check_dtype = np.float64
elif tensor_to_check_dtype == core.VarDesc.VarType.FP16:
tensor_to_check_dtype = np.float16
# set delta as np.float16, will automatic convert to float32, float64
delta = np.array(delta).astype(np.float16)
elif tensor_to_check_dtype == core.VarDesc.VarType.INT64:
tensor_to_check_dtype = np.int64
delta = self.lazy_share(delta)
else:
raise ValueError("Not supported data type " +
str(tensor_to_check_dtype))
def get_output():
sum = []
return_results = dict()
for name in (output_names):
return_results[name] = Manager().list()
def closure(**kwargs):
role = kwargs['role']
pfl_mpc.init("privc", role, "localhost", self.server,
int(self.port))
executor = Executor(place)
executor.run()
op.run(scope, place)
for name in output_names:
out = np.array(scope.find_var(name).get_tensor())
return_results[name].append(out)
ret = self.multi_party_run(target=closure)
self.assertEqual(ret[0], True)
for output_name in output_names:
plain = self.reconstruct(np.array(return_results[output_name]))
sum.append(plain.mean())
return (np.array(sum).sum() / len(output_names)).astype(np.float)
gradient_flat = np.zeros(shape=(tensor_size, ), dtype=np.float)
def __get_elem__(tensor, i):
if tensor_to_check_dtype == np.float16:
numpy_tensor = np.array(tensor).astype(np.float16)
numpy_tensor = numpy_tensor.flatten()
return numpy_tensor[i]
elif tensor_to_check_dtype == np.int64:
numpy_tensor = np.array(tensor).astype(np.int64)
numpy_tensor = numpy_tensor.flatten()
return numpy_tensor[i]
elif tensor_to_check_dtype == np.float32:
return tensor._get_float_element(i)
else:
return tensor._get_double_element(i)
def __set_elem__(tensor, i, e):
if tensor_to_check_dtype == np.float16:
numpy_tensor = np.array(tensor).astype(np.float16)
shape = numpy_tensor.shape
numpy_tensor = numpy_tensor.flatten()
numpy_tensor[i] = e
numpy_tensor = numpy_tensor.reshape(shape)
tensor.set(numpy_tensor, place)
elif tensor_to_check_dtype == np.int64:
numpy_tensor = np.array(tensor).astype(np.int64)
shape = numpy_tensor.shape
numpy_tensor = numpy_tensor.flatten()
numpy_tensor[i] = e
numpy_tensor = numpy_tensor.reshape(shape)
if input_to_check in transpose_input_list:
numpy_tensor = np.transpose(numpy_tensor, [1, 0])
numpy_tensor = np.array(numpy_tensor).reshape(input_shape)
tensor_input_origin = scope.find_var(
input_to_check).get_tensor()
tensor_input_origin.set(numpy_tensor, place)
elif tensor_to_check_dtype == np.float32:
tensor._set_float_element(i, e)
else:
tensor._set_double_element(i, e)
# we only compute gradient of one element each time.
# we use a for loop to compute the gradient of every element.
for i in six.moves.xrange(tensor_size):
if in_place:
set_input(scope, op, inputs, place)
# get one input element throw it's index i.
origin = __get_elem__(tensor_to_check, i)
x_pos = origin + delta
__set_elem__(tensor_to_check, i, x_pos)
y_pos = get_output()
if in_place:
set_input(scope, op, inputs, place)
x_neg = origin - delta
__set_elem__(tensor_to_check, i, x_neg)
y_neg = get_output()
__set_elem__(tensor_to_check, i, origin)
gradient_flat[i] = (y_pos - y_neg) / delta_origin / 2
return gradient_flat.reshape(input_plain_shape)
def check_grad_with_place(self,
place,
inputs_to_check,
output_names,
no_grad_set=None,
numeric_grad_delta=0.005,
in_place=False,
max_relative_error=0.005,
user_defined_grads=None,
check_dygraph=True):
self.scope = core.Scope()
op_inputs = self.inputs if hasattr(self, "inputs") else dict()
op_outputs = self.outputs if hasattr(self, "outputs") else dict()
op_attrs = self.attrs if hasattr(self, "attrs") else dict()
prog = Program()
block = prog.global_block()
op_init = block.append_op(
type="mpc_init",
#inputs=inputs,
#outputs=outputs,
attrs={"protocol_name": "privc"})
op_init.desc.infer_shape(block.desc)
mpc_protocol_index = MpcProtocols["PRIVC"].value
fluid.global_scope().var("mpc_protocol_index").get_tensor().set(
np.array((mpc_protocol_index)), fluid.CPUPlace())
self._check_grad_helper()
cache_list = None
if hasattr(self, "cache_name_list"):
cache_list = self.cache_name_list
self.op = create_op(self.scope,
self.op_type,
op_inputs,
op_outputs,
op_attrs,
cache_list=cache_list)
if no_grad_set is None:
no_grad_set = set()
for input_to_check in inputs_to_check:
set_input(self.scope, self.op, self.inputs, place)
tensor_to_check = self.scope.find_var(input_to_check).get_tensor()
tensor_size = six.moves.reduce(lambda a, b: a * b,
tensor_to_check.shape(), 1)
if tensor_size < 100:
self.__class__.input_shape_is_large = False
if not type(output_names) is list:
output_names = [output_names]
numeric_grads = user_defined_grads or [
self.get_numeric_gradient(place,
self.scope,
self.op,
self.inputs,
input_to_check,
output_names,
delta=numeric_grad_delta,
in_place=in_place)
for input_to_check in inputs_to_check
]
analytic_grads = self._get_gradient(inputs_to_check, place,
output_names, no_grad_set)
self._assert_is_close(numeric_grads, analytic_grads, inputs_to_check,
max_relative_error,
"Gradient Check On %s" % str(place))
