def assert_equal(actual, desired, err_msg=""):
"""
Asserts that two items are equal.
"""
# Case #1: dictionary .....
if isinstance(desired, dict):
if not isinstance(actual, dict):
raise AssertionError(repr(type(actual)))
assert_equal(len(actual), len(desired), err_msg)
for k, i in desired.items():
if k not in actual:
raise AssertionError("%s not in %s" % (k, actual))
assert_equal(actual[k], desired[k], "key=%r\n%s" % (k, err_msg))
return
# Case #2: lists .....
if isinstance(desired, (list, tuple)) and isinstance(actual, (list, tuple)):
return _assert_equal_on_sequences(actual, desired, err_msg="")
if not (isinstance(actual, ndarray) or isinstance(desired, ndarray)):
msg = build_err_msg([actual, desired], err_msg,)
if not desired == actual:
raise AssertionError(msg)
return
# Case #4. arrays or equivalent
if ((actual is masked) and not (desired is masked)) or (
(desired is masked) and not (actual is masked)
):
msg = build_err_msg([actual, desired], err_msg, header="", names=("x", "y"))
raise ValueError(msg)
actual = np.array(actual, copy=False, subok=True)
desired = np.array(desired, copy=False, subok=True)
(actual_dtype, desired_dtype) = (actual.dtype, desired.dtype)
if actual_dtype.char == "S" and desired_dtype.char == "S":
return _assert_equal_on_sequences(actual.tolist(), desired.tolist(), err_msg="")
return assert_array_equal(actual, desired, err_msg)
def assert_equal(actual, desired, err_msg=""):
"""
Asserts that two items are equal.
"""
# Case #1: dictionary .....
if isinstance(desired, dict):
if not isinstance(actual, dict):
raise AssertionError(repr(type(actual)))
assert_equal(len(actual), len(desired), err_msg)
for k, i in desired.items():
if k not in actual:
raise AssertionError("%s not in %s" % (k, actual))
assert_equal(actual[k], desired[k], "key=%r\n%s" % (k, err_msg))
return
# Case #2: lists .....
if isinstance(desired, (list, tuple)) and isinstance(actual, (list, tuple)):
return _assert_equal_on_sequences(actual, desired, err_msg="")
if not (isinstance(actual, ndarray) or isinstance(desired, ndarray)):
msg = build_err_msg([actual, desired], err_msg)
if not desired == actual:
raise AssertionError(msg)
return
# Case #4. arrays or equivalent
if ((actual is masked) and not (desired is masked)) or ((desired is masked) and not (actual is masked)):
msg = build_err_msg([actual, desired], err_msg, header="", names=("x", "y"))
raise ValueError(msg)
actual = np.array(actual, copy=False, subok=True)
desired = np.array(desired, copy=False, subok=True)
(actual_dtype, desired_dtype) = (actual.dtype, desired.dtype)
if actual_dtype.char == "S" and desired_dtype.char == "S":
return _assert_equal_on_sequences(actual.tolist(), desired.tolist(), err_msg="")
return assert_array_equal(actual, desired, err_msg)
def assert_array_compare(self, comparison, x, y, err_msg='', header='',
fill_value=True):
"""
Assert that a comparison of two masked arrays is satisfied elementwise.
"""
xf = self.filled(x)
yf = self.filled(y)
m = self.mask_or(self.getmask(x), self.getmask(y))
x = self.filled(self.masked_array(xf, mask=m), fill_value)
y = self.filled(self.masked_array(yf, mask=m), fill_value)
if (x.dtype.char != "O"):
x = x.astype(float_)
if isinstance(x, np.ndarray) and x.size > 1:
x[np.isnan(x)] = 0
elif np.isnan(x):
x = 0
if (y.dtype.char != "O"):
y = y.astype(float_)
if isinstance(y, np.ndarray) and y.size > 1:
y[np.isnan(y)] = 0
elif np.isnan(y):
y = 0
try:
cond = (x.shape == () or y.shape == ()) or x.shape == y.shape
if not cond:
msg = build_err_msg([x, y],
err_msg
+ '\n(shapes %s, %s mismatch)' % (x.shape,
y.shape),
header=header,
names=('x', 'y'))
assert cond, msg
val = comparison(x, y)
if m is not self.nomask and fill_value:
val = self.masked_array(val, mask=m)
if isinstance(val, bool):
cond = val
reduced = [0]
else:
reduced = val.ravel()
cond = reduced.all()
reduced = reduced.tolist()
if not cond:
match = 100-100.0*reduced.count(1)/len(reduced)
msg = build_err_msg([x, y],
err_msg
+ '\n(mismatch %s%%)' % (match,),
header=header,
names=('x', 'y'))
assert cond, msg
except ValueError:
msg = build_err_msg([x, y], err_msg, header=header, names=('x', 'y'))
raise ValueError(msg)
def assert_array_compare(self, comparison, x, y, err_msg='', header='',
fill_value=True):
"""
Assert that a comparison of two masked arrays is satisfied elementwise.
"""
xf = self.filled(x)
yf = self.filled(y)
m = self.mask_or(self.getmask(x), self.getmask(y))
x = self.filled(self.masked_array(xf, mask=m), fill_value)
y = self.filled(self.masked_array(yf, mask=m), fill_value)
if (x.dtype.char != "O"):
x = x.astype(float_)
if isinstance(x, np.ndarray) and x.size > 1:
x[np.isnan(x)] = 0
elif np.isnan(x):
x = 0
if (y.dtype.char != "O"):
y = y.astype(float_)
if isinstance(y, np.ndarray) and y.size > 1:
y[np.isnan(y)] = 0
elif np.isnan(y):
y = 0
try:
cond = (x.shape == () or y.shape == ()) or x.shape == y.shape
if not cond:
msg = build_err_msg([x, y],
err_msg
+ '\n(shapes %s, %s mismatch)' % (x.shape,
y.shape),
header=header,
names=('x', 'y'))
assert cond, msg
val = comparison(x, y)
if m is not self.nomask and fill_value:
val = self.masked_array(val, mask=m)
if isinstance(val, bool):
cond = val
reduced = [0]
else:
reduced = val.ravel()
cond = reduced.all()
reduced = reduced.tolist()
if not cond:
match = 100-100.0*reduced.count(1)/len(reduced)
msg = build_err_msg([x, y],
err_msg
+ '\n(mismatch %s%%)' % (match,),
header=header,
names=('x', 'y'))
assert cond, msg
except ValueError:
msg = build_err_msg([x, y], err_msg, header=header, names=('x', 'y'))
raise ValueError(msg)
def assert_almost_equal(a, b, rtol=None, atol=None, names=('a', 'b')):
"""Test that two numpy arrays are almost equal. Raise exception message if not.
Parameters
----------
a : np.ndarray
b : np.ndarray
threshold : None or float
The checking threshold. Default threshold will be used if set to ``None``.
"""
rtol = get_rtol(rtol)
atol = get_atol(atol)
if almost_equal(a, b, rtol, atol):
return
index, rel = find_max_violation(a, b, rtol, atol)
np.set_printoptions(threshold=4, suppress=True)
msg = npt.build_err_msg(
[a, b],
err_msg="Error %f exceeds tolerance rtol=%f, atol=%f. "
" Location of maximum error:%s, a=%f, b=%f" %
(rel, rtol, atol, str(index), a[index], b[index]),
names=names)
raise AssertionError(msg)
def fail_if_equal(
actual,
desired,
err_msg='',
):
"""
Raises an assertion error if two items are equal.
"""
if isinstance(desired, dict):
if not isinstance(actual, dict):
raise AssertionError(repr(type(actual)))
fail_if_equal(len(actual), len(desired), err_msg)
for k, i in desired.items():
if k not in actual:
raise AssertionError(repr(k))
fail_if_equal(actual[k], desired[k], 'key=%r\n%s' % (k, err_msg))
return
if isinstance(desired,
(list, tuple)) and isinstance(actual, (list, tuple)):
fail_if_equal(len(actual), len(desired), err_msg)
for k in range(len(desired)):
fail_if_equal(actual[k], desired[k], 'item=%r\n%s' % (k, err_msg))
return
if isinstance(actual, np.ndarray) or isinstance(desired, np.ndarray):
return fail_if_array_equal(actual, desired, err_msg)
msg = build_err_msg([actual, desired], err_msg)
if not desired != actual:
raise AssertionError(msg)
def assert_array_compare(comparison,
x,
y,
err_msg='',
verbose=True,
header='',
fill_value=True):
"""
Asserts that comparison between two masked arrays is satisfied.
The comparison is elementwise.
"""
# Allocate a common mask and refill
m = mask_or(getmask(x), getmask(y))
x = masked_array(x, copy=False, mask=m, keep_mask=False, subok=False)
y = masked_array(y, copy=False, mask=m, keep_mask=False, subok=False)
if ((x is masked) and not (y is masked)) or \
((y is masked) and not (x is masked)):
msg = build_err_msg([x, y],
err_msg=err_msg,
verbose=verbose,
header=header,
names=('x', 'y'))
raise ValueError(msg)
# OK, now run the basic tests on filled versions
return np.testing.assert_array_compare(comparison,
x.filled(fill_value),
y.filled(fill_value),
err_msg=err_msg,
verbose=verbose,
header=header)
def check_symbolic_forward(sym, location, expected, check_eps=1E-4, aux_states=None, ctx=None):
"""Compare foward call to expected value.
Parameters
---------
sym : Symbol
output symbol
location : list of np.ndarray or dict of str to np.ndarray
The evaluation point
- if type is list of np.ndarray
contain all the numpy arrays corresponding to `sym.list_arguments()`
- if type is dict of str to np.ndarray
contain the mapping between argument names and their values
expected : list of np.ndarray or dict of str to np.ndarray
The expected output value
- if type is list of np.ndarray
contain arrays corresponding to exe.outputs
- if type is dict of str to np.ndarray
contain mapping between sym.list_output() and exe.outputs
check_eps : float, optional
relative error to check to
aux_states : list of np.ndarray of dict, optional
- if type is list of np.ndarray
contain all the numpy arrays corresponding to sym.list_auxiliary_states
- if type is dict of str to np.ndarray
contain the mapping between names of auxiliary states and their values
ctx : Context, optional
running context
"""
if ctx is None:
ctx = default_context()
location = _parse_location(sym=sym, location=location, ctx=ctx)
aux_states = _parse_aux_states(sym=sym, aux_states=aux_states, ctx=ctx)
if isinstance(expected, dict):
expected = [expected[k] for k in sym.list_outputs()]
args_grad_data = {k:mx.nd.empty(v.shape, ctx=ctx) for k, v in location.items()}
executor = sym.bind(ctx=ctx, args=location, args_grad=args_grad_data, aux_states=aux_states)
for g in executor.grad_arrays:
if g:
g[:] = 0
executor.forward(is_train=False)
outputs = [x.asnumpy() for x in executor.outputs]
for output_name, expect, output in zip(sym.list_outputs(), expected, outputs):
rel = reldiff(expect, output)
if rel > check_eps:
np.set_printoptions(threshold=4, suppress=True)
msg = npt.build_err_msg([expect, output],
err_msg="In symbol \"%s\", ctx=%s, "
"forward check failed for \"%s\". "
"Rel Err=%f, Expected <=%f"
%(sym.name, str(ctx), output_name, rel, check_eps),
names=["EXPECTED", "FORWARD"])
raise Exception(msg)
def test_build_err_msg_no_verbose(self):
x = np.array([1.00001, 2.00002, 3.00003])
y = np.array([1.00002, 2.00003, 3.00004])
err_msg = 'There is a mismatch'
a = build_err_msg([x, y], err_msg, verbose=False)
b = '\nItems are not equal: There is a mismatch'
self.assertEqual(a, b)
def test_build_err_msg_no_verbose(self):
x = np.array([1.00001, 2.00002, 3.00003])
y = np.array([1.00002, 2.00003, 3.00004])
err_msg = 'There is a mismatch'
a = build_err_msg([x, y], err_msg, verbose=False)
b = '\nItems are not equal: There is a mismatch'
self.assertEqual(a, b)
def check_symbolic_forward(sym, location, expected, check_eps=1E-4, aux_states=None, ctx=mx.cpu()):
"""Compare foward call to expected value.
Parameters
---------
sym : Symbol
output symbol
location : list of np.ndarray or dict of str to np.ndarray
The evaluation point
- if type is list of np.ndarray
contain all the numpy arrays corresponding to `sym.list_arguments()`
- if type is dict of str to np.ndarray
contain the mapping between argument names and their values
expected : list of np.ndarray or dict of str to np.ndarray
The expected output value
- if type is list of np.ndarray
contain arrays corresponding to exe.outputs
- if type is dict of str to np.ndarray
contain mapping between sym.list_output() and exe.outputs
check_eps : float, optional
relative error to check to
aux_states : list of np.ndarray of dict, optional
- if type is list of np.ndarray
contain all the numpy arrays corresponding to sym.list_auxiliary_states
- if type is dict of str to np.ndarray
contain the mapping between names of auxiliary states and their values
ctx : Context, optional
running context
"""
location = _parse_location(sym=sym, location=location, ctx=ctx)
aux_states = _parse_aux_states(sym=sym, aux_states=aux_states, ctx=ctx)
if isinstance(expected, dict):
expected = [expected[k] for k in sym.list_outputs()]
args_grad_data = {k:mx.nd.empty(v.shape, ctx=ctx) for k, v in location.items()}
executor = sym.bind(ctx=ctx, args=location, args_grad=args_grad_data, aux_states=aux_states)
for g in executor.grad_arrays:
if g:
g[:] = 0
executor.forward(is_train=False)
outputs = [x.asnumpy() for x in executor.outputs]
for output_name, expect, output in zip(sym.list_outputs(), expected, outputs):
rel = reldiff(expect, output)
if rel > check_eps:
np.set_printoptions(threshold=4, suppress=True)
msg = npt.build_err_msg([expect, output],
err_msg="In symbol \"%s\", ctx=%s, "
"forward check failed for \"%s\". "
"Rel Err=%f, Expected <=%f"
%(sym.name, str(ctx), output_name, rel, check_eps),
names=["EXPECTED", "FORWARD"])
raise Exception(msg)
def test_build_err_msg_custom_precision(self):
x = np.array([1.000000001, 2.00002, 3.00003])
y = np.array([1.000000002, 2.00003, 3.00004])
err_msg = 'There is a mismatch'
a = build_err_msg([x, y], err_msg, precision=10)
b = ('\nItems are not equal: There is a mismatch\n ACTUAL: array(['
'1.000000001, 2.00002 , 3.00003 ])\n DESIRED: array(['
'1.000000002, 2.00003 , 3.00004 ])')
self.assertEqual(a, b)
def test_build_err_msg_custom_precision(self):
x = np.array([1.000000001, 2.00002, 3.00003])
y = np.array([1.000000002, 2.00003, 3.00004])
err_msg = 'There is a mismatch'
a = build_err_msg([x, y], err_msg, precision=10)
b = ('\nItems are not equal: There is a mismatch\n ACTUAL: array([ '
'1.000000001, 2.00002 , 3.00003 ])\n DESIRED: array([ '
'1.000000002, 2.00003 , 3.00004 ])')
self.assertEqual(a, b)
def test_build_err_msg_custom_names(self):
x = np.array([1.00001, 2.00002, 3.00003])
y = np.array([1.00002, 2.00003, 3.00004])
err_msg = 'There is a mismatch'
a = build_err_msg([x, y], err_msg, names=('FOO', 'BAR'))
b = ('\nItems are not equal: There is a mismatch\n FOO: array([ '
'1.00001, 2.00002, 3.00003])\n BAR: array([ 1.00002, 2.00003, '
'3.00004])')
self.assertEqual(a, b)
def assert_equal(actual, desired, err_msg=''):
"""
Asserts that two items are equal.
"""
# Case #1: dictionary .....
if isinstance(desired, dict):
if not isinstance(actual, dict):
raise AssertionError(repr(type(actual)))
assert_equal(len(actual), len(desired), err_msg)
for k, i in desired.items():
if k not in actual:
raise AssertionError(f"{k} not in {actual}")
assert_equal(actual[k], desired[k], f'key={k!r}\n{err_msg}')
return
# Case #2: lists .....
if isinstance(desired,
(list, tuple)) and isinstance(actual, (list, tuple)):
return _assert_equal_on_sequences(actual, desired, err_msg='')
if not (isinstance(actual, ndarray) or isinstance(desired, ndarray)):
msg = build_err_msg(
[actual, desired],
err_msg,
)
if not desired == actual:
raise AssertionError(msg)
return
# Case #4. arrays or equivalent
if ((actual is masked) and not (desired is masked)) or \
((desired is masked) and not (actual is masked)):
msg = build_err_msg([actual, desired],
err_msg,
header='',
names=('x', 'y'))
raise ValueError(msg)
actual = np.asanyarray(actual)
desired = np.asanyarray(desired)
(actual_dtype, desired_dtype) = (actual.dtype, desired.dtype)
if actual_dtype.char == "S" and desired_dtype.char == "S":
return _assert_equal_on_sequences(actual.tolist(),
desired.tolist(),
err_msg='')
return assert_array_equal(actual, desired, err_msg)
def test_build_err_msg_custom_names(self):
x = np.array([1.00001, 2.00002, 3.00003])
y = np.array([1.00002, 2.00003, 3.00004])
err_msg = 'There is a mismatch'
a = build_err_msg([x, y], err_msg, names=('FOO', 'BAR'))
b = ('\nItems are not equal: There is a mismatch\n FOO: array(['
'1.00001, 2.00002, 3.00003])\n BAR: array([1.00002, 2.00003, '
'3.00004])')
self.assertEqual(a, b)
def test_build_err_msg_defaults(self):
x = np.array([1.00001, 2.00002, 3.00003])
y = np.array([1.00002, 2.00003, 3.00004])
err_msg = 'There is a mismatch'
a = build_err_msg([x, y], err_msg)
b = ('\nItems are not equal: There is a mismatch\n ACTUAL: array(['
'1.00001, 2.00002, 3.00003])\n DESIRED: array([1.00002, '
'2.00003, 3.00004])')
assert_equal(a, b)
def test_build_err_msg_defaults(self):
x = np.array([1.00001, 2.00002, 3.00003])
y = np.array([1.00002, 2.00003, 3.00004])
err_msg = 'There is a mismatch'
a = build_err_msg([x, y], err_msg)
b = ('\nItems are not equal: There is a mismatch\n ACTUAL: array(['
'1.00001, 2.00002, 3.00003])\n DESIRED: array([1.00002, '
'2.00003, 3.00004])')
assert_equal(a, b)
def test_build_err_msg_defaults(self):
x = np.array([1.00001, 2.00002, 3.00003])
y = np.array([1.00002, 2.00003, 3.00004])
err_msg = "There is a mismatch"
a = build_err_msg([x, y], err_msg)
b = (
"\nItems are not equal: There is a mismatch\n ACTUAL: array([ "
"1.00001, 2.00002, 3.00003])\n DESIRED: array([ 1.00002, "
"2.00003, 3.00004])"
)
self.assertEqual(a, b)
def assert_almost_equal(actual, desired, decimal=7, err_msg="", verbose=True):
"""
Asserts that two items are almost equal.
The test is equivalent to abs(desired-actual) < 0.5 * 10**(-decimal).
"""
if isinstance(actual, np.ndarray) or isinstance(desired, np.ndarray):
return assert_array_almost_equal(actual, desired, decimal=decimal, err_msg=err_msg, verbose=verbose)
msg = build_err_msg([actual, desired], err_msg=err_msg, verbose=verbose)
if not round(abs(desired - actual), decimal) == 0:
raise AssertionError(msg)
def test_build_err_msg_custom_names(self):
x = np.array([1.00001, 2.00002, 3.00003])
y = np.array([1.00002, 2.00003, 3.00004])
err_msg = "There is a mismatch"
a = build_err_msg([x, y], err_msg, names=("FOO", "BAR"))
b = (
"\nItems are not equal: There is a mismatch\n FOO: array([ "
"1.00001, 2.00002, 3.00003])\n BAR: array([ 1.00002, 2.00003, "
"3.00004])"
)
self.assertEqual(a, b)
def assert_almost_equal(actual, desired, decimal=7, err_msg='', verbose=True):
"""
Asserts that two items are almost equal.
The test is equivalent to abs(desired-actual) < 0.5 * 10**(-decimal).
"""
if isinstance(actual, np.ndarray) or isinstance(desired, np.ndarray):
return assert_array_almost_equal(actual, desired, decimal=decimal,
err_msg=err_msg, verbose=verbose)
msg = build_err_msg([actual, desired],
err_msg=err_msg, verbose=verbose)
if not round(abs(desired - actual), decimal) == 0:
raise AssertionError(msg)
def assert_array_compare(comparison, x, y, err_msg="", verbose=True, header="", fill_value=True):
"""
Asserts that comparison between two masked arrays is satisfied.
The comparison is elementwise.
"""
# Allocate a common mask and refill
m = mask_or(getmask(x), getmask(y))
x = masked_array(x, copy=False, mask=m, keep_mask=False, subok=False)
y = masked_array(y, copy=False, mask=m, keep_mask=False, subok=False)
if ((x is masked) and not (y is masked)) or ((y is masked) and not (x is masked)):
msg = build_err_msg([x, y], err_msg=err_msg, verbose=verbose, header=header, names=("x", "y"))
raise ValueError(msg)
# OK, now run the basic tests on filled versions
return utils.assert_array_compare(
comparison, x.filled(fill_value), y.filled(fill_value), err_msg=err_msg, verbose=verbose, header=header
)
def assert_almost_equal(a, b, threshold=None):
"""Test that two numpy arrays are almost equal. Raise exception message if not.
Parameters
----------
a : np.ndarray
b : np.ndarray
threshold : None or float
The checking threshold. Default threshold will be used if set to None
"""
threshold = threshold or default_numerical_threshold()
rel = reldiff(a, b)
if np.isnan(rel) or rel > threshold:
np.set_printoptions(threshold=4, suppress=True)
msg = npt.build_err_msg([a, b],
err_msg="Rel Err=%f, Expected <=%f" % (rel, threshold),
names=["a", "b"])
raise Exception(msg)
return rel
def assert_almost_equal(a, b, threshold=None):
"""Test that two numpy arrays are almost equal. Raise exception message if not.
Parameters
----------
a : np.ndarray
b : np.ndarray
threshold : None or float
The checking threshold. Default threshold will be used if set to None
"""
threshold = threshold or default_numerical_threshold()
rel = reldiff(a, b)
if np.isnan(rel) or rel > threshold:
np.set_printoptions(threshold=4, suppress=True)
msg = npt.build_err_msg([a, b],
err_msg="Rel Err=%f, Expected <=%f" %
(rel, threshold),
names=["a", "b"])
raise Exception(msg)
return rel
def fail_if_equal(actual, desired, err_msg='',):
"""
Raises an assertion error if two items are equal.
"""
if isinstance(desired, dict):
if not isinstance(actual, dict):
raise AssertionError(repr(type(actual)))
fail_if_equal(len(actual), len(desired), err_msg)
for k, i in desired.items():
if k not in actual:
raise AssertionError(repr(k))
fail_if_equal(actual[k], desired[k], 'key=%r\n%s' % (k, err_msg))
return
if isinstance(desired, (list, tuple)) and isinstance(actual, (list, tuple)):
fail_if_equal(len(actual), len(desired), err_msg)
for k in range(len(desired)):
fail_if_equal(actual[k], desired[k], 'item=%r\n%s' % (k, err_msg))
return
if isinstance(actual, np.ndarray) or isinstance(desired, np.ndarray):
return fail_if_array_equal(actual, desired, err_msg)
msg = build_err_msg([actual, desired], err_msg)
if not desired != actual:
raise AssertionError(msg)
def assert_almost_equal(a, b, rtol=None, atol=None, names=('a', 'b')):
"""Test that two numpy arrays are almost equal. Raise exception message if not.
Parameters
----------
a : np.ndarray
b : np.ndarray
threshold : None or float
The checking threshold. Default threshold will be used if set to ``None``.
"""
rtol = get_rtol(rtol)
atol = get_atol(atol)
if almost_equal(a, b, rtol, atol):
return
index, rel = find_max_violation(a, b, rtol, atol)
np.set_printoptions(threshold=4, suppress=True)
msg = npt.build_err_msg([a, b],
err_msg="Error %f exceeds tolerance rtol=%f, atol=%f. "
" Location of maximum error:%s, a=%f, b=%f"
% (rel, rtol, atol, str(index), a[index], b[index]),
names=names)
raise AssertionError(msg)
def assert_within_tol(a, b, tol, err_msg=""):
if not within_tol(a, b, tol):
msg = build_err_msg((a, b), err_msg)
raise AssertionError(msg)
def check_numeric_gradient(sym, location, aux_states=None, numeric_eps=1e-4, check_eps=1e-2,
grad_nodes=None, use_forward_train=True, ctx=mx.cpu()):
"""Verify an operation by checking backward pass via finite difference method.
Based on Theano's `theano.gradient.verify_grad` [1]
Parameters
----------
sym : Symbol
Symbol containing op to test
location : list or tuple or dict
Argument values used as location to compute gradient
- if type is list of numpy.ndarray
inner elements should have the same the same order as mxnet.sym.list_arguments().
- if type is dict of str -> numpy.ndarray
maps the name of arguments to the corresponding numpy.ndarray.
*In either case, value of all the arguments must be provided.*
aux_states : ist or tuple or dict, optional
The auxiliary states required when generating the executor for the symbol
numeric_eps : float, optional
Delta for the finite difference method that approximates the gradient
check_eps : float, optional
relative error eps used when comparing numeric grad to symbolic grad
grad_nodes : None or list or tuple or dict, optional
Names of the nodes to check gradient on
use_forward_train : bool
Whether to use is_train=True when computing the finite-difference
ctx : Context, optional
Check the gradient computation on the specified device
References
---------
..[1] https://github.com/Theano/Theano/blob/master/theano/gradient.py
"""
def random_projection(shape):
"""Get a random weight matrix with not too small elements
Parameters
----------
shape : list or tuple
"""
# random_projection should not have elements too small,
# otherwise too much precision is lost in numerical gradient
plain = _rng.rand(*shape) + 0.1
return plain
location = _parse_location(sym=sym, location=location, ctx=ctx)
location_npy = {k:v.asnumpy() for k, v in location.items()}
aux_states = _parse_aux_states(sym=sym, aux_states=aux_states, ctx=ctx)
if aux_states is not None:
aux_states_npy = {k:v.asnumpy() for k, v in aux_states.items()}
else:
aux_states_npy = None
if grad_nodes is None:
grad_nodes = sym.list_arguments()
grad_req = {k: 'write' for k in grad_nodes}
elif isinstance(grad_nodes, (list, tuple)):
grad_nodes = list(grad_nodes)
grad_req = {k: 'write' for k in grad_nodes}
elif isinstance(grad_nodes, dict):
grad_req = grad_nodes.copy()
grad_nodes = grad_nodes.keys()
else:
raise ValueError
input_shape = {k: v.shape for k, v in location.items()}
_, out_shape, _ = sym.infer_shape(**input_shape)
proj = mx.sym.Variable("__random_proj")
out = mx.sym.sum(sym * proj)
out = mx.sym.MakeLoss(out)
location = dict(list(location.items()) +
[("__random_proj", mx.nd.array(random_projection(out_shape[0]), ctx=ctx))])
args_grad_npy = dict([(k, _rng.normal(0, 0.01, size=location[k].shape)) for k in grad_nodes]
+ [("__random_proj", _rng.normal(0, 0.01, size=out_shape[0]))])
args_grad = {k: mx.nd.array(v, ctx=ctx) for k, v in args_grad_npy.items()}
executor = out.bind(ctx, grad_req=grad_req,
args=location, args_grad=args_grad, aux_states=aux_states)
inps = executor.arg_arrays
if len(inps) != len(location):
raise ValueError("Executor arg_arrays and and location len do not match."
"Got %d inputs and %d locations"%(len(inps), len(location)))
assert len(executor.outputs) == 1
executor.forward(is_train=True)
executor.backward()
symbolic_grads = {k:executor.grad_dict[k].asnumpy() for k in grad_nodes}
numeric_gradients = numeric_grad(executor, location_npy, aux_states_npy,
eps=numeric_eps, use_forward_train=use_forward_train)
for name in grad_nodes:
fd_grad = numeric_gradients[name]
orig_grad = args_grad_npy[name]
sym_grad = symbolic_grads[name]
if grad_req[name] == 'write':
rel = reldiff(fd_grad, sym_grad)
arr_l = [fd_grad, sym_grad]
elif grad_req[name] == 'add':
rel = reldiff(fd_grad, sym_grad - orig_grad)
arr_l = [fd_grad, sym_grad - orig_grad]
elif grad_req[name] == 'null':
rel = reldiff(orig_grad, sym_grad)
arr_l = [orig_grad, sym_grad]
else:
raise ValueError
if np.isnan(rel) or rel > check_eps:
np.set_printoptions(threshold=4, suppress=True)
msg = npt.build_err_msg(arr_l,
err_msg="In symbol \"%s\", ctx=%s, "
"numeric check failed for \"%s\", grad_req= \"%s\". "
"Rel Err=%f, Expected <=%f"
%(sym.name, str(ctx), name, grad_req[name], rel, check_eps),
names=["NUMERICAL", "BACKWARD"])
raise Exception(msg)
def check_numeric_gradient(sym, location, aux_states=None, numeric_eps=1e-4, check_eps=1e-2,
grad_nodes=None, use_forward_train=True, ctx=None):
"""Verify an operation by checking backward pass via finite difference method.
Based on Theano's `theano.gradient.verify_grad` [1]
Parameters
----------
sym : Symbol
Symbol containing op to test
location : list or tuple or dict
Argument values used as location to compute gradient
- if type is list of numpy.ndarray
inner elements should have the same the same order as mxnet.sym.list_arguments().
- if type is dict of str -> numpy.ndarray
maps the name of arguments to the corresponding numpy.ndarray.
*In either case, value of all the arguments must be provided.*
aux_states : ist or tuple or dict, optional
The auxiliary states required when generating the executor for the symbol
numeric_eps : float, optional
Delta for the finite difference method that approximates the gradient
check_eps : float, optional
relative error eps used when comparing numeric grad to symbolic grad
grad_nodes : None or list or tuple or dict, optional
Names of the nodes to check gradient on
use_forward_train : bool
Whether to use is_train=True when computing the finite-difference
ctx : Context, optional
Check the gradient computation on the specified device
References
---------
..[1] https://github.com/Theano/Theano/blob/master/theano/gradient.py
"""
if ctx is None:
ctx = default_context()
def random_projection(shape):
"""Get a random weight matrix with not too small elements
Parameters
----------
shape : list or tuple
"""
# random_projection should not have elements too small,
# otherwise too much precision is lost in numerical gradient
plain = _rng.rand(*shape) + 0.1
return plain
location = _parse_location(sym=sym, location=location, ctx=ctx)
location_npy = {k:v.asnumpy() for k, v in location.items()}
aux_states = _parse_aux_states(sym=sym, aux_states=aux_states, ctx=ctx)
if aux_states is not None:
aux_states_npy = {k:v.asnumpy() for k, v in aux_states.items()}
else:
aux_states_npy = None
if grad_nodes is None:
grad_nodes = sym.list_arguments()
grad_req = {k: 'write' for k in grad_nodes}
elif isinstance(grad_nodes, (list, tuple)):
grad_nodes = list(grad_nodes)
grad_req = {k: 'write' for k in grad_nodes}
elif isinstance(grad_nodes, dict):
grad_req = grad_nodes.copy()
grad_nodes = grad_nodes.keys()
else:
raise ValueError
input_shape = {k: v.shape for k, v in location.items()}
_, out_shape, _ = sym.infer_shape(**input_shape)
proj = mx.sym.Variable("__random_proj")
out = mx.sym.sum(sym * proj)
out = mx.sym.MakeLoss(out)
location = dict(list(location.items()) +
[("__random_proj", mx.nd.array(random_projection(out_shape[0]), ctx=ctx))])
args_grad_npy = dict([(k, _rng.normal(0, 0.01, size=location[k].shape)) for k in grad_nodes]
+ [("__random_proj", _rng.normal(0, 0.01, size=out_shape[0]))])
args_grad = {k: mx.nd.array(v, ctx=ctx) for k, v in args_grad_npy.items()}
executor = out.bind(ctx, grad_req=grad_req,
args=location, args_grad=args_grad, aux_states=aux_states)
inps = executor.arg_arrays
if len(inps) != len(location):
raise ValueError("Executor arg_arrays and and location len do not match."
"Got %d inputs and %d locations"%(len(inps), len(location)))
assert len(executor.outputs) == 1
executor.forward(is_train=True)
executor.backward()
symbolic_grads = {k:executor.grad_dict[k].asnumpy() for k in grad_nodes}
numeric_gradients = numeric_grad(executor, location_npy, aux_states_npy,
eps=numeric_eps, use_forward_train=use_forward_train)
for name in grad_nodes:
fd_grad = numeric_gradients[name]
orig_grad = args_grad_npy[name]
sym_grad = symbolic_grads[name]
if grad_req[name] == 'write':
rel = reldiff(fd_grad, sym_grad)
arr_l = [fd_grad, sym_grad]
elif grad_req[name] == 'add':
rel = reldiff(fd_grad, sym_grad - orig_grad)
arr_l = [fd_grad, sym_grad - orig_grad]
elif grad_req[name] == 'null':
rel = reldiff(orig_grad, sym_grad)
arr_l = [orig_grad, sym_grad]
else:
raise ValueError
if np.isnan(rel) or rel > check_eps:
np.set_printoptions(threshold=4, suppress=True)
msg = npt.build_err_msg(arr_l,
err_msg="In symbol \"%s\", ctx=%s, "
"numeric check failed for \"%s\", grad_req= \"%s\". "
"Rel Err=%f, Expected <=%f"
%(sym.name, str(ctx), name, grad_req[name], rel, check_eps),
names=["NUMERICAL", "BACKWARD"])
raise Exception(msg)
def assert_within_tol(a,b,tol,err_msg=""):
if not within_tol(a,b,tol):
msg = build_err_msg((a,b),err_msg)
raise AssertionError(msg)
def check_symbolic_backward(sym, location, out_grads, expected, check_eps=1e-5,
aux_states=None, grad_req='write', ctx=None):
"""Compare backward call to expected value.
Parameters
---------
sym : Symbol
output symbol
location : list of np.ndarray or dict of str to np.ndarray
The evaluation point
- if type is list of np.ndarray
contain all the numpy arrays corresponding to mxnet.sym.list_arguments
- if type is dict of str to np.ndarray
contain the mapping between argument names and their values
out_grads : None or list of np.ndarray or dict of str to np.ndarray
numpy arrays corresponding to sym.outputs for incomming gradient
- if type is list of np.ndarray
contains arrays corresponding to exe.outputs
- if type is dict of str to np.ndarray
contains mapping between mxnet.sym.list_output() and Executor.outputs
expected : list of np.ndarray or dict of str to np.ndarray
expected gradient values
- if type is list of np.ndarray
contains arrays corresponding to exe.grad_arrays
- if type is dict of str to np.ndarray
contains mapping between sym.list_arguments() and exe.outputs
check_eps: float, optional
relative error to check to
aux_states : list of np.ndarray or dict of str to np.ndarray
grad_req : str or list of str or dict of str to str, optional
gradient requirements. 'write', 'add' or 'null'
ctx : Context, optional
running context
"""
if ctx is None:
ctx = default_context()
location = _parse_location(sym=sym, location=location, ctx=ctx)
aux_states = _parse_aux_states(sym=sym, aux_states=aux_states, ctx=ctx)
if isinstance(expected, (list, tuple)):
expected = {k:v for k, v in zip(sym.list_arguments(), expected)}
args_grad_npy = {k:_rng.normal(size=v.shape) for k, v in expected.items()}
args_grad_data = {k: mx.nd.array(v, ctx=ctx) for k, v in args_grad_npy.items()}
if isinstance(grad_req, str):
grad_req = {k:grad_req for k in sym.list_arguments()}
elif isinstance(grad_req, (list, tuple)):
grad_req = {k:v for k, v in zip(sym.list_arguments(), grad_req)}
executor = sym.bind(ctx=ctx, args=location, args_grad=args_grad_data, aux_states=aux_states)
executor.forward(is_train=True)
if isinstance(out_grads, (tuple, list)):
out_grads = [mx.nd.array(v, ctx=ctx) for v in out_grads]
elif isinstance(out_grads, (dict)):
out_grads = {k:mx.nd.array(v, ctx=ctx) for k, v in out_grads.items()}
else:
assert out_grads is None
executor.backward(out_grads)
grads = {k: v.asnumpy() for k, v in args_grad_data.items()}
for name in expected:
if grad_req[name] == 'write':
rel = reldiff(expected[name], grads[name])
arr_l = [expected[name], grads[name]]
elif grad_req[name] == 'add':
rel = reldiff(expected[name], grads[name] - args_grad_npy[name])
arr_l = [expected[name], grads[name] - args_grad_npy[name]]
elif grad_req[name] == 'null':
rel = reldiff(args_grad_npy[name], grads[name])
arr_l = [args_grad_npy[name], grads[name]]
else:
raise ValueError
if rel > check_eps:
np.set_printoptions(threshold=4, suppress=True)
msg = npt.build_err_msg(arr_l,
err_msg="In symbol \"%s\", ctx=%s, "
"backward check failed for \"%s\". "
"Rel Err=%f, Expected <=%f"
%(sym.name, str(ctx), name, rel, check_eps),
names=["EXPECTED", "BACKWARD"])
raise Exception(msg)
def check_symbolic_backward(sym, location, out_grads, expected, check_eps=1e-5,
aux_states=None, grad_req='write', ctx=mx.cpu()):
"""Compare backward call to expected value.
Parameters
---------
sym : Symbol
output symbol
location : list of np.ndarray or dict of str to np.ndarray
The evaluation point
- if type is list of np.ndarray
contain all the numpy arrays corresponding to mxnet.sym.list_arguments
- if type is dict of str to np.ndarray
contain the mapping between argument names and their values
out_grads : None or list of np.ndarray or dict of str to np.ndarray
numpy arrays corresponding to sym.outputs for incomming gradient
- if type is list of np.ndarray
contains arrays corresponding to exe.outputs
- if type is dict of str to np.ndarray
contains mapping between mxnet.sym.list_output() and Executor.outputs
expected : list of np.ndarray or dict of str to np.ndarray
expected gradient values
- if type is list of np.ndarray
contains arrays corresponding to exe.grad_arrays
- if type is dict of str to np.ndarray
contains mapping between sym.list_arguments() and exe.outputs
check_eps: float, optional
relative error to check to
aux_states : list of np.ndarray or dict of str to np.ndarray
grad_req : str or list of str or dict of str to str, optional
gradient requirements. 'write', 'add' or 'null'
ctx : Context, optional
running context
"""
location = _parse_location(sym=sym, location=location, ctx=ctx)
aux_states = _parse_aux_states(sym=sym, aux_states=aux_states, ctx=ctx)
if isinstance(expected, (list, tuple)):
expected = {k:v for k, v in zip(sym.list_arguments(), expected)}
args_grad_npy = {k:_rng.normal(size=v.shape) for k, v in expected.items()}
args_grad_data = {k: mx.nd.array(v, ctx=ctx) for k, v in args_grad_npy.items()}
if isinstance(grad_req, str):
grad_req = {k:grad_req for k in sym.list_arguments()}
elif isinstance(grad_req, (list, tuple)):
grad_req = {k:v for k, v in zip(sym.list_arguments(), grad_req)}
executor = sym.bind(ctx=ctx, args=location, args_grad=args_grad_data, aux_states=aux_states)
executor.forward(is_train=True)
if isinstance(out_grads, (tuple, list)):
out_grads = [mx.nd.array(v, ctx=ctx) for v in out_grads]
elif isinstance(out_grads, (dict)):
out_grads = {k:mx.nd.array(v, ctx=ctx) for k, v in out_grads.items()}
else:
assert out_grads is None
executor.backward(out_grads)
grads = {k: v.asnumpy() for k, v in args_grad_data.items()}
for name in expected:
if grad_req[name] == 'write':
rel = reldiff(expected[name], grads[name])
arr_l = [expected[name], grads[name]]
elif grad_req[name] == 'add':
rel = reldiff(expected[name], grads[name] - args_grad_npy[name])
arr_l = [expected[name], grads[name] - args_grad_npy[name]]
elif grad_req[name] == 'null':
rel = reldiff(args_grad_npy[name], grads[name])
arr_l = [args_grad_npy[name], grads[name]]
else:
raise ValueError
if rel > check_eps:
np.set_printoptions(threshold=4, suppress=True)
msg = npt.build_err_msg(arr_l,
err_msg="In symbol \"%s\", ctx=%s, "
"backward check failed for \"%s\". "
"Rel Err=%f, Expected <=%f"
%(sym.name, str(ctx), name, rel, check_eps),
names=["EXPECTED", "BACKWARD"])
raise Exception(msg)