def test_min():
x_np = np.array([[1, 2], [2, 1], [0, 0]])
x_grad1 = np.array([[0, 0], [0, 0], [1, 1]])
x_grad2 = np.array([[1, 0], [0, 1], [1, 1]])
x_grad3 = np.array([[0, 0], [0, 0], [1, 1]])
def red1(x):
return mp.min(x)
def red2(x):
return mp.min(x, axis=1)
def red3(x):
return mp.min(x, axis=1, keepdims=True)
def red4(x):
return mp.min(x, axis=0)
def red5(x):
return mp.min(x, axis=0, keepdims=True)
grad1 = grad(red1)
assert np.all(grad1(x_np).asnumpy() == x_grad1)
grad2 = grad(red2)
assert np.all(grad2(x_np).asnumpy() == x_grad2)
grad3 = grad(red3)
assert np.all(grad3(x_np).asnumpy() == x_grad2)
grad4 = grad(red4)
assert np.all(grad4(x_np).asnumpy() == x_grad3)
grad5 = grad(red5)
assert np.all(grad5(x_np).asnumpy() == x_grad3)
def test_min():
x_np = np.array([[1, 2], [2, 1], [0, 0]])
x_grad1 = np.array([[0, 0], [0, 0], [1, 1]])
x_grad2 = np.array([[1, 0], [0, 1], [1, 1]])
x_grad3 = np.array([[0, 0], [0, 0], [1, 1]])
def red1(x):
return mp.min(x)
def red2(x):
return mp.min(x, axis=1)
def red3(x):
return mp.min(x, axis=1, keepdims=True)
def red4(x):
return mp.min(x, axis=0)
def red5(x):
return mp.min(x, axis=0, keepdims=True)
grad1 = grad(red1)
assert np.all(grad1(x_np).asnumpy() == x_grad1)
grad2 = grad(red2)
assert np.all(grad2(x_np).asnumpy() == x_grad2)
grad3 = grad(red3)
assert np.all(grad3(x_np).asnumpy() == x_grad2)
grad4 = grad(red4)
assert np.all(grad4(x_np).asnumpy() == x_grad3)
grad5 = grad(red5)
assert np.all(grad5(x_np).asnumpy() == x_grad3)
def quick_grad_check(fun, arg0, extra_args=(), kwargs={}, verbose=True,
eps=EPS, rtol=RTOL, atol=ATOL, rs=None):
"""Checks the gradient of a function (w.r.t. to its first arg) in a random direction"""
if verbose:
print("Checking gradient of {0} at {1}".format(fun, arg0))
if rs is None:
rs = nnp.random.RandomState()
random_dir = rs.standard_normal(nnp.shape(arg0))
random_dir = random_dir / nnp.sqrt(nnp.sum(random_dir * random_dir))
if not extra_args == ():
unary_fun = lambda x : fun(arg0 + x * random_dir, extra_args)
numeric_grad = (unary_fun(eps/2) - unary_fun(-eps/2)) / eps
analytic_grad = np.sum(grad(fun)(arg0, extra_args) * random_dir)
else:
unary_fun = lambda x : fun(arg0 + x * random_dir)
numeric_grad = (unary_fun(eps/2) - unary_fun(-eps/2)) / eps
analytic_grad = np.sum(grad(fun)(arg0) * random_dir)
if isinstance(numeric_grad, minpy.array.Number):
assert abs((analytic_grad - numeric_grad).get_data(None)) < atol and abs((analytic_grad - numeric_grad).get_data(None)) < abs((analytic_grad * rtol).get_data(None)), \
"Check failed! nd={0}, ad={1}".format(numeric_grad, analytic_grad)
elif isinstance(numeric_grad, minpy.array.Array):
assert nnp.prod(nnp.shape(analytic_grad.asnumpy())[:]) == 1, "Currently only support check loss"
assert abs((analytic_grad - numeric_grad).asnumpy()) < atol and abs((analytic_grad - numeric_grad).asnumpy()) < abs((analytic_grad * rtol).asnumpy()), \
"Check failed! nd={0}, ad={1}".format(numeric_grad, analytic_grad)
else:
assert False
if verbose:
print("Gradient projection OK (numeric grad: {0}, analytic grad: {1})".format(
numeric_grad, analytic_grad))
def test_sum():
x_np = np.array([[1, 2], [3, 4], [5, 6]])
x_grad = np.array([[1, 1], [1, 1], [1, 1]])
def red1(x):
return mp.sum(x)
def red2(x):
return mp.sum(x, axis=0)
def red3(x):
return mp.sum(x, axis=0, keepdims=True)
grad1 = grad(red1)
assert np.all(grad1(x_np).asnumpy() == x_grad)
grad2 = grad(red2)
assert np.all(grad2(x_np).asnumpy() == x_grad)
grad3 = grad(red3)
assert np.all(grad3(x_np).asnumpy() == x_grad)
def test_customop():
x = np.array([[0.24854138, 1.94385293, 2.33848549, 2.75407309, 1.66905118],
[0.26498274, 1.60618255, 1.25387436, 2.9215846, 1.26427169],
[0.87108803, 1.45227827, 2.17339809, 0.50049702, 0.5883466],
[0.66406034, 0.48855862, 1.53960508, 0.66568797, 2.23948055],
[2.72220612, 1.82959485, 1.51552618, 1.54757016, 1.64023012],
[1.69430802, 2.21234513, 0.44159807, 1.94465274, 0.11623679],
[0.71774937, 1.99183721, 2.93154152, 0.23254174, 1.63623933],
[1.54450952, 2.32885258, 1.64220968, 1.66349828, 2.50975782],
[0.99172053, 2.60171951, 1.14377575, 0.28264201, 2.50368237],
[1.99669231, 2.16996937, 1.77290071, 1.34783694, 2.42391734]])
label = np.array([4, 0, 0, 1, 0, 4, 0, 2, 1, 3])
grad_func = grad(softmax)
# Check forward
assert rel_error(softmax(x, label), 2.16612911224) < 1e-12
expected_out = np.array([[0.00323393, 0.01761957, 0.02614461, 0.0396159, -0.08661401],
[-0.09591437, 0.01562194, 0.01098321, 0.05821122, 0.011098],
[-0.08735776, 0.02260642, 0.04649542, 0.00872727, 0.00952864],
[0.00992692, -0.09167096, 0.02382641, 0.00994309, 0.04797453],
[-0.05756653, 0.01738011, 0.01269563, 0.01310903, 0.01438177],
[0.02244518, 0.03767938, 0.00641325, 0.02883012, -0.09536792],
[-0.09406418, 0.02122317, 0.05431486, 0.00365391, 0.01487223],
[0.01242947, 0.02723256, -0.08629487, 0.01400002, 0.03263282],
[0.00820025, -0.05897573, 0.00954693, 0.00403532, 0.03719322],
[0.01982428, 0.02357495, 0.01584915, -0.08963896, 0.03039058]])
# Check backward
assert rel_error(grad_func(x, label), expected_out) < 1e-7
print('All passed!')
def test_context():
set_context(gpu(1)) # set the global context as gpu(1)
def sigmoid(x):
return 0.5 * (np.tanh(x / 2) + 1)
def predict(weights, inputs):
return sigmoid(np.dot(inputs, weights))
def training_loss(weights, inputs):
preds = predict(weights, inputs)
label_probabilities = preds * targets + (1 - preds) * (1 - targets)
l = -np.sum(np.log(label_probabilities))
return l
def training_accuracy(weights, inputs):
preds = predict(weights, inputs)
error = np.count_nonzero(np.argmax(preds, axis=1) - np.argmax(targets, axis=1))
return (256 - error) * 100 / 256.0
with gpu(0):
xshape = (256, 500)
wshape = (500, 250)
tshape = (256, 250)
inputs = random.rand(*xshape) - 0.5
targets = np.zeros(tshape)
truth = random.randint(0, 250, 256)
targets[np.arange(256), truth] = 1
weights = random.rand(*wshape) - 0.5
training_gradient_fun = grad(training_loss)
for i in range(20):
print('Trained loss accuracy #{}: {}%'.format(i, training_accuracy(weights, inputs)))
gr = training_gradient_fun(weights, inputs)
weights -= gr * 0.01
print("\nff and bp on {0}".format(weights.context))
print("\nexecute on cpu")
with cpu():
x_cpu = random.rand(32, 64) - 0.5
y_cpu = random.rand(64, 32) - 0.5
z_cpu = np.dot(x_cpu, y_cpu)
print('z_cpu.context = {0}'.format(z_cpu.context))
print("\nexecute on gpu(0)")
with gpu(0):
x_gpu0 = random.rand(32, 64) - 0.5
y_gpu0 = random.rand(64, 32) - 0.5
z_gpu0 = np.dot(x_gpu0, y_gpu0)
z_gpu0.asnumpy()
print('z_gpu0.context = {0}'.format(z_gpu0.context))
print("\n[use global context] execute on gpu(1)")
x_gpu1 = random.rand(32, 64) - 0.5
y_gpu1 = random.rand(64, 32) - 0.5
z_gpu1 = np.dot(x_gpu1, y_gpu1)
z_gpu1.asnumpy()
print('z_gpu1.context = {0}'.format(z_gpu1.context))
def test_autograd():
def func(x):
y = x + 1
return x
x = 1
gradient = grad(func)
print('Gradient should be 1, and it is actually', gradient(x))
def main():
x = random.rand(3, 3)
y = random.rand(3, 3)
print('x: {}'.format(x.asnumpy()))
print('y: {}'.format(y.asnumpy()))
g = core.grad(f, argnum=[0, 1])
gr = g(x, y)
print('grad_x: {}'.format(gr[0].asnumpy()))
print('grad_y: {}'.format(gr[1].asnumpy()))
def test_sum():
x_np = np.array([[1, 2], [3, 4], [5, 6]])
x_grad = np.array([[1, 1], [1, 1], [1, 1]])
def red1(x):
return mp.sum(x)
def red2(x):
return mp.sum(x, axis=0)
def red3(x):
return mp.sum(x, axis=0, keepdims=True)
grad1 = grad(red1)
assert np.all(grad1(x_np).asnumpy() == x_grad)
grad2 = grad(red2)
assert np.all(grad2(x_np).asnumpy() == x_grad)
grad3 = grad(red3)
assert np.all(grad3(x_np).asnumpy() == x_grad)
def quick_grad_check(fun,
arg0,
extra_args=(),
kwargs={},
verbose=True,
eps=EPS,
rtol=RTOL,
atol=ATOL,
rs=None):
"""Checks the gradient of a function (w.r.t. to its first arg) in a random direction"""
if verbose:
print("Checking gradient of {0} at {1}".format(fun, arg0))
if rs is None:
rs = nnp.random.RandomState()
random_dir = rs.standard_normal(nnp.shape(arg0))
random_dir = random_dir / nnp.sqrt(nnp.sum(random_dir * random_dir))
if not extra_args == ():
unary_fun = lambda x: fun(arg0 + x * random_dir, extra_args)
numeric_grad = (unary_fun(eps / 2) - unary_fun(-eps / 2)) / eps
analytic_grad = np.sum(grad(fun)(arg0, extra_args) * random_dir)
else:
unary_fun = lambda x: fun(arg0 + x * random_dir)
numeric_grad = (unary_fun(eps / 2) - unary_fun(-eps / 2)) / eps
analytic_grad = np.sum(grad(fun)(arg0) * random_dir)
if isinstance(numeric_grad, minpy.array.Number):
assert abs((analytic_grad - numeric_grad).get_data(None)) < atol and abs((analytic_grad - numeric_grad).get_data(None)) < abs((analytic_grad * rtol).get_data(None)), \
"Check failed! nd={0}, ad={1}".format(numeric_grad, analytic_grad)
elif isinstance(numeric_grad, minpy.array.Array):
assert nnp.prod(nnp.shape(analytic_grad.asnumpy())
[:]) == 1, "Currently only support check loss"
assert abs((analytic_grad - numeric_grad).asnumpy()) < atol and abs((analytic_grad - numeric_grad).asnumpy()) < abs((analytic_grad * rtol).asnumpy()), \
"Check failed! nd={0}, ad={1}".format(numeric_grad, analytic_grad)
else:
assert False
if verbose:
print("Gradient projection OK (numeric grad: {0}, analytic grad: {1})".
format(numeric_grad, analytic_grad))
def quick_grad_check(fun,
arg,
verbose=True,
eps=1e-2,
rtol=1e-2,
atol=1e-2,
rs=None):
# pylint: disable= too-many-arguments
"""
Checks the gradient of a function (w.r.t. to its first arg) in a random direction
Args:
fun:
The function for gradient checking
arg:
Gradient checking point
verbose:
Whether print some debug information
eps:
Epsilon in computing numerical gradient
rtol:
Relative tolerance
atol:
Absolute tolerance
rs:
RandomState used in generating random direction
"""
if rs is None:
rs = np.random.RandomState()
if isinstance(arg, Value): # convert it to numpy value
arg = arg.asnumpy()
random_dir = rs.standard_normal(np.shape(arg))
random_dir = random_dir / np.sqrt(np.sum(random_dir * random_dir))
grad_fun = grad(fun)
unary_fun = lambda x: fun(arg + x * random_dir).asnumpy()
numeric_grad = (unary_fun(eps / 2) - unary_fun(-eps / 2)) / eps
analytic_grad = np.sum(grad_fun(arg)[0].asnumpy() * random_dir)
passed = np.allclose(numeric_grad, analytic_grad, rtol=rtol, atol=atol)
if verbose:
if passed:
print("Gradient projection OK (numeric grad: {0}, analytic grad: {1})".format(\
numeric_grad, analytic_grad))
else:
print("Check failed! numeric={0}, analytic={1}".format(\
numeric_grad, analytic_grad))
return passed
def quick_grad_check(fun,
arg,
verbose=True,
eps=1e-2,
rtol=1e-2,
atol=1e-2,
rs=None):
"""
Checks the gradient of a function (w.r.t. to its first arg) in a random direction
Args:
fun:
The function for gradient checking
arg:
Gradient checking point
verbose:
Whether print some debug information
eps:
Epsilon in computing numerical gradient
rtol:
Relative tolerance
atol:
Absolute tolerance
rs:
RandomState used in generating random direction
"""
if rs is None:
rs = np.random.RandomState()
if isinstance(arg, Value): # convert it to numpy value
arg = arg.asnumpy()
random_dir = rs.standard_normal(np.shape(arg))
random_dir = random_dir / np.sqrt(np.sum(random_dir * random_dir))
grad_fun = grad(fun)
unary_fun = lambda x: fun(arg + x * random_dir).asnumpy()
numeric_grad = (unary_fun(eps / 2) - unary_fun(-eps / 2)) / eps
analytic_grad = np.sum(grad_fun(arg).asnumpy() * random_dir)
passed = np.allclose(numeric_grad, analytic_grad, rtol=rtol, atol=atol)
if verbose:
if passed:
print("Gradient projection OK (numeric grad: {0}, analytic grad: {1})".format(\
numeric_grad, analytic_grad))
else:
print("Check failed! numeric={0}, analytic={1}".format(\
numeric_grad, analytic_grad))
return passed
def test_op_statistics():
def sigmoid(x):
return 0.5 * (np.tanh(x / 2) + 1)
def predict(weights, inputs):
return sigmoid(np.dot(inputs, weights))
def training_loss(weights, inputs):
preds = predict(weights, inputs)
label_probabilities = preds * targets + (1 - preds) * (1 - targets)
l = -np.sum(np.log(label_probabilities))
return l
def training_accuracy(weights, inputs):
preds = predict(weights, inputs)
error = np.count_nonzero(
np.argmax(
preds, axis=1) - np.argmax(
targets, axis=1))
return (256 - error) * 100 / 256.0
np.record_op_stat()
xshape = (256, 500)
wshape = (500, 250)
tshape = (256, 250)
inputs = random.rand(*xshape) - 0.5
targets = np.zeros(tshape)
truth = random.randint(0, 250, 256)
targets[np.arange(256), truth] = 1
weights = random.rand(*wshape) - 0.5
training_gradient_fun = grad(training_loss)
for i in range(30):
print('Trained accuracy #{}: {}%'.format(i, training_accuracy(weights,
inputs)))
gr = training_gradient_fun(weights, inputs)
weights -= gr * 0.01
# Print Op Statistics Info
np.show_op_stat()
def test_mxnet_logistic():
def sigmoid(x):
return np.multiply(0.5, np.add(np.tanh(x), 1))
xshape = (256, 500)
#needs to reverse. because of mxnet's setting
wshape = (250, 500)
tshape = (256, 250)
inputs = random.rand(*xshape) - 0.5
targets = np.zeros(tshape)
truth = random.randint(0, 250, 256)
targets[np.arange(256), truth] = 1
weights = np.random.rand(*wshape) - 0.5
x = mx.sym.Variable(name='x')
fc = mx.sym.FullyConnected(name='fc', data=x, num_hidden=250)
act = mx.sym.Activation(data=fc, act_type='sigmoid')
f = core.Function(act, {'x': xshape})
def predict(weights, inputs):
#return f( data=[('x', inputs)], weight=[('fc_weight', weights)], ctx=mx.cpu())
return f(x=inputs, fc_weight=weights)
def training_loss(weights, inputs):
preds = predict(weights, inputs)
label_probabilities = preds * targets + (1 - preds) * (1 - targets)
return -np.sum(np.log(label_probabilities))
training_gradient_fun = core.grad(training_loss)
print('Initial loss: {}'.format(training_loss(weights, inputs)))
for i in range(100):
gr = training_gradient_fun(weights, inputs)
#print('Training gradient: {}'.format(gr))
weights -= gr * 0.1
if i % 10 == 0:
print('Trained loss: {}'.format(training_loss(weights, inputs)))
# The training loss should be around 300 in a bug-free Minpy
if (training_loss(weights, inputs)[0] > 600):
assert (False)
def test_slice():
def sigmoid(x):
return 0.5 * (np.tanh(x / 2) + 1)
def predict(weights, inputs):
# Test Slice
sliced_weights = weights[:, ::2]
y = sigmoid(np.dot(inputs, sliced_weights))
return y
def training_loss(weights, inputs):
preds = predict(weights, inputs)
label_probabilities = preds * targets + (1 - preds) * (1 - targets)
l = -np.sum(np.log(label_probabilities))
return l
def training_accuracy(weights, inputs):
preds = predict(weights, inputs)
error = np.count_nonzero(
np.argmax(preds, axis=1) - np.argmax(targets, axis=1))
return (256 - error) * 100 / 256.0
xshape = (256, 500)
# wshape = (500, 250)
wshape = (500, 500)
tshape = (256, 250)
inputs = random.rand(*xshape) - 0.5
targets = np.zeros(tshape)
truth = random.randint(0, 250, 256)
targets[np.arange(256), truth] = 1
weights = random.rand(*wshape) - 0.5
training_gradient_fun = grad(training_loss)
for i in range(20):
print('Trained loss accuracy #{}: {}%'.format(
i, training_accuracy(weights, inputs)))
gr = training_gradient_fun(weights, inputs)
print('Gradient Size', gr.shape)
print('Gradient example', gr[0, :10].asnumpy())
weights -= gr * 0.01
def test_logistic():
def sigmoid(x):
return 0.5 * (np.tanh(x / 2) + 1)
def predict(weights, inputs):
return sigmoid(np.dot(inputs, weights))
def training_loss(weights, inputs):
preds = predict(weights, inputs)
label_probabilities = preds * targets + (1 - preds) * (1 - targets)
l = -np.sum(np.log(label_probabilities))
return l
def training_accuracy(weights, inputs):
preds = predict(weights, inputs)
error = np.count_nonzero(
np.argmax(
preds, axis=1) - np.argmax(
targets, axis=1))
return (256 - error) * 100 / 256.0
xshape = (256, 500)
wshape = (500, 250)
tshape = (256, 250)
inputs = random.rand(*xshape) - 0.5
targets = np.zeros(tshape)
truth = random.randint(0, 250, 256)
targets[np.arange(256), truth] = 1
weights = random.rand(*wshape) - 0.5
training_gradient_fun = grad(training_loss)
for i in range(200):
print('Trained accuracy #{}: {}%'.format(
i, training_accuracy(weights, inputs)))
gr = training_gradient_fun(weights, inputs)
weights -= gr * 0.01
# The accuracy should be 100 in bug-free MinPy
if (training_accuracy(weights, inputs) < 95):
assert (False)
def test_logistic():
def sigmoid(x):
return 0.5 * (np.tanh(x / 2) + 1)
def predict(weights, inputs):
return sigmoid(np.dot(inputs, weights))
def training_loss(weights, inputs):
preds = predict(weights, inputs)
label_probabilities = preds * targets + (1 - preds) * (1 - targets)
l = -np.sum(np.log(label_probabilities))
return l
def training_accuracy(weights, inputs):
preds = predict(weights, inputs)
error = np.count_nonzero(
np.argmax(
preds, axis=1) - np.argmax(
targets, axis=1))
return (256 - error) * 100 / 256.0
xshape = (256, 500)
wshape = (500, 250)
tshape = (256, 250)
inputs = random.rand(*xshape) - 0.5
targets = np.zeros(tshape)
truth = random.randint(0, 250, 256)
targets[np.arange(256), truth] = 1
weights = random.rand(*wshape) - 0.5
training_gradient_fun = grad(training_loss)
for i in range(200):
print('Trained accuracy #{}: {}%'.format(i, training_accuracy(weights,
inputs)))
gr = training_gradient_fun(weights, inputs)
weights -= gr * 0.01
def test_slice():
def sigmoid(x):
return 0.5 * (np.tanh(x / 2) + 1)
def predict(weights, inputs):
# Test Slice
sliced_weights = weights[:, ::2]
y = sigmoid(np.dot(inputs, sliced_weights))
return y
def training_loss(weights, inputs):
preds = predict(weights, inputs)
label_probabilities = preds * targets + (1 - preds) * (1 - targets)
l = -np.sum(np.log(label_probabilities))
return l
def training_accuracy(weights, inputs):
preds = predict(weights, inputs)
error = np.count_nonzero(np.argmax(preds, axis=1) - np.argmax(targets, axis=1))
return (256 - error) * 100 / 256.0
xshape = (256, 500)
# wshape = (500, 250)
wshape = (500, 500)
tshape = (256, 250)
inputs = random.rand(*xshape) - 0.5
targets = np.zeros(tshape)
truth = random.randint(0, 250, 256)
targets[np.arange(256), truth] = 1
weights = random.rand(*wshape) - 0.5
training_gradient_fun = grad(training_loss)
for i in range(20):
print('Trained loss accuracy #{}: {}%'.format(i, training_accuracy(weights, inputs)))
gr = training_gradient_fun(weights, inputs)
print('Gradient Size', gr.shape)
print('Gradient example', gr[0,:10].asnumpy())
weights -= gr * 0.01
x = mx.sym.Variable(name='x')
fc = mx.sym.FullyConnected(name='fc', data=x)
#fc = mx.sym.FullyConnected(name='fc', data=x, num_hidden=inputs.shape[1])
act = mx.sym.Activation(data=fc, act_type='sigmoid'）
f = core.function(act)
def predict(weights, inputs):
return f(x=inputs, fc_weight=weights, ctx=mx.cpu())
def training_loss(weights, inputs):
preds = predict(weights, inputs)
label_probabilities = preds * targets + (1 - preds) * (1 - targets)
return -np.sum(np.log(label_probabilities))
xshape = (256, 500)
wshape = (500, 250)
tshape = (256, 250)
inputs = np.random.rand(*xshape) - 0.5
targets = np.random.randint(0, 2, size=tshape)
weights = np.random.rand(*wshape) - 0.5
training_gradient_fun = core.grad(training_loss)
print('Initial loss: {}'.format(training_loss(weights, inputs)))
for i in range(100):
gr = training_gradient_fun(weights, inputs)
#print('Training gradient: {}'.format(gr))
weights -= gr * 0.1
if i % 10 == 0:
print('Trained loss: {}'.format(training_loss(weights, inputs)))
preds = predict(weights, bias, inputs)
label_probabilities = preds * targets + (1 - preds) * (1 - targets)
l = -np.sum(np.log(label_probabilities))
return l
def training_accuracy(weights, bias, inputs):
preds = predict(weights, bias, inputs)
error = np.count_nonzero(np.argmax(preds, axis=1) - np.argmax(targets, axis=1))
return (256 - error) * 100 / 256.0
xshape = (256, 500)
wshape = (500, 250)
bshape = (250)
tshape = (256, 250)
inputs = random.rand(*xshape) - 0.5
targets = np.zeros(tshape)
truth = random.randint(0, 250, 256)
targets[np.arange(256), truth] = 1
weights = random.rand(*wshape) - 0.5
#bias = random.rand(bshape) - 0.5
#print bias.shape
bias = np.zeros(bshape)
print bias.shape
training_gradient_fun = grad(training_loss)
for i in range(20):
print('Trained loss accuracy #{}: {}%'.format(i, training_accuracy(weights, bias, inputs)))
gr = training_gradient_fun(weights, bias, inputs)
weights -= gr * 0.01
a.dtype
mx.nd.array(numpy.array([1, 2, 3]))
type(numpy.array([1, 2, 3]))
type(c)
np.ones((2, 3))
np.ones([2, 3])
mx.nd.ones([2, 3])
mx.nd.ones([2, 3]).asnumpy()
def foo(x):
return (5 * (x**2) + 3 * x + 2)
print(foo(4))
d_foo = grad(foo)
d_l_foo = grad_and_loss(foo)
d_foo(4)
d_l_foo(4)
# Symbol
a = mx.sym.Variable('a')
b = mx.sym.Variable('b')
c = a + b
# elemental wise times
d = a * b
# matrix multiplication
e = mx.sym.dot(a, b)
f = mx.sym.Reshape(d + e, shape=(1, 4))
# broadcast
g = mx.sym.broadcast_to(f, shape=(2, 4))
def build(self):
"""build network"""
self.buildLayers()
self.loss = self.loss_f
self.gradient = grad(self.loss_weights,
[0]) # diff with respect to weights
def test_zero_input_grad():
def foo1(x):
return 1
bar1 = grad(foo1)
assert bar1(0) == 0.0
def test_zero_input_grad():
def foo1(x):
return 1
bar1 = grad(foo1)
assert bar1(0) == 0.0
def training_accuracy(weights, inputs):
preds = predict(weights, inputs)
error = np.count_nonzero(np.argmax(preds, axis=1) - np.argmax(targets, axis=1))
return (256 - error) * 100 / 256.0
xshape = (256, 500)
# wshape = (500, 250)
wshape = (500, 500)
tshape = (256, 250)
inputs = random.rand(*xshape) - 0.5
targets = np.zeros(tshape)
truth = random.randint(0, 250, 256)
targets[np.arange(256), truth] = 1
weights = random.rand(*wshape) - 0.5
training_gradient_fun = grad(training_loss)
def NumpyVarToMinpy(var):
return minpy.array.Value.wrap(var)
def MinpyVarToNumpy(var):
return minpy.array.Value.wrap(var).get_data(ArrayType.NUMPY)
for i in range(20):
print('Trained loss accuracy #{}: {}%'.format(i, training_accuracy(weights, inputs)))
gr = training_gradient_fun(weights, inputs)
print('Gradient Size', gr.shape)
print('Gradient example', MinpyVarToNumpy(gr[0,:10]))
weights -= gr * 0.01
def test_context():
set_context(gpu(1)) # set the global context as gpu(1)
def sigmoid(x):
return 0.5 * (np.tanh(x / 2) + 1)
def predict(weights, inputs):
return sigmoid(np.dot(inputs, weights))
def training_loss(weights, inputs):
preds = predict(weights, inputs)
label_probabilities = preds * targets + (1 - preds) * (1 - targets)
l = -np.sum(np.log(label_probabilities))
return l
def training_accuracy(weights, inputs):
preds = predict(weights, inputs)
error = np.count_nonzero(
np.argmax(preds, axis=1) - np.argmax(targets, axis=1))
return (256 - error) * 100 / 256.0
with gpu(0):
xshape = (256, 500)
wshape = (500, 250)
tshape = (256, 250)
inputs = random.rand(*xshape) - 0.5
targets = np.zeros(tshape)
truth = random.randint(0, 250, 256)
targets[np.arange(256), truth] = 1
weights = random.rand(*wshape) - 0.5
training_gradient_fun = grad(training_loss)
for i in range(20):
print('Trained loss accuracy #{}: {}%'.format(
i, training_accuracy(weights, inputs)))
gr = training_gradient_fun(weights, inputs)
weights -= gr * 0.01
print("\nff and bp on {0}".format(weights.context))
print("\nexecute on cpu")
with cpu():
x_cpu = random.rand(32, 64) - 0.5
y_cpu = random.rand(64, 32) - 0.5
z_cpu = np.dot(x_cpu, y_cpu)
print('z_cpu.context = {0}'.format(z_cpu.context))
print("\nexecute on gpu(0)")
with gpu(0):
x_gpu0 = random.rand(32, 64) - 0.5
y_gpu0 = random.rand(64, 32) - 0.5
z_gpu0 = np.dot(x_gpu0, y_gpu0)
z_gpu0.asnumpy()
print('z_gpu0.context = {0}'.format(z_gpu0.context))
print("\n[use global context] execute on gpu(1)")
x_gpu1 = random.rand(32, 64) - 0.5
y_gpu1 = random.rand(64, 32) - 0.5
z_gpu1 = np.dot(x_gpu1, y_gpu1)
z_gpu1.asnumpy()
print('z_gpu1.context = {0}'.format(z_gpu1.context))
from minpy import core
import minpy.numpy as np
def f(x):
return x
if __name__ == '__main__':
inp = np.random.random((3, 2))
print(inp.asnumpy())
g0 = core.grad(f)(inp)
# All ones.
print(g0.asnumpy())
g_inj = np.random.random((3, 2))
print(g_inj.asnumpy())
injection = lambda *args, **kwargs: f(*args, **kwargs) * g_inj
g1 = core.grad(injection)(inp)
# Gradient will be as injected.
print(g1.asnumpy())
from __future__ import print_function
import minpy.numpy as np
from minpy.core import grad
def func(x):
y = x + 1
return x
x = 1
gradient = grad(func)
print('Gradient should be 1, and it is actually', gradient(x))
