def test_lr():
W = ad.Variable(name="W")
b = ad.Variable(name="b")
X = ad.Variable(name="X")
y_ = ad.Variable(name="y_")
ctx = ndarray.gpu(0)
# ini
x_val = np.linspace(0, 1, 100).reshape((100, 1))
y_val = x_val + 0.5
W_val = np.array([[0.1]])
b_val = np.array([0.1])
x_val = ndarray.array(x_val, ctx)
W_val = ndarray.array(W_val, ctx)
b_val = ndarray.array(b_val, ctx)
y_val = ndarray.array(y_val, ctx)
z = ad.matmul_op(X, W)
# z.shape = (100,1)
# b.shape = (1,1)
y = z + ad.broadcastto_op(b, z)
# y = (100,1)
y = ad.fullyactivation_forward_op(y, "NCHW", "relu")
loss = ad.matmul_op(y + (-1) * y_, y + (-1) * y_, trans_A=True) * (1 / 100)
# loss = ad.softmaxcrossentropy_op(y, y_)
grad_W, grad_b = ad.gradients(loss, [W, b])
executor = ad.Executor([loss, grad_W, grad_b], ctx)
aph = 1e-6
for i in range(100):
loss_val, grad_W_val, grad_b_val = executor.run(feed_dict={
X: x_val,
b: b_val,
W: W_val,
y_: y_val
})
grad_W_val = grad_W_val.asnumpy()
W_val = W_val.asnumpy()
W_val = W_val - aph * grad_W_val
W_val = ndarray.array(W_val, ctx)
grad_b_val = grad_b_val.asnumpy()
b_val = b_val.asnumpy()
b_val = b_val - aph * grad_b_val
b_val = ndarray.array(b_val, ctx)
print(W_val.asnumpy(), b_val.asnumpy())
def mnist_logreg(executor_ctx=None, num_epochs=10, print_loss_val_each_epoch=False):
# 训练逻辑回归模型
print("Build logistic regression model...")
W1 = ad.Variable(name="W1")
b1 = ad.Variable(name="b1")
X = ad.Variable(name="X")
y_ = ad.Variable(name="y_")
z1 = ad.matmul_op(X, W1)
y = z1 + ad.broadcastto_op(b1, z1)
loss = ad.softmaxcrossentropy_op(y, y_)
grad_W1, grad_b1 = ad.gradients(loss, [W1, b1])
executor = ad.Executor([loss, grad_W1, grad_b1, y], ctx=executor_ctx)
# Read input data
datasets = load_mnist_data("mnist.pkl.gz")
train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
# Set up minibatch
batch_size = 1000
n_train_batches = train_set_x.shape[0] // batch_size
n_valid_batches = valid_set_x.shape[0] // batch_size
print("Start training loop...")
# Initialize parameters
W1_val = np.zeros((784, 10))
b1_val = np.zeros((10))
X_val = np.empty(shape=(batch_size, 784), dtype=np.float32)
y_val = np.empty(shape=(batch_size, 10), dtype=np.float32)
valid_X_val = np.empty(shape=(batch_size, 784), dtype=np.float32)
valid_y_val = np.empty(shape=(batch_size, 10), dtype=np.float32)
if ndarray.is_gpu_ctx(executor_ctx):
W1_val = ndarray.array(W1_val, ctx=executor_ctx)
b1_val = ndarray.array(b1_val, ctx=executor_ctx)
X_val = ndarray.array(X_val, ctx=executor_ctx)
y_val = ndarray.array(y_val, ctx=executor_ctx)
lr = 1e-3
for i in range(num_epochs):
print("epoch %d" % i)
for minibatch_index in range(n_train_batches):
minibatch_start = minibatch_index * batch_size
minibatch_end = (minibatch_index + 1) * batch_size
X_val[:] = train_set_x[minibatch_start:minibatch_end]
y_val[:] = convert_to_one_hot(
train_set_y[minibatch_start:minibatch_end])
loss_val, grad_W1_val, grad_b1_val, _ = executor.run(
feed_dict = {X: X_val, y_: y_val, W1: W1_val, b1: b1_val})
# SGD update
if (executor_ctx is None):
W1_val = W1_val - lr * grad_W1_val
b1_val = b1_val - lr * grad_b1_val
else:
sgd_update_gpu(W1_val, grad_W1_val, lr)
sgd_update_gpu(b1_val, grad_b1_val, lr)
if print_loss_val_each_epoch:
if isinstance(loss_val, ndarray.NDArray):
print(loss_val.asnumpy())
else:
print(loss_val)
correct_predictions = []
for minibatch_index in range(n_valid_batches):
minibatch_start = minibatch_index * batch_size
minibatch_end = (minibatch_index + 1) * batch_size
valid_X_val[:] = valid_set_x[minibatch_start:minibatch_end]
valid_y_val[:] = convert_to_one_hot(
valid_set_y[minibatch_start:minibatch_end])
_, _, _, valid_y_predicted = executor.run(
feed_dict={
X: valid_X_val,
y_: valid_y_val,
W1: W1_val,
b1: b1_val},
convert_to_numpy_ret_vals=True)
correct_prediction = np.equal(
np.argmax(valid_y_val, 1),
np.argmax(valid_y_predicted, 1)).astype(np.float)
correct_predictions.extend(correct_prediction)
accuracy = np.mean(correct_predictions)
# validation set accuracy=0.928200
print("validation set accuracy=%f" % accuracy)
def mnist_mlp(executor_ctx=None, num_epochs=10, print_loss_val_each_epoch=False):
# 训练一个三层感知机模型
print("Build 3-layer MLP model...")
W1 = ad.Variable(name="W1")
W2 = ad.Variable(name="W2")
W3 = ad.Variable(name="W3")
b1 = ad.Variable(name="b1")
b2 = ad.Variable(name="b2")
b3 = ad.Variable(name="b3")
X = ad.Variable(name="X")
y_ = ad.Variable(name="y_")
# 下面是三层网络的激活函数，两个relu和一个softmax
# relu(X W1+b1)
z1 = ad.matmul_op(X, W1)
z2 = z1 + ad.broadcastto_op(b1, z1)
z3 = ad.relu_op(z2)
# relu(z3 W2+b2)
z4 = ad.matmul_op(z3, W2)
z5 = z4 + ad.broadcastto_op(b2, z4)
z6 = ad.relu_op(z5)
# softmax(z5 W2+b2)
z7 = ad.matmul_op(z6, W3)
y = z7 + ad.broadcastto_op(b3, z7)
loss = ad.softmaxcrossentropy_op(y, y_)
grad_W1, grad_W2, grad_W3, grad_b1, grad_b2, grad_b3 = ad.gradients(
loss, [W1, W2, W3, b1, b2, b3])
# 此处向前为符号定义
# 只声明，不操作
executor = ad.Executor(
[loss, grad_W1, grad_W2, grad_W3, grad_b1, grad_b2, grad_b3, y],
ctx=executor_ctx)
# Read input data
datasets = load_mnist_data("mnist.pkl.gz")
train_set_x, train_set_y = datasets[0]
valid_set_x, valid_set_y = datasets[1]
test_set_x, test_set_y = datasets[2]
# Set up minibatch
batch_size = 1000
n_train_batches = train_set_x.shape[0] // batch_size
n_valid_batches = valid_set_x.shape[0] // batch_size
print("Start training loop...")
# Initialize parameters
# 随机初始化网络中的w和b
rand = np.random.RandomState(seed=123)
W1_val = rand.normal(scale=0.1, size=(784, 256))
W2_val = rand.normal(scale=0.1, size=(256, 100))
W3_val = rand.normal(scale=0.1, size=(100, 10))
b1_val = rand.normal(scale=0.1, size=(256))
b2_val = rand.normal(scale=0.1, size=(100))
b3_val = rand.normal(scale=0.1, size=(10))
X_val = np.empty(shape=(batch_size, 784), dtype=np.float32)
y_val = np.empty(shape=(batch_size, 10), dtype=np.float32)
valid_X_val = np.empty(shape=(batch_size, 784), dtype=np.float32)
valid_y_val = np.empty(shape=(batch_size, 10), dtype=np.float32)
# todo 此处修改为cpu
W1_val = ndarray.array(W1_val, ctx=executor_ctx_cpu)
W2_val = ndarray.array(W2_val, ctx=executor_ctx_cpu)
W3_val = ndarray.array(W3_val, ctx=executor_ctx_cpu)
b1_val = ndarray.array(b1_val, ctx=executor_ctx_cpu)
b2_val = ndarray.array(b2_val, ctx=executor_ctx_cpu)
b3_val = ndarray.array(b3_val, ctx=executor_ctx_cpu)
X_val = ndarray.array(X_val, ctx=executor_ctx_cpu)
y_val = ndarray.array(y_val, ctx=executor_ctx_cpu)
# 此处以上将数据分别转化为cpu和gpu两种格式
lr = 1.0e-3
for i in range(num_epochs):
print("epoch %d" % i)
for minibatch_index in range(n_train_batches):
minibatch_start = minibatch_index * batch_size
minibatch_end = (minibatch_index + 1) * batch_size
X_val[:] = train_set_x[minibatch_start:minibatch_end]
y_val[:] = convert_to_one_hot(
train_set_y[minibatch_start:minibatch_end])
# 计算单步的梯度
loss_val, grad_W1_val, grad_W2_val, grad_W3_val, \
grad_b1_val, grad_b2_val, grad_b3_val, _ = executor.run(
feed_dict={
X: X_val,
y_: y_val,
W1: W1_val,
W2: W2_val,
W3: W3_val,
b1: b1_val,
b2: b2_val,
b3: b3_val})
# todo 更新sgd_update_gpu_on_cpu
def sgd_update_cpu(w1, w2, w3):
w1_gpu = ndarray.empty(w1.shape, executor_ctx)
w1.copyto(w1_gpu)
w2_gpu = ndarray.empty(w2.shape, executor_ctx)
w2.copyto(w2_gpu)
sgd_update_gpu(w1_gpu, w2_gpu, w3)
w1_gpu.copyto(w1)
w2_gpu.copyto(w2)
sgd_update_cpu(W1_val, grad_W1_val, lr)
sgd_update_cpu(W2_val, grad_W2_val, lr)
sgd_update_cpu(W3_val, grad_W3_val, lr)
sgd_update_cpu(b1_val, grad_b1_val, lr)
sgd_update_cpu(b2_val, grad_b2_val, lr)
sgd_update_cpu(b3_val, grad_b3_val, lr)
# sgd_update_gpu(W1_val, grad_W1_val, lr)
# sgd_update_gpu(W2_val, grad_W2_val, lr)
# sgd_update_gpu(W3_val, grad_W3_val, lr)
# sgd_update_gpu(b1_val, grad_b1_val, lr)
# sgd_update_gpu(b2_val, grad_b2_val, lr)
# sgd_update_gpu(b3_val, grad_b3_val, lr)
if print_loss_val_each_epoch:
if isinstance(loss_val, ndarray.NDArray):
print(loss_val.asnumpy())
else:
print(loss_val)
correct_predictions = []
for minibatch_index in range(n_valid_batches):
minibatch_start = minibatch_index * batch_size
minibatch_end = (minibatch_index + 1) * batch_size
valid_X_val[:] = valid_set_x[minibatch_start:minibatch_end]
valid_y_val[:] = convert_to_one_hot(
valid_set_y[minibatch_start:minibatch_end])
_, _, _, _, _, _, _, valid_y_predicted = executor.run(
feed_dict={
X: valid_X_val,
y_: valid_y_val,
W1: W1_val,
W2: W2_val,
W3: W3_val,
b1: b1_val,
b2: b2_val,
b3: b3_val},
convert_to_numpy_ret_vals=True)
correct_prediction = np.equal(
np.argmax(valid_y_val, 1),
np.argmax(valid_y_predicted, 1)).astype(np.float)
correct_predictions.extend(correct_prediction)
accuracy = np.mean(correct_predictions)
# validation set accuracy=0.970800
print("validation set accuracy=%f" % accuracy)