def forward(self, inputs):
x = autograd.matmul(inputs, self.w0)
x = autograd.add_bias(x, self.b0)
x = autograd.relu(x)
x = autograd.matmul(x, self.w1)
x = autograd.add_bias(x, self.b1)
return x
def onnx_loss(a,model,target):
'''
input:
a graph node dictionary
model: graph model
target: label
load other nodes of onnx
'''
for i in model.graph.node:
if (i.op_type == 'Constant'):
pass
# do nothing
if (i.op_type == 'LeakyRelu'):
a[str(i.output[0])] = autograd.relu(a[str(i.input[0])])
elif (i.op_type == 'Relu'):
a[str(i.output[0])] = autograd.relu(a[str(i.input[0])])
elif (i.op_type == 'Softmax'):
a[str(i.output[0])] = autograd.softmax(a[str(i.input[0])])
elif (i.op_type == 'Add'):
if(str(i.input[1])[-1] == 'b'):
a[str(i.output[0])] = autograd.add_bias(a[str(i.input[0])], a[str(i.input[1])])
else:
a[str(i.output[0])] = autograd.add(a[str(i.input[0])],a[str(i.input[1])])
elif (i.op_type == 'MatMul'):
a[str(i.output[0])] = autograd.matmul(a[str(i.input[0])], a[str(i.input[1])])
loss = autograd.cross_entropy(a['Y'], target)
return loss
def singa_to_onnx(niter, use_cpu=False):
if use_cpu:
print("Using CPU")
dev = device.get_default_device()
else:
print("Using GPU")
dev = device.create_cuda_gpu()
inputs = Tensor(
data=data,
device=dev,
requires_grad=False,
stores_grad=False,
name="input",
)
target = Tensor(
data=label,
device=dev,
requires_grad=False,
stores_grad=False,
name="target",
)
w0 = Tensor(shape=(2, 3), device=dev, requires_grad=True, stores_grad=True)
w0.gaussian(0.0, 0.1)
b0 = Tensor(shape=(3,), device=dev, requires_grad=True, stores_grad=True)
b0.set_value(0.0)
w1 = Tensor(shape=(3, 2), device=dev, requires_grad=True, stores_grad=True)
w1.gaussian(0.0, 0.1)
b1 = Tensor(shape=(2,), device=dev, requires_grad=True, stores_grad=True)
b1.set_value(0.0)
sgd = opt.SGD(0.1)
# training process
for i in range(100):
x = autograd.matmul(inputs, w0)
x = autograd.add_bias(x, b0)
x = autograd.relu(x)
x = autograd.matmul(x, w1)
x = autograd.add_bias(x, b1)
loss = autograd.softmax_cross_entropy(x, target)
for p, gp in autograd.backward(loss):
sgd.update(p, gp)
print("training loss = ", tensor.to_numpy(loss)[0])
sonnx.export([inputs], [x], file_path="mlp.onnx")
def _4d_matmul_helper(self, dev):
np_x1 = np.random.randn(2, 12, 256, 64).astype(np.float32)
np_x2 = np.random.randn(2, 12, 64, 256).astype(np.float32)
x1 = tensor.from_numpy(np_x1)
x1.to_device(dev)
x2 = tensor.from_numpy(np_x2)
x2.to_device(dev)
y = autograd.matmul(x1, x2)
np_y = np.matmul(np_x1, np_x2)
np.testing.assert_array_almost_equal(tensor.to_numpy(y), np_y)
np_x1 = np.random.randn(2, 12, 256, 64).astype(np.float32)
np_x2 = np.random.randn(2, 12, 64, 1024).astype(np.float32)
x1 = tensor.from_numpy(np_x1)
x1.to_device(dev)
x2 = tensor.from_numpy(np_x2)
x2.to_device(dev)
y = autograd.matmul(x1, x2)
np_y = np.matmul(np_x1, np_x2)
np.testing.assert_array_almost_equal(tensor.to_numpy(y), np_y)
def matmul_high_dim_helper(self, dev):
configs = [
[(1, 12, 7, 64), (1, 12, 64, 7)],
[(1, 7, 768), (768, 768)],
]
print()
for config in configs:
X = np.random.random(config[0]).astype(np.float32)
x = tensor.from_numpy(X)
x.to_device(dev)
W = np.random.random(config[1]).astype(np.float32)
w = tensor.from_numpy(W)
w.to_device(dev)
y_t = np.matmul(X, W)
y = autograd.matmul(x, w)
np.testing.assert_array_almost_equal(tensor.to_numpy(y), y_t, 3)
def _matmul_transpose_helper(self, dev):
X = np.random.random((1, 256, 12, 64)).astype(np.float32)
x = tensor.from_numpy(X)
x.to_device(dev)
W = np.random.random((1, 256, 12, 64)).astype(np.float32)
w = tensor.from_numpy(W)
w.to_device(dev)
X = np.transpose(X, (0, 2, 1, 3))
W = np.transpose(W, (0, 2, 1, 3))
W = np.transpose(W, (0, 1, 3, 2))
Y = np.matmul(X, W)
x = autograd.transpose(x, (0, 2, 1, 3))
w = autograd.transpose(w, (0, 2, 1, 3))
w = autograd.transpose(w, (0, 1, 3, 2))
y = autograd.matmul(x, w)
np.testing.assert_array_almost_equal(tensor.to_numpy(x), X)
np.testing.assert_array_almost_equal(tensor.to_numpy(w), W)
np.testing.assert_array_almost_equal(tensor.to_numpy(y), Y)
def run(model, modeldic, layer,inputs):
'''
input: input for singa model
load other nodes of onnx
'''
supportLayer = ['Linear','Conv','MaxPool','AveragePool','BatchNormalization']
#supportLayer = ['Conv', 'MaxPool', 'AveragePool', 'BatchNormalization']
oper=modeldic
for counter,i in enumerate(model.graph.input):
oper[i.name] = inputs[counter]
for i in model.graph.node:
if (i.op_type == 'Relu'):
oper[str(i.output[0])] = autograd.relu(oper[str(i.input[0])])
elif (i.op_type == 'Softmax'):
oper[str(i.output[0])] = autograd.softmax(oper[str(i.input[0])])
elif (i.op_type == 'Add'):
oper[str(i.output[0])] = autograd.add(oper[str(i.input[0])], oper[str(i.input[1])])
elif (i.op_type == 'MatMul'):
oper[str(i.output[0])] = autograd.matmul(oper[str(i.input[0])], oper[str(i.input[1])])
elif (i.op_type == 'Flatten'):
oper[str(i.output[0])] = autograd.flatten(oper[str(i.input[0])])
elif(i.op_type == 'Concat'):
oper[str(i.output[0])] = autograd.cat((oper[str(i.input[0])], oper[str(i.input[1])]),int(i.attribute[0].i))
elif(i.op_type == 'Tanh'):
oper[str(i.output[0])] = autograd.tanh(oper[str(i.input[0])])
elif (i.op_type == 'Sigmoid'):
oper[str(i.output[0])] = autograd.sigmoid(oper[str(i.input[0])])
elif (i.op_type == 'Mul'):
oper[str(i.output[0])] = autograd.mul(oper[str(i.input[0])],oper[str(i.input[1])])
elif (i.op_type in supportLayer):
oper[str(i.output[0])] = layer[str(i.output[0])](oper[str(i.input[0])])
out =[]
for counter,i in enumerate(model.graph.output):
out.append(modeldic[i.name])
return out
print("train_label_shape:", label.shape)
inputs = Tensor(data=data)
target = Tensor(data=label)
w0 = Tensor(shape=(2, 3), requires_grad=True, stores_grad=True)
w0.gaussian(0.0, 0.1)
b0 = Tensor(shape=(1, 3), requires_grad=True, stores_grad=True)
b0.set_value(0.0)
w1 = Tensor(shape=(3, 2), requires_grad=True, stores_grad=True)
w1.gaussian(0.0, 0.1)
b1 = Tensor(shape=(1, 2), requires_grad=True, stores_grad=True)
b1.set_value(0.0)
sgd = optimizer.SGD(0.05)
# training process
for i in range(1001):
x = autograd.matmul(inputs, w0)
x = autograd.add_bias(x, b0)
x = autograd.relu(x)
x = autograd.matmul(x, w1)
x = autograd.add_bias(x, b1)
x = autograd.softmax(x)
loss = autograd.cross_entropy(x, target)
for p, gp in autograd.backward(loss):
sgd.apply(0, gp, p, "")
if i % 100 == 0:
print("training loss = ", tensor.to_numpy(loss)[0])
print('train_label_shape:', label.shape)
inputs = Tensor(data=data)
target = Tensor(data=label)
w0 = Tensor(shape=(2, 3), requires_grad=True, stores_grad=True)
w0.gaussian(0.0, 0.1)
b0 = Tensor(shape=(1, 3), requires_grad=True, stores_grad=True)
b0.set_value(0.0)
w1 = Tensor(shape=(3, 2), requires_grad=True, stores_grad=True)
w1.gaussian(0.0, 0.1)
b1 = Tensor(shape=(1, 2), requires_grad=True, stores_grad=True)
b1.set_value(0.0)
sgd = optimizer.SGD(0.05)
# training process
for i in range(1001):
x = autograd.matmul(inputs, w0)
x = autograd.add_bias(x, b0)
x = autograd.relu(x)
x = autograd.matmul(x, w1)
x = autograd.add_bias(x, b1)
x = autograd.softmax(x)
loss = autograd.cross_entropy(x, target)
for p, gp in autograd.backward(loss):
sgd.apply(0, gp, p, '')
if (i % 100 == 0):
print('training loss = ', tensor.to_numpy(loss)[0])
w1 = Tensor(shape=(2, 2), requires_grad=True, stores_grad=True)
w1.gaussian(0.0, 0.1)
b1 = Tensor(shape=(1, 2), requires_grad=True, stores_grad=True)
b1.set_value(0.0)
w2 = Tensor(shape=(2, 2), requires_grad=True, stores_grad=True)
w2.gaussian(0.0, 0.1)
b2 = Tensor(shape=(1, 2), requires_grad=True, stores_grad=True)
b2.set_value(0.0)
sgd = optimizer.SGD(0.00)
# training process
for i in range(1):
x = autograd.matmul(inputs, w0)
x = autograd.add_bias(x, b0)
x = autograd.relu(x)
x2 = autograd.matmul(x, w2)
x2 = autograd.add_bias(x2, b2)
x1 = autograd.matmul(x, w1)
x1 = autograd.add_bias(x1, b1)
x = autograd.add(x1, x2)
y = autograd.softmax(x)
loss = autograd.cross_entropy(y, target)
gradient = autograd.backward(loss)
for p, gp in gradient:
sgd.apply(0, gp, p, '')
if (i % 100 == 0):
print('training loss = ', tensor.to_numpy(loss)[0])