def mock_embedding(name1, name2):
ids = xdl.convert_to_tensor(
np.array([[0, 0], [0, 1], [0, 2]], dtype=np.int64))
values = xdl.convert_to_tensor(np.array([1.0, 2.0, 3.0], dtype=np.float32))
segments = xdl.convert_to_tensor(np.array([3], dtype=np.int32))
sparse = xdl.SparseTensor(ids, values, segments)
emb = xdl.embedding(name1,
sparse,
xdl.Ones(),
embed_dim,
16,
'sum',
vtype='hash')
emb.set_shape((1, 3))
ids2 = xdl.convert_to_tensor(
np.array([[0, 1], [0, 2], [1, 1]], dtype=np.int64))
values2 = xdl.convert_to_tensor(np.array([1.0, 2.0, 3.0],
dtype=np.float32))
segments2 = xdl.convert_to_tensor(np.array([3], dtype=np.int32))
sparse2 = xdl.SparseTensor(ids2, values2, segments2)
emb2 = xdl.embedding(name2,
sparse2,
xdl.Ones(),
embed_dim,
16,
'sum',
vtype='hash')
emb2.set_shape((1, 3))
return [emb, emb2]
def test_all(self):
var = xdl.Variable(name="w", dtype=DataType.int32, shape=[4], initializer=xdl.Ones())
execute(xdl.variable_registers())
execute(xdl.global_initializers())
op = xdl.ps_pull_op(var_name="w", var_type="index", dtype=DataType.int32)
ret = execute(op)
self.assertTrue((ret == np.array([1,1,1,1])).all())
def test_all(self):
var = xdl.Variable(name="w",
dtype=DataType.float,
shape=[4, 1],
initializer=xdl.Ones())
execute(xdl.variable_registers())
execute(xdl.global_initializers())
op = xdl.ps_sparse_apply_ftrl_op(
learning_rate=np.array(0.1, dtype=np.float),
learning_rate_power=np.array(-0.5, dtype=np.float),
initial_accumulator_value=np.array(0.1, dtype=np.float),
l1_reg=np.array(0, dtype=np.float),
l2_reg=np.array(0, dtype=np.float),
grad=np.array([[1], [2]], dtype=np.float32),
indices=np.array([1, 2], dtype=np.int32),
var_name="w",
var_type="index")
execute(op)
ret = execute(var.value)
self.assertTrue((ret == np.array([[1], [0.6031424], [0.7450533], [1]],
dtype=np.float32)).all())
execute(op)
ret = execute(var.value)
self.assertTrue((ret == np.array([[1], [0.5341358], [0.6747804], [1]],
dtype=np.float32)).all())
def test_all(self):
var = xdl.Variable(name="w",
dtype=DataType.float,
shape=[4, 1],
initializer=xdl.Ones())
execute(xdl.variable_registers())
execute(xdl.global_initializers())
op = xdl.ps_sparse_apply_momentum_op(learning_rate=np.array(
0.5, dtype=np.float),
momentum=np.array(0.9,
dtype=np.float),
grad=np.array([[1], [2]],
dtype=np.float32),
indices=np.array([1, 2],
dtype=np.int32),
var_name="w",
var_type="index",
use_nesterov=False)
execute(op)
ret = execute(var.value)
self.assertTrue((ret == np.array([[1], [0.5], [0], [1]],
dtype=np.float32)).all())
execute(op)
ret = execute(var.value)
self.assertTrue((ret == np.array([[1], [-0.45], [-1.9], [1]],
dtype=np.float32)).all())
def test_all(self):
var = xdl.Variable(name="w",
dtype=DataType.float,
shape=[4],
initializer=xdl.Ones())
execute(xdl.variable_registers())
execute(xdl.global_initializers())
op = xdl.ps_dense_apply_adam_op(beta1=np.array(0.9, dtype=np.float),
beta2=np.array(0.999, dtype=np.float),
epsilon=np.array(1e-08,
dtype=np.float),
learning_rate=np.array(0.1,
dtype=np.float),
grad=np.array([1, 2, 3, 4],
dtype=np.float32),
lr_decay=True,
var_name="w",
var_type="index")
execute(op)
ret = execute(var.value)
self.assertTrue(
(ret == np.array([0.90000004, 0.90000004, 0.90000004, 0.90000004],
dtype=np.float32)).all())
execute(op)
ret = execute(var.value)
self.assertTrue(
(ret == np.array([0.8000001, 0.8000001, 0.8000001, 0.8],
dtype=np.float32)).all())
def eval_input_fn():
dense = xdl.mock_dense_op(shape=[1, 16], value=0.01)
labels = xdl.mock_dense_op(shape=[1, 1], value=1.0)
ids, values, segments = xdl.mock_sparse_op(dense_shape=[1, 16])
sparse = xdl.SparseTensor(ids, values, segments)
emb = xdl.embedding("sparse", sparse, xdl.Ones(), 1, 16, 'sum')
dense.set_shape([None, 16])
labels.set_shape([None, 1])
return [dense, emb], labels
def input_fn():
dense = xdl.mock_dense_op(shape=[1, 16], value=0.01)
indicator = xdl.mock_dense_op(shape=[5], value=0.0)
labels = xdl.mock_dense_op(shape=[5, 1], value=1.0)
ids, values, segments = xdl.mock_sparse_op(dense_shape=[1, 16])
sparse = xdl.SparseTensor(ids, values, segments)
sparse.set_shape([1, 16])
emb = xdl.embedding("sparse", sparse, xdl.Ones(), 1, 16, 'sum')
dense.set_shape([1, 16])
indicator.set_shape([5])
labels.set_shape([5, 1])
return [dense, emb, indicator], labels
def main():
dense = xdl.mock_dense_op(shape=[1, 16], value=0.01, name_="dense")
gear = xdl.mock_dense_op(shape=[1, 1], value=0.01, name_="gear")
labels = xdl.mock_dense_op(shape=[1, 1], value=1.0, name_="label")
ids, values, segments = xdl.mock_sparse_op(dense_shape=[1, 16], name_="wide")
sparse = xdl.SparseTensor(ids, values, segments)
emb = xdl.embedding("sparse", sparse, xdl.Ones(), 1, 16, 'sum')
gear.set_shape([None, 1])
dense.set_shape([None, 16])
labels.set_shape([None, 1])
with xdl.model_scope("ams_main"):
loss = ams_main(main_model)(dense, emb, labels, gear_inputs=[gear])
sess = xdl.TrainSession()
return sess.run(xdl.get_collection("gear_grad"))
def test_all(self):
var = xdl.Variable(name="w",
dtype=DataType.int32,
shape=[4, 2],
initializer=xdl.Ones())
execute(xdl.variable_registers())
execute(xdl.global_initializers())
op = xdl.ps_sparse_pull_op(var_name="w",
var_type="index",
save_ratio=1.0,
otype=DataType.int32,
ids=np.array([1, 3], dtype=np.int32))
ret = execute(op)
self.assertTrue((ret == np.array([[1, 1], [1, 1]])).all())
def test_all(self):
var = xdl.Variable(name="w",
dtype=DataType.float,
shape=[4],
initializer=xdl.Ones())
execute(xdl.variable_registers())
execute(xdl.global_initializers())
op = xdl.ps_apply_moving_average_op(var_name="w",
moment=0.8,
value=np.array([1, 2, 3, 3],
dtype=np.float32))
execute(op)
ret = execute(var.value)
self.assertTrue((ret == np.array([1., 1.2, 1.4, 1.4],
dtype=np.float32)).all())
def test_all(self):
var = xdl.Variable(name="w",
dtype=DataType.int64,
shape=[4, 8],
vtype=VarType.Hash,
initializer=xdl.Ones())
execute(xdl.variable_registers())
execute(xdl.global_initializers())
mark_op = xdl.ps_mark_op(var_name="w",
ids=np.array([[10, 10], [10, 10], [12, 12]],
dtype=np.int64),
pattern="g",
i=12)
execute(mark_op)
filter_op = xdl.ps_filter_op(var_name="w", pattern="i==g", i=12, d=0.5)
execute(filter_op)
def test_all(self):
var = xdl.Variable(name="w", dtype=DataType.float, shape=[4,1], initializer=xdl.Ones())
execute(xdl.variable_registers())
execute(xdl.global_initializers())
op = xdl.ps_sparse_apply_adagrad_op(
learning_rate=np.array(0.5, dtype=np.float),
initial_accumulator_value=np.array(0.0, dtype=np.float),
grad=np.array([[1],[2]], dtype=np.float32),
indices=np.array([1,2], dtype=np.int32),
var_name="w",
var_type="index")
execute(op)
ret = execute(var.value)
self.assertTrue((ret == np.array([[1],[0.5],[0.5],[1]], dtype=np.float32)).all())
execute(op)
ret = execute(var.value)
self.assertTrue((ret == np.array([[1],[0.14644662],[0.14644662],[1]], dtype=np.float32)).all())
def test_all(self):
var = xdl.Variable(name="w", dtype=xdl.DT_FLOAT, shape=[4], initializer=xdl.Ones())
execute(xdl.variable_registers())
execute(xdl.global_initializers())
op = xdl.ps_dense_apply_adagrad_op(
learning_rate=np.array(0.5, dtype=np.float),
initial_accumulator_value=np.array(0.0, dtype=np.float),
grad=np.array([1,2,3,4], dtype=np.float32),
var_name="w",
var_type="index")
execute(op)
ret = execute(var.value)
self.assertTrue((ret == np.array([0.5,0.5,0.5,0.5])).all())
execute(op)
ret = execute(var.value)
print(ret)
self.assertTrue((ret == np.array([0.14644662,0.14644662,0.14644662,0.14644662], dtype=np.float32)).all())
def test_all(self):
dense = xdl.mock_dense_op(shape=[1, 16], value=0.01, name_="dense")
labels = xdl.mock_dense_op(shape=[1, 1], value=1.0, name_="label")
ids = xdl.convert_to_tensor(
np.array([[0, 0], [0, 1], [0, 2]], dtype=np.int64))
values = xdl.convert_to_tensor(
np.array([1.0, 2.0, 3.0], dtype=np.float32))
segments = xdl.convert_to_tensor(np.array([3], dtype=np.int32))
sparse = xdl.SparseTensor(ids, values, segments)
emb = xdl.embedding("sparse",
sparse,
xdl.Ones(),
1,
16,
'sum',
vtype='hash')
loss = model(dense, emb, labels)
train_op = xdl.SGD(0.5).optimize()
sess = xdl.TrainSession()
_, l, g = sess.run(
[train_op, loss,
xdl.get_sparse_grads('sparse').grad])
self.assertTrue((l == np.array(0.0024364376, dtype=np.float32)).all())
self.assertTrue(
(g == np.array([[-0.002433472], [-0.004866944], [-0.007300416]],
dtype=np.float32)).all())
sparse_var = xdl.get_variable_by_name('sparse')
weights = sess.run(
sparse_var.gather(
np.array([[0, 0], [0, 1], [0, 2]], dtype=np.int64)))
self.assertTrue(
(weights == np.array([[1.0012168], [1.0024334], [1.0036502]],
dtype=np.float32)).all())
_, l, g = sess.run(
[train_op, loss,
xdl.get_sparse_grads('sparse').grad])
self.assertTrue((l == np.array(0.002395329, dtype=np.float32)).all())
self.assertTrue(
(g == np.array([[-0.0023924622], [-0.0047849244], [-0.0071773864]],
dtype=np.float32)).all())
weights = sess.run(
sparse_var.gather(
np.array([[0, 0], [0, 1], [0, 2]], dtype=np.int64)))
self.assertTrue(
(weights == np.array([[1.002413], [1.0048258], [1.0072389]],
dtype=np.float32)).all())
def test_all(self):
var = xdl.Variable(name="w", dtype=DataType.float, shape=[4], initializer=xdl.Ones())
execute(xdl.variable_registers())
execute(xdl.global_initializers())
op = xdl.ps_dense_apply_ftrl_op(
learning_rate=np.array(0.1, dtype=np.float),
learning_rate_power=np.array(-0.5, dtype=np.float),
initial_accumulator_value=np.array(0.1, dtype=np.float),
l1_reg=np.array(0, dtype=np.float),
l2_reg=np.array(0, dtype=np.float),
grad=np.array([1,2,3,4], dtype=np.float32),
var_name="w",
var_type="index")
execute(op)
ret = execute(var.value)
self.assertTrue((ret == np.array([0.6031424,0.7450533,0.7957225,0.8215], dtype=np.float32)).all())
execute(op)
ret = execute(var.value)
self.assertTrue((ret == np.array([0.5341358,0.6747804,0.7252074,0.75089955], dtype=np.float32)).all())
def main():
dense = xdl.mock_dense_op(shape=[1, 16], value=0.01, name_="dense")
labels = xdl.mock_dense_op(shape=[1, 1], value=1.0, name_="label")
ids = xdl.convert_to_tensor(
np.array([[0, 0], [0, 1], [0, 2]], dtype=np.int64))
values = xdl.convert_to_tensor(np.array([1.0, 2.0, 3.0], dtype=np.float32))
segments = xdl.convert_to_tensor(np.array([3], dtype=np.int32))
sparse = xdl.SparseTensor(ids, values, segments)
emb = xdl.embedding("sparse",
sparse,
xdl.Ones(),
1,
16,
'sum',
vtype='hash')
loss = model(dense, emb, labels)
train_op = xdl.SGD(0.5).optimize()
sess = xdl.TrainSession()
loss, gradients = sess.run([loss, xdl.get_sparse_grads('sparse').grad])
return loss, gradients
def test_all(self):
var = xdl.Variable(name="w",
dtype=DataType.float,
shape=[4],
initializer=xdl.Ones())
execute(xdl.variable_registers())
execute(xdl.global_initializers())
op = xdl.ps_dense_apply_momentum_op(learning_rate=np.array(
0.5, dtype=np.float),
momentum=np.array(0.9,
dtype=np.float),
grad=np.array([1, 2, 3, 4],
dtype=np.float32),
var_name="w",
var_type="index",
use_nesterov=False)
execute(op)
ret = execute(var.value)
self.assertTrue((ret == np.array([0.5, 0, -0.5, -1],
dtype=np.float32)).all())
execute(op)
ret = execute(var.value)
self.assertTrue((ret == np.array([-0.45, -1.9, -3.35, -4.8],
dtype=np.float32)).all())
def train():
batch = data_io.read()
print batch
embs = list()
for i in range(1, embs_len + 1):
name = "item_%d" % i
emb = xdl.embedding(name,
batch[name],
xdl.Ones(),
1,
1000,
'sum',
vtype='hash')
embs.append(emb)
print "emb =", name, ", shape =", emb.shape
print "origin batch[label].shape =", batch["label"].shape
loss, prop, label, indicator, din, dout, fc1_weight, fc1_bias, fc2_weight, fc2_bias = model(
embs, batch["label"], 4, 7)
train_op = xdl.SGD(0.5).optimize()
item1_grad = xdl.get_gradient('item_1')
item2_grad = xdl.get_gradient('item_2')
item3_grad = xdl.get_gradient('item_3')
item4_grad = xdl.get_gradient('item_4')
fc1_weight_grad = xdl.get_gradient('fc1_weight')
fc1_bias_grad = xdl.get_gradient('fc1_bias')
fc2_weight_grad = xdl.get_gradient('fc2_weight')
fc2_bias_grad = xdl.get_gradient('fc2_bias')
sess = xdl.TrainSession()
loop_num = 0
while not sess.should_stop():
if loop_num == 5:
break
print "\n>>>>>>>>>>>> loop_num = %d" % loop_num
result = sess.run([train_op, loss, prop, batch['label'], label, indicator, din, dout, \
batch['item_1'].ids, batch['item_1'].segments, batch['item_1'].values, \
batch['item_2'].ids, batch['item_2'].segments, batch['item_2'].values, \
batch['item_3'].ids, batch['item_3'].segments, batch['item_3'].values, \
batch['item_4'].ids, batch['item_4'].segments, batch['item_4'].values, \
item1_grad, item2_grad, item3_grad, item4_grad, \
fc1_weight, fc1_bias, fc1_weight_grad, fc1_bias_grad, \
fc2_weight, fc2_bias, fc2_weight_grad, fc2_bias_grad])
if result is None:
break
print "loss:", result[-31]
print "prop:", result[-30]
print "origin label:", result[-29]
print "label:", result[-28]
print "indicator:", result[-27]
print "din:", result[-26]
print "dout:", result[-25]
print "item_1: ids=", result[-24], "\n segments=", result[
-23], "\n values=", result[-22]
print "item_2: ids=", result[-21], "\n segments=", result[
-20], "\n values=", result[-19]
print "item_3: ids=", result[-18], "\n segments=", result[
-17], "\n values=", result[-16]
print "item_4: ids=", result[-15], "\n segments=", result[
-14], "\n values=", result[-13]
print "item1_grad", result[-12]
print "item2_grad", result[-11]
print "item1_grad", result[-10]
print "item2_grad", result[-9]
print "fc1_weight", result[-8]
print "fc1_bias", result[-7]
print "fc1_weight_grad", result[-6]
print "fc1_bias_grad", result[-5]
print "fc2_weight", result[-4]
print "fc2_bias", result[-3]
print "fc2_weight_grad", result[-2]
print "fc2_bias_grad", result[-1]
loop_num += 1
