def mock_embedding(name1, name2): ids = xdl.convert_to_tensor( np.array([[0, 0], [0, 1], [0, 2]], dtype=np.int64)) values = xdl.convert_to_tensor(np.array([1.0, 2.0, 3.0], dtype=np.float32)) segments = xdl.convert_to_tensor(np.array([3], dtype=np.int32)) sparse = xdl.SparseTensor(ids, values, segments) emb = xdl.embedding(name1, sparse, xdl.Ones(), embed_dim, 16, 'sum', vtype='hash') emb.set_shape((1, 3)) ids2 = xdl.convert_to_tensor( np.array([[0, 1], [0, 2], [1, 1]], dtype=np.int64)) values2 = xdl.convert_to_tensor(np.array([1.0, 2.0, 3.0], dtype=np.float32)) segments2 = xdl.convert_to_tensor(np.array([3], dtype=np.int32)) sparse2 = xdl.SparseTensor(ids2, values2, segments2) emb2 = xdl.embedding(name2, sparse2, xdl.Ones(), embed_dim, 16, 'sum', vtype='hash') emb2.set_shape((1, 3)) return [emb, emb2]
def test_all(self): var = xdl.Variable(name="w", dtype=DataType.int32, shape=[4], initializer=xdl.Ones()) execute(xdl.variable_registers()) execute(xdl.global_initializers()) op = xdl.ps_pull_op(var_name="w", var_type="index", dtype=DataType.int32) ret = execute(op) self.assertTrue((ret == np.array([1,1,1,1])).all())
def test_all(self): var = xdl.Variable(name="w", dtype=DataType.float, shape=[4, 1], initializer=xdl.Ones()) execute(xdl.variable_registers()) execute(xdl.global_initializers()) op = xdl.ps_sparse_apply_ftrl_op( learning_rate=np.array(0.1, dtype=np.float), learning_rate_power=np.array(-0.5, dtype=np.float), initial_accumulator_value=np.array(0.1, dtype=np.float), l1_reg=np.array(0, dtype=np.float), l2_reg=np.array(0, dtype=np.float), grad=np.array([[1], [2]], dtype=np.float32), indices=np.array([1, 2], dtype=np.int32), var_name="w", var_type="index") execute(op) ret = execute(var.value) self.assertTrue((ret == np.array([[1], [0.6031424], [0.7450533], [1]], dtype=np.float32)).all()) execute(op) ret = execute(var.value) self.assertTrue((ret == np.array([[1], [0.5341358], [0.6747804], [1]], dtype=np.float32)).all())
def test_all(self): var = xdl.Variable(name="w", dtype=DataType.float, shape=[4, 1], initializer=xdl.Ones()) execute(xdl.variable_registers()) execute(xdl.global_initializers()) op = xdl.ps_sparse_apply_momentum_op(learning_rate=np.array( 0.5, dtype=np.float), momentum=np.array(0.9, dtype=np.float), grad=np.array([[1], [2]], dtype=np.float32), indices=np.array([1, 2], dtype=np.int32), var_name="w", var_type="index", use_nesterov=False) execute(op) ret = execute(var.value) self.assertTrue((ret == np.array([[1], [0.5], [0], [1]], dtype=np.float32)).all()) execute(op) ret = execute(var.value) self.assertTrue((ret == np.array([[1], [-0.45], [-1.9], [1]], dtype=np.float32)).all())
def test_all(self): var = xdl.Variable(name="w", dtype=DataType.float, shape=[4], initializer=xdl.Ones()) execute(xdl.variable_registers()) execute(xdl.global_initializers()) op = xdl.ps_dense_apply_adam_op(beta1=np.array(0.9, dtype=np.float), beta2=np.array(0.999, dtype=np.float), epsilon=np.array(1e-08, dtype=np.float), learning_rate=np.array(0.1, dtype=np.float), grad=np.array([1, 2, 3, 4], dtype=np.float32), lr_decay=True, var_name="w", var_type="index") execute(op) ret = execute(var.value) self.assertTrue( (ret == np.array([0.90000004, 0.90000004, 0.90000004, 0.90000004], dtype=np.float32)).all()) execute(op) ret = execute(var.value) self.assertTrue( (ret == np.array([0.8000001, 0.8000001, 0.8000001, 0.8], dtype=np.float32)).all())
def eval_input_fn(): dense = xdl.mock_dense_op(shape=[1, 16], value=0.01) labels = xdl.mock_dense_op(shape=[1, 1], value=1.0) ids, values, segments = xdl.mock_sparse_op(dense_shape=[1, 16]) sparse = xdl.SparseTensor(ids, values, segments) emb = xdl.embedding("sparse", sparse, xdl.Ones(), 1, 16, 'sum') dense.set_shape([None, 16]) labels.set_shape([None, 1]) return [dense, emb], labels
def input_fn(): dense = xdl.mock_dense_op(shape=[1, 16], value=0.01) indicator = xdl.mock_dense_op(shape=[5], value=0.0) labels = xdl.mock_dense_op(shape=[5, 1], value=1.0) ids, values, segments = xdl.mock_sparse_op(dense_shape=[1, 16]) sparse = xdl.SparseTensor(ids, values, segments) sparse.set_shape([1, 16]) emb = xdl.embedding("sparse", sparse, xdl.Ones(), 1, 16, 'sum') dense.set_shape([1, 16]) indicator.set_shape([5]) labels.set_shape([5, 1]) return [dense, emb, indicator], labels
def main(): dense = xdl.mock_dense_op(shape=[1, 16], value=0.01, name_="dense") gear = xdl.mock_dense_op(shape=[1, 1], value=0.01, name_="gear") labels = xdl.mock_dense_op(shape=[1, 1], value=1.0, name_="label") ids, values, segments = xdl.mock_sparse_op(dense_shape=[1, 16], name_="wide") sparse = xdl.SparseTensor(ids, values, segments) emb = xdl.embedding("sparse", sparse, xdl.Ones(), 1, 16, 'sum') gear.set_shape([None, 1]) dense.set_shape([None, 16]) labels.set_shape([None, 1]) with xdl.model_scope("ams_main"): loss = ams_main(main_model)(dense, emb, labels, gear_inputs=[gear]) sess = xdl.TrainSession() return sess.run(xdl.get_collection("gear_grad"))
def test_all(self): var = xdl.Variable(name="w", dtype=DataType.int32, shape=[4, 2], initializer=xdl.Ones()) execute(xdl.variable_registers()) execute(xdl.global_initializers()) op = xdl.ps_sparse_pull_op(var_name="w", var_type="index", save_ratio=1.0, otype=DataType.int32, ids=np.array([1, 3], dtype=np.int32)) ret = execute(op) self.assertTrue((ret == np.array([[1, 1], [1, 1]])).all())
def test_all(self): var = xdl.Variable(name="w", dtype=DataType.float, shape=[4], initializer=xdl.Ones()) execute(xdl.variable_registers()) execute(xdl.global_initializers()) op = xdl.ps_apply_moving_average_op(var_name="w", moment=0.8, value=np.array([1, 2, 3, 3], dtype=np.float32)) execute(op) ret = execute(var.value) self.assertTrue((ret == np.array([1., 1.2, 1.4, 1.4], dtype=np.float32)).all())
def test_all(self): var = xdl.Variable(name="w", dtype=DataType.int64, shape=[4, 8], vtype=VarType.Hash, initializer=xdl.Ones()) execute(xdl.variable_registers()) execute(xdl.global_initializers()) mark_op = xdl.ps_mark_op(var_name="w", ids=np.array([[10, 10], [10, 10], [12, 12]], dtype=np.int64), pattern="g", i=12) execute(mark_op) filter_op = xdl.ps_filter_op(var_name="w", pattern="i==g", i=12, d=0.5) execute(filter_op)
def test_all(self): var = xdl.Variable(name="w", dtype=DataType.float, shape=[4,1], initializer=xdl.Ones()) execute(xdl.variable_registers()) execute(xdl.global_initializers()) op = xdl.ps_sparse_apply_adagrad_op( learning_rate=np.array(0.5, dtype=np.float), initial_accumulator_value=np.array(0.0, dtype=np.float), grad=np.array([[1],[2]], dtype=np.float32), indices=np.array([1,2], dtype=np.int32), var_name="w", var_type="index") execute(op) ret = execute(var.value) self.assertTrue((ret == np.array([[1],[0.5],[0.5],[1]], dtype=np.float32)).all()) execute(op) ret = execute(var.value) self.assertTrue((ret == np.array([[1],[0.14644662],[0.14644662],[1]], dtype=np.float32)).all())
def test_all(self): var = xdl.Variable(name="w", dtype=xdl.DT_FLOAT, shape=[4], initializer=xdl.Ones()) execute(xdl.variable_registers()) execute(xdl.global_initializers()) op = xdl.ps_dense_apply_adagrad_op( learning_rate=np.array(0.5, dtype=np.float), initial_accumulator_value=np.array(0.0, dtype=np.float), grad=np.array([1,2,3,4], dtype=np.float32), var_name="w", var_type="index") execute(op) ret = execute(var.value) self.assertTrue((ret == np.array([0.5,0.5,0.5,0.5])).all()) execute(op) ret = execute(var.value) print(ret) self.assertTrue((ret == np.array([0.14644662,0.14644662,0.14644662,0.14644662], dtype=np.float32)).all())
def test_all(self): dense = xdl.mock_dense_op(shape=[1, 16], value=0.01, name_="dense") labels = xdl.mock_dense_op(shape=[1, 1], value=1.0, name_="label") ids = xdl.convert_to_tensor( np.array([[0, 0], [0, 1], [0, 2]], dtype=np.int64)) values = xdl.convert_to_tensor( np.array([1.0, 2.0, 3.0], dtype=np.float32)) segments = xdl.convert_to_tensor(np.array([3], dtype=np.int32)) sparse = xdl.SparseTensor(ids, values, segments) emb = xdl.embedding("sparse", sparse, xdl.Ones(), 1, 16, 'sum', vtype='hash') loss = model(dense, emb, labels) train_op = xdl.SGD(0.5).optimize() sess = xdl.TrainSession() _, l, g = sess.run( [train_op, loss, xdl.get_sparse_grads('sparse').grad]) self.assertTrue((l == np.array(0.0024364376, dtype=np.float32)).all()) self.assertTrue( (g == np.array([[-0.002433472], [-0.004866944], [-0.007300416]], dtype=np.float32)).all()) sparse_var = xdl.get_variable_by_name('sparse') weights = sess.run( sparse_var.gather( np.array([[0, 0], [0, 1], [0, 2]], dtype=np.int64))) self.assertTrue( (weights == np.array([[1.0012168], [1.0024334], [1.0036502]], dtype=np.float32)).all()) _, l, g = sess.run( [train_op, loss, xdl.get_sparse_grads('sparse').grad]) self.assertTrue((l == np.array(0.002395329, dtype=np.float32)).all()) self.assertTrue( (g == np.array([[-0.0023924622], [-0.0047849244], [-0.0071773864]], dtype=np.float32)).all()) weights = sess.run( sparse_var.gather( np.array([[0, 0], [0, 1], [0, 2]], dtype=np.int64))) self.assertTrue( (weights == np.array([[1.002413], [1.0048258], [1.0072389]], dtype=np.float32)).all())
def test_all(self): var = xdl.Variable(name="w", dtype=DataType.float, shape=[4], initializer=xdl.Ones()) execute(xdl.variable_registers()) execute(xdl.global_initializers()) op = xdl.ps_dense_apply_ftrl_op( learning_rate=np.array(0.1, dtype=np.float), learning_rate_power=np.array(-0.5, dtype=np.float), initial_accumulator_value=np.array(0.1, dtype=np.float), l1_reg=np.array(0, dtype=np.float), l2_reg=np.array(0, dtype=np.float), grad=np.array([1,2,3,4], dtype=np.float32), var_name="w", var_type="index") execute(op) ret = execute(var.value) self.assertTrue((ret == np.array([0.6031424,0.7450533,0.7957225,0.8215], dtype=np.float32)).all()) execute(op) ret = execute(var.value) self.assertTrue((ret == np.array([0.5341358,0.6747804,0.7252074,0.75089955], dtype=np.float32)).all())
def main(): dense = xdl.mock_dense_op(shape=[1, 16], value=0.01, name_="dense") labels = xdl.mock_dense_op(shape=[1, 1], value=1.0, name_="label") ids = xdl.convert_to_tensor( np.array([[0, 0], [0, 1], [0, 2]], dtype=np.int64)) values = xdl.convert_to_tensor(np.array([1.0, 2.0, 3.0], dtype=np.float32)) segments = xdl.convert_to_tensor(np.array([3], dtype=np.int32)) sparse = xdl.SparseTensor(ids, values, segments) emb = xdl.embedding("sparse", sparse, xdl.Ones(), 1, 16, 'sum', vtype='hash') loss = model(dense, emb, labels) train_op = xdl.SGD(0.5).optimize() sess = xdl.TrainSession() loss, gradients = sess.run([loss, xdl.get_sparse_grads('sparse').grad]) return loss, gradients
def test_all(self): var = xdl.Variable(name="w", dtype=DataType.float, shape=[4], initializer=xdl.Ones()) execute(xdl.variable_registers()) execute(xdl.global_initializers()) op = xdl.ps_dense_apply_momentum_op(learning_rate=np.array( 0.5, dtype=np.float), momentum=np.array(0.9, dtype=np.float), grad=np.array([1, 2, 3, 4], dtype=np.float32), var_name="w", var_type="index", use_nesterov=False) execute(op) ret = execute(var.value) self.assertTrue((ret == np.array([0.5, 0, -0.5, -1], dtype=np.float32)).all()) execute(op) ret = execute(var.value) self.assertTrue((ret == np.array([-0.45, -1.9, -3.35, -4.8], dtype=np.float32)).all())
def train(): batch = data_io.read() print batch embs = list() for i in range(1, embs_len + 1): name = "item_%d" % i emb = xdl.embedding(name, batch[name], xdl.Ones(), 1, 1000, 'sum', vtype='hash') embs.append(emb) print "emb =", name, ", shape =", emb.shape print "origin batch[label].shape =", batch["label"].shape loss, prop, label, indicator, din, dout, fc1_weight, fc1_bias, fc2_weight, fc2_bias = model( embs, batch["label"], 4, 7) train_op = xdl.SGD(0.5).optimize() item1_grad = xdl.get_gradient('item_1') item2_grad = xdl.get_gradient('item_2') item3_grad = xdl.get_gradient('item_3') item4_grad = xdl.get_gradient('item_4') fc1_weight_grad = xdl.get_gradient('fc1_weight') fc1_bias_grad = xdl.get_gradient('fc1_bias') fc2_weight_grad = xdl.get_gradient('fc2_weight') fc2_bias_grad = xdl.get_gradient('fc2_bias') sess = xdl.TrainSession() loop_num = 0 while not sess.should_stop(): if loop_num == 5: break print "\n>>>>>>>>>>>> loop_num = %d" % loop_num result = sess.run([train_op, loss, prop, batch['label'], label, indicator, din, dout, \ batch['item_1'].ids, batch['item_1'].segments, batch['item_1'].values, \ batch['item_2'].ids, batch['item_2'].segments, batch['item_2'].values, \ batch['item_3'].ids, batch['item_3'].segments, batch['item_3'].values, \ batch['item_4'].ids, batch['item_4'].segments, batch['item_4'].values, \ item1_grad, item2_grad, item3_grad, item4_grad, \ fc1_weight, fc1_bias, fc1_weight_grad, fc1_bias_grad, \ fc2_weight, fc2_bias, fc2_weight_grad, fc2_bias_grad]) if result is None: break print "loss:", result[-31] print "prop:", result[-30] print "origin label:", result[-29] print "label:", result[-28] print "indicator:", result[-27] print "din:", result[-26] print "dout:", result[-25] print "item_1: ids=", result[-24], "\n segments=", result[ -23], "\n values=", result[-22] print "item_2: ids=", result[-21], "\n segments=", result[ -20], "\n values=", result[-19] print "item_3: ids=", result[-18], "\n segments=", result[ -17], "\n values=", result[-16] print "item_4: ids=", result[-15], "\n segments=", result[ -14], "\n values=", result[-13] print "item1_grad", result[-12] print "item2_grad", result[-11] print "item1_grad", result[-10] print "item2_grad", result[-9] print "fc1_weight", result[-8] print "fc1_bias", result[-7] print "fc1_weight_grad", result[-6] print "fc1_bias_grad", result[-5] print "fc2_weight", result[-4] print "fc2_bias", result[-3] print "fc2_weight_grad", result[-2] print "fc2_bias_grad", result[-1] loop_num += 1