def run():
user_ms = xdl.ModelServer(
"user_graph", user_graph_train, xdl.DataType.float,
xdl.ModelServer.Forward.UniqueCache(xdl.get_task_num()),
xdl.ModelServer.Backward.UniqueCache(xdl.get_task_num()))
xdl.current_env().start_model_server(user_ms)
ad_ms = xdl.ModelServer(
"ad_graph", ad_graph_train, xdl.DataType.float,
xdl.ModelServer.Forward.UniqueCache(xdl.get_task_num()),
xdl.ModelServer.Backward.UniqueCache(xdl.get_task_num()))
xdl.current_env().start_model_server(ad_ms)
batch = reader().read()
user0 = xdl.embedding("user0",
batch["user0"],
xdl.TruncatedNormal(stddev=0.001),
16,
2 * 1024 * 1024,
"sum",
vtype="hash")
user1 = xdl.embedding("user1",
batch["user1"],
xdl.TruncatedNormal(stddev=0.001),
16,
2 * 1024 * 1024,
"sum",
vtype="hash")
ad0 = batch["ad0"]
ad1 = batch["ad1"]
img0 = user_ms(batch["user_img"].ids)
ids0 = xdl.py_func(to_tf_segment_id, [batch["user_img"].segments],
[np.int32])[0]
img1 = ad_ms(batch["ad_img"].ids)
ids1 = xdl.py_func(to_tf_segment_id, [batch["ad_img"].segments],
[np.int32])[0]
label = batch['label']
loss = ams_main(main_model)(user0,
user1,
ad0,
ad1,
label,
ids0,
ids1,
gear_inputs=[img0, img1])
optimizer = xdl.Adam(0.0005).optimize()
run_ops = [loss, optimizer]
sess = xdl.TrainSession([])
while not sess.should_stop():
values = sess.run(run_ops)
if values is not None:
print 'loss: ', values[0]
def train():
images, labels = xdl.py_func(read_train, [],
output_type=[np.float32, np.float32])
images_test, labels_test = xdl.py_func(
read_test, [], output_type=[np.float32, np.float32])
with xdl.model_scope('train'):
loss = model(images, labels)
train_op = xdl.Adagrad(0.5).optimize()
train_sess = xdl.TrainSession()
with xdl.model_scope('test'):
accuracy = eval_model(images_test, labels_test)
eval_sess = xdl.TrainSession()
for _ in range(100):
for _ in range(1000):
train_sess.run(train_op)
print("accuracy %s" % eval_sess.run(accuracy))
def _py_func(self, fn, sparse_cnt=7):
types = []
for _ in range(sparse_cnt):
types.extend([np.int64, np.float32, np.int32])
types.extend([np.float32, np.float32, np.int32])
types.extend([np.int32 for _ in range(5)])
datas = xdl.py_func(fn, [], output_type=types)
sparse_tensors = []
for i in range(sparse_cnt):
sparse_tensors.append(xdl.SparseTensor(
datas[3 * i], datas[3 * i + 1], datas[3 * i + 2]))
return sparse_tensors + datas[sparse_cnt * 3:]
def _py_func(self, fn, sparse_cnt=7):
types = []
for _ in range(sparse_cnt):
types.extend([np.int64, np.float32, np.int32])
types.extend([np.float32, np.float32, np.int32])
types.extend([np.int32 for _ in range(8)])
datas = xdl.py_func(fn, [], output_type=types)
sparse_tensors = []
for i in range(sparse_cnt):
sparse_tensors.append(
xdl.SparseTensor(datas[3 * i], datas[3 * i + 1],
datas[3 * i + 2]))
return sparse_tensors + datas[sparse_cnt * 3:]
def train():
images, labels = xdl.py_func(read_train, [],
output_type=[np.float32, np.float32])
images_test, labels_test = xdl.py_func(
read_test, [], output_type=[np.float32, np.float32])
with xdl.model_scope('train'):
loss = model(images, labels)
train_op = xdl.Adagrad(0.5).optimize()
if xdl.get_task_index() == 0:
ckpt_hook = xdl.CheckpointHook(1000)
train_sess = xdl.TrainSession(hooks=[ckpt_hook])
else:
train_sess = xdl.TrainSession()
with xdl.model_scope('test'):
accuracy = eval_model(images_test, labels_test)
eval_sess = xdl.TrainSession()
for _ in range(100):
for _ in range(1000):
train_sess.run(train_op)
print("accuracy %s" % eval_sess.run(accuracy))
def train():
images, labels = xdl.py_func(read_train, [], output_type=[
np.float32, np.float32])
images_test, labels_test = xdl.py_func(
read_test, [], output_type=[np.float32, np.float32])
with xdl.model_scope('train'):
loss = model(images, labels)
train_op = xdl.Adagrad(0.5).optimize()
if xdl.get_task_index() == 0:
ckpt_hook = xdl.CheckpointHook(1000)
train_sess = xdl.TrainSession(hooks=[ckpt_hook])
else:
train_sess = xdl.TrainSession()
with xdl.model_scope('test'):
accuracy = eval_model(images_test, labels_test)
eval_sess = xdl.TrainSession()
for _ in range(100):
for _ in range(1000):
train_sess.run(train_op)
print("accuracy %s" % eval_sess.run(accuracy))
def run():
user_ms = xdl.ModelServer(
"user_graph", user_graph_train, xdl.DataType.float,
xdl.ModelServer.Forward.UniqueCache(xdl.get_task_num()),
xdl.ModelServer.Backward.UniqueCache(xdl.get_task_num()))
xdl.current_env().start_model_server(user_ms)
ad_ms = xdl.ModelServer(
"ad_graph", ad_graph_train, xdl.DataType.float,
xdl.ModelServer.Forward.UniqueCache(xdl.get_task_num()),
xdl.ModelServer.Backward.UniqueCache(xdl.get_task_num()))
xdl.current_env().start_model_server(ad_ms)
batch = reader().read()
user0 = xdl.embedding("user0", batch["user0"], xdl.TruncatedNormal(stddev=0.001),
16, 2 * 1024 * 1024, "sum", vtype="hash")
user1 = xdl.embedding("user1", batch["user1"], xdl.TruncatedNormal(stddev=0.001),
16, 2 * 1024 * 1024, "sum", vtype="hash")
ad0 = batch["ad0"]
ad1 = batch["ad1"]
img0 = user_ms(batch["user_img"].ids)
ids0 = xdl.py_func(to_tf_segment_id, [batch["user_img"].segments], [np.int32])[0]
img1 = ad_ms(batch["ad_img"].ids)
ids1 = xdl.py_func(to_tf_segment_id, [batch["ad_img"].segments], [np.int32])[0]
label = batch['label']
loss = ams_main(main_model)(user0, user1, ad0, ad1, label, ids0, ids1, gear_inputs=[img0, img1])
optimizer = xdl.Adam(0.0005).optimize()
run_ops = [loss, optimizer]
sess = xdl.TrainSession([])
while not sess.should_stop():
values = sess.run(run_ops)
if values is not None:
print 'loss: ', values[0]
def next_batch(self):
"""
data[0] -> click_seq
data[1] -> itemid
data[2] -> label
:return:
"""
types = []
for _ in range(2): #for click_seqs and itemid
types.extend([np.int32, np.float32, np.int32])
#types.append(np.int32) #for itemid
types.append(np.float32) # for label
types.append(np.int32) # for click seq
types.append(np.int32) # for itemid
datas =xdl.py_func(self.read_and_parse_data, [], output_type=types)
sparse_cnt = 2 #only click_seq and itemid is sparse
sparse_tensors = []
for i in range(sparse_cnt):
sparse_tensors.append(xdl.SparseTensor(
datas[3 * i], datas[3 * i + 1],
datas[3 * i + 2])) # a batch of sparse examples . ids/values/segments_index
return sparse_tensors + datas[sparse_cnt * 3:] #5 emelemts left
def test_py_func(self): op = xdl.py_func(func, [[10, 20, 30], [1, 2, 3]], [np.int64]) rst = xdl.execute(op) self.assertTrue((rst == np.array([21, 42, 63])).all())