from ddls.feeder.plan_maker import PlanMaker
from ddls.feeder.record_io import RecordIO, BinaryWrite
from ddls.hpps.tensor import Tensor
from ddls.feeder.feeder import Feeder
import numpy as np
################################################################
record_io = RecordIO('/tmp/sample1', BinaryWrite)
record_io.write_header(name=['ad', 'user'], type=[np.float32, np.float32])
ad_tensor = Tensor([300], np.float32)
ad_tensor.load_numpy(np.random.rand(300))
print(ad_tensor.asnumpy())
user_tensor = Tensor([400], np.float32)
user_tensor.load_numpy(np.random.rand(400))
for x in xrange(1, 1000):
record_io.write_sample({'ad': ad_tensor, 'user': user_tensor})
record_io.write_finalize()
##################################################################
from ddls.hpps.kv_table import KVTable, create_kv_table
from ddls.hpps.tensor import Tensor
import numpy as np
zoo_start()
array_table = create_array_table(size=100, type=np.float32, solver="default")
kv_table = create_kv_table(capacity=1000000,
value_len=2,
key_type=np.int64,
value_type=np.float32)
zoo_barrier()
data = Tensor(shape=[100], type=np.float32)
array_table.get(value=data)
print data.asnumpy()
grad = Tensor(shape=[100], type=np.float32)
grad.load_numpy(np.ones([100], np.float32))
array_table.add(grad=grad)
array_table.get(value=data)
print data.asnumpy()
id = Tensor(shape=[100], type=np.int64)
id.load_numpy(np.zeros([100], np.int64))
value = Tensor(shape=[1], type=np.float32)
kv_table.get(key=id, value=value)
print value.asnumpy()
id.load_numpy(np.ones([100], np.int64))
class Embedding(object):
""" The embedding model param manager, which is used for managing and
synchronizing the variables in sparse embedding.
"""
def __init__(self, capacity, value_len, kwargs={}, solver='adam'):
""" The constructor of Embedding
"""
self.kv_table = create_kv_table(capacity=capacity,
value_len=value_len,
key_type=np.int64,
value_type=np.float32,
kwargs=kwargs,
solver=solver)
self.wait_get_id = None
self.wait_add_id = None
self.value = Tensor([1], np.float32)
self.grad = Tensor([1], np.float32)
def get(self, key, next_key):
""" get current key's value and pre-get the next key.
Parameters
----------
key: The current iteration id key, which is Tensor instance
next_key: The next iteration id key
Returns
-------
Return value, which is numpy instance.
"""
assert isinstance(key, Tensor)
assert isinstance(next_key, Tensor)
if self.wait_get_id is None:
self.wait_get_id = self.kv_table.get_async(key=key,
value=self.value)
self.kv_table.get_async(key=key,
value=self.value,
wait_id=self.wait_get_id)
self.wait_get_id = self.kv_table.get_async(key=next_key,
value=self.value)
return self.value.asnumpy()
def add(self, key, grad, option={}):
""" add grad
Parameter
---------
key: The current iteration id key, which is Tensor instance
grad: The current iteration id grad, which is numpy instance
"""
assert isinstance(key, Tensor)
if self.wait_add_id is not None:
self.kv_table.wait(self.wait_add_id)
self.grad.reshape(grad.shape)
self.grad.load_numpy(grad)
self.wait_add_id = self.kv_table.add_async(key=key,
grad=self.grad,
option=option)