from ddls.feeder.plan_maker import PlanMaker from ddls.feeder.record_io import RecordIO, BinaryWrite from ddls.hpps.tensor import Tensor from ddls.feeder.feeder import Feeder import numpy as np ################################################################ record_io = RecordIO('/tmp/sample1', BinaryWrite) record_io.write_header(name=['ad', 'user'], type=[np.float32, np.float32]) ad_tensor = Tensor([300], np.float32) ad_tensor.load_numpy(np.random.rand(300)) print(ad_tensor.asnumpy()) user_tensor = Tensor([400], np.float32) user_tensor.load_numpy(np.random.rand(400)) for x in xrange(1, 1000): record_io.write_sample({'ad': ad_tensor, 'user': user_tensor}) record_io.write_finalize() ##################################################################
from ddls.hpps.kv_table import KVTable, create_kv_table from ddls.hpps.tensor import Tensor import numpy as np zoo_start() array_table = create_array_table(size=100, type=np.float32, solver="default") kv_table = create_kv_table(capacity=1000000, value_len=2, key_type=np.int64, value_type=np.float32) zoo_barrier() data = Tensor(shape=[100], type=np.float32) array_table.get(value=data) print data.asnumpy() grad = Tensor(shape=[100], type=np.float32) grad.load_numpy(np.ones([100], np.float32)) array_table.add(grad=grad) array_table.get(value=data) print data.asnumpy() id = Tensor(shape=[100], type=np.int64) id.load_numpy(np.zeros([100], np.int64)) value = Tensor(shape=[1], type=np.float32) kv_table.get(key=id, value=value) print value.asnumpy() id.load_numpy(np.ones([100], np.int64))
class Embedding(object): """ The embedding model param manager, which is used for managing and synchronizing the variables in sparse embedding. """ def __init__(self, capacity, value_len, kwargs={}, solver='adam'): """ The constructor of Embedding """ self.kv_table = create_kv_table(capacity=capacity, value_len=value_len, key_type=np.int64, value_type=np.float32, kwargs=kwargs, solver=solver) self.wait_get_id = None self.wait_add_id = None self.value = Tensor([1], np.float32) self.grad = Tensor([1], np.float32) def get(self, key, next_key): """ get current key's value and pre-get the next key. Parameters ---------- key: The current iteration id key, which is Tensor instance next_key: The next iteration id key Returns ------- Return value, which is numpy instance. """ assert isinstance(key, Tensor) assert isinstance(next_key, Tensor) if self.wait_get_id is None: self.wait_get_id = self.kv_table.get_async(key=key, value=self.value) self.kv_table.get_async(key=key, value=self.value, wait_id=self.wait_get_id) self.wait_get_id = self.kv_table.get_async(key=next_key, value=self.value) return self.value.asnumpy() def add(self, key, grad, option={}): """ add grad Parameter --------- key: The current iteration id key, which is Tensor instance grad: The current iteration id grad, which is numpy instance """ assert isinstance(key, Tensor) if self.wait_add_id is not None: self.kv_table.wait(self.wait_add_id) self.grad.reshape(grad.shape) self.grad.load_numpy(grad) self.wait_add_id = self.kv_table.add_async(key=key, grad=self.grad, option=option)