def create_kv_table(capacity, value_len, key_type, value_type, solver="", ps_mode="async", kwargs={}): """ create a new kv table Parameters --------- capacity: The key capacity value_len: The value length key_type: The key type value_type: The value type solver: The solver ps_mode: The ps model(sync/async) kwargs: The k/w config, The available key as follows: algo : assign/uniform/gaussian assigned_value : 0.1 mu : 0 sigma : 0.01 min : 0 max : 1 seed : 0 """ keys = [] values = [] for key, value in kwargs.iteritems(): keys.append(c_str(key)) values.append(c_str(value)) handle = ctypes.c_void_p() check_call( LIB.CreateKVTable(c_str(solver), c_str(ps_mode), capacity, value_len, _NP_2_DTYPE[key_type], _NP_2_DTYPE[value_type], len(keys), c_array(ctypes.c_char_p, keys), c_array(ctypes.c_char_p, values), ctypes.byref(handle))) return KVTable(handle)
def add(self, grad, option={}): """ push grads Parameters ---------- tensor: The array-like grads """ keys = [] values = [] for key, value in option.iteritems(): keys.append(c_str(key)) values.append(c_str(value)) check_call( LIB.ArrayTableAdd(self.handle, grad.handle, len(keys), c_array(ctypes.c_char_p, keys), c_array(ctypes.c_char_p, values)))
def store(self, uri): """ store table Parameters ---------- uri: The uri name """ check_call(LIB.TableStore(self.handle, c_str(uri)))
def load(self, uri): """ load table Parameters ---------- uri: The uri name """ check_call(LIB.TableLoad(self.handle, c_str(uri)))
def add(self, key, grad, option={}): """ push grads Parameters ---------- key: the uint32 or uint64 key tensor grad: The param grads """ keys = [] values = [] for key, value in option.iteritems(): keys.append(c_str(key)) values.append(c_str(value)) check_call( LIB.KVTableAdd(self.handle, key.handle, grad.handle, c_array(ctypes.c_char_p, keys), c_array(ctypes.c_char_p, values)))
def get_tensor(self, name): """ Return the tensor according to name """ out = ctypes.c_void_p() check_call( LIB.HPPS_BatchGetTensorFromKey(self.handle, c_str(name), ctypes.byref(out))) return Tensor(handle=out, shape=None, type=None)
def add_async(self, grad, option={}): """ push grads Parameters ---------- tensor: The array-like grads """ keys = [] values = [] for key, value in option.iteritems(): keys.append(c_str(key)) values.append(c_str(value)) id = ctypes.c_int() check_call( LIB.ArrayTableGetAsync(self.handle, grad.handle, ctypes.byref(id), len(keys), c_array(ctypes.c_char_p, keys), c_array(ctypes.c_char_p, values))) return id
def create_array_table(size, type, solver="", ps_mode="sync", kwargs={}): """ create a new array table Parameters --------- size: The array length type: The array data type solver: The user defined solver ps_mode: ps mode(sync/async) kwargs: The k/w config, The available keys as follows:: algo : assign/uniform/gaussian assigned_value : 0.1 mu : 0 sigma : 0.01 min : 0 max : 1 seed : 0 """ keys = [] values = [] for key, value in kwargs.iteritems(): keys.append(c_str(key)) values.append(c_str(value)) handle = ctypes.c_void_p() check_call( LIB.CreateArrayTable(c_str(solver), c_str(ps_mode), size, _NP_2_DTYPE[type], len(keys), c_array(ctypes.c_char_p, keys), c_array(ctypes.c_char_p, values), ctypes.byref(handle))) return ArrayTable(handle)
def zoo_set_log_file(log_file): """ Parameters ---------- log_file: The log file path for HPPS. """ check_call(LIB.HPPS_ZooSetLogFile(c_str(log_file)))
def set_uri(self, files): """ set the sample files for hpps Parameters --------- files: The files in array """ array = [] for file in files: array.append(c_str(file)) check_call( LIB.HPPS_PlanMakerSetURI(self.handle, len(array), c_array(ctypes.c_char_p, array)))
def write_sample(self, tensor_map): """ write one sample Parameters ---------- tensor_map: The name -> tensor map, each sample consists of mutiple tensors """ names = [] tensors = [] for name, tensor in tensor_map.iteritems(): names.append(c_str(name)) tensors.append(tensor.handle) check_call(LIB.HPPS_RecordIOWriteSample(self.handle, len(names), c_array(ctypes.c_char_p, names), c_array(ctypes.c_void_p, tensors)))
def write_header(self, name, type, level=None, is_aux_number=None): """ write header of record Parameters --------- name_type_map: tensor name -> type map """ names = [] types = [] levels = [] is_aux_numbers = [] for index in xrange(len(name)): names.append(c_str(name[index])) types.append(_NP_2_DTYPE[type[index]]) levels.append(0 if level is None else level[index]) is_aux_numbers.append(0 if is_aux_number is None else is_aux_number[index]) check_call(LIB.HPPS_RecordIOWriteHeader(self.handle, len(names), c_array(ctypes.c_char_p, names), c_array(ctypes.c_int, types), c_array(ctypes.c_int, levels), c_array(ctypes.c_int, is_aux_numbers)))
def zoo_start(args='-net_type=mpi -sync=true'): """Start the Zoo, all the Actors will be registered. """ check_call(LIB.HPPS_ZooStart(c_str(args)))
def __init__(self, uri, mode): """ init a new RecordIO instance """ self.handle = ctypes.c_void_p() check_call(LIB.HPPS_RecordIOCreate(c_str(uri), mode, ctypes.byref(self.handle)))