def test_all(self): var = xdl.Variable(name="w", dtype=DataType.int32, shape=[4], initializer=xdl.Ones()) execute(xdl.variable_registers()) execute(xdl.global_initializers()) op = xdl.ps_pull_op(var_name="w", var_type="index", dtype=DataType.int32) ret = execute(op) self.assertTrue((ret == np.array([1,1,1,1])).all())
def test_all(self): var = xdl.Variable(name="w", dtype=DataType.float, shape=[4, 1], initializer=xdl.Ones()) execute(xdl.variable_registers()) execute(xdl.global_initializers()) op = xdl.ps_sparse_apply_ftrl_op( learning_rate=np.array(0.1, dtype=np.float), learning_rate_power=np.array(-0.5, dtype=np.float), initial_accumulator_value=np.array(0.1, dtype=np.float), l1_reg=np.array(0, dtype=np.float), l2_reg=np.array(0, dtype=np.float), grad=np.array([[1], [2]], dtype=np.float32), indices=np.array([1, 2], dtype=np.int32), var_name="w", var_type="index") execute(op) ret = execute(var.value) self.assertTrue((ret == np.array([[1], [0.6031424], [0.7450533], [1]], dtype=np.float32)).all()) execute(op) ret = execute(var.value) self.assertTrue((ret == np.array([[1], [0.5341358], [0.6747804], [1]], dtype=np.float32)).all())
def start_model_server(self, model_server): if model_server.name() != self._task_name: return model_server = get_model_server_by_name(self._task_name) model_server_adapter = ModelServerAdapter( self._zk_addr, self._model_server.index(self._task_name) + 1, self._task_index, model_server.forward_cache, model_server.backward_cache, model_server.dtype()) model_server_adapter.init() model_server.init_server(model_server_adapter) with model_scope("ams_gear_forward"): while True: try: connect_to_client(self._zk_addr, '') if variable_registers() is not None: execute(variable_registers()) execute(global_initializers()) break except PsError as e: traceback.print_exc() time.sleep(10) model_server.run_server() while True: try: while True: print "RESTARTING CLIENT" if restart_client(): break time.sleep(10) execute_loop_wait() except PsError as e: traceback.print_exc() time.sleep(10)
def test_sync(self): s_enter_op = xdl.ps_synchronize_enter_op( id = np.array(0, dtype=np.int32), worker_count = np.array(10, dtype=np.int32)) finish_op = xdl.worker_report_finish_op(id = np.array(0, dtype=np.int32)) execute(s_enter_op) execute(finish_op)
def convert_ps_variable(var_list, ckpt_dir, output_dir): var_str_list = ','.join(var_list) op = xdl.ps_convert_ckpt_variable_op( variables=_string_to_int8(var_str_list), checkpoint_dir=_string_to_int8(ckpt_dir), output_dir=_string_to_int8(output_dir)) execute(op)
def test_all(self): var = xdl.Variable(name="w", dtype=DataType.float, shape=[2,2], initializer=xdl.Identity(np.array([[1,2],[3,4]], dtype=np.float32))) execute(xdl.variable_registers()) execute(xdl.global_initializers()) ret = execute(var.value) self.assertTrue((ret == np.array([[1,2],[3,4]])).all())
def test_all(self): var = xdl.Variable(name="w", dtype=DataType.float, shape=[4, 1], initializer=xdl.Ones()) execute(xdl.variable_registers()) execute(xdl.global_initializers()) op = xdl.ps_sparse_apply_momentum_op(learning_rate=np.array( 0.5, dtype=np.float), momentum=np.array(0.9, dtype=np.float), grad=np.array([[1], [2]], dtype=np.float32), indices=np.array([1, 2], dtype=np.int32), var_name="w", var_type="index", use_nesterov=False) execute(op) ret = execute(var.value) self.assertTrue((ret == np.array([[1], [0.5], [0], [1]], dtype=np.float32)).all()) execute(op) ret = execute(var.value) self.assertTrue((ret == np.array([[1], [-0.45], [-1.9], [1]], dtype=np.float32)).all())
def test_all(self): var = xdl.Variable(name="w", dtype=DataType.float, shape=[4], initializer=xdl.Ones()) execute(xdl.variable_registers()) execute(xdl.global_initializers()) op = xdl.ps_dense_apply_adam_op(beta1=np.array(0.9, dtype=np.float), beta2=np.array(0.999, dtype=np.float), epsilon=np.array(1e-08, dtype=np.float), learning_rate=np.array(0.1, dtype=np.float), grad=np.array([1, 2, 3, 4], dtype=np.float32), lr_decay=True, var_name="w", var_type="index") execute(op) ret = execute(var.value) self.assertTrue( (ret == np.array([0.90000004, 0.90000004, 0.90000004, 0.90000004], dtype=np.float32)).all()) execute(op) ret = execute(var.value) self.assertTrue( (ret == np.array([0.8000001, 0.8000001, 0.8000001, 0.8], dtype=np.float32)).all())
def test_async(self): as_enter_op = xdl.ps_asynchronize_enter_op( id = np.array(0, dtype=np.int32), staleness = np.array(32, dtype=np.int32), worker_count = np.array(10, dtype=np.int32)) s_leave_op = xdl.worker_report_finish_op(id = np.array(0, dtype=np.int32)) execute(as_enter_op) execute(s_leave_op)
def end(self): execute(xdl.worker_report_finish_op(id = np.array(self._index, dtype=np.int32))) if self._is_chief: finish_num = execute(self._op) while finish_num < math.ceil(self._finish_rate * self._worker_count / 100.0): print("Finish_num is [%ld] limit is [%d], waiting..." % (finish_num, math.ceil(self._finish_rate * self._worker_count / 100.0))) finish_num = execute(self._op) time.sleep(10) print("Finish_num is [%ld] limit is [%d], exiting..." % (finish_num, math.ceil(self._finish_rate * self._worker_count / 100.0)))
def test_not_init(self): var = xdl.Variable(name="w", dtype=DataType.int32, shape=[4], initializer=xdl.Zeros()) execute(var.var_register) op = xdl.ps_is_initialized_op(var_name="w") ret = execute(op) self.assertTrue((ret == np.array([0])).all())
def test_auc(self): labels = np.array([1, 0, 1, 1, 0, 0, 1, 1, 0, 0], dtype=np.float32) predicts = np.array([0.7, 0.2, 0.6, 0.8, 0.1, 0.2, 0.6, 0.9, 0.1, 0.1], dtype=np.float32) res = xdl.auc(predicts, labels) execute(xdl.variable_registers()) execute(xdl.global_initializers()) res = xdl.execute(res) print res
def _save_state(self): state = _string_to_int8_array(self._data_io.serialize_state(), _STATE_SIZE) update_op = xdl.ps_sparse_assign_op(var_name=self._state_var.name, var_type=self._state_var.vtype, ids=np.array([self._worker_index], dtype=np.int32), values=state) execute(update_op)
def _save_offset(self): offset = self._data_io.get_offset() update_op = xdl.ps_sparse_assign_op(var_name=self._offset_var.name, var_type=self._offset_var.vtype, ids=np.array([self._worker_index], dtype=np.int64), values=np.array([offset], dtype=np.int64)) execute(update_op)
def test_all(self): var = xdl.Variable(name="w", dtype=DataType.int32, shape=[4], initializer=xdl.Constant(8.0)) execute(xdl.variable_registers()) execute(xdl.global_initializers()) ret = execute(var.value) self.assertTrue((ret == np.array([8, 8, 8, 8])).all())
def _execute_sparse_assign(self, sparse_ids, sparse_vals): assert len(sparse_ids) == len(self._sparse_assign_ids) assert len(sparse_vals) == len(self._sparse_assign_values) for i in xrange(len(self._sparse_assign_ids)): ids = sparse_ids[i] values = sparse_vals[i] var = self._sparse_assign_vars[i] op = xdl.ps_sparse_assign_op(var_name=var.name, var_type=var.vtype, ids=ids, values=values) execute(op)
def test_all(self): var = xdl.Variable(name="w", dtype=DataType.int32, shape=[4, 2], initializer=xdl.Ones()) execute(xdl.variable_registers()) execute(xdl.global_initializers()) op = xdl.ps_sparse_pull_op(var_name="w", var_type="index", save_ratio=1.0, otype=DataType.int32, ids=np.array([1, 3], dtype=np.int32)) ret = execute(op) self.assertTrue((ret == np.array([[1, 1], [1, 1]])).all())
def _save_state(self): #state = _string_to_int8_array(self._data_io.serialize_state(), _STATE_SIZE) pb = self._data_io.serialize_state() state = _save(pb, _STATE_SIZE) if np.equal(state, self._last_state).all(): print "state equal to last, skip save ..." return self._last_state = state update_op = xdl.ps_sparse_assign_op(var_name=self._state_var.name, var_type=self._state_var.vtype, ids=np.array([self._worker_index], dtype=np.int32), values=state) execute(update_op)
def test_all(self): var = xdl.Variable(name="w", dtype=DataType.float, shape=[4,1], initializer=xdl.Ones()) execute(xdl.variable_registers()) execute(xdl.global_initializers()) op = xdl.ps_sparse_apply_adagrad_op( learning_rate=np.array(0.5, dtype=np.float), initial_accumulator_value=np.array(0.0, dtype=np.float), grad=np.array([[1],[2]], dtype=np.float32), indices=np.array([1,2], dtype=np.int32), var_name="w", var_type="index") execute(op) ret = execute(var.value) self.assertTrue((ret == np.array([[1],[0.5],[0.5],[1]], dtype=np.float32)).all()) execute(op) ret = execute(var.value) self.assertTrue((ret == np.array([[1],[0.14644662],[0.14644662],[1]], dtype=np.float32)).all())
def test_all(self): var = xdl.Variable(name="w", dtype=xdl.DT_FLOAT, shape=[4], initializer=xdl.Ones()) execute(xdl.variable_registers()) execute(xdl.global_initializers()) op = xdl.ps_dense_apply_adagrad_op( learning_rate=np.array(0.5, dtype=np.float), initial_accumulator_value=np.array(0.0, dtype=np.float), grad=np.array([1,2,3,4], dtype=np.float32), var_name="w", var_type="index") execute(op) ret = execute(var.value) self.assertTrue((ret == np.array([0.5,0.5,0.5,0.5])).all()) execute(op) ret = execute(var.value) print(ret) self.assertTrue((ret == np.array([0.14644662,0.14644662,0.14644662,0.14644662], dtype=np.float32)).all())
def barrier(task_id, task_num, name=None): global _BARRIER_NAME_2_ID global _BARRIER_TOKEN if name is None: name = '_default_barrier_' if name not in _BARRIER_NAME_2_ID: _BARRIER_NAME_2_ID[name] = len(_BARRIER_NAME_2_ID) barrier_id = _BARRIER_NAME_2_ID[name] barrier_op = xdl.worker_barrier_v2op(barrier_id=np.array(barrier_id, dtype=np.int32), task_id=np.array(task_id, dtype=np.int32), task_num=np.array(task_num, dtype=np.int32), token=np.array(_BARRIER_TOKEN, dtype=np.int32)) execute(barrier_op)
def create_session(self): state_op = self._state_var.gather( np.array([self._worker_index], dtype=np.int32)) state = execute(state_op) if (state == np.zeros((_STATE_SIZE), dtype=np.int8)).all(): return else: self._data_io.restore_from_state(_int8_array_to_string(state[0]))
def test_all(self): var = xdl.Variable(name="w", dtype=DataType.int32, shape=[4], initializer=xdl.Zeros()) execute(xdl.variable_registers()) execute(xdl.global_initializers()) op = xdl.ps_assign_add_op(var_name="w", var_type="index", delta=np.array([1,2,3,4], dtype=np.int32)) execute(op) ret = execute(var.value) self.assertTrue((ret == np.array([1,2,3,4])).all())
def create_session(self): state_op = self._state_var.gather( np.array([self._worker_index], dtype=np.int32)) state = execute(state_op) self._data_io.shutdown(True) if state.any(): #self._data_io.restore_from_state(_int8_array_to_string(state[0])) pb = _load(state) self._data_io.restore_from_state(pb) self._data_io.startup()
def test_all(self): var = xdl.Variable(name="w", dtype=DataType.float, shape=[4], initializer=xdl.Ones()) execute(xdl.variable_registers()) execute(xdl.global_initializers()) op = xdl.ps_dense_apply_ftrl_op( learning_rate=np.array(0.1, dtype=np.float), learning_rate_power=np.array(-0.5, dtype=np.float), initial_accumulator_value=np.array(0.1, dtype=np.float), l1_reg=np.array(0, dtype=np.float), l2_reg=np.array(0, dtype=np.float), grad=np.array([1,2,3,4], dtype=np.float32), var_name="w", var_type="index") execute(op) ret = execute(var.value) self.assertTrue((ret == np.array([0.6031424,0.7450533,0.7957225,0.8215], dtype=np.float32)).all()) execute(op) ret = execute(var.value) self.assertTrue((ret == np.array([0.5341358,0.6747804,0.7252074,0.75089955], dtype=np.float32)).all())
def run(self, v, run_option=None, run_statistic=None): run_item = [v] cbs = [] for hook in self._hooks: run, cb = hook.run(v) run_item.append(run) cbs.append(cb) results = execute(run_item, run_option, run_statistic) for i in range(len(cbs)): cbs[i](results[i + 1]) return results[0]
def execute_with_retry(ops, retry_cnt=6): ops = list(ops) if isinstance(ops, (list, tuple)) else [ops] i = 0 while i < retry_cnt: try: return execute(ops) except (PsError) as e: i = i + 1 if i == retry_cnt: raise e print('execute fail retry cnt[%d]' % i) time.sleep(30) restart_client()
def test_all(self): var = xdl.Variable(name="w", dtype=DataType.float, shape=[4,2], initializer=xdl.Zeros()) execute(xdl.variable_registers()) execute(xdl.global_initializers()) op = xdl.ps_sparse_assign_op( var_name="w", var_type="index", ids=np.array([1,2], dtype=np.int32), values=np.array([[1,2],[3,4]], dtype=np.float32)) execute(op) ret = execute(var.value) self.assertTrue((ret == np.array([[0,0],[1,2],[3,4],[0,0]], dtype=np.float32)).all())
def run(self, v, run_option=None, run_statistic=None, feed_dict=None): run_item = [v] cbs = [] for hook in self._hooks: run, cb = hook.run(v) run_item.append(run) cbs.append(cb) if feed_dict is None: results = execute(run_item, run_option, run_statistic) else: results = execute_with_feeds(run_item, run_option, run_statistic, feed_dict=feed_dict) for i in range(len(cbs)): cbs[i](results[i + 1]) return results[0]
def test_all(self): var = xdl.Variable(name="w", dtype=DataType.int64, shape=[4, 8], vtype=VarType.Hash, initializer=xdl.Ones()) execute(xdl.variable_registers()) execute(xdl.global_initializers()) mark_op = xdl.ps_mark_op(var_name="w", ids=np.array([[10, 10], [10, 10], [12, 12]], dtype=np.int64), pattern="g", i=12) execute(mark_op) filter_op = xdl.ps_filter_op(var_name="w", pattern="i==g", i=12, d=0.5) execute(filter_op)