def update(self): """Update parameters according to the installed optimizer and the gradients computed in the previous forward-backward batch. """ assert self.binded and self.params_initialized and self.optimizer_initialized self._params_dirty = True if self._update_on_kvstore: _update_params_on_kvstore(self._exec_group.param_arrays, self._exec_group.grad_arrays, self._kvstore) else: _update_params(self._exec_group.param_arrays, self._exec_group.grad_arrays, updater=self._updater, num_device=len(self._context), kvstore=self._kvstore)
def update(self): """Update parameters according to the installed optimizer and the gradients computed in the previous forward-backward batch. """ assert self.binded and self.params_initialized and self.optimizer_initialized self._params_dirty = True if self._update_on_kvstore: _update_params_on_kvstore(self._exec_group.param_arrays, self._exec_group.grad_arrays, self._kvstore, self._exec_group.param_names) else: _update_params(self._exec_group.param_arrays, self._exec_group.grad_arrays, updater=self._updater, num_device=len(self._context), kvstore=self._kvstore, param_names=self._exec_group.param_names)
def _run_sax(data_batch_zoo, marks, executor_manager, eval_metric, updater, ctx, kvstore, acc_hist, logger=None, callback=None, monitor=None, update_on_kvstore=None, is_train=False): if logger is None: logger = logging data_targets = [[e.arg_dict[name] for i, e in enumerate(executor_manager.execgrp.train_execs)] for name in ['c', 'h']] # for idx in range(len(data_targets[0])): # print 'data_targets c mean', data_targets[0][idx].asnumpy().mean() # print 'data_targets h mean', data_targets[1][idx].asnumpy().mean() # _load_general([c[idx]], [data_targets[0][idx]]) # _load_general([h[idx]], [data_targets[1][idx]]) for i, tg in enumerate(executor_manager.execgrp.data_arrays[1:]): for j, slice_array in enumerate(tg): assert isinstance(slice_array, tuple) array = slice_array[1] replace = 0 * array _load_general([replace], [array]) # Start Looping for t in range(len(marks)): m = marks[t] logger.debug('Time Step %d M %d', t, m) data_batch = data_batch_zoo[t] assert isinstance(m, int), 'Marks Type Error, %s provided' % type(m) #load in data executor_manager.load_data_batch(data_batch) executor_manager.forward(is_train=is_train) for num, ex in enumerate(executor_manager.curr_execgrp.train_execs): out = ex.outputs ccc = out[1] hhh = out[2] _load_general([ccc], [data_targets[0][num]]) _load_general([hhh], [data_targets[1][num]]) if is_train and m > 0: executor_manager.backward() logger.debug('Updateing weight...') #logger.debug('--------before update | grad check-------------') #for pari in zip(executor_manager.param_names, executor_manager.grad_arrays): # logger.debug('%s-%f', pari[0], pari[1][0].asnumpy().mean()) if update_on_kvstore: _update_params_on_kvstore(executor_manager.param_arrays, executor_manager.grad_arrays, kvstore) else: _update_params(executor_manager.param_arrays, executor_manager.grad_arrays, updater=updater, num_device=len(ctx), kvstore=kvstore) logger.debug('Done update') if monitor is not None: monitor.toc_print() if is_train: eval_metric.reset() if m == 1: executor_manager.update_metric(eval_metric, data_batch.label) name_value = eval_metric.get_name_value() for name, value in name_value: acc_hist.append(value) if is_train: logger.debug('[%02dth Step] %s:%f', t, name, value) # Time Step Callback if callback: callback = [callback] if not isinstance(callback, list) else callback for cb in callback: assert callable(cb), cb params = (t, m, eval_metric, locals()) cb(params) # end of all T return executor_manager, eval_metric, acc_hist
def run(self): data = self.model._init_iter(self.data, None, is_train=True) arg_names, param_names, aux_names = \ self.model._init_params(dict(data.provide_data+data.provide_label)) # create kvstore (kvstore, update_on_kvstore) = _create_kvstore( self.kv, len(self.ctxs), self.model.arg_params) self.executor_manager = DataParallelExecutorManager(symbol=self.sym, ctx=self.ctxs, train_data=self.data, param_names=param_names, arg_names=arg_names, aux_names=aux_names, logger=logger) self.executor_manager.set_params(self.model.arg_params, self.model.aux_params) if not update_on_kvstore: updater = get_updater(optimizer) if kvstore: _initialize_kvstore(kvstore=kvstore, param_arrays=self.executor_manager.param_arrays, arg_params=self.model.arg_params, param_names=self.executor_manager.param_names, update_on_kvstore=update_on_kvstore) if update_on_kvstore: kvstore.set_optimizer(self.optimizer) for e in self.before_training_extensions: e(self) while True: self.metric.reset() nbatch = 0 self.data.reset() for data_batch in self.data: self.executor_manager.load_data_batch(data_batch) self.executor_manager.forward(is_train=True) self.executor_manager.backward() if update_on_kvstore: _update_params_on_kvstore(self.executor_manager.param_arrays, self.executor_manager.grad_arrays, kvstore) else: _update_params(self.executor_manager.param_arrays, self.executor_manager.grad_arrays, updater=updater, num_device=len(self.model.ctx), kvstore=kvstore) # evaluate at end, so out_cpu_array can lazy copy self.metric.update(data_batch.label, self.executor_manager.cpu_output_arrays) self.status['iterations'] += 1 self.status['epoch_iterations'] += 1 self.log[self.status['iterations']] = dict(iterations=self.status['iterations']) self.current_log = self.log[self.status['iterations']] for e in self.batch_extensions: e(self) nbatch += 1 self.status['epochs'] += 1 self.status['epoch_iterations'] = 0 for e in self.epoch_extensions: e(self)