def update(self):
"""Updates parameters according to the installed optimizer and the gradients computed
in the previous forward-backward batch.
When KVStore is used to update parameters for multi-device or multi-machine training,
a copy of the parameters are stored in KVStore. Note that for `row_sparse` parameters,
this function does update the copy of parameters in KVStore, but doesn't broadcast the
updated parameters to all devices / machines. Please call `prepare` to broadcast
`row_sparse` parameters with the next batch of data.
See Also
----------
:meth:`BaseModule.update`.
"""
assert self.binded and self.params_initialized and self.optimizer_initialized
self._params_dirty = True
if self._update_on_kvstore:
_update_params_on_kvstore(self._exec_group.param_arrays,
self._exec_group.grad_arrays,
self._kvstore,
self._exec_group.param_names)
else:
_update_params(self._exec_group.param_arrays,
self._exec_group.grad_arrays,
updater=self._updater,
num_device=len(self._context),
kvstore=self._kvstore,
param_names=self._exec_group.param_names)
def update(self):
"""Update parameters according to the installed optimizer and the gradients computed
in the previous forward-backward batch.
"""
assert self.binded and self.params_initialized and self.optimizer_initialized
self._params_dirty = True
if self._update_on_kvstore:
try:
_update_params_on_kvstore(self._exec_group.param_arrays,
self._exec_group.grad_arrays,
self._kvstore)
except:
_update_params_on_kvstore(
self._exec_group.param_arrays,
self._exec_group.grad_arrays,
self._kvstore,
param_names=self._exec_group.param_names)
else:
_update_params(self._exec_group.param_arrays,
self._exec_group.grad_arrays,
updater=self._updater,
num_device=len(self._context),
kvstore=self._kvstore)
def update(self):
"""Update parameters according to the installed optimizer and the gradients computed
in the previous forward-backward batch.
"""
assert self.binded and self.params_initialized and self.optimizer_initialized
self._params_dirty = True
if self._update_on_kvstore:
_update_params_on_kvstore(self._exec_group.param_arrays,
self._exec_group.grad_arrays,
self._kvstore)
else:
_update_params(self._exec_group.param_arrays,
self._exec_group.grad_arrays,
updater=self._updater,
num_device=len(self._context),
kvstore=self._kvstore)
def update(self):
"""Update parameters according to the installed optimizer and the gradients computed
in the previous forward-backward batch.
"""
assert self.binded and self.params_initialized and self.optimizer_initialized
self._params_dirty = True
self.logger.debug("param_names = {}" + str(self._param_names))
# FIXME: takes 4 arguments but 3 given. (self._exec_group_param_names)
if self._update_on_kvstore:
_update_params_on_kvstore(self._exec_group.param_arrays,
self._exec_group.grad_arrays,
self._kvstore, self._param_names)
else:
_update_params(self._exec_group.param_arrays,
self._exec_group.grad_arrays,
updater=self._updater,
num_device=len(self._context),
kvstore=self._kvstore)
def _run_sax(data_batch_zoo, marks, executor_manager, eval_metric, updater, ctx, kvstore, acc_hist,
logger=None,
callback=None,
monitor=None,
update_on_kvstore=None,
is_train=False):
if logger is None:
logger = logging
data_targets = [[e.arg_dict[name] for i, e in enumerate(executor_manager.execgrp.train_execs)]
for name in ['c', 'h']]
# for idx in range(len(data_targets[0])):
# print 'data_targets c mean', data_targets[0][idx].asnumpy().mean()
# print 'data_targets h mean', data_targets[1][idx].asnumpy().mean()
# _load_general([c[idx]], [data_targets[0][idx]])
# _load_general([h[idx]], [data_targets[1][idx]])
for i, tg in enumerate(executor_manager.execgrp.data_arrays[1:]):
for j, slice_array in enumerate(tg):
assert isinstance(slice_array, tuple)
array = slice_array[1]
replace = 0 * array
_load_general([replace], [array])
# Start Looping
for t in range(len(marks)):
m = marks[t]
logger.debug('Time Step %d M %d', t, m)
data_batch = data_batch_zoo[t]
assert isinstance(m, int), 'Marks Type Error, %s provided' % type(m)
#load in data
executor_manager.load_data_batch(data_batch)
executor_manager.forward(is_train=is_train)
for num, ex in enumerate(executor_manager.curr_execgrp.train_execs):
out = ex.outputs
ccc = out[1]
hhh = out[2]
_load_general([ccc], [data_targets[0][num]])
_load_general([hhh], [data_targets[1][num]])
if is_train and m > 0:
executor_manager.backward()
logger.debug('Updateing weight...')
#logger.debug('--------before update | grad check-------------')
#for pari in zip(executor_manager.param_names, executor_manager.grad_arrays):
# logger.debug('%s-%f', pari[0], pari[1][0].asnumpy().mean())
if update_on_kvstore:
_update_params_on_kvstore(executor_manager.param_arrays,
executor_manager.grad_arrays,
kvstore)
else:
_update_params(executor_manager.param_arrays,
executor_manager.grad_arrays,
updater=updater,
num_device=len(ctx),
kvstore=kvstore)
logger.debug('Done update')
if monitor is not None:
monitor.toc_print()
if is_train:
eval_metric.reset()
if m == 1:
executor_manager.update_metric(eval_metric, data_batch.label)
name_value = eval_metric.get_name_value()
for name, value in name_value:
acc_hist.append(value)
if is_train:
logger.debug('[%02dth Step] %s:%f', t, name, value)
# Time Step Callback
if callback:
callback = [callback] if not isinstance(callback, list) else callback
for cb in callback:
assert callable(cb), cb
params = (t, m, eval_metric, locals())
cb(params)
# end of all T
return executor_manager, eval_metric, acc_hist
def run(self):
data = self.model._init_iter(self.data, None, is_train=True)
arg_names, param_names, aux_names = \
self.model._init_params(dict(data.provide_data+data.provide_label))
# create kvstore
(kvstore, update_on_kvstore) = _create_kvstore(
self.kv, len(self.ctxs), self.model.arg_params)
self.executor_manager = DataParallelExecutorManager(symbol=self.sym,
ctx=self.ctxs,
train_data=self.data,
param_names=param_names,
arg_names=arg_names,
aux_names=aux_names,
logger=logger)
self.executor_manager.set_params(self.model.arg_params, self.model.aux_params)
if not update_on_kvstore:
updater = get_updater(optimizer)
if kvstore:
_initialize_kvstore(kvstore=kvstore,
param_arrays=self.executor_manager.param_arrays,
arg_params=self.model.arg_params,
param_names=self.executor_manager.param_names,
update_on_kvstore=update_on_kvstore)
if update_on_kvstore:
kvstore.set_optimizer(self.optimizer)
for e in self.before_training_extensions:
e(self)
while True:
self.metric.reset()
nbatch = 0
self.data.reset()
for data_batch in self.data:
self.executor_manager.load_data_batch(data_batch)
self.executor_manager.forward(is_train=True)
self.executor_manager.backward()
if update_on_kvstore:
_update_params_on_kvstore(self.executor_manager.param_arrays,
self.executor_manager.grad_arrays,
kvstore)
else:
_update_params(self.executor_manager.param_arrays,
self.executor_manager.grad_arrays,
updater=updater,
num_device=len(self.model.ctx),
kvstore=kvstore)
# evaluate at end, so out_cpu_array can lazy copy
self.metric.update(data_batch.label, self.executor_manager.cpu_output_arrays)
self.status['iterations'] += 1
self.status['epoch_iterations'] += 1
self.log[self.status['iterations']] = dict(iterations=self.status['iterations'])
self.current_log = self.log[self.status['iterations']]
for e in self.batch_extensions:
e(self)
nbatch += 1
self.status['epochs'] += 1
self.status['epoch_iterations'] = 0
for e in self.epoch_extensions:
e(self)
