def _create_optimizer(name, solver, network, dataset, weight_decay, lr_decay,
lr_decay_interval, update_interval):
o = nnabla_pb2.Optimizer()
o.name = name
o.network_name = network.name
o.dataset_name = dataset.name
o.solver.type = re.sub(r'(|Cuda)$', '', str(solver.name))
if o.solver.type == 'Adadelta':
o.solver.adadelta_param.lr = solver.info['lr']
o.solver.adadelta_param.decay = solver.info['decay']
o.solver.adadelta_param.eps = solver.info['eps']
elif o.solver.type == 'Adagrad':
o.solver.adagrad_param.lr = solver.info['lr']
o.solver.adagrad_param.eps = solver.info['eps']
elif o.solver.type == 'Adam':
o.solver.adam_param.alpha = solver.info['alpha']
o.solver.adam_param.beta1 = solver.info['beta1']
o.solver.adam_param.beta2 = solver.info['beta2']
o.solver.adam_param.eps = solver.info['eps']
elif o.solver.type == 'Adamax':
o.solver.adamax_param.alpha = solver.info['alpha']
o.solver.adamax_param.beta1 = solver.info['beta1']
o.solver.adamax_param.beta2 = solver.info['beta2']
o.solver.adamax_param.eps = solver.info['eps']
elif o.solver.type == 'Momentum':
o.solver.momentum_param.lr = solver.info['lr']
o.solver.momentum_param.momentum = solver.info['momentum']
elif o.solver.type == 'Nesterov':
o.solver.nesterov_param.lr = solver.info['lr']
o.solver.nesterov_param.momentum = solver.info['momentum']
elif o.solver.type == 'RMSprop':
o.solver.rmsprop_param.lr = solver.info['lr']
o.solver.rmsprop_param.decay = solver.info['decay']
o.solver.rmsprop_param.eps = solver.info['eps']
elif o.solver.type == 'Sgd':
o.solver.sgd_param.lr = solver.info['lr']
inputs, outputs, params = _get_net_variables(network)
o.solver.weight_decay = weight_decay
o.solver.lr_decay = lr_decay
o.solver.lr_decay_interval = lr_decay_interval
o.update_interval = update_interval
for n, inp in enumerate(inputs):
d = o.data_variable.add()
d.variable_name = inp
d.data_name = dataset.variable[n]
for out in outputs:
d = o.loss_variable.add()
d.variable_name = out
for param in params:
d = o.parameter_variable.add()
d.variable_name = param
d.learning_rate_multiplier = 1.0
return o
def _create_optimizer(ctx, opti_d):
o = nnabla_pb2.Optimizer()
dataset = None
datasets = ctx.datasets
name = opti_d['name']
solver = opti_d['solver']
network = ctx.networks[opti_d['network']]
dataset_names = opti_d['dataset']
weight_decay = opti_d['weight_decay']
lr_decay = opti_d['lr_decay']
lr_decay_interval = opti_d['lr_decay_interval']
update_interval = opti_d['update_interval']
o.name = name
o.network_name = network.name
proto_network = ctx.proto_graphs[opti_d['network']].default_graph()
# Allow a list or tuple or a string for dataset names.
if isinstance(dataset_names, tuple):
dataset_names = list(dataset_names)
if isinstance(dataset_names, list):
for dataset_name in dataset_names:
if dataset_name in datasets:
o.dataset_name.append(dataset_name)
dataset = datasets[dataset_name]
else:
raise ValueError(
"Invalid dataset_name is found in optimizer: {}".format(
dataset_name))
elif isinstance(dataset_names, str):
dataset_name = dataset_names
if dataset_name in datasets:
o.dataset_name.append(dataset_name)
dataset = datasets[dataset_name]
if dataset is None:
raise ValueError("Dataset is not defined in optimizer.")
o.solver.type = re.sub(r'(|Cuda)$', '', str(solver.name))
if o.solver.type == 'Adadelta':
o.solver.adadelta_param.lr = solver.info['lr']
o.solver.adadelta_param.decay = solver.info['decay']
o.solver.adadelta_param.eps = solver.info['eps']
elif o.solver.type == 'Adagrad':
o.solver.adagrad_param.lr = solver.info['lr']
o.solver.adagrad_param.eps = solver.info['eps']
elif o.solver.type == 'AdaBelief':
o.solver.adabelief_param.alpha = solver.info['alpha']
o.solver.adabelief_param.beta1 = solver.info['beta1']
o.solver.adabelief_param.beta2 = solver.info['beta2']
o.solver.adabelief_param.eps = solver.info['eps']
o.solver.adabelief_param.wd = solver.info['wd']
o.solver.adabelief_param.amsgrad = solver.info['amsgrad']
o.solver.adabelief_param.weight_decouple = solver.info[
'weight_decouple']
o.solver.adabelief_param.fixed_decay = solver.info['fixed_decay']
o.solver.adabelief_param.rectify = solver.info['rectify']
elif o.solver.type == 'Adam':
o.solver.adam_param.alpha = solver.info['alpha']
o.solver.adam_param.beta1 = solver.info['beta1']
o.solver.adam_param.beta2 = solver.info['beta2']
o.solver.adam_param.eps = solver.info['eps']
elif o.solver.type == 'Adamax':
o.solver.adamax_param.alpha = solver.info['alpha']
o.solver.adamax_param.beta1 = solver.info['beta1']
o.solver.adamax_param.beta2 = solver.info['beta2']
o.solver.adamax_param.eps = solver.info['eps']
elif o.solver.type == 'Momentum':
o.solver.momentum_param.lr = solver.info['lr']
o.solver.momentum_param.momentum = solver.info['momentum']
elif o.solver.type == 'Nesterov':
o.solver.nesterov_param.lr = solver.info['lr']
o.solver.nesterov_param.momentum = solver.info['momentum']
elif o.solver.type == 'RMSprop':
o.solver.rmsprop_param.lr = solver.info['lr']
o.solver.rmsprop_param.decay = solver.info['decay']
o.solver.rmsprop_param.eps = solver.info['eps']
elif o.solver.type == 'RMSpropGraves':
o.solver.rmsprop_graves_param.lr = solver.info['lr']
o.solver.rmsprop_graves_param.decay = solver.info['decay']
o.solver.rmsprop_graves_param.momentum = solver.info['momentum']
o.solver.rmsprop_graves_param.eps = solver.info['eps']
elif o.solver.type == 'Sgd':
o.solver.sgd_param.lr = solver.info['lr']
o.solver.weight_decay = weight_decay
o.solver.lr_decay = lr_decay
o.solver.lr_decay_interval = lr_decay_interval
o.update_interval = update_interval
for var_name, data_name in opti_d.get('data_variables', {}).items():
d = o.data_variable.add()
d.variable_name = var_name
d.data_name = data_name
for loss_name in opti_d.get('loss_variables', proto_network.outputs):
d = o.loss_variable.add()
d.variable_name = loss_name
for param in proto_network.parameters.keys():
d = o.parameter_variable.add()
d.variable_name = param
d.learning_rate_multiplier = 1.0
for g_var in opti_d.get('generator_variables', []):
d = o.generator_variable.add()
d.variable_name = g_var
d.type = 'Constant'
d.multiplier = 0
solver.set_states_to_protobuf(o)
return o
def _create_optimizer_lite(opti):
proto_o = nnabla_pb2.Optimizer()
proto_o.name = opti.name
proto_o.solver.type = re.sub(r'(|Cuda)$', '', str(opti.solver.name))
opti.solver.set_states_to_protobuf(proto_o)
return proto_o
def _create_optimizer(datasets, name, solver, network, dataset_names,
weight_decay, lr_decay, lr_decay_interval,
update_interval):
o = nnabla_pb2.Optimizer()
o.name = name
o.network_name = network.name
dataset = None
# Allow a list or tuple or a string for dataset names.
if isinstance(dataset_names, tuple):
dataset_names = list(dataset_names)
if isinstance(dataset_names, list):
for dataset_name in dataset_names:
if dataset_name in datasets:
o.dataset_name.append(dataset_name)
dataset = datasets[dataset_name]
else:
raise ValueError(
"Invalid dataset_name is found in optimizer: {}".format(
dataset_name))
elif isinstance(dataset_names, str):
dataset_name = dataset_names
if dataset_name in datasets:
o.dataset_name.append(dataset_name)
dataset = datasets[dataset_name]
if dataset is None:
raise ValueError("Dataset is not defined in optimizer.")
o.solver.type = re.sub(r'(|Cuda)$', '', str(solver.name))
if o.solver.type == 'Adadelta':
o.solver.adadelta_param.lr = solver.info['lr']
o.solver.adadelta_param.decay = solver.info['decay']
o.solver.adadelta_param.eps = solver.info['eps']
elif o.solver.type == 'Adagrad':
o.solver.adagrad_param.lr = solver.info['lr']
o.solver.adagrad_param.eps = solver.info['eps']
elif o.solver.type == 'Adam':
o.solver.adam_param.alpha = solver.info['alpha']
o.solver.adam_param.beta1 = solver.info['beta1']
o.solver.adam_param.beta2 = solver.info['beta2']
o.solver.adam_param.eps = solver.info['eps']
elif o.solver.type == 'Adamax':
o.solver.adamax_param.alpha = solver.info['alpha']
o.solver.adamax_param.beta1 = solver.info['beta1']
o.solver.adamax_param.beta2 = solver.info['beta2']
o.solver.adamax_param.eps = solver.info['eps']
elif o.solver.type == 'Momentum':
o.solver.momentum_param.lr = solver.info['lr']
o.solver.momentum_param.momentum = solver.info['momentum']
elif o.solver.type == 'Nesterov':
o.solver.nesterov_param.lr = solver.info['lr']
o.solver.nesterov_param.momentum = solver.info['momentum']
elif o.solver.type == 'RMSprop':
o.solver.rmsprop_param.lr = solver.info['lr']
o.solver.rmsprop_param.decay = solver.info['decay']
o.solver.rmsprop_param.eps = solver.info['eps']
elif o.solver.type == 'RMSpropGraves':
o.solver.rmsprop_graves_param.lr = solver.info['lr']
o.solver.rmsprop_graves_param.decay = solver.info['decay']
o.solver.rmsprop_graves_param.momentum = solver.info['momentum']
o.solver.rmsprop_graves_param.eps = solver.info['eps']
elif o.solver.type == 'Sgd':
o.solver.sgd_param.lr = solver.info['lr']
inputs, outputs, params = _get_net_variables(network)
o.solver.weight_decay = weight_decay
o.solver.lr_decay = lr_decay
o.solver.lr_decay_interval = lr_decay_interval
o.update_interval = update_interval
for n, inp in enumerate(inputs):
d = o.data_variable.add()
d.variable_name = inp
d.data_name = dataset.variable[n]
for out in outputs:
d = o.loss_variable.add()
d.variable_name = out
for param in params:
d = o.parameter_variable.add()
d.variable_name = param
d.learning_rate_multiplier = 1.0
solver.set_states_to_protobuf(o)
return o