def _create_optimizer(name, solver, network, dataset, weight_decay, lr_decay, lr_decay_interval, update_interval): o = nnabla_pb2.Optimizer() o.name = name o.network_name = network.name o.dataset_name = dataset.name o.solver.type = re.sub(r'(|Cuda)$', '', str(solver.name)) if o.solver.type == 'Adadelta': o.solver.adadelta_param.lr = solver.info['lr'] o.solver.adadelta_param.decay = solver.info['decay'] o.solver.adadelta_param.eps = solver.info['eps'] elif o.solver.type == 'Adagrad': o.solver.adagrad_param.lr = solver.info['lr'] o.solver.adagrad_param.eps = solver.info['eps'] elif o.solver.type == 'Adam': o.solver.adam_param.alpha = solver.info['alpha'] o.solver.adam_param.beta1 = solver.info['beta1'] o.solver.adam_param.beta2 = solver.info['beta2'] o.solver.adam_param.eps = solver.info['eps'] elif o.solver.type == 'Adamax': o.solver.adamax_param.alpha = solver.info['alpha'] o.solver.adamax_param.beta1 = solver.info['beta1'] o.solver.adamax_param.beta2 = solver.info['beta2'] o.solver.adamax_param.eps = solver.info['eps'] elif o.solver.type == 'Momentum': o.solver.momentum_param.lr = solver.info['lr'] o.solver.momentum_param.momentum = solver.info['momentum'] elif o.solver.type == 'Nesterov': o.solver.nesterov_param.lr = solver.info['lr'] o.solver.nesterov_param.momentum = solver.info['momentum'] elif o.solver.type == 'RMSprop': o.solver.rmsprop_param.lr = solver.info['lr'] o.solver.rmsprop_param.decay = solver.info['decay'] o.solver.rmsprop_param.eps = solver.info['eps'] elif o.solver.type == 'Sgd': o.solver.sgd_param.lr = solver.info['lr'] inputs, outputs, params = _get_net_variables(network) o.solver.weight_decay = weight_decay o.solver.lr_decay = lr_decay o.solver.lr_decay_interval = lr_decay_interval o.update_interval = update_interval for n, inp in enumerate(inputs): d = o.data_variable.add() d.variable_name = inp d.data_name = dataset.variable[n] for out in outputs: d = o.loss_variable.add() d.variable_name = out for param in params: d = o.parameter_variable.add() d.variable_name = param d.learning_rate_multiplier = 1.0 return o
def _create_optimizer(ctx, opti_d): o = nnabla_pb2.Optimizer() dataset = None datasets = ctx.datasets name = opti_d['name'] solver = opti_d['solver'] network = ctx.networks[opti_d['network']] dataset_names = opti_d['dataset'] weight_decay = opti_d['weight_decay'] lr_decay = opti_d['lr_decay'] lr_decay_interval = opti_d['lr_decay_interval'] update_interval = opti_d['update_interval'] o.name = name o.network_name = network.name proto_network = ctx.proto_graphs[opti_d['network']].default_graph() # Allow a list or tuple or a string for dataset names. if isinstance(dataset_names, tuple): dataset_names = list(dataset_names) if isinstance(dataset_names, list): for dataset_name in dataset_names: if dataset_name in datasets: o.dataset_name.append(dataset_name) dataset = datasets[dataset_name] else: raise ValueError( "Invalid dataset_name is found in optimizer: {}".format( dataset_name)) elif isinstance(dataset_names, str): dataset_name = dataset_names if dataset_name in datasets: o.dataset_name.append(dataset_name) dataset = datasets[dataset_name] if dataset is None: raise ValueError("Dataset is not defined in optimizer.") o.solver.type = re.sub(r'(|Cuda)$', '', str(solver.name)) if o.solver.type == 'Adadelta': o.solver.adadelta_param.lr = solver.info['lr'] o.solver.adadelta_param.decay = solver.info['decay'] o.solver.adadelta_param.eps = solver.info['eps'] elif o.solver.type == 'Adagrad': o.solver.adagrad_param.lr = solver.info['lr'] o.solver.adagrad_param.eps = solver.info['eps'] elif o.solver.type == 'AdaBelief': o.solver.adabelief_param.alpha = solver.info['alpha'] o.solver.adabelief_param.beta1 = solver.info['beta1'] o.solver.adabelief_param.beta2 = solver.info['beta2'] o.solver.adabelief_param.eps = solver.info['eps'] o.solver.adabelief_param.wd = solver.info['wd'] o.solver.adabelief_param.amsgrad = solver.info['amsgrad'] o.solver.adabelief_param.weight_decouple = solver.info[ 'weight_decouple'] o.solver.adabelief_param.fixed_decay = solver.info['fixed_decay'] o.solver.adabelief_param.rectify = solver.info['rectify'] elif o.solver.type == 'Adam': o.solver.adam_param.alpha = solver.info['alpha'] o.solver.adam_param.beta1 = solver.info['beta1'] o.solver.adam_param.beta2 = solver.info['beta2'] o.solver.adam_param.eps = solver.info['eps'] elif o.solver.type == 'Adamax': o.solver.adamax_param.alpha = solver.info['alpha'] o.solver.adamax_param.beta1 = solver.info['beta1'] o.solver.adamax_param.beta2 = solver.info['beta2'] o.solver.adamax_param.eps = solver.info['eps'] elif o.solver.type == 'Momentum': o.solver.momentum_param.lr = solver.info['lr'] o.solver.momentum_param.momentum = solver.info['momentum'] elif o.solver.type == 'Nesterov': o.solver.nesterov_param.lr = solver.info['lr'] o.solver.nesterov_param.momentum = solver.info['momentum'] elif o.solver.type == 'RMSprop': o.solver.rmsprop_param.lr = solver.info['lr'] o.solver.rmsprop_param.decay = solver.info['decay'] o.solver.rmsprop_param.eps = solver.info['eps'] elif o.solver.type == 'RMSpropGraves': o.solver.rmsprop_graves_param.lr = solver.info['lr'] o.solver.rmsprop_graves_param.decay = solver.info['decay'] o.solver.rmsprop_graves_param.momentum = solver.info['momentum'] o.solver.rmsprop_graves_param.eps = solver.info['eps'] elif o.solver.type == 'Sgd': o.solver.sgd_param.lr = solver.info['lr'] o.solver.weight_decay = weight_decay o.solver.lr_decay = lr_decay o.solver.lr_decay_interval = lr_decay_interval o.update_interval = update_interval for var_name, data_name in opti_d.get('data_variables', {}).items(): d = o.data_variable.add() d.variable_name = var_name d.data_name = data_name for loss_name in opti_d.get('loss_variables', proto_network.outputs): d = o.loss_variable.add() d.variable_name = loss_name for param in proto_network.parameters.keys(): d = o.parameter_variable.add() d.variable_name = param d.learning_rate_multiplier = 1.0 for g_var in opti_d.get('generator_variables', []): d = o.generator_variable.add() d.variable_name = g_var d.type = 'Constant' d.multiplier = 0 solver.set_states_to_protobuf(o) return o
def _create_optimizer_lite(opti): proto_o = nnabla_pb2.Optimizer() proto_o.name = opti.name proto_o.solver.type = re.sub(r'(|Cuda)$', '', str(opti.solver.name)) opti.solver.set_states_to_protobuf(proto_o) return proto_o
def _create_optimizer(datasets, name, solver, network, dataset_names, weight_decay, lr_decay, lr_decay_interval, update_interval): o = nnabla_pb2.Optimizer() o.name = name o.network_name = network.name dataset = None # Allow a list or tuple or a string for dataset names. if isinstance(dataset_names, tuple): dataset_names = list(dataset_names) if isinstance(dataset_names, list): for dataset_name in dataset_names: if dataset_name in datasets: o.dataset_name.append(dataset_name) dataset = datasets[dataset_name] else: raise ValueError( "Invalid dataset_name is found in optimizer: {}".format( dataset_name)) elif isinstance(dataset_names, str): dataset_name = dataset_names if dataset_name in datasets: o.dataset_name.append(dataset_name) dataset = datasets[dataset_name] if dataset is None: raise ValueError("Dataset is not defined in optimizer.") o.solver.type = re.sub(r'(|Cuda)$', '', str(solver.name)) if o.solver.type == 'Adadelta': o.solver.adadelta_param.lr = solver.info['lr'] o.solver.adadelta_param.decay = solver.info['decay'] o.solver.adadelta_param.eps = solver.info['eps'] elif o.solver.type == 'Adagrad': o.solver.adagrad_param.lr = solver.info['lr'] o.solver.adagrad_param.eps = solver.info['eps'] elif o.solver.type == 'Adam': o.solver.adam_param.alpha = solver.info['alpha'] o.solver.adam_param.beta1 = solver.info['beta1'] o.solver.adam_param.beta2 = solver.info['beta2'] o.solver.adam_param.eps = solver.info['eps'] elif o.solver.type == 'Adamax': o.solver.adamax_param.alpha = solver.info['alpha'] o.solver.adamax_param.beta1 = solver.info['beta1'] o.solver.adamax_param.beta2 = solver.info['beta2'] o.solver.adamax_param.eps = solver.info['eps'] elif o.solver.type == 'Momentum': o.solver.momentum_param.lr = solver.info['lr'] o.solver.momentum_param.momentum = solver.info['momentum'] elif o.solver.type == 'Nesterov': o.solver.nesterov_param.lr = solver.info['lr'] o.solver.nesterov_param.momentum = solver.info['momentum'] elif o.solver.type == 'RMSprop': o.solver.rmsprop_param.lr = solver.info['lr'] o.solver.rmsprop_param.decay = solver.info['decay'] o.solver.rmsprop_param.eps = solver.info['eps'] elif o.solver.type == 'RMSpropGraves': o.solver.rmsprop_graves_param.lr = solver.info['lr'] o.solver.rmsprop_graves_param.decay = solver.info['decay'] o.solver.rmsprop_graves_param.momentum = solver.info['momentum'] o.solver.rmsprop_graves_param.eps = solver.info['eps'] elif o.solver.type == 'Sgd': o.solver.sgd_param.lr = solver.info['lr'] inputs, outputs, params = _get_net_variables(network) o.solver.weight_decay = weight_decay o.solver.lr_decay = lr_decay o.solver.lr_decay_interval = lr_decay_interval o.update_interval = update_interval for n, inp in enumerate(inputs): d = o.data_variable.add() d.variable_name = inp d.data_name = dataset.variable[n] for out in outputs: d = o.loss_variable.add() d.variable_name = out for param in params: d = o.parameter_variable.add() d.variable_name = param d.learning_rate_multiplier = 1.0 solver.set_states_to_protobuf(o) return o