def __init__(self, learning_rate, parameters, weight_decay=0.0, loss_scale=1.0, decay_filter=lambda x: 'beta' not in x.name and 'gamma' not in x.name): super(Optimizer, self).__init__() if isinstance(learning_rate, float): self.dynamic_lr = False self.gather = None self.assignadd = None self.global_step = None validator.check_number_range("learning rate", learning_rate, 0.0, float("inf"), Rel.INC_LEFT) else: self.dynamic_lr = True self.gather = P.GatherV2() self.assignadd = P.AssignAdd() self.global_step = Parameter(initializer(0, [1], mindspore.int32), name='global_step') if isinstance(learning_rate, Iterable): learning_rate = Tensor( np.array(list(learning_rate)).astype(np.float32)) elif isinstance(learning_rate, Tensor): if learning_rate.dim() > 1: raise ValueError( "Learning rate should be a 0 or 1 dim `Tensor`," f"but got {learning_rate.dim()}.") if learning_rate.dim() == 1 and learning_rate.size() < 2: logger.warning( "If want to use the dynamic learning rate, please make sure that the number " "of elements in the list, tuple or tensor passed is greater than 1." ) else: raise TypeError( "Learning rate should be float, Tensor or Iterable.") if loss_scale <= 0.0: raise ValueError( "Loss scale should be greater than 0, but got {}".format( loss_scale)) if weight_decay < 0.0: raise ValueError( "Weight decay should be equal or greater than 0, but got {}". format(weight_decay)) self.learning_rate = Parameter(learning_rate, name="learning_rate") self.parameters = ParameterTuple(parameters) self.reciprocal_scale = 1.0 / loss_scale self.weight_decay = weight_decay * loss_scale self.decay_flags = tuple(decay_filter(x) for x in self.parameters) if not self.parameters: raise ValueError("optimizer got an empty parameter list.")
def __init__(self, params, learning_rate=1e-3, beta1=0.9, beta2=0.999, eps=1e-8, use_locking=False, use_nesterov=False, weight_decay=0.0, loss_scale=1.0, decay_filter=lambda x: 'beta' not in x.name and 'gamma' not in x.name): super(Adam, self).__init__(learning_rate, params) _check_param_value(beta1, beta2, eps, weight_decay) validator.check_type("use_locking", use_locking, [bool]) validator.check_type("use_nesterov", use_nesterov, [bool]) validator.check_type("loss_scale", loss_scale, [float]) validator.check_number_range("loss_scale", loss_scale, 1.0, float("inf"), Rel.INC_LEFT) self.dynamic_lr = False if isinstance(learning_rate, Iterable) or \ (isinstance(learning_rate, Tensor) and learning_rate.dim() == 1): self.dynamic_lr = True self.gather = P.GatherV2() self.assignadd = P.AssignAdd() self.global_step = Parameter(initializer(0, [1], mstype.int32), name="global_step") self.axis = 0 self.beta1 = Tensor(beta1, mstype.float32) self.beta2 = Tensor(beta2, mstype.float32) self.beta1_power = Parameter(initializer(1, [1], mstype.float32), name="beta1_power") self.beta2_power = Parameter(initializer(1, [1], mstype.float32), name="beta2_power") self.eps = eps self.moment1 = self.parameters.clone(prefix="moment1", init='zeros') self.moment2 = self.parameters.clone(prefix="moment2", init='zeros') self.decay_tf = tuple(decay_filter(x) for x in self.parameters) self.hyper_map = C.HyperMap() self.opt = P.Adam(use_locking, use_nesterov) self.weight_decay = weight_decay * loss_scale self.reciprocal_scale = 1.0 / loss_scale self.pow = P.Pow() self.sqrt = P.Sqrt() self.one = Tensor(np.array([1.0]).astype(np.float32)) self.realdiv = P.RealDiv()
def _check_param_value(decay_steps, warmup_steps, start_learning_rate, end_learning_rate, power, beta1, beta2, eps, weight_decay): """Check the type of inputs.""" validator.check_type("decay_steps", decay_steps, [int]) validator.check_type("warmup_steps", warmup_steps, [int]) validator.check_type("start_learning_rate", start_learning_rate, [float]) validator.check_type("end_learning_rate", end_learning_rate, [float]) validator.check_type("power", power, [float]) validator.check_type("beta1", beta1, [float]) validator.check_type("beta2", beta2, [float]) validator.check_type("eps", eps, [float]) validator.check_type("weight_dacay", weight_decay, [float]) validator.check_number_range("decay_steps", decay_steps, 1, float("inf"), Rel.INC_LEFT) validator.check_number_range("beta1", beta1, 0.0, 1.0, Rel.INC_NEITHER) validator.check_number_range("beta2", beta2, 0.0, 1.0, Rel.INC_NEITHER) validator.check_number_range("eps", eps, 0.0, float("inf"), Rel.INC_NEITHER) validator.check_number_range("weight_decay", weight_decay, 0.0, float("inf"), Rel.INC_LEFT)
def _check_param_value(beta1, beta2, eps, weight_decay): """Check the type of inputs.""" validator.check_type("beta1", beta1, [float]) validator.check_type("beta2", beta2, [float]) validator.check_type("eps", eps, [float]) validator.check_type("weight_dacay", weight_decay, [float]) validator.check_number_range("beta1", beta1, 0.0, 1.0, Rel.INC_NEITHER) validator.check_number_range("beta2", beta2, 0.0, 1.0, Rel.INC_NEITHER) validator.check_number_range("eps", eps, 0.0, float("inf"), Rel.INC_NEITHER) validator.check_number_range("weight_decay", weight_decay, 0.0, float("inf"), Rel.INC_LEFT)
def __init__(self, learning_rate, parameters): super(Optimizer, self).__init__() if isinstance(learning_rate, float): validator.check_number_range("learning rate", learning_rate, 0.0, float("inf"), Rel.INC_LEFT) elif isinstance(learning_rate, Iterable): learning_rate = Tensor(np.array(list(learning_rate)).astype(np.float32)) elif isinstance(learning_rate, Tensor): if learning_rate.dim() > 1: raise ValueError("Learning rate should be a 0 or 1 dim `Tensor`," f"but got {learning_rate.dim()}.") else: raise TypeError("Learning rate should be float, Tensor or Iterable.") if isinstance(learning_rate, Tensor) and learning_rate.dim() == 1 and learning_rate.size() < 2: logger.warning("If want to use the dynamic learning rate, please make sure that " "the number of elements in the list, tuple or tensor passed is greater than 1.") self.learning_rate = Parameter(learning_rate, name="learning_rate") self.parameters = ParameterTuple(parameters) if not self.parameters: raise ValueError("optimizer got an empty parameter list.")
def __init__(self, params, learning_rate=1e-3, beta1=0.9, beta2=0.999, eps=1e-8, use_locking=False, use_nesterov=False, weight_decay=0.0, loss_scale=1.0, decay_filter=lambda x: 'beta' not in x.name and 'gamma' not in x.name): super(Adam, self).__init__(learning_rate, params, weight_decay, loss_scale, decay_filter) _check_param_value(beta1, beta2, eps, weight_decay) validator.check_type("use_locking", use_locking, [bool]) validator.check_type("use_nesterov", use_nesterov, [bool]) validator.check_type("loss_scale", loss_scale, [float]) validator.check_number_range("loss_scale", loss_scale, 1.0, float("inf"), Rel.INC_LEFT) self.beta1 = Tensor(beta1, mstype.float32) self.beta2 = Tensor(beta2, mstype.float32) self.beta1_power = Parameter(initializer(1, [1], mstype.float32), name="beta1_power") self.beta2_power = Parameter(initializer(1, [1], mstype.float32), name="beta2_power") self.eps = eps self.moment1 = self.parameters.clone(prefix="moment1", init='zeros') self.moment2 = self.parameters.clone(prefix="moment2", init='zeros') self.decay_tf = tuple(decay_filter(x) for x in self.parameters) self.hyper_map = C.HyperMap() self.opt = P.Adam(use_locking, use_nesterov) self.pow = P.Pow() self.sqrt = P.Sqrt() self.one = Tensor(np.array([1.0]).astype(np.float32)) self.realdiv = P.RealDiv()
def __init__(self, max_val=1.0, filter_size=11, filter_sigma=1.5, k1=0.01, k2=0.03): super(SSIM, self).__init__() validator.check_type('max_val', max_val, [int, float]) validator.check('max_val', max_val, '', 0.0, Rel.GT) self.max_val = max_val self.filter_size = validator.check_integer('filter_size', filter_size, 1, Rel.GE) self.filter_sigma = validator.check_float_positive( 'filter_sigma', filter_sigma) validator.check_type('k1', k1, [float]) self.k1 = validator.check_number_range('k1', k1, 0.0, 1.0, Rel.INC_NEITHER) validator.check_type('k2', k2, [float]) self.k2 = validator.check_number_range('k2', k2, 0.0, 1.0, Rel.INC_NEITHER) self.mean = P.DepthwiseConv2dNative(channel_multiplier=1, kernel_size=filter_size)