def __init__(self, params, learning_rate=0.1, decay=0.9, momentum=0.0, epsilon=1e-10, use_locking=False, centered=False, loss_scale=1.0, weight_decay=0.0): super(RMSProp, self).__init__(learning_rate, params, weight_decay, loss_scale) validator.check_value_type("decay", decay, [float], self.cls_name) validator.check_non_negative_float(decay, "decay", self.cls_name) validator.check_value_type("momentum", momentum, [float], self.cls_name) validator.check_non_negative_float(momentum, "momentum", self.cls_name) validator.check_value_type("epsilon", epsilon, [float], self.cls_name) validator.check_positive_float(epsilon, "epsilon", self.cls_name) validator.check_value_type("use_locking", use_locking, [bool], self.cls_name) validator.check_value_type("centered", centered, [bool], self.cls_name) self.centered = centered if centered: self.opt = P.ApplyCenteredRMSProp(use_locking) self.mg = self.parameters.clone(prefix="mean_grad", init='zeros') else: self.opt = P.ApplyRMSProp(use_locking) self.momentum = momentum self.ms = self.parameters.clone(prefix="mean_square", init='ones') self.moment = self.parameters.clone(prefix="moment", init='zeros') self.hyper_map = C.HyperMap() self.epsilon = epsilon self.decay = decay
def __init__(self, lr, decay, momentum, epsilon): super(NetCenteredRMSProp, self).__init__() self.rms_opt = P.ApplyCenteredRMSProp() self.lr = lr self.decay = decay self.momentum = momentum self.epsilon = epsilon
def __init__(self, use_centered): super(NetRMSProp, self).__init__() self.use_centered = use_centered if use_centered: self.rms_opt = P.ApplyCenteredRMSProp() else: self.rms_opt = P.ApplyRMSProp()
def __init__(self, params, learning_rate=0.1, decay=0.9, momentum=0.0, epsilon=1e-10, use_locking=False, centered=False, loss_scale=1.0, weight_decay=0.0, decay_filter=lambda x: 'beta' not in x.name and 'gamma' not in x.name): super(RMSProp, self).__init__(learning_rate, params) if isinstance(momentum, float) and momentum < 0.0: raise ValueError( "momentum should be at least 0.0, but got momentum {}".format( momentum)) if decay < 0.0: raise ValueError( "decay should be at least 0.0, but got dampening {}".format( decay)) self.decay = decay self.epsilon = epsilon validator.check_type("use_locking", use_locking, [bool]) validator.check_type("centered", centered, [bool]) self.centered = centered if centered: self.opt = P.ApplyCenteredRMSProp(use_locking) self.mg = self.parameters.clone(prefix="mean_grad", init='zeros') else: self.opt = P.ApplyRMSProp(use_locking) self.dynamic_lr = False if not isinstance(learning_rate, float): self.dynamic_lr = True self.gather = P.GatherV2() self.assignadd = P.AssignAdd() self.global_step = Parameter(initializer(0, [1], mstype.int32), name="global_step") self.axis = 0 self.one = Tensor(1, mstype.int32) self.momentum = momentum self.ms = self.parameters.clone(prefix="mean_square", init='zeros') self.moment = self.parameters.clone(prefix="moment", init='zeros') self.hyper_map = C.HyperMap() self.decay = decay self.decay_tf = tuple(decay_filter(x) for x in self.parameters) self.reciprocal_scale = 1.0 / loss_scale self.weight_decay = weight_decay * loss_scale
def __init__(self, lr, decay, momentum, epsilon, var, g, mg, rms, mom): super(NetCenteredRMSProp, self).__init__() self.rms_opt = P.ApplyCenteredRMSProp() self.lr = lr self.decay = decay self.momentum = momentum self.epsilon = epsilon self.var = var self.g = g self.mg = mg self.rms = rms self.mom = mom
def __init__(self, params, learning_rate=0.1, decay=0.9, momentum=0.0, epsilon=1e-10, use_locking=False, centered=False, loss_scale=1.0, weight_decay=0.0, decay_filter=lambda x: 'beta' not in x.name and 'gamma' not in x.name): super(RMSProp, self).__init__(learning_rate, params, weight_decay, loss_scale, decay_filter) if isinstance(momentum, float) and momentum < 0.0: raise ValueError( "momentum should be at least 0.0, but got momentum {}".format( momentum)) if decay < 0.0: raise ValueError( "decay should be at least 0.0, but got dampening {}".format( decay)) self.decay = decay self.epsilon = epsilon validator.check_value_type("use_locking", use_locking, [bool], self.cls_name) validator.check_value_type("centered", centered, [bool], self.cls_name) self.centered = centered if centered: self.opt = P.ApplyCenteredRMSProp(use_locking) self.mg = self.parameters.clone(prefix="mean_grad", init='zeros') else: self.opt = P.ApplyRMSProp(use_locking) self.momentum = momentum self.ms = self.parameters.clone(prefix="mean_square", init='zeros') self.moment = self.parameters.clone(prefix="moment", init='zeros') self.hyper_map = C.HyperMap() self.decay = decay
def __init__(self, params, learning_rate=0.1, decay=0.9, momentum=0.0, epsilon=1e-10, use_locking=False, centered=False, loss_scale=1.0, weight_decay=0.0, decay_filter=lambda x: 'beta' not in x.name and 'gamma' not in x.name): super(RMSProp, self).__init__(learning_rate, params, weight_decay, loss_scale, decay_filter) validator.check_value_type("decay", decay, [float], self.cls_name) validator.check_number_range("decay", decay, 0.0, float("inf"), Rel.INC_LEFT, self.cls_name) validator.check_value_type("momentum", momentum, [float], self.cls_name) validator.check_number_range("momentum", momentum, 0.0, float("inf"), Rel.INC_LEFT, self.cls_name) validator.check_value_type("epsilon", epsilon, [float], self.cls_name) validator.check_number_range("epsilon", epsilon, 0.0, float("inf"), Rel.INC_NEITHER, self.cls_name) validator.check_value_type("use_locking", use_locking, [bool], self.cls_name) validator.check_value_type("centered", centered, [bool], self.cls_name) self.centered = centered if centered: self.opt = P.ApplyCenteredRMSProp(use_locking) self.mg = self.parameters.clone(prefix="mean_grad", init='zeros') else: self.opt = P.ApplyRMSProp(use_locking) self.momentum = momentum self.ms = self.parameters.clone(prefix="mean_square", init='zeros') self.moment = self.parameters.clone(prefix="moment", init='zeros') self.hyper_map = C.HyperMap() self.epsilon = epsilon self.decay = decay
'desc_inputs': [[3, 2]], 'desc_bprop': [[3, 2]]}), ('ApplyFtrl', { 'block': P.ApplyFtrl(), 'desc_const': [0.001, 0.0, 0.0, -0.5], 'desc_inputs': [[3, 3], [3, 3], [3, 3], [3, 3]], 'desc_bprop': [3, 3], 'skip': ['backward']}), ('ApplyRMSProp', { 'block': P.ApplyRMSProp(), 'desc_const': [0.9, 0.0, 1e-10, 0.001], 'desc_inputs': [[3, 3], [3, 3], [3, 3], [3, 3]], 'desc_bprop': [3, 3], 'skip': ['backward']}), ('ApplyCenteredRMSProp', { 'block': P.ApplyCenteredRMSProp(), 'desc_const': [0.9, 0.0, 1e-10, 0.001], 'desc_inputs': [[3, 3], [3, 3], [3, 3], [3, 3], [3, 3]], 'desc_bprop': [3, 3], 'skip': ['backward']}), ('L2Loss_1', { 'block': P.L2Loss(), 'desc_inputs': [Tensor(np.array([1, 2, 3, 4]), mstype.float16)], 'desc_bprop': []}), ('L2Loss_2', { 'block': P.L2Loss(), 'desc_inputs': [Tensor(np.array([[1, 1], [2, 2], [3, 3], [4, 4]]), mstype.float16)], 'desc_bprop': []}), ] test_case_array_ops = [
def __init__(self, var): super(ApplyCenteredRMSPropNet, self).__init__() self.apply_centered_rms_prop = P.ApplyCenteredRMSProp() self.var = Parameter(var, name="var")