def __init__(self, record=None):
super(BCELoss, self).__init__(record)
self.sm_scalar = P.ScalarSummary()
self.cast = P.Cast()
self.record = record
self.weight = None
self.bce = P.BinaryCrossEntropy()
def __init__(self, tag1, tag2, tag3):
super(SummaryDemoTag, self).__init__()
self.s = P.ScalarSummary()
self.add = P.TensorAdd()
self.tag1 = tag1
self.tag2 = tag2
self.tag3 = tag3
def __init__(self, tag_tuple=None, scalar=1):
super(SummaryNet, self).__init__()
self.summary_s = P.ScalarSummary()
self.summary_i = P.ImageSummary()
self.summary_t = P.TensorSummary()
self.add = P.TensorAdd()
self.tag_tuple = tag_tuple
self.scalar = scalar
def __init__(self):
super(CrossEntropyLoss, self).__init__()
self.sm_scalar = P.ScalarSummary()
self.cross_entropy = P.SoftmaxCrossEntropyWithLogits()
self.mean = P.ReduceMean()
self.one_hot = P.OneHot()
self.on_value = Tensor(1.0, mstype.float32)
self.off_value = Tensor(0.0, mstype.float32)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._original_construct = super().construct
self.scalar_summary = P.ScalarSummary()
self.histogram_summary = P.HistogramSummary()
self.weight_names = [param.name for param in self.parameters]
self.gradient_names = [
param.name + ".gradient" for param in self.parameters
]
def __init__(self,
params,
learning_rate=0.1,
momentum=0.0,
dampening=0.0,
weight_decay=0.0,
nesterov=False,
loss_scale=1.0):
super(SGD_, self).__init__(params, learning_rate, momentum, dampening,
weight_decay, nesterov, loss_scale)
self.sm_scalar = P.ScalarSummary()
def __init__(self, network, optimizer, scale_update_cell=None):
super(GNMTTrainOneStepWithLossScaleCell,
self).__init__(auto_prefix=False)
self.network = network
self.network.set_grad()
self.network.add_flags(defer_inline=True)
self.weights = optimizer.parameters
self.optimizer = optimizer
self.grad = C.GradOperation(get_by_list=True, sens_param=True)
self.reducer_flag = False
self.all_reduce = P.AllReduce()
self.parallel_mode = _get_parallel_mode()
if self.parallel_mode not in ParallelMode.MODE_LIST:
raise ValueError("Parallel mode does not support: ",
self.parallel_mode)
if self.parallel_mode in [
ParallelMode.DATA_PARALLEL, ParallelMode.HYBRID_PARALLEL
]:
self.reducer_flag = True
self.grad_reducer = None
if self.reducer_flag:
mean = _get_gradients_mean()
degree = _get_device_num()
self.grad_reducer = DistributedGradReducer(optimizer.parameters,
mean, degree)
self.is_distributed = (self.parallel_mode != ParallelMode.STAND_ALONE)
self.clip_gradients = ClipGradients()
self.cast = P.Cast()
self.alloc_status = P.NPUAllocFloatStatus()
self.get_status = P.NPUGetFloatStatus()
self.clear_before_grad = P.NPUClearFloatStatus()
self.reduce_sum = P.ReduceSum(keep_dims=False)
self.depend_parameter_use = P.ControlDepend(depend_mode=1)
self.base = Tensor(1, mstype.float32)
self.less_equal = P.LessEqual()
self.hyper_map = C.HyperMap()
self.loss_scale = None
self.loss_scaling_manager = scale_update_cell
if scale_update_cell:
self.loss_scale = Parameter(Tensor(
scale_update_cell.get_loss_scale(), dtype=mstype.float32),
name="loss_scale")
self.add_flags(has_effect=True)
self.loss_scalar = P.ScalarSummary()
def __init__(self, num_class=10, channel=1):
super(LeNet5, self).__init__()
self.num_class = num_class
self.conv1 = conv(channel, 6, 5)
self.conv2 = conv(6, 16, 5)
self.fc1 = fc_with_initialize(16 * 5 * 5, 120)
self.fc2 = fc_with_initialize(120, 84)
self.fc3 = fc_with_initialize(84, self.num_class)
self.relu = nn.ReLU()
self.max_pool2d = nn.MaxPool2d(kernel_size=2, stride=2)
self.flatten = nn.Flatten()
self.scalar_summary = P.ScalarSummary()
self.image_summary = P.ImageSummary()
self.histogram_summary = P.HistogramSummary()
self.tensor_summary = P.TensorSummary()
self.channel = Tensor(channel)
def __init__(self, num_class=10, num_channel=1, include_top=True):
super(LeNet5, self).__init__()
self.conv1 = nn.Conv2d(num_channel, 6, 5, pad_mode='valid')
self.conv2 = nn.Conv2d(6, 16, 5, pad_mode='valid')
self.relu = nn.ReLU()
self.max_pool2d = nn.MaxPool2d(kernel_size=2, stride=2)
self.include_top = include_top
if self.include_top:
self.flatten = nn.Flatten()
self.fc1 = nn.Dense(16 * 5 * 5, 120, weight_init=Normal(0.02))
self.fc2 = nn.Dense(120, 84, weight_init=Normal(0.02))
self.fc3 = nn.Dense(84, num_class, weight_init=Normal(0.02))
self.scalar_summary = P.ScalarSummary()
self.image_summary = P.ImageSummary()
self.tensor_summary = P.TensorSummary()
self.channel = Tensor(num_channel)
def __init__(self,
params,
learning_rate,
momentum,
weight_decay=0.0,
loss_scale=1.0,
use_nesterov=False):
super(MyMomentum, self).__init__(learning_rate, params, weight_decay,
loss_scale)
if isinstance(momentum, float) and momentum < 0.0:
raise ValueError(
"momentum should be at least 0.0, but got momentum {}".format(
momentum))
self.momentum = Parameter(Tensor(momentum, mstype.float32),
name="momentum")
self.params = self.parameters
self.use_nesterov = check_bool(use_nesterov)
self.moments = self.params.clone(prefix="moments", init='zeros')
self.hyper_map = C.HyperMap()
self.opt = P.ApplyMomentum(use_nesterov=self.use_nesterov)
self.scalar_summary = P.ScalarSummary()
self.weight_names = [param.name for param in self.parameters]
def __init__(self,
params,
learning_rate=1e-3,
beta1=0.9,
beta2=0.999,
eps=1e-8,
use_locking=False,
use_nesterov=False,
weight_decay=0.0,
loss_scale=1.0):
super(Adam, self).__init__(learning_rate, params, weight_decay,
loss_scale)
_check_param_value(beta1, beta2, eps, weight_decay, self.cls_name)
validator.check_value_type("use_locking", use_locking, [bool],
self.cls_name)
validator.check_value_type("use_nesterov", use_nesterov, [bool],
self.cls_name)
validator.check_value_type("loss_scale", loss_scale, [float],
self.cls_name)
# validator.check_number_range("loss_scale", loss_scale, 1.0, float("inf"), Rel.INC_LEFT, self.cls_name)
self.beta1 = Tensor(beta1, mstype.float32)
self.beta2 = Tensor(beta2, mstype.float32)
self.beta1_power = Parameter(initializer(1, [1], mstype.float32))
self.beta2_power = Parameter(initializer(1, [1], mstype.float32))
self.eps = eps
self.moment1 = self.parameters.clone(prefix="moment1", init='zeros')
self.moment2 = self.parameters.clone(prefix="moment2", init='zeros')
self.hyper_map = C.HyperMap()
self.opt = P.Adam(use_locking, use_nesterov)
self.pow = P.Pow()
self.sqrt = P.Sqrt()
self.one = Tensor(np.array([1.0]).astype(np.float32))
self.realdiv = P.RealDiv()
self.lr_scalar = P.ScalarSummary()
def __init__(self):
super(ScalarSummaryNet, self).__init__()
self.summary = P.ScalarSummary()
def __init__(self, ):
super(SummaryDemo, self).__init__()
self.s = P.ScalarSummary()
self.histogram_summary = P.HistogramSummary()
self.add = P.TensorAdd()
def __init__(self,):
super(SummaryNet, self).__init__()
self.s = P.ScalarSummary()
self.add = P.TensorAdd()
def __init__(
self,
num_atomtypes=100,
dim_atomembedding=64,
min_rbf_dis=0.05,
max_rbf_dis=1,
num_rbf=32,
n_interactions=3,
n_heads=8,
max_cycles=10,
activation=Swish(),
output_dim=1,
self_dis=None,
rbf_sigma=None,
distance_expansion=LogGaussianDistribution,
cutoff=None,
cutoff_network=SmoothCutoff,
public_filter=True,
coupled_interactions=False,
trainable_gaussians=False,
use_pondering=True,
fixed_cycles=False,
rescale_rbf=True,
use_time_embedding=True,
use_all_interactions=True,
use_mcr=False,
debug=False,
):
super().__init__(
num_atomtypes=num_atomtypes,
dim_atomembedding=dim_atomembedding,
min_rbf_dis=min_rbf_dis,
max_rbf_dis=max_rbf_dis,
num_rbf=num_rbf,
output_dim=output_dim,
rbf_sigma=rbf_sigma,
distance_expansion=distance_expansion,
cutoff=cutoff,
cutoff_network=cutoff_network,
rescale_rbf=rescale_rbf,
use_all_interactions=use_all_interactions,
)
self.network_name = 'AirNet'
self.max_distance = max_rbf_dis
self.min_distance = min_rbf_dis
if self_dis is None:
self.self_dis = self.min_distance
else:
self.self_dis = self_dis
self.self_dis_tensor = Tensor([self.self_dis], ms.float32)
self.n_heads = n_heads
if use_time_embedding:
time_embedding = self._get_time_signal(max_cycles,
dim_atomembedding)
else:
time_embedding = [0 for _ in range(max_cycles)]
if public_filter:
inter_filter = False
self.filter = Filter(num_rbf, dim_atomembedding, None)
else:
inter_filter = True
self.filter = None
self.n_interactions = n_interactions
# block for computing interaction
if coupled_interactions:
# use the same SchNetInteraction instance (hence the same weights)
self.interactions = nn.CellList([
AirNetInteraction(
dim_atom_embed=dim_atomembedding,
num_rbf=num_rbf,
n_heads=n_heads,
activation=activation,
max_cycles=max_cycles,
time_embedding=time_embedding,
use_filter=inter_filter,
use_pondering=use_pondering,
fixed_cycles=fixed_cycles,
)
] * n_interactions)
else:
# use one SchNetInteraction instance for each interaction
self.interactions = nn.CellList([
AirNetInteraction(
dim_atom_embed=dim_atomembedding,
num_rbf=num_rbf,
n_heads=n_heads,
activation=activation,
max_cycles=max_cycles,
time_embedding=time_embedding,
use_filter=inter_filter,
use_pondering=use_pondering,
fixed_cycles=fixed_cycles,
) for i in range(n_interactions)
])
# readout layer
if self.use_all_interactions and n_interactions > 1:
if use_mcr:
self.gather_interactions = MultipleChannelRepresentation(
n_interactions, dim_atomembedding, 1, activation)
else:
self.gather_interactions = TensorSum()
else:
self.gather_interactions = None
readoutdim = int(dim_atomembedding / 2)
self.readout = AtomwiseReadout(dim_atomembedding, self.output_dim, [
readoutdim,
], activation)
if debug:
self.debug_fun = self._debug_fun
self.lmax_label = []
for i in range(n_interactions):
self.lmax_label.append('l' + str(i) + '_cycles')
self.fill = P.Fill()
self.concat = P.Concat(-1)
self.pack = P.Pack(-1)
self.reducesum = P.ReduceSum()
self.reducemax = P.ReduceMax()
self.tensor_summary = P.TensorSummary()
self.scalar_summary = P.ScalarSummary()
def __init__(self, value, tag_tuple):
super(SummaryDemoValueForSet, self).__init__()
self.s = P.ScalarSummary()
self.add = P.TensorAdd()
self.tag_tuple = tag_tuple
self.v = value
def __init__(self, value):
super(SummaryDemoValue, self).__init__()
self.s = P.ScalarSummary()
self.add = P.TensorAdd()
self.v = value
def __init__(self, tag_tuple):
super(SummaryDemoTagForSet, self).__init__()
self.s = P.ScalarSummary()
self.add = P.TensorAdd()
self.tag_tuple = tag_tuple
def __init__(self):
super().__init__()
self.scalar_summary = P.ScalarSummary()
self.image_summary = P.ImageSummary()
self.tensor_summary = P.TensorSummary()
self.histogram_summary = P.HistogramSummary()
c = x * y
return c
@ms_function
def f(x, y):
return clip_test(x, y)
@ms_function
def fd(x, y):
return grad_all(clip_test)(x, y)
print("forward: ", f(1.1, 0.1))
print("clip_gradient:", fd(1.1, 0.1))
summary = P.ScalarSummary()
def debug_gradient(dx):
""" debug_gradient """
summary("dx: ", dx)
return dx
debug = P.InsertGradientOf(debug_gradient)
def test_InsertGradientOf_3():
""" test_InsertGradientOf_3 """
def debug_test(x, y):
x = debug(x)