def construct(self, x):
"""stride"""
N, C, H, W = x.shape
leftup_idx_x = []
leftup_idx_y = []
nh = int(H / self.stride)
nw = int(W / self.stride)
for i in range(nh):
leftup_idx_x.append(i * self.stride)
for i in range(nw):
leftup_idx_y.append(i * self.stride)
NumBlock_x = len(leftup_idx_x)
NumBlock_y = len(leftup_idx_y)
zeroslike = P.ZerosLike()
cc_2 = P.Concat(axis=2)
cc_3 = P.Concat(axis=3)
unf_x = P.Zeros()((N, C, NumBlock_x * self.kernel_size,
NumBlock_y * self.kernel_size), mstype.float32)
N, C, H, W = unf_x.shape
for i in range(NumBlock_x):
for j in range(NumBlock_y):
unf_i = i * self.kernel_size
unf_j = j * self.kernel_size
org_i = leftup_idx_x[i]
org_j = leftup_idx_y[j]
fills = x[:, :, org_i:org_i + self.kernel_size,
org_j:org_j + self.kernel_size]
unf_x += cc_3((cc_3((zeroslike(unf_x[:, :, :, :unf_j]), cc_2((cc_2(
(zeroslike(unf_x[:, :, :unf_i, unf_j:unf_j + self.kernel_size]), fills)), zeroslike(
unf_x[:, :, unf_i + self.kernel_size:, unf_j:unf_j + self.kernel_size]))))),
zeroslike(unf_x[:, :, :, unf_j + self.kernel_size:])))
y = self.unfold(unf_x)
return y
def __init__(self,
low=None,
high=None,
seed=0,
dtype=mstype.float32,
name="Uniform"):
"""
Constructor of Uniform distribution.
"""
param = dict(locals())
super(Uniform, self).__init__(dtype, name, param)
if low is not None and high is not None:
self._low = convert_to_batch(low, self._broadcast_shape, dtype)
self._high = convert_to_batch(high, self._broadcast_shape, dtype)
check_greater(self.low, self.high, "low value", "high value")
else:
self._low = low
self._high = high
# ops needed for the class
self.const = P.ScalarToArray()
self.dtypeop = P.DType()
self.exp = P.Exp()
self.fill = P.Fill()
self.less = P.Less()
self.lessequal = P.LessEqual()
self.log = P.Log()
self.logicaland = P.LogicalAnd()
self.select = P.Select()
self.shape = P.Shape()
self.sq = P.Square()
self.sqrt = P.Sqrt()
self.uniform = P.UniformReal(seed=seed)
self.zeroslike = P.ZerosLike()
def __init__(self):
super(IGamma, self).__init__()
# const numbers
# If more data types are supported, this float max value need to be selected.
self.log_maxfloat32 = Tensor(np.log(np.finfo(np.float32).max),
mstype.float32)
# operations
self.logicaland = P.LogicalAnd()
self.logicalor = P.LogicalOr()
self.logicalnot = P.LogicalNot()
self.equal = P.Equal()
self.greater = P.Greater()
self.less = P.Less()
self.neg = P.Neg()
self.log = P.Log()
self.exp = P.Exp()
self.select = P.Select()
self.zeroslike = P.ZerosLike()
self.fill = P.Fill()
self.shape = P.Shape()
self.dtype = P.DType()
self.lgamma = LGamma()
self.const = P.ScalarToArray()
self.cast = P.Cast()
def __init__(self,
loc=None,
scale=None,
seed=0,
dtype=mstype.float32,
name="LogNormal"):
"""
Constructor of LogNormal distribution.
"""
super(LogNormal, self).__init__(distribution=msd.Normal(loc, scale, dtype=dtype),
bijector=msb.Exp(),
seed=seed, name=name)
self.log_2pi = np.log(2 * np.pi)
#ops needed for the class
self.exp = exp_generic
self.expm1 = expm1_generic
self.log = log_generic
self.const = P.ScalarToArray()
self.erf = P.Erf()
self.fill = P.Fill()
self.shape = P.Shape()
self.sq = P.Square()
self.sqrt = P.Sqrt()
self.zeroslike = P.ZerosLike()
def __init__(self, teacher_config, teacher_ckpt, student_config, is_training, use_one_hot_embeddings=False,
is_att_fit=True, is_rep_fit=True):
super(BertNetworkWithLoss_gd, self).__init__()
# load teacher model
self.teacher = BertModel(teacher_config, False, use_one_hot_embeddings)
param_dict = load_checkpoint(teacher_ckpt)
new_param_dict = {}
for key, value in param_dict.items():
new_key = re.sub('^bert.bert.', 'teacher.', key)
new_param_dict[new_key] = value
load_param_into_net(self.teacher, new_param_dict)
# no_grad
self.teacher.set_train(False)
params = self.teacher.trainable_params()
for param in params:
param.requires_grad = False
# student model
self.bert = TinyBertModel(student_config, is_training, use_one_hot_embeddings)
self.cast = P.Cast()
self.fit_dense = nn.Dense(student_config.hidden_size,
teacher_config.hidden_size).to_float(teacher_config.compute_type)
self.teacher_layers_num = teacher_config.num_hidden_layers
self.student_layers_num = student_config.num_hidden_layers
self.layers_per_block = int(self.teacher_layers_num / self.student_layers_num)
self.is_att_fit = is_att_fit
self.is_rep_fit = is_rep_fit
self.loss_mse = nn.MSELoss()
self.select = P.Select()
self.zeroslike = P.ZerosLike()
self.dtype = teacher_config.dtype
def __init__(self,
mean=None,
sd=None,
seed=0,
dtype=mstype.float32,
name="Normal"):
"""
Constructor of normal distribution.
"""
param = dict(locals())
valid_dtype = mstype.float_type
check_type(dtype, valid_dtype, type(self).__name__)
super(Normal, self).__init__(seed, dtype, name, param)
self.parameter_type = dtype
if mean is not None and sd is not None:
self._mean_value = cast_to_tensor(mean, self.parameter_type)
self._sd_value = cast_to_tensor(sd, self.parameter_type)
check_greater_zero(self._sd_value, "Standard deviation")
else:
self._mean_value = mean
self._sd_value = sd
#ops needed for the class
self.exp = exp_generic
self.expm1 = expm1_generic
self.log = log_generic
self.erf = erf_generic
self.squeeze = P.Squeeze(0)
self.cast = P.Cast()
self.const = P.ScalarToArray()
self.fill = P.Fill()
self.shape = P.Shape()
self.sq = P.Square()
self.sqrt = P.Sqrt()
self.zeroslike = P.ZerosLike()
def __init__(self, batch_size=4):
super(DiceLoss, self).__init__()
self.threshold0 = Tensor(0.5, mstype.float32)
self.zero_float32 = Tensor(0.0, mstype.float32)
self.k = int(640 * 640)
self.negative_one_int32 = Tensor(-1, mstype.int32)
self.batch_size = batch_size
self.concat = P.Concat()
self.less_equal = P.LessEqual()
self.greater = P.Greater()
self.reduce_sum = P.ReduceSum()
self.reduce_sum_keep_dims = P.ReduceSum(keep_dims=True)
self.reduce_mean = P.ReduceMean()
self.reduce_min = P.ReduceMin()
self.cast = P.Cast()
self.minimum = P.Minimum()
self.expand_dims = P.ExpandDims()
self.select = P.Select()
self.fill = P.Fill()
self.topk = P.TopK(sorted=True)
self.shape = P.Shape()
self.sigmoid = P.Sigmoid()
self.reshape = P.Reshape()
self.slice = P.Slice()
self.logical_and = P.LogicalAnd()
self.logical_or = P.LogicalOr()
self.equal = P.Equal()
self.zeros_like = P.ZerosLike()
self.add = P.TensorAdd()
self.gather = P.Gather()
def __init__(self, num_sampled, num_classes, num_true=1,
sampled_values=None, remove_accidental_hits=True, seed=0,
reduction='none'):
super(SampledSoftmaxLoss, self).__init__()
self.num_sampled = num_sampled
self.num_classes = num_classes
self.num_true = num_true
self.sampled_values = sampled_values
self.remove_accidental_hits = remove_accidental_hits
self.seed = seed
self.sampler = P.UniformSampler(
num_true,
num_sampled,
True,
num_classes,
seed,
remove_accidental_hits)
self.cast = P.Cast()
self.reshape = P.Reshape()
self.shape = P.Shape()
self.exp = P.Exp()
self.log = P.Log()
self.slice_op = P.Slice()
self.matmul = P.MatMul(False, True)
self.gather_v2 = P.GatherV2()
self.reduce_max_true = P.ReduceMax(True)
self.reduce_sum = P.ReduceSum()
self.reduce_sum_true = P.ReduceSum(True)
self.concat_dim0 = P.Concat(0)
self.concat_dim1 = P.Concat(1)
self.ones_like = P.OnesLike()
self.zeros_like = P.ZerosLike()
self.mul = P.Mul()
self.expand_dims = P.ExpandDims()
def __init__(self,
loc=None,
scale=None,
seed=0,
dtype=mstype.float32,
name="LogNormal"):
"""
Constructor of LogNormal distribution.
"""
super(LogNormal, self).__init__(distribution=msd.Normal(loc, scale, dtype=dtype),
bijector=msb.Exp(),
seed=seed, name=name)
# overwrite default_parameters and parameter_names
self._reset_parameters()
self._loc = self._add_parameter(loc, 'loc')
self._scale = self._add_parameter(scale, 'scale')
self.log_2pi = np.log(2 * np.pi)
#ops needed for the class
self.exp = exp_generic
self.expm1 = P.Expm1()
self.log = log_generic
self.const = P.ScalarToArray()
self.erf = P.Erf()
self.fill = P.Fill()
self.shape = P.Shape()
self.sq = P.Square()
self.sqrt = P.Sqrt()
self.cast = P.Cast()
self.squeeze = P.Squeeze(0)
self.zeroslike = P.ZerosLike()
def construct(self, x):
"""ipt"""
cc_2 = P.Concat(axis=2)
cc_3 = P.Concat(axis=3)
zeroslike = P.ZerosLike()
if self.output_shape[0] == -1:
large_x = self.fold(x)
N, C, H, _ = large_x.shape
leftup_idx = []
for i in range(0, H, self.kernel_size[0]):
leftup_idx.append(i)
NumBlock = len(leftup_idx)
fold_x = P.Zeros()(
(N, C, (NumBlock - 1) * self.stride + self.kernel_size[0],
(NumBlock - 1) * self.stride + self.kernel_size[0]),
mstype.float32)
for i in range(NumBlock):
for j in range(NumBlock):
fold_i = i * self.stride
fold_j = j * self.stride
org_i = leftup_idx[i]
org_j = leftup_idx[j]
fills = large_x[:, :, org_i:org_i + self.kernel_size[0],
org_j:org_j + self.kernel_size[1]]
fold_x += cc_3((cc_3((zeroslike(fold_x[:, :, :, :fold_j]), cc_2((cc_2((zeroslike(fold_x[:, :, :fold_i, fold_j:fold_j + self.kernel_size[1]]), fills)), zeroslike(fold_x[:, :, fold_i + self.kernel_size[0]:, fold_j:fold_j + self.kernel_size[1]]))))), zeroslike(fold_x[:, :, :, fold_j + self.kernel_size[1]:]))) #pylint: disable=line-too-long
y = fold_x
else:
NumBlock_x = int((self.output_shape[0] - self.kernel_size[0]) /
self.stride + 1)
NumBlock_y = int((self.output_shape[1] - self.kernel_size[1]) /
self.stride + 1)
large_shape = [
NumBlock_x * self.kernel_size[0],
NumBlock_y * self.kernel_size[1]
]
self.fold = _fold_(self.kernel_size, large_shape)
large_x = self.fold(x)
N, C, H, _ = large_x.shape
leftup_idx_x = []
leftup_idx_y = []
for i in range(NumBlock_x):
leftup_idx_x.append(i * self.kernel_size[0])
for i in range(NumBlock_y):
leftup_idx_y.append(i * self.kernel_size[1])
fold_x = P.Zeros()(
(N, C, (NumBlock_x - 1) * self.stride + self.kernel_size[0],
(NumBlock_y - 1) * self.stride + self.kernel_size[1]),
mstype.float32)
for i in range(NumBlock_x):
for j in range(NumBlock_y):
fold_i = i * self.stride
fold_j = j * self.stride
org_i = leftup_idx_x[i]
org_j = leftup_idx_y[j]
fills = large_x[:, :, org_i:org_i + self.kernel_size[0],
org_j:org_j + self.kernel_size[1]]
fold_x += cc_3((cc_3((zeroslike(fold_x[:, :, :, :fold_j]), cc_2((cc_2((zeroslike(fold_x[:, :, :fold_i, fold_j:fold_j + self.kernel_size[1]]), fills)), zeroslike(fold_x[:, :, fold_i + self.kernel_size[0]:, fold_j:fold_j + self.kernel_size[1]]))))), zeroslike(fold_x[:, :, :, fold_j + self.kernel_size[1]:]))) #pylint: disable=line-too-long
y = fold_x
return y
def __init__(self,
mean=None,
sd=None,
seed=0,
dtype=mstype.float32,
name="Normal"):
"""
Constructor of normal distribution.
"""
param = dict(locals())
super(Normal, self).__init__(dtype, name, param)
if mean is not None and sd is not None:
self._mean_value = convert_to_batch(mean, self._broadcast_shape,
dtype)
self._sd_value = convert_to_batch(sd, self._broadcast_shape, dtype)
check_greater_equal_zero(self._sd_value, "Standard deviation")
else:
self._mean_value = mean
self._sd_value = sd
#ops needed for the class
self.exp = P.Exp()
self.add = P.TensorAdd()
self.mul = P.Mul()
self.sq = P.Square()
self.log = P.Log()
self.sqrt = P.Sqrt()
self.realdiv = P.RealDiv()
self.expm1 = P.Expm1() if get_context(
'device_target') == 'Ascend' else self._expm1_by_step
self.normal = P.Normal(seed=seed)
self.shape = P.Shape()
self.zeroslike = P.ZerosLike()
self.const = P.ScalarToArray()
def __init__(self,
mean=None,
sd=None,
seed=0,
dtype=mstype.float32,
name="Normal"):
"""
Constructor of normal distribution.
"""
param = dict(locals())
super(Normal, self).__init__(dtype, name, param)
if mean is not None and sd is not None:
self._mean_value = convert_to_batch(mean, self._broadcast_shape, dtype)
self._sd_value = convert_to_batch(sd, self._broadcast_shape, dtype)
check_greater_equal_zero(self._sd_value, "Standard deviation")
else:
self._mean_value = mean
self._sd_value = sd
self.seed = seed
#ops needed for the class
self.const = P.ScalarToArray()
self.erf = P.Erf()
self.exp = P.Exp()
self.expm1 = self._expm1_by_step
self.fill = P.Fill()
self.log = P.Log()
self.shape = P.Shape()
self.sq = P.Square()
self.sqrt = P.Sqrt()
self.zeroslike = P.ZerosLike()
def __init__(self, config, batch_size, in_channels, feat_channels,
num_anchors, cls_out_channels):
super(RPN, self).__init__()
cfg_rpn = config
self.dtype = np.float32
self.ms_type = mstype.float32
self.num_bboxes = cfg_rpn.num_bboxes
self.slice_index = ()
self.feature_anchor_shape = ()
self.slice_index += (0, )
index = 0
for shape in cfg_rpn.feature_shapes:
self.slice_index += (self.slice_index[index] +
shape[0] * shape[1] * num_anchors, )
self.feature_anchor_shape += (shape[0] * shape[1] * num_anchors *
batch_size, )
index += 1
self.num_anchors = num_anchors
self.batch_size = batch_size
self.test_batch_size = cfg_rpn.test_batch_size
self.num_layers = 5
self.real_ratio = Tensor(np.ones((1, 1)).astype(self.dtype))
self.rpn_convs_list = nn.layer.CellList(
self._make_rpn_layer(self.num_layers, in_channels, feat_channels,
num_anchors, cls_out_channels))
self.transpose = P.Transpose()
self.reshape = P.Reshape()
self.concat = P.Concat(axis=0)
self.fill = P.Fill()
self.placeh1 = Tensor(np.ones((1, )).astype(self.dtype))
self.trans_shape = (0, 2, 3, 1)
self.reshape_shape_reg = (-1, 4)
self.reshape_shape_cls = (-1, )
self.rpn_loss_reg_weight = Tensor(
np.array(cfg_rpn.rpn_loss_reg_weight).astype(self.dtype))
self.rpn_loss_cls_weight = Tensor(
np.array(cfg_rpn.rpn_loss_cls_weight).astype(self.dtype))
self.num_expected_total = Tensor(
np.array(cfg_rpn.num_expected_neg * self.batch_size).astype(
self.dtype))
self.num_bboxes = cfg_rpn.num_bboxes
self.get_targets = BboxAssignSample(cfg_rpn, self.batch_size,
self.num_bboxes, False)
self.CheckValid = P.CheckValid()
self.sum_loss = P.ReduceSum()
self.loss_cls = P.SigmoidCrossEntropyWithLogits()
self.loss_bbox = P.SmoothL1Loss(beta=1.0 / 9.0)
self.squeeze = P.Squeeze()
self.cast = P.Cast()
self.tile = P.Tile()
self.zeros_like = P.ZerosLike()
self.loss = Tensor(np.zeros((1, )).astype(self.dtype))
self.clsloss = Tensor(np.zeros((1, )).astype(self.dtype))
self.regloss = Tensor(np.zeros((1, )).astype(self.dtype))
def __init__(self,
num_sampled,
num_classes,
num_true=1,
sampled_values=None,
remove_accidental_hits=True,
seed=0,
reduction='none'):
super(SampledSoftmaxLoss, self).__init__(reduction)
if num_true < 1:
raise ValueError(f"num_true {num_true} is less than 1.")
if seed < 0:
raise ValueError(f"seed {seed} is less than 0.")
if num_sampled > num_classes:
raise ValueError(
f"num_sampled {num_sampled} is great than num_classes {num_classes}."
)
if num_true > num_classes:
raise ValueError(
f"num_true {num_true} is great than num_classes {num_classes}."
)
if sampled_values is not None:
if not isinstance(sampled_values, (list, tuple)):
raise TypeError(
f"sampled_values {sampled_values} is not a list.")
if len(sampled_values) != 3:
raise ValueError(
f"sampled_values size {len(sampled_values)} is not 3.")
self.num_sampled = num_sampled
self.num_classes = num_classes
self.num_true = num_true
self.sampled_values = sampled_values
self.remove_accidental_hits = remove_accidental_hits
self.seed = seed
self.sampler = P.LogUniformCandidateSampler(num_true, num_sampled,
True, num_classes, seed)
self.cast = P.Cast()
self.reshape = P.Reshape()
self.shape = P.Shape()
self.exp = P.Exp()
self.log = P.Log()
self.slice_op = P.Slice()
self.matmul = P.MatMul(False, True)
self.gather_v2 = P.Gather()
self.reduce_max_true = P.ReduceMax(True)
self.reduce_sum = P.ReduceSum()
self.reduce_sum_true = P.ReduceSum(True)
self.concat_dim0 = P.Concat(0)
self.concat_dim1 = P.Concat(1)
self.ones_like = P.OnesLike()
self.zeros_like = P.ZerosLike()
self.mul = P.Mul()
self.expand_dims = P.ExpandDims()
self.dtype = P.DType()
self.compute_accidental_hits = P.ComputeAccidentalHits(num_true)
self.scatter_nd = P.ScatterNd()
def __init__(self, config, batch_size, in_channels, feat_channels,
num_anchors, cls_out_channels):
super(RPN, self).__init__()
cfg_rpn = config
self.cfg = config
self.num_bboxes = cfg_rpn.num_bboxes
self.feature_anchor_shape = cfg_rpn.feature_shapes
self.feature_anchor_shape = self.feature_anchor_shape[0] * \
self.feature_anchor_shape[1] * num_anchors * batch_size
self.num_anchors = num_anchors
self.batch_size = batch_size
self.test_batch_size = cfg_rpn.test_batch_size
self.num_layers = 1
self.real_ratio = Tensor(np.ones((1, 1)).astype(np.float16))
self.use_sigmoid_cls = config.use_sigmoid_cls
if config.use_sigmoid_cls:
self.reshape_shape_cls = (-1, )
self.loss_cls = P.SigmoidCrossEntropyWithLogits()
cls_out_channels = 1
else:
self.reshape_shape_cls = (-1, cls_out_channels)
self.loss_cls = nn.SoftmaxCrossEntropyWithLogits(sparse=True,
reduction="none")
self.rpn_convs_list = self._make_rpn_layer(self.num_layers, in_channels, feat_channels,\
num_anchors, cls_out_channels)
self.transpose = P.Transpose()
self.reshape = P.Reshape()
self.concat = P.Concat(axis=0)
self.fill = P.Fill()
self.placeh1 = Tensor(np.ones((1, )).astype(np.float16))
self.trans_shape = (0, 2, 3, 1)
self.reshape_shape_reg = (-1, 4)
self.softmax = nn.Softmax()
self.rpn_loss_reg_weight = Tensor(
np.array(cfg_rpn.rpn_loss_reg_weight).astype(np.float16))
self.rpn_loss_cls_weight = Tensor(
np.array(cfg_rpn.rpn_loss_cls_weight).astype(np.float16))
self.num_expected_total = Tensor(
np.array(cfg_rpn.num_expected_neg * self.batch_size).astype(
np.float16))
self.num_bboxes = cfg_rpn.num_bboxes
self.get_targets = BboxAssignSample(cfg_rpn, self.batch_size,
self.num_bboxes, False)
self.CheckValid = P.CheckValid()
self.sum_loss = P.ReduceSum()
self.loss_bbox = P.SmoothL1Loss(beta=1.0 / 9.0)
self.squeeze = P.Squeeze()
self.cast = P.Cast()
self.tile = P.Tile()
self.zeros_like = P.ZerosLike()
self.loss = Tensor(np.zeros((1, )).astype(np.float16))
self.clsloss = Tensor(np.zeros((1, )).astype(np.float16))
self.regloss = Tensor(np.zeros((1, )).astype(np.float16))
self.print = P.Print()
def __init__(self, stride):
"""Construct Zero class.
:param stride: stride of the output
"""
super(Zero, self).__init__()
self.zeroslike = P.ZerosLike()
self.stride = stride
self.shape = P.Shape()
def __init__(self, network, weights):
super(InversionLoss, self).__init__()
self._network = check_param_type('network', network, Cell)
self._mse_loss = MSELoss()
self._weights = check_param_multi_types('weights', weights,
[list, tuple])
self._get_shape = P.Shape()
self._zeros = P.ZerosLike()
self._device_target = context.get_context("device_target")
def __init__(self, network):
super(NetWithLossClass, self).__init__(auto_prefix=False)
self.loss = nn.SoftmaxCrossEntropyWithLogits(sparse=True)
self.network = network
self.reducesum = P.ReduceSum(keep_dims=False)
self.mul = P.Mul()
self.squeeze = P.Squeeze(axis=1)
self.zeroslike = P.ZerosLike()
self.concat = P.Concat(axis=1)
self.reciprocal = P.Reciprocal()
def __init__(self, teacher_config, teacher_ckpt, student_config, student_ckpt,
is_training, task_type, num_labels, use_one_hot_embeddings=False,
is_predistill=True, is_att_fit=True, is_rep_fit=True,
temperature=1.0, dropout_prob=0.1):
super(BertNetworkWithLoss_td, self).__init__()
# load teacher model
self.teacher = BertModelCLS(teacher_config, False, num_labels, dropout_prob,
use_one_hot_embeddings, "teacher")
param_dict = load_checkpoint(teacher_ckpt)
new_param_dict = {}
for key, value in param_dict.items():
new_key = re.sub('^bert.', 'teacher.', key)
new_param_dict[new_key] = value
load_param_into_net(self.teacher, new_param_dict)
# no_grad
self.teacher.set_train(False)
params = self.teacher.trainable_params()
for param in params:
param.requires_grad = False
# load student model
self.bert = BertModelCLS(student_config, is_training, num_labels, dropout_prob,
use_one_hot_embeddings, "student")
param_dict = load_checkpoint(student_ckpt)
if is_predistill:
new_param_dict = {}
for key, value in param_dict.items():
new_key = re.sub('tinybert_', 'bert_', 'bert.' + key)
new_param_dict[new_key] = value
load_param_into_net(self.bert, new_param_dict)
else:
new_param_dict = {}
for key, value in param_dict.items():
new_key = re.sub('tinybert_', 'bert_', key)
new_param_dict[new_key] = value
load_param_into_net(self.bert, new_param_dict)
self.cast = P.Cast()
self.fit_dense = nn.Dense(student_config.hidden_size,
teacher_config.hidden_size).to_float(teacher_config.compute_type)
self.teacher_layers_num = teacher_config.num_hidden_layers
self.student_layers_num = student_config.num_hidden_layers
self.layers_per_block = int(self.teacher_layers_num / self.student_layers_num)
self.is_predistill = is_predistill
self.is_att_fit = is_att_fit
self.is_rep_fit = is_rep_fit
self.task_type = task_type
self.temperature = temperature
self.loss_mse = nn.MSELoss()
self.select = P.Select()
self.zeroslike = P.ZerosLike()
self.dtype = student_config.dtype
self.num_labels = num_labels
self.dtype = teacher_config.dtype
self.soft_cross_entropy = SoftCrossEntropy()
def __init__(self, config, batch_size, num_bboxes, add_gt_as_proposals):
super(BboxAssignSample, self).__init__()
cfg = config
self.batch_size = batch_size
self.neg_iou_thr = Tensor(cfg.neg_iou_thr, mstype.float16)
self.pos_iou_thr = Tensor(cfg.pos_iou_thr, mstype.float16)
self.min_pos_iou = Tensor(cfg.min_pos_iou, mstype.float16)
self.zero_thr = Tensor(0.0, mstype.float16)
self.num_bboxes = num_bboxes
self.num_gts = cfg.num_gts
self.num_expected_pos = cfg.num_expected_pos
self.num_expected_neg = cfg.num_expected_neg
self.add_gt_as_proposals = add_gt_as_proposals
if self.add_gt_as_proposals:
self.label_inds = Tensor(np.arange(1, self.num_gts + 1))
self.concat = P.Concat(axis=0)
self.max_gt = P.ArgMaxWithValue(axis=0)
self.max_anchor = P.ArgMaxWithValue(axis=1)
self.sum_inds = P.ReduceSum()
self.iou = P.IOU()
self.greaterequal = P.GreaterEqual()
self.greater = P.Greater()
self.select = P.Select()
self.gatherND = P.GatherNd()
self.squeeze = P.Squeeze()
self.cast = P.Cast()
self.logicaland = P.LogicalAnd()
self.less = P.Less()
self.random_choice_with_mask_pos = P.RandomChoiceWithMask(self.num_expected_pos)
self.random_choice_with_mask_neg = P.RandomChoiceWithMask(self.num_expected_neg)
self.reshape = P.Reshape()
self.equal = P.Equal()
self.bounding_box_encode = BoundingBoxEncode()
self.scatterNdUpdate = P.ScatterNdUpdate()
self.scatterNd = P.ScatterNd()
self.logicalnot = P.LogicalNot()
self.tile = P.Tile()
self.zeros_like = P.ZerosLike()
self.assigned_gt_inds = Tensor(np.array(-1 * np.ones(num_bboxes), dtype=np.int32))
self.assigned_gt_zeros = Tensor(np.array(np.zeros(num_bboxes), dtype=np.int32))
self.assigned_gt_ones = Tensor(np.array(np.ones(num_bboxes), dtype=np.int32))
self.assigned_gt_ignores = Tensor(np.array(-1 * np.ones(num_bboxes), dtype=np.int32))
self.assigned_pos_ones = Tensor(np.array(np.ones(self.num_expected_pos), dtype=np.int32))
self.check_neg_mask = Tensor(np.array(np.ones(self.num_expected_neg - self.num_expected_pos), dtype=np.bool))
self.range_pos_size = Tensor(np.arange(self.num_expected_pos).astype(np.float16))
self.check_gt_one = Tensor(np.array(-1 * np.ones((self.num_gts, 4)), dtype=np.float16))
self.check_anchor_two = Tensor(np.array(-2 * np.ones((self.num_bboxes, 4)), dtype=np.float16))
self.print = P.Print()
def __init__(self, latent_prior='Normal', output_prior='Normal'):
super(ELBO, self).__init__()
self.sum = P.ReduceSum()
self.zeros = P.ZerosLike()
if latent_prior == 'Normal':
self.posterior = Normal()
else:
raise ValueError(
'The values of latent_prior now only support Normal')
if output_prior == 'Normal':
self.recon_loss = MSELoss(reduction='sum')
else:
raise ValueError(
'The values of output_dis now only support Normal')
def __init__(self,
low=None,
high=None,
seed=None,
dtype=mstype.float32,
name="Uniform"):
"""
Constructor of Uniform distribution.
"""
param = dict(locals())
valid_dtype = mstype.float_type
check_type(dtype, valid_dtype, type(self).__name__)
super(Uniform, self).__init__(seed, dtype, name, param)
self.parameter_type = set_param_type({
'low': low,
'high': high
}, self.dtype)
if low is not None and high is not None:
self._low = cast_to_tensor(low, self.parameter_type)
self._high = cast_to_tensor(high, self.parameter_type)
check_greater(self.low, self.high, "low value", "high value")
else:
self._low = low if low is None else cast_to_tensor(
low, self.parameter_type)
self._high = high if high is None else cast_to_tensor(
high, self.parameter_type)
self.default_parameters = [self.low, self.high]
self.parameter_names = ['low', 'high']
# ops needed for the class
self.exp = exp_generic
self.log = log_generic
self.squeeze = P.Squeeze(0)
self.cast = P.Cast()
self.const = P.ScalarToArray()
self.dtypeop = P.DType()
self.fill = P.Fill()
self.less = P.Less()
self.lessequal = P.LessEqual()
self.logicaland = P.LogicalAnd()
self.select = P.Select()
self.shape = P.Shape()
self.sq = P.Square()
self.sqrt = P.Sqrt()
self.zeroslike = P.ZerosLike()
self.uniform = C.uniform
self.sametypeshape = P.SameTypeShape()
def __init__(
self,
d_min=1e-3,
d_max=1.0,
num_rbf=32,
sigma=None,
trainable=False,
min_cutoff=False,
max_cutoff=False,
):
super().__init__()
if d_max <= d_min:
raise ValueError(
'The argument "d_max" must be larger' +
'than the argument "d_min" in LogGaussianDistribution!')
if d_min <= 0:
raise ValueError('The argument "d_min" must be ' +
' larger than 0 in LogGaussianDistribution!')
self.d_max = d_max
self.d_min = d_min / d_max
self.min_cutoff = min_cutoff
self.max_cutoff = max_cutoff
self.log = P.Log()
self.exp = P.Exp()
self.max = P.Maximum()
self.min = P.Minimum()
self.zeroslike = P.ZerosLike()
self.oneslike = P.OnesLike()
# linspace = nn.LinSpace(log_dmin,0,n_gaussians)
log_dmin = math.log(self.d_min)
# self.centers = linspace()
# self.ones = self.oneslike(self.centers)
centers = np.linspace(log_dmin, 0, num_rbf)
self.centers = Tensor(centers, ms.float32)
ones = np.ones_like(centers)
self.ones = Tensor(ones, ms.float32)
if sigma is None:
sigma = -log_dmin / (num_rbf - 1)
self.rescale = -0.5 / (sigma * sigma)
def __init__(self,
mean=None,
sd=None,
seed=None,
dtype=mstype.float32,
name="Normal"):
"""
Constructor of Normal.
"""
param = dict(locals())
valid_dtype = mstype.float_type
check_type(dtype, valid_dtype, type(self).__name__)
super(Normal, self).__init__(seed, dtype, name, param)
self.parameter_type = set_param_type(
{'mean': mean, 'sd': sd}, self.dtype)
if mean is not None and sd is not None:
self._mean_value = cast_to_tensor(mean, self.parameter_type)
self._sd_value = cast_to_tensor(sd, self.parameter_type)
check_greater_zero(self._sd_value, "Standard deviation")
else:
self._mean_value = mean if mean is None else cast_to_tensor(
mean, self.parameter_type)
self._sd_value = sd if sd is None else cast_to_tensor(
sd, self.parameter_type)
self.default_parameters = [self._mean_value, self._sd_value]
self.parameter_names = ['mean', 'sd']
# ops needed for the class
self.exp = exp_generic
self.expm1 = expm1_generic
self.log = log_generic
self.erf = P.Erf()
self.squeeze = P.Squeeze(0)
self.cast = P.Cast()
self.const = P.ScalarToArray()
self.fill = P.Fill()
self.shape = P.Shape()
self.sq = P.Square()
self.sqrt = P.Sqrt()
self.zeroslike = P.ZerosLike()
self.dtypeop = P.DType()
self.sametypeshape = P.SameTypeShape()
def construct(self, x, mean, variance, global_step):
if self.is_gpu:
if self.training:
batch_mean, batch_std, running_mean, running_std = self.bn_train(
x, mean, variance, global_step)
else:
batch_mean, batch_std, running_mean, running_std = self.bn_infer(
x, mean, variance, global_step)
else:
if self.training:
x_sum, x_square_sum = self.bn_reduce(x)
_, batch_mean, batch_std, running_mean, running_std, mean_updated, variance_updated = \
self.bn_update(x, x_sum, x_square_sum, mean, variance)
P.Assign()(mean, mean_updated)
P.Assign()(variance, variance_updated)
else:
batch_mean = P.ZerosLike()(variance)
batch_std = P.OnesLike()(variance)
running_mean = P.TensorAdd()(mean, 0.)
running_std = P.Sqrt()(P.TensorAdd()(variance, self.epsilon))
return batch_mean, batch_std, running_mean, running_std
def __init__(self):
super(MaskConv, self).__init__()
self.zeros = P.ZerosLike()
self.conv1 = nn.Conv2d(1,
32,
kernel_size=(41, 11),
stride=(2, 2),
pad_mode='pad',
padding=(20, 20, 5, 5))
self.bn1 = nn.BatchNorm2d(num_features=32)
self.conv2 = nn.Conv2d(32,
32,
kernel_size=(21, 11),
stride=(2, 1),
pad_mode='pad',
padding=(10, 10, 5, 5))
self.bn2 = nn.BatchNorm2d(num_features=32)
self.tanh = nn.Tanh()
self._initialize_weights()
self.module_list = nn.CellList(
[self.conv1, self.bn1, self.tanh, self.conv2, self.bn2, self.tanh])
def __init__(self,
low=None,
high=None,
seed=None,
dtype=mstype.float32,
name="Uniform"):
"""
Constructor of Uniform distribution.
"""
param = dict(locals())
param['param_dict'] = {'low': low, 'high': high}
valid_dtype = mstype.float_type
Validator.check_type_name("dtype", dtype, valid_dtype,
type(self).__name__)
super(Uniform, self).__init__(seed, dtype, name, param)
self._low = self._add_parameter(low, 'low')
self._high = self._add_parameter(high, 'high')
if self.low is not None and self.high is not None:
check_greater(self.low, self.high, 'low', 'high')
# ops needed for the class
self.exp = exp_generic
self.log = log_generic
self.squeeze = P.Squeeze(0)
self.cast = P.Cast()
self.const = P.ScalarToArray()
self.dtypeop = P.DType()
self.fill = P.Fill()
self.less = P.Less()
self.lessequal = P.LessEqual()
self.logicaland = P.LogicalAnd()
self.select = P.Select()
self.shape = P.Shape()
self.sq = P.Square()
self.zeroslike = P.ZerosLike()
self.uniform = C.uniform
update_accu_grads = C.MultitypeFuncGraph("update_accu_grads")
@update_accu_grads.register("Tensor", "Tensor")
def _update_accu_grads(accu_grad, grad):
succ = True
return F.depend(succ, F.assign(accu_grad, cast(grad, mstype.float32)))
accumulate_accu_grads = C.MultitypeFuncGraph("accumulate_accu_grads")
@accumulate_accu_grads.register("Tensor", "Tensor")
def _accumulate_accu_grads(accu_grad, grad):
succ = True
return F.depend(succ, F.assign_add(accu_grad, cast(grad, mstype.float32)))
zeroslike = P.ZerosLike()
reset_accu_grads = C.MultitypeFuncGraph("reset_accu_grads")
@reset_accu_grads.register("Tensor")
def _reset_accu_grads(accu_grad):
succ = True
return F.depend(succ, F.assign(accu_grad, zeroslike(accu_grad)))
class BertTrainAccumulationAllReducePostWithLossScaleCell(nn.Cell):
"""
Encapsulation class of bert network training.
Append an optimizer to the training network after that the construct
function can be called to create the backward graph.
def __init__(self,
batch_size,
seq_length,
vocab_size,
decoder,
beam_width=4,
length_penalty_weight=1.0,
max_decode_length=128,
sos_id=1,
eos_id=2,
compute_type=mstype.float32):
super(BeamSearchDecoder, self).__init__(auto_prefix=False)
self.seq_length = seq_length
self.batch_size = batch_size
self.vocab_size = vocab_size
self.beam_width = beam_width
self.length_penalty_weight = length_penalty_weight
self.max_decode_length = max_decode_length
self.decoder = decoder
self.add = P.TensorAdd()
self.expand = P.ExpandDims()
self.reshape = P.Reshape()
self.shape_flat = (-1, )
self.shape = P.Shape()
self.zero_tensor = Tensor(np.zeros([batch_size, beam_width]),
mstype.float32)
self.ninf_tensor = Tensor(np.full([batch_size, beam_width], -INF),
mstype.float32)
self.select = P.Select()
self.flat_shape = (batch_size, beam_width * vocab_size)
self.topk = P.TopK(sorted=True)
self.floor_div = P.FloorDiv()
self.vocab_size_tensor = Tensor(self.vocab_size, mstype.int32)
self.real_div = P.RealDiv()
self.mod = Mod()
self.equal = P.Equal()
self.eos_ids = Tensor(np.full([batch_size, beam_width], eos_id),
mstype.int32)
beam_ids = np.tile(
np.arange(beam_width).reshape((1, beam_width)), [batch_size, 1])
self.beam_ids = Tensor(beam_ids, mstype.int32)
batch_ids = np.arange(batch_size * beam_width).reshape(
(batch_size, beam_width)) // beam_width
self.batch_ids = Tensor(batch_ids, mstype.int32)
self.concat = P.Concat(axis=-1)
self.gather_nd = P.GatherNd()
self.greater_equal = P.GreaterEqual()
self.sub = P.Sub()
self.cast = P.Cast()
self.zeroslike = P.ZerosLike()
# init inputs and states
self.start_ids = Tensor(np.full([batch_size * beam_width, 1], sos_id),
mstype.int32)
self.init_seq = Tensor(np.full([batch_size, beam_width, 1], sos_id),
mstype.int32)
init_scores = np.tile(np.array([[0.] + [-INF] * (beam_width - 1)]),
[batch_size, 1])
self.init_scores = Tensor(init_scores, mstype.float32)
self.init_finished = Tensor(
np.zeros([batch_size, beam_width], dtype=np.bool))
self.init_length = Tensor(
np.zeros([batch_size, beam_width], dtype=np.int32))
self.length_penalty = LengthPenalty(weight=length_penalty_weight)
self.one = Tensor(1, mstype.int32)
def __init__(self):
super(ZerosLikeDynamicNet, self).__init__()
self.gpu_convert_to_dynamic_shape = inner.GpuConvertToDynamicShape()
self.zeros_like = P.ZerosLike()
