def __init__(self, config):
super(DeepFMModel, self).__init__()
self.batch_size = config.batch_size
self.field_size = config.data_field_size
self.vocab_size = config.data_vocab_size
self.emb_dim = config.data_emb_dim
self.deep_layer_dims_list, self.deep_layer_act = config.deep_layer_args
self.init_args = config.init_args
self.weight_bias_init = config.weight_bias_init
self.keep_prob = config.keep_prob
init_acts = [('W_l2', [self.vocab_size, 1], 'normal'),
('V_l2', [self.vocab_size, self.emb_dim], 'normal')]
var_map = init_var_dict(self.init_args, init_acts)
self.fm_w = var_map["W_l2"]
self.embedding_table = var_map["V_l2"]
" Deep Layers "
self.deep_input_dims = self.field_size * self.emb_dim
self.all_dim_list = [self.deep_input_dims
] + self.deep_layer_dims_list + [1]
self.dense_layer_1 = DenseLayer(self.all_dim_list[0],
self.all_dim_list[1],
self.weight_bias_init,
self.deep_layer_act,
self.keep_prob,
convert_dtype=True)
self.dense_layer_2 = DenseLayer(self.all_dim_list[1],
self.all_dim_list[2],
self.weight_bias_init,
self.deep_layer_act,
self.keep_prob,
convert_dtype=True)
self.dense_layer_3 = DenseLayer(self.all_dim_list[2],
self.all_dim_list[3],
self.weight_bias_init,
self.deep_layer_act,
self.keep_prob,
convert_dtype=True)
self.dense_layer_4 = DenseLayer(self.all_dim_list[3],
self.all_dim_list[4],
self.weight_bias_init,
self.deep_layer_act,
self.keep_prob,
convert_dtype=True)
self.dense_layer_5 = DenseLayer(self.all_dim_list[4],
self.all_dim_list[5],
self.weight_bias_init,
self.deep_layer_act,
self.keep_prob,
convert_dtype=True,
use_act=False)
" FM, linear Layers "
self.Gatherv2 = P.GatherV2()
self.Mul = P.Mul()
self.ReduceSum = P.ReduceSum(keep_dims=False)
self.Reshape = P.Reshape()
self.Square = P.Square()
self.Shape = P.Shape()
self.Tile = P.Tile()
self.Concat = P.Concat(axis=1)
self.Cast = P.Cast()
def __init__(self, config, batch_size, num_bboxes, add_gt_as_proposals):
super(BboxAssignSampleForRcnn, self).__init__()
cfg = config
self.use_ambigous_sample = cfg.use_ambigous_sample
self.batch_size = batch_size
self.neg_iou_thr = cfg.neg_iou_thr_stage2
self.pos_iou_thr = cfg.pos_iou_thr_stage2
self.min_pos_iou = cfg.min_pos_iou_stage2
self.num_gts = cfg.num_gts
self.num_bboxes = num_bboxes
self.num_expected_pos = cfg.num_expected_pos_stage2
self.num_expected_amb = cfg.num_expected_amb_stage2
self.num_expected_neg = cfg.num_expected_neg_stage2
self.num_expected_total = cfg.num_expected_total_stage2
self.add_gt_as_proposals = add_gt_as_proposals
self.label_inds = Tensor(np.arange(1, self.num_gts + 1).astype(np.int32))
self.add_gt_as_proposals_valid = Tensor(np.array(self.add_gt_as_proposals * np.ones(self.num_gts),
dtype=np.int32))
self.concat = P.Concat(axis=0)
self.max_gt = P.ArgMaxWithValue(axis=0)
self.max_anchor = P.ArgMaxWithValue(axis=1)
self.sum_inds = P.ReduceSum()
self.iou = P.IOU()
self.greaterequal = P.GreaterEqual()
self.greater = P.Greater()
self.select = P.Select()
self.gatherND = P.GatherNd()
self.gatherV2 = P.GatherV2()
self.squeeze = P.Squeeze()
self.cast = P.Cast()
self.logicaland = P.LogicalAnd()
self.less = P.Less()
self.random_choice_with_mask_pos = P.RandomChoiceWithMask(self.num_expected_pos)
self.random_choice_with_mask_amb = P.RandomChoiceWithMask(self.num_expected_amb)
self.random_choice_with_mask_neg = P.RandomChoiceWithMask(self.num_expected_neg)
self.reshape = P.Reshape()
self.equal = P.Equal()
self.bounding_box_encode = P.BoundingBoxEncode(means=(0.0, 0.0, 0.0, 0.0), stds=(0.1, 0.1, 0.2, 0.2))
self.concat_axis1 = P.Concat(axis=1)
self.logicalnot = P.LogicalNot()
self.tile = P.Tile()
# Check
self.check_gt_one = Tensor(np.array(-1 * np.ones((self.num_gts, 4)), dtype=np.float16))
self.check_anchor_two = Tensor(np.array(-2 * np.ones((self.num_bboxes, 4)), dtype=np.float16))
# Init tensor
self.assigned_gt_inds = Tensor(np.array(-1 * np.ones(num_bboxes), dtype=np.int32))
self.assigned_gt_zeros = Tensor(np.array(np.zeros(num_bboxes), dtype=np.int32))
self.assigned_gt_ones = Tensor(np.array(np.ones(num_bboxes), dtype=np.int32))
self.assigned_amb = Tensor(np.array(-3 * np.ones(num_bboxes), dtype=np.int32))
self.assigned_gt_ignores = Tensor(np.array(-1 * np.ones(num_bboxes), dtype=np.int32))
self.assigned_pos_ones = Tensor(np.array(np.ones(self.num_expected_pos), dtype=np.int32))
self.gt_ignores = Tensor(np.array(-1 * np.ones(self.num_gts), dtype=np.int32))
self.range_pos_size = Tensor(np.arange(self.num_expected_pos).astype(np.float16))
self.range_amb_size = Tensor(np.arange(self.num_expected_amb).astype(np.float16))
self.check_neg_mask = Tensor(np.array(np.ones(self.num_expected_neg - self.num_expected_pos), dtype=np.bool))
if self.use_ambigous_sample:
self.check_neg_mask = Tensor(
np.array(np.ones(self.num_expected_neg - self.num_expected_pos - self.num_expected_amb), dtype=np.bool))
check_neg_mask_ignore_end = np.array(np.ones(self.num_expected_neg), dtype=np.bool)
check_neg_mask_ignore_end[-1] = False
self.check_neg_mask_ignore_end = Tensor(check_neg_mask_ignore_end)
self.bboxs_neg_mask = Tensor(np.zeros((self.num_expected_neg, 4), dtype=np.float16))
self.bboxs_amb_mask = Tensor(np.zeros((self.num_expected_amb, 4), dtype=np.float16))
self.labels_neg_mask = Tensor(np.array(np.zeros(self.num_expected_neg), dtype=np.uint8))
self.labels_amb_mask = Tensor(np.array(np.zeros(self.num_expected_amb) + 2, dtype=np.uint8))
self.reshape_shape_pos = (self.num_expected_pos, 1)
self.reshape_shape_amb = (self.num_expected_amb, 1)
self.reshape_shape_neg = (self.num_expected_neg, 1)
self.scalar_zero = Tensor(0.0, dtype=mstype.float16)
self.scalar_neg_iou_thr = Tensor(self.neg_iou_thr, dtype=mstype.float16)
self.scalar_pos_iou_thr = Tensor(self.pos_iou_thr, dtype=mstype.float16)
self.scalar_min_pos_iou = Tensor(self.min_pos_iou, dtype=mstype.float16)
def __init__(self, config):
super(WideDeepModel, self).__init__()
self.batch_size = config.batch_size
host_device_mix = bool(config.host_device_mix)
parameter_server = bool(config.parameter_server)
parallel_mode = context.get_auto_parallel_context("parallel_mode")
is_auto_parallel = parallel_mode in (ParallelMode.SEMI_AUTO_PARALLEL,
ParallelMode.AUTO_PARALLEL)
if is_auto_parallel:
self.batch_size = self.batch_size * get_group_size()
is_field_slice = config.field_slice
sparse = config.sparse
self.field_size = config.field_size
self.vocab_size = config.vocab_size
self.vocab_cache_size = config.vocab_cache_size
self.emb_dim = config.emb_dim
self.deep_layer_dims_list = config.deep_layer_dim
self.deep_layer_act = config.deep_layer_act
self.init_args = config.init_args
self.weight_init, self.bias_init = config.weight_bias_init
self.weight_bias_init = config.weight_bias_init
self.emb_init = config.emb_init
self.drop_out = config.dropout_flag
self.keep_prob = config.keep_prob
self.deep_input_dims = self.field_size * self.emb_dim
self.layer_dims = self.deep_layer_dims_list + [1]
self.all_dim_list = [self.deep_input_dims] + self.layer_dims
init_acts = [('Wide_b', [1], self.emb_init)]
var_map = init_var_dict(self.init_args, init_acts)
self.wide_b = var_map["Wide_b"]
self.dense_layer_1 = DenseLayer(self.all_dim_list[0],
self.all_dim_list[1],
self.weight_bias_init,
self.deep_layer_act,
convert_dtype=True,
drop_out=config.dropout_flag)
self.dense_layer_2 = DenseLayer(self.all_dim_list[1],
self.all_dim_list[2],
self.weight_bias_init,
self.deep_layer_act,
convert_dtype=True,
drop_out=config.dropout_flag)
self.dense_layer_3 = DenseLayer(self.all_dim_list[2],
self.all_dim_list[3],
self.weight_bias_init,
self.deep_layer_act,
convert_dtype=True,
drop_out=config.dropout_flag)
self.dense_layer_4 = DenseLayer(self.all_dim_list[3],
self.all_dim_list[4],
self.weight_bias_init,
self.deep_layer_act,
convert_dtype=True,
drop_out=config.dropout_flag)
self.dense_layer_5 = DenseLayer(self.all_dim_list[4],
self.all_dim_list[5],
self.weight_bias_init,
self.deep_layer_act,
use_activation=False,
convert_dtype=True,
drop_out=config.dropout_flag)
self.wide_mul = P.Mul()
self.deep_mul = P.Mul()
self.reduce_sum = P.ReduceSum(keep_dims=False)
self.reshape = P.Reshape()
self.deep_reshape = P.Reshape()
self.square = P.Square()
self.shape = P.Shape()
self.tile = P.Tile()
self.concat = P.Concat(axis=1)
self.cast = P.Cast()
self.unique = P.Unique().shard(((1, ), ))
self.wide_gatherv2 = P.Gather()
self.deep_gatherv2 = P.Gather()
if is_auto_parallel and sparse and not is_field_slice and not parameter_server:
target = 'DEVICE'
if host_device_mix:
target = 'CPU'
self.wide_embeddinglookup = nn.EmbeddingLookup(
self.vocab_size,
1,
target=target,
slice_mode=nn.EmbeddingLookup.TABLE_ROW_SLICE)
if config.deep_table_slice_mode == "column_slice":
self.deep_embeddinglookup = nn.EmbeddingLookup(
self.vocab_size,
self.emb_dim,
target=target,
slice_mode=nn.EmbeddingLookup.TABLE_COLUMN_SLICE)
self.dense_layer_1.dropout.dropout.shard(
((1, get_group_size()), ))
self.dense_layer_1.matmul.shard(
((1, get_group_size()), (get_group_size(), 1)))
self.dense_layer_1.matmul.add_prim_attr(
"field_size", self.field_size)
self.deep_mul.shard(((1, 1, get_group_size()), (1, 1, 1)))
self.deep_reshape.add_prim_attr("skip_redistribution", True)
else:
self.deep_embeddinglookup = nn.EmbeddingLookup(
self.vocab_size,
self.emb_dim,
target=target,
slice_mode=nn.EmbeddingLookup.TABLE_ROW_SLICE)
self.reduce_sum.add_prim_attr("cross_batch", True)
self.embedding_table = self.deep_embeddinglookup.embedding_table
elif is_auto_parallel and host_device_mix and is_field_slice and config.full_batch and config.manual_shape:
manual_shapes = tuple((s[0] for s in config.manual_shape))
self.deep_embeddinglookup = nn.EmbeddingLookup(
self.vocab_size,
self.emb_dim,
slice_mode=nn.EmbeddingLookup.FIELD_SLICE,
manual_shapes=manual_shapes)
self.wide_embeddinglookup = nn.EmbeddingLookup(
self.vocab_size,
1,
slice_mode=nn.EmbeddingLookup.FIELD_SLICE,
manual_shapes=manual_shapes)
self.deep_mul.shard(
((1, get_group_size(), 1), (1, get_group_size(), 1)))
self.wide_mul.shard(
((1, get_group_size(), 1), (1, get_group_size(), 1)))
self.reduce_sum.shard(((1, get_group_size(), 1), ))
self.dense_layer_1.dropout.dropout.shard(((1, get_group_size()), ))
self.dense_layer_1.matmul.shard(
((1, get_group_size()), (get_group_size(), 1)))
self.embedding_table = self.deep_embeddinglookup.embedding_table
elif parameter_server:
cache_enable = self.vocab_cache_size > 0
target = 'DEVICE' if cache_enable else 'CPU'
if not cache_enable:
sparse = True
if is_auto_parallel and config.full_batch and cache_enable:
self.deep_embeddinglookup = nn.EmbeddingLookup(
self.vocab_size,
self.emb_dim,
target=target,
slice_mode=nn.EmbeddingLookup.TABLE_ROW_SLICE,
sparse=sparse,
vocab_cache_size=self.vocab_cache_size)
self.wide_embeddinglookup = nn.EmbeddingLookup(
self.vocab_size,
1,
target=target,
slice_mode=nn.EmbeddingLookup.TABLE_ROW_SLICE,
sparse=sparse,
vocab_cache_size=self.vocab_cache_size)
else:
self.deep_embeddinglookup = nn.EmbeddingLookup(
self.vocab_size,
self.emb_dim,
target=target,
sparse=sparse,
vocab_cache_size=self.vocab_cache_size)
self.wide_embeddinglookup = nn.EmbeddingLookup(
self.vocab_size,
1,
target=target,
sparse=sparse,
vocab_cache_size=self.vocab_cache_size)
self.embedding_table = self.deep_embeddinglookup.embedding_table
self.deep_embeddinglookup.embedding_table.set_param_ps()
self.wide_embeddinglookup.embedding_table.set_param_ps()
else:
self.deep_embeddinglookup = nn.EmbeddingLookup(
self.vocab_size,
self.emb_dim,
target='DEVICE',
sparse=sparse,
vocab_cache_size=self.vocab_cache_size)
self.wide_embeddinglookup = nn.EmbeddingLookup(
self.vocab_size,
1,
target='DEVICE',
sparse=sparse,
vocab_cache_size=self.vocab_cache_size)
self.embedding_table = self.deep_embeddinglookup.embedding_table
def __init__(self):
super(MSELoss, self).__init__()
self.reduce_sum = P.ReduceSum()
self.square = P.Square()
self.reduce_mean = P.ReduceMean()
def construct(self, logits, label):
label = self.one_hot(label,
F.shape(logits)[1], self.on_value, self.off_value)
loss = self.cross_entropy(logits, label)[0]
loss = P.RealDiv()(P.ReduceSum()(loss, -1), self.num)
return loss
def __init__(self, network):
super(FeatureCollectCell, self).__init__(auto_prefix=False)
self._network = network
self.shape = P.Shape()
self.sum = P.ReduceSum()
def __init__(self):
super(AxisListDefaultNet, self).__init__()
self.reduce_sum = P.ReduceSum()
def test_exec():
context.set_context(mode=context.GRAPH_MODE)
return test_exec_case
raise_set = [
('Squeeze_1_Error', {
'block': (lambda x: P.Squeeze(axis=1.2), {
'exception': ValueError
}),
'desc_inputs': [Tensor(np.ones(shape=[3, 1, 5]))]
}),
('Squeeze_2_Error', {
'block': (lambda x: P.Squeeze(axis=((1.2, 1.3))), {
'exception': ValueError
}),
'desc_inputs': [Tensor(np.ones(shape=[3, 1, 5]))]
}),
('ReduceSum_Error', {
'block': (lambda x: P.ReduceSum(keep_dims=1), {
'exception': ValueError
}),
'desc_inputs': [Tensor(np.ones(shape=[3, 1, 5]))]
}),
]
@mindspore_test(pipeline_for_verify_exception_for_case_by_case_config)
def test_check_exception():
return raise_set
def __init__(self):
super(CustNet3, self).__init__()
self.op = P.ReduceSum()
self.t1 = Tensor(np.ones([2, 2]), dtype=ms.int32)
self.t2 = Tensor(np.ones([1, 5]), dtype=ms.float16)
self.t2 = 1
def __init__(self):
super(SquaredLoss, self).__init__()
self.reshape = P.Reshape()
self.shape = P.Shape()
self.two = Tensor(np.array([2.0]).astype(np.float32))
self.reduce_sum = P.ReduceSum()
def __init__(self,
batch_size,
input_size,
hidden_size,
num_layers,
bidirectional=False,
batch_norm=False,
rnn_type='LSTM',
device_target="GPU"):
super(BatchRNN, self).__init__()
self.batch_size = batch_size
self.input_size = input_size
self.hidden_size = hidden_size
self.num_layers = num_layers
self.rnn_type = rnn_type
self.bidirectional = bidirectional
self.has_bias = True
self.is_batch_norm = batch_norm
self.num_directions = 2 if bidirectional else 1
self.reshape_op = P.Reshape()
self.shape_op = P.Shape()
self.sum_op = P.ReduceSum()
input_size_list = [input_size]
for i in range(num_layers - 1):
input_size_list.append(hidden_size)
layers = []
for i in range(num_layers):
layers.append(
nn.LSTMCell(input_size=input_size_list[i],
hidden_size=hidden_size,
bidirectional=bidirectional,
has_bias=self.has_bias))
weights = []
for i in range(num_layers):
weight_size = (input_size_list[i] +
hidden_size) * hidden_size * self.num_directions * 4
if self.has_bias:
if device_target == "GPU":
bias_size = self.num_directions * hidden_size * 4 * 2
else:
bias_size = self.num_directions * hidden_size * 4
weight_size = weight_size + bias_size
stdv = 1 / math.sqrt(hidden_size)
w_np = np.random.uniform(-stdv, stdv,
(weight_size, 1, 1)).astype(np.float32)
weights.append(
Parameter(initializer(Tensor(w_np), w_np.shape),
name="weight" + str(i)))
self.h, self.c = self.stack_lstm_default_state(
batch_size,
hidden_size,
num_layers=num_layers,
bidirectional=bidirectional)
self.lstms = layers
self.weight = ParameterTuple(tuple(weights))
if batch_norm:
batch_norm_layer = []
for i in range(num_layers - 1):
batch_norm_layer.append(nn.BatchNorm1d(hidden_size))
self.batch_norm_list = batch_norm_layer
def __init__(self, strategy0, strategy1):
super().__init__()
self.fc_nobias = P.MatMul(transpose_b=True).shard(strategy0)
self.reduce_sum = P.ReduceSum(keep_dims=False).shard(strategy1)
def __init__(self,
batch_size,
seq_length,
vocab_size,
decoder,
beam_width=4,
decoder_layers_nums=4,
length_penalty_weight=0.6,
cov_penalty_factor=0.1,
hidden_size=1024,
max_decode_length=64,
sos_id=2,
eos_id=3,
is_using_while=True,
compute_type=mstype.float32):
super(BeamSearchDecoder, self).__init__()
self.encoder_length = seq_length
self.hidden_size = hidden_size
self.batch_size = batch_size
self.vocab_size = vocab_size
self.beam_width = beam_width
self.decoder_layers_nums = decoder_layers_nums
self.length_penalty_weight = length_penalty_weight
self.cov_penalty_factor = cov_penalty_factor
self.max_decode_length = max_decode_length
self.decoder = decoder
self.is_using_while = is_using_while
self.add = P.Add()
self.expand = P.ExpandDims()
self.reshape = P.Reshape()
self.shape_flat = (-1, )
self.shape = P.Shape()
self.zero_tensor = Tensor(np.zeros([batch_size, beam_width]),
mstype.float32)
self.ninf_tensor = Tensor(np.full([batch_size, beam_width], -INF),
mstype.float32)
self.select = P.Select()
self.flat_shape = (batch_size, beam_width * vocab_size)
self.topk = P.TopK(sorted=True)
self.floor_div = P.FloorDiv()
self.vocab_size_tensor = Tensor(self.vocab_size, mstype.int32)
self.real_div = P.RealDiv()
self.mod = Mod()
self.equal = P.Equal()
self.eos_ids = Tensor(np.full([batch_size, beam_width], eos_id),
mstype.int32)
beam_ids = np.tile(
np.arange(beam_width).reshape((1, beam_width)), [batch_size, 1])
self.beam_ids = Tensor(beam_ids, mstype.int32)
batch_ids = np.arange(batch_size * beam_width).reshape(
(batch_size, beam_width)) // beam_width
self.batch_ids = Tensor(batch_ids, mstype.int32)
self.concat = P.Concat(axis=-1)
self.gather_nd = P.GatherNd()
self.start_ids = Tensor(np.full([batch_size * beam_width, 1], sos_id),
mstype.int32)
if self.is_using_while:
self.start = Tensor(0, dtype=mstype.int32)
self.init_seq = Tensor(
np.full([batch_size, beam_width, self.max_decode_length + 1],
sos_id), mstype.int32)
else:
self.init_seq = Tensor(
np.full([batch_size, beam_width, 1], sos_id), mstype.int32)
init_scores = np.tile(np.array([[0.] + [-INF] * (beam_width - 1)]),
[batch_size, 1])
self.init_scores = Tensor(init_scores, mstype.float32)
self.init_finished = Tensor(
np.zeros([batch_size, beam_width], dtype=np.bool))
self.init_length = Tensor(
np.zeros([batch_size, beam_width], dtype=np.int32))
self.length_penalty = LengthPenalty(weight=length_penalty_weight)
self.one = Tensor(1, mstype.int32)
self.prob_concat = P.Concat(axis=1)
self.cast = P.Cast()
self.decoder_hidden_state = Tensor(
np.zeros([
self.decoder_layers_nums, 2, self.batch_size * self.beam_width,
hidden_size
]), mstype.float32)
self.zeros_scores = Tensor(
np.zeros([batch_size, beam_width], dtype=np.float))
self.active_index = Tensor(
np.ones([batch_size, beam_width], dtype=np.int32))
self.init_zeros = Tensor(
np.zeros([batch_size, beam_width], dtype=np.int32))
self.init_ones = Tensor(
np.ones([batch_size, beam_width], dtype=np.float32))
self.accu_attn_scores = Tensor(
np.zeros([batch_size, beam_width, self.encoder_length],
dtype=np.float32))
self.zeros = Tensor([0], mstype.int32)
self.eos_tensor = Tensor(
np.full([batch_size, beam_width, beam_width], eos_id),
mstype.int32)
self.ones_3d = Tensor(
np.full([batch_size, beam_width, self.encoder_length], 1),
mstype.float32)
self.neg_inf_3d = Tensor(
np.full([batch_size, beam_width, self.encoder_length], -INF),
mstype.float32)
self.zeros_3d = Tensor(
np.full([batch_size, beam_width, self.encoder_length], 0),
mstype.float32)
self.zeros_2d = Tensor(
np.full([batch_size * beam_width, self.encoder_length], 0),
mstype.int32)
self.argmin = P.ArgMinWithValue(axis=1)
self.reducesum = P.ReduceSum()
self.div = P.Div()
self.shape_op = P.Shape()
self.mul = P.Mul()
self.log = P.Log()
self.less = P.Less()
self.tile = P.Tile()
self.noteq = P.Neg()
self.zeroslike = P.ZerosLike()
self.greater_equal = P.GreaterEqual()
self.sub = P.Sub()
def __init__(
self,
vocab_size,
embedding_size,
embedding_shape,
use_one_hot_embeddings=False,
initializer_range=0.02,
name='embedding_table',
is_expand=False,
batch_size=12,
damping=0.03,
loss_scale=1,
frequency=10,
):
super(Embedding_Thor, self).__init__()
self.vocab_size = vocab_size
self.use_one_hot_embeddings = use_one_hot_embeddings
self.embedding_table = Parameter(initializer(
TruncatedNormal(initializer_range), [vocab_size, embedding_size]),
name=name)
self.thor = True
self.is_expand = is_expand
self.expand = P.ExpandDims()
self.shape_flat = (-1, )
self.gather = P.GatherV2()
self.one_hot = P.OneHot()
self.on_value = Tensor(1.0, mstype.float32)
self.off_value = Tensor(0.0, mstype.float32)
self.array_mul = P.MatMul()
self.reshape = P.Reshape()
self.em_shape = tuple(embedding_shape)
self.shape = P.Shape()
self.loss_scale = Tensor(1 / loss_scale, mstype.float16)
self.matrix_A_inv = Parameter(Tensor(
np.zeros([vocab_size]).astype(np.float32)),
name='matrix_A_inv',
requires_grad=False)
self.matrix_G_inv = Parameter(Tensor(
np.zeros([embedding_size, embedding_size]).astype(np.float16)),
name="matrix_G_inv",
requires_grad=False)
self.A_inv_max = Parameter(initializer(0, [1], mstype.float32),
name="A_inv_max",
requires_grad=False)
self.G_inv_max = Parameter(initializer(0, [1], mstype.float32),
name="G_inv_max",
requires_grad=False)
self.fused_abs_max = P.CusFusedAbsMax1()
self.fake_G = Tensor(
np.zeros([embedding_size, embedding_size]).astype(np.float16))
self.dampingA = Tensor(np.ones([vocab_size]).astype(np.float32))
self.dampingG = Tensor(np.identity(embedding_size), mstype.float32)
self.cov_step = Parameter(initializer(0, [1], mstype.int32),
name="cov_step",
requires_grad=False)
self.freq = Tensor(frequency, mstype.int32)
self.axis = 0
self.damping = damping
self.gather = P.GatherV2()
self.sqrt = P.Sqrt()
self.mul = P.Mul()
self.cast = P.Cast()
self.cube_matmul = P.CusMatMulCube(transpose_a=True)
self.vector_matmul = P.CusBatchMatMul()
self.cholesky = P.CusCholeskyTrsm()
self.matrix_combine = P.CusMatrixCombine()
self.reduce_sum = P.ReduceSum(keep_dims=False)
self.inv = P.Inv()
self.getG = P.InsertGradientOf(self.save_gradient)
self.batch_size = batch_size
def __init__(self):
super().__init__()
self.normal_dist = msd.Normal(dtype=mstype.float32)
self.bernoulli_dist = msd.Bernoulli(dtype=mstype.float32)
self.reduce_sum = P.ReduceSum(keep_dims=True)
def __init__(
self,
in_channels,
out_channels,
activation=None,
has_bias=True,
weight_prior_fn=NormalPrior,
weight_posterior_fn=lambda name, shape: NormalPosterior(name=name, shape=shape),
bias_prior_fn=NormalPrior,
bias_posterior_fn=lambda name, shape: NormalPosterior(name=name, shape=shape)):
super(_DenseVariational, self).__init__()
self.in_channels = check_int_positive(in_channels)
self.out_channels = check_int_positive(out_channels)
self.has_bias = check_bool(has_bias)
if isinstance(weight_prior_fn, Cell):
self.weight_prior = weight_prior_fn
else:
self.weight_prior = weight_prior_fn()
for prior_name, prior_dist in self.weight_prior.name_cells().items():
if prior_name != 'normal':
raise TypeError("The type of distribution of `weight_prior_fn` should be `normal`")
if not (isinstance(getattr(prior_dist, '_mean_value'), Tensor) and
isinstance(getattr(prior_dist, '_sd_value'), Tensor)):
raise TypeError("The input form of `weight_prior_fn` is incorrect")
try:
self.weight_posterior = weight_posterior_fn(shape=[self.out_channels, self.in_channels], name='bnn_weight')
except TypeError:
raise TypeError('The type of `weight_posterior_fn` should be `NormalPosterior`')
for posterior_name, _ in self.weight_posterior.name_cells().items():
if posterior_name != 'normal':
raise TypeError("The type of distribution of `weight_posterior_fn` should be `normal`")
if self.has_bias:
if isinstance(bias_prior_fn, Cell):
self.bias_prior = bias_prior_fn
else:
self.bias_prior = bias_prior_fn()
for prior_name, prior_dist in self.bias_prior.name_cells().items():
if prior_name != 'normal':
raise TypeError("The type of distribution of `bias_prior_fn` should be `normal`")
if not (isinstance(getattr(prior_dist, '_mean_value'), Tensor) and
isinstance(getattr(prior_dist, '_sd_value'), Tensor)):
raise TypeError("The input form of `bias_prior_fn` is incorrect")
try:
self.bias_posterior = bias_posterior_fn(shape=[self.out_channels], name='bnn_bias')
except TypeError:
raise TypeError('The type of `bias_posterior_fn` should be `NormalPosterior`')
for posterior_name, _ in self.bias_posterior.name_cells().items():
if posterior_name != 'normal':
raise TypeError("The type of distribution of `bias_posterior_fn` should be `normal`")
self.activation = activation
if not self.activation:
self.activation_flag = False
else:
self.activation_flag = True
if isinstance(self.activation, str):
self.activation = get_activation(activation)
elif isinstance(self.activation, Cell):
self.activation = activation
else:
raise ValueError('The type of `activation` is wrong.')
self.matmul = P.MatMul(transpose_b=True)
self.bias_add = P.BiasAdd()
self.sum = P.ReduceSum()
def __init__(self, axis=-1):
super(Softmax, self).__init__()
self.max = P.ArgMaxWithValue(axis=axis, keep_dims=True)
self.sum = P.ReduceSum(keep_dims=True)
self.axis = axis
'desc_inputs': [[128, 5]],
'desc_bprop': [[128, 5], [128], [128]],
'skip': ['backward']
}), ('Abs', {
'block': P.Abs(),
'desc_inputs': [[4]],
'desc_bprop': [[4]]
}),
('CumSum', {
'block': P.CumSum(),
'desc_const': [0],
'desc_inputs': [Tensor(np.array([[3, 4], [1, 6]]).astype(np.float16))],
'desc_bprop': [Tensor(np.array([[3, 4], [4, 10]]).astype(np.float16))]
}),
('ReduceSum_3', {
'block': P.ReduceSum(),
'desc_const': [0],
'desc_inputs': [[3, 2]],
'desc_bprop': [[2]]
}),
('ReduceSum_4', {
'block': P.ReduceSum(keep_dims=True),
'desc_const': [0],
'desc_inputs': [[3, 2]],
'desc_bprop': [[1, 2]]
}),
('ReduceSum_5', {
'block': P.ReduceSum(keep_dims=True),
'desc_inputs': [[2, 3, 4]],
'desc_bprop': [[1, 1, 1]]
}),
def __init__(self):
super(AxisListEmptyNet, self).__init__()
self.reduce_sum = P.ReduceSum()
self.axis = []
def __init__(self):
super().__init__()
self.mul1 = P.Mul()
self.reduce_sum = P.ReduceSum(keep_dims=False)
self.mul2 = P.Mul()
def _get_square_sum(x):
norm = P.ReduceSum(False)(F.square(x), ())
norm = F.expand_dims(F.cast(norm, mstype.float32), 0)
return norm
def __init__(self,
in_channels,
out_channels,
kernel_size,
stride=1,
pad_mode='same',
padding=0,
dilation=1,
group=1,
has_bias=False,
weight_prior_fn=NormalPrior,
weight_posterior_fn=lambda name, shape: NormalPosterior(
name=name, shape=shape),
bias_prior_fn=NormalPrior,
bias_posterior_fn=lambda name, shape: NormalPosterior(
name=name, shape=shape)):
kernel_size = twice(kernel_size)
stride = twice(stride)
dilation = twice(dilation)
super(_ConvVariational, self).__init__(in_channels,
out_channels,
kernel_size,
stride,
pad_mode,
padding,
dilation,
group,
has_bias,
weight_init='normal',
bias_init='zeros')
if pad_mode not in ('valid', 'same', 'pad'):
raise ValueError(
'Attr \'pad_mode\' of \'Conv2d\' Op passed ' + str(pad_mode) +
', should be one of values in \'valid\', \'same\', \'pad\'.')
# convolution args
self.in_channels = in_channels
self.out_channels = out_channels
self.kernel_size = kernel_size
self.stride = stride
self.pad_mode = pad_mode
self.padding = padding
self.dilation = dilation
self.group = group
self.has_bias = has_bias
# distribution trainable parameters
self.shape = [
self.out_channels, self.in_channels // self.group,
*self.kernel_size
]
self.weight.requires_grad = False
if isinstance(weight_prior_fn, Cell):
if weight_prior_fn.__class__.__name__ != 'NormalPrior':
raise TypeError(
'The type of `weight_prior_fn` should be `NormalPrior`')
self.weight_prior = weight_prior_fn
else:
if weight_prior_fn.__name__ != 'NormalPrior':
raise TypeError(
'The type of `weight_prior_fn` should be `NormalPrior`')
self.weight_prior = weight_prior_fn()
try:
self.weight_posterior = weight_posterior_fn(shape=self.shape,
name='bnn_weight')
except TypeError:
raise TypeError(
'The type of `weight_posterior_fn` should be `NormalPosterior`'
)
if self.has_bias:
self.bias.requires_grad = False
if isinstance(bias_prior_fn, Cell):
if bias_prior_fn.__class__.__name__ != 'NormalPrior':
raise TypeError(
'The type of `bias_prior_fn` should be `NormalPrior`')
self.bias_prior = bias_prior_fn
else:
if bias_prior_fn.__name__ != 'NormalPrior':
raise TypeError(
'The type of `bias_prior_fn` should be `NormalPrior`')
self.bias_prior = bias_prior_fn()
try:
self.bias_posterior = bias_posterior_fn(
shape=[self.out_channels], name='bnn_bias')
except TypeError:
raise TypeError(
'The type of `bias_posterior_fn` should be `NormalPosterior`'
)
# mindspore operations
self.bias_add = P.BiasAdd()
self.conv2d = P.Conv2D(out_channel=self.out_channels,
kernel_size=self.kernel_size,
mode=1,
pad_mode=self.pad_mode,
pad=self.padding,
stride=self.stride,
dilation=self.dilation,
group=self.group)
self.log = P.Log()
self.sum = P.ReduceSum()
def __init__(self, config, batch_size, num_bboxes, add_gt_as_proposals):
super(BboxAssignSampleForRcnn, self).__init__()
cfg = config
self.batch_size = batch_size
self.neg_iou_thr = cfg.neg_iou_thr_stage2
self.pos_iou_thr = cfg.pos_iou_thr_stage2
self.min_pos_iou = cfg.min_pos_iou_stage2
self.num_gts = cfg.num_gts
self.num_bboxes = num_bboxes
self.num_expected_pos = cfg.num_expected_pos_stage2
self.num_expected_neg = cfg.num_expected_neg_stage2
self.num_expected_total = cfg.num_expected_total_stage2
self.add_gt_as_proposals = add_gt_as_proposals
self.label_inds = Tensor(
np.arange(1, self.num_gts + 1).astype(np.int32))
self.add_gt_as_proposals_valid = Tensor(
np.array(self.add_gt_as_proposals * np.ones(self.num_gts),
dtype=np.int32))
self.concat = P.Concat(axis=0)
self.max_gt = P.ArgMaxWithValue(axis=0)
self.max_anchor = P.ArgMaxWithValue(axis=1)
self.sum_inds = P.ReduceSum()
self.iou = P.IOU()
self.greaterequal = P.GreaterEqual()
self.greater = P.Greater()
self.select = P.Select()
self.gatherND = P.GatherNd()
self.squeeze = P.Squeeze()
self.cast = P.Cast()
self.logicaland = P.LogicalAnd()
self.less = P.Less()
self.random_choice_with_mask_pos = P.RandomChoiceWithMask(
self.num_expected_pos)
self.random_choice_with_mask_neg = P.RandomChoiceWithMask(
self.num_expected_neg)
self.reshape = P.Reshape()
self.equal = P.Equal()
self.bounding_box_encode = P.BoundingBoxEncode(means=(0.0, 0.0, 0.0,
0.0),
stds=(0.1, 0.1, 0.2,
0.2))
self.concat_axis1 = P.Concat(axis=1)
self.logicalnot = P.LogicalNot()
self.tile = P.Tile()
# Check
self.check_gt_one = Tensor(
np.array(-1 * np.ones((self.num_gts, 4)), dtype=np.float16))
self.check_anchor_two = Tensor(
np.array(-2 * np.ones((self.num_bboxes, 4)), dtype=np.float16))
# Init tensor
self.assigned_gt_inds = Tensor(
np.array(-1 * np.ones(num_bboxes), dtype=np.int32))
self.assigned_gt_zeros = Tensor(
np.array(np.zeros(num_bboxes), dtype=np.int32))
self.assigned_gt_ones = Tensor(
np.array(np.ones(num_bboxes), dtype=np.int32))
self.assigned_gt_ignores = Tensor(
np.array(-1 * np.ones(num_bboxes), dtype=np.int32))
self.assigned_pos_ones = Tensor(
np.array(np.ones(self.num_expected_pos), dtype=np.int32))
self.gt_ignores = Tensor(
np.array(-1 * np.ones(self.num_gts), dtype=np.int32))
self.range_pos_size = Tensor(
np.arange(self.num_expected_pos).astype(np.float16))
self.check_neg_mask = Tensor(
np.array(np.ones(self.num_expected_neg - self.num_expected_pos),
dtype=np.bool))
self.bboxs_neg_mask = Tensor(
np.zeros((self.num_expected_neg, 4), dtype=np.float16))
self.labels_neg_mask = Tensor(
np.array(np.zeros(self.num_expected_neg), dtype=np.uint8))
self.reshape_shape_pos = (self.num_expected_pos, 1)
self.reshape_shape_neg = (self.num_expected_neg, 1)
self.scalar_zero = Tensor(0.0, dtype=mstype.float16)
self.scalar_neg_iou_thr = Tensor(self.neg_iou_thr,
dtype=mstype.float16)
self.scalar_pos_iou_thr = Tensor(self.pos_iou_thr,
dtype=mstype.float16)
self.scalar_min_pos_iou = Tensor(self.min_pos_iou,
dtype=mstype.float16)
self.expand_dims = P.ExpandDims()
self.split = P.Split(axis=1, output_num=4)
self.concat_last_axis = P.Concat(axis=-1)
self.round = P.Round()
self.image_h_w = Tensor(
[cfg.img_height, cfg.img_width, cfg.img_height, cfg.img_width],
dtype=mstype.float16)
self.range = nn.Range(start=0, limit=cfg.num_expected_pos_stage2)
self.crop_and_resize = P.CropAndResize(method="bilinear_v2")
self.mask_shape = (cfg.mask_shape[0], cfg.mask_shape[1])
self.squeeze_mask_last = P.Squeeze(axis=-1)
def __init__(self, network, output_num):
super(OutputReduceSumCell, self).__init__()
self.output_num = output_num
self.network = network
self.reduce_sum = P.ReduceSum()
def __init__(self, config):
super(WideDeepModel, self).__init__()
self.batch_size = config.batch_size
host_device_mix = bool(config.host_device_mix)
parameter_server = bool(config.parameter_server)
parallel_mode = _get_parallel_mode()
is_auto_parallel = parallel_mode in (ParallelMode.SEMI_AUTO_PARALLEL,
ParallelMode.AUTO_PARALLEL)
if is_auto_parallel:
self.batch_size = self.batch_size * get_group_size()
self.field_size = config.field_size
self.vocab_size = config.vocab_size
self.emb_dim = config.emb_dim
self.deep_layer_dims_list = config.deep_layer_dim
self.deep_layer_act = config.deep_layer_act
self.init_args = config.init_args
self.weight_init, self.bias_init = config.weight_bias_init
self.weight_bias_init = config.weight_bias_init
self.emb_init = config.emb_init
self.drop_out = config.dropout_flag
self.keep_prob = config.keep_prob
self.deep_input_dims = self.field_size * self.emb_dim
self.layer_dims = self.deep_layer_dims_list + [1]
self.all_dim_list = [self.deep_input_dims] + self.layer_dims
init_acts = [('Wide_b', [1], self.emb_init)]
var_map = init_var_dict(self.init_args, init_acts)
self.wide_b = var_map["Wide_b"]
self.dense_layer_1 = DenseLayer(self.all_dim_list[0],
self.all_dim_list[1],
self.weight_bias_init,
self.deep_layer_act,
convert_dtype=True,
drop_out=config.dropout_flag)
self.dense_layer_2 = DenseLayer(self.all_dim_list[1],
self.all_dim_list[2],
self.weight_bias_init,
self.deep_layer_act,
convert_dtype=True,
drop_out=config.dropout_flag)
self.dense_layer_3 = DenseLayer(self.all_dim_list[2],
self.all_dim_list[3],
self.weight_bias_init,
self.deep_layer_act,
convert_dtype=True,
drop_out=config.dropout_flag)
self.dense_layer_4 = DenseLayer(self.all_dim_list[3],
self.all_dim_list[4],
self.weight_bias_init,
self.deep_layer_act,
convert_dtype=True,
drop_out=config.dropout_flag)
self.dense_layer_5 = DenseLayer(self.all_dim_list[4],
self.all_dim_list[5],
self.weight_bias_init,
self.deep_layer_act,
use_activation=False,
convert_dtype=True,
drop_out=config.dropout_flag)
self.wide_mul = P.Mul()
self.deep_mul = P.Mul()
self.reduce_sum = P.ReduceSum(keep_dims=False)
self.reshape = P.Reshape()
self.deep_reshape = P.Reshape()
self.square = P.Square()
self.shape = P.Shape()
self.tile = P.Tile()
self.concat = P.Concat(axis=1)
self.cast = P.Cast()
if is_auto_parallel and host_device_mix:
self.dense_layer_1.dropout.dropout_do_mask.set_strategy(
((1, get_group_size()), ))
self.dense_layer_1.matmul.set_strategy(
((1, get_group_size()), (get_group_size(), 1)))
self.deep_embeddinglookup = nn.EmbeddingLookup(
self.vocab_size,
self.emb_dim,
slice_mode=nn.EmbeddingLookUpSplitMode.TABLE_COLUMN_SLICE)
self.wide_embeddinglookup = nn.EmbeddingLookup(
self.vocab_size,
1,
slice_mode=nn.EmbeddingLookUpSplitMode.TABLE_ROW_SLICE)
self.deep_mul.set_strategy(((1, 1, get_group_size()), (1, 1, 1)))
self.deep_reshape.add_prim_attr("skip_redistribution", True)
self.reduce_sum.add_prim_attr("cross_batch", True)
self.embedding_table = self.deep_embeddinglookup.embedding_table
elif parameter_server:
self.deep_embeddinglookup = nn.EmbeddingLookup(
self.vocab_size, self.emb_dim)
self.wide_embeddinglookup = nn.EmbeddingLookup(self.vocab_size, 1)
self.embedding_table = self.deep_embeddinglookup.embedding_table
self.deep_embeddinglookup.embedding_table.set_param_ps()
self.wide_embeddinglookup.embedding_table.set_param_ps()
else:
self.deep_embeddinglookup = nn.EmbeddingLookup(self.vocab_size,
self.emb_dim,
target='DEVICE')
self.wide_embeddinglookup = nn.EmbeddingLookup(self.vocab_size,
1,
target='DEVICE')
self.embedding_table = self.deep_embeddinglookup.embedding_table
def __init__(self,
config,
representation_size,
batch_size,
num_classes,
target_means=(0., 0., 0., 0.),
target_stds=(0.1, 0.1, 0.2, 0.2)
):
super(Rcnn, self).__init__()
cfg = config
self.rcnn_loss_cls_weight = Tensor(np.array(cfg.rcnn_loss_cls_weight).astype(np.float16))
self.rcnn_loss_reg_weight = Tensor(np.array(cfg.rcnn_loss_reg_weight).astype(np.float16))
self.rcnn_fc_out_channels = cfg.rcnn_fc_out_channels
self.target_means = target_means
self.target_stds = target_stds
self.num_classes = num_classes
self.in_channels = cfg.rcnn_in_channels
self.train_batch_size = batch_size
self.test_batch_size = cfg.test_batch_size
shape_0 = (self.rcnn_fc_out_channels, representation_size)
weights_0 = initializer("XavierUniform", shape=shape_0[::-1], dtype=mstype.float16).to_tensor()
shape_1 = (self.rcnn_fc_out_channels, self.rcnn_fc_out_channels)
weights_1 = initializer("XavierUniform", shape=shape_1[::-1], dtype=mstype.float16).to_tensor()
self.shared_fc_0 = DenseNoTranpose(representation_size, self.rcnn_fc_out_channels, weights_0)
self.shared_fc_1 = DenseNoTranpose(self.rcnn_fc_out_channels, self.rcnn_fc_out_channels, weights_1)
cls_weight = initializer('Normal', shape=[num_classes, self.rcnn_fc_out_channels][::-1],
dtype=mstype.float16).to_tensor()
reg_weight = initializer('Normal', shape=[num_classes * 4, self.rcnn_fc_out_channels][::-1],
dtype=mstype.float16).to_tensor()
self.cls_scores = DenseNoTranpose(self.rcnn_fc_out_channels, num_classes, cls_weight)
self.reg_scores = DenseNoTranpose(self.rcnn_fc_out_channels, num_classes * 4, reg_weight)
self.flatten = P.Flatten()
self.relu = P.ReLU()
self.logicaland = P.LogicalAnd()
self.loss_cls = P.SoftmaxCrossEntropyWithLogits()
self.loss_bbox = P.SmoothL1Loss(beta=1.0)
self.reshape = P.Reshape()
self.onehot = P.OneHot()
self.greater = P.Greater()
self.cast = P.Cast()
self.sum_loss = P.ReduceSum()
self.tile = P.Tile()
self.expandims = P.ExpandDims()
self.gather = P.GatherNd()
self.argmax = P.ArgMaxWithValue(axis=1)
self.on_value = Tensor(1.0, mstype.float32)
self.off_value = Tensor(0.0, mstype.float32)
self.value = Tensor(1.0, mstype.float16)
self.num_bboxes = (cfg.num_expected_pos_stage2 + cfg.num_expected_neg_stage2) * batch_size
rmv_first = np.ones((self.num_bboxes, self.num_classes))
rmv_first[:, 0] = np.zeros((self.num_bboxes,))
self.rmv_first_tensor = Tensor(rmv_first.astype(np.float16))
self.num_bboxes_test = cfg.rpn_max_num * cfg.test_batch_size
range_max = np.arange(self.num_bboxes_test).astype(np.int32)
self.range_max = Tensor(range_max)
def __init__(self, network):
super(NetWithLoss, self).__init__()
self.sum = P.ReduceSum()
self.mean = P.ReduceMean()
self.net = network
def __init__(self):
super().__init__()
self.max = P.ReduceMax()
self.param = Parameter(Tensor(np.arange(2 * 2 * 2).reshape((2, 2, 2)), ms.float32), name="weight")
self.zero = Tensor(np.zeros(([2, 2, 2])), ms.float32)
self.reduce = P.ReduceSum()
from mindspore.ops.primitive import constexpr, PrimitiveWithInfer, prim_attr_register
from mindspore.ops._grad.grad_base import bprop_getters
from mindspore import Tensor, RowTensor, context
from mindspore.common.parameter import Parameter, ParameterTuple
from mindspore.common import dtype as mstype
from mindspore._checkparam import Validator as validator
from mindspore._checkparam import Rel
from mindspore.nn import Optimizer
from mindspore.nn import TrainOneStepCell, WithLossCell
from mindspore.nn.optim import Momentum
from mindspore.train import Model
from ....dataset_mock import MindData
context.set_context(mode=context.GRAPH_MODE, enable_sparse=True)
reduce_sum = P.ReduceSum()
unsorted_segment_sum = P.UnsortedSegmentSum()
transpose = P.Transpose()
shape_op = P.Shape()
reshape = P.Reshape()
size_op = P.Size()
invert_permutation = P.InvertPermutation()
logical_and = P.LogicalAnd()
def get_axis(x):
shape = shape_op(x)
length = F.tuple_len(shape)
perm = F.make_range(0, length)
return perm
def __init__(self, margin=0.0, reduction="mean"):
super(CosineEmbeddingLoss, self).__init__(reduction)
self.reduce_sum = P.ReduceSum()
self.maximum = P.Maximum()
validator.check_value_type("margin", margin, [float], self.cls_name)
self.margin = validator.check_number_range("margin", margin, -1.0, 1.0, Rel.INC_BOTH, self.cls_name)