def __init__(self, num_classes, num_boxes, neg_pre_positive, batch_size):
super(MultiBoxLoss, self).__init__()
self.num_classes = num_classes
self.num_boxes = num_boxes
self.neg_pre_positive = neg_pre_positive
self.notequal = P.NotEqual()
self.less = P.Less()
self.tile = P.Tile()
self.reduce_sum = P.ReduceSum()
self.reduce_mean = P.ReduceMean()
self.expand_dims = P.ExpandDims()
self.smooth_l1_loss = P.SmoothL1Loss()
self.cross_entropy = SoftmaxCrossEntropyWithLogits()
self.maximum = P.Maximum()
self.minimum = P.Minimum()
self.sort_descend = P.TopK(True)
self.sort = P.TopK(True)
self.gather = P.GatherNd()
self.max = P.ReduceMax()
self.log = P.Log()
self.exp = P.Exp()
self.concat = P.Concat(axis=1)
self.reduce_sum2 = P.ReduceSum(keep_dims=True)
self.idx = Tensor(
np.reshape(np.arange(batch_size * num_boxes), (-1, 1)), ms.int32)
def test_topk_big_2d():
context.set_context(mode=context.GRAPH_MODE, device_target="GPU")
x_np = np.random.rand(512, 1024).astype(np.float32)
k = 512
ms_output = P.TopK(True)(Tensor(x_np), k)
np_output = np.sort(x_np, axis=-1)[..., ::-1][..., 0:k]
assert np.allclose(ms_output[0].asnumpy(), np_output)
# sorted elements num greater than max thread per block
x_np = np.random.rand(128, 2048).astype(np.float32)
k = 1
ms_output = P.TopK(True)(Tensor(x_np), k)
np_output = np.sort(x_np, axis=-1)[..., ::-1][..., 0:k]
assert np.allclose(ms_output[0].asnumpy(), np_output)
x_np = np.random.rand(32, 2048).astype(np.float32)
k = 2048
ms_output = P.TopK(True)(Tensor(x_np), k)
np_output = np.sort(x_np, axis=-1)[..., ::-1][..., 0:k]
assert np.allclose(ms_output[0].asnumpy(), np_output)
# sorted elements num greater than max share memory per block
x_np = np.random.rand(16, 40960).astype(np.float32)
k = 1
ms_output = P.TopK(True)(Tensor(x_np), k)
np_output = np.sort(x_np, axis=-1)[..., ::-1][..., 0:k]
assert np.allclose(ms_output[0].asnumpy(), np_output)
def test_topk_1d():
context.set_context(mode=context.GRAPH_MODE, device_target="GPU")
x_np = np.random.rand(12).astype(np.float32)
k = 4
ms_output = P.TopK(True)(Tensor(x_np), k)
np_output = np.sort(x_np)[::-1][0:k]
assert np.allclose(ms_output[0].asnumpy(), np_output)
x_np = np.random.rand(1200).astype(np.float32)
k = 256
ms_output = P.TopK(True)(Tensor(x_np), k)
np_output = np.sort(x_np)[::-1][0:k]
assert np.allclose(ms_output[0].asnumpy(), np_output)
x_np = np.random.rand(250000).astype(np.float32)
k = 2000
ms_output = P.TopK(True)(Tensor(x_np), k)
np_output = np.sort(x_np)[::-1][0:k]
assert np.allclose(ms_output[0].asnumpy(), np_output)
x_np = np.random.rand(10240).astype(np.float32)
k = 4096
ms_output = P.TopK(True)(Tensor(x_np), k)
np_output = np.sort(x_np)[::-1][0:k]
assert np.allclose(ms_output[0].asnumpy(), np_output)
x_np = np.random.rand(720).astype(np.float32)
k = 720
ms_output = P.TopK(True)(Tensor(x_np), k)
np_output = np.sort(x_np)[::-1][0:k]
assert np.allclose(ms_output[0].asnumpy()[:k], np_output)
def test_topk_3d():
context.set_context(mode=context.GRAPH_MODE, device_target="GPU")
x_np = np.random.rand(2, 256, 128).astype(np.float32)
k = 4
ms_output = P.TopK(True)(Tensor(x_np), k)
np_output = np.sort(x_np, axis=-1)[..., ::-1][..., 0:k]
assert np.allclose(ms_output[0].asnumpy(), np_output)
x_np = np.random.rand(2, 3, 4).astype(np.float32)
k = 2
ms_output = P.TopK(True)(Tensor(x_np), k)
np_output = np.sort(x_np, axis=-1)[..., ::-1][..., 0:k]
assert np.allclose(ms_output[0].asnumpy(), np_output)
def __init__(self, config):
super(ClassificationLoss, self).__init__()
self.num_classes = config.NUM_CLASSES
self.num_boxes = config.NUM_SSD_BOXES
self.neg_pre_positive = config.NEG_PRE_POSITIVE
self.minimum = P.Minimum()
self.less = P.Less()
self.sort = P.TopK()
self.tile = P.Tile()
self.reduce_sum = P.ReduceSum()
self.reduce_mean = P.ReduceMean()
self.expand_dims = P.ExpandDims()
self.sort_descend = P.TopK(True)
self.cross_entropy = nn.SoftmaxCrossEntropyWithLogits(sparse=True)
def __init__(self, backbone, generate=False):
super(EvalNet, self).__init__(auto_prefix=False)
self.backbone = backbone
self.argmax = P.ArgMaxWithValue()
self.generate = generate
self.topk = P.TopK(sorted=True).shard(((1, 1),))
self.log_softmax = P.Softmax(axis=-1)
def __init__(self, batch_size=4):
super(DiceLoss, self).__init__()
self.threshold0 = Tensor(0.5, mstype.float32)
self.zero_float32 = Tensor(0.0, mstype.float32)
self.k = int(640 * 640)
self.negative_one_int32 = Tensor(-1, mstype.int32)
self.batch_size = batch_size
self.concat = P.Concat()
self.less_equal = P.LessEqual()
self.greater = P.Greater()
self.reduce_sum = P.ReduceSum()
self.reduce_sum_keep_dims = P.ReduceSum(keep_dims=True)
self.reduce_mean = P.ReduceMean()
self.reduce_min = P.ReduceMin()
self.cast = P.Cast()
self.minimum = P.Minimum()
self.expand_dims = P.ExpandDims()
self.select = P.Select()
self.fill = P.Fill()
self.topk = P.TopK(sorted=True)
self.shape = P.Shape()
self.sigmoid = P.Sigmoid()
self.reshape = P.Reshape()
self.slice = P.Slice()
self.logical_and = P.LogicalAnd()
self.logical_or = P.LogicalOr()
self.equal = P.Equal()
self.zeros_like = P.ZerosLike()
self.add = P.TensorAdd()
self.gather = P.Gather()
def set_train_local(self, config, training=False):
"""Set training flag."""
self.training_local = training
cfg = config
self.topK_stage1 = ()
self.topK_shape = ()
total_max_topk_input = 0
if not self.training_local:
self.num_pre = cfg.rpn_nms_pre
self.min_box_size = cfg.rpn_min_bbox_min_size
self.nms_thr = cfg.rpn_nms_thr
self.nms_post = cfg.rpn_nms_post
self.max_num = cfg.rpn_max_num
k_num = self.num_pre
total_max_topk_input = k_num
self.topK_stage1 = k_num
self.topK_shape = (k_num, 1)
self.topKv2 = P.TopK(sorted=True)
self.topK_shape_stage2 = (self.max_num, 1)
self.min_float_num = -65536.0
self.topK_mask = Tensor(self.min_float_num *
np.ones(total_max_topk_input, np.float16))
self.shape = P.Shape()
self.print = P.Print()
def generate_one_step(self,
input_ids: Tensor,
input_mask: Tensor,
beam_size=1):
"""
generate next token for only one step, use softmax to regularize logits
Arg:
input_ids (Tensor): (batch_size,seq_length) ids of input text
input_mask (Tensor): (batch_size,seq_length) mask of input text, 0 for mask, 1 for reserved
beam_size (int): int, beam_size for each candidate text
Return:
topk_indices (Tensor): (batch_size,beam_size), topk (k = beam_size) num of the next token indices for each batch
topk_logits (Tensor): (batch_size,beam_size), topk (k = beam_size) num of the next token logits(distribution) for each batch
"""
logits = self.decoder.predict(input_ids, input_mask)
last_token_pos_recorder = LastTokenPos(input_mask)
last_token_pos_list = last_token_pos_recorder.get_pos(shift=0)
return_last_logits = extract_single_token_logits(
logits, last_token_pos_list) #(batch_size,1,vocab_size)
return_last_logits = P.Reshape()(
return_last_logits,
(self.batch_size, self.vocab_size)) #(batch_size,vocab_size)
return_last_logits = P.Softmax()(return_last_logits)
topk_logits, topk_indices = P.TopK(sorted=True)(
return_last_logits, beam_size) #(batch_size,beam_size)
return topk_indices, topk_logits
def set_train_local(self, config, training=True):
"""Set training flag."""
self.training_local = training
cfg = config
self.topK_stage1 = ()
self.topK_shape = ()
total_max_topk_input = 0
if not self.training_local:
self.num_pre = cfg.rpn_nms_pre
self.min_box_size = cfg.rpn_min_bbox_min_size
self.nms_thr = cfg.rpn_nms_thr
self.nms_post = cfg.rpn_nms_post
self.nms_across_levels = cfg.rpn_nms_across_levels
self.max_num = cfg.rpn_max_num
for shp in self.feature_shapes:
k_num = min(self.num_pre, (shp[0] * shp[1] * 3))
total_max_topk_input += k_num
self.topK_stage1 += (k_num, )
self.topK_shape += ((k_num, 1), )
self.topKv2 = P.TopK(sorted=True)
self.topK_shape_stage2 = (self.max_num, 1)
self.min_float_num = -65536.0
self.topK_mask = Tensor(self.min_float_num *
np.ones(total_max_topk_input, np.float16))
def __init__(self):
super(Pca, self).__init__()
self.reduce_mean = ops.ReduceMean(keep_dims=True)
self.reshape = ops.Reshape()
self.matmul_a = ops.MatMul(transpose_a=True)
self.matmul_b = ops.MatMul(transpose_b=True)
self.top_k = ops.TopK(sorted=True)
self.gather = ops.GatherV2()
def __init__(self, network):
super(CenterFaceWithNms, self).__init__()
self.centerface_network = network
self.config = ConfigCenterface()
# two type of maxpool self.maxpool2d = nn.MaxPool2d(kernel_size=3, stride=1, pad_mode='same')
self.maxpool2d = P.MaxPoolWithArgmax(kernel_size=3, strides=1, pad_mode='same')
self.topk = P.TopK(sorted=True)
self.reshape = P.Reshape()
self.print = P.Print()
self.test_batch = self.config.test_batch_size
self.k = self.config.K
def __init__(self, network, k, num_eval_neg):
super(PredictWithSigmoid, self).__init__()
self.network = network
self.topk = P.TopK(sorted=True)
self.squeeze = P.Squeeze()
self.k = k
self.num_eval_neg = num_eval_neg
self.gather = P.Gather()
self.reshape = P.Reshape()
self.reducesum = P.ReduceSum(keep_dims=False)
self.notequal = P.NotEqual()
def __init__(self,
batch_size:int,
vocab_size:int,
k:int=0,
p:float=1.0,
temperature:float=1.0,
min_tokens_to_keep:int=1,
fp16:bool=False):
super(TopKTopP_Filter, self).__init__()
self.topK = P.TopK(sorted=True)
self.batch_size = batch_size
self.vocab_size = vocab_size
self.min_tokens_to_keep = min_tokens_to_keep
self.k = k
self.p = p
self.temp = temperature
self.fp16 = fp16
self.cumsum = P.CumSum()
self.onehot = P.OneHot()
self.cast = P.Cast()
self.expand_dims = P.ExpandDims()
self.device_target = get_context("device_target")
self.mask = Tensor(
np.zeros((batch_size, vocab_size)), dtype=mstype.float32)
self.on_value = Tensor(1.0, mstype.float32)
self.off_value = Tensor(0.0, mstype.float32)
self.softmax = P.Softmax()
self.safety_mask_left = np.zeros(
(batch_size, min_tokens_to_keep), dtype=float)
self.safety_mask_right = np.ones((batch_size,
vocab_size-min_tokens_to_keep),
dtype=float)
self.safety_mask = Tensor(np.concatenate((self.safety_mask_left,
self.safety_mask_right), axis=1),
dtype=mstype.float32)
self.NINF = float(-1e6)
self.expand_dims = P.ExpandDims()
assert self.temp > 0.0, 'temperature must be positive'
assert self.k >= 0, 'the top_k number must be no negative.'
if self.k > 0:
assert self.min_tokens_to_keep <= self.k, 'K must be larger than or equal to min_token_to_keep for top p sampling'
def test_mode_init(self, config):
self.test_batch_size = config.test_batch_size
self.split = P.Split(axis=0, output_num=self.test_batch_size)
self.split_shape = P.Split(axis=0, output_num=4)
self.split_scores = P.Split(axis=1, output_num=self.num_classes)
self.split_cls = P.Split(axis=0, output_num=self.num_classes - 1)
self.tile = P.Tile()
self.gather = P.GatherNd()
self.rpn_max_num = config.rpn_max_num
self.zeros_for_nms = Tensor(
np.zeros((self.rpn_max_num, 3)).astype(self.dtype))
self.ones_mask = np.ones((self.rpn_max_num, 1)).astype(np.bool)
self.zeros_mask = np.zeros((self.rpn_max_num, 1)).astype(np.bool)
self.bbox_mask = Tensor(
np.concatenate((self.ones_mask, self.zeros_mask, self.ones_mask,
self.zeros_mask),
axis=1))
self.nms_pad_mask = Tensor(
np.concatenate((self.ones_mask, self.ones_mask, self.ones_mask,
self.ones_mask, self.zeros_mask),
axis=1))
self.test_score_thresh = Tensor(
np.ones((self.rpn_max_num, 1)).astype(self.dtype) *
config.test_score_thr)
self.test_score_zeros = Tensor(
np.ones((self.rpn_max_num, 1)).astype(self.dtype) * 0)
self.test_box_zeros = Tensor(
np.ones((self.rpn_max_num, 4)).astype(self.dtype) * -1)
self.test_iou_thr = Tensor(
np.ones((self.rpn_max_num, 1)).astype(self.dtype) *
config.test_iou_thr)
self.test_max_per_img = config.test_max_per_img
self.nms_test = P.NMSWithMask(config.test_iou_thr)
self.softmax = P.Softmax(axis=1)
self.logicand = P.LogicalAnd()
self.oneslike = P.OnesLike()
self.test_topk = P.TopK(sorted=True)
self.test_num_proposal = self.test_batch_size * self.rpn_max_num
# b_shape[0] != x_shape[1]
('BiasAdd5', {
'block': (P.BiasAdd(), {
'exception': ValueError,
'error_keywords': ['BiasAdd']
}),
'desc_inputs': [
Tensor(np.ones([5, 3]).astype(np.float32)),
Tensor(np.ones([5]).astype(np.float32))
],
'skip': ['backward']
}),
# input x is scalar
('TopK0', {
'block': (TopKNet(P.TopK(), 5), {
'exception': TypeError,
'error_keywords': ['TopK']
}),
'desc_inputs': [5.0],
'skip': ['backward']
}),
# input x is Tensor(bool)
('TopK1', {
'block': (TopKNet(P.TopK(), 5), {
'exception': TypeError,
'error_keywords': ['TopK']
}),
'desc_inputs': [Tensor(np.ones([10]).astype(np.bool_))],
'skip': ['backward']
}),
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
from mindspore.ops import Primitive
from mindspore.ops import operations as P
TopK = P.TopK()
tuple_getitem = Primitive('tuple_getitem')
class FnDict:
def __init__(self):
self.fnDict = {}
def __call__(self, fn):
self.fnDict[fn.__name__] = fn
def __getitem__(self, name):
return self.fnDict[name]
def test_topk_split(tag):
'desc_inputs': [[128, 64, 32, 32], [64], [64], [64], [64]],
'desc_bprop': [[128, 64, 32, 32], [64], [64], [64], [64]],
'skip': []}),
('BatchNormGrad', {
'block': G.BatchNormGrad(),
'desc_inputs': [[128, 64, 32, 32], [128, 64, 32, 32], [64], [64], [64], [64]],
'desc_bprop': [[128, 64, 32, 32], [64], [64], [64], [64]],
'skip': ['backward']}),
('ApplyMomentum', {
'block': P.ApplyMomentum(),
'desc_inputs': [[128, 32, 32, 64], [128, 32, 32, 64],
[32, 32, 64], [32, 32, 64], [32, 32, 64]],
'desc_bprop': [[128, 32, 32, 64]],
'skip': ['backward']}),
('TopK', {
'block': P.TopK(),
'desc_const': [5],
'desc_inputs': [[20, 20, 10]],
'desc_bprop': [[20, 20, 5]],
'skip': ['backward']}),
('GatherV2_0', {
'block': P.GatherV2(),
'desc_const': [0],
'desc_inputs': [[3, 1, 2], Tensor(np.array([0, 1]).astype(np.int32))],
'desc_bprop': [[2, 1, 2]]}),
('GatherV2_1', {
'block': P.GatherV2(),
'desc_const': [2],
'desc_inputs': [[3, 1, 3], Tensor(np.array([0, 1]).astype(np.int32))],
'desc_bprop': [[3, 1, 2]]}),
('GatherV2_2', {
def __init__(self,
batch_size,
seq_length,
vocab_size,
decoder,
beam_width=4,
decoder_layers_nums=4,
length_penalty_weight=0.6,
cov_penalty_factor=0.1,
hidden_size=1024,
max_decode_length=64,
sos_id=2,
eos_id=3,
compute_type=mstype.float32):
super(BeamSearchDecoder, self).__init__()
self.encoder_length = seq_length
self.hidden_size = hidden_size
self.batch_size = batch_size
self.vocab_size = vocab_size
self.beam_width = beam_width
self.decoder_layers_nums = decoder_layers_nums
self.length_penalty_weight = length_penalty_weight
self.cov_penalty_factor = cov_penalty_factor
self.max_decode_length = max_decode_length
self.decoder = decoder
self.add = P.TensorAdd()
self.expand = P.ExpandDims()
self.reshape = P.Reshape()
self.shape_flat = (-1,)
self.shape = P.Shape()
self.zero_tensor = Tensor(np.zeros([batch_size, beam_width]), mstype.float32)
self.ninf_tensor = Tensor(np.full([batch_size, beam_width], -INF), mstype.float32)
self.select = P.Select()
self.flat_shape = (batch_size, beam_width * vocab_size)
self.topk = P.TopK(sorted=True)
self.floor_div = P.FloorDiv()
self.vocab_size_tensor = Tensor(self.vocab_size, mstype.int32)
self.real_div = P.RealDiv()
self.mod = Mod()
self.equal = P.Equal()
self.eos_ids = Tensor(np.full([batch_size, beam_width], eos_id), mstype.int32)
beam_ids = np.tile(np.arange(beam_width).reshape((1, beam_width)), [batch_size, 1])
self.beam_ids = Tensor(beam_ids, mstype.int32)
batch_ids = np.arange(batch_size * beam_width).reshape((batch_size, beam_width)) // beam_width
self.batch_ids = Tensor(batch_ids, mstype.int32)
self.concat = P.Concat(axis=-1)
self.gather_nd = P.GatherNd()
self.start = Tensor(0, dtype=mstype.int32)
self.start_ids = Tensor(np.full([batch_size * beam_width, 1], sos_id), mstype.int32)
self.init_seq = Tensor(np.full([batch_size, beam_width, self.max_decode_length], sos_id), mstype.int32)
init_scores = np.tile(np.array([[0.] + [-INF] * (beam_width - 1)]), [batch_size, 1])
self.init_scores = Tensor(init_scores, mstype.float32)
self.init_finished = Tensor(np.zeros([batch_size, beam_width], dtype=np.bool))
self.init_length = Tensor(np.zeros([batch_size, beam_width], dtype=np.int32))
self.length_penalty = LengthPenalty(weight=length_penalty_weight)
self.one = Tensor(1, mstype.int32)
self.prob_concat = P.Concat(axis=1)
self.cast = P.Cast()
self.decoder_hidden_state = Tensor(np.zeros([self.decoder_layers_nums, 2,
self.batch_size * self.beam_width,
hidden_size]), mstype.float32)
self.zeros_scores = Tensor(np.zeros([batch_size, beam_width], dtype=np.float))
self.active_index = Tensor(np.ones([batch_size, beam_width], dtype=np.int32))
self.init_zeros = Tensor(np.zeros([batch_size, beam_width], dtype=np.int32))
self.init_ones = Tensor(np.ones([batch_size, beam_width], dtype=np.float32))
self.accu_attn_scores = Tensor(np.zeros([batch_size, beam_width, self.encoder_length], dtype=np.float32))
self.zeros = Tensor([0], mstype.int32)
self.eos_tensor = Tensor(np.full([batch_size, beam_width, beam_width], eos_id), mstype.int32)
self.ones_3d = Tensor(np.full([batch_size, beam_width, self.encoder_length], 1), mstype.float32)
self.neg_inf_3d = Tensor(np.full([batch_size, beam_width, self.encoder_length], -INF), mstype.float32)
self.zeros_3d = Tensor(np.full([batch_size, beam_width, self.encoder_length], 0), mstype.float32)
self.zeros_2d = Tensor(np.full([batch_size * beam_width, self.encoder_length], 0), mstype.int32)
self.argmin = P.ArgMinWithValue(axis=1)
self.reducesum = P.ReduceSum()
self.div = P.Div()
self.shape_op = P.Shape()
self.mul = P.Mul()
self.log = P.Log()
self.less = P.Less()
self.tile = P.Tile()
self.noteq = P.Neg()
self.zeroslike = P.ZerosLike()
self.greater_equal = P.GreaterEqual()
self.sub = P.Sub()
def __init__(self, k):
super(Net, self).__init__()
self.topk = P.TopK(True)
self.k = k
def __init__(self, k):
super(KnnNet, self).__init__()
self.tile = P.Tile()
self.sum = P.ReduceSum()
self.topk = P.TopK()
self.k = k
def generate_beam_search(
model,
config,
input_ids,
input_mask,
cur_len,
max_length,
min_length,
do_sample,
early_stopping,
temperature,
top_k,
top_p,
repetition_penalty,
no_repeat_ngram_size,
pad_token_id,
eos_token_id,
#batch_size,
length_penalty,
num_beams: int,
#attention_mask,
use_cache):
generated_ids = []
max_length = min(max_length, config.seq_length)
batch_size = config.batch_size
vocab_size = config.vocab_size
assert batch_size == 1, "For now, it only generates 1 sentence per batch."
#initialize beam_score as 0 tensor
init_beam_prob = np.zeros((batch_size, num_beams), dtype=float)
reshape = P.Reshape()
squeeze_shape = (-1, )
top_k = P.TopK(sorted=False)
if do_sample is False:
init_beam_prob[:, 1:] = -1e9
# beam_scores in form of Tensor:
# beam_scores = Tensor(init_beam_prob,dtype=mstype.float32)
# beam_scores = reshape(beam_scores,squeeze_shape)
#Use numpy for now, since batch size is only 1
beam_scores = init_beam_prob
#beam_scores: shape (batch_size*num_beams,)
#cache states
past_states = None
done_sentences = [False for _ in range(batch_size)]
input_ids_expand = replicate_input(input_ids, time=num_beams)
log_softmax = P.LogSoftmax(axis=-1)
first_token = True
while cur_len < max_length:
lst_logits = []
generated_ids.append([])
for i in range(num_beams):
lst_logits.append(model.predict(input_ids_expand, input_mask))
tuple_logits = tuple(lst_logits)
concat = P.Concat(axis=0)
#concat from tuple of logits
logits = concat(tuple_logits)
next_token_logits = logits[::, cur_len, 0:vocab_size]
# (num_beams,vocab_size)
scores = log_softmax(next_token_logits)
candidates = None
sentence_prefix = None
squeezed_scores = reshape(scores, squeeze_shape)
#(num_beam*vocab_size)
#indices_np = None
if first_token:
first_token = False
values, indices = top_k(squeezed_scores[0:vocab_size], num_beams)
#indices (num_beams)
indices_np = indices.asnumpy()
values_np = indices.asnumpy()
candidates = indices_np.tolist()
#for the first token, we choose 0 as default for all situations since the model is not .
sentence_prefix = [0 for _ in range(num_beams)]
for i in range(num_beams):
beam_scores[i] += values_np[i]
generated_ids[-1].append(candidates[i])
else:
# need to choose top beams^2 prob of token
values, indices = top_k(squeezed_scores, num_beams * num_beams)
indices_np = indices.asnumpy()
indices_np = indices.asnumpy()
values_np = indices.asnumpy()
tmp_candidates = indices_np.tolist()
tmp_candidates_scores = []
for i in range(num_beams * num_beams):
sentence_index = indices_np[i] // vocab_size
# index of token, tmp_beam_score, sentence_index of token
tmp_candidates_scores.append(
(tmp_candidates[i] % vocab_size,
values_np[i] + beam_scores[sentence_index],
sentence_index))
#sort by beam_score
tmp_candidates_scores.sort(key=lambda x: x[1], reverse=True)
sentence_prefix = []
candidates = []
for i in range(num_beams):
sentence_prefix.append(tmp_candidates_scores[i][2])
candidates.append(tmp_candidates_scores[i][0])
beam_scores[i] += tmp_candidates_scores[i][1]
input_np = input_ids_expand.asnumpy()
#(num_beams,seq_length)
new_input = np.zeros_like(input_np)
for i in range(num_beams):
new_input[i] = input_np[sentence_prefix[i]]
new_input[i][cur_len] = candidates[i]
generated_ids[-1].append(candidates[i])
input_ids_expand = Tensor(input_np, dtype=mstype.float32)
cur_len += 1
pass
#(seq_length,num_beams) -> (num_beams,seq_length)
generated_ids_np = np.array(generated_ids).T
token_ids = generated_ids_np.tolist()
return token_ids
def __init__(self, config):
super(Mask_Rcnn_Resnet50, self).__init__()
self.train_batch_size = config.batch_size
self.num_classes = config.num_classes
self.anchor_scales = config.anchor_scales
self.anchor_ratios = config.anchor_ratios
self.anchor_strides = config.anchor_strides
self.target_means = tuple(config.rcnn_target_means)
self.target_stds = tuple(config.rcnn_target_stds)
# Anchor generator
anchor_base_sizes = None
self.anchor_base_sizes = list(
self.anchor_strides) if anchor_base_sizes is None else anchor_base_sizes
self.anchor_generators = []
for anchor_base in self.anchor_base_sizes:
self.anchor_generators.append(
AnchorGenerator(anchor_base, self.anchor_scales, self.anchor_ratios))
self.num_anchors = len(self.anchor_ratios) * len(self.anchor_scales)
featmap_sizes = config.feature_shapes
assert len(featmap_sizes) == len(self.anchor_generators)
self.anchor_list = self.get_anchors(featmap_sizes)
# Backbone resnet50
self.backbone = ResNetFea(ResidualBlockUsing,
config.resnet_block,
config.resnet_in_channels,
config.resnet_out_channels,
False)
# Fpn
self.fpn_ncek = FeatPyramidNeck(config.fpn_in_channels,
config.fpn_out_channels,
config.fpn_num_outs)
# Rpn and rpn loss
self.gt_labels_stage1 = Tensor(np.ones((self.train_batch_size, config.num_gts)).astype(np.uint8))
self.rpn_with_loss = RPN(config,
self.train_batch_size,
config.rpn_in_channels,
config.rpn_feat_channels,
config.num_anchors,
config.rpn_cls_out_channels)
# Proposal
self.proposal_generator = Proposal(config,
self.train_batch_size,
config.activate_num_classes,
config.use_sigmoid_cls)
self.proposal_generator.set_train_local(config, True)
self.proposal_generator_test = Proposal(config,
config.test_batch_size,
config.activate_num_classes,
config.use_sigmoid_cls)
self.proposal_generator_test.set_train_local(config, False)
# Assign and sampler stage two
self.bbox_assigner_sampler_for_rcnn = BboxAssignSampleForRcnn(config, self.train_batch_size,
config.num_bboxes_stage2, True)
self.decode = P.BoundingBoxDecode(max_shape=(768, 1280), means=self.target_means, \
stds=self.target_stds)
# Roi
self.roi_align = SingleRoIExtractor(config,
config.roi_layer,
config.roi_align_out_channels,
config.roi_align_featmap_strides,
self.train_batch_size,
config.roi_align_finest_scale,
mask=False)
self.roi_align.set_train_local(config, True)
self.roi_align_mask = SingleRoIExtractor(config,
config.roi_layer,
config.roi_align_out_channels,
config.roi_align_featmap_strides,
self.train_batch_size,
config.roi_align_finest_scale,
mask=True)
self.roi_align_mask.set_train_local(config, True)
self.roi_align_test = SingleRoIExtractor(config,
config.roi_layer,
config.roi_align_out_channels,
config.roi_align_featmap_strides,
1,
config.roi_align_finest_scale,
mask=False)
self.roi_align_test.set_train_local(config, False)
self.roi_align_mask_test = SingleRoIExtractor(config,
config.roi_layer,
config.roi_align_out_channels,
config.roi_align_featmap_strides,
1,
config.roi_align_finest_scale,
mask=True)
self.roi_align_mask_test.set_train_local(config, False)
# Rcnn
self.rcnn_cls = RcnnCls(config, self.train_batch_size, self.num_classes)
self.rcnn_mask = RcnnMask(config, self.train_batch_size, self.num_classes)
# Op declare
self.squeeze = P.Squeeze()
self.cast = P.Cast()
self.concat = P.Concat(axis=0)
self.concat_1 = P.Concat(axis=1)
self.concat_2 = P.Concat(axis=2)
self.reshape = P.Reshape()
self.select = P.Select()
self.greater = P.Greater()
self.transpose = P.Transpose()
# Test mode
self.test_batch_size = config.test_batch_size
self.split = P.Split(axis=0, output_num=self.test_batch_size)
self.split_shape = P.Split(axis=0, output_num=4)
self.split_scores = P.Split(axis=1, output_num=self.num_classes)
self.split_fb_mask = P.Split(axis=1, output_num=self.num_classes)
self.split_cls = P.Split(axis=0, output_num=self.num_classes-1)
self.tile = P.Tile()
self.gather = P.GatherNd()
self.rpn_max_num = config.rpn_max_num
self.zeros_for_nms = Tensor(np.zeros((self.rpn_max_num, 3)).astype(np.float16))
self.ones_mask = np.ones((self.rpn_max_num, 1)).astype(np.bool)
self.zeros_mask = np.zeros((self.rpn_max_num, 1)).astype(np.bool)
self.bbox_mask = Tensor(np.concatenate((self.ones_mask, self.zeros_mask,
self.ones_mask, self.zeros_mask), axis=1))
self.nms_pad_mask = Tensor(np.concatenate((self.ones_mask, self.ones_mask,
self.ones_mask, self.ones_mask, self.zeros_mask), axis=1))
self.test_score_thresh = Tensor(np.ones((self.rpn_max_num, 1)).astype(np.float16) * config.test_score_thr)
self.test_score_zeros = Tensor(np.ones((self.rpn_max_num, 1)).astype(np.float16) * 0)
self.test_box_zeros = Tensor(np.ones((self.rpn_max_num, 4)).astype(np.float16) * -1)
self.test_iou_thr = Tensor(np.ones((self.rpn_max_num, 1)).astype(np.float16) * config.test_iou_thr)
self.test_max_per_img = config.test_max_per_img
self.nms_test = P.NMSWithMask(config.test_iou_thr)
self.softmax = P.Softmax(axis=1)
self.logicand = P.LogicalAnd()
self.oneslike = P.OnesLike()
self.test_topk = P.TopK(sorted=True)
self.test_num_proposal = self.test_batch_size * self.rpn_max_num
# Improve speed
self.concat_start = min(self.num_classes - 2, 55)
self.concat_end = (self.num_classes - 1)
# Init tensor
roi_align_index = [np.array(np.ones((config.num_expected_pos_stage2 + config.num_expected_neg_stage2, 1)) * i,
dtype=np.float16) for i in range(self.train_batch_size)]
roi_align_index_test = [np.array(np.ones((config.rpn_max_num, 1)) * i, dtype=np.float16) \
for i in range(self.test_batch_size)]
self.roi_align_index_tensor = Tensor(np.concatenate(roi_align_index))
self.roi_align_index_test_tensor = Tensor(np.concatenate(roi_align_index_test))
roi_align_index_pos = [np.array(np.ones((config.num_expected_pos_stage2, 1)) * i,
dtype=np.float16) for i in range(self.train_batch_size)]
self.roi_align_index_tensor_pos = Tensor(np.concatenate(roi_align_index_pos))
self.rcnn_loss_cls_weight = Tensor(np.array(config.rcnn_loss_cls_weight).astype(np.float16))
self.rcnn_loss_reg_weight = Tensor(np.array(config.rcnn_loss_reg_weight).astype(np.float16))
self.rcnn_loss_mask_fb_weight = Tensor(np.array(config.rcnn_loss_mask_fb_weight).astype(np.float16))
self.argmax_with_value = P.ArgMaxWithValue(axis=1)
self.on_value = Tensor(1.0, mstype.float32)
self.off_value = Tensor(0.0, mstype.float32)
self.onehot = P.OneHot()
self.reducesum = P.ReduceSum()
self.sigmoid = P.Sigmoid()
self.expand_dims = P.ExpandDims()
self.test_mask_fb_zeros = Tensor(np.zeros((self.rpn_max_num, 28, 28)).astype(np.float16))
self.value = Tensor(1.0, mstype.float16)
def evaluation(d_id):
"""evaluation"""
context.set_context(mode=context.GRAPH_MODE,
device_target='Ascend',
device_id=d_id,
save_graphs=False)
print('********************** loading corpus ********************** ')
s_lc = time.time()
data_generator = DataGen(config)
queries = read_query(config, d_id)
print("loading corpus time (h):", (time.time() - s_lc) / 3600)
print('********************** loading model ********************** ')
s_lm = time.time()
model_onehop_bert = ModelOneHop(256)
param_dict = load_checkpoint(config.onehop_bert_path)
load_param_into_net(model_onehop_bert, param_dict)
model_twohop_bert = ModelOneHop(448)
param_dict2 = load_checkpoint(config.twohop_bert_path)
load_param_into_net(model_twohop_bert, param_dict2)
onehop = OneHopBert(config, model_onehop_bert)
twohop = TwoHopBert(config, model_twohop_bert)
print("loading model time (h):", (time.time() - s_lm) / 3600)
print('********************** evaluation ********************** ')
f_dev = open(config.dev_path, 'rb')
dev_data = json.load(f_dev)
f_dev.close()
q_gold = {}
q_2id = {}
for onedata in dev_data:
if onedata["question"] not in q_gold:
q_gold[onedata["question"]] = [
get_new_title(get_raw_title(item)) for item in onedata['path']
]
q_2id[onedata["question"]] = onedata['_id']
val, true_count, count, step = 0, 0, 0, 0
batch_queries = split_queries(config, queries)
output_path = []
for _, batch in enumerate(batch_queries):
print("###step###: ", str(step) + "_" + str(d_id))
query = batch[0]
temp_dict = {}
temp_dict['q_id'] = q_2id[query]
temp_dict['question'] = query
gold_path = q_gold[query]
input_ids_1, token_type_ids_1, input_mask_1 = data_generator.convert_onehop_to_features(
batch)
start = 0
TOTAL = len(input_ids_1)
split_chunk = 8
while start < TOTAL:
end = min(start + split_chunk - 1, TOTAL - 1)
chunk_len = end - start + 1
input_ids_1_ = input_ids_1[start:start + chunk_len]
input_ids_1_ = Tensor(input_ids_1_, mstype.int32)
token_type_ids_1_ = token_type_ids_1[start:start + chunk_len]
token_type_ids_1_ = Tensor(token_type_ids_1_, mstype.int32)
input_mask_1_ = input_mask_1[start:start + chunk_len]
input_mask_1_ = Tensor(input_mask_1_, mstype.int32)
cls_out = onehop(input_ids_1_, token_type_ids_1_, input_mask_1_)
if start == 0:
out = cls_out
else:
out = P.Concat(0)((out, cls_out))
start = end + 1
out = P.Squeeze(1)(out)
onehop_prob, onehop_index = P.TopK(sorted=True)(out, config.topk)
onehop_prob = P.Softmax()(onehop_prob)
sample, path_raw, last_out = data_generator.get_samples(
query, onehop_index, onehop_prob)
input_ids_2, token_type_ids_2, input_mask_2 = data_generator.convert_twohop_to_features(
sample)
start_2 = 0
TOTAL_2 = len(input_ids_2)
split_chunk = 8
while start_2 < TOTAL_2:
end_2 = min(start_2 + split_chunk - 1, TOTAL_2 - 1)
chunk_len = end_2 - start_2 + 1
input_ids_2_ = input_ids_2[start_2:start_2 + chunk_len]
input_ids_2_ = Tensor(input_ids_2_, mstype.int32)
token_type_ids_2_ = token_type_ids_2[start_2:start_2 + chunk_len]
token_type_ids_2_ = Tensor(token_type_ids_2_, mstype.int32)
input_mask_2_ = input_mask_2[start_2:start_2 + chunk_len]
input_mask_2_ = Tensor(input_mask_2_, mstype.int32)
cls_out = twohop(input_ids_2_, token_type_ids_2_, input_mask_2_)
if start_2 == 0:
out_2 = cls_out
else:
out_2 = P.Concat(0)((out_2, cls_out))
start_2 = end_2 + 1
out_2 = P.Softmax()(out_2)
last_out = Tensor(last_out, mstype.float32)
out_2 = P.Mul()(out_2, last_out)
val, true_count, topk_titles = eval_output(out_2, last_out, path_raw,
gold_path, val, true_count)
temp_dict['topk_titles'] = topk_titles
output_path.append(temp_dict)
step += 1
count += 1
return {
'val': val,
'count': count,
'true_count': true_count,
'path': output_path
}
def __init__(self, config):
super(Deeptext_VGG16, self).__init__()
self.train_batch_size = config.batch_size
self.num_classes = config.num_classes
self.anchor_scales = config.anchor_scales
self.anchor_ratios = config.anchor_ratios
self.anchor_strides = config.anchor_strides
self.target_means = tuple(config.rcnn_target_means)
self.target_stds = tuple(config.rcnn_target_stds)
# Anchor generator
anchor_base_sizes = None
self.anchor_base_sizes = list(
self.anchor_strides
) if anchor_base_sizes is None else anchor_base_sizes
self.anchor_generators = []
for anchor_base in self.anchor_base_sizes:
self.anchor_generators.append(
AnchorGenerator(anchor_base, self.anchor_scales,
self.anchor_ratios))
self.num_anchors = len(self.anchor_ratios) * len(self.anchor_scales)
featmap_sizes = config.feature_shapes
assert len(featmap_sizes) == len(self.anchor_generators)
self.anchor_list = self.get_anchors(featmap_sizes)
# Rpn and rpn loss
self.gt_labels_stage1 = Tensor(
np.ones((self.train_batch_size, config.num_gts)).astype(np.uint8))
self.rpn_with_loss = RPN(config, self.train_batch_size,
config.rpn_in_channels,
config.rpn_feat_channels, config.num_anchors,
config.rpn_cls_out_channels)
# Proposal
self.proposal_generator = Proposal(config, self.train_batch_size,
config.activate_num_classes,
config.use_sigmoid_cls)
self.proposal_generator.set_train_local(config, True)
self.proposal_generator_test = Proposal(config, config.test_batch_size,
config.activate_num_classes,
config.use_sigmoid_cls)
self.proposal_generator_test.set_train_local(config, False)
# Assign and sampler stage two
self.bbox_assigner_sampler_for_rcnn = BboxAssignSampleForRcnn(
config, self.train_batch_size, config.num_bboxes_stage2, True)
self.decode = P.BoundingBoxDecode(max_shape=(576, 960), means=self.target_means, \
stds=self.target_stds)
# Rcnn
self.rcnn = Rcnn(
config, config.rcnn_in_channels * config.roi_layer['out_size'] *
config.roi_layer['out_size'], self.train_batch_size,
self.num_classes)
# Op declare
self.squeeze = P.Squeeze()
self.cast = P.Cast()
self.concat = P.Concat(axis=0)
self.concat_1 = P.Concat(axis=1)
self.concat_2 = P.Concat(axis=2)
self.reshape = P.Reshape()
self.select = P.Select()
self.greater = P.Greater()
self.transpose = P.Transpose()
# Test mode
self.test_batch_size = config.test_batch_size
self.split = P.Split(axis=0, output_num=self.test_batch_size)
self.split_shape = P.Split(axis=0, output_num=4)
self.split_scores = P.Split(axis=1, output_num=self.num_classes)
self.split_cls = P.Split(axis=0, output_num=self.num_classes - 1)
self.tile = P.Tile()
self.gather = P.GatherNd()
self.rpn_max_num = config.rpn_max_num
self.zeros_for_nms = Tensor(
np.zeros((self.rpn_max_num, 3)).astype(np.float32))
self.ones_mask = np.ones((self.rpn_max_num, 1)).astype(np.bool)
self.zeros_mask = np.zeros((self.rpn_max_num, 1)).astype(np.bool)
self.bbox_mask = Tensor(
np.concatenate((self.ones_mask, self.zeros_mask, self.ones_mask,
self.zeros_mask),
axis=1))
self.nms_pad_mask = Tensor(
np.concatenate((self.ones_mask, self.ones_mask, self.ones_mask,
self.ones_mask, self.zeros_mask),
axis=1))
self.test_score_thresh = Tensor(
np.ones((self.rpn_max_num, 1)).astype(np.float32) *
config.test_score_thr)
self.test_score_zeros = Tensor(
np.ones((self.rpn_max_num, 1)).astype(np.float32) * 0)
self.test_box_zeros = Tensor(
np.ones((self.rpn_max_num, 4)).astype(np.float32) * -1)
self.test_iou_thr = Tensor(
np.ones((self.rpn_max_num, 1)).astype(np.float32) *
config.test_iou_thr)
self.test_max_per_img = config.test_max_per_img
self.nms_test = P.NMSWithMask(config.test_iou_thr)
self.softmax = P.Softmax(axis=1)
self.logicand = P.LogicalAnd()
self.oneslike = P.OnesLike()
self.test_topk = P.TopK(sorted=True)
self.test_num_proposal = self.test_batch_size * self.rpn_max_num
# Improve speed
self.concat_start = (self.num_classes - 2)
self.concat_end = (self.num_classes - 1)
# Init tensor
self.use_ambigous_sample = config.use_ambigous_sample
roi_align_index = [
np.array(np.ones((config.num_expected_pos_stage2 +
config.num_expected_neg_stage2, 1)) * i,
dtype=np.float32) for i in range(self.train_batch_size)
]
if self.use_ambigous_sample:
roi_align_index = [
np.array(np.ones((config.num_expected_pos_stage2 +
config.num_expected_amb_stage2 +
config.num_expected_neg_stage2, 1)) * i,
dtype=np.float32)
for i in range(self.train_batch_size)
]
roi_align_index_test = [np.array(np.ones((config.rpn_max_num, 1)) * i, dtype=np.float32) \
for i in range(self.test_batch_size)]
self.roi_align_index_tensor = Tensor(np.concatenate(roi_align_index))
self.roi_align_index_test_tensor = Tensor(
np.concatenate(roi_align_index_test))
self.roi_align4 = P.ROIAlign(pooled_width=7,
pooled_height=7,
spatial_scale=0.125)
self.roi_align5 = P.ROIAlign(pooled_width=7,
pooled_height=7,
spatial_scale=0.0625)
self.concat1 = P.Concat(axis=1)
self.roi_align_fuse = _conv(in_channels=1024,
out_channels=512,
kernel_size=1,
padding=0,
stride=1)
self.vgg16_feature_extractor = VGG16FeatureExtraction()
def __init__(self,
batch_size,
seq_length,
vocab_size,
decoder,
beam_width=4,
length_penalty_weight=1.0,
max_decode_length=128,
sos_id=1,
eos_id=2,
compute_type=mstype.float32):
super(BeamSearchDecoder, self).__init__(auto_prefix=False)
self.seq_length = seq_length
self.batch_size = batch_size
self.vocab_size = vocab_size
self.beam_width = beam_width
self.length_penalty_weight = length_penalty_weight
self.max_decode_length = max_decode_length
self.decoder = decoder
self.add = P.TensorAdd()
self.expand = P.ExpandDims()
self.reshape = P.Reshape()
self.shape_flat = (-1, )
self.shape = P.Shape()
self.zero_tensor = Tensor(np.zeros([batch_size, beam_width]),
mstype.float32)
self.ninf_tensor = Tensor(np.full([batch_size, beam_width], -INF),
mstype.float32)
self.select = P.Select()
self.flat_shape = (batch_size, beam_width * vocab_size)
self.topk = P.TopK(sorted=True)
self.floor_div = P.FloorDiv()
self.vocab_size_tensor = Tensor(self.vocab_size, mstype.int32)
self.real_div = P.RealDiv()
self.mod = Mod()
self.equal = P.Equal()
self.eos_ids = Tensor(np.full([batch_size, beam_width], eos_id),
mstype.int32)
beam_ids = np.tile(
np.arange(beam_width).reshape((1, beam_width)), [batch_size, 1])
self.beam_ids = Tensor(beam_ids, mstype.int32)
batch_ids = np.arange(batch_size * beam_width).reshape(
(batch_size, beam_width)) // beam_width
self.batch_ids = Tensor(batch_ids, mstype.int32)
self.concat = P.Concat(axis=-1)
self.gather_nd = P.GatherNd()
self.greater_equal = P.GreaterEqual()
self.sub = P.Sub()
self.cast = P.Cast()
self.zeroslike = P.ZerosLike()
# init inputs and states
self.start_ids = Tensor(np.full([batch_size * beam_width, 1], sos_id),
mstype.int32)
self.init_seq = Tensor(np.full([batch_size, beam_width, 1], sos_id),
mstype.int32)
init_scores = np.tile(np.array([[0.] + [-INF] * (beam_width - 1)]),
[batch_size, 1])
self.init_scores = Tensor(init_scores, mstype.float32)
self.init_finished = Tensor(
np.zeros([batch_size, beam_width], dtype=np.bool))
self.init_length = Tensor(
np.zeros([batch_size, beam_width], dtype=np.int32))
self.length_penalty = LengthPenalty(weight=length_penalty_weight)
self.one = Tensor(1, mstype.int32)
def __init__(self, w1, strategy1=None, strategy2=None):
super().__init__()
self.mul = P.Mul().shard(strategy1)
self.w1 = Parameter(w1, "w1")
self.topk = P.TopK().shard(strategy2)
