def construct(self, x, seq_lengths): """Defines the ReverseSequence operator computation performed.""" batch_size = x.shape[self.batch_dim] max_seq_len = x.shape[self.seq_dim] seq_lens_type = seq_lengths.dtype back = ops.Sub()(seq_lengths, ops.OnesLike()(seq_lengths)) batch_idx = self.make_shape((batch_size, max_seq_len), seq_lens_type, 0) forward_idx = self.make_shape((batch_size, max_seq_len), seq_lens_type, 1) back = back.view(-1, 1) reverse_idx = ops.Sub()(back, forward_idx) condition = ops.Less()(reverse_idx, ops.ZerosLike()(reverse_idx)) reverse_idx = ops.Select()(condition, forward_idx, reverse_idx) reverse_idx = ops.ExpandDims()(reverse_idx, 2) batch_idx = ops.ExpandDims()(batch_idx, 2) if self.batch_dim > self.seq_dim: batch_idx = ops.Transpose()(batch_idx, (1, 0, 2)) reverse_idx = ops.Transpose()(reverse_idx, (1, 0, 2)) x = ops.Transpose()(x, (1, 0, 2)) start_indices = ops.Concat(2)((batch_idx, reverse_idx)) output = ops.GatherNd()(x, start_indices) return output
def construct(self, category, subcategory, title, abstract): """ The news encoder is composed of title encoder, abstract encoder, category encoder and subcategory encoder. """ # Categories category_embedded = self.category_embedding( ops.reshape(category, (-1, ))) category_vector = self.category_dense(category_embedded) subcategory_embedded = self.sub_category_embedding( ops.reshape(subcategory, (-1, ))) subcategory_vector = self.subcategory_dense(subcategory_embedded) # title title_embedded = self.word_embedding( ops.reshape(title, self.title_shape)) title_feature = self.title_CNN(ops.Transpose()(title_embedded, (0, 2, 1))) title_vector = self.title_attention(ops.Transpose()(title_feature, (0, 2, 1))) # abstract abstract_embedded = self.word_embedding( ops.reshape(abstract, self.abstract_shape)) abstract_feature = self.abstract_CNN(ops.Transpose()(abstract_embedded, (0, 2, 1))) abstract_vector = self.abstract_attention(ops.Transpose()( abstract_feature, (0, 2, 1))) # total news_vector = self.total_attention( self.pack((category_vector, subcategory_vector, title_vector, abstract_vector))) return news_vector
def construct(self, hidden): h, c = hidden # h, c dim = 2 x b x hidden_dim h_in = P.Transpose()(h, (1, 0, 2)).view(-1, self.hidden_dim * 2) hidden_reduced_h = P.ReLU()(self.reduce_h(h)) hidden_reduced_h = P.ExpandDims()(hidden_reduced_h, 0) c_in = P.Transpose()(c, (1, 0, 2)).view(-1, self.hidden_dim * 2) hidden_reduced_c = P.ReLU()(self.reduce_c(c_in)) hidden_reduced_c = P.ExpandDims()(hidden_reduced_c, 0) return (hidden_reduced_h, hidden_reduced_c)
def construct(self, x, xr, h=None, seq_length=None): max_batch_size = x.shape[0] if self.batch_first else x.shape[1] if h is None: h = _init_state(self.num_layers * self.num_directions, max_batch_size, self.hidden_size, self.proj_size, x.dtype) if self.batch_first: x = P.Transpose()(x, (1, 0, 2)) xr = P.Transpose()(xr, (1, 0, 2)) x, h = self._stacked_bi_dynamic_rnn(x, xr, h, seq_length) if self.batch_first: x = P.Transpose()(x, (0, 2, 1, 3)) return x, h
def __init__(self): super(ImagePreProcess, self).__init__() self.transpose = ops.Transpose() self.perm_list = (0, 3, 1, 2) self.mean = Tensor(data_cfg.mean.reshape((1, 1, 1, 3))) self.std = Tensor(data_cfg.std.reshape((1, 1, 1, 3))) self.cast = ops.Cast()
def variable_recurrent(self, x, h, seq_length, w_ih, w_hh, b_ih, b_hh): '''recurrent steps with sequence length''' time_step = x.shape[0] h_t = h if self.is_lstm: hidden_size = h[0].shape[-1] zero_output = P.ZerosLike()(h_t[0]) else: hidden_size = h.shape[-1] zero_output = P.ZerosLike()(h_t) seq_length = P.Cast()(seq_length, mindspore.float32) seq_length = P.BroadcastTo((hidden_size, -1))(seq_length) seq_length = P.Cast()(seq_length, mindspore.int32) seq_length = P.Transpose()(seq_length, (1, 0)) outputs = [] state_t = h_t t = 0 while t < time_step: x_t = x[t:t + 1:1] x_t = P.Squeeze(0)(x_t) h_t = self.cell(x_t, state_t, w_ih, w_hh, b_ih, b_hh) seq_cond = seq_length > t if self.is_lstm: state_t_0 = P.Select()(seq_cond, h_t[0], state_t[0]) state_t_1 = P.Select()(seq_cond, h_t[1], state_t[1]) output = P.Select()(seq_cond, h_t[0], zero_output) state_t = (state_t_0, state_t_1) else: state_t = P.Select()(seq_cond, h_t, state_t) output = P.Select()(seq_cond, h_t, zero_output) outputs.append(output) t += 1 outputs = P.Stack()(outputs) return outputs, state_t
def variable_recurrent(self, x, h, seq_length): time_step = range(x.shape[0]) h_t = h if self.is_lstm: hidden_size = h[0].shape[-1] zero_output = P.ZerosLike()(h_t[0]) else: hidden_size = h.shape[-1] zero_output = P.ZerosLike()(h_t) seq_length = P.BroadcastTo((hidden_size, -1))(seq_length) seq_length = P.Transpose()(seq_length, (1, 0)) outputs = [] state_t = h_t for t in time_step: h_t = self.cell(x[t], state_t) seq_cond = seq_length > t if self.is_lstm: state_t_0 = P.Select()(seq_cond, h_t[0], state_t[0]) state_t_1 = P.Select()(seq_cond, h_t[1], state_t[1]) output = P.Select()(seq_cond, h_t[0], zero_output) state_t = (state_t_0, state_t_1) else: state_t = P.Select()(seq_cond, h_t, state_t) output = P.Select()(seq_cond, h_t, zero_output) outputs.append(output) outputs = P.Stack()(outputs) return outputs, state_t
def __init__(self): super(GatherFlipFeature, self).__init__() self.gather_nd = ops.GatherNd() self.transpose = ops.Transpose() self.perm_list = (1, 0, 2, 3) self.shape = ops.Shape() self.reshape = ops.Reshape()
def pixel_shuffle(tensor, scale_factor): """ Implementation of pixel shuffle using numpy Parameters: ----------- tensor: input tensor, shape is [N, C, H, W] scale_factor: scale factor to up-sample tensor Returns: -------- tensor: tensor after pixel shuffle, shape is [N, C/(r*r), r*H, r*W], where r refers to scale factor """ num, ch, height, width = tensor.shape # assert ch % (scale_factor * scale_factor) == 0 new_ch = ch // (scale_factor * scale_factor) new_height = height * scale_factor new_width = width * scale_factor reshape = ops.Reshape() tensor = reshape(tensor, (num, new_ch, scale_factor, scale_factor, height, width)) # new axis: [num, new_ch, height, scale_factor, width, scale_factor] transpose = ops.Transpose() tensor = transpose(tensor, (0, 1, 4, 2, 5, 3)) tensor = reshape(tensor, (num, new_ch, new_height, new_width)) return tensor
def __init__(self): super(TransposeGatherFeature, self).__init__() self.shape = ops.Shape() self.reshape = ops.Reshape() self.transpose = ops.Transpose() self.perm_list = (0, 2, 3, 1) self.gather_feat = GatherFeature()
def __init__(self, inc, outc, kernel_size=3, padding=1, stride=1, has_bias=False, modulation=True): super(DeformConv2d, self).__init__() self.kernel_size = kernel_size self.padding = padding self.stride = stride self.zero_padding = nn.Pad(((0, 0), (0, 0), (padding, padding), (padding, padding))) self.conv = nn.Conv2d(inc, outc, kernel_size=kernel_size, pad_mode='valid', padding=0, stride=kernel_size, has_bias=has_bias) self.p_conv = nn.Conv2d(inc, 2*kernel_size*kernel_size, kernel_size=self.kernel_size, pad_mode='pad', padding=self.padding, stride=self.stride) self.modulation = modulation if modulation: self.m_conv = nn.Conv2d(inc, kernel_size*kernel_size, kernel_size=self.kernel_size, pad_mode='valid', padding=0, stride=self.stride) if kernel_size % 2 == 0: raise ValueError("Only odd number is supported, but current kernel sizeis {}".format(kernel_size)) self.N = kernel_size * kernel_size self.begin = kernel_size // 2 self.sigmoid = ops.Sigmoid() self.dtype = ops.DType() self.perm_list = (0, 2, 3, 1) self.transpose = ops.Transpose() self.floor = ops.Floor() self.half = ops.Split(axis=-1, output_num=2) self.clip_value = ClipByValue() self.expand_dims = ops.ExpandDims() self.shape = ops.Shape() self.cast = ops.Cast() self._get_offset = GetOffsetPosition(self.begin, self.stride) self._get_surround = GetSurroundFeature() self._generate_fm = RegenerateFeatureMap(self.kernel_size)
def construct(self, x, h=None, seq_length=None): '''Defines the RNN like operators performed''' max_batch_size = x.shape[0] if self.batch_first else x.shape[1] num_directions = 2 if self.bidirectional else 1 if h is None: h = _init_state((self.num_layers * num_directions, max_batch_size, self.hidden_size), x.dtype, self.is_lstm) if self.batch_first: x = P.Transpose()(x, (1, 0, 2)) if self.bidirectional: x, h = self._stacked_bi_dynamic_rnn(x, h, seq_length) else: x, h = self._stacked_dynamic_rnn(x, h, seq_length) if self.batch_first: x = P.Transpose()(x, (1, 0, 2)) return x, h
def construct(self, x, h=None, seq_length=None): max_batch_size = x.shape[0] if self.batch_first else x.shape[1] num_directions = 2 if self.bidirectional else 1 if h is None: h = _init_state((self.num_layers * num_directions, max_batch_size, self.hidden_size), x.dtype, self.is_lstm) if self.batch_first: x = P.Transpose()(x, (1, 0, 2)) if self.bidirectional: x, h = self._stacked_bi_dynamic_rnn(x, h, seq_length, self._all_weights) else: x, h = self._stacked_dynamic_rnn(x, h, seq_length, self._all_weights) if self.batch_first: x = P.Transpose()(x, (1, 0, 2)) return x, h
def __init__(self, input_size, hidden_size, num_layers=1, has_bias=True, batch_first=False, dropout=0.0, bidirectional=False): super(StackLSTM, self).__init__() self.num_layers = num_layers self.batch_first = batch_first self.transpose = ops.Transpose() # direction number num_directions = 2 if bidirectional else 1 # input_size list input_size_list = [input_size] for i in range(num_layers - 1): input_size_list.append(hidden_size * num_directions) # layers layers = [] for i in range(num_layers): layers.append( nn.LSTMCell(input_size=input_size_list[i], hidden_size=hidden_size, has_bias=has_bias, batch_first=batch_first, bidirectional=bidirectional, dropout=dropout)) # weights weights = [] for i in range(num_layers): # weight size weight_size = (input_size_list[i] + hidden_size) * num_directions * hidden_size * 4 if has_bias: bias_size = num_directions * hidden_size * 4 weight_size = weight_size + bias_size # numpy weight stdv = 1 / math.sqrt(hidden_size) w_np = np.random.uniform(-stdv, stdv, (weight_size, 1, 1)).astype(np.float32) # lstm weight weights.append( Parameter(initializer(Tensor(w_np), w_np.shape), name="weight" + str(i))) # self.lstms = layers self.weight = ParameterTuple(tuple(weights))
def __init__(self): super(GetSurroundFeature, self).__init__() self.shape = ops.Shape() self.concat = ops.Concat(axis=1) self.reshape = ops.Reshape() self.half = ops.Split(axis=-1, output_num=2) self.tile = ops.Tile() self.gather_nd = ops.GatherNd() self.transpose = ops.Transpose() self.perm_list = (0, 2, 3, 1) self.order_list = (0, 3, 1, 2) self.expand_dims = ops.ExpandDims()
def __init__( self, input_size: int, hidden_size: int, nonlinearity: str = 'tanh', bias: bool = True, dropout: float = 0., batch_first: bool = False, ): super().__init__() self.rnn_cell = RNNCell(input_size, hidden_size, bias, nonlinearity) self.batch_first = batch_first self.transpose = P.Transpose() self.pack = P.Pack() self.dropout = nn.Dropout(1 - dropout)
def __init__(self, filters, n_filters, max_chars_per_token, char_embed_dim, n_chars, n_highway, output_dim, activation): super().__init__() self.max_chars_per_token = max_chars_per_token # activation for convolutions if activation == 'tanh': self._activation = nn.Tanh() elif activation == 'relu': self._activation = nn.ReLU() else: raise ValueError("Unknown activation") # init char_embedding self.char_embedding = Embedding(n_chars + 1, char_embed_dim, embedding_table=Uniform(1.0), padding_idx=0) # run convolutions convolutions = [] for (width, num) in filters: if activation == 'tanh': cnn_weight_init = Normal(np.sqrt(1.0 / width * char_embed_dim)) elif activation == 'relu': cnn_weight_init = Uniform(0.05) conv = nn.Conv1d(in_channels=char_embed_dim, out_channels=num, kernel_size=width, has_bias=True, weight_init=cnn_weight_init, pad_mode='valid') convolutions.append(conv) self._convolutions = nn.CellList(convolutions) # highway layers self._highways = HighWay(n_filters, n_highway, 'relu') # projection layer self._projection = nn.Dense(n_filters, output_dim, has_bias=True, weight_init=Normal(np.sqrt(1.0 / n_filters))) # array operations self.transpose = P.Transpose() self.concat = P.Concat(-1) self.max = P.ReduceMax()
def __init__(self, log_scale_min=-7.0, reduce=True): super(mix_gaussian_loss, self).__init__() self.log_scale_min = log_scale_min self.reduce = reduce self.transpose_op = P.Transpose() self.maximum = P.Maximum() self.tile = P.Tile() self.exp = P.Exp() self.logsoftmax = P.LogSoftmax(-1) self.expand_dims = P.ExpandDims() self.sums = P.ReduceSum() self.lse = log_sum_exp() self.sq = P.Square() self.sqrt = P.Sqrt() self.const = P.ScalarToArray() self.log = P.Log()
def __init__(self, num_classes=256, log_scale_min=-7.0, reduce=True): super(discretized_mix_logistic_loss, self).__init__() self.num_classes = num_classes self.log_scale_min = log_scale_min self.reduce = reduce self.transpose_op = P.Transpose() self.exp = P.Exp() self.sigmoid = P.Sigmoid() self.softplus = Stable_softplus() self.log = P.Log() self.cast = P.Cast() self.logsoftmax = P.LogSoftmax(-1) self.expand_dims = P.ExpandDims() self.tile = P.Tile() self.maximum = P.Maximum() self.sums = P.ReduceSum() self.lse = log_sum_exp() self.reshape = P.Reshape() self.factor = self.log(Tensor((self.num_classes - 1) / 2, ms.float32))
def __init__(self, log_scale_min=-7.0, reduce=True): super(mix_gaussian_loss, self).__init__() self.log_scale_min = log_scale_min self.reduce = reduce self.transpose_op = P.Transpose() self.maximum = P.Maximum() self.tile = P.Tile() self.exp = P.Exp() self.expand_dims = P.ExpandDims() self.sums = P.ReduceSum() self.lse = log_sum_exp() self.sq = P.Square() self.sqrt = P.Sqrt() self.const = P.ScalarToArray() self.log = P.Log() self.tensor_one = Tensor(1., ms.float32) if context.get_context("device_target") == "CPU": self.logsoftmax = log_softmax() else: self.logsoftmax = P.LogSoftmax(-1)
def __init__(self, batch_size, temperature=1, world_size=1): super(NT_Xent_Loss, self).__init__() # Parameters. self.LARGE_NUM = 1e9 self.batch_size = batch_size self.temperature = temperature self.world_size = world_size self.N = 2 * self.batch_size * self.world_size # Tail_Loss. self.criterion = CrossEntropyLoss(reduction="mean") self.norm = P.L2Normalize(axis=1) self.one_hot = P.OneHot() self.range = nn.Range(0, self.batch_size) self.one = Tensor(1.0, mstype.float32) self.zero = Tensor(0.0, mstype.float32) self.transpose = P.Transpose() self.matmul = nn.MatMul() # Operations. self.ones = P.Ones() self.zeros = P.Zeros() self.cat1 = P.Concat(axis=1)
def __init__(self, net_config, K=100, enable_nms_fp16=True): super(MultiPoseDecode, self).__init__() self.K = K self.nms = NMS(enable_nms_fp16=enable_nms_fp16) self.shape = ops.Shape() self.gather_topk = GatherTopK() self.gather_topk_channel = GatherTopKChannel() self.gather_by_ind = GatherFeatureByInd() self.half = ops.Split(axis=-1, output_num=2) self.half_first = ops.Split(axis=0, output_num=2) self.split = ops.Split(axis=-1, output_num=4) self.flip_lr = FlipLR() self.flip_lr_off = FlipLROff() self.flip_tensor = FlipTensor() self.concat = ops.Concat(axis=1) self.concat_a2 = ops.Concat(axis=2) self.concat_a3 = ops.Concat(axis=3) self.trans_gather_feature = TransposeGatherFeature() self.expand_dims = ops.ExpandDims() self.reshape = ops.Reshape() self.add = ops.TensorAdd() self.dtype = ops.DType() self.cast = ops.Cast() self.thresh = 0.1 self.transpose = ops.Transpose() self.perm_list = (0, 2, 1, 3) self.tile = ops.Tile() self.greater = ops.Greater() self.square = ops.Square() self.sqrt = ops.Sqrt() self.reduce_sum = ops.ReduceSum() self.min = ops.ArgMinWithValue(axis=3) self.max = ops.Maximum() self.hm_hp = net_config.hm_hp self.dense_hp = net_config.dense_hp self.reg_offset = net_config.reg_offset self.reg_hp_offset = net_config.reg_hp_offset self.hm_hp_ind = 3 if self.hm_hp else 2 self.reg_ind = self.hm_hp_ind + 1 if self.reg_offset else self.hm_hp_ind self.reg_hp_ind = self.reg_ind + 1 if self.reg_hp_offset else self.reg_ind
def __init__(self, vocab_size, embed_size, num_hiddens, num_layers, bidirectional, num_classes, weight, batch_size): super(SentimentNet, self).__init__() # Map words to vectors self.embedding = nn.Embedding(vocab_size, embed_size, embedding_table=weight) self.embedding.embedding_table.requires_grad = False self.trans = ops.Transpose() self.perm = (1, 0, 2) if context.get_context("device_target") in STACK_LSTM_DEVICE: # stack lstm by user self.encoder = StackLSTM(input_size=embed_size, hidden_size=num_hiddens, num_layers=num_layers, has_bias=True, bidirectional=bidirectional, dropout=0.0) self.h, self.c = stack_lstm_default_state(batch_size, num_hiddens, num_layers, bidirectional) else: # standard lstm self.encoder = nn.LSTM(input_size=embed_size, hidden_size=num_hiddens, num_layers=num_layers, has_bias=True, bidirectional=bidirectional, dropout=0.0) self.h, self.c = lstm_default_state(batch_size, num_hiddens, num_layers, bidirectional) self.concat = ops.Concat(1) if bidirectional: self.decoder = nn.Dense(num_hiddens * 4, num_classes) else: self.decoder = nn.Dense(num_hiddens * 2, num_classes)
if __name__ == '__main__': dataset_generator = DatasetGenerator(DATA_DIR, train=True, size=192) dataset = ds.GeneratorDataset(dataset_generator, ["hazy", "gt", "img_name"], shuffle=False) transforms_list = [ decode, (lambda img_name: set_random_seed(img_name, dataset_generator.get_seed())), py_trans.RandomCrop(192), py_trans.ToTensor(), ] compose_trans = Compose(transforms_list) dataset = dataset.map(operations=compose_trans, input_columns=["hazy"]) dataset = dataset.map(operations=compose_trans, input_columns=["gt"]) hazy_list, gt_list = [], [] for data in dataset.create_dict_iterator(): hazy_list.append(data['hazy']) gt_list.append(data['gt']) print("Transformed image Shape:", data['hazy'].shape, ", Transformed label:", data['gt'].shape) num_samples = 5 per = ops.Transpose() for i in range(num_samples): plt.subplot(2, num_samples, i+1) plt.imshow(per(hazy_list[i], (1, 2, 0)).asnumpy()) plt.subplot(2, num_samples, num_samples + i + 1) plt.imshow(per(gt_list[i], (1, 2, 0)).asnumpy()) plt.show()
def __init__(self, backbone, loss_fn): super(WithLossCell, self).__init__(auto_prefix=False) self._backbone = backbone self._loss_fn = loss_fn self.transpose = P.Transpose()
def construct(self, inputs, targets): """ Args: - inputs: feature matrix with shape (batch_size, feat_dim) - targets: ground truth labels with shape (num_classes) """ n = inputs.shape[0] # Compute pairwise distance, replace by the official when merged pow = P.Pow() sum = P.ReduceSum(keep_dims=True) expand = P.BroadcastTo((n, n)) transpose = P.Transpose() mul = P.Mul() add = P.Add() sqrt = P.Sqrt() equal = P.Equal() cat = P.Concat() ones_like = P.OnesLike() dist = pow(inputs, 2) dist = sum(dist, axis=1) dist = expand(dist) dist = dist + transpose(dist, (1, 0)) temp1 = P.matmul(inputs, transpose(inputs, (1, 0))) temp1 = mul(-2, temp1) dist = add(dist, temp1) dist = P.composite.clip_by_value( dist, clip_value_min=1e-12, clip_value_max=100000000 ) # for numerical stability, clip_value_max=? why must set? dist = sqrt(dist) # For each anchor, find the hardest positive and negative targets = expand(targets) mask = equal(targets, transpose(targets, (1, 0))) dist_ap = [] dist_an = [] # only for debugging ##################### # print("dist is") # print(dist.shape) # print(dist) # print("mask is") # print(mask.shape) # print(mask) # print(mask[0]) ##################### for i in range(n): minval = -1.0 maxval = -1.0 for j in range(n): if mask[i][j] and dist[i][j] > maxval: maxval = dist[i][j] if not mask[i][j] and (dist[i][j] < minval or minval == -1): minval = dist[i][j] if (not isinstance(minval, Tensor) or not isinstance(maxval, Tensor) or minval == -1.0 or maxval == -1.0): if self.error_msg is not None: print("Error Msg", file=self.error_msg) print("mask {} is".format(i), file=self.error_msg) print(mask[i], file=self.error_msg) print("dist is:", file=self.error_msg) print(dist[i], file=self.error_msg) print(maxval, file=self.error_msg) print(minval, file=self.error_msg) print(type(maxval), file=self.error_msg) print(type(minval), file=self.error_msg) self.error_msg.flush() # assert minval != -1.0 and isinstance(minval, Tensor) # assert maxval != -1.0 and isinstance(maxval, Tensor) dist_ap.append(maxval.asnumpy()) dist_an.append(minval.asnumpy()) dist_ap = Tensor(dist_ap, ms.float32) dist_an = Tensor(dist_an, ms.float32) # only for debugging ##################### # print(dist_ap) # print(dist_ap.shape) # print(dist_an) ##################### # Compute ranking hinge loss y = ones_like(dist_an) loss = self.ranking_loss(dist_an, dist_ap, y) # # compute accuracy # correct = torch.ge(dist_an, dist_ap).sum().item() return loss # class GradOriTripletLoss(nn.Cell) # def __init__(self, net): # super(GradOriTripletLoss, self).__init__() # self.net = net # self.grad_op = P.GradOperation(get_all=True) # # def construct(self, inputs, targets): # gradient_function = self.grad_op(self.net) # return gradient_function(inputs, targets)
def __init__(self, batch_size, from_tensor_width, to_tensor_width, from_seq_length, to_seq_length, num_attention_heads=1, size_per_head=512, query_act=None, key_act=None, value_act=None, has_attention_mask=False, attention_probs_dropout_prob=0.0, use_one_hot_embeddings=False, initializer_range=0.02, do_return_2d_tensor=False, use_relative_positions=False, compute_type=mstype.float32): super(BertAttention, self).__init__() self.batch_size = batch_size self.from_seq_length = from_seq_length self.to_seq_length = to_seq_length self.num_attention_heads = num_attention_heads self.size_per_head = size_per_head self.has_attention_mask = has_attention_mask self.use_relative_positions = use_relative_positions self.scores_mul = Tensor([1.0 / math.sqrt(float(self.size_per_head))], dtype=compute_type) self.reshape = ops.Reshape() self.shape_from_2d = (-1, from_tensor_width) self.shape_to_2d = (-1, to_tensor_width) weight = TruncatedNormal(initializer_range) units = num_attention_heads * size_per_head self.query_layer = nn.Dense(from_tensor_width, units, activation=query_act, weight_init=weight).to_float(compute_type) self.key_layer = nn.Dense(to_tensor_width, units, activation=key_act, weight_init=weight).to_float(compute_type) self.value_layer = nn.Dense(to_tensor_width, units, activation=value_act, weight_init=weight).to_float(compute_type) self.shape_from = (batch_size, from_seq_length, num_attention_heads, size_per_head) self.shape_to = ( batch_size, to_seq_length, num_attention_heads, size_per_head) self.matmul_trans_b = ops.BatchMatMul(transpose_b=True) self.multiply = ops.Mul() self.transpose = ops.Transpose() self.trans_shape = (0, 2, 1, 3) self.trans_shape_relative = (2, 0, 1, 3) self.trans_shape_position = (1, 2, 0, 3) #self.multiply_data = Tensor([-10000.0,], dtype=compute_type) self.multiply_data = Tensor([-10000.0,], dtype=mstype.float32) self.batch_num = batch_size * num_attention_heads self.matmul = ops.BatchMatMul() self.softmax = nn.Softmax() self.dropout = nn.Dropout(1 - attention_probs_dropout_prob) if self.has_attention_mask: self.expand_dims = ops.ExpandDims() self.sub = ops.Sub() self.add = ops.TensorAdd() self.cast = ops.Cast() self.get_dtype = ops.DType() if do_return_2d_tensor: self.shape_return = (batch_size * from_seq_length, num_attention_heads * size_per_head) else: self.shape_return = (batch_size, from_seq_length, num_attention_heads * size_per_head) self.cast_compute_type = SaturateCast(dst_type=compute_type) if self.use_relative_positions: self._generate_relative_positions_embeddings = \ RelaPosEmbeddingsGenerator(length=to_seq_length, depth=size_per_head, max_relative_position=16, initializer_range=initializer_range, use_one_hot_embeddings=use_one_hot_embeddings)
def __init__(self, num_classes, input_nc=1, padding=1, pad_mode='pad', has_bias=False, use_dropout=False): super(DFCNN, self).__init__() if pad_mode == 'pad': assert padding >= 0, "when the pad_mode is 'pad', the padding must be greater than or equal to 0!" if pad_mode == 'same' or pad_mode == 'valid': assert padding == 0, "when the pad_mode is 'same' or 'valid', the padding must be equal to 0!" self.use_dropout = use_dropout # structure # seq 1 self.conv11 = nn.Conv2d(in_channels=input_nc, out_channels=64, kernel_size=3, stride=1, padding=padding, has_bias=has_bias, pad_mode=pad_mode) self.bn11 = nn.BatchNorm2d(64) self.relu11 = nn.ReLU() self.conv12 = nn.Conv2d(in_channels=64, out_channels=64, kernel_size=3, stride=1, padding=padding, has_bias=has_bias, pad_mode=pad_mode) self.bn12 = nn.BatchNorm2d(64) self.relu12 = nn.ReLU() self.maxpool1 = nn.MaxPool2d(kernel_size=2, stride=2, pad_mode='valid') # seq 2 self.conv21 = nn.Conv2d(in_channels=64, out_channels=128, kernel_size=3, stride=1, padding=padding, has_bias=has_bias, pad_mode=pad_mode) self.bn21 = nn.BatchNorm2d(128) self.relu21 = nn.ReLU() self.conv22 = nn.Conv2d(in_channels=128, out_channels=128, kernel_size=3, stride=1, padding=padding, has_bias=has_bias, pad_mode=pad_mode) self.bn22 = nn.BatchNorm2d(128) self.relu22 = nn.ReLU() self.maxpool2 = nn.MaxPool2d(kernel_size=2, stride=2, pad_mode='valid') # seq 3 self.conv31 = nn.Conv2d(in_channels=128, out_channels=256, kernel_size=3, stride=1, padding=padding, has_bias=has_bias, pad_mode=pad_mode) self.bn31 = nn.BatchNorm2d(256) self.relu31 = nn.ReLU() self.conv32 = nn.Conv2d(in_channels=256, out_channels=256, kernel_size=3, stride=1, padding=padding, has_bias=has_bias, pad_mode=pad_mode) self.bn32 = nn.BatchNorm2d(256) self.relu32 = nn.ReLU() self.conv33 = nn.Conv2d(in_channels=256, out_channels=256, kernel_size=3, stride=1, padding=padding, has_bias=has_bias, pad_mode=pad_mode) self.bn33 = nn.BatchNorm2d(256) self.relu33 = nn.ReLU() self.maxpool3 = nn.MaxPool2d(kernel_size=2, stride=2, pad_mode='valid') # seq 4 self.conv41 = nn.Conv2d(in_channels=256, out_channels=512, kernel_size=3, stride=1, padding=padding, has_bias=has_bias, pad_mode=pad_mode) self.bn41 = nn.BatchNorm2d(512) self.relu41 = nn.ReLU() self.conv42 = nn.Conv2d(in_channels=512, out_channels=512, kernel_size=3, stride=1, padding=padding, has_bias=has_bias, pad_mode=pad_mode) self.bn42 = nn.BatchNorm2d(512) self.relu42 = nn.ReLU() self.conv43 = nn.Conv2d(in_channels=512, out_channels=512, kernel_size=3, stride=1, padding=padding, has_bias=has_bias, pad_mode=pad_mode) self.bn43 = nn.BatchNorm2d(512) self.relu43 = nn.ReLU() self.maxpool4 = nn.MaxPool2d(kernel_size=1, stride=1, pad_mode='valid') # seq 5 self.conv51 = nn.Conv2d(in_channels=512, out_channels=512, kernel_size=3, stride=1, padding=padding, has_bias=has_bias, pad_mode=pad_mode) self.bn51 = nn.BatchNorm2d(512) self.relu51 = nn.ReLU() self.conv52 = nn.Conv2d(in_channels=512, out_channels=512, kernel_size=3, stride=1, padding=padding, has_bias=has_bias, pad_mode=pad_mode) self.bn52 = nn.BatchNorm2d(512) self.relu52 = nn.ReLU() self.conv53 = nn.Conv2d(in_channels=512, out_channels=512, kernel_size=3, stride=1, padding=padding, has_bias=has_bias, pad_mode=pad_mode) self.bn53 = nn.BatchNorm2d(512) self.relu53 = nn.ReLU() self.maxpool5 = nn.MaxPool2d(kernel_size=1, stride=1, pad_mode='valid') self.bn = nn.BatchNorm2d(512) if self.use_dropout: self.drop1 = nn.Dropout(0.8) self.drop2 = nn.Dropout(0.8) self.drop3 = nn.Dropout(0.8) self.drop4 = nn.Dropout(0.8) self.drop5 = nn.Dropout(0.8) self.drop_fc1 = nn.Dropout(0.5) self.drop_fc2 = nn.Dropout(0.5) self.fc1 = nn.Dense(25 * 512, 4096, activation='relu') self.fc2 = nn.Dense(4096, 4096, activation='relu') self.fc3 = nn.Dense(4096, num_classes, activation='relu') # operation self.transpose = ops.Transpose() self.reshape = ops.Reshape()
def __init__(self, network, input_format="NCHW"): super(BuildEvalNetwork, self).__init__() self.network = network self.softmax = nn.Softmax(axis=1) self.transpose = ops.Transpose() self.format = input_format