def _euclidean_squared_distance(input1, input2): """Computes euclidean squared distance. Args: input1 : 2-D feature matrix. input2 : 2-D feature matrix. Returns: distance matrix. """ m, n = input1.shape[0], input2.shape[0] temp1 = math.reduce_sum(math.pow( input1, flow.constant_like(input1, 2, dtype=flow.float32)), axis=1) temp2 = math.reduce_sum(math.pow( input2, flow.constant_like(input2, 2, dtype=flow.float32)), axis=1) shape_tensor1 = flow.constant(value=0.0, dtype=flow.float32, shape=(m, n)) shape_tensor2 = flow.constant(value=0.0, dtype=flow.float32, shape=(n, m)) temp1 = flow.broadcast_like(temp1, like=shape_tensor1, broadcast_axes=[1]) temp2 = flow.transpose(flow.broadcast_like(temp2, like=shape_tensor2, broadcast_axes=[1]), perm=(1, 0)) dismat = math.add(temp1, temp2) return math.add( dismat, math.multiply(-2, flow.matmul(input1, flow.transpose(input2, perm=(1, 0)))))
def SplitLikeJob(x: oft.Numpy.Placeholder(x_shape, dtype=flow.float)): v = flow.get_variable( "x", shape=x_shape, dtype=flow.float, initializer=flow.constant_initializer(0), trainable=True, ) x += v like0 = flow.constant(0, dtype=flow.float, shape=like0_shape) like1 = flow.constant(0, dtype=flow.float, shape=like1_shape) with flow.scope.placement("gpu", "0:0"): y0, y1 = split_like(x, [like0, like1], "split_like") loss = y0 flow.optimizer.SGD( flow.optimizer.PiecewiseConstantScheduler([], [1e-4]), momentum=0 ).minimize(loss) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch(loss, test_global_storage.Setter("loss")) flow.watch_diff(loss, test_global_storage.Setter("loss_diff")) return y0, y1
def flow_net(var_name, random_mask): with flow.scope.placement(device_type, "0:0-0"): x = flow.get_variable( name=var_name, shape=x_shape, dtype=flow.float32, initializer=flow.ones_initializer(), trainable=True, ) constant_val = flow.constant(3.0, dtype=flow.float32, shape=(1, )) x = x * constant_val x = x * 2.0 if device_type == "gpu": x = flow.cast(x, flow.float16) x = flow.math.relu(x) x = flow.cast(x, flow.float) loss = flow.math.reduce_mean(x * random_mask) flow.optimizer.Adam( flow.optimizer.PiecewiseConstantScheduler([], [learning_rate]), beta1=beta1, beta2=beta2, epsilon=epsilon, do_bias_correction=True, ).minimize(loss) return x
def image_flip( image: BlobDef, flip_code: Union[int, BlobDef], name: Optional[str] = None ) -> BlobDef: assert isinstance(image, BlobDef) if name is None: name = id_util.UniqueStr("ImageFlip_") if not isinstance(flip_code, BlobDef): assert isinstance(flip_code, int) flip_code = flow.constant( flip_code, shape=(image.shape[0],), dtype=flow.int8, name="{}_FlipCode_".format(name), ) else: assert image.shape[0] == flip_code.shape[0] op = ( flow.user_op_builder(name) .Op("image_flip") .Input("in", [image]) .Input("flip_code", [flip_code]) .Output("out") .Build() ) return op.InferAndTryRun().SoleOutputBlob()
def object_bbox_flip( bbox: BlobDef, image_size: BlobDef, flip_code: Union[int, BlobDef], name: Optional[str] = None, ) -> BlobDef: assert isinstance(bbox, BlobDef) assert isinstance(image_size, BlobDef) assert bbox.shape[0] == image_size.shape[0] if name is None: name = id_util.UniqueStr("ObjectBboxFlip_") if not isinstance(flip_code, BlobDef): assert isinstance(flip_code, int) flip_code = flow.constant( flip_code, shape=(bbox.shape[0],), dtype=flow.int8, name="{}_FlipCode".format(name), ) else: assert bbox.shape[0] == flip_code.shape[0] op = ( flow.user_op_builder(name) .Op("object_bbox_flip") .Input("bbox", [bbox]) .Input("image_size", [image_size]) .Input("flip_code", [flip_code]) .Output("out") .Build() ) return op.InferAndTryRun().SoleOutputBlob()
def object_segm_poly_flip( poly: BlobDef, image_size: BlobDef, flip_code: Union[int, BlobDef], name: Optional[str] = None, ) -> BlobDef: assert isinstance(poly, BlobDef) assert isinstance(image_size, BlobDef) assert poly.shape[0] == image_size.shape[0] if name is None: name = id_util.UniqueStr("ObjectSegmPolyFilp_") if not isinstance(flip_code, BlobDef): assert isinstance(flip_code, int) flip_code = flow.constant( flip_code, shape=(poly.shape[0],), dtype=flow.int8, name="{}_FlipCode".format(name), ) else: assert poly.shape[0] == flip_code.shape[0] op = ( flow.user_op_builder(name) .Op("object_segmentation_polygon_flip") .Input("poly", [poly]) .Input("image_size", [image_size]) .Input("flip_code", [flip_code]) .Output("out") .Build() ) return op.InferAndTryRun().SoleOutputBlob()
def hybrid_concat_job( input_0_def: oft.ListNumpy.Placeholder(shape=static_shape, dtype=flow.float), input_1_def: oft.ListNumpy.Placeholder(shape=static_shape, dtype=flow.float), ): var = flow.get_variable( "var", shape=static_shape, dtype=flow.float, initializer=flow.random_uniform_initializer(), trainable=True, ) constant = flow.constant(1.0, dtype=flow.float, shape=rand_sub_shape) inputs = [ flow.cast_to_current_logical_view(input) for input in [var, input_0_def, input_1_def, constant] ] concated = flow.concat(inputs, axis=axis, max_dim_size=max_dim_size,) if verbose: print("concated static shape:", concated.shape) flow.optimizer.SGD( flow.optimizer.PiecewiseConstantScheduler([], [1e-3]), momentum=0 ).minimize(concated) flow.watch_diff(var, compare_var_diff) if max_dim_size is None: test_case.assertTrue( concated.shape[axis] == (static_shape[axis] * 3 + rand_sub_shape[axis]) ) else: test_case.assertTrue(concated.shape[axis] == max_dim_size) return var, concated
def build(self, inputs, targets): """ Args: inputs (torch.Tensor): feature matrix with shape (batch_size, feat_dim). targets (torch.LongTensor): ground truth labels with shape (num_classes). """ n = inputs.shape[0] dist = math.reduce_sum(math.pow( inputs, flow.constant_like(inputs, 2, dtype=flow.float32)), axis=1) shape_tensor = flow.constant(value=0.0, dtype=flow.float32, shape=(n, n)) dist = flow.broadcast_like(dist, like=shape_tensor, broadcast_axes=[1]) dist = math.add( dist, flow.transpose(dist, perm=(1, 0), batch_axis_non_change=True)) temp1 = math.multiply( -2, flow.matmul( inputs, flow.transpose(inputs, perm=(1, 0), batch_axis_non_change=True))) dist = math.add(dist, temp1) dist = math.sqrt(flow.clamp(dist, min_value=1e-12)) mask = math.equal( flow.broadcast_like(targets, like=shape_tensor, broadcast_axes=[1]), flow.transpose(flow.broadcast_like(targets, like=shape_tensor, broadcast_axes=[1]), perm=(1, 0), batch_axis_non_change=True)) mask_rev = math.not_equal( flow.broadcast_like(targets, like=shape_tensor, broadcast_axes=[1]), flow.transpose(flow.broadcast_like(targets, like=shape_tensor, broadcast_axes=[1]), perm=(1, 0), batch_axis_non_change=True)) dist_ap, dist_an = [], [] for i in range(n): temp_dist = flow.slice_v2(dist, [(i, i + 1, 1)]) temp_mask = flow.slice_v2(mask, [(i, i + 1, 1)]) temp_mask_rev = flow.slice_v2(mask_rev, [(i, i + 1, 1)]) dist_ap.append( math.reduce_max( flow.gather_nd(temp_dist, flow.where(temp_mask)))) dist_an.append( math.reduce_min( flow.gather_nd(temp_dist, flow.where(temp_mask_rev)))) dist_ap = flow.concat(dist_ap, 0) dist_an = flow.concat(dist_an, 0) y = flow.ones_like(dist_an) # return dist_an, dist_ap, y return self._MarginRankingLoss(dist_an, dist_ap, y)
def _CreateAttentionMaskFromInputMask(to_mask_blob, from_seq_length, to_seq_length): output = flow.cast(to_mask_blob, dtype=flow.float) output = flow.reshape(output, [-1, 1, to_seq_length]) zeros = flow.constant(0.0, dtype=flow.float, shape=[from_seq_length, to_seq_length]) output = zeros + output return output
def foo_job(): x = flow.constant(1, shape=(2, 5), dtype=flow.float) y = flow.get_variable( name="var", shape=(64, 4), dtype=flow.float, initializer=flow.zeros_initializer(), ) return x, y
def _CreateAttentionMaskFromInputMask(to_mask_blob, from_seq_length, to_seq_length): output = flow.cast(to_mask_blob, dtype=flow.float) output = flow.reshape(output, [-1, 1, to_seq_length]) zeros = flow.constant(0.0, dtype=flow.float, shape=[from_seq_length, to_seq_length]) attention_mask_blob = zeros + output attention_mask_blob = flow.reshape( attention_mask_blob, [-1, 1, from_seq_length, to_seq_length] ) attention_mask_blob = flow.cast(attention_mask_blob, dtype=flow.float) addr_blob = (attention_mask_blob - 1.0) * 10000.0 return addr_blob
def get_angles(pos, i, d_model): """ Compute angles The equation is 1 / 10000^(2i / d_model) :param pos: The position dims, shape=(position, 1) :param i: The d_model index, shape = (1, d_model) :param d_model: The hidden dims, int value :return: """ # Get constant value as d_model d_model_constant = flow.constant(d_model, dtype=flow.float32, shape=(1,), name="One_constant") constant_10000 = flow.constant(10000, dtype=flow.float32, shape=(1, d_model), name="constant_10000") constant_2 = flow.constant_scalar(2, dtype=flow.float32) # Compute angle_rates = 1 / 10000^(2i / d_model) angle_rates = 1 / flow.math.pow(constant_10000, (constant_2 * flow.math.floor(i / constant_2)) / d_model_constant) return pos * angle_rates
def positional_encoding(position, d_model, name="positional_encoding"): """ Do positional encoding :param position: The position :param d_model: The hidden dimension in model :return: shape like (1, position, d_model) """ with flow.scope.namespace(name): # shape = (position, 1) input_pos = flow.expand_dims(flow.range(position, dtype=flow.float32, name="pos"), axis=1) # shape = (1, d_model) input_d_model = flow.expand_dims(flow.range(d_model, dtype=flow.float32, name="d_model"), axis=0) angle_rads = get_angles(input_pos, input_d_model, d_model) # Get a even range like (0, 2, 4, 6, ....., d_model) even_range = flow.range(0, d_model, 2, dtype=flow.int32, name="even_range") # Do the sin in even indexes even_out = flow.math.sin(flow.gather(angle_rads, even_range, axis=1)) # Get a odd range like (1, 3, 5, 7, ....., d_model) odd_range = flow.range(1, d_model, 2, dtype=flow.int32, name="odd_range") # Do the cos in odd indexes odd_out = flow.math.cos(flow.gather(angle_rads, odd_range, axis=1)) # Initialize Position encode constant position_encode = flow.constant(0, dtype=flow.float32, shape=(d_model, position), name="pos_ende") # Due to the scatter only support row indexes, we need to transpose even_out = flow.tensor_scatter_nd_update(position_encode, flow.expand_dims(even_range, axis=1), flow.transpose(even_out, perm=[1, 0])) odd_out = flow.tensor_scatter_nd_update(position_encode, flow.expand_dims(odd_range, axis=1), flow.transpose(odd_out, perm=[1, 0])) # Add even indexes value and odd indexes value out = even_out + odd_out # Because We have transposed in even_out and odd_out, So we need to transpose back out = flow.transpose(out, perm=[1, 0]) # Expand dims in dim=0, we get shape like (1, position, d_model) out = flow.expand_dims(out, axis=0) return out
def att_distill(args, student_atts, teacher_atts): att_loss = 0. teacher_layer_num = len(teacher_atts) student_layer_num = len(student_atts) assert teacher_layer_num % student_layer_num == 0 layers_per_block = int(teacher_layer_num / student_layer_num) new_teacher_atts = [ teacher_atts[i * layers_per_block + layers_per_block - 1] for i in range(student_layer_num) ] for student_att, teacher_att in zip(student_atts, new_teacher_atts): student_att = flow.where( student_att <= flow.constant(-1e2, dtype=flow.float), flow.zeros_like(student_att), student_att) teacher_att = flow.where( teacher_att <= flow.constant(-1e2, dtype=flow.float), flow.zeros_like(teacher_att), teacher_att) tmp_loss = mseloss(student_att, teacher_att) att_loss += tmp_loss return att_loss
def __call__(self, x, enc_output, training, look_ahead_mask, padding_mask): """ Forward :param x: The input X :param pos_encoding: The positional encoding :param enc_output: The encoder output :param training: Whether training :param look_ahead_mask: The look ahead mask :param padding_mask: The padding mask :return: """ # Sequence length seq_len = x.shape[1] attention_weights = {} # Embedding with flow.scope.namespace("Decoder_Embedding"): x = EmbeddingLayer(x, vocab_size=self.target_vocab_size, embedding_size=self.d_model) d_model_constant = flow.constant(self.d_model, dtype=flow.float32, shape=(1,)) x *= flow.math.sqrt(d_model_constant) # print(x.shape) # Position encoding with flow.scope.namespace("Decoder_Position_encoding"): pos_encoding = flow.slice(self.pos_encoding, begin=[None, 0, None], size=[None, seq_len, None]) x += pos_encoding if training: x = flow.nn.dropout(x, rate=self.rate) # Decoding with flow.scope.namespace("Decoder_Multi_decoder"): for i in range(self.num_layers): with flow.scope.namespace('decoder_{}'.format(i)): x, block1, block2 = self.dec_layers[i](x, enc_output, training, look_ahead_mask, padding_mask) attention_weights['decoder_layer{}_block1'.format(i + 1)] = block1 attention_weights['decoder_layer{}_block2'.format(i + 1)] = block2 return x, attention_weights
def oneflow_Xmum( of_input_1: tp.Numpy.Placeholder(shape=input_1.shape, dtype=value_type["of_type"]), of_input_2: tp.Numpy.Placeholder(shape=input_2.shape, dtype=value_type["of_type"]), ) -> tp.Numpy: with flow.scope.placement(device_type, "0:0"): v1 = flow.get_variable( shape=input_1.shape, dtype=value_type["of_type"], initializer=flow.zeros_initializer(), name="x1_var", ) x1_var = of_input_1 + v1 if not dx_only: v2 = flow.get_variable( shape=input_2.shape, dtype=value_type["of_type"], initializer=flow.zeros_initializer(), name="x2_var", ) x2_var = of_input_2 + v2 else: x2_var = flow.constant(value=1.5, shape=of_input_2.shape, dtype=value_type["of_type"]) flow.watch_diff(x1_var, assert_prediction_grad) # Only Compare input1 Grad if compare_type == "maximum": of_Xmum_out = flow.math.maximum(x1_var, x2_var) elif compare_type == "minimum": of_Xmum_out = flow.math.minimum(x1_var, x2_var) with flow.scope.placement(device_type, "0:0"): flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler( [], [1e-3]), momentum=0).minimize(of_Xmum_out) return of_Xmum_out
def padded_cross_entropy_loss(logits, labels, smoothing, vocab_size): """Calculate cross entropy loss while ignoring padding. Args: logits: Tensor of size [batch_size, length_logits, vocab_size] labels: Tensor of size [batch_size, length_labels] smoothing: Label smoothing constant, used to determine the on and off values vocab_size: int size of the vocabulary Returns: Returns a float32 tensor with shape [batch_size, max(length_logits, length_labels)] """ with flow.scope.namespace("loss"): logits, labels = _pad_tensors_to_same_length(logits, labels) # Calculate smoothing cross entropy with flow.scope.namespace("smoothing_cross_entropy"): confidence = 1.0 - smoothing # low_confince = (1.0 - confidence) / flow.cast(vocab_size-1 ,dtype=flow.float32) low_confidence = (1.0 - confidence) / float(vocab_size - 1) soft_targets = flow.one_hot(flow.cast(labels, flow.int32), depth=vocab_size, on_value=confidence, off_value=low_confidence, dtype=flow.float32) xentropy = flow.nn.softmax_cross_entropy_with_logits( logits=logits, labels=soft_targets) normalizing_constant = -(confidence * math.log(confidence) + float(vocab_size - 1) * low_confidence * math.log(low_confidence + 1e-20)) xentropy -= normalizing_constant weights = flow.cast(flow.math.not_equal( labels, flow.constant(value=0, dtype=flow.float32, shape=labels.shape)), dtype=flow.float32) return xentropy * weights, weights
def scaled_dot_product_attention(query, key, value, mask=None): """ Build Attention Layer :param query: Query Matrix :param key: Key Matrix :param value: Value Matrix :param mask: The Mask :return: """ matmul_qk = flow.matmul(query, key, transpose_b=True) # scaled matmul_qk d_k = flow.constant(query.shape[-1], dtype=flow.float32) scaled_attention_logits = matmul_qk / flow.math.sqrt(d_k) # Add mask if mask is not None: scaled_attention_logits += (mask * 1e-9) attention_weights = flow.nn.softmax(scaled_attention_logits, axis=-1) out = flow.matmul(attention_weights, value) return out, attention_weights
def constant_scalar( value: Union[int, float], dtype: Optional[flow.dtype] = None, name: Optional[str] = None, ) -> oneflow._oneflow_internal.BlobDesc: """This operator creates a constant scalar Blob. Args: value (Union[int, float]): The constant value of Blob. dtype (Optional[flow.dtype], optional): The data type of Blob. Defaults to None. name (Optional[str], optional): The name for the operation. Defaults to None. Returns: oneflow._oneflow_internal.BlobDesc: The result blob. For example: .. code-block:: python import oneflow as flow import numpy as np import oneflow.typing as tp @flow.global_function() def constant_scalar_Job() -> tp.Numpy: constant_scalar = flow.constant_scalar(value=2.5, dtype=flow.float) return constant_scalar out = constant_scalar_Job() # out [2.5] """ return flow.constant(value, dtype=dtype, shape=[1])
def meanshift(x, rgb_range, rgb_mean, rgb_std, sign=-1, name="Meanshift"): # Concat the rgb_std _new_constant_std_0 = flow.constant_scalar(rgb_std[0], dtype=flow.float32, name=name + "_std_0") _new_constant_std_1 = flow.constant_scalar(rgb_std[1], dtype=flow.float32, name=name + "_std_1") _new_constant_std_2 = flow.constant_scalar(rgb_std[2], dtype=flow.float32, name=name + "_std_2") _std = flow.concat( inputs=[_new_constant_std_0, _new_constant_std_1, _new_constant_std_2], axis=0, ) _reshaped_std = flow.reshape(_std, (3, 1, 1, 1), name=name + "reshape_std") # Concat the rgb_mean _new_constant_mean_0 = flow.constant_scalar(rgb_mean[0], dtype=flow.float32, name=name + "_mean_0") _new_constant_mean_1 = flow.constant_scalar(rgb_mean[1], dtype=flow.float32, name=name + "_mean_1") _new_constant_mean_2 = flow.constant_scalar(rgb_mean[2], dtype=flow.float32, name=name + "_mean_2") _mean = flow.concat( inputs=[ _new_constant_mean_0, _new_constant_mean_1, _new_constant_mean_2 ], axis=0, ) _weight_ones = flow.constant(1.0, dtype=flow.float32, shape=(3, 3), name=name + "_ones") # Generate eye matrix # [[1, 0, 0], [[0, 0, 0], # [1, 1, 0], - [1, 0, 0], # [1, 1, 1]] [1, 1, 0]] weight = flow.math.tril(_weight_ones, 0) - flow.math.tril(_weight_ones, -1) weight = flow.reshape(weight, shape=(3, 3, 1, 1), name=name + "_reshaped_weight") weight = flow.math.divide(weight, _reshaped_std) bias = sign * rgb_range * _mean bias = flow.math.divide(bias, _std) _conv = flow.nn.conv2d(x, filters=weight, strides=1, padding="SAME", name=name + "_mean_shift_conv") output = flow.nn.bias_add(_conv, bias, data_format="NCHW", name=name + "_addbias") return output
def forward(self, inputs, targets): n = inputs.shape[0] # Compute pairwise distance, replace by the official when merged tempname = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S.%f') shape_tensor = flow.constant(value=0.0, dtype=flow.float32, shape=(n, n)) if self.distance == 'euclidean': blob_2 = flow.get_variable( "blob_2_" + tempname, shape=inputs.shape, initializer=flow.constant_initializer(2), dtype=inputs.dtype) dist = flow.math.pow(inputs, blob_2) dist = flow.math.reduce_sum(dist, axis=1, keepdims=True) dist = flow.broadcast_like(dist, shape_tensor) tempdist = flow.transpose(dist) dist = dist + tempdist inputs_t = flow.transpose(inputs) dist = addmm(dist, inputs, inputs_t, beta=1, alpha=-2) dist = flow.clamp(dist, min_value=1e-12) dist = flow.math.sqrt(dist) elif self.distance == 'cosine': #fnorm=flow.math.l2_normalize(inputs, axis=1) fnorm = flow.math.reduce_mean(flow.math.divide( inputs, flow.math.l2_normalize(inputs, axis=1)), axis=1, keepdims=True) expand_fnorm = flow.broadcast_like(fnorm, like=inputs, broadcast_axes=[1]) l2norm = flow.math.divide(inputs, expand_fnorm) l2norm_t = flow.transpose(l2norm, perm=(1, 0)) dist = flow.math.negative(flow.matmul(l2norm, l2norm_t)) # For each anchor, find the hardest positive and negative mask = math.equal( flow.broadcast_like(targets, like=shape_tensor, broadcast_axes=[1]), flow.transpose(flow.broadcast_like(targets, like=shape_tensor, broadcast_axes=[1]), perm=(1, 0), batch_axis_non_change=True)) mask_rev = math.not_equal( flow.broadcast_like(targets, like=shape_tensor, broadcast_axes=[1]), flow.transpose(flow.broadcast_like(targets, like=shape_tensor, broadcast_axes=[1]), perm=(1, 0), batch_axis_non_change=True)) dist_ap, dist_an = [], [] for i in range(n): temp_dist = flow.slice_v2(dist, [(i, i + 1, 1)]) temp_mask = flow.slice_v2(mask, [(i, i + 1, 1)]) temp_mask_rev = flow.slice_v2(mask_rev, [(i, i + 1, 1)]) temp_dist_ap = flow.expand_dims( math.reduce_max( flow.gather_nd(temp_dist, flow.where(temp_mask))), 0) temp_dist_an = flow.expand_dims( math.reduce_min( flow.gather_nd(temp_dist, flow.where(temp_mask_rev))), 0) dist_ap.append(temp_dist_ap) dist_an.append(temp_dist_an) dist_ap = flow.concat(dist_ap, 0) dist_an = flow.concat(dist_an, 0) y = flow.ones_like(dist_an) return self._MarginRankingLoss(dist_an, dist_ap, y)
def ctc_loss( log_probs: oneflow_api.BlobDesc, targets: oneflow_api.BlobDesc, input_lengths: oneflow_api.BlobDesc, target_lengths: oneflow_api.BlobDesc, blank: int = 0, reduction: str = "mean", zero_infinity: bool = False, name: Optional[str] = None, ) -> oneflow_api.BlobDesc: r"""Computes the CTC(Connectionist Temporal Classification) loss. This operator implements the CTC loss as presented in (Graves et al., 2006). Args: log_probs (oneflow_api.BlobDesc): A Blob of shape [input_length, batch_size, num_labels]. The logarithmized probabilities of the outputs (e.g. obtained with flow.nn.logsoftmax()). targets (oneflow_api.BlobDesc): A Blob of shape [batch_size, max_target_length]. It represent the target sequences. Each element in the target sequence is a class index. And the target index cannot be blank (default=0). input_lengths (oneflow_api.BlobDesc): A Blob of shape [batch_size]. It represent the lengths of the inputs. And the lengths are specified for each sequence to achieve masking under the assumption that sequences are padded to equal lengths. target_lengths (oneflow_api.BlobDesc): A Blob of shape [batch_size]. It represent lengths of the targets. Lengths are specified for each sequence to achieve masking under the assumption that sequences are padded to equal lengths. blank (int, optional): Blank label. Defaults to 0. reduction (str, optional): The reduce type, it can be the one of "none", "mean", "sum". "none": no reduction will be applied, "mean": the output losses will be divided by the target lengths and then the mean over the batch is taken, "sum": the output will be summed. Defaults to "mean". zero_infinity (bool, optional): Whether to zero infinite losses and the associated gradients. Infinite losses mainly occur when the inputs are too short to be aligned to the targets. Defaults to False. name (Optional[str], optional): The name for the operation. Defaults to None. Returns: oneflow_api.BlobDesc: The result Blob. For example: .. code-block:: python import oneflow as flow import oneflow.typing as tp import numpy as np @flow.global_function() def ctc_loss_job( log_probs: tp.Numpy.Placeholder(shape=(5, 2, 3)), targets: tp.Numpy.Placeholder(shape=(2, 3), dtype=flow.int32), input_lengths: tp.Numpy.Placeholder(shape=(2,), dtype=flow.int32), target_lengths: tp.Numpy.Placeholder(shape=(2,), dtype=flow.int32), ) -> tp.Numpy: loss = flow.ctc_loss( log_probs, targets, input_lengths, target_lengths, blank=0, reduction="none" ) return loss log_probs = np.array( [ [[-1.1031, -0.7998, -1.5200], [-0.9808, -1.1363, -1.1908]], [[-1.2258, -1.0665, -1.0153], [-1.1135, -1.2331, -0.9671]], [[-1.3348, -0.6611, -1.5118], [-0.9823, -1.2355, -1.0941]], [[-1.3850, -1.3273, -0.7247], [-0.8235, -1.4783, -1.0994]], [[-0.9049, -0.8867, -1.6962], [-1.4938, -1.3630, -0.6547]], ] ).astype(np.float32) targets = np.array([[1, 2, 2], [1, 2, 2]]).astype("int32") input_lengths = np.array([5, 5]).astype("int32") target_lengths = np.array([3, 3]).astype("int32") loss = ctc_loss_job(log_probs, targets, input_lengths, target_lengths) # loss [3.918017 2.907672] """ name = name if name is not None else id_util.UniqueStr("CTCLoss_") loss, _ = ( flow.user_op_builder(name) .Op("ctc_loss") .Input("log_probs", [log_probs]) .Input("targets", [targets]) .Input("input_lengths", [input_lengths]) .Input("target_lengths", [target_lengths]) .Output("loss") .Output("alpha") .Attr("blank", int(blank)) .Attr("zero_infinity", zero_infinity) .Build() .InferAndTryRun() .RemoteBlobList() ) if zero_infinity: cond = flow.math.equal( loss, flow.constant( float("inf"), dtype=loss.dtype, shape=loss.shape, name=name + "_constant", ), name=name + "_equal", ) loss = flow.where( cond, flow.zeros(dtype=loss.dtype, shape=loss.shape, name=name + "_zeros"), loss, name=name + "_where", ) if reduction == "mean": return flow.math.reduce_mean( flow.math.xdivy( loss, flow.cast( flow.math.clip_by_value( target_lengths, min_value=1, name=name + "_clip_by_value" ), dtype=log_probs.dtype, name=name + "_cast", ), name=name + "_xdivy", ), name=name + "_reduce_mean", ) elif reduction == "sum": return flow.math.reduce_sum(loss, name=name + "_reduce_sum") else: return loss
def constant_job(): with flow.scope.placement(device_type, "0:0"): return flow.constant(value, dtype=flow_type, shape=shape)
def batch_normalization( inputs: remote_blob_util.BlobDef, axis: int = -1, momentum: float = 0.99, epsilon: float = 0.001, center: bool = True, scale: bool = True, beta_initializer: Optional[op_conf_util.InitializerConf] = None, gamma_initializer: Optional[op_conf_util.InitializerConf] = None, beta_regularizer: Optional[op_conf_util.RegularizerConf] = None, gamma_regularizer: Optional[op_conf_util.RegularizerConf] = None, moving_mean_initializer: Optional[op_conf_util.InitializerConf] = None, moving_variance_initializer: Optional[op_conf_util.InitializerConf] = None, trainable: bool = True, training: bool = True, name: str = "BatchNorm", ) -> remote_blob_util.BlobDef: r"""Analogous to `tf.keras.layers.BatchNormalization <https://www.tensorflow.org/api_docs/python/tf/keras/layers/BatchNormalization>`_ Args: inputs (remote_blob_util.BlobDef): Input `Blob`. axis (int, optional): An int specifies the aixs that should be normalized . Default is -1, which normalizes the last axis. momentum (float, optional): A float specifies the momontum for the moving average. Defaults to 0.99. epsilon (float, optional): A small float added to avoid division by zero. Defaults to 0.001. center (bool, optional): A boolean specifies whether to add offset to normalized `Blob`. Defaults to True. scale (bool, optional): A boolean specifies whether to multiply normalized `Blob` by gamma. Defaults to True. beta_initializer (Optional[op_conf_util.InitializerConf], optional): Initializer for beta. Defaults to None. gamma_initializer (Optional[op_conf_util.InitializerConf], optional): Initializer for gamma. Defaults to None. beta_regularizer (Optional[op_conf_util.RegularizerConf], optional): Regularizer for beta. Defaults to None. gamma_regularizer (Optional[op_conf_util.RegularizerConf], optional): Regularizer for gamma. Defaults to None. moving_mean_initializer (Optional[op_conf_util.InitializerConf], optional): Initializer for moving mean. Defaults to None. moving_variance_initializer (Optional[op_conf_util.InitializerConf], optional): Initializer for moving variance. Defaults to None. trainable (bool, optional): A boolean specifies whether to train variables. Defaults to True. training (bool, optional): A boolean specifies whether now is training the model. Defaults to True. name (Optional[str], optional): This layer's name. Defaults to None. Returns: remote_blob_util.BlobDef: A `Blob` with same shape of input. Raises: ValueError: If axis is out of dimension of input. """ if axis < 0: axis += len(inputs.shape) assert axis >= 0 and axis < len(inputs.shape) params_shape = [inputs.shape[axis]] # Float32 required to avoid precision-loss when using fp16 input/output params_dtype = flow.float32 if inputs.dtype == flow.float16 else inputs.dtype if not flow.current_global_function_desc().IsTrainable() or not trainable: training = False with flow.scope.namespace(name): if center: beta = flow.get_variable( name="beta", shape=params_shape, dtype=params_dtype, initializer=beta_initializer or flow.zeros_initializer(), regularizer=beta_regularizer, trainable=trainable, distribute=distribute_util.broadcast(), reuse=False, ) else: beta = flow.constant(0, dtype=params_dtype, shape=params_shape, name="beta") if scale: gamma = flow.get_variable( name="gamma", shape=params_shape, dtype=params_dtype, initializer=gamma_initializer or flow.ones_initializer(), regularizer=gamma_regularizer, trainable=trainable, distribute=distribute_util.broadcast(), reuse=False, ) else: gamma = flow.constant(1, dtype=params_dtype, shape=params_shape, name="gamma") moving_mean = flow.get_variable( name="moving_mean", shape=params_shape, dtype=params_dtype, initializer=moving_mean_initializer or flow.zeros_initializer(), trainable=False, distribute=distribute_util.broadcast(), reuse=False, ) moving_variance = flow.get_variable( name="moving_variance", shape=params_shape, dtype=params_dtype, initializer=moving_variance_initializer or flow.ones_initializer(), trainable=False, distribute=distribute_util.broadcast(), reuse=False, ) if flow.current_scope().device_parallel_desc_symbol.device_tag == "cpu": if training: reduce_axis = [] for dim in range(len(inputs.shape)): if dim != axis: reduce_axis.append(dim) mean, variance = flow.nn.moments(inputs, reduce_axis, keepdims=False) def update_moving(moving, this_batch): moving_identity = flow.identity(moving) flow.assign( moving, momentum * moving_identity + (1 - momentum) * this_batch) update_moving(moving_mean, mean) update_moving(moving_variance, variance) return flow.nn.batch_normalization( x=inputs, mean=mean, variance=variance, offset=beta, scale=gamma, variance_epsilon=epsilon, axis=axis, name=name, ) else: mean = moving_mean variance = moving_variance return flow.nn.batch_normalization( x=inputs, mean=mean, variance=variance, offset=beta, scale=gamma, variance_epsilon=epsilon, axis=axis, name=name, ) else: builder = (flow.user_op_builder(name).Op("normalization").Input( "x", [inputs]).Input("moving_mean", [moving_mean]).Input( "moving_variance", [moving_variance]).Input("gamma", [gamma]).Input( "beta", [beta]).Output("y").Attr("axis", axis).Attr( "epsilon", epsilon).Attr("training", training).Attr("momentum", momentum)) if trainable and training: builder = builder.Output("mean").Output("inv_variance") return builder.Build().InferAndTryRun().RemoteBlobList()[0]
def foo_job(x_def: oft.Numpy.Placeholder(shape=slice_input.shape, dtype=flow.float)): y = x_def + flow.constant(1.0, shape=(1, ), dtype=flow.float) return y
def foo_job(x_def: oft.ListNumpy.Placeholder(shape=(5, 4), dtype=flow.float)): y = x_def * flow.constant(2.0, shape=(1, ), dtype=flow.float) return y
def foo_job(x_def: oft.Numpy.Placeholder(shape=(10, ), dtype=flow.float)): y = x_def + flow.constant(1.0, shape=(1, ), dtype=flow.float) test_case.assertTrue(np.allclose(y.numpy(0), output))
def ConstantJob(): with flow.scope.placement(device_type, "0:0"): x = flow.constant(6, dtype=flow.float, shape=(1024 * 1024 * 1024, 1024 * 1024 * 1024)) return x
def ConstantJob(): with flow.scope.placement(device_type, "0:0"): x = flow.constant(value, dtype=flow.float, shape=shape) y = flow.math.relu(x) z = flow.math.relu(y) return x
def foo_job(): x = flow.constant(1, shape=(2, 5), dtype=flow.float) return x