def hybrid_concat_job( input_0_def: oft.ListNumpy.Placeholder(shape=static_shape, dtype=flow.float), input_1_def: oft.ListNumpy.Placeholder(shape=static_shape, dtype=flow.float), ): var = flow.get_variable( "var", shape=static_shape, dtype=flow.float, initializer=flow.random_uniform_initializer(), trainable=True, ) constant = flow.constant(1.0, dtype=flow.float, shape=rand_sub_shape) inputs = [ flow.cast_to_current_logical_view(input) for input in [var, input_0_def, input_1_def, constant] ] concated = flow.concat(inputs, axis=axis, max_dim_size=max_dim_size) if verbose: print("concated static shape:", concated.shape) flow.optimizer.SGD( flow.optimizer.PiecewiseConstantScheduler([], [0.001]), momentum=0 ).minimize(concated) flow.watch_diff(var, compare_var_diff) if max_dim_size is None: test_case.assertTrue( concated.shape[axis] == static_shape[axis] * 3 + rand_sub_shape[axis] ) else: test_case.assertTrue(concated.shape[axis] == max_dim_size) return (var, concated)
def _hybrid_embedding(name, ids, embedding_size, vocab_size, hf_vocab_size): b, s = ids.shape ids = flow.flatten(ids) unique_ids, unique_ids_idx, _, _ = flow.experimental.unique_with_counts(ids) hf_vocab_size_constant = flow.constant(hf_vocab_size, dtype=flow.int32) hf_indices = flow.argwhere(flow.math.less(unique_ids, hf_vocab_size_constant)) lf_indices = flow.argwhere(flow.math.greater_equal(unique_ids, hf_vocab_size_constant)) hf_ids = flow.gather_nd(params=unique_ids, indices=hf_indices) lf_ids = flow.gather_nd(params=unique_ids, indices=lf_indices) hf_embedding_table = flow.get_variable( name=f'hf_{name}', shape=(hf_vocab_size, embedding_size), dtype=flow.float, initializer=flow.random_uniform_initializer(minval=-0.05, maxval=0.05), ) hf_embedding = flow.gather(params=hf_embedding_table, indices=hf_ids) lf_ids = lf_ids - hf_vocab_size_constant with flow.scope.placement('cpu', '0:0'): lf_embedding_table = flow.get_variable( name=f'lf_{name}', shape=(vocab_size - hf_vocab_size, embedding_size), dtype=flow.float, initializer=flow.random_uniform_initializer(minval=-0.05, maxval=0.05), ) lf_embedding = flow.gather(params=lf_embedding_table, indices=lf_ids) unique_embedding = flow.reshape(flow.zeros_like(unique_ids, dtype=flow.float), (-1, 1)) * flow.constant(0.0, dtype=flow.float, shape=(1,embedding_size)) unique_embedding = flow.tensor_scatter_nd_update(params=unique_embedding, updates=hf_embedding, indices=hf_indices) unique_embedding = flow.tensor_scatter_nd_update(params=unique_embedding, updates=lf_embedding, indices=lf_indices) unique_embedding = flow.gather(params=unique_embedding, indices=unique_ids_idx) unique_embedding = flow.cast_to_static_shape(unique_embedding) unique_embedding = flow.reshape(unique_embedding, shape=(b, s*embedding_size)) return unique_embedding
def _CreateAttentionMaskFromInputMask(to_mask_blob, from_seq_length, to_seq_length): output = flow.cast(to_mask_blob, dtype=flow.float) output = flow.reshape(output, [-1, 1, to_seq_length]) zeros = flow.constant(0.0, dtype=flow.float, shape=[from_seq_length, to_seq_length]) output = zeros + output return output
def foo_job(): x = flow.constant(1, shape=(2, 5), dtype=flow.float) y = flow.get_variable( name="var", shape=(64, 4), dtype=flow.float, initializer=flow.zeros_initializer(), ) return (x, y)
def _CreateAttentionMaskFromInputMask(to_mask_blob, from_seq_length, to_seq_length): output = flow.cast(to_mask_blob, dtype=flow.float) output = flow.reshape(output, [-1, 1, to_seq_length]) zeros = flow.constant(0.0, dtype=flow.float, shape=[from_seq_length, to_seq_length]) attention_mask_blob = zeros + output attention_mask_blob = flow.reshape( attention_mask_blob, [-1, 1, from_seq_length, to_seq_length] ) attention_mask_blob = flow.cast(attention_mask_blob, dtype=flow.float) addr_blob = (attention_mask_blob - 1.0) * 10000.0 return addr_blob
def SplitLikeJob(x: oft.Numpy.Placeholder(x_shape, dtype=flow.float)): v = flow.get_variable( "x", shape=x_shape, dtype=flow.float, initializer=flow.constant_initializer(0), trainable=True, ) x += v like0 = flow.constant(0, dtype=flow.float, shape=like0_shape) like1 = flow.constant(0, dtype=flow.float, shape=like1_shape) with flow.scope.placement("gpu", "0:0"): (y0, y1) = split_like(x, [like0, like1], "split_like") loss = y0 flow.optimizer.SGD( flow.optimizer.PiecewiseConstantScheduler([], [0.0001]), momentum=0 ).minimize(loss) flow.watch(x, test_global_storage.Setter("x")) flow.watch_diff(x, test_global_storage.Setter("x_diff")) flow.watch(loss, test_global_storage.Setter("loss")) flow.watch_diff(loss, test_global_storage.Setter("loss_diff")) return (y0, y1)
def oneflow_Xmum( of_input_1: tp.Numpy.Placeholder( shape=input_1.shape, dtype=value_type["of_type"] ), of_input_2: tp.Numpy.Placeholder( shape=input_2.shape, dtype=value_type["of_type"] ), ) -> tp.Numpy: with flow.scope.placement(device_type, "0:0"): v1 = flow.get_variable( shape=input_1.shape, dtype=value_type["of_type"], initializer=flow.zeros_initializer(), name="x1_var", ) x1_var = of_input_1 + v1 if not dx_only: v2 = flow.get_variable( shape=input_2.shape, dtype=value_type["of_type"], initializer=flow.zeros_initializer(), name="x2_var", ) x2_var = of_input_2 + v2 else: x2_var = flow.constant( value=1.5, shape=of_input_2.shape, dtype=value_type["of_type"] ) flow.watch_diff(x1_var, assert_prediction_grad) if compare_type == "maximum": of_Xmum_out = flow.math.maximum(x1_var, x2_var) elif compare_type == "minimum": of_Xmum_out = flow.math.minimum(x1_var, x2_var) with flow.scope.placement(device_type, "0:0"): flow.optimizer.SGD( flow.optimizer.PiecewiseConstantScheduler([], [0.001]), momentum=0 ).minimize(of_Xmum_out) return of_Xmum_out
def constant_scalar( value: Union[int, float], dtype: Optional[flow.dtype] = None, name: Optional[str] = None, ) -> oneflow._oneflow_internal.BlobDesc: """This operator creates a constant scalar Blob. Args: value (Union[int, float]): The constant value of Blob. dtype (Optional[flow.dtype], optional): The data type of Blob. Defaults to None. name (Optional[str], optional): The name for the operation. Defaults to None. Returns: oneflow._oneflow_internal.BlobDesc: The result blob. For example: .. code-block:: python import oneflow.compatible.single_client as flow import numpy as np import oneflow.compatible.single_client.typing as tp @flow.global_function() def constant_scalar_Job() -> tp.Numpy: constant_scalar = flow.constant_scalar(value=2.5, dtype=flow.float) return constant_scalar out = constant_scalar_Job() # out [2.5] """ return flow.constant(value, dtype=dtype, shape=[1])
def ConstantJob(): with flow.scope.placement(device_type, "0:0"): x = flow.constant(6, dtype=flow.float, shape=(1024 * 1024 * 1024, 1024 * 1024 * 1024)) return x
def foo_job(x_def: oft.Numpy.Placeholder(shape=input.shape[::-1], dtype=flow.float)): y = x_def + flow.constant(1.0, shape=(1, ), dtype=flow.float) return y
def foo_job(x_def: oft.ListNumpy.Placeholder(shape=(5, 4), dtype=flow.float)): y = x_def * flow.constant(2.0, shape=(1, ), dtype=flow.float) return y
def foo_job(x_def: oft.Numpy.Placeholder(shape=(10, ), dtype=flow.float)): y = x_def + flow.constant(1.0, shape=(1, ), dtype=flow.float) test_case.assertTrue(np.allclose(y.numpy(0), output))
def foo_job(): x = flow.constant(1, shape=(2, 5), dtype=flow.float) return x
def ctc_loss( log_probs: oneflow._oneflow_internal.BlobDesc, targets: oneflow._oneflow_internal.BlobDesc, input_lengths: oneflow._oneflow_internal.BlobDesc, target_lengths: oneflow._oneflow_internal.BlobDesc, blank: int = 0, reduction: str = "mean", zero_infinity: bool = False, name: Optional[str] = None, ) -> oneflow._oneflow_internal.BlobDesc: """Computes the CTC(Connectionist Temporal Classification) loss. This operator implements the CTC loss as presented in (Graves et al., 2006). Args: log_probs (oneflow._oneflow_internal.BlobDesc): A Blob of shape [input_length, batch_size, num_labels]. The logarithmized probabilities of the outputs (e.g. obtained with flow.nn.logsoftmax()). targets (oneflow._oneflow_internal.BlobDesc): A Blob of shape [batch_size, max_target_length]. It represent the target sequences. Each element in the target sequence is a class index. And the target index cannot be blank (default=0). input_lengths (oneflow._oneflow_internal.BlobDesc): A Blob of shape [batch_size]. It represent the lengths of the inputs. And the lengths are specified for each sequence to achieve masking under the assumption that sequences are padded to equal lengths. target_lengths (oneflow._oneflow_internal.BlobDesc): A Blob of shape [batch_size]. It represent lengths of the targets. Lengths are specified for each sequence to achieve masking under the assumption that sequences are padded to equal lengths. blank (int, optional): Blank label. Defaults to 0. reduction (str, optional): The reduce type, it can be the one of "none", "mean", "sum". "none": no reduction will be applied, "mean": the output losses will be divided by the target lengths and then the mean over the batch is taken, "sum": the output will be summed. Defaults to "mean". zero_infinity (bool, optional): Whether to zero infinite losses and the associated gradients. Infinite losses mainly occur when the inputs are too short to be aligned to the targets. Defaults to False. name (Optional[str], optional): The name for the operation. Defaults to None. Returns: oneflow._oneflow_internal.BlobDesc: The result Blob. For example: .. code-block:: python import oneflow.compatible.single_client as flow import oneflow.compatible.single_client.typing as tp import numpy as np @flow.global_function() def ctc_loss_job( log_probs: tp.Numpy.Placeholder(shape=(5, 2, 3)), targets: tp.Numpy.Placeholder(shape=(2, 3), dtype=flow.int32), input_lengths: tp.Numpy.Placeholder(shape=(2,), dtype=flow.int32), target_lengths: tp.Numpy.Placeholder(shape=(2,), dtype=flow.int32), ) -> tp.Numpy: loss = flow.ctc_loss( log_probs, targets, input_lengths, target_lengths, blank=0, reduction="none" ) return loss log_probs = np.array( [ [[-1.1031, -0.7998, -1.5200], [-0.9808, -1.1363, -1.1908]], [[-1.2258, -1.0665, -1.0153], [-1.1135, -1.2331, -0.9671]], [[-1.3348, -0.6611, -1.5118], [-0.9823, -1.2355, -1.0941]], [[-1.3850, -1.3273, -0.7247], [-0.8235, -1.4783, -1.0994]], [[-0.9049, -0.8867, -1.6962], [-1.4938, -1.3630, -0.6547]], ] ).astype(np.float32) targets = np.array([[1, 2, 2], [1, 2, 2]]).astype("int32") input_lengths = np.array([5, 5]).astype("int32") target_lengths = np.array([3, 3]).astype("int32") loss = ctc_loss_job(log_probs, targets, input_lengths, target_lengths) # loss [3.918017 2.907672] """ name = name if name is not None else id_util.UniqueStr("CTCLoss_") (loss, _) = (flow.user_op_builder(name).Op("ctc_loss").Input( "log_probs", [log_probs]).Input("targets", [targets]).Input("input_lengths", [ input_lengths ]).Input("target_lengths", [ target_lengths ]).Output("loss").Output("alpha").Attr( "max_target_length", int(targets.shape[1])).Attr( "blank", int(blank)).Attr( "zero_infinity", zero_infinity).Build().InferAndTryRun().RemoteBlobList()) if zero_infinity: cond = flow.math.equal( loss, flow.constant( float("inf"), dtype=loss.dtype, shape=loss.shape, name=name + "_constant", ), name=name + "_equal", ) loss = flow.where( cond, flow.zeros(dtype=loss.dtype, shape=loss.shape, name=name + "_zeros"), loss, name=name + "_where", ) if reduction == "mean": return flow.math.reduce_mean( flow.math.xdivy( loss, flow.cast( flow.math.clip_by_value(target_lengths, min_value=1, name=name + "_clip_by_value"), dtype=log_probs.dtype, name=name + "_cast", ), name=name + "_xdivy", ), name=name + "_reduce_mean", ) elif reduction == "sum": return flow.math.reduce_sum(loss, name=name + "_reduce_sum") else: return loss
def constant_job(): with flow.scope.placement(device_type, "0:0"): return flow.constant(value, dtype=flow_type, shape=shape)