def test_job(x: oft.Numpy.Placeholder(input_shape, dtype=flow.float32)):
v = flow.get_variable(
name="v",
shape=(1, ),
dtype=flow.float32,
initializer=flow.zeros_initializer(),
)
x = x + v
x1 = flow.identity(x)
x2 = flow.identity(x)
flow.watch_diff(x1, test_global_storage.Setter("x1_diff"))
flow.watch_diff(x2, test_global_storage.Setter("x2_diff"))
x1 = flow.cast(x1, data_type)
x2 = flow.cast(x2, data_type)
y1 = flow.layers.batch_normalization_relu(x1, axis=axis, name="BN1")
y2 = flow.math.relu(
flow.layers.batch_normalization(x2, axis=axis, name="BN2"))
y1 = flow.cast(y1, flow.float32)
y2 = flow.cast(y2, flow.float32)
flow.watch(y1, test_global_storage.Setter("y1"))
flow.watch(y2, test_global_storage.Setter("y2"))
y1 = flow.where(flow.math.greater(y2, v), y1, v)
y2 = flow.where(flow.math.greater(y1, v), y2, v)
loss = y1 + y2
flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler([],
[0.001]),
momentum=0).minimize(flow.math.reduce_sum(loss))
return loss
def do_where(condition, x, y):
with flow.scope.placement(device_type, "0:0"):
x_var = flow.get_variable(
"x",
shape=x.shape,
dtype=flow.float,
initializer=flow.constant_initializer(0),
)
x_var = flow.cast_to_current_logical_view(x_var)
x_var = x_var + x
y_var = flow.get_variable(
"y",
shape=y.shape,
dtype=flow.float,
initializer=flow.constant_initializer(0),
)
y_var = flow.cast_to_current_logical_view(y_var)
y_var = y_var + y
z = flow.where(condition, x_var, y_var)
with flow.scope.placement(device_type, "0:0"):
flow.optimizer.SGD(
flow.optimizer.PiecewiseConstantScheduler([], [0.001]), momentum=0
).minimize(z)
flow.watch_diff(x_var, dz_dx_watcher)
flow.watch_diff(y_var, dz_dy_watcher)
return z
def ctc_loss(
log_probs: oneflow._oneflow_internal.BlobDesc,
targets: oneflow._oneflow_internal.BlobDesc,
input_lengths: oneflow._oneflow_internal.BlobDesc,
target_lengths: oneflow._oneflow_internal.BlobDesc,
blank: int = 0,
reduction: str = "mean",
zero_infinity: bool = False,
name: Optional[str] = None,
) -> oneflow._oneflow_internal.BlobDesc:
"""Computes the CTC(Connectionist Temporal Classification) loss.
This operator implements the CTC loss as presented in (Graves et al., 2006).
Args:
log_probs (oneflow._oneflow_internal.BlobDesc): A Blob of shape [input_length, batch_size, num_labels]. The logarithmized probabilities of the outputs (e.g. obtained with flow.nn.logsoftmax()).
targets (oneflow._oneflow_internal.BlobDesc): A Blob of shape [batch_size, max_target_length]. It represent the target sequences. Each element in the target sequence is a class index. And the target index cannot be blank (default=0).
input_lengths (oneflow._oneflow_internal.BlobDesc): A Blob of shape [batch_size]. It represent the lengths of the inputs. And the lengths are specified for each sequence to achieve masking under the assumption that sequences are padded to equal lengths.
target_lengths (oneflow._oneflow_internal.BlobDesc): A Blob of shape [batch_size]. It represent lengths of the targets. Lengths are specified for each sequence to achieve masking under the assumption that sequences are padded to equal lengths.
blank (int, optional): Blank label. Defaults to 0.
reduction (str, optional): The reduce type, it can be the one of "none", "mean", "sum". "none": no reduction will be applied, "mean": the output losses will be divided by the target lengths and then the mean over the batch is taken, "sum": the output will be summed. Defaults to "mean".
zero_infinity (bool, optional): Whether to zero infinite losses and the associated gradients. Infinite losses mainly occur when the inputs are too short to be aligned to the targets. Defaults to False.
name (Optional[str], optional): The name for the operation. Defaults to None.
Returns:
oneflow._oneflow_internal.BlobDesc: The result Blob.
For example:
.. code-block:: python
import oneflow.compatible.single_client as flow
import oneflow.compatible.single_client.typing as tp
import numpy as np
@flow.global_function()
def ctc_loss_job(
log_probs: tp.Numpy.Placeholder(shape=(5, 2, 3)),
targets: tp.Numpy.Placeholder(shape=(2, 3), dtype=flow.int32),
input_lengths: tp.Numpy.Placeholder(shape=(2,), dtype=flow.int32),
target_lengths: tp.Numpy.Placeholder(shape=(2,), dtype=flow.int32),
) -> tp.Numpy:
loss = flow.ctc_loss(
log_probs, targets, input_lengths, target_lengths, blank=0, reduction="none"
)
return loss
log_probs = np.array(
[
[[-1.1031, -0.7998, -1.5200], [-0.9808, -1.1363, -1.1908]],
[[-1.2258, -1.0665, -1.0153], [-1.1135, -1.2331, -0.9671]],
[[-1.3348, -0.6611, -1.5118], [-0.9823, -1.2355, -1.0941]],
[[-1.3850, -1.3273, -0.7247], [-0.8235, -1.4783, -1.0994]],
[[-0.9049, -0.8867, -1.6962], [-1.4938, -1.3630, -0.6547]],
]
).astype(np.float32)
targets = np.array([[1, 2, 2], [1, 2, 2]]).astype("int32")
input_lengths = np.array([5, 5]).astype("int32")
target_lengths = np.array([3, 3]).astype("int32")
loss = ctc_loss_job(log_probs, targets, input_lengths, target_lengths)
# loss [3.918017 2.907672]
"""
name = name if name is not None else id_util.UniqueStr("CTCLoss_")
(loss, _) = (flow.user_op_builder(name).Op("ctc_loss").Input(
"log_probs",
[log_probs]).Input("targets", [targets]).Input("input_lengths", [
input_lengths
]).Input("target_lengths", [
target_lengths
]).Output("loss").Output("alpha").Attr(
"max_target_length",
int(targets.shape[1])).Attr(
"blank",
int(blank)).Attr(
"zero_infinity",
zero_infinity).Build().InferAndTryRun().RemoteBlobList())
if zero_infinity:
cond = flow.math.equal(
loss,
flow.constant(
float("inf"),
dtype=loss.dtype,
shape=loss.shape,
name=name + "_constant",
),
name=name + "_equal",
)
loss = flow.where(
cond,
flow.zeros(dtype=loss.dtype,
shape=loss.shape,
name=name + "_zeros"),
loss,
name=name + "_where",
)
if reduction == "mean":
return flow.math.reduce_mean(
flow.math.xdivy(
loss,
flow.cast(
flow.math.clip_by_value(target_lengths,
min_value=1,
name=name + "_clip_by_value"),
dtype=log_probs.dtype,
name=name + "_cast",
),
name=name + "_xdivy",
),
name=name + "_reduce_mean",
)
elif reduction == "sum":
return flow.math.reduce_sum(loss, name=name + "_reduce_sum")
else:
return loss
def do_where(condition, x, y):
return flow.where(condition, x, y)
def where_fn(
input_def: oft.ListNumpy.Placeholder(input_shape, dtype=flow.float)
):
return flow.where(input_def)