def hybrid_concat_job(
input_0_def: oft.ListNumpy.Placeholder(shape=static_shape, dtype=flow.float),
input_1_def: oft.ListNumpy.Placeholder(shape=static_shape, dtype=flow.float),
):
var = flow.get_variable(
"var",
shape=static_shape,
dtype=flow.float,
initializer=flow.random_uniform_initializer(),
trainable=True,
)
constant = flow.constant(1.0, dtype=flow.float, shape=rand_sub_shape)
inputs = [
flow.cast_to_current_logical_view(input)
for input in [var, input_0_def, input_1_def, constant]
]
concated = flow.concat(inputs, axis=axis, max_dim_size=max_dim_size)
if verbose:
print("concated static shape:", concated.shape)
flow.optimizer.SGD(
flow.optimizer.PiecewiseConstantScheduler([], [0.001]), momentum=0
).minimize(concated)
flow.watch_diff(var, compare_var_diff)
if max_dim_size is None:
test_case.assertTrue(
concated.shape[axis] == static_shape[axis] * 3 + rand_sub_shape[axis]
)
else:
test_case.assertTrue(concated.shape[axis] == max_dim_size)
return (var, concated)
def _hybrid_embedding(name, ids, embedding_size, vocab_size, hf_vocab_size):
b, s = ids.shape
ids = flow.flatten(ids)
unique_ids, unique_ids_idx, _, _ = flow.experimental.unique_with_counts(ids)
hf_vocab_size_constant = flow.constant(hf_vocab_size, dtype=flow.int32)
hf_indices = flow.argwhere(flow.math.less(unique_ids, hf_vocab_size_constant))
lf_indices = flow.argwhere(flow.math.greater_equal(unique_ids, hf_vocab_size_constant))
hf_ids = flow.gather_nd(params=unique_ids, indices=hf_indices)
lf_ids = flow.gather_nd(params=unique_ids, indices=lf_indices)
hf_embedding_table = flow.get_variable(
name=f'hf_{name}',
shape=(hf_vocab_size, embedding_size),
dtype=flow.float,
initializer=flow.random_uniform_initializer(minval=-0.05, maxval=0.05),
)
hf_embedding = flow.gather(params=hf_embedding_table, indices=hf_ids)
lf_ids = lf_ids - hf_vocab_size_constant
with flow.scope.placement('cpu', '0:0'):
lf_embedding_table = flow.get_variable(
name=f'lf_{name}',
shape=(vocab_size - hf_vocab_size, embedding_size),
dtype=flow.float,
initializer=flow.random_uniform_initializer(minval=-0.05, maxval=0.05),
)
lf_embedding = flow.gather(params=lf_embedding_table, indices=lf_ids)
unique_embedding = flow.reshape(flow.zeros_like(unique_ids, dtype=flow.float), (-1, 1)) * flow.constant(0.0, dtype=flow.float, shape=(1,embedding_size))
unique_embedding = flow.tensor_scatter_nd_update(params=unique_embedding, updates=hf_embedding, indices=hf_indices)
unique_embedding = flow.tensor_scatter_nd_update(params=unique_embedding, updates=lf_embedding, indices=lf_indices)
unique_embedding = flow.gather(params=unique_embedding, indices=unique_ids_idx)
unique_embedding = flow.cast_to_static_shape(unique_embedding)
unique_embedding = flow.reshape(unique_embedding, shape=(b, s*embedding_size))
return unique_embedding
def _CreateAttentionMaskFromInputMask(to_mask_blob, from_seq_length,
to_seq_length):
output = flow.cast(to_mask_blob, dtype=flow.float)
output = flow.reshape(output, [-1, 1, to_seq_length])
zeros = flow.constant(0.0,
dtype=flow.float,
shape=[from_seq_length, to_seq_length])
output = zeros + output
return output
def foo_job():
x = flow.constant(1, shape=(2, 5), dtype=flow.float)
y = flow.get_variable(
name="var",
shape=(64, 4),
dtype=flow.float,
initializer=flow.zeros_initializer(),
)
return (x, y)
def _CreateAttentionMaskFromInputMask(to_mask_blob, from_seq_length, to_seq_length):
output = flow.cast(to_mask_blob, dtype=flow.float)
output = flow.reshape(output, [-1, 1, to_seq_length])
zeros = flow.constant(0.0, dtype=flow.float, shape=[from_seq_length, to_seq_length])
attention_mask_blob = zeros + output
attention_mask_blob = flow.reshape(
attention_mask_blob, [-1, 1, from_seq_length, to_seq_length]
)
attention_mask_blob = flow.cast(attention_mask_blob, dtype=flow.float)
addr_blob = (attention_mask_blob - 1.0) * 10000.0
return addr_blob
def SplitLikeJob(x: oft.Numpy.Placeholder(x_shape, dtype=flow.float)):
v = flow.get_variable(
"x",
shape=x_shape,
dtype=flow.float,
initializer=flow.constant_initializer(0),
trainable=True,
)
x += v
like0 = flow.constant(0, dtype=flow.float, shape=like0_shape)
like1 = flow.constant(0, dtype=flow.float, shape=like1_shape)
with flow.scope.placement("gpu", "0:0"):
(y0, y1) = split_like(x, [like0, like1], "split_like")
loss = y0
flow.optimizer.SGD(
flow.optimizer.PiecewiseConstantScheduler([], [0.0001]), momentum=0
).minimize(loss)
flow.watch(x, test_global_storage.Setter("x"))
flow.watch_diff(x, test_global_storage.Setter("x_diff"))
flow.watch(loss, test_global_storage.Setter("loss"))
flow.watch_diff(loss, test_global_storage.Setter("loss_diff"))
return (y0, y1)
def oneflow_Xmum(
of_input_1: tp.Numpy.Placeholder(
shape=input_1.shape, dtype=value_type["of_type"]
),
of_input_2: tp.Numpy.Placeholder(
shape=input_2.shape, dtype=value_type["of_type"]
),
) -> tp.Numpy:
with flow.scope.placement(device_type, "0:0"):
v1 = flow.get_variable(
shape=input_1.shape,
dtype=value_type["of_type"],
initializer=flow.zeros_initializer(),
name="x1_var",
)
x1_var = of_input_1 + v1
if not dx_only:
v2 = flow.get_variable(
shape=input_2.shape,
dtype=value_type["of_type"],
initializer=flow.zeros_initializer(),
name="x2_var",
)
x2_var = of_input_2 + v2
else:
x2_var = flow.constant(
value=1.5, shape=of_input_2.shape, dtype=value_type["of_type"]
)
flow.watch_diff(x1_var, assert_prediction_grad)
if compare_type == "maximum":
of_Xmum_out = flow.math.maximum(x1_var, x2_var)
elif compare_type == "minimum":
of_Xmum_out = flow.math.minimum(x1_var, x2_var)
with flow.scope.placement(device_type, "0:0"):
flow.optimizer.SGD(
flow.optimizer.PiecewiseConstantScheduler([], [0.001]), momentum=0
).minimize(of_Xmum_out)
return of_Xmum_out
def constant_scalar(
value: Union[int, float],
dtype: Optional[flow.dtype] = None,
name: Optional[str] = None,
) -> oneflow._oneflow_internal.BlobDesc:
"""This operator creates a constant scalar Blob.
Args:
value (Union[int, float]): The constant value of Blob.
dtype (Optional[flow.dtype], optional): The data type of Blob. Defaults to None.
name (Optional[str], optional): The name for the operation. Defaults to None.
Returns:
oneflow._oneflow_internal.BlobDesc: The result blob.
For example:
.. code-block:: python
import oneflow.compatible.single_client as flow
import numpy as np
import oneflow.compatible.single_client.typing as tp
@flow.global_function()
def constant_scalar_Job() -> tp.Numpy:
constant_scalar = flow.constant_scalar(value=2.5,
dtype=flow.float)
return constant_scalar
out = constant_scalar_Job()
# out [2.5]
"""
return flow.constant(value, dtype=dtype, shape=[1])
def ConstantJob():
with flow.scope.placement(device_type, "0:0"):
x = flow.constant(6,
dtype=flow.float,
shape=(1024 * 1024 * 1024, 1024 * 1024 * 1024))
return x
def foo_job(x_def: oft.Numpy.Placeholder(shape=input.shape[::-1],
dtype=flow.float)):
y = x_def + flow.constant(1.0, shape=(1, ), dtype=flow.float)
return y
def foo_job(x_def: oft.ListNumpy.Placeholder(shape=(5, 4),
dtype=flow.float)):
y = x_def * flow.constant(2.0, shape=(1, ), dtype=flow.float)
return y
def foo_job(x_def: oft.Numpy.Placeholder(shape=(10, ),
dtype=flow.float)):
y = x_def + flow.constant(1.0, shape=(1, ), dtype=flow.float)
test_case.assertTrue(np.allclose(y.numpy(0), output))
def foo_job():
x = flow.constant(1, shape=(2, 5), dtype=flow.float)
return x
def ctc_loss(
log_probs: oneflow._oneflow_internal.BlobDesc,
targets: oneflow._oneflow_internal.BlobDesc,
input_lengths: oneflow._oneflow_internal.BlobDesc,
target_lengths: oneflow._oneflow_internal.BlobDesc,
blank: int = 0,
reduction: str = "mean",
zero_infinity: bool = False,
name: Optional[str] = None,
) -> oneflow._oneflow_internal.BlobDesc:
"""Computes the CTC(Connectionist Temporal Classification) loss.
This operator implements the CTC loss as presented in (Graves et al., 2006).
Args:
log_probs (oneflow._oneflow_internal.BlobDesc): A Blob of shape [input_length, batch_size, num_labels]. The logarithmized probabilities of the outputs (e.g. obtained with flow.nn.logsoftmax()).
targets (oneflow._oneflow_internal.BlobDesc): A Blob of shape [batch_size, max_target_length]. It represent the target sequences. Each element in the target sequence is a class index. And the target index cannot be blank (default=0).
input_lengths (oneflow._oneflow_internal.BlobDesc): A Blob of shape [batch_size]. It represent the lengths of the inputs. And the lengths are specified for each sequence to achieve masking under the assumption that sequences are padded to equal lengths.
target_lengths (oneflow._oneflow_internal.BlobDesc): A Blob of shape [batch_size]. It represent lengths of the targets. Lengths are specified for each sequence to achieve masking under the assumption that sequences are padded to equal lengths.
blank (int, optional): Blank label. Defaults to 0.
reduction (str, optional): The reduce type, it can be the one of "none", "mean", "sum". "none": no reduction will be applied, "mean": the output losses will be divided by the target lengths and then the mean over the batch is taken, "sum": the output will be summed. Defaults to "mean".
zero_infinity (bool, optional): Whether to zero infinite losses and the associated gradients. Infinite losses mainly occur when the inputs are too short to be aligned to the targets. Defaults to False.
name (Optional[str], optional): The name for the operation. Defaults to None.
Returns:
oneflow._oneflow_internal.BlobDesc: The result Blob.
For example:
.. code-block:: python
import oneflow.compatible.single_client as flow
import oneflow.compatible.single_client.typing as tp
import numpy as np
@flow.global_function()
def ctc_loss_job(
log_probs: tp.Numpy.Placeholder(shape=(5, 2, 3)),
targets: tp.Numpy.Placeholder(shape=(2, 3), dtype=flow.int32),
input_lengths: tp.Numpy.Placeholder(shape=(2,), dtype=flow.int32),
target_lengths: tp.Numpy.Placeholder(shape=(2,), dtype=flow.int32),
) -> tp.Numpy:
loss = flow.ctc_loss(
log_probs, targets, input_lengths, target_lengths, blank=0, reduction="none"
)
return loss
log_probs = np.array(
[
[[-1.1031, -0.7998, -1.5200], [-0.9808, -1.1363, -1.1908]],
[[-1.2258, -1.0665, -1.0153], [-1.1135, -1.2331, -0.9671]],
[[-1.3348, -0.6611, -1.5118], [-0.9823, -1.2355, -1.0941]],
[[-1.3850, -1.3273, -0.7247], [-0.8235, -1.4783, -1.0994]],
[[-0.9049, -0.8867, -1.6962], [-1.4938, -1.3630, -0.6547]],
]
).astype(np.float32)
targets = np.array([[1, 2, 2], [1, 2, 2]]).astype("int32")
input_lengths = np.array([5, 5]).astype("int32")
target_lengths = np.array([3, 3]).astype("int32")
loss = ctc_loss_job(log_probs, targets, input_lengths, target_lengths)
# loss [3.918017 2.907672]
"""
name = name if name is not None else id_util.UniqueStr("CTCLoss_")
(loss, _) = (flow.user_op_builder(name).Op("ctc_loss").Input(
"log_probs",
[log_probs]).Input("targets", [targets]).Input("input_lengths", [
input_lengths
]).Input("target_lengths", [
target_lengths
]).Output("loss").Output("alpha").Attr(
"max_target_length",
int(targets.shape[1])).Attr(
"blank",
int(blank)).Attr(
"zero_infinity",
zero_infinity).Build().InferAndTryRun().RemoteBlobList())
if zero_infinity:
cond = flow.math.equal(
loss,
flow.constant(
float("inf"),
dtype=loss.dtype,
shape=loss.shape,
name=name + "_constant",
),
name=name + "_equal",
)
loss = flow.where(
cond,
flow.zeros(dtype=loss.dtype,
shape=loss.shape,
name=name + "_zeros"),
loss,
name=name + "_where",
)
if reduction == "mean":
return flow.math.reduce_mean(
flow.math.xdivy(
loss,
flow.cast(
flow.math.clip_by_value(target_lengths,
min_value=1,
name=name + "_clip_by_value"),
dtype=log_probs.dtype,
name=name + "_cast",
),
name=name + "_xdivy",
),
name=name + "_reduce_mean",
)
elif reduction == "sum":
return flow.math.reduce_sum(loss, name=name + "_reduce_sum")
else:
return loss
def constant_job():
with flow.scope.placement(device_type, "0:0"):
return flow.constant(value, dtype=flow_type, shape=shape)
