def _hybrid_embedding(name, ids, embedding_size, vocab_size, hf_vocab_size):
b, s = ids.shape
ids = flow.flatten(ids)
unique_ids, unique_ids_idx, _, _ = flow.experimental.unique_with_counts(ids)
hf_vocab_size_constant = flow.constant(hf_vocab_size, dtype=flow.int32)
hf_indices = flow.argwhere(flow.math.less(unique_ids, hf_vocab_size_constant))
lf_indices = flow.argwhere(flow.math.greater_equal(unique_ids, hf_vocab_size_constant))
hf_ids = flow.gather_nd(params=unique_ids, indices=hf_indices)
lf_ids = flow.gather_nd(params=unique_ids, indices=lf_indices)
hf_embedding_table = flow.get_variable(
name=f'hf_{name}',
shape=(hf_vocab_size, embedding_size),
dtype=flow.float,
initializer=flow.random_uniform_initializer(minval=-0.05, maxval=0.05),
)
hf_embedding = flow.gather(params=hf_embedding_table, indices=hf_ids)
lf_ids = lf_ids - hf_vocab_size_constant
with flow.scope.placement('cpu', '0:0'):
lf_embedding_table = flow.get_variable(
name=f'lf_{name}',
shape=(vocab_size - hf_vocab_size, embedding_size),
dtype=flow.float,
initializer=flow.random_uniform_initializer(minval=-0.05, maxval=0.05),
)
lf_embedding = flow.gather(params=lf_embedding_table, indices=lf_ids)
unique_embedding = flow.reshape(flow.zeros_like(unique_ids, dtype=flow.float), (-1, 1)) * flow.constant(0.0, dtype=flow.float, shape=(1,embedding_size))
unique_embedding = flow.tensor_scatter_nd_update(params=unique_embedding, updates=hf_embedding, indices=hf_indices)
unique_embedding = flow.tensor_scatter_nd_update(params=unique_embedding, updates=lf_embedding, indices=lf_indices)
unique_embedding = flow.gather(params=unique_embedding, indices=unique_ids_idx)
unique_embedding = flow.cast_to_static_shape(unique_embedding)
unique_embedding = flow.reshape(unique_embedding, shape=(b, s*embedding_size))
return unique_embedding
def training_step(self, batch, optimizer_idx):
if optimizer_idx == 0:
(z, ) = batch
g_out = self._generator(z, trainable=True, const_init=True)
g_logits = self._discriminator(g_out,
trainable=False,
const_init=True)
g_loss = flow.nn.sigmoid_cross_entropy_with_logits(
flow.ones_like(g_logits),
g_logits,
name="Gloss_sigmoid_cross_entropy_with_logits",
)
return (g_loss, g_out)
elif optimizer_idx == 1:
(z, images) = batch
g_out = self._generator(z, trainable=False, const_init=True)
g_logits = self._discriminator(g_out,
trainable=True,
const_init=True)
d_loss_fake = flow.nn.sigmoid_cross_entropy_with_logits(
flow.zeros_like(g_logits),
g_logits,
name="Dloss_fake_sigmoid_cross_entropy_with_logits",
)
d_logits = self._discriminator(images,
trainable=True,
reuse=True,
const_init=True)
d_loss_real = flow.nn.sigmoid_cross_entropy_with_logits(
flow.ones_like(d_logits),
d_logits,
name="Dloss_real_sigmoid_cross_entropy_with_logits",
)
d_loss = d_loss_fake + d_loss_real
return d_loss
def test_discriminator(
z: oft.Numpy.Placeholder((self.batch_size, 100)),
images: oft.Numpy.Placeholder((self.batch_size, 1, 28, 28)),
label1: oft.Numpy.Placeholder((self.batch_size, 1)),
label0: oft.Numpy.Placeholder((self.batch_size, 1)),
):
g_out = self.generator(z, trainable=False, const_init=True)
g_logits = self.discriminator(g_out,
trainable=True,
const_init=True)
d_loss_fake = flow.nn.sigmoid_cross_entropy_with_logits(
flow.zeros_like(g_logits),
g_logits,
name="Dloss_fake_sigmoid_cross_entropy_with_logits",
)
d_logits = self.discriminator(images,
trainable=True,
reuse=True,
const_init=True)
d_loss_real = flow.nn.sigmoid_cross_entropy_with_logits(
flow.ones_like(d_logits),
d_logits,
name="Dloss_real_sigmoid_cross_entropy_with_logits",
)
d_loss = d_loss_fake + d_loss_real
flow.optimizer.SGD(flow.optimizer.PiecewiseConstantScheduler(
[], [self.lr]),
momentum=0).minimize(d_loss)
return d_loss
def reduce_std(
input_tensor: oneflow._oneflow_internal.BlobDesc,
axis: Optional[Union[int, Sequence[int]]] = None,
keepdims: bool = False,
name: Optional[str] = None,
) -> oneflow._oneflow_internal.BlobDesc:
"""This operator computes the standard deviation of input Blob along the specified axis
The equation is:
.. math::
out=\\sqrt{\\frac{1}{n}*\\sum_{i=1}^{n}(x_i-mean)^2}
Args:
input_tensor (oneflow._oneflow_internal.BlobDesc): A Blob
axis (Optional[Union[int, Sequence[int]]], optional): The dimension along which the standard deviation is computed. Defaults to None.
keepdims (bool, optional): Whether to keep the reduced dimension in the output Blob. Defaults to False.
name (Optional[str], optional): The name for the operation. Defaults to None.
Returns:
oneflow._oneflow_internal.BlobDesc: The result of standard deviation on the specified axis of input Blob
For example:
.. code-block:: python
import oneflow.compatible.single_client as flow
import numpy as np
import oneflow.compatible.single_client.typing as tp
@flow.global_function()
def reduce_std_Job(x: tp.Numpy.Placeholder((3, 3))
) -> tp.Numpy:
return flow.math.reduce_std(x, axis=1, keepdims=True)
x = np.array([[0, 5, 10], [5, 5, 5], [12, 3, 0]]).astype(np.float32)
out = reduce_std_Job(x)
# out [[4.0824833]
# [0. ]
# [5.0990195]]
"""
name = _gen_unique_name_if_need(name, "ReduceStd_")
axis = _check_axis(axis, input_tensor.shape)
if isinstance(axis, list) and len(axis) == 0:
return flow.zeros_like(input_tensor,
dtype=input_tensor.dtype,
name=name + "_zeros_like")
return flow.math.sqrt(
flow.math.reduce_variance(input_tensor, axis, keepdims,
name + "_reduce_variance"),
name + "_sqrt",
)
def train_discriminator(
z=flow.FixedTensorDef((self.batch_size, 100)),
images=flow.FixedTensorDef((self.batch_size, 1, 28, 28)),
):
g_out = self.generator(z, trainable=False)
g_logits = self.discriminator(g_out, trainable=True)
d_loss_fake = flow.nn.sigmoid_cross_entropy_with_logits(
flow.zeros_like(g_logits), g_logits, name="Dloss_fake_sigmoid_cross_entropy_with_logits"
)
d_logits = self.discriminator(images, trainable=True, reuse=True)
d_loss_real = flow.nn.sigmoid_cross_entropy_with_logits(
flow.ones_like(d_logits), d_logits, name="Dloss_real_sigmoid_cross_entropy_with_logits"
)
d_loss = d_loss_fake + d_loss_real
flow.losses.add_loss(d_loss)
return d_loss, d_loss_fake, d_loss_real