def __init__(
self,
num_features,
eps=1e-5,
momentum=0.1,
affine=True,
track_running_stats=True,
):
super().__init__(num_features, eps, momentum, affine,
track_running_stats)
self._training_op = (
flow.builtin_op("normalization").Input("x").Input("moving_mean").
Input("moving_variance").Input("gamma").Input("beta").Attr(
"axis",
1).Attr("epsilon",
eps).Attr("momentum",
momentum).Output("y").Output("mean").Output(
"inv_variance").Attr("training",
True).Build())
self._testing_op = (
flow.builtin_op("normalization").Input("x").Input("moving_mean").
Input("moving_variance").Input("gamma").Input("beta").Attr(
"axis", 1).Attr("epsilon", eps).Attr(
"momentum", momentum).Output("y").Attr("training",
False).Build())
def __init__(self, min_value=None, max_value=None) -> None:
super().__init__()
if min_value is not None:
floating_min_value = float(min_value)
integral_min_value = int(min_value)
if max_value is not None:
floating_max_value = float(max_value)
integral_max_value = int(max_value)
if min_value is not None and max_value is not None:
self._op = (
flow.builtin_op("clip_by_scalar").Input("x").Output("y").Attr(
"floating_min", floating_min_value).Attr(
"integral_min", integral_min_value).Attr(
"floating_max", floating_max_value).Attr(
"integral_max", integral_max_value).Build())
elif min_value is not None:
self._op = (flow.builtin_op("clip_by_scalar_min").Input(
"x").Output("y").Attr("floating_min", floating_min_value).Attr(
"integral_min", integral_min_value).Build())
elif max_value is not None:
self._op = (flow.builtin_op("clip_by_scalar_max").Input(
"x").Output("y").Attr("floating_max", floating_max_value).Attr(
"integral_max", integral_max_value).Build())
else:
raise ValueError(
"min_value and max_value cannot be None at the same time")
def __init__(
self,
normalized_shape: _shape_t,
eps: float = 1e-5,
elementwise_affine: bool = True,
) -> None:
super(LayerNorm, self).__init__()
if isinstance(normalized_shape, int):
# mypy error: incompatible types in assignment
normalized_shape = (normalized_shape, ) # type: ignore[assignment]
self.normalized_shape = tuple(
normalized_shape) # type: ignore[arg-type]
self.epsilon = eps
self.elementwise_affine = elementwise_affine
if self.elementwise_affine:
self.weight = flow.nn.Parameter(
flow.Tensor(*self.normalized_shape))
self.bias = flow.nn.Parameter(flow.Tensor(*self.normalized_shape))
else:
self.register_parameter("weight", None)
self.register_parameter("bias", None)
self.reset_parameters()
# An integer specifies which axis to normalize at first, defaults to 1.
self.begin_norm_axis = 1
# An integer specifies which axis params at, defaults to 1 in 'NCHW' format
self.begin_params_axis = 1
self._op = (flow.builtin_op("layer_norm").Input("x").Input(
"gamma").Input("beta").Output("y").Output("mean").Output(
"inv_variance").Output("normalized").Build())
self._op2 = (flow.builtin_op("layer_norm").Input("x").Output(
"y").Output("mean").Output("inv_variance").Build())
def __init__(
self,
in_channels: int,
out_channels: int,
kernel_size: _size_2_t,
stride: _size_2_t = 1,
padding: _size_2_t = 0,
dilation: _size_2_t = 1,
groups: int = 1,
bias: bool = True,
padding_mode: str = "zeros", # TODO: refine this type
):
super().__init__()
assert padding_mode == "zeros"
kernel_size = _pair(kernel_size)
stride = _pair(stride)
padding = _pair(padding)
dilation = _pair(dilation)
self.groups = groups
self.weight = flow.nn.Parameter(
flow.Tensor(out_channels, in_channels // groups, *kernel_size))
self.bias = None
self._bias_add_op = None
if bias:
self.bias = flow.nn.Parameter(flow.Tensor(out_channels))
self._bias_add_op = (flow.builtin_op("bias_add").Input("a").Input(
"b").Output("out").Attr("axis", 1).Build())
self._op = (flow.builtin_op("conv2d").Input("in").Input("weight").Attr(
"filters", out_channels).Attr("padding_before", padding).Attr(
"strides", stride).Attr("kernel_size", kernel_size).Attr(
"dilation_rate", dilation).Attr("groups", groups).Attr(
"data_format", "channels_first").Output("out").Build())
self.reset_parameters()
def __init__(self, copy):
super().__init__()
self._copy_op = flow.builtin_op("copy").Input("in").Output(
"out").Build()
self._cast_op = flow.builtin_op("cast").Input("in").Output(
"out").Build()
self.copy = copy
def __init__(self, p: float = 0.5, inplace: bool = False):
_DropoutNd.__init__(self, p, inplace)
if self.p == 1.0:
scale = 1.0
else:
scale = float(1.0 / (1.0 - self.p))
seed = random.randint(-sys.maxsize, sys.maxsize)
self._op = (
flow.builtin_op("dropout")
.Input("in")
.Input("mask")
.Output("out")
.Attr("scale", scale)
.Build()
)
self._mask_op = (
flow.builtin_op("random_mask_like")
.Input("like")
.Output("out")
.Attr("rate", self.p)
.Attr("seed", seed)
.Build()
)
def __init__(
self,
color_space: str = "BGR",
output_layout: str = "NCHW",
crop_h: int = 0,
crop_w: int = 0,
crop_pos_y: float = 0.5,
crop_pos_x: float = 0.5,
mean: Sequence[float] = [0.0],
std: Sequence[float] = [1.0],
output_dtype: flow.dtype = flow.float,
):
super().__init__()
self._op = (flow.builtin_op("crop_mirror_normalize_from_uint8").Input(
"in").Input("mirror").Output("out").Attr(
"color_space",
color_space).Attr("output_layout", output_layout).Attr(
"mean", mean).Attr("std", std).Attr("crop_h", crop_h).Attr(
"crop_w", crop_w).Attr("crop_pos_y", crop_pos_y).Attr(
"crop_pos_x",
crop_pos_x).Attr("output_dtype",
output_dtype).Build())
self._val_op = (
flow.builtin_op("crop_mirror_normalize_from_tensorbuffer").Input(
"in").Output("out").Attr("color_space", color_space).Attr(
"output_layout",
output_layout).Attr("mean", mean).Attr("std", std).Attr(
"crop_h", crop_h).Attr("crop_w", crop_w).Attr(
"crop_pos_y",
crop_pos_y).Attr("crop_pos_x", crop_pos_x).Attr(
"output_dtype", output_dtype).Build())
def __init__(self, dim: Optional[int] = None):
super().__init__()
self.axis = -1 if dim is None else dim
self._op = flow.builtin_op("softmax").Input("in").Output("out").Build()
self._transpose_op = (
flow.builtin_op("transpose").Input("input").Output("output").Attr(
"perm", []).Build())
def __init__(self, in_features: int, out_features: int, bias: bool = True) -> None:
super().__init__()
self.use_bias = bias
self.weight = flow.nn.Parameter(flow.Tensor(out_features, in_features))
self.bias = None
if bias:
self.bias = flow.nn.Parameter(flow.Tensor(out_features))
self._matmul_op = (
flow.builtin_op("matmul")
.Input("a")
.Input("b")
.Output("out")
.Attr("transpose_a", False)
.Attr("transpose_b", True)
.Attr("alpha", 1.0)
.Build()
)
self._broadcast_matmul_op = (
flow.builtin_op("broadcast_matmul")
.Input("a")
.Input("b")
.Output("out")
.Attr("transpose_a", False)
.Attr("transpose_b", True)
.Attr("alpha", 1.0)
.Build()
)
self.reset_parameters()
def __init__(self) -> None:
super().__init__()
self._matmul_op = (
flow.builtin_op("matmul")
.Input("a")
.Input("b")
.Output("out")
.Attr("transpose_a", False)
.Attr("transpose_b", False)
.Attr("alpha", 1.0)
.Build()
)
self._batch_matmul_op = (
flow.builtin_op("batch_matmul")
.Input("a")
.Input("b")
.Output("out")
.Attr("transpose_a", False)
.Attr("transpose_b", False)
.Attr("alpha", 1.0)
.Build()
)
self._broadcast_matmul_op = (
flow.builtin_op("broadcast_matmul")
.Input("a")
.Input("b")
.Output("out")
.Attr("transpose_a", False)
.Attr("transpose_b", False)
.Attr("alpha", 1.0)
.Build()
)
def __init__(
self,
parameters: Union[Iterator[Parameter], List[Dict]],
lr: float = 1e-3,
alpha: float = 0.99,
eps: float = 1e-8,
weight_decay: float = 0,
momentum: float = 0.0,
centered: bool = False,
scale: float = 1.0,
):
super().__init__()
assert lr >= 0.0, f"Invalid learning rate: {lr}"
assert alpha >= 0.0, f"Invalid alpha value: {alpha}"
assert eps >= 0.0, f"Invalid epsilon value: {eps}"
assert weight_decay >= 0.0, f"Invalid weight_decay value: {weight_decay}"
assert scale > 0.0, f"Invalid scale factor: {scale}"
assert momentum == 0.0, "Not support momentum greater than zeros now!"
self._default_options["lr"] = lr
self._default_options["alpha"] = alpha
self._default_options["eps"] = eps
self._default_options["weight_decay"] = weight_decay
self._default_options["centered"] = centered
self._default_options["scale"] = scale
# Add parameters
if isinstance(parameters, GeneratorType):
self._param_groups.append(
ParamGroup(parameters, self._default_options))
else: # List[Dict]
for param in parameters:
self._param_groups.append(
ParamGroup(param, self._default_options))
for param_group in self._param_groups:
for param in param_group.parameters:
assert param.is_leaf, "parameters must be leaf tensor"
self._state[param] = dict()
self._state[param][
"square_avg"] = flow.experimental.zeros_like(param)
if param_group.options["centered"]:
self._state[param][
"grad_avg"] = flow.experimental.zeros_like(param)
self._centered_rmsprop = (flow.builtin_op("rmsprop_update").Input(
"model").Input("model_diff").Input("learning_rate").Input(
"mean_square").Input("mean_gradient").Attr(
"centered", True).Attr("l1", 0.0).Attr("l2", 0.0).Build())
self._rmsprop = (
flow.builtin_op("rmsprop_update").Input("model").Input(
"model_diff").Input("learning_rate").Input("mean_square").Attr(
"centered", False).Attr("l1", 0.0).Attr("l2", 0.0).Build())
def test_dynamic_attrs(test_case):
x = (flow.builtin_op("constant").Output("out").Attr(
"is_floating_value",
True).Attr("floating_value",
3.0).Attr("dtype",
flow.float32).Attr("shape",
[2, 3]).Build())()[0]
op = flow.builtin_op("expand_dims").Input("in").Output("out").Build()
y = op(x, axis=1)[0]
test_case.assertEqual(y.shape, flow.Size((2, 1, 3)))
y = op(x, axis=2)[0]
test_case.assertEqual(y.shape, flow.Size((2, 3, 1)))
def test_stateful_local_kernel(test_case):
op1 = (flow.builtin_op("constant").Output("out").Attr(
"is_floating_value",
True).Attr("floating_value",
3.0).Attr("dtype", flow.float32).Attr("shape",
[1, 1]).Build())
op2 = (flow.builtin_op("matmul").Input("a").Input("b").Attr(
"transpose_a",
False).Attr("transpose_b",
False).Attr("alpha", float(1.0)).Output("out").Build())
x = op1()[0]
x = op2(x, x)[0]
def __init__(self, dim: int = None, keepdim: bool = False) -> None:
super().__init__()
self._op_softmax_last_dim = (
flow.builtin_op("argmax").Input("in").Output("out").Build())
self._expand_op = (
flow.builtin_op("expand_dims").Input("in").Output("out").Attr(
"axis", -1).Build())
self._flatten = (
flow.builtin_op("flatten").Input("in").Output("out").Attr(
"start_dim", 0).Attr("end_dim", -1).Build())
self.dim = dim
self.keepdim = keepdim
def __init__(
self,
parameters: Union[Iterator[Parameter], List[Dict]],
lr: float,
momentum: float = 0.0,
scale: float = 1.0,
):
super().__init__()
assert lr >= 0.0, f"Invalid learning rate: {lr}"
assert momentum >= 0.0, f"Invalid momentum: {momentum}"
assert scale >= 0.0, f"Invalid scale factor: {scale}"
self._default_options["lr"] = lr
self._default_options["scale"] = scale
if momentum != 0.0:
self._default_options["momentum"] = momentum
# Add parameters
if isinstance(parameters, GeneratorType):
self._param_groups.append(
ParamGroup(parameters, self._default_options))
else: # List[Dict]
for param in parameters:
self._param_groups.append(
ParamGroup(param, self._default_options))
for param_group in self._param_groups:
for param in param_group.parameters:
assert param.is_leaf, "parameters must be leaf tensor"
self._state[param] = dict()
if "momentum" in self._default_options:
self._state[param]["momentum_buf"] = flow.tmp.zeros(
# TODO: zeros module support flow.Size parameter
tuple(param.shape))
if "momentum" in self._default_options.keys():
self._op = (flow.builtin_op("momentum_update").Input(
"model").Input("model_diff").Input(
"learning_rate").Input("momentum").Attr(
"scale",
self._default_options["scale"]).Attr("l1", 0.0).Attr(
"l2",
0.0).Attr("beta",
self._default_options["momentum"]).Attr(
"weight_decay", 0.0).Build())
else:
self._op = (flow.builtin_op("sgd_update").Input("model").Input(
"model_diff").Input("learning_rate").Attr(
"scale", self._default_options["scale"]).Attr(
"weight_decay", 0.0).Attr("l1", 0.0).Attr("l2",
0.0).Build())
def __init__(
self,
parameters: Union[Iterator[Parameter], List[Dict]],
lr: float = 1e-3,
momentum: float = 0.0,
scale: float = 1.0,
):
super().__init__()
assert lr >= 0.0, f"Invalid learning rate: {lr}"
assert momentum >= 0.0, f"Invalid momentum: {momentum}"
assert scale >= 0.0, f"Invalid scale factor: {scale}"
self._default_options["lr"] = lr
self._default_options["scale"] = scale
self._default_options["momentum"] = momentum
# Add parameters
if isinstance(parameters, GeneratorType):
self.param_groups.append(ParamGroup(parameters, self._default_options))
else: # List[Dict]
for param in parameters:
self.param_groups.append(ParamGroup(param, self._default_options))
for param_group in self.param_groups:
for param in param_group.parameters:
assert param.is_leaf, "parameters must be leaf tensor"
self._state[param] = dict()
if param_group["momentum"] != 0.0:
self._state[param]["momentum_buf"] = flow.experimental.zeros_like(
param
)
self._momentum_sgd = (
flow.builtin_op("momentum_update")
.Input("model")
.Input("model_diff")
.Input("momentum")
.Attr("l1", 0.0)
.Attr("l2", 0.0)
.Attr("weight_decay", 0.0)
.Build()
)
self._sgd = (
flow.builtin_op("sgd_update")
.Input("model")
.Input("model_diff")
.Attr("weight_decay", 0.0)
.Attr("l1", 0.0)
.Attr("l2", 0.0)
.Build()
)
def __init__(
self,
ofrecord_dir: str,
batch_size: int = 1,
data_part_num: int = 1,
part_name_prefix: str = "part-",
part_name_suffix_length: int = -1,
random_shuffle: bool = False,
shuffle_buffer_size: int = 1024,
shuffle_after_epoch: bool = False,
random_seed: int = -1,
name: Optional[str] = None,
):
super().__init__()
seed, has_seed = mirrored_gen_random_seed(random_seed)
self._op = (flow.builtin_op("OFRecordReader", name).Output("out").Attr(
"data_dir",
ofrecord_dir).Attr("data_part_num", data_part_num).Attr(
"batch_size",
batch_size).Attr("part_name_prefix", part_name_prefix).Attr(
"random_shuffle", random_shuffle).Attr(
"shuffle_buffer_size", shuffle_buffer_size).Attr(
"shuffle_after_epoch", shuffle_after_epoch).Attr(
"part_name_suffix_length",
part_name_suffix_length).Attr("seed",
seed).Build())
def forward(self, x):
_, _, h, w = x.shape
if (self.padding[2] < h and self.padding[3] < h and self.padding[0] < w
and self.padding[1] < w):
if x.dtype in [flow.float32, flow.float16, flow.float64]:
floating_value = float(self.value)
integral_value = int(0)
else:
floating_value = float(0)
integral_value = int(self.value)
self._op = (
flow.builtin_op("constant_pad2d").Input("x").Output("y").Attr(
"padding",
self.padding).Attr("floating_value", floating_value).Attr(
"integral_value", integral_value).Build())
res = self._op(x)[0]
return res
else:
raise AssertionError(
"Padding size should be less than the corresponding input dimension. Please check."
)
def __init__(self, *dims) -> None:
super().__init__()
self.perm = list(*dims)
self._op = (
flow.builtin_op("transpose").Input("input").Output("output").Attr(
"perm", []).Build())
def __init__(self, num_parameters: int = 1, init: float = 0.25) -> None:
super().__init__()
self.num_parameters = num_parameters
self.weight = flow.nn.Parameter(
flow.Tensor(num_parameters, 1, 1).fill_(init))
self.op = flow.builtin_op("prelu").Input("x").Input("alpha").Output(
"y").Build()
def __init__(
self,
min_val: float = -1,
max_val: float = 1,
inplace: bool = False,
min_value: Optional[float] = None,
max_value: Optional[float] = None,
):
super().__init__()
if min_value is not None:
warnings.warn(
"keyword argument min_value is deprecated and rename to min_val"
)
min_val = min_value
if max_value is not None:
warnings.warn(
"keyword argument max_value is deprecated and rename to max_val"
)
max_val = max_value
self._op = (
flow.builtin_op("hardtanh")
.Input("in")
.Attr("min_val", min_val)
.Attr("max_val", max_val)
.Output("out")
.Build()
)
def _build_reduce_op(op_type_name, keepdims):
return (
flow.builtin_op(op_type_name)
.Input("input_tensor")
.Output("output_tensor")
.Attr("keepdims", keepdims)
.Build()
)
def __init__(self, start: Tuple[int, ...], stop: Tuple[int, ...],
step: Tuple[int, ...]) -> None:
super().__init__()
self._op = (flow.builtin_op("slice_update").Input("x").Input(
"update").Output("y").Attr("start",
start).Attr("stop",
stop).Attr("step",
step).Build())
def nms(boxes: Tensor, scores: Tensor, iou_threshold: float) -> Tensor:
scores_inds = flow_exp.argsort(scores, dim=0, descending=True)
boxes = flow._C.gather(boxes, scores_inds, axis=0)
_nms_op = (flow_exp.builtin_op("nms").Input("in").Output("out").Attr(
"iou_threshold", iou_threshold).Attr("keep_n", -1).Build())
keep = _nms_op(boxes)[0]
index = flow_exp.squeeze(flow_exp.argwhere(keep), dim=[1])
return flow._C.gather(scores_inds, index, axis=0)
def __init__(self) -> None:
super().__init__()
self._op = (
flow.builtin_op("scalar_mul_by_tensor")
.Input("x")
.Input("scalar")
.Output("y")
.Build()
)
def __init__(self, dtype: flow.dtype) -> None:
super().__init__()
self._op = (
flow.builtin_op("cast")
.Input("in")
.Output("out")
.Attr("dtype", dtype)
.Build()
)
def __init__(
self,
dim: Optional[int] = 1,
):
super().__init__()
self.dim = dim
self._op = (
flow.builtin_op("transpose").Input("input").Output("output").Attr(
"perm", []).Build())
def __init__(
self,
size: Union[_size_any_t, flow.Size],
value: Union[float, int],
dtype: Optional[flow.dtype],
device: Union[flow.device, str] = None,
requires_grad: bool = False,
) -> None:
super().__init__()
assert size is not None, "shape must not be None!"
assert isinstance(
size, (int, tuple, flow.Size)
), "shape should be int or tuple int!"
size = _single(size)
if dtype is None:
dtype = flow.float32
if device is None:
self.device = flow.device("cpu")
self.requires_grad = requires_grad
if dtype in [
flow.int,
flow.int64,
flow.int32,
flow.char,
flow.int8,
flow.long,
flow.uint8,
]:
floating_value = float(0)
integer_value = int(value)
is_floating_value = False
elif dtype in [
flow.float32,
flow.float,
flow.double,
flow.float64,
flow.float16,
flow.half,
]:
floating_value = float(value)
integer_value = int(0)
is_floating_value = True
else:
raise NotImplementedError("Unsupport data type")
self._op = (
flow.builtin_op("constant")
.Output("out")
.Attr("is_floating_value", is_floating_value)
.Attr("floating_value", floating_value)
.Attr("integer_value", integer_value)
.Attr("dtype", dtype)
.Attr("shape", size)
.Build()
)
def __init__(self,
axis: Optional[Union[int, Sequence[int]]] = None,
keepdims: bool = False) -> None:
super().__init__()
self.axis = axis
self.keepdims = keepdims
self._op = (flow.builtin_op("reduce_sum").Input("input_tensor").Output(
"output_tensor").Attr("keepdims", keepdims).Build())
def __init__(self) -> None:
super().__init__()
self._op = (
flow.builtin_op("broadcast_greater")
.Input("x")
.Input("y")
.Output("z")
.Build()
)
