def truncated_normal_initializer( mean: float = 0.0, stddev: float = 1.0) -> initializer_conf_util.InitializerConf: r"""Initializer that generates a truncated normal distribution. Args: mean (float, optional): A scalar (float). Defaults to 0.0. stddev (float, optional): A scalar (float). Defaults to 1.0. Returns: initializer_conf_util.InitializerConf: Initial configuration For example: Example 1: .. code-block:: python import oneflow as flow import oneflow.typing as tp def watch_handler(y: tp.Numpy): print("out", y) @flow.global_function() def truncated_normal_Job() -> None: init = flow.truncated_normal_initializer(mean=1, stddev=1) blob = flow.get_variable( "blob-weight", shape=(3, ), initializer=init, trainable=True ) flow.watch(blob, watch_handler) checkpoint = flow.train.CheckPoint() checkpoint.init() truncated_normal_Job() # out [1.8303236 0.09787154 0.83049864] Example 2: .. code-block:: python import oneflow as flow import numpy as np import oneflow.typing as tp @flow.global_function() def conv2d_truncated_normal_Job(x: tp.Numpy.Placeholder((1, 256, 32, 32)) ) -> tp.Numpy: initializer = flow.truncated_normal_initializer(mean=0, stddev=1) conv2d = flow.layers.conv2d( x, filters=128, kernel_size=3, strides=1, padding='SAME', kernel_initializer=initializer, name="Conv2d" ) return conv2d x = np.random.randn(1, 256, 32, 32).astype(np.float32) out = conv2d_truncated_normal_Job(x) # out.shape (1, 128, 32, 32) """ initializer = initializer_conf_util.InitializerConf() setattr(initializer.truncated_normal_conf, "mean", float(mean)) setattr(initializer.truncated_normal_conf, "std", float(stddev)) return initializer
def variance_scaling_initializer( scale: float = 1.0, mode: str = "fan_in", distribution: str = "truncated_normal", data_format: str = "", ) -> initializer_conf_util.InitializerConf: r"""Initializer that generates a truncated normal distribution or a random normal distribution or a random uniform distribution with a scale adapting to it. When the distribution is "truncated_normal" The equation is: .. math:: W\sim N(0, \sqrt{\frac{{scale}}{{n}}}) If mode is "fan_in", the "n" is the number of input units in the weight Blob. If mode is "fan_out", the "n" is the number of output units in the weight Blob. if mode is "fan_avg", the "n" is the average of the number of input and output units in the weight Blob Args: scale (float, optional): Scaling factor (positive float). Defaults to 1.0. mode (str, optional): One of "fan_in", "fan_out", "fan_avg". Defaults to "fan_in". distribution (str, optional): Random distribution to use. One of "truncated_normal",. Defaults to "truncated_normal". data_format (str, optional): A string be one of "N...C" or "NC...". Defaults to "". Returns: initializer_conf_util.InitializerConf: Initial configuration For example: Example 1: .. code-block:: python import oneflow as flow import oneflow.typing as tp def watch_handler(y: tp.Numpy): print("out", y) @flow.global_function() def variance_scale_Job() -> None: init = flow.variance_scaling_initializer(scale=2.0, mode="fan_avg") blob = flow.get_variable( "blob-weight", shape=(3, 3), initializer=init, trainable=True ) flow.watch(blob, watch_handler) checkpoint = flow.train.CheckPoint() checkpoint.init() variance_scale_Job() # out [[-0.13931477 0.12266728 -0.9434968 ] # [-0.49665168 0.10231158 -0.19194333] # [-0.7902896 -1.7034698 -0.38695997]] Example 2: .. code-block:: python import oneflow as flow import numpy as np import oneflow.typing as tp @flow.global_function() def conv2d_variance_scaling_Job(x: tp.Numpy.Placeholder((1, 256, 32, 32)) ) -> tp.Numpy: initializer = flow.variance_scaling_initializer(mode="fan_out") conv2d = flow.layers.conv2d( x, filters=128, kernel_size=3, strides=1, padding='SAME', kernel_initializer=initializer, name="Conv2d" ) return conv2d x = np.random.randn(1, 256, 32, 32).astype(np.float32) out = conv2d_variance_scaling_Job(x) # out.shape (1, 128, 32, 32) """ initializer = initializer_conf_util.InitializerConf() setattr(initializer.variance_scaling_conf, "scale", float(scale)) setattr( initializer.variance_scaling_conf, "variance_norm", _get_variance_norm(mode), ) setattr( initializer.variance_scaling_conf, "distribution", _get_random_distribution(distribution), ) setattr( initializer.variance_scaling_conf, "data_format", _get_data_format(data_format), ) return initializer
def random_normal_initializer( mean: float = 0.0, stddev: float = 1.0, seed: Optional[int] = None, dtype: Optional[flow.dtype] = None, ) -> initializer_conf_util.InitializerConf: r"""Initializer that generates blob with a normal distribution. Args: mean (float, optional): A python scalar. Mean of the random values to generate.. Defaults to 0.0. stddev (float, optional): A python scalar. Standard deviation of the random values to generate. Defaults to 1.0. seed (Optional[int], optional): None. Not support yet. Defaults to None. dtype (Optional[flow.dtype], optional): . Defaults to None. Returns: initializer_conf_util.InitializerConf: Initial configuration For example: Example 1: .. code-block:: python import oneflow as flow import oneflow.typing as tp def watch_handler(y: tp.Numpy): print("out", y) @flow.global_function() def random_normal_Job() -> None: init = flow.random_normal_initializer(mean=1, stddev=1) blob = flow.get_variable( "blob-weight", shape=(3, ), initializer=init, trainable=True ) flow.watch(blob, watch_handler) checkpoint = flow.train.CheckPoint() checkpoint.init() random_normal_Job() # out [1.4190257 2.7663114 1.7114428] Example 2: .. code-block:: python import oneflow as flow import numpy as np import oneflow.typing as tp @flow.global_function() def conv2d_random_normal_Job(x: tp.Numpy.Placeholder((1, 256, 32, 32)) ) -> tp.Numpy: initializer = flow.random_normal_initializer(mean=0, stddev=1) conv2d = flow.layers.conv2d( x, filters=128, kernel_size=3, strides=1, padding='SAME', kernel_initializer=initializer, name="Conv2d" ) return conv2d x = np.random.randn(1, 256, 32, 32).astype(np.float32) out = conv2d_random_normal_Job(x) # out.shape (1, 128, 32, 32) """ assert seed is None assert dtype is None if seed is not None: assert name is not None initializer = initializer_conf_util.InitializerConf() setattr(initializer.random_normal_conf, "mean", float(mean)) setattr(initializer.random_normal_conf, "std", float(stddev)) return initializer
def constant_initializer( value: float = 0, dtype: flow.dtype = flow.float ) -> initializer_conf_util.InitializerConf: r"""Initializer that generates blob with constant values. Args: value (float, optional): A Python scalar. All elements of the initialized variable . Defaults to 0. dtype (flow.dtype, optional): Default data type. Defaults to flow.float. Raises: NotImplementedError: Do not support such data type. Returns: initializer_conf_util.InitializerConf: An InitializerConf object. For example: Example 1: .. code-block:: python import oneflow as flow import oneflow.typing as tp def watch_handler(y: tp.Numpy): print("out", y) @flow.global_function() def constant_Job() -> None: init = flow.constant_initializer(2.5) blob = flow.get_variable( "blob-weight", shape=(3, ), initializer=init, trainable=True ) flow.watch(blob, watch_handler) checkpoint = flow.train.CheckPoint() checkpoint.init() constant_Job() # out [2.5 2.5 2.5] Example 2: .. code-block:: python import oneflow as flow import numpy as np import oneflow.typing as tp @flow.global_function() def conv2d_constant_Job(x: tp.Numpy.Placeholder((1, 256, 32, 32)) ) -> tp.Numpy: initializer = flow.constant_initializer(0.01) conv2d = flow.layers.conv2d( x, filters=128, kernel_size=3, strides=1, padding='SAME', kernel_initializer=initializer, name="Conv2d" ) return conv2d x = np.random.randn(1, 256, 32, 32).astype(np.float32) out = conv2d_constant_Job(x) # out.shape (1, 128, 32, 32) """ initializer = initializer_conf_util.InitializerConf() if dtype in [flow.float, flow.double]: setattr(initializer.constant_conf, "value", float(value)) elif dtype in [ flow.int8, flow.int32, flow.int64, ]: setattr(initializer.constant_int_conf, "value", int(value)) else: raise NotImplementedError("Do not support such data type") return initializer
def empty_initializer( dtype: flow.dtype = flow.float, ) -> initializer_conf_util.InitializerConf: initializer = initializer_conf_util.InitializerConf() empty_conf = initializer_conf_util.EmptyInitializerConf() initializer.empty_conf.CopyFrom(empty_conf) return initializer
def random_uniform_initializer(minval: float = 0, maxval: float = 1, dtype: flow.dtype = flow.float ) -> initializer_conf_util.InitializerConf: r"""Initializer that generates blobs with a uniform distribution. Args: minval (float, optional): A python scalar. Lower bound of the range of random values to generate. Defaults to 0. maxval (float, optional): A python scalar. Upper bound of the range of random values to generate. Defaults to 1. dtype (flow.dtype, optional): Default data type. Defaults to flow.float. Raises: NotImplementedError: Do not support such data type. Returns: initializer_conf_util.InitializerConf: Initial configuration For example: Example 1: .. code-block:: python import oneflow as flow import oneflow.typing as tp def watch_handler(y: tp.Numpy): print("out", y) @flow.global_function() def random_uniform_Job() -> None: init = flow.random_uniform_initializer(minval=0, maxval=0.5) blob = flow.get_variable( "blob-weight", shape=(3, ), initializer=init, trainable=True ) flow.watch(blob, watch_handler) checkpoint = flow.train.CheckPoint() checkpoint.init() random_uniform_Job() # out [0.07557311 0.3943565 0.31875622] Example 2: .. code-block:: python import oneflow as flow import numpy as np import oneflow.typing as tp @flow.global_function() def conv2d_random_uniform_Job(x: tp.Numpy.Placeholder((1, 256, 32, 32)) ) -> tp.Numpy: initializer = flow.random_uniform_initializer(minval=0, maxval=0.5) conv2d = flow.layers.conv2d( x, filters=128, kernel_size=3, strides=1, padding='SAME', kernel_initializer=initializer, name="Conv2d" ) return conv2d x = np.random.randn(1, 256, 32, 32).astype(np.float32) out = conv2d_random_uniform_Job(x) # out.shape (1, 128, 32, 32) """ assert minval <= maxval initializer = initializer_conf_util.InitializerConf() if dtype in [flow.float, flow.double]: setattr(initializer.random_uniform_conf, "min", float(minval)) setattr(initializer.random_uniform_conf, "max", float(maxval)) elif dtype in [ flow.int8, flow.int32, flow.int64, ]: setattr(initializer.random_uniform_int_conf, "min", int(minval)) setattr(initializer.random_uniform_int_conf, "max", int(maxval)) else: raise NotImplementedError("Do not support such data type") return initializer
def _get_initializer(): kernel_initializer = initializer_conf_util.InitializerConf() kernel_initializer.truncated_normal_conf.std = 0.816496580927726 return kernel_initializer
def _get_bias_initializer(): bias_initializer = initializer_conf_util.InitializerConf() bias_initializer.constant_conf.value = 0.0 return bias_initializer
def truncated_normal_initializer( stddev: float = 1.0, ) -> initializer_conf_util.InitializerConf: initializer = initializer_conf_util.InitializerConf() setattr(initializer.truncated_normal_conf, "std", float(stddev)) return initializer