def resnet50(args, data_dir):
(labels, images) = _data_load(args, data_dir)
g_output_key.append("input_img")
with flow.scope.namespace("Resnet"):
stem = resnet_stem(images)
body = resnet_conv_x_body(stem, lambda x: x)
pool5 = flow.nn.avg_pool2d(
body,
ksize=7,
strides=1,
padding="VALID",
data_format="NCHW",
name="pool5",
)
g_output_key.append("pool5")
fc1001 = flow.layers.dense(
flow.reshape(pool5, (pool5.shape[0], -1)),
units=1001,
use_bias=True,
kernel_initializer=flow.xavier_uniform_initializer(),
bias_initializer=flow.zeros_initializer(),
trainable=g_trainable,
name="fc1001",
)
g_output_key.append("fc1001")
loss = flow.nn.sparse_softmax_cross_entropy_with_logits(
labels, fc1001, name="softmax_loss")
g_output_key.append("cross_entropy")
return loss
def resnet50(images, trainable=True):
with flow.scope.namespace("Resnet"):
stem = resnet_stem(images)
body = resnet_conv_x_body(stem, lambda x: x)
pool5 = flow.nn.avg_pool2d(
body,
ksize=7,
strides=1,
padding="VALID",
data_format="NCHW",
name="pool5",
)
fc1001 = flow.layers.dense(
flow.reshape(pool5, (pool5.shape[0], -1)),
units=1001,
use_bias=True,
kernel_initializer=flow.xavier_uniform_initializer(),
bias_initializer=flow.zeros_initializer(),
trainable=trainable,
name="fc1001",
)
return fc1001
def resnet50(images,
cfg,
optimizer,
trainable=True,
need_transpose=False,
model_weight=True,
bn=True):
if need_transpose:
images = flow.transpose(images, name="transpose", perm=[0, 3, 1, 2])
global NAME_NUMBER
NAME_NUMBER = 0
stem = resnet_stem(images, bn, model_weight, optimizer)
body = resnet_conv_x_body(stem, cfg, bn, model_weight, optimizer,
lambda x: x)
pool5 = flow.nn.avg_pool2d(
body,
ksize=7,
strides=1,
padding="VALID",
data_format="NCHW",
name="pool5",
)
pool5 = flow.reshape(pool5, [pool5.shape[0], -1])
dense0 = flow.layers.dense(
inputs=pool5,
units=cfg[4],
use_bias=True,
kernel_initializer=flow.xavier_uniform_initializer(),
bias_initializer=flow.zeros_initializer(),
trainable=trainable,
name="dense0",
)
def getTypeAndShape(inputs, units):
in_shape = inputs.shape
in_num_axes = len(in_shape)
inputs = (flow.reshape(inputs,
(-1,
in_shape[-1])) if in_num_axes > 2 else inputs)
shape = (units, inputs.shape[1])
dtype = inputs.dtype
return shape, dtype
#添加dense层的Model weight
if model_weight == True:
shape_list = []
dtype_list = []
shape_weight, dtype = getTypeAndShape(pool5, cfg[4])
shape_list.append(shape_weight)
dtype_list.append(dtype)
modelWeight.addDense(dtype_old=dtype_list,
shape=shape_list,
optimizer=optimizer,
dense_num=1)
return dense0
def test_float_initializer(test_case):
initializers = [
flow.random_normal_initializer(mean=3, stddev=4),
flow.random_uniform_initializer(minval=-6, maxval=18),
flow.truncated_normal_initializer(mean=-5, stddev=8),
flow.xavier_uniform_initializer(data_format="NCHW"),
flow.xavier_uniform_initializer(data_format="NHWC"),
flow.xavier_normal_initializer(data_format="NCHW"),
flow.xavier_normal_initializer(data_format="NHWC"),
flow.constant_initializer(value=4),
flow.ones_initializer(),
flow.zeros_initializer(),
]
kaiming_args = GenArgDict(
OrderedDict(
shape=[SHAPE],
mode=["fan_in", "fan_out", "fan_avg"],
distribution=["random_normal", "random_uniform"],
data_format=["NCHW", "NHWC"],
negative_slope=[0.5],
))
vs_args = GenArgDict(
OrderedDict(
scale=[3.4],
mode=["fan_in", "fan_out", "fan_avg"],
distribution=[
"truncated_normal", "random_normal", "random_uniform"
],
data_format=["NCHW", "NHWC"],
))
for args in kaiming_args:
initializers.append(flow.kaiming_initializer(**args))
for args in vs_args:
initializers.append(flow.variance_scaling_initializer(**args))
for initializer in initializers:
CompareTwoDistribution(test_case, flow.float32, initializer)
def xavier_uniform_(self, gain=1.0, *, data_format="NCHW"):
assert gain == 1.0, "Only gain == 1.0 is supported now"
initializer_conf = flow.xavier_uniform_initializer(
data_format=data_format)
return self._init_by_initializer_conf(initializer_conf)
def conv3d(
inputs: remote_blob_util.BlobDef,
filters: int,
kernel_size: Union[int, Sequence[int]] = 1,
strides: Union[int, Sequence[int]] = 1,
padding: Union[str, Tuple[IntPair, IntPair, IntPair, IntPair,
IntPair]] = "VALID",
data_format: str = "NCDHW",
dilation_rate: Optional[Union[int, IntPair]] = None,
groups: int = 1,
activation: Optional[Callable[[remote_blob_util.BlobDef, str],
remote_blob_util.BlobDef]] = None,
use_bias: bool = True,
kernel_initializer: Optional[op_conf_util.InitializerConf] = None,
bias_initializer: Optional[op_conf_util.InitializerConf] = None,
kernel_regularizer: Optional[op_conf_util.RegularizerConf] = None,
bias_regularizer: Optional[op_conf_util.RegularizerConf] = None,
trainable: bool = True,
name: str = "Conv3d",
weight_name: Optional[str] = None,
bias_name: Optional[str] = None,
) -> remote_blob_util.BlobDef:
r"""3D convolution layer.
Args:
inputs (remote_blob_util.BlobDef): A 5D input `Blob`.
filters (int): An integer specifies the dimensionality of the output space.
kernel_size (Union[int, List[int], Sequence[int]], optional): An integer or tuple/list specifies the height and width of the convolution window.
When it is an integer, a square window is applied to the input. Defaults to 1.
strides (Union[int, List[int], Sequence[int]], optional): An integer or tuple/list specifies the strides of the convolution window along the height and width.
When it is an integer, the same value for the all spatial dimesions is applied. Defaults to 1.
padding (str, Tuple[IntPair, IntPair, IntPair, IntPair, IntPair], optional): padding: `string` `"SAME"` or `"SAME_LOWER"` or `"SAME_UPPER"` or `"VALID" or Tuple[IntPair, IntPair, IntPair, IntPair, IntPair]` indicating the type of padding algorithm to use, or a list indicating the explicit paddings at the start and end of each dimension. Defaults to "VALID".
data_format (str, optional): A string specifies the format of the input `Blob`, one of "NCDHW" or "NDHWC" (default: "NCDHW"). "NCDHW" cooresponds to channels_first, i.e. the input `Blob` with shape (batch_size, channels, depth, height, width).
"NDHWC" cooresponds to channels_last, i.e. the input `Blob` with shape (batch_size, channels, depth, height, width). Defaults to "NCDHW".
dilation_rate (int, optional): An integer or tuple/list specifies the dilation rate for the dilated convolution. When it is an integer, the same dilation rate is applied for the all dimensions. Defaults to 1.
groups (int, optional): A positive integer specifies number of groups for the Group conv. Defaults to 1.
activation (Optional[ Callable[[remote_blob_util.BlobDef, str], remote_blob_util.BlobDef] ], optional): Activation function. Defaults to None.
use_bias (bool, optional): A boolean specifies whether to use a bias vector. Defaults to True.
kernel_initializer (Optional[op_conf_util.InitializerConf], optional): Initializer for the kernel weights matrix. Defaults to None.
bias_initializer (Optional[op_conf_util.InitializerConf], optional): Initializer for the bias vector. Defaults to None.
kernel_regularizer (Optional[op_conf_util.RegularizerConf], optional): Regularizer for the kernel weights matrix. Defaults to None.
bias_regularizer (Optional[op_conf_util.RegularizerConf], optional): Regularizer for the bias vector . Defaults to None.
trainable (bool, optional): A boolean specifies whether to train variables. Defaults to True.
name (Optional[str], optional): This layer's name. Defaults to None.
weight_name (Optional[str], optional): This weight's name. Defaults to None.
bias_name (Optional[str], optional): This bias's name. Defaults to None.
Raises:
ValueError: If the type of kernel_size is not one of integer, list, tuple.
ValueError: The number of groups must be positive and number of filters must be divisible by it.
ValueError: If data_format is not one of 'NCDHW', 'NDHWC'.
ValueError: If number of input channels is not divisible by number of groups or less than number of groups.
ValueError: Number of group must be one when data_format is 'NDHWC'.
Returns:
remote_blob_util.BlobDef: A 5D `Blob` with the shape of (batch_size, filters, new_height, new_width).
"""
need_transpose = 0
if data_format.upper(
) == "NDHWC": # NDHWC is not supported before cudnn 8.0
need_transpose = 1
data_format = "NCDHW"
if need_transpose:
inputs = flow.transpose(inputs, perm=[0, 4, 1, 2, 3])
if isinstance(kernel_size, int):
kernel_size = (kernel_size, kernel_size, kernel_size)
else:
assert isinstance(kernel_size, (list, tuple))
assert len(kernel_size) == 3
kernel_size = tuple(kernel_size)
assert isinstance(groups, int)
assert groups > 0
assert groups <= filters
assert filters % groups == 0
if data_format.upper() == "NCDHW":
assert groups <= inputs.shape[1]
assert inputs.shape[1] % groups == 0
weight_shape = (filters, inputs.shape[1] // groups) + kernel_size
elif data_format.upper() == "NDHWC":
assert groups == 1
assert groups <= inputs.shape[3]
assert inputs.shape[3] % groups == 0
weight_shape = (
filters,
kernel_size[0],
kernel_size[1],
kernel_size[2],
inputs.shape[4] // groups,
)
else:
raise ValueError("data_format must be in NCHW or NHWC")
if kernel_initializer is None:
kernel_initializer = flow.xavier_uniform_initializer(
data_format=data_format)
if weight_name is None:
with flow.scope.namespace(name):
weight = flow.get_variable(
name="weight",
shape=weight_shape,
dtype=inputs.dtype,
initializer=kernel_initializer,
regularizer=kernel_regularizer,
trainable=trainable,
model_name="weight",
reuse=False,
)
else:
weight = flow.get_variable(
name=weight_name,
shape=weight_shape,
dtype=inputs.dtype,
initializer=kernel_initializer,
regularizer=kernel_regularizer,
trainable=trainable,
model_name="weight",
reuse=False,
)
output = flow.nn.conv3d(
inputs,
weight,
strides,
padding,
data_format,
dilation_rate,
groups=groups,
name=name,
)
if use_bias:
if bias_initializer is None:
bias_initializer = flow.constant_initializer(0)
if bias_name is None:
with flow.scope.namespace(name):
bias = flow.get_variable(
name="bias",
shape=(filters, ),
dtype=inputs.dtype,
initializer=bias_initializer,
regularizer=bias_regularizer,
trainable=trainable,
model_name="bias",
reuse=False,
)
else:
bias = flow.get_variable(
name=bias_name,
shape=(filters, ),
dtype=inputs.dtype,
initializer=bias_initializer,
regularizer=bias_regularizer,
trainable=trainable,
model_name="bias",
reuse=False,
)
with flow.scope.namespace(name):
output = flow.nn.bias_add(output,
bias,
data_format,
name="bias_add")
if callable(activation):
with flow.scope.namespace(name):
output = activation(output, name="activation")
if need_transpose:
output = flow.transpose(output, perm=[0, 2, 3, 4, 1])
return output
def build_network(self):
if self.need_transpose:
images = flow.transpose(self.images,
name="transpose",
perm=[0, 3, 1, 2])
else:
images = self.images
conv1 = _conv2d(images,
64,
kernel_size=7,
strides=2,
padding=([0, 0], [0, 0], [3, 3], [3, 3]),
groups=1,
use_bias=False,
trainable=self.trainable,
name="conv1")
bn1 = _batch_norm(conv1,
trainable=self.trainable,
training=self.training,
name="bn1")
relu = flow.nn.relu(bn1, name="relu1")
pad_before_max_pool = flow.pad(relu, ([0, 0], [0, 0], [1, 1], [1, 1]))
max_pool = flow.nn.max_pool2d(relu,
ksize=3,
strides=2,
padding="VALID",
data_format="NCHW",
name="max_pool")
layer1 = self._make_layer(max_pool,
64,
self.layers[0],
self.num_group,
layer_num="layer1")
layer2 = self._make_layer(layer1[-1],
128,
self.layers[1],
self.num_group,
strides=2,
layer_num="layer2")
layer3 = self._make_layer(layer2[-1],
256,
self.layers[2],
self.num_group,
strides=2,
layer_num="layer3")
layer4 = self._make_layer(layer3[-1],
512,
self.layers[3],
self.num_group,
strides=2,
layer_num="layer4")
# debug mode: dump data for debugging
# with flow.watch_scope(blob_watcher=blob_watched,
# diff_blob_watcher=diff_blob_watched):
# bn1_identity = flow.identity(layer4[-1], name="layer4_last_out")
avg_pool = flow.nn.avg_pool2d(layer4[-1],
7,
strides=1,
padding="VALID",
data_format="NCHW",
name="avg_pool")
reshape = flow.reshape(avg_pool, (avg_pool.shape[0], -1))
fc = flow.layers.dense(
reshape,
units=self.num_classes,
use_bias=True,
kernel_initializer=flow.xavier_uniform_initializer(),
bias_initializer=flow.zeros_initializer(),
trainable=self.trainable,
name="fc")
return fc
