def __init__(
self,
input_size,
output_size,
use_bias=True,
weights_initializer="xavier_uniform",
bias_initializer="zeros",
activation=None,
clip=None,
**kwargs,
):
super().__init__()
logger.debug(f" {self.name}")
logger.debug(" Dense")
self.dense = Dense(
input_size=input_size,
output_size=output_size,
use_bias=use_bias,
weights_initializer=weights_initializer,
bias_initializer=bias_initializer,
)
self.activation = get_activation(activation)
if clip is not None:
if isinstance(clip, (list, tuple)) and len(clip) == 2:
self.clip = partial(torch.clip, min=clip[0], max=clip[1])
else:
raise ValueError(
"The clip parameter of {} is {}. "
"It must be a list or a tuple of length 2.".format(
self.feature_name, self.clip))
else:
self.clip = None
def __init__(self, input_size, sequence_size, hidden_size, num_heads, fc_size, dropout=0.1):
super().__init__()
self.input_size = input_size
self.sequence_size = sequence_size
self.hidden_size = hidden_size
self.self_attention = MultiHeadSelfAttention(input_size, hidden_size, num_heads=num_heads)
self.dropout1 = nn.Dropout(dropout)
self.layernorm1 = nn.LayerNorm(hidden_size, eps=1e-6)
self.fully_connected = nn.Sequential(
nn.Linear(input_size, fc_size), get_activation("relu"), nn.Linear(fc_size, hidden_size)
)
self.dropout2 = nn.Dropout(dropout)
self.layernorm2 = nn.LayerNorm(hidden_size, eps=1e-6)
def __init__(
self,
input_size: int,
max_sequence_length: int,
hidden_size: int,
num_heads: int,
output_size: int,
dropout: float = 0.1,
):
super().__init__()
self.input_size = input_size
self.max_sequence_length = max_sequence_length
self.hidden_size = hidden_size
self.self_attention = MultiHeadSelfAttention(input_size,
hidden_size,
num_heads=num_heads)
self.dropout1 = nn.Dropout(dropout)
self.layernorm1 = nn.LayerNorm(hidden_size, eps=1e-6)
self.fully_connected = nn.Sequential(
nn.Linear(input_size, output_size), get_activation("relu"),
nn.Linear(output_size, hidden_size))
self.dropout2 = nn.Dropout(dropout)
self.layernorm2 = nn.LayerNorm(hidden_size, eps=1e-6)
def __init__(self, input_size, hidden_size=256, activation="tanh"):
super().__init__()
self.fc_layer1 = nn.Linear(input_size, hidden_size)
self.fc_layer1_activation = get_activation(activation)
self.fc_layer2 = nn.Linear(hidden_size, 1, bias=False)
def __init__(
self,
img_height: int,
img_width: int,
first_in_channels: int,
out_channels: int,
stride: int = 1,
batch_norm_momentum: float = 0.1,
batch_norm_epsilon: float = 0.001,
projection_shortcut: Optional[LudwigModule] = None,
):
"""Resnet blocks used for ResNet34 and smaller.
stride: A single int specifying the stride of the first convolution.
The last convolution will have stride of 1.
"""
super().__init__()
self._input_shape = (first_in_channels, img_height, img_width)
self.conv1 = Conv2DLayerFixedPadding(
img_height=img_height,
img_width=img_width,
in_channels=first_in_channels,
out_channels=out_channels,
kernel_size=3,
stride=stride,
)
in_channels, img_height, img_width = self.conv1.output_shape
self.norm1 = nn.BatchNorm2d(num_features=in_channels,
eps=batch_norm_epsilon,
momentum=batch_norm_momentum)
self.relu1 = get_activation("relu")
self.conv2 = Conv2DLayerFixedPadding(
img_height=img_height,
img_width=img_width,
in_channels=out_channels,
out_channels=out_channels,
kernel_size=3,
stride=1,
)
self.norm2 = nn.BatchNorm2d(num_features=out_channels,
eps=batch_norm_epsilon,
momentum=batch_norm_momentum)
self.relu2 = get_activation("relu")
for layer in [
self.conv1, self.norm1, self.relu1, self.conv2, self.norm2,
self.relu2
]:
logger.debug(f" {layer._get_name()}")
self._output_shape = self.conv2.output_shape
self.projection_shortcut = projection_shortcut
if self.projection_shortcut is not None and self.projection_shortcut.output_shape != self._output_shape:
raise ValueError(
f"Output shapes of ResnetBlock and projection_shortcut should "
f"match but are {self._output_shape} and "
f"{self.projection_shortcut.output_shape} respectively.")
if self.projection_shortcut is None and self._input_shape != self._output_shape:
self.projection_shortcut = Conv2DLayer(
img_height=self._input_shape[1],
img_width=self._input_shape[2],
in_channels=first_in_channels,
out_channels=out_channels,
kernel_size=1,
stride=stride,
)
def __init__(
self,
img_height: int,
img_width: int,
in_channels: int,
out_channels: int = 256,
kernel_size: Union[int, Tuple[int]] = 3,
stride: Union[int, Tuple[int]] = 1,
padding: Union[int, Tuple[int], str] = "valid",
dilation: Union[int, Tuple[int]] = 1,
groups: int = 1,
use_bias: bool = True,
padding_mode: str = "zeros",
norm: Optional[str] = None,
norm_params: Optional[Dict[str, Any]] = None,
activation: str = "relu",
dropout: float = 0,
pool_function: int = "max",
pool_kernel_size: Union[int, Tuple[int]] = None,
pool_stride: Optional[int] = None,
pool_padding: Union[int, Tuple[int]] = 0,
pool_dilation: Union[int, Tuple[int]] = 1,
):
super().__init__()
self.layers = torch.nn.ModuleList()
self._input_shape = (in_channels, img_height, img_width)
pool_stride = pool_stride or pool_kernel_size
self.layers.append(
nn.Conv2d(
in_channels=in_channels,
out_channels=out_channels,
kernel_size=kernel_size,
stride=stride,
padding=padding,
dilation=dilation,
groups=groups,
bias=use_bias,
padding_mode=padding_mode,
))
out_height, out_width = get_img_output_shape(img_height, img_width,
kernel_size, stride,
padding, dilation)
if norm and norm_params is None:
norm_params = {}
if norm == "batch":
# Batch norm over channels
self.layers.append(
nn.BatchNorm2d(num_features=out_channels, **norm_params))
elif norm == "layer":
# Layer norm over image height and width
self.layers.append(
nn.LayerNorm(normalized_shape=(out_height, out_width),
**norm_params))
self.layers.append(get_activation(activation))
if dropout > 0:
self.layers.append(nn.Dropout(dropout))
if pool_kernel_size is not None:
pool = partial(nn.MaxPool2d, dilation=pool_dilation)
if pool_function in {"average", "avg", "mean"}:
pool = nn.AvgPool2d
self.layers.append(
pool(kernel_size=pool_kernel_size,
stride=pool_stride,
padding=pool_padding))
out_height, out_width = get_img_output_shape(
img_height=out_height,
img_width=out_width,
kernel_size=pool_kernel_size,
stride=pool_stride,
padding=pool_padding,
dilation=pool_dilation,
)
for layer in self.layers:
logger.debug(f" {layer._get_name()}")
self._output_shape = (out_channels, out_height, out_width)
def __init__(
self,
in_channels=1,
out_channels=256,
max_sequence_length=None,
kernel_size=3,
strides=1,
padding="same",
dilation=1,
groups=1,
use_bias=True,
weights_initializer="xavier_uniform",
bias_initializer="zeros",
norm=None,
norm_params=None,
activation="relu",
dropout=0,
pool_function="max",
pool_size=2,
pool_strides=None,
pool_padding="valid",
):
super().__init__()
self.in_channels = in_channels
self.out_channels = out_channels
self.max_sequence_length = max_sequence_length
self.kernel_size = kernel_size
self.stride = strides
self.padding = padding
self.dilation = dilation
self.groups = groups
self.pool_size = pool_size
if pool_strides is None:
self.pool_strides = pool_size
else:
self.pool_strides = pool_strides
if pool_padding == "same" and pool_size is not None:
self.pool_padding = (self.pool_size - 1) // 2
else:
self.pool_padding = 0
self.layers = nn.ModuleList()
self.layers.append(
nn.Conv1d(
in_channels=in_channels,
out_channels=out_channels,
kernel_size=(kernel_size, ),
stride=(strides, ),
padding=padding,
dilation=(dilation, ),
))
if norm and norm_params is None:
norm_params = {}
if norm == "batch":
self.layers.append(
nn.BatchNorm1d(num_features=out_channels, **norm_params))
elif norm == "layer":
self.layers.append(
nn.LayerNorm(
normalized_shape=[out_channels, self.max_sequence_length],
**norm_params))
self.layers.append(get_activation(activation))
if dropout > 0:
self.layers.append(nn.Dropout(dropout))
if pool_size is not None:
pool = nn.MaxPool1d
if pool_function in {"average", "avg", "mean"}:
pool = nn.AvgPool1d
self.layers.append(
pool(kernel_size=self.pool_size,
stride=self.pool_strides,
padding=self.pool_padding))
for layer in self.layers:
logger.debug(f" {layer._get_name()}")
def __init__(
self,
img_height: int,
img_width: int,
first_in_channels: int,
out_channels: int,
resnet_size: int = 34,
kernel_size: Union[int, Tuple[int]] = 7,
conv_stride: Union[int, Tuple[int]] = 2,
first_pool_kernel_size: Union[int, Tuple[int]] = 3,
first_pool_stride: Union[int, Tuple[int]] = 2,
block_sizes: List[int] = None,
block_strides: List[Union[int, Tuple[int]]] = None,
batch_norm_momentum: float = 0.1,
batch_norm_epsilon: float = 0.001,
):
"""Creates a model obtaining an image representation.
Implements ResNet v2:
Identity Mappings in Deep Residual Networks
https://arxiv.org/pdf/1603.05027.pdf
by Kaiming He, Xiangyu Zhang, Shaoqing Ren, and Jian Sun, Jul 2016.
Args:
resnet_size: A single integer for the size of the ResNet model.
is_bottleneck: Use regular blocks or bottleneck blocks.
out_channels: The number of filters to use for the first block layer
of the model. This number is then doubled for each subsequent block
layer.
kernel_size: The kernel size to use for convolution.
conv_stride: stride size for the initial convolutional layer
first_pool_kernel_size: Pool size to be used for the first pooling layer.
If none, the first pooling layer is skipped.
first_pool_stride: stride size for the first pooling layer. Not used
if first_pool_kernel_size is None.
block_sizes: A list containing n values, where n is the number of sets of
block layers desired. Each value should be the number of blocks in the
i-th set.
block_strides: List of integers representing the desired stride size for
each of the sets of block layers. Should be same length as block_sizes.
Raises:
ValueError: if invalid version is selected.
"""
super().__init__()
self._input_shape = (first_in_channels, img_height, img_width)
is_bottleneck = self.get_is_bottleneck(resnet_size, block_sizes)
block_class = self.get_block_fn(is_bottleneck)
block_sizes, block_strides = self.get_blocks(resnet_size, block_sizes,
block_strides)
self.layers = torch.nn.ModuleList()
self.layers.append(
Conv2DLayerFixedPadding(
img_height=img_height,
img_width=img_width,
in_channels=first_in_channels,
out_channels=out_channels,
kernel_size=kernel_size,
stride=conv_stride,
))
in_channels, img_height, img_width = self.layers[-1].output_shape
self.layers.append(
nn.BatchNorm2d(num_features=out_channels,
eps=batch_norm_epsilon,
momentum=batch_norm_momentum))
self.layers.append(get_activation("relu"))
if first_pool_kernel_size:
self.layers.append(
nn.MaxPool2d(kernel_size=first_pool_kernel_size,
stride=first_pool_stride,
padding=1))
img_height, img_width = get_img_output_shape(
img_height=img_height,
img_width=img_width,
kernel_size=first_pool_kernel_size,
stride=first_pool_stride,
padding=1,
dilation=1,
)
for i, num_blocks in enumerate(block_sizes):
self.layers.append(
ResNetBlockLayer(
img_height=img_height,
img_width=img_width,
first_in_channels=in_channels,
out_channels=out_channels,
is_bottleneck=is_bottleneck,
block_fn=block_class,
num_blocks=num_blocks,
stride=block_strides[i],
batch_norm_momentum=batch_norm_momentum,
batch_norm_epsilon=batch_norm_epsilon,
))
out_channels *= 2
in_channels, img_height, img_width = self.layers[-1].output_shape
for layer in self.layers:
logger.debug(f" {layer._get_name()}")
self._output_shape = (in_channels, img_height, img_width)
logger.debug(" Dense")
self.dense = Dense(
input_size=input_size,
output_size=output_size,
use_bias=use_bias,
weights_initializer=weights_initializer,
bias_initializer=bias_initializer,
<<<<<<< HEAD
weights_regularizer=weights_regularizer,
bias_regularizer=bias_regularizer,
activity_regularizer=activity_regularizer,
=======
>>>>>>> upstream/master
)
self.activation = get_activation(activation)
if clip is not None:
if isinstance(clip, (list, tuple)) and len(clip) == 2:
self.clip = partial(torch.clip, min=clip[0], max=clip[1])
else:
raise ValueError(
"The clip parameter of {} is {}. "
"It must be a list or a tuple of length 2.".format(self.feature_name, self.clip)
)
else:
self.clip = None
<<<<<<< HEAD
=======
@property