def __init__(self, args):
super(SmallMotionEncoder, self).__init__()
cor_planes = args.corr_levels * (2 * args.corr_radius + 1)**2
self.convc1 = SparseConv(cor_planes, 96, 1, padding=0)
self.convf1 = SparseConv(2, 64, 7, padding=3)
self.convf2 = SparseConv(64, 32, 3, padding=1)
self.conv = SparseConv(128, 80, 3, padding=1)
def __init__(self, inplanes, planes, stride=1, downsample=None, N=2, M=4):
super(Bottleneck, self).__init__()
self.conv1 = SparseConv(inplanes,
planes,
kernel_size=1,
bias=False,
N=N,
M=M)
self.bn1 = nn.BatchNorm2d(planes)
self.conv2 = SparseConv(planes,
planes,
kernel_size=3,
stride=stride,
padding=1,
bias=False,
N=N,
M=M)
self.bn2 = nn.BatchNorm2d(planes)
self.conv3 = SparseConv(planes,
planes * 4,
kernel_size=1,
bias=False,
N=N,
M=M)
self.bn3 = nn.BatchNorm2d(planes * 4)
self.relu = nn.ReLU(inplace=True)
self.downsample = downsample
self.stride = stride
def __init__(self, args):
super(BasicMotionEncoder, self).__init__()
cor_planes = args.corr_levels * (2 * args.corr_radius + 1)**2
self.convc1 = SparseConv(cor_planes, 256, 1, padding=0)
self.convc2 = SparseConv(256, 192, 3, padding=1)
self.convf1 = SparseConv(2, 128, 7, padding=3)
self.convf2 = SparseConv(128, 64, 3, padding=1)
self.conv = SparseConv(64 + 192, 128 - 2, 3, padding=1)
def __init__(self, in_planes, planes, norm_fn='group', stride=1):
super(BottleneckBlock, self).__init__()
self.conv1 = SparseConv(in_planes,
planes // 4,
kernel_size=1,
padding=0)
self.conv2 = SparseConv(planes // 4,
planes // 4,
kernel_size=3,
padding=1,
stride=stride)
self.conv3 = SparseConv(planes // 4, planes, kernel_size=1, padding=0)
self.relu = nn.ReLU(inplace=True)
num_groups = planes // 8
if norm_fn == 'group':
self.norm1 = nn.GroupNorm(num_groups=num_groups,
num_channels=planes // 4)
self.norm2 = nn.GroupNorm(num_groups=num_groups,
num_channels=planes // 4)
self.norm3 = nn.GroupNorm(num_groups=num_groups,
num_channels=planes)
if not stride == 1:
self.norm4 = nn.GroupNorm(num_groups=num_groups,
num_channels=planes)
elif norm_fn == 'batch':
self.norm1 = nn.BatchNorm2d(planes // 4)
self.norm2 = nn.BatchNorm2d(planes // 4)
self.norm3 = nn.BatchNorm2d(planes)
if not stride == 1:
self.norm4 = nn.BatchNorm2d(planes)
elif norm_fn == 'instance':
self.norm1 = nn.InstanceNorm2d(planes // 4)
self.norm2 = nn.InstanceNorm2d(planes // 4)
self.norm3 = nn.InstanceNorm2d(planes)
if not stride == 1:
self.norm4 = nn.InstanceNorm2d(planes)
elif norm_fn == 'none':
self.norm1 = nn.Sequential()
self.norm2 = nn.Sequential()
self.norm3 = nn.Sequential()
if not stride == 1:
self.norm4 = nn.Sequential()
if stride == 1:
self.downsample = None
else:
self.downsample = nn.Sequential(
SparseConv(in_planes, planes, kernel_size=1, stride=stride),
self.norm4)
def __init__(self, args, hidden_dim=128, input_dim=128):
super(BasicUpdateBlock, self).__init__()
self.args = args
self.encoder = BasicMotionEncoder(args)
self.gru = SepConvGRU(hidden_dim=hidden_dim,
input_dim=128 + hidden_dim)
self.flow_head = FlowHead(hidden_dim, hidden_dim=256)
self.mask = nn.Sequential(SparseConv(128, 256, 3, padding=1),
nn.ReLU(inplace=True),
SparseConv(256, 64 * 9, 1, padding=0))
def __init__(self, hidden_dim=128, input_dim=192 + 128):
super(ConvGRU, self).__init__()
self.convz = SparseConv(hidden_dim + input_dim,
hidden_dim,
3,
padding=1)
self.convr = SparseConv(hidden_dim + input_dim,
hidden_dim,
3,
padding=1)
self.convq = SparseConv(hidden_dim + input_dim,
hidden_dim,
3,
padding=1)
def conv3x3(in_planes, out_planes, stride=1, N=2, M=4):
"""3x3 convolution with padding"""
return SparseConv(in_planes,
out_planes,
kernel_size=3,
stride=stride,
padding=1,
bias=False,
N=N,
M=M)
def __init__(self, output_dim=128, norm_fn='batch', dropout=0.0):
super(BasicEncoder, self).__init__()
self.norm_fn = norm_fn
if self.norm_fn == 'group':
self.norm1 = nn.GroupNorm(num_groups=8, num_channels=64)
elif self.norm_fn == 'batch':
self.norm1 = nn.BatchNorm2d(64)
elif self.norm_fn == 'instance':
self.norm1 = nn.InstanceNorm2d(64)
elif self.norm_fn == 'none':
self.norm1 = nn.Sequential()
self.conv1 = SparseConv(3, 64, kernel_size=7, stride=2, padding=3)
self.relu1 = nn.ReLU(inplace=True)
self.in_planes = 64
self.layer1 = self._make_layer(64, stride=1)
self.layer2 = self._make_layer(96, stride=2)
self.layer3 = self._make_layer(128, stride=2)
# output convolution
self.conv2 = SparseConv(128, output_dim, kernel_size=1)
self.dropout = None
if dropout > 0:
self.dropout = nn.Dropout2d(p=dropout)
for m in self.modules():
if isinstance(m, SparseConv):
nn.init.kaiming_normal_(m.weight,
mode='fan_out',
nonlinearity='relu')
elif isinstance(m,
(nn.BatchNorm2d, nn.InstanceNorm2d, nn.GroupNorm)):
if m.weight is not None:
nn.init.constant_(m.weight, 1)
if m.bias is not None:
nn.init.constant_(m.bias, 0)
def __init__(self, block, layers, num_classes=1000, N=2, M=4):
super(ResNetV1, self).__init__()
self.N = N
self.M = M
self.inplanes = 64
self.conv1 = SparseConv(3,
64,
kernel_size=7,
stride=2,
padding=3,
bias=False,
N=self.N,
M=self.M)
self.bn1 = nn.BatchNorm2d(64)
self.relu = nn.ReLU(inplace=True)
self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
self.layer1 = self._make_layer(block,
64,
layers[0],
N=self.N,
M=self.M)
self.layer2 = self._make_layer(block,
128,
layers[1],
stride=2,
N=self.N,
M=self.M)
self.layer3 = self._make_layer(block,
256,
layers[2],
stride=2,
N=self.N,
M=self.M)
self.layer4 = self._make_layer(block,
512,
layers[3],
stride=2,
N=self.N,
M=self.M)
self.avgpool = nn.AvgPool2d(7, stride=1)
self.fc = nn.Linear(512 * block.expansion, num_classes)
for m in self.modules():
if isinstance(m, SparseConv):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2. / n))
def _make_layer(self, block, planes, blocks, stride=1, N=2, M=4):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
SparseConv(self.inplanes,
planes * block.expansion,
kernel_size=1,
stride=stride,
bias=False,
N=N,
M=M),
nn.BatchNorm2d(planes * block.expansion),
)
layers = []
layers.append(
block(self.inplanes, planes, stride, downsample, N=N, M=M))
self.inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(block(self.inplanes, planes, N=N, M=M))
return nn.Sequential(*layers)
def __init__(self, hidden_dim=128, input_dim=192 + 128):
super(SepConvGRU, self).__init__()
self.convz1 = SparseConv(hidden_dim + input_dim,
hidden_dim, (1, 5),
padding=(0, 2))
self.convr1 = SparseConv(hidden_dim + input_dim,
hidden_dim, (1, 5),
padding=(0, 2))
self.convq1 = SparseConv(hidden_dim + input_dim,
hidden_dim, (1, 5),
padding=(0, 2))
self.convz2 = SparseConv(hidden_dim + input_dim,
hidden_dim, (5, 1),
padding=(2, 0))
self.convr2 = SparseConv(hidden_dim + input_dim,
hidden_dim, (5, 1),
padding=(2, 0))
self.convq2 = SparseConv(hidden_dim + input_dim,
hidden_dim, (5, 1),
padding=(2, 0))
def __init__(self, input_dim=128, hidden_dim=256):
super(FlowHead, self).__init__()
self.conv1 = SparseConv(input_dim, hidden_dim, 3, padding=1)
self.conv2 = SparseConv(hidden_dim, 2, 3, padding=1)
self.relu = nn.ReLU(inplace=True)