def softshrink(x, lambd, gpu_id):
nch = 2
nx = 161
# xs = 128
# ys = 160
ws = 5
t0 = time.time()
One = Variable(torch.ones(1).cuda(gpu_id), requires_grad=False)
Zero = Variable(torch.zeros(1).cuda(gpu_id), requires_grad=False)
lambd_t = One * lambd
x = x.view(nch, nx, 128, 160)
poolL2 = nn.LPPool2d(2, ws, stride=ws, ceil_mode=True)
xx_old = poolL2(x)
xx_old[xx_old == 0] = lambd_t / 1000
subgrad = torch.max(One - torch.div(lambd_t, xx_old), Zero)
xs = subgrad.shape[2]
ys = subgrad.shape[3]
subgrad = subgrad.view(nch, nx, xs * ys, -1).repeat(1, 1, 1, ws).view(
nch, nx, -1, ws * ys).repeat(1, 1, 1, ws).view(nch, nx, -1,
ws * ys)[:, :, :128, :]
x = (x * subgrad).view(nch, nx, -1, 20480).squeeze()
# print('total time per subgrad op: ', time.time()-t0)
# print('done!')
return x
def rn18_l4_1a_lppool(p: float):
n = 49**(1 / p)
model = rn18_l4_1a_nc_1k_scratch()
model.classifier = nn.Sequential(nn.LPPool2d(p, kernel_size=7),
LambdaLayer(lambda x: x / n),
nn.Flatten())
return model
def __init__(self,
plan,
conv,
num_classes=10,
dense_classifier=False,
path=False):
super(VGG, self).__init__()
layer_list = []
filters = 3
self.path = path
for spec in plan:
if spec == 'M':
if self.path:
layer_list.append(nn.LPPool2d(1.0, kernel_size=2,
stride=2))
else:
layer_list.append(nn.MaxPool2d(kernel_size=2, stride=2))
else:
layer_list.append(conv(filters, spec))
filters = spec
self.layers = nn.Sequential(*layer_list)
self.fc = layers.Linear(512, num_classes)
if dense_classifier:
self.fc = nn.Linear(512, num_classes)
self._initialize_weights()
def __init__(self):
super(NNPoolingModule, self).__init__()
self.input1d = torch.randn(1, 16, 50)
self.module1d = nn.ModuleList([
nn.MaxPool1d(3, stride=2),
nn.AvgPool1d(3, stride=2),
nn.LPPool1d(2, 3, stride=2),
nn.AdaptiveMaxPool1d(3),
nn.AdaptiveAvgPool1d(3),
])
self.input2d = torch.randn(1, 16, 30, 10)
self.module2d = nn.ModuleList([
nn.MaxPool2d((3, 2), stride=(2, 1)),
nn.AvgPool2d((3, 2), stride=(2, 1)),
nn.FractionalMaxPool2d(3, output_ratio=(0.5, 0.5)),
nn.LPPool2d(2, 3, stride=(2, 1)),
nn.AdaptiveMaxPool2d((5, 7)),
nn.AdaptiveAvgPool2d((7)),
])
self.input3d = torch.randn(1, 16, 20, 4, 4)
self.module3d = nn.ModuleList([
nn.MaxPool3d(2),
nn.AvgPool3d(2),
nn.FractionalMaxPool3d(2, output_ratio=(0.5, 0.5, 0.5)),
nn.AdaptiveMaxPool3d((5, 7, 9)),
nn.AdaptiveAvgPool3d((5, 7, 9)),
])
def __init__(self, useCuda, gpuDevice=0):
super(netOpenFace, self).__init__()
self.gpuDevice = gpuDevice
self.layer1 = Conv2d(3, 64, (7, 7), (2, 2), (3, 3))
self.layer2 = nn.BatchNorm2d(64)
self.layer3 = nn.ReLU()
self.layer4 = nn.MaxPool2d((3, 3), stride=(2, 2), padding=(1, 1))
self.layer5 = CrossMapLRN(5, 0.0001, 0.75, gpuDevice=gpuDevice)
self.layer6 = Conv2d(64, 64, (1, 1), (1, 1), (0, 0))
self.layer7 = nn.BatchNorm2d(64)
self.layer8 = nn.ReLU()
self.layer9 = Conv2d(64, 192, (3, 3), (1, 1), (1, 1))
self.layer10 = nn.BatchNorm2d(192)
self.layer11 = nn.ReLU()
self.layer12 = CrossMapLRN(5, 0.0001, 0.75, gpuDevice=gpuDevice)
self.layer13 = nn.MaxPool2d((3, 3), stride=(2, 2), padding=(1, 1))
self.layer14 = Inception(
192, (3, 5), (1, 1), (128, 32), (96, 16, 32, 64),
nn.MaxPool2d((3, 3), stride=(2, 2), padding=(0, 0)), True)
self.layer15 = Inception(256, (3, 5), (1, 1), (128, 64),
(96, 32, 64, 64),
nn.LPPool2d(2, (3, 3), stride=(3, 3)), True)
self.layer16 = Inception(
320, (3, 5), (2, 2), (256, 64), (128, 32, None, None),
nn.MaxPool2d((3, 3), stride=(2, 2), padding=(0, 0)), True)
self.layer17 = Inception(640, (3, 5), (1, 1), (192, 64),
(96, 32, 128, 256),
nn.LPPool2d(2, (3, 3), stride=(3, 3)), True)
self.layer18 = Inception(
640, (3, 5), (2, 2), (256, 128), (160, 64, None, None),
nn.MaxPool2d((3, 3), stride=(2, 2), padding=(0, 0)), True)
self.layer19 = Inception(1024, (3, ), (1, ), (384, ), (96, 96, 256),
nn.LPPool2d(2, (3, 3), stride=(3, 3)), True)
self.layer21 = Inception(
736, (3, ), (1, ), (384, ), (96, 96, 256),
nn.MaxPool2d((3, 3), stride=(2, 2), padding=(0, 0)), True)
self.layer22 = nn.AvgPool2d((3, 3), stride=(1, 1), padding=(0, 0))
self.layer25 = nn.Linear(736, 128)
#
# self.eval()
if useCuda:
self.cuda(gpuDevice)
def __init__(self):
super(Model, self).__init__()
self.pool_0 = nn.LPPool2d(norm_type=2, kernel_size=3)
self.pool_1 = nn.LPPool2d(norm_type=2, kernel_size=4, stride=2)
self.pool_2 = nn.LPPool2d(norm_type=1,
kernel_size=(1, 3),
stride=1,
ceil_mode=False)
self.pool_3 = nn.LPPool2d(norm_type=1,
kernel_size=(4, 5),
stride=(1, 2),
ceil_mode=True)
self.pool_4 = nn.LPPool2d(norm_type=1.2,
kernel_size=(5, 3),
stride=(2, 1),
ceil_mode=False)
self.pool_5 = nn.LPPool2d(norm_type=0.5,
kernel_size=2,
stride=1,
ceil_mode=True)
self.pool_6 = nn.LPPool2d(norm_type=0.1,
kernel_size=(5, 4),
stride=1,
ceil_mode=False)
def forward(self, x):
x = self.layers(x)
if self.path:
x = nn.LPPool2d(1.0, kernel_size=2)(x)
else:
x = nn.AvgPool2d(2)(x)
x = x.view(x.size(0), -1)
x = self.fc(x)
return x
def gram_nuclear_norm(y, gid):
'''y shape:
[m+p, n_features, T]'''
wsize = int((y.shape[2] + 1) / 2)
ksize = [1, wsize]
poolL2 = nn.LPPool2d(2, ksize, stride=1).cuda(gid) # ceil_mode?
avgpool = nn.AvgPool2d(ksize, stride=1).cuda(gid)
gnn = avgpool(poolL2(y)**2) # * wsize
return gnn
def forward(self, x):
out = F.relu(self.bn(self.conv(x)))
out = self.blocks(out)
if self.path:
out = nn.LPPool2d(1.0, out.size()[3])(out)
else:
out = F.avg_pool2d(out, out.size()[3])
out = out.view(out.size(0), -1)
out = self.fc(out)
return out
def __init__(self, num_classes=2):
super(ConvNet_v1, self).__init__()
self.features = nn.Sequential(
nn.Conv2d(9, 32, kernel_size=11, stride=4), nn.BatchNorm2d(32),
nn.ReLU(inplace=True), nn.LPPool2d(1, kernel_size=2, stride=2),
nn.Conv2d(32, 64, kernel_size=5, stride=2), nn.BatchNorm2d(64),
nn.ReLU(inplace=True), nn.MaxPool2d(kernel_size=2, stride=2),
nn.Conv2d(64, 64, kernel_size=3), nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=2, stride=2))
self.fc = nn.Sequential(nn.Linear(4 * 4 * 64, 256), nn.Sigmoid(),
nn.Linear(256, num_classes))
def __init__(self,
Chans,
kernLength1,
kernLength2,
dropoutRates=(0.25, 0.25),
F1=4,
D=2,
F2=8,
poolKern1=5,
poolKern2=8,
norm_rate=0.25,
dropoutType='Dropout'):
super(EEGNet_experimental, self).__init__()
#block1
time_padding = int((kernLength1 // 2))
self.conv1 = nn.Conv2d(in_channels=1,
out_channels=F1,
kernel_size=(1, kernLength1),
padding=(0, time_padding),
stride=1,
bias=False)
self.batchnorm1 = nn.BatchNorm2d(num_features=F1, affine=True)
self.depthwise1 = nn.Conv2d(in_channels=F1,
out_channels=F1 * D,
kernel_size=(Chans, 1),
groups=F1,
padding=0,
bias=False)
self.lppool = nn.LPPool2d(2, (1, 20),
stride=1) # convert to power, and pool
self.applyLog = logTransform()
self.batchnorm2 = nn.BatchNorm2d(num_features=F1 * D, affine=True)
self.activation_block1 = nn.ELU()
#self.avg_pool_block1 = nn.AvgPool2d((1,poolKern1))
self.avg_pool_block1 = nn.AdaptiveAvgPool2d((1, 32))
self.dropout_block1 = nn.Dropout(p=dropoutRates[0])
#block2
self.separable_block2 = deepwise_separable_conv(nin=F1 * D,
nout=F2,
kernelSize=kernLength2)
self.activation_block2 = nn.ELU()
self.avg_pool_block2 = nn.AdaptiveAvgPool2d((1, 10))
self.dropout_block2 = nn.Dropout(dropoutRates[1])
self.flatten = nn.Flatten()
# block 3
self.lstm = nn.LSTM(8, 20)
self.ln1 = nn.Linear(20, 1)
self.relu1 = nn.ReLU()
def __init__(self, num_classes=2):
super(ConvNet_v6, self).__init__()
self.layer1 = nn.Sequential(
nn.Conv2d(9, 32, kernel_size=11, stride=4, padding=10),
nn.ReLU(inplace=True), nn.LPPool2d(1, kernel_size=2, stride=2))
self.layer2 = nn.Sequential(
nn.Conv2d(32, 64, kernel_size=5, stride=2, padding=4),
nn.ReLU(inplace=True), nn.MaxPool2d(kernel_size=2, stride=2))
self.layer3 = nn.Sequential(
nn.Conv2d(64, 64, kernel_size=3, padding=1), nn.ReLU(inplace=True))
self.layer4 = nn.AdaptiveAvgPool2d(4)
self.fc = nn.Sequential(nn.Linear(4 * 4 * 64, 256), nn.Sigmoid(),
nn.Linear(256, num_classes))
def __init__(self,nb_classes=10, Chans=19, Samples=1000,
dropoutRates=(0.25,0.25), kernLength1=500,kernLength2=500, poolKern1=5,poolKern2=8, F1=4,
D=2, F2=8, norm_rate=0.25, dropoutType='Dropout'):
super(EEGNet_experimental,self).__init__()
self.Chans = Chans
self.Samples = Samples
self.output_sizes = {}
#block1
time_padding = int((kernLength1//2))
self.conv1 = nn.Conv2d(in_channels=1,out_channels=F1,kernel_size =(1,kernLength1),padding=(0,time_padding), stride=1,bias=False)
self.output_sizes['conv1']=convtransp_output_shape((Chans,Samples), kernel_size=(1,kernLength1), stride=1,
pad=(0,time_padding))
self.batchnorm1 = nn.BatchNorm2d(num_features=F1, affine=True)
self.depthwise1 = nn.Conv2d(in_channels=F1,out_channels=F1*D,kernel_size=(Chans,1),groups=F1,padding=0,bias=False)
self.lppool=nn.LPPool2d(2,(1,20),stride=1) # convert to power, and pool
self.applyLog=logTransform()
self.output_sizes['depthwise1'] = convtransp_output_shape(self.output_sizes['conv1'], kernel_size=(Chans,1),
stride=1, pad=0)
self.batchnorm2 = nn.BatchNorm2d(num_features=F1*D, affine=True)
self.activation_block1 = nn.ELU()
#self.avg_pool_block1 = nn.AvgPool2d((1,poolKern1))
self.output_sizes['avg_pool_block1'] = convtransp_output_shape(self.output_sizes['depthwise1'], kernel_size=(1, poolKern1),
stride=(1,poolKern1), pad=0)
self.avg_pool_block1 = nn.AdaptiveAvgPool2d((1,int(self.output_sizes['depthwise1'][1]/2))) # used to be 4
self.avg_pool_block1 = nn.AdaptiveAvgPool2d((1, 32))
self.output_sizes['avg_pool_block1'] = (1,32)
self.dropout_block1 = nn.Dropout(p=dropoutRates[0])
#block2
self.separable_block2 = deepwise_separable_conv(nin=F1*D,nout=F2,kernelSize=kernLength2)
self.output_sizes['separable_block2'] = self.separable_block2.get_output_size(self.output_sizes['avg_pool_block1'])
self.activation_block2 = nn.ELU()
# self.avg_pool_block2 = nn.AvgPool2d((1,poolKern2))
# self.output_sizes['avg_pool_block2'] = convtransp_output_shape(self.output_sizes['separable_block2'],
# kernel_size=(1, poolKern2),
# stride=(1, poolKern2), pad=0)
#self.avg_pool_block2 = nn.AdaptiveAvgPool2d((1,int(self.output_sizes['separable_block2'][1]/4)))
# self.output_sizes['avg_pool_block2'] = (1,int(self.output_sizes['separable_block2'][1]/4))
self.avg_pool_block2 = nn.AdaptiveAvgPool2d((1, 10))
self.output_sizes['avg_pool_block2'] = (1, 10)
self.dropout_block2 = nn.Dropout(dropoutRates[1])
self.flatten = nn.Flatten()
#n_size = self.get_features_dim(Chans,Samples)
#self.dense = nn.Linear(n_size,nb_classes)
# block 3
self.lstm = nn.LSTM(8, 20)
self.ln1 = nn.Linear(20, 1)
self.relu1 = nn.ReLU()
def __init__(self):
super(OpenFace, self).__init__()
self.layer1 = nn.Conv2d(3, 64, (7, 7), stride=(2, 2), padding=(3, 3))
self.layer2 = nn.BatchNorm2d(64)
self.layer3 = nn.ReLU()
self.layer4 = nn.MaxPool2d((3, 3), stride=(2, 2), padding=(1, 1))
self.layer5 = LRN(5)
self.layer6 = nn.Conv2d(64, 64, (1, 1))
self.layer7 = nn.BatchNorm2d(64)
self.layer8 = nn.ReLU()
self.layer9 = nn.Conv2d(64, 192, (3, 3), stride=(1, 1), padding=(1, 1))
self.layer10 = nn.BatchNorm2d(192)
self.layer11 = nn.ReLU()
self.layer12 = LRN(5)
self.layer13 = nn.MaxPool2d((3, 3), stride=(2, 2), padding=(1, 1))
self.layer14 = Inception(
192, (3, 5), (1, 1), (128, 32), (96, 16, 32, 64),
nn.MaxPool2d((3, 3), stride=(2, 2), padding=(0, 0)), True)
self.layer15 = Inception(256, (3, 5), (1, 1), (128, 64),
(96, 32, 64, 64),
nn.LPPool2d(2, (3, 3), stride=(3, 3)), True)
self.layer16 = Inception(
320, (3, 5), (2, 2), (256, 64), (128, 32, None, None),
nn.MaxPool2d((3, 3), stride=(2, 2), padding=(0, 0)), True)
self.layer17 = Inception(640, (3, 5), (1, 1), (192, 64),
(96, 32, 128, 256),
nn.LPPool2d(2, (3, 3), stride=(3, 3)), True)
self.layer18 = Inception(
640, (3, 5), (2, 2), (256, 128), (160, 64, None, None),
nn.MaxPool2d((3, 3), stride=(2, 2), padding=(0, 0)), True)
self.layer19 = Inception(1024, (3, ), (1, ), (384, ), (96, 96, 256),
nn.LPPool2d(2, (3, 3), stride=(3, 3)), True)
self.layer21 = Inception(
736, (3, ), (1, ), (384, ), (96, 96, 256),
nn.MaxPool2d((3, 3), stride=(2, 2), padding=(0, 0)), True)
self.layer22 = nn.AvgPool2d((3, 3), stride=(1, 1))
self.layer25 = nn.Linear(736, 128)
def __init__(self, inputdim, outputdim, **kwargs):
super().__init__()
self.features = nn.Sequential(
Block2D(1, 32),
nn.LPPool2d(4, (2, 4)),
Block2D(32, 128),
Block2D(128, 128),
nn.LPPool2d(4, (2, 4)),
Block2D(128, 128),
Block2D(128, 128),
nn.LPPool2d(4, (1, 4)),
nn.Dropout(0.3),
)
with torch.no_grad():
rnn_input_dim = self.features(torch.randn(1, 1, 500,
inputdim)).shape
rnn_input_dim = rnn_input_dim[1] * rnn_input_dim[-1]
self.gru = nn.GRU(rnn_input_dim,
128,
bidirectional=True,
batch_first=True)
self.temp_pool = LinearSoftPool()
self.outputlayer = nn.Linear(256, outputdim)
def __init__(self, num_classes=2):
super(ConvNet_v4, self).__init__()
self.layer1 = nn.Sequential(nn.Conv2d(9, 16, kernel_size=5),
nn.BatchNorm2d(16), nn.ReLU(inplace=True),
nn.LPPool2d(1, kernel_size=2, stride=2))
self.layer2 = nn.Sequential(nn.Conv2d(16, 24, kernel_size=5),
nn.BatchNorm2d(24), nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=2, stride=2))
self.layer3 = nn.Sequential(nn.Conv2d(72, 128, kernel_size=3),
nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=2, stride=2))
self.layer4 = nn.Sequential(nn.Conv2d(128, 128, kernel_size=3),
nn.ReLU(inplace=True),
nn.AdaptiveMaxPool2d(7))
self.fc = nn.Sequential(nn.Linear(7 * 7 * 128, 256), nn.Sigmoid(),
nn.Linear(256, num_classes))
def __init__(self, channel, reduction=16):
super(CALayer5, self).__init__()
# global average pooling: feature --> point
self.avg_pool = nn.AdaptiveAvgPool2d(1)
# global maximum pooling: feature --> point
self.max_pool = nn.AdaptiveMaxPool2d(1)
# global Lp pooling: feature --> point
self.lp_pool = nn.LPPool2d(2, kernel_size=3)
# define alpha and beta variables
self.alpha = nn.Parameter(data=torch.Tensor(), requires_grad=True)
# feature channel downscale and upscale --> channel weight
self.conv_du = nn.Sequential(
nn.Conv2d(channel, channel // reduction, 1, padding=0, bias=True),
nn.ReLU(inplace=True),
nn.Conv2d(channel // reduction, channel, 1, padding=0, bias=True),
nn.Sigmoid())
def __init__(self, mode='max3d', ch=None, bias=False):
super().__init__()
if ch is not None and ch[0] != ch[1]:
raise ValueError('input, output channels must be equal')
if mode == 'max':
self.down = nn.MaxPool2d(2, stride=2)
elif mode == 'max3d':
self.down = nn.MaxPool3d(2, stride=2)
elif mode == 'mean':
self.down = nn.AvgPool2d(2, stride=2)
elif mode == 'norm2':
self.down = nn.LPPool2d(2, 2, stride=2)
elif mode == 'conv':
self.down = conv1(ch, k=2, stride=2, bias=bias)
else:
raise ValueError("mode must be one of maxpool, norm2, conv")
def __init__(self, num_classes=2):
super(ConvNet_v3, self).__init__()
self.features = nn.Sequential(nn.Conv2d(9, 32, kernel_size=5),
nn.BatchNorm2d(32),
nn.ReLU(inplace=True),
nn.LPPool2d(1, kernel_size=2, stride=2),
nn.Conv2d(32, 64, kernel_size=5),
nn.BatchNorm2d(64),
nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=2, stride=2),
nn.Conv2d(64, 128, kernel_size=3),
nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=2, stride=2),
nn.Conv2d(128, 128, kernel_size=3),
nn.ReLU(inplace=True))
self.global_pooling = nn.AdaptiveMaxPool2d(7)
self.fc = nn.Sequential(nn.Linear(7 * 7 * 128, 256), nn.Sigmoid(),
nn.Linear(256, num_classes))
def __init__(self,
conv_1,
conv_3,
conv_5,
conv_3_max,
use_max_pool=True,
use_l2=False):
super(inception_b, self).__init__()
model_list = []
if (len(conv_1) != 0):
model_list.append(
nn.Sequential(nn.Conv2d(conv_1[0], conv_1[1], kernel_size=1),
nn.ReLU(inplace=1)))
if (len(conv_3) != 0):
model_list.append(
nn.Sequential(
nn.Conv2d(conv_3[0], conv_3[1], kernel_size=1),
nn.ReLU(inplace=True),
nn.Conv2d(conv_3[1], conv_3[2], kernel_size=3, padding=1),
nn.ReLU(inplace=True)))
if (len(conv_5) != 0):
model_list.append(
nn.Sequential(
nn.Conv2d(conv_5[0], conv_5[1], kernel_size=1),
nn.ReLU(inplace=True),
nn.Conv2d(conv_5[1], conv_5[2], kernel_size=5, padding=2),
nn.ReLU(inplace=True)))
if (use_max_pool):
model_list.append(
nn.Sequential(
nn.MaxPool2d(kernel_size=3, padding=1, stride=1),
nn.Conv2d(conv_3_max[0],
conv_3_max[1],
kernel_size=1,
padding=0), nn.ReLU(inplace=True)))
if (use_l2):
model_list.append(
nn.Sequential(
nn.LPPool2d(2, kernel_size=3, padding=1, stride=1),
nn.Conv2d(conv_3_max[0],
conv_3_max[1],
kernel_size=1,
padding=0), nn.ReLU(inplace=True)))
self.conv = nn.ModuleList(model_list)
def __init__(self,
Chans,
kernLength1,
kernLength2,
dropoutRates=(0.25, 0.25),
F1=4,
D=2,
F2=8,
poolKern1=5,
poolKern2=8,
norm_rate=0.25,
dropoutType='Dropout'):
super(model_simplify, self).__init__()
#block1
time_padding = int((kernLength1 // 2))
self.conv1 = nn.Conv2d(in_channels=1,
out_channels=F1,
kernel_size=(1, kernLength1),
padding=(0, time_padding),
stride=1,
bias=False)
self.batchnorm1 = nn.BatchNorm2d(num_features=F1, affine=True)
self.depthwise1 = nn.Conv2d(in_channels=F1,
out_channels=F1 * D,
kernel_size=(Chans, 1),
groups=F1,
padding=0,
bias=False)
self.lppool = nn.LPPool2d(2, (1, 20),
stride=1) # convert to power, and pool
self.applyLog = logTransform()
self.batchnorm2 = nn.BatchNorm2d(num_features=F1 * D, affine=True)
self.activation_block1 = nn.ELU()
#self.avg_pool_block1 = nn.AvgPool2d((1,poolKern1))
self.avg_pool_block1 = nn.AdaptiveAvgPool2d((1, 32))
self.dropout_block1 = nn.Dropout(p=dropoutRates[0])
# block 3
self.lstm = nn.LSTM(8, 20, bidirectional=True)
self.ln1 = nn.Linear(40, 1)
self.relu1 = nn.ReLU()
def __init__(self):
super(DiscriminatorNetwork, self).__init__()
self.resblock1 = ResBlockDown(1, 16)
self.resblock2 = ResBlockDown(16, 16)
self.resblock3 = ResBlockDown(16, 32)
self.resblock4 = ResBlockDown(32, 64)
self.resblock5 = ResBlockDown(64, 128)
self.resblock6 = ResBlockDown(128, 128)
self.resblock7 = ResBlockDown(128, 256)
self.resblock8 = ResBlock(256, 256)
self.resdownblocks = [
self.resblock1,
self.resblock2,
self.resblock3,
self.resblock4,
self.resblock5,
self.resblock6,
self.resblock7,
]
self.self_attention = SelfAttention(32)
self.global_sum_pooling = nn.LPPool2d(norm_type=1, kernel_size=(1, 4))
self.dense = spectral_norm(nn.Linear(256, 1))
def __init__(self):
super(HASHI, self).__init__()
self.conv1 = nn.Conv2d(3, 256, kernel_size=8, stride=1, padding=0, bias=False) # out 256x94x94
self.relu = nn.ReLU(inplace=True)
self.L2pool = nn.LPPool2d(2, 2, stride=2) # out 256x47x47
self.fc = nn.Linear(256*47*47, 2)
def __init__(self, args, gpuDevice=0):
super(netOpenFace, self).__init__()
self.gpuDevice = gpuDevice
self.layer1 = Conv2d(3, 64, (7, 7), (2, 2), (3, 3))
self.layer2 = BatchNorm(64)
self.layer3 = nn.ReLU()
self.layer4 = nn.MaxPool2d((3, 3), stride=(2, 2), padding=(1, 1))
self.layer5 = CrossMapLRN(5, 0.0001, 0.75, gpuDevice=gpuDevice)
# Inception (2)
self.layer6 = Conv2d(64, 64, (1, 1), (1, 1), (0, 0))
# self.layer6 = Conv2d(64, 64, (1,1), (1,1), (1,1))
self.layer7 = BatchNorm(64)
self.layer8 = nn.ReLU()
self.layer9 = Conv2d(64, 192, (3, 3), (1, 1), (1, 1))
self.layer10 = BatchNorm(192)
self.layer11 = nn.ReLU()
self.layer12 = CrossMapLRN(5, 0.0001, 0.75, gpuDevice=gpuDevice)
self.layer13 = nn.MaxPool2d((3, 3), stride=(2, 2), padding=(1, 1))
# Inception (3a)
# self.layer14 = Inception(192, (3,5), (1,1), (128,32), (96,16,32,64), nn.MaxPool2d((3,3), stride=(2,2), padding=(0,0)), True)
self.layer14 = Inception(
192, (3, 5), (1, 1), (128, 32), (96, 16, 32, 64),
nn.MaxPool2d((3, 3), stride=(1, 1), padding=(1, 1)), True)
# Inception (3b)
# self.layer15 = Inception(256, (3,5), (1,1), (128,64), (96,32,64,64), nn.LPPool2d(2, (3,3), stride=(3,3)), True)
self.layer15 = Inception(256, (3, 5), (1, 1), (128, 64),
(96, 32, 64, 64),
nn.LPPool2d(2, (3, 3), stride=(1, 1)), True)
# Inception (3c)
# self.layer16 = Inception(320, (3,5), (2,2), (256,64), (128,32,None,None), nn.MaxPool2d((3,3), stride=(2,2), padding=(0,0)), True)
self.layer16 = Inception(
320, (3, 5), (2, 2), (256, 64), (128, 32, None, None),
nn.MaxPool2d((3, 3), stride=(2, 2), padding=(1, 1)), True)
# Inception (4a)
# self.layer17 = Inception(640, (3,5), (1,1), (192,64), (96,32,128,256), nn.LPPool2d(2, (3,3), stride=(3,3)), True)
self.layer17 = Inception(640, (3, 5), (1, 1), (192, 64),
(96, 32, 128, 256),
nn.LPPool2d(2, (3, 3), stride=(1, 1)), True)
# Inception (4e)
# self.layer18 = Inception(640, (3,5), (2,2), (256,128), (160,64,None,None), nn.MaxPool2d((3,3), stride=(2,2), padding=(0,0)), True)
self.layer18 = Inception(
640, (3, 5), (2, 2), (256, 128), (160, 64, None, None),
nn.MaxPool2d((3, 3), stride=(2, 2), padding=(1, 1)), True)
# Inception (5a)
# self.layer19 = Inception(1024, (3,), (1,), (384,), (96,96,256), nn.LPPool2d(2, (3,3), stride=(3,3)), True)
self.layer19 = Inception(1024, (3, ), (1, ), (384, ), (96, 96, 256),
nn.LPPool2d(2, (3, 3), stride=(1, 1)), True)
# Inception (5b)
# self.layer21 = Inception(736, (3,), (1,), (384,), (96,96,256), nn.MaxPool2d((3,3), stride=(2,2), padding=(0,0)), True)
self.layer21 = Inception(
736, (3, ), (1, ), (384, ), (96, 96, 256),
nn.MaxPool2d((3, 3), stride=(1, 1), padding=(1, 1)), True)
self.layer22 = nn.AvgPool2d((3, 3), stride=(1, 1), padding=(0, 0))
self.layer25 = Linear(736, 128)
#
self.resize1 = nn.UpsamplingNearest2d(scale_factor=3)
self.resize2 = nn.AvgPool2d(4)
#
# self.eval()
if args.cuda:
self.cuda(gpuDevice)
def __init__(self,
n_conv,
n_pool,
n_fc,
conv_filters,
kernel_sizes,
p_kernels,
strides,
p_strides,
paddings,
fc_dims,
in_channels,
flat_dim,
dropout=.0,
batch_norm=False):
super(ConvNet, self).__init__()
self.n_conv = n_conv # integer
self.n_pool = n_pool # integer
self.n_fc = n_fc # integer
self.conv_filters = conv_filters # list with length n_conv
self.kernel_sizes = kernel_sizes # list with length n_conv (square filters)
self.p_kernels = p_kernels # list with length n_pool (square filters)
self.strides = strides # list with length n_conv
self.p_strides = p_strides # list with length n_pool
self.paddings = paddings # list with length n_conv
self.fc_dims = fc_dims # list with length n_fc
self.in_channels = in_channels # integer
self.flat_dim = flat_dim # integer
# convolutional layers
self.conv_layers = nn.ModuleList([
nn.Conv2d(self.in_channels,
self.conv_filters[0],
self.kernel_sizes[0],
stride=self.strides[0],
padding=self.paddings[0])
])
self.conv_layers.extend([
nn.Conv2d(self.conv_filters[i - 1],
self.conv_filters[i],
self.kernel_sizes[i],
stride=self.strides[i],
padding=self.paddings[i]) for i in range(1, self.n_conv)
])
# pooling layers
self.pool_layers = nn.ModuleList(
[nn.LPPool2d(2, self.p_kernels[0], stride=self.p_strides[0])])
self.pool_layers.extend([
nn.LPPool2d(2, self.p_kernels[i], stride=self.p_strides[i])
for i in range(1, self.n_pool)
])
# fully connected layers
self.fc_layers = nn.ModuleList(
[nn.Linear(self.flat_dim, self.fc_dims[0])])
self.fc_layers.extend([
nn.Linear(self.fc_dims[i - 1], self.fc_dims[i])
for i in range(1, self.n_fc)
])
def __append_layer(self, net_style, args_dict):
args_values_list = list(args_dict.values())
if net_style == "Conv2d":
self.layers.append(
nn.Conv2d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4], args_values_list[5],
args_values_list[6], args_values_list[7]))
elif net_style == "MaxPool2d":
self.layers.append(
nn.MaxPool2d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4], args_values_list[5]))
elif net_style == "Linear":
self.layers.append(
nn.Linear(args_values_list[0], args_values_list[1],
args_values_list[2]))
elif net_style == "reshape":
# 如果是特殊情况 reshape，就直接将目标向量尺寸传入
# print(type(args_values_list[0]))
self.layers.append(args_values_list[0])
elif net_style == "Conv1d":
self.layers.append(
nn.Conv1d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4], args_values_list[5],
args_values_list[6], args_values_list[7]))
elif net_style == "Conv3d":
self.layers.append(
nn.Conv3d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4], args_values_list[5],
args_values_list[6], args_values_list[7]))
elif net_style == "ConvTranspose1d":
self.layers.append(
nn.ConvTranspose1d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4], args_values_list[5],
args_values_list[6], args_values_list[7],
args_values_list[8]))
elif net_style == "ConvTranspose2d":
self.layers.append(
nn.ConvTranspose2d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4], args_values_list[5],
args_values_list[6], args_values_list[7],
args_values_list[8]))
elif net_style == "ConvTranspose3d":
self.layers.append(
nn.ConvTranspose3d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4], args_values_list[5],
args_values_list[6], args_values_list[7],
args_values_list[8]))
elif net_style == "Unfold":
self.layers.append(
nn.Unfold(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3]))
elif net_style == "Fold":
self.layers.append(
nn.Unfold(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4]))
elif net_style == "MaxPool1d":
self.layers.append(
nn.MaxPool1d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4], args_values_list[5]))
elif net_style == "MaxPool3d":
self.layers.append(
nn.MaxPool3d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4], args_values_list[5]))
elif net_style == "MaxUnpool1d":
self.layers.append(
nn.MaxUnpool1d(args_values_list[0], args_values_list[1],
args_values_list[2]))
elif net_style == "MaxUnpool2d":
self.layers.append(
nn.MaxUnpool2d(args_values_list[0], args_values_list[1],
args_values_list[2]))
elif net_style == "MaxUnpool3d":
self.layers.append(
nn.MaxUnpool3d(args_values_list[0], args_values_list[1],
args_values_list[2]))
elif net_style == "AvgPool1d":
self.layers.append(
nn.AvgPool1d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4]))
elif net_style == "AvgPool2d":
self.layers.append(
nn.AvgPool2d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4]))
elif net_style == "AvgPool3d":
self.layers.append(
nn.AvgPool3d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4]))
elif net_style == "FractionalMaxPool2d":
self.layers.append(
nn.FractionalMaxPool2d(args_values_list[0],
args_values_list[1],
args_values_list[2],
args_values_list[3],
args_values_list[4]))
elif net_style == "LPPool1d":
self.layers.append(
nn.LPPool1d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3]))
elif net_style == "LPPool2d":
self.layers.append(
nn.LPPool2d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3]))
elif net_style == "AdaptiveMaxPool1d":
self.layers.append(
nn.AdaptiveMaxPool1d(args_values_list[0], args_values_list[1]))
elif net_style == "AdaptiveMaxPool2d":
self.layers.append(
nn.AdaptiveMaxPool2d(args_values_list[0], args_values_list[1]))
elif net_style == "AdaptiveMaxPool3d":
self.layers.append(
nn.AdaptiveMaxPool3d(args_values_list[0], args_values_list[1]))
elif net_style == "AdaptiveAvgPool1d":
self.layers.append(nn.AdaptiveAvgPool1d(args_values_list[0]))
elif net_style == "AdaptiveAvgPool2d":
self.layers.append(nn.AdaptiveAvgPool2d(args_values_list[0]))
elif net_style == "AdaptiveAvgPool3d":
self.layers.append(nn.AdaptiveAvgPool3d(args_values_list[0]))
elif net_style == "ReflectionPad1d":
self.layers.append(nn.ReflectionPad1d(args_values_list[0]))
elif net_style == "ReflectionPad2d":
self.layers.append(nn.ReflectionPad2d(args_values_list[0]))
elif net_style == "ReplicationPad1d":
self.layers.append(nn.ReplicationPad1d(args_values_list[0]))
elif net_style == "ReplicationPad2d":
self.layers.append(nn.ReplicationPad2d(args_values_list[0]))
elif net_style == "ReplicationPad3d":
self.layers.append(nn.ReplicationPad3d(args_values_list[0]))
elif net_style == "ZeroPad2d":
self.layers.append(nn.ZeroPad2d(args_values_list[0]))
elif net_style == "ConstantPad1d":
self.layers.append(
nn.ConstantPad1d(args_values_list[0], args_values_list[1]))
elif net_style == "ConstantPad2d":
self.layers.append(
nn.ConstantPad2d(args_values_list[0], args_values_list[1]))
elif net_style == "ConstantPad3d":
self.layers.append(
nn.ConstantPad3d(args_values_list[0], args_values_list[1]))
elif net_style == "ELU":
self.layers.append(nn.ELU(args_values_list[0],
args_values_list[1]))
elif net_style == "Hardshrink":
self.layers.append(nn.Hardshrink(args_values_list[0]))
elif net_style == "Hardtanh":
self.layers.append(
nn.Hardtanh(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4]))
elif net_style == "LeakyReLU":
self.layers.append(
nn.LeakyReLU(args_values_list[0], args_values_list[1]))
elif net_style == "LogSigmoid":
self.layers.append(nn.LogSigmoid())
elif net_style == "PReLU":
self.layers.append(
nn.PReLU(args_values_list[0], args_values_list[1]))
elif net_style == "ReLU":
self.layers.append(nn.ReLU(args_values_list[0]))
elif net_style == "ReLU6":
self.layers.append(nn.ReLU6(args_values_list[0]))
elif net_style == "RReLU":
self.layers.append(
nn.RReLU(args_values_list[0], args_values_list[1],
args_values_list[2]))
elif net_style == "SELU":
self.layers.append(nn.SELU(args_values_list[0]))
elif net_style == "CELU":
self.layers.append(
nn.CELU(args_values_list[0], args_values_list[1]))
elif net_style == "Sigmoid":
self.layers.append(nn.Sigmoid())
elif net_style == "Softplus":
self.layers.append(
nn.Softplus(args_values_list[0], args_values_list[1]))
elif net_style == "Softshrink":
self.layers.append(nn.Softshrink(args_values_list[0]))
elif net_style == "Softsign":
self.layers.append(nn.Softsign())
elif net_style == "Tanh":
self.layers.append(nn.Tanh())
elif net_style == "Tanhshrink":
self.layers.append(nn.Tanhshrink())
elif net_style == "Threshold":
self.layers.append(
nn.Threshold(args_values_list[0], args_values_list[1],
args_values_list[2]))
elif net_style == "Softmin":
self.layers.append(nn.Softmin(args_values_list[0]))
elif net_style == "Softmax":
self.layers.append(nn.Softmax(args_values_list[0]))
elif net_style == "Softmax2d":
self.layers.append(nn.Softmax2d())
elif net_style == "LogSoftmax":
self.layers.append(nn.LogSoftmax(args_values_list[0]))
elif net_style == "AdaptiveLogSoftmaxWithLoss":
self.layers.append(
nn.AdaptiveLogSoftmaxWithLoss(args_values_list[0],
args_values_list[1],
args_values_list[2],
args_values_list[3],
args_values_list[4]))
elif net_style == "BatchNorm1d":
self.layers.append(
nn.BatchNorm1d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4]))
elif net_style == "BatchNorm2d":
self.layers.append(
nn.BatchNorm2d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4]))
elif net_style == "BatchNorm3d":
self.layers.append(
nn.BatchNorm3d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4]))
elif net_style == "GroupNorm":
self.layers.append(
nn.GroupNorm(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3]))
elif net_style == "InstanceNorm1d":
self.layers.append(
nn.InstanceNorm1d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4]))
elif net_style == "InstanceNorm2d":
self.layers.append(
nn.InstanceNorm2d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4]))
elif net_style == "InstanceNorm3d":
self.layers.append(
nn.InstanceNorm3d(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3],
args_values_list[4]))
elif net_style == "LayerNorm":
self.layers.append(
nn.LayerNorm(args_values_list[0], args_values_list[1],
args_values_list[2]))
elif net_style == "LocalResponseNorm":
self.layers.append(
nn.LocalResponseNorm(args_values_list[0], args_values_list[1],
args_values_list[2], args_values_list[3]))
elif net_style == "Linear":
self.layers.append(
nn.Linear(args_values_list[0], args_values_list[1],
args_values_list[2]))
elif net_style == "Dropout":
self.layers.append(
nn.Dropout(args_values_list[0], args_values_list[1]))
elif net_style == "Dropout2d":
self.layers.append(
nn.Dropout2d(args_values_list[0], args_values_list[1]))
elif net_style == "Dropout3d":
self.layers.append(
nn.Dropout3d(args_values_list[0], args_values_list[1]))
elif net_style == "AlphaDropout":
self.layers.append(
nn.AlphaDropout(args_values_list[0], args_values_list[1]))
def __init__(self, in_channels, init=False):
def scaled_tanh_nonlinearity():
return Expression(lambda x: 1.7156 * F.tanh(2 * x / 3))
# spatial layer with init from icp?!
super().__init__()
spatial_conv = nn.Sequential()
spatial_conv.add_module(
"spatial_conv",
torch.nn.Conv2d(in_channels=in_channels,
out_channels=in_channels,
kernel_size=(1, 1),
stride=1,
padding=0,
dilation=1,
groups=1,
bias=False))
spatial_conv.add_module("activation", scaled_tanh_nonlinearity())
self.spatial_conv = spatial_conv
temporal_conv = nn.Sequential()
NUM_FILTERS = [2, 4, 8]
DILATIONS = [1, 2, 4]
temp_in_chans = 1
for layer, (num_filters,
dilation) in enumerate(zip(NUM_FILTERS, DILATIONS)):
temporal_layer = nn.Conv2d(temp_in_chans,
num_filters,
dilation=dilation,
kernel_size=(1, 17),
bias=False)
# init temp layer with wavelets
if init:
pass
temporal_conv.add_module(f'temporal_conv{layer}', temporal_layer)
temporal_conv.add_module(f'activation{layer}',
scaled_tanh_nonlinearity())
temp_in_chans = num_filters
self.temporal_conv = temporal_conv
# pooling layer
# FIXME: this implementation does not follow published paper
self.l2_norm = nn.LPPool2d(2, (1, 10), stride=1)
# THIS IS THE PAPER IMPLEMENTATION
# 40 might be very aggressive for smaller sampling rates
# TODO: consider passing pool size and stride as an input argument
# self.l2_norm = nn.LPPool2d(2, (1, 40), stride=40)
# lstm
# FIXME: this implementation does not follow published paper
self.lstm = nn.LSTM(
input_size=in_channels * 8,
hidden_size=10,
batch_first=True,
# THIS IS THE PAPER IMPLEMENTATION
# self.lstm = nn.LSTM(input_size=in_channels * 8 * 25, hidden_size=10, batch_first=False,
num_layers=1,
bias=True)
# linear
# TODO: replace 1 with NxClasses for multi-task
self.fc = SequenceWise(nn.Linear(10, 1))
def rn18_l4_1a_lppool_p1_5():
model = rn18_l4_1a_nc_1k_scratch()
model.classifier = nn.Sequential(nn.LPPool2d(1.5, kernel_size=7),
LambdaLayer(lambda x: x / 13.390518),
nn.Flatten())
return model
# 线性层 class torch.nn.Linear(in_features, out_features, bias=True) torch.nn.functional.linear(input, weight, bias=None) # 卷积层 class torch.nn.Conv1d(in_channels, out_channels, kernel_size, stride=1, padding=0, dilation=1, groups=1, bias=True) # 对输入信号提供2维的幂平均池化操作 class torch.nn.LPPool2d(norm_type, kernel_size, stride=None, ceil_mode=False) torch.nn.functional.lp_pool2d(input, norm_type, kernel_size, stride=None, ceil_mode=False) class torch.nn.AdaptiveMaxPool1d(output_size, return_indices=False) torch.nn.functional.adaptive_max_pool1d(input, output_size, return_indices=False) class torch.nn.AdaptiveAvgPool1d(output_size)
def __init__(self,
sampling='down',
in_dim=64,
pool_ksize=2,
down_ch=1,
dconv_dim=1):
super(Multi_Attn, self).__init__()
self.dim = in_dim
self.pool_ksize = pool_ksize
self.sampling = sampling
# pool methods
self.max_pool = nn.MaxPool2d(pool_ksize, pool_ksize)
self.avg_pool = nn.AvgPool2d(pool_ksize, pool_ksize)
self.lp_pool = nn.LPPool2d(2, pool_ksize, pool_ksize)
self.pools = [self.max_pool, self.avg_pool, self.lp_pool]
self.xin_conv = nn.Conv2d(in_channels=in_dim,
out_channels=in_dim * 2,
kernel_size=4,
stride=2,
padding=1)
self.yin_conv = nn.Conv2d(in_channels=in_dim,
out_channels=in_dim * 2,
kernel_size=4,
stride=2,
padding=1)
self.q_conv = nn.Conv2d(in_channels=in_dim,
out_channels=in_dim // down_ch,
kernel_size=1)
self.k_conv = nn.Conv2d(in_channels=in_dim,
out_channels=in_dim // down_ch,
kernel_size=1)
self.v1_conv = nn.Conv2d(in_channels=in_dim,
out_channels=in_dim,
kernel_size=1)
self.v2_conv = nn.Conv2d(in_channels=in_dim,
out_channels=in_dim,
kernel_size=1)
self.out_deconv1_1 = nn.ConvTranspose2d(in_channels=in_dim,
out_channels=in_dim *
dconv_dim,
kernel_size=4,
stride=2,
padding=1)
self.out_deconv1_2 = nn.ConvTranspose2d(in_channels=in_dim,
out_channels=in_dim *
dconv_dim,
kernel_size=4,
stride=2,
padding=1)
self.out_deconv2_1 = nn.ConvTranspose2d(in_channels=in_dim,
out_channels=in_dim *
dconv_dim,
kernel_size=4,
stride=2,
padding=1)
self.out_deconv2_2 = nn.ConvTranspose2d(in_channels=in_dim,
out_channels=in_dim *
dconv_dim,
kernel_size=4,
stride=2,
padding=1)
self.softmax1 = nn.Softmax(dim=-1)
self.softmax2 = nn.Softmax(dim=1)
self.gamma1 = nn.Parameter(torch.zeros(1))
self.gamma2 = nn.Parameter(torch.zeros(1))
self.a1 = nn.Parameter(torch.ones(1))
self.a2 = nn.Parameter(torch.ones(1))
self.a3 = nn.Parameter(torch.ones(1))
self.b1 = nn.Parameter(torch.ones(1))
self.b2 = nn.Parameter(torch.ones(1))
self.b3 = nn.Parameter(torch.ones(1))
